diff --git a/.coveragerc b/.coveragerc index 87cdc8e..b346d37 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,5 +1,5 @@ [run] -source = dataflux +source = recordstream omit = */tests/* */examples/* diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 642d02c..4c4522a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,14 +1,14 @@ # ========================================================================= # AUTO-GENERATED FILE — DO NOT EDIT BY HAND -# Generated by: aisland jenkins scaffold --project dataflux +# Generated by: aisland jenkins scaffold recordstream # Source template: aisland/services/jenkins.py (JenkinsService.scaffold_pipelines) -# To regenerate: aisland jenkins scaffold --project dataflux --force +# To regenerate: aisland jenkins scaffold recordstream --force # ========================================================================= -name: Dataflux CI +name: Recordstream CI on: push: - branches: [ main, dev/main ] + branches: [ main ] pull_request: branches: [ main ] @@ -18,15 +18,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Mint GitHub App token for private Gearlux deps - id: app-token - uses: actions/create-github-app-token@v1 - with: - app-id: ${{ secrets.GEARLUX_APP_ID }} - private-key: ${{ secrets.GEARLUX_APP_PRIVATE_KEY }} - owner: Gearlux - - name: Configure git auth for Gearlux - run: git config --global url."https://x-access-token:${{ steps.app-token.outputs.token }}@github.com/".insteadOf "https://github.com/" - name: Set up Python 3.12 uses: actions/setup-python@v5 with: @@ -36,11 +27,7 @@ jobs: - name: Install dependencies run: | - # Internal Gearlux dependencies — installed FIRST with --no-deps so - # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. - uv pip install --system --no-deps git+https://github.com/Gearlux/log-flow.git@main - uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main - uv pip install --system -e ".[dev]" + uv pip install --system -e ".[dev,torch,keras]" - name: Run Isort run: isort --check-only . - name: Run Black @@ -56,15 +43,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Mint GitHub App token for private Gearlux deps - id: app-token - uses: actions/create-github-app-token@v1 - with: - app-id: ${{ secrets.GEARLUX_APP_ID }} - private-key: ${{ secrets.GEARLUX_APP_PRIVATE_KEY }} - owner: Gearlux - - name: Configure git auth for Gearlux - run: git config --global url."https://x-access-token:${{ steps.app-token.outputs.token }}@github.com/".insteadOf "https://github.com/" - name: Set up Python 3.12 uses: actions/setup-python@v5 with: @@ -74,15 +52,11 @@ jobs: - name: Install dependencies run: | - # Internal Gearlux dependencies — installed FIRST with --no-deps so - # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. - uv pip install --system --no-deps git+https://github.com/Gearlux/log-flow.git@main - uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main - uv pip install --system -e ".[dev]" + uv pip install --system -e ".[dev,torch,keras]" - name: Run Tests run: | if [ -d tests ] && find tests -name '*.py' | grep -q .; then - pytest tests --junitxml=test-report.xml --cov=dataflux --cov-report=xml --cov-report=term + pytest tests --junitxml=test-report.xml --cov=recordstream --cov-report=xml --cov-report=term else echo "No tests found. Skipping." fi @@ -99,15 +73,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Mint GitHub App token for private Gearlux deps - id: app-token - uses: actions/create-github-app-token@v1 - with: - app-id: ${{ secrets.GEARLUX_APP_ID }} - private-key: ${{ secrets.GEARLUX_APP_PRIVATE_KEY }} - owner: Gearlux - - name: Configure git auth for Gearlux - run: git config --global url."https://x-access-token:${{ steps.app-token.outputs.token }}@github.com/".insteadOf "https://github.com/" - name: Set up Python 3.12 uses: actions/setup-python@v5 with: @@ -117,11 +82,7 @@ jobs: - name: Install dependencies run: | - # Internal Gearlux dependencies — installed FIRST with --no-deps so - # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. - uv pip install --system --no-deps git+https://github.com/Gearlux/log-flow.git@main - uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main - uv pip install --system -e ".[dev]" + uv pip install --system -e ".[dev,torch,keras]" - name: Run Examples run: | for f in examples/*.py; do @@ -130,6 +91,15 @@ jobs: python3 "$f" fi done + # Directory examples opt in by shipping a run.py entry point; + # network-dependent / install-only example apps ship no run.py + # and are skipped by construction. + for f in examples/*/run.py; do + if [ -f "$f" ]; then + echo "Verifying $f..." + python3 "$f" + fi + done verify-notebooks: name: Verify Notebooks @@ -137,15 +107,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Mint GitHub App token for private Gearlux deps - id: app-token - uses: actions/create-github-app-token@v1 - with: - app-id: ${{ secrets.GEARLUX_APP_ID }} - private-key: ${{ secrets.GEARLUX_APP_PRIVATE_KEY }} - owner: Gearlux - - name: Configure git auth for Gearlux - run: git config --global url."https://x-access-token:${{ steps.app-token.outputs.token }}@github.com/".insteadOf "https://github.com/" - name: Set up Python 3.12 uses: actions/setup-python@v5 with: @@ -155,11 +116,7 @@ jobs: - name: Install dependencies run: | - # Internal Gearlux dependencies — installed FIRST with --no-deps so - # `-e .[dev]` below finds them pre-satisfied instead of hitting PyPI. - uv pip install --system --no-deps git+https://github.com/Gearlux/log-flow.git@main - uv pip install --system --no-deps git+https://github.com/Gearlux/confluid.git@main - uv pip install --system -e ".[dev]" + uv pip install --system -e ".[dev,torch,keras]" # Notebook-only dependencies live in the optional `[notebook]` extra # when a project ships notebooks; absence is not an error. uv pip install --system -e ".[notebook]" 2>/dev/null || echo "No [notebook] extras declared; continuing with defaults." diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..f51c172 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,101 @@ +# ========================================================================= +# HAND-WRITTEN workflow — NOT generated by `aisland jenkins scaffold`. +# (ci.yml / Jenkinsfile are generated; this release pipeline is separate so +# regeneration never clobbers it.) +# +# Publishes the `recordstream` distribution to PyPI when a version tag is pushed: +# +# git tag v0.1.0a1 && git push origin v0.1.0a1 +# +# Uses PyPI Trusted Publishing (OIDC, no stored API token). One-time setup on +# https://pypi.org/manage/account/publishing/ — add a "pending publisher": +# PyPI project name: recordstream +# Owner: Gearlux +# Repository: recordstream +# Workflow name: release.yml +# Environment: pypi +# +# Publish ORDER: loggair -> confluid -> liquifai are all on PyPI already, and +# this package declares floors on all three, so it publishes after them. The +# visual-editor package declares `recordstream>=0.1.0a1`, so it publishes after +# THIS one. +# ========================================================================= +name: Release to PyPI + +on: + push: + tags: ["v*"] + +jobs: + build: + name: Build & verify distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install packaging toolchain + run: python -m pip install build twine + - name: Guard - tag matches pyproject version + run: | + PKG_VERSION=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])") + TAG_VERSION="${GITHUB_REF_NAME#v}" + if [ "$PKG_VERSION" != "$TAG_VERSION" ]; then + echo "::error::Tag v$TAG_VERSION does not match pyproject version $PKG_VERSION" + exit 1 + fi + - name: Build sdist + wheel + run: python -m build + - name: Check package metadata + run: python -m twine check --strict dist/* + - name: Smoke-test the built wheel in a clean venv + # Runs from /tmp: with the repo root as cwd, `import recordstream` would find + # the SOURCE tree (cwd shadows site-packages) and silently test the repo + # instead of the wheel. This is the package-shadowing trap the workspace + # mandates warn about — the repo directory and the package share a name. + run: | + python -m venv /tmp/smoke + /tmp/smoke/bin/pip install --quiet dist/*.whl + cd /tmp + /tmp/smoke/bin/python - <<'EOF' + from importlib.metadata import version + + import numpy as np + + from recordstream import Image, Label, Stream, as_transform + + # A record is a plain dict of typed values; an op is type-dispatched. Running + # one record through a Stream exercises the engine, the item types and the + # op-family dispatch — the surfaces a broken wheel would take down. + records = [{"image": Image(np.zeros((4, 4, 3), dtype=np.float32)), + "class": Label("a", classes=["a", "b"])}] + stream = Stream(source=records, ops=[as_transform(lambda d: d + 1.0, handles=(Image,))]) + + out = list(stream) + assert len(out) == 1, out + assert float(out[0]["image"].max()) == 1.0 + assert out[0]["class"].value == "a" + assert len(stream) == 1 and stream[0]["class"].value == "a" # map-style protocol + print("wheel smoke-test OK, version:", version("recordstream")) + EOF + - uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/ + + publish: + name: Publish to PyPI + needs: [build] + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write # required for PyPI Trusted Publishing (OIDC) + steps: + - uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + - name: Publish + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.gitignore b/.gitignore index dae5208..81bde28 100644 --- a/.gitignore +++ b/.gitignore @@ -42,5 +42,13 @@ dir_store/ flake8.txt mypy.txt coverage.xml -test-report.xml +*test-report.xml coverage/ +/pyrightconfig.json + +# Run output — every run (training, evaluation, prediction, sweep cell) writes here. +# ONE ignored root per project: matrainer trackers default to `save_dir: ./runs`, and +# Lightning checkpoints follow the tracker's save_dir, so a whole run lands under one +# directory instead of scattering run-named trees into the repository root. +# See the workspace AGENTS.md mandate "Run Output Lands In A Gitignored `runs/`". +runs/ diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 8c67431..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "python.testing.pytestEnabled": true, - "python.testing.pytestArgs": [ - "tests" - ], - "python-envs.pythonProjects": [ - { - "path": ".", - "envManager": "ms-python.python:venv", - "packageManager": "ms-python.python:pip" - } - ], - "python.analysis.extraPaths": [ - "${workspaceFolder}/../logflow", - "${workspaceFolder}/../confluid", - "${workspaceFolder}/../liquify", - "${workspaceFolder}/../waivefront", - "${workspaceFolder}/../torpedo", - "${workspaceFolder}/../navigaitor", - "${workspaceFolder}/../aisland", - "${workspaceFolder}/../annotaide", - "${workspaceFolder}/../marainer", - "${workspaceFolder}/../waivefront-helios", - "${workspaceFolder}/../waivefront-rfuav" - ] -} diff --git a/AGENTS.md b/AGENTS.md index 6b9f4c1..05ec43b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,14 +1,67 @@ -# DataFlux Mandates +# RecordStream Mandates -- **Functional Purity:** Transforms are plain Python callables. Never introduce base classes or complex inheritance for data operations. -- **Sample Triplet:** All data flows through the `Sample(input, target, metadata)` NamedTuple. Never bypass metadata — full traceability is mandatory. -- **Lazy Evaluation:** Pipelines MUST remain lazy iterators until explicitly consumed. Never eagerly materialize entire datasets. +## Current state + +> **Renamed 2026-07-26 — `sampleflux` → `recordstream`, `Flux` → `Stream`.** The package was named +> for a data model it no longer has: the 2026-07-25 migration made the carrier a **record**, so the +> vocabulary is now one word per concept — a **`Stream`** of **`Record`**s. Import name, distribution +> name, GitHub repo, console script (`recordstream run`), every `recordstream-*` entry point, the +> `RECORDSTREAM_*` StreamStudio socket types, and the on-disk root attr (`recordstream_format`, +> value still `typedrecord-v1`) all moved together; `JointFlux.fluxes` is `JointStream.streams`. +> **No back-compat aliases** — a pre-rename config, saved canvas, or store must be re-pointed +> (a store missing `recordstream_format` raises the usual re-generate error). The word *sample* is +> now reserved for its OTHER meanings and was deliberately NOT renamed: a discrete-time signal +> sample (`samplerate`, `window_samples`, `num_iq_samples`), a stochastic draw (`Transform` still +> *samples* its params once per record), and external APIs (`sample_id` is LabelStudio's task key). + +Core engine feature-complete on the **record model**; the full surface (items · type dispatch · op families · collate · storage) is pinned by the mandates below. Shape in one pass: sources → ONE step-graph engine behind two facades (`Stream`/`JointStream` for the dataset surface, `FlowGraph` for a `flow:` document) → ops (native `Transform`s + bare library transforms) → storage sinks, with the runnable layer (`recordstream run`, `entrypoint` markers, `Sequence`/`Conditional`/`Switch`, `DatasetProcessor`) on top. Gotchas not covered by a mandate below: `ToTensor` emits a LIVE CHW-float `torch.Tensor` as a PLAIN record value (an `NDArrayItem` coerces through `np.asarray` and cannot hold one); `FormulaOp`'s sandbox adds the array reducers `amax`/`amin`/`mean`/`std`/`median`, function style; `Switch`'s knob is `select`; `HuggingFaceSource` yields keys `image`/`class` (+ metadata columns, default `"*"`); a `flow:` step carrying `bind:` MUST use the plain-mapping (`op:`) form — a nested mapping under a `!class:` marker is consumed by confluid as addressed config; the context ops + the flow⇄ops lowering pass were DELETED 2026-07-30 (one step-graph engine, see the mandate below). Executed proofs: `examples/record_pipeline.py` / `workflow_pipeline.py` / `storage_roundtrip.py`. + +- **The Runnable Protocol Lives Here (`recordstream.runnable` / `.workflow` / `.processing` / `.cli`, 2026-07):** recordstream owns the carrier-agnostic *runnable* layer (moved out of matrainer when matrainer became a pure tracking library). `recordstream.runnable`: the `TorchRunner` autograd marker + `ProgressReporting` progress-callback mixins (a GUI executor reads the duck-typed `__needs_autograd__` / `set_progress_callback`) — **the mixin keeps the framework name while its FLAG is named for what it decides (`__needs_autograd__`, renamed from `__torch_runner__` 2026-07-29, NO alias): autograd is a torch concept, but "is this a torch runner?" answered the wrong question at the one place it is read, and the merged runnables override it as a per-task property (`return self.task == "fit"`) that only reads correctly under the new name. It is a duck-typed CROSS-PACKAGE contract whose only reader is a GUI executor, and the read fails OPEN (`getattr(..., False)`) — so a renamed flag with an un-updated reader silently runs training under `inference_mode` until `loss.backward()` dies; change the two together or not at all**, AND the **`entrypoint(task, role, primary)` method-annotation mechanism** + `runnable_entrypoints(cls)` / `entrypoint_tasks(cls, role)` introspectors — a class that drives several capabilities off one `task` knob (the consumers' merged train+eval classes: `fit`/`evaluate`/`test`/`predict`) annotates each method with its `task` value and a `role` (`trainer`/`evaluator`/`predictor`), so a discovery consumer (navigaitor's config generator, a visual editor) learns from ONE class that it both trains and evaluates. **A merged runnable's `run()` MUST dispatch through `run_entrypoint(self, self.task)` — NEVER a hand-written `{task: method}` dict (2026-07-29):** the markers ARE the dispatch table (`run_entrypoint` builds `{declared task: method}` from `runnable_entrypoints(type(runnable))`, calls the match, and raises `ValueError` listing the declared tasks in DECLARATION order). A dict restates the same mapping a second time and the copies drift in the direction that bites: navigaitor pins `task:` from `entrypoint_tasks` (the markers), so a capability added to the markers and forgotten in the dict emits a GENERATED config that dies at dispatch with "unknown task" while discovery advertises it as supported — and nothing can test for it, because the dict is derived from nothing. Adding a capability is therefore ONE edit (decorate the method + extend the runnable's `task` Literal). **The four STANDARD tasks are the closed `RunnableTask = Literal["fit", "evaluate", "test", "predict"]` declared in this module (2026-08-06)** — a consumer's task alias is `MyTask = RunnableTask` so the set is written once; a runnable with a narrower capability set declares its own Literal (the type names the convention, the markers gate dispatch). Consequence to accept: the markers are now load-bearing at RUNTIME, so dropping an `@entrypoint` breaks the run instead of only emptying a picker. The lookup reads markers off raw function objects (`vars()`), so a dynamic `__needs_autograd__` property never fires during dispatch. The three merged consumer runnables (classification / segmentation / detection) all carried the identical five-line dict before this landed — do not reintroduce it. Rationale: `docs/architecture.md` §7. Pins: `tests/test_entrypoint.py` (dispatch, declaration-order error, subclass override, the added-capability regression, the property-getter guard). `recordstream.workflow`: the `Sequence`/`Conditional`/`Switch` combinators + `PathExists`/`Not`/`AllOf`/`AnyOf` predicates (a workflow is one Confluid document of `@configurable` runnables). `recordstream.processing`: `DatasetProcessor` (generic source→sink runner; zero-arg constructible, `stream` validated in `run()`). `recordstream.cli`: the `recordstream run ` liquifai app that binds the top-level `runnable:` key and calls `.run()` — the ONE runner for every kind of run (train/evaluate/predict/process); the matrainer CLI + per-verb vocabulary are retired. **A runner MUST build the bound node with `materialize_runnable()`, never a bare `flow()` (2026-07-29):** broadcasting (a top-level YAML key injecting into the same-named ctor param) only happens when a Fluid is built AGAINST its document. Liquifai's DI does that only for a command parameter annotated with a **configurable class** (`di.py` materializes the block with `context=`); a generic runner annotates `runnable: Any` — because the runnable is polymorphic — so DI hands over the raw Fluid and deep-flows it with NO document, and every top-level sibling is dropped SILENTLY (`train_set` -> `None`, `max_epochs: 3` -> the ctor default, the run proceeding as if configured). `materialize_runnable(node)` reaches the document back through `liquifai.context.get_context().config_data` and calls `materialize(node, context=document)`, falling back to `flow()` when there is no context or the root is a single `!class:` document (no siblings to lose). The verb commands therefore use `flow_mode="manual"` — liquifai's `"auto"` deep-flow is exactly the bare flow this replaces. This regressed when the workspace moved from per-verb CLIs (`def train(trainer: LightningTrainer)` — a configurable annotation, so DI broadcast) to ONE polymorphic runner; the example-config tests missed it because they load with `confluid.load(text, flow=True)`, which broadcasts by a different route. Consumers shipping their own CLI (`sonair train`) MUST call the same helper — do not re-derive it. Pins: `tests/test_cli_materialize.py` (incl. the executed bare-flow counterfactual). All exported at the package top level; entry-pointed `recordstream-processing`/`recordstream-workflow` + the `recordstream` console script + `liquifai.apps`. +- **RecordStream Is MODALITY-NEUTRAL — Signal-Domain Code Lives in waivefront (2026-07-18):** Every op/source/sink in this package MUST be meaningful for ANY modality (arrays, tensors, images, generic metadata). The signal-domain residents were MOVED OUT: the 1-D FFT family `FourierOp`/`InverseFourierOp`/`FftShiftOp`/`IfftShiftOp` + the calibration ops `WindowOp`/`SpectrumScalingOp` (numpy + torch variants) are now `waivefront.fourier` / `waivefront.fourier_torch`, the window/unit math module `windows.py` is `waivefront.windows`, the SigMF recording pair is `waivefront.sigmf`, and the annotation-join source `paired.py` is `waivefront.paired` (temporary home — flagged for redesign in root TASKS.md). When adding an op here, ask: does it make sense for an image dataset AND a waveform dataset AND a tabular one? If not, it belongs in the domain package. The engine's own docs (README, docs/*.md) stay UI-neutral as well — describe visual editors generically, never a specific GUI product (the UI/engine separation is deliberate). +- **RecordStream Is FRAMEWORK-NEUTRAL TOO — torch Is an EXTRA (2026-07-30):** The core engine is **numpy**; `torch` moved out of `dependencies` into `[project.optional-dependencies] torch`, so `import recordstream` pulls NO ML framework (measured). This is the framework axis of the modality-neutrality rule above: a Keras-only, TensorFlow-only or plain-numpy consumer was installing ~2GB it never called, and matrainer inherited it transitively. **`Stream` and `FlowGraph` no longer subclass `torch.utils.data.Dataset`** — they satisfy the `MapStyle` Protocol (`__len__` + `__getitem__`), which is ALL a `DataLoader` needs (it duck-types its argument; verified against a plain object). Nothing in the workspace does `isinstance(x, Dataset)` or subclasses `Stream`, so the base bought nothing but the dependency. Consequence to accept: torch's STUB still declares `Dataset[T]`, so a `DataLoader(stream)` call in TYPE-CHECKED code needs `cast(Any, stream)` — that is a stub's stricter view of a runtime contract that works, and the bridge belongs at the call site, never by re-coupling the engine. **`MapStyle` must be referenced as the real class, never a string forward-ref**, in any annotation a consumer might introspect: confluid evaluates annotations in the CONSUMER's namespace, so `"MapStyle"` in `RecordSource` raised `NameError` from a consumer's `__init__` scan. **Recognising a framework value never imports one** — `recordstream._compat.is_torch_tensor` consults `sys.modules` (a torch tensor cannot exist unless torch is already imported, so the check is exact, not a heuristic), the same instinct as the op-family MRO matchers. What legitimately needs torch: `recordstream.ops.torch.ToTensor` (lazily exported from `recordstream.ops` via a module `__getattr__`, raising an `ImportError` naming the extra) and `outputs.py`'s `classification_output` / `segmentation_output` builders (function-body imports; their `TypedDict`s stay module-level because they are typing-only and generic in the array type). Everything else returns numpy ON PURPOSE — see the `recordstream.batch` and class-balance mandates. **When adding code here, ask the framework question alongside the modality one:** does this work on a numpy-only install? If not, it goes behind the extra with a lazy import, never at module level. Verified by resolving each install shape into a clean set (bare -> no framework; `[torch]` -> torch). **Workspace-wide the extra is selected by `aisland framework`** — this project declares `[tool.aisland] frameworks = ["torch", "keras"]`, so `aisland framework set torch keras` installs `recordstream[dev,torch,keras]` and a selection without either installs `[dev]`; the same committed selection is what generated CI installs (`aisland jenkins scaffold recordstream --force` after changing the declaration — never hand-edit the three artifacts). Pins: `tests/test_optional_torch.py`. +- **The FRAMEWORK's Half Of Batching Lives Here Too — `recordstream.keras.RecordSequence` (2026-07-30):** Batching has two halves: WHAT a batch contains (`collate_records`) and WHICH ROWS go in which batch (order, slicing, short final batch, per-epoch reshuffle). torch gives the second half away — a `DataLoader` duck-types any `MapStyle` source and takes `collate_fn=collate_records` — so this package shipped only half a pair and the gap was invisible. **Keras 3 has no `DataLoader`** (`keras.utils.PyDataset.__getitem__` must return a whole BATCH), so that loop is `recordstream.keras.RecordSequence`, and the split is drawn exactly where torch draws it: **`transform` IS the `collate_fn` equivalent** — and since 2026-08-05 the COLLATE ITSELF is selectable here too (`collate=` takes a registered key or a function, resolved per batch so a key registered later still works), because a `PyDataset` otherwise had no way to say "don't stack" and the batch-shape choice was torch-only — a callable mapping one collated record to what the model consumes — so a task's batch SHAPE never enters this module (the first consumer had written the whole adapter in its training project, where ~60% of the lines mentioned nothing about its task while its torch twin was one `LazyClass(DataLoader, collate_fn=collate_records)` line). No `transform` = the batched record itself, which is also what `batches()` yields (the pairing half of prediction: a model emits `[N, ...]`, a `PredictionsSink` writes per record). **The module ALSO owns the `KERAS_BACKEND` ordering, and that is why it is a module and not a loose class:** Keras 3 reads the var at IMPORT time and defaults to `tensorflow`, which `recordstream[keras]` does not install (Keras is an API; the engine is the operator's choice), so a bare `import keras` dies with `ModuleNotFoundError: No module named 'tensorflow'` from inside `keras.src.tree.optree_impl` — verified in this venv. `os.environ.setdefault` to `_first_installed_backend()` (a `find_spec` probe, so nothing is imported just to look) must run in the LOWEST layer that imports keras, because import sorters put a library import ABOVE a first-party one: a consumer's own shim sorts BELOW `from recordstream.keras import RecordSequence` and loses the race. So **every consumer imports keras THROUGH `recordstream.keras`**; a project keeping its own shim re-exports from here. THREE invariants: (1) **`RecordSequence` is deliberately ABSENT from the package root** — `inspect.getmembers` (what `discovery.scan_module` and the GUI bridges call) getattrs every advertised name, so a PEP 562 root export (the `ops.ToTensor` pattern) would import keras on every discovery scan of a torch-only install; the import path IS the boundary marker; (2) it is **NOT `@configurable` and carries NO `category`** — engine plumbing a runnable builds in code, like `collate_records`; tagging it would put a keras import in the registry scan for a class no YAML wires; (3) the row order is a **lazy `@property`**, not constructor state — `len(source)` is real work for a deferred source (a `HuggingFaceSource` LOADS to answer it), so `RecordSequence()` builds zero-arg and a missing `source` is reported by `indices`. The extra names NO compute engine (`keras = ["keras>=3.0"]`). Rationale: `docs/architecture.md` §10. Usage: `docs/kinds.md`. Pins: `tests/test_keras_sequence.py` (task-free by design — a test there mentioning classes or `(x, y)` means the task leaked back in). +- **Op Consolidation (2026-07-18, updated 2026-07-25) — ONE Wiring Plane, No Twins:** `Tee` and `CaptureOutputOp` were DELETED (no aliases), and in the record migration `TransformChain` was DELETED too — **`Pipeline`** (`recordstream.transform`, `category="op"`, `group="compose"`) is THE sequential grouping op (an ordered `transforms` list appearing as one named config block / one canvas node; None-propagation, lazy marker flow, `close()` propagation; pins: `tests/test_pipeline.py`). Use `Pipeline` for grouping and the context ops (`Save`/`Use`/`MergeFields`) for real, isolated fan-out/fan-in. `CaptureOutputOp`'s job (record a live `@output`) is the context op `Capture`; the read-back idiom is `Apply(source=cell)`. `ConfigureOp` STAYS — its derive-the-value-FROM-the-record side-branch (`ops` compute chain → the `source`-keyed entry of the side-branch result → `setattr(target, param, value)` → apply `target` to the ORIGINAL record) is one node where cells need four, and it is the compiler target for canvas value chains; prefer `Apply` when the value already lives in a cell. Graph exporters emit ONLY context ops for wiring. Do NOT reintroduce a metadata-bus twin of a context op. +- **Functional Purity:** Transform kernels are plain Python callables — a `Transform` dispatches on value TYPE to a registered kernel function (`@Transform.kernel(ItemType)` / `register_kernel`), and ops in `recordstream.ops` stay plain callables (`record -> Optional[Record]`). The `Transform` base is a thin type-dispatch shell (it samples params once per record via `get_params`, then applies the per-type kernel to each handled value), NOT a deep inheritance hierarchy for data operations. Adding a data operation means registering a kernel or writing a callable op — never subclassing a behaviour-bearing base. +- **The RECORD Is THE Data Model (2026-07-25):** A record is a **PLAIN `dict`** — `recordstream.items.Record = Dict[str, Any]` — of **TYPED values**, each value owning its own metadata (an `Image` its `layout`, a `Label` its `classes`, a `Boxes` its `canvas`/`extras`). There is NO container class, NO `Sample`, NO role tags, NO `primary()` accessor, and NO `recordstream.bag` package — **key names carry meaning** (`"image"`, `"mask"`, `"bboxes"`, `"labels"`, `"class"` — the albumentations/torch-batch convention), and scalar side values are just more dict keys (`{"samplerate": 30.72e6}`). Items are HYBRID: array-backed items subclass `NDArrayItem` (an `np.ndarray` subclass whose declared `_item_attrs` survive numpy ops via `__array_finalize__` — `Image`/`Mask`); structured items are dataclass wrappers (`Boxes`/`Label`). **`Boxes` is PIXEL-ONLY (renamed from `Regions`, 2026-08-10, NO alias): half-open absolute-pixel `[x0, y0, x1, y1]` rows on the `(H, W)` `canvas` raster — the signal-domain time/frequency region item lives in the signal package and registers through the same `register_item` registry; a stored `typedrecord-v1` record carrying `__item_type__: "Regions"` fails loudly on decode and is re-generated (rationale: `docs/architecture.md` §14). `connected_component_boxes` (renamed WITH its contract from `connected_component_bboxes`) emits the SAME half-open xyxy order, `ConnectedComponents` fills `canvas` (empty masks included), and the geometry guards now correctly stay silent for a domain package's raster-independent region item.** `item_data`/`with_data` are the uniform payload accessors (kernels never special-case subclass vs wrapper) and **`item_value` is the one step further out — the SEMANTIC value whatever wrapper carried it (2026-08-02)**: a `Label`'s payload slot is `value`, not `data`, so `item_data(Label("cat"))` returns the `Label` ITSELF and a caller wanting the class id gets a 0-d object array. The rule (`MultiLabel` -> `.values`, `Label` -> `.value`, any other item -> `item_data`, a plain value verbatim) had been written out THREE times before it was extracted — `iter_key` (per record), `batch_values` (per batch) and `ops.image.ConvertToMask` (per field, the copy that prompted the extraction) — and `batch_values`'s docstring still claimed to be "the one place that knows how to get past a wrapper item" while two others did the same. Both former copies now delegate; each keeps only what is genuinely its own (the projection / that the values arrive collated). Use `item_data` inside a KERNEL, where the item type is already known and a `Label` cannot arrive; use `item_value` at a boundary that reads a CONFIGURED key, where a source may legitimately have wrapped anything. Do not re-derive the branch a fourth time; **`resolve_item`/`resolve_entry` are the same rule for the KEY question (2026-08-10)** — a `field=`-style op resolves its entry through them (explicit field must exist + match the item type with a `ValueError` naming owner/param/keys; blank field = first-of-type; `fallback=False` forbids the blank-field scan, `required=False` returns `None` — extracted from twelve per-class `_find_*` copies in one consumer package), never a hand-written find loop; `register_item`/`is_item`/`item_types`/`get_item_type` are the open item registry (a domain package adds one class + one decorator, no core edit; items are NOT confluid-`@configurable` — an ndarray subclass builds through `__new__`, which fights the `__init__` validation wrap). Ops are type-dispatched `Transform`s (`recordstream.transform`): `get_params(record)` samples shared parameters ONCE per record, then the per-type kernel (`@MyOp.kernel(ItemType)`, MRO-aware registry in `recordstream.dispatch`) applies to every handled value — unhandled values pass through; the `field=` ctor param pins an op to ONE named key (still type-gated). **Two sanctioned op shapes:** (1) same-type per-value edits register kernels; (2) type-CHANGING ops (read one key, write a differently-typed item — `Threshold`: array→`Mask`, `ConvertToImage`: array→`Image`, the target ops) subclass `Transform` and override `__call__`, declaring `handles`/`consumes`/`produces` truthfully as graph metadata. External libraries run **AS-IS** through the engine's op-family dispatch (mandate below) — there are NO adapter/wrapper classes and NO generated per-transform families. Import the whole surface from the PACKAGE TOP LEVEL (`from recordstream import Record, Image, Mask, Boxes, Label, Transform, Pipeline, as_transform, item_data, with_data, register_item, register_kernel, register_io, collate_records, ...`). recordstream ships ONLY generic items (`Image`/`Mask`/`Boxes`/`Label`) and **NO native augmentation ops** — domain items (a signal, a spectrogram) live in the domain package and register into the SAME registries. Serialization goes through the codec `recordstream/io.py` (`encode_item`/`decode_item`/`encode_record`/`decode_record`; a non-item value rides the `"plain"` type tag verbatim; `register_io` overrides per exact type). Usage: `docs/record-model.md`; rationale: `docs/architecture.md` → "One type-dispatched op engine"; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_items.py` / `tests/test_transform.py` / `tests/test_dispatch.py` / `tests/test_io.py`. Follow-ups (root TASKS.md): a torch-`Tensor`-subclass item base (torch payloads ride wrapper items for now), confluid-native item discovery. +- **Libraries Run AS-IS — the Op-Family Dispatch (`core._apply_op`, 2026-07-25):** `recordstream.core.families._apply_op(record, op)` is the engine's SINGLE op-application chokepoint, and it dispatches on the op's FAMILY (by MRO module name — `_is_albumentations` / `_is_torchvision_v2`, no eager library import), invoking each family the way its own library expects. THREE branches: (1) **albumentations** — the op receives EXACTLY its own kwarg vocabulary: the `_ALB_KEYS` (`image`/`mask`/`masks`/`bboxes`/`keypoints`/`labels`) present in the record, nothing else, so extra entries (scalars, domain items) never reach a library that would reject them; ONE call = ONE joint draw across those keys; array outputs are RE-WRAPPED in the incoming value's `NDArrayItem` type via `with_data` so `Image`/`Mask` types+metadata survive; zero known keys → debug log + pass-through. Box-carrying augmentation is a bare `A.Compose([...], bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"]))` dropped into the ops list — format handling is Compose's job in that library; seeding is the libraries' own mechanisms (`A.Compose(seed=...)` / `torch.manual_seed`). (2) **torchvision `transforms.v2`** — called on the record dict AS-IS (tv2 walks dicts natively, samples params once, transforms tensor/tv_tensor/PIL leaves and passes the rest through); layout conversions are the library's OWN transforms (`v2.ToImage()`), the engine NEVER converts silently. (3) **everything else** — `op(record)`, a native/wiring op `record -> Optional[Record]` where `None` = drop (filter semantics). **The families are an OPEN REGISTRY (2026-07-25): `register_op_family(name, matcher, invoker)`** (package-root export; `registered_op_families()` introspects) — the built-ins register through the SAME API at import (no privileged path), dispatch checks LAST-registered first (a more specific family shadows an earlier one), re-registering a name replaces in place, and matcher/invoker MUST be module-level functions (the spawn routes pickle them by reference and re-register inside workers via `_sync_op_families` — `_iter_parallel` and `Parallel.stream` pass `_extra_op_families()` along). NEVER add a wrapper/adapter class for a library — supporting a NEW library family (kornia, DALI, a fork) is ONE `register_op_family` call from ANY package (an MRO module-name matcher + the library's native calling convention), nothing else; a library convention that needs per-op config is a normal `Transform` op instead. In YAML, bare library transforms drop directly into `ops:` lists as `!class:albumentations.HorizontalFlip {p: 0.5}` (mapping form works — `Stream._check_ops_materialized` flows deferred markers at route entry; composing ops flow lazily too). **A geometry-changing transform WARNS when a `Boxes` sat out the call (2026-08-06):** the vocabulary rule above is what makes a bare library transform work unmodified, but a detection target rides as a `Boxes` item under a key of the pipeline's choosing — so it is not in that vocabulary, is not passed, and does not move. Measured: a bare `A.Resize` takes a 200x200 image to 64x64 and leaves the boxes on `[10, 10, 100, 100]`, and a bare `A.HorizontalFlip` mirrors the pixels while changing NO shape at all. Nothing errors either way — the shapes stay valid and only the coordinates become wrong. **The condition is the LIBRARY'S OWN taxonomy, not a raster comparison**: a `DualTransform` is by definition one that applies to boxes, an `ImageOnlyTransform` cannot touch geometry, so `_has_spatial_transform` matches `DualTransform` by MRO class NAME (import-free, like `_is_albumentations`) and recurses a `Compose` through `.transforms` — the flip case proves why a "did the size change" test is not enough, and reading the taxonomy means no list of transform names to drift. WARNING not error (a record may legitimately carry regions describing something else), once per transform TYPE (`_WARNED_SPATIAL` — the message is about the configuration), and silent when `bboxes` WAS passed. The message names both ways out: `bbox_params` on a `Compose`, or `ops.target.ResizeDetection` for a plain coupled resize. Docs: `docs/augmentation.md`; runnable proof: `examples/record_pipeline.py`; pins: `tests/test_op_families.py` (incl. `TestGeometryLeavingBoxesBehind`, which asserts the desync PREMISE before the guard). **The v2 family has the SAME gap by the other route and the same guard (2026-08-06):** v2 walks the record natively but transforms only its own `tv_tensors` TYPES, so a `Boxes` is passed through untouched — measured, `v2.Resize((64,64))` takes a 200x200 image to 64x64 with the boxes still on `[10,10,100,100]`, while the same transform over a `tv_tensors.BoundingBoxes` gives `[3.2, 3.2, 32, 32]`. `_is_v2_geometry` matches v2's private `torchvision.transforms.v2._geometry` module in the MRO (the library offers no public marker) and shares `_WARNED_SPATIAL` with the albumentations guard. **The behavioural test — apply the transform to a throwaway `BoundingBoxes` and see if they move — is deliberately NOT used: it would draw from the RNG and change the augmentation stream of the run being diagnosed.** The private path can go stale on a torchvision upgrade; it FAILS OPEN (no warning, nothing else changes), so `test_the_geometry_signal_still_matches_this_torchvision` asserts the CLASSIFICATION directly rather than only through a warning that would silently stop appearing (native + bare-albumentations + bare-tv2 in ONE ops list, re-wrap, YAML mapping form, spawn-parallel, and the registry: third-party family dispatch, last-wins shadowing, in-place replacement, spawn worker propagation). +- **OpenCV's Thread Pool Is The SECOND Fork Hazard, And The Guard Fires On USE (`core.families._disable_cv2_threading`, 2026-08-02):** the companion to the `ensure_materialized` mandate below, and **independent of it — neither fixes the other**. albumentations runs on OpenCV, whose thread pool is not fork-safe: once the PARENT has executed a cv2 op, a forked `DataLoader` worker inheriting that pool dies with a **SIGSEGV and no Python traceback**, surfacing only as `DataLoader worker exited unexpectedly` — the same symptom as the lazy-source hazard, which is exactly why they get confused for each other. Measured 2026-08-02 with `DataLoader(num_workers=2)` over a `Stream` whose ops list held an `A.Resize`: **with the source warmed and the pool ON the worker still SIGSEGVs, and with the pool off a cold source is still built in the child.** A consumer that forks needs BOTH guards. `cv2.setNumThreads(0)` is fired ONCE at the top of `_invoke_albumentations` — the one place this package INVOKES albumentations — and never at import: a process that never uses albumentations must not have its OpenCV settings changed by importing a data library, and cv2 must not become an import-time dependency of the engine. It costs nothing where it matters, because inside a worker the WORKER is the parallelism and cv2's own threads oversubscribe rather than help (albumentations' own docs recommend exactly this for multiprocessing loaders). Two cv2 quirks a test must not get wrong, both measured: `setNumThreads(0)` makes `getNumThreads()` report **1**, not 0; and `setNumThreads(4)` does not change what it reports at all. **Why the classification consumer never hit this and the segmentation one did:** classification resizes with `ConvertToImage` (PIL), while a per-pixel task must resize the image and its mask in ONE JOINT DRAW — which only a bare albumentations transform does. Segmentation is the first thing in the workspace to put cv2 on the worker path. Pins: `tests/test_fork_safety.py` (the forked-worker subprocess reproduction is the one that matters — removing the guard makes it FAIL, not pass differently). +- **Metadata Lives on the Value That Owns It — or as a Plain Record Key:** There is no per-record flat metadata dict object. Metadata is EITHER an attribute of the typed value it describes (an `Image` knows its `layout`, a `Boxes` its `canvas` + per-box `extras`, a `Label` its `classes` — carried by `_item_attrs`/dataclass fields, serialized per key) OR simply another record key when it describes the whole record (`record["samplerate"] = 30.72e6` — the `"plain"` codec tag stores/queries it). Read a value's metadata off the value (`record["image"].layout`), never from a side dict. **A `Boxes`' `canvas` is the load-bearing case of that rule, and EVERY op that makes or re-frames one fills it in (2026-08-06):** a box means nothing without the raster it is stated in, so `CocoToTorchVisionDetection` records the frame of the image its annotation describes, `MasksToDetectionBoxes` the shape of the mask its boxes were derived from (exact and free), and `ResizeDetection` the size it resized to — **including for an EMPTY target**, because a check that silently skips exactly the records with nothing to check reports a clean bill for the wrong reason. Previously ONLY the resize set it, so the frame was known exactly where it was least needed and unknown in the chain where boxes and pixels actually drift apart. The lookup (`ops.target._source_frame`) is deliberately NARROW — the `"image"` key, then the first `Image` item, then `None` — because a generic "first 2-D array" search reads a `Boxes`' own `[N, 4]` box array as an `N x 4` raster and records a confident lie (pinned). `None` stays an ordinary answer: nothing depends on the lookup succeeding. **The desync this makes detectable is otherwise SILENT:** an image-only resize (`ConvertToImage` and friends) moves pixels without moving boxes, every downstream SHAPE stays valid, only the coordinates are wrong, and a model trains against misplaced targets reporting nothing — so `ConvertToImage` WARNS once per op instance when it resizes a record carrying a `Boxes`, naming `ResizeDetection`. Once per INSTANCE, not per record: the message is about the configuration, and a copy per record only buries it. **`ops.image.image_frame(value)` is the shared "what raster is this" read** (PIL `.size` transposed, an `Image` item's DECLARED `layout`, else HWC) — it does NOT sniff a channel axis, because this package already carries three deliberately divergent channels-first heuristics and a guessing fourth would mislabel the very frame box coordinates are validated against. Consumer note: a downstream frame check that skipped on `canvas is None` now fires in chains it used to pass (verified on a real COCO-style set — the recorded frame matches the dataset's own `width`/`height` columns). Pins: `tests/test_convert_to_mask.py::TestBoxesKnowTheirFrame` (incl. the box-array-is-not-a-raster case and the once-per-op warning). NOTE for any test asserting on that warning: loggair is loguru, so `caplog` stays EMPTY, and its sink is ENQUEUED so `capfd` alone races it — add a sink and call `logger.complete()` (the mandated flush, never a sleep). Batching is `collate_records` (the registry's `"record"` default): payloads stacked per key via the codec, each item's declared attrs collected into per-record LISTS, plain values gathered into plain lists — the ONE batch convention. +- **Full Traceability Rides the Record:** Provenance is never dropped — everything that describes a value lives on the item that owns it or as its own record key. In service of this, `HuggingFaceSource.metadata_features` accepts the sentinel `METADATA_ALL_FEATURES = "*"` (bare or `["*"]`, and now the DEFAULT) meaning "every dataset column except `input_feature`/`target_feature`", resolved against the loaded dataset's `column_names` by the pure helper `_resolve_metadata_features` **lazily** (via the read-only `HuggingFaceSource.resolved_metadata_features` property — the `"*"` expansion needs the loaded columns, so it cannot happen in the lazy constructor); `None`/`[]` = no extra columns. Keep `"*"` as the one sentinel (a visual editor's metadata picker offers it) — don't add parallel magic strings. +- **ONE Execution Model — the STEP GRAPH; `ops:` Is Its Linear Spelling (2026-07-30, supersedes "The Context Is the Graph Data Plane" AND "`flow:` Documents ⇄ Flat Op Lists"):** There is ONE engine and ONE per-record kernel (`recordstream.flow.execute.run_steps_multi`). Both authoring forms parse to the SAME `FlowStep` list: an `ops:` list compiles to POSITIONAL steps (`core.linear_steps` → `s0`, `s1`, … — names that never surface, because nothing in an `ops:` document can reference a step; positional, not op-class-keyed, so the same op twice is two steps) and a `flow:` document parses to author-named steps with explicit `from:`/`merge_from:`/`bind:` edges. `Stream` and `FlowGraph` are two FACADES over that one kernel — `Stream` keeps the full dataset surface (`__len__`/`__getitem__`/`to_sink`/`project`/`map`/`filter`/`batch`/`parallel`, `JointStream`), `FlowGraph` loads a `flow:` document. **The lowering pass is DELETED (no back-compat):** `to_ops`, `from_ops`, `Stream.from_flow_yaml`, `recordstream.context` and the six context ops (`Save`/`Use`/`Drop`/`Apply`/`Capture`/`MergeFields`) are gone, along with the flow⇄ops parity suite and the `recordstream-ops-context` entry point. They encoded dataflow as imperative mutation of a per-record cell store, which destroys the dependency structure every consumer wants back (a compiler's reverse-dependency pruning walks `inputs`; a lowered list has none) — the visual editor was literally flattening its canvas graph and then lifting it back for readability. **HARD INVARIANTS:** (1) fan-out/fan-in/cross-step values are step GRAMMAR, never ops — `from:` is the fork, `merge_from:` the union (listed order, last-write-wins), `bind:` the cross-step value (`step` = whole record, `step[key]` = one entry, `step.attr` = the step op's live `@output`, read through wrapper chains by `flow._read_output`); (2) a step's `from:` MUST name an EARLIER step — document order IS the schedule, so cycles are inexpressible; (3) branch isolation is the ENVIRONMENT's job — a fan-out read deep-copies, and each expansion branch gets its own shallow env copy; (4) a straight chain takes the env-free FAST PATH (`flow.is_linear` → `_run_linear`), which MUST yield results identical to the general path (measured: the naive port cost +33% on a 23-step chain, the fast path brought it to +8%, and with real ops the difference is unmeasurable); (5) a BRANCHY graph has NO flat spelling — `FlowGraph.to_stream()` raises, and a consumer's ops-export must raise pointing at its flow export rather than inventing one. Do NOT reintroduce a lowering pass or a context/cell plane: that is a second execution model wearing the first one's clothes. A future runtime needing a flattened schedule owns that pass over its OWN IR, downstream of the graph. Rationale: `docs/architecture.md` §3. Usage: `docs/graph.md`. Pins: `tests/test_typed_flow.py` (`TestOneExecutor` — the ops→steps compilation, both spellings agreeing, the identity graph, the fast-path gate; `TestExpandingSteps`; `TestNativeExecution` — incl. `test_there_is_no_lowering_pass_left_to_call`). +- **1→N Expanding Steps Fork the REMAINING Subgraph (2026-07-30, supersedes the flat-engine pending-queue rule):** An op carrying `EXPANDS = True` yields N children from one record; the remaining steps then run ONCE PER CHILD over that child's own shallow copy of the step environment (independent name→result maps, shared values), DEPTH-FIRST so sibling order matches the nested-loop intuition. An empty expansion or a `None` child drops that branch. This works in EVERY route — serial, spawn-parallel (the worker returns a LIST), and inside a `flow:` graph (the old `FlowGraph` raised `NotImplementedError` on an expanding step; that limit is gone). CONSEQUENCES: (1) `__len__`/`__getitem__` RAISE on `Stream` AND `FlowGraph` when any step op expands — the expanded index map is unknowable up front, so the pipeline is ITERABLE-ONLY (iterate, wrap in a torch IterableDataset, window at the SOURCE for random access, or `list(...)`); (2) `run_steps` (the strict 1→1 twin used for indexing) raises rather than silently dropping siblings. Pins: `tests/test_typed_flow.py::TestExpandingSteps`. +- **Lazy Evaluation:** Pipelines MUST remain lazy iterators until explicitly consumed. Never eagerly materialize entire datasets. **This extends to construction:** a Source / Op constructor MUST do NO functional work — no `load_dataset`, file open, or network call in `__init__`. Defer materialization to a read-only `@property` that loads on first access and caches in a private `_backing` field, and make the class **zero-arg constructible** (every param defaulted, a required-at-use value validated lazily in the property with a clear error — never in `__init__`). This is the workspace "Lazy Initialization & Zero-Arg Construction" convention (see confluid `AGENTS.md`); `HuggingFaceSource` is the reference (`HuggingFaceSource()` builds with no network; `.dataset` loads on first use, `.resolved_metadata_features` derives lazily), mirroring `DatasetSplit`'s cached `train`/`val`/`test` views. **EVERY recordstream `@configurable` obeys this — ops, engines, sources, AND storage sinks/sources:** every constructor param is defaulted (zero-arg construction always works); an **op** validates its config lazily in `__call__` (e.g. `Threshold` the at-least-one-bound rule — `threshold_array` raises when both bounds are `None` — `EncodeTarget` the non-empty mapping, `FilterOp` the missing predicate — there, not in `__init__`); a **view source** defers validation + the index/offset precompute to a cached `@property` (`RangeSource.indices`, `ConcatSource.offsets`, `DatasetSplit._validate`/`_view`); **storage** defers the file open to `.open()`. **A `source:` slot holding a still-deferred `!class:X` marker RAISES with guidance, never flows (2026-08-10, user decision):** the parens-less spelling is a CONFIG error (`!class:X()` is the fix), and `Stream._guard_live_source` + the view sources' shared `sources/base.py::_guard_live_source` answer it with the same actionable `TypeError` (naming the slot, the deferred target, and the parens fix — `core.stream._fluid_source_guidance`). Do NOT "fix" a `got Class` failure by flowing the slot — the 2026-07-29 flow-first convention covers the FREE FUNCTIONS (`project`/`dataset_uri`/`LabelMap.encode`) and the `ops:` list (`_check_ops_materialized`), not `source:` slots. Pins: `tests/test_view_sources_deferred.py`. `tests/test_lazy_construction.py` walks the package and asserts `Cls()` succeeds for every `@configurable` (so a new class that adds a required ctor arg or does work in `__init__` fails there). Closed-`Literal` params (e.g. `Threshold.low_op`) keep their pydantic-at-construction enforcement — that is type validation (Schema Enforcement), not functional work, and zero-arg still holds because the default is a valid member. +- **Transforms Dispatch on Value TYPE via Kernels (`@Transform.kernel(ItemType)` / `register_kernel`):** A `Transform` declares which value TYPES it handles by registering a per-type kernel; it samples its parameters ONCE per record (`get_params(record)`), then applies the matching kernel to EVERY record value whose type it handles, passing untouched values through. Because the parameters are sampled once and shared, multi-key consistency is automatic — one drawn decision moves every handled value together (the torchvision-v2 model). Dispatch is MRO-aware (`recordstream.dispatch`): a kernel registered for a base item type also serves its subclasses, and a subclass transform inherits its base's kernels until it overrides them (memoized, cache cleared on registration). The **`field=`** ctor param pins an op to ONE named key (still type-gated) — it replaced the old `only=` list. A plain function becomes an op via `as_transform(fn, handles=(ItemType,), field=key)` (→ `FunctionTransform`); a type-changing shape (read one key, write a differently-typed item) subclasses `Transform` and overrides `__call__` instead of registering a same-type kernel, declaring `consumes`/`produces` truthfully. There is NO `Pipeline`-level coercion and NO adapter registry — bare library transforms enter through the ENGINE's op-family dispatch (`core._apply_op`), not through `Transform`. +- **Composing Ops Route Inner Ops Through `core._apply_op` (2026-07-19, rewritten 2026-07-25):** Every op that wraps/applies OTHER ops — `Pipeline`, `RandomApply`, `Enable`, `Parallel` (inline fallback; the streamed route already used `_worker_task`), `ConfigureOp` (compute chain AND `target`), and the context ops `Apply` / `Capture` — MUST apply an inner op via `recordstream.core.families._apply_op(record, op)`, NEVER `op(record)` directly. The chokepoint IS the op-family dispatch: it is what lets a bare albumentations transform (kwarg-vocabulary call + re-wrap) or a bare torchvision-v2 transform (dict call) nest inside a gate/chain/toggle exactly as it would sit in a bare `ops:` list — a raw `op(record)` call would hand an albumentations transform a positional dict it cannot accept. `_apply_op` also propagates `None` (FilterOp drop semantics) — a composing op's `__call__` therefore returns `Optional[Record]`. Pins: `tests/test_op_families.py` / `tests/test_pipeline.py` (bare library ops nested in composing ops). +- **Every Knob a Front-End Must Set Is a DECLARED Parameter — No Dynamic-Attribute Config (2026-07-27):** A user-facing switch/knob MUST be a **declared constructor parameter** (defaulted, `Args:`-documented), optionally exposed as a **settable property** when it needs validation. NEVER make an undeclared, post-construction-setattr attribute the config surface: only the YAML loader has a channel for unrecognised keys, so such a knob is invisible to `to_pydantic` (schema/form/canvas generators build a node with no widget), unconstructible from Python or a generated tool call (the generated config model forbids extras — `ValidationError: Extra inputs are not permitted`), and *silently dropped* by liquifai's bare-broadcast override path (`confluid.accepts_key` returns False for it). Post-construction setattr stays the mechanism a CONFIG LAYER uses to inject a declared key — it is not a substitute for declaring one. **`Enable` is the reference implementation** (2026-07-27 redesign): its toggle is the declared `enabled: bool = True` (settable property, non-bool raises `TypeError`), instances are told apart by the declared `name` which scopes the CLI flag to `--.enabled`, and the retired dynamic-toggle form (ANY boolean attribute name becoming the flag, e.g. a bare `visualize: false`) now raises `ValueError` on first record with the replacement spelling in the message rather than being silently ignored. Rationale + the evidence that killed the old design: `docs/architecture.md` §6. Pins: `tests/test_enable.py` (`TestIntrospectionContract` asserts `to_pydantic` fields and `accepts_key`/`accepts_broadcast` for every key). +- **Collation Is a Pluggable Registry, And The BATCH SHAPE Is A CHOICE (`recordstream.collate`; the choice landed 2026-08-05):** Batching a list of record dicts into ONE batched record goes through the registry — `register_collate(key)` / `get_collate(key)` / `collate(items, key=None)`. **TWO collates ship, differing in ONE decision — whether array payloads are STACKED — because that decision belongs to the MODEL, not to the data:** `"record"` (the default, `collate_records`) stacks what can stack; `"list"` (`collate_list` = `collate_records(items, stack=False)`) stacks nothing and leaves every key a per-record list with ITEMS KEPT AS ITEMS (so an `Image`'s `layout` survives per record). Left implicit the shape is decided by ACCIDENT — a column stacks if it holds array items and stays a list if it holds PLAIN values, so `ToTensor` running or not silently decides whether a detector gets its `List[Tensor]`; the key lets a consumer DECLARE it (raidar's torch loaders pass `collate_fn=collate_list`, `RecordSequence` takes `collate=`). **Every read-back helper accepts BOTH shapes** (`batch_values` unwraps a list-of-items element-wise, `batch_boxes` reads a batched `Boxes` or a list of them, `batch_metadata` transposes either) — that is what makes the collate a free choice rather than a fork in every consumer, and it is pinned by a parametrized test over both keys. **A stack failure now EXPLAINS itself** (`_stack_or_explain`): it names the key, the differing shapes and the `"list"` way out, where the raw `ValueError: all input arrays must have the same shape` from inside numpy named none of the three — this closed the old TASKS item about `_stack`'s unreachable "else a list" promise, by making the fallback a DECLARED mode rather than a silent type change. **`register_collate` is signature-PRESERVING** (`TypeVar` bound to `CollateFn`, not a flat `-> CollateFn`), so registering a collate no longer erases its own parameters — that is what lets `collate_list` call `collate_records(items, stack=False)` and type-check. The default `collate_records` behaviour is unchanged: per key (homogeneous key sets required — a mismatch raises), typed values encode through the io codec, payloads stack via `_stack` (torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a LIST of per-record values (decoded back into ONE batched item of the same type), and a `"plain"` value batches as the plain list. **The registry is ENGINE-INTERNAL — only shape-generic, parameter-free collates register (decided 2026-08-06, closing root TASKS item 41 as option (b)):** a TASK's batch shape (a fastai `(x, y)` tuple, a keras array pair) is parameterized by task-decided state (input/target keys, int-id vs multi-hot, a channels-last transpose) that a registry string cannot carry, so a consumer passes its callable straight to the slot that takes one (`DataLoader(collate_fn=...)`, `RecordSequence(transform=...)`) and never registers it — measured before deciding: no workspace consumer ever had. `collate_records` / `collate` / `get_collate` / `register_collate` / `registered_collates` are package-root exports. **The READ-BACK half lives beside it (`recordstream.batch`, 2026-07-29)** — `batch_values` (past the wrapper item: a `Label` -> `.value`, a `MultiLabel` -> `.values`, else `item_data`), `batch_boxes(batch, key)` (a collated `Boxes` column transposed into per-record `{boxes, labels}` dicts — see the detection note below), `multi_hot(batch, key, num_classes, dtype="float32")` (a `MultiLabel` column as an `[N, C]` matrix; out-of-range ids IGNORED, an empty label set is a meaningful all-zero row), `batch_tensor(batch, key, device=None, dtype=None)` (stack / `as_tensor` / optional dtype / optional device), `batch_metadata(batch, exclude=(...))` (the collate's transpose: the remaining columns back into N per-record dicts, `None` when nothing remains, ragged truncates rather than raising) and `per_record_predictions(preds)` (the transpose's PREDICTION-side twin, 2026-08-06: a model's batched output — a list, a `{"predictions": [...]}` wrapper, or a batched mapping with agreeing lengths — sliced into ONE entry per record for the per-record sink contract; extracted from three byte-identical consumer copies). They are the INVERSE of the collate rules and belong here because a consumer re-deriving them is re-deriving the collate — they were duplicated in two consumer packages before the move. **Only `batch_tensor` is torch:** `batch_values` / `multi_hot` / `batch_metadata` return plain values or NUMPY so a non-torch backend reuses them and converts in one line (`torch.as_tensor` shares memory; `tf.convert_to_tensor` is the TF twin) — a torch-typed `multi_hot` would have forced a second implementation for the next backend. **`dtype` is a PARAMETER, not an opinion** — the same knob as `device`: recordstream never decides the contract, it honours the one the caller names (a classifier passes `torch.int64` because a dataset yielding int32 label tensors is legal and `CrossEntropyLoss` rejects it with *"expected target dtype to be Long or Byte, but got Int"*; a segmenter passes the same for its pixel-class mask). What stays task-side is only WHICH call a trainer makes — both consumers' `_batch_target` wrappers were deleted 2026-07-29 when `dtype=` landed. Package-root exports; pins: `tests/test_batch.py`. **A DETECTION batch needs NO consumer collate, and that was measured (2026-08-05):** `collate_records` already batches one correctly — a variable-size image column stays a per-image LIST (`ToTensor` emits a live tensor as a PLAIN value, and plain values are gathered, never stacked) and a variable-N `Boxes` keeps per-record COLUMNS (its `boxes`/`labels` are declared attrs, which the collate lists rather than stacking). So the only piece that was missing is the inverse, `batch_boxes`, which is why it lives here: a detection consumer was carrying ~90 lines of its own collate (a `DetectionBatchInput` container, a `Boxes`->`{boxes,labels}` unwrap, a metadata transpose) that re-derived exactly these rules, and it was deleted in favour of `collate_records` + `batch_boxes` + `batch_metadata`. `batch_boxes` is FRAMEWORK-FREE like its neighbours (torch stays torch, numpy stays numpy — a target's dtype/device is the caller's contract), omits a field the item left `None` (so a training target is exactly `{boxes, labels}`), and leaves `canvas`/`extras` on the batched item (per-image frame metadata and an open dict are not per-box columns). **A variable-size `Image` ITEM column is batched by the `"list"` collate** — the default still raises for it, deliberately and with an explanatory message, because a caller who asked for stacking should hear that it could not happen rather than silently receive a different type. - **Serialization Symmetry:** Every pipeline configuration MUST be serializable via **Confluid** manifests for full reproducibility. - **Passive Introspection:** Pipeline discovery MUST use the `discovery` module for automatic JSON manifest generation. Never require manual tool definitions. -- **Storage Protocols:** All storage backends MUST implement the `DataSource`/`DataSink` protocols. Never couple the core engine to a specific format. -- **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them: `Flux` / `JointFlux` are `category="dataset"` (generic, task-agnostic engines); `FilterOp` / `WrappedOp` are `category="op"`. Preserve these tags when adding or renaming the classes — `tests/test_categories.py` pins them, and dropping a tag silently empties the corresponding picker. -- **Type Specs Live in `dataflux.typespec`:** The flexible array/tensor type system (`SampleType`, `ArrayType` with per-axis `Dim` ranges + dtype families + framework tags, `PythonType`, `UnionType`, `MappingType`/`ListType`, `AnyType`) describes what flows through a `Sample`. Ops/sources declare their contract via the **class attributes** `ACCEPTS` / `PRODUCES` (each a `SampleType`; absent ⇒ `Any`, fully backward-compatible). This is **NOT** a Functional-Purity violation: `ACCEPTS`/`PRODUCES` are plain class attributes (or the `@typed(...)` decorator that sets them), never a base class or inheritance — transforms remain plain callables. Matching is asymmetric: `consumer.accepts(producer)` (strict, runtime) vs `compatible(...)` (permissive, edit-time/discovery — `Any`/unknown on either side passes). When you add an op with a real type contract, declare it AND assert in tests that `PRODUCES.accepts(infer_sample_type(real_output))` (the type analogue of Pipeline Parity). -- **Stored Type Is Derived, Never a 4th Field:** A `Sample`'s type is reported by `Sample.describe()` — it returns the type stored in the reserved metadata keys `__features__` (a `datasets.Features` dict, the standard concrete description) + `__spec__` (sidecar refinements Features can't express: framework/ranges/`Any`/`Union`), else infers from the live data. Attach one with `Sample.with_type(...)` (copy-on-write). The pipeline only *maintains* a stored type that is already present (refresh from an op's `PRODUCES`, or drop it when the op declares none) — default pipelines stamp nothing, so metadata stays byte-identical and serialization is untouched. Never add a `spec` field to the `Sample` NamedTuple. +- **A Source/Op's `Args:` Docstring Is Its GUI Documentation:** Every node-facing class (Source / Op) MUST document each `__init__` parameter in a Google-style `Args:` block. That block is the single source of per-parameter help: `confluid.parse_param_docs` parses it into StreamStudio widget tooltips AND navigaitor's pydantic `Field(description=...)` (form-spec / visual editor). Keep each param's description on ONE physical line. An undocumented param silently shows no tooltip/description in either GUI — `recordstream/tests/test_node_docs.py` pins full coverage for the node-facing classes. +- **Storage Protocols — the Record Key-Group Layout (`typedrecord-v1`, 2026-07-25):** All storage backends MUST implement the `DataSource`/`DataSink` protocols (`storage/base.py`). Never couple the core engine to a specific format. Every sink ships with a matching source that reads its layout back into record dicts — `HDF5Sink`↔`HDF5Source`, `ZarrGroupSink`↔`ZarrGroupSource`, `ZarrBatchSink`↔`ZarrBatchSource` (batch is input-only), `DirectorySink`↔`DirectorySource`. When you add a sink, add (or justify the absence of) its source in the same change. **The layout:** root attr `recordstream_format = "typedrecord-v1"`; per record one group (`sNNNNNN` in HDF5, `record_NNNNNN` in Zarr; insertion order in the `__field_order__` attr) holding one subgroup per KEY with the `__item_type__` attr + the item's plain attrs natively (queryable), the payload as the `data` dataset, and array-valued attrs as datasets under `attrs/`; a `"plain"` value stores an array payload as `data` and any scalar/structured payload under the **`PLAIN_VALUE`** (`value`) attr (JSON-marked when structured — `split_attrs`/`restore_attrs` tuple-tag so tuples SURVIVE). There is NO `__role__` — roles are gone with the record model. Backends serialize ONLY through the `recordstream/io.py` codec (`encode_item`/`decode_item`), so externally-registered item types round-trip with zero storage edits. **NO backward compatibility (explicit user decision, 2026-07-25):** a store whose tag is `typedsample-v1` (or missing) raises `ValueError` via `storage/base.py::require_record_format` telling the user to re-generate with a current sink — never add a legacy-layout read path. `ZarrBatchSink` appends the FIRST record entry's payload per row + a one-time item template (`__field__` key + type + attrs of the first record); `ZarrBatchSource` rebuilds single-key records per row. **Discovery plumbing:** the storage SINKS carry `category="sink"` so a visual editor surfaces them as sink nodes docking into a `DatasetProcessor` runnable's `sink` slot; the matching SOURCES stay bare `@configurable` with no `category` (YAML `!class:` nodes, not canvas nodes). Because `recordstream.storage.*` is NOT re-exported from the package root and `scan_module` does not recurse submodules, **each storage module is entry-pointed** under `[project.entry-points."confluid.configurables"]` (`recordstream-storage-hdf5`/`-zarr`/`-directory`/`-query`) — add one for any new storage module, then reinstall the editable (`aisland setup`, never `--reinstall`). **Tensor→array conversion is shared:** array sinks convert payloads to numpy via `to_numpy` (in `storage/base.py`) before writing — both HDF5 and Zarr need it (zarr-v3 `create_array` can't read a torch tensor's dtype). Use zarr-v3 `create_array(..., overwrite=True)`, never the deprecated `create_dataset`. **`read_record_group(group, slices=…)` is the PUBLIC HDF5 row decoder (2026-08-10)** — `HDF5Source` iterates through it, and the `slices` map ({record key → slice of that field's `data` dataset}) is an h5py PARTIAL read for consumers that window large stored rows (a domain windowing source slices one window out of a whole-capture payload without materializing the row); the sliced decode is pinned identical to full-read-then-slice by `tests/test_typed_storage.py::TestReadRecordGroupSlices`. Do not re-derive the row-decode walk outside this function. +- **Metadata Is QUERYABLE Without Array Loads (`recordstream.storage.query`, 2026-07-17):** `recordstream.storage.query` defines the `SupportsMetadataScan` Protocol (`iter_metadata() -> (key, meta)` — NEVER loads arrays; free-function scanners `scan_hdf5_metadata` / `scan_zarr_metadata` read the record layout's attrs/`.zattrs`, requiring the `typedrecord-v1` tag; the protocol is STRUCTURAL, so external storage sources implement it without importing this module) + `MetadataFilterSource` (`category="source"`): `where` (the FormulaOp restricted namespace with metadata keys as variables; missing key = non-match, malformed = loud failure) AND-composed with a programmatic `predicate`; matching indices cached lazily; protocol-less sources fall back to full-iteration filtering via **`record_metadata(record)`** — a live record's queryable metadata in the SAME nested `{key: {attr: value}}` shape the scans yield (attrs via the io codec; a `"plain"` scalar contributes `{"value": }` under `PLAIN_VALUE`). A `where` expression addresses nested attrs as `.` (`_AttrView`; a Python-keyword key name like `class` is unaddressable in an expression — use `predicate`); array-valued attrs appear as shape/dtype stubs (presence/shape testable without an array read). Entry point `recordstream-storage-query`. No index sidecar in v1 (TASKS.md). The SigMF recording pair MOVED to **`waivefront.sigmf`** (2026-07-18) — SigMF is a waveform format, not engine-neutral; recordstream keeps ZERO knowledge of it. +- **Key Projection (`recordstream.projection`):** A source MAY implement the `SupportsProjection` Protocol (`project(keys) -> Iterator[Record]`) to yield partial records restricted to the requested KEYS **without building unrequested values** (e.g. an image dataset reads only the label column for a class-count walk — no decode). It's a `Protocol`, not a base class, so it does NOT violate Functional Purity. The primitive is deliberately general — any subset of record keys (bare strings; the old closed `ProjectionField` role Literal is DELETED with the roles themselves). **`project` materializes a DEFERRED source first (2026-07-29)** — a `!class:` marker from a config flows before the walk, matching `LabelMap.encode`, so a consumer no longer writes `flow(source)` at every call site to compensate for the inconsistency (flowing a live object is a no-op). Consumers use the helpers `project(source, keys)` / `iter_key(source, key)` (a `Label` unwraps to `.value`, other items to `item_data`, plain values pass verbatim), which fall back to full iteration + key-filtering for sources that don't implement it; `Stream.project(keys)` is the engine's implementation (runs the op chain, keeps only the requested keys). `num_classes(source, key="class")` is built on this — it always walks the `key` values and returns `max(class_id)+1`. Keep `num_classes` a **free function**, never a `Stream` method: integer class-id semantics are classification-specific, and adding it to the task-agnostic engine would make every `Stream` look classification-capable to duck-typed consumers. **`first_value(source, key)` is the ONE-PEEK primitive beside them (2026-08-02)** — the first non-`None` value under `key`, or `None` when there is none. It answers what a column's values ARE without walking the set, and it belongs here rather than in any consumer because it is `iter_key` plus a `next()`: it inherits all three of that helper's properties (a projection-aware source never builds the values it does not ask for, a deferred source is materialized first, the walk is lazy so a normal source costs ONE record) and its unwrapping rules are what make the answer meaningful — a `MultiLabel` arrives as its `.values` LIST, so a sequence IS a multi-label column, decided by the item type rather than by guessing what a list might mean. The canonical call site pairs it with `is_class_id` to decide whether the targets need a `LabelMap` at all. It was extracted from EIGHT byte-identical private copies in one consumer's training backends (2026-08-02); a consumer re-deriving it is re-deriving `iter_key`'s contract. Pins: `tests/test_projection.py`. +- **A Label Is ALWAYS Mappable To Ids — `Label` / `MultiLabel` + `is_class_id` (2026-07-29):** recordstream ships BOTH label items: `Label` (one class) and `MultiLabel` (several, `values: List[Any]`), each with `classes` and an `is_encoded` property. **`is_class_id(value)` is the ONE rule** for "is this an encoded id or a class NAME?" — an integer in ANY framework (Python `int`, numpy integer, a 0-d integer array/tensor, unwrapped via the `.item()` protocol so no framework is imported), with `bool` EXCLUDED (an `int` subclass, so a flag wired to the target key would silently become class 1). **`LabelMap.to_ids(target)` is the invariant made executable:** it accepts a `Label`/`MultiLabel` item, a bare name/id, or a sequence, and passes ALREADY-ENCODED values through — so it works on an integer-target dataset with an EMPTY map, and a consumer never branches on "names or ids?". This exists because consumers were sniffing types themselves (sonair had a `detect_target_kind` + a `_target_to_int` with the bool guard); both are DELETED — dispatch on the item/rule, never re-derive the check. `EncodeTarget`/`DecodeTarget` handle both items (`handles/consumes/produces = (Label, MultiLabel)`) and `iter_key` unwraps a `MultiLabel` to its `.values` list. Pins: `tests/test_labels.py`. +- **`LabelMap` Is the *Fittable* Companion to `EncodeTarget` (`recordstream.labels`):** `EncodeTarget` / `DecodeTarget` (`recordstream.ops.target`) carry a mapping that is **pinned in config, NOT fitted** (so train/eval/predict share one ordering). `LabelMap` is the piece that *creates* such a pin: `LabelMap.fit(targets)` derives a deterministic name→id mapping (sorted-unique ordering; accepts `Label`/`MultiLabel` items, bare values, or sequences — a multi-label dataset fits from the same call), and `save`/`load` persist it in matrainer's **`class_names.json` format** (`{"class_names": [...], "num_classes": N}`) — so the file matrainer writes next to a checkpoint and `LabelMap.load(...)` are the *same* file (one source of truth; eval reloads the training ordering instead of refitting on a subset). It exposes `num_classes` / `class_names` (id→name) / `inverse`, hands back the ops via `encode_op()` / `decode_op()`, and wraps a source in one call with **`encode(source) -> Stream`** — which SETS `Stream.class_names` so the vocabulary travels WITH the encoded data (2026-07-29). `Stream.class_names` is a DECLARED, validated `Optional[List[str]]` ctor slot, not a monkey-patched attribute (a consumer used to `setattr` it on and read it back with a `getattr` — an undeclared convention nothing could see); read it with the free function **`class_names(*sources)`** (`recordstream.projection`, beside `num_classes`), which takes several sources because a vocabulary is a property of the RUN rather than of whichever split carries it, skips `None` so `class_names(train, val, test)` needs no guards, and returns `None` when nothing carries one (an integer-labelled run is not an error). **Naming (2026-07-29):** `class_names`, NOT `label_names` — in HuggingFace `transformers`, `label_names` means "which input dict keys hold the labels", a different concept entirely; `class_names` is Keras's term, matches the `class_names.json` file and the `"class_names"` JSON key this already writes. `num_classes` likewise stays (timm / torchvision / torchmetrics-multiclass / HF `datasets.ClassLabel`); `num_labels` is reserved for the MULTI-LABEL count torchmetrics asks for (2026-07-29 — the `Stream(source=..., ops=[encode_op()])` idiom every consumer wrote; it flows a deferred source first). NOTE the asymmetry: `to_ids` passes an already-encoded id THROUGH, but the OP is a straight lookup, so `encode()`-ing an already-encoded set raises `KeyError` lazily while iterating — double-encoding fails loudly instead of silently remapping. Consumers ask `is_class_id` first. This does NOT contradict the "pinned, not fitted" op rule — fitting happens **once at train time**, then the mapping is pinned and persisted. **scikit-learn was DROPPED (2026-07-29)** — `LabelEncoder.classes_` is exactly `sorted(set(...))`, so the dependency made a DATA package require an ML library for one line; ordering is unchanged. Do not reintroduce it. Bare `@configurable` (NO discovery `category` — a config `!class:` node wired into a slot, like the storage classes, not a canvas node); zero-arg constructible (`LabelMap()` is empty, the properties validate lazily). +- **The MODEL BOUNDARY Lives Here Too — Contracts, Sinks, Dataset Normalization, Label Statistics (moved from a tracking library 2026-07-29):** four surfaces landed in recordstream in one pass, under one rule: **a package must not own a contract whose only reader lives elsewhere.** (1) **`ensure_record_dataset(source)` / `RecordSource`** (`recordstream.core`, package-root exports) — normalize a wired dataset slot (`Stream` / torch `Dataset` / bare source / list) into a map-style `Dataset` of records; a `Stream` returns AS-IS (identity matters — a label-encoding Stream carries its `class_names`), anything else is wrapped. It belongs beside `Stream` because that is the only type it knows. Consumers annotate dataset slots `Optional[Lazy[RecordSource]]` instead of inventing a union. (2) **`recordstream.outputs`** — the prediction-output contracts `ClassificationOutput` / `DetectionOutput` / `SegmentationOutput` / `RestorationOutput` (generic `TypedDict`s, parameterized by the array type so a non-torch backend declares the SAME contract) plus the torch builders `classification_output` / `segmentation_output` / `restoration_output` (`softmax`/`argmax` are library calls, not type declarations). Detection has NO builder on purpose — its boxes come from the detector's interface. **`RestorationOutput` carries ONE key (`image`) and that is the contract working, not a stub (2026-08-05):** the other three tasks emit something that needs interpreting, so their keys separate `logits` from `probs` from `class_idx`; an image-to-image model emits the answer, so what a consumer must be told is precisely that the array is a picture in the INPUT's value range — not logits to softmax, not a residual to add back. Its builder therefore transforms NOTHING and exists only so every backend spells the wrapping identically. Do NOT add a `residual` key (it is `input - image`, i.e. a second place for the two to disagree) and do NOT clamp inside the builder: a residual denoiser can legitimately overshoot `[0, 1]`, and a metric computed on clamped values is a different number, so that is the run's decision. (3) **`recordstream.predictions`** — the `PredictionsSink` Protocol (`write(prediction, metadata)` + `close()`, `@runtime_checkable`) and `ClassificationPredictionsSink` (top-k + label resolution -> a record threaded through `ops`, typically `RecordSinkOp`). (4) **`class_counts` / `inverse_frequency_weights`** (`recordstream.labels`, beside `LabelMap`) — see the balancing mandate below. Everything is exported from the package ROOT (the `recordstream` entry point + `__all__` carry them into discovery). **A consumer's stale `from matrainer.sinks import …` fails loudly — there are NO back-compat aliases.** Pins: `tests/test_record_source.py` / `tests/test_outputs.py` / `tests/test_predictions.py`. Rationale: `docs/architecture.md` §8. +- **TWO Sink Protocols, Deliberately (`DataSink` vs `PredictionsSink`, 2026-07-29):** `storage.base.DataSink` takes a whole `record` (`write(record)`) and is what `RecordSinkOp` adapts into an op chain; `predictions.PredictionsSink` takes a MODEL's output plus the metadata of the record it came from (`write(prediction, metadata)`) and builds the record itself — the two halves arrive separately because a model emits a BATCH while the sink contract is per-record. The split is load-bearing downstream: a visual editor surfaces `category="sink"` storage sinks as canvas nodes and deliberately excludes prediction sinks because their signature differs. Do NOT blur them, and do NOT tag a prediction sink `category="sink"`. Collapsing them (have the runnable build the record and write through `DataSink`) is a real option — it is filed in `TASKS.md`, not decided by drift. +- **Class-Balance Weights Are a LABEL STATISTIC, Not a Loss Concern (`recordstream.labels`, 2026-07-29):** `class_counts(targets, num_classes, label_map=None)` and `inverse_frequency_weights(...)` compute how often each class occurs and the inverse-frequency vector `w[c] = total / (num_classes * count[c])`. They live here because that is a property of the DATA — the same numbers serve `torch.nn`'s `weight=` constructor argument and Keras's `class_weight` on `fit()`. Three rules the signature encodes: (a) they take **already-walked targets, never a source** — a consumer walks the target stream ONCE and reuses that pass for the `LabelMap` fit, the class count AND the weights, so a convenience that walked internally would silently double the passes; (b) every target shape is accepted because `LabelMap.to_ids` normalizes it (a `Label`, a `MultiLabel` counting for every class it names, a bare id with an EMPTY map, a name with a fitted one) — the flattening consumers used to hand-write is now inside; (c) the return is **numpy** (`float32`, or `None` when nothing was counted, so "no weights" is distinguishable from "all-zero weights") — the same rule as `recordstream.batch`, where only `batch_tensor` is torch. An unobserved class gets `0.0`, never infinity; an out-of-range id is IGNORED rather than raising (a stray label must not abort a run). **What does NOT belong here: whether a loss takes weights and how to inject them.** That is a framework convention and lives in the consuming runnable as an overridable method (sonair's `accepts_class_weights` / `apply_class_weights`) — recordstream must never learn what a loss is. +- **`recordstream.sources` Is a PACKAGE, One Class Per Module — and the SUBMODULE Path Is the Canonical `!class:` Spelling (2026-08-01):** The 511-line `sources.py` is GONE; each source owns a file — `huggingface.py` (`HuggingFaceSource` + the `METADATA_ALL_FEATURES` sentinel + `_resolve_metadata_features`) / `split.py` (`DatasetSplit` + `SplitName` + the private `_SplitView`) / `range.py` (`RangeSource`) / `concat.py` (`ConcatSource`) — plus `base.py` for the one helper (`_pass_through`) the three view sources share, mirroring `recordstream.ops`. **The IMPORT surface is the package** (`from recordstream.sources import DatasetSplit` — `__init__.py` re-exports every public name), but the **CANONICAL path is the submodule** (`!class:recordstream.sources.split.DatasetSplit`), because `confluid.pydantic_export._qualname` builds a class's published path as `f"{cls.__module__}.{cls.__qualname__}"` — that string is the `!class:` tag a generated config emits, the form-spec / MCP schema path, and the key of navigaitor's `enrichment.yaml` `classes:` table (a stale key there silently drops a field alias instead of failing). The shorter `recordstream.sources.HuggingFaceSource` STILL resolves — `confluid.resolve_class` falls back to a module-path import and the package re-exports the name — so old hand-written configs keep loading; what moved is what GENERATORS write, which is why every such string workspace-wide was updated in the same change. **NEVER "fix" the path churn by pinning `__module__` back in `__init__.py`:** measured, that breaks `confluid.registry.key_for()` (`_entry_for_object` re-derives `f"{__module__}.{__qualname__}"` and misses the key stored when `@configurable` ran), so a class whose bare name later becomes ambiguous dumps the un-disambiguated `!class:Name()`, AND it breaks `inspect.getsource` (`OSError: could not find class definition`). **`__init__.py`'s `__all__` is LOAD-BEARING, not decoration:** `recordstream.discovery.scan_module` filters members on `member.__module__ == mod_name`, so it now returns `[]` for the package — a visual editor's node bridge surfaces these nodes ONLY through its second pass over `__all__`, so a new source re-exported but not listed there vanishes from the palette silently. ONE entry point covers the package (`recordstream-sources = "recordstream.sources"`) because `__init__.py` imports all four submodules — do NOT add per-submodule entry points. Adding a source = one new module + a re-export + an `__all__` entry. Rationale: `docs/architecture.md` §11; usage + the path table: `docs/sources.md`. +- **A Source NAMES The Data It Reads — `dataset_uri` + `dataset_url`, And A View Propagates Them VERBATIM (`recordstream.uri`, 2026-08-02):** `SupportsDatasetIdentity` is a `@runtime_checkable` Protocol with TWO properties, and the pair is deliberate rather than one field doing double duty: **`dataset_uri`** is the CANONICAL handle (machine-parseable, stable across machines, the string two runs are COMPARED on — `hf://datasets/ylecun/mnist?split=train`, matching the convention hosted tracking services already use for a dataset source) while **`dataset_url`** is a link a PERSON opens and is `None` whenever the data has no web page. Collapsing them forces a choice between a browsable string that lies about local data and a canonical one nobody can click. `HuggingFaceSource` implements both: a Hub repo id -> `hf://datasets/?…`, a local directory -> its `file://` URI, decided by `Path(path).exists()` (the same question `load_dataset` answers); the query params are SORTED so one configuration has exactly ONE string, and both read STORED CONFIG ONLY — asking never loads, so a source that is never iterated still names itself (`tests/test_dataset_uri.py::test_asking_for_identity_never_loads_the_dataset`). **`revision` is a DECLARED ctor param BECAUSE identity reads it** (it was in the removed `**kwargs`), and `load_options` merges it over `load_kwargs` as a read-only property rather than in `__init__` — a declared key may be set post-construction, and a dict assembled in the constructor would keep the value the object was born with. **Following happens in the FREE FUNCTIONS, not in every wrapper:** `dataset_uri(x)` / `dataset_url(x)` flow a deferred `!class:` marker (as `project` does) and then follow a `.source` attribute when the object holds no handle — depth-capped (`MAX_WRAPPER_DEPTH`) and cycle-safe. **A deferred marker is flowed ONLY when its target CLASS could answer (2026-08-24):** `_materialize` resolves the class without building (`uri._target_class` — class object, registry name, or importable dotted path; a builder FUNCTION stays conservative and builds) and answers `None` when it declares neither the identity properties nor a `source` slot (`hasattr` + `confluid.declares_key`) — a run's provenance capture used to construct a full `pl.Trainer` (and attempt a dataset-less `DataLoader`) per walk just to learn each names no dataset. Pins: the never-built group in `tests/test_dataset_uri.py` (the guard plus both still-builds con cases) — so `Stream` / `DatasetSplit` / `_SplitView` / `RangeSource` / `MetadataFilterSource` all work with ZERO code of their own, as does any third-party wrapper using that attribute name. **A wrapper must NOT decorate the URI it passes up** (no `#train`, no `#0:1000`): the handle identifies the DATASET, how much of it a run consumed is already recorded by the wrapper's own configuration, and decorating would mean one dataset reached two ways stops comparing equal — the single property the handle exists to have, and what makes a consumer's dedup exact. **`ConcatSource` answers `None` ON PURPOSE** — several datasets end to end are not one dataset, and picking a member would be a lie; `dataset_uris(source)` is the plural form that fans out over `.sources` (recursive, deduplicated). `None` is an ordinary answer everywhere (unconfigured source, in-memory stream, data with no page), never an error. Adding identity to a new source is TWO properties and no registration. Package-root exports + `__all__`; no entry point (the module holds no `@configurable`, same as `projection.py`). Rationale: `docs/architecture.md` §13. Usage: `docs/sources.md` → "Identifying a dataset". Pins: `tests/test_dataset_uri.py`. +- **A Lazy Source Must Not Be First READ In A Forked Child — `ensure_materialized` (2026-08-02):** every source here is lazy on purpose (the constructor does no work; the download / file open / client construction happens on first read), and there is exactly ONE place that is wrong: a forked worker process. `ensure_materialized(source)` reads ONE whole record so all of it happens in the caller's process, and returns the source so it composes with `ensure_record_dataset` — that one normalizes a source's TYPE, this one its STATE. **The failure it prevents was measured, not imagined:** a `DataLoader` worker was the first to touch a `HuggingFaceSource`, so `load_dataset` ran in the child and called `hf_hub_download` -> `httpx.Client()` -> `urllib.request.getproxies` -> `_scproxy` -> CoreFoundation, which is not fork-safe — **SIGSEGV with no Python traceback**, surfacing only as `DataLoader worker exited unexpectedly` (read from the macOS crash reports plus a `sitecustomize.py` stack probe running inside the worker). **It reads a whole RECORD and that is not laziness worth optimizing away:** `len(source)` was measured NOT to be enough — loading the dataset object is not the same as building what a read needs — and `first_value` is projection-aware, so it deliberately skips building the values it was not asked for. **Choosing spawn instead is not a general fix:** a framework may set the start method globally (fastai sets `fork` at import, measured), and a spawned worker cannot receive a model that lives on Apple's MPS (`_share_filename_: only available on CPU`). An empty source is a no-op, so a caller needs no guard for an unwired split. Consumers that fork call it beside `ensure_record_dataset` — and since 2026-08-06 the composition itself ships as **`prepare_record_dataset(source)`** = `ensure_materialized(ensure_record_dataset(source))` with `None` passing through (the `_prepare` helper every training runnable had re-composed; a consumer needing only one half still calls that half). **The torch loader triple ships beside it (`recordstream.loaders.loader_slots`, 2026-08-06):** the train/val/test `LazyClass(DataLoader, ...)` slots every torch training runnable declared identically, returned as the `LoaderSlots` NamedTuple (`slots.train`/`.val`/`.test`, typed `Lazy[DataLoader[Any]]`; train shuffled, eval not, `persistent_workers` derived from `num_workers` unless the caller NAMES it (`persistent_workers=` — `None` derives, an explicit value wins, and `True` with zero workers is refused at construction instead of at first iteration; the knob exists for a HOST fact no derivation can see, e.g. macOS terminating persistent workers slowly), `collate_fn=` the batch-shape choice, further DataLoader kwargs passing through to all three — `shuffle` refused as a shared kwarg because it is the one per-split decision the helper owns). The module is torch-ONLY and deliberately NOT package-root exported (the `ops.torch` pattern — a torch-free import of it raises an `ImportError` naming the extra); consumers import `from recordstream.loaders import loader_slots`. Config-side, the three runnable slots stay whole-value replaceable in YAML exactly as with the inline construction this replaces. Pins: `tests/test_record_source.py` (incl. the `len()`-is-not-enough premise), `tests/test_loaders.py`. +- **`recordstream.core` and `recordstream.flow` Are PACKAGES, Layered by IMPORT DIRECTION (2026-08-01):** The 713-line `core.py` and 708-line `flow.py` are GONE, split by COHESIVE UNIT (a class gets its own module when it dominates one; otherwise the unit is the boundary — this is NOT the literal one-class-per-file rule, which would have produced a 30-line `joint_stream.py` that `docs/architecture.md` §5 already rejected). The layering is the invariant, and imports run STRICTLY one way: **`core/`** = `families.py` (the op-family registry + the `_apply_op` chokepoint + the `EXPANDS` protocol — the BOTTOM of the op-facing layer, importing nothing from its siblings) -> `mapstyle.py` (`MapStyle` Protocol + `RecordSource`, pure types) -> `wrappers.py` (`FilterOp`/`WrappedOp`, §5) -> `stream.py` (`Stream` + `JointStream` + `linear_steps`/`_worker_task`/`ensure_record_dataset`, which live there because their DEPENDENCY puts them there — all three build or run a `Stream`); **`flow/`** = `steps.py` (`FlowStep` + the `bind:` grammar, pure data) -> `parse.py` (`parse_flow`, the only module that knows the DOCUMENT form) -> `execute.py` (the per-record kernel `run_steps_multi`/`run_steps`/`is_linear` + both routes + the spawn worker) -> `graph.py` (`FlowGraph`). `flow.execute` imports `core.families` at MODULE level; `core.stream` reaches `flow` only via BODY-LOCAL imports — reversing either closes the cycle §5 exists to prevent. **Canonical `!class:` paths are the SUBMODULE ones** (`recordstream.core.stream.Stream`, `recordstream.core.wrappers.FilterOp`, `recordstream.flow.graph.FlowGraph`) for the reason in the sources mandate above; the package spelling still resolves, and the IMPORT surface stays the package (`from recordstream.core import Stream`). **`core/__init__.py` re-exports PRIVATE names on purpose** (`_apply_op` + the spawn/registry helpers, `# noqa: F401`): they are the engine's internal cross-module surface — every composing op in `ops/` does `from recordstream.core import _apply_op` — but they MUST stay out of `__all__`, which is the palette. **THE TRAP, and it is silent: a re-exported name is a BINDING, not a view of the defining module.** `monkeypatch.setattr(recordstream.flow, "_result_readers", ...)` no longer reaches `flow/graph.py`, which bound the name at import — patch the module that USES a symbol (`recordstream.flow.graph`), never the one that defines it. `_OP_FAMILIES` is the one exception, and only because it is a MUTABLE list re-exported by identity, so `core._OP_FAMILIES[:] = snapshot` still restores the real registry (rebinding it would not). ONE entry point per package; `__all__` is load-bearing in both (`core.py` had none, so `Stream`/`JointStream` reached the palette purely through `scan_module`'s `__module__` filter — which now returns `[]`). Rationale: `docs/architecture.md` §12; pins: `tests/test_module_layout.py`. +- **Discovery Categories:** `@configurable` classes carry a confluid discovery `category` so navigaitor's `list_configurable_classes(category=...)` and the visual-editor form-spec (`get_node_form_spec`) can enumerate them. The recordstream buckets are deliberately split by ROLE, not lumped under "dataset": + - `category="engine"` — generic **composition primitives** that compose sources + ops: the dataset engines `Stream` / `JointStream` / `FlowGraph` (a `Stream` *implements* the `Dataset` interface but is conceptually the engine). They ARE canvas-composable in StreamStudio: the allowlist includes `engine`, and their source-typed constructor params render as wired sockets — `Stream.source` (single `RECORDSTREAM_SOURCE` input) + `Stream.ops` (dynamic `op_N` `RECORDSTREAM_OP` inputs), `JointStream.streams` (dynamic `source_N` `RECORDSTREAM_SOURCE` inputs). The higher-order op wrappers `FilterOp(p)` / `WrappedOp(f)` take a *raw Python callable* — they are NEITHER an op nor an engine, so they carry NO category (bare `@configurable`); excluded from StreamStudio as uncategorised (nothing to wire in a GUI). + - `category="source"` — concrete data **sources** that load/yield record dicts: `HuggingFaceSource` (and the domain packages' sources), **plus the view/derivation sources `DatasetSplit` / `RangeSource` / `ConcatSource` / `MetadataFilterSource`** — each yields records and is wired into a trainer's `source:` slot, deriving a *view* of other source(s) (train/val/test split · contiguous `[start:stop)` slice · concatenation · metadata filter) without applying ops, so they're sources, not engines. StreamStudio source nodes; navigaitor dataset-slot options. Their source-typed ctor params render as wired sockets on the canvas (`DatasetSplit.source`/`RangeSource.source` → a `RECORDSTREAM_SOURCE` input; `ConcatSource.sources` → dynamic `source_N` inputs). **`DatasetSplit` is `train`/`val`/`test`-aware**: ONE instance exposes the three splits as **cached `@property` views** (`split.train`/`.val`/`.test`) for CODE; in YAML a view is a `DatasetSplit` with `split=` set (select-one, `None`⇒`train`) — write the recipe once (`&split_recipe` on the train view) and `<<:`-merge it into the others; every view `!ref:`s the same upstream source, which therefore loads once (the attribute-reference spelling `!ref:my_split.train` is REFUSED by confluid since 2026-08 — record 19 phase 2 — and the refusal names this rewrite; `docs/sources.md`). `split` is the closed `SplitName = Literal["train","val","test"]`. (Range mode was extracted from DatasetSplit into `RangeSource`.) The view objects returned by the properties are a private `_SplitView` — NOT `@configurable` (only read off a live split, never built in a config). + - `category="op"` — concrete `Record → Optional[Record]` **ops**: EVERY op meant to be a canvas node MUST carry it. StreamStudio uses a POSITIVE allowlist `{op, source, engine, sink}`, so an UNtagged op silently vanishes from the palette — tag every new op `op`. Also give it a path-like **`group=`** (presentation-only; StreamStudio nests the palette as `Taidal/RecordStream/Op/`). The recordstream groups: `numpy` (`Threshold` — array→`Mask`, `ConnectedComponents` — `Mask`→`Boxes` via the shared `connected_component_boxes` helper, the torchvision Penn-Fudan-tutorial derivation) / `torch` (`ToTensor`) / `image` (`ConvertToImage`) / `structure` (the key-plumbing ops `RenameField`/`DropField`/`CopyField`/`SelectFields` from `recordstream.ops.structure` — rename or copy a record key, drop an entry, or narrow the record to a chosen key set (`RenameField` is also how a value routes into the albumentations key vocabulary); the six context ops `Save`/`Use`/`Drop`/`Apply`/`Capture`/`MergeFields`; and the target shapers `EncodeTarget` / `DecodeTarget` + the detection-target ops `CocoToTorchVisionDetection` / `MasksToDetectionBoxes` / `ResizeDetection` (`recordstream.ops.target` — the first two emit a `Boxes` detection target, lazy-importing torch: one from a HuggingFace / COCO `objects` annotation `{bbox, category}`, the other from a segmentation MASK; `ResizeDetection` is the COUPLED image+boxes resize for fixed-input-size detectors — PIL/uint8 image to `(height, width)` + the `Boxes` boxes scaled by the same factors, torch staying torch, `canvas` updated — run it BEFORE any float conversion such as `ToTensor`, and omit it for detectors that resize internally)) / `compose` (`Pipeline`/`Parallel`/`Enable`/`RandomApply`/`ConfigureOp` (the sanctioned per-record-parameter mechanism: `ops` compute-chain → the `source`-keyed entry → setattr as the wired `target` op's `param` → apply; StreamStudio renders `ops` as `op_N` sockets and `target` as ONE `RECORDSTREAM_OP` socket, and the ops-export embeds both as nested `!class:` instances) + its companion `FormulaOp` (`recordstream.ops.formula` — evaluates a restricted math formula over the `field`-keyed record entry; the canvas Math node's op form)) / `sink` (`RecordSinkOp` — adapt a `DataSink` as a pass-through op) / `debug` (`PrintRecordOp` = `recordstream.ops.debug`, a pass-through probe that logs/prints a per-record summary to the Loggair logger AND, via `to_console`, stdout; its `level` is restricted to `Literal["trace","debug"]` per the "Diagnostic Log Levels" mandate, console visibility comes from the `print`, and `limit` caps emissions on a large dataset). Pinned in `tests/test_categories.py`. An absent group just leaves the op directly under `…/Op`. +- **Generic MASK Conversion Lives Here Too — `ConvertToMask` (2026-08-02):** the segmentation counterpart of `ConvertToImage` and the same op SHAPE (read one field, write a differently-typed item under `output`): a mask-bearing field (an ndarray, a torch tensor, or the PIL image a source handed over) becomes an **`int64` `[H, W]` `Mask`** of per-pixel class ids — what a segmentation dataset actually ships (an Oxford-IIIT Pet trimap, Cityscapes label ids, a VOC segmentation map) turned into what a per-pixel loss consumes. It belongs HERE, not in a segmentation project: "a mask PNG's pixels are class ids" mentions no modality (the `Threshold` → `Mask` precedent), and a consumer owning it would be the third package to write the conversion. **It converts and NOTHING else, deliberately** — remapping the ids is `FormulaOp` over its output (`formula: a - 1` for a 1-based trimap) or `EncodeTarget` for a lookup table; resizing/augmenting it TOGETHER WITH THE IMAGE is a bare albumentations transform in the same ops list; dropping the source column is `DropField`. Do NOT grow it an `offset` / `mapping` / `dtype` knob: each one restates an op that already exists. **`output` defaults to `"mask"` and that is load-bearing, not a nicety** — it is albumentations' own key vocabulary (`_ALB_KEYS`), so the engine's op-family dispatch hands `image` AND `mask` to ONE call and a single joint draw moves both with the `Mask` type surviving the re-wrap (measured; an image-only transform like `Normalize` still touches the image alone). **`int64` is not a knob either:** a class-id map is integer by definition and it is what `torch.nn.CrossEntropyLoss` requires (*"expected target dtype to be Long or Byte, but got Int"*); a library that casts on the way past — albumentations returns int32 — is corrected at the MODEL boundary by `batch_tensor(..., dtype=...)`, where the caller names the contract (the `dtype`-is-a-parameter rule). It reads through **`item_value`, never `item_data`**, because a source that does not know a column is a mask ships it as a `Label` (`HuggingFaceSource` does this for every metadata column) — see the record-model mandate. Singleton axes are squeezed (`[H,W,1]` / `[1,H,W]` → `[H,W]`); an **RGB-encoded mask RAISES** rather than being collapsed, because picking one of three channels or decoding a palette is a decision the op must not make silently. Pins: `tests/test_convert_to_mask.py` (incl. the whole `preprocess` chain end to end, and that a `Normalize` leaves the mask untouched). Usage: `docs/image.md` → "Masks". +- **Generic Image Conversion Lives Here (`recordstream.ops.image`):** The single, modality-agnostic "any value → image" layer — `ConvertToImage` (`category="op"`, `group="image"`: reads an array-bearing key (`field=` or the first found) and writes an HWC-`uint8` `Image` item under `output` via normalize → colormap → optional `flip_vertical` → exact `width`/`height` or `max_size` resize; it does NOT publish pixel-dimension keys — the `Image` item's array SHAPE carries them) and the free function `normalize_to_uint8` (min-max value→`uint8` quantization; `vmin`/`vmax` default `None` = per-array auto-contrast, set them to pin a fixed scale across records — the standalone `NormalizeToUint8Op` op class was DELETED; only the function remains), plus the library functions `value_to_image` / `record_to_image` and the closed `Colormap` Literal + `COLORMAPS` tuple. **Array introspection helpers** `select_channel` / `channel_count` / `array_histogram` / `confusion_matrix_payload` / `confusion_matrices_payload` also live here — pure functions, NOT `@configurable` ops (they MEASURE/derive, they don't transform a record, so they're library helpers like `value_to_image`, never canvas nodes), backing StreamStudio's in-canvas viewer nodes (`streamstudio.nodes.ArrayHistogramViewerNode` / `ConfusionMatrixViewerNode`). `confusion_matrix_payload(matrix, class_names)` builds ONE render payload (raw counts + the `true`/`pred`/`all` normalizations, JSON-safe); `confusion_matrices_payload(metrics, class_names)` is the GENERIC extractor — it scans a metrics result (`name -> value`, e.g. an evaluator's full `all_metrics`) for EVERY confusion-matrix-shaped entry (square 2-D, by SHAPE not name) and returns one payload per match, so the viewer renders ALL confusion matrices from one generic all-metrics output (the metric-shape knowledge lives HERE, never in the evaluator). The rest: `select_channel(value, channel=-1)` reduces an arbitrary array/tensor to a 2-D `float32` map for one channel (`channel < 0` = mean across the channel axis), `channel_count` reports the channel count, and `array_histogram(value, bins, channel)` bins the values + summary stats over FINITE entries only (so the result is JSON-safe — no `NaN`/`±inf` leaks into `min`/`max`/`bin_edges`). It passes EXPLICIT `np.linspace` bin edges to `np.histogram`, NEVER `bins=, range=(lo,hi)`: numpy 2.2.x's uniform-bins fast path block-accumulates via `np.bincount` for arrays larger than its 65536-element block and miscomputes the bincount length on the workspace build, so `bins=` raises *"operands could not be broadcast together with shapes (256,) (257,) (256,)"* on any real image/spectrogram while passing on the small arrays unit tests happen to use — the explicit-edges (searchsorted) path sidesteps it (keep a >65536-element pin in the suite). The channel axis (`_channel_axis`) is the SMALLEST axis (the channels-are-fewest convention) — deliberately DISTINCT from `_render_rgb`'s `{1,3,4}`-membership heuristic (RGB-render-specific) and `streamstudio.nodes.RecordExtractorNode._as_2d`'s float-only mask rule; the divergence is documented so the three never look like an accidental disagreement. The quantization math is the free function `normalize_to_uint8` — the SINGLE source of truth called directly by the `value_to_image` renderer (2-D-map / float-array paths) and waivefront's LabelStudio renderer (it is the only normalization entry point); it replaced waivefront's old `normalize_dB_to_uint8`, which was modality-neutral and so belonged here, not in a signal package. It lives in recordstream (not waivefront) because the conversion is fully generic, so every project reuses ONE implementation; waivefront re-exports `value_to_image` / `record_to_image` / `Colormap` / `COLORMAPS` for back-compat. **Pillow is a runtime dependency**; matplotlib is imported lazily inside `_apply_colormap` (only non-`gray` colormaps need it, so the greyscale path stays matplotlib-free). **Text → image** also lives here: `draw_text(text, image=None, *, width/height/font_size/color/background/position/margin/wrap)` renders text onto an image (or a fresh `background` canvas) → an `(H,W,3)` uint8 array (PIL `ImageDraw`, word-wrap, 9-grid anchor), with the closed `TextPosition` Literal + `TEXT_POSITIONS` tuple. It's the home for StreamStudio's *Draw Text to Image* node (`streamstudio.nodes.DrawTextNode` — thin glue over it). Domain-specific rendering (spectrogram overlays, IQ panels) stays in the consuming package (waivefront's `RenderOverlays` / `RenderSignalPlot`), NOT here. + Rationale (history): engines were once `category="dataset"` vs uncategorised=sources — backwards (`Stream` the engine was the "dataset", the real `HuggingFaceSource` had no tag). Preserve these tags when adding/renaming classes — `tests/test_categories.py` pins them, and a missing/renamed tag silently empties the corresponding picker (or, for ops, drops the node from StreamStudio). +- **Type IS the Value's Python Class, Never a Separate Field:** A record value's type is its Python class (`Image`/`Mask`/`Boxes`/`Label` + domain items) — there is no stored-type sidecar and no reserved type-descriptor key in the record. A consumer reads a value's type by `type(value)` and its shape/dtype/framework off the value's own payload and attrs. An op that changes a value's type replaces the item under the same key (or writes its `output` key) — e.g. array → `Mask` → `Boxes`. Never carry a parallel type descriptor beside the record. (The on-disk `__item_type__` attr is the CODEC's reconstruction tag, not a record entry.) + +- **This Project's OWN mypy Config Must Be SELF-SUFFICIENT (2026-08-25):** it is type-checked twice with different configs, and the workspace-root one HIDES gaps in the other. `mypy recordstream` from the workspace root reads the shared `mypy.ini`, which already carries `[mypy-scipy.*]` and `[mypy-matplotlib.*]`; this project's own CI runs **`mypy .` from the project directory**, reads only `[[tool.mypy.overrides]]` in `pyproject.toml`, and is the config a PUBLISHED package has to satisfy. The gap is therefore invisible to every local check and surfaces only on the release PR — measured: PR #7's Quality Gates failed on three pre-existing errors (`keras` has no `py.typed`; `scipy.ndimage` stubs missing; `matplotlib` not found) while the workspace-root run reported `Success` on the same tree. **Every optional or lazily-imported third party needs an entry in the project's own override table, even when the workspace config already has one** — the `[keras]` / `[vision]` extras and the lazy `matplotlib` import in `ops.image._apply_colormap` are all correct by design; what was wrong was only that this project's config never learned about them. This does NOT relax the workspace mandate that mypy runs from the ROOT: that stays the routine invocation. Reproducing CI's `mypy .` is a DIAGNOSTIC for exactly this class of failure, not a new habit. **A NOTEBOOK must not require the workspace environment either (2026-08-25):** `notebooks/01 - cat_exploration.ipynb` RAISED on a missing `DATA_ROOT`, so Verify Notebooks failed on every runner — while passing locally, because the notebook walks up to the workspace `.env` and loads it. `DATA_ROOT` chooses WHERE the HF cache lives; it is a preference, not a requirement, so an unset one now falls through to huggingface_hub's own cache and a SET-but-missing one still raises (a misconfigured machine, not an absent preference). **Reproducing this needs a checkout with no `.env` in ANY parent** — `env -u DATA_ROOT` is not enough and will report a false pass, because the notebook re-reads the file itself; copy the notebook somewhere outside the workspace and execute it there. +- **The README Is The PyPI LANDING PAGE — Repo Links Are ABSOLUTE (2026-08-24):** `pyproject.toml` sets `readme = "README.md"`, so the README's rendered form IS the project page, and PyPI resolves a relative link against `pypi.org`, not against the repository — `docs/storage.md` becomes `pypi.org/docs/storage.md` and 404s. The link works perfectly on GitHub, so nothing in the repo catches it, which is why it is a test rather than a review habit: the README links to repo files as `https://github.com/Gearlux/recordstream/blob/main/` (26 links converted on 2026-08-24), while the `docs/*.md` pages are read on GitHub only and KEEP their relative links to each other. Spelling a README link absolutely also takes it out of reach of the existing rename check, so a second rule verifies each absolute self-link against the local tree — without it, converting the links would have traded a PyPI 404 for a GitHub one. Pins: `tests/test_docs_links.py::test_the_readme_links_to_repo_files_by_absolute_url` / `::test_readme_links_into_this_repo_name_a_file_that_exists`. +- **A Dependency FLOOR States What Is TESTED, Not The Oldest Release That Once Worked (2026-08-24):** the floors were `confluid>=0.1.0` / `loggair>=0.1.0` / `liquifai>=0.1.0` long after the workspace had moved on, and one of them described a combination NOBODY can install — `uv pip install recordstream confluid==0.1.0` fails with *"Because liquifai<=0.1.0 depends on confluid>=0.2.0 and you require confluid==0.1.0 … your requirements are unsatisfiable"*. A stale floor is not harmless: it is a claim about what this package supports that no test covers and no resolver honours (a lowest-direct resolve lands on 0.3.0 / 0.2.0 / 0.2.0 anyway). Raise a floor to the version the suite actually runs against when a release makes that possible, and state WHY in a comment when the reason is not the obvious one. **Below 1.0 a pre-release segment is load-bearing in the OTHER direction too:** under PEP 440 a plain `>=0.1.0` EXCLUDES every pre-release of 0.1.0, so a consumer depending on this package while it ships `0.1.0a1` must write `recordstream>=0.1.0a1` or resolve to nothing. Pin: `tests/test_packaging.py`. +- **Release Metadata Is Part Of The Package, Not Paperwork (2026-08-24):** `readme` / `license` / `license-files` / `classifiers` / `[project.urls]` are each optional to the BUILD — the wheel builds byte-identically without them — which is exactly why all four were missing at once and why the gap is invisible until the page is published (`twine check` reports only `long_description missing`). The `Development Status` classifier must agree with the version: an `aN`/`bN` version is `3 - Alpha`, a final one `4 - Beta`; they drift in opposite directions because the version moves every release and the classifier is written once. Pins: `tests/test_packaging.py::test_the_landing_page_metadata_is_present` / `::test_the_development_status_classifier_matches_the_version` / `::test_the_declared_readme_and_license_files_exist` / `::test_no_requirement_is_a_direct_url` (PyPI refuses any `Requires-Dist` carrying a URL — `warehouse/forklift/metadata.py`, "Can't have direct dependency"). **The sdist needs `MANIFEST.in` for anything nothing else references** — setuptools builds it from what is DECLARED (`readme` -> README.md, `license-files` -> LICENSE, `packages.find` + `package-data` -> the package), so `CHANGELOG.md` was silently absent from both projects' sdists until 2026-08-25. Pin: `tests/test_packaging.py::test_the_changelog_is_shipped_in_the_sdist`. ## Testing & Validation - **Pipeline Parity:** Test that serialized-then-deserialized pipelines produce identical output to the original. diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..15ef5ee --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,65 @@ +# Changelog + +All notable changes to this project are documented here. The format follows +[Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the versioning is +[Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.1.0a1] — 2026-08-25 + +First public pre-release. The surface it ships: + +### The record model +- A record is a plain `dict` of typed values — `Image`, `Mask`, `Boxes`, `Label`, + `MultiLabel` — each owning its own metadata. Key names carry meaning; there is no + wrapper container and no role tags. +- Array-backed items subclass `NDArrayItem`, so numpy operations preserve their declared + attributes. Structured items are dataclass wrappers. `register_item` opens the set to + any package. +- `recordstream.io` is the serialization codec, so an externally registered item type + round-trips through storage with no backend change. + +### The engine +- One step-graph engine behind two facades: `Stream` / `JointStream` for the dataset + surface (`__len__` / `__getitem__` / `.batch` / `.parallel` / `.project`), and + `FlowGraph` for a `flow:` document of named steps with `from:` / `merge_from:` / + `bind:` edges. An `ops:` list is the same engine's linear spelling. +- Ops dispatch on value type: a `Transform` samples its parameters once per record and + applies a per-type kernel to every value it handles. +- Bare albumentations and torchvision `transforms.v2` transforms run as-is through the + op-family dispatch — one call is one joint draw across image, mask and boxes. The + family registry (`register_op_family`) is open to other libraries. +- 1→N expanding ops fork the remaining subgraph, in every route including spawn-parallel. +- Multiprocessing uses the `spawn` context; `ensure_materialized` and the OpenCV thread + guard cover the two fork hazards a forked loader hits. + +### Storage +- HDF5, Zarr and Directory sinks, each with a matching source, over the `typedrecord-v1` + layout. +- Metadata is queryable without loading arrays: the `SupportsMetadataScan` protocol plus + `MetadataFilterSource`. + +### Sources and projection +- `HuggingFaceSource`, `DatasetSplit` train/val/test views, `RangeSource`, `ConcatSource`. +- Every source names the data it reads through `dataset_uri` / `dataset_url`, and a view + propagates the handle verbatim. +- Key projection (`project`, `iter_key`, `first_value`, `num_classes`, `class_names`), + the fittable `LabelMap`, and class-balance statistics. + +### Running +- `recordstream run ` runs any Confluid-wired runnable — a trainer, an + evaluator, a dataset processor, a workflow. The `@entrypoint` markers are the dispatch + table for a runnable that drives several tasks off one `task` knob. +- Workflow combinators (`Sequence` / `Conditional` / `Switch`) plus predicates, so a + resume-safe multi-stage pipeline is one document. + +### Batching +- `collate_records` (the `"record"` default) and `collate_list`, differing in one + decision — whether array payloads stack — plus the read-back half (`batch_values`, + `batch_boxes`, `batch_tensor`, `batch_metadata`, `multi_hot`). +- `recordstream.keras.RecordSequence` is the Keras `PyDataset` half, since Keras 3 has no + `DataLoader` to do the row scheduling. + +### Install shape +- The core engine is numpy and installs no ML framework. `[torch]` adds the pieces that + genuinely produce tensors; `[keras]` adds the `PyDataset` adapter and names no compute + engine; `[vision]` enables the bare torchvision transform family. diff --git a/Jenkinsfile b/Jenkinsfile index aed8f21..613078c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,8 +1,8 @@ // ========================================================================= // AUTO-GENERATED FILE — DO NOT EDIT BY HAND -// Generated by: aisland jenkins scaffold --project dataflux +// Generated by: aisland jenkins scaffold recordstream // Source template: aisland/services/jenkins.py (JenkinsService.scaffold_pipelines) -// To regenerate: aisland jenkins scaffold --project dataflux --force +// To regenerate: aisland jenkins scaffold recordstream --force // ========================================================================= pipeline { agent any @@ -30,12 +30,7 @@ pipeline { sh "${VENV_BIN}/pip install --upgrade pip uv" echo 'Installing Dependencies...' - // Internal Gearlux dependencies — installed FIRST with --no-deps - // so .[dev] below finds them pre-satisfied instead of hitting PyPI - // (Gearlux distribution names are intentionally unpublished on PyPI). - sh "${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/log-flow.git@main" - sh "${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/confluid.git@main" - sh "${VENV_BIN}/uv pip install -e .[dev]" + sh "${VENV_BIN}/uv pip install -e .[dev,torch,keras]" // Notebook-only extras (matplotlib, jupyter kernels, etc.) live // in the optional `[notebook]` extra when the project ships // notebooks; absence is not an error. @@ -51,7 +46,7 @@ pipeline { steps { script { sh "rm -f black-diff.txt black-checkstyle.xml" - def targets = sh(script: "for d in dataflux tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() + def targets = sh(script: "for d in recordstream tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() if (targets) { def exitCode = sh(script: "bash -c 'set -o pipefail; ${VENV_BIN}/black --check --diff ${targets} 2>&1 | tee black-diff.txt'", returnStatus: true) @@ -87,8 +82,8 @@ with open('black-checkstyle.xml', 'w') as f: script { if (fileExists('black-checkstyle.xml')) { recordIssues( - id: 'black-dataflux', - name: 'Black Formatting (Dataflux)', + id: 'black-recordstream', + name: 'Black Formatting (Recordstream)', tools: [checkStyle(pattern: 'black-checkstyle.xml')] ) } @@ -100,7 +95,7 @@ with open('black-checkstyle.xml', 'w') as f: steps { script { sh "rm -f isort-diff.txt isort-checkstyle.xml" - def targets = sh(script: "for d in dataflux tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() + def targets = sh(script: "for d in recordstream tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() if (targets) { def exitCode = sh(script: "bash -c 'set -o pipefail; ${VENV_BIN}/isort --check-only --diff ${targets} 2>&1 | tee isort-diff.txt'", returnStatus: true) @@ -136,8 +131,8 @@ with open('isort-checkstyle.xml', 'w') as f: script { if (fileExists('isort-checkstyle.xml')) { recordIssues( - id: 'isort-dataflux', - name: 'Isort Import Order (Dataflux)', + id: 'isort-recordstream', + name: 'Isort Import Order (Recordstream)', tools: [checkStyle(pattern: 'isort-checkstyle.xml')] ) } @@ -149,7 +144,7 @@ with open('isort-checkstyle.xml', 'w') as f: steps { script { sh "rm -f flake8.txt" - def targets = sh(script: "for d in dataflux tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() + def targets = sh(script: "for d in recordstream tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() if (targets) { sh "${VENV_BIN}/flake8 ${targets} --tee --output-file=flake8.txt" } else { @@ -162,8 +157,8 @@ with open('isort-checkstyle.xml', 'w') as f: script { if (fileExists('flake8.txt') && readFile('flake8.txt').trim()) { recordIssues( - id: 'flake8-dataflux', - name: 'Flake8 (Dataflux)', + id: 'flake8-recordstream', + name: 'Flake8 (Recordstream)', tools: [flake8(pattern: 'flake8.txt')] ) } @@ -192,8 +187,8 @@ with open('isort-checkstyle.xml', 'w') as f: script { if (fileExists('mypy.txt') && readFile('mypy.txt').trim()) { recordIssues( - id: 'mypy-dataflux', - name: 'Mypy (Dataflux)', + id: 'mypy-recordstream', + name: 'Mypy (Recordstream)', tools: [myPy(pattern: 'mypy.txt')] ) } @@ -208,7 +203,7 @@ with open('isort-checkstyle.xml', 'w') as f: steps { script { if (fileExists('tests') && sh(script: "find tests -name '*.py' | grep -q .", returnStatus: true) == 0) { - sh "${VENV_BIN}/pytest tests --junitxml=test-report.xml --cov=dataflux --cov-report=xml:coverage.xml --cov-report=term" + sh "${VENV_BIN}/pytest tests --junitxml=test-report.xml --cov=recordstream --cov-report=xml:coverage.xml --cov-report=term" } else { echo "No tests found in 'tests' directory. Skipping." } @@ -222,8 +217,8 @@ with open('isort-checkstyle.xml', 'w') as f: } if (fileExists('coverage.xml')) { recordCoverage( - id: 'coverage-dataflux', - name: 'Dataflux Coverage', + id: 'coverage-recordstream', + name: 'Recordstream Coverage', tools: [[parser: 'COBERTURA', pattern: 'coverage.xml']] ) } @@ -242,6 +237,15 @@ with open('isort-checkstyle.xml', 'w') as f: ${VENV_BIN}/python3 "$f" fi done + # Directory examples opt in by shipping a run.py entry point; + # network-dependent / install-only example apps ship no run.py + # and are skipped by construction. + for f in examples/*/run.py; do + if [ -f "$f" ]; then + echo "Verifying $f..." + ${VENV_BIN}/python3 "$f" + fi + done ''' } } @@ -289,13 +293,13 @@ with open('isort-checkstyle.xml', 'w') as f: post { always { - echo 'Dataflux Pipeline Complete.' + echo 'Recordstream Pipeline Complete.' } success { - echo 'Dataflux is healthy.' + echo 'Recordstream is healthy.' } failure { - echo 'Dataflux build failed. Please check linting or test failures.' + echo 'Recordstream build failed. Please check linting or test failures.' } } } diff --git a/Jenkinsfile.local b/Jenkinsfile.local index fe3f31e..52a0919 100644 --- a/Jenkinsfile.local +++ b/Jenkinsfile.local @@ -1,14 +1,14 @@ // ========================================================================= // AUTO-GENERATED FILE — DO NOT EDIT BY HAND -// Generated by: aisland jenkins scaffold --project dataflux +// Generated by: aisland jenkins scaffold recordstream // Source template: aisland/services/jenkins.py (JenkinsService.scaffold_pipelines) -// To regenerate: aisland jenkins scaffold --project dataflux --force +// To regenerate: aisland jenkins scaffold recordstream --force // ========================================================================= pipeline { agent { node { label 'built-in' - customWorkspace "${env.WORKSPACE_ROOT}/dataflux" + customWorkspace "${env.WORKSPACE_ROOT}/recordstream" } } @@ -42,9 +42,10 @@ pipeline { // Internal Gearlux dependencies — installed FIRST with --no-deps // so .[dev] below finds them pre-satisfied instead of hitting PyPI // (Gearlux distribution names are intentionally unpublished on PyPI). - sh "${VENV_BIN}/uv pip install --no-deps -e ${env.WORKSPACE_ROOT}/logflow" - sh "${VENV_BIN}/uv pip install --no-deps -e ${env.WORKSPACE_ROOT}/confluid" - sh "${VENV_BIN}/uv pip install -e .[dev]" + sh "if [ -f '${env.WORKSPACE_ROOT}/confluid/pyproject.toml' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/confluid'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/confluid.git@main; fi" + sh "if [ -f '${env.WORKSPACE_ROOT}/liquifai/pyproject.toml' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/liquifai'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/liquifai.git@main; fi" + sh "if [ -f '${env.WORKSPACE_ROOT}/loggair/pyproject.toml' ]; then ${VENV_BIN}/uv pip install --no-deps -e '${env.WORKSPACE_ROOT}/loggair'; else ${VENV_BIN}/uv pip install --no-deps git+https://github.com/Gearlux/loggair.git@main; fi" + sh "${VENV_BIN}/uv pip install -e .[dev,torch,keras]" // Notebook-only extras (matplotlib, jupyter kernels, etc.) live // in the optional `[notebook]` extra when the project ships // notebooks; absence is not an error. @@ -60,7 +61,7 @@ pipeline { steps { script { sh "rm -f black-diff.txt black-checkstyle.xml" - def targets = sh(script: "for d in dataflux tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() + def targets = sh(script: "for d in recordstream tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() if (targets) { def exitCode = sh(script: "bash -c 'set -o pipefail; ${VENV_BIN}/black --check --diff ${targets} 2>&1 | tee black-diff.txt'", returnStatus: true) @@ -96,8 +97,8 @@ with open('black-checkstyle.xml', 'w') as f: script { if (fileExists('black-checkstyle.xml')) { recordIssues( - id: 'black-dataflux', - name: 'Black Formatting (Dataflux)', + id: 'black-recordstream', + name: 'Black Formatting (Recordstream)', tools: [checkStyle(pattern: 'black-checkstyle.xml')] ) } @@ -109,7 +110,7 @@ with open('black-checkstyle.xml', 'w') as f: steps { script { sh "rm -f isort-diff.txt isort-checkstyle.xml" - def targets = sh(script: "for d in dataflux tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() + def targets = sh(script: "for d in recordstream tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() if (targets) { def exitCode = sh(script: "bash -c 'set -o pipefail; ${VENV_BIN}/isort --check-only --diff ${targets} 2>&1 | tee isort-diff.txt'", returnStatus: true) @@ -145,8 +146,8 @@ with open('isort-checkstyle.xml', 'w') as f: script { if (fileExists('isort-checkstyle.xml')) { recordIssues( - id: 'isort-dataflux', - name: 'Isort Import Order (Dataflux)', + id: 'isort-recordstream', + name: 'Isort Import Order (Recordstream)', tools: [checkStyle(pattern: 'isort-checkstyle.xml')] ) } @@ -158,7 +159,7 @@ with open('isort-checkstyle.xml', 'w') as f: steps { script { sh "rm -f flake8.txt" - def targets = sh(script: "for d in dataflux tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() + def targets = sh(script: "for d in recordstream tests examples; do if [ -d \"\$d\" ] && find \"\$d\" -name '*.py' | grep -q .; then printf \"%s \" \"\$d\"; fi; done || true", returnStdout: true).trim() if (targets) { sh "${VENV_BIN}/flake8 ${targets} --tee --output-file=flake8.txt" } else { @@ -171,8 +172,8 @@ with open('isort-checkstyle.xml', 'w') as f: script { if (fileExists('flake8.txt') && readFile('flake8.txt').trim()) { recordIssues( - id: 'flake8-dataflux', - name: 'Flake8 (Dataflux)', + id: 'flake8-recordstream', + name: 'Flake8 (Recordstream)', tools: [flake8(pattern: 'flake8.txt')] ) } @@ -187,7 +188,7 @@ with open('isort-checkstyle.xml', 'w') as f: // Run from workspace root so root mypy.ini is picked up and cross-project imports resolve. // tee streams errors to the Jenkins console; pipefail ensures mypy's exit code (not tee's) propagates // so the stage fails visibly when types break. - def exitCode = sh(script: "bash -c 'set -o pipefail; cd ${env.WORKSPACE_ROOT} && ${VENV_BIN}/mypy dataflux 2>&1 | tee ${env.WORKSPACE_ROOT}/dataflux/mypy.txt'", returnStatus: true) + def exitCode = sh(script: "bash -c 'set -o pipefail; cd ${env.WORKSPACE_ROOT} && ${VENV_BIN}/mypy recordstream 2>&1 | tee ${env.WORKSPACE_ROOT}/recordstream/mypy.txt'", returnStatus: true) if (exitCode != 0) { if (params.REPORT_ALL_WARNINGS) { unstable("Mypy found type errors. See console output above and Mypy report.") @@ -202,8 +203,8 @@ with open('isort-checkstyle.xml', 'w') as f: script { if (fileExists('mypy.txt') && readFile('mypy.txt').trim()) { recordIssues( - id: 'mypy-dataflux', - name: 'Mypy (Dataflux)', + id: 'mypy-recordstream', + name: 'Mypy (Recordstream)', tools: [myPy(pattern: 'mypy.txt')] ) } @@ -218,7 +219,7 @@ with open('isort-checkstyle.xml', 'w') as f: steps { script { if (fileExists('tests') && sh(script: "find tests -name '*.py' | grep -q .", returnStatus: true) == 0) { - sh "${VENV_BIN}/pytest tests --junitxml=test-report.xml --cov=dataflux --cov-report=xml:coverage.xml --cov-report=term" + sh "${VENV_BIN}/pytest tests --junitxml=test-report.xml --cov=recordstream --cov-report=xml:coverage.xml --cov-report=term" } else { echo "No tests found in 'tests' directory. Skipping." } @@ -232,8 +233,8 @@ with open('isort-checkstyle.xml', 'w') as f: } if (fileExists('coverage.xml')) { recordCoverage( - id: 'coverage-dataflux', - name: 'Dataflux Coverage', + id: 'coverage-recordstream', + name: 'Recordstream Coverage', tools: [[parser: 'COBERTURA', pattern: 'coverage.xml']] ) } @@ -252,6 +253,15 @@ with open('isort-checkstyle.xml', 'w') as f: ${VENV_BIN}/python3 "$f" fi done + # Directory examples opt in by shipping a run.py entry point; + # network-dependent / install-only example apps ship no run.py + # and are skipped by construction. + for f in examples/*/run.py; do + if [ -f "$f" ]; then + echo "Verifying $f..." + ${VENV_BIN}/python3 "$f" + fi + done ''' } } @@ -299,13 +309,13 @@ with open('isort-checkstyle.xml', 'w') as f: post { always { - echo 'Dataflux Pipeline Complete.' + echo 'Recordstream Pipeline Complete.' } success { - echo 'Dataflux is healthy.' + echo 'Recordstream is healthy.' } failure { - echo 'Dataflux build failed. Please check linting or test failures.' + echo 'Recordstream build failed. Please check linting or test failures.' } } } diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..70f8163 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Gert Behiels + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..e68f998 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,9 @@ +# What the sdist ships BEYOND what setuptools already infers. +# +# `readme = "README.md"` and `license-files = ["LICENSE"]` in pyproject already pull those +# two in, and `packages.find` + `[tool.setuptools.package-data]` pull the package. A +# CHANGELOG is referenced by nothing, so without this line it is silently absent from the +# sdist — which is the only copy of it a reader has while the repository is private. +# +# Pinned by tests/test_packaging.py::test_the_changelog_is_shipped_in_the_sdist +include CHANGELOG.md diff --git a/README.md b/README.md index 0f5a02e..59d7bb2 100644 --- a/README.md +++ b/README.md @@ -1,211 +1,178 @@ -# DataFlux +# RecordStream -**DataFlux** is a high-performance, functional data processing engine built for modern Machine Learning pipelines. It provides a clean, fluent API for streaming and transforming data from any source while maintaining strict compatibility with PyTorch and Hugging Face. +**RecordStream** is a high-performance, functional data processing engine built for modern Machine Learning pipelines. It provides a clean, fluent API for streaming and transforming data from any source while maintaining strict compatibility with PyTorch and Hugging Face. -Part of the **Modular Quartet**: `LogFlow`, `Confluid`, `Liquify`, and `DataFlux`. +Part of the **Modular Quartet**: `Loggair`, `Confluid`, `Liquifai`, and `RecordStream`. ## 🚀 Key Features -- **Functional Purity:** Transforms are simple Python callables. No complex base classes required. -- **Standardized Sample Triplet:** Standardizes on `(input, target, metadata)` for full traceability. -- **High Performance:** Native multiprocess support via `.parallel(workers=N)` using the safe `spawn` context. -- **Advanced Storage:** Built-in support for high-performance backends: - - **HDF5**: Clean, efficient read/write. - - **Zarr**: Cloud-native, concurrent storage (Group and Batch modes). - - **Directory**: Robust concurrent writing for irregular data lengths. -- **Passive Introspection:** Automatically generates JSON manifests for visual orchestration in **FluxStudio**. +- **A record is a plain dict:** the [record model](https://github.com/Gearlux/recordstream/blob/main/docs/record-model.md) — a `dict` of typed values (`Image`, `Mask`, `Boxes`, `Label`, `MultiLabel`, …), each owning its own metadata, with key names carrying meaning (`"image"`, `"mask"`, `"bboxes"`). No wrapper container, no role tags. +- **Libraries run AS-IS:** bare [albumentations and torchvision `transforms.v2`](https://github.com/Gearlux/recordstream/blob/main/docs/augmentation.md) transforms drop straight into any ops list — the engine invokes each op family natively (one call = one joint draw across image/mask/boxes). No adapter classes anywhere. +- **Type-dispatched native ops:** a `Transform` samples its parameters once per record and applies a per-type kernel to every value it handles — teach an existing op a new value type with one `@MyOp.kernel(NewType)` registration. +- **Graph pipelines:** readable [`flow:` documents](https://github.com/Gearlux/recordstream/blob/main/docs/graph.md) of named steps — `from:` forks, `merge_from:` merges, `bind:` feeds one step's value into another's parameter. An `ops:` list is the same engine's linear spelling; both parse to one step graph. +- **High Performance:** Native multiprocess support via `.parallel(workers=N)` using the safe `spawn` context; [1→N expanding ops](https://github.com/Gearlux/recordstream/blob/main/docs/kinds.md#1n-expanding-ops-iterable-only-pipelines) flatten in every route. +- **Advanced Storage:** HDF5, Zarr and Directory backends with matching read-back sources and [metadata-only querying](https://github.com/Gearlux/recordstream/blob/main/docs/storage.md#queryable-metadata-recordstreamstoragequery) — filter stored datasets without loading a single array. +- **Passive Introspection:** ops declare the value types they [handle / consume / produce](https://github.com/Gearlux/recordstream/blob/main/docs/record-model.md) and are discoverable by category for visual editors and schema generators. - **100% Reproducibility:** Entire pipelines are serializable via **Confluid** manifests. -## 🎯 Design Goals & Requirements - -### Stream Engine -- **Functional API:** Provide a lazy, chainable pipeline API (`map`, `filter`, `batch`). -- **Standardized Samples:** Use the `Sample(input, target, metadata)` triplet as the primary data unit. -- **Parallel Execution:** Support high-performance multiprocess execution via `.parallel(workers=N)` using the `spawn` context. - -### Storage -- **High-Performance Sinks:** Native support for HDF5 (sequential), Zarr (concurrent), and Directory (irregular) storage. -- **JointFlux Pattern:** Support aggregating multiple heterogeneous data sources into a single stream, preserving per-source transform chains. - -### Metadata & Discovery -- **Passive Introspection:** Automatically discover available tools and ops for serialized manifests. -- **Discovery Categories:** `@configurable` classes are tagged with a confluid `category` (`Flux`/`JointFlux` → `dataset`, `FilterOp`/`WrappedOp` → `op`) so tools like navigaitor's `list_configurable_classes(category=...)` enumerate them by kind. -- **Serialization Symmetry:** Ensure full-pipeline states are serializable and reconstructible via Confluid. - ## 🛠 Quick Start -```python -import numpy as np -from dataflux.core import Flux - -# 1. Define a simple transformation -def normalize(data: np.ndarray, mean: float = 0.0): - return data - mean - -# 2. Build a pipeline -raw_data = [np.random.randn(10) for _ in range(100)] - -flux = Flux(raw_data) \ - .map(normalize, mean=0.5) \ - .filter(lambda s: s.input.mean() > 0) \ - .parallel(workers=4) - -# 3. Collect or stream -for sample in flux: - print(sample.input.shape) -``` - -## 🏷 Type Specs - -`dataflux.typespec` describes *what flows through a `Sample`* and lets ops declare what they accept/produce, so tools like FluxStudio can filter which nodes may connect. It is flexible by design — N-dimensional arrays across numpy/torch/tensorflow, **per-axis bounded ranges**, dtype families, images, and arbitrary Python types — and anything left unspecified defaults to `Any`. +One pipeline mixing a **bare albumentations Compose** (image + mask + boxes move together in one draw), a **bare torchvision v2 transform**, and a **native op** — no wrappers (mirrors [`examples/record_pipeline.py`](https://github.com/Gearlux/recordstream/blob/main/examples/record_pipeline.py)): ```python -from dataflux.typespec import SampleType, ArrayType, Dim, PythonType, UnionType - -# "a 2-D float array whose first axis is 1–10, second axis any size" -ArrayType(shape=(Dim.range(1, 10), Dim.any("N")), dtype="floating") -ArrayType.parse("3 h w", dtype="float32", framework="torch") # jaxtyping-style shorthand -ArrayType.image("CHW", channels=3, dtype="float32", framework="torch") # an image convenience +import albumentations as A +import numpy as np +from recordstream import Stream, Image, Label, Mask, as_transform + +records = [ + { + "image": Image(rng.random((16, 20, 3)).astype(np.float32)), # typed: knows its layout + "mask": Mask((rng.random((16, 20)) > 0.5).astype(np.uint8)), + "bboxes": [[2, 3, 6, 7]], # albumentations vocabulary + "labels": ["drone"], + "class": Label("drone_x", classes=["noise", "drone_x"]), # typed: knows its vocab + "gain_db": -3.0, # a scalar is just another key + } + for rng in (np.random.default_rng(i) for i in range(100)) +] + +stream = Stream( + source=records, + ops=[ + A.Compose( # bare albumentations — as-is + [A.HorizontalFlip(p=0.5)], + bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"]), + ), + A.GaussNoise(p=1.0), # image only (its own kwarg vocabulary) + as_transform(lambda d: d - 0.5, handles=(Image,)), # native: a plain function op + ], +).parallel(workers=4) + +for record in stream: + print(record["image"].shape, record["class"].value) # image+mask+boxes flipped together ``` -**Declare an op's contract** with the class attributes `ACCEPTS` / `PRODUCES` (each a `SampleType`; both default to `Any`, so annotating is optional and backward-compatible). No base class — transforms stay plain callables: +The same ops list in Confluid YAML — bare library transforms are ordinary `!class:` nodes: -```python -@configurable -class StandardizeOp: - ACCEPTS = SampleType(input=UnionType((ArrayType(dtype="numeric"), PythonType("PIL.Image.Image")))) - PRODUCES = SampleType(input=ArrayType(dtype="floating", frameworks={"numpy"})) - def __call__(self, sample): ... +```yaml +ops: + - !class:albumentations.HorizontalFlip + p: 0.5 + - !class:albumentations.GaussNoise + p: 1.0 + - !class:recordstream.ops.numpy.Threshold + low_level: 0.5 ``` -Matching is asymmetric: `consumer.accepts(producer)` is strict (used at runtime against a concrete inferred type); `compatible(consumer, producer)` is permissive (used at edit time — `Any`/unknown on either side passes). A `Sample`'s own type comes from `sample.describe()` — it returns a type stored in the reserved metadata keys `__features__` (a `datasets.Features` dict) + `__spec__` (sidecar refinements), or infers one from the live data; attach a stored type with `sample.with_type(SampleType(...))`. - -## 📦 Storage Integration +### Toggling a branch from the CLI (`Enable`) -DataFlux makes it easy to move data between different formats: +Wrap any stretch of an ops list in `Enable` to switch the whole chain on or off from one flag. +The toggle is the declared `enabled` parameter; `name` identifies the wrapper so several of them +toggle independently: -```python -from dataflux.storage.hdf5 import HDF5Source -from dataflux.storage.zarr import ZarrGroupSink - -# Stream from HDF5 to Zarr in parallel -Flux.from_source(HDF5Source("input.h5")) \ - .parallel(workers=8) \ - .map(heavy_op) \ - .to_sink(ZarrGroupSink("output.zarr")) +```yaml +ops: + - !class:recordstream.ops.numpy.Threshold {low_level: 0.5} + - !class:recordstream.ops.enable.Enable + name: visualize # ← names THIS wrapper; scopes its CLI flag + enabled: false # ← off by default; the chain below is skipped + ops: + - !class:recordstream.ops.image.ConvertToImage {} + - !class:recordstream.ops.debug.PrintRecordOp {} ``` -## ✂️ Train / Val Splitting - -`DatasetSplit` carves a subset view out of any indexable source (implementing `__len__` and `__getitem__`). It supports three modes: - -1. **Fraction mode** — pick a reproducible train/val split from a single source: - - ```yaml - hf_train: !class:dataflux.sources.HuggingFaceSource() - path: mnist - split: train - - train_set: !class:dataflux.sources.DatasetSplit() - source: !ref:hf_train - split: train - val_fraction: 0.1 - seed: 42 - - val_set: !class:dataflux.sources.DatasetSplit() - source: !ref:hf_train - split: val - val_fraction: 0.1 - seed: 42 - ``` - - Same seed + same source length ⇒ deterministic, disjoint, complementary views. - -2. **Range mode** — explicit slice: - - ```yaml - first_half: !class:dataflux.sources.DatasetSplit() - source: !ref:hf_train - start: 0 - end: 5000 - ``` - -3. **HuggingFace native slicing** (alternative, no `DatasetSplit` needed): - - ```yaml - train_src: !class:dataflux.sources.HuggingFaceSource() - path: mnist - split: "train[:90%]" - val_src: !class:dataflux.sources.HuggingFaceSource() - path: mnist - split: "train[90%:]" - ``` - -> **Note on `!ref:`** — Confluid `!ref:` resolves to the same live object as the referenced key, so a single `HuggingFaceSource` is loaded once and shared by both splits. Use `!clone:` when you want an independent deep copy instead. +```bash +recordstream run pipeline.yaml --visualize.enabled true # this wrapper only +recordstream run pipeline.yaml --visualize.enabled+ # polarity shorthand → True +recordstream run pipeline.yaml --enabled false # broadcast: every Enable off +``` -## 🔗 Paired Join (Binary ↔ Annotations) +Inner ops are not materialized until the wrapper first fires, so gating an expensive chain with +`enabled: false` costs nothing at startup. In Python the same wrapper is one call — +`Enable(ops=[...], name="visualize", enabled=False)` — which is what lets a visual editor or a +generated tool schema set the toggle too (see [docs/architecture.md](https://github.com/Gearlux/recordstream/blob/main/docs/architecture.md#6-every-knob-is-a-declared-parameter--the-enable-toggle-2026-07-27)). -`PairedSource` joins a primary `DataSource` (e.g. raw binary samples) with a secondary mapping-shaped annotation store via a key function. It generalises the common "I have data, and I have a sidecar file of annotations that covers some of it" pattern. Three join policies cover the scenarios we actually see in ML research: +### Inference as an op (`ModelPredict`) -| Policy | Iterates | Use case | -|---|---|---| -| `left_outer` (default) | Every primary sample; attaches annotation when the key matches | Process everything, use labels where available | -| `inner` | Only primary samples whose key is in the store | Train/evaluate on the labeled subset | -| `right_driven` | Every key in the annotation store; resolves the primary sample via `primary_resolver(key, primary)` | Very sparse labels where full-primary enumeration is costly | +A pipeline can carry its own inference: `recordstream.ops.predict.ModelPredict` runs any +callable model wrapper on each record and stamps the prediction back as a record field — +a class `Label`, `Boxes`, an int class mask, or the restored image, by `kind`. The model's +heavy work (build the network, load `checkpoint_path`) happens in its `solidify()`, called +lazily on the first record; the op itself imports no ML framework. ```yaml -primary: !class:waivefront.rfuav.data.source.RFUAVSource() - root: /Volumes/Data/RFUAV - window_samples: 1000000 - -labels: !class:annotaide.store.JSONFileAnnotationStore() - path: /Volumes/Data/RFUAV-labels - -paired: !class:dataflux.paired.PairedSource() - primary: !ref:primary - secondary: !ref:labels - key_fn: "waivefront.rfuav.keys:sample_window_key" - policy: left_outer +pipeline: !class:recordstream.core.stream.Stream + source: !class:recordstream.sources.huggingface.HuggingFaceSource {path: ylecun/mnist, split: test} + ops: + - !class:recordstream.ops.image.ConvertToImage {width: 224, height: 224} + - !class:recordstream.ops.predict.ModelPredict + model: !class: {checkpoint_path: runs/checkpoints/mnist/last.ckpt} + kind: classification # or detection / segmentation / restoration ``` -Annotation records are **flattened into `Sample.metadata`**, so a detection record `{bboxes, labels, scores}` shows up as three independent metadata keys. Two `metadata` keys are always populated: `annotated: bool` and `annotation_key: str`. Optional `prefix` and `store_full_under` parameters shape the layout. +A viewer reads the stamped `predict*` fields back as layers; `recordstream run` executes +the same document offline. -### Coarser-granularity keys (broadcast and slicing) +## 📚 Documentation -`key_fn` is free to return a coarser key than the sample granularity. When multiple primary samples map to the same key, they all look up the same record: +| Page | Covers | +|---|---| +| [docs/record-model.md](https://github.com/Gearlux/recordstream/blob/main/docs/record-model.md) | The record data model: a plain dict of typed values, type-dispatched ops and kernels, mixing libraries as-is, custom item types, engines, storage layout | +| [docs/kinds.md](https://github.com/Gearlux/recordstream/blob/main/docs/kinds.md) | Writing ops (kernels, `field=`, type-changing ops), the collate registry (`collate_records`) + its read-back (`batch_values` / `batch_tensor` / `batch_metadata`), the Keras `RecordSequence` adapter, 1→N expanding ops | +| [docs/graph.md](https://github.com/Gearlux/recordstream/blob/main/docs/graph.md) | `flow:` documents + the `FlowGraph` engine, `ops:` as the linear spelling of the same step graph, expanding (1→N) steps, `Stream.from_ops_yaml` | +| [docs/sources.md](https://github.com/Gearlux/recordstream/blob/main/docs/sources.md) | `HuggingFaceSource`, `DatasetSplit` train/val/test views, `RangeSource`, `ConcatSource`, Confluid `!ref:` sharing, dataset identity (`dataset_uri` / `dataset_url`) | +| [docs/storage.md](https://github.com/Gearlux/recordstream/blob/main/docs/storage.md) | HDF5 / Zarr / Directory sinks & sources (`typedrecord-v1`), array-valued item attributes, the `SupportsMetadataScan` protocol + `MetadataFilterSource` querying | +| [docs/projection.md](https://github.com/Gearlux/recordstream/blob/main/docs/projection.md) | Key projection (`SupportsProjection`), lazy key walks (`iter_key`), one-peek `first_value`, `num_classes`, the fittable `LabelMap`, class-balance weights | +| [docs/predictions.md](https://github.com/Gearlux/recordstream/blob/main/docs/predictions.md) | The model boundary: prediction-output contracts (`ClassificationOutput` & co), `ensure_record_dataset`, the `PredictionsSink` protocol + the classification sink | +| [docs/image.md](https://github.com/Gearlux/recordstream/blob/main/docs/image.md) | Generic value→image conversion (`ConvertToImage`, `normalize_to_uint8`), mask→class-id conversion (`ConvertToMask`), array introspection helpers | +| [docs/configure.md](https://github.com/Gearlux/recordstream/blob/main/docs/configure.md) | Per-record op parameters (`ConfigureOp` and the `Capture`/`Apply` context ops) | +| [docs/runnable.md](https://github.com/Gearlux/recordstream/blob/main/docs/runnable.md) | Runnables (`run()` + `recordstream run`), the `@entrypoint` task/role markers + `run_entrypoint` dispatch with a worked example, `TorchRunner` / `ProgressReporting` | +| [docs/workflow.md](https://github.com/Gearlux/recordstream/blob/main/docs/workflow.md) | Workflow combinators (`Sequence`/`Conditional`/`Switch` + predicates): resume-safe multi-stage pipelines as ONE document | +| [docs/augmentation.md](https://github.com/Gearlux/recordstream/blob/main/docs/augmentation.md) | Augmentation via bare albumentations / torchvision `transforms.v2` — the op-family dispatch, key vocabulary, bbox recipes, seeding | +| [docs/architecture.md](https://github.com/Gearlux/recordstream/blob/main/docs/architecture.md) | Architecture decision records — the *why* behind non-obvious mechanisms (e.g. why collation is a pluggable registry) | -- **Without `extract_fn`** — the record is broadcast identically into every matching sample's metadata (e.g. a scalar pack-level class label inherited by every window of that pack). -- **With `extract_fn`** — the record is projected per sample. The callable is invoked as `extract_fn(record, sample) -> dict | None`; returning `None` marks the sample unannotated (and filters it under `policy="inner"`). Use this when a pack-level annotation carries time-ranged content that must be trimmed to each window's bounds. +## 🧭 Scope: a modality-neutral engine -Multi-granularity joins (e.g. pack-level + window-level annotations merged together) compose by chaining `PairedSource` instances — the output of one is itself a `DataSource` that the next can consume. +RecordStream deliberately contains **no domain-specific code** — every op, source and sink in this package is meaningful for any modality (arrays, tensors, images, generic metadata). Domain packages build on it and keep their own vocabulary: -### Callable resolution +- Signal/waveform items and ops (spectrograms, FFT windows, recording formats) live in the domain package, which registers its item types into the same registries. +- Task-specific trainers, collates and models live in their consuming projects. -`key_fn`, `extract_fn`, and `primary_resolver` all accept either a callable **or** a `"module:function"` string path resolved through `dataflux.discovery.resolve_callable`. The string form is what survives YAML round-trip via Confluid. +## 🌐 Ecosystem Integration -See [`examples/paired_annotations.py`](examples/paired_annotations.py) for a runnable end-to-end walkthrough of all four scenarios. +RecordStream is designed to sit between your data catalog and your training loop, acting as the high-performance "glue" for ML pipelines: -## 🌐 Ecosystem Integration +- **Hugging Face** for community datasets and Arrow/Parquet loading — `HuggingFaceSource` turns a `datasets.Dataset` into record dicts of typed values with full metadata traceability, and [names the dataset it reads](https://github.com/Gearlux/recordstream/blob/main/docs/sources.md#identifying-a-dataset) so a run record can point at it (see [docs/sources.md](https://github.com/Gearlux/recordstream/blob/main/docs/sources.md)). +- **Confluid** for configuration: every pipeline is a YAML document, every op a `!class:` node — including bare library transforms — every run reproducible. +- **PyTorch**: `Stream` and `FlowGraph` implement the `Dataset` protocol (`__len__`/`__getitem__`/`.batch`/`.parallel`) and plug straight into a `DataLoader` with a [registry collate](https://github.com/Gearlux/recordstream/blob/main/docs/kinds.md#batching--collate_records--the-collate-registry-recordstreamcollate) (`collate_records` is the default). +- **Keras 3**: no `DataLoader` exists to do the batching, so [`RecordSequence`](https://github.com/Gearlux/recordstream/blob/main/docs/kinds.md#keras-recordsequence--the-batching-half-the-framework-leaves-to-you) is the `keras.utils.PyDataset` half — row order, slicing, per-epoch reshuffle, `collate_records` — and a `transform` callable supplies the batch shape, exactly as `collate_fn` does for torch. +- **Augmentation libraries**: [albumentations](https://albumentations.ai) and torchvision `transforms.v2` transforms run **as-is** in any ops list — the engine speaks each library's native convention (kwarg vocabulary vs dict walk), so there is nothing to wrap (see [docs/augmentation.md](https://github.com/Gearlux/recordstream/blob/main/docs/augmentation.md)). -DataFlux is designed to sit between your data catalog and your training loop, acting as the high-performance "glue" for ML pipelines. +## 🔧 Installation -### Intake (Data Discovery & Catalogs) -- **Use Intake for:** Data discovery, remote storage abstraction (S3/GCS), and sharing "canned" datasets via YAML catalogs. -- **Integration:** Wrap an Intake driver in a DataFlux `DataSource` to gain functional `.map()`, `.filter()`, and `.parallel()` capabilities on cataloged data. +RecordStream is on PyPI as a pre-release, so `pip` needs `--pre` to see it: -### Hugging Face (Community & Standardized Datasets) -- **Use Hugging Face for:** Accessing community datasets and leveraging the `datasets` library for efficient Arrow/Parquet loading. -- **Integration:** Use DataFlux to transform `datasets.Dataset` objects into standardized `Sample` triplets, ensuring metadata traceability that often goes missing in simple dictionary-based records. +```bash +pip install --pre recordstream +``` -### DataFlux (The Functional Engine) -- **Use DataFlux for:** The "inner loop" of your experiment. When you need high-performance multiprocess streaming, per-sample metadata preservation, and 100% reproducible pipelines via **Confluid** serialization. +The core engine is **numpy**, and installs no ML framework. A framework arrives only with the extra +that needs it: -## 🔧 Installation +| Extra | Provides | +|---|---| +| `torch` | The pieces that genuinely produce tensors — the `ToTensor` op, `batch_tensor`, and the `classification_output` / `segmentation_output` builders | +| `keras` | `recordstream.keras` — the `RecordSequence` `PyDataset` adapter and the `KERAS_BACKEND` ordering. Keras 3 is an API, so this names no compute engine; it runs on whichever of torch / TensorFlow / JAX you have | ```bash -pip install git+https://github.com/Gearlux/dataflux.git@main +pip install --pre "recordstream[torch]" ``` +Everything else works without either. A `Stream` is map-style (`__len__`/`__getitem__`), so a +`DataLoader` still accepts one directly on a torch install; `batch_values`, `multi_hot` and the +class-balance statistics return numpy, so a non-torch backend converts in one line. Reaching for +`recordstream.ops.ToTensor` without the extra raises an `ImportError` naming it. + ## 📄 License MIT diff --git a/TASKS.md b/TASKS.md new file mode 100644 index 0000000..7ba91f2 --- /dev/null +++ b/TASKS.md @@ -0,0 +1,26 @@ +# recordstream — backlog + +Open work for this project. Cross-cutting / multi-project initiatives live in the +workspace root `TASKS.md`. Completed items are not archived here — git history is the record. + +- [ ] **A torch-less install WARNS on every discovery scan** @low — the documented default install (`pip install --pre recordstream`, no extra) leaves `recordstream.ops.torch` unimportable, so `confluid.registry.load_configurables` logs `WARNING … entry point 'recordstream-ops-torch' … failed to import: No module named 'torch'` every time anything scans the registry. Measured 2026-08-24 in a clean venv. Discovery itself degrades correctly — it skips the module and continues — so this is noise, not breakage, but it fires on the SUPPORTED configuration and the workspace's own "Diagnostic Log Levels" mandate reserves `warning` for conditions an operator must act on. Two ways out, and they belong to different packages: demote the skip to `debug` in confluid (it cannot tell a missing optional extra from a real defect), or make the entry point importable without torch here. Decide which layer owns it before editing either. @2026-08-24 +- [ ] **`materialize_runnable` can shrink away now that liquifai 0.2.0 is on PyPI** @low @refactor — `recordstream/cli.py:33` exists because liquifai's DI dropped every top-level config key for an `Any`-annotated command parameter; liquifai 0.1.1 fixed that at its own layer (`di.deep_flow` now takes the document) and 0.2.0 is published, so the two `flow_mode="manual"` commands can go back to `flow_mode="auto"`. The floor was raised to `liquifai>=0.2.0` on 2026-08-24, which removes the blocker the docstring names. It is NOT urgent: building against the document is correct under either liquifai, so keeping the helper costs only the indirection. A consumer shipping its own CLI calls the same helper, so revert both sides together. @2026-08-24 +- [ ] **`Stream.source`'s annotation is narrower than its documented contract** @low @refactor — the docstring says "any iterable **or indexable** dataset (duck-typed)" but the annotation is `Optional[Iterable[Any]]`. A torch map-style `Dataset` iterates at runtime via the legacy `__getitem__` protocol, which mypy does not model, so passing one is statically invalid though perfectly correct — `recordstream.ensure_record_dataset` (moved in from a consumer 2026-07-29) hits exactly this and carries a documented `cast`. Attempted 2026-07-29: widening to `Union[Iterable[Any], Indexable]` with a `Protocol` BREAKS `confluid.to_pydantic` for every Stream (`SchemaError: Error building "model" validator ... Field "source"` — pydantic cannot schema a bare Protocol), so it needs either a pydantic-friendly spelling or an entry in confluid's opaque-type coercion (`_is_opaque_type` -> `Any`, the same escape hatch the torchvision `Callable`/enum landmines use). Not worth a schema regression for a type nicety; revisit if a second consumer hits the cast. +- [ ] **`to_tensor`'s `normalize` heuristic silently corrupts already-standardized floats** @bug — `recordstream/ops/torch.py::to_tensor` does `elif normalize and tensor.max() > 1.0: tensor = tensor / 255.0`, i.e. it infers "a float whose max exceeds 1 must be 0-255 pixels". An ImageNet-standardized array (range ~[-2.12, 2.64]) satisfies that test, so chaining a `Normalize` op before `ToTensor` divides the standardized values by 255 and squashes them to ~[-0.01, 0.01] — no error, no warning, just a model that learns nothing. Hit for real 2026-07-29 while wiring a consumer's example config; the caller's fix is `ToTensor(normalize=false)`, which is correct but only discoverable by inspecting the tensor. Options: gate the rescale on an INTEGER dtype only (what the docstring already claims — "scale integer pixel inputs"), or keep the heuristic and warn when it fires on a float input. Changing it is a behaviour change for anyone relying on the 0-255-float path, so it needs a decision rather than a quiet edit. +- [ ] **Redesign `waivefront.paired` (`AnnotationJoinSource`)** — moved out of recordstream verbatim 2026-07-18; the user judges the implementation too complex for the pattern it serves (three policies + broadcast/extract projection + string-callable resolution in one class). Rethink the decomposition (join policy vs record projection vs key derivation), possibly as smaller composable sources/ops; keep the public surface stable until then. @medium @refactor +- [ ] **Evaluate consolidating the two callable-resolution grammars** @refactor — `recordstream.discovery.resolve_callable` (`"module:qualname"`, plus `.py`-file and `__main__` handling) overlaps confluid's `resolve_class` module-path branch / `!ref:` grammar (`"module.attr"`) for plain importable functions — two spellings of one job. The non-overlapping remainder (`get_callable_path` string *production*, `scan_module`, `ACCEPTS`/`PRODUCES` schemas) stays in recordstream; decide whether the resolution half should delegate to confluid. Flagged 2026-07-20 while writing the discovery architecture record. @low +- [ ] **Sweep existing user docs for dependent-project mentions** @docs — audit each project's README/`docs/*.md`/examples for names of its own consumers (per the 2026-07-20 "Docs Never Name Dependent Projects" mandate) and genericize; published projects were already swept 2026-07-14, the internal ones (recordstream, waivefront, matrainer, …) were not. @low +- [ ] **Refactor source-level windowing into a reusable expanding `WindowOp`** — the one-capture→N-windows loop now lives ONCE in `waivefront/sources/base.py` (`iter_windows` + `ChunkAnnotations`, shared by `waivefront.sources.rfuav.RFUAVSource` and `waivefront.sources.hdf5_window.HDF5WindowSource`, both with EXACT locator-index `__len__`/`__getitem__` since 2026-08-10), and 1→N expanding ops HAVE landed (`EXPANDS = True`). What remains is the extraction itself: a reusable expanding op (`HDF5Source → IQWindowOp` as the streaming path), keeping the windowed sources for random-access training paths. Flagged 2026-07-17 during the FlowGraph plan; deferred by user decision. @medium @refactor +- [ ] **Decide whether `PredictionsSink` collapses into `DataSink`** @medium @refactor — since 2026-07-29 this package carries TWO sink protocols: `storage.base.DataSink.write(record)` (adapted into op chains by `RecordSinkOp`) and `predictions.PredictionsSink.write(prediction, metadata)`. The split is real — a model emits a BATCH while the sink contract is per-record, so the prediction and its record's metadata arrive separately — and it is load-bearing downstream, where a visual editor's node palette keys off the signature difference. The alternative: have the consuming runnable build the record (it already holds both halves — it slices the batch per record before calling `write`) and write through the ordinary `DataSink`, deleting `PredictionsSink` and letting prediction sinks become ordinary `category="sink"` canvas nodes. That touches the predict path of every consuming runnable, which is why it was NOT bundled into the move. Decide deliberately; do not let the two protocols blur by drift. Rationale for the current state: `docs/architecture.md` §8. +- [ ] **RecordStream Phase 3:** Implement high-performance GPU processing and prefetching. @medium @performance +- [ ] **Typed-bag PoC → torch-`Tensor`-subclass item base:** array-backed items are `np.ndarray` subclasses only; add a `TensorItem` base (torch `__torch_function__` attr-preservation) so torch payloads can be array items instead of riding wrapper `Signal`s. @low @ml +- [ ] **Typed-bag PoC → confluid-native item-type discovery:** item types register in a local `register_item` registry (an `np.ndarray` subclass fights confluid's `__init__` validation wrap); make item types `@configurable(category="itemtype")` + entry-pointed so navigaitor/StreamStudio enumerate them as socket types. @low @tooling +- [ ] **Record model → inverse-transform path:** design an inverse/back-projection hook on `recordstream.transform.Transform` (the old `Transform.decode` stub was dropped in the record conversion) for visualization / inference back-projection (e.g. Spectrogram→Signal, region px→signal coords). @low @ml +- [ ] **S3 storage backend:** extend the `DataSource`/`DataSink` protocols (HDF5/Zarr/Directory exist) with cloud-native S3 support. @feature +- [ ] **Stratified splits** — per-class balanced train/val/test on `DatasetSplit` so small classes don't fall entirely into one partition. @feature +- [ ] **K-fold cross-validation** — a `KFoldSplit` companion yielding fold views; a single config loops over folds. @feature +- [ ] **Grouped splits** — honour a `group_by` metadata key (patient id, source file) so records from one group never leak across train/val. @feature +- [ ] **Pre-computed split manifests** — export the train/val index lists + seed as Confluid artifacts for reproducibility and dataset cards. @low @feature +- [ ] **Auto-split when `val_set` is missing** — implicit fraction split if only `train_set` is wired; deliberately deferred in favour of explicit YAML, revisit for ergonomics. @low +- [ ] **Give `Stream`'s streamed route a graph shape** @low — a stream-level op (`Parallel`, anything exposing `.stream`) sees the WHOLE stream, so it cannot be a step in the per-record graph; `Stream._iter_streamed` therefore splits the chain at each one and runs the per-record segments through the kernel. That is correct but it is a second iteration strategy. Modelling a stream-level stage as a step KIND would make it one shape again. Not urgent: the split is small and the semantics are clear. +- [ ] **`docs/configure.md` still describes `ConfigureOp` beside the deleted context ops** @low — the op itself is unchanged and stays (its compute-chain side-branch is one node where `bind:` needs a producer step), but the surrounding prose should be re-read now that `Apply`/`Capture` are gone. diff --git a/dataflux/__init__.py b/dataflux/__init__.py deleted file mode 100644 index 3a7c6de..0000000 --- a/dataflux/__init__.py +++ /dev/null @@ -1,46 +0,0 @@ -""" -DataFlux: Modular, functional data pipelines. -""" - -from dataflux.core import Flux, JointFlux, WrappedOp -from dataflux.ops import RescaleOp, StandardizeOp, ToTensorOp -from dataflux.paired import PairedSource -from dataflux.sample import Sample -from dataflux.sources import DatasetSplit, HuggingFaceSource -from dataflux.typespec import ( - AnyType, - ArrayType, - Dim, - ListType, - MappingType, - PythonType, - SampleType, - UnionType, - infer_sample_type, - infer_type, - typed, -) - -__all__ = [ - "AnyType", - "ArrayType", - "DatasetSplit", - "Dim", - "Flux", - "HuggingFaceSource", - "JointFlux", - "ListType", - "MappingType", - "PairedSource", - "PythonType", - "RescaleOp", - "Sample", - "SampleType", - "StandardizeOp", - "ToTensorOp", - "UnionType", - "WrappedOp", - "infer_sample_type", - "infer_type", - "typed", -] diff --git a/dataflux/core.py b/dataflux/core.py deleted file mode 100644 index 971fb9c..0000000 --- a/dataflux/core.py +++ /dev/null @@ -1,437 +0,0 @@ -import concurrent.futures -import json -import multiprocessing -from contextlib import nullcontext -from functools import lru_cache -from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, Iterator, List, Optional, Tuple, Union, cast - -import torch.utils.data -from confluid import configurable -from confluid.fluid import Fluid as _ConfluidFluid -from logflow import get_logger - -from dataflux.sample import FEATURES_KEY, SPEC_KEY, TYPE_KEYS, Sample - -if TYPE_CHECKING: # pragma: no cover - typing only - from dataflux.typespec import SampleType - -logger = get_logger(__name__) - - -@lru_cache(maxsize=None) -def _serialized_type_keys(produces: "SampleType") -> Tuple[str, str]: - """Serialize an op's ``PRODUCES`` to the two stored-type JSON strings, memoized per spec so the - ``datasets.Features`` build happens once per distinct spec rather than once per sample.""" - features, extras = produces.to_hf_features() - return json.dumps(features.to_dict()), json.dumps(extras) - - -def _refresh_type(sample: Sample, op: Any) -> Sample: - """Keep a sample's stored type honest after an op — only when the sample already carries one. - - Default (untracked) pipelines never stamp a type, so this is a no-op and metadata is byte-identical - to before. When a stored type IS present (set via :meth:`Sample.with_type` or loaded from a typed - dataset), an op that declares ``PRODUCES`` refreshes it; an op that declares none drops it so - :meth:`Sample.describe` falls back to inference rather than reporting a stale type. - """ - if not any(key in sample.metadata for key in TYPE_KEYS): - return sample - produces = getattr(op, "PRODUCES", None) - if produces is not None: - features_key, spec_key = _serialized_type_keys(produces) - return sample._replace(metadata={**sample.metadata, FEATURES_KEY: features_key, SPEC_KEY: spec_key}) - return sample._replace(metadata={k: v for k, v in sample.metadata.items() if k not in TYPE_KEYS}) - - -def _apply_op(sample: Sample, op: Any) -> Optional[Sample]: - """Apply one op and refresh the stored type. The single op-application chokepoint shared by the - sequential, parallel (via :func:`_worker_task`), streamed, and random-access (``__getitem__``) paths.""" - result = op(sample) - if result is None: - return None - return _refresh_type(result, op) - - -def _describe_deferred_source(source: Any) -> str: - """Return a human-friendly description of a still-deferred Confluid source. - - Surfaces the tag/target so the error explains WHAT was deferred instead - of just noting it isn't a live object. - """ - target = getattr(source, "target", "") - target_name = target if isinstance(target, str) else getattr(target, "__qualname__", str(target)) - return f"{type(source).__name__}(target={target_name!r})" - - -def _fluid_source_guidance(source: Any) -> str: - """Build an actionable message when Flux.source is still a Confluid Fluid.""" - return ( - f"Flux.source is still a deferred Confluid marker: {_describe_deferred_source(source)}. " - "Confluid has not materialized it yet. Fixes: (a) in YAML, write the source as " - "`!class:X()` (with parens) instead of `!class:X` so it becomes an Instance and is " - "materialized at load time; (b) or call `flow(source)` on the source before handing " - "it to Flux." - ) - - -def _fluid_op_guidance(op: Any, index: int) -> str: - """Build an actionable message when a Flux op is still a Confluid Fluid.""" - return ( - f"Flux.ops[{index}] is still a deferred Confluid marker: {_describe_deferred_source(op)}. " - "Ops must be live callables at iteration time. Fixes: (a) in YAML, write each op as " - "`!class:X()` (with parens) so it becomes an Instance and is materialized at load " - "time; (b) or call `flow(op)` on the op before handing it to Flux." - ) - - -def _check_ops_materialized(ops: List[Any]) -> None: - """Raise a single actionable error if any op is still a Confluid Fluid marker.""" - for i, op in enumerate(ops): - if isinstance(op, _ConfluidFluid): - raise TypeError(_fluid_op_guidance(op, i)) - - -@configurable(category="op") -class FilterOp: - """Configurable filter operation.""" - - def __init__(self, p: Callable[[Sample], bool]): - self.p = p - - def __call__(self, s: Sample) -> Optional[Sample]: - return s if self.p(s) else None - - -@configurable(category="op") -class WrappedOp: - """Configurable transformation wrapper with smart mapping.""" - - def __init__(self, f: Union[str, Callable], s: str, kw: Dict[str, Any]): - from dataflux.discovery import get_callable_path - - # EXPLICIT: Always store the string path for serialization - self.f = get_callable_path(f) if callable(f) else f - self.s = s - self.kw = kw - # Internal cache for the live callable - self._func_cache: Optional[Callable] = None - - @property - def func(self) -> Callable: - if self._func_cache is None: - from dataflux.discovery import resolve_callable - - self._func_cache = resolve_callable(self.f) - return self._func_cache - - def __call__(self, sample: Sample) -> Optional[Sample]: - try: - if self.s == "input": - new_input = self.func(sample.input, **self.kw) - return sample._replace(input=new_input) - elif self.s == "target": - new_target = self.func(sample.target, **self.kw) - return sample._replace(target=new_target) - elif self.s == "all": - return cast(Sample, self.func(sample, **self.kw)) - return sample - except Exception as e: - raise e - - -def _worker_task(sample: Sample, ops: List[Any]) -> Optional[Sample]: - """Top-level helper for multiprocess workers. Must be at top level for pickling.""" - current_sample: Optional[Sample] = sample - for op in ops: - if current_sample is None: - return None - current_sample = _apply_op(current_sample, op) - return current_sample - - -@configurable(category="dataset") -class JointFlux: - """ - Aggregates multiple Flux streams into a single joint stream. - Each sub-flux maintains its own unique transformation chain. - """ - - def __init__(self, fluxes: List["Flux"]) -> None: - self.fluxes = fluxes - - def __iter__(self) -> Iterator[Sample]: - """Iterate through all sub-fluxes sequentially.""" - for flux in self.fluxes: - yield from flux - - def __len__(self) -> int: - """Total length is the sum of all sub-fluxes.""" - return sum(len(f) for f in self.fluxes) - - -@configurable(category="dataset") -class Flux(torch.utils.data.Dataset[Sample]): - """ - The primary stream engine for DataFlux. - Wraps any iterable or indexed dataset and provides a functional API. - - Annotation design (kept intentionally ``Any``): - Per the DataFlux mandate "Functional Purity: Transforms are plain - Python callables. Never introduce base classes or complex inheritance - for data operations.", ``source`` is duck-typed (any iterable; the - Indexable protocol if ``__getitem__``/``__len__`` are present) and - ``ops`` is a list of bare callables ``Sample -> Optional[Sample]``. - No ``Source`` or ``Op`` ABC is introduced. - - Downstream auto-gen pydantic mirrors (``confluid.to_pydantic``) - coerce abstract iterable types to ``Any`` so identity-tracked - serialization (e.g. shared-source dataset-split patterns in - navigaitor) works correctly — see - ``confluid/pydantic_export.py:_ITER_TYPES_AS_ANY``. - """ - - def __init__( - self, - source: Optional[Iterable[Any]] = None, - ops: Optional[List[Any]] = None, - chunk_size: Optional[int] = 0, - ) -> None: - self.source = source - self.ops: List[Any] = ops or [] - self._workers = 1 - self._chunk_size = chunk_size or 0 - # Populated on first random access when the source is iterable-only - # (has ``__len__`` but not ``__getitem__``). - self._indexable_cache: Optional[List[Any]] = None - - def _guard_live_source(self) -> Any: - """Return the source, surfacing a clear error when it's still a Fluid marker. - - Flux does not materialize deferred Confluid markers itself — that's - Confluid's job — but if a user hands Flux a deferred Class/Instance - marker we raise with an actionable message instead of letting the - failure surface as ``num_samples=0`` or a generic ``TypeError`` deep - inside torch's DataLoader. - """ - if isinstance(self.source, _ConfluidFluid): - raise TypeError(_fluid_source_guidance(self.source)) - return self.source - - @classmethod - def from_source(cls, source: Any) -> "Flux": - """Create a Flux from a DataSource.""" - return cls(source=source) - - @classmethod - def joint(cls, fluxes: List["Flux"]) -> "Flux": - """Create a new Flux that aggregates multiple other Flux streams.""" - return cls(source=JointFlux(fluxes)) - - def __len__(self) -> int: - """Return the length of the underlying source if available. - - Surfaces a clear error when the source is still a deferred Confluid - marker so downstream callers (e.g. torch's DataLoader) don't end up - reporting the opaque ``num_samples=0``. - """ - from collections.abc import Sized - - source = self._guard_live_source() - if isinstance(source, Sized): - return len(source) - return 0 - - def __getitem__(self, index: int) -> Sample: - """Random access: get the i-th sample with ops applied. - - Supports three source shapes: - - - **Indexable** (``__getitem__`` present) — delegates directly. - - **Iterable with ``__len__``** (map-style-but-stream, like - :class:`waivefront.regions_source.RegionsJsonSource`) — materializes - the full source into a list on first access, caches it on the Flux - instance, and indexes into the cache on every subsequent call. - The list is built once per Flux lifetime, not once per epoch. - - **Bare iterator** (no ``__len__``) — raises ``TypeError``. Caching - a one-shot iterator silently would consume the user's source; if - random access is genuinely needed, either give the source a - ``__len__`` or wrap with ``list(source)`` explicitly at the call - site. - """ - source = self._guard_live_source() - if source is None: - raise TypeError("Flux source is None — cannot index. Pass a DataSource / iterable to Flux(source=...).") - - if hasattr(source, "__getitem__"): - raw = source[index] - elif hasattr(source, "__len__"): - if self._indexable_cache is None: - logger.debug( - f"Flux: materializing iterable-only source " f"{type(source).__name__} for map-style random access." - ) - self._indexable_cache = list(source) - raw = self._indexable_cache[index] - else: - raise TypeError( - f"Flux source {type(source).__name__} does not support indexing and has no __len__ " - "(bare iterator). Map-style DataLoader random access is unsafe on a one-shot " - "iterator; give the source a __len__ (then Flux caches on first access) or wrap " - "it in ``list(...)`` before handing it to Flux." - ) - _check_ops_materialized(self.ops) - sample = Sample.from_any(raw) - for op in self.ops: - result = _apply_op(sample, op) - if result is None: - raise IndexError(f"Sample {index} filtered out by {op}") - sample = result - return sample - - def to_sink(self, sink: Any) -> None: - """Write the entire flux to a DataSink.""" - from dataflux.storage.base import Storage - - # Open sink if it's a context-aware storage; otherwise no-op context. - target_sink: Any = sink if isinstance(sink, Storage) else nullcontext() - - with target_sink: - for sample in self: - sink.write(sample) - sink.flush() - - def parallel(self, workers: int = 4) -> "Flux": - """ - Enable multiprocess execution for the pipeline. - - Args: - workers: Number of worker processes to spawn. - """ - self._workers = workers - return self - - def batch(self, chunk_size: int) -> "Flux": - """ - Group samples into chunks (lists of N samples). - - Args: - chunk_size: Number of samples per chunk. - """ - self._chunk_size = chunk_size - return self - - def map(self, func: Callable, select: str = "input", **kwargs: Any) -> "Flux": - """ - Append a transformation to the flux. - """ - op = WrappedOp(func, select, kwargs) - self.ops.append(op) - return self - - def filter(self, predicate: Callable[[Sample], bool]) -> "Flux": - """Filter the flux based on a predicate.""" - self.ops.append(FilterOp(predicate)) - return self - - def __iter__(self) -> Iterator[Any]: - """Execute the pipeline lazily. - - Routing: - * Any op exposes a callable ``stream`` attribute (e.g. - :class:`dataflux.ops.parallel.Parallel`) → :meth:`_iter_streamed`, - which composes the upstream iterator through stream-level ops. - * Else ``self._workers > 1`` → legacy :meth:`_iter_parallel`. - * Else :meth:`_iter_sequential`. - """ - if not self._guard_live_source(): - return - - if any(hasattr(op, "stream") and callable(op.stream) for op in self.ops): - it = self._iter_streamed() - elif self._workers > 1: - it = self._iter_parallel() - else: - it = self._iter_sequential() - - if self._chunk_size > 0: - batch = [] - for sample in it: - batch.append(sample) - if len(batch) == self._chunk_size: - yield batch - batch = [] - if batch: - yield batch - else: - yield from it - - def _iter_streamed(self) -> Iterator[Sample]: - """Mixed per-sample / stream-level op chain. - - Per-sample ops are applied via ``op(sample)``. Ops that implement - ``.stream(sample_iter)`` (e.g. - :class:`dataflux.ops.parallel.Parallel`) are handed the upstream - generator and yield transformed samples themselves. ``None`` results - are filtered, matching :meth:`_iter_sequential`. - """ - source = self._guard_live_source() - if source is None: - return - _check_ops_materialized(self.ops) - - def to_samples() -> Iterator[Sample]: - for item in source: - yield Sample.from_any(item) - - def per_sample(stream: Iterator[Optional[Sample]], op: Any) -> Iterator[Optional[Sample]]: - for s in stream: - if s is None: - continue - yield _apply_op(s, op) - - stream: Iterator[Optional[Sample]] = to_samples() - for op in self.ops: - if hasattr(op, "stream") and callable(op.stream): - stream = op.stream(stream) - else: - stream = per_sample(stream, op) - - for s in stream: - if s is not None: - yield s - - def _iter_sequential(self) -> Iterator[Sample]: - """Standard single-threaded execution.""" - source = self._guard_live_source() - if source is None: - return - _check_ops_materialized(self.ops) - for item in source: - sample = Sample.from_any(item) - result = _worker_task(sample, self.ops) - if result is not None: - yield result - - def _iter_parallel(self) -> Iterator[Sample]: - """Multiprocess execution engine.""" - source = self._guard_live_source() - if source is None: - return - _check_ops_materialized(self.ops) - - # We use 'spawn' to be consistent with LogFlow and prevent CI deadlocks - ctx = multiprocessing.get_context("spawn") - - with concurrent.futures.ProcessPoolExecutor(max_workers=self._workers, mp_context=ctx) as executor: - futures = [] - for item in source: - sample = Sample.from_any(item) - futures.append(executor.submit(_worker_task, sample, self.ops)) - - for future in futures: - result = future.result() - if result is not None: - yield result - - def collect(self) -> List[Sample]: - """Materialize the full flux into a list.""" - return list(self) diff --git a/dataflux/hf_core.py b/dataflux/hf_core.py deleted file mode 100644 index 471181f..0000000 --- a/dataflux/hf_core.py +++ /dev/null @@ -1,209 +0,0 @@ -"""Hugging Face Datasets based implementation of DataFlux core.""" - -from typing import Any, Callable, Dict, Iterator, List, Union, cast - -import datasets -import numpy as np -import torch -from confluid import configurable -from logflow import get_logger - -from dataflux.sample import Sample - -logger = get_logger(__name__) - - -def _encode_complex(val: Any) -> Any: - """Recursively encode complex types into Arrow-compatible dicts.""" - # Use np.iscomplexobj to catch arrays and scalars (numpy or python) - if np.iscomplexobj(val) or (isinstance(val, torch.Tensor) and val.is_complex()): - kind = "numpy" if not isinstance(val, torch.Tensor) else "torch" - if np.isscalar(val): - scalar = cast(complex, val) - return { - "_complex_": True, - "real": float(scalar.real), - "imag": float(scalar.imag), - "dtype": str(getattr(val, "dtype", "complex128")), - "kind": "scalar", - } - - real = val.real - imag = val.imag - if kind == "torch": - real = real.numpy() - imag = imag.numpy() - dtype = str(val.dtype) - else: - dtype = str(val.dtype) - - return { - "_complex_": True, - "real": real, - "imag": imag, - "dtype": dtype, - "kind": kind, - } - - if isinstance(val, dict): - return {k: _encode_complex(v) for k, v in val.items()} - if isinstance(val, (list, tuple)): - return [_encode_complex(v) for v in val] - return val - - -def _decode_complex(val: Any) -> Any: - """Recursively decode complex types from Arrow-compatible dicts.""" - if isinstance(val, dict) and val.get("_complex_"): - real = val["real"] - imag = val["imag"] - dtype_str = val.get("dtype", "complex128") - kind = val.get("kind", "numpy") - - # When coming back from Arrow/HF, arrays are often lists - if isinstance(real, list): - real = np.array(real) - imag = np.array(imag) - - if kind == "scalar": - return complex(real, imag) - - # Build intermediate numpy complex array - res_np = real + 1j * imag - if kind == "torch": - torch_dtype = torch.complex64 if "complex64" in dtype_str else torch.complex128 - # Convert to appropriate numpy complex first to ensure precision, then to torch - np_dtype = np.complex64 if "complex64" in dtype_str else np.complex128 - return torch.from_numpy(res_np.astype(np_dtype)).to(torch_dtype) - - return res_np.astype(dtype_str) - - if isinstance(val, dict): - return {k: _decode_complex(v) for k, v in val.items()} - if isinstance(val, list): - return [_decode_complex(v) for v in val] - return val - - -def _sample_to_dict(sample: Sample) -> Dict[str, Any]: - """Flatten Sample triplet into a dictionary for HF datasets.""" - d = { - "input": _encode_complex(sample.input), - "target": _encode_complex(sample.target), - } - # Flatten metadata - if sample.metadata: - for k, v in sample.metadata.items(): - # Avoid collisions with reserved keys - if k in ("input", "target"): - k = f"meta_{k}" - d[k] = _encode_complex(v) - return d - - -def _dict_to_sample(d: Dict[str, Any]) -> Sample: - """Reconstruct Sample triplet from an HF dataset row.""" - input_val = _decode_complex(d.get("input")) - target_val = _decode_complex(d.get("target")) - metadata = {k: _decode_complex(v) for k, v in d.items() if k not in ("input", "target")} - # Unflatten meta_ collisions - if "meta_input" in metadata: - metadata["input"] = metadata.pop("meta_input") - if "meta_target" in metadata: - metadata["target"] = metadata.pop("meta_target") - return Sample(input=input_val, target=target_val, metadata=metadata) - - -def wrap_op(op: Callable) -> Callable[[Dict[str, Any]], Dict[str, Any]]: - """Wrap a legacy Sample-based op so it can be used with HF datasets.map().""" - - def wrapper(row: Dict[str, Any]) -> Dict[str, Any]: - sample = _dict_to_sample(row) - result = op(sample) - if result is None: - # For HF filter ops or flat maps - return row # Filtering is separate in HF - return _sample_to_dict(result) - - return wrapper - - -@configurable -class HFFlux: - """ - Flux engine backed by Hugging Face `datasets`. - Provides a high-performance implementation of the DataFlux API. - """ - - def __init__(self, dataset: Union[datasets.Dataset, datasets.IterableDataset]) -> None: - self._dataset = dataset - - @property - def dataset(self) -> Union[datasets.Dataset, datasets.IterableDataset]: - return self._dataset - - @classmethod - def from_source(cls, source: Any) -> "HFFlux": - """Create an HFFlux from any iterable source. - - An in-memory sequence (``list``/``tuple``) is already fully - materialized, so we build a map-style ``datasets.Dataset`` for it — - this enables ``len()`` and integer indexing without writing any cache - files. This does NOT violate the lazy-evaluation mandate: the data is - already in memory, so there is no lazy iterator left to consume eagerly. - - Any other source (generator, streaming DataFlux source) is treated as - lazy and backed by an ``IterableDataset`` to avoid materializing it and - to avoid large cache files / permission issues on external drives - (like /Volumes/Store). - """ - if hasattr(source, "_dataset") and isinstance(source._dataset, (datasets.Dataset, datasets.IterableDataset)): - return cls(source._dataset) - - if isinstance(source, (list, tuple)): - rows = [_sample_to_dict(Sample.from_any(item)) for item in source] - return cls(datasets.Dataset.from_list(rows)) - - def gen() -> Iterator[Dict[str, Any]]: - for item in source: - yield _sample_to_dict(Sample.from_any(item)) - - ds = datasets.IterableDataset.from_generator(gen) - return cls(ds) - - def map(self, func: Callable, **kwargs: Any) -> "HFFlux": - """ - Apply a transformation to the dataset. - If `func` expects a Sample, it should be wrapped. - """ - # Default behavior: assume func handles dicts (HF style) - # We can add a 'wrapper' mode if needed for compatibility with old ops. - return HFFlux(self._dataset.map(func, **kwargs)) - - def filter(self, predicate: Callable, **kwargs: Any) -> "HFFlux": - """Filter the dataset.""" - return HFFlux(self._dataset.filter(predicate, **kwargs)) - - def parallel(self, workers: int = 4) -> "HFFlux": - """Set default number of processes for subsequent operations.""" - # datasets.map uses `num_proc` parameter. - # We can store this or apply it to the next operation. - # For now, this is just for API compatibility. - return self - - def __iter__(self) -> Iterator[Sample]: - for row in self._dataset: - yield _dict_to_sample(row) - - def __len__(self) -> int: - return len(self._dataset) - - def __getitem__(self, index: int) -> Sample: - return _dict_to_sample(self._dataset[index]) - - def collect(self) -> List[Sample]: - return list(self) - - def info(self) -> Any: - """Expose HF dataset info (schema, metadata).""" - return self._dataset.info diff --git a/dataflux/ops/__init__.py b/dataflux/ops/__init__.py deleted file mode 100644 index 12097e9..0000000 --- a/dataflux/ops/__init__.py +++ /dev/null @@ -1,38 +0,0 @@ -""" -DataFlux operations. - -Submodules: - - dataflux.ops.numpy: RescaleOp, StandardizeOp, ClipPercentilesOp, - ReplaceNonFiniteOp, ThresholdOp, ConnectedComponentsOp (ndarray) - - dataflux.ops.torch: RescaleOp, StandardizeOp, ToTensorOp (tensor) - - dataflux.ops.tee: Tee (fan-out branching) - - dataflux.ops.parallel: Parallel (worker-pool sub-pipeline) - - dataflux.ops.copy: CopySampleOp, CopyInputOp, CopyTargetOp, CopyMetadataOp - - dataflux.ops.swap: SwapInputTargetOp - - dataflux.ops.stash: StashInputOp, UnstashInputOp - -Flat imports default to torch variants for the data ops; flow / copy / -swap / stash utilities are field-agnostic. -""" - -from dataflux.ops.copy import CopyInputOp, CopyMetadataOp, CopySampleOp, CopyTargetOp -from dataflux.ops.parallel import Parallel -from dataflux.ops.stash import StashInputOp, UnstashInputOp -from dataflux.ops.swap import SwapInputTargetOp -from dataflux.ops.tee import Tee -from dataflux.ops.torch import RescaleOp, StandardizeOp, ToTensorOp - -__all__ = [ - "CopyInputOp", - "CopyMetadataOp", - "CopySampleOp", - "CopyTargetOp", - "Parallel", - "RescaleOp", - "StandardizeOp", - "StashInputOp", - "SwapInputTargetOp", - "Tee", - "ToTensorOp", - "UnstashInputOp", -] diff --git a/dataflux/ops/copy.py b/dataflux/ops/copy.py deleted file mode 100644 index 6a4c333..0000000 --- a/dataflux/ops/copy.py +++ /dev/null @@ -1,55 +0,0 @@ -"""Defensive deepcopy ops. - -Use these when a downstream op mutates ``sample.input`` / ``sample.target`` -in place and you want subsequent branches (or external references) to see -the pre-mutation value. ``Tee`` shares a single ``Sample`` across all -branches by design, so isolation is opt-in via these ops. -""" - -import copy -from typing import Any - -from confluid import configurable - -from dataflux.sample import Sample - - -@configurable -class CopySampleOp: - """Deepcopy of input, target, and metadata.""" - - def __call__(self, sample: Sample) -> Sample: - return Sample( - input=copy.deepcopy(sample.input), - target=copy.deepcopy(sample.target), - metadata=copy.deepcopy(sample.metadata), - ) - - -@configurable -class CopyInputOp: - """Deepcopy ``sample.input``.""" - - def __call__(self, sample: Sample) -> Sample: - return sample._replace(input=copy.deepcopy(sample.input)) - - -@configurable -class CopyTargetOp: - """Deepcopy ``sample.target``.""" - - def __call__(self, sample: Sample) -> Sample: - return sample._replace(target=copy.deepcopy(sample.target)) - - -@configurable -class CopyMetadataOp: - """Deepcopy ``sample.metadata``. - - The replacement dict is a fresh object, so subsequent in-place writes - on the new metadata won't be seen by other holders of the old dict. - """ - - def __call__(self, sample: Sample) -> Sample: - new_meta: Any = copy.deepcopy(sample.metadata) - return sample._replace(metadata=new_meta) diff --git a/dataflux/ops/numpy.py b/dataflux/ops/numpy.py deleted file mode 100644 index fd37034..0000000 --- a/dataflux/ops/numpy.py +++ /dev/null @@ -1,350 +0,0 @@ -import os -import re -from typing import List, Sequence, Tuple, Union - -import numpy as np -from confluid import configurable -from logflow import get_logger - -from dataflux.sample import Sample -from dataflux.typespec import ArrayType, PythonType, SampleType, UnionType - -# Common shorthands for the numpy ops' declared types. -_NDARRAY = ArrayType(frameworks={"numpy"}) -_NUMERIC_OR_PIL = UnionType((ArrayType(dtype="numeric", frameworks={"numpy"}), PythonType("PIL.Image.Image"))) - -logger = get_logger(__name__) - - -_EXPR_PATTERN = re.compile(r"\{(\w+)\}|\$(\w+)") - - -def resolve_expression(value: str, sample: Sample) -> str: - """Substitute ``{key}`` from ``sample.metadata`` and ``$NAME`` from ``os.environ``. - - Returns the substituted string verbatim — the caller is responsible for - any further casting (e.g. ``float(...)`` for a numeric expression). - - Examples: - ``"5.5"`` → ``"5.5"`` (no substitution) - ``"{reference_snr_level}"`` → ``str(metadata["reference_snr_level"])`` - ``"-{reference_snr_level}"`` → ``"-"`` (sign passes through to ``float()``) - ``"$REF_SNR"`` → ``os.environ["REF_SNR"]`` - - Raises: - KeyError: A referenced metadata key or environment variable is missing. - """ - - def _repl(match: "re.Match[str]") -> str: - meta_key = match.group(1) - env_name = match.group(2) - if meta_key is not None: - if meta_key not in sample.metadata: - raise KeyError( - f"resolve_expression: metadata key {meta_key!r} missing in {value!r}; " - f"available keys: {sorted(sample.metadata)}" - ) - return str(sample.metadata[meta_key]) - assert env_name is not None - if env_name not in os.environ: - raise KeyError(f"resolve_expression: environment variable {env_name!r} missing in {value!r}") - return os.environ[env_name] - - return _EXPR_PATTERN.sub(_repl, value) - - -@configurable -class StandardizeOp: - """ - Standardizes ndarray values with given mean and standard deviation. - - Formula: output = (input - mean) / std - - mean/std can be a single float (applied uniformly) or a sequence of - per-channel values that broadcasts over [C, H, W] format. - - Handles PIL images by converting to ndarray first. - """ - - ACCEPTS = SampleType(input=_NUMERIC_OR_PIL) - PRODUCES = SampleType(input=ArrayType(dtype="floating", frameworks={"numpy"})) - - def __init__(self, mean: Union[float, Sequence[float]], std: Union[float, Sequence[float]]): - self.mean = mean - self.std = std - - def __call__(self, sample: Sample) -> Sample: - arr = sample.input - - # Handle PIL / PngImageFile - if hasattr(arr, "convert"): - arr = np.array(arr) - - if not isinstance(arr, np.ndarray): - raise TypeError(f"StandardizeOp expects an np.ndarray, got {type(arr).__name__}") - - if arr.dtype == np.float64: - arr = arr.astype(np.float64) - else: - arr = arr.astype(np.float32) - - if isinstance(self.mean, (int, float)): - mean_a = np.array([self.mean], dtype=arr.dtype) - else: - mean_a = np.array(self.mean, dtype=arr.dtype) - - if isinstance(self.std, (int, float)): - std_a = np.array([self.std], dtype=arr.dtype) - else: - std_a = np.array(self.std, dtype=arr.dtype) - - # Reshape to [C, 1, 1, ...] for broadcasting over [C, H, W] - mean_a = mean_a.reshape(-1, *([1] * (arr.ndim - 1))) - std_a = std_a.reshape(-1, *([1] * (arr.ndim - 1))) - - arr = (arr - mean_a) / std_a - - return sample._replace(input=arr) - - -def _require_ndarray(sample: Sample, op_name: str) -> np.ndarray: - arr = sample.input - if not isinstance(arr, np.ndarray): - raise TypeError(f"{op_name} expects an np.ndarray on sample.input, got {type(arr).__name__}") - return arr - - -@configurable -class ClipPercentilesOp: - """Clip ``sample.input`` to ``[p_low, p_high]`` percentiles of finite values. - - Percentiles are computed over only finite entries — ``inf`` / ``-inf`` / - ``nan`` are excluded from the percentile estimate. ``np.clip`` then maps - ``+inf`` to the upper bound and ``-inf`` to the lower bound; ``nan`` - survives unchanged. Chain :class:`ReplaceNonFiniteOp` upstream if remaining - ``nan`` values matter. - - Args: - low: Lower percentile in ``[0, 100)``. Default ``2.0``. - high: Upper percentile in ``(0, 100]``, must be ``> low``. Default ``98.0``. - """ - - ACCEPTS = SampleType(input=_NDARRAY) - PRODUCES = SampleType(input=_NDARRAY) - - def __init__(self, low: float = 2.0, high: float = 98.0) -> None: - if not (0.0 <= low < high <= 100.0): - raise ValueError(f"ClipPercentilesOp: require 0 <= low < high <= 100; got low={low}, high={high}") - self.low = float(low) - self.high = float(high) - - def __call__(self, sample: Sample) -> Sample: - arr = _require_ndarray(sample, "ClipPercentilesOp") - finite = np.isfinite(arr) - if not finite.any(): - logger.warning("ClipPercentilesOp: input is entirely non-finite; passing through") - return sample - lo = float(np.percentile(arr[finite], self.low)) - hi = float(np.percentile(arr[finite], self.high)) - return sample._replace(input=np.clip(arr, lo, hi)) - - -@configurable -class RescaleOp: - """Affine rescale ``sample.input`` from ``[in_min, in_max]`` to ``[out_min, out_max]``. - - The default ``out_min=0.0`` / ``out_max=1.0`` covers the common - ``[0, 255] -> [0, 1]`` image-normalization case. PIL inputs are - converted to ndarray; integer dtypes are promoted to ``float32`` - (``float64`` is preserved). - - Args: - in_min: Lower edge of the input range. Required. - in_max: Upper edge of the input range, must be ``> in_min``. Required. - out_min: Lower edge of the output range. Default ``0.0``. - out_max: Upper edge of the output range, must be ``> out_min``. Default ``1.0``. - clip: When True (default), clamp values outside ``[in_min, in_max]`` - before rescaling. When False, extrapolate linearly. - """ - - ACCEPTS = SampleType(input=_NUMERIC_OR_PIL) - PRODUCES = SampleType(input=ArrayType(dtype="floating", frameworks={"numpy"})) - - def __init__( - self, - in_min: float, - in_max: float, - out_min: float = 0.0, - out_max: float = 1.0, - clip: bool = True, - ) -> None: - if not (in_min < in_max): - raise ValueError(f"RescaleOp: require in_min < in_max; got in_min={in_min}, in_max={in_max}") - if not (out_min < out_max): - raise ValueError(f"RescaleOp: require out_min < out_max; got out_min={out_min}, out_max={out_max}") - self.in_min = float(in_min) - self.in_max = float(in_max) - self.out_min = float(out_min) - self.out_max = float(out_max) - self.clip = bool(clip) - - def __call__(self, sample: Sample) -> Sample: - arr = sample.input - if hasattr(arr, "convert"): - arr = np.array(arr) - if not isinstance(arr, np.ndarray): - raise TypeError(f"RescaleOp expects an np.ndarray, got {type(arr).__name__}") - arr = arr.astype(np.float64 if arr.dtype == np.float64 else np.float32) - src = np.clip(arr, self.in_min, self.in_max) if self.clip else arr - scaled = (src - self.in_min) / (self.in_max - self.in_min) - out = scaled * (self.out_max - self.out_min) + self.out_min - return sample._replace(input=out) - - -@configurable -class ReplaceNonFiniteOp: - """Replace ``inf`` / ``-inf`` / ``nan`` entries in ``sample.input``. - - Args: - value: Replacement specifier. Either: - - * a ``float`` / ``int`` — literal replacement value; - * the string ``"min"`` — replace with the array's finite min; - * the string ``"max"`` — replace with the array's finite max. - - Default ``"min"``. - """ - - ACCEPTS = SampleType(input=_NDARRAY) - PRODUCES = SampleType(input=_NDARRAY) - - def __init__(self, value: Union[float, int, str] = "min") -> None: - if isinstance(value, str) and value not in ("min", "max"): - raise ValueError(f"ReplaceNonFiniteOp: value string must be 'min' or 'max'; got {value!r}") - self.value = value - - def __call__(self, sample: Sample) -> Sample: - arr = _require_ndarray(sample, "ReplaceNonFiniteOp") - non_finite = ~np.isfinite(arr) - if not non_finite.any(): - return sample - if isinstance(self.value, str): - finite = ~non_finite - if not finite.any(): - logger.warning("ReplaceNonFiniteOp: array is entirely non-finite; passing through") - return sample - finite_values = arr[finite] - repl = float(finite_values.min() if self.value == "min" else finite_values.max()) - else: - repl = float(self.value) - return sample._replace(input=np.where(non_finite, repl, arr)) - - -@configurable -class ThresholdOp: - """Threshold ``sample.input`` (ndarray) into a boolean mask: ``input > value``. - - ``value`` is either a numeric literal or a string expression resolved via - :func:`resolve_expression` against ``sample.metadata`` and ``os.environ``: - - * ``5.5`` or ``"5.5"`` — fixed threshold - * ``"{reference_snr_level}"`` — looks up ``metadata["reference_snr_level"]`` - * ``"-{reference_snr_level}"`` — negated lookup (the leading ``-`` is - carried through ``float(...)`` after substitution) - * ``"$REF_SNR"`` / ``"-$REF_SNR"`` — environment-variable lookup - - Records the resolved threshold under ``metadata["threshold"]`` for traceability. - """ - - ACCEPTS = SampleType(input=_NDARRAY) - PRODUCES = SampleType(input=ArrayType(dtype="bool", frameworks={"numpy"})) - - def __init__(self, value: Union[float, int, str] = 0.0) -> None: - self.value = value - - def _resolve(self, sample: Sample) -> float: - if isinstance(self.value, (int, float)): - return float(self.value) - if not isinstance(self.value, str): - raise TypeError(f"ThresholdOp.value must be a number or expression string; got {type(self.value).__name__}") - resolved = resolve_expression(self.value, sample) - try: - return float(resolved) - except ValueError as exc: - raise ValueError( - f"ThresholdOp: expression {self.value!r} resolved to {resolved!r}, " f"which is not a number" - ) from exc - - def __call__(self, sample: Sample) -> Sample: - arr = sample.input - if not isinstance(arr, np.ndarray): - raise TypeError(f"ThresholdOp expects an np.ndarray on sample.input, got {type(arr).__name__}") - threshold = self._resolve(sample) - sample.metadata["threshold"] = threshold - return sample._replace(input=arr > threshold) - - -@configurable -class ConnectedComponentsOp: - """Label connected ``True`` regions of a boolean mask into bin-bbox tuples. - - Reads ``sample.input`` as a 2-D boolean ndarray; writes ``sample.input`` as - a list of ``(row_min, row_max, col_min, col_max)`` integer tuples (inclusive - bounds). Components smaller than ``min_area_bins`` are dropped. - - ``connectivity`` selects the neighborhood: - - * ``4`` — orthogonal neighbors only (N/S/E/W); diagonally touching - components stay separate. - * ``8`` — orthogonal + diagonal neighbors; diagonally touching - components merge. - - Requires ``scipy`` (install via ``pip install data-flux[vision]``). - """ - - ACCEPTS = SampleType(input=ArrayType(ndim=2, dtype="bool", frameworks={"numpy"})) - PRODUCES = SampleType(input=PythonType("list")) - - def __init__(self, min_area_bins: int = 1, connectivity: int = 4) -> None: - if min_area_bins < 1: - raise ValueError(f"min_area_bins must be >= 1; got {min_area_bins!r}") - if connectivity not in (4, 8): - raise ValueError(f"connectivity must be 4 or 8; got {connectivity!r}") - self.min_area_bins = int(min_area_bins) - self.connectivity = int(connectivity) - - def __call__(self, sample: Sample) -> Sample: - try: - from scipy.ndimage import find_objects, generate_binary_structure, label - except ImportError as exc: - raise ImportError( - "ConnectedComponentsOp requires scipy. " - "Install with `pip install data-flux[vision]` or add scipy to your environment." - ) from exc - - mask = sample.input - if not isinstance(mask, np.ndarray): - raise TypeError(f"ConnectedComponentsOp expects an np.ndarray on sample.input, got {type(mask).__name__}") - if mask.ndim != 2: - raise ValueError(f"ConnectedComponentsOp expects a 2-D mask; got shape {mask.shape}") - - structure = generate_binary_structure(2, 1 if self.connectivity == 4 else 2) - labels, n_components = label(mask, structure=structure) - bboxes: List[Tuple[int, int, int, int]] = [] - if n_components > 0: - for idx, sl in enumerate(find_objects(labels), start=1): - if sl is None: - continue - row_slice, col_slice = sl - area = int((labels[row_slice, col_slice] == idx).sum()) - if area < self.min_area_bins: - continue - bboxes.append( - ( - int(row_slice.start), - int(row_slice.stop) - 1, - int(col_slice.start), - int(col_slice.stop) - 1, - ) - ) - return sample._replace(input=bboxes) diff --git a/dataflux/ops/stash.py b/dataflux/ops/stash.py deleted file mode 100644 index 9adcfef..0000000 --- a/dataflux/ops/stash.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Stash / unstash ``sample.input`` to / from ``metadata``. - -Use ``StashInputOp(key)`` to snapshot the current ``sample.input`` under a -metadata key without changing ``sample.input``. Use ``UnstashInputOp(key)`` -later (e.g. inside another ``Tee`` branch) to restore that value into -``sample.input``. - -``UnstashInputOp`` defaults to ``copy=True`` (deepcopy) so two branches -that both unstash the same key are independent — each gets its own array -to mutate. Without the copy, an in-place op like ``ClipPercentilesOp`` -in the first branch would silently corrupt the stashed value seen by the -second branch. -""" - -import copy as _copy - -from confluid import configurable - -from dataflux.sample import Sample - - -@configurable -class StashInputOp: - """Copy ``sample.input`` into ``metadata[key]``; ``sample.input`` unchanged. - - Args: - key: Metadata key to write. - copy: When ``True``, deepcopy ``sample.input`` before stashing. - Defaults to ``False`` (cheap pointer alias) — the typical case - is that downstream ops use ``sample._replace(input=...)`` and - don't mutate the shared array in place. - """ - - def __init__(self, key: str, copy: bool = False) -> None: - self.key = key - self.copy = copy - - def __call__(self, sample: Sample) -> Sample: - sample.metadata[self.key] = _copy.deepcopy(sample.input) if self.copy else sample.input - return sample - - -@configurable -class UnstashInputOp: - """Set ``sample.input := metadata[key]``. - - Args: - key: Metadata key to read. - copy: When ``True`` (default), deepcopy the stashed value before - assigning. This prevents two branches that unstash the same - key from corrupting each other through downstream in-place - mutations. Set ``False`` only when the caller has audited - that no downstream op mutates the array in place. - """ - - def __init__(self, key: str, copy: bool = True) -> None: - self.key = key - self.copy = copy - - def __call__(self, sample: Sample) -> Sample: - value = sample.metadata[self.key] - if self.copy: - value = _copy.deepcopy(value) - return sample._replace(input=value) diff --git a/dataflux/ops/swap.py b/dataflux/ops/swap.py deleted file mode 100644 index 142e85a..0000000 --- a/dataflux/ops/swap.py +++ /dev/null @@ -1,17 +0,0 @@ -"""``SwapInputTargetOp`` — exchange ``sample.input`` and ``sample.target``. - -Useful when an op operates on ``sample.input`` but you want it applied to -the target instead: swap, run the op, swap back. -""" - -from confluid import configurable - -from dataflux.sample import Sample - - -@configurable -class SwapInputTargetOp: - """Exchange ``sample.input`` ↔ ``sample.target``. Metadata unchanged.""" - - def __call__(self, sample: Sample) -> Sample: - return sample._replace(input=sample.target, target=sample.input) diff --git a/dataflux/ops/tee.py b/dataflux/ops/tee.py deleted file mode 100644 index 9a70fed..0000000 --- a/dataflux/ops/tee.py +++ /dev/null @@ -1,47 +0,0 @@ -"""``Tee`` — fan-out side effects across N branches on a single sample. - -Branches run sequentially on the **same** ``Sample`` object and **same** -``metadata`` dict. There are no automatic copies; if isolation is needed, -place an explicit ``CopyInputOp`` / ``CopySampleOp`` at the start of a -branch. Later branches see what earlier branches wrote to ``metadata``. - -If any op in any branch returns ``None``, ``Tee`` propagates ``None`` -(consistent with ``FilterOp`` semantics — the whole sample is dropped). -""" - -from typing import Any, List, Optional - -from confluid import configurable, flow -from confluid.fluid import Fluid - -from dataflux.sample import Sample - - -@configurable -class Tee: - """Run N op-list branches sequentially on the same sample / metadata.""" - - def __init__(self, branches: List[List[Any]]) -> None: - self.branches = [list(b) for b in branches] - - def __call__(self, sample: Sample) -> Optional[Sample]: - current: Optional[Sample] = sample - for branch in self.branches: - for i, op in enumerate(branch): - if current is None: - return None - if isinstance(op, Fluid): - op = flow(op) - branch[i] = op - if op is None: - continue - current = op(current) - return current - - def close(self) -> None: - """Propagate close() to inner ops that own resources.""" - for branch in self.branches: - for op in branch: - close_fn = getattr(op, "close", None) - if callable(close_fn): - close_fn() diff --git a/dataflux/ops/torch.py b/dataflux/ops/torch.py deleted file mode 100644 index c4e7443..0000000 --- a/dataflux/ops/torch.py +++ /dev/null @@ -1,150 +0,0 @@ -from typing import Sequence, Union - -import numpy as np -import torch -from confluid import configurable - -from dataflux.sample import Sample -from dataflux.typespec import ArrayType, PythonType, SampleType, UnionType - -_TORCH = ArrayType(frameworks={"torch"}) -_TORCH_FLOAT = ArrayType(dtype="floating", frameworks={"torch"}) - - -@configurable -class ToTensorOp: - """ - Converts input (PIL Image, NumPy array, etc.) to a Torch Tensor. - """ - - ACCEPTS = SampleType(input=UnionType((PythonType("PIL.Image.Image"), ArrayType(frameworks={"numpy"})))) - PRODUCES = SampleType(input=_TORCH) - - def __init__(self, normalize: bool = True): - self.normalize = normalize - - def __call__(self, sample: Sample) -> Sample: - img = sample.input - - # Handle PIL / PngImageFile - if hasattr(img, "convert"): - # Ensure grayscale or RGB as needed, but for generic we just array it - img = np.array(img) - - # Convert to Tensor - if isinstance(img, np.ndarray): - # Standard Vision format: [H, W, C] -> [C, H, W] - if img.ndim == 3: - img = img.transpose(2, 0, 1) - elif img.ndim == 2: - img = img[np.newaxis, :] - - tensor = torch.from_numpy(img) - else: - tensor = torch.as_tensor(img) - - # Normalize 0-255 to 0-1 - if self.normalize and tensor.dtype == torch.uint8: - tensor = tensor.float() / 255.0 - elif self.normalize and tensor.max() > 1.0: - # Fallback for floats that are still in 0-255 range - tensor = tensor / 255.0 - - return sample._replace(input=tensor) - - -@configurable -class RescaleOp: - """Affine rescale a torch.Tensor from ``[in_min, in_max]`` to ``[out_min, out_max]``. - - The default ``out_min=0.0`` / ``out_max=1.0`` covers the common - ``[0, 255] -> [0, 1]`` image-normalization case. Integer dtypes are - promoted to ``float32`` (``float64`` is preserved). - - Args: - in_min: Lower edge of the input range. Required. - in_max: Upper edge of the input range, must be ``> in_min``. Required. - out_min: Lower edge of the output range. Default ``0.0``. - out_max: Upper edge of the output range, must be ``> out_min``. Default ``1.0``. - clip: When True (default), clamp values outside ``[in_min, in_max]`` - before rescaling. When False, extrapolate linearly. - """ - - ACCEPTS = SampleType(input=_TORCH) - PRODUCES = SampleType(input=_TORCH_FLOAT) - - def __init__( - self, - in_min: float, - in_max: float, - out_min: float = 0.0, - out_max: float = 1.0, - clip: bool = True, - ) -> None: - if not (in_min < in_max): - raise ValueError(f"RescaleOp: require in_min < in_max; got in_min={in_min}, in_max={in_max}") - if not (out_min < out_max): - raise ValueError(f"RescaleOp: require out_min < out_max; got out_min={out_min}, out_max={out_max}") - self.in_min = float(in_min) - self.in_max = float(in_max) - self.out_min = float(out_min) - self.out_max = float(out_max) - self.clip = bool(clip) - - def __call__(self, sample: Sample) -> Sample: - tensor = sample.input - if not isinstance(tensor, torch.Tensor): - raise TypeError(f"RescaleOp expects a torch.Tensor, got {type(tensor).__name__}") - if tensor.dtype != torch.float32 and tensor.dtype != torch.float64: - tensor = tensor.float() - src = tensor.clamp(self.in_min, self.in_max) if self.clip else tensor - scaled = (src - self.in_min) / (self.in_max - self.in_min) - out = scaled * (self.out_max - self.out_min) + self.out_min - return sample._replace(input=out) - - -@configurable -class StandardizeOp: - """ - Standardizes tensor values with given mean and standard deviation. - - Formula: output = (input - mean) / std - - mean/std can be a single float (applied uniformly) or a sequence of - per-channel values that broadcasts over [C, H, W] format. - """ - - ACCEPTS = SampleType(input=_TORCH) - PRODUCES = SampleType(input=_TORCH_FLOAT) - - def __init__(self, mean: Union[float, Sequence[float]], std: Union[float, Sequence[float]]): - self.mean = mean - self.std = std - - def __call__(self, sample: Sample) -> Sample: - tensor = sample.input - - if not isinstance(tensor, torch.Tensor): - raise TypeError(f"StandardizeOp expects a torch.Tensor, got {type(tensor).__name__}") - - if tensor.dtype != torch.float32 and tensor.dtype != torch.float64: - tensor = tensor.float() - - mean_t = torch.tensor( - [self.mean] if isinstance(self.mean, (int, float)) else self.mean, - dtype=tensor.dtype, - device=tensor.device, - ) - std_t = torch.tensor( - [self.std] if isinstance(self.std, (int, float)) else self.std, - dtype=tensor.dtype, - device=tensor.device, - ) - - # Reshape to [C, 1, 1, ...] for broadcasting over [C, H, W] - mean_t = mean_t.view(-1, *([1] * (tensor.ndim - 1))) - std_t = std_t.view(-1, *([1] * (tensor.ndim - 1))) - - tensor = (tensor - mean_t) / std_t - - return sample._replace(input=tensor) diff --git a/dataflux/paired.py b/dataflux/paired.py deleted file mode 100644 index bd3e30a..0000000 --- a/dataflux/paired.py +++ /dev/null @@ -1,197 +0,0 @@ -"""Keyed-join source pairing a primary DataSource with a secondary annotation mapping.""" - -from typing import Any, Callable, Dict, Iterator, Optional, Tuple, Union - -from confluid import configurable -from logflow import get_logger - -from dataflux.discovery import get_callable_path, resolve_callable -from dataflux.sample import Sample - -logger = get_logger(__name__) - -VALID_POLICIES = ("left_outer", "inner", "right_driven") - - -@configurable -class PairedSource: - """Pair a primary DataSource with a secondary annotation mapping via a key function. - - Produces ``Sample`` values where the matched annotation record is flattened into - ``Sample.metadata``. Supports three join policies: - - - ``left_outer``: iterate primary; attach annotation when the key matches, - otherwise emit the sample unannotated. Preserves primary's ``__len__`` and - ``__getitem__``. (Scenario A.) - - ``inner``: same as left_outer, filtered to annotated samples only. - (Scenario B.) - - ``right_driven``: iterate ``secondary.keys()``; resolve each primary sample - via ``primary_resolver(key, primary)``. Use when annotations are sparse. - - Coarser-granularity joins are expressed by returning a coarser key from - ``key_fn`` so multiple primary samples map to the same annotation record. - Use ``extract_fn`` to project the record down to each sample's scope (e.g. - trim a pack-level time-ranged annotation to a single window). Returning - ``None`` from ``extract_fn`` marks the sample as unannotated. - - Args: - primary: Any iterable (or ``DataSource``) yielding raw items that - ``Sample.from_any`` can coerce into samples. - secondary: For ``left_outer``/``inner``, a mapping-like object supporting - ``__contains__`` and ``__getitem__``. For ``right_driven``, an object - also supporting ``keys()``. - key_fn: ``"module:function"`` path (or a callable) producing the join key - from a sample. Signature: ``(sample: Sample) -> str``. - policy: One of ``"left_outer"``, ``"inner"``, ``"right_driven"``. - extract_fn: Optional ``"module:function"`` path (or callable) called as - ``extract_fn(record, sample) -> dict | None`` to project the record - per sample. Returning ``None`` marks the sample unannotated. - prefix: Optional string prefix applied to every annotation field when - flattening into ``Sample.metadata``. - store_full_under: If set, also stash the (extracted) record under - ``Sample.metadata[store_full_under]``. - primary_resolver: Required for ``right_driven``. ``"module:function"`` - path (or callable) invoked as ``primary_resolver(key, primary)`` - to fetch the primary sample for a given annotation key. - """ - - def __init__( - self, - primary: Any, - secondary: Any, - key_fn: Union[str, Callable[[Sample], str]], - policy: str = "left_outer", - extract_fn: Optional[Union[str, Callable[[Dict[str, Any], Sample], Optional[Dict[str, Any]]]]] = None, - prefix: str = "", - store_full_under: Optional[str] = None, - primary_resolver: Optional[Union[str, Callable[[str, Any], Any]]] = None, - ) -> None: - if policy not in VALID_POLICIES: - raise ValueError(f"Invalid policy {policy!r}; must be one of {VALID_POLICIES}") - - if policy in ("left_outer", "inner"): - if not hasattr(secondary, "__contains__") or not hasattr(secondary, "__getitem__"): - raise TypeError( - f"policy={policy!r} requires secondary to support __contains__ and __getitem__; " - f"got {type(secondary).__name__}" - ) - - if policy == "right_driven": - if primary_resolver is None: - raise ValueError("policy='right_driven' requires primary_resolver") - if not hasattr(secondary, "keys"): - raise TypeError( - f"policy='right_driven' requires secondary to support keys(); " f"got {type(secondary).__name__}" - ) - - self.primary = primary - self.secondary = secondary - self.key_fn = get_callable_path(key_fn) if callable(key_fn) else key_fn - self.policy = policy - self.extract_fn = get_callable_path(extract_fn) if callable(extract_fn) else extract_fn - self.prefix = prefix - self.store_full_under = store_full_under - self.primary_resolver = get_callable_path(primary_resolver) if callable(primary_resolver) else primary_resolver - - self._key_fn_cache: Optional[Callable[[Sample], str]] = None - self._extract_fn_cache: Optional[Callable[[Dict[str, Any], Sample], Optional[Dict[str, Any]]]] = None - self._primary_resolver_cache: Optional[Callable[[str, Any], Any]] = None - self._inner_length: Optional[int] = None - - @property - def _resolved_key_fn(self) -> Callable[[Sample], str]: - if self._key_fn_cache is None: - self._key_fn_cache = resolve_callable(self.key_fn) - return self._key_fn_cache - - @property - def _resolved_extract_fn( - self, - ) -> Optional[Callable[[Dict[str, Any], Sample], Optional[Dict[str, Any]]]]: - if self.extract_fn is None: - return None - if self._extract_fn_cache is None: - self._extract_fn_cache = resolve_callable(self.extract_fn) - return self._extract_fn_cache - - @property - def _resolved_primary_resolver(self) -> Callable[[str, Any], Any]: - if self.primary_resolver is None: - raise ValueError("primary_resolver is not set") - if self._primary_resolver_cache is None: - self._primary_resolver_cache = resolve_callable(self.primary_resolver) - return self._primary_resolver_cache - - def _attach(self, sample: Sample, record: Optional[Dict[str, Any]], key: str) -> Sample: - metadata = dict(sample.metadata) - metadata["annotation_key"] = key - metadata["annotated"] = record is not None - - if record is not None: - for k, v in record.items(): - metadata[f"{self.prefix}{k}"] = v - if self.store_full_under is not None: - metadata[self.store_full_under] = record - - return sample._replace(metadata=metadata) - - def _lookup(self, sample: Sample) -> Tuple[str, Optional[Dict[str, Any]]]: - key = self._resolved_key_fn(sample) - if key not in self.secondary: - return key, None - record: Optional[Dict[str, Any]] = self.secondary[key] - extract_fn = self._resolved_extract_fn - if extract_fn is not None and record is not None: - record = extract_fn(record, sample) - return key, record - - def __iter__(self) -> Iterator[Sample]: - if self.policy == "right_driven": - yield from self._iter_right_driven() - return - - for item in self.primary: - sample = Sample.from_any(item) - key, record = self._lookup(sample) - if self.policy == "inner" and record is None: - continue - yield self._attach(sample, record, key) - - def _iter_right_driven(self) -> Iterator[Sample]: - resolver = self._resolved_primary_resolver - for key in self.secondary.keys(): - raw = resolver(key, self.primary) - sample = Sample.from_any(raw) - record: Optional[Dict[str, Any]] = self.secondary[key] - extract_fn = self._resolved_extract_fn - if extract_fn is not None and record is not None: - record = extract_fn(record, sample) - if record is None: - # extract_fn signalled "not applicable"; skip this entry - continue - yield self._attach(sample, record, key) - - def __len__(self) -> int: - if self.policy == "left_outer": - return len(self.primary) - if self.policy == "right_driven": - return len(list(self.secondary.keys())) - - if self._inner_length is None: - count = 0 - for item in self.primary: - sample = Sample.from_any(item) - _, record = self._lookup(sample) - if record is not None: - count += 1 - self._inner_length = count - return self._inner_length - - def __getitem__(self, index: int) -> Sample: - if self.policy != "left_outer": - raise TypeError(f"__getitem__ is only supported for policy='left_outer'; got {self.policy!r}") - if not hasattr(self.primary, "__getitem__"): - raise TypeError("primary must support __getitem__ for PairedSource.__getitem__") - sample = Sample.from_any(self.primary[index]) - key, record = self._lookup(sample) - return self._attach(sample, record, key) diff --git a/dataflux/sample.py b/dataflux/sample.py deleted file mode 100644 index 4998844..0000000 --- a/dataflux/sample.py +++ /dev/null @@ -1,70 +0,0 @@ -import json -from typing import TYPE_CHECKING, Any, Dict, NamedTuple, Tuple - -if TYPE_CHECKING: # pragma: no cover - typing only - from dataflux.typespec import SampleType - -# Reserved metadata keys carrying a sample's stored type description (JSON strings so they survive -# every storage backend's metadata round-trip — HDF5 attrs / Zarr attrs / Directory YAML / HF / Confluid). -# ``__features__`` holds a ``datasets.Features`` dict (the standard, concrete structural description); -# ``__spec__`` holds the sidecar refinements Features can't express (framework / ranges / Any / Union). -FEATURES_KEY = "__features__" -SPEC_KEY = "__spec__" -TYPE_KEYS = (FEATURES_KEY, SPEC_KEY) - - -# Standardized Sample: (input, target, metadata) -# This allows DataFlux to handle complex pipelines while remaining -# compatible with simple PyTorch/HF (input, target) pairs. -class Sample(NamedTuple): - input: Any - target: Any = None - metadata: Dict[str, Any] = {} - - def to_tuple(self) -> Tuple[Any, Any, Dict[str, Any]]: - return (self.input, self.target, self.metadata) - - def describe(self) -> "SampleType": - """Return this sample's :class:`~dataflux.typespec.SampleType`. - - Prefers the stored type (the reserved metadata keys, set explicitly via :meth:`with_type` or - carried by a serialized dataset); otherwise infers it from the live ``input`` / ``target``. - """ - from dataflux.typespec import SampleType, infer_sample_type - - raw_features = self.metadata.get(FEATURES_KEY) - raw_extras = self.metadata.get(SPEC_KEY) - if raw_features is not None or raw_extras is not None: - features = json.loads(raw_features) if isinstance(raw_features, str) else (raw_features or {}) - extras = json.loads(raw_extras) if isinstance(raw_extras, str) else raw_extras - return SampleType.from_hf_features(features, extras) - return infer_sample_type(self) - - def with_type(self, sample_type: "SampleType") -> "Sample": - """Return a copy carrying ``sample_type`` in the reserved metadata keys (copy-on-write, so the - original sample's metadata is not mutated).""" - features, extras = sample_type.to_hf_features() - metadata = {**self.metadata, FEATURES_KEY: json.dumps(features.to_dict()), SPEC_KEY: json.dumps(extras)} - return self._replace(metadata=metadata) - - @classmethod - def from_any(cls, obj: Any) -> "Sample": - """Coerce raw data from various sources into a Sample.""" - if isinstance(obj, cls): - return obj - if isinstance(obj, tuple): - if len(obj) >= 3: - return cls(obj[0], obj[1], obj[2] or {}) - if len(obj) == 2: - return cls(obj[0], obj[1], {}) - if len(obj) == 1: - return cls(obj[0], None, {}) - # Empty tuple - return cls(None, None, {}) - if isinstance(obj, dict): - return cls( - input=obj.get("input"), - target=obj.get("target"), - metadata=obj.get("metadata", {}), - ) - return cls(obj, None, {}) diff --git a/dataflux/sources.py b/dataflux/sources.py deleted file mode 100644 index 1dff834..0000000 --- a/dataflux/sources.py +++ /dev/null @@ -1,231 +0,0 @@ -import random -from typing import Any, Iterator, List, Optional - -from confluid import configurable -from logflow import get_logger - -from dataflux.sample import Sample - -logger = get_logger(__name__) - - -@configurable -class HuggingFaceSource: - """ - DataFlux Source for Hugging Face Datasets. - Configurable mapping of dataset features to DataFlux Sample triplets. - """ - - def __init__( - self, - path: str, - split: str = "train", - input_feature: str = "image", - target_feature: str = "label", - metadata_features: Optional[List[str]] = None, - count: Optional[int] = None, - name: Optional[str] = None, - **kwargs: Any, - ) -> None: - from datasets import load_dataset - - self.path = path - self.split = split - self.input_feature = input_feature - self.target_feature = target_feature - self.metadata_features = metadata_features or [] - self.count = count - - logger.info(f"HuggingFaceSource: Loading {path} ({split})...") - self._dataset = load_dataset(path, name=name, split=split, **kwargs) - - def __iter__(self) -> Iterator[Sample]: - counter = 0 - limit = self.count or len(self._dataset) - - for item in self._dataset: - if counter >= limit: - break - - # 1. Extract Input - input_val = item.get(self.input_feature) - - # 2. Extract Target - target_val = item.get(self.target_feature) - - # 3. Build Metadata - metadata = {f: item.get(f) for f in self.metadata_features} - metadata["hf_path"] = self.path - metadata["hf_split"] = self.split - - yield Sample(input=input_val, target=target_val, metadata=metadata) - counter += 1 - - def __getitem__(self, index: int) -> Sample: - item = self._dataset[index] - metadata = {f: item.get(f) for f in self.metadata_features} - metadata["hf_path"] = self.path - metadata["hf_split"] = self.split - return Sample( - input=item.get(self.input_feature), - target=item.get(self.target_feature), - metadata=metadata, - ) - - def __len__(self) -> int: - # A ``count`` of 0 (or None) means "all samples", matching __iter__'s - # ``limit = self.count or len(...)``. Returning a bare ``self.count`` here - # would report 0 for the common "0 == unlimited" case, making the source - # look empty (e.g. a downstream len()-based stepper raising ``len == 0``) - # even though iteration yields every sample. - return self.count or len(self._dataset) - - -@configurable -class DatasetSplit: - """ - Selects a subset view of an indexable source (e.g. ``HuggingFaceSource``). - - Supports three mutually exclusive modes: - - 1. **Fraction mode** — pick a reproducible train/val split from a single source. - Pass ``split`` (``"train"`` or ``"val"``), ``val_fraction``, and ``seed``. - Two ``DatasetSplit`` instances sharing a source (by Python identity or - by ``!ref:`` in YAML) and the *same* ``seed`` + ``val_fraction`` yield - disjoint, complementary views. With Confluid ``!ref:``, the underlying - source is loaded exactly once and shared by identity between the splits. - - 2. **Range mode** — plain index slice ``[start:end)`` over the source. - Pass ``start`` and/or ``end``. - - 3. **Passthrough** — no split args yields a full view (rarely useful, - mostly for symmetry in YAML templates). - - The wrapped source must implement ``__len__`` and ``__getitem__``. - Lazy evaluation is preserved: only index arithmetic happens at - construction time; samples are produced on demand. - - Args: - source: The underlying indexable source. - split: ``"train"`` or ``"val"`` (fraction mode only). - val_fraction: Fraction of samples assigned to the ``"val"`` view - (fraction mode). Must be in ``(0, 1)``. - seed: Seed for the deterministic shuffle (fraction mode). - Required when ``val_fraction`` is set so that sibling splits - stay consistent. - start: Inclusive start index (range mode). - end: Exclusive end index (range mode). - - Example (fraction mode, YAML):: - - hf_train: !class:dataflux.sources.HuggingFaceSource() - path: mnist - split: train - - train_set: !class:dataflux.sources.DatasetSplit() - source: !ref:hf_train - split: train - val_fraction: 0.1 - seed: 42 - - val_set: !class:dataflux.sources.DatasetSplit() - source: !ref:hf_train - split: val - val_fraction: 0.1 - seed: 42 - - Alternative (no shared load, HuggingFace native slicing):: - - # Loads the HF dataset twice — kept for clarity if sharing is not desired. - train_src: !class:dataflux.sources.HuggingFaceSource() - path: mnist - split: "train[:90%]" - val_src: !class:dataflux.sources.HuggingFaceSource() - path: mnist - split: "train[90%:]" - """ - - def __init__( - self, - source: Any, - split: Optional[str] = None, - val_fraction: Optional[float] = None, - seed: Optional[int] = None, - start: Optional[int] = None, - end: Optional[int] = None, - ) -> None: - if not hasattr(source, "__len__") or not hasattr(source, "__getitem__"): - raise TypeError( - "DatasetSplit requires a source supporting __len__ and __getitem__; " f"got {type(source).__name__}" - ) - - fraction_args = val_fraction is not None or split is not None - range_args = start is not None or end is not None - if fraction_args and range_args: - raise ValueError( - "DatasetSplit accepts either fraction-mode args (split, val_fraction, seed) " - "or range-mode args (start, end), not both." - ) - - self.source = source - self.split = split - self.val_fraction = val_fraction - self.seed = seed - self.start = start - self.end = end - - self._indices: List[int] = self._compute_indices() - logger.debug( - "DatasetSplit: mode=%s size=%d source_size=%d", - "fraction" if fraction_args else ("range" if range_args else "passthrough"), - len(self._indices), - len(source), - ) - - def _compute_indices(self) -> List[int]: - n = len(self.source) - - # Fraction mode - if self.val_fraction is not None or self.split is not None: - if self.val_fraction is None: - raise ValueError("DatasetSplit fraction mode requires `val_fraction`.") - if self.seed is None: - raise ValueError("DatasetSplit fraction mode requires `seed` so that sibling splits stay consistent.") - if not (0.0 < self.val_fraction < 1.0): - raise ValueError(f"val_fraction must be in (0, 1); got {self.val_fraction}") - if self.split not in ("train", "val"): - raise ValueError(f"split must be 'train' or 'val' in fraction mode; got {self.split!r}") - - rng = random.Random(self.seed) - shuffled = list(range(n)) - rng.shuffle(shuffled) - val_count = max(1, int(round(n * self.val_fraction))) - train_count = n - val_count - if self.split == "train": - return shuffled[:train_count] - return shuffled[train_count:] - - # Range mode - if self.start is not None or self.end is not None: - start = 0 if self.start is None else self.start - end = n if self.end is None else self.end - if start < 0: - start = max(0, n + start) - if end < 0: - end = max(0, n + end) - start = max(0, min(start, n)) - end = max(start, min(end, n)) - return list(range(start, end)) - - # Passthrough - return list(range(n)) - - def __iter__(self) -> Iterator[Sample]: - for idx in self._indices: - yield Sample.from_any(self.source[idx]) - - def __getitem__(self, index: int) -> Sample: - return Sample.from_any(self.source[self._indices[index]]) - - def __len__(self) -> int: - return len(self._indices) diff --git a/dataflux/storage/base.py b/dataflux/storage/base.py deleted file mode 100644 index 609ef49..0000000 --- a/dataflux/storage/base.py +++ /dev/null @@ -1,45 +0,0 @@ -from typing import Any, Iterator, Protocol, runtime_checkable - -from dataflux.sample import Sample - - -@runtime_checkable -class DataSource(Protocol): - """Minimum contract for a DataFlux data source.""" - - def __iter__(self) -> Iterator[Sample]: - """Iterate over samples in the source.""" - ... - - def __len__(self) -> int: - """Total number of samples available.""" - ... - - -@runtime_checkable -class DataSink(Protocol): - """Minimum contract for a DataFlux data sink.""" - - def write(self, sample: Sample) -> None: - """Write a single sample to the sink.""" - ... - - def flush(self) -> None: - """Ensure all pending writes are committed to storage.""" - ... - - -class Storage: - """Base class for storage backends providing context manager support.""" - - def open(self) -> "Storage": - return self - - def close(self) -> None: - pass # pragma: no cover - - def __enter__(self) -> "Storage": - return self.open() - - def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: - self.close() diff --git a/dataflux/storage/directory.py b/dataflux/storage/directory.py deleted file mode 100644 index 53d8517..0000000 --- a/dataflux/storage/directory.py +++ /dev/null @@ -1,59 +0,0 @@ -from pathlib import Path -from typing import Union - -import confluid -import numpy as np - -from dataflux.sample import Sample -from dataflux.storage.base import DataSink, Storage - - -@confluid.configurable -class DirectorySink(Storage, DataSink): - """ - High-concurrency sink that stores each Sample in its own directory. - Perfect for irregular data lengths and massive parallel writing. - """ - - def __init__(self, path: Union[str, Path], overwrite: bool = False, use_npz: bool = True) -> None: - self.path = Path(path) - self.overwrite = overwrite - self.use_npz = use_npz - self._counter = 0 - - def open(self) -> "DirectorySink": - if self.overwrite and self.path.exists(): - # In a real app, we'd clear the directory - pass - self.path.mkdir(parents=True, exist_ok=True) - return self - - def write(self, sample: Sample) -> None: - """Write a sample to its own subdirectory.""" - # Use a zero-padded index for sorting - sample_dir = self.path / f"{self._counter:06d}" - sample_dir.mkdir(parents=True, exist_ok=True) - - # 1. Save Metadata (YAML via Confluid) - if sample.metadata: - meta_path = sample_dir / "metadata.yaml" - meta_path.write_text(confluid.dump(sample.metadata)) - - # 2. Save Input and Target (Numpy) - if self.use_npz: - # Combined file - np.savez( - sample_dir / "sample.npz", - data=sample.input, - target=sample.target if sample.target is not None else np.array([]), - ) - else: - # Separate files - np.save(sample_dir / "data.npy", sample.input) - if sample.target is not None: - np.save(sample_dir / "target.npy", sample.target) - - self._counter += 1 - - def flush(self) -> None: - pass # Filesystem handles immediate writes diff --git a/dataflux/storage/hdf5.py b/dataflux/storage/hdf5.py deleted file mode 100644 index 53c5031..0000000 --- a/dataflux/storage/hdf5.py +++ /dev/null @@ -1,134 +0,0 @@ -from pathlib import Path -from typing import Any, Iterator, Optional, Union - -import h5py -import torch -from confluid import configurable -from logflow import get_logger - -from dataflux.sample import Sample -from dataflux.storage.base import DataSink, DataSource, Storage - -logger = get_logger("dataflux.storage.hdf5") - - -def to_numpy(data: Any) -> Any: - """Utility to convert torch tensors to numpy arrays for HDF5 storage.""" - if isinstance(data, torch.Tensor): - return data.detach().cpu().numpy() - return data - - -@configurable -class HDF5Source(Storage, DataSource): - """Clean, high-performance HDF5 data source.""" - - def __init__( - self, - path: Union[str, Path], - sample_key: str = "data", - target_key: Optional[str] = "target", - ) -> None: - self.path = Path(path) - self.sample_key = sample_key - self.target_key = target_key - self._file: Optional[h5py.File] = None - - def open(self) -> "HDF5Source": - if self._file is None: - self._file = h5py.File(self.path, "r") - return self - - def close(self) -> None: - if self._file: - self._file.close() - self._file = None - - def __iter__(self) -> Iterator[Sample]: - self.open() - if self._file is None: - return - - prefixes = sorted([k.split("_data")[0] for k in self._file.keys() if k.endswith("_data")]) - - for pref in prefixes: - data = self._file[f"{pref}_data"][()] - target = self._file[f"{pref}_target"][()] if f"{pref}_target" in self._file else None - metadata = dict(self._file[f"{pref}_data"].attrs) - # Source returns Tensors to match schema - yield Sample(input=torch.from_numpy(data), target=target, metadata=metadata) - - def __len__(self) -> int: - self.open() - if self._file is None: - return 0 - return len([k for k in self._file.keys() if k.endswith("_data")]) - - -@configurable -class HDF5Sink(Storage, DataSink): - """High-performance HDF5 data sink focused on Sample triplets.""" - - def __init__( - self, - path: Union[str, Path], - compression: Optional[str] = "gzip", - overwrite: bool = False, - ) -> None: - self.path = Path(path) - self.compression = compression - self.overwrite = overwrite - self._file: Optional[h5py.File] = None - self._counter = 0 - - def open(self) -> "HDF5Sink": - if self._file is None: - mode = "w" if self.overwrite and self._counter == 0 else "a" - self.path.parent.mkdir(parents=True, exist_ok=True) - logger.info(f"Opening HDF5 file for writing: {self.path} (mode={mode})") - self._file = h5py.File(self.path, mode) - return self - - def close(self) -> None: - if self._file: - self._file.close() - self._file = None - - def write(self, sample: Sample) -> None: - self.open() - if self._file is None: - return - - prefix = f"{self._counter:05d}" - - # Convert tensors to numpy for h5py - input_data = to_numpy(sample.input) - target_data = to_numpy(sample.target) - - # 1. Write Data - kwargs = {} - if self.compression and hasattr(input_data, "shape") and len(input_data.shape) > 0: - kwargs["compression"] = self.compression - - ds = self._file.create_dataset(f"{prefix}_data", data=input_data, **kwargs) - - # 2. Write Attributes (Metadata) - for k, v in sample.metadata.items(): - try: - ds.attrs[k] = v - except Exception: - ds.attrs[k] = str(v) - - # 3. Write Target - if target_data is not None: - t_kwargs = {} - if self.compression and hasattr(target_data, "shape") and len(target_data.shape) > 0: - t_kwargs["compression"] = self.compression - - self._file.create_dataset(f"{prefix}_target", data=target_data, **t_kwargs) - - self._counter += 1 - - def flush(self) -> None: - if self._file: - self._file.flush() diff --git a/dataflux/storage/intake.py b/dataflux/storage/intake.py deleted file mode 100644 index 9e96edf..0000000 --- a/dataflux/storage/intake.py +++ /dev/null @@ -1,152 +0,0 @@ -"""Generic adapter that exposes any intake DataSource as a DataFlux DataSource. - -This bridge lets DataFlux pipelines consume any data exposed via the -`intake `_ catalog system. It is deliberately -container-agnostic: ``xarray.DataArray`` / ``xarray.Dataset``, ``numpy.ndarray``, -``pandas.DataFrame``, and arbitrary Python objects all map cleanly onto the -DataFlux ``Sample(input, target, metadata)`` triplet. - -The ``intake`` import is lazy so DataFlux's hard dependency surface is unchanged -— users only pay the cost when they actually construct an ``IntakeSource``. -""" - -from pathlib import Path -from typing import Any, Iterator, Optional, Union - -import confluid -import numpy as np -import torch -from logflow import get_logger - -from dataflux.sample import Sample -from dataflux.storage.base import DataSource, Storage - -logger = get_logger(__name__) - -_INTAKE_INSTALL_HINT = "IntakeSource requires the 'intake' package. " "Install it with: pip install 'intake>=2.0'" - - -@confluid.configurable -class IntakeSource(Storage, DataSource): - """Wrap an intake DataSource so it satisfies the DataFlux ``DataSource`` protocol. - - Each intake partition becomes one ``Sample``. Mapping rules: - - - ``xarray.DataArray`` / ``xarray.Dataset``: ``Sample.input`` is the - underlying numpy array (wrapped as a torch tensor when possible), - ``Sample.metadata`` is ``dict(da.attrs)``, and if ``target_attr`` is - set, ``Sample.target`` is read from ``attrs[target_attr]``. - - ``numpy.ndarray``: ``Sample.input`` is the array as a torch tensor; - no target, empty metadata. - - Everything else: routed through :py:meth:`dataflux.sample.Sample.from_any`. - - Construct with either a (catalog_path, source_name) pair (preferred for - Confluid serialization symmetry) or a pre-instantiated intake source via - ``source`` (handy for tests and ad-hoc use). - - Args: - catalog_path: Path to an intake catalog YAML. - source_name: Key inside the catalog identifying the source. - source: Pre-instantiated intake DataSource. Mutually exclusive with - the (catalog_path, source_name) pair. - target_attr: Name of an attribute key whose value should be lifted to - ``Sample.target`` for xarray containers. ``None`` means - no target. - """ - - def __init__( - self, - catalog_path: Optional[Union[str, Path]] = None, - source_name: Optional[str] = None, - source: Any = None, - target_attr: Optional[str] = None, - ) -> None: - if source is None and (catalog_path is None or source_name is None): - raise ValueError( - "IntakeSource requires either a (catalog_path, source_name) pair " - "or a pre-instantiated `source` object." - ) - if source is not None and (catalog_path is not None or source_name is not None): - raise ValueError("IntakeSource: pass either (catalog_path, source_name) or `source`, not both.") - - try: - import intake # noqa: F401 - except ImportError as e: # pragma: no cover - exercised via test_missing_intake_dep - raise ImportError(_INTAKE_INSTALL_HINT) from e - - self.catalog_path: Optional[str] = str(catalog_path) if catalog_path is not None else None - self.source_name = source_name - self.target_attr = target_attr - self._source = source - self._catalog: Any = None - - def _resolve_source(self) -> Any: - if self._source is not None: - return self._source - import intake - - logger.info(f"IntakeSource: opening catalog {self.catalog_path!r}, source {self.source_name!r}") - self._catalog = intake.open_catalog(self.catalog_path) - self._source = self._catalog[self.source_name] - return self._source - - def open(self) -> "IntakeSource": - self._resolve_source() - return self - - def close(self) -> None: - src = self._source - if src is not None and hasattr(src, "_close"): - try: - src._close() - except Exception as e: # pragma: no cover - defensive - logger.warning(f"IntakeSource: source close raised {e!r}") - # Only drop catalog-derived sources; user-supplied ones stay alive. - if self._catalog is not None: - self._source = None - self._catalog = None - - def __iter__(self) -> Iterator[Sample]: - src = self._resolve_source() - npart = self._npartitions(src) - for i in range(npart): - yield self._to_sample(src.read_partition(i)) - - def __len__(self) -> int: - return self._npartitions(self._resolve_source()) - - @staticmethod - def _npartitions(src: Any) -> int: - # intake 2.x lazily populates the schema; npartitions reads 0 until discover() runs. - if hasattr(src, "discover"): - try: - src.discover() - except Exception as e: # pragma: no cover - defensive - logger.debug(f"IntakeSource: discover() raised {e!r}") - n = getattr(src, "npartitions", None) - if n is None or n == 0: - schema = src._get_schema() - n = getattr(schema, "npartitions", 1) or 1 - return int(n) - - def _to_sample(self, value: Any) -> Sample: - # xarray: DataArray and Dataset both expose .values + .attrs. - if hasattr(value, "values") and hasattr(value, "attrs"): - attrs = dict(value.attrs) - data = value.values - tensor = self._array_to_tensor(data) - target = attrs.get(self.target_attr) if self.target_attr else None - return Sample(input=tensor, target=target, metadata=attrs) - if isinstance(value, np.ndarray): - return Sample(input=self._array_to_tensor(value), target=None, metadata={}) - return Sample.from_any(value) - - @staticmethod - def _array_to_tensor(array: np.ndarray) -> Any: - # Complex dtypes are preserved; non-complex go through torch.from_numpy directly. - try: - return torch.from_numpy(array) - except TypeError: - # Older torch versions can't ingest some dtypes (e.g. complex on CPU < 1.8). - # Fall back to returning the numpy array; downstream ops can handle it. - return array diff --git a/dataflux/storage/zarr.py b/dataflux/storage/zarr.py deleted file mode 100644 index 0a4ace3..0000000 --- a/dataflux/storage/zarr.py +++ /dev/null @@ -1,120 +0,0 @@ -from pathlib import Path -from typing import List, Optional, Union - -import confluid -import zarr - -from dataflux.sample import Sample -from dataflux.storage.base import DataSink, Storage - - -@confluid.configurable -class ZarrGroupSink(Storage, DataSink): - """ - Stores each sample as a unique array within a Zarr group. - Supports variable lengths while keeping data in a single bundle. - """ - - def __init__(self, path: Union[str, Path], overwrite: bool = False) -> None: - self.path = str(path) - self.overwrite = overwrite - self._root: Optional[zarr.Group] = None - self._counter = 0 - - def open(self) -> "ZarrGroupSink": - if self._root is None: - self._root = zarr.open_group(self.path, mode="a") - if self.overwrite: - # In a real app, we'd clear the group - pass - return self - - def write(self, sample: Sample) -> None: - self.open() - if self._root is None: - raise RuntimeError("Zarr group not open") - # Use require_group to handle existing nodes safely - name = f"sample_{self._counter:06d}" - grp = self._root.require_group(name) - - # 1. Save data and target (Explicit shape/dtype for Zarr v3) - # Use require_dataset or delete if exists - if "data" in grp: - del grp["data"] - grp.create_dataset( - "data", - data=sample.input, - shape=sample.input.shape, - dtype=sample.input.dtype, - ) - - if sample.target is not None: - if "target" in grp: - del grp["target"] - grp.create_dataset( - "target", - data=sample.target, - shape=sample.target.shape, - dtype=sample.target.dtype, - ) - - # 2. Save metadata as Zarr attributes (.zattrs) - if sample.metadata: - grp.attrs.update(sample.metadata) - - self._counter += 1 - - def flush(self) -> None: - pass # pragma: no cover - - -@confluid.configurable -class ZarrBatchSink(Storage, DataSink): - """ - Optimized for uniform data. Appends samples into a single large Zarr array. - """ - - def __init__( - self, - path: Union[str, Path], - shape: List[int], - dtype: str = "float32", - chunks: Optional[List[int]] = None, - overwrite: bool = False, - ) -> None: - self.path = str(path) - self.shape = tuple(shape) - self.dtype = dtype - self.chunks = tuple(chunks) if chunks else None - self.overwrite = overwrite - self._data_arr: Optional[zarr.Array] = None - self._target_arr: Optional[zarr.Array] = None - self._counter = 0 - - def open(self) -> "ZarrBatchSink": - if self._data_arr is None: - # We create a resizable array (unlimited along first dimension) - self._data_arr = zarr.open_array( - store=f"{self.path}/data", - mode="a" if not self.overwrite else "w", - shape=(0,) + self.shape, - chunks=(1,) + self.shape if not self.chunks else self.chunks, - dtype=self.dtype, - ) - return self - - def write(self, sample: Sample) -> None: - self.open() - if self._data_arr is None: - raise RuntimeError("Zarr array not open") - # Append to the primary array - # Zarr handles the resizing and chunking internally - self._data_arr.append([sample.input], axis=0) - - # Note: Handling metadata in a single-array sink requires - # a separate attribute list or sidecar file. - # For simplicity, we attach to the array attributes. - self._counter += 1 - - def flush(self) -> None: - pass # pragma: no cover diff --git a/dataflux/typespec.py b/dataflux/typespec.py deleted file mode 100644 index e23ee42..0000000 --- a/dataflux/typespec.py +++ /dev/null @@ -1,769 +0,0 @@ -"""Type-spec system: describe and match the types flowing through a :class:`~dataflux.sample.Sample`. - -Two gaps this fills: - -1. A ``Sample`` carries no description of *what kind of data* sits in its ``input`` / ``target``. -2. Ops and sources don't declare which input/target types they accept or produce. - -The model is a small set of frozen value objects (no base class — see DataFlux "Functional Purity" -mandate; these are values, not data ops) describing one slot: - -* :class:`AnyType` — matches everything (the default when nothing is declared). -* :class:`ArrayType` — an N-D array/tensor across frameworks (numpy / torch / tensorflow): optional - rank, per-axis :class:`Dim` constraints (exact / bounded-range / unbounded), optional dtype - (concrete ``"float32"`` or a *family* ``"floating"`` / ``"integer"`` / ``"numeric"`` …), optional - framework set, optional ``semantic`` tag. ``ArrayType.image(...)`` is a convenience for images. -* :class:`PythonType` — a non-array Python value by qualname (``"PIL.Image.Image"``, ``"dict"`` …). -* :class:`UnionType` — any-of. -* :class:`MappingType` / :class:`ListType` — mirror ``datasets`` struct / ``Sequence`` so structured - targets (e.g. detection ``{boxes, labels}``) get a real type and bridge 1:1 to HF ``Features``. -* :class:`SampleType` — the ``(input, target)`` pair an op/source declares or a sample reports. - -**Matching** is asymmetric (covariant): ``consumer.accepts(producer)`` is True iff every concrete -value the producer can emit is acceptable to the consumer. Two flavours share the leaf logic: - -* ``accepts`` — *strict*; used by the runtime check where the producer is a concrete inferred type. -* ``compatible`` — *permissive*; used at edit-time (FluxStudio canvas) and for discovery filtering: - ``Any``/unknown on **either** side ⇒ compatible (honours "if not defined, assume Any"), and an - unbounded producer axis against a bounded consumer axis is a soft-pass (the runtime check still - catches an actual out-of-range value). - -Prior art borrowed rather than reinvented: the ``None``-dim base case mirrors ``tf.TensorShape`` -(``Dim`` adds the bounded-range extension tf/HF/jaxtyping lack); dtype family names align with -``numpy.isdtype`` / the array-api taxonomy; ``ArrayType.parse`` accepts a jaxtyping-style shape -string; and :meth:`SampleType.from_hf_features` / :meth:`SampleType.to_hf_features` bridge to the -``datasets.Features`` we already depend on (used for the concrete per-sample stored type). - -Everything is JSON round-trippable (``to_dict`` / :func:`type_from_dict` / :func:`sampletype_from_dict`) -so specs ride the discovery manifest and can be re-implemented by FluxStudio's JS connection-validator. -""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import TYPE_CHECKING, AbstractSet, Any, Callable, Dict, FrozenSet, List, Optional, Tuple, TypeVar, Union - -import numpy as np - -if TYPE_CHECKING: # pragma: no cover - typing only - from dataflux.sample import Sample - -# A single-slot type spec. Defined as a forward-ref union so leaf classes can annotate it before the -# alias is bound at runtime (annotations are strings under ``from __future__ import annotations``). -TypeSpec = Union["AnyType", "ArrayType", "PythonType", "UnionType", "MappingType", "ListType"] - -C = TypeVar("C") - -__all__ = [ - "Dim", - "AnyType", - "ArrayType", - "PythonType", - "UnionType", - "MappingType", - "ListType", - "SampleType", - "TypeSpec", - "typed", - "accepts", - "compatible", - "infer_type", - "infer_sample_type", - "canonical_dtype", - "type_to_dict", - "type_from_dict", - "sampletype_from_dict", -] - - -# -------------------------------------------------------------------------------------------------- -# dtype canonicalization + families (array-api-aligned names) -# -------------------------------------------------------------------------------------------------- - -_DTYPE_ALIASES: Dict[str, str] = { - "double": "float64", - "single": "float32", - "half": "float16", - "bool_": "bool", - "boolean": "bool", -} - -_FLOATING = {"float16", "bfloat16", "float32", "float64"} -_INTEGER = {"int8", "int16", "int32", "int64"} -_UNSIGNED = {"uint8", "uint16", "uint32", "uint64"} -_COMPLEX = {"complex64", "complex128"} -_DTYPE_FAMILIES: Dict[str, FrozenSet[str]] = { - "floating": frozenset(_FLOATING), - "integer": frozenset(_INTEGER), - "unsigned": frozenset(_UNSIGNED), - "bool": frozenset({"bool"}), - "complex": frozenset(_COMPLEX), - "numeric": frozenset(_FLOATING | _INTEGER | _UNSIGNED), -} - - -def canonical_dtype(x: Any) -> str: - """Normalize a dtype (str, numpy dtype/scalar-type, or torch dtype) to a canonical lowercase name. - - Family names (``"floating"``, ``"integer"``, ``"numeric"`` …) pass through unchanged so they can - be used as relaxed dtype constraints on an :class:`ArrayType`. - """ - if isinstance(x, str): - s = x.lower() - return _DTYPE_ALIASES.get(s, s) - if isinstance(x, np.dtype): - return str(x.name) - if isinstance(x, type) and issubclass(x, np.generic): - return str(np.dtype(x).name) - # torch dtype (lazy — torch is a hard dep but we avoid importing it at module load) - try: - import torch - - if isinstance(x, torch.dtype): - return str(x).replace("torch.", "") - except ImportError: # pragma: no cover - torch is a hard dep - pass - return str(x).lower() - - -def _dtype_accepts(consumer: str, producer: str) -> bool: - """True iff a ``producer`` concrete/family dtype satisfies a ``consumer`` dtype constraint.""" - c = canonical_dtype(consumer) - p = canonical_dtype(producer) - if c == p: - return True - fam = _DTYPE_FAMILIES.get(c) - if fam is not None: - # producer may itself be a (sub)family name or a concrete member - if p in fam: - return True - pfam = _DTYPE_FAMILIES.get(p) - return pfam is not None and pfam <= fam - return False - - -# -------------------------------------------------------------------------------------------------- -# Spec value objects -# -------------------------------------------------------------------------------------------------- - - -@dataclass(frozen=True) -class Dim: - """A single axis-size constraint: the closed interval ``[min, max]`` (``None`` = unbounded). - - ``name`` is informational only (never matched). ``Dim.any()`` is exactly tf's ``None`` dim. - """ - - min: Optional[int] = None - max: Optional[int] = None - name: Optional[str] = None - - @classmethod - def exact(cls, n: int, name: Optional[str] = None) -> "Dim": - return cls(n, n, name) - - @classmethod - def any(cls, name: Optional[str] = None) -> "Dim": - return cls(None, None, name) - - @classmethod - def range(cls, lo: Optional[int], hi: Optional[int], name: Optional[str] = None) -> "Dim": - return cls(lo, hi, name) - - def accepts(self, other: "Dim") -> bool: - """Strict: ``other``'s whole possible interval lies within this interval.""" - lo = self.min if self.min is not None else 0 - o_lo = other.min if other.min is not None else 0 - if o_lo < lo: - return False - if self.max is not None and (other.max is None or other.max > self.max): - return False - return True - - def compatible(self, other: "Dim") -> bool: - """Permissive: the two intervals could overlap (an unbounded ``other`` is a soft-pass).""" - c_lo = self.min if self.min is not None else 0 - o_lo = other.min if other.min is not None else 0 - if self.max is not None and o_lo > self.max: - return False - if other.max is not None and other.max < c_lo: - return False - return True - - def to_dict(self) -> Dict[str, Any]: - return {"min": self.min, "max": self.max, "name": self.name} - - @classmethod - def from_dict(cls, d: Dict[str, Any]) -> "Dim": - return cls(d.get("min"), d.get("max"), d.get("name")) - - -@dataclass(frozen=True) -class AnyType: - """Matches everything; the default when an op/source/sample declares no type.""" - - def to_dict(self) -> Dict[str, Any]: - return {"kind": "any"} - - -@dataclass(frozen=True) -class ArrayType: - """An N-D array/tensor constraint across numpy / torch / tensorflow. - - All fields are optional and each ``None`` means "unconstrained": - - * ``ndim`` — required rank (derived from ``shape`` when that is given). - * ``shape`` — per-axis :class:`Dim` tuple. - * ``dtype`` — canonical name (``"float32"``) or family (``"floating"``/``"numeric"`` …). - * ``frameworks`` — allowed framework set (``{"numpy", "torch"}`` …). - * ``semantic`` — free-form tag (e.g. ``"image"``); display/inference hint, never matched. - """ - - ndim: Optional[int] = None - shape: Optional[Tuple[Dim, ...]] = None - dtype: Optional[str] = None - # Accept any set on construction (ergonomic ``frameworks={"torch"}``); ``__post_init__`` freezes it. - frameworks: Optional[AbstractSet[str]] = None - semantic: Optional[str] = None - - def __post_init__(self) -> None: - if self.shape is not None: - shape = tuple(self.shape) - object.__setattr__(self, "shape", shape) - if self.ndim is None: - object.__setattr__(self, "ndim", len(shape)) - elif self.ndim != len(shape): - raise ValueError(f"ArrayType: ndim={self.ndim} disagrees with shape length {len(shape)}") - if self.frameworks is not None and not isinstance(self.frameworks, frozenset): - object.__setattr__(self, "frameworks", frozenset(self.frameworks)) - if self.dtype is not None: - object.__setattr__(self, "dtype", canonical_dtype(self.dtype)) - - @classmethod - def image( - cls, - layout: str = "CHW", - channels: Union[int, Tuple[int, ...]] = 3, - dtype: Optional[str] = None, - framework: Optional[str] = None, - ) -> "ArrayType": - """Rank-3 image convenience. ``channels`` as a tuple is treated as the inclusive range - ``[min, max]`` (a pragmatic approximation — pass an exact :class:`Dim` via the constructor - for anything finer).""" - if isinstance(channels, int): - cdim = Dim.exact(channels, "C") - else: - cdim = Dim(min(channels), max(channels), "C") - h, w = Dim.any("H"), Dim.any("W") - up = layout.upper() - if up == "CHW": - shape: Tuple[Dim, ...] = (cdim, h, w) - elif up == "HWC": - shape = (h, w, cdim) - else: - raise ValueError(f"ArrayType.image: layout must be 'CHW' or 'HWC', got {layout!r}") - fws = frozenset({framework}) if framework else None - return cls(ndim=3, shape=shape, dtype=dtype, frameworks=fws, semantic="image") - - @classmethod - def parse( - cls, - spec: str, - dtype: Optional[str] = None, - framework: Optional[str] = None, - semantic: Optional[str] = None, - ) -> "ArrayType": - """Build from a jaxtyping-style shape string: space-separated axes where a bare int is an - exact size, ``lo-hi`` a bounded range (open ends allowed, e.g. ``"1-"``), and any other token - a named free axis. Example: ``ArrayType.parse("3 h w", dtype="float32")``. - """ - dims: List[Dim] = [] - for tok in spec.split(): - if tok.startswith("*"): - raise ValueError("ArrayType.parse: variadic '*' axes are not supported in v1") - if tok.isdigit(): - dims.append(Dim.exact(int(tok))) - elif "-" in tok[1:] and all(part == "" or part.isdigit() for part in tok.split("-", 1)): - lo_s, hi_s = tok.split("-", 1) - dims.append(Dim.range(int(lo_s) if lo_s else None, int(hi_s) if hi_s else None)) - else: - dims.append(Dim.any(tok)) - fws = frozenset({framework}) if framework else None - return cls(ndim=len(dims), shape=tuple(dims), dtype=dtype, frameworks=fws, semantic=semantic) - - def to_dict(self) -> Dict[str, Any]: - return { - "kind": "array", - "ndim": self.ndim, - "shape": [d.to_dict() for d in self.shape] if self.shape is not None else None, - "dtype": self.dtype, - "frameworks": sorted(self.frameworks) if self.frameworks is not None else None, - "semantic": self.semantic, - } - - -@dataclass(frozen=True) -class PythonType: - """A non-array Python value, matched by exact qualname (e.g. ``"PIL.Image.Image"``, ``"dict"``).""" - - qualname: str - - def to_dict(self) -> Dict[str, Any]: - return {"kind": "python", "qualname": self.qualname} - - -@dataclass(frozen=True) -class UnionType: - """Any-of. Stored as a tuple so the value object stays hashable/immutable.""" - - members: Tuple[TypeSpec, ...] - - def __post_init__(self) -> None: - object.__setattr__(self, "members", tuple(self.members)) - - def to_dict(self) -> Dict[str, Any]: - return {"kind": "union", "members": [type_to_dict(m) for m in self.members]} - - -@dataclass(frozen=True) -class ListType: - """A homogeneous sequence (mirrors ``datasets.Sequence`` / ``List``).""" - - item: TypeSpec - - def to_dict(self) -> Dict[str, Any]: - return {"kind": "list", "item": type_to_dict(self.item)} - - -@dataclass(frozen=True) -class MappingType: - """A struct with named fields (mirrors a nested ``datasets.Features`` / Python ``dict`` schema). - - Fields are stored as a sorted tuple of ``(name, spec)`` pairs so the object stays hashable. - Build via :meth:`of`. - """ - - fields: Tuple[Tuple[str, TypeSpec], ...] - - def __post_init__(self) -> None: - items = self.fields.items() if isinstance(self.fields, dict) else self.fields - object.__setattr__(self, "fields", tuple(sorted(items, key=lambda kv: kv[0]))) - - @classmethod - def of(cls, fields: Dict[str, TypeSpec]) -> "MappingType": - return cls(tuple(sorted(fields.items(), key=lambda kv: kv[0]))) - - def field_map(self) -> Dict[str, TypeSpec]: - return dict(self.fields) - - def to_dict(self) -> Dict[str, Any]: - return {"kind": "mapping", "fields": {k: type_to_dict(v) for k, v in self.fields}} - - -@dataclass(frozen=True) -class SampleType: - """The ``(input, target)`` type pair an op/source declares or a sample reports.""" - - input: TypeSpec = field(default_factory=AnyType) - target: TypeSpec = field(default_factory=AnyType) - - def accepts(self, producer: "SampleType") -> bool: - """Strict: this consumer accepts every value ``producer`` can emit (both slots).""" - return _accepts(self.input, producer.input, permissive=False) and _accepts( - self.target, producer.target, permissive=False - ) - - def compatible(self, producer: "SampleType") -> bool: - """Permissive: edit-time/discovery wiring — ``Any``/unknown on either side passes.""" - return _accepts(self.input, producer.input, permissive=True) and _accepts( - self.target, producer.target, permissive=True - ) - - def to_dict(self) -> Dict[str, Any]: - return {"kind": "sample", "input": type_to_dict(self.input), "target": type_to_dict(self.target)} - - @classmethod - def from_dict(cls, d: Dict[str, Any]) -> "SampleType": - return cls(type_from_dict(d["input"]), type_from_dict(d["target"])) - - # -- datasets.Features bridge (the concrete per-sample stored representation) ------------------ - - @classmethod - def from_hf_features(cls, features: Any, extras: Optional[Dict[str, Any]] = None) -> "SampleType": - """Build from a ``datasets.Features`` (or its ``.to_dict()`` form) keyed by ``"input"`` / - ``"target"``, applying the sidecar ``extras`` that Features can't express (framework, ranges, - or a full ``typespec`` override for Any/Union/family-only slots).""" - from datasets import Features - - if not isinstance(features, Features): - features = Features.from_dict(features) - extras = extras or {} - return cls( - input=_slot_from_features(features.get("input"), extras.get("input")), - target=_slot_from_features(features.get("target"), extras.get("target")), - ) - - def to_hf_features(self) -> Tuple[Any, Dict[str, Any]]: - """Split into a ``datasets.Features`` (the Feature-expressible part) and an ``extras`` dict - (framework/range refinements + a ``typespec`` fallback for slots Features can't represent).""" - from datasets import Features - - feats: Dict[str, Any] = {} - extras: Dict[str, Any] = {} - for slot, spec in (("input", self.input), ("target", self.target)): - feat, slot_extra = _feature_from_typespec(spec) - if feat is not None: - feats[slot] = feat - if slot_extra: - extras[slot] = slot_extra - return Features(feats), extras - - -# -------------------------------------------------------------------------------------------------- -# Matching (module-level dispatch — no base class) -# -------------------------------------------------------------------------------------------------- - - -def accepts(consumer: TypeSpec, producer: TypeSpec) -> bool: - """Strict single-slot match: does ``consumer`` accept every value ``producer`` can emit?""" - return _accepts(consumer, producer, permissive=False) - - -def compatible(consumer: TypeSpec, producer: TypeSpec) -> bool: - """Permissive single-slot match (edit-time / discovery): ``Any``/unknown on either side passes.""" - return _accepts(consumer, producer, permissive=True) - - -def _accepts(consumer: TypeSpec, producer: TypeSpec, *, permissive: bool) -> bool: - if isinstance(consumer, AnyType): - return True - if isinstance(producer, AnyType): - return permissive - if isinstance(producer, UnionType): - return all(_accepts(consumer, m, permissive=permissive) for m in producer.members) - if isinstance(consumer, UnionType): - return any(_accepts(m, producer, permissive=permissive) for m in consumer.members) - if isinstance(consumer, ArrayType) and isinstance(producer, ArrayType): - return _array_accepts(consumer, producer, permissive) - if isinstance(consumer, PythonType) and isinstance(producer, PythonType): - return consumer.qualname == producer.qualname - if isinstance(consumer, ListType) and isinstance(producer, ListType): - return _accepts(consumer.item, producer.item, permissive=permissive) - if isinstance(consumer, MappingType) and isinstance(producer, MappingType): - pm = producer.field_map() - for key, cval in consumer.fields: - if key not in pm or not _accepts(cval, pm[key], permissive=permissive): - return False - return True - return False - - -def _array_accepts(consumer: ArrayType, producer: ArrayType, permissive: bool) -> bool: - if consumer.ndim is not None: - if producer.ndim is None: - if not permissive: - return False - elif producer.ndim != consumer.ndim: - return False - if consumer.shape is not None: - producer_shape = producer.shape - if producer_shape is None and producer.ndim is not None and producer.ndim == len(consumer.shape): - # A shapeless producer of matching rank carries no per-axis info — equivalent to all - # unbounded dims. This keeps matching symmetric across the Features round-trip, where a - # 1-D ``ArrayType(shape=None)`` stores as a Sequence and reads back as ``shape=(Dim.any(),)``. - producer_shape = tuple(Dim.any() for _ in consumer.shape) - if producer_shape is None: - if not permissive: - return False - elif len(producer_shape) != len(consumer.shape): - return False - else: - for cdim, pdim in zip(consumer.shape, producer_shape): - ok = cdim.compatible(pdim) if permissive else cdim.accepts(pdim) - if not ok: - return False - if consumer.dtype is not None: - if producer.dtype is None: - if not permissive: - return False - elif not _dtype_accepts(consumer.dtype, producer.dtype): - return False - if consumer.frameworks is not None: - if producer.frameworks is None: - if not permissive: - return False - elif permissive: - if not (producer.frameworks & consumer.frameworks): - return False - elif not (producer.frameworks <= consumer.frameworks): - return False - return True - - -# -------------------------------------------------------------------------------------------------- -# Inference from live values -# -------------------------------------------------------------------------------------------------- - - -def infer_type(value: Any) -> TypeSpec: - """Infer a concrete :data:`TypeSpec` from a live value. ``None`` ⇒ :class:`AnyType` (so an unset - target stays permissive). torch/tf/PIL are imported lazily and guarded.""" - if value is None: - return AnyType() - if isinstance(value, np.ndarray): - return ArrayType( - ndim=value.ndim, - shape=tuple(Dim.exact(int(s)) for s in value.shape), - dtype=canonical_dtype(value.dtype), - frameworks=frozenset({"numpy"}), - ) - if isinstance(value, np.generic): - return ArrayType(ndim=0, shape=(), dtype=canonical_dtype(value.dtype), frameworks=frozenset({"numpy"})) - try: - import torch - - if isinstance(value, torch.Tensor): - return ArrayType( - ndim=value.dim(), - shape=tuple(Dim.exact(int(s)) for s in tuple(value.shape)), - dtype=canonical_dtype(value.dtype), - frameworks=frozenset({"torch"}), - ) - except ImportError: # pragma: no cover - torch is a hard dep - pass - try: - import tensorflow as tf - - if isinstance(value, tf.Tensor): # pragma: no cover - tensorflow is optional / not installed here - shape = tuple(Dim.exact(int(s)) if s is not None else Dim.any() for s in value.shape) - return ArrayType( - ndim=len(shape), - shape=shape, - dtype=canonical_dtype(value.dtype.name), - frameworks=frozenset({"tensorflow"}), - ) - except ImportError: - pass - try: - from PIL import Image - - if isinstance(value, Image.Image): - return PythonType("PIL.Image.Image") - except ImportError: # pragma: no cover - Pillow optional - pass - if isinstance(value, bool): - return PythonType("bool") - if isinstance(value, int): - return PythonType("int") - if isinstance(value, float): - return PythonType("float") - if isinstance(value, str): - return PythonType("str") - if isinstance(value, dict): - return PythonType("dict") - if isinstance(value, list): - return PythonType("list") - if isinstance(value, tuple): - return PythonType("tuple") - return PythonType(type(value).__qualname__) - - -def infer_sample_type(sample: "Sample") -> SampleType: - """Infer the :class:`SampleType` of a live sample (duck-typed: reads ``.input`` / ``.target``).""" - return SampleType(input=infer_type(sample.input), target=infer_type(sample.target)) - - -# -------------------------------------------------------------------------------------------------- -# JSON (de)serialization -# -------------------------------------------------------------------------------------------------- - - -def type_to_dict(t: TypeSpec) -> Dict[str, Any]: - return t.to_dict() - - -def type_from_dict(d: Dict[str, Any]) -> TypeSpec: - kind = d["kind"] - if kind == "any": - return AnyType() - if kind == "python": - return PythonType(d["qualname"]) - if kind == "array": - shape_d = d.get("shape") - shape = tuple(Dim.from_dict(x) for x in shape_d) if shape_d is not None else None - fw = d.get("frameworks") - return ArrayType( - ndim=d.get("ndim"), - shape=shape, - dtype=d.get("dtype"), - frameworks=frozenset(fw) if fw is not None else None, - semantic=d.get("semantic"), - ) - if kind == "union": - return UnionType(tuple(type_from_dict(m) for m in d["members"])) - if kind == "list": - return ListType(type_from_dict(d["item"])) - if kind == "mapping": - return MappingType.of({k: type_from_dict(v) for k, v in d["fields"].items()}) - raise ValueError(f"type_from_dict: unknown kind {kind!r}") - - -def sampletype_from_dict(d: Dict[str, Any]) -> SampleType: - return SampleType.from_dict(d) - - -# -------------------------------------------------------------------------------------------------- -# datasets.Features bridge helpers -# -------------------------------------------------------------------------------------------------- - - -def _slot_from_features(feat: Any, extra: Optional[Dict[str, Any]]) -> TypeSpec: - """Reconstruct one slot's :data:`TypeSpec` from its Feature plus sidecar ``extras`` (recursively).""" - extra = extra or {} - if "typespec" in extra: - return type_from_dict(extra["typespec"]) - if feat is None: - return AnyType() - return _apply_extras(_typespec_from_feature(feat), extra) - - -def _apply_extras(spec: TypeSpec, extra: Dict[str, Any]) -> TypeSpec: - """Layer the sidecar refinements Features can't hold back onto a Feature-derived spec, recursing - into mapping fields / list items so per-field framework/range info survives the round-trip.""" - if not extra: - return spec - if "typespec" in extra: - return type_from_dict(extra["typespec"]) - if isinstance(spec, ArrayType): - frameworks = frozenset(extra["frameworks"]) if "frameworks" in extra else spec.frameworks - shape = spec.shape - bounds = extra.get("shape_bounds") - if bounds and shape is not None: - new_shape = list(shape) - for idx_s, (lo, hi) in bounds.items(): - new_shape[int(idx_s)] = Dim.range(lo, hi, new_shape[int(idx_s)].name) - shape = tuple(new_shape) - return ArrayType( - ndim=spec.ndim, - shape=shape, - dtype=spec.dtype, - frameworks=frameworks, - semantic=extra.get("semantic", spec.semantic), - ) - if isinstance(spec, MappingType) and "fields" in extra: - sub = extra["fields"] - return MappingType.of({k: _apply_extras(v, sub.get(k, {})) for k, v in spec.field_map().items()}) - if isinstance(spec, ListType) and "item" in extra: - return ListType(_apply_extras(spec.item, extra["item"])) - return spec - - -def _typespec_from_feature(feat: Any) -> TypeSpec: - from datasets import Features, Image, Value - - if isinstance(feat, (Features, dict)): - return MappingType.of({k: _typespec_from_feature(v) for k, v in feat.items()}) - inner = feat[0] if isinstance(feat, list) else getattr(feat, "feature", None) - is_sequence = isinstance(feat, list) or feat.__class__.__name__ in ("Sequence", "List", "LargeList") - if inner is not None and is_sequence: - # A sequence of a numeric scalar is canonically a 1-D array (that's how HF stores a [N] tensor); - # only a sequence of non-scalars (struct / nested array / string) is a true ListType. - if isinstance(inner, Value) and inner.dtype not in ("string", "large_string"): - return ArrayType(ndim=1, shape=(Dim.any(),), dtype=canonical_dtype(inner.dtype)) - return ListType(_typespec_from_feature(inner)) - if isinstance(feat, Image): - return PythonType("PIL.Image.Image") - shape = getattr(feat, "shape", None) - dtype = getattr(feat, "dtype", None) - if shape is not None and dtype is not None: - dims = tuple(Dim.any() if s is None else Dim.exact(int(s)) for s in shape) - return ArrayType(ndim=len(dims), shape=dims, dtype=canonical_dtype(dtype)) - if isinstance(feat, Value): - if feat.dtype in ("string", "large_string"): - return PythonType("str") - return ArrayType(ndim=0, dtype=canonical_dtype(feat.dtype)) - return AnyType() - - -def _feature_from_typespec(spec: TypeSpec) -> Tuple[Any, Dict[str, Any]]: - """Return ``(feature_or_None, extras)`` for one slot. ``feature`` is ``None`` when the spec can't - be a concrete ``Features`` entry (Any/Union/family-dtype/unknown-rank/exotic python type).""" - from datasets import Array2D, Array3D, Array4D, Array5D, Features, Image, Sequence, Value - - if isinstance(spec, PythonType): - if spec.qualname == "PIL.Image.Image": - return Image(), {} - if spec.qualname == "str": - return Value("string"), {} - return None, {"typespec": spec.to_dict()} - if isinstance(spec, ListType): - inner_feat, inner_extra = _feature_from_typespec(spec.item) - if inner_feat is None: - return None, {"typespec": spec.to_dict()} - return Sequence(inner_feat), ({"item": inner_extra} if inner_extra else {}) - if isinstance(spec, MappingType): - feats: Dict[str, Any] = {} - sub: Dict[str, Any] = {} - for key, val in spec.fields: - feat, extra = _feature_from_typespec(val) - if feat is None: - return None, {"typespec": spec.to_dict()} - feats[key] = feat - if extra: - sub[key] = extra - return Features(feats), ({"fields": sub} if sub else {}) - if isinstance(spec, ArrayType): - if spec.ndim is None or spec.dtype is None or spec.dtype in _DTYPE_FAMILIES: - return None, {"typespec": spec.to_dict()} - extras: Dict[str, Any] = {} - if spec.frameworks is not None: - extras["frameworks"] = sorted(spec.frameworks) - if spec.semantic is not None: - extras["semantic"] = spec.semantic - if spec.ndim == 0: - return Value(spec.dtype), extras - if spec.ndim == 1: - return Sequence(Value(spec.dtype)), extras - arrcls = {2: Array2D, 3: Array3D, 4: Array4D, 5: Array5D}.get(spec.ndim) - if arrcls is None: - return None, {"typespec": spec.to_dict()} - shape_list: List[Optional[int]] = [] - bounds: Dict[str, List[Optional[int]]] = {} - if spec.shape is None: - shape_list = [None] * spec.ndim # rank known but no per-axis info -> all dynamic - else: - for i, d in enumerate(spec.shape): - if d.min is not None and d.min == d.max: - shape_list.append(int(d.min)) - else: - shape_list.append(None) - if d.min is not None or d.max is not None: - bounds[str(i)] = [d.min, d.max] - if bounds: - extras["shape_bounds"] = bounds - return arrcls(shape=tuple(shape_list), dtype=spec.dtype), extras - # AnyType / UnionType - return None, {"typespec": spec.to_dict()} - - -# -------------------------------------------------------------------------------------------------- -# Declaration decorator -# -------------------------------------------------------------------------------------------------- - - -def typed(*, accepts: Optional[SampleType] = None, produces: Optional[SampleType] = None) -> Callable[[C], C]: - """Class decorator that sets ``ACCEPTS`` / ``PRODUCES`` on an op/source (ergonomic alternative to - plain class attributes). Validates the arguments are :class:`SampleType` at decoration time.""" - - def deco(cls: C) -> C: - if accepts is not None: - if not isinstance(accepts, SampleType): - raise TypeError(f"@typed(accepts=...) expects a SampleType, got {type(accepts).__name__}") - setattr(cls, "ACCEPTS", accepts) - if produces is not None: - if not isinstance(produces, SampleType): - raise TypeError(f"@typed(produces=...) expects a SampleType, got {type(produces).__name__}") - setattr(cls, "PRODUCES", produces) - return cls - - return deco diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..c7fcdf2 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,1369 @@ +# RecordStream architecture + +The *why* behind recordstream's module boundaries and mechanisms. The user-facing documentation +([README](../README.md), the per-topic `docs/*.md`) shows **how to use** each surface; this +document records **why the surface is shaped the way it is** — so a reader who asks "why does this +module exist?" finds the answer here instead of reverse-engineering it from git history. + +Maintenance rules: + +- Every record keeps the five elements **Context → Decision → Consequences → Example → What you + may change**, dated (see the workspace `AGENTS.md` → "Architecture Decisions Are Documented"). +- A change that alters a mechanism updates its record **in the same change**. +- A superseded decision is **deleted**, not archived: whatever it still binds is folded into its + successor record. History lives in git, not here. + +## The system at a glance + +| Layer | Modules | What it is | Where the *why* lives | +|---|---|---|---| +| Data model | `items.py`, `io.py` | A record is a plain `dict` of typed values; one codec serializes any value | [§1](#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25) | +| Native ops | `transform.py`, `dispatch.py`, `ops/*` | Type-dispatched `Transform`s (kernels, `field=`) + structural/compose ops | [§1](#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25) | +| Library interop | `core._apply_op`, `register_op_family` | External libraries run as-is via the op-family dispatch — no adapters | [§1](#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25) | +| Engines | `core/` (`Stream`/`JointStream`), `flow/` (`FlowGraph`) | One op-application chokepoint, four routes; one per-record kernel behind two authoring forms | [§1](#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25), [§3](#3-the-graph-is-the-execution-model--the-lowering-pass-was-deleted-2026-07-30), [§5](#5-the-engines-own-callable-wrappers-live-in-core-2026-07-20-still-true-after-the-2026-08-01-package-split) | +| Graph wiring | `flow/steps.py`, `flow/parse.py` | Fan-out/fan-in/cross-step values are step GRAMMAR (`from:`/`merge_from:`/`bind:`), never ops | [§3](#3-the-graph-is-the-execution-model--the-lowering-pass-was-deleted-2026-07-30) | +| Batching | `collate.py` | Grouping is the engine's; stacking is a pluggable registry | [§2](#2-batching-is-two-stage-collation-is-a-pluggable-registry-recordstreamcollate-2026-07-17) | +| Storage & query | `storage/*` | The `typedrecord-v1` key-group layout over the codec; metadata scans without array loads | [§1](#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25) (contracts) + [storage.md](storage.md) | +| Introspection & serialization | `discovery.py` | Callable↔string identity + registration-free module scans | [§4](#4-callablestring-serialization--passive-introspection-recordstreamdiscovery-2026-07-20) | +| Runnables & workflows | `runnable.py`, `workflow.py`, `processing.py`, `cli.py` | `run()` objects, entry-point markers, combinators, the one `recordstream run` runner | [§7](#7-the-entrypoint-markers-are-the-dispatch-table-run_entrypoint-2026-07-29) + [runnable.md](runnable.md), [workflow.md](workflow.md) | +| Model boundary | `outputs.py`, `predictions.py`, `core.ensure_record_dataset`, `labels.class_counts` | Dataset normalization in, prediction contracts + sinks out, class-balance statistics | [§8](#8-the-model-boundary-belongs-to-the-package-that-reads-it-2026-07-29) + [predictions.md](predictions.md) | + +--- + +## 1. The record data model and the type-dispatched op engine (2026-07-25) + +### Context + +The rejected alternative was a bespoke record container: typed items (that part was right) +wrapped in a `Record` class with per-key role tags, plus an adapter registry that wrapped every +external library transform in an adapter object before it could touch a record (two adapter +classes, a coercion registry, and ~170 generated per-transform wrapper ops — all maintenance +surface). The container was the friction point: role tags duplicated what key names already say +(`"mask"` *is* the mask), and dict-native libraries — torchvision `transforms.v2` walks dicts, +albumentations takes named kwargs — were kept at arm's length from a carrier they could have +consumed directly. With two op-authoring surfaces (native transforms vs the adapter/generated +families), "where does augmentation come from?" had three answers. + +### Decision + +Collapse to ONE carrier and ONE op engine: + +- **A record is a plain `dict`** — `recordstream.items.Record = Dict[str, Any]` — of **typed + values** (`Image`/`Mask`/`Boxes`/`Label`, base `NDArrayItem`; open registry `register_item`; + uniform payload accessors `item_data`/`with_data`). No container class, no roles, no + `primary()`: **key names carry meaning** (`"image"`, `"mask"`, `"bboxes"`, `"class"`), and a + scalar side value is just another key. Metadata is attrs on the typed value (`Image.layout`, + `Label.classes`) or more dict keys (`"samplerate": 30.72e6`). Items are deliberately NOT + confluid-`@configurable`: an ndarray subclass builds through `__new__`, which fights the + `__init__` validation wrap — they live in their own registry. +- **Native ops are type-dispatched `Transform`s** (`recordstream/transform.py`): + `get_params(record)` draws shared parameters ONCE per record, per-type kernels + (`@MyOp.kernel(ItemType)`, MRO-aware registry in `recordstream/dispatch.py`) apply to every + handled value, `field=` pins one key. The second sanctioned shape — type-CHANGING ops + (`Threshold`: array→`Mask`, `ConvertToImage`: array→`Image`, `ConnectedComponents`: + `Mask`→`Boxes`, the target ops) — overrides `__call__`, resolves its source by an explicit + `field=` or the first value of the natural type, and raises a `ValueError` naming the record's + keys on every miss. +- **External libraries run AS-IS through the engine's op-family dispatch** + (`recordstream.core.families._apply_op`): an albumentations op receives exactly its own kwarg vocabulary + (`image`/`mask`/`masks`/`bboxes`/`keypoints`/`labels` keys present in the record; one call = + one joint draw; array outputs re-wrapped in the incoming `NDArrayItem` type so `Image`/`Mask` + survive); a torchvision-v2 op is called on the dict as-is; everything else is `op(record)` with + `None` = drop. **The families are an open registry** — `register_op_family(name, matcher, + invoker)`; the built-ins register through the same API, dispatch checks last-registered first, + and matchers/invokers are module-level functions so spawn workers rebuild the registry. Family + detection is by MRO module name — no eager imports, no adapters, no generated wrappers. + Box-carrying augmentation is the library's own `A.Compose(..., bbox_params=...)`; seeding is + the libraries' own mechanisms. +- **`Pipeline(transforms=[...])`** (`recordstream/transform.py`) is THE sequential composer; every + composing op routes inner ops through `_apply_op`, so bare library transforms nest anywhere a + native op does. +- **Tensors are plain values.** `ToTensor` writes a LIVE CHW-float `torch.Tensor` under its key — + a record value can be anything (`collate_records` stacks tensors natively, storage converts via + `to_numpy` on write, a downstream tv2 op transforms them as-is). An `Image` itself cannot hold + a tensor (`NDArrayItem.__new__` runs `np.asarray`); a typed tensor ITEM base is a tracked + follow-up (root `TASKS.md`). +- **Storage is the record key-group layout** (`typedrecord-v1`): everything serializes through + the `recordstream/io.py` codec; plain values ride the `"plain"` tag; NO backward compatibility + with the pre-record layout (an old/untagged store raises via + `storage/base.py::require_record_format` — an explicit decision: re-generate, never accrete + legacy readers). +- **Projection and collation are key-addressed**: `project(source, keys)` / `iter_key` / + `num_classes(key="class")`; the collate registry's default is `"record"` = `collate_records`. + +### Consequences + +- Zero adapter surface: a new library version's transforms are available the moment the library + is — nothing to regenerate; a NEW library family is one `register_op_family` call from any + package. +- Cross-key consistency is the LIBRARY's own joint draw (albumentations Compose / tv2's dict + walk) for augmentation, and `get_params`-once for native ops — one mechanism per world, both + automatic. +- YAML needs no special forms: a bare `!class:albumentations.HorizontalFlip {p: 0.5}` sits in an + `ops:` list like any native op (deferred markers flow at route entry). +- The albumentations vocabulary is load-bearing: a value augments only if it rides one of the + library's key names — routing is an explicit `RenameField`, never engine magic. +- **Contracts that outlive refactors:** the HALF-OPEN pixel xyxy `(x0, y0, x1, y1)` order of + `connected_component_boxes` (every `Boxes` producer emits it; a downstream back-projection + reads exactly that order); the `typedrecord-v1` tag + no-back-compat rule; the albumentations + key vocabulary; the `"module:qualname"` callable-path format (§4). +- Anything that used the old container API must migrate — there are deliberately no aliases and + no legacy read path. + +### Example + +One `Stream` ops list mixing both worlds, no wrappers: + +```python +import albumentations as A +from recordstream import Stream, Image, as_transform + +stream = Stream(source=records, ops=[ + A.Compose([A.HorizontalFlip(p=0.5)], + bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"])), + A.GaussNoise(p=1.0), # bare library op — as-is + as_transform(lambda d: d - 0.5, handles=(Image,)), # native type-dispatched op +]) +``` + +The same shape in YAML: + +```yaml +ops: + - !class:albumentations.HorizontalFlip + p: 0.5 + - !class:recordstream.ops.numpy.Threshold + low_level: 0.5 +``` + +### What you may change (and where it's documented) + +- **A new item type** — one class + `@register_item` (array-backed: subclass `NDArrayItem`, + declare `_item_attrs`); usage in [record-model.md](record-model.md). +- **A new per-type behaviour for an existing op** — `@Op.kernel(ItemType)`, no core edit. +- **A new library family** — one `register_op_family(name, matcher, invoker)` call from any + package (MRO module-name matcher + the library's native calling convention; module-level + functions so spawn workers rebuild the registry). Never an adapter/wrapper class. The + built-ins register through the same API; dispatch is last-registered-first, so + forks/extensions shadow their base library by registering later. Usage: + [record-model.md](record-model.md) → "A new library family". +- **The `typedrecord-v1` tag and the no-back-compat rule are contracts** — changing the on-disk + layout means a NEW tag and a re-generation story, never a silent dual-read path. + +--- + +## 2. Batching is two-stage; collation is a pluggable registry (`recordstream.collate`, 2026-07-17) + +### Context + +Turning N pipeline items into one batched carrier has two distinct halves: + +1. **Grouping** — the engine yields groups of N items (`Stream.batch` / `FlowGraph.batch` yield + `list`s, and a torch `DataLoader` hands its `collate_fn` a list). +2. **Stacking** — a *collate function* turns one group into one batched carrier. + +The engine owns grouping; it must NOT own stacking, because stacking is task-shaped: historically +every consuming project shipped its own task collate (classification, segmentation, detection), +and divergent batched-metadata conventions emerged between them. + +### Decision + +`recordstream/collate.py` is a **pluggable registry of collate functions keyed by representation**: +`register_collate(key)` / `get_collate(key)` / `collate(items, key=None)`, where an omitted key +uses the default **`"record"`** collate (`collate_records`) — N plain record dicts into ONE +batched record: per key, typed values encode through the `recordstream/io.py` codec, payloads stack +(torch → stacked tensor, numpy → stacked array, else a list), each declared item attr becomes a +LIST of per-record values (decoded back into one batched item of the same type), and a +`"plain"`-tagged value batches as the plain list. Batches must be key-homogeneous — a mismatch +raises. Consuming projects register task aliases (`"detection"`, `"yolo"`, …) **additively**; +re-registering a key deliberately overwrites so a consumer can replace a default. The divergent +consumer conventions were deliberately NOT unified here — the registry is an addressable home +consumers opt into, not a forced migration. + +The open, string-keyed half of the registry exists first and foremost for **AI-callable tools** +(the workspace converges on an MCP tool surface — see the root `AGENTS.md` end-goal): a JSON tool +argument can carry `"collate": "yolo"` but never a Python function object, and a tool schema can +offer the legal values only if the set is discoverable at runtime (`registered_collates()`). In +ordinary Python (and in YAML via a dotted `!ref:` to the function), passing the collate function +directly remains the normal path. + +### Consequences + +- The engine stays task-agnostic: recordstream stacks by key + item type, never + classification/detection/…. +- Item metadata batches deterministically: per-record attrs become lists on the ONE batched item + (`batch["image"].layout == ["HWC", "HWC", ...]`), plain values become plain lists — there is + no second batched-metadata convention in this package. +- A task whose batch shape the generic rules cannot express (detection's ragged per-record + boxes) opts OUT entirely and emits its model family's native contract — see the worked + example in [record-model.md](record-model.md) → "Batching". +- Registration happens at module import, so a key exists only after its defining module has been + imported. + +### Example + +```python +from torch.utils.data import DataLoader + +from recordstream import Stream, collate, collate_records, get_collate, register_collate + +stream = Stream(source=my_source, ops=[...]) + +batch = collate([stream[0], stream[1]]) # the "record" default +batch["image"].shape # stacked payloads, one batched Image +batch["image"].layout # per-record attrs -> a list + +loader = DataLoader(stream, batch_size=8, collate_fn=collate_records) + + +# A task alias registers additively (runs when the defining module is imported). +@register_collate("yolo") +def yolo_collate(items): + ... # stack to the task's own batch layout + + +loader = DataLoader(stream, batch_size=8, collate_fn=get_collate("yolo")) +``` + +### Addendum: the READ-BACK lives here too (`recordstream.batch`, 2026-07-29) + +**Context.** Every model boundary has to undo the three rules above: get past a wrapper item, +turn a per-record list into one tensor, transpose the leftover columns into per-record dicts for +a predictions sink. That is not task knowledge — it is the collate's own convention read +backwards. Two consumer packages had independently written it: one for classification, one for +segmentation, with two near-identical private `_batch_metadata` implementations and two separate +test files pinning them. A third consumer would have written a third. + +**Decision.** `recordstream.batch` ships the inverse beside the collate — `batch_values`, +`batch_tensor`, `batch_metadata` — and it carries **no dtype or shape opinion**. Rule 2 above +says turning class names into an `[N]` int64 tensor is "the model boundary's one explicit step, +not a generic-engine guess"; that still holds. What moved is *reading*, not *shaping*. + +**Consequences.** The three shapes a consumer actually wants — a classifier's `[N]` int64 ids, a +multi-label trainer's `[N, C]` float multi-hot, a segmenter's `[N, H, W]` int64 mask — all start +from `batch_values` and are shaped by a small task-specific function the consumer keeps. Folding +those three into one shared helper would produce a function whose body is a task switch, which +is the thing the collate registry exists to avoid. + +**Example.** + +```python +from recordstream import batch_tensor, batch_values, batch_metadata + +x = batch_tensor(batch, "image", device=self.device) # generic: one [N, 3, H, W] tensor +meta = batch_metadata(batch, exclude=("image", "class")) # generic: N per-record dicts + +# task-specific, stays in the consumer: +ids = torch.as_tensor(batch_values(batch, "class")) # a classifier's [N] class ids +mask = batch_tensor(batch, "target").long() # a segmenter's [N, H, W] int64 mask +``` + +### What you may change (and where it's documented) + +- **Plugging in your own batch layout** is the supported extension point — decorate a function + with `@register_collate("your-key")` and select it via `get_collate`/`collate`. Usage: + [kinds.md](kinds.md); the detection walkthrough: [record-model.md](record-model.md). +- **Changing the default collate's semantics** (how `"record"` stacks, the attrs-become-lists + convention) is an architectural change: every batch consumer depends on it. Update this record + and the recordstream `AGENTS.md` metadata mandate together. +- **Adding a reader** to `recordstream.batch` is fine when it is the collate read backwards. + Adding one that shapes for a task (promotes a dtype, builds a multi-hot) is not — that belongs + to the consumer, or the helper becomes a task switch. + +--- + +## 3. The graph IS the execution model — the lowering pass was deleted (2026-07-30) + +*(Supersedes "The per-record Context is an ambient wiring plane", 2026-07-17.)* + +### Context + +Between 2026-07-17 and 2026-07-30 this package had two ways to run a pipeline. A `flow:` +document (named steps, explicit `from:`/`merge_from:`/`bind:` edges) was the readable authoring +form; a flat `ops:` list was the execution form. A **lowering pass** (`to_ops`) compiled the +first into the second by inserting six *context ops* — `Save`/`Use`/`Drop`/`Apply`/`Capture`/ +`MergeFields` — that moved records through an ambient per-record cell store, and a **lifting +pass** (`from_ops`) reconstructed a flow document from such a list. Execution parity in both +directions was a pinned contract with its own suite. + +The arrangement was coherent but it cost a second executor (`FlowGraph` duplicating `Stream`'s +iteration, length, indexing and batching while being strictly less capable — no `to_sink`, no +`project`, and a `NotImplementedError` on 1→N expanding steps), a permanent parity tax on every +change to an op's semantics, and an ambient `contextvars` plane that nothing in the workspace's +15 real configs ever used. Adoption told the story plainly: every config on disk was a linear +`ops:` list; zero were `flow:` documents; the only producer of branchy pipelines — the visual +editor — compiled its canvas graph *down* to context ops and then lifted it *back* to a flow +document purely for readability. + +The decisive argument was about the consumer nobody had built yet. A lowered list re-encodes +dataflow as imperative mutation of named cells, which is exactly the information a compiler +needs and cannot recover: reverse-dependency analysis walks `node.inputs` backwards from the +outputs, and a flat list has no inputs. Handing a compiler the lowered form means asking it to +run the lifting pass first to rebuild what was just destroyed. + +### Decision + +**One execution model: the step graph.** Both spellings parse to the same `FlowStep` list and run +through the same per-record kernel (`recordstream.flow.execute.run_steps_multi`). + +- An `ops:` list compiles to positional steps (`core.linear_steps` — `s0`, `s1`, …) whose names + never surface. A sequence IS a graph; no lifting is involved. +- A `flow:` document parses to the same steps with author-chosen names and explicit edges. +- The kernel takes an **env-free fast path** for a straight chain (`is_linear`), so the linear + case carries none of the graph bookkeeping. +- `to_ops`, `from_ops`, `Stream.from_flow_yaml`, `recordstream.context` and + `recordstream.ops.context` are **deleted**, with no back-compat shims. + +Fan-out, fan-in and cross-step values are expressed as step GRAMMAR rather than as ops: `from:` +is the fork, `merge_from:` the union, `bind:` the cross-step value (including a producer's live +`@output` via `step.attr`). Branch isolation, which the cell store provided by deep-copying on +read, is now a property of the environment: each expansion branch gets its own shallow copy of +the step env, and a fan-out read copies. + +### Consequences + +- **One executor.** `Stream` and `FlowGraph` are two facades over one kernel; the parity suite is + gone because there is nothing left to keep in parity. +- **Expanding ops work everywhere.** The graph gained 1→N support (the remaining subgraph runs per + child, depth-first) that the old `FlowGraph` refused outright. +- **A branchy pipeline has no flat spelling — deliberately.** `FlowGraph.to_stream()` raises for + one, and a visual editor's ops-export raises pointing at its flow export. This is the honest + consequence of deleting the pass that manufactured such a spelling. +- **Compilation becomes possible.** A backend reads `FlowGraph.steps` and maps each step to an IR + node with real `inputs`; reverse-dependency pruning runs on the result. +- **Measured cost:** on a 23-step pipeline of trivial ops the graph engine was 1.41x the old flat + loop; hoisting a per-record analysis pass and adding the linear fast path brought it to 1.02x, + and with real ops in the chain the difference is not measurable. +- **Lost with the cell store:** a hand-written wiring op that stashed a value under its own cell + name. Anything that must persist belongs in the record; anything that wires belongs in the + grammar. + +### Example + +```yaml +# Fan-out -> two branches -> fan-in, entirely in step grammar. No cells, no snapshots. +flow: + spec: !class:mypkg.MakeSpectrogram {} + masked: !class:recordstream.ops.numpy.Threshold {low_level: 0.5, from: spec} + boost: !class:mypkg.Boost {from: spec} # second reader of `spec` = the fork + out: {from: boost, merge_from: [masked]} # union, last-write-wins +outputs: out +``` + +```python +# The same graph, and what a compiler front end reads off it. +from recordstream.flow import parse_flow + +steps, outputs = parse_flow(doc["flow"], doc["outputs"]) +for step in steps: + print(step.name, "<-", step.from_, step.merge_from) # every edge, explicit +# out <- boost ('masked',) +``` + +### What you may change (and where it's documented) + +- **Adding a step-grammar key** is an architectural change: it widens the contract every consumer + (the engine, a compiler front end, a visual editor's compiler) reads. Update this record, the + `AGENTS.md` flow mandate, and [graph.md](graph.md) together. +- **The linear fast path** (`is_linear`) is an optimization, not a semantic: it must produce + results identical to the general path, and the suite pins that both spellings agree. +- **Do NOT reintroduce a lowering pass.** A flat list that encodes branches as cell mutations is + a second execution model wearing the first one's clothes; the reason it was removed is written + above. If a future runtime genuinely needs a flattened schedule, it owns that pass — over its + own IR, downstream of the graph. + +--- + +## 4. Callable↔string serialization + passive introspection (`recordstream.discovery`, 2026-07-20) + +### Context + +Two workspace mandates — *Serialization Symmetry* (every pipeline round-trips through Confluid +YAML) and *Passive Introspection* (tools discover pipeline pieces without hand-written +definitions) — need a bridge the Confluid registry deliberately does not provide. The registry is +a **curated, opt-in catalog**: classes *and* builder functions participate, but only after an +explicit `@configurable`/`register()`, keyed by name/category/task/role, resolving *strings → +callables* for config materialization. What it does NOT do: produce a string **from** a live +callable (the dump direction a bare-function value like a mapped transform needs), resolve a +callable out of a plain `.py` script or `__main__`, or walk a module to introspect every callable +*defined in it* — registered or not. + +### Decision + +`recordstream/discovery.py` is one small stdlib-only module with **two halves**: + +- **Serialization** — `get_callable_path(fn)` → an importable `"module:qualname"` string + (resolving `__main__` to the script filename so the path survives process boundaries) and + `resolve_callable(path)` back to the live object (module import, `.py`-file load, or an + already-callable passthrough). +- **Introspection** — `introspect_callable(fn)` → a JSON-serializable schema (path, name, doc, + per-parameter type/default/required), and `scan_module(module_or_py)` applying it to every + callable *defined in* a module (`__module__`-filtered, so imports don't leak in). + +Curated discovery (MCP form-specs, task/category option pickers) deliberately does **not** use +this module — it builds on the Confluid registry. The two surfaces answer different questions: +`scan_module` reflects over *a module, no curation required*; the registry resolves *a curated +name/category*. + +### Consequences + +- `WrappedOp` stores its callable as the string path and resolves it lazily — which is exactly + what makes it pickle across `spawn` workers and serialize into YAML verbatim. +- The dotted-path idiom became the workspace's generic **string-callable hook pattern**: + consuming packages resolve their own hook knobs (metadata encoders, exporter callables) through + `resolve_callable` instead of hand-rolling import dances. +- A visual editor's node bridge scans op/source modules and auto-generates one node (plus its + property-panel widgets) per callable — no manual node definitions anywhere. +- The `__module__ == module` filter in `scan_module` is a real contract: a class defined + elsewhere and merely *imported* into a module is invisible to it (registry-based passes exist + for that case). +- **One acknowledged overlap**: `resolve_callable`'s plain module-import branch resolves the same + importable-function targets confluid's `resolve_class` module-path branch / `!ref:` grammar can + — two spellings of one job. The non-overlapping remainder (path *production*, `.py`-file and + `__main__` handling, module scans) is why the module exists; whether the resolution half should + delegate to confluid is a tracked follow-up in the root `TASKS.md`. + +### Example + +```python +import numpy as np + +from recordstream.discovery import get_callable_path, resolve_callable, scan_module + +path = get_callable_path(np.sqrt) # "numpy:sqrt" — YAML/pickle-safe identity +fn = resolve_callable(path) # back to the live callable +fn is resolve_callable(fn) # an already-callable argument passes through + +schemas = scan_module("recordstream.ops.numpy") # one JSON schema per op defined there +``` + +### What you may change (and where it's documented) + +- **Adding a string-callable knob to your own class**: reuse `resolve_callable` (the + `WrappedOp.f` pattern) — never write a bespoke import dance. +- **The `"module:qualname"` format and the module-local scan filter are contracts** — serialized + pipelines and node bridges depend on both; changing either is an architectural change that must + update this record. + +--- + +## 5. The engine's own callable wrappers live in `core` (2026-07-20; still true after the 2026-08-01 package split) + +> **Note (2026-08-01).** `core.py` became the `core/` PACKAGE (§11). Everything below holds +> unchanged — the split preserved exactly the property this record protects. `FilterOp` and +> `WrappedOp` moved to `core/wrappers.py` and `JointStream` sits beside `Stream` in +> `core/stream.py`, still inside `core`, still importing one way. What a config SPELLS changed +> (`!class:recordstream.core.wrappers.FilterOp`); what depends on whom did not. + +### Context + +Three classes sit in `core` next to the `Stream` engine that look, at first glance, like they +belong elsewhere: `FilterOp` and `WrappedOp` (op-shaped, so why not `ops/`?) and `JointStream` +(a second engine in the engine module). + +### Decision + +They stay in `core` because of **who constructs them and which way imports flow**. All three +are the construction targets of `Stream`'s own fluent API — `.filter(pred)` appends a `FilterOp`, +`.map(fn)` appends a `WrappedOp`, `Stream.joint([...])` wraps a `JointStream` — so the engine itself +instantiates them. And `core` is the *bottom* of the op-facing layer: every composing op in +`ops/` imports `core._apply_op` (the op-family dispatch chokepoint); moving `FilterOp`/`WrappedOp` +into `ops/` would make `core` import from `ops` and close an import cycle. `JointStream` is +`Stream`'s iteration-only fan-in sibling (`category="engine"`), 20 lines that exist to be +`Stream.joint`'s return value — a module of its own would be structure for structure's sake +(`FlowGraph` earns its separate module by size and its own document grammar). + +`FilterOp`/`WrappedOp` carry **no discovery category** on purpose: they wrap a *raw Python +callable*, which no GUI can wire, so they are neither canvas ops nor sources — bare +`@configurable` keeps them YAML-round-trippable while the positive category allowlist keeps them +off visual canvases. + +### Consequences + +- `ops/` stays a pure consumer of `core` — the layering is one-directional. +- `WrappedOp` is a package-root export (the public "lift a plain function" surface, and its + stored-string `f` is the reference use of the discovery serialization half); `FilterOp` is not + root-exported (normally reached via `Stream.filter`; importable as `recordstream.core.wrappers.FilterOp`). +- `JointStream` is YAML-addressable (`!class:recordstream.core.stream.JointStream()`) and canvas-composable as + an engine node; its indexable counterpart for raw sources is `ConcatSource`. + +### Example + +```python +stream = ( + Stream(source=src) + .map(np.sqrt, key="image") # appends WrappedOp(f="numpy:sqrt", key="image") + .filter(lambda r: float(r["image"].max()) > 0) # appends FilterOp(p=...) +) +both = Stream.joint([stream_a, stream_b]) # Stream(source=JointStream([stream_a, stream_b])) +``` + +### What you may change (and where it's documented) + +- **A new engine-constructed helper** (another fluent-API target) belongs in `core` for the + same import-direction reason; an op users wire *directly* (YAML/canvas) belongs in `ops/` with + a category and group. +- **Do not add a discovery category to `FilterOp`/`WrappedOp`** — surfacing a raw-callable + parameter on a canvas is a dead widget; the taxonomy is pinned in `tests/test_categories.py`. + +## 6. Every knob is a DECLARED parameter — the `Enable` toggle (2026-07-27) + +### Context + +`Enable` gates an inner ops list behind one boolean. Its original design leaned on Confluid's +post-construction paradigm: **any** boolean attribute set on the instance was the toggle, and that +attribute's NAME became the CLI flag — `visualize: false` in YAML produced `--visualize`, and a +`name:` was only needed to disambiguate two wrappers. Nothing about the toggle was declared; it +existed purely as a runtime attribute Confluid setattr'd from an unrecognised YAML key. + +That works for exactly one front-end — hand-written YAML — because only the YAML loader has a +channel for undeclared keys. Every other caller reads the *signature*: + +- `to_pydantic(Enable).model_fields` returned `['ops']`, so a schema/form/canvas generator built a + node with no toggle at all — the wrapper rendered as a pass-through and then raised at runtime. +- `Enable(ops=[...], visualize=True)` raised `ValidationError: Extra inputs are not permitted` + (the generated config model forbids extras), so neither Python nor a generated tool call could + construct a toggled wrapper — the only spelling was construct-then-setattr. +- `confluid.accepts_key(Enable, "visualize")` was `False`, so liquifai *silently dropped* the bare + broadcast the docstring advertised (`--visualize true`). Only `--.` landed, and + only via the addressed branch's "the key is already in the YAML kwargs" escape hatch. + +A knob that only YAML can reach is a knob three of the four front-ends cannot offer. + +### Decision + +The toggle is a **declared, defaulted constructor parameter** — `enabled: bool = True` — exposed as +a **settable property**, and instance identity is the **declared `name`**, which scopes the flag to +`--.enabled`. Dynamic toggle naming is retired. + +A property rather than a plain attribute for two reasons: `confluid.accepts_key` admits "public +settable class attributes", so the property keeps `enabled` overridable independently of the +signature; and it gives ONE funnel to reject a non-bool, so a quoted YAML `enabled: "true"` fails +at its `file:line` instead of being silently truthy. + +The retired form is not silently ignored: a stray public boolean attribute (what `visualize: false` +now lands as) raises on first record with the replacement spelling in the message. + +### Consequences + +- One declaration serves every front-end: YAML key, `--enabled` / `--.enabled` override, + Python kwarg, generated tool/form schema, canvas widget. No front-end-specific glue. +- The generalisable rule: **if a front-end must set it, declare it.** A value that only ever + arrives via post-construction setattr is reachable from YAML alone. +- Breaking change: `visualize: false` (and any other dynamic toggle name) must become + `name: visualize` + `enabled: false`; the CLI flag becomes `--visualize.enabled`. +- `flag_name` is gone — with a fixed toggle name there is nothing to introspect. +- Strictness is deliberate: `enabled` accepts only `bool`. Every CLI form already delivers a real + bool (`--enabled true`, `--enabled=false`, `--enabled+`, `enabled=true`), so the rejection only + catches genuinely ambiguous config. + +### Example + +```yaml +- !class:recordstream.ops.enable.Enable + name: visualize + enabled: false + ops: [ !class:recordstream.ops.image.ConvertToImage {} ] +``` + +```bash +recordstream run pipeline.yaml --visualize.enabled true # addressed: this wrapper +recordstream run pipeline.yaml --enabled false # broadcast: every wrapper +``` + +```python +op = Enable(ops=[convert], name="visualize", enabled=False) # one call — no setattr step +op.enabled = True # property setter; non-bool raises + +to_pydantic(Enable).model_fields # {'ops', 'enabled', 'name'} — the schema surface +accepts_broadcast(Enable, "enabled") # True — the bare --enabled form now lands +``` + +### What you may change (and where it's documented) + +- **Adding a knob to any op**: declare it in `__init__` with a default and an `Args:` line. Reach + for post-construction setattr only for values a *config layer* injects, never for a user-facing + switch. Usage lives in the project README (`Toggling a branch from the CLI`). +- **Distinguishing instances**: use `name:` — Confluid reads it for hierarchy labelling and + liquifai for `--.` addressing. Do not invent a per-class flag vocabulary; that was + the retired design. +- **More than one switch in a chain**: use several `Enable` wrappers with distinct names rather + than teaching one wrapper several toggles — each name is independently addressable, and the + broadcast form still flips them all. + +### Amendment: a DERIVED value is a knob nobody can reach either (2026-08-22) + +`loader_slots` derived `persistent_workers` from `num_workers` (`!= 0`) and its docstring said +there was "deliberately no separate knob, because the pairing never varies". The pairing does vary, +along an axis the derivation cannot see: macOS terminates persistent workers slowly enough that a +short run spends longer stopping than training, so a config there wants workers WITHOUT persistence. +Reaching that meant replacing all three loader slots wholesale — twelve YAML lines that also had to +restate `collate_fn`, because a replaced slot loses the code default and torch's `default_collate` +then crashes on string metadata. + +The reason a plain top-level key was not enough is worth stating, because the neighbouring case +looks identical and behaves differently: a flat config key DOES reach these code-created markers by +broadcasting — `multiprocessing_context: fork` lands in all three slots with no parameter anywhere +(measured) — but only for a kwarg the marker does not already carry. `persistent_workers` is BAKED +here at construction, so there was nothing for a config to reach. + +The rule above answers it unchanged: **if a front-end must set it, declare it.** The parameter is +`persistent_workers: Optional[bool] = None` — `None` derives exactly as before (so nothing that +does not pass it changes), an explicit value wins, and `True` with `num_workers=0` raises at +construction rather than inside torch at first iteration, which is the one invariant the +derived-only version had protected by construction. The same parameter is declared by every +consuming runnable, so it broadcasts from a flat config like `batch_size` does. + +The generalisation for the next time: deriving a value is not a way to avoid declaring it. A +derivation is a good DEFAULT and a bad ONLY option — the moment one caller knows something the +derivation cannot, the undeclared value costs a wholesale replacement of the object that holds it. + +## 7. The `@entrypoint` markers ARE the dispatch table (`run_entrypoint`, 2026-07-29) + +### Context + +A merged train+eval runnable exposes several capabilities from ONE class and selects between them +with a single `task` knob. Two readers need to know the task→capability mapping: the runnable's own +`run()`, which must call the right method, and a discovery consumer (a config generator, a visual +editor), which must know that one class both trains and evaluates and which `task` value means +"evaluate". The `@entrypoint(task, role, primary)` marker was introduced for the second reader only; +`run()` carried its own copy: + +```python +dispatch = {"fit": self.fit, "evaluate": self.evaluate, "test": self.test, "predict": self.predict} +``` + +So every merged runnable stated the same mapping twice — once in the decorators, once in the dict — +and three consumer packages carried that same five-line block verbatim. The two copies drift in a +direction that bites: a config generator pins `task:` from `entrypoint_tasks` (the markers), so a +capability added to the markers and forgotten in the dict yields a *generated* config that dies at +dispatch with "unknown task" while discovery advertises it as supported. Nothing could catch that — +the dict is not derived from anything, so no test can compare it to a source of truth. + +### Decision + +The markers are the ONE table, and `run_entrypoint(runnable, task)` is their runtime half: it builds +`{declared task: method name}` from `runnable_entrypoints(type(runnable))`, calls the match, and +raises `ValueError` on an unknown task listing the declared ones in declaration order. A merged +runnable's `run()` is then `run_entrypoint(self, self.task)` — the decorators are the only place the +mapping exists. + +The lookup reads markers off raw function objects via `vars()` (as `runnable_entrypoints` already +did), so a dynamic `__needs_autograd__` property never fires during dispatch. + +### Consequences + +- Adding a capability is ONE edit: decorate a method. Discovery and dispatch cannot disagree, + because they read the same annotations. +- The error message doubles as the class's capability list, in declaration order rather than the + sorted order a set would give. +- What is lost: the dict form let a type checker verify `self.fit` exists; `getattr(self, name)()` + is `Any`. Cheap here — the methods are decorated in the same file, and a wrong name would have to + survive its own `@entrypoint` line. +- Cost is one MRO walk per `run()` — once per training run. +- The markers are now load-bearing at RUNTIME, not just for discovery: dropping an `@entrypoint` + breaks the run, where before it only emptied a picker. That is the intended direction (a silent + discovery gap becomes a loud dispatch failure), but it means the decorators are no longer + optional metadata for a class that dispatches this way. + +### Example + +```python +from recordstream import TorchRunner, entrypoint, run_entrypoint + +class Classifier(TorchRunner): + def __init__(self, task: str = "fit") -> None: + self.task = task + + def run(self) -> None: + run_entrypoint(self, self.task) # no second copy of the mapping + + @entrypoint("fit", role="trainer", primary=True) + def fit(self) -> None: ... + + @entrypoint("test", role="evaluator", primary=True) + def test(self) -> None: ... +``` + +```python +>>> Classifier(task="test").run() # calls Classifier.test() +>>> Classifier(task="export").run() +ValueError: Unknown task 'export'; expected one of ['fit', 'test']. +``` + +### What you may change (and where it's documented) + +- **Adding a capability**: decorate the method with `@entrypoint("", role=..., primary=...)` + and extend the runnable's own `task` Literal. Nothing else — usage lives in `docs/runnable.md`. +- **A capability that is NOT config-selectable**: leave it undecorated and call it directly; the + marker means "reachable through `task:`", so decorating a helper would advertise it to config + generators as a runnable capability. +- **A different dispatch policy** (aliases, a default task, a per-role default): build it on top of + `runnable_entrypoints` rather than beside it — the invariant to preserve is that the markers stay + the only place the mapping is written down. + +## 8. The autograd marker is named for the framework; its FLAG for what it decides (2026-07-29) + +### Context + +`TorchRunner` exists so a GUI executor — which evaluates graph nodes under +`torch.inference_mode()` for cheap, grad-free runs — can tell "this run does gradient descent" +from "this run is inference-only" and re-enable autograd around the former. The mixin set a flag +named after ITSELF, `__torch_runner__`, and that name answers a question nobody asks at the one +place it is read: + +```python +torch_runner = bool(getattr(runnable, "__torch_runner__", False)) # "is this a torch runner?" +``` + +Every runnable in this workspace is a torch runnable, so read literally the flag is always true — +yet it is deliberately false for an evaluator, and the merged train+eval runnables override it as +a per-task property whose body (`return self.task == "fit"`) contradicts its own name: predicting +with a torch model does not stop the object from being "a torch runner". The name described the +declaring class instead of the decision the reader makes with it. + +### Decision + +Keep the CLASS name (`TorchRunner` — autograd is a torch concept, and a non-torch backend would +not inherit this mixin at all), rename the FLAG to **`__needs_autograd__`**. The two names then +answer different questions on purpose: which framework's execution mode is at stake, and whether +this particular run needs gradients. + +No compatibility alias. The flag is a duck-typed contract with exactly one reader, so the rename +lands in both packages at once — consistent with the workspace's no-back-compat precedent. + +### Consequences + +- The dynamic per-task override reads as what it means, which is where the old name hurt most. +- **The read fails OPEN** (`getattr(runnable, "__needs_autograd__", False)`): a reader left on the + old name sees `False` for every runnable and silently executes training under `inference_mode` + until `loss.backward()` raises *"element 0 of tensors does not require grad"*. That is why the + rename is all-or-nothing across the reader and the declarer — never a partial rollout. +- An external duck-typed implementer (an object that sets the flag without inheriting the mixin) + must be updated by hand; there is no import to break and therefore no compile-time signal. + +### Example + +```python +class TorchRunner: + __needs_autograd__: bool = True # inherited by trainers and workflow combinators + + +class Classifier(TorchRunner, L.LightningModule): + @property + def __needs_autograd__(self) -> bool: # type: ignore[override] + """Only ``fit`` needs autograd; evaluate / test / predict are inference-only.""" + return self.task == "fit" +``` + +```python +# the executor side (one reader, no import of this package) +if getattr(runnable, "__needs_autograd__", False): + with torch.inference_mode(False), torch.enable_grad(): + runnable.run() +else: + runnable.run() +``` + +### What you may change (and where it's documented) + +- **A runnable that never trains**: do not inherit `TorchRunner` at all — the absent flag is the + statement. Usage lives in `docs/runnable.md`. +- **A runnable that sometimes trains**: override `__needs_autograd__` as a property, as above. +- **Another execution-mode marker** (a "needs a GPU", "must run single-process" flag): follow the + same rule — name the class for the concern, the flag for the decision the executor makes, and + remember that a duck-typed read of a missing flag is silent. + +## 8. The model boundary belongs to the package that reads it (2026-07-29) + +### Context + +Four surfaces used to live in the workspace's experiment-**tracking** library: a dataset +normalizer (`ensure_record_dataset`), the prediction-output contracts (`ClassificationOutput` & +co) with their torch builders, a predictions sink (`PredictionsSink` + +`ClassificationPredictionsSink`), and class-imbalance weighting (`apply_class_weights` and its +inverse-frequency arithmetic). + +None of them tracked anything. The normalizer's whole body was "already a `Stream`? else wrap in +one". The sink's own module docstring justified its placement circularly — it lived there +*because the contract and the output type lived there* — while its body was record plumbing plus a +numpy `argsort`, threading its result through recordstream ops. And the sink advertised itself as +modality-neutral while building its diagnostics from `pack_id` / `iq_file` / +`window_start_sample`: signal-domain keys, in a class a tabular classifier was supposed to reuse. + +The pattern underneath: each of these describes a boundary whose only READER is elsewhere, and a +contract that outlives its reader accumulates justifications instead of users. + +### Decision + +A package owns a contract when it owns the reader. So: + +- `ensure_record_dataset` / `RecordSource` land beside `Stream` — the only type they know. +- `recordstream.outputs` holds the contracts *and* their torch builders, because + `recordstream.predictions` — the sink that reads `probs` by name — is right next to it. +- `recordstream.predictions` holds the sink and the `PredictionsSink` protocol. +- `class_counts` / `inverse_frequency_weights` land beside `LabelMap`, because how often each + class occurs is a statistic over the labels. + +The line is drawn at the *framework convention*, not at "does this import torch" (this package +already hard-depends on torch — a `Stream` IS a `torch.utils.data.Dataset`). What did NOT move: +whether a loss accepts a `weight` argument and how to inject it. That is `torch.nn`'s constructor +convention — Keras takes `class_weight` on `fit()` — so it lives in the consuming runnable as an +overridable method, and this package never learns what a loss is. + +### Consequences + +- The weights come back as **numpy**, matching `recordstream.batch` (only `batch_tensor` is + torch). A torch caller writes `torch.as_tensor(w)`; a Keras backend feeds the same array to + `fit(class_weight=…)`. One statistic, no framework baked in. +- `inverse_frequency_weights` absorbed the `LabelMap.to_ids` flattening consumers used to write by + hand, so a multi-label target counts for every class it names with no call-site branch. +- The engine's own rules bit immediately and usefully: the package-wide zero-arg-construction + sweep failed on the imported sink (`ops` was a required constructor argument), so the check + moved to `write()` where it belongs. Stricter host, better tenant. +- Two sink protocols now coexist (`DataSink.write(record)` vs + `PredictionsSink.write(prediction, metadata)`). That is deliberate — a model emits a batch while + the sink contract is per-record, so the halves arrive separately — and load-bearing downstream, + where a visual editor's node palette keys off the distinction. Collapsing them is filed in + `TASKS.md` rather than left to drift. +- No back-compat aliases: a stale import from the old location fails loudly. + +### Example + +```python +# a trainer, walking its targets exactly once and reusing that pass three ways +targets = self._walk_targets(self.train_set) # ONE pass +self.label_map = LabelMap.fit(targets) # (1) the encoding +num_classes = self.label_map.num_classes # (2) the head size +weights = inverse_frequency_weights(targets, num_classes, self.label_map) # (3) the balance + +if weights is not None: + self.apply_class_weights(weights) # framework hook — torch: loss.weight = ... +``` + +```python +# the boundary on the way out +def predict_step(self, batch, batch_idx): + out = classification_output(self(x)) # recordstream.outputs + self.predictions_sink.write(out, metadata) # recordstream.predictions + return out +``` + +### What you may change (and where it's documented) + +- **A new prediction contract**: add it to `recordstream.outputs`, generic in the array type, and + give it a builder only if the payload is DERIVED (logits → probs). A payload the model hands you + directly (boxes) gets no builder. Usage lives in `docs/predictions.md`. +- **Another task's predictions sink**: implement `PredictionsSink` beside the classification one, + or in the domain package when it needs domain geometry (a detector's back-projection to + time/frequency does). +- **A different balancing policy** (effective-number, sqrt-inverse): add it beside + `inverse_frequency_weights` in `recordstream.labels` as another statistic returning numpy. Do + NOT add the injection here — that stays a per-backend method on the runnable. + +--- + +## 9. torch is an extra; the engine is numpy (2026-07-30) + +### Context + +`recordstream` declared `torch` as a hard dependency, so `import recordstream` imported ~2GB of +PyTorch — and every package built on it inherited that transitively, declaring no torch of its +own. That was fine +while every consumer was a Lightning trainer. It stopped being fine when a second training engine +landed: a Keras-on-TensorFlow install, or a plain-numpy dataset-conversion job, paid for a +framework it never called. + +Auditing what actually needed torch found the coupling was almost entirely nominal: + +- **`Stream` and `FlowGraph` subclassed `torch.utils.data.Dataset`.** This was the expensive + line, and it bought nothing. `Dataset` is an empty base — `DataLoader` duck-types its argument, + needing only `__len__` and `__getitem__` (verified against a plain class with those two + methods and no base). Nothing in the workspace does `isinstance(x, Dataset)`, and nothing + subclasses `Stream`. +- **`storage/base.py` and `ops/image.py` imported torch for `isinstance(x, torch.Tensor)` alone** — + to decide whether a payload needed `.detach().cpu().numpy()` before being written or rendered. +- Only `ops/torch.py` (`ToTensor`) and `outputs.py`'s `softmax`/`argmax` builders genuinely + compute with it. + +The two isinstance sites are the interesting case, because the naive fix — a lazy in-function +`import torch` — still *imports torch* the first time a record is written. + +### Decision + +**`torch` moved from `dependencies` to `[project.optional-dependencies] torch`, and the core +imports no framework.** Four mechanisms, one per coupling: + +1. **The `Dataset` base is dropped** in favour of a `MapStyle` Protocol (`__len__` + + `__getitem__`) — the engine still *says* "map-style dataset" in its own vocabulary, and + `RecordSource = Union[MapStyle, Iterable[Record]]` stays the contract `ensure_record_dataset` + enforces. +2. **Type identity without an import**: `recordstream._compat.is_torch_tensor` consults + `sys.modules` rather than importing. This is exact, not a heuristic — *a torch tensor cannot + exist in a process that has not imported torch*, so the absence of the module proves the + negative. It is the same instinct as the op-family matchers, which identify an albumentations + or torchvision transform by its MRO module name. +3. **`ToTensor` is a lazy export** — `recordstream.ops` maps it in `_OPTIONAL_OPS` and resolves it + in a PEP 562 module `__getattr__`, raising an `ImportError` that names the extra instead of a + traceback from three libraries down. `__dir__` still advertises it so completion works. +4. **`outputs.py` splits by what needs a runtime**: the `TypedDict` contracts stay module-level + (they are typing-only, and generic in the array type), while `classification_output` / + `segmentation_output` import torch in the function body — they are the only part that computes. + +### Consequences + +- **`DataLoader(stream)` now needs `cast(Any, stream)` in type-checked code.** torch's *stub* + declares `Dataset[T]`; the runtime accepts any map-style object. This is a stub's stricter view + of a contract that works, and the bridge belongs at the four call sites (all in tests) rather + than in the engine — re-adding the base to satisfy a stub would restore the 2GB dependency to + silence a type checker. +- **`MapStyle` must be referenced as the real class in any annotation a consumer introspects, never + a string forward-ref.** confluid evaluates annotations in the *consumer's* namespace, so + `RecordSource = Union["MapStyle", ...]` raised `NameError: name 'MapStyle' is not defined` from + a consumer's `__init__` scan, three packages away. +- **The numpy-return rule elsewhere is now load-bearing, not stylistic.** `batch_values`, + `multi_hot`, `batch_metadata` and the class-balance statistics return numpy precisely so this + boundary holds; only `batch_tensor` is torch. +- **`recordstream.ops.torch` cannot be eagerly imported by anything in the package** — a new + convenience re-export there would silently undo all of the above. + +### Example + +```python +# storage/base.py — recognise a tensor without importing torch +from recordstream._compat import is_torch_tensor + +def to_numpy(data): + return data.detach().cpu().numpy() if is_torch_tensor(data) else np.asarray(data) +``` + +```python +# recordstream/ops/__init__.py — the op is reachable, the import is not eager +_OPTIONAL_OPS = {"ToTensor": ("recordstream.ops.torch", "torch")} + +def __getattr__(name): + entry = _OPTIONAL_OPS.get(name) + if entry is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + module_path, extra = entry + try: + return getattr(importlib.import_module(module_path), name) + except ImportError as exc: + raise ImportError(f"recordstream.ops.{name} needs the {extra!r} extra: " + f"pip install 'recordstream[{extra}]'") from exc +``` + +### What you may change (and where it's documented) + +- **Add another optional-framework op**: put it in its own module, add one `_OPTIONAL_OPS` entry + and one `__all__` entry, and declare the extra in `pyproject.toml`. No other edit — the error + message and `dir()` follow from the mapping. +- **Recognise another framework's tensor type** (a TF tensor, a jax array): add a sibling to + `_compat.py` using the same `sys.modules` rule. Do not add a module-level import of that + framework anywhere in the core. +- **Need a torch-typed return from an existing helper**: add a `dtype=`/`device=` parameter and + keep the numpy default, as `batch_tensor` does — do not change an existing numpy return, or the + next backend has to reimplement it. +- **Installation** is documented in the README's Installation section; what each extra provides + is the `pyproject.toml` comment beside it. + +## 10. The framework's batching half lives beside the collate (`recordstream.keras`, 2026-07-30) + +### Context + +Batching a record source has two halves: **what one batch contains** (recordstream's +`collate_records`) and **which rows go in which batch** (the row order, the slicing, the short +final batch, the per-epoch reshuffle). For torch, the second half is free — a `DataLoader` does it, +duck-typing any `MapStyle` source (§9) and taking `collate_fn=collate_records` for the first half. +So recordstream shipped only half of the pair, and nobody noticed the other half was missing. + +Keras 3 has no `DataLoader`. `keras.utils.PyDataset.__getitem__` must return a whole BATCH, so a +consumer has to write that loop itself. The first one did, in a training project — a +`RecordSequence(keras.utils.PyDataset)` inside an image classifier — and the result read as if the +adapter were part of the task. It was not: sixty percent of it (row order, `np.arange`, the rng, +`on_epoch_end`, `collate_records([source[i] for i in rows])`) mentioned nothing about +classification, while its torch twin in the same project was a single +`LazyClass(DataLoader, shuffle=True, collate_fn=collate_records)` line. A second Keras consumer +would have copied the file. + +### Decision + +**`recordstream.keras.RecordSequence` owns the DataLoader half; the consumer passes the batch +shape in.** The split is drawn exactly where torch draws it: `transform` is the `collate_fn` +equivalent, a callable mapping one collated record to what the model consumes. With no +`transform`, `__getitem__` hands over the batched record — the identity, which is also what +`batches()` yields for pairing per-record predictions with per-record metadata. + +The module also owns the **`KERAS_BACKEND` ordering**, which is why it exists as one module rather +than a class dropped somewhere. Keras 3 reads that variable at import time and defaults to +`tensorflow`, which `recordstream[keras]` does not install (Keras is an API; the compute engine is +the operator's choice), so a bare `import keras` dies inside `keras.src.tree.optree_impl` with +`ModuleNotFoundError: No module named 'tensorflow'`. A `setdefault` to the first backend actually +present — probed with `find_spec`, so nothing is imported just to look — has to run in the LOWEST +layer that imports keras: import sorters put a library import above a first-party one, so a +consumer's own shim sorts BELOW `from recordstream.keras import RecordSequence` and would lose the +race. + +### Consequences + +- **`RecordSequence` is absent from the package root, deliberately.** `inspect.getmembers` — what + `discovery.scan_module` and the GUI bridges call — getattrs every name a module advertises, so a + PEP 562 lazy export at the root (the `recordstream.ops.ToTensor` pattern) would import keras on + every discovery scan of a torch-only install. The import path is the boundary marker: + `from recordstream.keras import RecordSequence`. +- **It is not `@configurable` and carries no discovery `category`.** It is engine plumbing a + runnable builds in code, like `collate_records`; tagging it would put a keras import in the + registry scan for a class no YAML wires. +- **The row order is a lazy `@property`, not constructor state.** `len(source)` is real work for a + deferred source (a `HuggingFaceSource` LOADS its dataset to answer it), and recordstream + constructors do none — so `RecordSequence()` builds zero-arg and a missing `source` is reported + by `indices` with a clear message. +- **A consumer's keras imports now route through recordstream.** A training project keeps its own + one-line shim for spelling, but the ordering rule has one home; a project that imports keras + ahead of `recordstream.keras` reintroduces the TensorFlow failure. +- **The extra names no compute engine.** `keras = ["keras>=3.0"]` only; torch/TF/jax come from + whichever consumer extra selected one, and `_first_installed_backend` adapts to what is there. + +### Example + +```python +# The consumer supplies the SHAPE; the engine supplies the batching. +from recordstream.keras import RecordSequence + +def to_xy(batch): # the classification decision, 3 lines + x = np.asarray(batch_values(batch, "image"), dtype="float32") + return x, np.asarray(batch_values(batch, "class"), dtype="int64") + +seq = RecordSequence(stream, batch_size=32, shuffle=True, transform=to_xy) +model.fit(seq, epochs=3) + +# ...and the torch twin, for the symmetry this restores: +loader = DataLoader(cast(Any, stream), batch_size=32, shuffle=True, collate_fn=collate_records) +``` + +### What you may change (and where it's documented) + +- **Add another framework's batching adapter** (a JAX/`grain` sampler, a TF `tf.data` generator): + a sibling module behind its own extra, same split — the engine owns row order + collate, the + caller owns the batch shape via a `transform`-shaped parameter. Do not grow `RecordSequence` a + framework switch. +- **`PyDataset`'s prefetch knobs are already declared** (`workers` / `use_multiprocessing` / + `max_queue_size`, forwarded to `super().__init__()` at Keras's own defaults). Any further + passthrough follows the same rule — a named, defaulted, `Args:`-documented parameter, never a + `**kwargs` escape hatch, per the declared-parameter mandate (§6). Pinned by + `test_every_knob_is_a_declared_parameter`. +- **Usage** is [docs/kinds.md](kinds.md#keras-recordsequence--the-batching-half-the-framework-leaves-to-you); + what the extra provides is the `pyproject.toml` comment beside it. + +## 11. Sources are a package, one class per module — and the module path is the contract (2026-08-01) + +### Context + +`recordstream/sources.py` had grown to four unrelated `@configurable` classes in 511 lines: a +concrete loader (`HuggingFaceSource`, which reaches the network) and three pure view sources that +only do index arithmetic (`DatasetSplit`, `RangeSource`, `ConcatSource`). Nothing tied them +together beyond the word "source" — editing the HF metadata resolution meant scrolling past the +split partitioner, and a reader looking for the concat offsets had to know it was the last class +in the file. `recordstream.ops` had already been a package for exactly this reason. + +The split is not free, because in this workspace a class's **module path is a published +contract**. `confluid.pydantic_export._qualname` builds it as `f"{cls.__module__}.{cls.__qualname__}"`, +and that string is what a generated config emits as its `!class:` tag, what a form-spec / MCP +schema reports, and what keys the discovery-service enrichment table. Moving a class to a +submodule therefore changes the tag every generator writes. + +### Decision + +**One class per module under `recordstream/sources/`, and the submodule path is canonical.** +`huggingface.py` / `split.py` / `range.py` / `concat.py`, plus a `base.py` holding the one helper +the three view sources share. `__init__.py` re-exports every public name so +`from recordstream.sources import DatasetSplit` is unchanged. + +The alternative — pinning `__module__` back to `recordstream.sources` in `__init__.py` so no +downstream string moves — was measured and rejected. It breaks +`confluid.registry.key_for()`: `_entry_for_object` finds an entry by recomputing +`f"{cls.__module__}.{cls.__qualname__}"` and comparing against the key stored when +`@configurable` ran, so a rewritten `__module__` misses. The fallout is silent — with a namesake +registered, a pinned class dumps the ambiguous `!class:Thing()` while an unpinned twin correctly +dumps its disambiguated `!class:__main__.Thing~2()`. It also breaks `inspect.getsource`, which +searches `__init__.py` and raises `OSError: could not find class definition`. + +### Consequences + +- **Both spellings still resolve.** `confluid.resolve_class` falls back to + `importlib.import_module(module_path)` + `getattr`, and the package re-exports every name, so a + hand-written `!class:recordstream.sources.HuggingFaceSource` in an old config keeps loading. + What *changed* is what generators WRITE, so every such string in the workspace was updated in + the same change — including the discovery-service enrichment key, whose miss would have silently + dropped a field alias rather than failing. +- **`__init__.py`'s `__all__` became load-bearing.** `recordstream.discovery.scan_module` filters + members on `member.__module__ == mod_name`, so it now returns `[]` for the package. A visual + editor's node bridge survives only through its second pass, which walks `__all__` — verified by + running that bridge before and after and diffing the registered node keys (identical). +- **One entry point for the package, not one per submodule.** Unlike `recordstream.ops.*`, where + each module is entry-pointed, `recordstream/sources/__init__.py` imports all four submodules, so + importing the package registers every `@configurable`. Adding per-submodule entry points would + only re-scan the same classes. +- **A new source is a new file.** There is no longer a "where in the file" question, and a source + that needs a heavy import keeps it out of its siblings' import path. + +### Example + +```yaml +# canonical — what a generator emits, matching cls.__module__ +train_set: !class:recordstream.sources.huggingface.HuggingFaceSource + path: ylecun/mnist + split: train + +my_split: !class:recordstream.sources.split.DatasetSplit() + source: !ref:train_set + val_fraction: 0.1 + seed: 42 +``` + +```python +# the import surface is the package, unchanged by the split +from recordstream.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource +``` + +### What you may change (and where it's documented) + +- **Add a source**: one new module under `recordstream/sources/`, its class re-exported from + `__init__.py` AND listed in `__all__` — the second half is what puts it in a visual editor's + palette, and forgetting it fails silently. +- **Share code between view sources**: `base.py`. It is private to the package; a helper a + consumer should call belongs at the package root instead. +- **Usage** is [docs/sources.md](sources.md). + +## 12. `core` and `flow` are packages too — layered by import direction (2026-08-01) + +### Context + +`core.py` was 713 lines holding four unrelated things: the op-family registry and the +`_apply_op` chokepoint, the `MapStyle` Protocol, the two fluent-API callable wrappers, and the +`Stream` engine. `flow.py` was 708: the step model, the document parser, the per-record kernel, +and `FlowGraph`. Both had crossed the line where a reader looking for one thing scrolls past +three others — the same threshold that made `sources.py` a package (§11). + +Unlike `sources.py`, neither file is class-dominated: roughly 45% of `core.py` was free +functions. A literal one-class-per-module rule would have produced a 30-line `joint_stream.py` +and two ~35-line wrapper modules, which §5 had already considered and rejected as "structure for +structure's sake". And the split had to preserve something load-bearing: §5's whole argument is +that `core` is the BOTTOM of the op-facing layer, so a naive split risked closing the very +import cycle that record exists to prevent. + +### Decision + +**One module per cohesive unit, layered so imports run strictly one way.** A class gets its own +module when it dominates one (`Stream`, `FlowGraph`); otherwise the unit is the boundary. + + core/ families.py → mapstyle.py → wrappers.py → stream.py + flow/ steps.py → parse.py → execute.py → graph.py + +`core/families.py` is the bottom: the registry, the built-in families, `_apply_op`, and the +`EXPANDS` protocol — everything about applying ONE op to ONE record, importing nothing from its +siblings. `flow/execute.py` imports it directly, which is what keeps `flow` from reaching back +into `core.stream`; `core.stream` reaches `flow` only through body-local imports, exactly as +`core.py` did. + +Three functions sit where their DEPENDENCY puts them rather than where their name suggests: +`ensure_record_dataset` is in `stream.py` (its whole body builds a `Stream`, and putting it in +`mapstyle.py` beside the `RecordSource` type it consumes would have made a pure type module +import the engine); `linear_steps` and `_worker_task` likewise, because both compile or run an +ops LIST, which is `Stream`'s spelling of a pipeline. + +### Consequences + +- **The canonical `!class:` path moved** — `recordstream.core.stream.Stream`, + `recordstream.core.wrappers.FilterOp`, `recordstream.flow.graph.FlowGraph`. The package + spelling still resolves (§11), so hand-written configs keep loading, but generators emit the + new one, so all 64 downstream files were updated in the same change. +- **A monkeypatch must now name the module that USES a symbol, not the one that defines it.** + This is the one behaviour change with teeth. `flow/graph.py` does + `from recordstream.flow.execute import _result_readers`, which BINDS the name — so patching + `recordstream.flow` (which worked when both lived in one module) silently misses. The suite + caught it immediately; a test that had been asserting a performance property was suddenly + asserting nothing. `recordstream/core/__init__.py` carries a note saying so. +- **`_OP_FAMILIES` is the exception, and only because it is mutable.** The registry list is + re-exported by identity, so `core._OP_FAMILIES[:] = snapshot` still restores the real registry. + Rebinding it (`core._OP_FAMILIES = []`) would not. +- **`core/__init__.py` re-exports PRIVATE names deliberately** (`_apply_op` and friends), marked + `# noqa: F401`. They are the engine's internal cross-module surface — every composing op in + `ops/` imports `_apply_op` from `recordstream.core` — so the package boundary has to carry them + even though `__all__` (and therefore a visual editor's palette) must not. +- **`__all__` became load-bearing in both packages**, for the reason §11 gives. `core.py` had + none at all, so `Stream` and `JointStream` reached the palette purely through + `scan_module`'s `__module__` filter; after the split that pass returns `[]`. Verified by + diffing the registered node keys before and after (identical). + +### Example + +```yaml +# canonical — what a generator emits, matching cls.__module__ +train_set: !class:recordstream.core.stream.Stream() + source: !ref:hf_train + ops: !ref:preprocess +``` + +```python +# the import surface is the package, unchanged by the split +from recordstream.core import Stream, ensure_record_dataset +from recordstream.flow import FlowGraph, run_steps_multi + +# ...but a test double names the USER of a symbol, not its definition: +monkeypatch.setattr(recordstream.flow.graph, "_result_readers", counting) # ✓ +monkeypatch.setattr(recordstream.flow, "_result_readers", counting) # ✗ silently misses +``` + +### What you may change (and where it's documented) + +- **Add an engine-constructed helper**: `core/wrappers.py` if it wraps a raw callable, else the + module whose layer it belongs to — never a new module above `stream.py`, which would invert the + direction the layering protects (§5). +- **Add an op family**: `register_op_family` from anywhere; `core/families.py` only holds the + built-ins, and they register through the same public API (§1). +- **Split `flow/execute.py` further** if a third route appears — but `is_linear` must stay the + single gate, and the routes must keep agreeing record-for-record (§3). + +## 13. Dataset identity is a protocol, and a view propagates it verbatim (`recordstream.uri`, 2026-08-02) + +### Context + +A source knew what it read; nothing else could ask. `HuggingFaceSource` logged +`Loading mnist (train)...` and stamped `hf_path` / `hf_split` onto every record, but both are +per-record payload — a consumer wanting to answer "which data produced this?" for the RUN had to +reach into a record, or reach into the source's constructor arguments and reassemble an +identifier by hand. Every such consumer reassembled it differently, so two answers to the same +question could not be compared. + +What the question needs is one string per dataset that is stable across machines and across the +wrappers a config puts in front of a source — a trainer's `train_set` is typically a stream over +a split view over the real source, and all three read the same dataset. + +### Decision + +**A source may expose `dataset_uri` and `dataset_url`; free functions read them, and following +wrappers happens in the functions rather than in every wrapper.** + +Two properties, because they answer different questions. `dataset_uri` is the CANONICAL handle +— machine-parseable, stable, the string two runs are compared on. `dataset_url` is a link a +person can open, and `None` whenever the data has no web page. Collapsing them would force a +choice between a browsable string that lies about local data and a canonical one nobody can +click. + +`dataset_uri(source)` materializes a deferred source (as `project` does), reads the property, +and — when the object has none — follows a `.source` attribute and asks again, depth-capped and +cycle-safe. Every view source in this package uses that attribute name, so all of them work with +no code of their own, and so does any third-party wrapper that follows the convention. + +**A wrapper propagates the URI unchanged.** Decorating it with the slice or the split the wrapper +applies was considered and rejected: the handle identifies the *dataset*, and how much of it a +run consumed is already recorded by the wrapper's own configuration. Decorating would mean one +dataset reached two ways no longer compares equal — the single property the handle exists to +have. + +A source holding SEVERAL datasets (`ConcatSource`) answers `None` rather than picking a member; +`dataset_uris` is the plural form that fans out over them. + +### Consequences + +- **Asking is free and never loads.** The properties read stored configuration, so a tracking + layer can identify a dataset that the run has not touched yet — and a source that is never + iterated still names itself. +- **A new source needs one property, not a registration.** `SupportsDatasetIdentity` is a + `Protocol`, so a domain package opts in by defining the members. +- **A consumer derives the classification from the URI, not from the source.** The scheme is the + kind, the path is the name, a `split` parameter is the split — so a consumer recording datasets + needs no knowledge of any particular source type, and a new source type needs no change there. +- **The query string is sorted.** Two identically-configured sources must produce one string; + parameter order that depended on insertion would silently break comparison. +- **`None` is an ordinary answer**, not an error: an unconfigured source, a stream over an + in-memory list, and data with no web page all legitimately have nothing to say. + +### Example + +```python +from recordstream import HuggingFaceSource, Stream, dataset_uri, dataset_uris, dataset_url + +source = HuggingFaceSource(path="ylecun/mnist", split="train") +source.dataset_uri # 'hf://datasets/ylecun/mnist?split=train' +source.dataset_url # 'https://huggingface.co/datasets/ylecun/mnist/viewer/default/train' + +# a view reports the same DATASET — the slice it takes is its own configuration, not identity +dataset_uri(Stream(source=RangeSource(source=source, stop=100))) +# 'hf://datasets/ylecun/mnist?split=train' + +# several datasets end to end are not one dataset +dataset_uri(ConcatSource(sources=[a, b])) # None +dataset_uris(ConcatSource(sources=[a, b])) # ['hf://datasets/…', 'file:///…'] +``` + +### What you may change (and where it's documented) + +- **Give a new source an identity**: add the two properties. Keep them cheap and side-effect free + — they are asked of sources that may never be read. +- **Recognise a new wrapper shape**: `recordstream/uri.py` follows `.source`; a wrapper using a + different attribute name implements the properties itself instead. +- **Usage** is [docs/sources.md](sources.md#identifying-a-dataset). + +## 14. `Boxes` is pixel-only — the signal-domain region type moved out (2026-08-10) + +### Context + +The structured item was born as `Regions`, and its own docstring sanctioned TWO coordinate +systems in one field: pixel `[x0, y0, x1, y1]` rows on an image raster, or signal +`[f0, f1, t0, t1]` rows in time/frequency. Nothing on the item said which one a given +instance held — consumers disambiguated by record key and by which op produced the value. +Every mechanical consumer in this package (the detection target ops, `ResizeDetection`'s +scaling, the three geometry-desync guards, `canvas` itself — an `(H, W)` raster) assumed the +pixel reading; a `Regions` carrying signal coordinates satisfied `isinstance` checks written +for a contract it did not hold. A third convention hid inside the same type: +`connected_component_bboxes` emitted INCLUSIVE `(row_min, row_max, col_min, col_max)` bin +tuples, axis-swapped from every other producer, reconciled by a transpose at exactly one +call site. + +### Decision + +The item is **`Boxes`**, and it is **pixel-only**: half-open absolute-pixel +`[x0, y0, x1, y1]` rows (x rightward, y downward), `labels`/`scores`, the `(H, W)` `canvas` +frame, per-box `extras`. A domain package needing a different coordinate system registers its +OWN item through the SAME open `register_item` registry — exactly the modality-neutrality +story the item registry exists for; this engine keeps zero knowledge of it. +`connected_component_boxes` (renamed WITH its contract, so stale callers break loudly) now +emits the same half-open xyxy convention, `ConnectedComponents` fills `canvas` in (empty +masks included), and the one reconciling transpose in `masks_to_detection` is gone. There is +NO back-compat alias in either direction (the workspace rename convention): a stored +`typedrecord-v1` record carrying `__item_type__: "Regions"` fails loudly on decode and is +re-generated with a current sink. + +### Consequences + +- One name, one convention: `isinstance(value, Boxes)` now implies the pixel contract the + geometry guards and detection consumers were already assuming. +- The guards correctly go SILENT for a domain package's region item — physical-unit + coordinates are raster-independent, so "the pixels moved and the boxes did not" was a false + alarm for them all along. +- Every `Boxes` producer emits the same row shape; the `+1`/transpose fix-ups that existed + only to bridge `connected_component_bboxes`' divergent order are deleted rather than moved. +- Stored datasets from before the rename must be re-generated (loud `KeyError`/`TypeError` on + decode — the established `typedrecord-v1` no-back-compat rule). + +### Example + +```python +from recordstream import Boxes, Mask +from recordstream.ops.numpy import ConnectedComponents + +out = ConnectedComponents()({"mask": Mask(mask_2d)}) +out["boxes"] # Boxes(boxes=[(x0, y0, x1, y1), ...], canvas=mask_2d.shape) +mask_2d[y0:y1, x0:x1] # half-open: covers the component exactly +``` + +### What you may change (and where it's documented) + +- **Add a new pixel-box producer**: emit half-open xyxy and FILL `canvas` in (the + metadata-on-the-value mandate); the pins live in `tests/test_typed_generic_ops.py`. +- **A new coordinate system** is a NEW registered item in the owning domain package, never a + second meaning for `Boxes` — that is the mistake this record exists to prevent. +- **Usage** is [docs/record-model.md](record-model.md); the batch read-back is `batch_boxes` + ([docs/kinds.md](kinds.md)). diff --git a/docs/augmentation.md b/docs/augmentation.md new file mode 100644 index 0000000..4931d2b --- /dev/null +++ b/docs/augmentation.md @@ -0,0 +1,228 @@ +# Augmentation — well-known libraries run AS-IS + +RecordStream does not reimplement augmentations, and it does not wrap them either. A bare +[albumentations](https://albumentations.ai) transform or a bare torchvision `transforms.v2` +transform drops **as-is** into any ops list — `Stream(ops=[...])`, a `Pipeline`, a `flow:` step, +inside `RandomApply` / `Enable` — and the engine's op-family dispatch +(`recordstream.core.families._apply_op`) invokes it the way its own library expects. There are no adapter +classes and no generated per-transform op families. + +```python +import albumentations as A +from torchvision.transforms import v2 +from recordstream import Stream, Pipeline + +stream = Stream(source=records, ops=[ + A.HorizontalFlip(p=0.5), # bare albumentations + A.GaussNoise(p=1.0), # bare albumentations + my_native_op, # native recordstream op — same list +]) + +Pipeline([v2.ToImage(), v2.RandomCrop(8)])(record) # bare torchvision v2 +``` + +Torchvision is optional (`pip install "recordstream[vision]"`); albumentations is a core +dependency. The family check is by MRO module name — neither library is imported until you +actually put one of its transforms in a pipeline. + +## How each family is invoked + +- **albumentations** dispatches by KWARG NAME: the op receives exactly its own target keys + present in the record — `image` / `mask` / `masks` / `bboxes` / `keypoints` / `labels` — and + nothing else, so extra record entries (scalars, domain items) never reach a library that would + reject them. One call = **one joint draw** across those keys: image, mask and boxes move with + the same decision. Array outputs are re-wrapped in the incoming value's item type, so an + `Image` / `Mask` keeps its type and metadata through the library. A record with none of the + known keys passes through untouched (logged at debug). +- **torchvision `transforms.v2`** natively walks dicts: the op is called on the record as-is, + samples its parameters once, transforms tensor / tv_tensor / PIL leaves and passes everything + else (labels, scalars) through. +- **everything else** is a native/wiring op `record -> Optional[Record]` (`None` drops the + record). + +## The key vocabulary — and routing into it + +Key names carry meaning: albumentations sees only its own vocabulary, so a value augments only if +it rides one of those keys. If your pipeline produced the value under another name, route it with +`RenameField` (`recordstream.ops.structure`) before the library op: + +```yaml +ops: + - !class:recordstream.ops.structure.RenameField {src: spec_view, dst: image} + - !class:albumentations.GaussNoise + p: 1.0 +``` + +## Boxes: use the library's own Compose + +Box-carrying augmentation is albumentations' `Compose` job — drop a prebuilt `A.Compose` with its +own `bbox_params` into the ops list (the record supplies `bboxes` + `labels` under exactly those +keys): + +```python +import albumentations as A + +flip = A.Compose( + [A.HorizontalFlip(p=1.0)], + bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"]), +) +out = Pipeline([flip])(record) # image + mask + bboxes flipped together, one draw +``` + +Format handling (`pascal_voc` / `coco` / `yolo` / `albumentations`) is `BboxParams`' knob — the +engine adds nothing on top. The detection-target ops (`CocoToTorchVisionDetection` / +`MasksToDetectionBoxes`) produce a `Boxes` item for the training boundary; the plain +`bboxes`/`labels` list keys are the augmentation-time form the library consumes. + +For a plain deterministic resize of the `Boxes` form there is `ResizeDetection` +(`recordstream.ops.target`) — the detection twin of the joint image+mask draw: it resizes the +image (PIL or uint8 array) to a fixed `(height, width)` AND scales the `Boxes` boxes by the +same factors in one coupled step, recording the new frame in `canvas`. Fixed-input-size +detectors need it; detectors that resize internally simply omit it. Run it BEFORE any float +conversion (e.g. before `ToTensor`): + +```yaml +ops: + - !class:recordstream.ops.target.CocoToTorchVisionDetection { bbox_format: xywh, label_offset: 1 } + - !class:recordstream.ops.target.ResizeDetection { width: 256, height: 256 } + - !class:recordstream.ops.torch.ToTensor { mode: RGB, normalize: true } +``` + +### Boxes carry the frame they are stated in + +A box is only meaningful against a raster, so a `Boxes` records that raster in `canvas` — +`(H, W)` — and every op that makes or re-frames one fills it in: `CocoToTorchVisionDetection` +from the image the annotation describes, `MasksToDetectionBoxes` from the mask the boxes were +derived from, `ResizeDetection` from the size it resized to (including for an empty target, so a +negative example is not the one record whose frame is unknown). It stays `None` only when no +image is in the record to read. + +That makes a desync *detectable*, which matters because it is otherwise silent. An **image-only** +resize moves pixels without moving boxes: + +```yaml +ops: + - !class:recordstream.ops.target.CocoToTorchVisionDetection { bbox_format: xywh } + - !class:recordstream.ops.image.ConvertToImage { width: 256, height: 256 } # ← boxes left behind +``` + +Every shape downstream stays valid — only the coordinates are wrong — so a model trains happily +against misplaced targets. `ConvertToImage` warns once per op when it resizes a record carrying a +`Boxes`, and a consumer that must be certain compares `canvas` against the image itself +(`recordstream.ops.image.image_frame` reads the `(H, W)` of either). Use `ResizeDetection`, which +moves both. + +**A bare library transform has the same gap**, for the reason that makes the dispatch work: an +albumentations op receives exactly its own key vocabulary, and a `Boxes` is not in it. So a bare +`A.Resize` resizes the image and leaves the boxes; a bare `A.HorizontalFlip` mirrors the pixels +and leaves them — *without changing any shape at all*. The engine warns once per transform type +when a geometry-changing transform runs while a `Boxes` sat out the call, deciding "geometry- +changing" by the library's own `DualTransform` / `ImageOnlyTransform` split (so `Normalize` and +friends stay silent). Speak the library's vocabulary and it moves them for you, in the same draw: + +```yaml +ops: + - !class:recordstream.ops.structure.RenameField { src: my_boxes, dst: bboxes } + - !class:albumentations.Compose + transforms: [!class:albumentations.HorizontalFlip { p: 0.5 }] + bbox_params: !class:albumentations.BboxParams { format: pascal_voc, label_fields: [labels] } +``` + +**torchvision v2 has it too, by the other route.** v2 walks the record natively but transforms +only its OWN `tv_tensors` types, and a `Boxes` is not one — so `v2.Resize` moves the pixels and +leaves the boxes, while the same transform over a `tv_tensors.BoundingBoxes` rescales them +correctly. The engine warns once per transform type here as well, using v2's geometric-transform +grouping so `ColorJitter` and `Normalize` stay silent. Carry boxes in v2's own type when you want +v2 to move them: + +```python +from torchvision import tv_tensors + +record["boxes"] = tv_tensors.BoundingBoxes(boxes, format="XYXY", canvas_size=(h, w)) +``` + +Either way, `ResizeDetection` remains the plain coupled resize over the `Boxes` form. + +## YAML — bare library transforms are ordinary `!class:` nodes + +No library-specific serialization format — a transform is a Confluid `!class:` node like any op, +in mapping form or call form. `Stream` flows deferred markers at route entry, and composing ops +(`Pipeline` / `Enable` / `RandomApply`) flow theirs lazily: + +```yaml +ops: + - !class:albumentations.HorizontalFlip + p: 0.5 + - !class:albumentations.GaussNoise {p: 1.0} + - !class:recordstream.ops.numpy.Threshold + low_level: 0.5 +``` + +## Layout contract (the main footgun) + +The two libraries disagree about layout, and the engine keeps each library's native convention +instead of hiding it — **conversions are always explicit library transforms, never silent**: + +- **albumentations** consumes and emits numpy **HWC** — run it while your values are still numpy + arrays (an `Image`/`Mask` is an ndarray subclass, so it feeds straight in). +- **torchvision v2** wants **CHW tensors** — put the library's own `v2.ToImage()` (numpy HWC → + CHW tv_tensor) in the list first, then any v2 transform; exactly like a plain torchvision + pipeline. + +Don't chain one library's output straight into the other without an explicit conversion step. + +## Randomness & seeding + +Stochasticity lives where each library puts it — the engine adds no seed plumbing: + +- albumentations: `A.Compose(seed=N)` on a prebuilt Compose (individual transforms keep their own + `p`). +- torchvision v2: the global torch RNG — `torch.manual_seed(N)`. +- per-record gating of any op (native or library): `RandomApply(op=..., probability=..., + random_state=N)`. + +## Multiprocessing: two fork hazards, both handled + +A `DataLoader` worker is often a **forked** child (torch's default on Linux, and on macOS whenever +something in the process has set `fork` — `import fastai` does). A fork inherits memory but not +threads, so two things on this path would otherwise crash the child with a **SIGSEGV and no Python +traceback**, surfacing only as `DataLoader worker exited unexpectedly`: + +| hazard | guard | who calls it | +|---|---|---| +| OpenCV's thread pool, inherited across the fork (albumentations runs on cv2) | `cv2.setNumThreads(0)` | the engine, automatically, the first time it invokes an albumentations op | +| a lazy source first built in the child — the download reaches `_scproxy`, which is not fork-safe | `ensure_materialized(source)` | **you**, in the parent, before building the loader | + +The first needs nothing from you. The second does: + +```python +from recordstream import ensure_materialized, ensure_record_dataset + +dataset = ensure_materialized(ensure_record_dataset(train_set)) # reads ONE record, in the parent +loader = DataLoader(dataset, num_workers=4, collate_fn=collate_records) +``` + +They are **independent** — neither fixes the other. With the source warmed but the cv2 pool on, the +worker still dies; with the pool off but the source cold, it is still built in the child. Setting +`num_workers=0` avoids both by not forking at all, which is why the crash looks intermittent and +gets misread as a flaky test rather than an ordering bug. + +Turning cv2's pool off costs nothing where it matters: inside a worker the *worker* is the +parallelism, so cv2's own threads oversubscribe rather than help. The engine does it at the point +of use, never at import — a process that never touches albumentations keeps its OpenCV settings. + +## Other libraries — register an op family + +albumentations and torchvision v2 are the built-in families, registered through the same OPEN +registry any package can use: `register_op_family(name, matcher, invoker)` teaches the engine a +new library's native calling convention (kornia, DALI, a fork extending albumentations, a +signal-processing library), and bare ops of that library then sit in ANY ops list — every engine +route and composing op, including spawn-parallel workers. Full example + rules: +[record-model.md → "A new library family"](record-model.md#a-new-library-family). + +## Example + +[`examples/record_pipeline.py`](../examples/record_pipeline.py) — the tour: a bare +`A.Compose` with `bbox_params` + `A.GaussNoise` + a native type-dispatched op in ONE `Pipeline` +(image/mask/bboxes moved jointly, types preserved), `field=` pinning, and torchvision v2 as-is +after an explicit `v2.ToImage()`. diff --git a/docs/configure.md b/docs/configure.md new file mode 100644 index 0000000..d31da7d --- /dev/null +++ b/docs/configure.md @@ -0,0 +1,93 @@ +# Per-record op parameters (`ConfigureOp` / `Apply` / `Capture`) + +Some op parameters are only known *per record*. Two mechanisms cover this: + +- **`ConfigureOp(ops, target, param, source)`** — runs the `ops` compute-chain on the record as a SIDE branch (its transformations are discarded — the original record continues); the `source`-keyed entry of the chain's final record becomes the VALUE (payload-unwrapped via `item_data`), which is set as the `param` attribute of `target` — post-construction configuration, the confluid paradigm — and then `target` is applied to the original record. Use it when the value is *derived from the record itself* (e.g. a threshold from the record's own max) — the whole derivation reads as one node/YAML block. +- **`Capture` + `Apply`** (`recordstream.ops.context`, see [graph.md](graph.md)) — when the value is an op's runtime **`@output`** (possibly stochastic — a random draw that can't be recomputed): `Capture(op, output, name)` applies the producer and records its live `@output` into a Context cell; a later `Apply(op, param, source)` sets the consumer's `param` from that cell and applies it. This is what graph exporters emit for `@output` → param wires, and the preferred form whenever the value already lives in a cell. + +Concretely — a producer that draws a random gain per record and publishes what it ACTUALLY drew +as a confluid `@output` (apply `@output` UNDER `@property`), and a consumer whose `level` gets set +per record. The pair is deliberately a roundtrip: compensating with the captured gain restores the +original image, which proves the LIVE draw — not a recomputation — reached the consumer +(verified: `np.allclose(out["image"], original)` holds for every record): + +```python +# mypackage/ops.py +from confluid import configurable, output +from recordstream import Record, item_data, with_data +import numpy as np + +@configurable(category="op", random=True) +class AugmentOp: + """Scale the image by a random gain drawn per record. + + Args: + max_gain: Upper bound of the uniform gain draw. + """ + + def __init__(self, max_gain: float = 2.0) -> None: + self.max_gain = max_gain + self._applied = 1.0 + self._rng = np.random.default_rng() + + @property + @output + def applied_level(self) -> float: + """The gain the LAST call actually drew — the live @output that Capture records.""" + return self._applied + + def __call__(self, record: Record) -> Record: + self._applied = float(self._rng.uniform(1.0, self.max_gain)) + img = record["image"] + return {**record, "image": with_data(img, item_data(img) * self._applied)} + +@configurable(category="op") +class CompensateOp: + """Divide the image by ``level`` — undo a gain applied earlier in the chain. + + Args: + level: The gain to divide out; set per record by Apply (or ConfigureOp). + """ + + def __init__(self, level: float = 1.0) -> None: + self.level = level + + def __call__(self, record: Record) -> Record: + img = record["image"] + return {**record, "image": with_data(img, item_data(img) / float(self.level))} +``` + +```yaml +ops: + # AugmentOp draws a random gain each call; capture the LIVE @output into a cell. + - !class:recordstream.ops.context.Capture + op: !class:mypackage.ops.AugmentOp {} # or the registered short name: !class:AugmentOp {} + output: applied_level + name: __captured_level + # …then inject the captured value into the consumer's parameter, per record. + - !class:recordstream.ops.context.Apply + op: !class:mypackage.ops.CompensateOp {} + param: level + source: __captured_level +``` + +Reading the pair: `Capture` runs `AugmentOp` once (the record's image is scaled by, say, 1.7×) and +stores `applied_level` = 1.7 in the `__captured_level` cell; `Apply` does +`setattr(compensate_op, "level", 1.7)` and then runs it — post-construction configuration, the +confluid paradigm. Because the value is read off the op AFTER it ran, a stochastic draw is captured +exactly; recomputing it (the naive alternative) would draw a DIFFERENT number. + +A self-contained `ConfigureOp` example — derive a per-record threshold from the record's own statistics: + +```yaml +ops: + - !class:recordstream.ops.configure.ConfigureOp + ops: + - !class:recordstream.ops.formula.FormulaOp {field: image, formula: "amax(a) * 0.5"} + source: image + target: !class:recordstream.ops.numpy.Threshold + low_op: ">=" + param: low_level +``` + +Both mechanisms leave the record's own entries untouched: `ConfigureOp`'s compute chain runs on a side-branch copy, and `Capture`/`Apply` move values through the per-record Context. All inner ops (the compute chain, `target`, the wrapped ops of `Capture`/`Apply`) are applied through the engine's op-family dispatch, so a bare library transform works in any of these slots too. diff --git a/docs/graph.md b/docs/graph.md new file mode 100644 index 0000000..f941a83 --- /dev/null +++ b/docs/graph.md @@ -0,0 +1,126 @@ +# Graph pipelines — `flow:` documents and the `FlowGraph` engine + +A pipeline is a **graph of named steps**. There is ONE engine and ONE execution model; `ops:` +and `flow:` are two spellings of it, and which one you write is purely about whether the +pipeline branches. + +`recordstream.core` and `recordstream.flow` are packages with **one module per cohesive unit**. +Import from the package — `from recordstream.core import Stream` / `from recordstream.flow import +FlowGraph` — but spell the **submodule** path in a config, because that is what `cls.__module__` +says and what a generated config emits: + +| class | module | `!class:` path | +| --- | --- | --- | +| `Stream`, `JointStream` | `core/stream.py` | `recordstream.core.stream.Stream` | +| `FilterOp`, `WrappedOp` | `core/wrappers.py` | `recordstream.core.wrappers.FilterOp` | +| `FlowGraph` | `flow/graph.py` | `recordstream.flow.graph.FlowGraph` | + +The shorter `!class:recordstream.core.Stream` still resolves (Confluid falls back to a +module-path import, and each package re-exports every name), so an older config keeps loading. +Rationale: [docs/architecture.md §12](architecture.md#12-core-and-flow-are-packages-too--layered-by-import-direction-2026-08-01). + +## `ops:` — the linear spelling + +A straight chain is a graph where every step reads the one before it, so it needs no names: + +```yaml +ops: + - !class:recordstream.ops.image.ConvertToImage {width: 224, height: 224} + - !class:albumentations.Normalize {mean: [0.485, 0.456, 0.406], std: [0.229, 0.224, 0.225]} + - !class:recordstream.ops.torch.ToTensor {normalize: false} +``` + +The engine compiles that list into positional steps (`s0`, `s1`, `s2`) and runs it on the same +kernel a `flow:` document uses. The names never surface — nothing in an `ops:` document can +reference a step. Repeats stay distinct: the same op twice in a row is two steps. + +## `flow:` — the named spelling, for branchy pipelines + +When a pipeline forks, merges, or feeds one step's value into another's parameter, the steps +need names — and the name is how a later step refers to an earlier one: + +```yaml +flow: + spec: !class:mypkg.MakeSpectrogram {} # input: the source record + masked: !class:recordstream.ops.numpy.Threshold {low_level: 0.5, from: spec} # 2nd reader of `spec` = fan-out + thresh: !class:recordstream.ops.formula.FormulaOp {formula: "amax(a) * 0.6", field: image, from: spec} + gated: # a step with bind: uses the plain-mapping form + op: !class:recordstream.ops.numpy.Threshold {output: gated_mask} + from: spec + bind: + low_level: thresh[image] # per-record param := the `image` entry of thresh's result + out: {from: gated, merge_from: [masked]} # fan-in (no op) +outputs: out +``` + +Two YAML spelling rules (both verified): SCALAR/list reserved keys (`from:`, `merge_from:`) may +ride inside a `!class:` marker's mapping alongside its kwargs — but **`bind:` (a nested mapping) +MUST use the plain-mapping step form** (`op:` + reserved keys, the `gated` step above): a nested +mapping under a `!class:` marker is consumed by Confluid as addressed configuration and never +reaches the step grammar. Write bind refs in block style or quoted — `{low_level: thresh[image]}` +inline is a YAML parse error (`[` opens a flow sequence). + +Step grammar (three reserved keys, stripped before the op is built): + +- **`from:`** — the input step (omitted = previous step; must name an *earlier* step, so document + order is the schedule and cycles are inexpressible). +- **`merge_from:`** — fan-in: UNION the named steps' record ENTRIES into this step's incoming + record, in listed order, last-write-wins on a key collision. +- **`bind:`** — `{param: ref}` per-record parameters: a bare `step` binds the step's WHOLE result + record, `step[key]` the named ENTRY of its record, and `step.attr` the step op's live + `@output` (read after it ran — stochastic-correct). + +A plain-mapping step with no op (`out: {from: a, merge_from: [b]}`) is a pure fan-in; `{}` is the +identity (names the source). `outputs:` picks the yielded step (default: the last). Steps apply +their ops through the engine's op-family dispatch, so bare library transforms sit in flow steps +too. + +## Running one + +```python +from recordstream import FlowGraph, Stream +from recordstream.sources import HuggingFaceSource + +graph = FlowGraph.from_yaml("graph.yaml", source=HuggingFaceSource(path="ylecun/mnist")) +for record in graph: + ... + +graph.parallel(4) # spawn workers, one future per source record +len(graph); graph[3] # map-style access (unavailable if a step op is 1→N expanding) +``` + +`FlowGraph` is a map-style dataset like `Stream` (`__len__`/`__getitem__`/`.batch`/`.parallel`). +Both classes hold a step graph and call the same per-record kernel; a straight chain takes an +env-free fast path in that kernel, so `ops:` costs no more to run than it ever did. + +A LINEAR graph converts to a `Stream` (`graph.to_stream()`) because an op list can express a +straight chain. A branchy one does not — and there is no lowering pass that would manufacture a +flat spelling for it. That pass existed until 2026-07-30 (`to_ops`/`from_ops` plus six context +ops that re-encoded dataflow as imperative mutations of a per-record cell store); it was deleted +because it destroyed the very structure every consumer — a compiler, a visual editor, a reader — +wants back. Rationale: [architecture.md](architecture.md). + +## Expanding (1→N) steps + +A step whose op carries `EXPANDS = True` yields several records from one. The remaining subgraph +runs once per child over its own shallow copy of the step environment, depth-first, so sibling +order matches the nested-loop intuition. Such a pipeline is ITERABLE-ONLY: `__len__`/`__getitem__` +raise, because the expanded index map is unknowable up front. + +## Reattach an ops-only YAML (`Stream.from_ops_yaml`) + +A `{ops: [!class:…()]}` document — e.g. one exported by a visual editor — can be attached to any +source: + +```python +from recordstream import Stream +from recordstream.sources import HuggingFaceSource + +stream = Stream.from_ops_yaml("ops.yaml", source=HuggingFaceSource(path="ylecun/mnist")) +``` + +The helper **materializes** the deferred `!class:` markers eagerly (via `confluid.load`) so +a broken op fails at load time with the YAML in hand. It is a convenience, not a necessity: +`Stream` also flows any still-deferred marker in place at engine-route entry, which is what lets a +bare mapping-form `!class:albumentations.HorizontalFlip {p: 0.5}` sit directly in an `ops:` list. +`FlowGraph.from_ops_yaml` loads the same document as a linear step graph. diff --git a/docs/image.md b/docs/image.md new file mode 100644 index 0000000..e36dc2a --- /dev/null +++ b/docs/image.md @@ -0,0 +1,68 @@ +# Image conversion (`recordstream.ops.image`) + +The single, modality-agnostic "any value → image" layer — generic so every consuming project (spectrogram previews, dataset browsers, GUI viewers) reuses one implementation. Domain-specific rendering (overlays, signal plots) stays in the consuming package. + +```python +from recordstream.ops.image import ConvertToImage, value_to_image + +# Op: an array-bearing record value (2-D map / CHW tensor / PIL / bool mask) -> an Image item. +op = ConvertToImage( + colormap="viridis", # closed `Colormap` Literal -> enumerable in GUIs / schemas + width=1024, height=512, # exact resize when both > 0; else bound longest side by max_size + flip_vertical=True, # e.g. a spectrogram stores row 0 = f_min but display wants f_max on top + field="spec", # source key; blank picks the first array-bearing value + output="image", # key the HWC-uint8 Image item is written to +) +record = op(record) # adds record["image"]; the pixel dimensions live in its array shape + +# Library function for ad-hoc previews (PIL / tensor / ndarray / mask -> (H, W, 3) uint8): +rgb = value_to_image(some_value, colormap="magma", max_size=512) + +# normalize_to_uint8: the standalone min-max value -> uint8 quantization step +# (decoupled from colormap / PIL). vmin/vmax default None = per-array auto-contrast; +# set them to pin a fixed scale across records (out-of-range values clamp). +from recordstream.ops.image import normalize_to_uint8 + +u8 = normalize_to_uint8(arr) # auto per-array min/max +u8 = normalize_to_uint8(arr, vmin=-80.0, vmax=0.0) # fixed dB window across a dataset +``` + +`record_to_image(record, ...)` renders a record's first array-bearing (2-D / 3-D) value the same way — the ad-hoc whole-record preview for viewer tooling. Pillow is a runtime dependency; matplotlib is imported lazily (only non-`gray` colormaps need it). + +## Masks (`ConvertToMask`) + +The segmentation counterpart, and the same shape of op — read one field, write a differently-typed item under `output`. A segmentation dataset ships its target as a greyscale/paletted PNG whose pixel values *are* the class ids (an Oxford-IIIT Pet trimap, Cityscapes label ids, a VOC segmentation map); this turns that payload into the `int64` `[H, W]` `Mask` every per-pixel loss expects. + +```python +from recordstream.ops.image import ConvertToMask + +op = ConvertToMask( + field="segmentation_mask", # source key; blank picks the first array/PIL-bearing value + output="mask", # key the int64 Mask item is written to +) +``` + +It converts and **nothing else**, because the rest of the chain is ops that already exist: + +| you want | use | +| --- | --- | +| remap the ids (a 1-based trimap → 0-based) | `FormulaOp(field="mask", formula="a - 1")` | +| remap through a lookup table (Cityscapes id → trainId) | `EncodeTarget` | +| resize / augment it **together with the image** | a bare `albumentations` transform in the same ops list | +| drop the source column | `DropField(key="segmentation_mask")` | + +That fourth row is why `output` defaults to `"mask"`: it is albumentations' own key vocabulary, so the engine's op-family dispatch hands `image` **and** `mask` to one call — a single joint draw moves both, and the `Mask` type survives the round trip. An image-only transform (`Normalize`) still touches the image alone. + +```yaml +# the target half of a segmentation `preprocess` chain +- !class:recordstream.ops.image.ConvertToMask {field: segmentation_mask, output: mask} +- !class:recordstream.ops.formula.FormulaOp {field: mask, formula: a - 1} +- !class:recordstream.ops.structure.DropField {key: segmentation_mask} +- !class:albumentations.Resize {height: 224, width: 224} # image AND mask +``` + +`int64` is not a knob: a class-id map is integer by definition, and it is what `torch.nn.CrossEntropyLoss` requires (it rejects int32 with *"expected target dtype to be Long or Byte, but got Int"*). Libraries that cast on the way past — albumentations returns int32 — are corrected at the model boundary with `batch_tensor(batch, "mask", dtype=torch.int64)`, where the caller names the contract. An RGB-encoded mask is **refused** rather than collapsed: picking one of three channels is a decision the op must not make silently. + +## Introspection helpers + +Pure library functions (not ops) also live here, backing viewer tooling: `select_channel` (reduce an array/tensor to a 2-D float32 map for one channel; negative = mean across channels), `channel_count`, `array_histogram` (finite-only binning + summary stats, JSON-safe), `confusion_matrix_payload` / `confusion_matrices_payload` (render payloads for every confusion-matrix-shaped entry in a metrics result), and `draw_text` (text → `(H, W, 3)` uint8 image with word-wrap and 9-grid anchoring, plus the closed `TextPosition` Literal). diff --git a/docs/kinds.md b/docs/kinds.md new file mode 100644 index 0000000..bf87b5c --- /dev/null +++ b/docs/kinds.md @@ -0,0 +1,172 @@ +# Ops, batching & expanding ops (`recordstream.transform` / `recordstream.collate`) + +## What an op processes — dispatch on value type + +A **record** is a plain dict of typed values (`Image`, `Mask`, `Boxes`, `Label`, … — see [record-model.md](record-model.md)). A native op is a `Transform`: it declares which value TYPES it handles and registers a per-type **kernel**; it samples its parameters ONCE per record (`get_params`), then applies the matching kernel to every value whose type it handles, passing untouched values through: + +```python +from recordstream import Record, Transform, Image + +class Recenter(Transform): + handles = (Image,) # which value types this op touches + + def get_params(self, record: Record) -> dict: + return {"mean": 0.5} # sampled ONCE per record, shared across values + +@Recenter.kernel(Image) # per-type behaviour +def _(value, params): + return value - params["mean"] +``` + +Because the parameters are sampled once and shared, an op that handles several types moves those values **consistently** — one drawn decision applies to every handled value in the record. Dispatch is MRO-aware: a kernel registered for a base item type also serves its subclasses, and a subclass transform inherits its base's kernels until it overrides them. + +Two smaller shapes round it out: + +- **A plain function** becomes an op via `as_transform(fn, handles=(Image,), field="image")` — `field=` pins the op to one named key (still type-gated). +- **A type-changing op** — read one key, write a differently-typed item (`Threshold`: array → `Mask`, `ConvertToImage`: array → `Image`, `ConnectedComponents`: `Mask` → `Boxes`) — subclasses `Transform` and overrides `__call__` instead of registering a same-type kernel. + +Bare library transforms (torchvision `transforms.v2` walking the dict natively, albumentations dispatching by keyword name) drop straight into any ops list **as-is** — the engine's op-family dispatch invokes each one the way its own library expects. See [record-model.md](record-model.md#mixing-libraries--as-is-no-adapters) and [augmentation.md](augmentation.md). + +```python +import albumentations as A +from recordstream import Pipeline + +out = Pipeline([ + A.HorizontalFlip(p=1.0), # image + mask + bboxes together (one library draw) + A.GaussNoise(p=1.0), # image only — its own kwarg vocabulary + Recenter(), # native op — same list +])(record) +# record["class"] (a Label) is untouched: no kernel handles it, no library key names it. +``` + +## Batching — `collate_records` & the collate registry (`recordstream.collate`) + +Ops are per-record; batching is a separate stage. **`collate_records`** (the registry's `"record"` default) stacks N record dicts into ONE batched record: per key, typed payloads stack (torch → stacked tensor, numpy → stacked array, else a list) and each item's declared attrs become per-record lists, decoded back into one batched item of the same type; plain values batch as plain lists. Batches must carry the same keys — a mismatch raises. + +```python +from recordstream import collate_records +from torch.utils.data import DataLoader + +batch = collate_records(list(stream)) # ONE batched record: payloads stacked per key +loader = DataLoader(stream, collate_fn=collate_records) +``` + +### The batch SHAPE is a choice — `"record"` vs `"list"` + +Whether a column is STACKED is a requirement of the **model**, not a property of the data: a +torchvision detector takes `List[Tensor]` (its images differ in size), a classifier takes one +`[N, C, H, W]` tensor. Two collates ship, differing in exactly that: + +```python +from recordstream import collate_list, collate_records + +collate_records(records)["image"] # Image (2, 3, 8, 8) — stacked +collate_list(records)["image"] # [Image (3, 8, 8), Image (3, 12, 12)] — per record, items kept +``` + +Declare it where the batch is built — `DataLoader(stream, collate_fn=collate_list)` on torch, +`RecordSequence(source, collate="list")` on Keras. Left implicit, the shape is decided by +accident: under the default collate a column stacks if it holds array *items* and stays a list if +it holds *plain* values, so whether an op like `ToTensor` ran ends up choosing for you. + +Two things make the choice free: + +* **every read-back helper accepts both shapes** — `batch_values` unwraps a list-of-items + element-wise, `batch_boxes` reads a batched `Boxes` or a list of them, `batch_metadata` + transposes either — so a consumer never branches on which collate ran; +* **a stack failure explains itself**, naming the key, the differing shapes and the `"list"` way + out, rather than surfacing numpy's bare *"all input arrays must have the same shape"*. + +The registry is additive, so a task can register its own convention too: + +```python +from recordstream import get_collate, register_collate + +@register_collate("yolo") # task aliases are additive +def yolo_collate(items): ... +loader = DataLoader(stream, collate_fn=get_collate("yolo")) +``` + +The string keys primarily target the MCP tool surface (JSON-serializable, enumerable collate selection) — in Python, passing the function directly stays the normal path. The full rationale is recorded in [architecture.md](architecture.md#2-batching-is-two-stage-collation-is-a-pluggable-registry-recordstreamcollate-2026-07-17). + +### Reading a batch back (`recordstream.batch`) + +The inverse of `collate_records`, shipped alongside it so a model boundary never re-derives the convention: + +```python +from recordstream import batch_values, batch_tensor, batch_metadata, multi_hot + +batch_values(batch, "class") # past the wrapper item: a Label -> its .value list +batch_tensor(batch, "image", device=model.device) # ONE torch tensor, stacked + moved +batch_tensor(batch, "class", dev, dtype=torch.int64) # ...with the dtype your loss requires +multi_hot(batch, "class", num_classes) # a MultiLabel column as an [N, C] numpy matrix +batch_metadata(batch, exclude=("image", "class")) # the remaining columns transposed into N dicts +per_record_predictions(model_output) # a BATCHED model output sliced into one entry per record +``` + +Only `batch_tensor` is torch; the rest return plain values or numpy, so a non-torch backend reuses them and converts in one line. `dtype` is a parameter, not an opinion — the same knob as `device`. What stays task-side is only WHICH call a trainer makes. + +### Keras: `RecordSequence` — the batching half the framework leaves to you + +A `DataLoader` needs one thing from recordstream (`collate_fn=collate_records`) and does the rest itself: row order, batch slicing, the short final batch, the per-epoch reshuffle. Keras 3 has no `DataLoader` — `keras.utils.PyDataset.__getitem__` must return a whole **batch** — so that loop is `RecordSequence`, in `recordstream.keras`: + +```python +from recordstream.keras import RecordSequence + +def to_xy(batch): # your collate_fn equivalent + return batch_values(batch, "image"), batch_values(batch, "class") + +train = RecordSequence(stream, batch_size=32, shuffle=True, transform=to_xy) +model.fit(train, epochs=3) # reshuffles between epochs itself +``` + +`PyDataset`'s prefetch knobs are declared parameters, at Keras's own defaults — pass `workers=4` to overlap a slow record walk (decode, resize, remote read) with the training step, `max_queue_size=` to cap how many prefetched batches may wait, and `use_multiprocessing=True` only when the per-record work is GIL-bound (each process re-pickles the sequence and its source). + +`transform` is the whole task-facing surface: it maps one collated record to what the model consumes, so the shape decision stays in your code exactly as it does with a `DataLoader`. Omit it and `__getitem__` hands over the batched record itself — which is also what `seq.batches()` yields, the pairing half of prediction (a model emits `[N, ...]` while a [`PredictionsSink`](predictions.md) writes per record, so you need the batch its output came from to read that batch's [`batch_metadata`](#reading-a-batch-back-recordstreambatch)). + +Needs the extra — `pip install "recordstream[keras]"`. Importing `recordstream.keras` is also what sets `KERAS_BACKEND` (Keras reads it at import time and would otherwise default to TensorFlow, which this extra does not install), so it must be the first keras-touching import in a process; never `import keras` ahead of it. Why the adapter lives here rather than in a training project is recorded in [architecture.md](architecture.md#10-the-frameworks-batching-half-lives-beside-the-collate-recordstreamkeras-2026-07-30). + + +## 1→N expanding ops (iterable-only pipelines) + +An op may return **several** carriers — a windowing op splitting one capture into N windows marks itself with `EXPANDS = True` and returns an iterable of records: + +```python +from typing import Iterator +from confluid import configurable +from recordstream import Record +from recordstream.items import item_data, with_data + +@configurable(category="op") +class SlidingWindow: + EXPANDS = True # the explicit 1→N marker + + def __call__(self, record: Record) -> Iterator[Record]: + item = record["signal"] + for w in sliding_windows(item_data(item), self.size, self.stride): + yield {**record, "signal": with_data(item, w)} +``` + +Expansion is flattened in every iteration route — sequential, spawn-parallel, and streamed — depth-first, so sibling order matches the nested-loop intuition. Each child continues through the remaining ops with its own (shallow-copied) Context; a child filtered to `None` just drops. + +A pipeline containing an expanding op is **ITERABLE-ONLY**: `len(stream)` / `stream[i]` raise a clear `TypeError` (the expanded length is unknowable up front). Iterate it, wrap it in a torch `IterableDataset`, window at the source for random access, or materialize with `list(stream)`. `FlowGraph` steps are strictly 1→1 (a named step has one result) — expanding pipelines belong to the `Stream` engine. + +## The training-side data helpers (`prepare_record_dataset`, `recordstream.loaders`) + +Two helpers every training runnable composes, written once here: + +```python +from recordstream import prepare_record_dataset +from recordstream.loaders import loader_slots # torch-only module, not in the package root + +dataset = prepare_record_dataset(source) # ensure_record_dataset + ensure_materialized; None passes through +slots = loader_slots(batch_size=32, num_workers=0) # deferred DataLoader triple (train shuffled) +loader = flow(slots.train, dataset=dataset) # the dataset arrives at run time +``` + +`prepare_record_dataset` normalizes a wired source's TYPE and its STATE in the calling process +(the fork-safety pair — see the `ensure_materialized` docs). `loader_slots` returns the +train/val/test `LazyClass(DataLoader, ...)` markers as a named tuple; further `DataLoader` +kwargs pass through to all three, and a config can still replace any individual loader slot +wholesale. The module imports torch, so it is deliberately NOT re-exported from the package +root — `import recordstream` stays framework-free. diff --git a/docs/predictions.md b/docs/predictions.md new file mode 100644 index 0000000..fd3ea26 --- /dev/null +++ b/docs/predictions.md @@ -0,0 +1,106 @@ +# The model boundary: datasets in, predictions out + +RecordStream owns the data on both sides of a model: the dataset a trainer consumes, and the +prediction it emits. This page covers the three surfaces that sit on that boundary. + +## `ensure_record_dataset` — normalize a wired dataset slot + +A config can wire a dataset slot to a `Stream`, another torch `Dataset`, a bare source, or a plain +list of records. Normalize once, and the rest of the pipeline (target detection, label encoding, +collate, metrics) can assume record items unconditionally: + +```python +from recordstream import RecordSource, ensure_record_dataset + +dataset = ensure_record_dataset(self.train_set) # -> a map-style Dataset of records +``` + +A `Stream` comes back **as-is** — identity matters, because a label-encoding Stream carries its +`class_names` and re-wrapping would lose it. Anything else is wrapped in a `Stream`, which makes it +both map-style and record-yielding. + +`RecordSource` is the contract it enforces (`Dataset | Iterable[Record]`), named once so a consumer +annotates `Optional[Lazy[RecordSource]]` instead of inventing its own union. + +## Prediction-output contracts (`recordstream.outputs`) + +What a model's eval-mode `forward` returns, declared as typed dicts so metrics, sinks and +visualizers read known keys instead of guessing whether a tensor is logits, probabilities, or +argmax'd class ids: + +| Contract | Keys | +|---|---| +| `ClassificationOutput` | `logits` `[B, C]`, `probs` `[B, C]`, `class_idx` `[B]` | +| `DetectionOutput` | `boxes` `[N, 4]` xyxy absolute pixels, `scores` `[N]`, `labels` `[N]` | +| `SegmentationOutput` | `logits` `[B, C, H, W]`, `probs` `[B, C, H, W]`, `mask` `[B, H, W]` | +| `RestorationOutput` | `image` `[B, C, H, W]` — the restored image, in the input's value range | + +That guess is not hypothetical: two independently-written detector wrappers agree that `boxes` is +xyxy in absolute pixels only because `DetectionOutput` says so. + +`RestorationOutput` carries **one** key while the others carry three, and that asymmetry is the +contract doing its job rather than an unfinished row. The other tasks emit something that needs +interpreting — `logits` are not `probs` are not `class_idx` — whereas an image-to-image model emits +the answer directly, so the thing a consumer needs told is exactly that: the array under `image` is +a picture in the input's units, not logits to be softmaxed and not a residual to be added back. +`restoration_output()` therefore transforms nothing; it exists so every backend spells the wrapping +identically. The residual is deliberately absent — it is `input - image`, and a key a consumer can +compute is a second place for the two to disagree. + +Each contract is **generic in the array type**, so the same declaration describes a torch run and a +numpy/TF/JAX one: + +```python +from recordstream import ClassificationOutput + +def predict(self, x) -> ClassificationOutput[np.ndarray]: ... # a non-torch backend +``` + +A bare `ClassificationOutput` means "whatever array type". The **builders** are the only torch part +— `softmax` and `argmax` are library calls, not type declarations: + +```python +from recordstream import classification_output + +def predict_step(self, batch, batch_idx): + return classification_output(self(x)) # logits -> the full contract +``` + +Detection deliberately has no builder: its boxes come from the detector's own interface, so the +dict is built inline at the call site rather than invented from logits. + +## Predictions sinks (`recordstream.predictions`) + +A predict/test loop calls `predictions_sink.write(prediction, metadata)` once per record and +`close()` at the end. `PredictionsSink` is that contract as a `@runtime_checkable` Protocol — +annotate the slot with it rather than `Any`, so a use site can't call `.write` on something that is +still a deferred marker. + +`ClassificationPredictionsSink` is the classification implementation: read `probs` / `class_idx`, +resolve the class id to a label, build a top-k list, and thread a record through your ops. + +```yaml +predictions_sink: !class:recordstream.predictions.ClassificationPredictionsSink + class_names: !ref:class_id_to_label # {0: "bird", 1: "cat", ...} + top_k: 5 + confidence_threshold: 0.0 # skip predictions below this top-1 probability + ops: + - !class:recordstream.ops.sink.RecordSinkOp + sink: !class:mypkg.JsonSink() { path: ./predictions } +``` + +Each written record carries the original metadata plus `predicted_class_id`, +`predicted_class_label`, `predicted_confidence` and `predicted_top_k`, under a single `"metadata"` +key. Diagnostics identify a record by its **ordinal** in the run — the sink is modality-neutral, so +it never assumes a metadata key exists. + +Like every configurable here it is **zero-arg constructible**: `ops` is required to *run*, not to +*build*, so the non-empty check fires on the first `write` rather than in `__init__`. + +### Why this is a second sink protocol + +`recordstream.storage.base.DataSink` takes a whole record (`write(record)`) and is what +`RecordSinkOp` adapts into an op chain. A predictions sink instead receives the model's output plus +the metadata of the record it came from, and builds the record itself — the two halves arrive +separately because a model emits a batch while the sink contract is per-record. Keep the two +distinct; collapsing them is tracked as a deliberate decision, not something to do by drift. diff --git a/docs/projection.md b/docs/projection.md new file mode 100644 index 0000000..44cfa1c --- /dev/null +++ b/docs/projection.md @@ -0,0 +1,120 @@ +# Key projection, class counting & label maps (`recordstream.projection` / `recordstream.labels`) + +## Key projection + +Walking a source for a single record key (the classic case: counting classes from the label key) shouldn't pay to build the values you don't need. `recordstream.projection` adds an opt-in protocol plus lazy helpers, all **key-addressed** — any subset of record keys: + +```python +from recordstream import first_value, project, iter_key, num_classes + +# A source MAY implement SupportsProjection (`project(keys)`) to skip building +# unrequested values — e.g. an image dataset reads only the label column for a +# class-count walk, never decoding an image. +for record in project(my_source, ("class",)): + ... # partial records carrying only the "class" entry + +labels = list(iter_key(my_source, "class")) # lazy; a Label unwraps to .value, a MultiLabel to + # its .values LIST, other items to their payload, + # plain values verbatim +first = first_value(my_source, "class") # ONE peek — the cheapest question about a column +n = num_classes(my_source, key="class") # max(class_id) + 1 — always walks +``` + +A **deferred** source — a `!class:` marker straight out of a config — is materialized first, so a caller never has to know which entry point flows and which doesn't (flowing a live object is a no-op). Sources that don't implement `SupportsProjection` still work via a correct full-iteration fallback (just without the skip-decode speedup); `Stream.project(keys)` is the engine's implementation — it runs the op chain, then keeps only the requested keys. `num_classes` is a free function, not a `Stream` method: integer class-id semantics are classification-specific, so the task-agnostic engine doesn't advertise it. + +`first_value` answers what the values in a column *are* without walking the set — one peek is enough to learn a kind. On a label column it decides both questions a consumer has before it can encode anything: + +```python +from recordstream import first_value, is_class_id + +target = first_value(train_source, "class") +multilabel = isinstance(target, (list, tuple, set)) # a MultiLabel arrives as its .values LIST, + # so a sequence IS multi-label — the item + # type, never a guess about what a list means +probe = next(iter(target), None) if multilabel else target +needs_a_label_map = probe is not None and not is_class_id(probe) +``` + +It stops at the first hit, so it is one record on a normal source; a missing or all-`None` column costs one full pass and answers `None` rather than raising. + +## `LabelMap` — fittable name↔id encoding + +When a dataset's label is a class **name** rather than an integer id, `LabelMap` turns it into the pinned encoding the `EncodeTarget` / `DecodeTarget` ops need — the *fittable* companion to those ops. Fit it once (deterministic `sorted(set(...))` ordering), persist it in the `class_names.json` format, and reload it at eval/predict so every stage shares one ordering: + +```python +from recordstream import LabelMap, Stream, iter_key + +lm = LabelMap.fit(iter_key(train_source, "class")) # {"bird": 0, "cat": 1, "dog": 2} +lm.num_classes # 3 +lm.class_names # ["bird", "cat", "dog"] (id -> name) +lm.save("class_names.json") # {"class_names": [...], "num_classes": N} + +encoded = lm.encode(train_source) # a Stream whose "class" Labels carry int ids +# (the long form, when you need to pin the field or tolerate unknowns: +# Stream(source=train_source, ops=[lm.encode_op(ignore_unknown=True)])) + +# Later, at eval time — reload the SAME ordering instead of refitting: +lm2 = LabelMap.load("class_names.json") +``` + +`LabelMap.fit` is the *only* place a mapping is derived from data; everywhere downstream the mapping is pinned, so train / eval / predict never disagree. `fit` is pure stdlib — no ML library is pulled in to sort a set of names. + +### Multi-label targets and `to_ids` + +`fit` accepts a `MultiLabel` (or any sequence) just as readily as a single `Label`, so a +multi-label dataset builds its vocabulary from the same call — every distinct member becomes +one class: + +```python +from recordstream import LabelMap, MultiLabel + +lm = LabelMap.fit([MultiLabel(["cat", "dog"]), MultiLabel(["bird"])]) +lm.class_names # ["bird", "cat", "dog"] +``` + +`to_ids(target)` is the one accessor a consumer needs — it always returns a **list of int class +ids**, whatever it is handed, so there is no name-vs-id and no single-vs-multi branch at the call +site: + +```python +lm.to_ids(Label("cat")) # [1] a name -> its id +lm.to_ids(Label(1)) # [1] an ALREADY-encoded id passes through +lm.to_ids(MultiLabel(["cat", "dog"])) # [1, 2] +lm.to_ids("dog") # [2] a bare value works too +``` + +Because encoded ids pass through untouched, `LabelMap().to_ids(...)` (an *empty* map) is a valid +way to normalize an already-encoded dataset to id lists — useful for counting classes or +class-frequency statistics without fitting anything. + +## Class-balance weights (`class_counts` / `inverse_frequency_weights`) + +Training on a skewed label distribution biases a model toward the majority class. The remedy is +per-class weights, and *deriving* them is a statistic over the labels — so it lives here, beside +the `LabelMap` that normalizes the targets: + +```python +from recordstream import LabelMap, inverse_frequency_weights, iter_key + +targets = list(iter_key(train_source, "class")) # walk ONCE — see the note below +weights = inverse_frequency_weights(targets, num_classes=3, label_map=lm) +# array([0.667, 2.0, 1.0], dtype=float32) rare classes weigh more +``` + +`w[c] = total / (num_classes * count[c])`, so a class at exactly the mean frequency gets `1.0`. A +class observed **zero** times gets `0.0` (never infinity), an id outside `[0, num_classes)` is +ignored rather than raising, and the whole call returns `None` when nothing was counted — so "no +weights" stays distinguishable from "all-zero weights". `class_counts(...)` exposes the raw +histogram if you want it. + +Three things the signature is deliberate about: + +- **It takes already-walked targets, not a source.** A trainer typically walks the target stream + once and reuses that single pass to fit the `LabelMap`, derive `num_classes`, *and* weigh the + classes. A convenience that walked internally would quietly double the passes over the dataset. +- **Every target shape works**, because `LabelMap.to_ids` normalizes it: a `Label`, a `MultiLabel` + (counting for every class it names), a bare id with no map at all, a name with a fitted map. +- **It returns numpy**, like everything in `recordstream.batch` except `batch_tensor`. What a + framework does with the vector is its own convention — `torch.nn` takes a `weight` tensor in the + loss constructor, Keras takes `class_weight` on `fit()` — so that last step belongs to the + consuming trainer, not here. diff --git a/docs/record-model.md b/docs/record-model.md new file mode 100644 index 0000000..690425e --- /dev/null +++ b/docs/record-model.md @@ -0,0 +1,567 @@ +# The record model — THE recordstream data model + +A record is a **plain `dict`** of **typed values**. Import the whole surface from the PACKAGE TOP +LEVEL (`from recordstream import Record, Image, Mask, Boxes, Label, Transform, Pipeline, +as_transform, item_data, item_value, with_data, register_item, register_kernel, register_io, collate_records, ...`). +The design rationale is recorded in +[architecture.md](architecture.md#1-the-record-data-model-and-the-type-dispatched-op-engine-2026-07-25). + +## Why + +If everything that is not literally the model input or target — a segmentation mask, +region boxes, a signal's samplerate, an image's layout, a label's class names — is jammed into one +flat `metadata` dict keyed by string, it is disconnected from the value it describes. And if the +carrier is a bespoke container class, every external library needs an adapter before it can touch it. + +The record model fixes both. **A record is a plain dict, values are typed, and metadata lives on the +value it describes** — an `Image` carries its `layout`, a `Label` its `classes`. **Key names carry +meaning** (`"image"`, `"mask"`, `"bboxes"`, `"labels"`, `"class"` — the same convention as every torch +batch dict and albumentations' keyword vocabulary), so libraries that already understand dicts or +named kwargs run **as-is**, with no wrapper anywhere. A scalar side value is just another key: + +```python +record = { + "image": Image(rgb_hwc), # typed: knows its layout + "mask": Mask(seg_hw), # shares the image's frame + "bboxes": [[2, 3, 6, 7]], # albumentations vocabulary + "labels": ["drone"], + "class": Label("drone_x", classes=["noise", "drone_x"]), + "samplerate": 30.72e6, # a plain value is just another key +} +``` + +There is deliberately **no container class** — `Record` is a type alias (`Dict[str, Any]` in +`recordstream.items`), ops receive and return ordinary dicts, and `None` means "drop this record" +(filter semantics). + +## The pieces + +### Items — typed values that own their metadata + +recordstream is **modality-neutral**, so its core ships only generic items — images, masks, boxes, +labels. (Domain items — a signal, a spectrogram — live in the domain package; see below.) + +```python +from recordstream import Image, Mask, Boxes, Label, MultiLabel + +Image(rgb_hwc, layout="HWC") # an image knows its layout ("HWC" default / "CHW") +Mask(seg_hw) # a mask shares its image's frame +Boxes(boxes=[[10,10,40,40]], labels=[1], canvas=(64, 64)) # half-open pixel xyxy on a raster +Label("drone_x", classes=["noise", "drone_x"]) # ONE class for this record +MultiLabel(["drone_x", "jammer"], classes=[...]) # SEVERAL classes for this record +``` + +`Label` / `MultiLabel` hold either class **names** or already-encoded **ids** — `.is_encoded` +(built on the free function `is_class_id`) is the single rule that decides which, and +[`LabelMap.to_ids`](projection.md) is the single way to get ids out. Multi-label is its own +ITEM rather than a `Label` holding a list, because only the type distinguishes a genuine +multi-label target from an ordinary sequence value that happens to sit under the target key. + +Items are **hybrid**: array-backed items (`Image`, `Mask`) subclass `NDArrayItem` — an `np.ndarray` +subclass whose declared `_item_attrs` survive numpy operations via `__array_finalize__` — so a +type-agnostic operation touches them as an array; structured items (`Boxes`, `Label`, `MultiLabel`) +are dataclass wrappers (a bounding-box set is not an array). A uniform payload accessor hides the difference from +kernels: + +```python +from recordstream import item_data, item_value, with_data +item_data(Image(arr)) # -> the plain ndarray +with_data(Image(a, layout="CHW"), b) # a copy carrying b, layout preserved +``` + +`item_value` is the same question one step further out, and the difference is the label items: a +`Label`'s payload slot is `value`, not `data`, so `item_data(Label("cat"))` hands the `Label` back +while `item_value(Label("cat"))` gives you `"cat"` (and a `MultiLabel` its `.values` list). Use +`item_data` in a kernel, where the item type is already known; use `item_value` when you want *the +value* whatever wrapper carried it — which is what `iter_key`, `batch_values` and `ConvertToMask` all +want, and why the rule is one function rather than three copies of it. + +`register_item` / `is_item` / `item_types` / `get_item_type` / `item_type_names` are the open item +registry — the extensibility surface a domain package or user type plugs into (one class + one +decorator, no core edit). + +An op with a `field=`-style knob resolves the entry it reads with `resolve_item` (or +`resolve_entry` when it also needs the key back) — an explicit field must exist and hold the +expected item type (each miss raises a `ValueError` naming the op, the parameter and the record's +keys), while a blank field falls back to the first value of that type: + +```python +from recordstream import Image, resolve_item +image = resolve_item(record, self.image_field, Image, owner="SaveImage", param="image_field") +``` + +`fallback=False` makes a blank field an error too (for ops whose key is mandatory config); +`required=False` turns every miss into `None` (the probe form). Twelve per-class `_find_*` +copies in one consumer package predated the extraction — don't re-derive the branch. + +### Ops — type dispatch with once-per-record parameters + +A `Transform` (`recordstream.transform`) samples its parameters ONCE per record +(`get_params(record)`), then applies a per-type **kernel** to every value whose type it handles +(`@MyOp.kernel(ItemType)`, resolved MRO-aware by `recordstream.dispatch`). Values it does not handle +pass through. Because the parameters are sampled once and shared, one op moves every handled value +with the SAME decision — the torchvision-v2 model. Targeting is by TYPE; the `field=` constructor +parameter pins an op to one named key when a record holds several values of a handled type. + +```python +import numpy as np +from recordstream import Image, Record, Transform + +class Brighten(Transform): + handles = (Image,) + + def __init__(self, strength: float = 0.1, field: str | None = None) -> None: + super().__init__(field=field) + self.strength = strength + self._rng = np.random.default_rng(7) + + def get_params(self, record: Record) -> dict: + return {"offset": self._rng.uniform(0.0, self.strength)} # drawn ONCE per record + +@Brighten.kernel(Image) +def _brighten_image(value: Image, params: dict) -> Image: + return Image(np.asarray(value) + params["offset"], layout=value.layout) +``` + +The second sanctioned op shape is the **type-changing op** — read one key, write a differently-typed +item (`Threshold`: array → `Mask`, `ConvertToImage`: array → `Image`, `ConnectedComponents`: +`Mask` → `Boxes`, the target ops). It subclasses `Transform` and overrides `__call__` instead of +registering a same-type kernel, declaring `handles` / `consumes` / `produces` truthfully as graph +metadata (next section). + +### Declaring an op's type interface — `handles` / `consumes` / `optional` / `produces` + +Every `Transform` carries four class-level tuples of item types. They are the op's **type +interface**: what a reader (or a machine — a visual editor's typed sockets, a pipeline linter) +learns about the op without executing it or loading the kernel registry. + +| Attribute | Meaning | Enforced at runtime? | +|---|---|---| +| `handles` | The value types this op processes — every record value of one of these types is touched, everything else passes through. | Only by `FunctionTransform` / `as_transform` (`isinstance(value, self.handles)` is its application gate). For a kernel op, actual dispatch is the kernel registry (`dispatch(type(self), type(value))`) — `handles` must MIRROR the registered kernels. | +| `consumes` | The input types the op NEEDS to do useful work (its required inputs). Convention: an empty `consumes` means "same as `handles`". | No — declarative. | +| `optional` | Input types the op uses when present but works without (e.g. a geometric op that also moves a `Mask` if the record has one). | No — declarative. | +| `produces` | The types the op ADDS or CHANGES — its output contract (what a downstream op can rely on finding). | No — declarative. | + +Concretely, `ToTensor` declares: + +```python +class ToTensor(Transform): + handles = (NDArrayItem,) # touches array-backed values + consumes = (NDArrayItem,) # needs at least one array-bearing key to act on + produces = (torch.Tensor,) # writes a LIVE CHW float tensor under `output` (or in place) +``` + +(A `produces` entry need not be a registered item type — `ToTensor`'s output is a plain +record value, which is exactly what the declaration should say.) + +**When `handles` and `consumes` differ.** They coincide for a simple one-type op (`Threshold`, +the FFT ops), and diverge in two directions: + +- **Optional riders — `handles` ⊃ `consumes`.** A joint geometric op MAY move several types with + one draw but only REQUIRES one of them: + + ```python + class JointFlip(Transform): + handles = (Image, Mask, Boxes) # everything ONE draw may move + consumes = (Image,) # the only input it needs to be useful + optional = (Mask, Boxes) # moved together with the image when present + ``` + + A record with just an `Image` is fine; a record that also carries a `Mask`/`Boxes` gets them + moved consistently. Declaring `consumes = handles` here would wrongly tell a reader (or a + pipeline linter) that a mask is required. + +- **Read-only reference inputs — `consumes` ⊃ `handles`.** An op may NEED a value it never + changes. A denoiser that estimates the noise floor from the signal but excludes the + ground-truth ON regions when a mask is available follows the same logic with `optional` + (a real op: `handles = consumes = (Signal,)`, `optional = (Mask, GridMask)`, `produces = + (Signal,)` — the mask is read, never written). The required-reference variant looks like: + + ```python + class ScaleBoxesToImage(Transform): + handles = (Boxes,) # the only type it CHANGES + consumes = (Boxes, Image) # ...but it cannot run without the reference Image (its shape) + ``` + +In short: `handles` = "what I write", `consumes` = "what must be present", `optional` = "what I +use when present" — the three answer different questions, and only collapse into one tuple for +the simplest ops. + +**Limiting a multi-input op to NAMED keys.** The type interface says *what kinds* of values an +op works with; *which record entry* each input comes from is CONFIG. A single-input op uses the +base `field=` param (one key, still type-gated). A multi-input op declares **one `_field` +constructor param per input slot** — defaulting to the conventional key name, resolved and +validated lazily in `__call__`: + +```python +class KeepBoxesOnMask(Transform): + """Drop boxes whose center pixel is OFF in the activity mask. + + Args: + mask_field: Record key of the activity Mask to test against. Defaults to "mask". + boxes_field: Record key of the Boxes to filter. Defaults to "boxes". + output: Key the filtered Boxes are written to; blank (default) replaces boxes_field in place. + """ + + handles = (Boxes,) # the only type it CHANGES + consumes = (Mask, Boxes) # both inputs must be present + produces = (Boxes,) + + def __init__(self, mask_field: str = "mask", boxes_field: str = "boxes", output: str = "") -> None: + super().__init__() + self.mask_field = mask_field + self.boxes_field = boxes_field + self.output = output + + def __call__(self, record: Record) -> Record: + for name, want in ((self.mask_field, Mask), (self.boxes_field, Boxes)): + if name not in record: + raise ValueError(f"{type(self).__name__}: no {name!r} key in record (keys: {list(record)})") + if not isinstance(record[name], want): + raise TypeError(f"{type(self).__name__}: {name!r} is {type(record[name]).__name__}, expected {want.__name__}") + mask, boxes = record[self.mask_field], record[self.boxes_field] + keep = [b for b in boxes.boxes if mask[int((b[1] + b[3]) / 2), int((b[0] + b[2]) / 2)]] + out = Boxes(boxes=keep, labels=boxes.labels, scores=boxes.scores, canvas=boxes.canvas) + return {**record, (self.output or self.boxes_field): out} +``` + +So a record carrying several masks and several region sets is disambiguated entirely in config — +the op looks ONLY at the named entries: + +```yaml +- !class:mypkg.KeepBoxesOnMask + mask_field: activity_mask # not the segmentation mask under "mask" + boxes_field: predictions # not the ground truth under "boxes" +``` + +This is the established pattern for every shipped multi-input op (e.g. the region→target ops +take `image_field="image"` + `boxes_field="regions"` + `output="target"`). Two rules keep it +predictable: the defaults are the CONVENTIONAL key names (so the common record shape needs zero +config), and a wrong/missing key fails lazily in `__call__` with the key list in the message — +never silently falls back to a different entry when an explicit name was given. + +Rules of use: + +- **Declare truthfully or not at all.** Nothing validates these tuples against the op's behavior, + so wrong metadata is worse than missing metadata — it misleads both readers and any tool that + consumes it. A kernel op's `handles` changes when its kernel registrations change; keep them in + sync (an externally-registered kernel widens the REAL dispatch without widening `handles` — that + is fine, `handles` documents the op author's contract, the registry documents the deployment). +- **Kernel ops rarely need more than `handles`** — dispatch and pass-through already follow from + the registry; `consumes`/`produces` earn their keep on type-CHANGING ops, where the `__call__` + override hides the type flow that kernels would have made explicit. +- **These tuples never gate execution** (except the `FunctionTransform` case above). If an op must + refuse to run without an input, validate lazily in `__call__` with a clear error — the same + lazy-validation convention every op follows. + +**recordstream ships no native augmentation ops** — geometric/photometric augmentation comes from +torchvision `transforms.v2` / albumentations run as-is (next section); native ops exist only where +no library covers them. + +### Mixing libraries — as-is, no adapters + +The engine's single op-application chokepoint, `recordstream.core.families._apply_op(record, op)`, dispatches +on the op's FAMILY (by MRO module name, no eager import) and invokes each family the way its own +library expects: + +- **albumentations** — the op receives exactly its own kwarg vocabulary: the + `image`/`mask`/`masks`/`bboxes`/`keypoints`/`labels` keys present in the record, nothing else. One + call = one joint draw across them; array outputs are re-wrapped in the incoming value's item type, + so an `Image`/`Mask` keeps its type and metadata through the library. +- **torchvision `transforms.v2`** — called on the record dict as-is (tv2 walks dicts natively). + Layout conversions are the library's own transforms (`v2.ToImage()`) — the engine never converts + silently. +- **everything else** — `op(record)`; `None` drops the record. + +So bare library transforms sit in one list with native ops — in `Stream(ops=[...])`, in a `Pipeline`, +in a `flow:` step: + +```python +import albumentations as A +from recordstream import Pipeline + +Pipeline([ + A.Compose( # box-carrying augmentation: the library's own Compose + [A.HorizontalFlip(p=1.0)], + bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"]), + ), + A.GaussNoise(p=1.0), # image only — its own kwarg vocabulary + Brighten(strength=0.2), # native type-dispatched op +])(record) +# image + mask + bboxes flipped together (one joint draw); record["class"] untouched. +``` + +The same holds in YAML — a bare library transform is an ordinary `!class:` node in an `ops:` list +(the engine flows deferred markers at route entry): + +```yaml +ops: + - !class:albumentations.HorizontalFlip + p: 0.5 + - !class:albumentations.GaussNoise + p: 1.0 +``` + +See [augmentation.md](augmentation.md) for the full key-vocabulary / bbox / seeding recipes. +Runnable end-to-end: [`examples/record_pipeline.py`](../examples/record_pipeline.py). + +### `Pipeline` — the sequential composer + +`Pipeline(transforms=[...])` (`recordstream.transform`, `@configurable(category="op", +group="compose")`) wraps an ordered op list so it appears as one named block in a config and one +node on a visual canvas: zero-arg/lazy (config-deferred markers flow on first call), entries applied +through `_apply_op` (so bare library transforms nest exactly as in a bare ops list), `None` +propagation (a filter-drop stops the chain), and `close()` propagation to inner ops that own +resources. + +## Extending it + +### A custom op from a plain function + +```python +from recordstream import as_transform, Image +brighten = as_transform(lambda d: d + 0.1, handles=(Image,), field="image") +``` + +### A custom item type + a kernel for an existing op — no core edit + +```python +from dataclasses import dataclass, field +from recordstream import register_item +from mypkg.transforms import MyGeoTransform # any Transform subclass + +@register_item +@dataclass +class Keypoints: + data: list = field(default_factory=list) # a `data` field = the payload slot + +@MyGeoTransform.kernel(Keypoints) +def _(value, params): + return move_points(value, params) +``` + +Dispatch is MRO-aware: a kernel registered for a base item type also serves its subclasses, and a +subclass transform inherits its base's kernels until it overrides them. + +### Domain items live in the domain package + +The same mechanism, applied across packages: because recordstream is modality-neutral, a signal-domain +package defines its own items (a signal, a spectrogram) and its own type-changing ops, registers +them with `register_item`, and they become first-class record values — dispatchable, collatable, +storable — with no core edit. + +### A new library family + +Supporting a new external transform library (kornia, DALI, an albumentations fork, a +signal-processing library) is NOT an adapter class — it is one **registered op family**: a matcher +that recognises the library's op objects plus an invoker that applies one op with the library's own +calling convention. Every engine route (sequential, spawn-parallel, streamed, random-access) and +every composing op picks it up at once, because they all funnel through `_apply_op`: + +```python +from recordstream import register_op_family + +def is_kornia(op) -> bool: + # Keep the matcher IMPORT-FREE: inspect MRO module names, never import the library. + return any(c.__module__.startswith("kornia.augmentation") for c in type(op).__mro__) + +def invoke_kornia(record, op): + # kornia augmentations are nn.Modules over batched BCHW tensors — one draw per call. + img = record["image"] # a CHW torch.Tensor (e.g. after ToTensor) + out = op(img.unsqueeze(0)).squeeze(0) + return {**record, "image": out} + +register_op_family("kornia", is_kornia, invoke_kornia) + +# From here on, bare kornia ops sit in ANY ops list — Stream, Pipeline, RandomApply, flow steps: +stream = Stream(source=records, ops=[ToTensor(field="image"), K.RandomHorizontalFlip(p=1.0)]) +``` + +The rules: dispatch checks families **last-registered first**, so a more specific family (say a +fork extending albumentations) registers after the built-ins and wins the overlap; re-registering a +name replaces that family in place; matcher and invoker must be **module-level functions** — the +spawn-parallel routes pickle them by reference to rebuild the registry inside worker processes +(defining them in a script's `__main__` or a REPL breaks `.parallel()`; a module import side effect +is the sanctioned place, exactly like `register_item`/`register_kernel`). The built-in +`albumentations` / `torchvision_v2` families register through this same API at import — there is no +privileged code path. When a library's convention needs per-op configuration instead (which key to +read, per-op state), write a normal `Transform` op that wraps it explicitly — the registry is for +AS-IS drop-in. + +## Engines — Stream and FlowGraph carry the record + +Every carrier is a plain dict, and every route applies ops through `_apply_op` — sequential, +spawn-parallel, streamed, and random-access (`__getitem__`) alike, in `Stream` and in `FlowGraph`. +Composing ops (`Pipeline`, `RandomApply`, `Enable`, `Parallel`, `ConfigureOp`, the context ops +`Apply`/`Capture`) route their inner ops through the same chokepoint, so a bare library transform +nests anywhere a native op does. + +```python +Stream(source=my_source, ops=[A.GaussNoise(p=1.0), Brighten()]).to_sink(HDF5Sink(path="out.h5")) +``` + +`Stream.map(func, key=None)` lifts a plain function over one record entry (`key=None` hands it the +whole dict — internally a `WrappedOp`, which stores the callable as its importable path so it +pickles across `spawn` workers); `Stream.project(keys)` yields partial records restricted to the +requested keys (see [projection.md](projection.md)). + +### Graph fan-in (`merge_from`) and entry binds (`step[key]`) + +In a `flow:` document, the fan-in is **`merge_from`** — the UNION of the named steps' record +entries, in slot order, last-write-wins on a key collision. The idiom for a derived-entry branch: +produce, `SelectFields` the new key(s), merge: + +```yaml +flow: + start: {} + masked: {op: !class:recordstream.ops.numpy.Threshold(low_level=0.5), from: start} + mask_only: {op: !class:recordstream.ops.structure.SelectFields(keys: [mask]), from: masked} + boosted: {op: !class:mypkg.Boost(), from: start} + out: {from: boosted, merge_from: [mask_only]} +``` + +`bind:` references have three shapes: a bare `step` binds the step's WHOLE result record, +`step[key]` binds the named ENTRY of that step's record, and `step.attr` binds the step op's live +`@output`. All three are read by the engine directly from the step grammar — there is no lowering +to a flat op list (the pass that did that, and the six context ops it emitted, were deleted +2026-07-30). See [graph.md](graph.md). + +## Storage — the record key-group layout + +All four backends (`HDF5Sink`↔`HDF5Source`, `ZarrGroupSink`↔`ZarrGroupSource`, +`ZarrBatchSink`↔`ZarrBatchSource`, `DirectorySink`↔`DirectorySource`) write a record in ONE logical +schema: per record, one group per KEY carrying the value's registered type name (`__item_type__`), +the payload as a `data` dataset, and its attrs (scalars natively — queryable; arrays as sub-datasets +under `attrs/`; structured values JSON-tagged so tuples survive). A plain (non-item) value rides the +`"plain"` type tag — an array payload as `data`, a scalar under the `value` attr. Key order is +preserved in `__field_order__`; the store is stamped `recordstream_format = "typedrecord-v1"`. + +Backends never inspect item internals — everything serializes through the item codec +(`recordstream/io.py`: `encode_item` / `decode_item` / `encode_record` / `decode_record`), so an +externally-registered item type round-trips with zero storage edits; +`register_io(MyItem, encode=..., decode=...)` overrides the default structural codec when needed. +Decoding requires the item type to be registered (imported) in the reading process — the same +contract as Confluid's `!class:`. + +```python +sink = HDF5Sink(path="out.h5", overwrite=True) +with sink: + for record in stream: + sink.write(record) +back = list(HDF5Source(path="out.h5")) # exact records: keys, types, order, tuple attrs +``` + +`ZarrBatchSink` (the uniform single-array sink) appends the FIRST record entry's payload per row and +stores a one-time item template — per-record attr variation needs `ZarrGroupSink`. + +**No backward compatibility:** a store stamped with the pre-record `typedsample-v1` tag (or carrying +no tag) raises a `ValueError` telling you to re-generate it with a current sink +(`storage/base.py::require_record_format`) — there is no legacy read path. + +### Querying record stores without loading arrays + +The metadata scans yield the nested `{key: {attr: value}}` shape, and a `where` expression +addresses it as `.` (a plain scalar entry appears under its `value` attr): + +```python +fast = MetadataFilterSource(source=HDF5Source(path="out.h5"), where="signal.samplerate > 1e6") +``` + +Array-valued attrs appear as shape/dtype stubs (presence/shape testable, never loaded). A key named +like a Python keyword (e.g. `class`) can't be addressed in an expression — use the programmatic +`predicate` or a non-keyword key name. Live records expose the same nested shape via +`recordstream.storage.query.record_metadata(record)`. See [storage.md](storage.md). + +## Batching — `collate_records` and the collate registry + +A torch `DataLoader` (or `Stream.batch`) hands a collate function a LIST of N records and expects +ONE object back. `collate_records` — the registry's `"record"` default — folds per key with three +rules (all records must share the same key set; a mismatch raises): + +1. **array-backed item** → payloads stacked into one array/tensor with a leading batch dim, SAME + item type back; each declared attr becomes a per-record list; +2. **wrapper item** (`Label`, `Boxes`) → ONE item whose fields are per-record LISTS — deliberately + not auto-tensorized (turning class names into an `[N]` int64 tensor is the model boundary's one + explicit step, not a generic-engine guess); +3. **plain value** → a plain list. + +```python +records = [{"image": Image(...2×2×3...), "class": Label(i % 2, classes=["noise", "drone"]), + "snr_db": 10.0 * i} for i in range(3)] +batch = collate_records(records) +# image: Image (3, 2, 2, 3) layout: ['HWC', 'HWC', 'HWC'] +# class: Label value=[0, 1, 0] classes: [['noise', 'drone'], ×3] +# snr_db: [0.0, 10.0, 20.0] +``` + +### Reading the batch back — `batch_values` / `batch_tensor` / `batch_metadata` + +Every model boundary has to undo those three rules, so `recordstream.batch` ships the inverse +next to the collate that wrote it: + +```python +from recordstream import batch_values, batch_tensor, batch_metadata, multi_hot + +batch_values(batch, "class") # [0, 1, 0] — past the wrapper item +batch_tensor(batch, "image", device=dev) # [N, 3, H, W] torch tensor +batch_tensor(batch, "class", dev, dtype=torch.int64) # [N] class ids +multi_hot(batch, "class", num_classes=3) # [N, 3] numpy multi-hot +batch_metadata(batch, exclude=("image", "class")) # [{"snr_db": 0.0}, ...] +``` + +`batch_values` is the one that knows how to get *past* an item — a `Label` yields its `.value`, +a `MultiLabel` its `.values`, an array item its stacked payload, a plain value its list. +`multi_hot` renders a `MultiLabel` column as an `[N, C]` matrix. `batch_metadata` transposes +the remaining columns back into N dicts so a predictions sink can pair a model's output with +the record it came from. + +**Only `batch_tensor` is torch.** The others return plain values or numpy, so a non-torch +backend uses the same code and converts in one line (`torch.as_tensor(m)`, which shares memory, +or the TensorFlow/JAX equivalent). A torch-typed `multi_hot` would have forced a second +implementation for the next backend. + +`dtype` is a **parameter, not an opinion** — the same knob as `device`. recordstream never +decides the contract; the caller names the one its loss requires. That matters: a dataset +yielding int32 label tensors is legal, and `CrossEntropyLoss` refuses it with *"expected scalar +type Long but found Int"*, so a classifier passes `dtype=torch.int64` and a segmenter does the +same for its pixel-class mask. What stays task-side is only *which* call to make. + +### When the generic rules cannot work: register a task collate + +Stacking is task-shaped, and detection is the canonical failure: each record carries a DIFFERENT +number of boxes, and rule 2 can only give you `Boxes(boxes=[<1 box>, <3 boxes>])` — per-record +lists no detection model accepts. A detection model family has its own batch contract (stacked +images + RAGGED per-record target dicts), so the task package registers a collate that produces +exactly that: + +```python +from recordstream import Image, Boxes, collate, register_collate + +@register_collate("detection") +def detection_collate(items): + """The torchvision detection contract: stacked images + ragged per-record targets.""" + images = torch.stack([torch.as_tensor(np.asarray(r["image"])).permute(2, 0, 1) for r in items]) + targets = [ + {"boxes": torch.as_tensor(r["target"].boxes, dtype=torch.float32).reshape(-1, 4), + "labels": torch.as_tensor(r["target"].labels, dtype=torch.int64)} + for r in items + ] + metadata = [{k: v for k, v in r.items() if k not in ("image", "target")} for r in items] + return {"images": images, "targets": targets, "metadata": metadata} + +batch = collate(records, key="detection") # or: DataLoader(..., collate_fn=get_collate("detection")) +# images: [2, 3, 4, 4] — uniform, so stacked +# targets[0]: {'boxes': [1, 4], 'labels': [1]} +# targets[1]: {'boxes': [3, 4], 'labels': [3]} — raggedness PRESERVED, per record +``` + +The registration is what "solves" detection: the registry lets the task OPT OUT of the generic +folding entirely and emit its model family's native batch shape — while the engine keeps owning +only the GROUPING (yielding lists of records) and never grows task knowledge. Registration is +additive (an import side effect of the task package); a config wires the collate by reference +(`collate_fn: !ref:mypkg.detection_collate`) like any other slot. See [kinds.md](kinds.md). + +## What is NOT here yet (follow-ups) + +A torch-`Tensor`-subclass item base (torch payloads currently ride in wrapper items or as plain +values) and confluid-native item-type discovery. See the root `TASKS.md`. diff --git a/docs/runnable.md b/docs/runnable.md new file mode 100644 index 0000000..3d5cc40 --- /dev/null +++ b/docs/runnable.md @@ -0,0 +1,154 @@ +# Runnables and entry points + +A **runnable** is any object exposing a no-arg `run()` — a trainer, an evaluator, a dataset +processor, a workflow. It is the unit `recordstream run` executes: + +```yaml +# config.yaml — the ONE runner shape for every kind of run +runnable: !class:mypkg.Classifier + task: fit # ← the one knob: fit / evaluate / test / predict + train_set: !ref:train_split # a DatasetSplit with `split: train` +``` + +```bash +python -m recordstream.cli run config.yaml # builds `runnable:`, calls .run() +``` + +### Top-level keys reach the runnable + +The runnable is built **against the whole document**, so a *flat* config works: a top-level +key broadcasts into the same-named constructor parameter, with no nesting and no `!ref:`. + +```yaml +runnable: !class:mypkg.Classifier + model: !lazy:mypkg.Backbone { name: resnet18 } + +train_set: !class:recordstream.sources.huggingface.HuggingFaceSource { path: mnist, split: train } +max_epochs: 3 # -> Classifier(max_epochs=3) +batch_size: 32 # -> Classifier(batch_size=32) +``` + +If you write your own runner CLI, build the bound node with +`recordstream.cli.materialize_runnable(node)` rather than a bare `flow()`. A bare flow builds +the node in isolation, so every top-level key above is dropped — and dropped *silently*: +`train_set` becomes `None` and `max_epochs` quietly falls back to its default, leaving a run +that looks configured and is not. + +```python +from recordstream.cli import materialize_runnable + +@app.script_command(flow_mode="manual") # "auto" is the bare flow this replaces +def run(runnable: Any) -> None: + materialize_runnable(runnable).run() +``` + +## The problem entry points solve + +A merged train+eval class exposes SEVERAL capabilities from one class, dispatched off its +`task` knob. Without extra information, a discovery consumer (a config generator, a visual +editor) would have to *assume* one class per capability — it cannot know that `Classifier` +both trains and evaluates, nor which `task` value means "evaluate". The `@entrypoint` marker +declares exactly that, per method. + +## A straightforward example + +```python +from recordstream import ProgressReporting, TorchRunner, entrypoint, run_entrypoint + +class Classifier(TorchRunner, ProgressReporting): + """One class, four capabilities — run() dispatches off the ``task`` knob.""" + + def __init__(self, task: str = "fit"): + self.task = task + + def run(self) -> None: + run_entrypoint(self, self.task) # the markers below ARE the dispatch table + + @entrypoint("fit", role="trainer", primary=True) + def fit(self) -> None: ... # gradient training + + @entrypoint("evaluate", role="evaluator") + def evaluate(self) -> None: ... # metrics over the VALIDATION split + + @entrypoint("test", role="evaluator", primary=True) + def test(self) -> None: ... # metrics over the held-out TEST split + + @entrypoint("predict", role="predictor", primary=True) + def predict(self) -> None: ... # stream predictions +``` + +Each marker states three things: the **`task` value** that reaches this method through +`run()`, the **`role`** capability label (conventionally `"trainer"` / `"evaluator"` / +`"predictor"`; free-form for new capabilities), and — when several methods share a role — +which one is the **`primary`** (here `test` is the default evaluator; `evaluate` is the +secondary, validation-split variant). + +## What the introspectors return + +Real output for the class above (these are executed facts, not sketches): + +```python +>>> from recordstream import runnable_entrypoints, entrypoint_tasks +>>> runnable_entrypoints(Classifier) +{'fit': {'task': 'fit', 'role': 'trainer', 'primary': True}, + 'evaluate': {'task': 'evaluate', 'role': 'evaluator', 'primary': False}, + 'test': {'task': 'test', 'role': 'evaluator', 'primary': True}, + 'predict': {'task': 'predict', 'role': 'predictor', 'primary': True}} + +>>> entrypoint_tasks(Classifier, "trainer") +['fit'] +>>> entrypoint_tasks(Classifier, "evaluator") +['test', 'evaluate'] # PRIMARY FIRST — "run this as an evaluator" means task: test +>>> entrypoint_tasks(Classifier, "exporter") +[] # unknown role: empty, never an error +``` + +`runnable_entrypoints` walks the MRO (an inherited entry point is found; a subclass override +wins) and reads the marker off the raw function object, so property getters never fire. + +## Dispatching: `run_entrypoint` + +`run()` above dispatches *through* the markers rather than restating them: + +```python +>>> Classifier(task="test").run() # calls Classifier.test() +>>> Classifier(task="export").run() +ValueError: Unknown task 'export'; expected one of ['fit', 'evaluate', 'test', 'predict']. +``` + +The declared tasks are listed in **declaration order**, so the error reads as the class's +capability list. The return value of the entry-point method is passed through. + +Write the `run()` body this way rather than as a hand-written `{task: method}` dict. The dict +states the same mapping a second time, and the copies drift in one direction that bites: a +config generator pins `task:` from `entrypoint_tasks` — the markers — so a capability added to +the markers and forgotten in the dict produces a *generated* config that dies at dispatch with +"unknown task" while discovery advertises it as supported. With `run_entrypoint` there is one +table, and adding a fifth `@entrypoint` method is all that adding a fifth capability takes. + +## How a consumer uses this + +A config generator asked for "an evaluator config for `Classifier`" calls +`entrypoint_tasks(Classifier, "evaluator")[0]` → `"test"` and pins `task: test` in the YAML +it emits — one `recordstream run` then dispatches correctly with no human editing. The same +walk over every discovered class tells a visual editor which classes to offer in a +"trainer" picker versus an "evaluator" picker, even when both answers are the same class. + +## The two marker mixins + +Orthogonal to entry points, a runnable may inherit two stateless mixins: + +- **`TorchRunner`** — declares "my `run()` needs autograd" (`__needs_autograd__ = True`, + duck-typed). A GUI executor that evaluates nodes under `torch.inference_mode()` re-enables + autograd for the duration of `run()`. Inference-only runnables deliberately do NOT inherit + it. A merged class can even make it dynamic — a property returning `self.task == "fit"`, + so the same class trains under autograd and predicts under inference mode. The class and + the flag are named for different things on purpose: the *class* for the framework whose + execution mode is at stake (autograd is a torch concept), the *flag* for what it decides — + which is what makes the dynamic property above read correctly. +- **`ProgressReporting`** — a framework-free progress sink: the executor injects + `(value, total, desc) -> None` via `set_progress_callback()`, the runnable drains it via + `self._report_progress(step, total, "epoch 3")` from its loop. With no sink injected + (a plain CLI run) every call is a silent no-op. + +Pins: `tests/test_runnable.py` / `tests/test_entrypoint.py`. diff --git a/docs/sources.md b/docs/sources.md new file mode 100644 index 0000000..b87f6f6 --- /dev/null +++ b/docs/sources.md @@ -0,0 +1,195 @@ +# Sources — HuggingFace, splits, ranges, concatenation (`recordstream.sources`) + +`recordstream.sources` is a package with **one class per module**. Import from the package — +`from recordstream.sources import HuggingFaceSource, DatasetSplit, RangeSource, ConcatSource` — +but spell the **submodule** path in a config, because that is what `cls.__module__` says and +what a generated config emits: + +| class | module | `!class:` path | +| --- | --- | --- | +| `HuggingFaceSource` | `huggingface.py` | `recordstream.sources.huggingface.HuggingFaceSource` | +| `DatasetSplit` | `split.py` | `recordstream.sources.split.DatasetSplit` | +| `RangeSource` | `range.py` | `recordstream.sources.range.RangeSource` | +| `ConcatSource` | `concat.py` | `recordstream.sources.concat.ConcatSource` | + +The shorter `!class:recordstream.sources.HuggingFaceSource` still resolves (Confluid falls back to +a module-path import, and the package re-exports every name), so an older config keeps loading — +but a generated one will use the submodule spelling. Rationale: +[docs/architecture.md §11](architecture.md#11-sources-are-a-package-one-class-per-module--and-the-module-path-is-the-contract-2026-08-01). + +## Hugging Face datasets + +`HuggingFaceSource` turns any `datasets.Dataset` (a Hub repo id or a local imagefolder path) into plain record dicts of typed values: the `input_feature` column becomes an `Image` under the record key `"image"`, the `target_feature` column a `Label` under `"class"`, and each kept metadata column its own `Label` entry keyed by the column name (plus the source-provenance `hf_path` / `hf_split` entries) — traceability that often goes missing in bare dictionary loading. + +- **`metadata_features` (which extra columns become record entries):** the sentinel **`"*"`** (or `["*"]`, the default) keeps **every column except `input_feature` / `target_feature`** — the full-traceability option, resolved against the dataset's real columns at load; an explicit list keeps exactly those columns; `None` / `[]` keep none. + +```yaml +hf_train: !class:recordstream.sources.huggingface.HuggingFaceSource() + path: ylecun/mnist + input_feature: image + target_feature: label + metadata_features: ["*"] # keep every other column as its own record entry (the default) +``` + +> **Use the namespaced repo id** (`ylecun/mnist`, never the legacy bare `mnist`): current `huggingface_hub` rejects namespace-less ids (`HfUriError: Repository id must be 'namespace/name'`, measured 2026-08-06). Worse than the hard failure is the soft one — with a stale local cache present, `datasets` logs "couldn't be found on the Hugging Face Hub" and silently loads the cached copy, so a bare id can appear to work on one machine and fail on a fresh one. + +> **Lazy & zero-arg construction** — `HuggingFaceSource` follows the workspace lazy-init convention: the constructor does no work (no network), so `HuggingFaceSource()` is valid and building one is free. The dataset is downloaded only on first access to the read-only `.dataset` property (cached thereafter; reset `_dataset` to reload), and `.resolved_metadata_features` (the `"*"` expansion) is derived lazily from the loaded columns. `path` is therefore optional at construction and validated lazily — accessing `.dataset` with an empty `path` raises a clear `ValueError`. + +## Train / val / test splitting (`DatasetSplit`) + +`DatasetSplit` partitions any indexable source (implementing `__len__` and `__getitem__`) into reproducible **train / val / test** views. It is a `source` (`category="source"`) — it yields records and is wired into a trainer's `source:` slot — and it applies no ops, so it's a source, not an engine. + +**Property API (preferred).** Configure **one** `DatasetSplit` with a `seed` and the held-out fraction(s), then read the three cached views off it — `split.train` / `split.val` / `split.test`: + +```python +from recordstream import DatasetSplit +split = DatasetSplit(source=src, val_fraction=0.1, test_fraction=0.1, seed=42) +split.train # ≈80% — the remainder split.val # ≈10% split.test # ≈10% +``` + +The views are disjoint and complementary, computed once over a single deterministic shuffle (cached). In a config each view is a `DatasetSplit` with its `split` selector set — write the recipe once (a YAML anchor on the first view) and merge it (`<<:`) into the others. Every view references the *same* `hf_train` (`!ref:` shares the instance), so the upstream source is loaded **exactly once**; a `DatasetSplit`'s own partition is one seeded shuffle over `len(source)`: + +```yaml +hf_train: !class:recordstream.sources.huggingface.HuggingFaceSource() + path: ylecun/mnist + split: train + +train_set: !class:recordstream.core.stream.Stream() + source: &split_recipe !class:recordstream.sources.split.DatasetSplit() + source: !ref:hf_train + val_fraction: 0.1 + test_fraction: 0.1 + seed: 42 + split: train +val_set: !class:recordstream.core.stream.Stream() + source: !class:recordstream.sources.split.DatasetSplit() + <<: *split_recipe + split: val +test_set: !class:recordstream.core.stream.Stream() + source: !class:recordstream.sources.split.DatasetSplit() + <<: *split_recipe + split: test +``` + +(Reading a view by attribute reference — `!ref:my_split.train` — is no longer a config spelling; the config engine refuses it and names this rewrite.) + +Omit `test_fraction` for a plain two-way train/val split; omit both fractions and `train` is the whole source (`val`/`test` empty). + +**Select-one API.** Passing `split` makes the `DatasetSplit` *itself* iterate that one view (`split=None` ⇒ `train`), so it's directly usable as a single `source:`. `split` is the closed `Literal["train", "val", "test"]`, exported as `recordstream.SplitName`. + +```yaml +val_set: !class:recordstream.sources.split.DatasetSplit() + source: !ref:hf_train + split: val + val_fraction: 0.1 + seed: 42 +``` + +## Range & concatenation sources + +- **`RangeSource(source, start, stop)`** — a contiguous index slice `[start:stop)` over a source (negatives count from the end; clamped). The plain-slice counterpart to `DatasetSplit`. + + ```yaml + first_half: !class:recordstream.sources.range.RangeSource() + source: !ref:hf_train + start: 0 + stop: 5000 + ``` + +- **`ConcatSource(sources)`** — joins multiple indexable sources into one longer indexable source (the indexable counterpart to `JointStream`, which is iteration-only). Because it's indexable, a `ConcatSource` can itself be wrapped by `DatasetSplit` / `RangeSource`. + + ```yaml + combined: !class:recordstream.sources.concat.ConcatSource() + sources: + - !ref:train_main + - !ref:extra_shard + ``` + +**HuggingFace native slicing** (alternative, no RecordStream split needed): `split: "train[:90%]"` / `"train[90%:]"` on two `HuggingFaceSource`s. + +> **Never mark a nested source `_partial_: true`.** A partial is a *deferred marker*, not an +> instance — a `source:` slot holding one fails at first use with an error naming the slot and +> the fix (the same guidance `Stream` gives), because nothing will flow it for you. A plain +> `_target_:` source is built at load time and is what the slot wants. The examples above use +> `${ref:…}` to a top-level source instead, which is also what lets several wrappers share one +> loaded source. + +## Identifying a dataset + +A source can name the data it reads, so a run record, a report, or a log line can point at it. +Two handles, because they answer different questions: + +| property | what it is | when it is `None` | +| --- | --- | --- | +| `dataset_uri` | the **canonical** identifier — machine-parseable and stable, the string two runs are compared on | the source has no dataset configured | +| `dataset_url` | a link a **person** can open | the data has no web page (anything local) | + +```python +from recordstream import HuggingFaceSource, dataset_uri, dataset_url + +source = HuggingFaceSource(path="ylecun/mnist", split="train") +source.dataset_uri # 'hf://datasets/ylecun/mnist?split=train' +source.dataset_url # 'https://huggingface.co/datasets/ylecun/mnist/viewer/default/train' +``` + +What `HuggingFaceSource` produces, for each way it can be configured: + +| configuration | `dataset_uri` | `dataset_url` | +| --- | --- | --- | +| `path="ylecun/mnist", split="train"` | `hf://datasets/ylecun/mnist?split=train` | `…/ylecun/mnist/viewer/default/train` | +| `+ name="fashion", revision="abc123"` | `hf://datasets/ylecun/mnist?name=fashion&revision=abc123&split=train` | `…/ylecun/mnist/viewer/fashion/train` | +| `path="/data/imagefolder"` | `file:///data/imagefolder?split=train` | `None` | +| `path=""` | `None` | `None` | + +Both read **stored configuration only** — asking never loads, downloads or opens anything, so a +source that is never iterated still names itself. Whether a `path` is a Hub repo id or a local +directory is decided by whether it exists on disk, the same question `load_dataset` answers. The +query parameters are sorted, so one configuration has exactly one URI. + +**Use the free functions rather than the attributes when the source may be wrapped or deferred.** +`dataset_uri(x)` / `dataset_url(x)` materialize a `!class:` marker straight out of a config, and +follow a view's `.source` to the dataset underneath: + +```python +from recordstream import RangeSource, Stream, dataset_uri + +# a view reports the same DATASET — the slice it takes is its own configuration, not identity +dataset_uri(Stream(source=RangeSource(source=source, stop=100))) +# 'hf://datasets/ylecun/mnist?split=train' +``` + +That verbatim propagation is deliberate: one dataset reached two ways must compare equal. How +much of it a run consumed is recorded by the wrapper's own settings (`start` / `stop`, the split +fractions), not by mangling the handle. + +A source holding **several** datasets declines to be one of them — `ConcatSource` answers `None`, +and `dataset_uris` is the plural form: + +```python +from recordstream import ConcatSource, dataset_uri, dataset_uris + +dataset_uri(ConcatSource(sources=[a, b])) # None — a concatenation is not one dataset +dataset_uris(ConcatSource(sources=[a, b])) # ['hf://datasets/…', 'file:///…'] +``` + +**Adding identity to your own source** is defining the two properties — `SupportsDatasetIdentity` +is a `Protocol`, so there is nothing to register: + +```python +@configurable(category="source") +class MyStoreSource: + def __init__(self, bucket: str = "", prefix: str = "") -> None: + self.bucket, self.prefix = bucket, prefix + + @property + def dataset_uri(self) -> Optional[str]: + return f"s3://{self.bucket}/{self.prefix}" if self.bucket else None + + @property + def dataset_url(self) -> Optional[str]: + return None # no web page: honest, and never an error +``` + +Rationale: [docs/architecture.md §13](architecture.md#13-dataset-identity-is-a-protocol-and-a-view-propagates-it-verbatim-recordstreamuri-2026-08-02). + +> **Note on `!ref:`** — Confluid `!ref:` resolves to the same live object as the referenced key, so a single `HuggingFaceSource` is loaded once and shared. Write the marker again when you want an independent instance instead. diff --git a/docs/storage.md b/docs/storage.md new file mode 100644 index 0000000..8401912 --- /dev/null +++ b/docs/storage.md @@ -0,0 +1,116 @@ +# Storage — sinks, sources and queryable metadata (`recordstream.storage`) + +> Runnable tour: [`examples/storage_roundtrip.py`](../examples/storage_roundtrip.py) — the same +> records through all three sink/source pairs (typed values + a plain scalar, byte-identical +> round-trips) plus a `MetadataFilterSource` query that never loads an array. + +RecordStream makes it easy to move data between different formats: + +```python +from recordstream.storage.hdf5 import HDF5Source +from recordstream.storage.zarr import ZarrGroupSink + +# Stream from HDF5 to Zarr in parallel +Stream.from_source(HDF5Source("input.h5")) \ + .parallel(workers=8) \ + .map(heavy_op, key="image") \ + .to_sink(ZarrGroupSink("output.zarr")) +``` + +## Sinks and their matching sources + +Every sink has a source that reads its layout back into record dicts of typed values: + +| Backend | Sink | Source | Round-trips | +|---|---|---|---| +| HDF5 (sequential) | `HDF5Sink` | `HDF5Source` | all keys + item types + attrs | +| Zarr group (one group / record) | `ZarrGroupSink` | `ZarrGroupSource` | all keys + item types + attrs | +| Zarr batch (one stacked array) | `ZarrBatchSink` | `ZarrBatchSource` | first record entry only (uniform shape) | +| Directory (one dir / record) | `DirectorySink` | `DirectorySource` | all keys + item types + attrs | + +```python +from recordstream.storage.zarr import ZarrGroupSink, ZarrGroupSource + +Stream(records).to_sink(ZarrGroupSink("ds.zarr", overwrite=True)) +for record in ZarrGroupSource("ds.zarr"): # exact keys, item types and attrs reconstructed + ... +``` + +All four backends share ONE logical schema — the **record key-group layout**, stamped +`recordstream_format = "typedrecord-v1"`: per record, one group per KEY carrying the value's +registered type name, the payload as a `data` dataset, and its attrs (scalars natively; structured +values JSON-tagged so tuples survive); a plain (non-item) value rides the `"plain"` type tag — an +array payload as `data`, a scalar under the `value` attr. Everything serializes through the item +codec (`recordstream/io.py`), so an externally-registered item type round-trips with zero storage +edits (see [record-model.md](record-model.md#storage--the-record-key-group-layout)). + +> **No backward compatibility.** A store whose format tag is missing or pre-record +> (`typedsample-v1`) raises a `ValueError` telling you to re-generate it with a current sink — +> there is no legacy read path. + +> Domain-specific storage formats implement the same `DataSink`/`DataSource` protocols in their own +> package — e.g. a waveform-recording format pair lives in the signal-domain package, not here. The +> engine never couples to a specific format. + +## Array-valued item attributes + +Each key is stored as its own group: the item's payload as a `data` dataset and its scalar +attributes as HDF5 **attributes**. HDF5 caps attribute size, so any **array-valued attribute** +(`np.ndarray` / `torch.Tensor`, e.g. a per-record weight map riding an item's attrs) is written as +its own sub-dataset under `attrs/` instead — a large array never overflows the attribute limit, and +`HDF5Source` restores every attribute on read. A segmentation mask is not an attribute at all: it +is a first-class `Mask` value under its own key with its own payload. + +```python +from recordstream import Image, Mask, item_data + +record = {"image": Image(data), "mask": Mask(mask_2d)} +Stream([record]).to_sink(HDF5Sink("ds.h5", overwrite=True)) +loaded = next(iter(HDF5Source("ds.h5"))) +item_data(loaded["mask"]) # the full mask array, byte-exact (not a truncated repr) +loaded["image"].layout # item attrs round-trip too +``` + +## Partial payload reads (`read_record_group`) + +`recordstream.storage.hdf5.read_record_group(group, slices=...)` is the public form of the +HDF5 row decoder `HDF5Source` iterates through. `slices` maps a record KEY to a slice applied +to that field's `data` dataset **at read time** — an h5py partial read, so only the requested +span of the payload ever leaves the file. A consumer windowing large stored rows (e.g. a +whole-capture signal archived once, re-windowed onto many grids at read time) decodes each +window without materializing the row; the decoded item is identical to a full read followed by +an in-memory slice, and attrs are never sliced: + +```python +import h5py +from recordstream.storage.hdf5 import read_record_group + +with h5py.File("ds.h5", "r") as handle: + window = read_record_group(handle["s000000"], slices={"signal": slice(1000, 2000)}) +``` + +## Queryable metadata (`recordstream.storage.query`) + +Filter stored records by metadata predicates *without loading arrays*: sources implementing the +`SupportsMetadataScan` protocol (`iter_metadata()`) scan only attrs / `.zattrs` / sidecar JSON — +`HDF5Source` and `ZarrGroupSource` both do (and external storage sources can implement the +structural protocol without importing this module), so **record-layout HDF5/Zarr files are +queryable with no extra index**: + +```python +from recordstream.storage.query import MetadataFilterSource + +view = MetadataFilterSource(source=HDF5Source(path="d.h5"), where="signal.samplerate > 1e6") +len(view) # matches counted from a metadata-only scan +stream = Stream(source=view, ops=[...]) # arrays load ONLY for matching records +``` + +The scans yield the nested `{key: {attr: value}}` shape, and a `where` expression addresses it as +`.` (a plain scalar entry appears under its `value` attr — `"snr_db.value > 10"`). +`where` uses the FormulaOp restricted namespace with metadata keys as variables (a missing key = +non-matching, a malformed expression fails loudly); a programmatic `predicate=` composes with AND; +sources without the protocol fall back to full-iteration filtering via `record_metadata(record)` — +the same nested shape derived from a live record. Array-valued attrs appear in the scan as +shape/dtype stub strings (`""`), so queries can test presence +without a single array read. A key named like a Python keyword (e.g. `class`) can't be addressed +in an expression — use the programmatic `predicate` or a non-keyword key name. diff --git a/docs/workflow.md b/docs/workflow.md new file mode 100644 index 0000000..2e34cfc --- /dev/null +++ b/docs/workflow.md @@ -0,0 +1,62 @@ +# Workflows — composing runnables + +The workflow combinators (`recordstream.workflow`) are the RUNNABLE-level analogue of the +composing ops: they HOLD other runnables and orchestrate them, so a multi-stage pipeline +(prepare → train → evaluate) is ONE Confluid document executed by the same +`recordstream run workflow.yaml` as any single runnable. + +| Combinator | Runs | +|---|---| +| `Sequence(steps=[...])` | every step in order — the workflow itself (`None` steps skip) | +| `Conditional(condition, if_true, if_false)` | one of two branches, by a condition (`None` branch = do nothing, the sequence continues) | +| `Switch(select, cases, default)` | one of several branches, keyed by the select's value (coerced to `str`) | + +Conditions are `@configurable` predicates — a no-arg `__call__() -> bool`: `PathExists(path)` +(the canonical cache check), `Not(condition)`, `AllOf(conditions)`, `AnyOf(conditions)` — or any +zero-arg callable, a plain `bool`, or a deferred marker resolving to one. + +## The compelling case: a resume-safe pipeline + +Re-run the SAME document after a crash (or just again tomorrow) and it skips the work whose +artifact already exists — *memoise and continue*: + +```yaml +runnable: !class:recordstream.workflow.Sequence + steps: + # Train ONLY when the checkpoint is missing. On a cache hit the !lazy: branch is + # not just skipped — it is never even BUILT (no model / dataset materialised). + - !class:recordstream.workflow.Conditional + condition: !class:recordstream.workflow.PathExists + path: $MODEL_ROOT/run1/model.ckpt + if_true: null # cache hit -> skip, Sequence continues + if_false: !lazy:TrainModel # @configurable classes resolve by registered NAME + ckpt: $MODEL_ROOT/run1/model.ckpt + + # Always runs; the report FORMAT is a Switch on a plain config value — one key a + # CLI override can flip (--select text) without touching the workflow shape. + - !class:recordstream.workflow.Switch + select: json + cases: + json: !lazy:Evaluate { report: $MODEL_ROOT/run1/report.json, fmt: json } + text: !lazy:Evaluate { report: $MODEL_ROOT/run1/report.txt, fmt: text } +``` + +**The guarantee:** only the selected branch's `run()` is ever called, and a branch wired +`!lazy:` is only *constructed* when selected. [`examples/workflow_pipeline.py`](../examples/workflow_pipeline.py) +runs this exact shape twice and asserts both: pass 2 re-evaluates without retraining, and the +trainer class records exactly ONE construction across both passes. + +## Semantics worth knowing + +- **Branches are lazy twice over**: unchosen `!lazy:` branches are never built; chosen ones are + flowed inside `run()` (zero-arg construction of the combinators themselves does no work). +- **`Conditional` with a `None` branch is "skip and continue"** — the enclosing `Sequence` + proceeds to the next step; nothing blocks. +- **`Switch` select** may be a plain value (`select: json` — overridable from a CLI), a + predicate, or any zero-arg callable; an unmatched key (or `None`) runs `default` (`None` = + no-op). +- **GUI cooperation is inherited**: the combinators mix in `TorchRunner` (a wrapped trainer's + `loss.backward()` survives an inference-mode executor) and `ProgressReporting` (the injected + progress callback is FORWARDED to whichever branch is running). + +Runnables and the `task`/`role` entry-point markers: [runnable.md](runnable.md). diff --git a/examples/advanced_storage_demo.py b/examples/advanced_storage_demo.py deleted file mode 100644 index 845a31f..0000000 --- a/examples/advanced_storage_demo.py +++ /dev/null @@ -1,45 +0,0 @@ -from pathlib import Path - -import numpy as np - -from dataflux.core import Flux -from dataflux.sample import Sample -from dataflux.storage.directory import DirectorySink -from dataflux.storage.zarr import ZarrBatchSink, ZarrGroupSink - - -def main() -> None: - # --- 1. DirectorySink (Irregular Lengths) --- - print("--- Testing DirectorySink ---") - dir_path = Path("examples/dir_store") - samples = [ - Sample(input=np.random.randn(5), metadata={"id": "tiny"}), - Sample(input=np.random.randn(50), metadata={"id": "large"}), - ] - - sink_dir = DirectorySink(dir_path, overwrite=True) - Flux(samples).to_sink(sink_dir) - print(f"Created {len(list(dir_path.glob('*')))} sample folders.") - - # --- 2. ZarrGroupSink (Irregular Lengths, Single Bundle) --- - print("\n--- Testing ZarrGroupSink ---") - zarr_group_path = "examples/group_store.zarr" - sink_group = ZarrGroupSink(zarr_group_path, overwrite=True) - Flux(samples).to_sink(sink_group) - print("Zarr group write complete.") - - # --- 3. ZarrBatchSink (Uniform Lengths, Optimized) --- - print("\n--- Testing ZarrBatchSink ---") - zarr_batch_path = "examples/batch_store.zarr" - # Uniform samples: 10 elements each - uniform_samples = [Sample(input=np.random.randn(10).astype(np.float32)) for _ in range(10)] - - sink_batch = ZarrBatchSink(zarr_batch_path, shape=[10], overwrite=True) - Flux(uniform_samples).to_sink(sink_batch) - print("Zarr batch append complete.") - - print("\nAll advanced storage sinks verified!") - - -if __name__ == "__main__": - main() diff --git a/examples/basic_pipeline.py b/examples/basic_pipeline.py deleted file mode 100644 index 4e1313f..0000000 --- a/examples/basic_pipeline.py +++ /dev/null @@ -1,59 +0,0 @@ -import confluid # type: ignore[import-not-found] -import numpy as np - -from dataflux.core import Flux - - -# 1. Define simple functional transformations -def add_noise(data: np.ndarray, std: float = 0.1) -> np.ndarray: - noise = np.random.normal(0, std, data.shape) - return np.asarray(data + noise) - - -def multiply(data: np.ndarray, factor: float = 2.0) -> np.ndarray: - return data * factor - - -def main() -> None: - # 2. Create raw data source - raw_data = [(np.array([1.0, 2.0]), 1), (np.array([3.0, 4.0]), 0)] - - # 3. Build the Flux pipeline - # For robust serialization, we should use strings for function references - # or ensure the classes/functions are part of the registry. - pipeline = Flux(raw_data).map(multiply, factor=10.0).map(add_noise, std=0.01) - - # 4. Serialize the Pipeline - # We set source=None before serialization to only serialize the "recipe" - # and avoid serializing raw numpy data which is not YAML-safe. - print("\n--- Serialized DataFlux Pipeline ---") - yaml_state = "" - try: - pipeline.source = None - yaml_state = confluid.dump(pipeline) - print(yaml_state) - except Exception as e: - print(f"Serialization failed: {e}") - - # 5. Reconstruct and Execute - print("\n--- Reconstructing Pipeline from YAML ---") - try: - if yaml_state: - # Explicitly pass as YAML string (containing \n ensures - # load treats it as YAML) or ensure it's handled by path vs yaml logic. - new_pipeline = confluid.load(yaml_state) - - new_pipeline.source = raw_data - for sample in new_pipeline: - print(f"Reconstructed Output: {sample.input}") - else: - print("No YAML state to reconstruct.") - except Exception as e: - print(f"Reconstruction failed: {e}") - import traceback - - traceback.print_exc() - - -if __name__ == "__main__": - main() diff --git a/examples/cache_pipeline.py b/examples/cache_pipeline.py index 884fdcc..2becf28 100644 --- a/examples/cache_pipeline.py +++ b/examples/cache_pipeline.py @@ -6,7 +6,7 @@ 3. The cache enforces an LRU budget — the oldest entry is evicted when a new one would push total usage past ``max_bytes``. -Runs end-to-end with no external data and no DataFlux pipeline. +Runs end-to-end with no external data and no RecordStream pipeline. """ import tempfile @@ -14,11 +14,11 @@ from pathlib import Path from typing import Callable -from dataflux.storage.cache import CacheBudgetExceeded, DiskCache +from recordstream.storage.cache import CacheBudgetExceeded, DiskCache def main() -> None: - with tempfile.TemporaryDirectory(prefix="dataflux-cache-demo-") as tmp: + with tempfile.TemporaryDirectory(prefix="recordstream-cache-demo-") as tmp: cache = DiskCache(Path(tmp), max_bytes=300) print(f"Cache root: {cache.root} (max_bytes={cache.max_bytes})") diff --git a/examples/dataset_split.yaml b/examples/dataset_split.yaml deleted file mode 100644 index c1ae87b..0000000 --- a/examples/dataset_split.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# DataFlux DatasetSplit Example -# -# Shows how to use DatasetSplit to carve a single HuggingFace source -# into disjoint train/val views sharing the same seed. -# -# Reload with `confluid.load(...)` and iterate the Fluxes. - -hf_train: !class:dataflux.sources.HuggingFaceSource() - path: mnist - split: train - input_feature: image - target_feature: label - -hf_test: !class:dataflux.sources.HuggingFaceSource() - path: mnist - split: test - input_feature: image - target_feature: label - -# 90/10 train/val split, deterministic via seed=42 -train_set: !class:dataflux.core.Flux() - source: !class:dataflux.sources.DatasetSplit() - source: !ref:hf_train - split: train - val_fraction: 0.1 - seed: 42 - -val_set: !class:dataflux.core.Flux() - source: !class:dataflux.sources.DatasetSplit() - source: !ref:hf_train - split: val - val_fraction: 0.1 - seed: 42 - -# Test set stays untouched until final evaluation. -test_set: !class:dataflux.core.Flux() - source: !ref:hf_test diff --git a/examples/discovery_demo.py b/examples/discovery_demo.py index 42e4169..476cf40 100644 --- a/examples/discovery_demo.py +++ b/examples/discovery_demo.py @@ -1,25 +1,23 @@ +"""Passive introspection: scan a module and get a JSON schema per callable (no manual tool defs). + +Standalone, zero-arg, exit 0. +""" + import json -from pathlib import Path -from dataflux.discovery import scan_module +from recordstream.discovery import scan_module def main() -> None: - # 1. Path to our basic pipeline script - pipeline_script = Path(__file__).parent / "basic_pipeline.py" - - # 2. Scan the module for callables - print(f"--- Scanning Module: {pipeline_script.name} ---") - schemas = scan_module(pipeline_script) + module = "recordstream.ops.numpy" + print(f"--- Scanning Module: {module} ---") + schemas = scan_module(module) - # 3. Print the results as formatted JSON - # This is exactly what FluxStudio will see print(json.dumps(schemas, indent=2)) - # 4. Verification - found_ops = [s["name"] for s in schemas] - assert "multiply" in found_ops - assert "add_noise" in found_ops + found = [s["name"] for s in schemas] + assert "Threshold" in found + assert "ConnectedComponents" in found print("\nDiscovery Engine Verified!") diff --git a/examples/hdf5_pipeline.py b/examples/hdf5_pipeline.py deleted file mode 100644 index dcd9099..0000000 --- a/examples/hdf5_pipeline.py +++ /dev/null @@ -1,51 +0,0 @@ -from pathlib import Path - -import confluid # type: ignore[import-not-found] -import numpy as np - -from dataflux.core import Flux -from dataflux.storage.hdf5 import HDF5Sink, HDF5Source - - -# 1. Define a simple transform -def rescale(data: np.ndarray, scale: float = 1.0) -> np.ndarray: - return data * scale - - -def main() -> None: - # 2. Setup paths - h5_path = Path("examples/test_data.h5") - - # 3. Create synthetic data and write to HDF5 - print(f"--- Writing synthetic data to {h5_path} ---") - raw_data = [np.random.randn(10) for _ in range(5)] - - # We use overwrite=True to ensure a fresh file - sink = HDF5Sink(h5_path, overwrite=True) - Flux(raw_data).to_sink(sink) - - # Force close if not already closed by context - sink.close() - - # 4. Serialize the Pipeline - pipeline = Flux().map(rescale, scale=100.0) - print("\n--- Serialized DataFlux Pipeline ---") - yaml_state = confluid.dump(pipeline) - print(yaml_state) - - # 5. Read back and process using reconstructed pipeline - print("\n--- Reading back through Flux + HDF5Source ---") - source = HDF5Source(h5_path) - - # Reconstruct the processing logic from YAML - new_pipeline = confluid.load(yaml_state) - new_pipeline.source = source # Assign the live source - - for i, sample in enumerate(new_pipeline): - print(f"Sample {i}: mean={sample.input.mean():.2f}") - - print("\nHDF5 end-to-end flow verified!") - - -if __name__ == "__main__": - main() diff --git a/examples/intake_pipeline.py b/examples/intake_pipeline.py deleted file mode 100644 index a90baec..0000000 --- a/examples/intake_pipeline.py +++ /dev/null @@ -1,103 +0,0 @@ -"""IntakeSource adapter demo. - -Shows how the generic ``dataflux.storage.intake.IntakeSource`` adapter consumes -any intake DataSource — using a tiny in-process driver here so the example needs -no external data or services. - -Demonstrates: -1. Wrapping an ``intake.source.base.DataSource`` that yields ``xarray.DataArray`` - partitions. -2. ``target_attr`` lifting the per-partition label from xarray attrs into - ``Sample.target``. -3. Round-tripping a Flux pipeline through ``HDF5Sink`` → ``HDF5Source`` to show - that the metadata flowing in via xarray attrs survives storage. -""" - -import tempfile -from pathlib import Path -from typing import Any - -import intake -import numpy as np -import xarray as xr - -from dataflux.core import Flux -from dataflux.storage.hdf5 import HDF5Sink, HDF5Source -from dataflux.storage.intake import IntakeSource - - -class _DemoXArraySource(intake.source.base.DataSource): - """Tiny in-process intake DataSource yielding 4 xarray.DataArray partitions.""" - - container = "xarray" - name = "_demo_xarray" - version = "0.1" - partition_access = True - - def __init__(self, n: int = 4, label: str = "demo", metadata: Any = None) -> None: - super().__init__(metadata=metadata or {}) - self._n = n - self._label = label - - def _get_schema(self) -> Any: - return intake.source.base.Schema( - datashape=None, - dtype=str(np.dtype(np.float32)), - shape=(8,), - npartitions=self._n, - extra_metadata={}, - ) - - def _get_partition(self, i: int) -> xr.DataArray: - return xr.DataArray( - np.arange(8, dtype=np.float32) + (i * 100), - dims=["x"], - attrs={"label": self._label, "partition_index": i, "samplerate": 1000.0}, - ) - - def _close(self) -> None: - pass - - -def main() -> None: - with tempfile.TemporaryDirectory(prefix="dataflux-intake-demo-") as tmp: - # 1. Build an intake source and wrap it. - intake_src = _DemoXArraySource(n=4, label="demo") - df_src = IntakeSource(source=intake_src, target_attr="label") - print(f"IntakeSource length: {len(df_src)} partitions") - - # 2. Iterate and inspect the first sample. - first = next(iter(df_src)) - print( - "First sample:", - "input.shape=", - tuple(first.input.shape), - "input.dtype=", - first.input.dtype, - "target=", - first.target, - "metadata.keys=", - sorted(first.metadata.keys()), - ) - - # 3. Pipe through Flux → HDF5Sink and read back. - h5_path = Path(tmp) / "intake_roundtrip.h5" - Flux.from_source(df_src).to_sink(HDF5Sink(h5_path, overwrite=True)) - - loaded = list(HDF5Source(h5_path)) - print(f"Wrote {len(loaded)} samples to {h5_path}") - print( - "Roundtripped sample 2:", - "shape=", - tuple(loaded[2].input.shape), - "samplerate(meta)=", - loaded[2].metadata.get("samplerate"), - "label(meta)=", - loaded[2].metadata.get("label"), - ) - - print("OK — intake_pipeline.py finished.") - - -if __name__ == "__main__": - main() diff --git a/examples/paired_annotations.py b/examples/paired_annotations.py deleted file mode 100644 index 37f104c..0000000 --- a/examples/paired_annotations.py +++ /dev/null @@ -1,227 +0,0 @@ -"""PairedSource walkthrough: binary-first, annotation-first, broadcast, slicing. - -Runs standalone with no external data. Demonstrates the four scenarios the -``PairedSource`` primitive is designed for, using a tiny in-memory primary -source and a dict-shaped annotation store. -""" - -from typing import Any, Dict, Iterator, Optional - -import confluid # type: ignore[import-not-found] - -from dataflux.paired import PairedSource -from dataflux.sample import Sample - - -@confluid.configurable -class WindowedSource: - """Primary source yielding N windows over a mock pack.""" - - def __init__(self, pack_id: str = "demo:pack1", n_windows: int = 6, samples_per_window: int = 100) -> None: - self.pack_id = pack_id - self.n_windows = n_windows - self.samples_per_window = samples_per_window - - def __len__(self) -> int: - return self.n_windows - - def __getitem__(self, idx: int) -> Sample: - return Sample( - input=f"iq_window_{idx}", - target=None, - metadata={ - "pack_id": self.pack_id, - "window_start_sample": idx * self.samples_per_window, - "window_end_sample": (idx + 1) * self.samples_per_window, - "samplerate": 1_000_000.0, - }, - ) - - def __iter__(self) -> Iterator[Sample]: - for i in range(self.n_windows): - yield self[i] - - -@confluid.configurable -class DictStore: - """Mapping-shaped secondary for demos.""" - - def __init__(self, records: Optional[Dict[str, Dict[str, Any]]] = None) -> None: - self.records = records or {} - - def __contains__(self, key: str) -> bool: - return key in self.records - - def __getitem__(self, key: str) -> Dict[str, Any]: - return self.records[key] - - def keys(self) -> Any: - return self.records.keys() - - -# Module-level callables so they survive Confluid YAML round-trip via -# dataflux.discovery.resolve_callable("examples.paired_annotations:"). -def window_key(sample: Sample) -> str: - return f"{sample.metadata['pack_id']}:win{sample.metadata['window_start_sample']:08d}" - - -def pack_key(sample: Sample) -> str: - return str(sample.metadata["pack_id"]) - - -def resolve_by_window_key(key: str, primary: WindowedSource) -> Sample: - for item in primary: - if window_key(item) == key: - return item - raise KeyError(key) - - -def slice_intervals(record: Dict[str, Any], sample: Sample) -> Optional[Dict[str, Any]]: - """Trim per-pack time intervals down to each window's range.""" - samplerate = sample.metadata["samplerate"] - win_start = sample.metadata["window_start_sample"] / samplerate - win_end = sample.metadata["window_end_sample"] / samplerate - trimmed = [] - for iv in record.get("intervals", []): - s, e = max(iv["start_s"], win_start), min(iv["end_s"], win_end) - if e > s: - trimmed.append({**iv, "start_s": s, "end_s": e}) - if not trimmed: - return None - out = {k: v for k, v in record.items() if k != "intervals"} - out["intervals"] = trimmed - return out - - -def scenario_a_binary_first() -> None: - """Scenario A: iterate all samples; attach annotation when available.""" - print("\n=== Scenario A: binary-first, annotations optional ===") - primary = WindowedSource(n_windows=4) - store = DictStore({"demo:pack1:win00000100": {"label": "dji_mavic", "score": 0.92}}) - - paired = PairedSource(primary=primary, secondary=store, key_fn=window_key) - - for s in paired: - flag = "ANNOTATED" if s.metadata["annotated"] else " -" - label = s.metadata.get("label", "") - print(f" [{flag}] window_start={s.metadata['window_start_sample']:>4} label={label!r}") - - -def scenario_b_annotation_first() -> None: - """Scenario B: only emit samples that have an annotation.""" - print("\n=== Scenario B: annotation-first, curated labeled subset ===") - primary = WindowedSource(n_windows=6) - store = DictStore( - { - "demo:pack1:win00000000": {"label": "wifi"}, - "demo:pack1:win00000300": {"label": "lora"}, - } - ) - - paired = PairedSource(primary=primary, secondary=store, key_fn=window_key, policy="inner") - - for s in paired: - print(f" key={s.metadata['annotation_key']:<30} label={s.metadata['label']!r}") - print(f" -> {len(list(paired))} samples (out of {len(primary)} in primary)") - - -def scenario_c1_broadcast() -> None: - """Scenario C1: one annotation per pack, broadcast to every window.""" - print("\n=== Scenario C1: pack-level broadcast ===") - primary = WindowedSource(n_windows=4) - store = DictStore({"demo:pack1": {"drone": "DJI Mavic 3 Pro", "operator": "alice"}}) - - paired = PairedSource(primary=primary, secondary=store, key_fn=pack_key) - - for s in paired: - print( - f" win={s.metadata['window_start_sample']:>4} " - f"drone={s.metadata['drone']!r} operator={s.metadata['operator']!r}" - ) - - -def scenario_c2_slicing() -> None: - """Scenario C2: pack-level time-ranged annotation, sliced per window.""" - print("\n=== Scenario C2: pack-level time intervals, sliced per window ===") - primary = WindowedSource(n_windows=6, samples_per_window=100) - # Samplerate is 1 MHz and windows are 100 samples = 100 us each, so: - # win0 = [0, 100us], win1 = [100us, 200us], ..., win5 = [500us, 600us]. - # An interval at [150us, 470us] overlaps windows 1, 2, 3, 4. - store = DictStore( - { - "demo:pack1": { - "drone": "mavic", - "intervals": [{"start_s": 150e-6, "end_s": 470e-6, "label": "active_emission"}], - } - } - ) - - paired = PairedSource( - primary=primary, - secondary=store, - key_fn=pack_key, - extract_fn=slice_intervals, - ) - - for s in paired: - win = s.metadata["window_start_sample"] - if s.metadata["annotated"]: - iv = s.metadata["intervals"][0] - print( - f" win_start={win:>4} drone={s.metadata['drone']!r} " - f"active=[{iv['start_s']*1e6:.1f}us, {iv['end_s']*1e6:.1f}us]" - ) - else: - print(f" win_start={win:>4} no overlap") - - -def scenario_d_right_driven() -> None: - """Scenario D: iterate the annotation store, resolve primary on demand.""" - print("\n=== Scenario D: right-driven (sparse labels, large primary) ===") - primary = WindowedSource(n_windows=1000) # pretend this is huge - store = DictStore( - { - "demo:pack1:win00000000": {"label": "wifi"}, - "demo:pack1:win00050000": {"label": "lora"}, - "demo:pack1:win00099900": {"label": "dji_ocusync"}, - } - ) - - paired = PairedSource( - primary=primary, - secondary=store, - key_fn=window_key, - policy="right_driven", - primary_resolver=resolve_by_window_key, - ) - - for s in paired: - print(f" key={s.metadata['annotation_key']:<30} label={s.metadata['label']!r}") - - -def scenario_e_confluid_roundtrip() -> None: - """Show that the pipeline survives YAML serialization via Confluid.""" - print("\n=== Scenario E: Confluid YAML round-trip ===") - primary = WindowedSource(n_windows=3) - store = DictStore({"demo:pack1:win00000000": {"label": "wifi"}}) - paired = PairedSource(primary=primary, secondary=store, key_fn=window_key) - - yaml_state = confluid.dump(paired) - print(yaml_state) - restored = confluid.load(yaml_state) - print(f" restored.policy = {restored.policy!r}") - print(f" restored.key_fn = {restored.key_fn!r}") - - -def main() -> None: - scenario_a_binary_first() - scenario_b_annotation_first() - scenario_c1_broadcast() - scenario_c2_slicing() - scenario_d_right_driven() - scenario_e_confluid_roundtrip() - print("\nOK — paired_annotations.py finished.") - - -if __name__ == "__main__": - main() diff --git a/examples/parallel_hdf5_stream.py b/examples/parallel_hdf5_stream.py deleted file mode 100644 index 1b1fe37..0000000 --- a/examples/parallel_hdf5_stream.py +++ /dev/null @@ -1,50 +0,0 @@ -import time -from pathlib import Path - -import numpy as np - -from dataflux.core import Flux -from dataflux.storage.hdf5 import HDF5Sink, HDF5Source - - -def heavy_rescale(data: np.ndarray, factor: float = 1.0) -> np.ndarray: - """A simulated heavy transformation.""" - time.sleep(0.05) # Simulate CPU work - return data * factor - - -def main() -> None: - h5_path = Path("examples/stream_results.h5") - num_samples = 50 - - print(f"--- Starting Parallel Stream to {h5_path} ---") - print(f"Generating {num_samples} samples with 4 parallel workers...") - - # 1. Setup Source (Synthetic) - raw_data = [np.random.randn(1000) for _ in range(num_samples)] - - # 2. Setup Sink - sink = HDF5Sink(h5_path, overwrite=True) - - # 3. Build and Execute Pipeline - # Processing happens in parallel, Writing happens sequentially in the main process - start = time.time() - Flux(raw_data).parallel(workers=4).map(heavy_rescale, factor=255.0).to_sink(sink) - - duration = time.time() - start - print(f"Streaming completed in {duration:.2f}s") - - # 4. Verify the results - print("\n--- Verifying HDF5 Sink Content ---") - source = HDF5Source(h5_path) - with source: - print(f"Total samples in file: {len(source)}") - sample = next(iter(source)) - print(f"First sample shape: {sample.input.shape}") - print(f"First sample max value: {sample.input.max():.2f}") - - print("\nParallel HDF5 Streaming Verified!") - - -if __name__ == "__main__": - main() diff --git a/examples/parallel_pipeline.py b/examples/parallel_pipeline.py deleted file mode 100644 index 1c68165..0000000 --- a/examples/parallel_pipeline.py +++ /dev/null @@ -1,45 +0,0 @@ -import time - -import numpy as np - -from dataflux.core import Flux - - -def heavy_computation(data: np.ndarray, intensity: int = 10) -> np.ndarray: - """Simulate a heavy CPU-bound transformation.""" - # Artificial delay to simulate processing time - time.sleep(0.1) - return data * intensity - - -def main() -> None: - # 1. Create a large synthetic dataset - print("--- Creating data source (20 items) ---") - raw_data = [np.random.randn(100) for _ in range(20)] - - # 2. Sequential Execution - print("\n--- Running Sequentially ---") - start = time.time() - sequential_pipeline = Flux(raw_data).map(heavy_computation, intensity=2) - results_seq = sequential_pipeline.collect() - duration_seq = time.time() - start - print(f"Sequential duration: {duration_seq:.2f}s") - - # 3. Parallel Execution (4 workers) - print("\n--- Running in Parallel (4 workers) ---") - start = time.time() - # .parallel() enables the multiprocess engine - parallel_pipeline = Flux(raw_data).map(heavy_computation, intensity=2).parallel(workers=4) - results_par = parallel_pipeline.collect() - duration_par = time.time() - start - print(f"Parallel duration: {duration_par:.2f}s") - - # 4. Verification - assert len(results_seq) == len(results_par) - speedup = duration_seq / duration_par - print(f"\nSpeedup: {speedup:.1f}x") - print("Multiprocessing Verified!") - - -if __name__ == "__main__": - main() diff --git a/examples/record_pipeline.py b/examples/record_pipeline.py new file mode 100644 index 0000000..8970ec6 --- /dev/null +++ b/examples/record_pipeline.py @@ -0,0 +1,114 @@ +"""The record data model: a plain dict of typed values, type-dispatched ops, libraries as-is. + +Demonstrates the modality-neutral core of the engine: + +1. a record is a PLAIN ``dict`` of TYPED values, each owning its metadata — an ``Image`` + carries its layout, a ``Label`` its classes; scalar side values are just more keys; +2. the HEADLINE — ONE pipeline mixing a BARE albumentations transform (invoked natively by + the engine's op-family dispatch: it receives exactly its own ``image``/``mask``/``bboxes`` + keys, one call = one joint draw) with native ops. recordstream ships NO augmentation of its + own and NO adapter classes — the libraries run as-is; +3. cross-key consistency — ONE ``A.Compose`` draw moves image, mask and bboxes together, + the Label untouched; +4. a native op in the torchvision-v2 authoring style: params drawn once per record in + ``get_params``, a kernel per value type, ``field=`` pinning it to one key; +5. a torchvision ``transforms.v2`` transform as-is — after the EXPLICIT ``v2.ToImage()`` + conversion, exactly like a plain torchvision pipeline (the engine never converts silently). + +Standalone, zero-arg, exit 0 (CI runs every ``examples/*.py``). +""" + +import albumentations as A +import numpy as np +import torch +from torchvision.transforms import v2 + +from recordstream import Image, Label, Mask, Pipeline, Record, Transform, as_transform + + +def make_record(rng: np.random.Generator) -> Record: + """A detection record: image, mask, boxes (albumentations vocabulary), and a class label.""" + return { + "image": Image(rng.random((16, 20, 3)).astype(np.float32)), + "mask": Mask((rng.random((16, 20)) > 0.5).astype(np.uint8)), + "bboxes": [[2, 3, 6, 7]], + "labels": ["drone"], + "class": Label("drone_x", classes=["noise", "drone_x"]), + "gain_db": -3.0, # a scalar side value is just another key + } + + +class Brighten(Transform): + """Add a per-record random offset to every Image value (the tv2 authoring pattern). + + Args: + strength: Maximum brightness offset drawn per record. + field: Apply only to this record key. None (default) = every Image value. + """ + + handles = (Image,) + + def __init__(self, strength: float = 0.1, field: str = None) -> None: # type: ignore[assignment] + super().__init__(field=field) + self.strength = strength + self._rng = np.random.default_rng(7) + + def get_params(self, record: Record) -> dict: + return {"offset": self._rng.uniform(0.0, self.strength)} # drawn ONCE per record + + +@Brighten.kernel(Image) +def _brighten_image(value: Image, params: dict) -> Image: + return Image(np.asarray(value) + params["offset"], layout=value.layout) + + +def main() -> None: + rng = np.random.default_rng(0) + + # 1. The record: a plain dict of typed values. + record = make_record(rng) + print("record keys: ", list(record)) + print("image meta: ", f"layout={record['image'].layout} class vocab={record['class'].classes}") + + # 2+3. HEADLINE — bare albumentations (its OWN Compose carries bbox_params) + a native + # op in ONE Pipeline. The engine invokes each op family natively — no wrappers. + flip = A.Compose( + [A.HorizontalFlip(p=1.0)], + bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"]), + ) + out = Pipeline([flip, A.GaussNoise(p=1.0), Brighten(strength=0.2)])(record) + assert out is not None + print("\n--- bare albumentations + native op in one pipeline ---") + print("image ->", type(out["image"]).__name__, np.asarray(out["image"]).shape, "(flipped + noised + brightened)") + print("mask ->", type(out["mask"]).__name__, "(flipped with the image — one joint draw)") + print("bboxes ->", record["bboxes"], "->", [[round(v) for v in b] for b in out["bboxes"]], "(W=20)") + print("class ->", type(out["class"]).__name__, repr(out["class"].value), "(no handler — untouched)") + assert np.array_equal(np.asarray(out["mask"]), np.asarray(record["mask"])[:, ::-1]) + assert [round(v) for v in out["bboxes"][0]] == [14, 3, 18, 7] + assert out["class"].value == "drone_x" and out["gain_db"] == -3.0 + assert isinstance(out["image"], Image) and isinstance(out["mask"], Mask) # types survive the library + + # 4. field= pins a type-dispatched op to ONE key (here a no-op: "class" is not an Image). + untouched = Brighten(strength=0.2, field="class")(record) + assert untouched is not None and np.array_equal(np.asarray(untouched["image"]), np.asarray(record["image"])) + + # 5. torchvision v2 as-is: the EXPLICIT conversion first (v2.ToImage: numpy HWC -> CHW + # tv_tensor), then any v2 transform — the engine passes the dict straight through. + tv_out = Pipeline([v2.ToImage()])({"image": np.asarray(record["image"])}) + assert tv_out is not None and isinstance(tv_out["image"], torch.Tensor) and tv_out["image"].shape == (3, 16, 20) + cropped = Pipeline([v2.RandomCrop(8)])(tv_out) + assert cropped is not None and tuple(cropped["image"].shape) == (3, 8, 8) + print("\n--- torchvision v2 as-is (explicit ToImage conversion) ---") + print("image ->", type(cropped["image"]).__name__, tuple(cropped["image"].shape)) + + # 6. A custom function op — no library, no core edit. + doubled = as_transform(lambda d: d * 2, handles=(Mask,), field="mask")(record) + assert doubled is not None + print("\n--- custom function op ---") + print("mask doubled:", np.array_equal(np.asarray(doubled["mask"]), np.asarray(record["mask"]) * 2)) + + print("\nOK") + + +if __name__ == "__main__": + main() diff --git a/examples/storage_roundtrip.py b/examples/storage_roundtrip.py new file mode 100644 index 0000000..2823412 --- /dev/null +++ b/examples/storage_roundtrip.py @@ -0,0 +1,98 @@ +"""Storage round-trip: every sink has a matching source, one codec serializes everything. + +The storage tour of the record model (`typedrecord-v1` — the key-group layout): + +1. Write the SAME records through all three sink/source pairs — `HDF5Sink`↔`HDF5Source`, + `ZarrGroupSink`↔`ZarrGroupSource`, `DirectorySink`↔`DirectorySource` — and read them back + IDENTICAL: typed values keep their type AND their metadata attrs (`Image.layout`, + `Label.classes`), and a plain scalar entry (`snr_db`) rides the same layout via the + ``"plain"`` codec tag. +2. Query a store's metadata WITHOUT loading a single array: `MetadataFilterSource` over the + HDF5 store filters on the scalar entry (`snr_db.value > 10`) through the + `SupportsMetadataScan` protocol — attrs only, payloads untouched until iteration. + +Everything is written to a temp directory — examples leave no artifacts behind. + +Standalone, zero-arg, exit 0 (CI runs every ``examples/*.py``). +""" + +import tempfile +from pathlib import Path +from typing import List, Tuple, Union + +import numpy as np + +from recordstream import Image, Label, Record +from recordstream.storage.directory import DirectorySink, DirectorySource +from recordstream.storage.hdf5 import HDF5Sink, HDF5Source +from recordstream.storage.query import MetadataFilterSource +from recordstream.storage.zarr import ZarrGroupSink, ZarrGroupSource + +#: The three backends are independent classes (no shared sink/source base beyond ``Storage``), +#: so the pair list is spelled as a union to keep ``write``/``flush``/iteration typed. +AnySink = Union[HDF5Sink, ZarrGroupSink, DirectorySink] +AnySource = Union[HDF5Source, ZarrGroupSource, DirectorySource] + + +def make_records(n: int = 4) -> list: + rng = np.random.default_rng(0) + return [ + { + "image": Image(rng.random((8, 10, 3)).astype(np.float32)), # typed: knows its layout + "class": Label(i % 2, classes=["noise", "drone"]), # typed: carries its vocabulary + "snr_db": float(5 * i), # plain scalar — just another key + } + for i in range(n) + ] + + +def assert_roundtrip(original: Record, restored: Record) -> None: + assert set(restored) == set(original) + assert isinstance(restored["image"], Image) and restored["image"].layout == "HWC" + assert np.array_equal(np.asarray(restored["image"]), np.asarray(original["image"])) + assert isinstance(restored["class"], Label) + assert restored["class"].value == original["class"].value + assert restored["class"].classes == original["class"].classes + assert restored["snr_db"] == original["snr_db"] # the "plain" codec tag + + +def main() -> None: + records = make_records() + + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + pairs: List[Tuple[AnySink, AnySource]] = [ + (HDF5Sink(path=work / "store.h5"), HDF5Source(path=work / "store.h5")), + (ZarrGroupSink(path=work / "store.zarr"), ZarrGroupSource(path=work / "store.zarr")), + (DirectorySink(path=work / "store_dir"), DirectorySource(path=work / "store_dir")), + ] + + # 1. Every sink has a matching source — write, read back, byte-identical typed records. + for sink, source in pairs: + with sink: + for record in records: + sink.write(record) + sink.flush() + with source: + restored = list(source) + assert len(restored) == len(records) + for original, back in zip(records, restored): + assert_roundtrip(original, back) + print( + f" {type(sink).__name__:14} -> {type(source).__name__:16} round-trip OK " + f"({len(restored)} records; Image+attrs, Label+vocab, plain snr_db)" + ) + + # 2. Metadata-only querying: filter the HDF5 store on the plain scalar WITHOUT + # loading arrays (the SupportsMetadataScan protocol reads attrs only; a plain + # scalar is addressable as .value). + filtered = MetadataFilterSource(source=HDF5Source(path=work / "store.h5"), where="snr_db.value > 10") + hits = list(filtered) + assert [r["snr_db"] for r in hits] == [15.0] + print(f" MetadataFilterSource(where='snr_db.value > 10') -> {len(hits)} record (snr_db=15.0)") + + print("OK") + + +if __name__ == "__main__": + main() diff --git a/examples/workflow_pipeline.py b/examples/workflow_pipeline.py new file mode 100644 index 0000000..6a0f2cc --- /dev/null +++ b/examples/workflow_pipeline.py @@ -0,0 +1,140 @@ +"""A resume-safe train → evaluate workflow — ONE Confluid document of runnables. + +The compelling case for the workflow combinators: a pipeline you can re-run after a crash +(or a second `recordstream run`) that SKIPS the work whose artifact already exists and +carries on — *memoise and continue*, expressed declaratively: + +1. ``Sequence`` drives the stages in order (the workflow itself). +2. ``Conditional`` + ``PathExists`` guard the expensive stage: train ONLY when the + checkpoint is missing. On a cache hit the branch is not just skipped — wired + ``!lazy:``, it is **never even constructed** (no model, no dataset materialised). +3. ``Switch`` picks the report format off a plain config value — one key a CLI override + can flip (``--select text``) without touching the workflow shape. + +This script runs the SAME document twice and proves both guarantees: the second pass +evaluates again but does not retrain, and the trainer class records exactly ONE +construction across both passes. + +Standalone, zero-arg, exit 0 (CI runs every ``examples/*.py``). +""" + +import tempfile +from pathlib import Path + +import confluid +from confluid import configurable +from confluid.fluid import Fluid + +# --------------------------------------------------------------------------------------- +# Three tiny stub runnables (a runnable = any object with a no-arg run()). Real pipelines +# put a trainer / DatasetProcessor here; stubs keep the example self-contained and fast. +# --------------------------------------------------------------------------------------- + + +@configurable +class TrainModel: + """Stub trainer: 'trains' by writing the checkpoint artifact. + + Args: + ckpt: Path the checkpoint artifact is written to. + """ + + #: Instrumentation for THIS example: counts constructions, to prove the ``!lazy:`` + #: guarantee (an unchosen branch is never built). Not a pattern for real runnables. + builds = 0 + + def __init__(self, ckpt: str = "") -> None: + type(self).builds += 1 + self.ckpt = ckpt + + def run(self) -> None: + Path(self.ckpt).write_text("weights") + print(f" [train] wrote {Path(self.ckpt).name}") + + +@configurable +class Evaluate: + """Stub evaluator: reads the checkpoint, writes a report in the given format. + + Args: + ckpt: Checkpoint artifact to 'evaluate'. + report: Path the report is written to. + fmt: Report format tag written into the file. + """ + + def __init__(self, ckpt: str = "", report: str = "", fmt: str = "text") -> None: + self.ckpt = ckpt + self.report = report + self.fmt = fmt + + def run(self) -> None: + weights = Path(self.ckpt).read_text() + Path(self.report).write_text(f"[{self.fmt}] accuracy of {weights!r}: 0.93") + print(f" [evaluate] wrote {Path(self.report).name} ({self.fmt})") + + +def workflow_yaml(work: Path) -> str: + """The whole pipeline — stages, the cache guard, AND the format switch — as ONE document.""" + return f""" +runnable: !class:recordstream.workflow.Sequence + steps: + # Stage 1 — the expensive stage, guarded: train ONLY when the checkpoint is missing. + # On a cache hit the !lazy: branch is never even BUILT (no model materialised). + - !class:recordstream.workflow.Conditional + condition: !class:recordstream.workflow.PathExists + path: {work / "model.ckpt"} + if_true: null # cache hit -> skip, Sequence continues + if_false: !lazy:TrainModel # @configurable classes resolve by registered NAME + ckpt: {work / "model.ckpt"} + + # Stage 2 — always runs; the report FORMAT is a Switch on a plain config value + # (override from a CLI with --select text — the workflow shape never changes). + - !class:recordstream.workflow.Switch + select: json + cases: + json: !lazy:Evaluate + ckpt: {work / "model.ckpt"} + report: {work / "report.json"} + fmt: json + text: !lazy:Evaluate + ckpt: {work / "model.ckpt"} + report: {work / "report.txt"} + fmt: text +""" + + +def run_document(path: Path) -> None: + """What ``recordstream run `` does: bind the top-level ``runnable:`` and run it.""" + loaded = confluid.load(str(path)) + runnable = loaded["runnable"] + if isinstance(runnable, Fluid): + runnable = confluid.flow(runnable) + runnable.run() + + +def main() -> None: + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + doc = work / "workflow.yaml" + doc.write_text(workflow_yaml(work)) + + print("--- pass 1: nothing cached -> trains, then evaluates ---") + run_document(doc) + assert (work / "model.ckpt").exists() and (work / "report.json").exists() + assert TrainModel.builds == 1 + + (work / "report.json").unlink() # so pass 2 visibly re-evaluates + + print("--- pass 2: checkpoint exists -> SKIPS training, evaluates again ---") + run_document(doc) + assert (work / "report.json").exists() + + # The two guarantees: no retrain (artifact-guarded), and the unchosen !lazy: + # branch was never CONSTRUCTED on pass 2 — builds stayed at one. + assert TrainModel.builds == 1, f"trainer was rebuilt: {TrainModel.builds}" + print("\ntrainer constructions across both passes:", TrainModel.builds) + print("OK") + + +if __name__ == "__main__": + main() diff --git a/manifests/datasets/flux.yaml b/manifests/datasets/flux.yaml deleted file mode 100644 index 5b31ab5..0000000 --- a/manifests/datasets/flux.yaml +++ /dev/null @@ -1,18 +0,0 @@ -description: "DataFlux Flux — primary stream engine wrapping any iterable or indexed dataset with a functional op chain." -use_case: "The fundamental DataFlux container. Wrap a Source (HuggingFaceSource / RegionsJsonSource / a list of Samples) and apply zero or more Ops lazily on __getitem__. Composable via JointFlux for source-mixing or DatasetSplit for train/val partitioning. Supports map-style (__getitem__) when the underlying source has __len__." -category: "Generic" -flux_type: "Dataset" -class_path: "dataflux.core.Flux" -params: - source: - type: "Iterable[Sample] | torch.utils.data.Dataset" - required: true - description: "Underlying data source — anything iterable yielding Samples, or a torch Dataset." - ops: - type: "List[dataflux.core.Op] | null" - default: null - description: "Ordered list of ops applied lazily on __getitem__ / iteration. Empty / null = pass-through." - chunk_size: - type: "int | null" - default: 0 - description: "Optional chunking hint for streaming sources. 0 = no chunking." diff --git a/manifests/datasets/joint_flux.yaml b/manifests/datasets/joint_flux.yaml deleted file mode 100644 index 5dfe49a..0000000 --- a/manifests/datasets/joint_flux.yaml +++ /dev/null @@ -1,10 +0,0 @@ -description: "Aggregate multiple Flux streams into a single joint sequential stream." -use_case: "Use to mix sources during training (e.g. fold a small slice of the test set into train). Each sub-flux keeps its own op chain — JointFlux only sequences iteration. Pair with DatasetSplit when you want a reproducible disjoint train/val partition with shared random seed." -category: "Generic" -flux_type: "Dataset" -class_path: "dataflux.core.JointFlux" -params: - fluxes: - type: "List[dataflux.core.Flux]" - required: true - description: "Sub-fluxes whose iteration is concatenated end-to-end." diff --git a/manifests/sources/dataset_split.yaml b/manifests/sources/dataset_split.yaml deleted file mode 100644 index d83ca92..0000000 --- a/manifests/sources/dataset_split.yaml +++ /dev/null @@ -1,27 +0,0 @@ -description: "Select a reproducible subset view of an indexable source." -use_case: "Three modes: (1) fraction-mode train/val split with shared seed (two DatasetSplits with same seed + val_fraction yield disjoint complementary views — wire via !ref to share the underlying source); (2) explicit indices for a fixed subset; (3) range selection. Heavily used in the rfuav templates to fold-test-into-train." -category: "Generic" -flux_type: "DataSource" -class_path: "dataflux.sources.DatasetSplit" -params: - source: - type: "Indexable source" - required: true - description: "Underlying source with __len__ and __getitem__ (e.g. a Flux wrapping a RegionsJsonSource)." - split: - type: "str" - default: "train" - choices: ["train", "val"] - description: "Which side of the partition this view returns. With shared seed across two DatasetSplits, 'train' and 'val' are guaranteed disjoint and complementary." - val_fraction: - type: "float" - default: 0.2 - description: "Fraction of the source going to the 'val' branch (the rest goes to 'train')." - seed: - type: "int" - default: 42 - description: "Random seed for the shuffle. Must match across paired splits to get a clean partition." - indices: - type: "List[int] | null" - default: null - description: "Explicit indices mode (mutually exclusive with split/val_fraction)." diff --git a/manifests/sources/huggingface_source.yaml b/manifests/sources/huggingface_source.yaml deleted file mode 100644 index 65d0bd2..0000000 --- a/manifests/sources/huggingface_source.yaml +++ /dev/null @@ -1,34 +0,0 @@ -description: "Hugging Face Datasets adapter — maps HF dataset features to DataFlux Sample triplets." -use_case: "Standard adapter for any HF Hub dataset (e.g. kitofrank/RFUAV). Configure which feature column maps to Sample.input, which to Sample.target, and which (if any) to Sample.metadata." -category: "HuggingFace" -flux_type: "DataSource" -class_path: "dataflux.sources.HuggingFaceSource" -params: - path: - type: "str" - required: true - description: "HF dataset path — repo identifier (kitofrank/RFUAV) or local imagefolder path." - split: - type: "str" - default: "train" - description: "HF split name (train / validation / test / etc.)." - input_feature: - type: "str" - default: "image" - description: "Feature column to map to Sample.input." - target_feature: - type: "str" - default: "label" - description: "Feature column to map to Sample.target." - metadata_features: - type: "List[str] | null" - default: null - description: "Feature columns preserved on Sample.metadata." - count: - type: "int | null" - default: null - description: "Optional cap on the number of samples (useful for fast smoke runs)." - name: - type: "str | null" - default: null - description: "Optional HF subset name (e.g. for multi-config datasets)." diff --git a/notebooks/01 - cat_exploration.ipynb b/notebooks/01 - cat_exploration.ipynb index ec27f49..8f8a616 100644 --- a/notebooks/01 - cat_exploration.ipynb +++ b/notebooks/01 - cat_exploration.ipynb @@ -21,7 +21,7 @@ "from dotenv import load_dotenv\n", "\n", "# Walk up from the notebook to find the workspace .env (alongside the top-level\n", - "# CLAUDE.md / AGENTS.md). Loading it before importing `datasets` is required \u2014\n", + "# CLAUDE.md / AGENTS.md). Loading it before importing `datasets` is required —\n", "# huggingface_hub reads HF_HOME at import time of its cache module.\n", "here = Path.cwd()\n", "for candidate in [here, *here.parents]:\n", @@ -29,16 +29,21 @@ " load_dotenv(candidate / \".env\")\n", " break\n", "\n", + "# DATA_ROOT chooses WHERE the HF cache lives; it is not required to run this\n", + "# notebook. On a workstation it points the cache at the configured volume so a\n", + "# large dataset does not land in the home directory. Where it is unset — a CI\n", + "# runner, or a fresh clone with no .env — huggingface_hub uses its own default\n", + "# cache and the notebook runs unchanged. A DATA_ROOT that is SET but missing is\n", + "# still an error: that is a misconfigured machine, not an absent preference.\n", "data_root = os.getenv(\"DATA_ROOT\")\n", - "if not data_root:\n", - " raise RuntimeError(\"DATA_ROOT is not set \u2014 add it to the workspace .env (e.g. DATA_ROOT=/Volumes/Store).\")\n", - "if not Path(data_root).exists():\n", - " raise RuntimeError(f\"DATA_ROOT={data_root!r} does not exist on this machine \u2014 mount the volume or update .env.\")\n", + "if data_root:\n", + " if not Path(data_root).exists():\n", + " raise RuntimeError(f\"DATA_ROOT={data_root!r} does not exist on this machine — mount the volume or update .env.\")\n", + " # Point HF cache at the configured volume regardless of any inherited HF_HOME.\n", + " os.environ[\"HF_HOME\"] = str(Path(data_root) / \"huggingface\")\n", "\n", - "# Point HF cache at the configured volume regardless of any inherited HF_HOME.\n", - "os.environ[\"HF_HOME\"] = str(Path(data_root) / \"huggingface\")\n", - "print(f\"DATA_ROOT = {data_root}\")\n", - "print(f\"HF_HOME = {os.environ['HF_HOME']}\")" + "print(f\"DATA_ROOT = {data_root or '(unset — using the default HF cache)'}\")\n", + "print(f\"HF_HOME = {os.environ.get('HF_HOME', '(default)')}\")" ] }, { diff --git a/pyproject.toml b/pyproject.toml index cc002a7..c35cd53 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,26 +1,69 @@ [project] -name = "data-flux" -version = "0.1.0" +name = "recordstream" +version = "0.1.0a1" description = "Clean, functional data pipelines for ML research and production." -authors = [{ name = "Taidal", email = "info@gearlux.ai" }] +readme = "README.md" +authors = [{ name = "Gert Behiels" }] +license = "MIT" +license-files = ["LICENSE"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Software Development :: Libraries :: Python Modules", + "Typing :: Typed", +] +# Floors are what this package is actually TESTED against, not the oldest release that +# once worked: `confluid==0.1.0` is provably unsatisfiable here (liquifai requires +# confluid>=0.2.0), so the old `>=0.1.0` described a combination nobody could install. dependencies = [ - "confluid>=0.1.0", - "log-flow>=0.1.0", + "confluid>=0.3.0", + "loggair>=0.2.0", "numpy", + "Pillow", "h5py", "zarr", - "torch", "typing-extensions", "datasets", "albumentations", "orjson", "fsspec", - "cloudpathlib" + "cloudpathlib", + # liquifai powers the generic `recordstream run ` CLI (recordstream.cli) that + # runs any Confluid-wired runnable; it also brings `rich` (used by DatasetProcessor's + # optional console progress bar). liquifai depends only on confluid/loggair/rich — no cycle. + "liquifai>=0.2.0", ] requires-python = ">=3.12" +[project.urls] +Homepage = "https://github.com/Gearlux/recordstream" +Repository = "https://github.com/Gearlux/recordstream.git" +Issues = "https://github.com/Gearlux/recordstream/issues" +Changelog = "https://github.com/Gearlux/recordstream/blob/main/CHANGELOG.md" + [project.optional-dependencies] +# The torch-shaped surface, NOT the engine: `recordstream.ops.torch` (ToTensor), +# `batch_tensor`, and the `outputs` builders. The core is numpy — `Stream` is a plain +# map-style class, and the two places that recognise a torch tensor do it through +# `_compat.is_torch_tensor`, which consults sys.modules rather than importing. So a +# consumer on Keras/JAX installs recordstream without a 2GB framework it never calls. +torch = ["torch"] +# The Keras-shaped surface: `recordstream.keras` (the KERAS_BACKEND ordering + `RecordSequence`, +# the `PyDataset` a `DataLoader` would have been). Keras 3 is an API, not a runtime, so this +# names NO compute engine — `recordstream.keras` defaults KERAS_BACKEND to whichever of +# torch/tensorflow/jax is actually installed, and the consumer's own extra picks one. +keras = ["keras>=3.0"] dev = [ + # Release tooling: `python -m build` produces the sdist + wheel, `twine check` + # validates the metadata PyPI will re-validate on upload. + "build>=1.0.0", + "twine>=5.0.0", "black>=24.0.0,<25.0.0", "isort>=5.13.0,<6.0.0", "flake8>=6.0.0,<8.0.0", @@ -28,9 +71,15 @@ dev = [ "mypy>=1.0.0,<2.0.0", "pytest>=7.0.0,<9.0.0", "pytest-cov>=4.0.0,<7.0.0", - "Pillow", - "intake>=2.0", - "xarray", + # CI installs `.[dev]` (unit-tests AND verify-examples), so torchvision here keeps the + # bare torchvision-v2 op family + examples exercised in CI without a workflow edit. + "torchvision", + # Same reason: `ops.image._apply_colormap` lazily imports matplotlib for every + # non-gray colormap, and the gray path is matplotlib-free BY DESIGN — so without this + # the colormap half of ConvertToImage / value_to_image goes untested in CI. It stays + # out of `dependencies` deliberately: a consumer rendering only grayscale must not be + # made to install a plotting library. + "matplotlib", ] notebook = [ "matplotlib", @@ -39,34 +88,97 @@ notebook = [ ] vision = [ "scipy", + # Bare torchvision transforms.v2 ops run as-is through the engine's op-family dispatch + # (recordstream.core.families._apply_op); the extra makes `pip install recordstream[vision]` the + # documented way to enable them. + "torchvision", ] [build-system] -requires = ["setuptools>=64", "wheel"] +requires = ["setuptools>=77", "wheel"] build-backend = "setuptools.build_meta" # CONFLUID-CONFIGURABLE DISCOVERY -# Sources / ops live in separate submodules that `dataflux/__init__.py` +# Sources / ops live in separate submodules that `recordstream/__init__.py` # does not eagerly import. Listing them here makes navigaitor discover # every ``@configurable`` on bootstrap without hardcoding the list. [project.entry-points."confluid.configurables"] -dataflux = "dataflux" -dataflux-core = "dataflux.core" -dataflux-sources = "dataflux.sources" -dataflux-ops-parallel = "dataflux.ops.parallel" -dataflux-ops-tee = "dataflux.ops.tee" -dataflux-ops-stash = "dataflux.ops.stash" -dataflux-ops-numpy = "dataflux.ops.numpy" -dataflux-ops-torch = "dataflux.ops.torch" -dataflux-ops-copy = "dataflux.ops.copy" -dataflux-ops-swap = "dataflux.ops.swap" +recordstream = "recordstream" +# ONE entry point per engine PACKAGE (core / flow / sources are each one class per cohesive +# module, all re-exported by the package __init__, so importing it registers every +# @configurable). Unlike recordstream.ops.*, do NOT add per-submodule entry points: scan_module's +# __module__ filter finds nothing in a package __init__, so a visual editor's node bridge +# surfaces these through its second pass over ``__all__`` — which is why that list is load-bearing. +recordstream-core = "recordstream.core" +recordstream-sources = "recordstream.sources" +recordstream-ops-parallel = "recordstream.ops.parallel" +recordstream-ops-enable = "recordstream.ops.enable" +recordstream-ops-random-apply = "recordstream.ops.random_apply" +# ConfigureOp (per-record parameter injection — derive an op's parameter from the record); entry-point +# changes need an editable reinstall before StreamStudio/navigaitor discovery sees the module. +recordstream-ops-configure = "recordstream.ops.configure" +recordstream-ops-formula = "recordstream.ops.formula" +# The FlowGraph engine (flow: named-step documents) + the per-record kernel both authoring +# forms run on. There is no flow<->ops lowering pass — it was deleted 2026-07-30 (architecture §3). +recordstream-flow = "recordstream.flow" +# The queryable-metadata scan protocol + MetadataFilterSource view source +recordstream-storage-query = "recordstream.storage.query" +recordstream-ops-sink = "recordstream.ops.sink" +recordstream-ops-predict = "recordstream.ops.predict" +recordstream-ops-numpy = "recordstream.ops.numpy" +recordstream-ops-torch = "recordstream.ops.torch" +recordstream-ops-target = "recordstream.ops.target" +recordstream-ops-image = "recordstream.ops.image" +# PrintRecordOp (log/print a per-record summary). Entry-point changes need an editable reinstall +# before StreamStudio/navigaitor discovery sees the module (`aisland setup`, never --reinstall). +recordstream-ops-debug = "recordstream.ops.debug" +# Storage SINKS (HDF5Sink / ZarrGroupSink / ZarrBatchSink / DirectorySink) carry +# category="sink" so StreamStudio surfaces them as DatasetProcessor sink nodes. They live under +# recordstream.storage.* (NOT re-exported from the package root), and scan_module does not recurse +# submodules, so each storage module needs its own entry point. The matching SOURCES in these +# modules stay uncategorised, so the positive {op,source,engine,sink} allowlist surfaces only the +# tagged sinks. (Entry-point changes need an editable reinstall — `aisland setup`, never --reinstall.) +recordstream-storage-hdf5 = "recordstream.storage.hdf5" +recordstream-storage-zarr = "recordstream.storage.zarr" +recordstream-storage-directory = "recordstream.storage.directory" +# The record model's op surface: Transform (type-dispatched record ops) + Pipeline (THE +# compose op) live in recordstream.transform. Entry-point changes need an editable reinstall +# before StreamStudio/navigaitor discovery sees the module (`aisland setup`, never --reinstall). +recordstream-transform = "recordstream.transform" +# Structure ops (RenameField/DropField/CopyField/SelectFields) — reshape a record's entries. +recordstream-ops-structure = "recordstream.ops.structure" +# The runnable orchestration layer: DatasetProcessor (generic source→sink runner) and the +# workflow combinators (Sequence/Conditional/Switch + PathExists/Not/AllOf/AnyOf predicates). +# Discovered as @configurable runnables; run via `recordstream run`. +recordstream-processing = "recordstream.processing" +recordstream-workflow = "recordstream.workflow" + +# The `recordstream` console script — `recordstream run ` runs any Confluid-wired +# runnable (a trainer, an evaluator, a DatasetProcessor, a workflow). +[project.scripts] +recordstream = "recordstream.cli:main" + +# Liquifai app registration: lets `liquifai-install-completions` discover this CLI via an +# instant entry-point metadata read instead of a subprocess probe. Name = the binary name; +# value = the LiquifyApp instance. +[project.entry-points."liquifai.apps"] +recordstream = "recordstream.cli:app" + +# Which of this project's extras are ML FRAMEWORKS, for `aisland framework`. Selecting a name +# here is what makes `aisland setup` add the matching extra to this project's install. +# +# The core engine is numpy — `torch` powers only `ToTensor` and the output builders, and `keras` +# only `recordstream.keras`, so a workspace that selects no framework still gets a fully working +# record engine. +[tool.aisland] +frameworks = ["torch", "keras"] [tool.setuptools.packages.find] where = ["."] -include = ["dataflux*"] +include = ["recordstream*"] [tool.setuptools.package-data] -dataflux = ["py.typed"] +recordstream = ["py.typed"] [tool.black] line-length = 120 @@ -79,13 +191,26 @@ line_length = 120 testpaths = ["tests"] python_files = ["test_*.py"] +# This project is type-checked TWICE with different configs, and only one of them is +# authoritative for a release. From the workspace root a shared `mypy.ini` supplies these +# ignores, so a gap here is invisible there; this project's OWN CI runs `mypy .` from the +# project directory, reads only the table below, and is what a published package must +# satisfy. Every optional or lazily-imported third party therefore needs an entry HERE, +# even when the workspace config already carries one. [[tool.mypy.overrides]] module = [ + "albumentations.*", "datasets.*", "h5py.*", + # Lazily imported inside `ops.image._apply_colormap` (only a non-gray colormap needs + # it), so it is not a declared dependency at all — nothing installs it for CI. + "matplotlib.*", + # The `[keras]` extra. Keras ships no `py.typed`, so it is untyped rather than absent. + "keras.*", "PIL.*", + # The `[vision]` extra, behind a try/except in `ops.numpy.connected_component_boxes`. + "scipy.*", + "torchvision.*", "zarr.*", - "intake.*", - "xarray.*", ] ignore_missing_imports = true diff --git a/recordstream/__init__.py b/recordstream/__init__.py new file mode 100644 index 0000000..b25a548 --- /dev/null +++ b/recordstream/__init__.py @@ -0,0 +1,219 @@ +""" +RecordStream: Modular, functional data pipelines. + +The data model is the RECORD: a record is a plain ``dict`` of typed values (each value +owning its metadata — an ``Image`` its layout, a ``Label`` its classes), and ops dispatch +on value TYPE (the torchvision-v2 model). Bare albumentations / torchvision ``transforms.v2`` +transforms drop into any ops list AS-IS — the engine invokes each op family natively +(``recordstream.core.families._apply_op``). Import the whole surface from the package top level +(``from recordstream import Record, Image, Transform, Pipeline, ...``). +""" + +# --- shared infrastructure ----------------------------------------------------------------- +from recordstream.batch import ( + batch_boxes, + batch_metadata, + batch_tensor, + batch_values, + multi_hot, + per_record_predictions, +) +from recordstream.collate import ( + collate, + collate_list, + collate_records, + get_collate, + register_collate, + registered_collates, +) +from recordstream.core import ( + FilterOp, + JointStream, + RecordSource, + Stream, + WrappedOp, + ensure_materialized, + ensure_record_dataset, + prepare_record_dataset, + register_op_family, + registered_op_families, +) + +# --- the record data model + transforms + item codec ---------------------------------------- +from recordstream.dispatch import dispatch, register_kernel, registered_kernels +from recordstream.flow import FlowGraph +from recordstream.io import ( + EncodedField, + EncodedItem, + decode_item, + decode_record, + encode_item, + encode_record, + register_io, +) +from recordstream.items import ( + Boxes, + Image, + Label, + Mask, + MultiLabel, + NDArrayItem, + Record, + get_item_type, + is_class_id, + is_item, + item_data, + item_type_names, + item_types, + item_value, + register_item, + resolve_entry, + resolve_item, + with_data, +) +from recordstream.labels import LabelMap, class_counts, inverse_frequency_weights +from recordstream.outputs import ( + ClassificationOutput, + DetectionOutput, + DetectionPredictions, + RestorationOutput, + SegmentationOutput, + classification_output, + restoration_output, + segmentation_output, +) +from recordstream.predictions import ClassificationPredictionsSink, PredictionsSink +from recordstream.processing import DatasetProcessor +from recordstream.projection import ( + SupportsProjection, + class_names, + first_value, + iter_key, + num_classes, + num_mask_classes, + project, +) +from recordstream.runnable import ( + ProgressCallback, + ProgressReporting, + RunnableTask, + TorchRunner, + entrypoint, + entrypoint_tasks, + run_entrypoint, + runnable_entrypoints, +) +from recordstream.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource, SplitName +from recordstream.transform import FunctionTransform, Pipeline, Transform, as_transform +from recordstream.uri import SupportsDatasetIdentity, dataset_uri, dataset_uris, dataset_url +from recordstream.workflow import AllOf, AnyOf, Conditional, Not, PathExists, Sequence, Switch + +__all__ = [ + # ---- record data model ---- + "Record", + "NDArrayItem", + "Image", + "Mask", + "Boxes", + "Label", + "MultiLabel", + "is_class_id", + "register_item", + "item_types", + "item_type_names", + "get_item_type", + "is_item", + "item_data", + "item_value", + "resolve_entry", + "resolve_item", + "with_data", + "Transform", + "Pipeline", + "FunctionTransform", + "as_transform", + "dispatch", + "register_kernel", + "registered_kernels", + "EncodedItem", + "EncodedField", + "register_io", + "encode_item", + "decode_item", + "encode_record", + "decode_record", + # ---- shared infrastructure ---- + "Stream", + "JointStream", + "RecordSource", + "ensure_materialized", + "ensure_record_dataset", + "prepare_record_dataset", + "FilterOp", + "WrappedOp", + "register_op_family", + "registered_op_families", + "FlowGraph", + "collate", + "batch_metadata", + "batch_boxes", + "batch_tensor", + "batch_values", + "multi_hot", + "per_record_predictions", + "collate_list", + "collate_records", + "get_collate", + "register_collate", + "registered_collates", + "LabelMap", + "class_counts", + "inverse_frequency_weights", + # ---- prediction contracts + sinks ---- + "ClassificationOutput", + "DetectionOutput", + "DetectionPredictions", + "RestorationOutput", + "SegmentationOutput", + "classification_output", + "restoration_output", + "segmentation_output", + "PredictionsSink", + "ClassificationPredictionsSink", + # ---- sources ---- + "HuggingFaceSource", + "DatasetSplit", + "RangeSource", + "ConcatSource", + "SplitName", + # ---- dataset identity ---- + "SupportsDatasetIdentity", + "dataset_uri", + "dataset_uris", + "dataset_url", + # ---- projection ---- + "SupportsProjection", + "first_value", + "iter_key", + "class_names", + "num_classes", + "num_mask_classes", + "project", + # ---- runnable protocol + orchestration ---- + "TorchRunner", + "ProgressReporting", + "ProgressCallback", + "RunnableTask", + "entrypoint", + "entrypoint_tasks", + "run_entrypoint", + "runnable_entrypoints", + "DatasetProcessor", + "Sequence", + "Conditional", + "Switch", + "PathExists", + "Not", + "AllOf", + "AnyOf", +] diff --git a/recordstream/_compat.py b/recordstream/_compat.py new file mode 100644 index 0000000..586fb8d --- /dev/null +++ b/recordstream/_compat.py @@ -0,0 +1,35 @@ +"""Answering "is this value from an optional framework?" without importing that framework. + +recordstream's core is numpy. A handful of places still need to recognise a torch tensor — +storage converting a payload before writing, image conversion detaching one — and doing that +with a module-level ``import torch`` made a 2GB framework a hard dependency of a package whose +own work is arrays. + +The trick is that the question answers itself: **a torch tensor cannot exist unless torch has +already been imported.** So consulting ``sys.modules`` is exact, not a heuristic — if the module +is absent the value is provably not one of its types, and if it is present we do a real +``isinstance`` with no import of our own. + +This is the same instinct as the op-family matchers in :mod:`recordstream.core`, which identify +an albumentations or torchvision transform by its MRO module name rather than importing either. +""" + +import sys +from typing import Any + +__all__ = ["is_torch_tensor"] + + +def is_torch_tensor(value: Any) -> bool: + """True when ``value`` is a ``torch.Tensor``, without importing torch. + + Exact rather than duck-typed: when ``torch`` is already loaded this is a real + ``isinstance`` check; when it is not, no torch tensor can exist in the process, so the + answer is ``False``. + + Example:: + + payload = value.detach().cpu().numpy() if is_torch_tensor(value) else np.asarray(value) + """ + torch = sys.modules.get("torch") + return torch is not None and isinstance(value, torch.Tensor) diff --git a/recordstream/batch.py b/recordstream/batch.py new file mode 100644 index 0000000..2f0628a --- /dev/null +++ b/recordstream/batch.py @@ -0,0 +1,302 @@ +"""Reading a batched record back — the inverse of :func:`~recordstream.collate.collate_records`. + +:mod:`recordstream.collate` writes the batch convention; this module reads it. They are two +halves of ONE piece of knowledge (an item's payload stacks, its declared attrs become +per-record lists, a plain value becomes a plain list), so they live side by side — a consumer +that had to re-derive the read-back would be re-deriving the collate. + +The primitives are deliberately TASK-AGNOSTIC. They answer "what did the collate put under +this key?", never "which of these does my loss want?" — a trainer picks the call and names +the dtype its contract requires: + +* :func:`batch_values` — the raw values, past the wrapper item. Framework-free. +* :func:`multi_hot` — a :class:`~recordstream.MultiLabel` column as an ``[N, C]`` matrix. + Framework-free (numpy). +* :func:`batch_boxes` — a :class:`~recordstream.Boxes` column as per-record + ``{boxes, labels}`` dicts. Framework-free. +* :func:`batch_tensor` — the torch adapter: stack, optional dtype, optional device. +* :func:`batch_metadata` — the collate's transpose, for prediction sinks. Framework-free. + +**Only `batch_tensor` is torch.** Everything else returns plain values or numpy, so a +non-torch backend uses the same code and converts in one line +(``tf.convert_to_tensor(m)`` / ``torch.as_tensor(m)``, the latter sharing memory). ``dtype`` +is a PARAMETER, not an opinion — the same knob as ``device``: recordstream never decides the +contract, it honours the one the caller names. + +Typical use at a model boundary:: + + x = batch_tensor(batch, "image", device=self.device) # [N, 3, H, W] + y = batch_tensor(batch, "class", device=self.device, dtype=torch.int64) # [N] class ids + y = torch.as_tensor(multi_hot(batch, "class", num_classes)).to(self.device) # [N, C] + meta = batch_metadata(batch, exclude=("image", "class")) # per-record dicts for a sink +""" + +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional + +import numpy as np + +from recordstream.items import Boxes, Record, is_item, item_value + +if TYPE_CHECKING: # torch is imported lazily at call time — this is annotation-only + from torch import Tensor + +__all__ = ["batch_boxes", "batch_metadata", "batch_tensor", "batch_values", "multi_hot", "per_record_predictions"] + +#: The per-box PARALLEL ARRAY fields of a :class:`~recordstream.Boxes`, in the order a +#: per-record dict presents them. ``canvas`` and ``extras`` are deliberately absent: the first is +#: per-IMAGE frame metadata and the second an open dict, neither of which is a per-box column — +#: read them off the batched item itself (``batch[key].canvas`` is the per-record list). +_BOX_FIELDS = ("boxes", "labels", "scores") + + +def batch_values(batch: Record, key: str) -> Any: + """The raw batched values under ``key``, unwrapped from their item type. + + Getting *past* a wrapper item is :func:`~recordstream.item_value`'s rule: a + :class:`~recordstream.Label` yields its ``.value`` (the collate leaves it a per-record LIST + — a wrapper item's payload is not stacked), a :class:`~recordstream.MultiLabel` its + ``.values`` (a list OF lists), and anything else its payload (an array item yields its + stacked payload, a plain value the per-record list the collate gathered). What is THIS + function's own is the BATCH reading — that the values arrive already collated. + + No torch, no stacking, no dtype opinion — just the values. Use :func:`batch_tensor` when + a tensor is what you need. + + **Both collates read the same here.** Under ``"record"`` a wrapper item's column arrives as + ONE batched item; under ``"list"`` (:func:`~recordstream.collate_list`) it arrives as a LIST + of per-record items. The second shape is unwrapped ELEMENT-WISE, so a caller gets + ``[0, 1]`` either way and never branches on which collate ran — the property that makes the + collate a free choice rather than a fork in every consumer. + + Example:: + + batch_values(collate_records([{"class": Label(0)}, {"class": Label(1)}]), "class") + # [0, 1] + """ + value = batch[key] + if isinstance(value, list) and any(is_item(entry) for entry in value): + return [item_value(entry) for entry in value] + return item_value(value) + + +def multi_hot(batch: Record, key: str, num_classes: int, dtype: Any = "float32") -> np.ndarray: + """A :class:`~recordstream.MultiLabel` column as an ``[N, num_classes]`` multi-hot matrix. + + The encoding a multi-label target needs: row ``i`` has a 1 in every column that record's + label set contains. It is the natural rendering of :class:`~recordstream.MultiLabel`, so it + belongs beside the item rather than in whichever consumer needed it first. + + **Returns NUMPY, deliberately.** Nothing about counting labels into a matrix is + framework-specific, and numpy is what every framework converts from in one line — + ``torch.as_tensor(m)`` (which shares memory, no copy) or the TensorFlow/JAX equivalent. A + torch-typed return would have forced a second implementation for the next backend. + + Args: + batch: A batched record (the output of :func:`~recordstream.collate_records`). + key: The record key holding the multi-label target. + num_classes: Matrix width. Ids outside ``[0, num_classes)`` are IGNORED rather than + raising — a stray label must not abort a training run (the same rule + ``matrainer.torch.inverse_frequency_weights`` applies to class counting). + dtype: Result dtype, default ``"float32"`` — the multi-label losses + (``BCEWithLogitsLoss`` and friends) want float targets shaped like the logits, not + integer class ids. + + Returns: + An ``[N, num_classes]`` numpy array. A record whose label set is empty yields an + all-zero row, which is a meaningful multi-label target (this record has no classes) and + not an error. + + Example:: + + y = torch.as_tensor(multi_hot(batch, "class", num_classes=3)).to(self.device) + # MultiLabel([0, 2]), MultiLabel([1]) -> [[1, 0, 1], [0, 1, 0]] + """ + value = batch_values(batch, key) + rows = value if isinstance(value, list) else [value] + out = np.zeros((len(rows), int(num_classes)), dtype=dtype) + for row, ids in enumerate(rows): + for class_id in ids if isinstance(ids, (list, tuple, set)) else [ids]: + index = int(class_id) + if 0 <= index < num_classes: + out[row, index] = 1 + return out + + +def batch_boxes(batch: Record, key: str) -> List[Dict[str, Any]]: + """A collated :class:`~recordstream.Boxes` column back into PER-RECORD dicts. + + The collate cannot stack a box set — every record has its own N — so it leaves each + declared attr as a per-record LIST (``boxes`` = ``[[N0, 4], [N1, 4], …]``). That is the + right batch, and it is also not what a model takes: every detection interface in use wants + ONE dict per image. This is that transpose, and it belongs beside :func:`batch_metadata` + (which transposes the same way for the remaining columns) rather than in whichever consumer + needed it first — a consumer re-deriving it is re-deriving the collate. + + **Framework-free, deliberately.** The values are handed back EXACTLY as the record carried + them — torch stays torch, numpy stays numpy — because a detection target's dtype and device + are the caller's contract, not this module's (the same rule that keeps + :func:`batch_values` framework-free and confines torch to :func:`batch_tensor`). A torch + backend moves the dicts to its device in one comprehension; a numpy one uses them as they + are. + + Args: + batch: A batched record — from EITHER collate (``"record"`` leaves one batched + :class:`~recordstream.Boxes` with per-record columns; ``"list"`` leaves a list of + per-record ``Boxes``; both are read here). + key: The record key holding the collated :class:`~recordstream.Boxes`. + + Returns: + One dict per record, carrying whichever of ``boxes`` / ``labels`` / ``scores`` that + record actually has — a field left ``None`` on the item is OMITTED rather than handed + over as ``None``, so a prediction-free training target is exactly ``{boxes, labels}``. + ``canvas`` and ``extras`` stay on the batched item (per-image frame metadata and an + open dict are not per-box columns); read them off ``batch[key]``. + + Raises: + TypeError: when ``key`` does not hold a :class:`~recordstream.Boxes`. + ValueError: when the item is not COLLATED (its ``boxes`` is not a per-record list) — + passing a single record's ``Boxes`` here is the mistake the message names. + + Example:: + + targets = batch_boxes(batch, "target") # [{"boxes": [N0, 4], "labels": [N0]}, …] + targets = [{k: v.to(device) for k, v in t.items()} for t in targets] # a torch caller + """ + item = batch[key] + # The "list" collate leaves a LIST of per-record Boxes; the default leaves ONE batched + # Boxes whose attrs are per-record lists. Both mean the same thing, so both read the same + # — a consumer never branches on which collate ran. + if isinstance(item, list): + if not all(isinstance(entry, Boxes) for entry in item): + raise TypeError(f"batch_boxes: {key!r} holds a list whose entries are not all Boxes.") + return [ + {name: getattr(entry, name) for name in _BOX_FIELDS if getattr(entry, name, None) is not None} + for entry in item + ] + if not isinstance(item, Boxes): + raise TypeError(f"batch_boxes: {key!r} holds {type(item).__name__}, not a Boxes.") + if not isinstance(item.boxes, list): + raise ValueError( + f"batch_boxes: {key!r} is not a COLLATED Boxes — its `boxes` is " + f"{type(item.boxes).__name__}, not the per-record list collate_records leaves. " + "Pass the batched record, not a single record's Boxes." + ) + columns = {name: getattr(item, name) for name in _BOX_FIELDS if isinstance(getattr(item, name, None), list)} + return [ + {name: values[index] for name, values in columns.items() if values[index] is not None} + for index in range(len(item.boxes)) + ] + + +def batch_tensor(batch: Record, key: str, device: Any = None, dtype: Any = None) -> "Tensor": + """The batched values under ``key`` as ONE torch tensor. + + Normalizes the two shapes the collate can leave behind — a stacked array payload, or a + per-record LIST (a wrapper item's payload, or a plain value) — into a single tensor. An + already-stacked tensor is used verbatim; anything else goes through the cheap, + memory-sharing ``torch.as_tensor``. + + This is the TORCH adapter over :func:`batch_values`. A non-torch backend calls + ``batch_values`` (or :func:`multi_hot`) and converts with its own one-liner; nothing here + is duplicated for it. + + Args: + batch: A batched record (the output of :func:`~recordstream.collate_records`). + key: The record key to read. + device: Optional target device. A tensor built HERE from a per-record list is created + on the CPU regardless of a framework's own batch move, so pass the module's device + when the result feeds a model. + dtype: Optional target dtype — a PARAMETER, not an opinion: the caller names the + contract its loss requires and this honours it. Pass ``torch.int64`` for class ids + (``CrossEntropyLoss`` raises *"expected target dtype to be Long or Byte, but got Int"* on an + int32 target, and a dataset yielding int32 label tensors is perfectly legal) or for + a pixel-class mask. ``None`` keeps whatever the values carry. + + Returns: + A ``torch.Tensor``. + + Example:: + + x = batch_tensor(batch, "image", device=self.device) # [N, 3, H, W] + y = batch_tensor(batch, "class", device=self.device, dtype=torch.int64) # [N] class ids + """ + import torch # local: recordstream stays importable without touching torch + + value = batch_values(batch, key) + if isinstance(value, list): + tensor = torch.stack([torch.as_tensor(v) for v in value]) + elif isinstance(value, torch.Tensor): + tensor = value + else: + tensor = torch.as_tensor(np.asarray(value)) + if dtype is not None and tensor.dtype != dtype: + tensor = tensor.to(dtype) + return tensor if device is None else tensor.to(device) + + +def per_record_predictions(preds: Any) -> List[Any]: + """Split a model's BATCHED prediction output into ONE entry per record. + + The sink contract (``write(prediction, metadata)``) is per-record, but a model emits the + whole batch at once — a classifier's ``probs`` ``[N, C]``, a restorer's ``image`` + ``[N, C, H, W]``. This is the pairing half of prediction, the same transpose + :func:`batch_metadata` performs for the metadata columns, which is why it lives beside it: + a consumer re-deriving the slicing re-derives what a batched output IS. It was extracted + from three byte-identical private copies in the training projects (2026-08-06). + + Handled shapes, in order: + + * a **list** — already per-record (a detector's per-image dicts), returned as-is; + * a ``{"predictions": [...]}`` **wrapper** — that list (a detection convention); + * a **batched mapping** whose values agree on one length ``N`` (a + :class:`~recordstream.ClassificationOutput` and friends) — sliced row-wise into ``N`` + mappings; + * anything else — treated as ONE prediction, wrapped in a singleton list. Handing an + unrecognised batch over whole (the pre-extraction bug) wrote ONE entry for an N-record + batch, with the sink reading row 0 as if it were the whole prediction. + """ + if isinstance(preds, list): + return preds + if isinstance(preds, dict): + if "predictions" in preds: + return list(preds["predictions"]) + lengths = {len(v) for v in preds.values() if hasattr(v, "__len__")} + if len(lengths) == 1: + n = lengths.pop() + return [{k: v[i] for k, v in preds.items()} for i in range(n)] + return [preds] + + +def batch_metadata(batch: Record, exclude: Iterable[str] = ()) -> Optional[List[Dict[str, Any]]]: + """Per-record metadata dicts recovered from a batched record — the collate's transpose. + + The collate turns N records into ONE record of per-key columns; this turns those columns + back into N dicts, so a predictions sink can correlate a model's per-record output with + the record it came from. Keys named in ``exclude`` (typically the input and target keys, + which the model already consumed) are left out. + + A hand-built batch that instead carries a single list-valued ``"metadata"`` key is + returned verbatim — that shape is already the answer. + + Args: + batch: A batched record. + exclude: Keys to omit from the per-record dicts. + + Returns: + One dict per record, or ``None`` when nothing remains after ``exclude`` (there is no + metadata to correlate, which a caller reads as "skip"). The list is as long as the + SHORTEST column — a ragged batch truncates rather than raising, so a metadata detail + cannot take down an inference run. + + Example:: + + batch_metadata(batch, exclude=("image", "class")) # [{"idx": 0}, {"idx": 1}] + """ + if "metadata" in batch and isinstance(batch["metadata"], list): + return list(batch["metadata"]) + skip = set(exclude) + columns = {key: batch_values(batch, key) for key in batch if key not in skip} + if not columns: + return None + n = min(len(v) for v in columns.values()) + return [{k: columns[k][i] for k in columns} for i in range(n)] diff --git a/recordstream/cli.py b/recordstream/cli.py new file mode 100644 index 0000000..3b07b56 --- /dev/null +++ b/recordstream/cli.py @@ -0,0 +1,120 @@ +"""The ``recordstream`` CLI — a generic runner for any Confluid-wired runnable. + +``recordstream run `` loads a Confluid YAML that binds a *runnable* +object (anything exposing a no-arg ``run()``) under the top-level ``runnable:`` +key, materializes it against the whole document (so the flat config's top-level +keys broadcast into its constructor), and calls ``run()``. This is the single +entry point that replaces bespoke per-verb CLIs: a training run, an evaluation, a +dataset conversion, or a whole :mod:`~recordstream.workflow` are all just +runnables — the ``!class:`` the YAML roots on decides what happens. + +Example:: + + # convert.yaml + runnable: !class:recordstream.processing.DatasetProcessor + stream: !class:recordstream.Stream { source: !class:my.Source(), ops: [...] } + sink: !class:recordstream.storage.HDF5Sink { path: out.h5 } + + recordstream run convert.yaml +""" + +from typing import Any + +from liquifai import LiquifyApp +from loggair import get_logger + +logger = get_logger(__name__) + +app = LiquifyApp(name="recordstream") + +__all__ = ["app", "main", "materialize_runnable", "run"] + + +def materialize_runnable(runnable: Any) -> Any: + """Build a bound ``runnable:`` node **with the flat config's keys broadcast in**. + + Why this exists instead of a bare ``flow()``: broadcasting (a top-level YAML key + injecting into the same-named constructor parameter) only happens when a Fluid is + built *against the document it came from*. Liquifai's DI does that for a command + parameter annotated with a **configurable class** — it materializes the block with + ``context=``. A generic runner cannot use such an annotation: its + parameter is ``Any`` precisely because the runnable is polymorphic, so DI falls back + to handing over the raw Fluid and deep-flowing it with no context, and every + top-level sibling is silently dropped. + + Silently is the operative word — a dropped ``train_set:`` surfaces later as an empty + dataset, and a dropped ``max_epochs: 3`` does not surface at all: the run proceeds on + the constructor default and looks configured. + + So this reaches back to the loaded document through liquifai's context and calls + ``load(node, context=document)``. Nested stubs still ride the normal deferred + path — a ``!lazy:`` marker stays deferred for the runnable to flow at run time. + + **liquifai 0.1.1 fixes this at its own layer** (``di.deep_flow`` now takes the + document), so once that release is on PyPI and the floors are raised, the runners can + go back to ``flow_mode="auto"`` and this helper can shrink away. It stays until then + because generated CI clones each local-sourced dependency's ``main`` — reverting early + would silently regress to dropping every top-level key. Keeping it is harmless + meanwhile: building against the document is correct under either liquifai. + + Args: + runnable: The value liquifai bound to the command's ``runnable`` parameter — + typically a :class:`~confluid.fluid.Fluid`, but a live object (already built) + passes through untouched. + + Returns: + The built runnable. Falls back to a plain ``flow()`` when there is no liquifai + context or its config is not a mapping (a YAML whose root is a single ``!class:`` + document has no siblings to broadcast, so there is nothing to lose). + + Example:: + + @app.script_command(flow_mode="manual") + def run(runnable: Any) -> None: + runnable = materialize_runnable(runnable) + runnable.run() + """ + from confluid import flow, load + from confluid.fluid import Fluid + from liquifai.context import get_context + + if not isinstance(runnable, Fluid): + return runnable + + context = get_context() + document = getattr(context, "config_data", None) if context is not None else None + if isinstance(document, dict): + return load(runnable, context=document) + return flow(runnable) + + +@app.script_command(flow_mode="manual") +def run(runnable: Any) -> None: + """Run any Confluid-instantiated object that exposes ``.run()``. + + The YAML config binds the object under the top-level ``runnable:`` key. + ``flow_mode="manual"`` because :func:`materialize_runnable` does the building + itself — liquifai's auto deep-flow would bare-flow the node and drop every + broadcast top-level key (see that function's docstring). + """ + if runnable is None: + logger.error("'runnable' was not bound — provide one under 'runnable:' in your YAML.") + return + + runnable = materialize_runnable(runnable) + + label = runnable.__class__.__name__ + run_method = getattr(runnable, "run", None) + if not callable(run_method): + logger.error(f"The injected runnable ({label}) does not implement 'run()'.") + return + logger.info(f"recordstream running: {label}") + run_method() + + +def main() -> None: + app.run() + + +if __name__ == "__main__": + main() diff --git a/recordstream/collate.py b/recordstream/collate.py new file mode 100644 index 0000000..2b403bd --- /dev/null +++ b/recordstream/collate.py @@ -0,0 +1,222 @@ +"""The pluggable collate registry — batch builders keyed by representation. + +Batching in recordstream is two-stage: the engine groups carriers (``Stream.batch`` / +``FlowGraph.batch`` yield ``list``\\ s of N items) and a COLLATE function stacks a group +into one batched carrier. The registry keys serve surfaces that pass JSON-serializable +NAMES rather than function objects (an AI-callable tool, ``RecordSequence``'s ``collate=`` +string form), enumerating the legal values via :func:`registered_collates`; in Python (and +in YAML via a dotted ``!ref:`` to the function), passing a collate function directly is the +normal path. + +TWO collates are registered here, and they differ in ONE decision — whether array payloads +are STACKED — because that decision belongs to the model, not to the data: + +* ``"record"`` (the default) — array payloads stacked per key, per-record item attrs as + lists, plain values gathered into lists. What a classifier or a dense-target detector wants. +* ``"list"`` — nothing stacked; every key becomes a per-record list, items kept as items. + What a model taking variable-size inputs wants (a torchvision detector's ``List[Tensor]``). + +**The registry is engine-internal — only shape-generic, parameter-free collates register +(decided 2026-08-06, closing the open question).** A TASK's batch shape (a fastai +``(x, y)`` tuple, a keras ``(inputs, targets)`` array pair) is parameterized by task-decided +state — which keys are input and target, an int-id vs multi-hot target, a channels-last +transpose — that a bare registry string cannot carry: a registered ``"tuple"`` with baked +default keys would be the silent-wrong-keys trap. Consumers therefore pass their callable +straight to the slot that takes one (``DataLoader(collate_fn=...)``, +``RecordSequence(transform=...)``) and never register it. Measured before deciding: no +workspace consumer had ever registered one. +""" + +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple, TypeVar + +from loggair import get_logger + +from recordstream.io import PLAIN_TYPE, EncodedItem, decode_item, encode_item +from recordstream.items import Record + +logger = get_logger(__name__) + +CollateFn = Callable[[Sequence[Any]], Any] + +#: Bound to `CollateFn` but PRESERVED through the decorator, so registering a collate does +#: not erase its own signature — `collate_records`' `stack=` stays visible to callers and +#: to a type checker (a plain `-> CollateFn` return flattened every registered collate to +#: the loose one-argument protocol). +F = TypeVar("F", bound=CollateFn) + +_REGISTRY: Dict[str, CollateFn] = {} + +__all__ = [ + "CollateFn", + "collate", + "collate_list", + "collate_records", + "get_collate", + "register_collate", + "registered_collates", +] + + +def register_collate(key: str) -> Callable[[F], F]: + """Register a collate function under ``key``. + + For SHAPE-GENERIC, parameter-free collates only (see the module docstring) — a + task-shaped collate carries task state a key cannot, and is passed as a callable to + the slot that takes one instead of being registered. Usable as a decorator:: + + @register_collate("record") + def collate_records(items): ... + + Re-registering a key overwrites it (logged at debug — a deliberate replacement of a + default is legal). + """ + + def _register(fn: F) -> F: + if key in _REGISTRY: + logger.debug(f"collate registry: overwriting existing collate for key {key!r}") + _REGISTRY[key] = fn + return fn + + return _register + + +def get_collate(key: str) -> CollateFn: + """The collate registered under ``key``; a miss names the known keys.""" + try: + return _REGISTRY[key] + except KeyError: + known = ", ".join(sorted(_REGISTRY)) or "" + raise KeyError(f"no collate registered for {key!r} (known: {known})") from None + + +def registered_collates() -> Tuple[str, ...]: + """The registered keys (sorted).""" + return tuple(sorted(_REGISTRY)) + + +def collate(items: Sequence[Any], key: Optional[str] = None) -> Any: + """Collate ``items`` into one batched carrier. + + ``key`` picks a registered collate explicitly; omitted, the default ``"record"`` collate + is used (every carrier is a plain dict). An empty batch raises. + """ + if not items: + raise ValueError("collate: cannot collate an empty batch") + return get_collate(key or "record")(items) + + +def _stack(values: List[Any]) -> Any: + """Best-effort stacking: torch tensors -> stacked tensor, numpy -> stacked array, else a list.""" + first = values[0] + try: + import torch + + if isinstance(first, torch.Tensor): + return torch.stack(list(values)) + except ImportError: # pragma: no cover - torch is a hard dep today, defensive only + pass + try: + import numpy as np + + if isinstance(first, np.ndarray): + return np.stack(list(values)) + if isinstance(first, (int, float)) and all(isinstance(v, (int, float)) for v in values): + return np.asarray(values) + except ImportError: # pragma: no cover - numpy is a hard dep, defensive only + pass + return list(values) + + +@register_collate("record") +def collate_records(items: Sequence[Record], stack: bool = True) -> Record: + """The record collate: N record dicts → ONE batched record dict. + + Per key (union of keys is NOT taken — every record must carry the same keys, a + mismatch raises): typed values encode through :func:`~recordstream.io.encode_item`, + payloads are stacked via :func:`_stack` (torch → stacked tensor, numpy → stacked + array, else a list) and each declared item attr becomes a LIST of per-record values, + decoding back into ONE batched item of the same type. A ``"plain"``-tagged value + (a scalar / string / bare value) batches as the plain LIST of per-record values. + + Args: + items: The records to batch. + stack: Whether array payloads are STACKED into one array/tensor. ``False`` is the + registered ``"list"`` collate (:func:`collate_list`) — see it for why the choice + belongs to the caller rather than to the data. + """ + if not items: + raise ValueError("collate_records: cannot collate an empty batch") + first = items[0] + if not isinstance(first, dict): + raise TypeError(f"collate_records: expected record dicts, got {type(first).__name__}") + keys = list(first.keys()) + for i, record in enumerate(items): + if not isinstance(record, dict) or list(record.keys()) != keys: + raise ValueError( + f"collate_records: item {i} keys {list(record.keys()) if isinstance(record, dict) else '?'} " + f"do not match the batch keys {keys} — collate requires a homogeneous batch." + ) + batched: Record = {} + for key in keys: + if not stack: + # The values VERBATIM, one per record — items stay items, so per-record metadata + # (an `Image`'s layout) survives instead of being flattened into one batched item. + batched[key] = [record[key] for record in items] + continue + encoded = [encode_item(record[key]) for record in items] + type_name = encoded[0].type_name + if type_name == PLAIN_TYPE: + batched[key] = [e.payload for e in encoded] + continue + payloads = [e.payload for e in encoded] + stacked_payload = _stack_or_explain(payloads, key) if encoded[0].payload is not None else None + batched_attrs = {name: [e.attrs.get(name) for e in encoded] for name in encoded[0].attrs} + batched[key] = decode_item(EncodedItem(type_name=type_name, payload=stacked_payload, attrs=batched_attrs)) + return batched + + +@register_collate("list") +def collate_list(items: Sequence[Record]) -> Record: + """The UNSTACKED collate: N record dicts → one record whose every key is a per-record LIST. + + The sibling of :func:`collate_records`, and the reason the registry exists: **the batch + SHAPE is a model requirement, not a property of the data.** A torchvision detector takes + ``List[Tensor]`` because its images differ in size; a classifier takes one ``[N, C, H, W]`` + tensor. Left implicit, that choice is made by accident — under the default collate a column + stacks if it holds array ITEMS and stays a list if it holds PLAIN values, so the shape ends + up decided by whether an op like ``ToTensor`` happened to run. This key lets the consumer + DECLARE it instead. + + Two consequences worth knowing: + + * a variable-size column is fine here (nothing is stacked), where the default collate + raises — and the type does not depend on the data, because you asked for lists; + * items stay ITEMS (a list of :class:`~recordstream.Image`, not a list of bare arrays), so + per-record metadata survives. The read-back helpers (:func:`~recordstream.batch_values`, + :func:`~recordstream.batch_boxes`, :func:`~recordstream.batch_metadata`) accept BOTH + shapes, so a consumer reads the batch the same way under either collate. + + Example:: + + DataLoader(stream, collate_fn=collate_list) # or: collate(items, key="list") + """ + return collate_records(items, stack=False) + + +def _stack_or_explain(payloads: List[Any], key: str) -> Any: + """:func:`_stack`, but a stacking failure names the KEY, the shapes and the way out. + + Raw, the failure is ``ValueError: all input arrays must have the same shape`` from inside + numpy — which says nothing about which column, what shapes, or what to do. A variable-size + column is a legitimate batch; it just is not a STACKED one. + """ + try: + return _stack(payloads) + except (ValueError, RuntimeError) as exc: + shapes = [getattr(p, "shape", None) for p in payloads] + raise ValueError( + f"collate_records: cannot stack the {key!r} column — its payloads have differing " + f"shapes {shapes}. Either make them uniform upstream (a resize op), or batch with " + f'the "list" collate (`collate_list` / `collate(items, key="list")`), which keeps ' + f"every column as a per-record list." + ) from exc diff --git a/recordstream/core/__init__.py b/recordstream/core/__init__.py new file mode 100644 index 0000000..bc39387 --- /dev/null +++ b/recordstream/core/__init__.py @@ -0,0 +1,86 @@ +""" +RecordStream's engine core — the ``Stream`` facade and the machinery it runs on. + +One module per cohesive unit, re-exported here so ``from recordstream.core import Stream`` +is unchanged; the canonical dotted path a config / form-spec / MCP schema spells out is the +SUBMODULE one (``!class:recordstream.core.stream.Stream``), because that is what +``cls.__module__`` says. Both resolve — ``confluid.resolve_class`` falls back to a +module-path import, and this package re-exports every name — but a GENERATED config uses the +submodule spelling. + +Submodules, bottom of the layer first (imports run strictly one way): + + - recordstream.core.families: the op-family registry + ``_apply_op``, the single + op-application chokepoint every composing op routes through, plus the ``EXPANDS`` + protocol. Imports nothing from the rest of core. + - recordstream.core.mapstyle: the ``MapStyle`` Protocol + the ``RecordSource`` union — + "a dataset", said structurally so the engine never imports a framework. + - recordstream.core.wrappers: ``FilterOp`` / ``WrappedOp``, the targets of ``Stream``'s + fluent ``.filter()`` / ``.map()`` (``docs/architecture.md`` §5). + - recordstream.core.stream: ``Stream`` + ``JointStream``, the ops-list plumbing + (``linear_steps`` / ``_worker_task``) and ``ensure_record_dataset``. + +``__all__`` below is load-bearing, not decoration: a visual editor's node bridge scans this +module in two passes, and the first (``recordstream.discovery.scan_module``) filters on +``member.__module__``, so it sees NOTHING here now that the classes live in submodules. The +second pass — the one that surfaces ``Stream`` / ``JointStream`` as engine nodes — walks +exactly this ``__all__``. +""" + +from recordstream.core.families import ( # noqa: F401 — see the internal-surface note below + _ALB_KEYS, + _OP_FAMILIES, + OpInvoker, + OpMatcher, + _apply_op, + _expand, + _extra_op_families, + _is_albumentations, + _is_torchvision_v2, + _op_expands, + _sync_op_families, + register_op_family, + registered_op_families, +) +from recordstream.core.mapstyle import MapStyle, RecordSource +from recordstream.core.stream import ( # noqa: F401 — see the internal-surface note below + JointStream, + Stream, + _check_ops_materialized, + _fluid_source_guidance, + _worker_task, + ensure_materialized, + ensure_record_dataset, + linear_steps, + prepare_record_dataset, +) +from recordstream.core.wrappers import FilterOp, WrappedOp + +__all__ = [ + "FilterOp", + "JointStream", + "MapStyle", + "OpInvoker", + "OpMatcher", + "RecordSource", + "Stream", + "WrappedOp", + "ensure_materialized", + "ensure_record_dataset", + "linear_steps", + "prepare_record_dataset", + "register_op_family", + "registered_op_families", +] + +# The private names re-exported above are the engine's INTERNAL cross-module surface: the op +# dispatch `_apply_op` that every composing op in `recordstream.ops` imports, the spawn-worker +# helpers, and the registry list / family predicates the suite snapshots. They stay OUT of +# `__all__` (a leading underscore already keeps them off a visual editor's palette), but +# `from recordstream.core import _apply_op` must keep working — so they are re-exported +# deliberately, with the `noqa` marking that as intent rather than a stray unused import. +# +# NOTE for test doubles: these are BOUND NAMES, not views of the defining module. Patching +# `recordstream.core._apply_op` does NOT affect the copy `flow.execute` already imported — +# patch the module that USES it. The one exception is `_OP_FAMILIES`, a mutable list whose +# identity is shared, so `core._OP_FAMILIES[:] = snapshot` still restores the real registry. diff --git a/recordstream/core/families.py b/recordstream/core/families.py new file mode 100644 index 0000000..07959ed --- /dev/null +++ b/recordstream/core/families.py @@ -0,0 +1,336 @@ +"""Applying ONE op to ONE record — the op-family registry and the dispatch chokepoint. + +The bottom of the op-facing layer: every composing op in :mod:`recordstream.ops` reaches +:func:`_apply_op` from here, and nothing here imports back out (see ``docs/architecture.md`` +§5 on the import direction this protects). Also home to the ``EXPANDS`` protocol +(:func:`_op_expands` / :func:`_expand`), because "does this op expand, and how do I run it?" +is the same question as "how do I apply this op" — the graph kernel asks all three together. +""" + +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, cast + +from loggair import get_logger + +from recordstream.items import Boxes, NDArrayItem, Record, with_data + +logger = get_logger(__name__) + + +def _op_expands(op: Any) -> bool: + """True when an op is a 1→N expanding op (explicit ``EXPANDS = True`` class attribute).""" + return bool(getattr(op, "EXPANDS", False)) + + +def _expand(op: Any, record: Any) -> List[Any]: + """Run a 1→N EXPANDING op and return its flattened children.""" + raw = op(record) + if raw is None: + return [] + return [child for child in raw if child is not None] + + +#: An op-family MATCHER recognises a library's op objects. Keep it IMPORT-FREE — inspect +#: ``type(op).__mro__`` module names rather than importing the library. +OpMatcher = Callable[[Any], bool] +#: An op-family INVOKER applies one foreign op with its library's native calling +#: convention: ``(record, op) -> Optional[Record]`` (``None`` = drop the record). +OpInvoker = Callable[[Record, Any], Optional[Record]] + +#: The registered op families, in registration order. Dispatch checks LAST-registered +#: first, so a later (more specific) family can shadow an earlier one. +_OP_FAMILIES: List[Tuple[str, OpMatcher, OpInvoker]] = [] + + +def register_op_family(name: str, matcher: OpMatcher, invoker: OpInvoker) -> None: + """Teach the engine to invoke a NEW library's ops natively — the open extension point. + + ``matcher(op) -> bool`` recognises the family's op objects (keep it import-free — + inspect ``type(op).__mro__`` module names); ``invoker(record, op)`` applies one op with + the library's own calling convention and returns the new record (``None`` drops it). + Re-registering a ``name`` REPLACES that family in place; otherwise the family is + appended, and dispatch checks last-registered first (a more specific family shadows an + earlier one — register yours after the built-ins to win an overlap). + + Both callables MUST be module-level functions (picklable by reference): the engine's + spawn-parallel routes ship non-builtin families to worker processes by pickling them. + + Example — kornia augmentations (``nn.Module``s over batched BCHW tensors):: + + def is_kornia(op) -> bool: + return any(c.__module__.startswith("kornia.augmentation") for c in type(op).__mro__) + + def invoke_kornia(record, op): + img = record["image"] # a CHW torch.Tensor (e.g. after ToTensor) + out = op(img.unsqueeze(0)).squeeze(0) # kornia draws once per batch call + return {**record, "image": out} + + register_op_family("kornia", is_kornia, invoke_kornia) + """ + entry = (str(name), matcher, invoker) + for i, (existing, _, _) in enumerate(_OP_FAMILIES): + if existing == name: + _OP_FAMILIES[i] = entry + return + _OP_FAMILIES.append(entry) + + +def registered_op_families() -> Tuple[str, ...]: + """The registered op-family names, in registration/dispatch-precedence order.""" + return tuple(name for name, _, _ in _OP_FAMILIES) + + +def _sync_op_families(families: Optional[List[Tuple[str, OpMatcher, OpInvoker]]]) -> None: + """Merge families shipped from the parent process into this process's registry. + + Spawn workers import this module (built-ins present) but never re-run the user's + registration side effects — the parallel routes therefore pass the parent's + non-builtin entries along and merge them here (idempotent by name). + """ + for name, matcher, invoker in families or []: + register_op_family(name, matcher, invoker) + + +def _extra_op_families() -> List[Tuple[str, OpMatcher, OpInvoker]]: + """The non-builtin registry entries — what a spawn worker cannot rebuild by import alone.""" + return [entry for entry in _OP_FAMILIES if entry[0] not in _BUILTIN_FAMILIES] + + +#: The record keys albumentations understands — its OWN target vocabulary. An albumentations +#: op receives exactly these keys (the ones present) and nothing else, so extra record +#: entries (scalars, domain items) never reach a library that would reject them. +_ALB_KEYS: Tuple[str, ...] = ("image", "mask", "masks", "bboxes", "keypoints", "labels") + + +def _is_albumentations(op: Any) -> bool: + """True for an albumentations transform / ``Compose`` — by MRO module name (no import here).""" + return any(getattr(cls, "__module__", "").startswith("albumentations") for cls in type(op).__mro__) + + +#: Whether :func:`_disable_cv2_threading` has already run. Module-level, so the cost of the +#: guarantee is one bool check per op application. +_CV2_THREADING_DISABLED = False + + +def _disable_cv2_threading() -> None: + """Turn OpenCV's internal thread pool off, ONCE, the first time albumentations is used. + + **This prevents a SIGSEGV, not a slowdown.** albumentations runs on OpenCV, whose thread pool + is not fork-safe: once a parent process has executed a cv2 op, a FORKED child inheriting that + pool crashes with *"DataLoader worker ... is killed by signal: Segmentation fault: 11"* — no + Python traceback, because the child never gets to raise. Measured on macOS, 2026-08-02, with a + ``DataLoader(num_workers=2)`` over a ``Stream`` whose ops list contained ``A.Resize``: it + segfaults reliably with the pool on and passes reliably with it off. + + It is the SECOND fork hazard on this path and is independent of the first + (:func:`~recordstream.ensure_materialized`, which warms a lazy source so the child does not + run a download through the non-fork-safe ``_scproxy``). Neither fixes the other: with the + source warmed and the pool ON the worker still dies, and with the pool off a cold source is + still built in the child. A consumer that forks needs both. + + Turning the pool off costs nothing where it matters. Inside a ``DataLoader`` worker the + WORKER is the parallelism — cv2's own threads oversubscribe the machine rather than help — + which is why albumentations' own documentation recommends exactly this for multiprocessing + loaders. + + Done HERE, at the one place this package invokes albumentations, rather than at import: a + process that never uses albumentations must not have its OpenCV settings changed by importing + a data library, and cv2 must not become an import-time dependency of the engine. + """ + global _CV2_THREADING_DISABLED + if _CV2_THREADING_DISABLED: + return + _CV2_THREADING_DISABLED = True # set FIRST: a cv2-less install must not retry on every record + try: + import cv2 + + cv2.setNumThreads(0) + except Exception as exc: # pragma: no cover - albumentations without cv2 is not a real install + logger.debug(f"could not disable OpenCV threading ({exc}); a forked DataLoader worker may crash.") + + +#: Transform classes already warned about (see :func:`_warn_if_boxes_are_left_behind`). Keyed +#: by CLASS, not instance: the message describes a configuration pattern, and two `Resize`s in +#: one chain have the same thing wrong with them. +_WARNED_SPATIAL: Set[type] = set() + + +def _has_spatial_transform(op: Any, depth: int = 0) -> bool: + """True when ``op`` contains a transform that would MOVE boxes, per albumentations' own taxonomy. + + The library already draws this line: a ``DualTransform`` is defined as one that applies to + boxes and masks as well as the image (``Resize``, ``HorizontalFlip``, ``RandomCrop``), while + an ``ImageOnlyTransform`` cannot touch geometry (``Normalize``, ``ColorJitter``). Reading THAT + distinction is why this needs no list of transform names to drift out of date, and no guess + about what a given transform does. + + Matched by MRO class NAME rather than `isinstance`, for the same reason + :func:`_is_albumentations` matches by module name: recognising a library must never import + one. A ``Compose`` is recursed through its ``transforms``, depth-capped against a cycle. + """ + if any(cls.__name__ == "DualTransform" for cls in type(op).__mro__): + return True + if depth >= 4: + return False + children = getattr(op, "transforms", None) + if not children: + return False + return any(_has_spatial_transform(child, depth + 1) for child in children) + + +def _warn_if_boxes_are_left_behind(record: Record, op: Any, passed: Dict[str, Any]) -> None: + """Warn once when a geometry-changing transform ran while a ``Boxes`` sat out the call. + + This family passes the op EXACTLY the keys of albumentations' own vocabulary, which is what + lets a bare library transform work unmodified — but a detection target rides as a ``Boxes`` + item under a key of the pipeline's choosing, so it is not in that vocabulary and does not get + passed. Measured: a bare ``A.Resize`` moves a 200x200 image to 64x64 and leaves the boxes on + ``[10, 10, 100, 100]``; a bare ``A.HorizontalFlip`` mirrors the pixels and leaves the boxes + where they were WITHOUT changing the raster at all — which is why the condition here is the + library's spatial/photometric taxonomy and not "did the image size change". + + Nothing errors either way: the shapes stay valid and only the coordinates become wrong, so a + model trains against misplaced targets and reports nothing. It stays a WARNING rather than an + error because a record may legitimately carry boxes describing something other than the + image being augmented — this family cannot know, and refusing the call would break a pipeline + that is right. + + The fix is to speak the library's vocabulary: put boxes under ``bboxes`` with their + ``labels`` and declare ``bbox_params`` on the ``Compose``, and the library moves them in the + same joint draw. For a plain deterministic resize, ``ops.target.ResizeDetection`` does the + coupled step over the ``Boxes`` form directly. + """ + if "bboxes" in passed or type(op) in _WARNED_SPATIAL: + return + keys = [key for key, value in record.items() if isinstance(value, Boxes)] + if not keys or not _has_spatial_transform(op): + return + _WARNED_SPATIAL.add(type(op)) + logger.warning( + f"albumentations {type(op).__name__} changes GEOMETRY, but this record's detection " + f"boxes ({keys}) ride as a Boxes item, which is not in the library's key vocabulary " + f"({', '.join(_ALB_KEYS)}) — so the pixels moved and the boxes did not. Put boxes under " + f"'bboxes' + 'labels' with A.Compose(..., bbox_params=A.BboxParams(...)) so the library " + f"moves them in the same draw, or use recordstream.ops.target.ResizeDetection for a " + f"plain coupled resize. (Warned once per transform type.)" + ) + + +def _invoke_albumentations(record: Record, op: Any) -> Optional[Record]: + """albumentations dispatches by KWARG NAME: hand the op exactly its own target keys + present in the record (one call = one joint draw across them); array outputs are + re-wrapped in the incoming value's item type (``with_data``) so ``Image``/``Mask`` + keep their type and metadata. Box-carrying augmentation belongs in albumentations' own + ``A.Compose(..., bbox_params=...)`` — format handling is Compose's job in that library. + """ + _disable_cv2_threading() + kwargs = {k: record[k] for k in _ALB_KEYS if k in record} + if not kwargs: + logger.debug( + f"albumentations op {type(op).__name__} received no known keys " + f"({', '.join(_ALB_KEYS)}) — record keys: {list(record)}; passing through." + ) + return record + _warn_if_boxes_are_left_behind(record, op, kwargs) + out = op(**kwargs) + merged = dict(record) + for key, value in out.items(): + original = record.get(key) + if isinstance(original, NDArrayItem) and not isinstance(value, NDArrayItem): + value = with_data(original, value) + merged[key] = value + return merged + + +def _is_torchvision_v2(op: Any) -> bool: + """True for a torchvision ``transforms.v2`` transform — by MRO module name (no import here).""" + return any(getattr(cls, "__module__", "").startswith("torchvision.transforms.v2") for cls in type(op).__mro__) + + +#: torchvision v2's geometric transforms all live in ONE private module — the closest thing the +#: library has to albumentations' `DualTransform` marker. Private, so this can go stale across a +#: torchvision release; it FAILS OPEN (no warning, nothing else changes), which is the right +#: direction for a diagnostic. +_V2_GEOMETRY_MODULE = "torchvision.transforms.v2._geometry" + + +def _is_v2_geometry(op: Any, depth: int = 0) -> bool: + """True when a v2 transform (or one nested in a ``Compose``) changes GEOMETRY. + + The behavioural test — apply it to a throwaway ``BoundingBoxes`` and see whether they move — + would be authoritative and is deliberately NOT used: running a transform speculatively draws + from the RNG, which would change the augmentation stream of the run being diagnosed. A + diagnostic must not alter what it observes. + """ + if any(getattr(cls, "__module__", "") == _V2_GEOMETRY_MODULE for cls in type(op).__mro__): + return True + if depth >= 4: + return False + children = getattr(op, "transforms", None) + if not children: + return False + return any(_is_v2_geometry(child, depth + 1) for child in children) + + +def _invoke_torchvision_v2(record: Record, op: Any) -> Optional[Record]: + """torchvision v2 natively walks a dict: params sampled once, tensor/tv_tensor/PIL + leaves transformed, everything else passed through — called as-is. + + "Everything else passed through" is where detection boxes fall: v2 recognises its OWN + ``tv_tensors`` types, and a :class:`~recordstream.Boxes` is not one, so a geometric + transform moves the pixels and leaves the boxes — the same silent desync the albumentations + family has, reached by a different route (there the boxes are not in the key vocabulary; here + they are not in the TYPE vocabulary). Measured: ``v2.Resize((64, 64))`` takes a 200x200 image + to 64x64 with the boxes still on ``[10, 10, 100, 100]``, while the same transform over a + ``tv_tensors.BoundingBoxes`` correctly rescales them to ``[3.2, 3.2, 32, 32]``. + """ + _warn_if_v2_leaves_boxes_behind(record, op) + return cast(Record, op(record)) + + +def _warn_if_v2_leaves_boxes_behind(record: Record, op: Any) -> None: + """The v2 twin of :func:`_warn_if_boxes_are_left_behind` — once per transform type.""" + if type(op) in _WARNED_SPATIAL: + return + keys = [key for key, value in record.items() if isinstance(value, Boxes)] + if not keys or not _is_v2_geometry(op): + return + _WARNED_SPATIAL.add(type(op)) + logger.warning( + f"torchvision v2 {type(op).__name__} changes GEOMETRY, but this record's detection boxes " + f"({keys}) ride as a Boxes item, which is not one of v2's tv_tensors types — so v2 " + f"passes them through untouched while the pixels move. Carry boxes as " + f"torchvision.tv_tensors.BoundingBoxes(..., format=…, canvas_size=…) so v2 transforms " + f"them in the same call, or use recordstream.ops.target.ResizeDetection for a plain " + f"coupled resize. (Warned once per transform type.)" + ) + + +# The built-in families register through the SAME open registry third parties use — +# one mechanism, no privileged code path. Registered at import, so spawn workers +# rebuild them by importing this module. +register_op_family("albumentations", _is_albumentations, _invoke_albumentations) +register_op_family("torchvision_v2", _is_torchvision_v2, _invoke_torchvision_v2) +_BUILTIN_FAMILIES: Tuple[str, ...] = ("albumentations", "torchvision_v2") + + +def _apply_op(record: Record, op: Any) -> Optional[Record]: + """Apply one op to the record dict — the engine's op-FAMILY dispatch. + + The single op-application chokepoint shared by the sequential, parallel (via + :func:`~recordstream.core.stream._worker_task`), streamed, and random-access + (``__getitem__``) paths; composing ops (``Pipeline`` / ``Parallel`` / ``Enable`` / + ``RandomApply`` / ``ConfigureOp``) route their inner ops through here so every op is + applied identically. Each op family is invoked the way its library expects — no + wrapper/adapter classes: the registered families (:func:`register_op_family`; built-ins + ``albumentations`` / ``torchvision_v2``) are checked LAST-registered first, and an op + matching none of them is a native/wiring op called ``op(record) -> Optional[Record]`` + (``None`` drops the record — filter semantics). + """ + for _name, matcher, invoker in reversed(_OP_FAMILIES): + try: + matched = matcher(op) + except Exception: # pragma: no cover - a defensive matcher never breaks dispatch + matched = False + if matched: + return invoker(record, op) + return cast(Optional[Record], op(record)) diff --git a/recordstream/core/mapstyle.py b/recordstream/core/mapstyle.py new file mode 100644 index 0000000..419cf6b --- /dev/null +++ b/recordstream/core/mapstyle.py @@ -0,0 +1,41 @@ +"""What recordstream MEANS by "a dataset", said structurally — no framework import. + +Pure types: the :class:`MapStyle` Protocol and the :data:`RecordSource` union built on it. +Deliberately depends on nothing but ``typing`` + the record alias, so every other core +module can import it without ordering constraints. The function that NORMALIZES a wired slot +into one of these (``ensure_record_dataset``) lives beside ``Stream`` instead, because +building a ``Stream`` is its whole body. +""" + +from typing import Any, Iterable, Protocol, Union, runtime_checkable + +from recordstream.items import Record + + +@runtime_checkable +class MapStyle(Protocol): + """A map-style dataset: ``len(ds)`` and ``ds[i]``. + + What recordstream MEANS by "a dataset", said structurally so the engine never imports a + framework to express it. ``Stream`` used to inherit ``torch.utils.data.Dataset``, which made + torch a hard dependency of a package whose own work is numpy — for nothing: that base is not + load-bearing. ``DataLoader`` duck-types its argument (a plain object with these two methods + works), nothing in the workspace does ``isinstance(x, Dataset)``, and the annotation is the + only thing the inheritance ever bought. + """ + + def __len__(self) -> int: ... + + def __getitem__(self, index: int) -> Any: ... + + +#: What a wired dataset slot may hold — the contract ``ensure_record_dataset`` enforces, +#: named ONCE here rather than restated by every consumer: anything MAP-STYLE (``__len__`` + +#: ``__getitem__`` — which a ``Stream`` is), or any iterable of records (a recordstream +#: source, a plain list of record dicts). Consumers annotate their slots +#: ``Optional[Partial[RecordSource]]`` — ``Partial`` because they flow the slot at run time. +#: +#: Expressed with the structural :class:`MapStyle` rather than ``torch.utils.data.Dataset`` so the +#: engine can say "a dataset" without importing a framework; torch's ``DataLoader`` is itself +#: duck-typed and consumes either. +RecordSource = Union[MapStyle, Iterable[Record]] diff --git a/recordstream/core/stream.py b/recordstream/core/stream.py new file mode 100644 index 0000000..fa0c549 --- /dev/null +++ b/recordstream/core/stream.py @@ -0,0 +1,533 @@ +"""The ``Stream`` engine, its fan-in sibling ``JointStream``, and the ops-list plumbing. + +``Stream`` is the dataset-surface facade over the one step-graph kernel: an ``ops:`` list is +compiled to POSITIONAL steps by :func:`linear_steps` and run by +:mod:`recordstream.flow`, exactly as a ``flow:`` document's author-named steps are. The two +spellings are the same thing (``docs/architecture.md`` §3). + +What else lives here and why: + +* ``JointStream`` — 20 lines that exist to be ``Stream.joint``'s return value (§5). +* :func:`linear_steps` / :func:`_worker_task` — both compile or run an ops LIST, which is + ``Stream``'s spelling of a pipeline. +* :func:`ensure_record_dataset` — its whole body is "already a Stream? else wrap in one". +* the deferred-source guidance helpers — they phrase ``Stream``'s own Fluid errors. + +The imports of :mod:`recordstream.flow` are body-local ON PURPOSE: flow imports the op +dispatch from :mod:`recordstream.core.families` at module level, so a top-level import back +would close the cycle. +""" + +import concurrent.futures +import multiprocessing +from contextlib import nullcontext +from typing import Any, Callable, Collection, Iterable, Iterator, List, Optional, Sequence, Tuple, Union, cast + +from confluid import configurable +from confluid import load as _confluid_load +from confluid.fluid import Fluid as _ConfluidFluid +from loggair import get_logger + +from recordstream.core.families import OpInvoker, OpMatcher, _extra_op_families, _op_expands, _sync_op_families +from recordstream.core.mapstyle import RecordSource +from recordstream.core.wrappers import FilterOp, WrappedOp +from recordstream.items import Record + +logger = get_logger(__name__) + + +def _describe_deferred_source(source: Any) -> str: + """Return a human-friendly description of a still-deferred Confluid source. + + Surfaces the tag/target so the error explains WHAT was deferred instead + of just noting it isn't a live object. + """ + target = getattr(source, "target", "") + target_name = target if isinstance(target, str) else getattr(target, "__qualname__", str(target)) + return f"{type(source).__name__}(target={target_name!r})" + + +def _fluid_source_guidance(source: Any, slot: str = "Stream.source") -> str: + """Build an actionable message when a source slot is still a Confluid Fluid. + + ``slot`` names the owning slot (``"Stream.source"``, ``"RangeSource.source"``, …) so the + view sources raise the SAME guidance Stream does — a still-deferred ``source:`` is a + CONFIG error, never something the engine flows. + + The trigger CHANGED with confluid's marker merge (2026-08-11): a parens-less + ``!class:X`` used to leave a deferred stub here, and this message used to say "add + parens". Both spellings build now, so the only way to reach this is asking for + deferral explicitly — ``_partial_: true`` (``!lazy:``) — or wiring a hand-built + ``PartialClass(...)``. Telling the user to add parens would now be advice that + changes nothing. + """ + return ( + f"{slot} is still a deferred Confluid marker: {_describe_deferred_source(source)}. " + "Confluid was told NOT to build it. Fixes: (a) in YAML, drop `_partial_: true` from " + "the source so it is built at load time — a source slot needs a live object, and " + "nothing here will flow it for you; (b) or call `flow(source)` yourself before " + "wiring it in." + ) + + +def _fluid_op_guidance(op: Any, index: int) -> str: + """Build an actionable message when a Stream op marker cannot be materialized.""" + return ( + f"Stream.ops[{index}] is a deferred Confluid marker that could not be materialized: " + f"{_describe_deferred_source(op)}. Fixes: (a) in YAML, drop `_partial_: true` from " + "the op so it is built at load time; (b) or call `flow(op)` on the op before handing " + "it to Stream." + ) + + +def _check_ops_materialized(ops: List[Any]) -> None: + """Flow any still-deferred Confluid op markers IN PLACE at engine-route entry. + + The same lazy-flow convention the composing ops (``Pipeline`` / ``Enable`` / + ``RandomApply``) use — so a YAML ops doc may list bare ``!class:`` mapping-form + entries (e.g. a bare albumentations transform) directly under ``ops:``. The in-place + write is the cache: later routes (and the spawn pickler) see live ops. A marker that + cannot build raises ONE actionable error naming the offending index. + """ + from confluid import flow + + for i, op in enumerate(ops): + if isinstance(op, _ConfluidFluid): + try: + ops[i] = flow(op) + except Exception as exc: + raise TypeError(_fluid_op_guidance(op, i)) from exc + + +def linear_steps(ops: Sequence[Any]) -> Tuple[List[Any], str]: + """Compile a flat op list into the linear step graph the engine executes. + + A sequence IS a graph — every step reads the previous one — so an ``ops:`` list needs no + lifting to run on the graph kernel, just names. The names are positional (``s0``, ``s1``, + …) and never surface: nothing in an ``ops:`` document can reference a step, so they exist + only to key the step environment. Positional (not op-class) naming is deliberate — the + same op twice in a row is two distinct steps, which a name-keyed mapping would collapse. + + Returns ``(steps, output_step)``; an empty list yields ``([], "")``, the identity graph. + """ + from recordstream.flow import FlowStep + + steps = [FlowStep(name=f"s{i}", op=op, from_=None, bind={}, merge_from=()) for i, op in enumerate(ops)] + return cast(List[Any], steps), (steps[-1].name if steps else "") + + +def _worker_task( + record: Any, ops: List[Any], families: Optional[List[Tuple[str, OpMatcher, OpInvoker]]] = None +) -> Optional[Any]: + """Single-result worker for STRICTLY 1→1 op lists (the ``Parallel`` op's contract). + + Kept for callers that need exactly one carrier back; expanding ops raise here — + route expanding pipelines through the iterating engine. + """ + from recordstream.flow import run_steps + + _sync_op_families(families) + steps, outputs = linear_steps(ops) + return run_steps(record, steps, outputs) + + +@configurable(category="engine") +class JointStream: + """ + Aggregates multiple Stream streams into a single joint stream. + Each sub-stream maintains its own unique transformation chain. + + The iteration-only fan-in engine behind :meth:`Stream.joint`: each sub-stream applies + its OWN op chain, so differently-processed streams concatenate lazily without + materialization. For an indexable (random-access) concatenation of raw sources, + use ``ConcatSource`` instead. + + Args: + streams: The Stream streams to concatenate; iteration walks them in order and length is their sum. + Defaults to ``None`` ⇒ an empty joint stream (zero-arg construction). + """ + + def __init__(self, streams: Optional[List["Stream"]] = None) -> None: + # Partial / zero-arg: store config only; no sub-streams ⇒ an empty stream. + self.streams = streams if streams is not None else [] + + def __iter__(self) -> Iterator[Record]: + """Iterate through all sub-streams sequentially.""" + for stream in self.streams: + yield from stream + + def __len__(self) -> int: + """Total length is the sum of all sub-streams.""" + return sum(len(f) for f in self.streams) + + +@configurable(category="engine") +class Stream: + """ + The primary stream engine for RecordStream. + Wraps any iterable or indexed dataset and provides a functional API. + + Every carrier is a plain record ``dict`` of typed values, and every op is applied + through the op-FAMILY dispatch (:func:`~recordstream.core.families._apply_op`) — so + native recordstream ops, bare albumentations transforms, and bare torchvision + ``transforms.v2`` transforms all sit in ONE ``ops`` list as-is. ``source`` is + duck-typed (any iterable; the Indexable protocol if ``__getitem__``/``__len__`` are + present). + + Args: + source: Any iterable or indexable dataset (duck-typed) yielding record dicts; ``None`` = empty stream. + ops: Ordered ops applied lazily on access — native ops and bare library transforms alike (``None`` = no ops). + chunk_size: Parallel-processing chunk size; ``0`` (the default) processes sequentially. + class_names: Optional ordered class vocabulary this stream's labels index into. Set by + :meth:`~recordstream.LabelMap.encode` so the vocabulary travels WITH the encoded + data — a consumer that needs to name a predicted class id, or persist the mapping + beside a checkpoint, reads it via :func:`~recordstream.class_names` instead of + being handed a separate LabelMap it has to keep in sync. + """ + + def __init__( + self, + source: Optional[Iterable[Any]] = None, + ops: Optional[List[Any]] = None, + chunk_size: Optional[int] = 0, + class_names: Optional[List[str]] = None, + ) -> None: + self.source = source + self.ops: List[Any] = ops or [] + self.class_names: Optional[List[str]] = class_names + self._workers = 1 + self._chunk_size = chunk_size or 0 + # Populated on first random access when the source is iterable-only + # (has ``__len__`` but not ``__getitem__``). + self._indexable_cache: Optional[List[Any]] = None + + def _guard_live_source(self) -> Any: + """Return the source, surfacing a clear error when it's still a Fluid marker.""" + if isinstance(self.source, _ConfluidFluid): + raise TypeError(_fluid_source_guidance(self.source)) + return self.source + + @classmethod + def from_source(cls, source: Any) -> "Stream": + """Create a Stream from a DataSource.""" + return cls(source=source) + + @classmethod + def joint(cls, streams: List["Stream"]) -> "Stream": + """Create a new Stream that aggregates multiple other Stream streams.""" + return cls(source=JointStream(streams)) + + @classmethod + def from_ops_yaml(cls, path: str, source: Optional[Iterable[Any]] = None) -> "Stream": + """Attach an ops-only Confluid YAML (e.g. one exported by a pipeline-authoring tool) to ``source``. + + ``path`` is the ``{ops: [!class:...()]}`` document produced by an external graph + exporter's ops-export. Op markers are materialized to live callables before being + attached (``confluid.load`` leaves ``!class:`` markers nested under a mapping key + deferred, so ``confluid.load`` flows them into live ops). + """ + loaded = _confluid_load(path) + raw_ops = loaded.get("ops", []) if isinstance(loaded, dict) else [] + ops = list(_confluid_load(raw_ops)) + return cls(source=source, ops=ops) + + @property + def _expands(self) -> bool: + """True when any (materialized) op is a 1→N expanding op — the pipeline is then iterable-only.""" + return any(not isinstance(op, _ConfluidFluid) and _op_expands(op) for op in self.ops) + + def _guard_not_expanding(self, operation: str) -> None: + if self._expands: + culprit = next( + type(op).__name__ for op in self.ops if not isinstance(op, _ConfluidFluid) and _op_expands(op) + ) + raise TypeError( + f"Stream.{operation}: the pipeline contains the 1→N expanding op {culprit!r}, so the " + "expanded length/index mapping is unknowable up front — the pipeline is ITERABLE-ONLY. " + "Iterate it (or wrap in a torch IterableDataset); for random access, window/expand at " + "the source instead, or materialize with list(stream) first." + ) + + def __len__(self) -> int: + """Return the length of the underlying source if available.""" + from collections.abc import Sized + + source = self._guard_live_source() + self._guard_not_expanding("__len__") + if isinstance(source, Sized): + return len(source) + return 0 + + def __getitem__(self, index: int) -> Any: + """Random access: get the i-th record with ops applied.""" + source = self._guard_live_source() + if source is None: + raise TypeError("Stream source is None — cannot index. Pass a DataSource / iterable to Stream(source=...).") + self._guard_not_expanding("__getitem__") + + if hasattr(source, "__getitem__"): + raw = source[index] + elif hasattr(source, "__len__"): + if self._indexable_cache is None: + logger.debug(f"Stream: materializing iterable-only source {type(source).__name__} for random access.") + self._indexable_cache = list(source) + raw = self._indexable_cache[index] + else: + raise TypeError( + f"Stream source {type(source).__name__} does not support indexing and has no __len__ " + "(bare iterator). Map-style DataLoader random access is unsafe on a one-shot " + "iterator; give the source a __len__ (then Stream caches on first access) or wrap " + "it in ``list(...)`` before handing it to Stream." + ) + from recordstream.flow import run_steps + + _check_ops_materialized(self.ops) + steps, outputs = linear_steps(self.ops) + record = run_steps(raw, steps, outputs) + if record is None: + raise IndexError(f"Record {index} filtered out by the pipeline") + return cast(Record, record) + + def to_sink(self, sink: Any) -> None: + """Write the entire stream to a DataSink.""" + from recordstream.storage.base import Storage + + target_sink: Any = sink if isinstance(sink, Storage) else nullcontext() + + with target_sink: + for record in self: + sink.write(record) + sink.flush() + + def parallel(self, workers: int = 4) -> "Stream": + """Enable multiprocess execution for the pipeline.""" + self._workers = workers + return self + + def batch(self, chunk_size: int) -> "Stream": + """Group records into chunks (lists of N records).""" + self._chunk_size = chunk_size + return self + + def map(self, func: Callable, key: Optional[str] = None, **kwargs: Any) -> "Stream": + """Append a transformation to the stream. + + ``key`` names the record entry whose payload ``func`` transforms; ``None`` hands + ``func`` the whole record dict. + """ + op = WrappedOp(func, key, kwargs) + self.ops.append(op) + return self + + def filter(self, predicate: Callable[[Record], bool]) -> "Stream": + """Filter the stream based on a predicate.""" + self.ops.append(FilterOp(predicate)) + return self + + def __iter__(self) -> Iterator[Any]: + """Execute the pipeline lazily.""" + if not self._guard_live_source(): + return + + if any(hasattr(op, "stream") and callable(op.stream) for op in self.ops): + it = self._iter_streamed() + elif self._workers > 1: + it = self._iter_parallel() + else: + it = self._iter_sequential() + + if self._chunk_size > 0: + batch = [] + for record in it: + batch.append(record) + if len(batch) == self._chunk_size: + yield batch + batch = [] + if batch: + yield batch + else: + yield from it + + def _iter_streamed(self) -> Iterator[Record]: + """Mixed per-record / stream-level op chain (a stream-level op exposes ``.stream``). + + A stream-level op (``Parallel``) sees the WHOLE stream rather than one record, so it + cannot be a step in the per-record graph — the chain is split at each such op and the + per-record runs between them go through the ordinary kernel. Records travel as plain + records: the per-record Context they used to be paired with is gone, and with it the + "cells cannot cross a stream-op boundary" restriction that pairing imposed. + """ + from recordstream.flow import run_steps_multi + + source = self._guard_live_source() + if source is None: + return + _check_ops_materialized(self.ops) + + def per_record(stream: Iterator[Optional[Record]], op: Any) -> Iterator[Optional[Record]]: + steps, outputs = linear_steps([op]) + for record in stream: + if record is None: + continue + yield from run_steps_multi(record, steps, outputs) + + carried: Iterator[Optional[Record]] = iter(source) + for op in self.ops: + if hasattr(op, "stream") and callable(op.stream): + carried = op.stream(carried) + else: + carried = per_record(carried, op) + + for record in carried: + if record is not None: + yield record + + def _iter_sequential(self) -> Iterator[Record]: + """Standard single-threaded execution — the flat op list run as a linear step graph.""" + from recordstream.flow import _result_readers, run_steps_multi + + source = self._guard_live_source() + if source is None: + return + _check_ops_materialized(self.ops) + # Compile + analyse ONCE per iteration, never per record (see run_steps_multi). + steps, outputs = linear_steps(self.ops) + readers = _result_readers(steps, outputs) + for item in source: + yield from run_steps_multi(item, steps, outputs, readers) + + def _iter_parallel(self) -> Iterator[Record]: + """Multiprocess execution engine.""" + source = self._guard_live_source() + if source is None: + return + _check_ops_materialized(self.ops) + + # We use 'spawn' to be consistent with Loggair and prevent CI deadlocks + ctx = multiprocessing.get_context("spawn") + + from recordstream.flow import _graph_worker_task + + steps, outputs = linear_steps(self.ops) + with concurrent.futures.ProcessPoolExecutor(max_workers=self._workers, mp_context=ctx) as executor: + futures = [] + extra_families = _extra_op_families() # ship third-party op families to the workers + for item in source: + futures.append(executor.submit(_graph_worker_task, item, steps, outputs, extra_families)) + + for future in futures: + yield from future.result() + + def collect(self) -> List[Record]: + """Materialize the full stream into a list.""" + return list(self) + + def project(self, keys: Collection[str]) -> Iterator[Record]: + """Yield pipeline-output records carrying only ``keys`` (the projection primitive). + + Implements :class:`recordstream.projection.SupportsProjection`. Stream must run its op + chain to produce each record (an op may consume the input), so this is the generic + "iterate, then keep only the requested keys" form. Partial: a generator. + """ + want = set(keys) + for record in self: + yield {k: v for k, v in record.items() if k in want} + + +def ensure_materialized(source: RecordSource) -> RecordSource: + """Read ONE whole record, so everything this source builds lazily is built in THIS process. + + The companion to :func:`ensure_record_dataset`: that one normalizes a source's TYPE, this one + normalizes its STATE. Returns the source, so it composes. + + A source in this package is lazy on purpose — a constructor does no work, and the download / + file open / client construction happens on first read. That is right until the first read + happens somewhere it must not, and there is one such place: **a forked child process.** + + Measured, on macOS, 2026-08-02. A ``DataLoader`` worker was the first to touch a + ``HuggingFaceSource``, so ``load_dataset`` ran in the child, called ``hf_hub_download`` -> + ``httpx.Client()`` -> ``urllib.request.getproxies`` -> ``_scproxy`` -> CoreFoundation, which + is not fork-safe: **SIGSEGV**, with no Python traceback, surfacing only as + ``DataLoader worker exited unexpectedly``. Calling this in the parent first makes the child + inherit a source that needs nothing, and the crash does not happen. The consumer cannot + always choose spawn instead — a framework may set the start method globally (fastai sets + ``fork`` at import), and spawning has its own cost (a model on Apple's MPS cannot be shared + to a spawned worker at all). + + It reads a whole RECORD rather than asking a cheaper question, and that is the point: + ``len(source)`` was measured NOT to be enough, because loading the dataset object is not the + same as building everything a read needs. :func:`~recordstream.first_value` is no substitute + either — it is projection-aware, so it deliberately avoids building the values it was not + asked for. + + An empty source is not an error: there is nothing to build, and a caller should not need a + guard for a split that happens to have no rows. + """ + try: + if hasattr(source, "__getitem__"): + source[0] # type: ignore[index] + else: + next(iter(source), None) # type: ignore[call-overload] + except (IndexError, StopIteration): + pass # empty source — nothing to warm, and not an error + return source + + +def ensure_record_dataset(source: Optional[Union[_ConfluidFluid, RecordSource]]) -> "Stream": + """Normalize any wired source into a map-style ``Dataset`` that yields record dicts. + + A wired ``train_set`` / ``val_set`` / ``test_set`` may be a :class:`Stream`, another torch + ``Dataset``, a recordstream source (``HuggingFaceSource``), or a plain list — and its items + may be record dicts or raw rows. A ``Stream`` already coerces every item to a record, so: + + * a ``Stream`` is returned as-is (already a ``Dataset`` of records; this preserves a + subclass's own wrap, e.g. a label-encoding Stream with its ``class_names``), and + * anything else is wrapped in a ``Stream``, which makes it both a map-style ``Dataset`` + AND a record-yielding one. + + Calling this once up front lets the rest of a training pipeline (target detection, label + fitting/encoding, collate, metrics) assume record items — no per-call "is this a record?" + checks. It lives beside :class:`Stream` because that is the only type it knows: the whole + body is "already a Stream? else wrap in one". + + The parameter admits a ``Fluid`` and ``None`` because both genuinely occur at the call + sites: a config hands over a deferred marker, and an optional split may be unwired (which + yields an empty stream, so a caller needs no guard). + + A DEFERRED source (a ``!class:`` marker straight out of a config) is materialized first, + matching :func:`~recordstream.project` and :meth:`~recordstream.LabelMap.encode`. Without + it, wrapping a marker produced a ``Stream`` whose source was still a Fluid — which fails + later, at first iteration, with an error about the Stream rather than about the config that + caused it. Flowing a live object is a no-op. + """ + from confluid import flow + + source = flow(source) + if isinstance(source, Stream): + return source + # `cast`: a map-style `Dataset` iterates through Python's legacy `__getitem__` protocol, + # which mypy does not model — so it is not `Iterable` statically even though `Stream` + # consumes it correctly at runtime (`Stream.source` accepts "any iterable or indexable + # dataset"). Widening that annotation with a Protocol breaks `to_pydantic` for every + # Stream, so the exception is documented here instead. See TASKS.md. + return Stream(source=cast(Iterable[Any], source)) + + +def prepare_record_dataset(source: Optional[Union[_ConfluidFluid, RecordSource]]) -> Optional["Stream"]: + """Normalize a wired dataset slot's TYPE and its STATE, in THIS process. + + The composition every forking consumer writes: + :func:`ensure_record_dataset` normalizes the TYPE (any wired source into a record-yielding + map-style ``Stream``) and :func:`ensure_materialized` normalizes the STATE (one whole record + read here, so a lazy source is built in the CALLING process rather than in a forked + ``DataLoader`` worker — where a first read SIGSEGVs through ``_scproxy`` / CoreFoundation on + macOS with no Python traceback; the full account lives on :func:`ensure_materialized`). + + ``None`` passes through, so an unwired optional split (``val_set``) needs no guard at the + call site. This function exists because the pair was re-composed identically in every + training runnable across the workspace ("``_prepare``"); a consumer needing only one half + still calls that half directly. + """ + if source is None: + return None + return cast("Stream", ensure_materialized(ensure_record_dataset(source))) diff --git a/recordstream/core/wrappers.py b/recordstream/core/wrappers.py new file mode 100644 index 0000000..bd2879a --- /dev/null +++ b/recordstream/core/wrappers.py @@ -0,0 +1,88 @@ +"""The engine's own callable wrappers — ``FilterOp`` and ``WrappedOp``. + +Op-shaped, but NOT in :mod:`recordstream.ops`: both are construction targets of ``Stream``'s +fluent API (``.filter`` appends a ``FilterOp``, ``.map`` a ``WrappedOp``), and moving them +into ``ops/`` would make core import from a package that imports core. They carry no +discovery ``category`` on purpose — they wrap a RAW Python callable, which no GUI can wire. +Rationale: ``docs/architecture.md`` §5. +""" + +from typing import Any, Callable, Dict, Optional, Union, cast + +from confluid import configurable + +from recordstream.items import Record, item_data, with_data + + +@configurable +class FilterOp: + """Configurable filter operation. + + The op form of :meth:`~recordstream.core.Stream.filter` — a predicate gate over the + stream: the record passes when the predicate returns ``True`` and is dropped otherwise + (``__call__`` returns ``None``, which every engine route treats as "skip this record"). + + Args: + p: Predicate ``record -> bool``; the record passes through when it returns ``True``, else is dropped. + Defaults to ``None`` (zero-arg construction); a predicate must be set before the op runs. + """ + + def __init__(self, p: Optional[Callable[[Record], bool]] = None): + # Partial / zero-arg: store config only; a missing predicate is validated lazily in __call__. + self.p = p + + def __call__(self, record: Record) -> Optional[Record]: + if self.p is None: + raise ValueError("FilterOp.p (predicate) is not set — provide a record->bool callable before use.") + return record if self.p(record) else None + + +@configurable +class WrappedOp: + """Configurable transformation wrapper with smart mapping. + + The op form of :meth:`~recordstream.core.Stream.map` — lifts a plain function over one + record value. The callable is ALWAYS stored as its importable ``module:function`` path + (via :mod:`recordstream.discovery`), so the op pickles across ``spawn`` workers and + serializes into Confluid YAML verbatim; the live function resolves lazily on first + call. + + Args: + f: The wrapped callable, or its importable ``module:function`` path (stored as a string for serialization). + Defaults to ``""`` (zero-arg construction); resolving an empty path fails lazily on first call. + key: The record key whose value payload the function transforms (item metadata preserved). + ``None`` (default) = the function receives the WHOLE record dict and returns the new record. + kw: Extra keyword arguments forwarded to the wrapped callable on every call (defaults to none). + """ + + def __init__(self, f: Union[str, Callable] = "", key: Optional[str] = None, kw: Optional[Dict[str, Any]] = None): + from recordstream.discovery import get_callable_path + + # Partial / zero-arg: store config only (the empty-path default resolves lazily via the `func` + # property). EXPLICIT: always store the string path for serialization. + self.f = get_callable_path(f) if callable(f) else f + self.key = key + self.kw = dict(kw) if kw else {} + # Internal cache for the live callable + self._func_cache: Optional[Callable] = None + + @property + def func(self) -> Callable: + if self._func_cache is None: + from recordstream.discovery import resolve_callable + + self._func_cache = resolve_callable(self.f) + return self._func_cache + + def __call__(self, record: Record) -> Optional[Record]: + if self.key is None: + return cast(Optional[Record], self.func(record, **self.kw)) + if self.key not in record: + raise KeyError(f"WrappedOp: record has no key {self.key!r} (keys: {list(record)})") + value = record[self.key] + new_data = self.func(item_data(value), **self.kw) + try: + new_value = with_data(value, new_data) + except TypeError: + new_value = new_data # a plain (non-item) value is replaced verbatim + return {**record, self.key: new_value} diff --git a/dataflux/discovery.py b/recordstream/discovery.py similarity index 58% rename from dataflux/discovery.py rename to recordstream/discovery.py index da58265..c769ca2 100644 --- a/dataflux/discovery.py +++ b/recordstream/discovery.py @@ -1,16 +1,46 @@ +"""Passive introspection for RecordStream callables. + +A round-trip bridge between live Python callables (sources, ops, plain +functions) and JSON-serializable schemas, so downstream tools can discover and +wire pipeline pieces without any hand-written tool definitions (the workspace's +"Passive Introspection" mandate). Two halves: + +* **Serialization** (callable <-> string): :func:`get_callable_path` turns a + callable into an importable ``"module:qualname"`` key and + :func:`resolve_callable` imports it back. This is how a pipeline step is + referenced in a Confluid manifest and resurrected later for reproducibility. +* **Discovery** (callable -> JSON schema): :func:`introspect_callable` reflects + a single callable into a schema (signature + docstring), and + :func:`scan_module` does the same for every callable *defined in* a module — a + visual editor's node bridge reads these to auto-generate canvas nodes and + their property panels. + +The serialization half doubles as the workspace's generic string-callable hook +pattern (:class:`~recordstream.core.wrappers.WrappedOp` stores its ``f`` this way; consuming +packages reuse it for their own dotted-path hooks). Curated discovery (MCP +form-specs, option pickers) builds on the Confluid registry instead — which +registers classes AND builder functions, but only opt-in by name; this module is +the registration-free complement (reflect over ANY callable, no curation, plus +the callable→string dump direction the registry doesn't offer). +""" + import importlib import importlib.util import inspect import os import sys from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Union, cast +from typing import Any, Callable, Dict, List, Union, cast def get_callable_path(func: Callable) -> str: """ - Convert a function/class into an importable string path. - Avoids '__main__' by resolving the script's filename. + Convert a function/class into an importable string path (``"module:qualname"``). + Avoids '__main__' by resolving the script's filename, so the path stays + re-importable from another process. + + Use: the serialization key that lets a discovered callable be referenced in + a Confluid manifest and rehydrated later via :func:`resolve_callable`. """ if not callable(func): raise TypeError(f"Object {func} is not callable") @@ -35,7 +65,14 @@ def get_callable_path(func: Callable) -> str: def resolve_callable(path: Union[str, Callable]) -> Callable: - """Resolve an importable string path back into a callable.""" + """Resolve an importable string path back into a callable (inverse of + :func:`get_callable_path`). Handles a normal module import, a ``.py`` file + path loaded via ``spec_from_file_location``, and a ``.py``-suffix fallback; + an already-callable argument is returned unchanged. + + Use: rehydrating a serialized pipeline — turning the stored ``"module:func"`` + string back into the live op/source. + """ if callable(path): return path @@ -73,8 +110,12 @@ def resolve_callable(path: Union[str, Callable]) -> Callable: def introspect_callable(func: Callable) -> Dict[str, Any]: """ - Build a JSON-serializable schema for a callable. - Used by FluxStudio to render nodes and property panels. + Build a JSON-serializable schema for a callable by reflecting over its + signature: ``path``, ``name``, ``doc``, per-parameter info (``name`` / + ``type`` / ``default`` / ``required``, skipping ``self`` / ``cls`` / + ``*args`` / ``**kwargs``). + + Use: a visual editor reads this to render a node and its property-panel widgets. """ try: sig = inspect.signature(func) @@ -99,21 +140,19 @@ def introspect_callable(func: Callable) -> Dict[str, Any]: "name": getattr(func, "__name__", str(func)), "doc": func.__doc__.strip() if func.__doc__ else "", "parameters": params, - "accepts": _spec_dict(getattr(func, "ACCEPTS", None)), - "produces": _spec_dict(getattr(func, "PRODUCES", None)), } -def _spec_dict(spec: Any) -> Optional[Dict[str, Any]]: - """JSON-serialize a declared ``ACCEPTS`` / ``PRODUCES`` (a :class:`~dataflux.typespec.SampleType`), - or ``None`` when undeclared. Duck-typed so ``discovery`` needn't import ``typespec``.""" - to_dict = getattr(spec, "to_dict", None) - return cast(Dict[str, Any], to_dict()) if callable(to_dict) else None - - def scan_module(path_or_name: Union[str, Path]) -> List[Dict[str, Any]]: """ - Scan a module or script file and return schemas for locally callables. + Scan a module (by import name) or a ``.py`` script (by path) and return an + :func:`introspect_callable` schema for every callable *defined in* that + module — names merely imported into it are filtered out by checking + ``member.__module__ == mod_name``. + + Use: the entry point for whole-module discovery — a visual editor's bridge + calls this to auto-generate one node per source/op, fulfilling the + "never require manual tool definitions" mandate. """ is_py = isinstance(path_or_name, Path) or (isinstance(path_or_name, str) and path_or_name.endswith(".py")) diff --git a/recordstream/dispatch.py b/recordstream/dispatch.py new file mode 100644 index 0000000..ae33c16 --- /dev/null +++ b/recordstream/dispatch.py @@ -0,0 +1,82 @@ +"""The kernel registry — type dispatch for ops (the torchvision-v2 ``_KERNEL_REGISTRY`` pattern). + +An op does not hard-code how to handle each value type. Instead a kernel is registered +per ``(transform class, item type)`` pair, and :func:`dispatch` looks one up — walking the +value's MRO so a kernel registered for a base item type also serves its subclasses. This is +the same registry idea as :mod:`recordstream.collate` (batching keyed by representation), +applied to per-type op behaviour. + +Registration is open: a downstream package teaches an existing op about a new value +type with one decorator and NO core edit — + + from mypkg.transforms import Denoise # any Transform subclass + from mypkg.items import IQSignal # any registered item type + + @Denoise.kernel(IQSignal) + def _(value, params): + return denoise_iq(value, strength=params["strength"]) + +The transform base exposes ``.kernel(item_type)`` as a thin wrapper over +:func:`register_kernel`; both are documented so either entry point works. +""" + +from typing import Any, Callable, Dict, Optional, Tuple + +__all__ = ["Kernel", "register_kernel", "get_kernel", "dispatch", "registered_kernels"] + +#: A kernel maps ``(value, params) -> value`` — the per-type behaviour of one op. +Kernel = Callable[[Any, Dict[str, Any]], Any] + +_KERNEL_REGISTRY: Dict[Tuple[type, type], Kernel] = {} +# Memoized MRO-resolution results (``None`` = a resolved miss). Cleared on every registration. +_DISPATCH_CACHE: Dict[Tuple[type, type], Optional[Kernel]] = {} + + +def register_kernel(transform_cls: type, item_cls: type) -> Callable[[Kernel], Kernel]: + """Register a kernel for ``(transform_cls, item_cls)`` (usable as a decorator). + + Re-registering the same pair overwrites (a consumer may deliberately replace a kernel). + """ + + def _register(fn: Kernel) -> Kernel: + _KERNEL_REGISTRY[(transform_cls, item_cls)] = fn + _DISPATCH_CACHE.clear() # a new registration may change what an MRO walk resolves + return fn + + return _register + + +def get_kernel(transform_cls: type, item_cls: type) -> Optional[Kernel]: + """The kernel registered EXACTLY for ``(transform_cls, item_cls)`` (no MRO walk); ``None`` if absent.""" + return _KERNEL_REGISTRY.get((transform_cls, item_cls)) + + +def dispatch(transform_cls: type, item_cls: type) -> Optional[Kernel]: + """The kernel handling ``item_cls`` for ``transform_cls``, resolved by MRO, or ``None``. + + Resolution walks the transform's MRO (a subclass transform inherits its base's kernels + unless it overrides them) and, for each, the item's MRO (a kernel on a base item type + serves subclasses). The MOST specific transform wins; within a transform, the most + specific item type wins. ``None`` means "this op does not handle this value" — + the caller passes the entry through untouched. Results are memoized (see + :data:`_DISPATCH_CACHE`), invalidated on every :func:`register_kernel`. + """ + key = (transform_cls, item_cls) + if key in _DISPATCH_CACHE: + return _DISPATCH_CACHE[key] + resolved: Optional[Kernel] = None + for t_cls in transform_cls.__mro__: + for i_cls in item_cls.__mro__: + kernel = _KERNEL_REGISTRY.get((t_cls, i_cls)) + if kernel is not None: + resolved = kernel + break + if resolved is not None: + break + _DISPATCH_CACHE[key] = resolved + return resolved + + +def registered_kernels() -> Tuple[Tuple[str, str], ...]: + """Every registered ``(transform-name, item-name)`` pair (sorted) — for introspection / tests.""" + return tuple(sorted((t.__name__, i.__name__) for (t, i) in _KERNEL_REGISTRY)) diff --git a/recordstream/flow/__init__.py b/recordstream/flow/__init__.py new file mode 100644 index 0000000..7fcfde2 --- /dev/null +++ b/recordstream/flow/__init__.py @@ -0,0 +1,86 @@ +"""The ``flow:`` document and the :class:`FlowGraph` engine. + +A **flow document** is the named-step form of a pipeline: a mapping of ``step-name → op``, +where a step's name is how later steps reference its result. It is the spelling to reach for +when a pipeline BRANCHES; a straight chain is written as a plain ``ops:`` list, which the +engine compiles to positional steps (``recordstream.core.linear_steps``). Both parse to the +same :class:`FlowStep` list and run through the same per-record kernel — there is ONE +execution model, and no lowering pass between the two forms (the flow⇄ops converters and the +per-record context ops they emitted were deleted 2026-07-30; see ``docs/architecture.md`` §3). + +.. code-block:: yaml + + flow: + spec: !class:mypkg.MakeSpectrogram() # input: the source record + rescaled: !class:recordstream.ops.numpy.Threshold() # input: previous step + masked: !class:mypkg.Segment() {from: spec} # 2nd reader of spec = fan-out + out: {from: masked, merge_from: [rescaled]} # fan-in (no op) + outputs: out + +Step grammar (the three RESERVED step keys, stripped before the op is built): + +- ``from:`` — the step supplying this step's input record. Omitted = the previous step + (the first step reads the source record). Must name an EARLIER step: document order is + the schedule, so forward references are errors and cycles are inexpressible. +- ``merge_from:`` — fan-in: UNION the named steps' record entries into this step's incoming + record before the op runs (listed order, last-write-wins on a key collision). +- ``bind:`` — ``{param: ref}`` per-record parameters: ``ref`` is a step name (the step's + whole result record), ``step[key]`` (one entry of it), or ``step.attr`` (the step op's + live ``@output`` after it ran — read through wrapper chains by ``_read_output``). + +A step may be a plain mapping with no op (``out: {from: a, merge_from: [b]}``) — a pure +fan-in/identity step; ``{}`` is the identity (used to give the source a referable name). +``outputs:`` names the step whose result the pipeline yields (default: the last step). + +Step results are freed automatically: ``_result_readers`` counts each step's readers +slot-granularly and the kernel drops a result after its last one. A straight chain needs no +environment at all — :func:`is_linear` routes it to ``_run_linear``. + +Submodules, bottom of the layer first (imports run strictly one way): + + - recordstream.flow.steps: the step MODEL — ``FlowStep``, the ``bind:`` reference + grammar, ``RESERVED_STEP_KEYS``. Pure data; imports nothing from its siblings. + - recordstream.flow.parse: ``parse_flow`` — the only module that knows the DOCUMENT form. + - recordstream.flow.execute: the per-record kernel (``run_steps_multi`` / ``run_steps`` / + ``is_linear`` + the two routes) and the spawn-worker entry point. + - recordstream.flow.graph: ``FlowGraph``, the engine facade. + +The canonical dotted path for a config is the SUBMODULE one +(``!class:recordstream.flow.graph.FlowGraph``); the package re-export keeps +``from recordstream.flow import FlowGraph`` and the older spelling working. ``__all__`` is +load-bearing — a visual editor's node bridge surfaces ``FlowGraph`` through it, because +``scan_module``'s ``__module__`` filter no longer sees anything in this package. +""" + +from recordstream.flow.execute import ( # noqa: F401 — see the internal-surface note below + _graph_worker_task, + _result_readers, + _run_from, + _run_linear, + is_linear, + run_steps, + run_steps_multi, +) +from recordstream.flow.graph import FlowGraph +from recordstream.flow.parse import parse_flow +from recordstream.flow.steps import ( # noqa: F401 — see the internal-surface note below + _MISSING, + RESERVED_STEP_KEYS, + FlowStep, + _BindRef, + _read_output, + _split_bind_ref, +) + +__all__ = ["FlowGraph", "FlowStep", "parse_flow", "run_steps", "run_steps_multi", "is_linear", "RESERVED_STEP_KEYS"] + +# The private names re-exported above are the engine's INTERNAL cross-module surface: the +# kernel's two routes, the reader accounting `core.stream` imports, the spawn-worker entry +# point, and the bind-grammar helpers the suite pins. They stay OUT of `__all__` (a leading +# underscore already keeps them off a visual editor's palette), but +# `from recordstream.flow import _result_readers` must keep working — so they are re-exported +# deliberately, with the `noqa` marking that as intent rather than a stray unused import. +# +# NOTE for test doubles: these are BOUND NAMES, not views of the defining module. Patching +# `recordstream.flow._result_readers` does NOT affect the copy `flow.graph` already imported — +# patch the module that USES it (see `tests/test_typed_flow.py`). diff --git a/recordstream/flow/execute.py b/recordstream/flow/execute.py new file mode 100644 index 0000000..482b5b5 --- /dev/null +++ b/recordstream/flow/execute.py @@ -0,0 +1,270 @@ +"""The per-record KERNEL — the one execution model, shared by both authoring forms. + +``Stream``'s positional steps and a ``flow:`` document's author-named steps arrive here as +the same :class:`FlowStep` list (``docs/architecture.md`` §3). Two routes, and they MUST +agree record-for-record: :func:`_run_linear` for a straight chain (no step environment at +all — what keeps an ``ops:`` list cheap) and :func:`_run_from` for a graph that forks, +merges or binds. :func:`is_linear` is the gate between them. + +Result lifetimes are automatic: :func:`_result_readers` counts each step's readers +slot-granularly and the kernel drops a result after its last one. +""" + +from copy import deepcopy +from typing import Any, Dict, List, Optional, Sequence, Tuple, cast + +from loggair import get_logger + +from recordstream.core.families import OpInvoker, OpMatcher, _apply_op, _expand, _op_expands, _sync_op_families +from recordstream.flow.steps import _MISSING, FlowStep, _read_output, _split_bind_ref +from recordstream.items import Record + +logger = get_logger(__name__) + + +def _result_readers(steps: Sequence[FlowStep], outputs: str) -> Dict[str, List[Tuple[int, str]]]: + """Step-result cell -> ordered ``(consumer_index, slot)`` reads. + + Slot granularity matters: one consumer step may read the SAME producer through several + slots (its input AND a ``bind`` param), and only the ``"in"`` slot of the immediately + following step can ride the linear stream. Slots: ``"in"`` (input), ``"merge"``, + ``"bind"``, and the final ``"out"`` read at index ``len(steps)``. A ``bind`` step-result + reference counts; an ``@output`` (``step.attr``) reference does NOT. + + NO steps is the identity graph (a bare ``ops: []``): nothing is produced, so nothing is + read — and there is no output step to account for. + """ + if not steps: + return {} + readers: Dict[str, List[Tuple[int, str]]] = {s.name: [] for s in steps} + for i, step in enumerate(steps): + implicit = steps[i - 1].name if i > 0 else None + source = step.from_ or implicit + if source is not None: + readers[source].append((i, "in")) + for ref in step.merge_from: + readers[ref].append((i, "merge")) + for ref in step.bind.values(): + parsed = _split_bind_ref(ref) + if parsed.attr is None: + readers[parsed.step].append((i, "bind")) + readers[outputs].append((len(steps), "out")) + return readers + + +def run_steps_multi( + seed: Any, + steps: Sequence[FlowStep], + outputs: str, + readers: Optional[Dict[str, List[Tuple[int, str]]]] = None, +) -> List[Record]: + """Run ONE source record through the parsed steps, returning EVERY resulting record. + + The engine's per-record kernel, module-level so a spawn worker can pickle a reference to + it. Usually one record back, zero when a step filtered (an op returned ``None``), several + when a 1→N EXPANDING step fired. + + ``readers`` is the slot-granular reader accounting from :func:`_result_readers`; it + depends only on ``(steps, outputs)``, so a caller running many records MUST compute it + once and pass it in — recomputing per record is an O(steps²) tax on every record (it was + measured at 3.4 µs/record on a 23-step pipeline, roughly half the graph engine's total + overhead over a flat op list). + + EXPANSION semantics: a step whose op carries ``EXPANDS`` yields N children, and the + REMAINING subgraph runs once per child over its own shallow copy of the step environment + (independent name→result maps, shared values — the graph twin of ``Context.copy()``). + Traversal is DEPTH-FIRST, so sibling order matches the nested-loop intuition and the flat + engine's documented order. An empty expansion or a ``None`` child just drops that branch. + + NO steps is the IDENTITY graph — the seed comes straight back. That is what makes a bare + ``Stream(source=..., ops=[])`` yield its source unchanged once the flat engine routes + through this kernel. + """ + if not steps: + return [] if seed is None else [cast(Record, seed)] + out: List[Record] = [] + if is_linear(steps, outputs): + _run_linear(seed, steps, 0, out) + return out + if readers is None: + readers = _result_readers(steps, outputs) + base_remaining = {name: len(idx) for name, idx in readers.items()} + _run_from(0, seed, steps, outputs, {}, base_remaining, None, out) + return out + + +def is_linear(steps: Sequence[FlowStep], outputs: str) -> bool: + """True when the graph is a straight chain — no named reference reaches back. + + Every step reads the one before it, nothing binds, nothing merges, and the yielded step + is the last one. Such a graph needs no step ENVIRONMENT at all: the record can ride a + local variable exactly as it did in the flat op loop, which is what keeps an ``ops:`` + list as cheap to run as before it became a graph (the env bookkeeping measured ~33% + of engine overhead on a 23-step chain). + """ + if not steps or outputs != steps[-1].name: + return False + return all(s.from_ is None and not s.bind and not s.merge_from for s in steps) + + +def _run_linear( + record: Any, + steps: Sequence[FlowStep], + index: int, + out: List[Record], +) -> None: + """Run a straight chain from ``steps[index:]`` — the env-free path (see :func:`is_linear`). + + Same expansion contract as :func:`_run_from`: a 1→N step forks the remaining chain, + depth-first, so sibling order matches the nested-loop intuition. + """ + for i in range(index, len(steps)): + op = steps[i].op + if op is None: + continue + if _op_expands(op): + for child in _expand(op, record): + _run_linear(child, steps, i + 1, out) + return + result = _apply_op(record, op) + if result is None: + return + record = result + out.append(cast(Record, record)) + + +def _run_from( + index: int, + seed: Any, + steps: Sequence[FlowStep], + outputs: str, + env: Dict[str, Any], + remaining: Dict[str, int], + prev: Optional[str], + out: List[Record], +) -> None: + """Run ``steps[index:]`` over ``env``, appending every surviving result to ``out``. + + Recurses ONCE PER CHILD at an expanding step (recursion depth = the number of expanding + steps on the path, not the record count), which is what gives depth-first sibling order + for free. + + Each expansion branch gets its OWN shallow copy of the step environment (independent + name→result maps, shared values), so siblings cannot see each other's results. + """ + + def read_result(name: str, *, copy: bool) -> Any: + value = env[name] + remaining[name] -= 1 + if remaining[name] <= 0: + del env[name] + elif copy: + value = deepcopy(value) + return value + + for i in range(index, len(steps)): + step = steps[i] + # 1. the input record (implicit stream reads move; explicit fan-out reads copy) + if step.from_ is not None: + record = read_result(step.from_, copy=True) + elif prev is not None: + record = read_result(prev, copy=False) + else: + record = seed + + # 2. fan-in: UNION the merge_from steps' entries (slot order, last wins) + if step.merge_from: + if not isinstance(record, dict): + raise TypeError( + f"flow step {step.name!r}: merge_from is the record fan-in but the carrier is " + f"{type(record).__name__} — expected a record dict." + ) + merged = dict(record) + for ref in step.merge_from: + value = read_result(ref, copy=True) + if not isinstance(value, dict): + raise TypeError( + f"flow step {step.name!r}: merge_from step {ref!r} holds " + f"{type(value).__name__}, expected a record" + ) + merged.update(value) + record = merged + + # 3. per-record parameter binds + if step.op is not None: + op = step.op + for param, ref in step.bind.items(): + parsed = _split_bind_ref(ref) + if parsed.attr is not None: + producer = next(s for s in steps if s.name == parsed.step) + value = _read_output(producer.op, parsed.attr) + if value is _MISSING: + raise AttributeError( + f"flow step {step.name!r}: bind {param}={ref!r} — " + f"step {parsed.step!r} op has no @output attribute {parsed.attr!r}" + ) + else: + value = read_result(parsed.step, copy=False) + if isinstance(value, dict) and parsed.key: + # "step[key]" = the named entry; bare "step" = the whole record. + value = value[parsed.key] + setattr(op, param, value) + + # 4. a 1→N step forks the REMAINING subgraph, one branch per child + if _op_expands(op): + for child in _expand(op, record): + child_env = dict(env) + child_env[step.name] = child + _run_from(i + 1, seed, steps, outputs, child_env, dict(remaining), step.name, out) + return + + result = _apply_op(record, op) + if result is None: + return + record = result + + env[step.name] = record + prev = step.name + + if outputs in env: + out.append(cast(Record, env[outputs])) + + +def run_steps( + seed: Any, + steps: Sequence[FlowStep], + outputs: str, + readers: Optional[Dict[str, List[Tuple[int, str]]]] = None, +) -> Optional[Record]: + """Strictly 1→1 twin of :func:`run_steps_multi` — one result back, or ``None``. + + For callers that need exactly one carrier (indexing, a single-record probe). An expanding + step RAISES here rather than silently dropping its siblings; route those through + :func:`run_steps_multi`. + """ + for step in steps: + if step.op is not None and _op_expands(step.op): + raise TypeError( + f"flow step {step.name!r}: {type(step.op).__name__!r} is a 1→N expanding op, which " + "this strictly 1→1 route cannot carry — iterate the graph instead." + ) + results = run_steps_multi(seed, steps, outputs, readers) + return results[0] if results else None + + +def _graph_worker_task( + seed: Any, + steps: Sequence[FlowStep], + outputs: str, + families: Optional[List[Tuple[str, OpMatcher, OpInvoker]]] = None, +) -> List[Record]: + """Spawn-worker entry point: re-register third-party op families, then run one record. + + Module-level for pickling (the same constraint + :func:`recordstream.core.stream._worker_task` obeys). Returns a LIST because an expanding + step makes one seed yield several records. ``readers`` is deliberately NOT passed across + the boundary — it is cheap to derive once per worker call relative to the process hop, and + shipping it would add a second pickled structure that must stay in sync with ``steps``. + """ + _sync_op_families(families) + return run_steps_multi(seed, steps, outputs) diff --git a/recordstream/flow/graph.py b/recordstream/flow/graph.py new file mode 100644 index 0000000..ba89602 --- /dev/null +++ b/recordstream/flow/graph.py @@ -0,0 +1,237 @@ +"""``FlowGraph`` — the engine facade over a ``flow:`` document. + +The named-step twin of ``Stream``: same kernel (:mod:`recordstream.flow.execute`), different +authoring form. It earns its own module by size and by owning a document grammar of its own +(``docs/architecture.md`` §5); ``Stream`` reaches it only through body-local imports, which +is what keeps the core→flow direction one-way. +""" + +import concurrent.futures +import multiprocessing +from typing import Any, Dict, Iterator, List, Optional, Tuple, Union, cast + +from confluid import configurable +from confluid import load as _confluid_load +from loggair import get_logger + +from recordstream.core.families import _extra_op_families, _op_expands +from recordstream.flow.execute import _graph_worker_task, _result_readers, is_linear, run_steps, run_steps_multi +from recordstream.flow.parse import parse_flow +from recordstream.flow.steps import FlowStep +from recordstream.items import Record + +logger = get_logger(__name__) + + +@configurable(category="engine") +class FlowGraph: + """Named-step graph engine — executes a ``flow:`` document natively. + + The named-step twin of :class:`~recordstream.core.Stream`, over the SAME kernel: steps run + in document order against a per-record environment of named results, with fan-out isolation + (copy-on-read, move on last read) and automatic result lifetimes. A LINEAR graph converts + to a Stream (:meth:`to_stream`); a branchy one has no flat spelling by design. + + Args: + source: Any iterable or indexable dataset (duck-typed) yielding record dicts; ``None`` = empty stream. + flow: The flow mapping (step-name -> op / marker / step mapping) or a parsed list of FlowStep. + outputs: Name of the step whose result is yielded. Blank (default) = the last step. + chunk_size: Batch size for chunked iteration; ``0`` (the default) yields single records. + """ + + def __init__( + self, + source: Optional[Any] = None, + flow: Optional[Union[Dict[str, Any], List[FlowStep]]] = None, + outputs: str = "", + chunk_size: int = 0, + ) -> None: + # Partial / zero-arg: store config only; parsing/validation happen in the cached property. + self.source = source + self.flow = flow + self.outputs = str(outputs) + self._chunk_size = int(chunk_size) + self._workers = 1 + self._parsed: Optional[Tuple[List[FlowStep], str]] = None + self._readers: Optional[Dict[str, List[Tuple[int, str]]]] = None + + # -- parsing ----------------------------------------------------------- + + @property + def steps(self) -> List[FlowStep]: + """The parsed, validated steps (cached; recomputed only if ``flow`` is reassigned).""" + return self._ensure_parsed()[0] + + @property + def output_step(self) -> str: + """The resolved output step name.""" + return self._ensure_parsed()[1] + + def _ensure_parsed(self) -> Tuple[List[FlowStep], str]: + if self._parsed is None: + if self.flow is None: + raise ValueError("FlowGraph.flow is not set — provide a flow mapping or FlowStep list.") + if isinstance(self.flow, list) and all(isinstance(s, FlowStep) for s in self.flow): + names = [s.name for s in self.flow] + out = self.outputs or (names[-1] if names else "") + if out not in names: + raise ValueError(f"FlowGraph: outputs {out!r} does not name a step ({names!r})") + self._parsed = (list(self.flow), out) + else: + self._parsed = parse_flow(cast(Dict[str, Any], self.flow), self.outputs) + return self._parsed + + def _ensure_readers(self) -> Dict[str, List[Tuple[int, str]]]: + """The reader accounting, computed ONCE per graph (see :func:`run_steps`).""" + if self._readers is None: + steps, outputs = self._ensure_parsed() + self._readers = _result_readers(steps, outputs) + return self._readers + + @classmethod + def from_yaml(cls, path: str, source: Optional[Any] = None) -> "FlowGraph": + """Build a FlowGraph from a ``{flow: {...}, outputs: ...}`` YAML document (or inline string). + + Uses ``confluid.load(until="settled")`` so step markers stay UNbuilt until :func:`parse_flow` + pops the reserved step keys and flows each op itself. + """ + doc = _confluid_load(path, until="settled") + if not isinstance(doc, dict) or "flow" not in doc: + raise ValueError(f"FlowGraph.from_yaml: {path!r} has no 'flow:' mapping") + return cls(source=source, flow=doc["flow"], outputs=str(doc.get("outputs", "") or "")) + + @classmethod + def from_ops_yaml(cls, path: str, source: Optional[Any] = None) -> "FlowGraph": + """Load a flat ``{ops: [...]}`` YAML document as a LINEAR step graph. + + No lifting is involved: a sequence IS a graph, so the op list becomes positional + steps (``recordstream.core.linear_steps``) — the same compilation a ``Stream``'s + ``ops`` list goes through, because they are the same thing spelled two ways. + """ + from recordstream.core import Stream, linear_steps + + stream = Stream.from_ops_yaml(path, source=source) + steps, outputs = linear_steps(stream.ops) + return cls(source=source, flow=steps, outputs=outputs) + + # -- execution --------------------------------------------------------- + + def _run(self, seed: Any) -> Optional[Any]: + """Run one record through the steps; ``None`` = filtered (an op returned None).""" + steps, outputs = self._ensure_parsed() + return run_steps(seed, steps, outputs, self._ensure_readers()) + + def __iter__(self) -> Iterator[Any]: + if self.source is None: + return + it = self._iter_records() + if self._chunk_size > 0: + batch: List[Record] = [] + for record in it: + batch.append(record) + if len(batch) == self._chunk_size: + yield batch + batch = [] + if batch: + yield batch + else: + yield from it + + def _iter_records(self) -> Iterator[Record]: + if self._workers > 1: + yield from self._iter_parallel() + return + assert self.source is not None + steps, outputs = self._ensure_parsed() + readers = self._ensure_readers() + for item in self.source: + yield from run_steps_multi(item, steps, outputs, readers) + + def _iter_parallel(self) -> Iterator[Record]: + """Multiprocess execution — the graph's OWN spawn pool, one future per source record. + + Mirrors :meth:`recordstream.core.Stream._iter_parallel`: ``spawn`` (consistent with + Loggair, no CI deadlocks), third-party op families shipped to the workers by + reference. The steps pickle because their ops already must; the source never crosses + the boundary (only the seed record does). + """ + assert self.source is not None + steps, outputs = self._ensure_parsed() + ctx = multiprocessing.get_context("spawn") + + with concurrent.futures.ProcessPoolExecutor(max_workers=self._workers, mp_context=ctx) as executor: + extra_families = _extra_op_families() + futures = [ + executor.submit(_graph_worker_task, item, steps, outputs, extra_families) for item in self.source + ] + for future in futures: + yield from future.result() + + @property + def _expands(self) -> bool: + """True when any step op is 1→N — the length/index map is then unknowable.""" + return any(step.op is not None and _op_expands(step.op) for step in self._ensure_parsed()[0]) + + def _guard_not_expanding(self, operation: str) -> None: + if self._expands: + raise TypeError( + f"FlowGraph.{operation} is unavailable: a step op is 1→N EXPANDING, so the " + "expanded length/index map is unknowable. Iterate the graph, wrap it in a torch " + "IterableDataset, window at the SOURCE for random access, or call .collect()." + ) + + def __len__(self) -> int: + from collections.abc import Sized + + self._guard_not_expanding("__len__") + if isinstance(self.source, Sized): + return len(self.source) + return 0 + + def __getitem__(self, index: int) -> Any: + if self.source is None: + raise TypeError("FlowGraph source is None — cannot index.") + self._guard_not_expanding("__getitem__") + if hasattr(self.source, "__getitem__"): + raw = self.source[index] + else: + raise TypeError( + f"FlowGraph source {type(self.source).__name__} does not support indexing; " + "wrap it in a list or use iteration." + ) + result = self._run(raw) + if result is None: + raise IndexError(f"Record {index} filtered out by the flow") + return result + + def parallel(self, workers: int = 4) -> "FlowGraph": + """Enable multiprocess execution on the graph's own spawn pool.""" + self._workers = workers + return self + + def batch(self, chunk_size: int) -> "FlowGraph": + """Group yielded records into lists of ``chunk_size``.""" + self._chunk_size = chunk_size + return self + + def collect(self) -> List[Any]: + """Materialize the full stream into a list.""" + return list(self) + + def to_stream(self) -> Any: + """The ``Stream`` twin of a LINEAR graph — same source, same ops, same engine. + + Only a straight chain converts: a `Stream` carries an op LIST, which cannot express + fan-out. A branchy graph has no flat spelling (that is what the deleted lowering pass + manufactured, at the cost of destroying the structure), so it raises. + """ + from recordstream.core import Stream + + steps, outputs = self._ensure_parsed() + if not is_linear(steps, outputs): + raise TypeError( + "FlowGraph.to_stream: this graph is not a straight chain (it forks or merges), " + "and a Stream's ops list cannot express that. Iterate the FlowGraph directly — " + "it is the same engine." + ) + return Stream(source=self.source, ops=[s.op for s in steps if s.op is not None]) diff --git a/recordstream/flow/parse.py b/recordstream/flow/parse.py new file mode 100644 index 0000000..f9b43cb --- /dev/null +++ b/recordstream/flow/parse.py @@ -0,0 +1,110 @@ +"""Parsing a ``flow:`` mapping into ordered, validated :class:`FlowStep`\\ s. + +The only module that knows the DOCUMENT form. It builds ops (flowing confluid markers per +step, because confluid does not auto-flow two-levels-nested markers) and enforces the +grammar's one structural rule: a reference must name an EARLIER step, so document order IS +the schedule and cycles are inexpressible. +""" + +from typing import Any, Dict, List, Optional, Tuple + +from confluid import flow +from confluid.fluid import Fluid as _ConfluidFluid + +from recordstream.flow.steps import RESERVED_STEP_KEYS, FlowStep, _check_reserved_collision, _parse_bind_ref + + +def parse_flow(flow_doc: Any, outputs: str = "", build: bool = True) -> Tuple[List[FlowStep], str]: + """Parse a flow mapping into ordered :class:`FlowStep`\\ s + the resolved output step name. + + ``flow_doc`` is the ``flow:`` mapping — step values may be confluid markers (from + ``resolve()``/``load()``), plain dicts (pure fan-in steps, or programmatic + ``{"op": , "from": ...}`` form), or live op callables. Reserved keys are popped; + markers are flowed per step (confluid does not auto-flow two-levels-nested markers). + Validates: step names carry no dots, every reference points to an EARLIER step. + + ``build=False`` keeps a marker step UNBUILT (the op stays a Fluid marker) — for + structural consumers (converters/importers) that must not materialize ops. + """ + if not isinstance(flow_doc, dict) or not flow_doc: + raise ValueError("flow: expected a non-empty mapping of step-name -> op") + + steps: List[FlowStep] = [] + seen: List[str] = [] + for name, value in flow_doc.items(): + name = str(name) + if "." in name: + raise ValueError(f"flow: step name {name!r} may not contain '.' (reserved for @output refs)") + if name in seen: + raise ValueError(f"flow: duplicate step name {name!r}") + + reserved: Dict[str, Any] = {} + op: Optional[Any] + if isinstance(value, _ConfluidFluid): + for key in RESERVED_STEP_KEYS: + if key in value.kwargs: + reserved[key] = value.kwargs.pop(key) + op = flow(value) if build else value + elif isinstance(value, dict): + extra = value.get("op") + reserved = {k: v for k, v in value.items() if k in RESERVED_STEP_KEYS} + unknown = [k for k in value if k not in RESERVED_STEP_KEYS and k != "op"] + if unknown: + raise ValueError( + f"flow step {name!r}: unknown step key(s) {unknown!r} — a plain-mapping step " + f"accepts only {RESERVED_STEP_KEYS!r} and 'op'" + ) + op = flow(extra) if (build and isinstance(extra, _ConfluidFluid)) else extra + elif callable(value): + op = value + elif value is None: + op = None + else: + raise TypeError(f"flow step {name!r}: expected an op, a marker, or a mapping — got {type(value).__name__}") + + if op is not None and not isinstance(op, _ConfluidFluid) and not callable(op): + raise TypeError(f"flow step {name!r}: op is not callable ({type(op).__name__})") + if op is not None and not isinstance(op, _ConfluidFluid): + _check_reserved_collision(op, name) + + from_ = reserved.get("from") + if from_ is not None and str(from_) not in seen: + raise ValueError( + f"flow step {name!r}: from: {from_!r} does not name an EARLIER step " + f"(document order is the schedule; steps so far: {seen!r})" + ) + merge_raw = reserved.get("merge_from") + merge_from: Tuple[str, ...] = () + if merge_raw is not None: + merge_from = (str(merge_raw),) if isinstance(merge_raw, str) else tuple(str(r) for r in merge_raw) + for ref in merge_from: + if ref not in seen: + raise ValueError( + f"flow step {name!r}: merge_from: {ref!r} does not name an EARLIER step " + f"(document order is the schedule; steps so far: {seen!r})" + ) + bind_raw = reserved.get("bind") or {} + if not isinstance(bind_raw, dict): + raise TypeError(f"flow step {name!r}: bind must be a mapping of param -> step[.output]") + bind: Dict[str, str] = {} + for param, ref in bind_raw.items(): + _parse_bind_ref(str(ref), seen) # validates + bind[str(param)] = str(ref) + if bind and op is None: + raise ValueError(f"flow step {name!r}: bind requires an op to configure") + + steps.append( + FlowStep( + name=name, + op=op, + from_=None if from_ is None else str(from_), + bind=bind, + merge_from=merge_from, + ) + ) + seen.append(name) + + out = str(outputs) if outputs else steps[-1].name + if out not in seen: + raise ValueError(f"flow: outputs {out!r} does not name a step (steps: {seen!r})") + return steps, out diff --git a/recordstream/flow/steps.py b/recordstream/flow/steps.py new file mode 100644 index 0000000..9974615 --- /dev/null +++ b/recordstream/flow/steps.py @@ -0,0 +1,90 @@ +"""The step MODEL — what a parsed flow step is, and how its references are spelled. + +Pure data + string grammar: no execution, no confluid, no op dispatch. Everything else in +:mod:`recordstream.flow` builds on this, so it deliberately sits at the bottom and imports +nothing from its siblings. +""" + +import inspect +from typing import Any, Dict, NamedTuple, Optional, Sequence, Tuple + +RESERVED_STEP_KEYS = ("from", "merge_from", "bind") +"""Step-grammar keys stripped from a step mapping before the op is constructed.""" + +_MISSING = object() + + +def _read_output(op: Any, name: str) -> Any: + """Read attribute ``name`` off ``op``, looking through ``target``/``op`` wrapper chains. + + Backs the ``bind: {param: "step.attr"}`` grammar — the step op's live ``@output`` after it + ran. The wrapper walk matters because a step op may be a composing op (``ConfigureOp`` + wrapping the real op in ``target``). Returns ``_MISSING`` when absent. + """ + cur, seen = op, set() + while cur is not None and id(cur) not in seen: + seen.add(id(cur)) + value = getattr(cur, name, _MISSING) + if value is not _MISSING: + return value + cur = getattr(cur, "target", None) or getattr(cur, "op", None) + return _MISSING + + +class FlowStep(NamedTuple): + """One parsed step of a flow document.""" + + name: str + op: Optional[Any] # live op callable; None = pure fan-in / identity step + from_: Optional[str] # None = previous step (first step: the source record) + bind: Dict[str, str] # param -> "step" | "step.attr" | "step[key]" + merge_from: Tuple[str, ...] = () # typed fan-in: union these steps' FIELDS, in slot order + + +class _BindRef(NamedTuple): + """A parsed ``bind:`` reference.""" + + step: str + attr: Optional[str] # "step.attr" = the step op's @output attribute + key: Optional[str] # "step[key]" = the named ENTRY of the step's record result + + +def _split_bind_ref(ref: str) -> _BindRef: + """Split a bind reference into its three shapes: ``step`` / ``step.attr`` / ``step[key]``.""" + text = str(ref) + if text.endswith("]") and "[" in text: + head, _, inner = text[:-1].partition("[") + if head and inner and "." not in head: + return _BindRef(head, None, inner) + head, dot, attr = text.partition(".") + return _BindRef(head, attr if dot else None, None) + + +def _parse_bind_ref(ref: str, known: Sequence[str]) -> _BindRef: + parsed = _split_bind_ref(ref) + if parsed.step not in known: + raise ValueError( + f"flow: bind reference {ref!r} does not name an earlier step " + f"(known steps at this point: {list(known)!r})" + ) + return parsed + + +def _check_reserved_collision(op: Any, step_name: str) -> None: + """Raise if the op's constructor has a param named like a reserved step key. + + Reserved keys are stripped from the step mapping before the op is built, so such a + param could never be configured inline — fail loudly instead of silently stealing it. + """ + try: + params = inspect.signature(type(op).__init__).parameters + except (TypeError, ValueError): # pragma: no cover - C-extension ctor + return + clash = [k for k in RESERVED_STEP_KEYS if k in params] + if clash: + raise ValueError( + f"flow step {step_name!r}: op {type(op).__name__!r} has constructor parameter(s) " + f"{clash!r} that collide with reserved flow step keys {RESERVED_STEP_KEYS!r} — " + "such an op cannot be configured in a flow document; rename the parameter or " + "wire the op in the flat ops form instead." + ) diff --git a/recordstream/io.py b/recordstream/io.py new file mode 100644 index 0000000..d2726e0 --- /dev/null +++ b/recordstream/io.py @@ -0,0 +1,141 @@ +"""The item codec registry — how a typed value serializes, for EVERY storage backend. + +Storage backends never inspect item internals: they call :func:`encode_item` to get a flat +:class:`EncodedItem` (registered type name + array payload + scalar attrs) and +:func:`decode_item` to rebuild the item. The DEFAULT structural codec covers both item +shapes (an :class:`~recordstream.items.NDArrayItem` subclass → the array + its declared +attrs; a dataclass wrapper with a ``data`` field → the payload + the remaining fields), so +an externally-registered item type — a domain package's signal item, a user type — serializes +with ZERO storage-code changes. :func:`register_io` overrides the codec for types whose +structure the default cannot capture (e.g. a payload-less wrapper with non-scalar fields). + +A PLAIN (non-item) record value — a float, a string, a bare array — encodes under the +pseudo type tag ``"plain"`` and decodes back verbatim, so scalar metadata keys ride the +same layout as typed values. + +The registered TYPE NAME (via :func:`~recordstream.items.register_item` / +:func:`~recordstream.items.get_item_type`) is the on-disk type tag — decoding requires the +item type to be registered (imported) in the reading process, exactly like the confluid +``!class:`` contract. +""" + +from dataclasses import dataclass +from dataclasses import fields as dataclass_fields +from dataclasses import is_dataclass +from typing import Any, Callable, Dict, Tuple, cast + +from recordstream.items import NDArrayItem, Record, get_item_type, is_item, item_data + +__all__ = [ + "EncodedItem", + "EncodedField", + "register_io", + "encode_item", + "decode_item", + "encode_record", + "decode_record", +] + +#: The on-disk type tag for a plain (non-item) record value — stored and restored verbatim. +PLAIN_TYPE = "plain" + + +@dataclass(frozen=True) +class EncodedItem: + """One value, flattened for storage: registered type name (or ``"plain"``) + payload + scalar attrs.""" + + type_name: str + payload: Any # ndarray / tensor / scalar / None + attrs: Dict[str, Any] + + +@dataclass(frozen=True) +class EncodedField: + """One named entry of a record: the encoded value plus its key.""" + + key: str + item: EncodedItem + + +#: Encoder: item -> (payload, attrs). Decoder: (payload, attrs) -> item. +Encoder = Callable[[Any], Tuple[Any, Dict[str, Any]]] +Decoder = Callable[[Any, Dict[str, Any]], Any] + +_CODECS: Dict[type, Tuple[Encoder, Decoder]] = {} + + +def register_io(item_cls: type, *, encode: Encoder, decode: Decoder) -> None: + """Override the codec for ``item_cls`` (exact type — no MRO walk; a codec is a per-type contract). + + ``encode(item) -> (payload, attrs)`` and ``decode(payload, attrs) -> item``. Registering + replaces any previous codec for the type. + """ + _CODECS[item_cls] = (encode, decode) + + +def encode_item(item: Any) -> EncodedItem: + """Flatten one value for storage (registered codec first, else the default structural codec). + + A value that is not a registered item type encodes as ``"plain"`` — payload verbatim. + """ + codec = _CODECS.get(type(item)) + if codec is not None: + payload, attrs = codec[0](item) + return EncodedItem(type_name=type(item).__name__, payload=payload, attrs=attrs) + if not is_item(item): + return EncodedItem(type_name=PLAIN_TYPE, payload=item, attrs={}) + return EncodedItem(type_name=type(item).__name__, payload=_payload(item), attrs=_attrs(item)) + + +def decode_item(encoded: EncodedItem) -> Any: + """Rebuild a value from its encoded form (an item type must be registered in this process).""" + if encoded.type_name == PLAIN_TYPE: + return encoded.payload + cls = cast(Any, get_item_type(encoded.type_name)) + codec = _CODECS.get(cls) + if codec is not None: + return codec[1](encoded.payload, dict(encoded.attrs)) + if issubclass(cls, NDArrayItem): + return cls(encoded.payload, **encoded.attrs) + if _has_data_field(cls): + return cls(data=encoded.payload, **encoded.attrs) + return cls(**encoded.attrs) + + +def encode_record(record: Record) -> Tuple[EncodedField, ...]: + """Encode every entry of a record, in insertion order.""" + return tuple(EncodedField(key=key, item=encode_item(value)) for key, value in record.items()) + + +def decode_record(fields: Tuple[EncodedField, ...]) -> Record: + """Rebuild a record dict from encoded fields (order preserved).""" + return {field.key: decode_item(field.item) for field in fields} + + +# --- the default structural codec ------------------------------------------- +def _payload(item: Any) -> Any: + """The array/data payload to store — the array for array items, ``.data`` for wrappers, else ``None``.""" + if isinstance(item, NDArrayItem): + return item_data(item) + if is_dataclass(item) and not isinstance(item, type) and _has_data_field(type(item)): + return getattr(item, "data") + return None + + +def _attrs(item: Any) -> Dict[str, Any]: + """The reconstruction attributes (everything but the payload).""" + if isinstance(item, NDArrayItem): + return {name: getattr(item, name, None) for name in type(item)._item_attrs} + if is_dataclass(item) and not isinstance(item, type): + return {f.name: getattr(item, f.name) for f in dataclass_fields(item) if f.name != "data"} + return {} + + +def _has_data_field(cls: type) -> bool: + return is_dataclass(cls) and any(f.name == "data" for f in dataclass_fields(cls)) + + +# --- optional helper: an item with a python-object payload (e.g. Boxes boxes) ---- +def default_encoded_attrs(item: Any) -> Dict[str, Any]: + """The default codec's attrs view of ``item`` — reusable inside a custom encoder.""" + return _attrs(item) diff --git a/recordstream/items.py b/recordstream/items.py new file mode 100644 index 0000000..e701c11 --- /dev/null +++ b/recordstream/items.py @@ -0,0 +1,464 @@ +"""Typed values — the vocabulary a record is made of, each value OWNING its metadata. + +A record is a plain ``dict`` (the :data:`Record` alias) whose values are TYPED: an +:class:`Image` carries its ``layout``, a :class:`Label` its ``classes``, a +:class:`Boxes` its ``canvas`` reference frame. Ops dispatch on these types (the +torchvision-v2 ``tv_tensors`` idea) — there is no wrapper container and no role tags; +key names ("image", "mask", "label") carry meaning, exactly like every torch batch dict. + +The item model is HYBRID (the workspace decision): + +* **Array-backed items subclass the payload** (:class:`NDArrayItem`, an ``np.ndarray`` + subclass) so a type-agnostic operation touches them AS an array while their extra + attributes survive numpy operations (``__array_finalize__``). ``Image`` / ``Mask`` are + these. +* **Structured items are dataclass wrappers** (:class:`Boxes` / :class:`Label`) — a + bounding-box set or a class label is not an array; a wrapper is also the right home for a + payload a domain package does not want to subclass (e.g. complex-IQ signal data, where + subclassing an ``np.complex64`` ndarray and preserving attributes through arithmetic is + fragile). + +This module is MODALITY-NEUTRAL — only generic items live here (images, masks, boxes, +labels). Domain items (a signal, a spectrogram) live in the domain package and register +into the SAME registry, per the workspace modality-neutral mandate. That IS the +extensibility story below. + +Both shapes present a uniform payload accessor via :func:`item_data` / :func:`with_data`, so +a transform kernel never has to special-case "is this a subclass or a wrapper". + +Extensibility: any type decorated with :func:`register_item` becomes a first-class item — +the dispatch registry (:mod:`recordstream.dispatch`) and a visual editor's socket-type map can +see it. A downstream package (a signal item, a user type) adds one class + one decorator, +no core edit. + +NOTE (scope): array items are ``np.ndarray`` subclasses only; a torch-``Tensor``-subclass +item base (via ``__torch_function__``) is a documented follow-up — torch payloads ride in +wrapper items. Items are registered in the local :func:`register_item` registry rather than +carried on the confluid ``@configurable`` registry (an ``np.ndarray`` subclass builds +through ``__new__``, which fights confluid's ``__init__`` validation wrap). +""" + +from dataclasses import dataclass, field, fields, is_dataclass, replace +from typing import Any, Dict, List, Literal, Optional, Tuple, Type, TypeVar, cast, overload + +import numpy as np + +_ItemT = TypeVar("_ItemT") + +#: A record record — a PLAIN dict of typed values. There is deliberately no container +#: class: ops receive and return ordinary dicts, so library transforms that already +#: understand dicts (torchvision v2) or named kwargs (albumentations) run as-is. +Record = Dict[str, Any] + +__all__ = [ + "Record", + "NDArrayItem", + "Image", + "Mask", + "Boxes", + "Label", + "MultiLabel", + "is_class_id", + "register_item", + "item_types", + "item_type_names", + "get_item_type", + "is_item", + "item_data", + "item_value", + "resolve_entry", + "resolve_item", + "with_data", +] + +# --------------------------------------------------------------------------- +# Item registry — the extensibility surface. A registered type is a first-class +# item the dispatch registry and a visual editor's socket-type map see. +# --------------------------------------------------------------------------- +_ITEM_TYPES: Dict[str, type] = {} + + +def register_item(cls: Type[Any]) -> Type[Any]: + """Register ``cls`` as a first-class item type (usable as a class decorator). + + Re-registering the same name overwrites (consumers may deliberately replace a type). + """ + _ITEM_TYPES[cls.__name__] = cls + return cls + + +def item_types() -> Tuple[type, ...]: + """Every registered item type (registration order).""" + return tuple(_ITEM_TYPES.values()) + + +def get_item_type(name: str) -> type: + """The registered item type named ``name`` (a miss names the known types).""" + try: + return _ITEM_TYPES[name] + except KeyError: + known = ", ".join(sorted(_ITEM_TYPES)) or "" + raise KeyError(f"no item type registered as {name!r} (known: {known})") from None + + +def item_type_names() -> Tuple[str, ...]: + """The registered item type NAMES (sorted) — the enumerable socket-type vocabulary.""" + return tuple(sorted(_ITEM_TYPES)) + + +def is_item(obj: Any) -> bool: + """True if ``obj`` is an instance of a registered item type.""" + types = tuple(_ITEM_TYPES.values()) + return bool(types) and isinstance(obj, types) + + +# --------------------------------------------------------------------------- +# Array-backed items — np.ndarray subclasses that preserve their extra attributes. +# --------------------------------------------------------------------------- +class NDArrayItem(np.ndarray): + """Base for array-backed items: an ``np.ndarray`` subclass whose declared extra + attributes (``_item_attrs``) survive numpy operations via ``__array_finalize__``. + + Subclasses declare their metadata attributes as ``_item_attrs`` plus a class-level + default for each:: + + class Image(NDArrayItem): + _item_attrs = ("layout",) + layout = "HWC" + + img = Image(rgb_hwc) # img.layout == "HWC" + img = Image(rgb_chw, layout="CHW") + flipped = np.flip(img, axis=1) # still an Image, flipped.layout == "CHW" + """ + + _item_attrs: Tuple[str, ...] = () + + def __new__(cls, data: Any, **attrs: Any) -> "NDArrayItem": + unknown = set(attrs) - set(cls._item_attrs) + if unknown: + raise TypeError( + f"{cls.__name__}: unexpected attributes {sorted(unknown)} (allowed: {list(cls._item_attrs)})" + ) + obj = np.asarray(data).view(cls) + for name in cls._item_attrs: + setattr(obj, name, attrs[name] if name in attrs else getattr(cls, name, None)) + return obj + + def __array_finalize__(self, obj: Any) -> None: + # Called on every construction path (view, slice, ufunc output). Carry the extra + # attributes forward from the source array (class default when absent). + if obj is None: + return + for name in getattr(type(self), "_item_attrs", ()): + setattr(self, name, getattr(obj, name, getattr(type(self), name, None))) + + +@register_item +class Image(NDArrayItem): + """An image array. ``layout`` is ``"HWC"`` (numpy convention, default) or ``"CHW"``.""" + + _item_attrs = ("layout",) + layout: str = "HWC" + + +@register_item +class Mask(NDArrayItem): + """A segmentation / activity mask array (same spatial frame as its sibling image).""" + + +# --------------------------------------------------------------------------- +# Structured items — dataclass wrappers (not arrays). +# --------------------------------------------------------------------------- +@register_item +@dataclass +class Boxes: + """A set of pixel-space bounding boxes on an image raster, with optional labels and scores. + + Pixel-ONLY by contract: boxes are HALF-OPEN ``[x0, y0, x1, y1]`` rows in absolute + pixels (x rightward, y downward). A domain package needing a different coordinate + system (e.g. time/frequency regions on a waveform) registers its OWN item type — + this one is what every image-geometry op and detection consumer dispatches on. + + Attributes: + boxes: The boxes — half-open absolute-pixel ``[x0, y0, x1, y1]`` rows, as a + list OR an ``[N, 4]`` array/tensor (a detection pipeline keeps its framework's type; + annotated ``Any`` because list, ndarray and tensor share no useful protocol). + labels: Optional per-box class labels (list or ``[N]`` array/tensor, like ``boxes``). + scores: Optional per-box confidence scores (list or ``[N]`` array/tensor). + canvas: Optional ``(H, W)`` reference frame — the raster the boxes are stated in, + so a geometric transform (flip / resize) has a self-contained frame. + extras: Auxiliary PER-BOX parallel arrays and box-set measurements keyed by name — + item-scoped metadata that travels WITH the boxes it describes. + """ + + boxes: Any = field(default_factory=list) + labels: Optional[Any] = None + scores: Optional[Any] = None + canvas: Optional[Tuple[int, int]] = None + extras: Dict[str, Any] = field(default_factory=dict) + + +def is_class_id(value: Any) -> bool: + """True when ``value`` is an ENCODED class id (an integer), not a class name. + + The ONE rule for "is this label already encoded?" — so consumers dispatch on + it instead of re-deriving a type check each time (a trainer used to sniff + ``isinstance(target, str)`` itself). + + Recognises an integer in ANY framework: a Python ``int``, a numpy integer, + and a **0-dimensional integer array or tensor** — a dataset that yields + ``Label(torch.tensor(3))`` is as encoded as one yielding ``Label(3)``, and + treating the tensor as a class NAME would send it through a LabelMap and + key the mapping on ``"tensor(3)"``. + + ``bool`` is deliberately excluded: it is an ``int`` subclass, so a boolean + flag mistakenly wired to the target key would silently become class id 1 and + train without complaint. + """ + if isinstance(value, bool): + return False + if isinstance(value, (int, np.integer)): + return True + # 0-d array / tensor (numpy, torch, …) — unwrap via the array-scalar protocol + # rather than importing a framework, so this stays modality- and engine-neutral. + unwrap = getattr(value, "item", None) + if callable(unwrap) and getattr(value, "ndim", None) == 0: + try: + return is_class_id(unwrap()) + except Exception: # pragma: no cover - defensive: exotic 0-d payload + return False + return False + + +@register_item +@dataclass +class Label: + """A single classification label plus its class vocabulary. + + The label is either a class NAME (needs a + :class:`~recordstream.labels.LabelMap` to encode) or an already-encoded + class ID — :attr:`is_encoded` is the one place that distinction is decided. + + Attributes: + value: The label (a class id or name). + classes: Optional ordered class vocabulary this label indexes into. + """ + + value: Any = None + classes: Optional[List[Any]] = None + + @property + def is_encoded(self) -> bool: + """True when :attr:`value` is already a class id rather than a name.""" + return is_class_id(self.value) + + +@register_item +@dataclass +class MultiLabel: + """Several classification labels for one record, plus their class vocabulary. + + The multi-label counterpart of :class:`Label` — a record belonging to more + than one class. Giving it a TYPE is what lets consumers dispatch on the + item instead of sniffing ``isinstance(value, (list, tuple, set))``, which + cannot distinguish a genuine multi-label target from an ordinary sequence + value that happens to sit under the target key. + + Like :class:`Label`, its values are always mappable to class ids through a + :class:`~recordstream.labels.LabelMap`. + + Attributes: + values: The labels (class ids or names). Order is not significant. + classes: Optional ordered class vocabulary these labels index into. + """ + + values: List[Any] = field(default_factory=list) + classes: Optional[List[Any]] = None + + @property + def is_encoded(self) -> bool: + """True when every value is already a class id (vacuously true when empty).""" + return all(is_class_id(v) for v in self.values) + + +# --------------------------------------------------------------------------- +# Uniform payload accessors — so kernels never special-case subclass vs wrapper. +# --------------------------------------------------------------------------- +def item_data(item: Any) -> Any: + """The underlying payload of an item: the plain array (array items) or ``.data`` (wrappers).""" + if isinstance(item, NDArrayItem): + return item.view(np.ndarray) + if is_dataclass(item) and any(f.name == "data" for f in fields(item)): + return getattr(item, "data") + return item + + +def item_value(item: Any) -> Any: + """The semantic VALUE of a record entry — one step further past a wrapper than :func:`item_data`. + + The difference is the label items, and it is the whole reason this exists beside + :func:`item_data`: a :class:`Label`'s payload slot is ``value``, not ``data``, so + ``item_data`` hands the ``Label`` itself back. A consumer that wants *the class id* — or + *the mask array*, without caring which wrapper carried it — wants this instead. + + The rule: a :class:`MultiLabel` yields its ``.values`` list, a :class:`Label` its + ``.value``, any other registered item its payload via :func:`item_data`, and a plain value + passes through verbatim. + + It is ONE function because the rule was written out three times — + :func:`~recordstream.projection.iter_key` (per record), + :func:`~recordstream.batch.batch_values` (per batch) and, the copy that prompted the + extraction, an op reading a mask that a source had wrapped in a ``Label``. Those state + WHERE they read; the unwrapping itself never differed. + + Example:: + + item_value(Label("cat")) # 'cat' (item_data returns the Label) + item_value(Mask(np.zeros((4, 4)))) # the ndarray + item_value(30.72e6) # 30720000.0 + """ + if isinstance(item, MultiLabel): + return item.values + if isinstance(item, Label): + return item.value + return item_data(item) + + +def with_data(item: _ItemT, new_data: Any) -> _ItemT: + """A copy of ``item`` carrying ``new_data`` as its payload, metadata preserved (same type). + + Works for both shapes: an array item is rebuilt with its declared attributes; a wrapper + with a ``data`` field is ``dataclasses.replace``\\ d. An item with no payload slot raises. + """ + if isinstance(item, NDArrayItem): + attrs = {name: getattr(item, name, None) for name in type(item)._item_attrs} + return cast(_ItemT, type(item)(new_data, **attrs)) + if is_dataclass(item) and not isinstance(item, type) and any(f.name == "data" for f in fields(item)): + return cast(_ItemT, replace(cast(Any, item), data=new_data)) + raise TypeError(f"with_data: {type(item).__name__} has no payload slot to replace") + + +_ResolvedT = TypeVar("_ResolvedT") + + +@overload +def resolve_entry( + record: Record, + field: Optional[str], + item_type: Type[_ResolvedT], + *, + owner: str, + param: str = ..., + fallback: bool = ..., + required: Literal[True] = ..., +) -> Tuple[str, _ResolvedT]: ... + + +@overload +def resolve_entry( + record: Record, + field: Optional[str], + item_type: Type[_ResolvedT], + *, + owner: str, + param: str = ..., + fallback: bool = ..., + required: Literal[False] = ..., +) -> Optional[Tuple[str, _ResolvedT]]: ... + + +def resolve_entry( + record: Record, + field: Optional[str], + item_type: Type[_ResolvedT], + *, + owner: str, + param: str = "field", + fallback: bool = True, + required: bool = True, +) -> Optional[Tuple[str, _ResolvedT]]: + """Resolve the ``(key, value)`` entry of ``item_type`` a field-pinned op reads. + + THE record-key resolution every ``field=``-style op used to re-derive per class + (twelve byte-parallel private ``_find_*`` copies in one consumer package before the + extraction — the same story as :func:`item_value` and ``first_value``): an explicit + ``field`` must exist and hold an ``item_type`` value — each miss raises a + ``ValueError`` naming ``owner``, ``param`` and the record's keys — while a blank + ``field`` falls back to the FIRST ``item_type`` value in record order. + + Args: + record: The record dict being resolved against. + field: The configured record key; blank/``None`` engages the first-of-type fallback. + item_type: The item class the entry must be an instance of. + owner: The op/class name error messages lead with (e.g. ``"SaveImage"``). + param: The ctor-param name error messages cite (e.g. ``"image_field"``). + fallback: When False, a blank ``field`` is an error like any other missing key — + for ops whose key is mandatory config rather than a convenience default. + required: When False, every miss returns ``None`` instead of raising (the probe + form — "use it if the record has one"). + + Returns: + ``(key, value)`` — the resolved record key and its typed value; ``None`` only + when ``required=False`` missed. + """ + type_name = item_type.__name__ + if field or not fallback: + if field not in record: + if not required: + return None + raise ValueError(f"{owner}: {param} {field!r} not in record (keys: {list(record)})") + value = record[field] + if not isinstance(value, item_type): + if not required: + return None + raise ValueError(f"{owner}: {param} {field!r} is {type(value).__name__}, expected {type_name}") + return field, value + for key, value in record.items(): + if isinstance(value, item_type): + return key, value + if not required: + return None + raise ValueError(f"{owner}: no {type_name} entry in record (keys: {list(record)})") + + +@overload +def resolve_item( + record: Record, + field: Optional[str], + item_type: Type[_ResolvedT], + *, + owner: str, + param: str = ..., + fallback: bool = ..., + required: Literal[True] = ..., +) -> _ResolvedT: ... + + +@overload +def resolve_item( + record: Record, + field: Optional[str], + item_type: Type[_ResolvedT], + *, + owner: str, + param: str = ..., + fallback: bool = ..., + required: Literal[False] = ..., +) -> Optional[_ResolvedT]: ... + + +def resolve_item( + record: Record, + field: Optional[str], + item_type: Type[_ResolvedT], + *, + owner: str, + param: str = "field", + fallback: bool = True, + required: bool = True, +) -> Optional[_ResolvedT]: + """The value half of :func:`resolve_entry` — the common case when the key is not needed.""" + entry = resolve_entry( + record, field, item_type, owner=owner, param=param, fallback=fallback, required=cast(Any, required) + ) + return None if entry is None else entry[1] diff --git a/recordstream/keras.py b/recordstream/keras.py new file mode 100644 index 0000000..b5497a5 --- /dev/null +++ b/recordstream/keras.py @@ -0,0 +1,211 @@ +"""The Keras boundary: the backend default, the ``keras`` handle, and the batching adapter. + +Three things live here, in the order they have to happen. + +**1. The backend default.** Keras 3 reads ``KERAS_BACKEND`` **at import time** and defaults to +``tensorflow``, which ``recordstream[keras]`` does not install (Keras 3 is an API, not a runtime — +it needs exactly one of torch / TensorFlow / JAX underneath, and which one is the operator's +choice). A bare ``import keras`` on a torch-backed install therefore dies with +``ModuleNotFoundError: No module named 'tensorflow'`` raised from inside +``keras.src.tree.optree_impl`` — a traceback that says nothing about what to do. So this module +sets a default the install can actually honour, the first backend that is PRESENT, and only when +the operator has not chosen: ``setdefault`` never overrides an explicit ``KERAS_BACKEND=jax``. + +**2. The ``keras`` handle.** Consumers import keras THROUGH here (``from recordstream.keras import +keras``) so the ordering above cannot be got wrong by an import that happens to land first — which +is a real hazard, because import sorters group a library import ahead of a first-party one. + +**3. The batching adapter.** For torch, recordstream supplies one function and the framework does +the rest: hand :func:`~recordstream.collate.collate_records` to a ``DataLoader`` as its +``collate_fn`` and torch owns the row order, the batch slicing and the per-epoch reshuffle (a +``Stream`` is map-style, which is all a ``DataLoader`` needs — see +:class:`~recordstream.core.mapstyle.MapStyle`). Keras 3 has no ``DataLoader``: +``keras.utils.PyDataset.__getitem__`` must return a whole BATCH, so somebody has to write that +loop. :class:`RecordSequence` is that loop and nothing else — row order, slicing, reshuffle, +``collate_records`` — the DataLoader half, kept beside the collate half it calls instead of +re-appearing in every training project. + +What a batch BECOMES stays the caller's, exactly as ``collate_fn`` is the caller's on the torch +side: ``transform`` maps one collated record to what the model consumes, so task shapes (a +classifier's ``(x, y)`` tuple, a multi-input model's dict) never enter this module. + +Import it by PATH — ``RecordSequence`` is deliberately absent from the package root, because +``inspect.getmembers`` (what :func:`recordstream.discovery.scan_module` and the GUI bridges use) +getattrs every name a module advertises, so a lazy root export would import keras on every +discovery scan of a torch-only install:: + + from recordstream.keras import RecordSequence + + seq = RecordSequence(stream, batch_size=32, shuffle=True, transform=to_xy) + model.fit(seq, epochs=3) +""" + +import importlib.util +import os +from typing import Any, Callable, Iterator, Optional, Union, cast + +import numpy as np + +from recordstream.collate import CollateFn, get_collate +from recordstream.core import MapStyle +from recordstream.items import Record + +#: Compute backends Keras 3 can run on, in the order this package prefers them. torch first +#: because it is the one every other recordstream extra already implies; the rest are honoured +#: whenever an operator has them installed. +_BACKENDS = ("torch", "tensorflow", "jax") + + +def _first_installed_backend() -> str: + """The first of :data:`_BACKENDS` actually importable, else Keras's own default. + + Picking a backend that is PRESENT rather than one we assume: hard-coding ``torch`` would fail + on a TensorFlow-only install exactly as Keras's own ``tensorflow`` default fails on the + torch-only install this exists to fix. Uses ``find_spec`` so nothing is imported just to look, + which matters because it runs at import time of this module. + + Example:: + + os.environ.setdefault("KERAS_BACKEND", _first_installed_backend()) # "torch" here + """ + for name in _BACKENDS: + if importlib.util.find_spec(name) is not None: + return name + return "tensorflow" # Keras's own default — let IT raise, naming the package to install + + +#: Set BEFORE keras is imported, and only if unset — an explicit choice always wins. +os.environ.setdefault("KERAS_BACKEND", _first_installed_backend()) + +import keras # noqa: E402 - deliberately after the environment default above + +__all__ = ["RecordSequence", "keras", "keras_backend"] + + +def keras_backend() -> str: + """The Keras backend actually in effect (``torch`` / ``tensorflow`` / ``jax``). + + Reads Keras rather than the environment variable, so it reports what is LOADED — Keras + consults ``KERAS_BACKEND`` once at import, and changing it afterwards has no effect. Worth + logging at the start of a run whose engine is selectable. + """ + return str(keras.backend.backend()) + + +class RecordSequence(keras.utils.PyDataset): + """A map-style record source as a ``keras.utils.PyDataset`` of collated batches. + + The DataLoader half of Keras batching: row order, batch slicing, per-epoch reshuffle, and + the collate. It is deliberately task-blind — ``transform`` is the caller's + ``collate_fn``-equivalent and decides what the model actually receives. + + Args: + source: Any map-style record source (a ``Stream`` is one). + batch_size: Rows per batch. A short final batch is yielded as-is, never padded. + shuffle: Reshuffle the row order at the end of every epoch (training). + seed: Shuffle seed, so a shuffled run is reproducible. + transform: Maps one collated record batch to what the model consumes. ``None`` hands + over the batched record itself. + collate: Which batch shape to build — a registered KEY (``"record"`` stacks what can + stack; ``"list"`` keeps every column as a per-record list) or a collate function. + The torch half takes this choice as ``DataLoader(collate_fn=…)``; this is the same + choice on the Keras side, so an engine that needs ``List[Tensor]`` inputs is not + forced to the stacking default. See :mod:`recordstream.collate`. + workers: ``PyDataset`` prefetch workers. ``1`` (Keras's own default) loads batches on + the calling thread; higher values overlap the record walk with the training step, + which is what a slow source (decode, resize, remote read) needs. + use_multiprocessing: Run those workers as PROCESSES instead of threads. Each one + re-pickles this object and its source, so it is the wrong default for a source + holding an open handle — reach for it only when the per-record work is + GIL-bound and does not parallelize with threads. + max_queue_size: How many prefetched batches may wait. The memory ceiling of + prefetching: batches are held whole, so a large value on large batches is a + real footprint. + """ + + def __init__( + self, + source: Optional[MapStyle] = None, + batch_size: int = 32, + shuffle: bool = False, + seed: int = 0, + transform: Optional[Callable[[Record], Any]] = None, + collate: Union[str, CollateFn] = "record", + workers: int = 1, + use_multiprocessing: bool = False, + max_queue_size: int = 10, + ) -> None: + # Keras's own defaults, restated so they are DECLARED parameters rather than reachable + # only through `**kwargs` — the every-knob-is-a-declared-parameter rule (architecture + # §6): a form/schema generator enumerates the signature, and an undeclared knob is + # invisible to it. + super().__init__(workers=workers, use_multiprocessing=use_multiprocessing, max_queue_size=max_queue_size) + self.source = source + self.batch_size = int(batch_size) + self.shuffle = bool(shuffle) + self.seed = int(seed) + self.transform = transform + # Stored VERBATIM (a key or a function); resolved per batch by `_collate`, so a key + # registered AFTER this object was built still resolves. + self.collate: Union[str, CollateFn] = collate + self._rng = np.random.default_rng(seed) + self._indices: Optional[np.ndarray] = None + + @property + def indices(self) -> np.ndarray: + """Row order for the current epoch — built on first use, reshuffled by ``on_epoch_end``. + + Built lazily rather than in the constructor because ``len(source)`` is real work for a + deferred source (a ``HuggingFaceSource`` loads its dataset to answer it), and recordstream + constructors do none. It is also where a missing ``source`` is reported, so zero-arg + construction stays possible. + """ + if self._indices is None: + if self.source is None: + raise RuntimeError("RecordSequence: no 'source' to batch — wire the dataset first.") + order = np.arange(len(self.source)) + if self.shuffle: + self._rng.shuffle(order) + self._indices = order + return self._indices + + @property + def _collate(self) -> CollateFn: + """The chosen collate — a registered key resolved on use, or the function as given.""" + return get_collate(self.collate) if isinstance(self.collate, str) else self.collate + + def __len__(self) -> int: + """Number of batches — Keras asks once per epoch.""" + return int(np.ceil(len(self.indices) / self.batch_size)) + + def batch(self, index: int) -> Record: + """The collated record batch at ``index``, BEFORE ``transform``.""" + rows = self.indices[index * self.batch_size : (index + 1) * self.batch_size] + source = cast(MapStyle, self.source) # the `indices` read above validated it + # Declared local, not a bare return: `@register_collate` types every registered collate + # as `CollateFn = Callable[[Sequence[Any]], Any]` — deliberately loose, because the + # registry holds task collates with divergent conventions — which erases + # `collate_records`' own `-> Record` at the call site. + collated: Record = self._collate([source[int(i)] for i in rows]) + return collated + + def batches(self) -> Iterator[Record]: + """Every collated batch, in the current epoch's order. + + The pairing half of prediction: a model emits ``[N, ...]`` while a + :class:`~recordstream.predictions.PredictionsSink` writes per record, so the caller needs + the batch its output came from to read that batch's metadata back + (:func:`~recordstream.batch.batch_metadata`). + """ + for index in range(len(self)): + yield self.batch(index) + + def __getitem__(self, index: int) -> Any: + """What Keras feeds the model: the collated batch, mapped by ``transform``.""" + batch = self.batch(index) + return batch if self.transform is None else self.transform(batch) + + def on_epoch_end(self) -> None: + """Reshuffle between epochs when training. Keras calls this itself.""" + if self.shuffle: + self._rng.shuffle(self.indices) diff --git a/recordstream/labels.py b/recordstream/labels.py new file mode 100644 index 0000000..86597c8 --- /dev/null +++ b/recordstream/labels.py @@ -0,0 +1,327 @@ +"""``LabelMap`` — a bidirectional class-name ↔ integer-id map. + +The *fittable* companion to the config-pinned :class:`~recordstream.ops.target.EncodeTarget` / +:class:`~recordstream.ops.target.DecodeTarget`. Those ops carry an explicit ``mapping`` that is +**pinned in config, NOT fitted** at run time, so train / eval / predict share one identical +label→id ordering. :class:`LabelMap` is the piece that *produces* such a pinned mapping: + +* :meth:`LabelMap.fit` derives a deterministic name→id mapping from a stream of raw targets + (backed by scikit-learn's ``LabelEncoder``) — the one-time fit that happens at **train** time. +* :meth:`LabelMap.save` / :meth:`LabelMap.load` persist it (in matrainer's ``class_names.json`` + format) so **eval / predict** reload the *same* mapping rather than refitting on a subset. +* :meth:`LabelMap.encode_op` / :meth:`LabelMap.decode_op` hand back the recordstream ops that apply it. + +So fitting happens once, then the mapping is pinned/persisted — it does NOT contradict the +"mapping pinned in config, not fitted" discipline of the ops; it is how the pin gets created. + +Zero-arg constructible (``LabelMap()`` succeeds with an empty mapping) and side-effect-free in +``__init__`` per the workspace "Partial Initialization & Zero-Arg Construction" convention; the +non-empty requirement is validated lazily in the properties, not in the constructor. + +A label is ALWAYS mappable to ids: :meth:`LabelMap.to_ids` accepts a ``Label`` / ``MultiLabel`` +item, a bare name or id, or a sequence of those, and passes already-encoded values through — so a +consumer never branches on "are these names or ids?". +""" + +import json +from pathlib import Path +from typing import TYPE_CHECKING, Any, Dict, Iterable, Iterator, List, Optional, Sequence, Union + +import numpy as np +from confluid import configurable + +from recordstream.items import Label, MultiLabel, is_class_id +from recordstream.ops.target import DecodeTarget, EncodeTarget + +if TYPE_CHECKING: # Stream imports labels indirectly — keep this annotation-only + from recordstream.core import Stream + + +def _iter_label_values(target: Any) -> Iterator[Any]: + """Yield the individual label values of ``target``, whatever shape it takes. + + A :class:`~recordstream.MultiLabel` yields each of its values, a + :class:`~recordstream.Label` its single value, a bare sequence its elements, and anything + else itself. One walker so :meth:`LabelMap.fit` and :meth:`LabelMap.to_ids` agree on what + "the labels of this target" means. + """ + if isinstance(target, MultiLabel): + yield from target.values + elif isinstance(target, Label): + yield target.value + elif isinstance(target, (list, tuple, set)): + yield from target + elif target is not None: + yield target + + +@configurable +class LabelMap: + """Bidirectional class-name ↔ integer-id map (the fittable companion to ``EncodeTarget``). + + Holds an explicit name→id ``mapping`` (pinned in config), or one fitted from a target stream + via :meth:`fit`. Exposes :attr:`num_classes` / :attr:`class_names`, builds the + :class:`~recordstream.ops.target.EncodeTarget` / :class:`~recordstream.ops.target.DecodeTarget` + that apply it, and round-trips to disk in matrainer's ``class_names.json`` format. + + Args: + mapping: Explicit name→id lookup, e.g. ``{"cat": 0, "dog": 1}``. ``None`` (default) builds an + empty map — valid to construct (zero-arg convention), but the properties raise until it + is populated (by passing a mapping, or via :meth:`fit` / :meth:`from_class_names`). + """ + + def __init__(self, mapping: Optional[Dict[str, int]] = None) -> None: + # Partial / zero-arg: store config only. An empty map is a valid object; the non-empty + # requirement is enforced lazily in the properties, never here. + self.mapping: Dict[str, int] = {str(k): int(v) for k, v in mapping.items()} if mapping else {} + + def _require(self) -> Dict[str, int]: + if not self.mapping: + raise ValueError( + "LabelMap is empty — pass a `mapping`, or build one via LabelMap.fit(targets) / " + "LabelMap.from_class_names(names) / LabelMap.load(path) before use." + ) + return self.mapping + + @property + def num_classes(self) -> int: + """Class count = ``max(id) + 1`` (covers the largest id even if some don't appear).""" + return max(self._require().values()) + 1 + + @property + def class_names(self) -> List[str]: + """``id → name`` list (index == class id). Ids without a name fall back to ``str(id)``.""" + inverse = self.inverse + return [inverse.get(i, str(i)) for i in range(self.num_classes)] + + @property + def inverse(self) -> Dict[int, str]: + """``id → name`` lookup (the inverse of :attr:`mapping`).""" + return {v: k for k, v in self._require().items()} + + def encode_op(self, ignore_unknown: bool = False, default: Any = 0) -> EncodeTarget: + """Return an :class:`~recordstream.ops.target.EncodeTarget` transform that maps name → id via this map.""" + return EncodeTarget(mapping=dict(self._require()), ignore_unknown=ignore_unknown, default=default) + + def encode(self, source: Any) -> "Stream": + """Wrap ``source`` in a :class:`~recordstream.Stream` that applies this map's encode op. + + The one-call form of the two-step idiom every consumer of a name-labelled dataset + writes — ``Stream(source=source, ops=[label_map.encode_op()])``. It lives here because + it is a :class:`LabelMap` operation over a source, not knowledge about any particular + task: a classifier, a detector and a tagger all need the identical wrap. + + Args: + source: Any source/stream the engine accepts. A deferred ``!class:`` marker is + flowed first, so a config-wired source works without the caller flowing it. + + Returns: + A :class:`~recordstream.Stream` yielding the same records with their labels mapped + to integer ids, and carrying this map's ``class_names`` so the vocabulary travels + with the encoded data (read it back with :func:`~recordstream.class_names`). + Which key is encoded follows :class:`EncodeTarget`'s own rule (its blank + ``field`` picks the first :class:`~recordstream.Label`); pass a configured + ``encode_op()`` into a ``Stream`` yourself when you need to pin a different key or + tolerate unknowns. + + Raises: + KeyError: lazily, while iterating, when a label is not in the mapping. That + includes an ALREADY-ENCODED id — unlike :meth:`to_ids`, which passes ids through, + the op is a straight lookup, so double-encoding fails loudly instead of silently + remapping. Wrap a source only when its labels are names (ask + :func:`~recordstream.is_class_id`), or build the op with ``ignore_unknown=True``. + + Example:: + + label_map = LabelMap.fit(iter_key(train_source, "class")) + train_set = label_map.encode(train_source) + """ + from confluid import flow + + from recordstream.core import Stream + + return Stream(source=flow(source), ops=[self.encode_op()], class_names=self.class_names) + + def decode_op(self, ignore_unknown: bool = False, default: Any = None) -> DecodeTarget: + """Return a :class:`~recordstream.ops.target.DecodeTarget` transform that maps id → name via this map.""" + return DecodeTarget(mapping=dict(self.inverse), ignore_unknown=ignore_unknown, default=default) + + @classmethod + def fit(cls, targets: Iterable[Any]) -> "LabelMap": + """Fit a deterministic name→id map from a stream of raw targets. + + Ordering is sorted-unique, so the same set of labels always yields the same mapping — + train and (a refit on the same labels at) eval agree. In practice eval should :meth:`load` + the pinned training map rather than refit on a subset. + + A :class:`~recordstream.MultiLabel` (or any sequence) target contributes EVERY one of its + labels, so a multi-label dataset fits from the same call as a single-label one. + + (This used to delegate to scikit-learn's ``LabelEncoder``, whose ``classes_`` is exactly + ``sorted(set(...))`` — the dependency bought nothing but made a data package require an ML + library, so it was dropped. Ordering is unchanged.) + + Args: + targets: Iterable of raw labels — names, ids, ``Label``/``MultiLabel`` items, or + sequences of any of those. Must yield at least one label. + """ + labels: List[str] = [] + for target in targets: + labels.extend(str(v) for v in _iter_label_values(target)) + if not labels: + raise ValueError("LabelMap.fit: no targets to fit on (empty stream).") + return cls(mapping={name: idx for idx, name in enumerate(sorted(set(labels)))}) + + @classmethod + def from_class_names(cls, names: Sequence[str]) -> "LabelMap": + """Build a map from an ordered ``id → name`` list (the inverse of :attr:`class_names`). + + Args: + names: Ordered class names; the list index becomes the class id. Must be non-empty. + """ + if not names: + raise ValueError("LabelMap.from_class_names: `names` is empty.") + return cls(mapping={str(name): int(i) for i, name in enumerate(names)}) + + def to_ids(self, target: Any) -> List[int]: + """Class ids for ``target`` — the "a label is ALWAYS mappable to ints" contract. + + Accepts every shape a target takes: a :class:`~recordstream.Label` or + :class:`~recordstream.MultiLabel` item, a bare name/id, or a sequence of those. Values + that are ALREADY encoded (:func:`~recordstream.items.is_class_id`) pass through, so this + works on an integer-target dataset even when the map is EMPTY — which is what lets a + consumer stop branching on "are these names or ids?" entirely. + + Raises: + ValueError: A class NAME arrived but this map is empty (nothing to encode with). + KeyError: A name is not in the mapping. + + Example:: + + LabelMap({"cat": 0, "dog": 1}).to_ids(Label("dog")) # [1] + LabelMap().to_ids(Label(2)) # [2] — no map needed + LabelMap({"a": 0, "b": 1}).to_ids(MultiLabel(["a", "b"])) # [0, 1] + """ + ids: List[int] = [] + for value in _iter_label_values(target): + if is_class_id(value): + ids.append(int(value)) + continue + if not self.mapping: + raise ValueError( + f"LabelMap.to_ids: {value!r} is a class NAME but this LabelMap is empty — " + "fit or load a mapping first (LabelMap.fit(targets) / LabelMap.load(path))." + ) + name = str(value) + if name not in self.mapping: + raise KeyError(f"LabelMap.to_ids: {name!r} is not in the mapping (classes: {list(self.mapping)[:8]})") + ids.append(self.mapping[name]) + return ids + + def save(self, path: Union[str, Path]) -> None: + """Persist as ``{"class_names": [...], "num_classes": N}`` — matrainer's ``class_names.json`` format. + + Args: + path: Destination file. Parent directories are created as needed. + """ + out = Path(path).expanduser() + out.parent.mkdir(parents=True, exist_ok=True) + payload = {"class_names": self.class_names, "num_classes": self.num_classes} + out.write_text(json.dumps(payload, indent=2, sort_keys=True)) + + @classmethod + def load(cls, path: Union[str, Path]) -> "LabelMap": + """Restore from a ``class_names.json``-shaped file written by :meth:`save` or matrainer. + + Args: + path: Source file shaped ``{"class_names": [...]}`` (the ``num_classes`` key is optional; + the ordering of ``class_names`` is authoritative). + """ + data = json.loads(Path(path).expanduser().read_text()) + names = data.get("class_names") + if not names: + raise ValueError(f"LabelMap.load: {path} has no non-empty 'class_names' list.") + return cls.from_class_names([str(n) for n in names]) + + +def class_counts(targets: Iterable[Any], num_classes: int, label_map: Optional[LabelMap] = None) -> np.ndarray: + """How often each class id occurs in ``targets`` — the label statistic behind class balancing. + + Every target shape is accepted, because :meth:`LabelMap.to_ids` normalizes them: a + :class:`~recordstream.Label` or :class:`~recordstream.MultiLabel` item, a bare name/id, or a + sequence. A multi-label target counts for EVERY class it names. ``None`` targets are skipped. + + Args: + targets: Already-walked target values (see the note on walking below). + num_classes: Width of the returned vector. Ids outside ``[0, num_classes)`` are IGNORED + rather than raising — a stray label must not abort a training run. + label_map: Map for class-NAME targets. Omit for integer targets: an empty + :class:`LabelMap` passes already-encoded ids through, which is the ``to_ids`` contract. + + Returns: + A ``float64`` vector of length ``num_classes``. + + Note: + This takes ALREADY-WALKED targets, not a source, on purpose. A caller typically walks the + target stream once (``iter_key(source, key)``) and reuses that single pass for several + answers — fitting a :class:`LabelMap`, deriving the class count, and weighting — and a + convenience that walked internally would silently double the passes over the dataset. + + Example:: + + class_counts([Label("cat"), Label("dog"), Label("cat")], 2, LabelMap({"cat": 0, "dog": 1})) + # array([2., 1.]) + """ + mapper = label_map if label_map is not None else LabelMap() + counts = np.zeros(int(num_classes), dtype=np.float64) + for target in targets: + if target is None: + continue + for class_id in mapper.to_ids(target): + if 0 <= class_id < num_classes: + counts[class_id] += 1.0 + return counts + + +def inverse_frequency_weights( + targets: Iterable[Any], num_classes: int, label_map: Optional[LabelMap] = None +) -> Optional[np.ndarray]: + """Per-class weights inversely proportional to observed frequency. + + ``w[c] = total / (num_classes * count[c])`` — a class at exactly the mean frequency gets + ``1.0``, rarer classes more, commoner classes less. Training on a skewed label distribution + biases a model toward the majority class; feeding these weights to a loss (or a framework's + ``class_weight`` knob) is the standard remedy. + + This is a statistic OVER THE DATA, which is why it lives here rather than beside a loss: what + a consuming framework then does with the vector — ``torch.nn``'s ``weight=`` constructor + argument, Keras's ``class_weight`` on ``fit()`` — is that framework's convention, and the + numbers are the same either way. Hence the **numpy** return (the same rule as + :mod:`recordstream.batch`: only ``batch_tensor`` is torch); a torch caller writes + ``torch.as_tensor(weights)``. + + Args: + targets: Already-walked target values (see :func:`class_counts` on why not a source). + num_classes: Width of the returned vector. + label_map: Map for class-NAME targets; omit for integer targets. + + Returns: + A ``float32`` vector of length ``num_classes``, or ``None`` when nothing was counted (an + empty or fully out-of-range target set) — so a caller can tell "no weights" from + "all-zero weights". A class observed **zero** times gets weight ``0.0``, not infinity. + + Example:: + + inverse_frequency_weights([Label(0), Label(0), Label(0), Label(1)], num_classes=2) + # array([0.6667, 2.0], dtype=float32) + """ + counts = class_counts(targets, num_classes, label_map) + total = float(counts.sum()) + if total <= 0: + return None + with np.errstate(divide="ignore", invalid="ignore"): + weights = np.where(counts > 0, total / (num_classes * counts), 0.0) + return weights.astype(np.float32) + + +__all__ = ["LabelMap", "class_counts", "inverse_frequency_weights"] diff --git a/recordstream/loaders.py b/recordstream/loaders.py new file mode 100644 index 0000000..95e1ff4 --- /dev/null +++ b/recordstream/loaders.py @@ -0,0 +1,115 @@ +"""Deferred torch ``DataLoader`` slots for a training runnable — the torch half of batching. + +Batching has two halves (see ``docs/architecture.md`` §10): WHAT a batch contains (a collate) +and WHICH ROWS go in which batch (order, slicing, the short final batch, the per-epoch +reshuffle). torch's second half is the ``DataLoader``, and every training runnable in the +workspace baked the same three deferred loader slots into its constructor — train shuffled, +val/test not, one shared kwarg set. :func:`loader_slots` is that construction written once. + +The slots are :class:`confluid.PartialClass` markers, not live loaders, on purpose: the lazy-init +mandate forbids functional work in a constructor, and the dataset does not exist yet — the run +method flows each slot with ``dataset=`` at run time (``flow(self.train_loader, dataset=ds)``). + +**This module is torch-only and deliberately NOT re-exported from the package root** (the +``recordstream.ops.torch`` pattern): ``import recordstream`` keeps pulling no ML framework, and +a consumer — which is by definition a torch trainer — imports it directly:: + + from recordstream.loaders import loader_slots +""" + +from typing import Any, Callable, List, NamedTuple, Optional + +from confluid import Partial, PartialClass + +from recordstream.collate import collate_records +from recordstream.items import Record + +try: + from torch.utils.data import DataLoader +except ImportError as exc: # pragma: no cover - exercised only on a torch-free install + raise ImportError( + "recordstream.loaders needs torch (it declares DataLoader slots) — install `recordstream[torch]`." + ) from exc + +__all__ = ["LoaderSlots", "loader_slots"] + + +class LoaderSlots(NamedTuple): + """The three deferred loader markers, addressed by split (``slots.train`` / ``.val`` / ``.test``).""" + + train: Partial[DataLoader[Any]] + val: Partial[DataLoader[Any]] + test: Partial[DataLoader[Any]] + + +def loader_slots( + batch_size: int, + num_workers: int, + *, + collate_fn: Callable[[List[Record]], Record] = collate_records, + persistent_workers: Optional[bool] = None, + **loader_kw: Any, +) -> LoaderSlots: + """The train/val/test deferred ``DataLoader`` triple every torch training runnable declares. + + Configurability is TWO-CHANNELLED, and this helper narrows neither: + + * **From code** — any further ``DataLoader`` kwarg (``pin_memory``, ``drop_last``, + ``prefetch_factor``, a ``worker_init_fn``) passes through ``**loader_kw`` and is baked + into all three markers. + * **From config** — the runnable's ``train_loader`` / ``val_loader`` / ``test_loader`` + slots stay whole-value replaceable in YAML (``train_loader: !class:torch.utils.data.DataLoader + {shuffle: false, pin_memory: true, ...}``), exactly as with the inline construction this + replaces; the run method only injects ``dataset=`` at flow time, so every knob a replaced + slot sets survives. + + Args: + batch_size: Rows per batch, baked into all three loaders. + num_workers: Worker processes per loader. ``persistent_workers`` DERIVES from it + (``num_workers != 0``) unless the caller says otherwise. + persistent_workers: Override the derived pairing. ``None`` (the default) derives as + above and is what nearly every run wants; an explicit value is for the case the + derivation cannot see — a HOST fact. macOS terminates persistent workers slowly + enough that a short run spends longer stopping than training, so a config there + says ``persistent_workers: false`` while keeping its workers. ``True`` with + ``num_workers=0`` is refused HERE (torch raises for that pairing when the loader + is first iterated, minutes into a run); it is the one invariant the previously + derived-only value protected by construction. + collate_fn: The batch-shape choice (see the collate registry) — ``collate_records`` + stacks, ``collate_list`` does not (what a detection consumer passes), and a task + collate is any callable. + **loader_kw: Further ``DataLoader`` kwargs, baked into ALL THREE markers. ``shuffle`` + is refused here because it is the ONE per-split kwarg this helper owns (train + ``True``, eval ``False``) — a different split policy is a whole-slot replacement, + not a shared kwarg. + + Returns: + A :class:`LoaderSlots` named tuple — three ``PartialClass(DataLoader, ...)`` markers + (``slots.train`` with ``shuffle=True``, ``slots.val`` / ``slots.test`` with + ``shuffle=False``). Assign them to the runnable's ``train_loader`` / ``val_loader`` / + ``test_loader`` slots and flow each with ``dataset=`` at run time. + """ + if "shuffle" in loader_kw: + raise ValueError( + "loader_slots: 'shuffle' is per-split (train shuffles, val/test do not) and cannot be " + "a shared kwarg — replace the individual loader slot instead " + "(e.g. train_loader: !class:torch.utils.data.DataLoader {shuffle: false, ...})." + ) + if persistent_workers and num_workers == 0: + raise ValueError( + "loader_slots: persistent_workers=True needs num_workers > 0 — torch keeps worker " + "processes alive between epochs and there are none. Raise num_workers, or leave " + "persistent_workers unset to derive it." + ) + shared = dict( + collate_fn=collate_fn, + batch_size=batch_size, + num_workers=num_workers, + persistent_workers=(num_workers != 0) if persistent_workers is None else persistent_workers, + **loader_kw, + ) + return LoaderSlots( + train=PartialClass(DataLoader, shuffle=True, **shared), + val=PartialClass(DataLoader, shuffle=False, **shared), + test=PartialClass(DataLoader, shuffle=False, **shared), + ) diff --git a/recordstream/ops/__init__.py b/recordstream/ops/__init__.py new file mode 100644 index 0000000..2066188 --- /dev/null +++ b/recordstream/ops/__init__.py @@ -0,0 +1,87 @@ +""" +RecordStream operations (record-dict ops). + +Submodules: + - recordstream.ops.numpy: Threshold, ConnectedComponents (+ threshold_array / + connected_component_boxes / resolve_expression helpers) + - recordstream.ops.torch: ToTensor (+ to_tensor helper) + - recordstream.ops.image: ConvertToImage, ConvertToMask (+ value_to_image / normalize_to_uint8 …) + - recordstream.ops.target: EncodeTarget, DecodeTarget, + CocoToTorchVisionDetection, MasksToDetectionBoxes + - recordstream.ops.structure: RenameField, DropField, CopyField, SelectFields + - recordstream.ops.parallel: Parallel (worker-pool sub-pipeline) + - recordstream.ops.enable: Enable (toggle an op-list via one named CLI flag) + - recordstream.ops.random_apply: RandomApply (gate any op behind a Bernoulli flip) + - recordstream.ops.configure: ConfigureOp (per-record parameter injection) + - recordstream.ops.formula: FormulaOp (math formula over one record entry) + - recordstream.ops.sink: RecordSinkOp (adapt a DataSink as a pass-through op) + - recordstream.ops.debug: PrintRecordOp (per-record summary probe) + +The sequential composer ``Pipeline`` lives in :mod:`recordstream.transform` (package-root +export) — one list mixing native ops with bare albumentations / torchvision-v2 transforms. +""" + +import importlib +from typing import Any, Dict, List, Tuple + +from recordstream.ops.configure import ConfigureOp +from recordstream.ops.debug import PrintRecordOp +from recordstream.ops.enable import Enable +from recordstream.ops.formula import FormulaOp +from recordstream.ops.image import ConvertToImage, ConvertToMask +from recordstream.ops.numpy import ConnectedComponents, Threshold +from recordstream.ops.parallel import Parallel +from recordstream.ops.random_apply import RandomApply +from recordstream.ops.sink import RecordSinkOp +from recordstream.ops.structure import CopyField, DropField, RenameField, SelectFields +from recordstream.ops.target import CocoToTorchVisionDetection, DecodeTarget, EncodeTarget, MasksToDetectionBoxes + +__all__ = [ + "CocoToTorchVisionDetection", + "ConfigureOp", + "ConnectedComponents", + "ConvertToImage", + "ConvertToMask", + "CopyField", + "DecodeTarget", + "DropField", + "Enable", + "EncodeTarget", + "FormulaOp", + "MasksToDetectionBoxes", + "Parallel", + "PrintRecordOp", + "RandomApply", + "RenameField", + "RecordSinkOp", + "SelectFields", + "Threshold", + "ToTensor", +] + + +#: Ops whose module needs an optional framework — resolved on first attribute access (PEP 562) +#: so ``import recordstream`` never imports one. ``ToTensor`` is the only such op today: its +#: module is torch by definition, and eagerly re-exporting it here was the single line that made +#: torch a hard dependency of the whole engine. +_OPTIONAL_OPS: Dict[str, Tuple[str, str]] = {"ToTensor": ("recordstream.ops.torch", "torch")} + + +def __getattr__(name: str) -> Any: + """Resolve an optional-framework op on first use, or name the extra that provides it.""" + entry = _OPTIONAL_OPS.get(name) + if entry is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + module_path, extra = entry + try: + return getattr(importlib.import_module(module_path), name) + except ImportError as exc: + raise ImportError( + f"recordstream.ops.{name} needs the {extra!r} extra: pip install 'recordstream[{extra}]'\n" + f"(underlying import error: {exc})" + ) from exc + + +def __dir__() -> List[str]: + """Advertise the lazily-resolved names to ``dir()`` / tab-completion.""" + return sorted({*globals(), *_OPTIONAL_OPS}) diff --git a/recordstream/ops/configure.py b/recordstream/ops/configure.py new file mode 100644 index 0000000..0d3a489 --- /dev/null +++ b/recordstream/ops/configure.py @@ -0,0 +1,102 @@ +"""``ConfigureOp`` — per-record parameter injection. + +Some op parameters are only known per record (a threshold derived from the record's own +max, a crop length derived from its duration). ``ConfigureOp`` runs a ``compute`` op-chain +that derives the value FROM the record, injects it as a constructor attribute of the +``target`` op (post-construction configuration — the confluid paradigm), then applies +``target`` to the ORIGINAL record. + +Modality-neutral — it threads any record through any ops — so it lives in core +recordstream (compose group, alongside ``Pipeline`` / ``Enable`` / ``RandomApply``). +""" + +from typing import Any, List, Optional, cast + +from confluid import configurable, flow +from confluid.fluid import Fluid + +from recordstream.items import Record, item_data + + +@configurable(category="op", group="compose") +class ConfigureOp: + """Compute a value from the record and inject it as a parameter of a target op. + + The ``ops`` chain runs on the incoming record as a SIDE branch — its transformations + are discarded (the original record continues). The ``source``-keyed entry of the + chain's final record becomes the VALUE (payload-unwrapped via ``item_data``): it is + set as the ``param`` attribute of ``target``, then ``target`` is applied to the + original record. + + Confluid ``!class:`` / ``!lazy:`` markers in ``ops`` / ``target`` are flowed lazily at + first call (like ``Pipeline``), so a ``ConfigureOp()`` built from YAML costs nothing. + + YAML — a per-record threshold derived from the record's own statistics: + + .. code-block:: yaml + + - !class:recordstream.ops.configure.ConfigureOp + ops: + - !class:recordstream.ops.formula.FormulaOp {field: image, formula: "amax(a) * 0.5"} + source: image + target: !class:recordstream.ops.numpy.Threshold + low_op: ">=" + param: low_level + + Args: + ops: Value-computing op-chain run on a side-branch copy of the record. Empty = the incoming record. + target: The op to configure and apply; required at call time, validated lazily. + param: Target attribute name to set with the computed value (e.g. ``low_level``). + source: Record key of the side-branch result holding the computed value; required at call time. + """ + + def __init__( + self, + ops: Optional[List[Any]] = None, + target: Optional[object] = None, + param: str = "", + source: str = "", + ) -> None: + # Partial / zero-arg: store config only; target/param/source are validated at first call. + self.ops = list(ops) if ops else [] + self.target = target + self.param = str(param) + self.source = str(source) + + def __call__(self, record: Record) -> Optional[Record]: + if self.target is None: + raise ValueError("ConfigureOp: a 'target' op is required") + if not self.param: + raise ValueError("ConfigureOp: 'param' (the target attribute to set) is required") + if not self.source: + raise ValueError("ConfigureOp: 'source' (the record key holding the computed value) is required") + if isinstance(self.target, Fluid): + self.target = flow(self.target) + # _apply_op = the engine's op-family dispatch, so bare library transforms + # work in the compute chain and as target exactly as in a bare ops list. + from recordstream.core import _apply_op + + current: Record = record + for i, op in enumerate(self.ops): + if isinstance(op, Fluid): + op = flow(op) + self.ops[i] = op + if op is None: + continue + result = _apply_op(current, op) + if result is None: + return None # the compute chain filtered the record (FilterOp semantics) + current = result + if self.source not in current: + raise KeyError(f"ConfigureOp: side-branch result has no key {self.source!r} (keys: {list(current)})") + value = item_data(current[self.source]) + target = cast(Any, self.target) + setattr(target, self.param, value) + return _apply_op(record, target) + + def close(self) -> None: + """Propagate close() to inner ops that own resources.""" + for op in [*self.ops, self.target]: + close_fn = getattr(op, "close", None) + if callable(close_fn): + close_fn() diff --git a/recordstream/ops/debug.py b/recordstream/ops/debug.py new file mode 100644 index 0000000..9aed78c --- /dev/null +++ b/recordstream/ops/debug.py @@ -0,0 +1,115 @@ +"""Record inspection / debug ops.""" + +from typing import Any, Literal, Optional + +from confluid import configurable +from loggair import get_logger + +from recordstream.items import Record, is_item, item_data + +logger = get_logger(__name__) + +# Per-record output is DIAGNOSTIC, so the logger level is restricted to trace/debug (the workspace +# "Diagnostic Log Levels" mandate — never info/warning for per-iteration events). Console visibility +# comes from ``to_console`` (a plain ``print``), independent of the log level. +LogLevel = Literal["trace", "debug"] + +_MAX_VALUE_REPR = 200 + + +def _summarize(value: Any) -> str: + """A compact one-line description: array shape/dtype + a length-capped value preview (large + arrays elided by numpy), else a length-capped ``repr``.""" + if value is None: + return "None" + shape = getattr(value, "shape", None) + dtype = getattr(value, "dtype", None) + if shape is not None and dtype is not None: + values = _cap(_array_values_repr(value)) + return f"{type(value).__name__}(shape={tuple(shape)}, dtype={dtype}, values={values})" + return _cap(repr(value)) + + +def _cap(text: str) -> str: + """Truncate a repr to ``_MAX_VALUE_REPR`` chars with an ellipsis marker.""" + return text if len(text) <= _MAX_VALUE_REPR else text[:_MAX_VALUE_REPR] + "…" + + +def _array_values_repr(value: Any) -> str: + """Compact one-line repr of an array's values (numpy / torch / anything array-like).""" + try: + import numpy as np + + return np.array2string(np.asarray(value), threshold=20, separator=", ").replace("\n", " ") + except Exception: # noqa: BLE001 - best-effort preview; fall back to plain repr + return repr(value) + + +def _summarize_metadata(metadata: Any) -> str: + """Summarise a metadata payload — each VALUE compacted so a large array shows shape/dtype.""" + if isinstance(metadata, dict): + return "{" + ", ".join(f"{key!r}: {_summarize(value)}" for key, value in metadata.items()) + "}" + if isinstance(metadata, list): + return f"[batch of {len(metadata)} metadata dicts]" + return _summarize(metadata) + + +@configurable(category="op", group="debug") +class PrintRecordOp: + """Log / print a summary of each record passing through (a pass-through op). + + A pipeline probe: emits a compact description of the record — each typed value's + shape+dtype plus a length-capped value preview (large arrays elided), and the plain + entries (scalars etc., summarised the same way) — to the Loggair logger (the LOG file + + console) and, by default, to stdout via ``print`` (so it shows in a terminal / a GUI + node output panel regardless of log level). The record is returned UNCHANGED. + + Args: + label: A prefix identifying this probe in the output (e.g. "after-impairments"). + level: Loggair level for the logged line — "trace" or "debug" (per-record output is + diagnostic, so info/warning are deliberately not offered; use ``to_console`` to see it). + include_data: Include the typed item values (shape+dtype + value preview). + include_metadata: Include the plain (non-item) entries — scalars, strings, side values. + to_console: Also ``print`` the line to stdout — guaranteed console / node-panel visibility, + independent of the log level. Set False to log only. + limit: Stop emitting after this many records (None = every record) — avoids flooding on a + large dataset; the op still passes EVERY record through unchanged. + """ + + def __init__( + self, + label: str = "record", + level: LogLevel = "debug", + include_data: bool = True, + include_metadata: bool = True, + to_console: bool = True, + limit: Optional[int] = None, + ) -> None: + self.label = label + self.level = level + self.include_data = include_data + self.include_metadata = include_metadata + self.to_console = to_console + self.limit = limit + self._count = 0 # runtime probe counter — NOT config (per-instance, per-process) + + def __call__(self, record: Record) -> Record: + if self.limit is None or self._count < self.limit: + message = self._format(record) + getattr(logger, self.level)(message) # level is a closed Literal, so this method exists + if self.to_console: + print(message) + self._count += 1 + return record + + def _format(self, record: Record) -> str: + parts = [f"[{self.label} #{self._count}]"] + for key, value in record.items(): + typed = is_item(value) + if typed and not self.include_data: + continue + if not typed and not self.include_metadata: + continue + tag = type(value).__name__ if typed else "plain" + parts.append(f"{key}[{tag}]={_summarize(item_data(value))}") + return " ".join(parts) diff --git a/recordstream/ops/enable.py b/recordstream/ops/enable.py new file mode 100644 index 0000000..29d904b --- /dev/null +++ b/recordstream/ops/enable.py @@ -0,0 +1,158 @@ +"""``Enable`` — toggle one or more ops on/off from a single declared flag. + +A compose-group op (alongside ``Pipeline`` / ``Parallel``): wrap an inner op-list +so the whole chain can be switched on or off from ONE boolean, ``enabled``. +Modality-neutral — it threads any record through any ops — so it lives in core +recordstream, not a domain package. +""" + +from typing import Any, List, Optional + +from confluid import configurable +from loggair import get_logger + +from recordstream.items import Record + +logger = get_logger(__name__) + + +@configurable(category="op", group="compose") +class Enable: + """Wrap one or more ops so the whole chain can be switched on or off. + + ``ops`` is a list; even a single-op guard uses ``ops: [op]``. The wrapper + threads each record through every op in sequence — same semantics as + listing them inline in ``Stream.ops`` — so a whole visualization chain + shares one toggle instead of needing a wrapper per op. + + The toggle is ``enabled``: a DECLARED constructor parameter exposed as a + settable property. Being declared is what makes it reachable from every + front-end — a YAML key, a CLI override, a Python kwarg, a generated + tool/form schema, a canvas widget — through the same introspection every + other ``@configurable`` parameter uses. There is no dynamic toggle-attribute + naming: an unrecognised boolean key on this class is an error, not a flag. + + Several wrappers in one pipeline are told apart by ``name``, which scopes + the CLI flag to that instance (``--.enabled``); a bare ``--enabled`` + still broadcasts to every wrapper at once. + + YAML: + + .. code-block:: yaml + + - !class:recordstream.ops.enable.Enable + name: visualize # ← names THIS instance; scopes its CLI flag + enabled: false + ops: + - !class:recordstream.ops.image.ConvertToImage {} + - !class:recordstream.ops.debug.PrintRecordOp {} + + CLI: + + .. code-block:: bash + + # Targeted — only the wrapper named `visualize` flips. + recordstream run pipeline.yaml --visualize.enabled true + recordstream run pipeline.yaml --visualize.enabled+ # polarity shorthand → True + recordstream run pipeline.yaml --visualize.enabled- # polarity shorthand → False + + # Broadcast — every Enable in the config flips. + recordstream run pipeline.yaml --enabled false + + Python: + + .. code-block:: python + + op = Enable(ops=[convert, save], name="visualize", enabled=False) + op.enabled = True # plain attribute write (validated: must be a bool) + + Inner ops stay deferred (not materialized) until the wrapper actually + fires for the first time, so guarding expensive-to-construct ops with + ``enabled: false`` costs nothing at startup. + + Constraints: + * ``ops`` is required and must be a non-empty list — validated **lazily** + on first call (zero-arg construction stays valid per the recordstream + "Partial Initialization & Zero-Arg Construction" convention). + * ``enabled`` must be a ``bool``; a non-bool raises ``TypeError`` at set time. + * Any OTHER boolean attribute set on the wrapper raises ``ValueError`` on + first call. That is the migration guard for the retired dynamic-toggle + form (``visualize: false`` as a bare kwarg), which would otherwise be + accepted silently by the post-construction paradigm and never read. + + Args: + ops: Non-empty list of ops (native or bare library transforms) gated by the toggle. + enabled: Whether the wrapped ops fire. Settable post-construction, from YAML, + and from the CLI (``--enabled`` / ``--.enabled``). + name: Identifier for THIS instance — scopes its CLI flag to ``--.enabled`` + and labels it in ``--help``. Empty (the default) leaves it unnamed, reachable + only by the broadcast form. + """ + + def __init__(self, ops: Optional[List] = None, enabled: bool = True, name: str = "") -> None: + # Partial / zero-arg: store config only; `ops` non-emptiness and stray-toggle + # rejection are enforced lazily on first call. + self.ops: List = list(ops) if ops else [] + self.name = name + self.enabled = enabled + self._checked = False + + @property + def enabled(self) -> bool: + """Whether the wrapped ops fire for each record (the one toggle).""" + return self._enabled + + @enabled.setter + def enabled(self, value: Any) -> None: + # A settable property — not a plain attribute — so `confluid.accepts_key` + # reports it settable and the CLI/YAML override paths admit `enabled`. + if not isinstance(value, bool): + raise TypeError(f"Enable.enabled must be a bool; got {type(value).__name__} ({value!r}).") + self._enabled = value + + def _check(self) -> None: + """Partial one-time validation, run on the first record.""" + if not self.ops: + raise ValueError("Enable requires a non-empty 'ops' list.") + stray = [key for key, value in vars(self).items() if isinstance(value, bool) and not key.startswith("_")] + if stray: + raise ValueError( + f"Enable: unexpected boolean attribute(s) {stray} — the toggle is 'enabled'. " + f"Dynamic toggle names are retired: write `name: {stray[0]}` + `enabled: ` " + f"and toggle it with `--{stray[0]}.enabled true`." + ) + + def __call__(self, record: Record) -> Optional[Record]: + if not self._checked: + self._check() + self._checked = True + if not self.enabled: + return record + from confluid import flow + from confluid.fluid import Fluid + + # _apply_op = the engine's op-family dispatch, so bare library transforms + # run under the toggle exactly as in a bare ops list. + from recordstream.core import _apply_op + + current: Optional[Record] = record + for i, op in enumerate(self.ops): + if current is None: + return None + if isinstance(op, Fluid): + op = flow(op) + self.ops[i] = op + if op is None: + continue + current = _apply_op(current, op) + return current + + def close(self) -> None: + """Propagate close to inner ops that own resources (e.g. RecordSinkOp).""" + for op in self.ops: + close_fn = getattr(op, "close", None) + if callable(close_fn): + close_fn() + + +__all__ = ["Enable"] diff --git a/recordstream/ops/formula.py b/recordstream/ops/formula.py new file mode 100644 index 0000000..24051a7 --- /dev/null +++ b/recordstream/ops/formula.py @@ -0,0 +1,65 @@ +"""``FormulaOp`` — evaluate a math formula over one record entry. + +The op-form of a visual canvas *Math* node: a restricted Python expression over one +named variable bound to the ``field``-keyed record value (plus the stdlib ``math`` +namespace and the scalar helpers ``abs``/``min``/``max``/``round``/``pow`` — no builtins, +so ``__import__``/``open``/``exec`` are unavailable). Its main consumer is an ops-export's +value-chain compilation: an on-canvas ``… → Extract → Math → widget`` wire becomes +``ConfigureOp(ops=[…, FormulaOp(field, formula)], target=…, param=…)``, so the per-record +value survives serialization. +""" + +import math as _math +from typing import Any, Dict + +import numpy as _np +from confluid import configurable + +from recordstream.items import Record, item_data, with_data + +# Every public ``math`` symbol + the scalar built-in helpers, mirroring the canvas Math +# node's namespace. The bound variable shadows same-named constants (e.g. ``e``). +_FORMULA_NAMESPACE: Dict[str, Any] = {k: getattr(_math, k) for k in dir(_math) if not k.startswith("_")} +_FORMULA_NAMESPACE.update({"abs": abs, "min": min, "max": max, "round": round, "pow": pow}) +# Array reducers, FUNCTION style (``amax(a) * 0.5``) — pre-bound numpy callables whose +# internal lazy imports resolve via numpy's own globals. The ATTRIBUTE form (``a.max()``) +# is NOT guaranteed under the sandbox: numpy's C reductions lazy-import through the +# CALLING frame, whose ``__builtins__`` is empty here (KeyError: '__import__') unless some +# earlier code already warmed that import in this process. Teach the function form. +_FORMULA_NAMESPACE.update({"amax": _np.max, "amin": _np.min, "mean": _np.mean, "std": _np.std, "median": _np.median}) + + +@configurable(category="op", group="compose") +class FormulaOp: + """Replace the ``field``-keyed record value with ``formula`` evaluated over it. + + Args: + formula: Expression over ``var`` — math.*, abs/min/max/round/pow + reducers amax/amin/mean/std/median. + field: Record key whose value the formula reads and replaces; required at call time. + var: Variable name the incoming value binds to. Defaults to ``a``. + """ + + def __init__(self, formula: str = "a", field: str = "", var: str = "a") -> None: + # Partial / zero-arg: store config only; formula and field are validated at first call. + self.formula = str(formula) + self.field = str(field) + self.var = str(var) + + def __call__(self, record: Record) -> Record: + if not self.formula.strip(): + raise ValueError("FormulaOp: 'formula' must be a non-empty expression") + if not self.field: + raise ValueError("FormulaOp: 'field' (the record key to evaluate over) is required") + if self.field not in record: + raise KeyError(f"FormulaOp: record has no key {self.field!r} (keys: {list(record)})") + item = record[self.field] + namespace = {**_FORMULA_NAMESPACE, self.var: item_data(item)} + try: + value = eval(self.formula, {"__builtins__": {}}, namespace) # noqa: S307 - restricted namespace + except Exception as exc: + raise ValueError(f"FormulaOp: formula {self.formula!r} failed: {exc}") from exc + try: + new_value = with_data(item, value) + except TypeError: + new_value = value # a plain (non-item) value is replaced verbatim + return {**record, self.field: new_value} diff --git a/recordstream/ops/image.py b/recordstream/ops/image.py new file mode 100644 index 0000000..7e34412 --- /dev/null +++ b/recordstream/ops/image.py @@ -0,0 +1,844 @@ +"""Generic, modality-agnostic image conversion for RecordStream pipelines. + +This is the single home for "turn an arbitrary value into an image": the +:class:`ConvertToImage` op plus the library functions +(:func:`value_to_image` / :func:`record_to_image`) that back it and the GUI +record preview. It lives in recordstream (not waivefront) because the conversion is +fully generic — a 2-D map, a CHW tensor, a PIL image, a boolean mask all render +the same way regardless of domain — so every project (waivefront's spectrogram +render, any image dataset preview, GUI viewer nodes) reuses ONE implementation. + +Domain-specific rendering stays in the consuming package: waivefront's +``RenderOverlaysOp`` draws signal-region rectangles on top of the PIL image this +op produces, and ``RenderSignalPlotOp`` builds IQ time/freq/constellation panels. +Those need signal semantics; this op does not. + +PIL is a hard dependency here (used directly for the image rendering). Matplotlib +is imported lazily inside :func:`_apply_colormap` — only non-``"gray"`` colormaps +need it, so the pure-greyscale path stays matplotlib-free. +""" + +from typing import Any, Dict, List, Literal, Optional, Sequence, Tuple, get_args + +import numpy as np +from confluid import configurable +from loggair import get_logger +from PIL import Image, ImageDraw + +from recordstream._compat import is_torch_tensor +from recordstream.items import Boxes +from recordstream.items import Image as ImageItem +from recordstream.items import Mask as MaskItem +from recordstream.items import NDArrayItem, Record, is_item, item_data, item_value +from recordstream.transform import Transform + +logger = get_logger("recordstream.ops.image") + + +def normalize_to_uint8( + arr: np.ndarray, + vmin: Optional[float] = None, + vmax: Optional[float] = None, +) -> np.ndarray: + """Min-max normalize ``arr`` to ``uint8`` in ``[0, 255]``. + + ``vmin`` / ``vmax`` pin the scale when given (clamping out-of-range values); otherwise the + array's finite min / max are used. Non-finite entries are folded to the bounds; a degenerate + range yields all-zeros. The single quantization source of truth (the 2-D-map / float-array + paths of :func:`value_to_image` call it directly). + """ + arr = np.asarray(arr).astype(np.float32) + finite = arr[np.isfinite(arr)] + lo = float(vmin) if vmin is not None else (float(finite.min()) if finite.size else 0.0) + hi = float(vmax) if vmax is not None else (float(finite.max()) if finite.size else 0.0) + if hi <= lo: + return np.zeros(arr.shape, dtype=np.uint8) + filled = np.nan_to_num(arr, nan=lo, posinf=hi, neginf=lo) + norm = (filled - lo) / (hi - lo) + return np.asarray(np.clip(norm, 0.0, 1.0) * 255.0, dtype=np.uint8) + + +# Closed set of supported matplotlib colormaps — the SINGLE source of truth for every colormap knob +# across the workspace (``value_to_image`` / ``record_to_image`` / ``ConvertToImage`` and, via +# re-export, waivefront's renderers) AND for GUI colormap dropdowns (which read ``COLORMAPS``). +# A closed ``Literal`` (never a bare ``str``) makes the choice self-documenting and machine- +# introspectable: visual-editor palettes, navigaitor's form-spec, and MCP tool schemas enumerate the +# options straight from the annotation via ``typing.get_args`` instead of hard-coding a parallel list +# that silently drifts. ``"gray"`` is the greyscale path (special-cased in ``_apply_colormap``); every +# other name resolves through ``matplotlib.colormaps[name]``. Per the workspace "closed Literal" +# mandate, derive the runtime tuple FROM the Literal (``get_args``) — never restate the values. +Colormap = Literal[ + "viridis", + "plasma", + "inferno", + "magma", + "cividis", + "gray", + "hot", + "cool", + "jet", + "turbo", + "twilight", + "hsv", +] +COLORMAPS: Tuple[Colormap, ...] = get_args(Colormap) + + +def _apply_colormap(spec_u8: np.ndarray, colormap: Colormap) -> Image.Image: + """Turn a ``(H, W)`` uint8 magnitude map into an RGB PIL image. + + ``colormap="gray"`` reproduces the greyscale-to-RGB path (matplotlib-free). + Any other name is resolved through ``matplotlib.colormaps[name]`` (lazily + imported) so standard cmaps (``"hot"``, ``"viridis"``, ``"magma"``, + ``"plasma"``, ``"inferno"``, ``"turbo"``, …) are supported. + """ + if colormap == "gray": + return Image.fromarray(spec_u8, mode="L").convert("RGB") + import matplotlib + + cmap = matplotlib.colormaps[colormap] + rgba = cmap(spec_u8.astype(np.float32) / 255.0) + rgb = (rgba[..., :3] * 255.0).astype(np.uint8) + return Image.fromarray(rgb, mode="RGB") + + +def _text_to_image(text: str, width: int = 512, height: int = 160) -> np.ndarray: + """Render a short string to an ``(H, W, 3)`` uint8 image (non-image fallback).""" + img = Image.new("RGB", (width, height), color=(30, 30, 30)) + draw = ImageDraw.Draw(img) + max_chars = max(1, width // 7) + lines = [text[i : i + max_chars] for i in range(0, min(len(text), max_chars * 8), max_chars)] + draw.multiline_text((6, 6), "\n".join(lines) or "", fill=(220, 220, 220)) + return np.array(img) + + +def image_frame(value: Any) -> Optional[Tuple[int, int]]: + """The ``(H, W)`` raster of an image-bearing value, or ``None`` when it is not one. + + The reference frame a :class:`~recordstream.Boxes`' boxes are stated in is a raster, so + "what raster is this?" is asked wherever boxes and pixels have to agree — the coupled + image+boxes resize reads it to derive its scale factors, the ops that CREATE a target read + it to record the frame on the item, and any consumer comparing the two reads it to notice a + desync. It was written out per call site before it was extracted. + + It reads the DECLARED layout and does not guess: an :class:`~recordstream.Image` item is + trusted for its ``layout``, a PIL image for its ``size`` (which is ``(W, H)`` — the one + transposed convention here), and a bare array is read as ``HWC``, the layout ``Image`` + documents and the one every pre-tensor path in this package produces. It deliberately does + NOT sniff a channel axis: this package already carries three separate, deliberately + divergent channels-first heuristics, and a fourth guessing one HERE would silently mislabel + the frame that box coordinates are validated against. + """ + payload = item_data(value) if is_item(value) else value + if hasattr(payload, "size") and hasattr(payload, "convert"): # PIL: size is (W, H) + width, height = payload.size + return int(height), int(width) + shape = getattr(payload, "shape", None) + if shape is None or len(shape) not in (2, 3): + return None + if isinstance(value, ImageItem) and getattr(value, "layout", "HWC") == "CHW" and len(shape) == 3: + return int(shape[1]), int(shape[2]) + return int(shape[0]), int(shape[1]) + + +def _render_rgb(value: Any, colormap: Colormap) -> np.ndarray: + """Render an arbitrary value to an ``(H, W, 3)`` uint8 RGB image WITHOUT resizing. + + The core of :func:`value_to_image` factored out so callers that need their + own resize policy (e.g. :class:`ConvertToImage`'s exact ``width``/``height``) + don't pay a double resize. Handles PIL images, torch tensors, numpy arrays + (2-D maps → ``colormap``; 3-D → image with channel coercion; bool → 0/255); + anything else falls back to a text rendering of its ``repr``. + """ + data: Any = value + + if hasattr(data, "convert"): # PIL.Image.Image + data = np.array(data.convert("RGB")) + elif is_torch_tensor(data): + data = data.detach().cpu().numpy() + + if not isinstance(data, np.ndarray): + return _text_to_image(repr(data)) + + arr = np.squeeze(np.asarray(data)) + + if arr.dtype == np.bool_: + arr = arr.astype(np.uint8) * 255 + + if arr.ndim == 2: + return np.array(_apply_colormap(normalize_to_uint8(arr), colormap)) + if arr.ndim == 3: + # Normalize channel position to trailing (HWC). + if arr.shape[0] in (1, 3, 4) and arr.shape[2] not in (1, 3, 4): + arr = np.transpose(arr, (1, 2, 0)) + channels = arr.shape[2] + if channels == 3: + pass + elif channels == 1: + arr = np.repeat(arr, 3, axis=2) + elif channels >= 4: + arr = arr[..., :3] + else: # 2 channels (or other) — replicate the first + arr = np.repeat(arr[..., :1], 3, axis=2) + return arr if arr.dtype == np.uint8 else normalize_to_uint8(arr) + return _text_to_image(f"input ndim={arr.ndim}, shape={arr.shape}") + + +def _bound_longest_side(rgb: np.ndarray, max_size: int) -> np.ndarray: + """Downscale an ``(H, W, 3)`` image so its longest side is ≤ ``max_size`` (aspect preserved).""" + height_px, width_px = rgb.shape[:2] + longest = max(height_px, width_px) + if max_size <= 0 or longest <= max_size: + return rgb.astype(np.uint8) + scale = max_size / longest + resized = Image.fromarray(rgb).resize( + (max(1, int(width_px * scale)), max(1, int(height_px * scale))), + Image.Resampling.BILINEAR, + ) + return np.array(resized).astype(np.uint8) + + +def value_to_image(value: Any, colormap: Colormap = "viridis", max_size: int = 512) -> np.ndarray: + """Render an arbitrary value (any record entry) to an ``(H, W, 3)`` uint8 RGB image. + + A generic, modality-agnostic preview usable from any RecordStream pipeline (and + by a GUI record extractor, which renders the selected field). Handles: + + * ``PIL.Image`` — converted to RGB; + * ``torch.Tensor`` — detached to numpy (CHW collapsed to HWC below); + * ``np.ndarray`` — 2-D maps go through ``colormap`` (one of the supported + colormaps — see ``Colormap``; ``"gray"`` for greyscale); 3-D arrays are + treated as images (a leading channel axis is transposed to trailing, + 1/2/4-channel coerced to 3); boolean masks become 0/255; floating arrays + are min-max normalized. + + Anything else (e.g. a bbox list) falls back to a text rendering of its + ``repr`` so the caller still shows *something* rather than erroring. + ``max_size`` bounds the longest side. + + Args: + value: The value to render (image / tensor / ndarray / mask, else a text repr of its ``repr``). + colormap: Colormap applied to 2-D maps — one of the supported names (see ``Colormap``; ``"gray"`` = greyscale). + max_size: Maximum length in pixels of the longest image side; larger renders are downscaled. + """ + return _bound_longest_side(_render_rgb(value, colormap), max_size) + + +def record_to_image(record: Record, colormap: Colormap = "viridis", max_size: int = 512) -> np.ndarray: + """Render a record's first array-bearing value to an ``(H, W, 3)`` uint8 RGB image for display. + + Thin wrapper over :func:`value_to_image` (which does the modality-agnostic rendering) + applied to the payload of the record's first array-bearing (2-D / 3-D) value. Use + :func:`value_to_image` directly to render an arbitrary value. + + Args: + record: The record to preview; its first array-bearing (2-D / 3-D) value is rendered. + colormap: Colormap applied to 2-D maps — one of the supported names (see ``Colormap``; ``"gray"`` = greyscale). + max_size: Maximum length in pixels of the longest image side; larger renders are downscaled. + """ + for value in record.values(): + arr = _coerce_to_ndarray(item_data(value)) + if arr is not None and arr.ndim in (2, 3): + return value_to_image(item_data(value), colormap=colormap, max_size=max_size) + raise ValueError(f"record_to_image: no array-bearing value in record (keys: {list(record)})") + + +# --------------------------------------------------------------------------- # +# Array introspection helpers — channel selection + histogram. +# +# These back GUI "Array / Tensor Histogram" viewer nodes (and are usable +# from any pipeline / notebook): a generic, modality-agnostic way to look at the +# RAW numeric values of an array/tensor — pick a channel, render it, and bin its +# values. Pure functions (NOT @configurable ops): they measure/derive, they don't +# transform a record, so they're library helpers like value_to_image — not canvas +# nodes. They live here (not in the GUI node) so the computation is reusable +# and unit-tested, per the workspace "rendering/analysis lives in recordstream" mandate. +# --------------------------------------------------------------------------- # + + +def _coerce_to_ndarray(value: Any) -> Optional[np.ndarray]: + """Best-effort view of an arbitrary value as a numeric ``np.ndarray`` for analysis. + + PIL image → RGB array, ``torch.Tensor`` → detached numpy, complex array → + magnitude (``abs``), list/scalar → ``np.asarray``. Returns ``None`` when the + value cannot sensibly be viewed as a numeric array (string/bytes/None, or an + object-dtype array such as a list of ragged things). + """ + data: Any = value + if data is None or isinstance(data, (str, bytes)): + return None + if hasattr(data, "convert"): # PIL.Image.Image + data = np.array(data.convert("RGB")) + elif is_torch_tensor(data): + data = data.detach().cpu().numpy() + try: + arr = np.asarray(data) + except Exception: # pragma: no cover - defensive: exotic objects np can't view + return None + if arr.dtype == object: + return None + if np.iscomplexobj(arr): + arr = np.abs(arr) + return arr + + +def _squeeze_to_3d(arr: np.ndarray) -> np.ndarray: + """Squeeze size-1 axes, then drop leading axes until at most 3-D (a ``[B,C,H,W]`` → first item).""" + arr = np.squeeze(arr) + while arr.ndim > 3: + arr = arr[0] + return arr + + +def _channel_axis(shape: Tuple[int, ...]) -> int: + """Index of the channel axis of a 3-D shape: the SMALLEST axis (channels-are-fewest convention). + + Deliberately distinct from the other two channel heuristics in this workspace, each scoped to a + narrower job: :func:`_render_rgb`'s ``{1,3,4}``-membership test is RGB-render-specific (it only + recognises 1/3/4-channel *images*), and a GUI extractor's float-only mask rule is + mask-specific (float-only). For a general N-channel feature map (e.g. an 8-channel tensor) the + smallest-axis rule is the most defensible default; documented here so the three never look like an + accidental disagreement. + """ + return int(np.argmin(shape)) + + +def select_channel(value: Any, channel: int = -1) -> np.ndarray: + """Reduce an arbitrary array/tensor to a single 2-D ``float32`` map for the given channel. + + The view used both for rendering one channel and for the per-pixel hover readout: + + * a 2-D array passes through; a 1-D array becomes a ``(1, N)`` strip; a scalar a ``(1, 1)`` cell; + * a 3-D array selects ``channel`` along its channel axis (the smallest axis — see + :func:`_channel_axis`); ``channel < 0`` collapses that axis by **mean** (an "all channels" view); + * higher-rank arrays drop leading axes to 3-D first; complex data is magnitude (``abs``). + + Out-of-range ``channel`` is clamped into ``[0, channels-1]``. A non-numeric value yields a ``(1, 1)`` + zero map (so callers always get a real 2-D array). + + Args: + value: The array / tensor / PIL image / scalar to view. + channel: Channel index to select; ``-1`` (default) means "all" → mean across the channel axis. + """ + arr = _coerce_to_ndarray(value) + if arr is None: + return np.zeros((1, 1), dtype=np.float32) + arr = _squeeze_to_3d(np.asarray(arr, dtype=np.float32)) + if arr.ndim == 0: + return arr.reshape(1, 1) + if arr.ndim == 1: + return arr.reshape(1, -1) + if arr.ndim == 2: + return arr + # 3-D: the channel axis is the smallest axis. + caxis = _channel_axis(arr.shape) + n_channels = arr.shape[caxis] + if channel is None or channel < 0: + return np.asarray(arr.mean(axis=caxis), dtype=np.float32) + idx = min(max(int(channel), 0), n_channels - 1) + return np.asarray(np.take(arr, idx, axis=caxis), dtype=np.float32) + + +def channel_count(value: Any) -> int: + """Number of channels of an array/tensor: 1 for ≤2-D data, the smallest-axis size for 3-D, 0 for non-arrays.""" + arr = _coerce_to_ndarray(value) + if arr is None: + return 0 + sq = _squeeze_to_3d(np.asarray(arr)) + return int(sq.shape[_channel_axis(sq.shape)]) if sq.ndim == 3 else 1 + + +def array_histogram(value: Any, bins: int = 256, channel: int = -1) -> Dict[str, Any]: + """Bin the values of an array/tensor into a histogram + summary statistics. + + Counts and statistics are taken over **finite** values only (``NaN`` / ``±inf`` are dropped, so + the result is always JSON-safe — no non-finite floats leak into ``min``/``max``/``bin_edges``). + When ``channel >= 0`` the histogram is of that single channel's plane; ``channel < 0`` histograms + **every** element across all channels. + + Returns a dict with ``counts`` (length ``bins``), ``bin_edges`` (length ``bins+1``), ``min`` / + ``max`` / ``mean`` / ``std`` (``None`` when there are no finite values), ``count`` (number of + finite values) and ``channels`` (detected channel count). A degenerate all-equal array bins into + the first bin over a unit-wide range. + + Args: + value: The array / tensor / PIL image / scalar to histogram. + bins: Number of histogram bins (clamped to at least 1). + channel: Channel to histogram; ``-1`` (default) histograms all elements across channels. + """ + bins = max(1, int(bins)) + arr = _coerce_to_ndarray(value) + channels = channel_count(value) + if arr is None: + flat = np.empty((0,), dtype=np.float32) + elif channel is not None and channel >= 0: + flat = select_channel(value, channel).astype(np.float32).ravel() + else: + flat = np.asarray(arr, dtype=np.float32).ravel() + finite = flat[np.isfinite(flat)] + if finite.size == 0: + edges = np.linspace(0.0, 1.0, bins + 1) + return { + "counts": [0] * bins, + "bin_edges": edges.tolist(), + "min": None, + "max": None, + "mean": None, + "std": None, + "count": 0, + "channels": channels, + } + lo = float(finite.min()) + hi = float(finite.max()) + # A flat array (all values equal) has a zero-width range — pin a deterministic unit range so the + # single populated bin is predictable (np.histogram would otherwise auto-pad to lo±0.5). + hi_edge = hi if hi > lo else lo + 1.0 + # Pass EXPLICIT bin edges (np.linspace), NOT `bins=, range=(lo, hi)`. numpy 2.2.x's uniform + # fast path block-accumulates with `np.bincount(...)` for arrays larger than its 65536-element + # block, and on the workspace build that miscomputes the bincount length so `n += bincount(...)` + # dies with "operands could not be broadcast together with shapes (256,) (257,) (256,)" — i.e. it + # fails on any real image/spectrogram (>65536 px) while passing on the small arrays unit tests use. + # The explicit-edges path (searchsorted) sidesteps that bug and is otherwise identical: the last + # bin is closed, so values == hi are still counted (sum(counts) == finite.size). + edges = np.linspace(lo, hi_edge, bins + 1) + counts, edges = np.histogram(finite, bins=edges) + return { + "counts": counts.astype(int).tolist(), + "bin_edges": edges.astype(float).tolist(), + "min": lo, + "max": hi, + "mean": float(finite.mean()), + "std": float(finite.std()), + "count": int(finite.size), + "channels": channels, + } + + +def confusion_matrix_payload( + matrix: Any, + class_names: Optional[Sequence[Any]] = None, +) -> Dict[str, Any]: + """Structure a confusion matrix + class names into a JSON-safe payload for a GUI viewer. + + Backs GUI *Confusion Matrix* viewer nodes. + The MATH that lives here is the three normalizations (the viewer toggles between them WITHOUT a + re-run — the JS only colours + labels + hovers): ``true`` (each row / actual-class sums to 1), + ``pred`` (each column / predicted-class sums to 1) and ``all`` (the whole matrix sums to 1). Every + float is finite-checked (``NaN``/``±inf`` → ``None``, never a misleading substitute) so the payload + survives ComfyUI's ``json.dumps`` websocket encoding — mirroring :func:`array_histogram`. A row / + column whose count-sum is ``0`` normalises to ``None`` (undefined, not ``0``). + + Args: + matrix: A square ``N×N`` confusion matrix (integer counts) as an array / tensor / nested list. + class_names: Optional length-``N`` class labels; defaults to ``["0", "1", …, "N-1"]``. + + Returns a dict with ``counts`` (``N×N`` ints), ``normalized`` (``{"true","pred","all"}``, each + ``N×N`` floats or ``None``), ``class_names`` (length ``N``), ``n_classes``, and ``total``. A + non-square / empty / non-2-D input yields ``{"n_classes": 0, ...}`` + a ``message``. + """ + arr = _coerce_to_ndarray(matrix) + if arr is None or arr.ndim != 2 or arr.shape[0] != arr.shape[1] or arr.shape[0] == 0: + shape = None if arr is None else tuple(int(d) for d in arr.shape) + return { + "counts": [], + "normalized": {"true": [], "pred": [], "all": []}, + "class_names": [], + "n_classes": 0, + "total": 0, + "message": f"not a square 2-D confusion matrix (shape {shape})", + } + + counts = np.asarray(arr).astype(np.int64) + n = int(counts.shape[0]) + total = int(counts.sum()) + row_sums = counts.sum(axis=1) # per true class + col_sums = counts.sum(axis=0) # per predicted class + + def _normed(divisor: np.ndarray) -> list: + # Element-wise count / divisor; a 0 divisor (empty row/col/matrix) -> None (undefined). + out: list = [] + for i in range(n): + row: list = [] + for j in range(n): + d = float(divisor[i, j]) + row.append(_sanitize_finite(counts[i, j] / d) if d != 0.0 else None) + out.append(row) + return out + + names = [str(c) for c in class_names] if class_names is not None else [str(i) for i in range(n)] + # Pad / trim to exactly N so the viewer always has one label per row/column. + names = (names + [str(i) for i in range(len(names), n)])[:n] + + return { + "counts": [[int(c) for c in row] for row in counts.tolist()], + "normalized": { + "true": _normed(np.broadcast_to(row_sums.reshape(n, 1), (n, n))), + "pred": _normed(np.broadcast_to(col_sums.reshape(1, n), (n, n))), + "all": _normed(np.full((n, n), float(total))), + }, + "class_names": names, + "n_classes": n, + "total": total, + } + + +def _is_square_2d(value: Any) -> bool: + """True when ``value`` views as a square ``N×N`` (``N>=1``) numeric array — confusion-matrix shape.""" + arr = _coerce_to_ndarray(value) + return arr is not None and arr.ndim == 2 and arr.shape[0] == arr.shape[1] and arr.shape[0] >= 1 + + +def confusion_matrices_payload(metrics: Any, class_names: Optional[Sequence[Any]] = None) -> List[Dict[str, Any]]: + """Extract EVERY confusion matrix from a metrics result and build a render payload for each. + + The generic counterpart to :func:`confusion_matrix_payload`: a model evaluator emits its FULL + metric results (``name -> value``; scalars, vectors, AND `N×N` matrices) with NO knowledge of which + is a confusion matrix — this scans them and renders all CONFUSION-MATRIX-SHAPED entries (square 2-D, + ``_is_square_2d``, by SHAPE not name), returning one :func:`confusion_matrix_payload` per match + (each tagged with its metric ``name``) in dict order, or ``[]`` when none. A bare square-2D + ``metrics`` (not a dict) is treated as a single matrix named ``"confusion_matrix"``. This is what + lets a GUI *Confusion Matrix* viewer render ALL matrices from one all-metrics output (there + can be several). ``class_names`` labels every matrix the same way (they share the class set). + + Args: + metrics: An evaluator's metric results — a ``dict`` of ``name -> value`` (the usual form), or a + single ``N×N`` matrix. + class_names: Optional length-``N`` class labels applied to each matrix; defaults to indices. + """ + items = list(metrics.items()) if isinstance(metrics, dict) else [("confusion_matrix", metrics)] + out: List[Dict[str, Any]] = [] + for name, value in items: + if _is_square_2d(value): + payload = confusion_matrix_payload(value, class_names=class_names) + payload["name"] = str(name) + out.append(payload) + return out + + +def _sanitize_finite(x: float) -> Optional[float]: + """A finite float rounded for compactness, or ``None`` for ``NaN``/``±inf`` (JSON-safe).""" + v = float(x) + return round(v, 6) if np.isfinite(v) else None + + +# --------------------------------------------------------------------------- # +# Text → image rendering — draw text onto an image (or a fresh canvas). +# --------------------------------------------------------------------------- # + +# Closed 9-grid set of text anchor positions (a closed Literal per the workspace mandate, so the +# choice is a dropdown in GUIs / navigaitor enumerated from one source of truth). +TextPosition = Literal[ + "top-left", + "top", + "top-right", + "center-left", + "center", + "center-right", + "bottom-left", + "bottom", + "bottom-right", +] +TEXT_POSITIONS: Tuple[TextPosition, ...] = get_args(TextPosition) + + +def _text_anchor_xy(position: str, block_w: int, block_h: int, img_w: int, img_h: int, margin: int) -> Tuple[int, int]: + """Top-left ``(x, y)`` for a ``block_w × block_h`` text block per the 9-grid ``position`` + margin.""" + if "left" in position: + x: float = margin + elif "right" in position: + x = img_w - block_w - margin + else: # "top" / "bottom" / "center" (no left/right) → horizontally centered + x = (img_w - block_w) / 2 + if position.startswith("top"): + y: float = margin + elif position.startswith("bottom"): + y = img_h - block_h - margin + else: # "center-*" / left / right (no top/bottom) → vertically centered + y = (img_h - block_h) / 2 + return int(round(x)), int(round(y)) + + +def _wrap_text(draw: "ImageDraw.ImageDraw", text: str, font: Any, max_width: int) -> str: + """Greedy word-wrap so each line fits ``max_width`` px (explicit newlines preserved).""" + out: list = [] + for paragraph in text.split("\n"): + line = "" + for word in paragraph.split(" "): + trial = f"{line} {word}".strip() + if line and draw.textlength(trial, font=font) > max_width: + out.append(line) + line = word + else: + line = trial + out.append(line) + return "\n".join(out) + + +def draw_text( + text: str, + image: Optional[Any] = None, + *, + width: int = 512, + height: int = 256, + font_size: int = 24, + color: str = "white", + background: str = "black", + position: TextPosition = "top-left", + margin: int = 8, + wrap: bool = True, +) -> np.ndarray: + """Render ``text`` onto ``image`` (or a fresh ``background`` canvas) → an ``(H, W, 3)`` uint8 RGB array. + + The single, modality-agnostic "draw text on an image" renderer (a GUI *Draw Text to Image* + node is thin glue over it). When ``image`` is ``None`` a blank ``(height, width)`` canvas of color + ``background`` is created; otherwise the value is coerced to an RGB image (via :func:`_render_rgb`, + so PIL / ndarray / tensor / 2-D maps all work) and drawn on a copy. The text is word-wrapped to the + image width (``wrap``; explicit newlines kept) and anchored per the 9-grid ``position`` with a + ``margin`` inset. Uses PIL's sized default bitmap font. + + Args: + text: The text to draw (multi-line allowed). + image: Background image (PIL / ndarray / tensor / 2-D map); ``None`` makes a blank canvas. + width: Blank-canvas width in pixels (used only when ``image`` is ``None``). + height: Blank-canvas height in pixels (used only when ``image`` is ``None``). + font_size: Font size in points. + color: Text color — any PIL color name or hex (``"white"`` / ``"#ffcc00"`` / ...). + background: Canvas color when ``image`` is ``None`` — any PIL color name or hex. + position: Anchor of the text block — one of the 9-grid ``TextPosition`` values. + margin: Inset in pixels from the edges for non-centered anchors. + wrap: Word-wrap long lines to fit the image width. + """ + from PIL import ImageFont + + if image is None: + img = Image.new("RGB", (max(1, int(width)), max(1, int(height))), color=background) + else: + img = Image.fromarray(_render_rgb(image, "gray")) + draw = ImageDraw.Draw(img) + try: + font = ImageFont.load_default(size=int(font_size)) + except TypeError: # Pillow < 10.1 has no sized default font — fall back to the fixed bitmap font + font = ImageFont.load_default() + rendered = _wrap_text(draw, text, font, max(1, img.width - 2 * margin)) if wrap else text + # multiline_textbbox is relative to the anchor; subtract its offset so the block's top-left lands at (x, y). + bbox = draw.multiline_textbbox((0, 0), rendered, font=font) + block_w, block_h = int(bbox[2] - bbox[0]), int(bbox[3] - bbox[1]) + x, y = _text_anchor_xy(position, block_w, block_h, img.width, img.height, margin) + draw.multiline_text((x - bbox[0], y - bbox[1]), rendered, fill=color, font=font) + return np.array(img) + + +@configurable(category="op", group="image") +class ConvertToImage(Transform): + """An array-bearing field → an ``Image`` item. + + Reads an array-bearing field from the record and writes a fresh + :class:`~recordstream.Image` item (HWC ``uint8`` RGB) under ``output`` (it is the + pipeline's working image). Any other field passes through untouched. + + Rendering is byte-identical to the legacy op — it reuses the SAME + :func:`value_to_image` core (:func:`_render_rgb` → optional flip → resize): a 2-D map is + colormapped, a 3-D array treated as an image, a boolean mask becomes 0/255, floats are + min-max normalized. Sizing matches the legacy op: + + * ``width`` and ``height`` both > 0 → resize to exactly that raster; + * otherwise → bound the longest side by ``max_size``, preserving aspect. + + Unlike the legacy op it does NOT publish ``image_width_px`` / ``image_height_px`` — the + ``Image`` item's array SHAPE carries the pixel dimensions, so a downstream consumer + (e.g. a back-projection) reads them straight off the payload; there is no shared + metadata dict to publish into in the record model. + + Args: + colormap: Colormap applied to 2-D maps — a supported ``Colormap`` name (``"gray"`` = greyscale). + width: Exact output width in pixels; resize to ``(width, height)`` when both width and height are > 0. + height: Exact output height in pixels; resize to ``(width, height)`` when both width and height are > 0. + max_size: When ``width``/``height`` aren't both set, bound the longest side to this many pixels (aspect kept). + flip_vertical: Mirror the image top-to-bottom (e.g. spectrogram row 0 = f_min → display f_max at the top). + field: Name of the source field to render; blank (default) picks the first array-bearing item in the record. + output: Name of the key the ``Image`` item is written to (added if new). + """ + + handles = (NDArrayItem,) + consumes = (NDArrayItem,) + produces = (ImageItem,) + + def __init__( + self, + colormap: Colormap = "gray", + width: int = 0, + height: int = 0, + max_size: int = 512, + flip_vertical: bool = False, + field: str = "", + output: str = "image", + ) -> None: + super().__init__() + self.colormap: Colormap = colormap + self.width = int(width) + self.height = int(height) + self.max_size = int(max_size) + self.flip_vertical = bool(flip_vertical) + self.field = field + self.output = output + # Private, so it stays out of the config surface (it is not a knob) — see + # `_warn_if_it_desyncs_boxes`, which reports the configuration once, not per record. + self._warned_about_boxes = False + + def _find_source(self, record: Record) -> Any: + """Resolve the payload to render (``self.field`` or the first array-bearing item).""" + if self.field: + if self.field not in record: + raise ValueError(f"ConvertToImage: field {self.field!r} not in record (keys: {list(record)})") + return item_data(record[self.field]) + for _key, item in record.items(): + arr = _coerce_to_ndarray(item_data(item)) + if arr is not None and arr.ndim in (2, 3): + return item_data(item) + raise ValueError(f"ConvertToImage: no array-bearing field in record (keys: {list(record)})") + + def _warn_if_it_desyncs_boxes(self, record: Record, before: Tuple[int, int], after: Tuple[int, int]) -> None: + """Warn ONCE when this op resized the pixels of a record whose boxes describe them. + + This op resizes the IMAGE and nothing else, which is correct for what it is — but a + record carrying a :class:`~recordstream.Boxes` states its boxes in a raster, and moving + the pixels out from under them leaves the two disagreeing with no error of its own: every + shape stays valid and only the coordinates become wrong. Downstream that surfaces as a + model quietly training against misplaced targets, which is the expensive way to find out. + + It is a WARNING and not an error because this op cannot know what the boxes describe — a + record may legitimately carry boxes belonging to a different key than the field being + rendered — so the condition is likely, not certain. It fires once per op instance: the + message is about the CONFIGURATION, so a second copy per record only buries it. + """ + if self._warned_about_boxes or before == after: + return + keys = [key for key, value in record.items() if isinstance(value, Boxes)] + if not keys: + return + self._warned_about_boxes = True + logger.warning( + f"ConvertToImage resized {before} -> {after} (H, W) on a record whose {keys} " + f"carries detection boxes — this op moves PIXELS ONLY, so those boxes now describe " + f"a raster that no longer exists. Use recordstream.ops.target.ResizeDetection, which " + f"moves the image and its boxes in one coupled step. (Warned once per op.)" + ) + + def __call__(self, record: Record) -> Record: + rgb = _render_rgb(self._find_source(record), self.colormap) + if self.flip_vertical: + rgb = rgb[::-1, :, :] + if self.width > 0 and self.height > 0: + out_arr = np.array( + Image.fromarray(rgb).resize((self.width, self.height), resample=Image.Resampling.BILINEAR) + ) + else: + out_arr = _bound_longest_side(rgb, self.max_size) + self._warn_if_it_desyncs_boxes(record, rgb.shape[:2], out_arr.shape[:2]) + return {**record, self.output: ImageItem(out_arr, layout="HWC")} + + +@configurable(category="op", group="image") +class ConvertToMask(Transform): + """A mask-bearing field → a :class:`~recordstream.Mask` item of per-pixel class ids. + + The segmentation counterpart of :class:`ConvertToImage`, and the same shape of op: it reads + ONE field and writes a differently-typed item under ``output``, leaving every other key + untouched. A segmentation dataset ships its target as a greyscale/paletted PNG whose pixel + values ARE the class ids (the Oxford-IIIT Pet trimap, Cityscapes label ids, a VOC + segmentation map); this turns that payload into the ``int64`` ``[H, W]`` array every + per-pixel loss expects. + + **It converts and nothing else** — deliberately, because recordstream already owns the rest: + + * **remapping** the ids (1-based trimap → 0-based class ids, a Cityscapes id → trainId + table) is :class:`~recordstream.ops.formula.FormulaOp` over this op's ``output`` + (``formula: a - 1``) or :class:`~recordstream.ops.target.EncodeTarget`; + * **resizing / augmenting** it *together with the image* is a bare albumentations transform + dropped into the same ops list — the engine's op-family dispatch hands it the ``image`` + and ``mask`` keys in ONE call, so a single joint draw moves both and the ``Mask`` type + survives the round trip. That is why ``output`` defaults to ``"mask"``: it is + albumentations' own key vocabulary, so the very next op in the chain finds it. + + ``int64`` is not a knob: a class-id mask is integer by definition, and it is the dtype + ``torch.nn.CrossEntropyLoss`` requires (it rejects int32 with *"expected target dtype to be + Long or Byte, but got Int"*). A library that casts on the way past — albumentations returns + int32 — is corrected at the model boundary by ``batch_tensor(..., dtype=...)``, where the + caller names the contract. + + Args: + field: Name of the source field to read; blank (default) picks the first array/PIL-bearing item. + output: Name of the key the ``Mask`` item is written to (added if new); defaults to ``mask``. + """ + + handles = (NDArrayItem,) + consumes = (NDArrayItem,) + produces = (MaskItem,) + + def __init__(self, field: str = "", output: str = "mask") -> None: + # Partial / zero-arg: store config only. A missing/unusable field is reported at call time. + super().__init__() + self.field = field + self.output = output + + def _find_source(self, record: Record) -> Any: + """Resolve the payload to convert (``self.field``, else the first array/PIL-bearing item). + + The "array or PIL" rule is :class:`~recordstream.ops.torch.ToTensor`'s, not a third + spelling: a mask arrives either already decoded (an ndarray) or as the PIL image a + source handed over, and both are equally normal. + + Unwrapping goes through :func:`~recordstream.item_value` rather than + :func:`~recordstream.item_data`, and the difference is load-bearing here: a source that + does not know a column is a mask hands it over as a :class:`~recordstream.Label` (this + is what ``HuggingFaceSource`` does for every metadata column), whose payload slot is + ``value``, not ``data`` — ``item_data`` would return the ``Label`` itself and the PIL + image inside it would never be found. + """ + if self.field: + if self.field not in record: + raise ValueError(f"ConvertToMask: field {self.field!r} not in record (keys: {list(record)})") + return item_value(record[self.field]) + for _key, item in record.items(): + data = item_value(item) + if isinstance(data, np.ndarray) or is_torch_tensor(data) or hasattr(data, "convert"): + return data + raise ValueError(f"ConvertToMask: no array/PIL-bearing field in record (keys: {list(record)})") + + def __call__(self, record: Record) -> Record: + value = self._find_source(record) + if hasattr(value, "convert"): # a PIL image — an L / P mode plane is already the id map + value = np.asarray(value) + elif is_torch_tensor(value): + value = value.detach().cpu().numpy() + arr = np.asarray(value) + # Squeeze SINGLETON axes only, so a mask stored as [H, W, 1] or [1, H, W] lands as + # [H, W]. An RGB-encoded mask is deliberately NOT collapsed: picking one of three + # channels (or looking up a palette) is a decision this op must not make silently. + if arr.ndim > 2: + arr = np.squeeze(arr) + if arr.ndim != 2: + raise ValueError( + f"ConvertToMask: expected a 2-D [H, W] class-id map, got shape {tuple(np.shape(value))}. " + "An RGB-encoded mask needs a channel/palette decode first (e.g. select_channel)." + ) + return {**record, self.output: MaskItem(arr.astype(np.int64))} + + +__all__ = [ + "Colormap", + "COLORMAPS", + "ConvertToImage", + "ConvertToMask", + "normalize_to_uint8", + "value_to_image", + "record_to_image", + "select_channel", + "channel_count", + "array_histogram", + "confusion_matrix_payload", + "confusion_matrices_payload", + "draw_text", + "TextPosition", + "TEXT_POSITIONS", +] diff --git a/recordstream/ops/numpy.py b/recordstream/ops/numpy.py new file mode 100644 index 0000000..0e54805 --- /dev/null +++ b/recordstream/ops/numpy.py @@ -0,0 +1,321 @@ +import operator +import os +import re +from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Union + +import numpy as np +from confluid import configurable +from loggair import get_logger + +from recordstream.items import Boxes, Mask, NDArrayItem, Record, item_data +from recordstream.transform import Transform + +logger = get_logger(__name__) + + +_EXPR_PATTERN = re.compile(r"\{(\w+)\}|\$(\w+)") + + +def resolve_expression(value: str, meta: Optional[Dict[str, Any]] = None) -> str: + """Substitute ``{key}`` from ``meta`` and ``$NAME`` from ``os.environ``. + + Returns the substituted string verbatim — the caller is responsible for any further + casting (e.g. ``float(...)`` for a numeric expression). In the record model an item + owns its own metadata (there is no shared metadata dict), so ``meta`` is usually empty and + only literals / ``$ENV`` expressions resolve; a ``{key}`` bound then raises ``KeyError``. + + Args: + value: Expression string with ``{meta_key}`` and/or ``$ENV_VAR`` placeholders. + meta: Metadata dict supplying the ``{key}`` substitutions (defaults to empty). + + Raises: + KeyError: A referenced metadata key or environment variable is missing. + """ + meta = meta or {} + + def _repl(match: "re.Match[str]") -> str: + meta_key = match.group(1) + env_name = match.group(2) + if meta_key is not None: + if meta_key not in meta: + raise KeyError( + f"resolve_expression: metadata key {meta_key!r} missing in {value!r}; " + f"available keys: {sorted(meta)}" + ) + return str(meta[meta_key]) + assert env_name is not None + if env_name not in os.environ: + raise KeyError(f"resolve_expression: environment variable {env_name!r} missing in {value!r}") + return os.environ[env_name] + + return _EXPR_PATTERN.sub(_repl, value) + + +# Threshold comparison selectors. Closed ``Literal``s so GUIs / schema generators render the +# choice as a dropdown and the allowed operators stay machine-introspectable via +# ``typing.get_args(...)``. Two distinct types because the lower bound only sensibly uses +# ``>`` / ``>=`` and the upper bound only ``<`` / ``<=``. +LowComparison = Literal[">", ">="] +HighComparison = Literal["<", "<="] + +_LOW_COMPARISONS: Dict[str, Callable[[Any, float], Any]] = {">": operator.gt, ">=": operator.ge} +_HIGH_COMPARISONS: Dict[str, Callable[[Any, float], Any]] = {"<": operator.lt, "<=": operator.le} + + +def _resolve_bound(bound: Union[float, int, str], meta: Optional[Dict[str, Any]]) -> float: + """Resolve a threshold bound (literal / numeric / ``resolve_expression`` string) to a float.""" + if isinstance(bound, str): + resolved = resolve_expression(bound, meta) + try: + return float(resolved) + except (TypeError, ValueError) as exc: + raise ValueError( + f"threshold: expression {bound!r} resolved to {resolved!r}, which is not a number" + ) from exc + try: + return float(bound) + except (TypeError, ValueError) as exc: + raise TypeError(f"threshold bounds must be a number or expression string; got {type(bound).__name__}") from exc + + +def threshold_array( + arr: np.ndarray, + low_level: Optional[Union[float, int, str]] = None, + high_level: Optional[Union[float, int, str]] = None, + low_op: LowComparison = ">", + high_op: HighComparison = "<", + meta: Optional[Dict[str, Any]] = None, +) -> np.ndarray: + """Threshold ``arr`` into a boolean mask using one or both bounds. + + * only ``low_level`` → ``arr low_level`` (values above the floor) + * only ``high_level`` → ``arr high_level`` (values below the ceiling) + * both → both conditions AND-ed together (band-pass) + + At least one of ``low_level`` / ``high_level`` MUST be provided. + """ + if not isinstance(arr, np.ndarray): + raise TypeError(f"threshold_array expects an np.ndarray, got {type(arr).__name__}") + if isinstance(low_level, str) and low_level.strip() == "": + low_level = None + if isinstance(high_level, str) and high_level.strip() == "": + high_level = None + + mask: Optional[np.ndarray] = None + if low_level is not None: + low = _resolve_bound(low_level, meta) + if np.isnan(low): + logger.warning(f"threshold_array: resolved low_level is NaN ({low_level!r}); no values pass the floor.") + else: + mask = _LOW_COMPARISONS[low_op](arr, low) + if high_level is not None: + high = _resolve_bound(high_level, meta) + if np.isnan(high): + logger.warning(f"threshold_array: resolved high_level is NaN ({high_level!r}); no values pass the ceiling.") + else: + below = _HIGH_COMPARISONS[high_op](arr, high) + mask = below if mask is None else (mask & below) + if mask is None: + raise ValueError("threshold_array requires at least one of 'low_level' / 'high_level'") + return mask + + +@configurable(category="op", group="numpy") +class Threshold(Transform): + """An array-bearing field → a boolean ``Mask`` item. + + Reads the array at ``field`` (blank = the first array-bearing item in the record) and thresholds + it into a boolean mask with the bound / comparison / expression math (:func:`threshold_array`), + writing a :class:`~recordstream.Mask` item under ``output`` (a threshold mask is an + intermediate that a later op — e.g. :class:`ConnectedComponents` — consumes). Any other key + passes through untouched. + + Each bound is a numeric literal or a ``resolve_expression`` string — ``5.5`` / ``"5.5"`` + (literal) or ``"$REF_SNR"`` (environment variable). NOTE: ``{meta_key}`` expressions have no + metadata source in the record model, so only literals and ``$ENV`` resolve here. + + Args: + low_level: Lower bound (numeric literal or ``$ENV`` expression) compared with ``low_op`` when set; + ``None`` disables the lower bound. + high_level: Upper bound (numeric literal or ``$ENV`` expression) compared with ``high_op`` when set; + ``None`` disables the upper bound. + low_op: Lower-bound comparison — ``">"`` (strict, default) or ``">="`` (inclusive). + high_op: Upper-bound comparison — ``"<"`` (strict, default) or ``"<="`` (inclusive). + field: Name of the array field to threshold; blank (default) picks the first array-bearing item. + output: Name of the key the boolean ``Mask`` item is written to (added if new). + """ + + handles = (NDArrayItem,) + consumes = (NDArrayItem,) + produces = (Mask,) + + def __init__( + self, + low_level: Optional[Union[float, int, str]] = None, + high_level: Optional[Union[float, int, str]] = None, + low_op: LowComparison = ">", + high_op: HighComparison = "<", + field: str = "", + output: str = "mask", + ) -> None: + super().__init__() + self.low_level = low_level + self.high_level = high_level + self.low_op = low_op + self.high_op = high_op + self.field = field + self.output = output + + def _find_array(self, record: Record) -> np.ndarray: + """Resolve the array to threshold (``self.field`` or the first array-bearing item).""" + if self.field: + if self.field not in record: + raise ValueError(f"Threshold: field {self.field!r} not in record (keys: {list(record)})") + data = item_data(record[self.field]) + if not isinstance(data, np.ndarray): + raise TypeError(f"Threshold: field {self.field!r} payload is {type(data).__name__}, expected an array") + return data + for _key, item in record.items(): + data = item_data(item) + if isinstance(data, np.ndarray): + return data + raise ValueError(f"Threshold: no array-bearing field in record (keys: {list(record)})") + + def __call__(self, record: Record) -> Record: + arr = self._find_array(record) + mask = threshold_array(arr, self.low_level, self.high_level, self.low_op, self.high_op) + return {**record, self.output: Mask(mask)} + + +def connected_component_boxes( + mask: np.ndarray, min_area_bins: int = 1, connectivity: int = 4 +) -> List[Tuple[int, int, int, int]]: + """Label connected ``True`` regions of a 2-D bool mask → HALF-OPEN xyxy ``(x0, y0, x1, y1)`` tuples. + + The pixel-box convention every :class:`~recordstream.Boxes` producer emits: x = column, + y = row, far edges EXCLUSIVE — ``mask[y0:y1, x0:x1]`` covers the component exactly. + (Renamed from ``connected_component_bboxes``, which returned INCLUSIVE + ``(row_min, row_max, col_min, col_max)`` tuples — the rename makes a stale caller fail + loudly instead of silently mis-reading axes.) + + Components smaller than ``min_area_bins`` are dropped. ``connectivity`` is ``4`` + (orthogonal neighbors) or ``8`` (orthogonal + diagonal). Shared by :class:`ConnectedComponents` + AND :func:`recordstream.ops.target.masks_to_detection` (its ``connected=True`` mode). Requires + ``scipy`` (``pip install recordstream[vision]``). + """ + if min_area_bins < 1: + raise ValueError(f"min_area_bins must be >= 1; got {min_area_bins!r}") + if connectivity not in (4, 8): + raise ValueError(f"connectivity must be 4 or 8; got {connectivity!r}") + try: + from scipy.ndimage import find_objects, generate_binary_structure, label + except ImportError as exc: + raise ImportError( + "connected-components labeling requires scipy. " + "Install with `pip install recordstream[vision]` or add scipy to your environment." + ) from exc + + structure = generate_binary_structure(2, 1 if connectivity == 4 else 2) + labels, n_components = label(mask, structure=structure) + bboxes: List[Tuple[int, int, int, int]] = [] + if n_components > 0: + for idx, sl in enumerate(find_objects(labels), start=1): + if sl is None: + continue + row_slice, col_slice = sl + area = int((labels[row_slice, col_slice] == idx).sum()) + if area < min_area_bins: + continue + bboxes.append( + ( + int(col_slice.start), + int(row_slice.start), + int(col_slice.stop), + int(row_slice.stop), + ) + ) + return bboxes + + +@configurable(category="op", group="numpy") +class ConnectedComponents(Transform): + """A boolean ``Mask`` → a ``Boxes`` item. + + Reads the :class:`~recordstream.Mask` at ``field`` (blank = the first ``Mask`` in the record, else the + first array-bearing item) as a 2-D boolean array and labels its connected ``True`` regions into + HALF-OPEN pixel xyxy ``(x0, y0, x1, y1)`` tuples via :func:`connected_component_boxes`, + writing them as a :class:`~recordstream.Boxes` item under ``output`` with ``canvas`` set to + the mask's shape (RAW detections, not model predictions). Any other key passes through. + + Components smaller than ``min_area_bins`` are dropped; ``connectivity`` selects the 4- or + 8-neighborhood. Requires ``scipy`` (``pip install recordstream[vision]``). + + Args: + min_area_bins: Minimum component area in bins; smaller connected regions are dropped (``>= 1``). + connectivity: Pixel neighborhood — ``4`` (orthogonal only) or ``8`` (orthogonal + diagonal). + field: Name of the ``Mask`` field to label; blank (default) picks the first ``Mask`` (else first array). + output: Name of the key the ``Boxes`` item is written to (added if new). + """ + + handles = (Mask,) + consumes = (Mask,) + produces = (Boxes,) + + def __init__( + self, + min_area_bins: int = 1, + connectivity: int = 4, + field: str = "", + output: str = "boxes", + ) -> None: + super().__init__() + self.min_area_bins = int(min_area_bins) + self.connectivity = int(connectivity) + self.field = field + self.output = output + + def _find_mask(self, record: Record) -> np.ndarray: + """Resolve the mask to label (``self.field``, else the first ``Mask``, else the first array).""" + if self.field: + if self.field not in record: + raise ValueError(f"ConnectedComponents: field {self.field!r} not in record (keys: {list(record)})") + data = item_data(record[self.field]) + else: + data = None + for _key, item in record.items(): + if isinstance(item, Mask): + data = item_data(item) + break + if data is None: + for _key, item in record.items(): + payload = item_data(item) + if isinstance(payload, np.ndarray): + data = payload + break + if data is None: + raise ValueError( + f"ConnectedComponents: no Mask or array-bearing field in record (keys: {list(record)})" + ) + if not isinstance(data, np.ndarray): + raise TypeError(f"ConnectedComponents expects an np.ndarray mask, got {type(data).__name__}") + if data.ndim != 2: + raise ValueError(f"ConnectedComponents expects a 2-D mask; got shape {data.shape}") + return data + + def __call__(self, record: Record) -> Record: + mask = self._find_mask(record) + boxes = connected_component_boxes(mask, self.min_area_bins, self.connectivity) + # canvas is filled in even for an EMPTY box set — a frame check that silently skips + # exactly the records with nothing to check reports a clean bill for the wrong reason. + return {**record, self.output: Boxes(boxes=list(boxes), canvas=(mask.shape[0], mask.shape[1]))} + + +__all__ = [ + "resolve_expression", + "threshold_array", + "connected_component_boxes", + "LowComparison", + "HighComparison", + "Threshold", + "ConnectedComponents", +] diff --git a/dataflux/ops/parallel.py b/recordstream/ops/parallel.py similarity index 55% rename from dataflux/ops/parallel.py rename to recordstream/ops/parallel.py index ac57f69..cda3d6a 100644 --- a/dataflux/ops/parallel.py +++ b/recordstream/ops/parallel.py @@ -1,18 +1,18 @@ """``Parallel`` — explicit parallel sub-pipeline op. -Place inside a :class:`~dataflux.core.Flux`'s ops list to dispatch each -upstream sample through an inner sub-pipeline (``self.ops``) in a +Place inside a :class:`~recordstream.core.stream.Stream`'s ops list to dispatch each +upstream record through an inner sub-pipeline (``self.ops``) in a spawn-context worker pool. Bounded prefetch caps outstanding work so the executor queue can't grow unboundedly with source length. Falls back to inline sequential application when invoked as a regular -per-sample op (e.g. via :meth:`Flux.__getitem__`) so random access remains +per-record op (e.g. via :meth:`Stream.__getitem__`) so random access remains correct. Note: Do not nest a ``Parallel`` op inside another ``Parallel.ops`` — workers - must not themselves spawn workers. ``Tee``, ``Enable``, and any - pickle-safe per-sample op are fine inside. + must not themselves spawn workers. ``Pipeline``, ``Enable``, and any + pickle-safe per-record op are fine inside. """ from __future__ import annotations @@ -25,54 +25,62 @@ from confluid import configurable, flow from confluid.fluid import Fluid -from dataflux.core import _worker_task -from dataflux.sample import Sample +from recordstream.core import _worker_task +from recordstream.items import Record -@configurable +@configurable(category="op", group="compose") class Parallel: """Run an inner op sub-pipeline in a worker pool with bounded prefetch. Args: - ops: Sequential sub-pipeline applied to each sample inside a worker. + ops: Sequential sub-pipeline applied to each record inside a worker. workers: Number of worker processes (spawn context). Must be >= 1. """ - def __init__(self, ops: List[Any], workers: int = 4) -> None: - if workers < 1: - raise ValueError(f"Parallel(workers={workers!r}): must be >= 1") - self.ops = list(ops) + def __init__(self, ops: Optional[List[Any]] = None, workers: int = 4) -> None: + # Partial / zero-arg: store config only; ``workers >= 1`` is validated lazily in ``stream``. + self.ops = list(ops) if ops else [] self.workers = int(workers) def _materialize_ops(self) -> None: # Confluid post-construction paradigm leaves nested ops as Fluid - # markers; resolve them in-place on first use, mirroring Tee. + # markers; resolve them in-place on first use, mirroring Pipeline. for i, op in enumerate(self.ops): if isinstance(op, Fluid): self.ops[i] = flow(op) - def __call__(self, sample: Sample) -> Optional[Sample]: - # Inline fallback for non-streaming callers (e.g. Flux.__getitem__). + def __call__(self, record: Record) -> Optional[Record]: + # Inline fallback for non-streaming callers (e.g. Stream.__getitem__). Routed through + # _apply_op — the same op-family dispatch the streamed route's _worker_task uses — + # so bare library transforms behave identically. + from recordstream.core import _apply_op + self._materialize_ops() - current: Optional[Sample] = sample + current: Optional[Record] = record for op in self.ops: if current is None: return None - current = op(current) + current = _apply_op(current, op) return current - def stream(self, samples: Iterable[Optional[Sample]]) -> Iterator[Optional[Sample]]: + def stream(self, records: Iterable[Optional[Record]]) -> Iterator[Optional[Record]]: """Stream-level dispatch with bounded prefetch (in-order yield).""" + if self.workers < 1: + raise ValueError(f"Parallel(workers={self.workers!r}): must be >= 1") self._materialize_ops() ctx = multiprocessing.get_context("spawn") limit = max(2 * self.workers, self.workers + 1) + from recordstream.core import _extra_op_families + with concurrent.futures.ProcessPoolExecutor(max_workers=self.workers, mp_context=ctx) as executor: - pending: "deque[concurrent.futures.Future[Optional[Sample]]]" = deque() - for s in samples: + pending: "deque[concurrent.futures.Future[Optional[Record]]]" = deque() + extra_families = _extra_op_families() # ship third-party op families to the workers + for s in records: if s is None: continue - pending.append(executor.submit(_worker_task, s, self.ops)) + pending.append(executor.submit(_worker_task, s, self.ops, extra_families)) if len(pending) >= limit: yield pending.popleft().result() while pending: diff --git a/recordstream/ops/predict.py b/recordstream/ops/predict.py new file mode 100644 index 0000000..46bf16f --- /dev/null +++ b/recordstream/ops/predict.py @@ -0,0 +1,195 @@ +"""Run a task model over records — inference as ONE op in a pipeline. + +This is what lets a graph (or any config) CARRY its inference: a source, its +preprocessing, and a ``ModelPredict`` op wired to a project's model wrapper compose +into one runnable pipeline — a viewer executes it per record and reads the stamped +fields back as layers, and the same document runs offline with ``recordstream run``. +The op lives here (not in a viewer or a project package) because it is fully generic: +the model is any callable ``model(batch)``, duck-typed — no torch import of its own. +""" + +from typing import Any, Dict, Literal, get_args + +import numpy as np +from confluid import configurable +from loggair import get_logger + +from recordstream.items import Boxes, Label + +logger = get_logger(__name__) + +#: What a model's output means — how it is stamped back onto the record. +PredictKind = Literal["classification", "detection", "segmentation", "restoration"] + + +def _to_numpy(value: Any) -> np.ndarray: + if hasattr(value, "detach"): + value = value.detach().cpu() + return np.asarray(value) # type: ignore[no-any-return] + + +def _batch(x: Any) -> Any: + return x[None] if hasattr(x, "shape") else np.asarray(x)[None] + + +def _first(out: Any) -> Any: + """The first element of a batched output (a sequence, a batched array, or a mapping of batched arrays).""" + if isinstance(out, (list, tuple)): + return out[0] + if isinstance(out, dict): + return {k: (v[0] if hasattr(v, "__len__") and len(v) else v) for k, v in out.items()} + if hasattr(out, "shape") and len(out.shape) > 0: + return out[0] + return out + + +def _fields(out: Any, *names: str) -> Any: + """``out[name]`` / ``out.name`` for the first name present.""" + for name in names: + if isinstance(out, dict) and name in out: + return out[name] + if hasattr(out, name): + return getattr(out, name) + raise ValueError(f"model output carries none of {names} (got {type(out).__name__})") + + +def _softmax(logits: np.ndarray) -> np.ndarray: + shifted = logits - logits.max(axis=-1, keepdims=True) + exp = np.exp(shifted) + return np.asarray(exp / exp.sum(axis=-1, keepdims=True), dtype=np.float64) + + +def _image_hw(value: Any) -> "tuple[int, int]": + """The (H, W) of an image field — HWC, HW, or CHW (1/3 channels first).""" + arr = _to_numpy(value) + if arr.ndim == 3 and arr.shape[0] in (1, 3) and arr.shape[-1] not in (1, 3): + return int(arr.shape[1]), int(arr.shape[2]) + return int(arr.shape[0]), int(arr.shape[1]) + + +@configurable(category="op") +class ModelPredict: + """Run a task model on ONE record and stamp its prediction as a record field. + + The model is any callable ``model(batch)`` — typically a project's checkpointed + wrapper. Its real work (build the network, load the checkpoint) happens in its + ``solidify()``, called lazily on the first record, so constructing this op is free. + What gets stamped follows the field conventions a viewer reads back as layers: + + - ``classification``: ``output`` = the predicted class as a :class:`~recordstream.items.Label` + (output shape ``[1, C]`` probs/logits, or a mapping/object with ``probs``/``logits``); + - ``detection``: ``output`` = a :class:`~recordstream.items.Boxes` (per-image + ``boxes``/``scores``/``labels``, pixel xyxy); + - ``segmentation``: ``_mask`` = an int class mask ``[H, W]`` (argmax over + ``[C, H, W]`` logits when needed); + - ``restoration``: ``output`` = the restored image array (channels-first is moved last). + + Args: + model: The callable model (a checkpointed wrapper; solidified on first use). + kind: What the model's output means — ``classification`` / ``detection`` / + ``segmentation`` / ``restoration``. + key: Record field fed to the model. + output: Record field stamped with the prediction (``segmentation`` stamps + ``_mask``). Keep the ``predict`` prefix — that is what marks a + field as a prediction downstream. + device: Where a torch model runs (``cpu`` / ``cuda`` / ``mps``); ignored otherwise. + """ + + def __init__( + self, + model: Any = None, # any callable model wrapper — naming a real type would force a torch-shaped import + kind: PredictKind = "classification", + key: str = "image", + output: str = "predict", + device: str = "cpu", + ) -> None: + if kind not in get_args(PredictKind): + raise ValueError(f"ModelPredict kind must be one of {get_args(PredictKind)}, got {kind!r}") + self.model = model + self.kind = kind + self.key = key + self.output = output + self.device = device + self._ready: Any = None + + def _model(self) -> Any: + if self._ready is None: + model = self.model + if model is None: + raise ValueError("ModelPredict needs 'model' (a callable model wrapper)") + if hasattr(model, "solidify") and callable(model.solidify): + built = model.solidify() + model = built if built is not None else model + if hasattr(model, "eval") and callable(model.eval): + model.eval() + if self.device and hasattr(model, "to") and callable(model.to): + model.to(self.device) + self._ready = model + return self._ready + + def _call(self, batch: Any) -> Any: + model = self._model() + try: + import torch # noqa: F401 + + with torch.no_grad(): + if hasattr(batch, "to") and self.device: + batch = batch.to(self.device) + return model(batch) + except ImportError: + return model(batch) + + def __call__(self, record: Dict[str, Any]) -> Dict[str, Any]: + if self.key not in record: + raise ValueError(f"ModelPredict: record has no field {self.key!r} (fields: {sorted(record)})") + out = self._call(_batch(record[self.key])) + if self.kind == "classification": + return {**record, self.output: self._classification(out)} + if self.kind == "detection": + return {**record, self.output: self._detection(out, record[self.key])} + if self.kind == "segmentation": + return {**record, f"{self.output}_mask": self._segmentation(out)} + return {**record, self.output: self._restoration(out)} + + def _classification(self, out: Any) -> Label: + values = out + if isinstance(out, dict) or (not hasattr(out, "shape") and hasattr(out, "probs")): + values = _fields(out, "probs", "logits") + scores = _to_numpy(_first(values)).astype(np.float64).reshape(-1) + if scores.min() < 0.0 or scores.sum() > 1.0001: + scores = _softmax(scores) + return Label(int(np.argmax(scores))) + + def _detection(self, out: Any, image: Any) -> Boxes: + first = _first(out) + boxes = _to_numpy(_fields(first, "boxes")).reshape(-1, 4) + scores = _to_numpy(_fields(first, "scores")).reshape(-1) if _has(first, "scores") else np.ones(len(boxes)) + labels = ( + _to_numpy(_fields(first, "labels")).reshape(-1).astype(int) + if _has(first, "labels") + else np.zeros(len(boxes), dtype=int) + ) + return Boxes(boxes=boxes.tolist(), labels=labels.tolist(), scores=scores.tolist(), canvas=_image_hw(image)) + + def _segmentation(self, out: Any) -> np.ndarray: + values = out + if isinstance(out, dict) or (not hasattr(out, "shape") and (hasattr(out, "mask") or hasattr(out, "logits"))): + values = _fields(out, "mask", "logits", "probs") + arr = _to_numpy(_first(values)) + return np.argmax(arr, axis=0).astype(np.int64) if arr.ndim == 3 else arr.astype(np.int64) + + def _restoration(self, out: Any) -> np.ndarray: + values = out + if isinstance(out, dict) or (not hasattr(out, "shape") and hasattr(out, "image")): + values = _fields(out, "image") + arr = _to_numpy(_first(values)) + if arr.ndim == 3 and arr.shape[0] in (1, 3) and arr.shape[-1] not in (1, 3): + arr = np.moveaxis(arr, 0, -1) + return arr + + +def _has(out: Any, name: str) -> bool: + return (isinstance(out, dict) and name in out) or hasattr(out, name) + + +__all__ = ["ModelPredict", "PredictKind"] diff --git a/recordstream/ops/random_apply.py b/recordstream/ops/random_apply.py new file mode 100644 index 0000000..822bb0c --- /dev/null +++ b/recordstream/ops/random_apply.py @@ -0,0 +1,80 @@ +"""``RandomApply`` — apply an op with a given probability. + +A compose-group op (alongside ``Enable`` / ``Pipeline`` / ``Parallel``): +wrap any single op (native or a bare library transform) so it fires only *p* +fraction of the time. Records that are skipped pass through unchanged. + +Modality-neutral — it threads any record through any op — so it lives +in core recordstream, not a domain package. +""" + +import random +from typing import Optional + +from confluid import configurable +from loggair import get_logger + +from recordstream.items import Record + +logger = get_logger(__name__) + + +@configurable(category="op", group="compose", random=True) +class RandomApply: + """Gate any op behind a Bernoulli coin flip. + + On each call, a uniform ``U ~ [0, 1)`` is drawn; if ``U < probability`` + the inner ``op`` is applied, otherwise the record passes through unchanged. + + ``op`` is flowed lazily on first use (Confluid ``!class:`` / ``!lazy:`` + markers are resolved at call-time, not at construction), so building a + ``RandomApply()`` with no arguments costs nothing. + + YAML: + + .. code-block:: yaml + + - !class:recordstream.ops.random_apply.RandomApply + probability: 0.5 + op: !class:albumentations.HorizontalFlip {p: 1.0} + + Args: + op: Inner op to gate (native op or bare library transform). Defaults to ``None`` + (identity); validated lazily on first call. + probability: Gate probability in ``[0, 1]``. ``0.0`` = never apply; + ``1.0`` = always apply. Defaults to ``0.5``. + random_state: Seed for the Bernoulli gate RNG. ``None`` = non-deterministic (default). + """ + + def __init__( + self, + op: Optional[object] = None, + probability: float = 0.5, + random_state: Optional[int] = None, + ) -> None: + self.op = op + self.probability = probability + self.random_state = random_state + self._gate_rng: Optional[random.Random] = None + + def __call__(self, record: Record) -> Optional[Record]: + if self.op is None: + raise ValueError("RandomApply requires 'op' to be set before calling.") + if self._gate_rng is None: + self._gate_rng = random.Random(self.random_state) + if self._gate_rng.random() >= self.probability: + return record + from confluid import flow + from confluid.fluid import Fluid + + # _apply_op is the engine's op-family dispatch — routing through it (instead of + # op(record)) lets a bare albumentations / torchvision-v2 transform nest inside + # the gate exactly as it would sit in a bare ops list. + from recordstream.core import _apply_op + + op = flow(self.op) if isinstance(self.op, Fluid) else self.op + self.op = op # cache the flowed op so we only flow once + return _apply_op(record, op) + + +__all__ = ["RandomApply"] diff --git a/recordstream/ops/sink.py b/recordstream/ops/sink.py new file mode 100644 index 0000000..aad62af --- /dev/null +++ b/recordstream/ops/sink.py @@ -0,0 +1,71 @@ +"""``RecordSinkOp`` — adapt a :class:`~recordstream.storage.base.DataSink` as a pass-through op. + +Lets any storage sink (``HDF5Sink``, ``ZarrGroupSink``, a domain package's JSON +sinks …) slot into a record-based op chain: on first call it opens the +sink, every call writes the record and returns it unchanged, and ``close()`` +flushes + closes. Modality-neutral (duck-typed ``open``/``write``/``close``), +so it lives in core recordstream. +""" + +from typing import Any + +from confluid import configurable +from loggair import get_logger + +from recordstream.items import Record + +logger = get_logger(__name__) + + +@configurable(category="op", group="sink") +class RecordSinkOp: + """Adapter: wrap a :class:`recordstream.storage.base.DataSink` as a pass-through op. + + Sinks implement the ``open()`` / ``write(record)`` / ``close()`` protocol and + are normally attached to a :class:`recordstream.processing.DatasetProcessor` as + the stream's terminal sink. This adapter lets the same sinks slot into any + record-based op chain (e.g. persisting a prediction pipeline's outputs + mid-chain). + + On the first call the adapter calls ``sink.open()`` (when present); each + subsequent call forwards the record to ``sink.write(record)`` and returns + the record unchanged. ``close()`` flushes (when present) and closes the + underlying sink — propagated by the composing ops at end-of-run. + + YAML:: + + - !class:recordstream.ops.sink.RecordSinkOp + sink: !class:recordstream.storage.hdf5.HDF5Sink + path: ./records.h5 + + Args: + sink: A DataSink-like object exposing ``write(record)`` (and optionally ``open``/``flush``/``close``). + """ + + def __init__(self, sink: Any = None) -> None: + # Partial / zero-arg: store config only; a non-None sink is required lazily in __call__. + self.sink = sink + self._opened = False + + def __call__(self, record: Record) -> Record: + if self.sink is None: + raise ValueError("RecordSinkOp requires a non-None 'sink'.") + if not self._opened: + opener = getattr(self.sink, "open", None) + if callable(opener): + opener() + self._opened = True + self.sink.write(record) + return record + + def close(self) -> None: + flush = getattr(self.sink, "flush", None) + if callable(flush): + flush() + closer = getattr(self.sink, "close", None) + if callable(closer): + closer() + self._opened = False + + +__all__ = ["RecordSinkOp"] diff --git a/recordstream/ops/structure.py b/recordstream/ops/structure.py new file mode 100644 index 0000000..2ee8217 --- /dev/null +++ b/recordstream/ops/structure.py @@ -0,0 +1,105 @@ +"""Structure ops — reshape a record dict's entries. + +Plumbing over the plain-dict carrier: RENAME, COPY, DROP, or SELECT named entries. Each op +is a thin copy-on-write dict expression — no payload is touched. + +All ops are lazy / zero-arg constructible (config validated in ``__call__``) and +``@configurable(category="op", group="structure")`` so they surface as canvas nodes. +""" + +from typing import List, Optional + +from confluid import configurable + +from recordstream.items import Record + +__all__ = ["RenameField", "DropField", "CopyField", "SelectFields"] + + +@configurable(category="op", group="structure") +class RenameField: + """Rename a record entry. Renaming onto an existing key replaces it. + + The sanctioned way to avoid a deliberate fan-in collision: rename on the producing branch + BEFORE the merge, instead of a merge-policy knob. Also the way to route a value into an + albumentations op's vocabulary (``image`` / ``mask`` / ``bboxes``). + + Args: + src: The entry to rename. + dst: The new key. + """ + + def __init__(self, src: str = "", dst: str = "") -> None: + self.src = src + self.dst = dst + + def __call__(self, record: Record) -> Record: + if not self.src or not self.dst: + raise ValueError("RenameField: both 'src' and 'dst' are required") + if self.src not in record: + raise KeyError(f"RenameField: unknown key {self.src!r} (keys: {list(record)})") + return {(self.dst if k == self.src else k): v for k, v in record.items()} + + +@configurable(category="op", group="structure") +class DropField: + """Remove an entry from the record (e.g. free a heavy signal after its spectrogram is derived). + + Args: + key: The entry to remove. Missing keys raise unless ``missing_ok``. + missing_ok: Silently pass through when the entry is absent (default False). + """ + + def __init__(self, key: str = "", missing_ok: bool = False) -> None: + self.key = key + self.missing_ok = missing_ok + + def __call__(self, record: Record) -> Record: + if not self.key: + raise ValueError("DropField: 'key' (the entry to remove) is required") + if self.key not in record: + if self.missing_ok: + return record + raise KeyError(f"DropField: unknown key {self.key!r} (keys: {list(record)})") + return {k: v for k, v in record.items() if k != self.key} + + +@configurable(category="op", group="structure") +class CopyField: + """Duplicate an entry under a new key (same value object; values are treated as immutable). + + Args: + src: The entry to copy. + dst: The key of the copy. An existing ``dst`` is replaced. + """ + + def __init__(self, src: str = "", dst: str = "") -> None: + self.src = src + self.dst = dst + + def __call__(self, record: Record) -> Record: + if not self.src or not self.dst: + raise ValueError("CopyField: both 'src' and 'dst' are required") + if self.src not in record: + raise KeyError(f"CopyField: unknown key {self.src!r} (keys: {list(record)})") + return {**record, self.dst: record[self.src]} + + +@configurable(category="op", group="structure") +class SelectFields: + """Keep ONLY the named entries (order = the given order); everything else is dropped. + + Args: + keys: The entries to keep. Unknown keys raise (a silent miss hides a typo). + """ + + def __init__(self, keys: Optional[List[str]] = None) -> None: + self.keys = list(keys) if keys else [] + + def __call__(self, record: Record) -> Record: + if not self.keys: + raise ValueError("SelectFields: 'keys' (the entries to keep) is required") + missing = [k for k in self.keys if k not in record] + if missing: + raise KeyError(f"SelectFields: unknown keys {missing} (keys: {list(record)})") + return {k: record[k] for k in self.keys} diff --git a/recordstream/ops/target.py b/recordstream/ops/target.py new file mode 100644 index 0000000..a2fb1f8 --- /dev/null +++ b/recordstream/ops/target.py @@ -0,0 +1,552 @@ +"""Target-shaping transforms over plain-dict records. + +* :class:`EncodeTarget` / :class:`DecodeTarget` map a class-name ``Label`` to a class-id + ``Label`` and back through an explicit lookup ``mapping`` — the declarative analogue of + scikit-learn's ``LabelEncoder``. The mapping is pinned in config, NOT fitted, so + train / eval / predict share one identical ordering. +* :class:`CocoToTorchVisionDetection` turns a HuggingFace / COCO ``objects`` annotation + (``{bbox, category}``) into a torchvision detection target rendered as a + :class:`~recordstream.Boxes` item. +* :class:`MasksToDetectionBoxes` derives detection boxes from a segmentation ``Mask``. + +The detection conversions are the modality-neutral, image-detection counterparts of +a signal package's domain-specific region ops. The encoded target value is written verbatim; wrap +it into a framework tensor downstream (e.g. a collate function) when a loss needs one. +""" + +from typing import Any, Dict, Literal, Optional, Tuple + +import numpy as np +from confluid import configurable + +from recordstream.items import Boxes, Label, Mask, MultiLabel, Record, item_data, resolve_item +from recordstream.transform import Transform + +#: COCO / HuggingFace bounding-box layouts (all in absolute pixels). Closed set so a typo +#: fails at the call site and UIs / form-specs enumerate the choices. +BBoxFormat = Literal["xywh", "xyxy", "cxcywh"] + + +def _source_frame(record: Record) -> Optional[Tuple[int, int]]: + """The ``(H, W)`` raster a record's boxes are stated in, or ``None`` when it cannot be read. + + An annotation gives coordinates in the pixel space of the image it annotates, so the frame + is the IMAGE's — which the record carries but the annotation does not. It is looked up under + the ``"image"`` key first (this engine's declared key vocabulary — what the op-family + dispatch hands to a library, what the coupled resize defaults to, what the dataset sources + yield), then as the first :class:`~recordstream.Image` item. + + It is deliberately narrow: only a declared image is trusted. A generic "first array with two + dimensions" search would happily read a ``Boxes``' own ``[N, 4]`` box array as an ``N x 4`` + raster and record a confident lie. ``None`` is an ordinary answer — it leaves ``canvas`` + exactly as it was before this was recorded at all, so nothing depends on the lookup + succeeding. + """ + from recordstream.items import Image as ImageItem + from recordstream.ops.image import image_frame + + candidate = record.get("image") + if candidate is not None: + frame = image_frame(candidate) + if frame is not None: + return frame + for value in record.values(): + if isinstance(value, ImageItem): + frame = image_frame(value) + if frame is not None: + return frame + return None + + +def _lookup(value: Any, mapping: Dict[Any, Any], ignore_unknown: bool, default: Any, op_name: str) -> Any: + """Return ``mapping[value]``, or ``default`` when missing and ``ignore_unknown``. + + A plain module-level function shared by :class:`EncodeTarget` / :class:`DecodeTarget`. + """ + if value in mapping: + return mapping[value] + if ignore_unknown: + return default + record_keys = list(mapping)[:8] + suffix = "..." if len(mapping) > 8 else "" + raise KeyError( + f"{op_name}: value {value!r} not in mapping (keys: {record_keys}{suffix}). " + "Pass ignore_unknown=True to substitute `default` instead." + ) + + +def _find_label_key(record: Record, field: Optional[str], op_name: str) -> str: + """Resolve the KEY of the label field to shape (``field`` or the first label item). + + Matches a :class:`~recordstream.Label` OR a :class:`~recordstream.MultiLabel` — both are + label items, and a multi-label target must be shaped through the same op. Shared by + :class:`EncodeTarget` / :class:`DecodeTarget`, which carried byte-parallel copies. + Deliberately NOT :func:`~recordstream.resolve_entry`: the gate is a type TUPLE and the + wrong-type miss is a ``TypeError`` (the callers' pinned API contract), both outside the + one-type/``ValueError`` shape the shared resolver pins. + """ + if field: + if field not in record: + raise ValueError(f"{op_name}: field {field!r} not in record (keys: {list(record)})") + item = record[field] + if not isinstance(item, (Label, MultiLabel)): + raise TypeError(f"{op_name}: field {field!r} is {type(item).__name__}, expected a Label or MultiLabel") + return field + for key, _item in ((k, v) for k, v in record.items() if isinstance(v, (Label, MultiLabel))): + return key + raise ValueError(f"{op_name}: no Label/MultiLabel field in record (keys: {list(record)})") + + +def coco_to_detection( + objects: Any, + bbox_key: str = "bbox", + category_key: str = "category", + bbox_format: BBoxFormat = "xywh", + label_offset: int = 0, +) -> Dict[str, Any]: + """Convert a COCO / HuggingFace ``objects`` mapping to ``{"boxes": [N,4] xyxy, "labels": [N]}`` tensors. + + Each box is, by COCO convention, ``[x, y, w, h]`` in absolute pixels; ``category`` is an + integer class id. An empty annotation yields empty ``[0,4]`` / ``[0]`` tensors (the + negative-example contract torchvision detectors accept). + """ + import torch + + if not isinstance(objects, dict): + raise TypeError( + f"coco_to_detection: expected a COCO/HF objects mapping " + f"(a dict with {bbox_key!r}/{category_key!r}); got {type(objects).__name__}." + ) + raw_boxes = objects.get(bbox_key) or [] + raw_labels = objects.get(category_key) or [] + + if len(raw_boxes): + boxes = torch.as_tensor(raw_boxes, dtype=torch.float32).reshape(-1, 4) + if bbox_format == "xywh": # COCO: top-left + size + x, y, w, h = boxes.unbind(-1) + boxes = torch.stack([x, y, x + w, y + h], dim=-1) + elif bbox_format == "cxcywh": # center + size + cx, cy, w, h = boxes.unbind(-1) + boxes = torch.stack([cx - w / 2, cy - h / 2, cx + w / 2, cy + h / 2], dim=-1) + # "xyxy": already in the output layout + else: + boxes = torch.zeros((0, 4), dtype=torch.float32) + + if len(raw_labels): + labels = torch.as_tensor(list(raw_labels), dtype=torch.int64).reshape(-1) + label_offset + else: + labels = torch.zeros((0,), dtype=torch.int64) + + return {"boxes": boxes, "labels": labels} + + +def masks_to_detection( + mask: Any, + label: int = 1, + connected: bool = False, + min_area: int = 1, + connectivity: int = 4, +) -> Dict[str, Any]: + """Derive ``{"boxes": [N,4] xyxy, "labels": [N]}`` tensors from a 2-D integer segmentation mask. + + ``connected=False`` (default) — an instance mask: each distinct non-zero pixel value is + one object. ``connected=True`` — binarize then split into connected components. Every box + gets class id ``label``. An empty mask yields empty ``[0,4]`` / ``[0]`` tensors. + """ + import torch + + if hasattr(mask, "convert"): # PIL image (e.g. an 'L' instance mask) + mask = np.array(mask) + mask = np.asarray(mask) + if mask.ndim != 2: + raise TypeError( + f"masks_to_detection: expected a 2-D segmentation mask; got shape {getattr(mask, 'shape', None)}." + ) + + boxes: list = [] + if connected: + from recordstream.ops.numpy import connected_component_boxes + + for x0, y0, x1, y1 in connected_component_boxes(mask != 0, min_area, connectivity): + boxes.append((float(x0), float(y0), float(x1), float(y1))) + else: + for value in np.unique(mask): + if int(value) == 0: + continue + ys, xs = np.where(mask == value) + if int(ys.size) < min_area: + continue + boxes.append((float(xs.min()), float(ys.min()), float(xs.max() + 1), float(ys.max() + 1))) + + if boxes: + boxes_t = torch.tensor(boxes, dtype=torch.float32) + labels_t = torch.full((len(boxes),), label, dtype=torch.int64) + else: + boxes_t = torch.zeros((0, 4), dtype=torch.float32) + labels_t = torch.zeros((0,), dtype=torch.int64) + return {"boxes": boxes_t, "labels": labels_t} + + +@configurable(category="op", group="structure") +class EncodeTarget(Transform): + """A class-NAME ``Label`` → a class-ID ``Label``. + + Reads a :class:`~recordstream.Label` field (``field``; blank picks the first ``Label``) + whose ``.value`` is a raw class name and maps it to its class id through the config-pinned + ``mapping`` — the declarative ``LabelEncoder`` analogue. The result is a new + :class:`~recordstream.Label` (carrying the source label's ``classes`` vocabulary) written + under ``output`` — blank (default) replaces the source field in place. + + Args: + mapping: Lookup from raw label name → class id, e.g. ``{"DJI AVATA2": 2, ...}``. Must be non-empty. + ignore_unknown: When ``False`` (default), raise on a label missing from ``mapping``; when + ``True``, substitute ``default``. + default: Value written for an unknown label when ``ignore_unknown=True`` (default ``0``). + field: ``Label`` field to encode; blank (default) picks the first ``Label`` field. + output: Key the encoded ``Label`` is written to; blank (default) replaces the source field in place. + """ + + handles = (Label, MultiLabel) + consumes = (Label, MultiLabel) + produces = (Label, MultiLabel) + + def __init__( + self, + mapping: Optional[Dict[Any, Any]] = None, + ignore_unknown: bool = False, + default: Any = 0, + field: str = "", + output: str = "", + ) -> None: + super().__init__() + # Partial / zero-arg: store config only; the non-empty requirement is validated lazily in __call__. + self.mapping = dict(mapping) if mapping else {} + self.ignore_unknown = bool(ignore_unknown) + self.default = default + self.field = str(field) + self.output = str(output) + + def _find_label(self, record: Record) -> str: + """Resolve the KEY of the label field to encode (via the shared :func:`_find_label_key`).""" + return _find_label_key(record, self.field, "EncodeTarget") + + def __call__(self, record: Record) -> Record: + if not self.mapping: + raise ValueError("EncodeTarget: mapping must contain at least one entry.") + key = self._find_label(record) + label = record[key] + out_key = self.output or key + if isinstance(label, MultiLabel): + values = [_lookup(v, self.mapping, self.ignore_unknown, self.default, "EncodeTarget") for v in label.values] + return {**record, out_key: MultiLabel(values, classes=label.classes)} + encoded = _lookup(label.value, self.mapping, self.ignore_unknown, self.default, "EncodeTarget") + return {**record, out_key: Label(encoded, classes=label.classes)} + + +@configurable(category="op", group="structure") +class DecodeTarget(Transform): + """A class-ID ``Label`` → a class-NAME ``Label`` (inverse of :class:`EncodeTarget`). + + Reads a :class:`~recordstream.Label` field (``field``; blank picks the first ``Label``) + whose ``.value`` is an encoded class id and maps it back to its label name through + ``mapping`` — the readback half used in prediction / reporting. The result is a new + :class:`~recordstream.Label` written under ``output`` (blank replaces in place). + + Args: + mapping: Lookup from class id → label name, e.g. ``{2: "DJI AVATA2", ...}``. Must be non-empty. + ignore_unknown: When ``False`` (default), raise on an id missing from ``mapping``; when + ``True``, substitute ``default``. + default: Value written for an unknown id when ``ignore_unknown=True`` (default ``None``). + field: ``Label`` field to decode; blank (default) picks the first ``Label`` field. + output: Key the decoded ``Label`` is written to; blank (default) replaces the source field in place. + """ + + handles = (Label, MultiLabel) + consumes = (Label, MultiLabel) + produces = (Label, MultiLabel) + + def __init__( + self, + mapping: Optional[Dict[Any, Any]] = None, + ignore_unknown: bool = False, + default: Any = None, + field: str = "", + output: str = "", + ) -> None: + super().__init__() + # Partial / zero-arg: store config only; the non-empty requirement is validated lazily in __call__. + self.mapping = dict(mapping) if mapping else {} + self.ignore_unknown = bool(ignore_unknown) + self.default = default + self.field = str(field) + self.output = str(output) + + def _find_label(self, record: Record) -> str: + """Resolve the KEY of the label field to decode (via the shared :func:`_find_label_key`).""" + return _find_label_key(record, self.field, "DecodeTarget") + + def __call__(self, record: Record) -> Record: + if not self.mapping: + raise ValueError("DecodeTarget: mapping must contain at least one entry.") + key = self._find_label(record) + label = record[key] + out_key = self.output or key + if isinstance(label, MultiLabel): + values = [_lookup(v, self.mapping, self.ignore_unknown, self.default, "DecodeTarget") for v in label.values] + return {**record, out_key: MultiLabel(values, classes=label.classes)} + decoded = _lookup(label.value, self.mapping, self.ignore_unknown, self.default, "DecodeTarget") + return {**record, out_key: Label(decoded, classes=label.classes)} + + +@configurable(category="op", group="structure") +class CocoToTorchVisionDetection(Transform): + """A COCO / HF ``objects`` annotation → a target ``Boxes``. + + Reads a source field (``field``; blank picks the first :class:`~recordstream.Label`, else the + first field) carrying a HuggingFace / COCO ``objects`` mapping and rewrites it to the + torchvision detection target, riding as a :class:`~recordstream.Boxes` item under + ``output`` (``boxes`` = the ``[N, 4]`` float32 xyxy tensor, ``labels`` = the ``[N]`` int64 + class-id tensor). An empty annotation yields empty ``[0,4]`` / ``[0]`` + tensors (the negative-example contract). + + Args: + bbox_key: Key in the objects mapping holding per-box coordinates (default ``"bbox"``). + category_key: Key holding the per-box integer class ids (default ``"category"``). + bbox_format: Box layout in pixels — ``xywh`` (COCO, default), ``xyxy``, or ``cxcywh``; output is xyxy. + label_offset: Added to each class id (default ``0``). Set ``1`` to reserve class ``0`` for background. + field: Source field with the objects mapping; blank (default) picks the first ``Label``, else the first field. + output: Key the target ``Boxes`` is written to (added if new). + """ + + handles = (Label,) + consumes = (Label,) + produces = (Boxes,) + + def __init__( + self, + bbox_key: str = "bbox", + category_key: str = "category", + bbox_format: BBoxFormat = "xywh", + label_offset: int = 0, + field: str = "", + output: str = "target", + ) -> None: + super().__init__() + self.bbox_key = str(bbox_key) + self.category_key = str(category_key) + self.bbox_format = bbox_format + self.label_offset = int(label_offset) + self.field = str(field) + self.output = str(output) + + def _find_source(self, record: Record) -> str: + """Resolve the KEY of the source field (``self.field``, else the first ``Label``, else the first field).""" + if self.field: + if self.field not in record: + raise ValueError( + f"CocoToTorchVisionDetection: field {self.field!r} not in record (keys: {list(record)})" + ) + return self.field + for key, _item in ((k, v) for k, v in record.items() if isinstance(v, Label)): + return key + for key in record: + return key + raise ValueError("CocoToTorchVisionDetection: record is empty — no source field to read") + + def __call__(self, record: Record) -> Record: + key = self._find_source(record) + item = record[key] + objects = item.value if isinstance(item, Label) else item_data(item) + target = coco_to_detection(objects, self.bbox_key, self.category_key, self.bbox_format, self.label_offset) + # `canvas` IS the frame the boxes are stated in — a COCO box is in the source image's + # pixel space, so it is knowable here and recording it costs one lookup. Left None when + # no image is in the record, which is what every Boxes carried before this. + return { + **record, + self.output: Boxes(boxes=target["boxes"], labels=target["labels"], canvas=_source_frame(record)), + } + + +@configurable(category="op", group="structure") +class MasksToDetectionBoxes(Transform): + """A segmentation ``Mask`` → a target ``Boxes``. + + Reads the :class:`~recordstream.Mask` at ``field`` (blank = the first ``Mask`` in the record, + else the first array-bearing item) as a 2-D integer mask and derives one tight + ``[x0,y0,x1,y1]`` box per object. The target rides as a :class:`~recordstream.Boxes` item + under ``output``. An empty mask yields empty ``[0,4]`` / ``[0]`` tensors. + + Args: + label: Foreground class id assigned to every derived box (default ``1``; class 0 = background). + connected: True = connected-components on a binary mask; False (default) = each non-zero value is one instance. + min_area: Drop objects whose mask area (in pixels) is below this (default ``1``). + connectivity: Connected-components neighborhood when ``connected=True`` — ``4`` or ``8`` (default ``4``). + field: Name of the ``Mask`` field to read; blank (default) picks the first ``Mask`` (else the first array). + output: Key the target ``Boxes`` is written to (added if new). + """ + + handles = (Mask,) + consumes = (Mask,) + produces = (Boxes,) + + def __init__( + self, + label: int = 1, + connected: bool = False, + min_area: int = 1, + connectivity: int = 4, + field: str = "", + output: str = "target", + ) -> None: + super().__init__() + self.label = int(label) + self.connected = bool(connected) + self.min_area = int(min_area) + self.connectivity = int(connectivity) + self.field = str(field) + self.output = str(output) + + def _find_mask(self, record: Record) -> np.ndarray: + """Resolve the mask array (``self.field``, else the first ``Mask``, else the first array-bearing item).""" + if self.field: + if self.field not in record: + raise ValueError(f"MasksToDetectionBoxes: field {self.field!r} not in record (keys: {list(record)})") + data = item_data(record[self.field]) + else: + data = None + for _key, item in ((k, v) for k, v in record.items() if isinstance(v, Mask)): + data = item_data(item) + break + if data is None: + for _key, item in record.items(): + payload = item_data(item) + if isinstance(payload, np.ndarray): + data = payload + break + if data is None: + raise ValueError( + f"MasksToDetectionBoxes: no Mask or array-bearing field in record (keys: {list(record)})" + ) + if not isinstance(data, np.ndarray): + raise TypeError(f"MasksToDetectionBoxes: expected an np.ndarray mask, got {type(data).__name__}") + return data + + def __call__(self, record: Record) -> Record: + mask = self._find_mask(record) + target = masks_to_detection(mask, self.label, self.connected, self.min_area, self.connectivity) + # The boxes were derived FROM this mask, so its shape is the frame exactly — no lookup, + # no fallback, nothing to be wrong about. + height, width = int(mask.shape[0]), int(mask.shape[1]) + return { + **record, + self.output: Boxes(boxes=target["boxes"], labels=target["labels"], canvas=(height, width)), + } + + +@configurable(category="op", group="structure") +class ResizeDetection(Transform): + """Resize an image AND scale its detection-target boxes in ONE coupled step. + + The detection twin of the joint image+mask draw: a fixed-input-size detector needs the image + resized, and a resize that moved the pixels without moving the boxes would silently train on + misplaced targets. Reads the image under ``input_key`` (PIL or a uint8 HWC/2-D array), + resizes it to ``(height, width)`` (bilinear, PIL), and scales the + :class:`~recordstream.Boxes` under ``target_key`` by the same factors — torch boxes + stay torch, numpy stays numpy. The resized ``Boxes`` records the new frame in ``canvas``. + + Ops that need no fixed size (torchvision detectors resize internally) simply omit this op — + it exists for the detectors that require pre-sized square inputs. + + Args: + width: Target width in pixels; required at use (validated lazily, ``0`` = unset). + height: Target height in pixels; required at use (validated lazily, ``0`` = unset). + input_key: Record key carrying the image (default ``"image"``). + target_key: Record key carrying the target ``Boxes``; a record without it resizes the image alone. + """ + + consumes = (Boxes,) + produces = (Boxes,) + + def __init__( + self, + width: int = 0, + height: int = 0, + input_key: str = "image", + target_key: str = "target", + ) -> None: + super().__init__() + self.width = int(width) + self.height = int(height) + self.input_key = str(input_key) + self.target_key = str(target_key) + + def _resize_image(self, payload: Any) -> Any: + from PIL import Image as PILImage + + if hasattr(payload, "convert"): # PIL + return payload.resize((self.width, self.height), PILImage.Resampling.BILINEAR) + array = np.asarray(payload) + if array.dtype != np.uint8: + raise TypeError( + f"ResizeDetection: expected a PIL image or a uint8 array under {self.input_key!r}; " + f"got dtype {array.dtype}. Run it BEFORE any float conversion (e.g. before ToTensor)." + ) + resized = PILImage.fromarray(array).resize((self.width, self.height), PILImage.Resampling.BILINEAR) + return np.array(resized) + + @staticmethod + def _scale_boxes(boxes: Any, sx: float, sy: float) -> Any: + """Scale ``[N, 4]`` xyxy boxes by per-axis factors, preserving the array framework.""" + from recordstream._compat import is_torch_tensor + + if is_torch_tensor(boxes): + import torch + + return boxes * torch.tensor([sx, sy, sx, sy], dtype=boxes.dtype) + return np.asarray(boxes, dtype=np.float64).reshape(-1, 4) * np.array([sx, sy, sx, sy]) + + def __call__(self, record: Record) -> Record: + if self.width < 1 or self.height < 1: + raise ValueError(f"ResizeDetection needs positive width/height (got {self.width}x{self.height}).") + if self.input_key not in record: + raise ValueError(f"ResizeDetection: field {self.input_key!r} not in record (keys: {list(record)})") + item = record[self.input_key] + payload = item_data(item) + frame = _source_frame({"image": item}) + if frame is None: + raise ValueError( + f"ResizeDetection: the value under {self.input_key!r} is not an image " + f"(got {type(payload).__name__}) — it has no raster to resize." + ) + orig_h, orig_w = frame + resized = self._resize_image(payload) + merged = dict(record) + from recordstream.items import NDArrayItem, with_data + + merged[self.input_key] = with_data(item, resized) if isinstance(item, NDArrayItem) else resized + + target = resolve_item( + record, self.target_key, Boxes, owner="ResizeDetection", param="target_key", fallback=False, required=False + ) + if target is not None: + import dataclasses + + # An EMPTY target is re-framed too. Scaling no boxes is a no-op, but leaving the + # canvas behind would make a negative example the ONE record in a set whose frame is + # unknown — and a frame check that silently skips exactly the records with nothing + # to check is a check that reports a clean bill for the wrong reason. + sx, sy = float(self.width) / float(orig_w), float(self.height) / float(orig_h) + boxes = self._scale_boxes(target.boxes, sx, sy) if len(target.boxes) else target.boxes + merged[self.target_key] = dataclasses.replace(target, boxes=boxes, canvas=(self.height, self.width)) + return merged + + +__all__ = [ + "EncodeTarget", + "DecodeTarget", + "CocoToTorchVisionDetection", + "MasksToDetectionBoxes", + "ResizeDetection", + "coco_to_detection", + "masks_to_detection", +] diff --git a/recordstream/ops/torch.py b/recordstream/ops/torch.py new file mode 100644 index 0000000..791567f --- /dev/null +++ b/recordstream/ops/torch.py @@ -0,0 +1,113 @@ +from typing import Any, Optional + +import numpy as np +import torch +from confluid import configurable + +from recordstream.items import NDArrayItem, Record, item_data +from recordstream.transform import Transform + + +def to_tensor(img: Any, normalize: bool = True, mode: Optional[str] = None) -> torch.Tensor: + """Convert a PIL image / NumPy array to a CHW ``torch.Tensor``. + + ``mode`` (e.g. ``"RGB"``, forcing 3 channels) coerces a PIL payload directly — and a + ``uint8`` ARRAY payload through a PIL round-trip, because a decoding source may hand over + the already-arrayed pixels: a mixed-mode dataset (RGB + RGBA + grayscale rows) then reaches + the model with ragged channel counts unless the coercion applies to arrays too. (Found the + hard way: cppe-5 ships 3-, 4- and 1-channel images, and a ``mode="RGB"`` that silently + skipped arrays crashed torchvision's normalize mid-epoch with a channel mismatch.) A + non-``uint8`` array with ``mode`` set is left as-is — PIL cannot represent it faithfully. + + An ``[H, W, C]`` array is transposed to ``[C, H, W]`` (a 2-D array gets a leading channel + axis). With ``normalize`` an integer / 0-255-float payload is scaled into ``[0, 1]``. + """ + if hasattr(img, "convert"): + if mode is not None: + img = img.convert(mode) + img = np.array(img) + elif mode is not None and isinstance(img, np.ndarray) and img.dtype == np.uint8: + from PIL import Image as PILImage + + # A trailing singleton channel axis defeats `fromarray` — squeeze it to the 2-D form. + arr = img[..., 0] if (img.ndim == 3 and img.shape[2] == 1) else img + img = np.array(PILImage.fromarray(arr).convert(mode)) + + if isinstance(img, np.ndarray): + if img.ndim == 3: + img = img.transpose(2, 0, 1) + elif img.ndim == 2: + img = img[np.newaxis, :] + tensor = torch.from_numpy(img) + else: + tensor = torch.as_tensor(img) + + if normalize and tensor.dtype == torch.uint8: + tensor = tensor.float() / 255.0 + elif normalize and tensor.max() > 1.0: + tensor = tensor / 255.0 + return tensor + + +@configurable(category="op", group="torch") +class ToTensor(Transform): + """An array-bearing field → a LIVE CHW-float ``torch.Tensor`` record value. + + Reads the payload of an array-bearing field (blank ``field`` picks the first array/PIL-bearing + item — typically the :class:`~recordstream.Image` a :class:`~recordstream.ops.image.ConvertToImage` + produced), runs the HWC→CHW transpose + ``normalize`` conversion (:func:`to_tensor`), and writes + the resulting ``torch.Tensor`` back AS-IS. By default it REPLACES the resolved field in place + (``output`` blank); set ``output`` to write a NEW key instead. Any other key passes + through untouched. + + The output is a PLAIN record value (a record holds arbitrary values — the ``"plain"`` codec + tag covers storage): ``collate_records`` stacks torch tensors natively (``torch.stack``), a + torchvision ``transforms.v2`` op downstream transforms it as-is, and array sinks convert via + ``to_numpy`` on write. It is deliberately NOT wrapped in an :class:`~recordstream.Image` — an + ``NDArrayItem`` coerces through ``np.asarray`` and cannot hold a live tensor. + + Args: + normalize: When ``True`` (default), scale integer pixel inputs into the ``[0, 1]`` float range. + mode: Optional PIL mode to convert a PIL payload to (e.g. ``"RGB"`` forces 3 channels); ``None`` = as-is. + field: Name of the source field to tensorize; blank (default) picks the first array/PIL-bearing item. + output: Key the tensor is written to; blank (default) replaces the source field in place. + """ + + handles = (NDArrayItem,) + consumes = (NDArrayItem,) + produces = (torch.Tensor,) + + def __init__( + self, + normalize: bool = True, + mode: Optional[str] = None, + field: str = "", + output: str = "", + ) -> None: + super().__init__() + self.normalize = bool(normalize) + self.mode = mode + self.field = field + self.output = output + + def _find_field(self, record: Record) -> str: + """Resolve the KEY of the field to tensorize (``self.field`` or the first array/PIL item).""" + if self.field: + if self.field not in record: + raise ValueError(f"ToTensor: field {self.field!r} not in record (keys: {list(record)})") + return self.field + for key, item in record.items(): + data = item_data(item) + if isinstance(data, np.ndarray) or hasattr(data, "convert"): + return key + raise ValueError(f"ToTensor: no array-bearing field in record (keys: {list(record)})") + + def __call__(self, record: Record) -> Record: + key = self._find_field(record) + data = item_data(record[key]) + tensor = to_tensor(data, self.normalize, self.mode) + out_key = self.output or key + return {**record, out_key: tensor} + + +__all__ = ["ToTensor", "to_tensor"] diff --git a/recordstream/outputs.py b/recordstream/outputs.py new file mode 100644 index 0000000..572b8f8 --- /dev/null +++ b/recordstream/outputs.py @@ -0,0 +1,163 @@ +"""Prediction-output contracts — what a model hands to a sink, metric, or visualizer. + +Every model's eval-mode ``forward`` should return one of these — or a task-specific superset. +The keys *are* the documentation: downstream metrics, sinks, visualizers, and evaluators +inspect known keys instead of guessing whether a tensor is logits, probabilities, or argmax'd +class ids. That guess is not hypothetical: two independently-written detector wrappers agree +that ``boxes`` is xyxy in absolute pixels only because :class:`DetectionOutput` says so. + +**Generic in the array type.** Each contract is a generic ``TypedDict`` parameterized by the +array type it carries, so the SAME contract describes a torch run, a numpy/JAX run, or a +TensorFlow one:: + + ClassificationOutput[Tensor] # a torch model + ClassificationOutput[np.ndarray] # a numpy / TF / JAX model + +The parameter is optional — a bare ``ClassificationOutput`` means "whatever array type". + +**Why here.** These describe the boundary between a model and whatever consumes its output, +and the consumer this package ships is :mod:`recordstream.predictions` — the sink that reads +``probs`` / ``class_idx`` by name. A contract owned by a different package than its reader is +how "it lives there because that other thing lives there" starts. + +The BUILDERS (bottom of this module) are necessarily per-framework — ``softmax`` and ``argmax`` +are library calls, not type declarations — and are torch, like the rest of this package's tensor +surface. A backend on another framework adds its own builders beside these; it does NOT redefine +the contracts. +""" + +from typing import TYPE_CHECKING, Generic, List, TypedDict, TypeVar + +if TYPE_CHECKING: # torch is imported inside the BUILDERS — the contracts are typing-only + from torch import Tensor + +#: The array type a contract carries — ``torch.Tensor``, ``np.ndarray``, a TF/JAX array. +ArrayT = TypeVar("ArrayT") + +__all__ = [ + "ArrayT", + "ClassificationOutput", + "DetectionOutput", + "DetectionPredictions", + "RestorationOutput", + "SegmentationOutput", + "classification_output", + "restoration_output", + "segmentation_output", +] + + +class ClassificationOutput(TypedDict, Generic[ArrayT]): + """Per-record classification prediction. + + Keys: + logits: ``[B, C]`` float — raw pre-softmax scores. + probs: ``[B, C]`` float — softmax over the last dim. + class_idx: ``[B]`` int64 — argmax of ``logits`` along the last dim. + """ + + logits: ArrayT + probs: ArrayT + class_idx: ArrayT + + +class DetectionOutput(TypedDict, Generic[ArrayT]): + """Per-image object detection prediction. + + Keys: + boxes: ``[N, 4]`` float32 — xyxy in **absolute pixels**. + scores: ``[N]`` float32 — confidence in ``[0, 1]``. + labels: ``[N]`` int64 — class ids. Class 0 is conventionally reserved + for background in torchvision-style detectors. + """ + + boxes: ArrayT + scores: ArrayT + labels: ArrayT + + +class SegmentationOutput(TypedDict, Generic[ArrayT]): + """Per-pixel segmentation prediction. + + Keys: + logits: ``[B, C, H, W]`` float — raw pre-softmax scores. + probs: ``[B, C, H, W]`` float — softmax over the channel dim. + mask: ``[B, H, W]`` int64 — argmax across channels. + """ + + logits: ArrayT + probs: ArrayT + mask: ArrayT + + +class RestorationOutput(TypedDict, Generic[ArrayT]): + """Per-record image-to-image restoration prediction. + + Keys: + image: ``[B, C, H, W]`` float — the restored image, in the SAME value range as the + model's input (``[0, 1]`` for a pipeline that ends in ``ToTensor(normalize=True)``). + + **One key, and that is the whole point of the contract.** The other tasks here carry three + because their raw output needs interpreting — ``logits`` are not ``probs`` are not + ``class_idx``. A restoration model emits the answer directly, so what a consumer needs told + is precisely that: the array under ``image`` is a picture in the input's units, NOT logits + to be softmaxed and NOT a residual to be added back. A sink that writes a PNG, a PSNR metric + and a viewer all read it the same way because this says so. + + The residual (what the model removed) is deliberately absent: it is ``input - image``, and a + contract key that a consumer can compute is a second place for the two to disagree. + """ + + image: ArrayT + + +#: A detector's per-image results — one :class:`DetectionOutput` per image in the batch. +DetectionPredictions = List[DetectionOutput] + + +def classification_output(logits: "Tensor") -> "ClassificationOutput[Tensor]": + """Build a full :class:`ClassificationOutput` from raw ``[B, C]`` logits. + + Example:: + + def predict_step(self, batch, batch_idx): + return classification_output(self(x)) # what a predictions sink reads + """ + import torch + import torch.nn.functional as F + + probs = F.softmax(logits, dim=-1) + class_idx = torch.argmax(logits, dim=-1).to(torch.int64) + return ClassificationOutput(logits=logits, probs=probs, class_idx=class_idx) + + +def segmentation_output(logits: "Tensor") -> "SegmentationOutput[Tensor]": + """Build a full :class:`SegmentationOutput` from raw ``[B, C, H, W]`` logits.""" + import torch + import torch.nn.functional as F + + probs = F.softmax(logits, dim=1) + mask = torch.argmax(logits, dim=1).to(torch.int64) + return SegmentationOutput(logits=logits, probs=probs, mask=mask) + + +def restoration_output(image: "Tensor") -> "RestorationOutput[Tensor]": + """Build a :class:`RestorationOutput` from a restored ``[B, C, H, W]`` image. + + It transforms nothing — unlike its two siblings, which softmax and argmax — because a + restoration model's output already IS the contract's one key. The builder exists anyway so + every backend spells the wrapping identically and a reader of ``predict_step`` sees the same + shape of line in all four tasks:: + + def predict_step(self, batch, batch_idx): + return restoration_output(self(x)) # what a predictions sink reads + + Clamping is deliberately NOT done here: whether a restored image may leave ``[0, 1]`` is the + model's business (a residual denoiser can legitimately overshoot, and a metric computed on + clamped values is a different number), so a run that wants it wires it where it decides that. + """ + return RestorationOutput(image=image) + + +# Detection deliberately has NO builder: boxes come from the detector's own interface, so the +# dict is built inline at the call site rather than invented from nothing here. diff --git a/recordstream/predictions.py b/recordstream/predictions.py new file mode 100644 index 0000000..29e395e --- /dev/null +++ b/recordstream/predictions.py @@ -0,0 +1,206 @@ +"""Predictions sinks — where a predict/test loop's per-record output goes. + +A runnable's predict loop calls ``predictions_sink.write(prediction, metadata)`` once per +record and ``predictions_sink.close()`` at end of run. This module carries the protocol that +states that contract plus the classification sink built on it. + +**Why this is a SECOND sink protocol.** :class:`recordstream.storage.base.DataSink` takes a +whole ``record`` (``write(record)``) and is what ``RecordSinkOp`` adapts into an op chain. +A predictions sink instead receives the MODEL's output plus the metadata of the record it came +from, and builds the record itself — the two halves arrive separately because a model emits a +batch while the sink contract is per-record. The split is deliberate and load-bearing +elsewhere: a visual editor surfaces ``category="sink"`` storage sinks as canvas nodes and +excludes prediction sinks precisely because their signature differs. Collapsing them (have the +runnable build the record and write through ``DataSink``) is a real option, tracked in +``TASKS.md`` — until then, do not blur the two. +""" + +from typing import Any, Dict, List, Optional, Protocol, runtime_checkable + +import numpy as np +from confluid import configurable +from loggair import get_logger + +from recordstream.items import Record + +logger = get_logger(__name__) + + +@runtime_checkable +class PredictionsSink(Protocol): + """What a runnable's ``predictions_sink`` slot must provide. + + Structural, not a base class: a sink is anything that can take one record's prediction plus + that record's metadata, and be closed at the end. Naming it here — beside the sink this + package ships — means consuming runnables annotate ``Optional[Partial[PredictionsSink]]`` + instead of ``Any``, which declared nothing and let a use site call ``.write`` on a slot that + might still be a deferred marker. + + ``@runtime_checkable`` so an ``isinstance`` guard is available; note that only the METHOD + NAMES are checked at runtime, never their signatures. + """ + + def write(self, prediction: Any, metadata: Dict[str, Any]) -> None: + """Record one prediction alongside the metadata of the record it came from.""" + ... + + def close(self) -> None: + """Flush and release whatever the sink holds open.""" + ... + + +@configurable +class ClassificationPredictionsSink: + """Lift classification predictions into a record and run ops over it. + + Bridges a predict/test loop's ``predictions_sink`` contract — ``write(prediction, metadata)`` + per record — to record ops, giving the train→eval→predict triad a uniform shape across + tasks. + + For each call: + + 1. Read the model's :class:`~recordstream.outputs.ClassificationOutput` — ``probs`` ``[C]`` + and ``class_idx`` scalar, per record. + 2. Resolve the int class id to a human-readable label via ``class_names``, and build a top-k + list (the ``top_k`` highest-probability classes with their probabilities + labels). + 3. Build a fresh record carrying the original metadata plus the prediction columns under a + single ``"metadata"`` key: + + * ``predicted_class_id`` (int) + * ``predicted_class_label`` (str) + * ``predicted_confidence`` (float — top-1 probability) + * ``predicted_top_k`` (list of ``{class_id, label, probability}``, descending) + + 4. Thread that record through ``ops`` — typically just ``RecordSinkOp(sink=…)`` to dump JSON. + + Modality-neutral: it reads only the prediction contract and the metadata it is handed, so a + classifier over images, spectrograms or tabular rows uses it unchanged. Diagnostics identify + a record by its ORDINAL in the run for the same reason — no domain key names appear here. + + Zero-arg constructible, like every configurable in this package: ``ops`` is required to RUN, + not to BUILD, so the non-empty check fires on the first :meth:`write` with a clear message + rather than in ``__init__`` (which would make the class unbuildable by a schema/form + generator that instantiates with defaults to introspect it). + + Args: + ops: Ops to run on each per-prediction record. Required by the time the sink is written + to; validated there, not at construction. + class_names: Optional ``{int_class_id: str}`` map converting the model's int64 class ids + back to human-readable strings. Without it, labels become ``str(class_id)``. YAML int + keys land here as strings (config loaders stringify mapping keys); both forms are + accepted and normalized to ``str`` internally. + top_k: Number of top-probability predictions recorded per record. Defaults to ``1``; + ``top_k > num_classes`` is silently clamped. + confidence_threshold: Predictions whose top-1 probability is below this are skipped + entirely (no record produced, no ops run). Default ``0.0`` keeps every prediction. + """ + + def __init__( + self, + ops: Optional[List[Any]] = None, + class_names: Optional[Dict[Any, str]] = None, + top_k: int = 1, + confidence_threshold: float = 0.0, + ) -> None: + if top_k < 1: + raise ValueError(f"top_k must be >= 1, got {top_k}.") + self.ops: List[Any] = list(ops or []) + self.top_k = int(top_k) + self.confidence_threshold = float(confidence_threshold) + # Normalize keys to `str` so YAML-loaded maps (always stringified) and Python-constructed + # maps (which may use int keys) are both addressable by the same lookup. + self.class_names: Dict[str, str] = {str(k): str(v) for k, v in (class_names or {}).items()} + #: How many predictions have been offered — the ordinal a diagnostic names. + self._seen = 0 + + def _label_for(self, class_id: int) -> str: + return self.class_names.get(str(int(class_id)), str(int(class_id))) + + def write(self, prediction: Dict[str, Any], metadata: Dict[str, Any]) -> None: + from confluid import flow + from confluid.fluid import Fluid + + if not self.ops: + raise ValueError( + "ClassificationPredictionsSink: 'ops' is empty — wire at least one op to receive " + "the prediction records (typically RecordSinkOp(sink=...))." + ) + + where = f"record #{self._seen}" + self._seen += 1 + + probs = prediction.get("probs") + class_idx = prediction.get("class_idx") + if probs is None or class_idx is None: + logger.warning( + f"ClassificationPredictionsSink: {where} is missing 'probs' or 'class_idx' " + f"(got keys: {sorted(prediction.keys())}); skipping." + ) + return + + probs_arr = probs.detach().cpu().numpy() if hasattr(probs, "detach") else np.asarray(probs) + # Tolerate both [C] (single-record) and [1, C] (batched-with-1) shapes. + if probs_arr.ndim == 2 and probs_arr.shape[0] == 1: + probs_arr = probs_arr[0] + if probs_arr.ndim != 1: + logger.warning( + f"ClassificationPredictionsSink: {where} has probs of shape {probs_arr.shape}, " + f"expected [C] or [1, C]; skipping." + ) + return + + n_classes = int(probs_arr.shape[0]) + top1_id = int(class_idx.item()) if hasattr(class_idx, "item") else int(class_idx) + top1_prob = float(probs_arr[top1_id]) + if top1_prob < self.confidence_threshold: + logger.debug( + f"ClassificationPredictionsSink: {where} top1_prob={top1_prob:.3f} " + f"< threshold={self.confidence_threshold:.3f}; skipping." + ) + return + + k = min(self.top_k, n_classes) + top_k_ids = np.argsort(-probs_arr)[:k] # descending, first k + top_k_list = [ + { + "class_id": int(cid), + "label": self._label_for(int(cid)), + "probability": float(probs_arr[cid]), + } + for cid in top_k_ids + ] + + new_metadata = dict(metadata) + new_metadata["predicted_class_id"] = top1_id + new_metadata["predicted_class_label"] = self._label_for(top1_id) + new_metadata["predicted_confidence"] = top1_prob + new_metadata["predicted_top_k"] = top_k_list + + logger.debug( + f"ClassificationPredictionsSink: {where} top1={new_metadata['predicted_class_label']!r} " + f"({top1_prob:.3f}), top_k={k}" + ) + + # The prediction metadata rides a single "metadata" key on a plain record. Downstream ops + # (typically `RecordSinkOp` wrapping a JSON sink) read it via `record["metadata"]`. There + # is no input/target on a prediction-only record. + record: Record = {"metadata": new_metadata} + for i, op in enumerate(self.ops): + if isinstance(op, Fluid): + op = flow(op) + self.ops[i] = op + record = op(record) + + def close(self) -> None: + """Propagate ``close()`` to ops that own resources (e.g. wrapped sinks). + + The sink itself buffers nothing, but the wrapped ops typically do (e.g. ``RecordSinkOp`` + wrapping a buffered/file-handle sink). + """ + for op in self.ops: + close_fn = getattr(op, "close", None) + if callable(close_fn): + close_fn() + + +__all__ = ["ClassificationPredictionsSink", "PredictionsSink"] diff --git a/recordstream/processing.py b/recordstream/processing.py new file mode 100644 index 0000000..7f8ebb0 --- /dev/null +++ b/recordstream/processing.py @@ -0,0 +1,153 @@ +"""Generic source→sink pipeline runner. + +:class:`DatasetProcessor` orchestrates a :class:`~recordstream.core.stream.Stream` from source +to sink — a runnable that drives whole-dataset processing (windowing, format +conversion, data acquisition) with an optional console progress bar. It is the +generic, modality-neutral data-pipeline runner: it iterates the stream and writes +each item to the sink, carrier-agnostic (it never inspects item internals), so it +works for any ``Stream`` regardless of what flows through it. + +Wired as the ``runnable:`` object of a config and run via ``recordstream run``, or +docked into a visual-editor canvas as a runnable node. +""" + +from contextlib import nullcontext +from typing import Any, Iterable, Iterator, Optional + +from confluid import configurable +from loggair import get_logger +from rich.progress import BarColumn, MofNCompleteColumn, Progress, TextColumn, TimeRemainingColumn + +from recordstream.core import Stream +from recordstream.runnable import ProgressReporting +from recordstream.storage.base import Storage + +logger = get_logger(__name__) + + +def _stream_total(stream: Stream) -> Optional[int]: + """``len(stream.source)`` when the source is sized, else ``None`` (a glob-based / streaming source).""" + try: + return len(stream.source) # type: ignore[arg-type] + except (TypeError, AttributeError): + return None + + +@configurable +class DatasetProcessor(ProgressReporting): + """Orchestrate a RecordStream pipeline from source to sink. + + Args: + stream: The :class:`~recordstream.core.stream.Stream` to execute. Required to run; + defaulted to ``None`` for zero-arg construction (validated in + :meth:`run`, the workspace lazy-construction rule). + sink: Optional sink; when absent, records are materialized to a list. + show_progress: If ``True``, wrap iteration with a ``rich.progress`` bar. + The total is derived from ``len(stream.source)`` when available; an + unsized source falls back to a count-only bar. Default ``False``. + progress_desc: Optional label for the progress bar (defaults to + ``"DatasetProcessor"``). Ignored when ``show_progress`` is ``False``. + """ + + def __init__( + self, + stream: Optional[Stream] = None, + sink: Optional[Any] = None, + show_progress: bool = False, + progress_desc: Optional[str] = None, + ) -> None: + self.stream = stream + self.sink = sink + self.show_progress = show_progress + self.progress_desc = progress_desc + + def run(self) -> None: + logger.info("Starting DatasetProcessor...") + if self.stream is None: + raise ValueError("DatasetProcessor.run() requires a 'stream' — none was configured.") + # Confluid keeps Class kwargs deferred (post-construction paradigm) so + # when the processor was loaded from YAML, ``self.stream``, its source, + # its ops, and ``self.sink`` may all be Fluid stubs. Materialize them + # here so callers don't need to know. + from confluid import flow + from confluid.fluid import Fluid + + stream = flow(self.stream) if isinstance(self.stream, Fluid) else self.stream + if isinstance(stream.source, Fluid): + stream.source = flow(stream.source) + stream.ops = [flow(op) if isinstance(op, Fluid) else op for op in stream.ops] + self.stream = stream + sink = flow(self.sink) if isinstance(self.sink, Fluid) else self.sink + + iterator = self._wrap_progress(stream) + # Drive an executor's progress bar (a GUI canvas) per item — independent of the console + # ``show_progress`` rich bar; a no-op when no progress callback was injected. + total = _stream_total(stream) + desc = self.progress_desc or "DatasetProcessor" + + if sink: + logger.info(f"Streaming data to sink: {sink.__class__.__name__}") + # Replicates recordstream.core.stream.Stream.to_sink so we can iterate through + # our progress wrapper while preserving the Storage context + flush. + sink_ctx: Any = sink if isinstance(sink, Storage) else nullcontext() + count = 0 + with sink_ctx: + for record in iterator: + sink.write(record) + count += 1 + self._report_progress(count, total, desc) + sink.flush() + logger.info(f"Streamed {count} record(s) to sink.") + else: + logger.info("No sink provided. Materializing data in-memory.") + results = [] + for count, record in enumerate(iterator, start=1): + results.append(record) + self._report_progress(count, total, desc) + logger.info(f"Processed {len(results)} records.") + + logger.info("Processing complete.") + + def _wrap_progress(self, stream: Stream) -> Iterable[Any]: + """Wrap the stream iterator in rich.progress when ``show_progress`` is enabled. + + Source length is probed defensively — not every DataSource implements + ``__len__`` (e.g. glob-based streaming sources). Missing lengths + degrade to a count-only bar instead of breaking the run. + """ + if not self.show_progress: + return stream + desc = self.progress_desc or "DatasetProcessor" + return _ProgressIter(stream, total=_stream_total(stream), desc=desc) + + +class _ProgressIter: + """Iterable wrapper that opens a ``rich.progress`` bar at ``__iter__`` time. + + Kept as a class (not a generator) so ``DatasetProcessor._wrap_progress`` + can return a value that is truthy to ``bool()`` even when empty, matching + the contract of raw ``Stream`` which behaves like a ``Sized`` (``Stream`` + subclasses ``torch.utils.data.Dataset``). + """ + + def __init__(self, stream: Stream, total: Optional[int], desc: str) -> None: + self._stream = stream + self._total = total + self._desc = desc + + def __iter__(self) -> Iterator[Any]: + with Progress( + TextColumn("[progress.description]{task.description}"), + BarColumn(), + MofNCompleteColumn(), + TextColumn("•"), + TimeRemainingColumn(), + transient=True, + ) as progress: + task = progress.add_task(self._desc, total=self._total) + for record in self._stream: + yield record + progress.update(task, advance=1) + + +__all__ = ["DatasetProcessor"] diff --git a/recordstream/projection.py b/recordstream/projection.py new file mode 100644 index 0000000..5a4f4ed --- /dev/null +++ b/recordstream/projection.py @@ -0,0 +1,228 @@ +"""Key projection for RecordStream sources — read only the record keys you need. + +Walking a source for a single key (the canonical case: counting classes from the label +key) should not pay for constructing the values you don't need — e.g. decoding image +inputs you are about to throw away. This module adds an **opt-in** projection protocol +plus walk helpers that any consumer can use against any source, with a correct (if +unoptimized) fallback for sources that don't implement the protocol. + +The primitive is deliberately general (any subset of record KEYS); :func:`num_classes` +is one helper built on top of it. + +Design notes +------------ +* :class:`SupportsProjection` is a ``Protocol`` (never a base class), so a source + opts in by *defining* ``project``, not by inheriting. +* Every public function is a lazy generator (**Partial Evaluation** mandate) — + nothing materializes the whole source. +* :func:`num_classes` (integer class-id semantics) is a free function, *not* a + method on the generic :class:`~recordstream.core.stream.Stream` engine — counting classes is + a classification concern, and bolting it onto the task-agnostic engine would + make every ``Stream`` look classification-capable to duck-typed consumers. +""" + +from typing import Any, Collection, Iterator, List, Optional, Protocol, runtime_checkable + +from recordstream.items import Record, item_value + + +@runtime_checkable +class SupportsProjection(Protocol): + """A source that can yield partial records restricted to the requested keys. + + Implementers SHOULD avoid building unrequested values — e.g. skip decoding the + input image when only the label key is asked for; that efficiency is the whole + point of the protocol. ``keys`` is a subset of the source's record keys. + """ + + def project(self, keys: Collection[str]) -> Iterator[Record]: ... + + +def project(source: Any, keys: Collection[str]) -> Iterator[Record]: + """Yield partial records from ``source`` carrying only ``keys``. + + Uses the source's own ``project`` when it implements :class:`SupportsProjection` (the + efficient path that skips building unrequested values); otherwise falls back to a full + iteration that keeps only the requested keys. Partial: a generator. + + A DEFERRED source (a ``!class:`` marker straight out of a config) is materialized first, + so a caller never has to remember which entry point flows and which does not — + :meth:`~recordstream.LabelMap.encode` already did, and every consumer of this one was + writing ``flow(source)`` at the call site to compensate. Flowing a live object is a no-op. + """ + from confluid import flow + + source = flow(source) + want = frozenset(keys) + if isinstance(source, SupportsProjection): + yield from source.project(want) + return + for record in source: + yield {k: v for k, v in record.items() if k in want} + + +def iter_key(source: Any, key: str) -> Iterator[Any]: + """Lazily yield each record's ``key`` VALUE (skipping other-key construction when supported). + + Unwrapping is :func:`~recordstream.items.item_value`'s rule, not a second spelling of it: a + :class:`~recordstream.items.Label` yields its ``.value`` (the class id / name), a + :class:`~recordstream.items.MultiLabel` its ``.values`` list, any other registered item its + payload, and a plain value passes through verbatim. What is THIS function's own is only the + projection — a record without ``key`` yields ``None``. + """ + for record in project(source, (key,)): + yield item_value(record.get(key)) + + +def first_value(source: Any, key: str) -> Any: + """The first non-``None`` value under ``key`` in ``source`` — ``None`` when there is none. + + The cheapest possible question about a column: ONE peek, which is all a consumer needs to + learn the KIND of the values without walking the set. The canonical use is a label column, + where the peek decides whether the targets are class NAMES needing a + :class:`~recordstream.labels.LabelMap` or ids that pass straight through (ask + :func:`~recordstream.items.is_class_id` of the answer), and whether the column is + multi-label (a :class:`~recordstream.items.MultiLabel` arrives here as its ``values`` + LIST, so a sequence IS multi-label — the item type, not a guess about what a list means). + + Built on :func:`iter_key`, so all three of its properties carry over: a + projection-aware source never builds the values this does not ask for, a deferred source + is materialized first, and the walk stops at the first hit — a missing or all-``None`` + column costs one full pass and answers ``None`` rather than raising. + """ + for value in iter_key(source, key): + if value is not None: + return value + return None + + +def _to_int(value: Any) -> int: + """Coerce a single target into a Python ``int`` class id. + + Handles plain ``int``, numpy scalars, and 0-d / single-element torch tensors + (via ``.item()``). Rejects ``bool`` (an ``int`` subclass — accepting it would + silently turn a boolean target into class 0/1) and anything that isn't a + scalar so callers fail loudly instead of miscounting. + """ + if isinstance(value, bool): + raise TypeError(f"target {value!r} is a bool, not a class id") + if isinstance(value, int): + return value + item = getattr(value, "item", None) + if callable(item): + try: + result = item() + except Exception as exc: # pragma: no cover - exotic array/tensor types + raise TypeError(f"could not read a scalar class id from target {value!r}: {exc}") from exc + if isinstance(result, bool): + raise TypeError(f"target {value!r} resolved to a bool, not a class id") + if isinstance(result, int): + return result + if isinstance(result, float) and result.is_integer(): + return int(result) + raise TypeError(f"target {value!r} did not yield an integer class id (got {result!r})") + raise TypeError(f"target {value!r} of type {type(value).__name__} is not a scalar class id") + + +def class_names(*sources: Any) -> Optional[List[str]]: + """The class vocabulary carried by the first of ``sources`` that has one. + + The naming counterpart of :func:`num_classes`: that one WALKS a source to count classes, + this one READS the vocabulary a source already carries — set by + :meth:`~recordstream.LabelMap.encode` when it wrapped the source, so the names travel with + the encoded data rather than in a LabelMap the consumer has to keep alongside it. + + Takes several sources because a vocabulary is a property of the RUN, not of whichever + split happens to carry it: a config may encode only the train set, or hand the eval path a + pre-encoded test set. ``None`` entries are skipped, so the common + ``class_names(train_set, val_set, test_set)`` needs no guards at the call site. + + Args: + *sources: Datasets / streams to consult, in priority order. ``None`` values are ignored. + + Returns: + The names as a list of ``str``, or ``None`` when no source carries a usable vocabulary + — a source with no labels, or an unencoded one, is not an error. + + Example:: + + names = class_names(train_set, val_set, test_set) # ['bird', 'cat', 'dog'] or None + """ + for source in sources: + names = getattr(source, "class_names", None) + if not names: + continue + try: + return [str(n) for n in names] + except TypeError: # not iterable — a source using the name for something else + continue + return None + + +def num_classes(source: Any, key: str = "class") -> int: + """Derive the number of classes by walking **every** ``key`` value in ``source``. + + Always walks the full label stream (key-restricted, so other values are never + constructed when the source supports projection) and returns + ``max(class_id) + 1`` — the classifier-head size needed to cover the largest + label, robust to a class id that happens not to appear in this split. Raises + ``ValueError`` if the source yields no values (or a ``None`` value) under ``key``. + + This is the engine behind a dataset's lazy ``num_classes()`` method. + + Args: + source: The source to walk (any iterable of records; projection-aware when supported). + key: The record key holding the class label. Defaults to ``"label"``. + """ + highest = -1 + for target in iter_key(source, key): + if target is None: + raise ValueError(f"num_classes: encountered a record with no {key!r} value — cannot derive a class count.") + cid = _to_int(target) + if cid > highest: + highest = cid + if highest < 0: + raise ValueError(f"num_classes: source yielded no {key!r} values — cannot derive a class count.") + return highest + 1 + + +def num_mask_classes(source: Any, key: str = "mask") -> int: + """The per-pixel twin of :func:`num_classes` — walk a MASK column and return ``max + 1``. + + A per-pixel target is an ``[H, W]`` array of class ids rather than one id, so the count is + the largest id anywhere in the column plus one. Same contract as :func:`num_classes` + otherwise: the walk is key-restricted (a projection-aware source never decodes the image + beside the mask), it covers EVERY record so a class appearing only in the last one still + sizes the head, and an empty column raises rather than returning a plausible number. + + It is a separate function rather than a widening of :func:`num_classes`, and the reason is + that function's strictness: ``_to_int`` REJECTS a non-scalar target on purpose, so a + classification run that is accidentally handed arrays fails loudly instead of miscounting. + Teaching it to take the max of whatever it gets would trade that guard away for both tasks. + + Args: + source: The source to walk (any iterable of records; projection-aware when supported). + key: The record key holding the per-pixel mask. Defaults to ``"mask"``. + """ + import numpy as np + + highest = -1 + for mask in iter_key(source, key): + if mask is None: + raise ValueError(f"num_mask_classes: a record has no {key!r} value — cannot derive a class count.") + arr = np.asarray(mask) + if arr.size == 0: + continue # an empty mask constrains nothing; a column of them raises below + highest = max(highest, int(arr.max())) + if highest < 0: + raise ValueError(f"num_mask_classes: source yielded no usable {key!r} masks — cannot derive a class count.") + return highest + 1 + + +__all__ = [ + "SupportsProjection", + "project", + "iter_key", + "num_classes", + "num_mask_classes", +] diff --git a/dataflux/py.typed b/recordstream/py.typed similarity index 100% rename from dataflux/py.typed rename to recordstream/py.typed diff --git a/recordstream/runnable.py b/recordstream/runnable.py new file mode 100644 index 0000000..0764ea3 --- /dev/null +++ b/recordstream/runnable.py @@ -0,0 +1,228 @@ +"""The runnable protocol: gradient + progress marker mixins for long-running objects. + +A *runnable* is any object exposing a no-arg ``run(self)`` — a trainer, an +evaluator, a dataset processor, a workflow. This module carries the two stateless +mixins a runnable inherits so a GUI executor (one that runs the object as a graph +node) can cooperate with it WITHOUT this package importing the GUI framework: + +* :class:`TorchRunner` — marks a runnable whose ``run()`` needs autograd (it + performs gradient-based optimization). The class is named for the framework it + concerns; the flag it sets says what it MEANS — a GUI executor that evaluates nodes + under ``torch.inference_mode()`` reads the duck-typed ``__needs_autograd__`` flag + and re-enables autograd for the duration of ``run()``. +* :class:`ProgressReporting` — gives a runnable a framework-free progress callback. + The executor injects a ``(value, total, desc) -> None`` sink via + :meth:`~ProgressReporting.set_progress_callback`; the runnable drains it from its + loop via :meth:`~ProgressReporting._report_progress` (a silent no-op when no sink + was injected, so a plain CLI run is unaffected). + +Both are pure marker/utility mixins (no required state) so they compose cleanly +with any base class (a ``torch.nn.Module``, a ``LightningModule``, a plain object). +This package declares only *that* a runnable needs autograd / can report progress; +the framework-specific bridges (e.g. a Lightning ``Callback`` that maps per-batch +hooks onto :meth:`~ProgressReporting._report_progress`) live in the consuming +framework package, not here. + +It also carries the :func:`entrypoint` method marker: a runnable that exposes SEVERAL +capabilities from one class — the merged train+eval classes drive ``fit`` / +``evaluate`` / ``test`` / ``predict`` off a single ``task`` knob — annotates each such +method with the ``task`` value it runs and a ``role`` label (``"trainer"`` / +``"evaluator"`` / ``"predictor"``). A discovery consumer (a config generator, a visual +editor) reads these via :func:`runnable_entrypoints` to learn that one class both +trains and evaluates, instead of assuming a separate class per role — and the runnable's +own ``run()`` dispatches through :func:`run_entrypoint`, so the markers ARE the dispatch +table rather than a description of one kept in sync by hand. Straightforward worked +example (the class + the exact introspector outputs): ``docs/runnable.md``. +""" + +from typing import Any, Callable, Dict, List, Literal, Optional + +from loggair import get_logger + +logger = get_logger(__name__) + +#: Attribute stamped on a method by :func:`entrypoint`. +_ENTRYPOINT_ATTR = "__runnable_entrypoint__" + +#: The four standard tasks of a merged train+eval runnable — the ``task`` values its +#: ``@entrypoint`` markers declare and its ``run()`` dispatches on. Declared HERE because they are +#: this module's vocabulary (the markers/dispatch above speak them); a consumer's own task alias +#: is typically ``MyTask = RunnableTask`` so the closed set is written exactly once. A runnable +#: with a narrower or wider capability set declares its own Literal instead — the type names the +#: CONVENTION, it does not gate dispatch (``run_entrypoint`` reads the markers, not this). +RunnableTask = Literal["fit", "evaluate", "test", "predict"] + +#: A progress sink: ``(value, total, description) -> None``. ``value`` / ``total`` are +#: floats in the same unit (optimizer steps, records); ``description`` is a short stage label. +ProgressCallback = Callable[[float, float, str], None] + + +class TorchRunner: + """Mixin marking a runnable whose ``run()`` performs gradient-based optimization. + + Pure marker — no state, no ``__init__`` — so it composes cleanly with any base + (``torch.nn.Module``, ``pytorch_lightning.LightningModule``, …). It lets a GUI + executor distinguish "this run needs autograd" from "this run is inference-only". + + Why it exists: a GUI executor may call ``run()`` from inside a graph evaluation + wrapped in ``torch.inference_mode()`` for cheap, grad-free node evaluation. Under + inference mode every tensor created — model parameters, forward activations, the + loss — is an inference tensor with no autograd graph, so ``loss.backward()`` dies + with *"element 0 of tensors does not require grad and does not have a grad_fn"*. + The executor reads the duck-typed ``__needs_autograd__`` flag (no hard import on the + GUI side) and re-enables normal autograd for the duration of ``run()``. + + Note the deliberate split between the two names: the CLASS is named for the framework + whose execution mode is at stake (autograd is a torch concept, and a non-torch backend + would not inherit this mixin at all), while the FLAG is named for what it decides — + "this run needs autograd". A runnable that only sometimes trains overrides the flag as + a property (``return self.task == "fit"``), which reads correctly only under that name. + + Inference-only runnables (a pure evaluator, a dataset processor) deliberately do + NOT inherit this — they run as-is under the executor's inference mode. + """ + + __needs_autograd__: bool = True + + +class ProgressReporting: + """Mixin giving a runnable a framework-free progress callback. + + Pure mixin — no ``__init__``, no required state. The executor injects a sink via + :meth:`set_progress_callback`; the runnable drains it from its loop via + :meth:`_report_progress`. When no sink is injected (a plain CLI run, or any + non-GUI caller) every call is a silent no-op, so behaviour is unchanged. + """ + + #: Set by the executor via :meth:`set_progress_callback`; ``None`` ⇒ progress reporting is a no-op. + _progress_callback: Optional[ProgressCallback] = None + + def set_progress_callback(self, callback: Optional[ProgressCallback]) -> None: + """Inject (or clear, with ``None``) the progress sink the executor drains. Idempotent.""" + self._progress_callback = callback + + def _report_progress(self, value: float, total: Optional[float], desc: str = "") -> None: + """Report ``value`` of ``total`` to the injected callback; no-op when unset or invalid. + + Never raises — a broken progress sink must never abort a run (it is cosmetic). A ``None`` or + non-positive ``total`` (an unbounded / unknown length, e.g. a streaming source) is skipped so + the executor's bar stays in its indeterminate state rather than dividing by zero. + """ + callback = self._progress_callback + if callback is None or not total or total <= 0: + return + try: + callback(float(value), float(total), desc) + except Exception: # noqa: BLE001 - progress reporting must never break the run + logger.debug("progress callback raised; ignoring", exc_info=True) + + +def entrypoint(task: str, role: str = "runnable", primary: bool = False) -> Callable: + """Mark a runnable method as a named capability entry point. + + A class that drives several capabilities off one ``task`` knob (the merged train+eval + runnables: ``fit`` / ``evaluate`` / ``test`` / ``predict``) annotates each entry-point + method so a discovery consumer (a config generator, a visual editor) can learn — from + ONE class — which capabilities it exposes, instead of assuming a separate class per role. + + Example — one class, four capabilities (full walkthrough in ``docs/runnable.md``):: + + class Classifier(TorchRunner, ProgressReporting): + def run(self) -> None: + run_entrypoint(self, self.task) # the markers below ARE the dispatch table + + @entrypoint("fit", role="trainer", primary=True) + def fit(self) -> None: ... + + @entrypoint("evaluate", role="evaluator") # validation-split metrics + def evaluate(self) -> None: ... + + @entrypoint("test", role="evaluator", primary=True) # held-out metrics = the default evaluator + def test(self) -> None: ... + + @entrypoint("predict", role="predictor", primary=True) + def predict(self) -> None: ... + + entrypoint_tasks(Classifier, "evaluator") # -> ["test", "evaluate"] (primary first) + entrypoint_tasks(Classifier, "trainer") # -> ["fit"] + + A consumer asked for "an evaluator config" takes ``entrypoint_tasks(cls, "evaluator")[0]`` + and pins ``task: test`` in the config it emits. + + Args: + task: The ``task`` value the runnable's ``run()`` dispatches to for this method + (e.g. ``"fit"`` / ``"test"``). + role: A capability label — conventionally ``"trainer"`` (fits/trains), + ``"evaluator"`` (computes metrics over a held-out set), or ``"predictor"`` + (streams predictions). Free-form so new capabilities need no change here. + primary: When several methods share a ``role``, marks the default one (e.g. ``test`` + is the primary ``"evaluator"`` over ``evaluate``/validate). + """ + + def deco(fn: Callable) -> Callable: + setattr(fn, _ENTRYPOINT_ATTR, {"task": task, "role": role, "primary": primary}) + return fn + + return deco + + +def runnable_entrypoints(cls: type) -> Dict[str, Dict[str, object]]: + """Return ``{method_name: {"task", "role", "primary"}}`` for every :func:`entrypoint` method. + + Walks the MRO so an inherited entry point is found, and reads the marker off the raw + function object (never triggering a property getter). + """ + out: Dict[str, Dict[str, object]] = {} + for klass in reversed(cls.__mro__): + for name, attr in vars(klass).items(): + meta = getattr(attr, _ENTRYPOINT_ATTR, None) + if meta is not None: + out[name] = dict(meta) + return out + + +def entrypoint_tasks(cls: type, role: str) -> List[str]: + """Return the ``task`` values of ``cls``'s entry points with the given ``role``, primary first.""" + matches = [(name, meta) for name, meta in runnable_entrypoints(cls).items() if meta.get("role") == role] + matches.sort(key=lambda nm: not nm[1].get("primary", False)) # primary (True) sorts first + return [str(meta["task"]) for _, meta in matches] + + +def run_entrypoint(runnable: object, task: str) -> Any: + """Call ``runnable``'s :func:`entrypoint` method whose declared task is ``task``. + + This is the RUNTIME half of the marker: a merged train+eval class's ``run()`` is + ``run_entrypoint(self, self.task)``, so the ``@entrypoint`` decorators ARE the dispatch + table instead of merely describing one. A hand-written ``{task: method}`` dict states the + same mapping a second time, and the two drift silently in one direction that matters — a + config generator pins ``task:`` from :func:`entrypoint_tasks` (the markers), so a + capability added to the markers but forgotten in the dict yields a generated config that + fails at dispatch with "unknown task" while discovery advertises it as supported. + + Unknown tasks raise :class:`ValueError` listing the declared ones in DECLARATION order + (``runnable_entrypoints`` preserves it), which reads as the class's capability list. + + Args: + runnable: The instance to dispatch on (its ``type()`` carries the markers). + task: The ``task`` value to run, matched against each entry point's declared task. + + Returns: + Whatever the entry-point method returns (``None`` for the merged runnables). + """ + by_task = {str(meta["task"]): name for name, meta in runnable_entrypoints(type(runnable)).items()} + name = by_task.get(task) + if name is None: + raise ValueError(f"Unknown task {task!r}; expected one of {list(by_task)}.") + return getattr(runnable, name)() + + +__all__ = [ + "ProgressCallback", + "ProgressReporting", + "RunnableTask", + "TorchRunner", + "entrypoint", + "entrypoint_tasks", + "run_entrypoint", + "runnable_entrypoints", +] diff --git a/recordstream/sources/__init__.py b/recordstream/sources/__init__.py new file mode 100644 index 0000000..b6e092d --- /dev/null +++ b/recordstream/sources/__init__.py @@ -0,0 +1,35 @@ +""" +RecordStream sources — the classes that yield (or derive a view of) record dicts. + +One class per module, re-exported here so ``from recordstream.sources import X`` keeps +working; the canonical dotted path a config / form-spec / MCP schema spells out is the +SUBMODULE one (``!class:recordstream.sources.huggingface.HuggingFaceSource``), exactly as +for :mod:`recordstream.ops`. Both spellings resolve — ``confluid.resolve_class`` falls back +to a module-path import, and this package re-exports every name — but generated configs and +the enrichment table key on the submodule path, because that is what ``cls.__module__`` says. + +Submodules: + - recordstream.sources.huggingface: HuggingFaceSource (+ the METADATA_ALL_FEATURES sentinel) + - recordstream.sources.split: DatasetSplit (+ the SplitName Literal) + - recordstream.sources.range: RangeSource (a contiguous ``[start:stop)`` slice) + - recordstream.sources.concat: ConcatSource (several indexable sources end to end) + +``__all__`` below is load-bearing, not decoration: a visual editor's node bridge scans this +module in two passes, and the first (``recordstream.discovery.scan_module``) filters on +``member.__module__``, so it sees NOTHING here now that the classes live in submodules. The +second pass — the one that surfaces these nodes — walks exactly this ``__all__``. +""" + +from recordstream.sources.concat import ConcatSource +from recordstream.sources.huggingface import METADATA_ALL_FEATURES, HuggingFaceSource +from recordstream.sources.range import RangeSource +from recordstream.sources.split import DatasetSplit, SplitName + +__all__ = [ + "ConcatSource", + "DatasetSplit", + "HuggingFaceSource", + "METADATA_ALL_FEATURES", + "RangeSource", + "SplitName", +] diff --git a/recordstream/sources/base.py b/recordstream/sources/base.py new file mode 100644 index 0000000..197f02a --- /dev/null +++ b/recordstream/sources/base.py @@ -0,0 +1,33 @@ +"""Shared internals for the view sources (``split`` / ``range`` / ``concat``).""" + +from typing import Any + +from confluid.fluid import Fluid as _ConfluidFluid + +from recordstream.core import _fluid_source_guidance + + +def _guard_live_source(source: Any, slot: str) -> None: + """Raise Stream's actionable deferred-marker error when ``source`` is still a Fluid. + + A still-deferred marker in a ``source:`` slot is a CONFIG error — the slot needs a live + object — and the view sources answer it exactly as ``Stream._guard_live_source`` does, + naming the slot, the deferred target and the fix (drop ``_partial_: true``), instead of + the cryptic ``got Partial`` a bare ``hasattr`` check produces. Deliberately + message-only: the slot is never flowed here (the raise-with-guidance convention for + ``source:`` slots, distinct from the free functions ``project`` / ``dataset_uri``, + which do materialize a marker first). + """ + if isinstance(source, _ConfluidFluid): + raise TypeError(_fluid_source_guidance(source, slot=slot)) + + +def _pass_through(item: Any) -> Any: + """Pass a wrapped source's item through verbatim. + + Every carrier is a plain dict; the view sources + (:class:`~recordstream.sources.DatasetSplit` / :class:`~recordstream.sources.RangeSource` / + :class:`~recordstream.sources.ConcatSource`) only slice/index, they never inspect payloads, + so a source's records flow through them unchanged. + """ + return item diff --git a/recordstream/sources/concat.py b/recordstream/sources/concat.py new file mode 100644 index 0000000..6f51dfc --- /dev/null +++ b/recordstream/sources/concat.py @@ -0,0 +1,73 @@ +"""``ConcatSource`` — several indexable sources presented end to end as one.""" + +import bisect +from typing import Any, Iterator, List, Optional + +from confluid import configurable + +from recordstream.items import Record +from recordstream.sources.base import _guard_live_source, _pass_through + + +@configurable(category="source") +class ConcatSource: + """Concatenates multiple indexable sources into one longer indexable source. + + The indexable counterpart to :class:`recordstream.core.JointStream` (which is iteration-only): + ``len`` is the sum of the parts and ``source[i]`` maps a global index onto the owning + sub-source, so a ``ConcatSource`` can itself be wrapped by + :class:`~recordstream.sources.DatasetSplit` / :class:`~recordstream.sources.RangeSource`. + (Distinct from an annotation-join source, which *column-joins* annotations onto records — + this one *concatenates* sequences end to end.) + + Each sub-source must implement ``__len__`` and ``__getitem__``. + + Args: + sources: The indexable sources to concatenate, walked in order (defaults to ``None`` ⇒ empty). + """ + + def __init__(self, sources: Optional[List[Any]] = None) -> None: + # Partial / zero-arg: store config only; sub-source validation + the cumulative-offset precompute + # are deferred to the ``offsets`` property so sources can be configured post-construction. + self.sources = list(sources) if sources else [] + self._offsets: Optional[List[int]] = None + + @property + def offsets(self) -> List[int]: + """Cumulative END offsets per sub-source, computed lazily on first access and cached. + + Computing them validates each sub-source (``__len__`` / ``__getitem__``); enables an + O(log k) global-index → (sub-source, local index) map. + """ + if self._offsets is None: + offsets: List[int] = [] + total = 0 + for i, src in enumerate(self.sources): + _guard_live_source(src, f"ConcatSource.sources[{i}]") + if not hasattr(src, "__len__") or not hasattr(src, "__getitem__"): + raise TypeError( + "ConcatSource requires sources supporting __len__ and __getitem__; " + f"source[{i}] is {type(src).__name__}" + ) + total += len(src) + offsets.append(total) + self._offsets = offsets + return self._offsets + + def __len__(self) -> int: + return self.offsets[-1] if self.offsets else 0 + + def __getitem__(self, index: int) -> Any: + n = len(self) + if index < 0: + index += n + if not 0 <= index < n: + raise IndexError(index) + j = bisect.bisect_right(self.offsets, index) + start = self.offsets[j - 1] if j > 0 else 0 + return _pass_through(self.sources[j][index - start]) + + def __iter__(self) -> Iterator[Record]: + for src in self.sources: + for item in src: + yield _pass_through(item) diff --git a/recordstream/sources/huggingface.py b/recordstream/sources/huggingface.py new file mode 100644 index 0000000..111c65f --- /dev/null +++ b/recordstream/sources/huggingface.py @@ -0,0 +1,309 @@ +"""``HuggingFaceSource`` — a Hugging Face dataset as a stream of record dicts.""" + +from pathlib import Path +from typing import Any, Collection, Dict, Iterator, List, Optional +from urllib.parse import quote, urlencode + +from confluid import configurable +from loggair import get_logger + +from recordstream.items import Image, Label, Record + +logger = get_logger(__name__) + +#: Scheme of the canonical identifier for a Hub dataset. Matches the convention hosted +#: tracking services already use for a dataset source, so a URI recorded here is the one +#: their UI expects rather than a spelling invented for this package. +HF_URI_PREFIX = "hf://datasets/" + +#: Where a Hub dataset is browsable. The ``/viewer//`` suffix opens the +#: dataset viewer on exactly the rows this source reads. +HF_BROWSE_PREFIX = "https://huggingface.co/datasets/" + +#: The config name the Hub viewer uses when a dataset declares no named configs. +HF_DEFAULT_CONFIG = "default" + +# Sentinel for ``HuggingFaceSource.metadata_features`` meaning "every dataset column except the +# input/target features" — the full-traceability option, kept OPT-IN (``None`` / ``[]`` still = no +# extra metadata) so existing configs are unaffected. Resolved against the loaded dataset's +# ``column_names`` at construction. Accepted bare (``"*"``) or as the one-element list (``["*"]``); +# Visual editors offer it as a selectable "*" entry in a metadata picker. +METADATA_ALL_FEATURES = "*" + + +def _resolve_metadata_features( + requested: Optional[List[str] | str], + column_names: Optional[List[str]], + input_feature: str, + target_feature: str, +) -> List[str]: + """Resolve a ``metadata_features`` spec into a concrete, order-preserving column list. + + ``None`` / ``[]`` -> ``[]`` (no extra metadata — the backward-compatible default). The sentinel + ``"*"`` (bare or inside a list) -> every column in ``column_names`` except ``input_feature`` / + ``target_feature`` (full traceability). An explicit list of names is used verbatim. ``"*"`` may + be combined with extra names (union, order-preserving: the "rest" first, then the extras). + """ + if not requested: + return [] + if isinstance(requested, str): + requested = [requested] + if METADATA_ALL_FEATURES not in requested: + return list(requested) + excluded = {input_feature, target_feature} + rest = [c for c in (column_names or []) if c not in excluded] + extras = [r for r in requested if r != METADATA_ALL_FEATURES and r not in excluded and r not in rest] + return rest + extras + + +@configurable(category="source") +class HuggingFaceSource: + """ + RecordStream Source for Hugging Face Datasets, yielding plain record dicts. + + Key mapping (the record layout): + + * the ``input_feature`` value (image / array) -> an :class:`~recordstream.Image` under the + record key ``"image"``; + * the ``target_feature`` value (label) -> a :class:`~recordstream.Label` under the record key + ``"class"``; + * each ``metadata_features`` column -> its own :class:`~recordstream.Label` keyed by the column + name, plus the source-provenance ``hf_path`` / ``hf_split`` Labels. + + Partial & zero-arg per the workspace class-design convention (see confluid AGENTS.md + "Partial Initialization & Zero-Arg Construction"): the constructor only stores values and + does NO functional work — ``HuggingFaceSource()`` is valid, and the dataset is downloaded + only on first access to :attr:`dataset` (cached thereafter; reset ``_dataset`` to reload). + ``path`` is therefore optional at construction and validated lazily when the data is needed. + + It also carries its own IDENTITY (:mod:`recordstream.uri`): :attr:`dataset_uri` names the + dataset canonically (``hf://datasets/ylecun/mnist?split=train``, or a ``file://`` URI for a + local imagefolder) and :attr:`dataset_url` links to the Hub viewer for the same rows. Both + read stored configuration only — asking either never loads anything. + + Args: + path: HF dataset identifier — a Hub repo id (e.g. ``kitofrank/RFUAV``) or a local imagefolder path. + split: HF split name (``train`` / ``validation`` / ``test`` / etc.). + input_feature: Dataset feature column mapped onto the ``"image"`` record key (an ``Image`` item). + target_feature: Dataset feature column mapped onto the ``"class"`` record key (a ``Label`` item). + metadata_features: Columns -> per-column ``Label`` entries; ``None``=none, ``"*"``=all-but-i/o, else a list. + count: Optional cap on the number of records yielded (useful for fast smoke runs). + name: Optional HF subset/config name (e.g. for multi-config datasets). + revision: Optional dataset git revision (branch / tag / commit). Declared rather than + left to ``load_kwargs`` because it is part of the source's IDENTITY — ``dataset_uri`` + reads it. + load_kwargs: The remaining ``datasets.load_dataset`` options (``token``, ``cache_dir``, + ``trust_remote_code``, …) as an explicit mapping. A dict rather than ``**kwargs``: + see the note in ``__init__`` — a ``**kwargs`` constructor accepts every broadcast key + there is, and ``load_dataset`` turns an unknown keyword into a builder config name. + """ + + def __init__( + self, + path: str = "", + split: str = "train", + input_feature: str = "image", + target_feature: str = "label", + metadata_features: Optional[List[str] | str] = "*", + count: Optional[int] = None, + name: Optional[str] = None, + revision: Optional[str] = None, + load_kwargs: Optional[Dict[str, Any]] = None, + ) -> None: + # Partial constructor: store config only — never load here. Real work (the network/disk + # download) is deferred to the ``dataset`` property so the object is cheap to build and + # configurable post-construction. + self.path = path + self.split = split + self.input_feature = input_feature + self.target_feature = target_feature + # Stored as the RAW spec (``None`` / ``"*"`` / list) — resolved against the loaded dataset's + # columns lazily by the ``resolved_metadata_features`` property, not eagerly here. + self.metadata_features = metadata_features + self.count = count + self.name = name + self.revision = revision + # The long tail of `datasets.load_dataset` options (``token``, ``cache_dir``, + # ``trust_remote_code``), as an EXPLICIT dict rather than the `**kwargs` this took until + # 2026-08-02. That `**kwargs` was a live hazard, not merely untidy: confluid/liquifai + # broadcast a key into any node whose constructor ACCEPTS it, and a `**kwargs` constructor + # accepts every key there is — so unrelated run identity landed in `load_dataset`, which + # turns unknown keyword arguments into a builder CONFIG NAME. A single + # `sonair train … --run_name my_run` therefore looked for `mnist` under a config named + # `default-`, missed the cache, and went to the Hub. Measured: the same + # command passes with no name override and fails with one. + self._load_kwargs = dict(load_kwargs or {}) + # Partial cache for the materialized dataset (see the ``dataset`` property). + self._dataset: Any = None + + @property + def load_options(self) -> Dict[str, Any]: + """``load_kwargs`` with the declared ``revision`` folded in — what reaches ``load_dataset``. + + Recomputed on every read rather than merged in ``__init__``, per the workspace + derived-state rule: ``revision`` is a DECLARED parameter, so the config layer may set it + post-construction (that is how a broadcast key arrives), and a dict assembled once in the + constructor would silently keep the value the object was born with. + """ + options = dict(self._load_kwargs) + if self.revision is not None: + options["revision"] = self.revision + return options + + @property + def dataset(self) -> Any: + """The HF dataset, loaded on first access and cached. Resetting ``_dataset`` to None reloads. + + Raises ``ValueError`` if ``path`` was never set — the zero-arg constructor allows building an + unconfigured source, but materializing one without a dataset id cannot succeed. + """ + if self._dataset is None: + if not self.path: + raise ValueError( + "HuggingFaceSource.path is empty — set it (constructor arg, YAML, or configure()) " + "before iterating or indexing the source." + ) + from datasets import load_dataset + + # The identifier goes in the LOAD line: it is the one moment a reader of the log + # can tie the run to a specific dataset, and the browsable URL is what makes that + # tie followable rather than merely recorded. + logger.info(f"HuggingFaceSource: Loading {self.dataset_url or self.dataset_uri}...") + self._dataset = load_dataset(self.path, name=self.name, split=self.split, **self.load_options) + return self._dataset + + # -- identity (see recordstream.uri) ------------------------------------------------------ + + @property + def _identity_query(self) -> str: + """The ``name`` / ``revision`` / ``split`` selection as a sorted query string. + + Sorted so two identically-configured sources produce the SAME string — a URI whose + parameter order depended on insertion would not compare equal to itself. + """ + parts: Dict[str, str] = {} + if self.name: + parts["name"] = str(self.name) + if self.revision: + parts["revision"] = str(self.revision) + if self.split: + parts["split"] = str(self.split) + return urlencode(sorted(parts.items())) + + @property + def dataset_uri(self) -> Optional[str]: + """Canonical identifier for the dataset this source reads — ``None`` without a ``path``. + + A Hub repo id becomes ``hf://datasets/?…``; a local directory becomes its + ``file://`` URI. Which one applies is decided by whether ``path`` exists on disk — + the same question ``datasets.load_dataset`` itself answers. Pure string work over the + stored configuration: nothing is loaded, so an unconsumed source still answers. + """ + if not self.path: + return None + local = Path(self.path) + base = local.resolve().as_uri() if local.exists() else HF_URI_PREFIX + quote(str(self.path).strip("/")) + query = self._identity_query + return f"{base}?{query}" if query else base + + @property + def dataset_url(self) -> Optional[str]: + """Browsable Hub link, or ``None`` for a local dataset (which has no web page). + + Points at the dataset VIEWER on this source's config + split when a split is + configured, so the link opens on the rows this source reads rather than on the + repository's front page. + """ + if not self.path or Path(self.path).exists(): + return None + page = HF_BROWSE_PREFIX + quote(str(self.path).strip("/")) + if not self.split: + return page + config = quote(str(self.name)) if self.name else HF_DEFAULT_CONFIG + return f"{page}/viewer/{config}/{quote(str(self.split))}" + + @property + def resolved_metadata_features(self) -> List[str]: + """``metadata_features`` resolved against the live dataset's columns (expands the ``"*"`` sentinel). + + Partial because the ``"*"`` expansion needs the loaded dataset's ``column_names``; ``None`` / ``[]`` + stays "no extra metadata" (backward-compatible). + """ + return _resolve_metadata_features( + self.metadata_features, getattr(self.dataset, "column_names", None), self.input_feature, self.target_feature + ) + + def _to_record( + self, + item: Any, + metadata_features: List[str], + keys: Optional[Collection[str]] = None, + ) -> Record: + """Assemble one record dict from a raw HF row dict (see the class docstring for the key mapping). + + ``keys`` gates which record entries are built (``None`` = all) — the projection path + (:meth:`project`) passes only the requested ones, so an unwanted image is never decoded. + ``metadata_features`` arrives pre-filtered on the projection path. + """ + record: Record = {} + if keys is None or "image" in keys: + # The input value (image/array) becomes an ``Image`` item; a PIL image / list is coerced + # to an ndarray by ``Image.__new__`` (np.asarray), preserving the default HWC layout. + record["image"] = Image(item.get(self.input_feature)) + if keys is None or "class" in keys: + record["class"] = Label(item.get(self.target_feature)) + # Each requested metadata column rides its OWN Label entry keyed by the column name (the + # metadata a value needs travels WITH it). Source provenance follows the same shape. + for feature in metadata_features: + record[feature] = Label(item.get(feature)) + if keys is None or "hf_path" in keys: + record["hf_path"] = Label(self.path) + if keys is None or "hf_split" in keys: + record["hf_split"] = Label(self.split) + return record + + def __iter__(self) -> Iterator[Record]: + dataset = self.dataset + metadata_features = self.resolved_metadata_features + limit = self.count or len(dataset) + + for counter, item in enumerate(dataset): + if counter >= limit: + break + yield self._to_record(item, metadata_features) + + def __getitem__(self, index: int) -> Record: + return self._to_record(self.dataset[index], self.resolved_metadata_features) + + def project(self, keys: Collection[str]) -> Iterator[Record]: + """Yield key-restricted records — the ``SupportsProjection`` efficient path. + + Only the requested keys are built, so a label-only walk (e.g. :func:`~recordstream.num_classes`) + skips decoding the image entirely: ``"image"`` -> the input feature, ``"class"`` -> the target + Label, plus any requested metadata-column / provenance keys. + """ + want = frozenset(keys) + dataset = self.dataset + # Resolve (and pre-filter) the metadata columns only when a key beyond the fixed image/class + # pair is requested — the "*" expansion needs the loaded dataset's columns. + meta_requested = bool(want - {"image", "class"}) + metadata_features = [f for f in self.resolved_metadata_features if f in want] if meta_requested else [] + limit = self.count or len(dataset) + for counter, item in enumerate(dataset): + if counter >= limit: + break + yield self._to_record(item, metadata_features, keys=want) + + def __len__(self) -> int: + # A ``count`` of 0 (or None) means "all records", matching __iter__'s + # ``limit = self.count or len(...)``. Returning a bare ``self.count`` here + # would report 0 for the common "0 == unlimited" case, making the source + # look empty (e.g. a downstream len()-based stepper raising ``len == 0``) + # even though iteration yields every record. + # + # A ``count`` LARGER than the split is clamped, never reported verbatim: a map-style + # consumer trusts ``len()`` for its index space, so a lying length surfaces as an + # ``IndexError`` deep inside a DataLoader worker, one epoch in. (Found the hard way: + # ``count: 32`` over cppe-5's 29-row test split killed the first validation pass.) + n = len(self.dataset) + return min(self.count, n) if self.count else n diff --git a/recordstream/sources/range.py b/recordstream/sources/range.py new file mode 100644 index 0000000..05e9783 --- /dev/null +++ b/recordstream/sources/range.py @@ -0,0 +1,70 @@ +"""``RangeSource`` — a contiguous ``[start:stop)`` index slice over an indexable source.""" + +from typing import Any, Iterator, List, Optional + +from confluid import configurable +from loggair import get_logger + +from recordstream.items import Record +from recordstream.sources.base import _guard_live_source, _pass_through + +logger = get_logger(__name__) + + +@configurable(category="source") +class RangeSource: + """A contiguous index slice ``[start:stop)`` over an indexable source. + + The plain-slice counterpart to :class:`~recordstream.sources.DatasetSplit` (which shuffles + + partitions) — extracted from DatasetSplit's old "range mode". Negative ``start`` / + ``stop`` count from the end; both are clamped to ``[0, len(source)]``. Partial: only index + arithmetic happens up front; records are produced on demand. + + The wrapped source must implement ``__len__`` and ``__getitem__``. + + Args: + source: The underlying indexable source (defaults to ``None``; validated lazily on first use). + start: Inclusive start index (``None`` ⇒ 0; a negative value counts from the end). + stop: Exclusive stop index (``None`` ⇒ len(source); a negative value counts from the end). + """ + + def __init__(self, source: Any = None, start: Optional[int] = None, stop: Optional[int] = None) -> None: + # Partial / zero-arg: store config only; the index arithmetic (and source validation) is deferred + # to the ``indices`` property so the source can be configured post-construction. + self.source = source + self.start = start + self.stop = stop + self._indices: Optional[List[int]] = None + + @property + def indices(self) -> List[int]: + """The contiguous ``[start:stop)`` source indices, computed lazily on first access and cached.""" + if self._indices is None: + source = self.source + _guard_live_source(source, "RangeSource.source") + if source is None or not hasattr(source, "__len__") or not hasattr(source, "__getitem__"): + raise TypeError( + "RangeSource requires a source supporting __len__ and __getitem__; " f"got {type(source).__name__}" + ) + n = len(source) + s = 0 if self.start is None else self.start + e = n if self.stop is None else self.stop + if s < 0: + s = max(0, n + s) + if e < 0: + e = max(0, n + e) + s = max(0, min(s, n)) + e = max(s, min(e, n)) + self._indices = list(range(s, e)) + logger.debug("RangeSource: size=%d source_size=%d", len(self._indices), n) + return self._indices + + def __iter__(self) -> Iterator[Record]: + for idx in self.indices: + yield _pass_through(self.source[idx]) + + def __getitem__(self, index: int) -> Any: + return _pass_through(self.source[self.indices[index]]) + + def __len__(self) -> int: + return len(self.indices) diff --git a/recordstream/sources/split.py b/recordstream/sources/split.py new file mode 100644 index 0000000..5bbf6bf --- /dev/null +++ b/recordstream/sources/split.py @@ -0,0 +1,196 @@ +"""``DatasetSplit`` — reproducible train/val/test views over an indexable source.""" + +import random +from typing import Any, Dict, Iterator, List, Literal, Optional, get_args + +from confluid import configurable + +from recordstream.items import Record +from recordstream.sources.base import _guard_live_source, _pass_through + +# Closed set of split names for DatasetSplit's fraction mode (workspace mandate: prefer +# closed Literals over bare strings — self-documenting + machine-introspectable by UIs / +# navigaitor form-spec / MCP schemas via ``typing.get_args``). The runtime-validation tuple +# is derived from the Literal so there is ONE source of truth — never restate the values. +SplitName = Literal["train", "val", "test"] +_SPLIT_NAMES = get_args(SplitName) + + +@configurable(category="source") +class DatasetSplit: + """ + Splits an indexable source into reproducible ``train`` / ``val`` / ``test`` views. + + A ``source`` (it yields records and is wired into a trainer's ``source:`` slot), + not an engine — it applies no ops, it just exposes a reproducible partition of another + source. (For a contiguous index slice use :class:`~recordstream.sources.RangeSource`; to + concatenate several sources use :class:`~recordstream.sources.ConcatSource`.) + + **Property API (preferred).** Configure ONE ``DatasetSplit`` with ``seed`` and the + held-out fraction(s) (``val_fraction`` and/or ``test_fraction``) and read the three + cached view sources off it:: + + split = DatasetSplit(source=src, val_fraction=0.1, test_fraction=0.1, seed=42) + split.train # ≈80% — the remainder + split.val # ≈10% + split.test # ≈10% + + The views are disjoint and complementary, computed once (cached) over a single + deterministic shuffle. In a config each view is a ``DatasetSplit`` of its own with the + ``split`` selector set — the recipe is written once (a YAML anchor) and merged (``<<:``) + into the other views. Every view references the SAME ``source`` (``!ref:`` shares the + instance), so the upstream source is loaded exactly once; a ``DatasetSplit``'s own + partition is one seeded shuffle over ``len(source)``, cheap to repeat:: + + train_set: !class:recordstream.core.stream.Stream() + source: &split_recipe !class:recordstream.sources.split.DatasetSplit() + source: !ref:hf_train + val_fraction: 0.1 + test_fraction: 0.1 + seed: 42 + split: train + val_set: !class:recordstream.core.stream.Stream() + source: !class:recordstream.sources.split.DatasetSplit() + <<: *split_recipe + split: val + + (Reading a view by attribute reference — ``!ref:my_split.train`` — is no longer a + config spelling; the config engine refuses it and names this rewrite.) + + **Select-one API.** Passing ``split`` makes the ``DatasetSplit`` itself iterate that one + view (``split=None`` ⇒ ``train``), so it is directly usable as a single ``source:``. + + Omit ``test_fraction`` for a plain two-way train/val split; omit both fractions for a + degenerate split where ``train`` is the whole source and ``val`` / ``test`` are empty. + + The wrapped source must implement ``__len__`` and ``__getitem__``. Partial: only index + arithmetic happens up front; records are produced on demand. + + Args: + source: The underlying indexable source (defaults to ``None``; validated lazily on first use). + split: View this iterates as a source — ``train`` / ``val`` / ``test`` (``None`` ⇒ ``train``). + val_fraction: Fraction of records assigned to the ``val`` view. Must be in ``(0, 1)``. + test_fraction: Fraction of records assigned to the ``test`` view. Must be in ``(0, 1)``. + seed: Seed for the deterministic shuffle. Required when any fraction is set. + """ + + def __init__( + self, + source: Any = None, + split: Optional[SplitName] = None, + val_fraction: Optional[float] = None, + test_fraction: Optional[float] = None, + seed: Optional[int] = None, + ) -> None: + # Partial / zero-arg: store config only. All validation is deferred to first materialization + # (``_validate``, invoked from ``_view``) so the source can be configured post-construction. + self.source = source + self.split = split + self.val_fraction = val_fraction + self.test_fraction = test_fraction + self.seed = seed + # Cache of materialized split views. Underscore-prefixed so confluid's + # vars(obj)-based discovery / dump ignores it (the `train`/`val`/`test` + # @property descriptors live on the class, not in vars(obj), so they never + # surface as configurable attributes either). + self._views: Dict[str, "_SplitView"] = {} + + def _validate(self) -> None: + """Validate the (post-construction) configuration. Called lazily before the first partition.""" + source = self.source + _guard_live_source(source, "DatasetSplit.source") + if source is None or not hasattr(source, "__len__") or not hasattr(source, "__getitem__"): + raise TypeError( + "DatasetSplit requires a source supporting __len__ and __getitem__; " f"got {type(source).__name__}" + ) + if self.split is not None and self.split not in _SPLIT_NAMES: + raise ValueError(f"split must be one of {_SPLIT_NAMES}; got {self.split!r}") + if (self.val_fraction is not None or self.test_fraction is not None) and self.seed is None: + raise ValueError("DatasetSplit requires `seed` when a fraction is set, so the partition is reproducible.") + if self.val_fraction is not None and not (0.0 < self.val_fraction < 1.0): + raise ValueError(f"val_fraction must be in (0, 1); got {self.val_fraction}") + if self.test_fraction is not None and not (0.0 < self.test_fraction < 1.0): + raise ValueError(f"test_fraction must be in (0, 1); got {self.test_fraction}") + if (self.val_fraction or 0.0) + (self.test_fraction or 0.0) >= 1.0: + raise ValueError( + "val_fraction + test_fraction must be < 1 (to leave a non-empty train split); " + f"got val_fraction={self.val_fraction}, test_fraction={self.test_fraction}" + ) + + def _partition(self) -> Dict[str, List[int]]: + """Deterministically partition the source indices into ``train`` / ``val`` / ``test``. + + One shuffle seeded by ``seed`` (skipped when no fraction is set, so the degenerate + "all train" case keeps source order); layout is ``[train | val | test]``. ``max(1, …)`` + guarantees a held-out split gets at least one record on tiny sources. + """ + n = len(self.source) + val_fraction = self.val_fraction or 0.0 + test_fraction = self.test_fraction or 0.0 + shuffled = list(range(n)) + if val_fraction or test_fraction: + random.Random(self.seed).shuffle(shuffled) + val_count = max(1, int(round(n * val_fraction))) if val_fraction else 0 + test_count = max(1, int(round(n * test_fraction))) if test_fraction else 0 + train_count = max(0, n - val_count - test_count) + return { + "train": shuffled[:train_count], + "val": shuffled[train_count : train_count + val_count], + "test": shuffled[train_count + val_count :], + } + + def _view(self, split: SplitName) -> "_SplitView": + if split not in self._views: + self._validate() + self._views[split] = _SplitView(self.source, self._partition()[split]) + return self._views[split] + + @property + def train(self) -> "_SplitView": + """Cached training-split view (the remainder after ``val`` / ``test`` are held out).""" + return self._view("train") + + @property + def val(self) -> "_SplitView": + """Cached validation-split view (≈ ``val_fraction`` of the source).""" + return self._view("val") + + @property + def test(self) -> "_SplitView": + """Cached test-split view (≈ ``test_fraction`` of the source).""" + return self._view("test") + + def __iter__(self) -> Iterator[Record]: + return iter(self._view(self.split or "train")) + + def __getitem__(self, index: int) -> Any: + return self._view(self.split or "train")[index] + + def __len__(self) -> int: + return len(self._view(self.split or "train")) + + +class _SplitView: + """An indexable view of ``source`` restricted (and reordered) to ``indices``. + + Internal to :class:`DatasetSplit` — produced by its ``train`` / ``val`` / ``test`` + properties (in a config, select a view with the ``split`` parameter). Deliberately + NOT a ``@configurable``: it is never constructed directly in a config, only read off a + live ``DatasetSplit`` instance, so it carries no discovery surface of its own. It stays + in this module for the same reason — it is DatasetSplit's own return type, not a + separately-exported source. + """ + + def __init__(self, source: Any, indices: List[int]) -> None: + self.source = source + self.indices = indices + + def __iter__(self) -> Iterator[Record]: + for idx in self.indices: + yield _pass_through(self.source[idx]) + + def __getitem__(self, index: int) -> Any: + return _pass_through(self.source[self.indices[index]]) + + def __len__(self) -> int: + return len(self.indices) diff --git a/recordstream/storage/base.py b/recordstream/storage/base.py new file mode 100644 index 0000000..043db98 --- /dev/null +++ b/recordstream/storage/base.py @@ -0,0 +1,154 @@ +import json +from typing import Any, Dict, Iterator, Protocol, Self, Tuple, runtime_checkable + +import numpy as np + +from recordstream._compat import is_torch_tensor +from recordstream.items import Record + +#: Root-attribute format tag stamped on stores written in the record key-group layout. +TYPED_FORMAT = "typedrecord-v1" + +#: Attr/entry name a ``"plain"``-tagged SCALAR record value is stored under (the array +#: payload of a plain value rides the regular ``data`` slot instead). +PLAIN_VALUE = "value" + +#: Prefix marking a JSON-encoded structured attr value (list/tuple/dict/None) in plain attrs. +_JSON_MARK = "__json__:" + + +def require_record_format(found: Any, where: str) -> None: + """Raise unless ``found`` is the record-layout tag ``"typedrecord-v1"``. + + There is deliberately NO backward compatibility with pre-record layouts: a store whose + ``recordstream_format`` tag is missing or different was written before the plain-dict + record model and must be re-generated with a current sink. + """ + if found == TYPED_FORMAT: + return + detail = "no recordstream_format tag" if found is None else f"recordstream_format={found!r}" + raise ValueError( + f"{where}: {detail} — expected {TYPED_FORMAT!r}. Pre-record-model datasets are not " + "readable/appendable; re-generate them with a current sink." + ) + + +def to_numpy(data: Any) -> Any: + """Convert a torch tensor to a numpy array for array-storage backends (HDF5 / Zarr). + + Detaches and moves to CPU first so tensors carrying grad or living on a GPU + convert cleanly. Non-tensor values pass through unchanged. + """ + if is_torch_tensor(data): + return data.detach().cpu().numpy() + return data + + +@runtime_checkable +class DataSource(Protocol): + """Minimum contract for a RecordStream data source.""" + + def __iter__(self) -> Iterator[Record]: + """Iterate over records in the source.""" + ... + + def __len__(self) -> int: + """Total number of records available.""" + ... + + +@runtime_checkable +class DataSink(Protocol): + """Minimum contract for a RecordStream data sink.""" + + def write(self, record: Record) -> None: + """Write a single record to the sink.""" + ... + + def flush(self) -> None: + """Ensure all pending writes are committed to storage.""" + ... + + +# -------------------------------------------------------------------------------------- +# The shared attr wire-format for the record key-group layout (HDF5 attrs / Zarr .zattrs +# / directory JSON all speak it): scalars stay native (queryable), array values become +# separate datasets, and structured values (list/tuple/dict/None) ride a JSON string with +# TUPLE TAGGING so a round-trip preserves tuple-ness (Boxes.canvas == (H, W), not [H, W]). +# -------------------------------------------------------------------------------------- +def split_attrs(attrs: Dict[str, Any]) -> Tuple[Dict[str, Any], Dict[str, Any]]: + """Split an encoded value's attrs into ``(plain, arrays)`` for storage. + + ``plain`` holds natively-storable scalars/strings plus JSON-marked structured values; + ``arrays`` holds ndarray/Tensor attr values (stored as their own datasets). + """ + plain: Dict[str, Any] = {} + arrays: Dict[str, Any] = {} + for key, value in attrs.items(): + if isinstance(value, np.ndarray) or is_torch_tensor(value): + arrays[key] = to_numpy(value) + elif isinstance(value, np.generic): + plain[key] = value.item() + elif isinstance(value, (bool, int, float, str)): + plain[key] = value + else: + plain[key] = _JSON_MARK + json.dumps(_tag_json(value)) + return plain, arrays + + +def restore_attrs(plain: Dict[str, Any], arrays: Dict[str, Any]) -> Dict[str, Any]: + """Rebuild an encoded value's attrs dict from :func:`split_attrs`' two halves.""" + attrs: Dict[str, Any] = {} + for key, value in plain.items(): + if isinstance(value, np.generic): + value = value.item() + if isinstance(value, bytes): # h5py may hand string attrs back as bytes + value = value.decode("utf-8") + if isinstance(value, str) and value.startswith(_JSON_MARK): + attrs[key] = _untag_json(json.loads(value[len(_JSON_MARK) :])) + else: + attrs[key] = value + attrs.update(arrays) + return attrs + + +def _tag_json(value: Any) -> Any: + """JSON-safe view of ``value`` with tuples tagged (``{"__tuple__": [...]}``) so they survive.""" + if isinstance(value, tuple): + return {"__tuple__": [_tag_json(v) for v in value]} + if isinstance(value, list): + return [_tag_json(v) for v in value] + if isinstance(value, dict): + return {k: _tag_json(v) for k, v in value.items()} + if isinstance(value, np.generic): + return value.item() + return value + + +def _untag_json(value: Any) -> Any: + """Reverse :func:`_tag_json` (restores tuples).""" + if isinstance(value, dict): + if set(value.keys()) == {"__tuple__"}: + return tuple(_untag_json(v) for v in value["__tuple__"]) + return {k: _untag_json(v) for k, v in value.items()} + if isinstance(value, list): + return [_untag_json(v) for v in value] + return value + + +class Storage: + """Base class for storage backends providing context manager support.""" + + def open(self) -> Self: + return self + + def close(self) -> None: + pass # pragma: no cover + + def __enter__(self) -> Self: + # `Self`, not `Storage`: `with HDF5Sink(...) as sink` must keep the concrete + # backend type so `sink.write(...)` type-checks at the call site. + return self.open() + + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + self.close() diff --git a/dataflux/storage/cache.py b/recordstream/storage/cache.py similarity index 99% rename from dataflux/storage/cache.py rename to recordstream/storage/cache.py index 036888c..4c5b36b 100644 --- a/dataflux/storage/cache.py +++ b/recordstream/storage/cache.py @@ -11,7 +11,7 @@ from pathlib import Path from typing import Callable, List, Tuple, Union -from logflow import get_logger +from loggair import get_logger logger = get_logger(__name__) diff --git a/recordstream/storage/directory.py b/recordstream/storage/directory.py new file mode 100644 index 0000000..bb2970a --- /dev/null +++ b/recordstream/storage/directory.py @@ -0,0 +1,162 @@ +import json +from pathlib import Path +from typing import Any, Dict, Iterator, Union + +import confluid +import numpy as np + +from recordstream.io import PLAIN_TYPE, EncodedItem, decode_item, encode_item +from recordstream.items import Record +from recordstream.storage.base import ( + PLAIN_VALUE, + TYPED_FORMAT, + DataSink, + DataSource, + Storage, + require_record_format, + restore_attrs, + split_attrs, + to_numpy, +) + +#: Record-layout filenames inside each per-record directory. +_FIELDS_JSON = "fields.json" +_FIELDS_NPZ = "fields.npz" + + +# category="sink": surfaced by visual editors as a sink node docking into a DatasetProcessor's sink slot. +@confluid.configurable(category="sink") +class DirectorySink(Storage, DataSink): + """ + High-concurrency sink that stores each record in its own directory. + Perfect for irregular data lengths and massive parallel writing. + """ + + def __init__(self, path: Union[str, Path] = "", overwrite: bool = False, use_npz: bool = True) -> None: + # Partial / zero-arg: store config only; the directory is created lazily in open(). + self.path = Path(path) + self.overwrite = overwrite + self.use_npz = use_npz + self._counter = 0 + + def open(self) -> "DirectorySink": + if self.overwrite and self.path.exists(): + # In a real app, we'd clear the directory + pass + self.path.mkdir(parents=True, exist_ok=True) + return self + + def write(self, record: Any) -> None: + """Write a record to its own subdirectory.""" + if not isinstance(record, dict): + raise TypeError(f"DirectorySink: expected a record dict, got {type(record).__name__}") + self.open() + self._write_record(record) + + def _write_record(self, record: Record) -> None: + """One record in the key-group layout: ``fields.json`` + ``fields.npz``. + + ``fields.json`` describes every entry (order, item type, plain attrs — a ``"plain"`` + value's non-array payload rides its ``attrs`` under ``"value"``, JSON-marked when + structured); ``fields.npz`` carries the array halves — payloads keyed by record key, + array-valued attrs keyed ``.``. Every value serializes through the + :mod:`recordstream.io` codec, so externally-registered item types round-trip with no + storage edits. + """ + record_dir = self.path / f"{self._counter:06d}" + record_dir.mkdir(parents=True, exist_ok=True) + + spec: Dict[str, Any] = {"recordstream_format": TYPED_FORMAT, "fields": []} + payloads: Dict[str, Any] = {} + for key, value in record.items(): + encoded = encode_item(value) + if encoded.type_name == PLAIN_TYPE: + plain, arrays = split_attrs({PLAIN_VALUE: encoded.payload}) + has_payload = bool(arrays) + spec["fields"].append( + { + "key": key, + "type": encoded.type_name, + "attrs": plain, + "array_attrs": [], + "has_payload": has_payload, + } + ) + if has_payload: + payloads[key] = np.asarray(arrays[PLAIN_VALUE]) + continue + plain, arrays = split_attrs(encoded.attrs) + spec["fields"].append( + { + "key": key, + "type": encoded.type_name, + "attrs": plain, + "array_attrs": sorted(arrays), + "has_payload": encoded.payload is not None, + } + ) + if encoded.payload is not None: + payloads[key] = np.asarray(to_numpy(encoded.payload)) + for name, attr_value in arrays.items(): + payloads[f"{key}.{name}"] = np.asarray(attr_value) + + (record_dir / _FIELDS_JSON).write_text(json.dumps(spec, indent=2)) + if payloads: + np.savez(record_dir / _FIELDS_NPZ, **payloads) + self._counter += 1 + + def flush(self) -> None: + pass # Filesystem handles immediate writes + + +@confluid.configurable +class DirectorySource(Storage, DataSource): + """Read records written by :class:`DirectorySink` (one ``fields.json`` + ``fields.npz`` per record). + + The matching source of the sink's record layout (one directory per record, sorted by the + zero-padded name, so read order matches write order). + + Args: + path: Root directory written by DirectorySink. + """ + + def __init__(self, path: Union[str, Path] = "") -> None: + # Partial / zero-arg: store config only; the directory is scanned lazily on iteration. + self.path = Path(path) + + def _record_dirs(self) -> list: + if not self.path.exists(): + raise FileNotFoundError(f"DirectorySource: {self.path} does not exist") + return sorted(p for p in self.path.iterdir() if p.is_dir() and (p / _FIELDS_JSON).exists()) + + def __iter__(self) -> Iterator[Record]: + for record_dir in self._record_dirs(): + yield self._read(record_dir) + + def __len__(self) -> int: + return len(self._record_dirs()) + + @staticmethod + def _read(record_dir: Path) -> Record: + spec = json.loads((record_dir / _FIELDS_JSON).read_text()) + require_record_format(spec.get("recordstream_format"), "DirectorySource") + npz_path = record_dir / _FIELDS_NPZ + payloads = dict(np.load(npz_path, allow_pickle=False)) if npz_path.exists() else {} + record: Record = {} + payload: Any + for entry in spec["fields"]: + key = entry["key"] + if entry["type"] == PLAIN_TYPE: + # A plain value: array payload in the npz, non-array payload restored from the + # ``value`` attr (see DirectorySink._write_record). + if entry["has_payload"]: + payload = payloads[key] + else: + payload = restore_attrs(dict(entry["attrs"]), {}).get(PLAIN_VALUE) + record[key] = decode_item(EncodedItem(type_name=PLAIN_TYPE, payload=payload, attrs={})) + continue + arrays = {name: payloads[f"{key}.{name}"] for name in entry["array_attrs"]} + attrs = restore_attrs(dict(entry["attrs"]), arrays) + payload = payloads[key] if entry["has_payload"] else None + record[key] = decode_item(EncodedItem(type_name=entry["type"], payload=payload, attrs=attrs)) + return record diff --git a/recordstream/storage/hdf5.py b/recordstream/storage/hdf5.py new file mode 100644 index 0000000..8d11cba --- /dev/null +++ b/recordstream/storage/hdf5.py @@ -0,0 +1,218 @@ +import json +from pathlib import Path +from typing import Any, Dict, Iterator, Optional, Union + +import h5py +import numpy as np +from confluid import configurable +from loggair import get_logger + +from recordstream.io import PLAIN_TYPE, EncodedItem, decode_item, encode_item +from recordstream.items import Record +from recordstream.storage.base import ( + PLAIN_VALUE, + TYPED_FORMAT, + DataSink, + DataSource, + Storage, + require_record_format, + restore_attrs, + split_attrs, + to_numpy, +) + +logger = get_logger("recordstream.storage.hdf5") + +#: Reserved key-group attr names in the record layout (never item attrs). +_TYPE_ATTR = "__item_type__" +_ORDER_ATTR = "__field_order__" + + +def read_record_group(group: h5py.Group, slices: Optional[Dict[str, slice]] = None) -> Record: + """Decode one ``sNNNNNN`` record group of the record key-group layout. + + ``slices`` maps a record KEY to a slice applied to that field's ``data`` dataset at + read time — an h5py partial read, so only the requested span of the payload ever + leaves the file. This is what lets a windowing source slice one window out of a + large stored row (e.g. a whole-capture IQ array) without materializing the row: + the decoded item is identical to reading the full payload and slicing in memory + (pinned by the storage tests). Keys absent from ``slices`` (and every attr) are + read in full; a sliced key whose group has no ``data`` dataset raises ``KeyError``. + """ + order = json.loads(group.attrs[_ORDER_ATTR]) + record: Record = {} + payload: Any + for name in order: + fgrp = group[name] + type_name = str(fgrp.attrs[_TYPE_ATTR]) + window = (slices or {}).get(name) + if type_name == PLAIN_TYPE: + # A plain value: array payload as the ``data`` dataset, scalar payload as the + # ``value`` attr (JSON-marked when structured) — see HDF5Sink._write_record. + if "data" in fgrp: + payload = fgrp["data"][window] if window is not None else fgrp["data"][()] + else: + if window is not None: + raise KeyError(f"read_record_group: field {name!r} has no 'data' dataset to slice") + payload = restore_attrs({PLAIN_VALUE: fgrp.attrs[PLAIN_VALUE]}, {})[PLAIN_VALUE] + record[name] = decode_item(EncodedItem(type_name=type_name, payload=payload, attrs={})) + continue + plain = {k: v for k, v in fgrp.attrs.items() if k != _TYPE_ATTR} + arrays: Dict[str, Any] = {} + agrp = fgrp.get("attrs") + if isinstance(agrp, h5py.Group): + for key, dset in agrp.items(): + arrays[key] = dset[()] + if window is not None: + if "data" not in fgrp: + raise KeyError(f"read_record_group: field {name!r} has no 'data' dataset to slice") + payload = fgrp["data"][window] + else: + payload = fgrp["data"][()] if "data" in fgrp else None + attrs = restore_attrs(dict(plain), arrays) + record[name] = decode_item(EncodedItem(type_name=type_name, payload=payload, attrs=attrs)) + return record + + +@configurable +class HDF5Source(Storage, DataSource): + """Read records written by :class:`HDF5Sink` (the record key-group layout). + + Args: + path: Path to the HDF5 file written by HDF5Sink. + """ + + def __init__(self, path: Union[str, Path] = "") -> None: + # Partial / zero-arg: store config only; the file is opened lazily in open() (an unset path + # surfaces there, not in __init__). + self.path = Path(path) + self._file: Optional[h5py.File] = None + + def open(self) -> "HDF5Source": + if self._file is None: + handle = h5py.File(self.path, "r") + found = handle.attrs.get("recordstream_format") + if found != TYPED_FORMAT: + handle.close() + require_record_format(found, "HDF5Source") + self._file = handle + return self + + def close(self) -> None: + if self._file: + self._file.close() + self._file = None + + def __iter__(self) -> Iterator[Record]: + self.open() + if self._file is None: + return + for name in sorted(k for k in self._file.keys() if k.startswith("s")): + yield read_record_group(self._file[name]) + + def __len__(self) -> int: + self.open() + if self._file is None: + return 0 + return len([k for k in self._file.keys() if k.startswith("s")]) + + def iter_metadata(self) -> "Iterator[tuple[str, dict]]": + """(prefix, metadata) per record WITHOUT loading data arrays (SupportsMetadataScan). + + Array-valued metadata appears as shape/dtype stub strings — see + :func:`recordstream.storage.query.scan_hdf5_metadata`. + """ + from recordstream.storage.query import scan_hdf5_metadata + + yield from scan_hdf5_metadata(self.path) + + +# category="sink": surfaced by visual editors as a sink node docking into a DatasetProcessor's sink slot. +@configurable(category="sink") +class HDF5Sink(Storage, DataSink): + """High-performance HDF5 data sink for plain record dicts.""" + + def __init__( + self, + path: Union[str, Path] = "", + compression: Optional[str] = "gzip", + overwrite: bool = False, + ) -> None: + # Partial / zero-arg: store config only; the file is opened lazily in open(). + self.path = Path(path) + self.compression = compression + self.overwrite = overwrite + self._file: Optional[h5py.File] = None + self._counter = 0 + + def open(self) -> "HDF5Sink": + if self._file is None: + mode = "w" if self.overwrite and self._counter == 0 else "a" + self.path.parent.mkdir(parents=True, exist_ok=True) + logger.info(f"Opening HDF5 file for writing: {self.path} (mode={mode})") + self._file = h5py.File(self.path, mode) + return self + + def close(self) -> None: + if self._file: + self._file.close() + self._file = None + + def write(self, record: Any) -> None: + self.open() + if self._file is None: + return + if not isinstance(record, dict): + raise TypeError(f"HDF5Sink: expected a record dict, got {type(record).__name__}") + self._write_record(record) + + def _write_record(self, record: Record) -> None: + """One record in the key-group layout. + + Layout: root attr ``recordstream_format = "typedrecord-v1"``; per record a group + ``sNNNNNN`` (attr ``__field_order__`` preserves insertion order) holding one subgroup + per KEY with the ``__item_type__`` attr + the item's plain attrs, the payload as + ``data``, and array-valued attrs as datasets under ``attrs/``. A ``"plain"`` value + stores an array payload as ``data`` and any other payload as the ``value`` attr + (JSON-marked when structured). Every value serializes through the + :mod:`recordstream.io` codec, so externally-registered item types round-trip with no + storage edits. + """ + assert self._file is not None + existing = self._file.attrs.get("recordstream_format") + if existing is None and len(self._file) == 0: + self._file.attrs["recordstream_format"] = TYPED_FORMAT + elif existing != TYPED_FORMAT: + require_record_format(existing, "HDF5Sink") + + group = self._file.create_group(f"s{self._counter:06d}") + group.attrs[_ORDER_ATTR] = json.dumps(list(record.keys())) + for key, value in record.items(): + encoded = encode_item(value) + fgrp = group.create_group(key) + fgrp.attrs[_TYPE_ATTR] = encoded.type_name + if encoded.type_name == PLAIN_TYPE: + plain, arrays = split_attrs({PLAIN_VALUE: encoded.payload}) + if arrays: + arr = np.asarray(arrays[PLAIN_VALUE]) + kwargs = {"compression": self.compression} if self.compression and arr.ndim > 0 else {} + fgrp.create_dataset("data", data=arr, **kwargs) + else: + fgrp.attrs[PLAIN_VALUE] = plain[PLAIN_VALUE] + continue + plain, arrays = split_attrs(encoded.attrs) + for name, attr_value in plain.items(): + fgrp.attrs[name] = attr_value + if encoded.payload is not None: + payload = np.asarray(to_numpy(encoded.payload)) + kwargs = {"compression": self.compression} if self.compression and payload.ndim > 0 else {} + fgrp.create_dataset("data", data=payload, **kwargs) + for name, attr_value in arrays.items(): + arr = np.asarray(attr_value) + kwargs = {"compression": self.compression} if self.compression and arr.ndim > 0 else {} + fgrp.create_dataset(f"attrs/{name}", data=arr, **kwargs) + self._counter += 1 + + def flush(self) -> None: + if self._file: + self._file.flush() diff --git a/recordstream/storage/query.py b/recordstream/storage/query.py new file mode 100644 index 0000000..26bc0a8 --- /dev/null +++ b/recordstream/storage/query.py @@ -0,0 +1,246 @@ +"""Queryable metadata — filter stored records by metadata predicates WITHOUT loading arrays. + +Two pieces (mirroring the ``recordstream.projection`` protocol-plus-fallback design): + +- :class:`SupportsMetadataScan` — a source opts in by implementing + ``iter_metadata() -> Iterator[(key, metadata_dict)]`` that reads ONLY the metadata + (HDF5 attrs, Zarr ``.zattrs``, a sidecar JSON) — never the data arrays. + Free-function scanners for the shipped sources live here (``scan_hdf5_metadata`` / + ``scan_zarr_metadata``); any external storage source can implement the protocol + directly (it is structural — no import of this module required). +- :class:`MetadataFilterSource` — a view source (``category="source"``) yielding only + the records whose metadata passes a predicate: the YAML-friendly ``where`` expression + (the same restricted-eval namespace as ``FormulaOp`` — metadata keys become variables) + and/or a programmatic ``predicate`` callable. The matching index set is computed + lazily from the metadata scan (cached), so arrays load only for matches; a source + without the protocol falls back to a full iteration filter. + +Record-layout HDF5/Zarr stores are queryable with NO extra index — their metadata already +lives in attrs/``.zattrs``. (A ``.metaindex`` sidecar accelerator is a TASKS.md follow-up +if scans ever become hot.) +""" + +import json +from typing import Any, Callable, Dict, Iterator, List, Optional, Protocol, Tuple, cast, runtime_checkable + +import h5py +import numpy as np +from confluid import configurable +from loggair import get_logger + +from recordstream.io import PLAIN_TYPE, encode_item +from recordstream.items import Record +from recordstream.ops.formula import _FORMULA_NAMESPACE +from recordstream.storage.base import PLAIN_VALUE, require_record_format, restore_attrs + +logger = get_logger("recordstream.storage.query") + +__all__ = [ + "MetadataFilterSource", + "SupportsMetadataScan", + "record_metadata", + "scan_hdf5_metadata", + "scan_zarr_metadata", +] + + +class _AttrView(dict): + """A metadata sub-dict that ALSO answers attribute access — so a record scan's per-key + attrs evaluate naturally in a ``where`` expression (``signal.samplerate > 1e6``) while + staying a plain dict for programmatic predicates.""" + + def __getattr__(self, name: str) -> Any: + try: + return self[name] + except KeyError as exc: # pragma: no cover - mirrors normal attribute-miss semantics + raise AttributeError(name) from exc + + +def _viewed(metadata: Dict[str, Any]) -> Dict[str, Any]: + """Wrap dict-valued entries in :class:`_AttrView` (one level — the record key/attr shape).""" + return {k: _AttrView(v) if isinstance(v, dict) else v for k, v in metadata.items()} + + +def record_metadata(record: Record) -> Dict[str, Dict[str, Any]]: + """A live record's queryable metadata: ``{key: {attr: value}}`` (attrs via the io codec, + payloads untouched) — the same nested shape the record storage scans yield. A ``"plain"`` + value contributes ``{"value": }`` when its payload is a scalar, else ``{}``.""" + out: Dict[str, Dict[str, Any]] = {} + for key, value in record.items(): + encoded = encode_item(value) + if encoded.type_name == PLAIN_TYPE: + payload = encoded.payload + if isinstance(payload, np.generic): + payload = payload.item() + out[key] = {PLAIN_VALUE: payload} if isinstance(payload, (bool, int, float, str)) else {} + else: + out[key] = dict(encoded.attrs) + return out + + +@runtime_checkable +class SupportsMetadataScan(Protocol): + """A source that can enumerate per-record metadata WITHOUT loading data arrays.""" + + def iter_metadata(self) -> Iterator[Tuple[str, Dict[str, Any]]]: + """Yield ``(record key, metadata dict)`` pairs, array payloads untouched.""" + ... # pragma: no cover - protocol + + +def scan_hdf5_metadata(path: Any) -> Iterator[Tuple[str, Dict[str, Any]]]: + """Scan an ``HDF5Sink`` file's metadata WITHOUT loading payload arrays. + + Per record the NESTED shape ``{key: {attr: value}}`` (plain attrs decoded; array-valued + attrs as SHAPE/DTYPE stubs ``""`` — queries can test + presence/shape without an array read; a ``"plain"`` value's scalar payload appears under + its ``value`` attr) — a ``where`` expression addresses it as ``"."`` + (e.g. ``"signal.samplerate > 1e6"``). + """ + with h5py.File(str(path), "r") as handle: + require_record_format(handle.attrs.get("recordstream_format"), "scan_hdf5_metadata") + for name in sorted(k for k in handle.keys() if k.startswith("s")): + group = handle[name] + nested: Dict[str, Any] = {} + for field in json.loads(group.attrs["__field_order__"]): + fgrp = group[field] + plain = {k: v for k, v in fgrp.attrs.items() if k != "__item_type__"} + attrs = restore_attrs(dict(plain), {}) + agrp = fgrp.get("attrs") + if isinstance(agrp, h5py.Group): + for key, dset in agrp.items(): + attrs[key] = f"" + nested[field] = attrs + yield name, nested + + +def scan_zarr_metadata(path: Any) -> Iterator[Tuple[str, Dict[str, Any]]]: + """Scan a ``ZarrGroupSink`` store's metadata: ``.zattrs`` only, no payload arrays. + + Yields the same NESTED ``{key: {attr: value}}`` shape as the HDF5 scan (array-valued + attrs as name stubs). + """ + import zarr + + root = zarr.open_group(str(path), mode="r") + require_record_format(root.attrs.get("recordstream_format"), "scan_zarr_metadata") + for name in sorted(root.group_keys()): + group = cast(Any, root[name]) + nested: Dict[str, Any] = {} + for field in json.loads(group.attrs["__field_order__"]): + fgrp = group[field] + plain = {k: v for k, v in dict(fgrp.attrs).items() if k != "__item_type__"} + attrs = restore_attrs(plain, {}) + if "attrs" in fgrp: + for key in fgrp["attrs"].array_keys(): + attrs[key] = f"" + nested[field] = attrs + yield name, nested + + +def _where_predicate(where: str) -> Callable[[Dict[str, Any]], bool]: + """Compile a ``where`` expression into a metadata predicate. + + The expression evaluates in the FormulaOp restricted namespace (``math.*`` + + ``abs``/``min``/``max``/``round``/``pow``, no builtins) with the metadata KEYS bound + as variables — e.g. ``"snr_db > 10 and drone == 'DJI'"`` (a flat external scan) or + ``"signal.samplerate > 1e6"`` (a record scan's per-key attrs). A missing key/attr + (NameError/AttributeError) means the record does not match (logged at debug); any + other evaluation error raises (a malformed expression must fail loudly). + + NOTE: a record key named like a Python keyword (e.g. ``class``) cannot be addressed in + an expression — query such keys via the programmatic ``predicate`` (metadata is plain + nested dicts there), or give queryable keys non-keyword names. + """ + + def _predicate(metadata: Dict[str, Any]) -> bool: + # Dict-valued entries (the record scans' per-key attr dicts) evaluate through + # _AttrView so "." reads naturally; flat external metadata is untouched. + namespace = {**_FORMULA_NAMESPACE, **_viewed(metadata)} + try: + return bool(eval(where, {"__builtins__": {}}, namespace)) # noqa: S307 - restricted namespace + except (NameError, AttributeError, KeyError) as exc: + logger.debug(f"MetadataFilterSource: where={where!r} — {exc}; record treated as non-matching") + return False + except Exception as exc: + raise ValueError(f"MetadataFilterSource: where expression {where!r} failed: {exc}") from exc + + return _predicate + + +@configurable(category="source") +class MetadataFilterSource: + """A view source yielding only the records whose metadata matches. + + Filtering uses the wrapped source's :class:`SupportsMetadataScan` protocol when + available (metadata-only scan — data arrays load ONLY for matching records, via the + source's ``__getitem__``), else falls back to full-iteration filtering (the + projection-module pattern). Match criteria compose with AND: the ``where`` expression + and the programmatic ``predicate`` must both pass when both are set. + + Args: + source: The wrapped source; required at use time, validated lazily. + where: Restricted boolean expression over metadata keys (e.g. ``"snr_db > 10"``). Blank = no expression. + predicate: Programmatic ``metadata -> bool`` callable (not serialized; the ``FilterOp.p`` convention). + """ + + def __init__( + self, + source: Optional[Any] = None, + where: str = "", + predicate: Optional[Callable[[Dict[str, Any]], bool]] = None, + ) -> None: + # Partial / zero-arg: store config only; matching indices compute lazily on first access. + self.source = source + self.where = str(where) + self.predicate = predicate + self._matches: Optional[List[int]] = None + + def _match(self, metadata: Dict[str, Any]) -> bool: + if self.where and not _where_predicate(self.where)(metadata): + return False + if self.predicate is not None and not self.predicate(metadata): + return False + return True + + @property + def matches(self) -> List[int]: + """Indices of matching records (computed once per instance; ``_matches = None`` resets).""" + if self._matches is None: + if self.source is None: + raise ValueError("MetadataFilterSource: a 'source' is required") + if not self.where and self.predicate is None: + raise ValueError("MetadataFilterSource: set 'where' and/or 'predicate' — an empty filter is a bug") + if isinstance(self.source, SupportsMetadataScan): + self._matches = [i for i, (_key, meta) in enumerate(self.source.iter_metadata()) if self._match(meta)] + else: + logger.debug( + f"MetadataFilterSource: {type(self.source).__name__} has no iter_metadata — " + "falling back to full-iteration filtering (arrays load for every record)." + ) + self._matches = [i for i, record in enumerate(self.source) if self._match(record_metadata(record))] + return self._matches + + def __iter__(self) -> Iterator[Record]: + matches = self.matches # validates the source before iteration + source: Any = self.source + if hasattr(source, "__getitem__"): + for index in matches: + yield cast(Record, source[index]) + else: + match_set = set(matches) + for i, record in enumerate(source): + if i in match_set: + yield cast(Record, record) + + def __len__(self) -> int: + return len(self.matches) + + def __getitem__(self, index: int) -> Record: + source_index = self.matches[index] + source: Any = self.source + if hasattr(source, "__getitem__"): + return cast(Record, source[source_index]) + for i, record in enumerate(source): + if i == source_index: + return cast(Record, record) + raise IndexError(index) diff --git a/recordstream/storage/zarr.py b/recordstream/storage/zarr.py new file mode 100644 index 0000000..1d967d1 --- /dev/null +++ b/recordstream/storage/zarr.py @@ -0,0 +1,292 @@ +import json +from pathlib import Path +from typing import Any, Dict, Iterator, List, Optional, Union, cast + +import confluid +import numpy as np +import zarr + +from recordstream.io import PLAIN_TYPE, EncodedItem, decode_item, encode_item +from recordstream.items import Record +from recordstream.storage.base import ( + PLAIN_VALUE, + TYPED_FORMAT, + DataSink, + DataSource, + Storage, + require_record_format, + restore_attrs, + split_attrs, + to_numpy, +) + +#: Reserved key-group attr names in the record layout (never item attrs). +_TYPE_ATTR = "__item_type__" +_ORDER_ATTR = "__field_order__" + + +def _read_record(grp: "zarr.Group") -> Record: + """Decode one ``record_NNNNNN`` group of the record key-group layout.""" + order = json.loads(str(grp.attrs[_ORDER_ATTR])) + record: Record = {} + payload: Any + for name in order: + fgrp = cast(zarr.Group, grp[name]) + fattrs = dict(fgrp.attrs) + type_name = str(fattrs[_TYPE_ATTR]) + if type_name == PLAIN_TYPE: + # A plain value: array payload as the ``data`` array, scalar payload as the + # ``value`` attr (JSON-marked when structured) — see ZarrGroupSink._write_record. + if "data" in fgrp.array_keys(): + payload = np.asarray(cast(zarr.Array, fgrp["data"])[:]) + else: + payload = restore_attrs({PLAIN_VALUE: fattrs[PLAIN_VALUE]}, {})[PLAIN_VALUE] + record[name] = decode_item(EncodedItem(type_name=type_name, payload=payload, attrs={})) + continue + plain = {k: v for k, v in fattrs.items() if k != _TYPE_ATTR} + arrays: Dict[str, Any] = {} + if "attrs" in fgrp: + agrp = cast(zarr.Group, fgrp["attrs"]) + for key in agrp.array_keys(): + arrays[key] = np.asarray(cast(zarr.Array, agrp[key])[:]) + payload = np.asarray(cast(zarr.Array, fgrp["data"])[:]) if "data" in fgrp.array_keys() else None + attrs = restore_attrs(plain, arrays) + record[name] = decode_item(EncodedItem(type_name=type_name, payload=payload, attrs=attrs)) + return record + + +# category="sink": surfaced by visual editors as a sink node docking into a DatasetProcessor's sink slot. +@confluid.configurable(category="sink") +class ZarrGroupSink(Storage, DataSink): + """ + Stores each record as a unique group within a Zarr group. + Supports variable lengths while keeping data in a single bundle. + """ + + def __init__(self, path: Union[str, Path] = "", overwrite: bool = False) -> None: + # Partial / zero-arg: store config only; the group is opened lazily in open(). + self.path = str(path) + self.overwrite = overwrite + self._root: Optional[zarr.Group] = None + self._counter = 0 + + def open(self) -> "ZarrGroupSink": + if self._root is None: + self._root = zarr.open_group(self.path, mode="a") + if self.overwrite: + # In a real app, we'd clear the group + pass + return self + + def write(self, record: Any) -> None: + self.open() + if self._root is None: + raise RuntimeError("Zarr group not open") + if not isinstance(record, dict): + raise TypeError(f"ZarrGroupSink: expected a record dict, got {type(record).__name__}") + self._write_record(record) + + def _write_record(self, record: Record) -> None: + """One record in the key-group layout (the Zarr twin of HDF5Sink._write_record).""" + assert self._root is not None + existing = self._root.attrs.get("recordstream_format") + if existing is None and not any(True for _ in self._root.group_keys()): + self._root.attrs["recordstream_format"] = TYPED_FORMAT + elif existing != TYPED_FORMAT: + require_record_format(existing, "ZarrGroupSink") + + grp = self._root.require_group(f"record_{self._counter:06d}") + grp.attrs[_ORDER_ATTR] = json.dumps(list(record.keys())) + for key, value in record.items(): + encoded = encode_item(value) + fgrp = grp.require_group(key) + fgrp.attrs[_TYPE_ATTR] = encoded.type_name + if encoded.type_name == PLAIN_TYPE: + plain, arrays = split_attrs({PLAIN_VALUE: encoded.payload}) + if arrays: + fgrp.create_array("data", data=np.asarray(arrays[PLAIN_VALUE]), overwrite=True) + else: + fgrp.attrs[PLAIN_VALUE] = plain[PLAIN_VALUE] + continue + plain, arrays = split_attrs(encoded.attrs) + fgrp.attrs.update(plain) + if encoded.payload is not None: + fgrp.create_array("data", data=np.asarray(to_numpy(encoded.payload)), overwrite=True) + for name, attr_value in arrays.items(): + fgrp.create_array(f"attrs/{name}", data=np.asarray(attr_value), overwrite=True) + self._counter += 1 + + def flush(self) -> None: + pass # pragma: no cover + + +@confluid.configurable +class ZarrGroupSource(Storage, DataSource): + """Read records written by :class:`ZarrGroupSink` (one Zarr group per record). + + Mirrors the group sink's key-group layout; groups are iterated in sorted name + order so the read order matches the write order. + + Args: + path: Path to the Zarr group written by ZarrGroupSink. + """ + + def __init__(self, path: Union[str, Path] = "") -> None: + # Partial / zero-arg: store config only; the group is opened lazily in open(). + self.path = str(path) + self._root: Optional[zarr.Group] = None + + def open(self) -> "ZarrGroupSource": + if self._root is None: + root = zarr.open_group(self.path, mode="r") + require_record_format(root.attrs.get("recordstream_format"), "ZarrGroupSource") + self._root = root + return self + + def close(self) -> None: + self._root = None + + def __iter__(self) -> Iterator[Record]: + self.open() + if self._root is None: + return + for name in sorted(self._root.group_keys()): + yield _read_record(cast(zarr.Group, self._root[name])) + + def __len__(self) -> int: + self.open() + if self._root is None: + return 0 + return len(list(self._root.group_keys())) + + def iter_metadata(self) -> "Iterator[tuple[str, dict]]": + """(group name, ``.zattrs`` metadata) per record WITHOUT loading arrays (SupportsMetadataScan).""" + from recordstream.storage.query import scan_zarr_metadata + + yield from scan_zarr_metadata(self.path) + + +# category="sink": surfaced by visual editors as a sink node docking into a DatasetProcessor's sink slot. +@confluid.configurable(category="sink") +class ZarrBatchSink(Storage, DataSink): + """ + Optimized for uniform data. Appends records into a single large Zarr array. + """ + + def __init__( + self, + path: Union[str, Path] = "", + shape: Optional[List[int]] = None, + dtype: str = "float32", + chunks: Optional[List[int]] = None, + overwrite: bool = False, + ) -> None: + # Partial / zero-arg: store config only; the array is created lazily in open() (an unset + # path / shape surfaces there). + self.path = str(path) + self.shape = tuple(shape) if shape else () + self.dtype = dtype + self.chunks = tuple(chunks) if chunks else None + self.overwrite = overwrite + self._data_arr: Optional[zarr.Array] = None + self._target_arr: Optional[zarr.Array] = None + self._counter = 0 + + def open(self) -> "ZarrBatchSink": + if self._data_arr is None: + # We create a resizable array (unlimited along first dimension) + self._data_arr = zarr.open_array( + store=f"{self.path}/data", + mode="a" if not self.overwrite else "w", + shape=(0,) + self.shape, + chunks=(1,) + self.shape if not self.chunks else self.chunks, + dtype=self.dtype, + ) + return self + + def write(self, record: Any) -> None: + self.open() + if self._data_arr is None: + raise RuntimeError("Zarr array not open") + if not isinstance(record, dict): + raise TypeError(f"ZarrBatchSink: expected a record dict, got {type(record).__name__}") + if not record: + raise ValueError("ZarrBatchSink: cannot write an empty record") + + # The batch sink stores ONE uniform array: the FIRST record entry's payload per row + # (insertion order), plus a one-time item template (type/key/attrs of the FIRST record) + # so the source can rebuild typed rows. Uniform-batch by design — per-record attr + # variation does not fit a single stacked array; use ZarrGroupSink for that. + key, value = next(iter(record.items())) + encoded = encode_item(value) + existing = self._data_arr.attrs.get("recordstream_format") + if existing is None: + if self._data_arr.shape[0] > 0: + require_record_format(None, "ZarrBatchSink") + plain, arrays = split_attrs(encoded.attrs) + if arrays: + raise TypeError( + "ZarrBatchSink: array-valued item attrs do not fit the single-array batch " + "layout — use ZarrGroupSink." + ) + self._data_arr.attrs.update( + {"recordstream_format": TYPED_FORMAT, _TYPE_ATTR: encoded.type_name, "__field__": key, **plain} + ) + elif existing != TYPED_FORMAT: + require_record_format(existing, "ZarrBatchSink") + self._data_arr.append([np.asarray(to_numpy(encoded.payload))], axis=0) + self._counter += 1 + + def flush(self) -> None: + pass # pragma: no cover + + +@confluid.configurable +class ZarrBatchSource(Storage, DataSource): + """Read records written by :class:`ZarrBatchSink` (one stacked array). + + The batch sink appends every record's first entry's payload along axis 0 of a + single ``data`` array plus a one-time uniform item template, so this source + yields single-key records — one per row of the leading axis. + + Args: + path: Path to the Zarr store written by ZarrBatchSink (the directory holding the ``data`` array). + """ + + def __init__(self, path: Union[str, Path] = "") -> None: + # Partial / zero-arg: store config only; the array is opened lazily in open(). + self.path = str(path) + self._data_arr: Optional[zarr.Array] = None + + def open(self) -> "ZarrBatchSource": + if self._data_arr is None: + arr = zarr.open_array(store=f"{self.path}/data", mode="r") + require_record_format(arr.attrs.get("recordstream_format"), "ZarrBatchSource") + self._data_arr = arr + return self + + def close(self) -> None: + self._data_arr = None + + def __iter__(self) -> Iterator[Record]: + self.open() + if self._data_arr is None: + return + attrs = dict(self._data_arr.attrs) + # Record batch rows: rebuild each row as the stored item type under the stored key + # (uniform template — see ZarrBatchSink.write). + field = str(attrs["__field__"]) + type_name = str(attrs[_TYPE_ATTR]) + item_attrs = restore_attrs( + {k: v for k, v in attrs.items() if k not in ("recordstream_format", _TYPE_ATTR, "__field__")}, {} + ) + for i in range(self._data_arr.shape[0]): + payload = np.asarray(self._data_arr[i]) + item = decode_item(EncodedItem(type_name=type_name, payload=payload, attrs=item_attrs)) + yield {field: item} + + def __len__(self) -> int: + self.open() + if self._data_arr is None: + return 0 + return int(self._data_arr.shape[0]) diff --git a/recordstream/transform.py b/recordstream/transform.py new file mode 100644 index 0000000..bf7f6d3 --- /dev/null +++ b/recordstream/transform.py @@ -0,0 +1,173 @@ +"""``Transform`` — type-dispatched record ops with once-per-record parameters, plus ``Pipeline``. + +A transform records its random / configured parameters ONCE per record (:meth:`Transform.get_params`), +then walks the dict and, for each value whose type it handles, applies the registered +kernel (:mod:`recordstream.dispatch`). Values it does not handle pass through untouched. + +Two properties fall out of this shape for free: + +* **Cross-field consistency.** Because params are sampled once and shared, one op moves + every handled value with the SAME decision (two ``Signal`` values in one record get the + same drawn SNR) — the torchvision-v2 model. +* **Open extension.** A new value type is taught to an existing op with one + ``@MyOp.kernel(NewType)`` registration and no core edit. + +Targeting is by TYPE; the ``field`` parameter pins an op to one named key when a record +holds several values of a handled type. + +recordstream ships NO native augmentation ops — geometric/photometric augmentation comes from +the libraries (torchvision ``transforms.v2`` / albumentations) dropped into an ops list +AS-IS; the engine invokes each op family natively (see ``recordstream.core.families._apply_op``). +There are no wrapper/adapter classes. +""" + +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple + +from confluid import configurable + +from recordstream.dispatch import Kernel, dispatch, register_kernel +from recordstream.items import Record, item_data, with_data + +__all__ = [ + "Transform", + "Pipeline", + "FunctionTransform", + "as_transform", +] + + +class Transform: + """Base class for type-dispatched record ops (see the module docstring). + + Subclasses declare ``handles`` (the value types they process) and register a kernel per + type via ``@MyOp.kernel(ItemType)``. Override :meth:`get_params` to record shared + parameters once per record. + + **The type-interface attributes** (``handles`` / ``consumes`` / ``optional`` / + ``produces``) describe the op to readers and machines (a visual editor's typed sockets, + a pipeline linter) WITHOUT executing it. They are DECLARATIVE: nothing validates them + against the op's behavior, and the base ``__call__`` dispatches on the KERNEL REGISTRY, + never on ``handles`` — the one exception is :class:`FunctionTransform`, whose + ``handles`` IS its ``isinstance`` application gate. Declare them truthfully or not at + all; a kernel op keeps ``handles`` mirroring its registered kernels, and a + type-CHANGING op (``__call__`` override) states its ``consumes``/``produces`` type flow + explicitly because no kernel registration reveals it. Full guidance: + ``docs/record-model.md`` → "Declaring an op's type interface". + + Args: + field: Apply only to this record key (still type-gated). None (default) = every value of a handled type. + """ + + #: Value types this op processes (everything else passes through). DECLARATIVE for + #: kernel ops (the kernel registry decides dispatch — keep this mirroring it); + #: ENFORCED only by FunctionTransform, where it is the isinstance application gate. + handles: Tuple[type, ...] = () + #: Type interface — input types the op NEEDS to do useful work. Empty = same as + #: ``handles``. Never gates execution; validate a hard requirement lazily in __call__. + consumes: Tuple[type, ...] = () + #: Type interface — input types used when present but not required (e.g. a geometric + #: op that also moves a Mask if the record has one). Never gates execution. + optional: Tuple[type, ...] = () + #: Type interface — value types this op ADDS or CHANGES (its output contract; what a + #: downstream op can rely on finding). Never gates execution. + produces: Tuple[type, ...] = () + + def __init__(self, field: Optional[str] = None) -> None: + self.field = field + + @classmethod + def kernel(cls, item_type: type) -> Callable[[Kernel], Kernel]: + """Register a kernel for ``item_type`` on this op (decorator over :func:`register_kernel`).""" + return register_kernel(cls, item_type) + + def get_params(self, record: Record) -> Dict[str, Any]: + """Record the shared parameters for one call. Default: no params.""" + return {} + + def __call__(self, record: Record) -> Optional[Record]: + params = self.get_params(record) + out = dict(record) + for key, value in record.items(): + if self.field is not None and key != self.field: + continue + kernel = dispatch(type(self), type(value)) + if kernel is None: + continue + out[key] = kernel(value, params) + return out + + +@configurable(category="op", group="compose") +class Pipeline: + """Sequential application of ops — ``Pipeline([a, b, c])(record)`` is ``c(b(a(record)))``. + + THE compose-group unit: wrap an ordered list of ops so they appear as one named block in + a config and one node on a visual canvas. Entries may be native ops, BARE library + transforms (torchvision ``transforms.v2`` / albumentations — invoked natively by the + engine's op-family dispatch), or config-deferred markers (built on first use). If any op + returns ``None`` the chain stops and propagates ``None`` (filter-drop semantics). + + Args: + transforms: Ordered ops applied in sequence (bare library transforms allowed). Defaults to ``[]`` (identity). + """ + + def __init__(self, transforms: Optional[Sequence[Any]] = None) -> None: + # Partial / zero-arg: store config only; marker flow happens on first call. + self.transforms: List[Any] = list(transforms) if transforms else [] + + def __call__(self, record: Record) -> Optional[Record]: + from confluid import flow + from confluid.fluid import Fluid + + # _apply_op = the engine's op-family dispatch, so a bare albumentations / + # torchvision-v2 transform nests here exactly as in a bare ops list. + from recordstream.core import _apply_op + + current: Optional[Record] = record + for i, op in enumerate(self.transforms): + if current is None: + return None + if isinstance(op, Fluid): + op = flow(op) + self.transforms[i] = op + if op is None: + continue + current = _apply_op(current, op) + return current + + def close(self) -> None: + """Propagate close() to inner ops that own resources (e.g. a sink op).""" + for op in self.transforms: + close_fn = getattr(op, "close", None) + if callable(close_fn): + close_fn() + + def __repr__(self) -> str: + return f"Pipeline([{', '.join(type(t).__name__ for t in self.transforms)}])" + + +class FunctionTransform(Transform): + """An op that applies one plain function ``fn(data) -> data`` to every handled value. + + The escape hatch for custom ops: no kernel registration, no subclass — wrap a + function and say which value types it applies to (via :func:`as_transform`). + """ + + def __init__(self, fn: Callable[[Any], Any], handles: Sequence[type], field: Optional[str] = None) -> None: + super().__init__(field=field) + self._fn = fn + self.handles = tuple(handles) + + def __call__(self, record: Record) -> Optional[Record]: + out = dict(record) + for key, value in record.items(): + if self.field is not None and key != self.field: + continue + if isinstance(value, self.handles): + out[key] = with_data(value, self._fn(item_data(value))) + return out + + +def as_transform(fn: Callable[[Any], Any], handles: Sequence[type], field: Optional[str] = None) -> FunctionTransform: + """Wrap a plain ``fn(data) -> data`` as a :class:`FunctionTransform` over ``handles``.""" + return FunctionTransform(fn, handles, field=field) diff --git a/recordstream/uri.py b/recordstream/uri.py new file mode 100644 index 0000000..400db9e --- /dev/null +++ b/recordstream/uri.py @@ -0,0 +1,231 @@ +"""Dataset identity — the URI/URL a source carries for the data it reads. + +A run record that says *which* data produced it needs a stable, comparable handle on +that data. A source already knows one: a Hub repo id, a directory on disk, a query +against a store. This module is the opt-in protocol for exposing it plus the walk +helpers a consumer uses, with the same shape as :mod:`recordstream.projection` — +a ``Protocol`` (never a base class), free functions that materialize a deferred +source first, and ``None`` for "this source has no such handle", which is not an error. + +Two handles, not one, because they answer different questions: + +* **``dataset_uri``** — the CANONICAL identifier: machine-parseable, stable across + machines, and the thing two runs are compared on. ``hf://datasets/ylecun/mnist?split=train``. +* **``dataset_url``** — a link a HUMAN can open, or ``None`` when the data has no web + page (a local directory has none, and inventing one would be a lie). + +Design notes +------------ +* **A wrapper propagates the URI VERBATIM.** The handle identifies the *dataset*; how + much of it a run consumed (a split, a slice, a filter) is a different fact, carried + by the wrapper's own configuration. Decorating the string would mean the same dataset + reached through two wrappers no longer compares equal, which is the one property the + handle exists to have. +* **Following happens HERE, not in every wrapper.** :func:`dataset_uri` follows a + ``.source`` attribute when the object holds no handle of its own, so every view + source in this package — and any third-party wrapper using the same attribute name — + works with no code of its own. +* **A concatenation declines.** Several datasets end to end are not one dataset, so the + singular functions answer ``None`` for a source holding ``.sources``; use + :func:`dataset_uris`, which fans out over the members. +""" + +from typing import Any, List, Optional, Protocol, runtime_checkable + +#: How many ``.source`` hops :func:`dataset_uri` will follow before giving up. Wrapping +#: is shallow in practice (a split over a range over a source is already unusual); the +#: cap is what keeps a cyclic ``source`` reference from hanging a tracking call. +MAX_WRAPPER_DEPTH = 16 + + +@runtime_checkable +class SupportsDatasetIdentity(Protocol): + """A source that can name the dataset it reads. + + Both members are properties and both may answer ``None`` — a source configured + with no dataset yet (the zero-arg construction convention) has no identity, and a + dataset with no web page has no URL. Implementations MUST be cheap and side-effect + free: this is asked of a source that may never be iterated, so it reads stored + configuration and never loads, downloads or opens anything. + """ + + @property + def dataset_uri(self) -> Optional[str]: + """Canonical, machine-parseable identifier for the data this source reads.""" + ... + + @property + def dataset_url(self) -> Optional[str]: + """Browsable link to the data, or ``None`` when it has no web page.""" + ... + + +def _identity(source: Any, attribute: str) -> Optional[str]: + """Read ``attribute`` off ``source``, following ``.source`` wrappers. + + The shared body of :func:`dataset_uri` and :func:`dataset_url`: materialize a + deferred source, read the handle if it has one, otherwise take one hop into the + wrapped source and ask again. + """ + current = _materialize(source) + for _ in range(MAX_WRAPPER_DEPTH): + if current is None: + return None + value = getattr(current, attribute, None) + if value: + return str(value) + wrapped = getattr(current, "source", None) + if wrapped is None or wrapped is current: + return None + current = _materialize(wrapped) + return None + + +def _materialize(node: Any) -> Any: + """Build ``node`` only if it is a DEFERRED config marker; hand a live object back untouched. + + Narrower than a bare ``flow()``, and the difference is the "asking never loads" property. + ``flow()`` on a LIVE object still runs confluid's post-construction ``solidify()`` hook, so a + source that grows one would be materialized merely by being asked its name. A marker has + nothing to read until it is built, and building one is cheap by the lazy-construction rule. + + A deferred marker is built only when its target CLASS could possibly answer — it declares + the identity protocol (a ``dataset_uri`` / ``dataset_url`` property) or a ``source`` slot + the walk would follow. Anything else answers ``None`` WITHOUT being constructed: a run's + provenance capture walks every configured slot, and it used to build a full ``pl.Trainer`` + (and attempt a dataset-less ``DataLoader``) per walk just to learn each names no dataset. + An unresolvable target keeps the old build path, so nothing that answered before answers + differently now. + """ + from confluid import Fluid, flow + + if not isinstance(node, Fluid): + return node + target = _target_class(node) + if target is not None and not _has_identity_surface(target): + return None + return flow(node) + + +def _target_class(marker: Any) -> Optional[type]: + """The marker's target as a CLASS, resolved without building — ``None`` when it cannot be. + + A builder FUNCTION target also answers ``None`` (its return type is unknowable statically), + so the caller stays conservative and builds, exactly as before the guard existed. + """ + target = getattr(marker, "target", None) + if isinstance(target, type): + return target + if isinstance(target, str): + from confluid.registry import resolve_class + + resolved = resolve_class(target) + if isinstance(resolved, type): + return resolved + module, _, name = target.rpartition(".") + if module: + try: + import importlib + + candidate = getattr(importlib.import_module(module), name, None) + except Exception: + return None + if isinstance(candidate, type): + return candidate + return None + + +def _has_identity_surface(cls: type) -> bool: + """Whether ``cls`` declares anything the identity walk could read. + + ``hasattr`` catches the protocol properties and a class-level ``source``; + ``declares_key`` catches a ``source`` constructor parameter / body slot. + """ + if hasattr(cls, "dataset_uri") or hasattr(cls, "dataset_url") or hasattr(cls, "source"): + return True + from confluid import declares_key + + return declares_key(cls, "source") + + +def dataset_uri(source: Any) -> Optional[str]: + """The canonical URI identifying the dataset ``source`` reads, or ``None``. + + A DEFERRED source (a ``!class:`` marker straight out of a config) is materialized + first, exactly as :func:`~recordstream.projection.project` does, so a caller never + has to remember which entry point flows and which does not. + + A wrapper that holds no URI of its own but wraps another source via ``.source`` — + every view source in this package does — reports the wrapped source's URI + unchanged. A wrapper holding SEVERAL sources answers ``None``; ask + :func:`dataset_uris` instead. + + Args: + source: Any source, view, stream, or deferred config marker. ``None`` is accepted + and answers ``None``, so a caller needs no guard for an unwired slot. + + Example:: + + dataset_uri(HuggingFaceSource(path="ylecun/mnist")) # hf://datasets/ylecun/mnist?split=train + dataset_uri(Stream(source=split.train)) # the same string — the split is not part of it + """ + return _identity(source, "dataset_uri") + + +def dataset_url(source: Any) -> Optional[str]: + """The browsable URL for the dataset ``source`` reads, or ``None``. + + The human-facing twin of :func:`dataset_uri`, with identical flowing and wrapper + rules. ``None`` is the ordinary answer for data with no web page — a directory on + disk, a store on a mounted volume — and never an error. + + Args: + source: Any source, view, stream, or deferred config marker. + """ + return _identity(source, "dataset_url") + + +def dataset_uris(source: Any) -> List[str]: + """Every dataset URI reachable from ``source``, in order, deduplicated. + + The plural form, for a source that concatenates several datasets: it fans out over + a ``.sources`` list (recursively, so a concatenation of concatenations works) while + a single source yields its one URI. Sources with no URI contribute nothing rather + than a ``None`` entry, so the result is directly usable. + + Args: + source: Any source, view, stream, or deferred config marker. + + Example:: + + dataset_uris(ConcatSource(sources=[a, b])) # ['hf://datasets/…', 'file:///…'] + """ + found: List[str] = [] + + def visit(node: Any, depth: int) -> None: + if node is None or depth > MAX_WRAPPER_DEPTH: + return + node = _materialize(node) + uri = dataset_uri(node) + if uri: + if uri not in found: + found.append(uri) + return + # No single identity: a concatenation is the case worth descending into. `.source` + # is already covered by `dataset_uri` above, so only the plural attribute is left. + members = getattr(node, "sources", None) + if isinstance(members, (list, tuple)): + for member in members: + visit(member, depth + 1) + + visit(source, 0) + return found + + +__all__ = [ + "MAX_WRAPPER_DEPTH", + "SupportsDatasetIdentity", + "dataset_uri", + "dataset_uris", + "dataset_url", +] diff --git a/recordstream/workflow.py b/recordstream/workflow.py new file mode 100644 index 0000000..9c944df --- /dev/null +++ b/recordstream/workflow.py @@ -0,0 +1,272 @@ +"""Higher-order runnables — compose other runnables into a workflow. + +A *runnable* is any object exposing a no-arg ``run(self)`` (a trainer, an +evaluator, a :class:`~recordstream.processing.DatasetProcessor`). This module adds +Confluid-``@configurable`` *combinators* that HOLD other runnables and orchestrate +them — the runnable-level analogue of the composing ops (``Pipeline`` / +``Parallel`` / ``Enable``): + +* :class:`Sequence` — run a list of runnables in order (the workflow itself). +* :class:`Conditional` — run one of two runnables depending on a condition. +* :class:`Switch` — run one of several runnables keyed by a select value. + +Conditions are themselves Confluid-``@configurable`` *predicates* — a no-arg +``__call__(self) -> bool`` (:class:`PathExists` / :class:`Not` / :class:`AllOf` +/ :class:`AnyOf`) — so a whole workflow (steps, branches, AND the conditions +that pick them) serialises to ONE Confluid YAML document, runs via the generic +``recordstream run workflow.yaml`` runner, and — being plain ``@configurable`` +classes — is surfaced by discovery / a visual editor with no bespoke glue. + +Branches are held as INSTANCES and only ``run()`` when selected. Example:: + + !class:recordstream.workflow.Sequence + steps: + - !lazy:DownloadData + - !class:recordstream.workflow.Conditional + condition: !class:recordstream.workflow.PathExists { path: $MODEL_ROOT/model.ckpt } + if_false: !lazy:TrainModel # cache miss -> train + if_true: null # cache hit -> skip, fall through + - !lazy:Evaluate + +The guarantee: **only the selected branch's ``run()`` is ever called** — an +unchosen branch is never run (and, wired ``!lazy:``, never even built, so no +model / dataset is materialised). This is the *memoise-and-continue* answer to +"don't recompute, move on": the next ``steps:`` entry runs regardless, because +``Sequence`` drives them in order — no execution-blocking, no dead branches. +Runnable proof (the resume-safe train→evaluate pipeline, run twice, both +guarantees asserted): ``examples/workflow_pipeline.py``; usage: ``docs/workflow.md``. + +All combinators are zero-arg constructible and do NO functional work in +``__init__`` (the workspace lazy-construction convention); branches and +conditions are flowed lazily inside ``run()`` (a Confluid ``!class:`` / ``!lazy:`` +member arrives as a deferred ``Fluid`` stub and is materialised on demand). + +The combinators inherit :class:`~recordstream.runnable.TorchRunner` and +:class:`~recordstream.runnable.ProgressReporting` so a workflow runs correctly on a +GUI canvas: a combinator may wrap a *trainer*, so it declares ``__needs_autograd__`` +(the executor re-enables autograd for the whole run — otherwise an inner +``loss.backward()`` dies under the executor's inference mode; restoring autograd is +harmless for an inner evaluator), and it FORWARDS the executor-injected progress +callback to whichever branch is running (:func:`_run`). +""" + +from pathlib import Path +from typing import Any, Dict, List, Optional + +from confluid import configurable, flow +from confluid.fluid import Fluid +from loggair import get_logger + +from recordstream.runnable import ProgressReporting, TorchRunner + +logger = get_logger(__name__) + + +def _resolve(value: Any) -> Any: + """Flow a possibly-deferred Confluid value to a live object (idempotent on live ones).""" + return flow(value) if isinstance(value, Fluid) else value + + +def _evaluate_condition(condition: Any) -> bool: + """Evaluate a workflow condition to a ``bool``. + + Accepts a ``@configurable`` predicate (no-arg ``__call__ -> bool``), any + zero-arg callable, a plain ``bool``, or a deferred ``Fluid`` resolving to one + of those. ``None`` is falsy. + """ + condition = _resolve(condition) + if condition is None: + return False + if callable(condition): + return bool(condition()) + return bool(condition) + + +def _run(runnable: Any, progress_callback: Any = None) -> None: + """Flow ``runnable`` (if deferred) and call its ``run()``; ``None`` is a no-op. + + When ``progress_callback`` is supplied (the combinator's OWN callback, injected by a GUI + executor via :meth:`ProgressReporting.set_progress_callback`) it is FORWARDED to the branch + first, so a canvas progress bar tracks whichever runnable is executing inside the workflow + (the combinator itself has no loop to report from). + """ + runnable = _resolve(runnable) + if runnable is None: + return + run = getattr(runnable, "run", None) + if not callable(run): + raise TypeError(f"workflow branch {type(runnable).__name__!r} has no callable run() method") + if progress_callback is not None: + setter = getattr(runnable, "set_progress_callback", None) + if callable(setter): + setter(progress_callback) + run() + + +@configurable +class Sequence(TorchRunner, ProgressReporting): + """Run a list of runnables in order — the workflow itself. + + Args: + steps: Runnables (or deferred ``!lazy:`` / ``!class:`` markers) to run in + order. A ``None`` entry is skipped. An empty list is a no-op. + """ + + def __init__(self, steps: Optional[List[Any]] = None) -> None: + # Partial / zero-arg: store config only; branches are flowed in run(). + self.steps: List[Any] = list(steps) if steps else [] + + def run(self) -> None: + total = len(self.steps) + for i, step in enumerate(self.steps): + resolved = _resolve(step) + self.steps[i] = resolved # cache the flowed step so we only flow once + if resolved is None: + continue + logger.info(f"Sequence step {i + 1}/{total}: {type(resolved).__name__}") + _run(resolved, self._progress_callback) + + +@configurable +class Conditional(TorchRunner, ProgressReporting): + """Run one of two runnables depending on a condition. + + The runnable-level ``if``/``else``: a runnable that, on a condition, triggers + another runnable held as an instance. + + Args: + condition: A predicate (no-arg ``__call__ -> bool``), zero-arg callable, + ``bool``, or deferred ``Fluid`` resolving to one of those. ``None`` is + falsy. + if_true: Runnable to run when the condition holds. ``None`` = do nothing. + if_false: Runnable to run otherwise. ``None`` = do nothing (skip and let + an enclosing :class:`Sequence` continue to the next step). + """ + + def __init__(self, condition: Any = None, if_true: Any = None, if_false: Any = None) -> None: + self.condition = condition + self.if_true = if_true + self.if_false = if_false + + def run(self) -> None: + chosen = self.if_true if _evaluate_condition(self.condition) else self.if_false + branch = _resolve(chosen) + if branch is None: + logger.debug("Conditional: selected branch is None — nothing to run.") + return + logger.info(f"Conditional -> {type(branch).__name__}") + _run(branch, self._progress_callback) + + +@configurable +class Switch(TorchRunner, ProgressReporting): + """Run one of several runnables keyed by a select's value. + + Args: + select: A no-arg callable / predicate / deferred value producing the + case KEY (coerced to ``str``). ``None`` (or a ``None`` result) selects + ``default``. + cases: Mapping of key -> runnable. The runnable whose key matches the + select runs; an unmatched key falls back to ``default``. + default: Runnable to run when no case matches. ``None`` = no-op. + """ + + def __init__( + self, + select: Any = None, + cases: Optional[Dict[str, Any]] = None, + default: Any = None, + ) -> None: + self.select = select + self.cases: Dict[str, Any] = dict(cases) if cases else {} + self.default = default + + def run(self) -> None: + key = self._select() + chosen = self.cases.get(key, self.default) if key is not None else self.default + branch = _resolve(chosen) + if branch is None: + logger.debug(f"Switch: no branch for key {key!r} and no default — nothing to run.") + return + logger.info(f"Switch[{key!r}] -> {type(branch).__name__}") + _run(branch, self._progress_callback) + + def _select(self) -> Optional[str]: + selector = _resolve(self.select) + if selector is None: + return None + value = selector() if callable(selector) else selector + return None if value is None else str(value) + + +@configurable +class PathExists: + """Predicate: ``True`` iff ``path`` exists on disk. + + The canonical cache check — pair with :class:`Conditional` to skip a step + whose output artifact (a checkpoint, a converted dataset) is already present. + + Args: + path: Filesystem path to test. Empty / ``None`` -> ``False``. + """ + + def __init__(self, path: str = "") -> None: + self.path = path + + def __call__(self) -> bool: + return bool(self.path) and Path(self.path).exists() + + +@configurable +class Not: + """Predicate: the negation of another condition. + + Args: + condition: The condition to negate (predicate / callable / ``bool`` / + ``Fluid``). ``None`` is falsy, so ``Not(None)`` is ``True``. + """ + + def __init__(self, condition: Any = None) -> None: + self.condition = condition + + def __call__(self) -> bool: + return not _evaluate_condition(self.condition) + + +@configurable +class AllOf: + """Predicate: ``True`` iff EVERY sub-condition is truthy (logical AND). + + An empty list is ``True`` (vacuous truth). + + Args: + conditions: Conditions to AND together (each a predicate / callable / + ``bool`` / ``Fluid``). + """ + + def __init__(self, conditions: Optional[List[Any]] = None) -> None: + self.conditions: List[Any] = list(conditions) if conditions else [] + + def __call__(self) -> bool: + return all(_evaluate_condition(c) for c in self.conditions) + + +@configurable +class AnyOf: + """Predicate: ``True`` iff at least ONE sub-condition is truthy (logical OR). + + An empty list is ``False``. + + Args: + conditions: Conditions to OR together (each a predicate / callable / + ``bool`` / ``Fluid``). + """ + + def __init__(self, conditions: Optional[List[Any]] = None) -> None: + self.conditions: List[Any] = list(conditions) if conditions else [] + + def __call__(self) -> bool: + return any(_evaluate_condition(c) for c in self.conditions) + + +__all__ = ["Sequence", "Conditional", "Switch", "PathExists", "Not", "AllOf", "AnyOf"] diff --git a/tests/_fixtures.py b/tests/_fixtures.py new file mode 100644 index 0000000..eac19a0 --- /dev/null +++ b/tests/_fixtures.py @@ -0,0 +1,75 @@ +"""Test-local record-model fixtures. + +``FixtureFlip`` is the former native ``HorizontalFlip`` kept ONLY as a test fixture: recordstream +ships no native augmentation transforms (geometric/photometric augmentation comes from +torchvision v2 / albumentations invoked natively by the engine's op-family dispatch), but the +kernel-dispatch machinery (once-per-record params, per-type kernels, MRO resolution, the +``field=`` pin) still needs a fully native transform to pin. +""" + +from typing import Any, Dict, Optional + +import numpy as np + +from recordstream import Boxes, Image, Mask, Record, Transform, item_data, with_data + + +class FixtureFlip(Transform): + """Horizontal flip with ONE shared decision across Image + Mask + Boxes (test fixture).""" + + handles = (Image, Mask, Boxes) + consumes = (Image,) + optional = (Mask, Boxes) + produces = (Image, Mask, Boxes) + + def __init__(self, p: float = 0.5, field: Optional[str] = None) -> None: + super().__init__(field=field) + self.p = p + + def get_params(self, record: Record) -> Dict[str, Any]: + do = float(np.random.random()) < self.p + return {"do": do, "width": _reference_width(record)} + + +@FixtureFlip.kernel(Image) +def _flip_image(item: Image, params: Dict[str, Any]) -> Image: + if not params["do"]: + return item + axis = 2 if getattr(item, "layout", "HWC") == "CHW" else 1 + return with_data(item, np.flip(item_data(item), axis=axis).copy()) + + +@FixtureFlip.kernel(Mask) +def _flip_mask(item: Mask, params: Dict[str, Any]) -> Mask: + if not params["do"]: + return item + return with_data(item, np.flip(item_data(item), axis=1).copy()) + + +@FixtureFlip.kernel(Boxes) +def _flip_regions(item: Boxes, params: Dict[str, Any]) -> Boxes: + if not params["do"]: + return item + width = params.get("width") or (item.canvas[1] if item.canvas else None) + if width is None: + raise ValueError("FixtureFlip: no reference width to flip Boxes") + boxes = [[width - box[2], box[1], width - box[0], box[3]] for box in item.boxes] + return Boxes(boxes=boxes, labels=item.labels, scores=item.scores, canvas=item.canvas) + + +def _reference_width(record: Record) -> Optional[int]: + """The horizontal extent to flip boxes against — from the first Image/Mask, or a Boxes canvas.""" + for _, item in record.items(): + if isinstance(item, Image): + arr = item_data(item) + axis = 2 if getattr(item, "layout", "HWC") == "CHW" else 1 + if arr.ndim > axis: + return int(arr.shape[axis]) + if isinstance(item, Mask): + arr = item_data(item) + if arr.ndim >= 2: + return int(arr.shape[1]) + for _, item in record.items(): + if isinstance(item, Boxes) and item.canvas: + return int(item.canvas[1]) + return None diff --git a/tests/test_batch.py b/tests/test_batch.py new file mode 100644 index 0000000..92186ca --- /dev/null +++ b/tests/test_batch.py @@ -0,0 +1,400 @@ +"""Reading a batched record back (``recordstream.batch``) — the inverse of ``collate_records``. + +These pin the CONVENTION, not a task: the helpers answer "what did the collate put under this +key?" and hand back values / one tensor / per-record dicts. Shaping for a particular loss is +the caller's job, so there is deliberately no dtype promotion or multi-hot test here. +""" + +from typing import Any + +import numpy as np +import pytest +import torch + +from recordstream import ( + Image, + Label, + Mask, + MultiLabel, + Record, + batch_metadata, + batch_tensor, + batch_values, + collate_records, + multi_hot, +) + +# --------------------------------------------------------------------------- # +# batch_values — getting past the wrapper item +# --------------------------------------------------------------------------- # + + +def _chw(size: int = 4) -> np.ndarray: + return np.random.rand(3, size, size).astype("float32") + + +def test_label_values_come_back_as_the_per_record_list() -> None: + """A wrapper item's payload is NOT stacked by the collate — it stays a list.""" + batch = collate_records([{"class": Label(i % 2)} for i in range(4)]) + assert batch_values(batch, "class") == [0, 1, 0, 1] + + +def test_multilabel_values_come_back_as_a_list_of_lists() -> None: + batch = collate_records([{"class": MultiLabel([0, 2])}, {"class": MultiLabel([1])}]) + assert batch_values(batch, "class") == [[0, 2], [1]] + + +def test_array_item_values_come_back_as_the_stacked_payload() -> None: + batch = collate_records([{"image": Image(_chw(), layout="CHW")} for _ in range(3)]) + values = batch_values(batch, "image") + assert getattr(values, "shape", None) == (3, 3, 4, 4) + + +def test_plain_values_come_back_as_the_gathered_list() -> None: + batch = collate_records([{"idx": i} for i in range(3)]) + assert list(batch_values(batch, "idx")) == [0, 1, 2] + + +# --------------------------------------------------------------------------- # +# batch_tensor — one tensor, whichever shape the collate left +# --------------------------------------------------------------------------- # + + +def test_stacked_array_payload_becomes_a_tensor() -> None: + batch = collate_records([{"image": Image(_chw(8), layout="CHW")} for _ in range(4)]) + x = batch_tensor(batch, "image") + assert isinstance(x, torch.Tensor) and x.shape == (4, 3, 8, 8) + + +def test_per_record_list_is_stacked() -> None: + """The Label path: a list of scalars becomes an [N] tensor.""" + batch = collate_records([{"class": Label(i % 3)} for i in range(6)]) + y = batch_tensor(batch, "class") + assert y.shape == (6,) and y.tolist() == [0, 1, 2, 0, 1, 2] + + +def test_a_prestacked_tensor_is_used_verbatim() -> None: + """A hand-built batch (tests, a custom collate) must not be re-stacked.""" + pinned = torch.arange(4) + assert batch_tensor({"class": Label(value=pinned)}, "class") is pinned + + +def test_mask_item_stacks_without_a_dtype_opinion() -> None: + """No int64 promotion here — that is the segmenter's model boundary, not the convention.""" + batch = collate_records([{"target": Mask(np.zeros((4, 4), dtype="uint8"))} for _ in range(2)]) + assert batch_tensor(batch, "target").dtype is torch.uint8 + + +def test_device_moves_the_result() -> None: + batch = collate_records([{"class": Label(i)} for i in range(2)]) + assert batch_tensor(batch, "class", device="cpu").device.type == "cpu" + + +def test_dtype_is_honoured() -> None: + """`dtype` is a PARAMETER, not an opinion — the caller names its loss's contract.""" + batch = collate_records([{"class": Label(torch.tensor(i, dtype=torch.int32))} for i in range(3)]) + + assert batch_tensor(batch, "class").dtype is torch.int32 # unasked: as-is + assert batch_tensor(batch, "class", dtype=torch.int64).dtype is torch.int64 + + +def test_dtype_matters_because_the_loss_rejects_the_wrong_one() -> None: + """Why the knob exists: int32 class ids are legal and CrossEntropyLoss refuses them.""" + import torch.nn as nn + + batch = collate_records([{"class": Label(torch.tensor(i, dtype=torch.int32))} for i in range(3)]) + logits = torch.randn(3, 4) + + # Matched loosely on purpose: torch reworded this in 2.13 ("expected scalar type Long but + # found Int" -> "expected target dtype to be Long or Byte, but got Int"). The REJECTION is + # the contract this test pins; the exact phrasing is torch's to change. + with pytest.raises(RuntimeError, match="(scalar type|target dtype).*Long"): + nn.CrossEntropyLoss()(logits, batch_tensor(batch, "class")) + + nn.CrossEntropyLoss()(logits, batch_tensor(batch, "class", dtype=torch.int64)) # no raise + + +# --------------------------------------------------------------------------- # +# multi_hot — MultiLabel rendered as a matrix, framework-free +# --------------------------------------------------------------------------- # + + +def test_multi_hot_marks_every_label_of_every_record() -> None: + batch = collate_records([{"class": MultiLabel([0, 2])}, {"class": MultiLabel([1])}]) + + assert multi_hot(batch, "class", 3).tolist() == [[1.0, 0.0, 1.0], [0.0, 1.0, 0.0]] + + +def test_multi_hot_returns_numpy_so_any_framework_can_use_it() -> None: + """A torch return type would have forced a second implementation for the next backend.""" + batch = collate_records([{"class": MultiLabel([0])}]) + result = multi_hot(batch, "class", 2) + + assert isinstance(result, np.ndarray) + assert result.dtype == np.float32 # what the multi-label losses consume + assert torch.as_tensor(result).dtype is torch.float32 # one line into torch + + +def test_multi_hot_dtype_is_selectable() -> None: + assert multi_hot(collate_records([{"class": MultiLabel([0])}]), "class", 2, dtype="int64").dtype == np.int64 + + +def test_an_empty_label_set_is_an_all_zero_row_not_an_error() -> None: + """ "This record has no classes" is a meaningful multi-label target.""" + batch = collate_records([{"class": MultiLabel([1])}, {"class": MultiLabel([])}]) + + assert multi_hot(batch, "class", 2).tolist() == [[0.0, 1.0], [0.0, 0.0]] + + +def test_out_of_range_ids_are_ignored_rather_than_raising() -> None: + """A stray label must not abort a training run.""" + batch = collate_records([{"class": MultiLabel([0, 99, -1])}]) + + assert multi_hot(batch, "class", 2).tolist() == [[1.0, 0.0]] + + +def test_multi_hot_width_is_num_classes_not_the_observed_max() -> None: + """The head's width decides the matrix, never the batch's contents.""" + batch = collate_records([{"class": MultiLabel([0])}]) + + assert multi_hot(batch, "class", 5).shape == (1, 5) + + +# --------------------------------------------------------------------------- # +# batch_metadata — the collate's transpose +# --------------------------------------------------------------------------- # + + +def test_columns_transpose_into_per_record_dicts() -> None: + records: list = [ + {"image": Image(_chw(), layout="CHW"), "class": Label(0), "idx": i, "src": f"f{i}"} for i in range(3) + ] + metas = batch_metadata(collate_records(records), exclude=("image", "class")) + assert metas == [{"idx": 0, "src": "f0"}, {"idx": 1, "src": "f1"}, {"idx": 2, "src": "f2"}] + + +def test_excluded_keys_are_omitted() -> None: + batch = collate_records([{"image": Image(_chw(), layout="CHW"), "idx": i} for i in range(2)]) + assert batch_metadata(batch, exclude=("image",)) == [{"idx": 0}, {"idx": 1}] + assert "image" not in (batch_metadata(batch, exclude=("image",)) or [{}])[0] + + +def test_no_remaining_keys_returns_none() -> None: + """`None` is the "nothing to correlate" answer a sink reads as skip.""" + batch = collate_records([{"image": Image(_chw(), layout="CHW")} for _ in range(2)]) + assert batch_metadata(batch, exclude=("image",)) is None + + +def test_a_hand_built_metadata_key_is_returned_verbatim() -> None: + batch: Record = {"image": torch.zeros(2, 3, 4, 4), "metadata": [{"idx": 0}, {"idx": 1}]} + assert batch_metadata(batch, exclude=("image",)) == [{"idx": 0}, {"idx": 1}] + + +def test_a_ragged_batch_truncates_rather_than_raising() -> None: + """A metadata detail must never take down an inference run.""" + batch: Record = {"a": [1, 2, 3], "b": [1, 2]} + assert batch_metadata(batch) == [{"a": 1, "b": 1}, {"a": 2, "b": 2}] + + +def test_a_label_column_contributes_its_values() -> None: + """Metadata carried as a Label unwraps like any other key.""" + batch = collate_records([{"image": Image(_chw(), layout="CHW"), "split": Label("train")} for _ in range(2)]) + assert batch_metadata(batch, exclude=("image",)) == [{"split": "train"}, {"split": "train"}] + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) + + +# --------------------------------------------------------------------------- # +# batch_boxes — the collate's transpose for a region-set column +# --------------------------------------------------------------------------- # +class TestBatchBoxes: + """A collated `Boxes` back into the per-record dicts every detection interface takes.""" + + def _batch(self, counts: tuple = (1, 3), scores: bool = False) -> Any: + import torch + + from recordstream import Boxes, collate_records + + records = [ + { + "target": Boxes( + boxes=torch.rand(n, 4), + labels=torch.zeros(n, dtype=torch.int64), + scores=torch.ones(n) if scores else None, + ) + } + for n in counts + ] + return collate_records(records) + + def test_it_transposes_a_variable_n_column_into_per_record_dicts(self) -> None: + from recordstream import batch_boxes + + targets = batch_boxes(self._batch(counts=(1, 3)), "target") + assert len(targets) == 2 + assert [tuple(t["boxes"].shape) for t in targets] == [(1, 4), (3, 4)] + assert [tuple(t["labels"].shape) for t in targets] == [(1,), (3,)] + + def test_an_absent_field_is_OMITTED_not_handed_over_as_none(self) -> None: + """A training target is exactly {boxes, labels} — a `None` scores key would reach a model.""" + from recordstream import batch_boxes + + assert set(batch_boxes(self._batch(), "target")[0]) == {"boxes", "labels"} + assert set(batch_boxes(self._batch(scores=True), "target")[0]) == {"boxes", "labels", "scores"} + + def test_values_keep_their_framework(self) -> None: + """Framework-free by rule: the caller owns dtype and device, as with `batch_values`.""" + import torch + + from recordstream import batch_boxes + + assert isinstance(batch_boxes(self._batch(), "target")[0]["boxes"], torch.Tensor) + + def test_numpy_boxes_stay_numpy(self) -> None: + import numpy as np + + from recordstream import Boxes, batch_boxes, collate_records + + batch = collate_records([{"target": Boxes(boxes=np.zeros((2, 4)), labels=np.zeros(2))}]) + assert isinstance(batch_boxes(batch, "target")[0]["boxes"], np.ndarray) + + def test_a_wrong_type_raises_naming_it(self) -> None: + import pytest + + from recordstream import Label, batch_boxes, collate_records + + with pytest.raises(TypeError, match="not a Boxes"): + batch_boxes(collate_records([{"target": Label(0)}]), "target") + + def test_an_uncollated_regions_raises_naming_the_mistake(self) -> None: + import numpy as np + import pytest + + from recordstream import Boxes, batch_boxes + + with pytest.raises(ValueError, match="not a COLLATED Boxes"): + batch_boxes({"target": Boxes(boxes=np.zeros((2, 4)))}, "target") + + +# --------------------------------------------------------------------------- # +# The collate is a CHOICE — and both choices read back the same +# --------------------------------------------------------------------------- # +class TestCollateIsAChoice: + """`"record"` stacks, `"list"` does not — and every read-back helper accepts both. + + That last property is what makes the choice free rather than a fork in every consumer: a + trainer picks the batch shape its MODEL needs and reads the batch the same way either way. + """ + + def _records(self, sizes: tuple = (8, 8)) -> Any: + import torch + + from recordstream import Boxes, Image, Label + + return [ + { + "image": Image(np.zeros((3, s, s), dtype="float32"), layout="CHW"), + "target": Boxes(boxes=torch.rand(n, 4), labels=torch.zeros(n, dtype=torch.int64)), + "class": Label(i), + } + for i, (s, n) in enumerate(zip(sizes, (1, 3))) + ] + + def test_both_keys_are_registered(self) -> None: + from recordstream import registered_collates + + assert {"record", "list"} <= set(registered_collates()) + + def test_record_stacks_and_list_does_not(self) -> None: + from recordstream import Image, collate_list, collate_records + + stacked = collate_records(self._records())["image"] + listed = collate_list(self._records())["image"] + assert isinstance(stacked, Image) and stacked.shape == (2, 3, 8, 8) + assert isinstance(listed, list) and [v.shape for v in listed] == [(3, 8, 8), (3, 8, 8)] + + def test_the_list_collate_keeps_items_as_items(self) -> None: + """Per-record metadata survives — a list of bare arrays would drop every `layout`.""" + from recordstream import Image, collate_list + + column = collate_list(self._records())["image"] + assert all(isinstance(v, Image) for v in column) + assert [v.layout for v in column] == ["CHW", "CHW"] + + def test_a_variable_size_column_is_a_CHOICE_not_a_crash(self) -> None: + """The whole point: ragged is fine when you asked for lists, and the default explains + itself instead of raising numpy's shape error from three frames down.""" + from recordstream import collate_list, collate_records + + ragged = self._records(sizes=(8, 12)) + with pytest.raises(ValueError, match=r"cannot stack the 'image' column"): + collate_records(ragged) + assert len(collate_list(ragged)["image"]) == 2 + + def test_the_stack_error_names_the_shapes_and_the_way_out(self) -> None: + from recordstream import collate_records + + with pytest.raises(ValueError) as excinfo: + collate_records(self._records(sizes=(8, 12))) + message = str(excinfo.value) + assert "(3, 8, 8)" in message and "(3, 12, 12)" in message + assert '"list"' in message, "the message must name the collate that CAN batch this" + + @pytest.mark.parametrize("collate_key", ["record", "list"]) + def test_every_read_back_helper_accepts_both_collates(self, collate_key: str) -> None: + """`batch_values` / `batch_boxes` / `batch_metadata` give the SAME answer either way.""" + from recordstream import batch_boxes, batch_metadata, batch_values, collate + + batch = collate(self._records(), key=collate_key) + assert batch_values(batch, "class") == [0, 1] + targets = batch_boxes(batch, "target") + assert [t["boxes"].shape[0] for t in targets] == [1, 3] + assert batch_metadata(batch, exclude=("image", "target")) == [{"class": 0}, {"class": 1}] + + +# --------------------------------------------------------------------------- # +# per_record_predictions — the prediction-side transpose +# --------------------------------------------------------------------------- # + + +class TestPerRecordPredictions: + """Batched model output -> one entry per record, for the per-record sink contract. + + Ported from the consumer suites when the three byte-identical private copies were + extracted (2026-08-06) — these are the cases those copies exercised. + """ + + def test_a_list_is_already_per_record(self) -> None: + from recordstream import per_record_predictions + + items = [{"boxes": [1]}, {"boxes": [2]}] + assert per_record_predictions(items) is items + + def test_a_predictions_wrapper_is_unwrapped(self) -> None: + from recordstream import per_record_predictions + + assert per_record_predictions({"predictions": [1, 2, 3]}) == [1, 2, 3] + + def test_a_batched_mapping_is_sliced_row_wise(self) -> None: + """A ClassificationOutput-shaped batch becomes N per-record mappings.""" + from recordstream import per_record_predictions + + batch = {"probs": np.eye(3, dtype="float32"), "class_idx": np.array([0, 1, 2])} + rows = per_record_predictions(batch) + assert len(rows) == 3 + assert rows[1]["class_idx"] == 1 + assert np.allclose(rows[2]["probs"], [0.0, 0.0, 1.0]) + + def test_a_mapping_with_disagreeing_lengths_is_one_prediction(self) -> None: + """Row-slicing is only safe when every column agrees on N.""" + from recordstream import per_record_predictions + + batch = {"a": np.zeros(2), "b": np.zeros(3)} + assert per_record_predictions(batch) == [batch] + + def test_anything_else_is_one_prediction(self) -> None: + from recordstream import per_record_predictions + + assert per_record_predictions(1.5) == [1.5] diff --git a/tests/test_cache.py b/tests/test_cache.py index 45ae16b..f817241 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -8,7 +8,7 @@ import pytest -from dataflux.storage.cache import CacheBudgetExceeded, DiskCache +from recordstream.storage.cache import CacheBudgetExceeded, DiskCache def _write(payload: bytes) -> Callable[[Path], None]: diff --git a/tests/test_categories.py b/tests/test_categories.py index 4197e8f..dd11a2b 100644 --- a/tests/test_categories.py +++ b/tests/test_categories.py @@ -1,5 +1,5 @@ -# mypy: disable-error-code="attr-defined" -"""Discovery-category coverage for dataflux ``@configurable`` classes. +# mypy: disable-error-code="attr-defined,union-attr" +"""Discovery-category coverage for recordstream ``@configurable`` classes. These ``category=`` tags drive navigaitor's ``list_configurable_classes(category=...)`` MCP tool and, downstream, the visual-editor form-spec picker (``get_node_form_spec``). @@ -9,27 +9,145 @@ from confluid.registry import get_registry -from dataflux.core import FilterOp, Flux, JointFlux, WrappedOp +from recordstream import Pipeline +from recordstream.core import FilterOp, JointStream, Stream, WrappedOp +from recordstream.ops.configure import ConfigureOp +from recordstream.ops.debug import PrintRecordOp +from recordstream.ops.enable import Enable +from recordstream.ops.formula import FormulaOp +from recordstream.ops.image import ConvertToImage, ConvertToMask +from recordstream.ops.numpy import ConnectedComponents, Threshold +from recordstream.ops.parallel import Parallel +from recordstream.ops.random_apply import RandomApply +from recordstream.ops.sink import RecordSinkOp +from recordstream.ops.structure import CopyField, DropField, RenameField, SelectFields +from recordstream.ops.target import CocoToTorchVisionDetection, DecodeTarget, EncodeTarget, MasksToDetectionBoxes +from recordstream.ops.torch import ToTensor +from recordstream.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource +from recordstream.storage.directory import DirectorySink +from recordstream.storage.hdf5 import HDF5Sink, HDF5Source +from recordstream.storage.zarr import ZarrBatchSink, ZarrGroupSink -def test_dataset_classes_tagged() -> None: - """``Flux`` / ``JointFlux`` are the generic (task-agnostic) dataset engines.""" - assert Flux.__confluid_category__ == "dataset" - assert JointFlux.__confluid_category__ == "dataset" +def test_engine_classes_tagged() -> None: + assert Stream.__confluid_category__ == "engine" + assert JointStream.__confluid_category__ == "engine" + + +def test_raw_callable_wrappers_uncategorised() -> None: + assert getattr(FilterOp, "__confluid_category__", None) is None + assert getattr(WrappedOp, "__confluid_category__", None) is None + assert FilterOp.__confluid_configurable__ is True + assert WrappedOp.__confluid_configurable__ is True + + +def test_source_classes_tagged() -> None: + assert HuggingFaceSource.__confluid_category__ == "source" + assert DatasetSplit.__confluid_category__ == "source" + assert RangeSource.__confluid_category__ == "source" + assert ConcatSource.__confluid_category__ == "source" def test_op_classes_tagged() -> None: - """``FilterOp`` / ``WrappedOp`` are pipeline ops.""" - assert FilterOp.__confluid_category__ == "op" - assert WrappedOp.__confluid_category__ == "op" + for cls in ( + Threshold, + ConnectedComponents, + ToTensor, + ConvertToImage, + ConvertToMask, + Enable, + Pipeline, + Parallel, + RandomApply, + RecordSinkOp, + EncodeTarget, + DecodeTarget, + CocoToTorchVisionDetection, + MasksToDetectionBoxes, + ConfigureOp, + FormulaOp, + RenameField, + DropField, + CopyField, + SelectFields, + PrintRecordOp, + ): + assert cls.__confluid_category__ == "op", cls.__name__ + + +def test_random_apply_random_tagged() -> None: + assert RandomApply.__confluid_random__ is True + + +def test_storage_sink_classes_tagged() -> None: + assert HDF5Sink.__confluid_category__ == "sink" + assert ZarrGroupSink.__confluid_category__ == "sink" + assert ZarrBatchSink.__confluid_category__ == "sink" + assert DirectorySink.__confluid_category__ == "sink" + assert getattr(HDF5Source, "__confluid_category__", None) is None + + +def test_op_group_tags() -> None: + assert Threshold.__confluid_group__ == "numpy" + assert ConnectedComponents.__confluid_group__ == "numpy" + assert ToTensor.__confluid_group__ == "torch" + assert ConvertToImage.__confluid_group__ == "image" + assert ConvertToMask.__confluid_group__ == "image" + assert SelectFields.__confluid_group__ == "structure" + assert PrintRecordOp.__confluid_group__ == "debug" + assert EncodeTarget.__confluid_group__ == "structure" + assert DecodeTarget.__confluid_group__ == "structure" + assert CocoToTorchVisionDetection.__confluid_group__ == "structure" + assert MasksToDetectionBoxes.__confluid_group__ == "structure" + assert Parallel.__confluid_group__ == "compose" + assert Enable.__confluid_group__ == "compose" + assert Pipeline.__confluid_group__ == "compose" + assert RandomApply.__confluid_group__ == "compose" + assert ConfigureOp.__confluid_group__ == "compose" + assert FormulaOp.__confluid_group__ == "compose" + assert RecordSinkOp.__confluid_group__ == "sink" def test_categories_enumerable_via_registry() -> None: - """Importing the classes registers them; the category index must surface them. + registry = get_registry() + assert {"Stream", "JointStream"} <= registry.list_classes(category="engine") + assert "DatasetSplit" not in registry.list_classes(category="engine") + assert not ({"FilterOp", "WrappedOp"} & registry.list_classes(category="engine")) + assert {"HuggingFaceSource", "DatasetSplit", "RangeSource", "ConcatSource"} <= registry.list_classes( + category="source" + ) + assert { + "Threshold", + "ConnectedComponents", + "ToTensor", + "ConvertToImage", + "ConvertToMask", + "Enable", + "Pipeline", + "RecordSinkOp", + "EncodeTarget", + "DecodeTarget", + "CocoToTorchVisionDetection", + "MasksToDetectionBoxes", + } <= registry.list_classes(category="op") + assert {"HDF5Sink", "ZarrGroupSink", "ZarrBatchSink", "DirectorySink"} <= registry.list_classes(category="sink") + assert "RecordSinkOp" not in registry.list_classes(category="sink") + - The navigaitor picker queries ``list_classes(category=...)``, so the index — - not just the class attribute — has to carry the tag. - """ +def test_groups_enumerable_via_registry() -> None: registry = get_registry() - assert {"Flux", "JointFlux"} <= registry.list_classes(category="dataset") - assert {"FilterOp", "WrappedOp"} <= registry.list_classes(category="op") + assert {"Threshold", "ConnectedComponents"} <= registry.list_classes(group="numpy") + assert {"ToTensor"} <= registry.list_classes(group="torch") + assert {"ConvertToImage", "ConvertToMask"} <= registry.list_classes(group="image") + assert {"Parallel", "Enable", "Pipeline", "RandomApply", "ConfigureOp", "FormulaOp"} <= registry.list_classes( + group="compose" + ) + assert {"RecordSinkOp"} <= registry.list_classes(group="sink") + assert { + "EncodeTarget", + "DecodeTarget", + "CocoToTorchVisionDetection", + "MasksToDetectionBoxes", + "SelectFields", + } <= registry.list_classes(group="structure") + assert "Pipeline" in registry.list_classes(category="op", group="compose") diff --git a/tests/test_cli_materialize.py b/tests/test_cli_materialize.py new file mode 100644 index 0000000..05f961f --- /dev/null +++ b/tests/test_cli_materialize.py @@ -0,0 +1,128 @@ +"""``materialize_runnable`` — the flat config's top-level keys must reach the runnable. + +The regression this pins cost a debugging session and would have cost worse: a bare +``flow()`` on the bound ``runnable:`` node drops every broadcast sibling SILENTLY. A +dropped ``train_set:`` eventually surfaces as an empty dataset; a dropped +``max_epochs: 3`` never surfaces at all — the run proceeds on the constructor default +and looks configured. + +Why it happened: liquifai's DI materializes a command parameter *against the loaded +document* only when the parameter is annotated with a configurable class. A generic +runner annotates ``runnable: Any`` precisely because the runnable is polymorphic, so DI +hands over the raw Fluid and deep-flows it with no document — the very genericity that +makes one runner serve every task is what disabled broadcasting. +""" + +from typing import Any, Optional + +import confluid +import pytest +from confluid import configurable +from liquifai.context import LiquifyContext, get_context, set_context + +from recordstream.cli import materialize_runnable + + +@configurable +class _Runner: + """A stand-in runnable with the ergonomic-knob shape a flat config drives.""" + + def __init__(self, dataset: Optional[Any] = None, max_epochs: int = 10, name: str = "default") -> None: + self.dataset = dataset + self.max_epochs = max_epochs + self.name = name + + def run(self) -> str: + return self.name + + +FLAT_CONFIG = """ +runnable: !class:tests.test_cli_materialize._Runner +max_epochs: 3 +name: from_the_flat_config +dataset: !class:recordstream.Stream +""" + + +@pytest.fixture +def liquifai_context() -> Any: + """Install/remove a liquifai context the way the app does around a command.""" + previous = get_context() + + def _install(config_data: Any) -> None: + ctx = LiquifyContext(name="test") + ctx.config_data = config_data + set_context(ctx) + + yield _install + set_context(previous) + + +def _node(text: str = FLAT_CONFIG) -> Any: + """The document + its ``runnable:`` node, loaded the way liquifai loads it (until="document").""" + document = confluid.load(text, until="document") + return document, document["runnable"] + + +def test_top_level_keys_broadcast_into_the_runnable(liquifai_context: Any) -> None: + """THE regression: a bare flow() would leave every one of these at its default.""" + document, node = _node() + liquifai_context(document) + + runner = materialize_runnable(node) + + assert runner.max_epochs == 3, "a bare flow() leaves this at the ctor default of 10 — silently" + assert runner.name == "from_the_flat_config" + assert runner.dataset is not None + + +def test_a_bare_flow_would_have_dropped_them(liquifai_context: Any) -> None: + """The counterfactual, executed — so the pin above cannot silently stop meaning anything.""" + from confluid import flow + + _, node = _node() + dropped = flow(node) + + assert dropped.max_epochs == 10 and dropped.name == "default" and dropped.dataset is None + + +def test_a_live_object_passes_through_untouched(liquifai_context: Any) -> None: + """Only a Fluid needs building — an already-built runnable must not be rebuilt.""" + liquifai_context({"max_epochs": 3}) + live = _Runner(max_epochs=99) + + assert materialize_runnable(live) is live + assert live.max_epochs == 99 + + +def test_no_liquifai_context_falls_back_to_flow(liquifai_context: Any) -> None: + """Called outside an app (a library user, a test) — build it, don't raise.""" + set_context(None) + _, node = _node() + + runner = materialize_runnable(node) + + assert isinstance(runner, _Runner) + assert runner.max_epochs == 10 # nothing to broadcast FROM + + +def test_a_root_fluid_document_falls_back_to_flow(liquifai_context: Any) -> None: + """A YAML whose root is a single `!class:` has no siblings — nothing is lost.""" + document = confluid.load("!class:tests.test_cli_materialize._Runner\nmax_epochs: 5\n", until="document") + liquifai_context(document) # not a dict + + runner = materialize_runnable(document) + + assert isinstance(runner, _Runner) and runner.max_epochs == 5 + + +def test_the_built_runnable_still_runs(liquifai_context: Any) -> None: + """End of the line: what the CLI does after building.""" + document, node = _node() + liquifai_context(document) + + assert materialize_runnable(node).run() == "from_the_flat_config" + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/test_cli_run.py b/tests/test_cli_run.py new file mode 100644 index 0000000..70b8697 --- /dev/null +++ b/tests/test_cli_run.py @@ -0,0 +1,44 @@ +"""Tests for the `recordstream run` CLI dispatch (recordstream.cli.run).""" + +from typing import List + +from confluid import Target + +from recordstream.cli import run + + +def test_run_calls_runnable_run() -> None: + calls: List[str] = [] + + class R: + def run(self) -> None: + calls.append("ran") + + run(R()) + assert calls == ["ran"] + + +def test_run_flows_deferred_marker() -> None: + from confluid import configurable + + log: List[str] = [] + + @configurable + class R: + def __init__(self, tag: str = "") -> None: + self.tag = tag + + def run(self) -> None: + log.append(self.tag) + + # A deferred Confluid marker is flowed before run() is called. + run(Target(R, tag="x")) + assert log == ["x"] + + +def test_run_none_is_noop() -> None: + run(None) # must not raise + + +def test_run_non_runnable_is_handled() -> None: + run(object()) # no .run() -> logged + returns, never raises diff --git a/tests/test_convert_to_mask.py b/tests/test_convert_to_mask.py new file mode 100644 index 0000000..a8a31d9 --- /dev/null +++ b/tests/test_convert_to_mask.py @@ -0,0 +1,347 @@ +"""``ConvertToMask`` — a mask-bearing field to an int64 class-id ``Mask`` item. + +The segmentation counterpart of ``ConvertToImage``, and the front half of the target +pipeline a per-pixel task drives: the source hands over a greyscale/paletted PNG, this +turns it into the ``[H, W]`` int64 array a per-pixel loss consumes, and every remaining +step is an op that already existed (``FormulaOp`` remaps the ids, a bare albumentations +transform resizes it jointly with the image, ``collate_records`` stacks it). + +Also pins :func:`recordstream.item_value`, extracted here because the op needed the same +"get past a wrapper item" rule ``iter_key`` and ``batch_values`` already had. +""" + +from contextlib import contextmanager +from typing import Any, Dict, Iterator, List, Tuple + +import numpy as np +import pytest +from PIL import Image as PILImage + +from recordstream import Boxes, Image, Label, Mask, MultiLabel, collate_records, item_data, item_value +from recordstream.core import _apply_op +from recordstream.ops import ConvertToMask, DropField, FormulaOp + + +def _trimap(height: int = 30, width: int = 20) -> PILImage.Image: + """An Oxford-IIIT-Pet-style trimap: an L-mode PNG whose pixels are 1-based class ids.""" + return PILImage.fromarray(np.random.randint(1, 4, (height, width)).astype(np.uint8), mode="L") + + +# --------------------------------------------------------------------------- # +# item_value — the unwrapping rule the op shares with iter_key / batch_values +# --------------------------------------------------------------------------- # +class TestItemValue: + def test_a_label_yields_its_value_where_item_data_yields_the_label(self) -> None: + # The whole reason the function exists: a Label's payload slot is `value`, not + # `data`, so `item_data` cannot see into it. + label = Label("cat") + assert item_value(label) == "cat" + assert item_data(label) is label + + def test_a_multilabel_yields_its_values_list(self) -> None: + assert item_value(MultiLabel(["cat", "dog"])) == ["cat", "dog"] + + def test_an_array_item_yields_its_payload(self) -> None: + value = item_value(Mask(np.zeros((3, 3), dtype=np.int64))) + assert type(value) is np.ndarray + assert value.shape == (3, 3) + + def test_a_plain_value_passes_through(self) -> None: + assert item_value(30.72e6) == 30.72e6 + assert item_value(None) is None + + def test_iter_key_and_batch_values_agree_with_it(self) -> None: + """The two former copies now route through it — so they cannot drift from it.""" + from recordstream import batch_values, iter_key + + records = [{"class": Label(0)}, {"class": Label(1)}] + assert list(iter_key(records, "class")) == [0, 1] + assert list(batch_values(collate_records(records), "class")) == [0, 1] + + +# --------------------------------------------------------------------------- # +# ConvertToMask +# --------------------------------------------------------------------------- # +class TestConvertToMask: + def test_a_pil_mask_becomes_an_int64_mask_item(self) -> None: + out = ConvertToMask(field="segmentation_mask")({"segmentation_mask": _trimap()}) + mask = out["mask"] + assert isinstance(mask, Mask) + assert mask.shape == (30, 20) + assert mask.dtype == np.int64 + + def test_it_reads_a_mask_a_source_wrapped_in_a_label(self) -> None: + """The real shape: a source that does not know a column is a mask ships it as a Label. + + ``HuggingFaceSource`` does exactly this for every metadata column, so reading through + ``item_data`` (which hands the Label straight back) found nothing at all. + """ + record = {"image": Image(np.zeros((30, 20, 3), np.uint8)), "segmentation_mask": Label(_trimap())} + out = ConvertToMask(field="segmentation_mask")(record) + assert isinstance(out["mask"], Mask) + assert set(np.unique(np.asarray(out["mask"]))) <= {1, 2, 3} + + def test_a_blank_field_picks_the_first_array_or_pil_bearing_item(self) -> None: + out = ConvertToMask()({"note": "text", "seg": np.zeros((4, 5), dtype=np.uint8)}) + assert out["mask"].shape == (4, 5) + + def test_the_output_key_defaults_to_the_albumentations_vocabulary(self) -> None: + """``mask`` by default — so a bare albumentations transform finds it with no rename.""" + assert ConvertToMask().output == "mask" + assert "mask" in ConvertToMask()({"seg": np.zeros((4, 5))}) + + def test_the_output_key_is_configurable(self) -> None: + out = ConvertToMask(field="seg", output="target")({"seg": np.zeros((4, 5))}) + assert "target" in out and "mask" not in out + + def test_the_source_field_survives_so_a_later_op_can_read_it(self) -> None: + out = ConvertToMask(field="seg")({"seg": np.zeros((4, 5))}) + assert "seg" in out # dropping it is DropField's job, not this op's + + def test_other_keys_pass_through_untouched(self) -> None: + image = Image(np.zeros((4, 5, 3), np.uint8)) + out = ConvertToMask(field="seg")({"seg": np.zeros((4, 5)), "image": image, "id": 7}) + assert out["image"] is image + assert out["id"] == 7 + + @pytest.mark.parametrize("shape", [(4, 5, 1), (1, 4, 5)]) + def test_singleton_axes_are_squeezed(self, shape: Tuple[int, ...]) -> None: + assert ConvertToMask()({"seg": np.zeros(shape, dtype=np.uint8)})["mask"].shape == (4, 5) + + def test_an_rgb_mask_is_refused_rather_than_silently_collapsed(self) -> None: + """Picking one of three channels is a decision the op must not make for the user.""" + with pytest.raises(ValueError, match="2-D"): + ConvertToMask()({"seg": np.zeros((4, 5, 3), dtype=np.uint8)}) + + def test_a_torch_tensor_payload_is_accepted(self) -> None: + torch = pytest.importorskip("torch") + out = ConvertToMask()({"seg": torch.randint(0, 3, (4, 5))}) + assert isinstance(out["mask"], Mask) + assert out["mask"].dtype == np.int64 + + # ---- lazy / zero-arg construction (workspace mandate) ------------------- # + def test_zero_arg_construction_works(self) -> None: + assert ConvertToMask().field == "" + + def test_a_missing_named_field_is_reported_at_call_time(self) -> None: + with pytest.raises(ValueError, match="not in record"): + ConvertToMask(field="nope")({"seg": np.zeros((4, 5))}) + + def test_a_record_with_no_candidate_is_reported_at_call_time(self) -> None: + with pytest.raises(ValueError, match="no array/PIL-bearing field"): + ConvertToMask()({"note": "text"}) + + +# --------------------------------------------------------------------------- # +# The chain it belongs to — this is what a segmentation config actually writes +# --------------------------------------------------------------------------- # +class TestTheSegmentationTargetPipeline: + def test_convert_remap_drop_resize_collate(self) -> None: + """End to end, with every step after the conversion an op that already existed. + + This is the ``preprocess`` chain of a segmentation config, run in Python: the point + being that ``ConvertToMask`` is the ONLY piece segmentation needed added. + """ + albumentations = pytest.importorskip("albumentations") + torch = pytest.importorskip("torch") + from recordstream import batch_tensor + + record: Dict[str, Any] = { + "image": Image(np.random.randint(0, 255, (30, 20, 3), dtype=np.uint8)), + "segmentation_mask": Label(_trimap()), + } + chain = [ + ConvertToMask(field="segmentation_mask"), + FormulaOp(field="mask", formula="a - 1"), # 1-based trimap -> 0-based class ids + DropField(key="segmentation_mask"), # a PIL left in the record would break collate + albumentations.Resize(height=16, width=16), # ONE joint draw over image AND mask + ] + for op in chain: + applied = _apply_op(record, op) + assert applied is not None # no op in this chain drops a record + record = applied + + assert set(record) == {"image", "mask"} + assert record["image"].shape == (16, 16, 3) + assert record["mask"].shape == (16, 16) + assert set(np.unique(np.asarray(record["mask"]))) <= {0, 1, 2} + + batch = collate_records([record, record]) + assert batch["image"].shape == (2, 16, 16, 3) + assert batch["mask"].shape == (2, 16, 16) + # The dtype the per-pixel CrossEntropy contract needs — named by the CALLER, because + # albumentations casts the mask to int32 on the way past. + assert batch_tensor(batch, "mask", dtype=torch.int64).dtype == torch.int64 + + def test_a_normalize_leaves_the_mask_alone(self) -> None: + """Albumentations applies an image-only transform to the image only — pinned, because + a Normalize that reached the mask would turn class ids into floats silently.""" + albumentations = pytest.importorskip("albumentations") + record = { + "image": Image(np.random.randint(0, 255, (8, 8, 3), dtype=np.uint8)), + "mask": Mask(np.random.randint(0, 3, (8, 8)).astype(np.int64)), + } + out = _apply_op(record, albumentations.Normalize(mean=[0.5] * 3, std=[0.5] * 3)) + assert out is not None + assert out["image"].dtype == np.float32 + assert np.array_equal(np.asarray(out["mask"]), np.asarray(record["mask"])) + + +# --------------------------------------------------------------------------- # +# num_mask_classes — the per-pixel twin of num_classes +# --------------------------------------------------------------------------- # +class TestNumMaskClasses: + def test_it_is_the_largest_id_anywhere_plus_one(self) -> None: + from recordstream import num_mask_classes + + records = [ + {"mask": Mask(np.array([[0, 1], [1, 0]], dtype=np.int64))}, + {"mask": Mask(np.array([[0, 2], [2, 2]], dtype=np.int64))}, # 2 appears only here + ] + assert num_mask_classes(records) == 3 + + def test_a_class_in_the_last_record_still_sizes_the_head(self) -> None: + """The walk covers EVERY record — a rare class must not be missed by an early exit.""" + from recordstream import num_mask_classes + + records = [{"mask": Mask(np.zeros((2, 2), dtype=np.int64))} for _ in range(50)] + records[-1] = {"mask": Mask(np.array([[0, 7], [0, 0]], dtype=np.int64))} + assert num_mask_classes(records) == 8 + + def test_the_key_is_configurable(self) -> None: + from recordstream import num_mask_classes + + assert num_mask_classes([{"target": Mask(np.array([[0, 4]], dtype=np.int64))}], key="target") == 5 + + def test_an_empty_source_raises_rather_than_guessing(self) -> None: + from recordstream import num_mask_classes + + with pytest.raises(ValueError, match="no usable"): + num_mask_classes([]) + + def test_a_record_without_a_mask_raises(self) -> None: + """Silently skipping it would under-count the head against a partially-labelled set.""" + from recordstream import num_mask_classes + + with pytest.raises(ValueError, match="no 'mask' value"): + num_mask_classes([{"mask": Mask(np.zeros((2, 2), dtype=np.int64))}, {"mask": None}]) + with pytest.raises(ValueError, match="no 'mask' value"): + num_mask_classes([{"image": Image(np.zeros((2, 2, 3), np.uint8))}]) + + def test_num_classes_still_refuses_an_array_target(self) -> None: + """The reason this is a separate function: the scalar guard must stay strict. + + A classification run accidentally handed masks has to fail loudly rather than report + whatever the first array's `.item()` would have been. + """ + from recordstream import num_classes + + with pytest.raises((TypeError, ValueError)): + num_classes([{"class": Mask(np.array([[0, 1], [1, 0]], dtype=np.int64))}]) + + +@contextmanager +def _captured_warnings() -> Iterator[List[str]]: + """Collect loggair WARNING records emitted inside the block. + + `caplog` cannot see these: loggair is loguru, which does not propagate to stdlib logging. + Neither can `capfd` alone — the sink is ENQUEUED, so the write lands on another thread after + the assertion runs (this looked exactly like "the warning never fired"). `logger.complete()` + is the deterministic flush the workspace mandates instead of sleeping. + """ + from loguru import logger + + collected: List[str] = [] + sink_id = logger.add(lambda message: collected.append(str(message)), level="WARNING") + try: + yield collected + logger.complete() + finally: + logger.remove(sink_id) + + +class TestBoxesKnowTheirFrame: + """`canvas` is the raster a `Boxes`' boxes are stated in — so every op that makes or + re-frames one records it, and the op that moves pixels ALONE says so. + + Before this, only the coupled resize set `canvas`, which meant the frame was knowable + exactly when it was least needed (a resize that already moved the boxes correctly) and + unknown in the chain where boxes and pixels can actually drift apart. + """ + + def test_coco_boxes_record_the_image_they_annotate(self) -> None: + from recordstream.ops.target import CocoToTorchVisionDetection + + record = { + "image": Image(np.zeros((300, 400, 3), dtype="uint8")), + "objects": Label({"bbox": [[10.0, 10.0, 20.0, 20.0]], "category": [1]}), + } + out = CocoToTorchVisionDetection(field="objects")(record) + assert out["target"].canvas == (300, 400), "the annotation's frame is the image's" + + def test_mask_derived_boxes_record_the_mask(self) -> None: + from recordstream.ops.target import MasksToDetectionBoxes + + mask = np.zeros((64, 96), dtype="int64") + mask[10:20, 30:40] = 1 + out = MasksToDetectionBoxes()({"mask": Mask(mask)}) + assert out["target"].canvas == (64, 96), "the boxes were derived FROM this raster" + + def test_a_record_with_no_image_still_works(self) -> None: + """`None` is an ordinary answer — the lookup failing must not fail the op.""" + from recordstream.ops.target import CocoToTorchVisionDetection + + out = CocoToTorchVisionDetection(field="objects")( + {"objects": Label({"bbox": [[1.0, 2.0, 3.0, 4.0]], "category": [0]})} + ) + assert out["target"].canvas is None + + def test_a_regions_box_array_is_never_mistaken_for_a_raster(self) -> None: + """The narrow lookup's whole point: an `[N, 4]` box array is 2-D and must not be read + as an N x 4 image, which a generic first-array search would do confidently.""" + from recordstream.ops.target import CocoToTorchVisionDetection + + record = { + "objects": Label({"bbox": [[1.0, 2.0, 3.0, 4.0]], "category": [0]}), + "other": Boxes(boxes=np.zeros((7, 4), dtype="float32"), labels=np.zeros((7,), dtype="int64")), + } + assert CocoToTorchVisionDetection(field="objects")(record)["target"].canvas is None + + def test_an_EMPTY_target_is_re_framed_too(self) -> None: + """A negative example must not be the one record whose frame is unknown.""" + from recordstream.ops.target import ResizeDetection + + record = { + "image": Image(np.zeros((100, 100, 3), dtype="uint8")), + "target": Boxes(boxes=np.zeros((0, 4), dtype="float32"), labels=np.zeros((0,), dtype="int64")), + } + out = ResizeDetection(width=64, height=32)(record) + assert out["target"].canvas == (32, 64) + + def test_an_image_only_resize_WARNS_when_it_desyncs_boxes(self) -> None: + from recordstream.ops.image import ConvertToImage + + record = { + "image": Image(np.zeros((200, 200, 3), dtype="uint8")), + "target": Boxes(boxes=np.array([[10.0, 10.0, 50.0, 50.0]]), labels=np.array([1])), + } + op = ConvertToImage(field="image", width=64, height=64) + with _captured_warnings() as warnings: + op(record) + assert len(warnings) == 1 + assert "ResizeDetection" in warnings[0] and "PIXELS ONLY" in warnings[0] + assert "(200, 200) -> (64, 64)" in warnings[0], "the message states both rasters" + # Once per op instance: the message is about the CONFIGURATION, not about this record. + with _captured_warnings() as second: + op(record) + assert second == [] + + def test_no_warning_without_boxes_or_without_a_resize(self) -> None: + from recordstream.ops.image import ConvertToImage + + image = Image(np.zeros((200, 200, 3), dtype="uint8")) + boxes = Boxes(boxes=np.array([[1.0, 2.0, 3.0, 4.0]]), labels=np.array([1])) + with _captured_warnings() as warnings: + ConvertToImage(field="image", width=64, height=64)({"image": image}) # resized, no boxes + ConvertToImage(field="image", max_size=999)({"image": image, "target": boxes}) # boxes, no resize + assert warnings == [] diff --git a/tests/test_coverage_gap.py b/tests/test_coverage_gap.py deleted file mode 100644 index 2d0f568..0000000 --- a/tests/test_coverage_gap.py +++ /dev/null @@ -1,156 +0,0 @@ -from pathlib import Path -from typing import Any - -import numpy as np -import pytest - -from dataflux.core import Flux -from dataflux.discovery import get_callable_path, resolve_callable -from dataflux.sample import Sample -from dataflux.storage.base import Storage -from dataflux.storage.directory import DirectorySink -from dataflux.storage.hdf5 import HDF5Sink, HDF5Source -from dataflux.storage.zarr import ZarrBatchSink, ZarrGroupSink - - -def test_storage_base_close() -> None: - # hits base.py:36 - s = Storage() - s.close() - - -def test_directory_open_overwrite(tmp_path: Path) -> None: - # hits directory.py:27 (pass) - d = tmp_path / "ovr" - d.mkdir() - sink = DirectorySink(d, overwrite=True) - sink.open() - - -def test_hdf5_none_file() -> None: - # hits hdf5.py:45, 59, 90 - source = HDF5Source("nonexistent.h5") - # Manually ensure _file is None (it is by default) - assert source._file is None - # We need to bypass open() or mock it to stay None - # But __iter__ calls open(). - # Let's mock open to do nothing - source.open = lambda: source # type: ignore - assert list(source) == [] # hits line 45 - assert len(source) == 0 # hits line 59 - - sink = HDF5Sink("nonexistent.h5") - sink.open = lambda: sink # type: ignore - sink.write(Sample(input=1)) # hits line 90 - - -def test_resolve_callable_import_error() -> None: - # hits discovery.py:57 - with pytest.raises(ImportError, match="Cannot resolve"): - resolve_callable("nonexistent_mod_totally:func") - - -def test_get_callable_path_main_no_file(monkeypatch: Any) -> None: - # hits discovery.py:33 (pass) - import sys - import types - - m = types.ModuleType("__main__") - # No __file__ attribute - monkeypatch.setitem(sys.modules, "__main__", m) - - def local_f() -> None: - pass - - local_f.__module__ = "__main__" - # Should fallback to valueerror or just skip the main block if it fails - try: - get_callable_path(local_f) - except ValueError: - pass - - -def test_hdf5_metadata_exception(tmp_path: Path) -> None: - # hits hdf5.py:except branch in metadata loop - h5_path = tmp_path / "meta_err.h5" - sink = HDF5Sink(h5_path) - # Metadata that h5py might not like (e.g. nested dict) - sample = Sample(input=np.array([1]), metadata={"bad": {"nested": "value"}}) - sink.write(sample) - sink.close() - - -def test_zarr_group_open_overwrite(tmp_path: Path) -> None: - # hits zarr.py:29 (pass) - z_path = tmp_path / "z_ovr.zarr" - z_path.mkdir() - sink = ZarrGroupSink(z_path, overwrite=True) - sink.open() - - -def test_flux_iter_none_source() -> None: - # hits core.py:183, 193 - f = Flux(None) - assert list(f._iter_sequential()) == [] - assert list(f._iter_parallel()) == [] - - -def test_zarr_group_flush() -> None: - # hits zarr.py:56 - sink = ZarrGroupSink("test.zarr") - sink.flush() - - -def test_sample_from_any_tuple_1() -> None: - # hits sample.py:22 - s = Sample.from_any((1,)) - assert s.input == 1 - assert s.target is None - - -def test_sample_from_any_empty_tuple() -> None: - # hits sample.py new branch - s = Sample.from_any(()) - assert s.input is None - - -def test_optional_context_manager_direct() -> None: - # hits core.py:177 - from dataflux.core import Flux - - f = Flux([1]) - - class SimpleSink: - def write(self, s: Any) -> None: - pass - - def flush(self) -> None: - pass - - f.to_sink(SimpleSink()) - - -def test_discovery_main_failure(monkeypatch: Any) -> None: - # hits discovery.py:30 (pass) - import sys - import types - - m = types.ModuleType("__main__") - # NO __file__ - monkeypatch.setitem(sys.modules, "__main__", m) - - def f() -> None: - pass - - f.__module__ = "__main__" - try: - get_callable_path(f) - except ValueError: - pass - - -def test_zarr_batch_chunks(tmp_path: Path) -> None: - # hits zarr.py: chunks logic - p = tmp_path / "chunks.zarr" - sink = ZarrBatchSink(p, shape=[10], chunks=[1, 10], overwrite=True) - sink.write(Sample(input=np.zeros(10))) diff --git a/tests/test_dataset_uri.py b/tests/test_dataset_uri.py new file mode 100644 index 0000000..2541d77 --- /dev/null +++ b/tests/test_dataset_uri.py @@ -0,0 +1,280 @@ +"""Dataset identity — ``recordstream.uri`` and ``HuggingFaceSource``'s implementation of it.""" + +from pathlib import Path +from typing import Any, List, Optional + +import pytest + +from recordstream import ( + ConcatSource, + DatasetSplit, + HuggingFaceSource, + RangeSource, + Stream, + SupportsDatasetIdentity, + dataset_uri, + dataset_uris, + dataset_url, +) +from recordstream.uri import MAX_WRAPPER_DEPTH + + +class _FakeSource: + """A minimal indexable source, so a view can wrap something with a known length.""" + + def __init__(self, size: int = 6, uri: Optional[str] = None) -> None: + self._size = size + self.dataset_uri = uri # type: ignore[assignment] + self.dataset_url = None # type: ignore[assignment] + + def __len__(self) -> int: + return self._size + + def __getitem__(self, index: int) -> Any: + return {"class": index} + + def __iter__(self) -> Any: + return iter(self[i] for i in range(self._size)) + + +# --- HuggingFaceSource's identity ------------------------------------------------------------ + + +def test_a_hub_repo_id_becomes_an_hf_uri_and_a_viewer_url() -> None: + source = HuggingFaceSource(path="ylecun/mnist", split="train") + assert source.dataset_uri == "hf://datasets/ylecun/mnist?split=train" + assert source.dataset_url == "https://huggingface.co/datasets/ylecun/mnist/viewer/default/train" + + +def test_config_name_and_revision_ride_the_uri_and_the_url() -> None: + source = HuggingFaceSource(path="ylecun/mnist", name="fashion", split="test", revision="abc123") + assert source.dataset_uri == "hf://datasets/ylecun/mnist?name=fashion&revision=abc123&split=test" + assert source.dataset_url == "https://huggingface.co/datasets/ylecun/mnist/viewer/fashion/test" + + +def test_query_parameters_are_sorted_so_one_config_has_exactly_one_uri() -> None: + """Two identically-configured sources must produce the SAME string, whatever the kwarg order.""" + one = HuggingFaceSource(path="a/b", split="train", name="cfg", revision="r1") + other = HuggingFaceSource(path="a/b", revision="r1", name="cfg", split="train") + assert one.dataset_uri == other.dataset_uri + + +def test_a_local_directory_becomes_a_file_uri_with_no_browsable_url(tmp_path: Path) -> None: + source = HuggingFaceSource(path=str(tmp_path), split="train") + assert source.dataset_uri == f"{tmp_path.resolve().as_uri()}?split=train" + # Data on disk has no web page, and inventing a Hub URL for it would be a lie. + assert source.dataset_url is None + + +def test_an_unconfigured_source_has_no_identity() -> None: + source = HuggingFaceSource() + assert source.dataset_uri is None + assert source.dataset_url is None + + +def test_a_source_without_a_split_links_to_the_dataset_page() -> None: + source = HuggingFaceSource(path="ylecun/mnist", split="") + assert source.dataset_uri == "hf://datasets/ylecun/mnist" + assert source.dataset_url == "https://huggingface.co/datasets/ylecun/mnist" + + +def test_asking_for_identity_never_loads_the_dataset() -> None: + """The whole point of reading stored config: a source that is never iterated still answers.""" + source = HuggingFaceSource(path="ylecun/mnist", split="train") + assert source.dataset_uri is not None + assert source._dataset is None # nothing was materialized + + +def test_huggingface_source_satisfies_the_protocol() -> None: + assert isinstance(HuggingFaceSource(path="a/b"), SupportsDatasetIdentity) + + +# --- the load-option surface `revision` is shared with ---------------------------------------- + + +def test_revision_reaches_both_the_uri_and_the_load_options() -> None: + source = HuggingFaceSource(path="a/b", revision="v2") + assert "revision=v2" in (source.dataset_uri or "") + assert source.load_options == {"revision": "v2"} + + +def test_load_options_merges_the_explicit_dict_under_the_declared_revision() -> None: + source = HuggingFaceSource(path="a/b", revision="v2", load_kwargs={"token": "t", "cache_dir": "/c"}) + assert source.load_options == {"token": "t", "cache_dir": "/c", "revision": "v2"} + + +def test_a_revision_set_after_construction_is_not_stale() -> None: + """`revision` is a DECLARED param, so the config layer may set it post-construction.""" + source = HuggingFaceSource(path="a/b") + assert source.load_options == {} + source.revision = "v3" + assert source.load_options == {"revision": "v3"} + assert "revision=v3" in (source.dataset_uri or "") + + +def test_the_constructor_does_not_accept_arbitrary_keywords() -> None: + """A `**kwargs` constructor accepts every broadcast key there is; this one must not. + + The rejection is what keeps an unrelated broadcast key (a run name) out of + ``load_dataset``, which reads an unknown keyword as a builder CONFIG NAME and so misses + the cache entirely. The exception type is confluid's validation wrap, not a bare + ``TypeError``, so the assertion is on the rejected KEY being named. + """ + with pytest.raises(Exception) as raised: + HuggingFaceSource(path="a/b", run_name="my_run") # type: ignore[call-arg] + assert "run_name" in str(raised.value) + + +# --- the free functions ---------------------------------------------------------------------- + + +def test_the_free_functions_read_a_sources_own_identity() -> None: + source = HuggingFaceSource(path="ylecun/mnist", split="train") + assert dataset_uri(source) == source.dataset_uri + assert dataset_url(source) == source.dataset_url + + +def test_none_and_an_identity_less_source_answer_none() -> None: + assert dataset_uri(None) is None + assert dataset_url(None) is None + assert dataset_uri(_FakeSource()) is None + + +@pytest.mark.parametrize( + "wrap", + [ + pytest.param(lambda src: Stream(source=src), id="stream"), + pytest.param(lambda src: RangeSource(source=src, start=0, stop=2), id="range"), + pytest.param(lambda src: DatasetSplit(source=src, val_fraction=0.5, seed=0), id="split"), + pytest.param(lambda src: Stream(source=RangeSource(source=src, start=0, stop=2)), id="nested"), + ], +) +def test_a_wrapper_reports_the_wrapped_datasets_identity_verbatim(wrap: Any) -> None: + """A view identifies the same DATASET; how much of it this run reads is a separate fact.""" + source = HuggingFaceSource(path="ylecun/mnist", split="train") + wrapped = wrap(source) + assert dataset_uri(wrapped) == "hf://datasets/ylecun/mnist?split=train" + assert dataset_url(wrapped) == "https://huggingface.co/datasets/ylecun/mnist/viewer/default/train" + + +def test_a_split_view_reports_the_split_sources_identity() -> None: + """The three views are private objects only ever read off a live split — they follow too. + + Wrapping a local fake rather than a Hub source on purpose: reading ``.train`` PARTITIONS, + which calls ``len(source)``, which would make a Hub source download. + """ + inner = _FakeSource(size=8, uri="hf://datasets/a/b?split=train") + split = DatasetSplit(source=inner, val_fraction=0.25, seed=0) + assert dataset_uri(split.train) == "hf://datasets/a/b?split=train" + assert dataset_uri(split.val) == dataset_uri(split.train) + + +def test_a_wrapper_with_its_own_identity_wins_over_the_one_it_wraps() -> None: + inner = HuggingFaceSource(path="ylecun/mnist", split="train") + outer = _FakeSource(uri="store://curated/v2") + outer.source = inner # type: ignore[attr-defined] + assert dataset_uri(outer) == "store://curated/v2" + + +def test_a_cyclic_wrapper_chain_terminates() -> None: + """The depth cap exists so a tracking call can never hang on a self-referential source.""" + node = _FakeSource() + node.source = node # type: ignore[attr-defined] + assert dataset_uri(node) is None + + +def test_a_chain_deeper_than_the_cap_gives_up_rather_than_walking_forever() -> None: + deepest = HuggingFaceSource(path="a/b", split="train") + chain: Any = deepest + for _ in range(MAX_WRAPPER_DEPTH + 2): + wrapper = _FakeSource() + wrapper.source = chain # type: ignore[attr-defined] + chain = wrapper + assert dataset_uri(chain) is None + + +def test_a_deferred_config_marker_is_materialized_before_being_asked() -> None: + """A `!class:` marker straight out of a config must not need a `flow()` at the call site.""" + from confluid import load + + node = load( + "!class:recordstream.sources.huggingface.HuggingFaceSource()\n path: ylecun/mnist\n split: train\n", + until="document", + ) + assert dataset_uri(node) == "hf://datasets/ylecun/mnist?split=train" + + +# --- concatenation --------------------------------------------------------------------------- + + +def test_a_concatenation_declines_to_name_one_dataset() -> None: + concat = ConcatSource(sources=[HuggingFaceSource(path="a/b"), HuggingFaceSource(path="c/d")]) + assert dataset_uri(concat) is None + assert dataset_url(concat) is None + + +def test_dataset_uris_fans_out_over_the_members() -> None: + concat = ConcatSource(sources=[HuggingFaceSource(path="a/b"), HuggingFaceSource(path="c/d")]) + assert dataset_uris(concat) == ["hf://datasets/a/b?split=train", "hf://datasets/c/d?split=train"] + + +def test_dataset_uris_deduplicates_and_descends_into_nested_concatenations() -> None: + same = HuggingFaceSource(path="a/b") + inner = ConcatSource(sources=[same, HuggingFaceSource(path="c/d")]) + outer = ConcatSource(sources=[inner, HuggingFaceSource(path="a/b")]) + assert dataset_uris(outer) == ["hf://datasets/a/b?split=train", "hf://datasets/c/d?split=train"] + + +def test_dataset_uris_of_a_single_source_is_its_one_uri() -> None: + source = HuggingFaceSource(path="ylecun/mnist", split="train") + assert dataset_uris(source) == ["hf://datasets/ylecun/mnist?split=train"] + + +def test_dataset_uris_skips_members_that_have_no_identity() -> None: + concat = ConcatSource(sources=[_FakeSource(), HuggingFaceSource(path="a/b")]) + found: List[str] = dataset_uris(concat) + assert found == ["hf://datasets/a/b?split=train"] + + +# --- asking never builds: a deferred marker's CLASS is checked before construction ------------- + +_BUILDS = {"count": 0} + + +class _NoDatasetSurface: + """No identity properties, no ``source`` slot — the shape of a deferred trainer/loader.""" + + def __init__(self, max_epochs: int = 1) -> None: + _BUILDS["count"] += 1 + self.max_epochs = max_epochs + + +def test_a_deferred_marker_whose_class_has_no_dataset_surface_is_never_built() -> None: + from confluid import PartialClass + + _BUILDS["count"] = 0 + assert dataset_uri(PartialClass(_NoDatasetSurface, max_epochs=3)) is None + assert dataset_url(PartialClass(_NoDatasetSurface, max_epochs=3)) is None + assert _BUILDS["count"] == 0, "the class answers the question; nothing is constructed" + + +def test_a_deferred_source_with_an_identity_property_still_builds_and_answers() -> None: + class _IdSource: + @property + def dataset_uri(self) -> str: + return "hf://datasets/x" + + from confluid import Target + + assert dataset_uri(Target(_IdSource)) == "hf://datasets/x" + + +def test_a_deferred_wrapper_with_a_source_slot_is_still_built_and_followed() -> None: + class _Wrap: + def __init__(self, source: Any = None) -> None: + self.source = source + + from confluid import Target + + inner = _FakeSource(uri="hf://datasets/inner") + assert dataset_uri(Target(_Wrap, source=inner)) == "hf://datasets/inner" diff --git a/tests/test_discovery.py b/tests/test_discovery.py index 7c303f6..7d5797c 100644 --- a/tests/test_discovery.py +++ b/tests/test_discovery.py @@ -4,11 +4,11 @@ import pytest -from dataflux.discovery import get_callable_path, introspect_callable, resolve_callable, scan_module +from recordstream.discovery import get_callable_path, introspect_callable, resolve_callable, scan_module def sample_func(a: int, b: str = "default") -> str: - """Sample docstring.""" + """Record docstring.""" return f"{a}-{b}" @@ -45,7 +45,7 @@ def test_resolve_callable_errors() -> None: # Test AttributeError with pytest.raises(AttributeError): - resolve_callable("dataflux.discovery:nonexistent_func") + resolve_callable("recordstream.discovery:nonexistent_func") def test_introspect_errors() -> None: @@ -111,7 +111,7 @@ def test_resolve_callable_direct() -> None: def test_introspect_callable() -> None: schema = introspect_callable(sample_func) assert schema["name"] == "sample_func" - assert schema["doc"] == "Sample docstring." + assert schema["doc"] == "Record docstring." assert len(schema["parameters"]) == 2 p0 = schema["parameters"][0] @@ -160,7 +160,7 @@ class ClassInScript: assert "ClassInScript" in names # Standard module scan - schemas_self = scan_module("dataflux.discovery") + schemas_self = scan_module("recordstream.discovery") names_self = [s["name"] for s in schemas_self] assert "scan_module" in names_self assert "get_callable_path" in names_self diff --git a/tests/test_dispatch.py b/tests/test_dispatch.py new file mode 100644 index 0000000..9088dc9 --- /dev/null +++ b/tests/test_dispatch.py @@ -0,0 +1,56 @@ +"""The kernel registry — exact + MRO dispatch, transform-MRO inheritance, override, cache.""" + +from typing import Any, Dict + +from recordstream import Boxes, Image, Label, Mask, Transform +from recordstream.dispatch import dispatch, get_kernel, register_kernel, registered_kernels +from tests._fixtures import FixtureFlip + + +class TestDispatch: + def test_exact_hit(self) -> None: + assert get_kernel(FixtureFlip, Image) is not None + assert dispatch(FixtureFlip, Image) is get_kernel(FixtureFlip, Image) + + def test_miss_returns_none(self) -> None: + # FixtureFlip has no Label kernel — a Label value passes through. + assert dispatch(FixtureFlip, Label) is None + assert get_kernel(FixtureFlip, Label) is None + + def test_item_mro_walk(self) -> None: + class SubMask(Mask): + pass + + # No kernel for SubMask, but its base Mask has one — the MRO walk resolves it. + assert get_kernel(FixtureFlip, SubMask) is None + assert dispatch(FixtureFlip, SubMask) is get_kernel(FixtureFlip, Mask) + + def test_transform_mro_inheritance(self) -> None: + class TunedFlip(FixtureFlip): + pass + + # A subclass transform inherits its base's kernels until it overrides them. + assert dispatch(TunedFlip, Image) is get_kernel(FixtureFlip, Image) + + def test_override_wins_and_invalidates_cache(self) -> None: + class T(Transform): + pass + + assert dispatch(T, Image) is None # populate the cache with a miss + + @register_kernel(T, Image) + def _kernel(item: Any, params: Dict[str, Any]) -> Any: + return item + + assert dispatch(T, Image) is _kernel # cache was cleared on registration + + def test_plain_value_type_misses(self) -> None: + # A plain (non-item) value type — e.g. float — has no kernel: the op passes it through. + assert dispatch(FixtureFlip, float) is None + + def test_registered_kernels_lists_pairs(self) -> None: + pairs = registered_kernels() + assert ("FixtureFlip", "Image") in pairs + assert ("FixtureFlip", "Boxes") in pairs + assert dispatch(FixtureFlip, Label) is None # FixtureFlip does not handle Label + assert dispatch(FixtureFlip, Boxes) is get_kernel(FixtureFlip, Boxes) diff --git a/tests/test_docs_links.py b/tests/test_docs_links.py new file mode 100644 index 0000000..897f90b --- /dev/null +++ b/tests/test_docs_links.py @@ -0,0 +1,193 @@ +"""The documentation's internal links resolve. + +A dead docs link is the kind of rot nobody notices until a reader hits one, and +it is created by exactly the ordinary edits this project does constantly: renaming +a heading, splitting a page, moving a section between guides. Nothing else checks +it — CI runs the test suite and the examples, and a broken `[text](other.md#x)` +breaks neither. + +Three rules, one per failure mode: + +* a relative link names a file that exists, +* an `#anchor` names a heading that exists in that file, +* every docs page is reachable from the README index. + +The README is the PyPI landing page (``readme = "README.md"``), and PyPI resolves a +relative link against ``pypi.org``, not against the repository — so two further rules +apply to it alone: it links to repo files by ABSOLUTE GitHub URL, and each such URL +names a file that exists here. The ``docs/*.md`` pages are read on GitHub only and +keep their relative links. + +Fenced code blocks are excluded from the scan — see :func:`_strip_code`. + +The CI workflow is generated and must not be hand-edited, so these live as tests +— which is also where they belong: they run locally on the same command. +""" + +import re +from pathlib import Path +from typing import List, Set + +import pytest + +_REPO = Path(__file__).resolve().parent.parent +_DOCS = _REPO / "docs" +_README = _REPO / "README.md" + +#: `[text](target)` where the target is a relative path — absolute URLs are excluded +#: by requiring the target not to contain `://`. +_RELATIVE_LINK = re.compile(r"\[[^\]]*\]\((?!\w+://)([^)\s]+)\)") + + +def _slug(heading: str) -> str: + """GitHub's anchor slug for a heading: lowercase, punctuation dropped, spaces hyphenated. + + Punctuation removal is what makes `## Registering a class you don't own` reachable + as `#registering-a-class-you-dont-own` and a backticked `## \\`flow()\\` finishes...` + reachable without the backticks or parens. + """ + return re.sub(r"[^\w\s-]", "", heading.lower()).replace(" ", "-") + + +def _headings(path: Path) -> Set[str]: + return {_slug(line.lstrip("#").strip()) for line in path.read_text().splitlines() if line.startswith("#")} + + +def _markdown_files() -> List[Path]: + return sorted(_DOCS.glob("*.md")) + [_README] + + +def _strip_code(text: str) -> str: + """Blank out fenced code blocks before scanning for links. + + Python subscript-then-call — `PartialClass[Metric](SomeClass)` — is + indistinguishable from a markdown link to a regex, so a code sample containing + one gets reported as a link to a file named `SomeClass`. Found exactly that way: + the first run of this check against another project's docs produced a false + positive, which in a link checker is worse than a miss, because it trains the + reader to ignore it. + """ + out, fenced = [], False + for line in text.splitlines(): + if line.lstrip().startswith("```"): + fenced = not fenced + out.append("") + continue + out.append("" if fenced else line) + return "\n".join(out) + + +def _links(path: Path) -> List[str]: + return _RELATIVE_LINK.findall(_strip_code(path.read_text())) + + +@pytest.mark.parametrize("path", _markdown_files(), ids=lambda p: p.name) +def test_relative_links_name_a_file_that_exists(path: Path) -> None: + """A `[text](other.md)` must point at a real file. + + Catches a page that was renamed or deleted while something still pointed at it. + """ + missing = [] + for target in _links(path): + if target.startswith("#"): + continue # same-page anchor, checked below + file_part = target.split("#", 1)[0] + if not (path.parent / file_part).resolve().exists(): + missing.append(target) + + assert not missing, f"{path.name} links to files that do not exist: {missing}" + + +@pytest.mark.parametrize("path", _markdown_files(), ids=lambda p: p.name) +def test_anchors_name_a_heading_that_exists(path: Path) -> None: + """A `#anchor` must match a heading in the file it points at. + + This is the half that rots silently: renaming a heading leaves every link to it + pointing at the top of the page instead of failing, so the reader lands + somewhere plausible and never reports it. + """ + dead = [] + for target in _links(path): + if "#" not in target: + continue + file_part, _, anchor = target.partition("#") + if not anchor: + continue + target_file = path if not file_part else (path.parent / file_part).resolve() + if not target_file.exists(): + continue # reported by the test above; don't double-fail + if anchor not in _headings(target_file): + dead.append(target) + + assert not dead, f"{path.name} links to headings that do not exist: {dead}" + + +def test_every_docs_page_is_listed_in_the_readme_index() -> None: + """The README is the docs index; a page missing from it is a page nobody finds. + + `architecture.md` is exempt: it is the rationale record for maintainers, and the + README index is the user-facing table. + """ + exempt = {"architecture.md"} + listed = set(re.findall(r"docs/([a-z0-9_-]+\.md)", _README.read_text())) + present = {p.name for p in _DOCS.glob("*.md")} - exempt + + assert not (present - listed), f"docs pages missing from the README index: {sorted(present - listed)}" + + +def test_the_slug_rule_matches_githubs() -> None: + """Guard the slug helper itself — a wrong rule makes every test above vacuous. + + If `_slug` stopped stripping punctuation, every anchor would fail to match and + the suite would report dead links everywhere; if it over-stripped, nothing would + match and everything would pass. Either way the failure is in the helper the + other tests trust, so it needs assertions that do not depend on them. + + Deliberately literal rather than read from this repo's headings: the same file + is used across projects, and a fixture that names a particular page would make + it non-portable for no gain. + + The three cases are the punctuation classes that actually occur in headings — + an apostrophe, code backticks with parens, and a comma plus an em dash (which + leaves a double hyphen, matching GitHub). + """ + assert _slug("Registering a class you don't own") == "registering-a-class-you-dont-own" + assert _slug("`flow()` finishes the object") == "flow-finishes-the-object" + assert _slug("Bare, addressed, glob — the scoping model") == "bare-addressed-glob--the-scoping-model" + + +#: `https://github.com/Gearlux/recordstream/blob/main/` — a link from the README +#: back into this repository. Any `#anchor` is captured separately so it can be dropped. +_SELF_REPO_LINK = re.compile(r"https://github\.com/Gearlux/recordstream/blob/main/([^)\s#]+)") + + +def test_the_readme_links_to_repo_files_by_absolute_url() -> None: + """The README carries no relative link to a repo file. + + It is the PyPI landing page, and PyPI resolves `docs/storage.md` against + `pypi.org/docs/storage.md`, which does not exist. The reader gets a 404 from a + link that works perfectly on GitHub, so nothing in the repo can catch it — which + is why this rule is a test rather than a review habit. + + Same-page anchors (`#installation`) are fine: they resolve on the rendered page. + """ + relative = [t for t in _links(_README) if not t.startswith("#")] + + assert not relative, ( + f"README links to repo files relatively; PyPI cannot resolve these: {relative}. " + f"Use https://github.com/Gearlux/recordstream/blob/main/." + ) + + +def test_readme_links_into_this_repo_name_a_file_that_exists() -> None: + """An absolute self-link still has to point at something. + + Spelling the link absolutely takes it out of reach of + `test_relative_links_name_a_file_that_exists`, so without this rule the rename + check silently stops covering the README — trading a PyPI 404 for a GitHub one. + """ + missing = [ + target for target in _SELF_REPO_LINK.findall(_strip_code(_README.read_text())) if not (_REPO / target).exists() + ] + + assert not missing, f"README links to repo files that do not exist: {missing}" diff --git a/tests/test_enable.py b/tests/test_enable.py new file mode 100644 index 0000000..0191309 --- /dev/null +++ b/tests/test_enable.py @@ -0,0 +1,134 @@ +"""``Enable`` — the one-flag op-list toggle. + +Pins the toggle contract: the flag is the DECLARED ``enabled`` parameter (a settable +property), several wrappers are told apart by ``name`` (which scopes the CLI flag to +``--.enabled``), and the retired dynamic-toggle form (any boolean attribute name +becoming the flag) is rejected loudly instead of being silently ignored. +""" + +from typing import List + +import pytest +from confluid import accepts_broadcast, accepts_key, to_pydantic + +from recordstream import Record +from recordstream.ops.enable import Enable + + +def _tag(record: Record) -> Record: + return {**record, "seen": True} + + +class TestEnableToggle: + def test_default_is_enabled(self) -> None: + # Zero-config wrapper runs its ops — the toggle only ever has to be written to turn it OFF. + assert Enable(ops=[_tag])({"x": 1}) == {"x": 1, "seen": True} + + def test_disabled_passes_through(self) -> None: + op = Enable(ops=[_tag], enabled=False) + assert op({"x": 1}) == {"x": 1} + assert op.enabled is False + + def test_constructible_from_python_in_one_call(self) -> None: + # The whole wrapper — ops, toggle, name — comes from the constructor, so a + # generated tool/form/canvas call can build it without post-construction setattr. + op = Enable(ops=[_tag], enabled=False, name="visualize") + assert (op.name, op.enabled, op.ops) == ("visualize", False, [_tag]) + + def test_post_construction_setattr_is_the_yaml_path(self) -> None: + # What `enabled: false` in YAML does when confluid setattr's it after __init__. + op = Enable(ops=[_tag]) + op.enabled = False + assert op({"x": 1}) == {"x": 1} + op.enabled = True + assert op({"x": 1}) == {"x": 1, "seen": True} + + def test_named_wrappers_toggle_independently(self) -> None: + # `name` scopes the CLI flag (--overlay.enabled vs --labelstudio.enabled); here we + # simulate the post-override state the two addressed writes produce. + overlay = Enable(ops=[_tag], name="overlay", enabled=True) + labelstudio = Enable(ops=[_tag], name="labelstudio", enabled=False) + assert overlay({"x": 1}) == {"x": 1, "seen": True} + assert labelstudio({"x": 1}) == {"x": 1} + + def test_non_bool_toggle_raises(self) -> None: + # No silent truthiness: a quoted YAML bool / typo'd value fails at set time. + with pytest.raises(TypeError, match="must be a bool"): + Enable(ops=[_tag], enabled="true") # type: ignore[arg-type] + op = Enable(ops=[_tag]) + with pytest.raises(TypeError, match="must be a bool"): + op.enabled = 1 # type: ignore[assignment] + + +class TestIntrospectionContract: + """The toggle must be reachable from every front-end, not just YAML.""" + + def test_declared_parameters_are_the_schema(self) -> None: + # to_pydantic drives navigaitor's MCP/form schemas and StreamStudio's widgets — + # a toggle absent here is a toggle no GUI or tool call can set. + assert set(to_pydantic(Enable).model_fields) == {"ops", "enabled", "name"} + + @pytest.mark.parametrize("key", ["ops", "enabled", "name"]) + def test_keys_are_cli_settable(self, key: str) -> None: + # accepts_key gates liquifai's addressed `--.` writes; + # accepts_broadcast gates the bare `--` form. + assert accepts_key(Enable, key) is True + assert accepts_broadcast(Enable, key) is True + + def test_retired_dynamic_toggle_name_is_not_settable(self) -> None: + assert accepts_key(Enable, "visualize") is False + assert accepts_broadcast(Enable, "visualize") is False + + +class TestLazyValidation: + def test_empty_ops_raises_on_first_call(self) -> None: + op = Enable() # zero-arg construction stays valid + with pytest.raises(ValueError, match="non-empty 'ops'"): + op({"x": 1}) + + def test_stray_boolean_attribute_raises_with_migration_hint(self) -> None: + # The retired form (`visualize: false` as a bare YAML kwarg) lands as a + # post-construction attribute nothing reads — reject it instead of silently + # running the ops the user meant to gate. + op = Enable(ops=[_tag]) + op.visualize = False # type: ignore[attr-defined] # what the old YAML form produced + with pytest.raises(ValueError, match=r"unexpected boolean attribute\(s\) \['visualize'\]"): + op({"x": 1}) + + def test_migration_hint_names_the_replacement_spelling(self) -> None: + op = Enable(ops=[_tag]) + op.visualize = False # type: ignore[attr-defined] + with pytest.raises(ValueError) as excinfo: + op({"x": 1}) + message = str(excinfo.value) + assert "name: visualize" in message and "enabled:" in message + assert "--visualize.enabled" in message + + def test_validation_runs_once_then_stays_out_of_the_hot_path(self) -> None: + op = Enable(ops=[_tag]) + assert op({"x": 1}) == {"x": 1, "seen": True} + # A stray attribute set AFTER the first record is not re-scanned per record. + op.visualize = False # type: ignore[attr-defined] + assert op({"x": 2}) == {"x": 2, "seen": True} + + +class TestOpsChain: + def test_ops_run_in_sequence(self) -> None: + def _bump(record: Record) -> Record: + return {**record, "x": record["x"] + 1} + + op = Enable(ops=[_bump, _bump, _tag]) + assert op({"x": 1}) == {"x": 3, "seen": True} + + def test_close_propagates_to_inner_ops(self) -> None: + closed: List[str] = [] + + class _Sink: + def __call__(self, record: Record) -> Record: + return record + + def close(self) -> None: + closed.append("sink") + + Enable(ops=[_Sink()]).close() + assert closed == ["sink"] diff --git a/tests/test_entrypoint.py b/tests/test_entrypoint.py new file mode 100644 index 0000000..7f46a31 --- /dev/null +++ b/tests/test_entrypoint.py @@ -0,0 +1,151 @@ +"""Tests for the runnable entry-point marker (entrypoint / runnable_entrypoints / run_entrypoint).""" + +import pytest + +from recordstream.runnable import entrypoint, entrypoint_tasks, run_entrypoint, runnable_entrypoints + + +class _Runnable: + @entrypoint("fit", role="trainer") + def fit(self) -> None: ... + + @entrypoint("evaluate", role="evaluator") + def evaluate(self) -> None: ... + + @entrypoint("test", role="evaluator", primary=True) + def test(self) -> None: ... + + @entrypoint("predict", role="predictor") + def predict(self) -> None: ... + + def not_an_entrypoint(self) -> None: ... + + +def test_runnable_entrypoints_finds_all_marked_methods() -> None: + eps = runnable_entrypoints(_Runnable) + assert set(eps) == {"fit", "evaluate", "test", "predict"} + assert eps["fit"] == {"task": "fit", "role": "trainer", "primary": False} + assert eps["test"]["primary"] is True + assert "not_an_entrypoint" not in eps + + +def test_entrypoint_tasks_by_role_primary_first() -> None: + assert entrypoint_tasks(_Runnable, "trainer") == ["fit"] + # test is primary → sorts first among evaluators. + assert entrypoint_tasks(_Runnable, "evaluator") == ["test", "evaluate"] + assert entrypoint_tasks(_Runnable, "predictor") == ["predict"] + assert entrypoint_tasks(_Runnable, "nonexistent") == [] + + +def test_entrypoints_are_inherited_via_mro() -> None: + class _Sub(_Runnable): + @entrypoint("fit", role="trainer") + def fit(self) -> None: ... # override keeps the marker + + eps = runnable_entrypoints(_Sub) + assert set(eps) == {"fit", "evaluate", "test", "predict"} + + +def test_marker_does_not_break_calling_the_method() -> None: + calls = [] + + class _R: + @entrypoint("fit", role="trainer") + def fit(self) -> None: + calls.append("fit") + + _R().fit() + assert calls == ["fit"] + + +# ---- run_entrypoint: the markers ARE the dispatch table ---------------------- # + + +class _Dispatching: + """The merged train+eval shape: one ``task`` knob, ``run()`` dispatching off the markers.""" + + def __init__(self, task: str = "fit") -> None: + self.task = task + self.calls: list = [] + + def run(self) -> object: + return run_entrypoint(self, self.task) + + @entrypoint("fit", role="trainer", primary=True) + def fit(self) -> str: + self.calls.append("fit") + return "fitted" + + @entrypoint("evaluate", role="evaluator") + def evaluate(self) -> None: + self.calls.append("evaluate") + + @entrypoint("test", role="evaluator", primary=True) + def test(self) -> None: + self.calls.append("test") + + @entrypoint("predict", role="predictor", primary=True) + def predict(self) -> None: + self.calls.append("predict") + + +@pytest.mark.parametrize("task", ["fit", "evaluate", "test", "predict"]) +def test_run_entrypoint_calls_the_method_declaring_the_task(task: str) -> None: + runnable = _Dispatching(task=task) + runnable.run() + assert runnable.calls == [task] + + +def test_run_entrypoint_returns_the_method_result() -> None: + assert _Dispatching(task="fit").run() == "fitted" + + +def test_run_entrypoint_rejects_an_unknown_task_listing_the_declared_ones() -> None: + runnable = _Dispatching(task="export") + with pytest.raises(ValueError) as excinfo: + runnable.run() + message = str(excinfo.value) + assert "Unknown task 'export'" in message + # Declaration order, so the list reads as the class's capability list. + assert "['fit', 'evaluate', 'test', 'predict']" in message + assert runnable.calls == [] + + +def test_a_new_entrypoint_dispatches_with_no_other_change() -> None: + """The regression a hand-written ``{task: method}`` dict allowed: marker added, dict forgotten.""" + + class _WithExport(_Dispatching): + @entrypoint("export", role="exporter", primary=True) + def export(self) -> None: + self.calls.append("export") + + # The config generator pins `task:` from the markers — dispatch must agree with it. + assert entrypoint_tasks(_WithExport, "exporter") == ["export"] + runnable = _WithExport(task="export") + runnable.run() + assert runnable.calls == ["export"] + + +def test_run_entrypoint_dispatches_to_a_subclass_override() -> None: + class _Override(_Dispatching): + @entrypoint("fit", role="trainer", primary=True) + def fit(self) -> str: + self.calls.append("fit-override") + return "overridden" + + runnable = _Override(task="fit") + assert runnable.run() == "overridden" + assert runnable.calls == ["fit-override"] + + +def test_run_entrypoint_never_fires_a_property_getter() -> None: + """The merged runnables carry a dynamic ``__needs_autograd__`` property; lookup must not touch it.""" + + class _WithProperty(_Dispatching): + @property + def __needs_autograd__(self) -> bool: + raise AssertionError("property getter fired during dispatch") + + runnable = _WithProperty(task="test") + runnable.run() + assert runnable.calls == ["test"] diff --git a/tests/test_flux.py b/tests/test_flux.py deleted file mode 100644 index 5a9f0a4..0000000 --- a/tests/test_flux.py +++ /dev/null @@ -1,263 +0,0 @@ -from typing import Any - -import numpy as np -import pytest - -from dataflux.core import Flux -from dataflux.sample import Sample - - -def test_basic_flux() -> None: - source = [np.array([1, 2, 3]), np.array([4, 5, 6])] - pipeline = Flux(source) - results = list(pipeline) - assert len(results) == 2 - assert isinstance(results[0], Sample) - assert np.array_equal(results[0].input, source[0]) - - -def double_it(x: np.ndarray) -> np.ndarray: - return x * 2 - - -def is_greater_than_two(s: Sample) -> bool: - # Use np.any() or similar to ensure a single boolean is returned for mypy - return bool(np.any(s.input > 2)) - - -def test_flux_map() -> None: - source = [np.array([1, 2, 3])] - pipeline = Flux(source).map(double_it) - results = list(pipeline) - assert np.array_equal(results[0].input, np.array([2, 4, 6])) - - -def test_flux_filter() -> None: - source = [np.array([1]), np.array([2]), np.array([3]), np.array([4])] - pipeline = Flux(source).filter(is_greater_than_two) - results = list(pipeline) - assert len(results) == 2 - assert results[0].input == 3 - - -class MockSink: - def __init__(self) -> None: - self.written: list[Sample] = [] - - def write(self, sample: Sample) -> None: - self.written.append(sample) - - def flush(self) -> None: - pass - - -def test_flux_to_sink() -> None: - source = [np.array([1, 2]), np.array([3, 4])] - sink = MockSink() - Flux(source).to_sink(sink) - assert len(sink.written) == 2 - assert np.array_equal(sink.written[0].input, source[0]) - - -def full_transform(s: Sample) -> Sample: - return s._replace(input=s.input * 2) - - -def test_wrapped_op_all() -> None: - source = [np.array([10])] - pipeline = Flux(source).map(full_transform, select="all") - results = pipeline.collect() - assert results[0].input == 20 - - -def test_filter_op() -> None: - from dataflux.core import FilterOp - - op = FilterOp(lambda s: bool(s.input > 5)) - s1 = Sample(input=10) - s2 = Sample(input=2) - assert op(s1) == s1 - assert op(s2) is None - - -def fail_op(x: Any) -> Any: - raise ValueError("Intentional failure") - - -def test_wrapped_op_error() -> None: - source = [np.array([1])] - pipeline = Flux(source).map(fail_op) - with pytest.raises(ValueError, match="Intentional failure"): - pipeline.collect() - - -def target_transform(t: Any) -> Any: - return t + 10 - - -def test_wrapped_op_target() -> None: - source = [Sample(input=1, target=5)] - # select="target" hits lines 69-70 - pipeline = Flux(source).map(target_transform, select="target") - results = pipeline.collect() - assert results[0].target == 15 - - -def test_wrapped_op_fallback() -> None: - # select="unknown" hits line 73 - source = [Sample(input=1)] - pipeline = Flux(source).map(lambda x: x, select="unknown") - results = pipeline.collect() - assert results[0].input == 1 - - -def test_worker_task_none() -> None: - from dataflux.core import _worker_task - - # hits line 83 by using two ops, first returning None - assert _worker_task(Sample(input=1), [lambda s: None, lambda s: s]) is None - - -def test_flux_from_source() -> None: - # hits from_source classmethod - f = Flux.from_source([1, 2, 3]) - assert len(f) == 3 - - -def test_flux_len_fallback() -> None: - # hits line 134 (len 0 if no source or no len) - f = Flux() - assert len(f) == 0 - f2 = Flux(iter([1, 2])) # iter doesn't have len - assert len(f2) == 0 - - -# --- Indexed access (__getitem__) --- - - -def test_getitem_basic() -> None: - source = [np.array([10]), np.array([20]), np.array([30])] - flux = Flux(source) - sample = flux[1] - assert isinstance(sample, Sample) - assert np.array_equal(sample.input, np.array([20])) - - -def test_getitem_with_ops() -> None: - source = [np.array([1]), np.array([2]), np.array([3])] - flux = Flux(source).map(double_it) - assert np.array_equal(flux[0].input, np.array([2])) - assert np.array_equal(flux[2].input, np.array([6])) - - -def test_getitem_no_source() -> None: - flux = Flux() - with pytest.raises(TypeError): - flux[0] - - -def test_getitem_non_indexable_source() -> None: - flux = Flux(iter([1, 2, 3])) # iterators don't support indexing - with pytest.raises(TypeError): - flux[0] - - -def test_getitem_out_of_range() -> None: - source = [np.array([1])] - flux = Flux(source) - with pytest.raises(IndexError): - flux[5] - - -class _IterableWithLen: - """Iterable source with ``__iter__`` + ``__len__`` but NO ``__getitem__``. - - Mirrors the shape of ``waivefront.regions_source.RegionsJsonSource``. - Tracks how many times ``__iter__`` is invoked so tests can prove Flux - materializes only once per Flux lifetime. - """ - - def __init__(self, items: list) -> None: - self._items = items - self.iter_calls = 0 - - def __iter__(self) -> Any: - self.iter_calls += 1 - return iter(self._items) - - def __len__(self) -> int: - return len(self._items) - - -def test_getitem_iterable_with_len_materializes_once() -> None: - """Flux caches iterable-only sources on first __getitem__ and reuses the cache.""" - source = _IterableWithLen([np.array([1]), np.array([2]), np.array([3])]) - flux = Flux(source) - - # Cache is empty until first random access. - assert flux._indexable_cache is None - assert source.iter_calls == 0 - - s0 = flux[0] - s1 = flux[1] - s2 = flux[2] - - assert np.array_equal(s0.input, np.array([1])) - assert np.array_equal(s1.input, np.array([2])) - assert np.array_equal(s2.input, np.array([3])) - # The source was iterated exactly once across the three random-access calls. - assert source.iter_calls == 1 - assert flux._indexable_cache is not None - assert len(flux._indexable_cache) == 3 - - -def test_getitem_indexable_source_does_not_populate_cache() -> None: - """List-backed (indexable) sources take the fast path and don't trigger the cache.""" - source = [np.array([1]), np.array([2])] - flux = Flux(source) - _ = flux[0] - _ = flux[1] - # Fast path: cache is never populated. - assert flux._indexable_cache is None - - -def test_getitem_on_deferred_fluid_source_raises_actionable_error() -> None: - """When the user hands Flux a still-deferred Confluid Class marker, indexing - must raise with a human-readable hint — not the cryptic ``num_samples=0`` - or ``does not support indexing``. - """ - from confluid import Class - - class Dummy: - def __init__(self, x: int = 0) -> None: - self.x = x - - # Deliberately wrong type — this test verifies Flux raises a clear error - # when handed a still-deferred Confluid marker as the source. - flux = Flux(Class(Dummy, x=1)) # type: ignore[arg-type] - with pytest.raises(TypeError, match="deferred Confluid marker"): - flux[0] - with pytest.raises(TypeError, match="deferred Confluid marker"): - len(flux) - - -def test_deferred_fluid_op_raises_actionable_error() -> None: - """Same guard applies to ops: a still-deferred Class in ops[] surfaces a - clear message pointing the YAML author at the `!class:X()` fix instead of - letting DataLoader workers report `'Class' object is not callable`. - """ - from confluid import Class - - class DummyOp: - def __init__(self, factor: int = 1) -> None: - self.factor = factor - - def __call__(self, sample: Sample) -> Sample: - return sample - - source = [np.array([1]), np.array([2])] - flux = Flux(source, ops=[Class(DummyOp, factor=2)]) - with pytest.raises(TypeError, match=r"Flux\.ops\[0\] is still a deferred Confluid marker"): - flux[0] - with pytest.raises(TypeError, match=r"Flux\.ops\[0\] is still a deferred Confluid marker"): - list(flux) diff --git a/tests/test_fork_safety.py b/tests/test_fork_safety.py new file mode 100644 index 0000000..249cdb7 --- /dev/null +++ b/tests/test_fork_safety.py @@ -0,0 +1,158 @@ +"""The two fork hazards on the DataLoader path, and the guards that remove them. + +A ``DataLoader`` worker may be a FORKED child — torch's default start method on Linux, and what +you get on macOS whenever something in the process has set ``fork`` (fastai does, at import). A +forked child inherits the parent's memory but NOT its threads, so anything holding a thread pool +or a system handle across the fork can die there. When it does it dies with a **SIGSEGV and no +Python traceback**, surfacing only as ``DataLoader worker exited unexpectedly`` — which is why +both guards are pinned here rather than left to be re-diagnosed. + +The two are INDEPENDENT and neither fixes the other: + +* :func:`~recordstream.ensure_materialized` — a lazy source built in the child runs its download + through ``_scproxy`` / CoreFoundation, which is not fork-safe. +* ``cv2.setNumThreads(0)`` — OpenCV's thread pool, inherited across a fork, crashes the child. + albumentations runs on OpenCV, so any ops list containing one is exposed. + +The subprocess tests are the ones that matter: they reproduce the real crash, so removing a guard +makes them fail rather than making them pass differently. They are subprocesses because a SIGSEGV +takes the whole interpreter down — an in-process test could not report it. +""" + +import subprocess +import sys +import textwrap +from typing import Any + +import numpy as np +import pytest + +from recordstream import Image, Mask, ensure_materialized + +albumentations = pytest.importorskip("albumentations") +pytest.importorskip("torch") + + +def _run(body: str) -> subprocess.CompletedProcess: + """Run ``body`` in a fresh interpreter that forks its DataLoader workers.""" + script = ( + textwrap.dedent( + """ + import multiprocessing as mp + mp.set_start_method("fork", force=True) # what fastai does at import + import numpy as np, torch + from torch.utils.data import DataLoader + from recordstream import Image, Mask, collate_records + from recordstream.core import _apply_op + import albumentations as A + """ + ) + + textwrap.dedent(body) + ) + return subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, timeout=300) + + +# --------------------------------------------------------------------------- # +# cv2's thread pool +# --------------------------------------------------------------------------- # +class TestOpenCVThreadingIsDisabled: + def test_applying_an_albumentations_op_turns_the_pool_off(self, monkeypatch: pytest.MonkeyPatch) -> None: + """The guard fires on USE, not on import — a process that never touches albumentations + must not have its OpenCV settings changed by importing a data library. + + The module flag is reset first because the guard is deliberately ONE-TIME (its cost is + then a single bool check per op); by the time this runs, another test has usually already + tripped it. + """ + import cv2 + + from recordstream.core import _apply_op, families + + monkeypatch.setattr(families, "_CV2_THREADING_DISABLED", False) + record = { + "image": Image(np.zeros((8, 8, 3), np.uint8)), + "mask": Mask(np.zeros((8, 8), np.int64)), + } + + _apply_op(record, albumentations.Resize(height=4, width=4)) + # `setNumThreads(0)` means "no pool — run on the calling thread", and cv2 then REPORTS 1 + # rather than 0 (measured). Asserting 0 would look right and always fail. + assert cv2.getNumThreads() == 1 + + def test_the_guard_runs_only_once(self, monkeypatch: pytest.MonkeyPatch) -> None: + """So the cost of the guarantee is one bool check per op, not a cv2 call per record.""" + from recordstream.core import _apply_op, families + + monkeypatch.setattr(families, "_CV2_THREADING_DISABLED", False) + calls: list = [] + monkeypatch.setattr(families, "_disable_cv2_threading", lambda: calls.append(1)) + + record = {"image": Image(np.zeros((8, 8, 3), np.uint8))} + resize = albumentations.Resize(height=4, width=4) + for _ in range(3): + _apply_op(record, resize) + # The invoker calls it every time; the FUNCTION is what short-circuits, so the guarantee + # holds for a record that arrives after something else re-enabled the pool. + assert len(calls) == 3 + + def test_a_forked_worker_survives_an_albumentations_op(self) -> None: + """The real crash, reproduced. Without the guard this dies with + *"DataLoader worker ... is killed by signal: Segmentation fault: 11"*. + """ + result = _run( + """ + records = [{"image": Image(np.zeros((32, 32, 3), np.uint8)), + "mask": Mask(np.zeros((32, 32), np.int64))} for _ in range(8)] + resize = A.Resize(height=16, width=16) + # The PARENT applies one first — which is what creates the thread pool that the + # forked child would inherit. Without that this passes vacuously. + _apply_op(records[0], resize) + + class _DS(torch.utils.data.Dataset): + def __len__(self): return len(records) + def __getitem__(self, i): return _apply_op(records[i], resize) + + loader = DataLoader(_DS(), batch_size=2, num_workers=2, collate_fn=collate_records) + n = sum(1 for _ in loader) + print("BATCHES", n) + """ + ) + assert result.returncode == 0, f"forked worker died:\n{result.stderr[-2000:]}" + assert "BATCHES 4" in result.stdout + + +# --------------------------------------------------------------------------- # +# a lazily-built source +# --------------------------------------------------------------------------- # +class TestEnsureMaterialized: + def test_it_reads_one_whole_record(self) -> None: + """A whole RECORD, not a cheaper question: loading a dataset object is not the same as + building everything a read needs, so `len()` was measured NOT to be enough.""" + reads: list = [] + + class _Lazy: + def __getitem__(self, index: int) -> dict: + reads.append(index) + return {"image": Image(np.zeros((4, 4, 3), np.uint8))} + + def __len__(self) -> int: + return 4 + + source = _Lazy() + assert ensure_materialized(source) is source, "it must return the source, so it composes" + assert reads == [0] + + def test_an_empty_source_is_not_an_error(self) -> None: + """A split that happens to have no rows has nothing to warm, and a caller should not + need a guard for it.""" + assert ensure_materialized([]) == [] + + def test_an_iterable_only_source_is_warmed_too(self) -> None: + consumed = [] + + def _gen() -> Any: + consumed.append(1) + yield {"image": Image(np.zeros((4, 4, 3), np.uint8))} + + ensure_materialized(_gen()) + assert consumed == [1] diff --git a/tests/test_hf_core.py b/tests/test_hf_core.py deleted file mode 100644 index 339d5c7..0000000 --- a/tests/test_hf_core.py +++ /dev/null @@ -1,38 +0,0 @@ -from typing import Any, Dict - -from dataflux.hf_core import HFFlux -from dataflux.sample import Sample - - -def test_hfflux_from_list() -> None: - raw_data = [ - Sample(input="data1", target=0, metadata={"id": "s1"}), - Sample(input="data2", target=1, metadata={"id": "s2"}), - ] - flux = HFFlux.from_source(raw_data) - assert len(flux) == 2 - assert flux[0].input == "data1" - assert flux[0].metadata["id"] == "s1" - - -def test_hfflux_map() -> None: - raw_data = [Sample(input=1), Sample(input=2)] - flux = HFFlux.from_source(raw_data) - - # HF style map expects a dict and returns a dict - def double(example: Dict[str, Any]) -> Dict[str, Any]: - example["input"] = example["input"] * 2 - return example - - mapped = flux.map(double) - assert mapped[0].input == 2 - assert mapped[1].input == 4 - - -def test_hfflux_filter() -> None: - raw_data = [Sample(input=1), Sample(input=2), Sample(input=3)] - flux = HFFlux.from_source(raw_data) - - filtered = flux.filter(lambda x: x["input"] > 1) - assert len(filtered) == 2 - assert filtered[0].input == 2 diff --git a/tests/test_huggingface_source.py b/tests/test_huggingface_source.py new file mode 100644 index 0000000..51ce8e4 --- /dev/null +++ b/tests/test_huggingface_source.py @@ -0,0 +1,27 @@ +"""HuggingFaceSource unit pins that need no Hub access (the dataset cache is stubbed).""" + +from recordstream.sources.huggingface import HuggingFaceSource + + +def _with_rows(n: int, count: int = 0) -> HuggingFaceSource: + source = HuggingFaceSource(path="stub/dataset", count=count) + source._dataset = list(range(n)) # the lazy cache — a list satisfies len()/iteration + return source + + +def test_len_reports_the_dataset_when_count_is_unset() -> None: + """``count`` of 0 (or None) means "all records" — a bare ``self.count`` would report 0 and + make the source look empty to a len()-based consumer.""" + assert len(_with_rows(29)) == 29 + + +def test_len_reports_count_when_it_caps() -> None: + assert len(_with_rows(29, count=10)) == 10 + + +def test_len_is_clamped_when_count_exceeds_the_split() -> None: + """A ``count`` larger than the split must never be reported verbatim: a map-style consumer + trusts ``len()`` for its index space, so a lying length surfaces as an ``IndexError`` deep + inside a DataLoader worker, one epoch in (found the hard way — ``count: 32`` over cppe-5's + 29-row test split killed the first validation pass).""" + assert len(_with_rows(29, count=32)) == 29 diff --git a/tests/test_intake.py b/tests/test_intake.py deleted file mode 100644 index 2e65aa7..0000000 --- a/tests/test_intake.py +++ /dev/null @@ -1,156 +0,0 @@ -"""Tests for the generic intake → DataFlux adapter.""" - -from pathlib import Path -from typing import Any - -import confluid -import intake -import intake.source.base -import numpy as np -import pytest -import torch -import xarray as xr - -from dataflux.sample import Sample -from dataflux.storage.intake import IntakeSource - - -class _XArrayPartitionedSource(intake.source.base.DataSource): - """Minimal in-process intake DataSource for testing. - - Yields ``n_partitions`` tiny 1-D xarray.DataArray objects with attrs. - """ - - container = "xarray" - name = "test_xarray_partitioned" - version = "0.1" - partition_access = True - - def __init__(self, n_partitions: int = 3, label: str = "alpha", metadata: Any = None) -> None: - super().__init__(metadata=metadata or {}) - self._n = n_partitions - self._label = label - - def _get_schema(self) -> Any: - return intake.source.base.Schema( - datashape=None, - dtype=str(np.dtype(np.float32)), - shape=(8,), - npartitions=self._n, - extra_metadata={}, - ) - - def _get_partition(self, i: int) -> xr.DataArray: - arr = np.arange(8, dtype=np.float32) + (i * 100) - return xr.DataArray( - arr, - dims=["x"], - attrs={"label": self._label, "partition_index": i, "samplerate": 100.0}, - ) - - def _close(self) -> None: - pass - - -def test_intake_source_iterates_xarray_partitions_as_samples() -> None: - src = _XArrayPartitionedSource(n_partitions=4) - df_src = IntakeSource(source=src) - samples = list(df_src) - assert len(samples) == 4 - assert all(isinstance(s, Sample) for s in samples) - assert all(isinstance(s.input, torch.Tensor) for s in samples) - assert all(s.input.shape == (8,) for s in samples) - # Metadata propagates from xarray attrs. - assert samples[0].metadata["label"] == "alpha" - assert samples[0].metadata["samplerate"] == 100.0 - assert samples[2].metadata["partition_index"] == 2 - - -def test_intake_source_uses_target_attr() -> None: - src = _XArrayPartitionedSource(n_partitions=2, label="bravo") - df_src = IntakeSource(source=src, target_attr="label") - samples = list(df_src) - assert all(s.target == "bravo" for s in samples) - - -def test_intake_source_target_none_when_attr_missing() -> None: - src = _XArrayPartitionedSource(n_partitions=1) - df_src = IntakeSource(source=src, target_attr="not_a_real_attr") - s = next(iter(df_src)) - assert s.target is None - # The metadata still includes the actual attrs; it's only target lookup that misses. - assert "label" in s.metadata - - -def test_intake_source_len_matches_npartitions() -> None: - src = _XArrayPartitionedSource(n_partitions=7) - df_src = IntakeSource(source=src) - assert len(df_src) == 7 - - -def test_intake_source_handles_numpy_partition() -> None: - class NDArraySource(intake.source.base.DataSource): - container = "ndarray" - name = "test_ndarray" - version = "0.1" - partition_access = True - - def _get_schema(self) -> Any: - return intake.source.base.Schema( - datashape=None, - dtype=str(np.dtype(np.float64)), - shape=(3,), - npartitions=2, - extra_metadata={}, - ) - - def _get_partition(self, i: int) -> np.ndarray: - return np.array([i, i + 1, i + 2], dtype=np.float64) - - def _close(self) -> None: - pass - - df_src = IntakeSource(source=NDArraySource()) - samples = list(df_src) - assert len(samples) == 2 - assert isinstance(samples[0].input, torch.Tensor) - assert samples[0].input.tolist() == [0.0, 1.0, 2.0] - assert samples[0].metadata == {} - - -def test_intake_source_requires_either_catalog_or_source() -> None: - with pytest.raises(ValueError, match="catalog_path"): - IntakeSource() - - -def test_intake_source_rejects_both_catalog_and_source() -> None: - src = _XArrayPartitionedSource() - with pytest.raises(ValueError, match="not both"): - IntakeSource(catalog_path="a.yml", source_name="x", source=src) - - -def test_intake_source_partial_pair_raises() -> None: - with pytest.raises(ValueError): - IntakeSource(catalog_path="a.yml") - with pytest.raises(ValueError): - IntakeSource(source_name="x") - - -def test_intake_source_configurable_yaml_roundtrip() -> None: - df_src = IntakeSource(catalog_path="cat.yml", source_name="entry", target_attr="label") - state = confluid.dump(df_src) - restored = confluid.load(state) - assert isinstance(restored, IntakeSource) - assert restored.catalog_path == "cat.yml" - assert restored.source_name == "entry" - assert restored.target_attr == "label" - - -def test_intake_source_reset_after_close(tmp_path: Path) -> None: - src = _XArrayPartitionedSource(n_partitions=2) - df_src = IntakeSource(source=src) - samples_a = list(df_src) - df_src.close() - # User-supplied source is preserved; iterating again still works. - samples_b = list(df_src) - assert len(samples_a) == len(samples_b) == 2 diff --git a/tests/test_io.py b/tests/test_io.py new file mode 100644 index 0000000..615f99a --- /dev/null +++ b/tests/test_io.py @@ -0,0 +1,115 @@ +"""The item codec registry (``recordstream.io``) — default structural codec, overrides, records, +the ``"plain"`` codec path.""" + +from dataclasses import dataclass + +import numpy as np +import pytest + +from recordstream import ( + Boxes, + EncodedItem, + Image, + Label, + decode_item, + decode_record, + encode_item, + encode_record, + register_io, + register_item, +) +from recordstream.io import PLAIN_TYPE + + +@register_item +@dataclass +class _IoBlob: + """A data-bearing wrapper (the shape a domain signal item takes).""" + + data: object = None + rate: float = 1.0 + + +class TestDefaultCodec: + def test_array_item_round_trip(self) -> None: + img = Image(np.arange(12, dtype=np.float32).reshape(2, 2, 3), layout="CHW") + enc = encode_item(img) + assert enc.type_name == "Image" and enc.attrs == {"layout": "CHW"} + back = decode_item(enc) + assert isinstance(back, Image) and back.layout == "CHW" + assert np.array_equal(np.asarray(back), np.asarray(img)) + + def test_wrapper_item_round_trip(self) -> None: + blob = _IoBlob(np.ones(4), rate=48000.0) + enc = encode_item(blob) + assert enc.type_name == "_IoBlob" and enc.attrs == {"rate": 48000.0} + assert np.array_equal(enc.payload, np.ones(4)) + back = decode_item(enc) + assert isinstance(back, _IoBlob) and back.rate == 48000.0 + + def test_payloadless_item_round_trip(self) -> None: + lab = Label("drone", classes=["a", "drone"]) + enc = encode_item(lab) + assert enc.payload is None and enc.attrs == {"value": "drone", "classes": ["a", "drone"]} + back = decode_item(enc) + assert isinstance(back, Label) and back.value == "drone" and back.classes == ["a", "drone"] + + def test_unknown_type_name_raises(self) -> None: + with pytest.raises(KeyError, match="no item type registered"): + decode_item(EncodedItem(type_name="Nope", payload=None, attrs={})) + + +class TestPlainCodec: + def test_scalar_round_trips_verbatim(self) -> None: + for value in (-3.0, 7, "text", True): + enc = encode_item(value) + assert enc.type_name == PLAIN_TYPE and enc.payload == value and enc.attrs == {} + assert decode_item(enc) == value + + def test_bare_array_round_trips_verbatim(self) -> None: + arr = np.arange(6).reshape(2, 3) # a bare ndarray is NOT a registered item -> "plain" + enc = encode_item(arr) + assert enc.type_name == PLAIN_TYPE + back = decode_item(enc) + assert type(back) is np.ndarray and np.array_equal(back, arr) + + def test_none_is_plain(self) -> None: + enc = encode_item(None) + assert enc.type_name == PLAIN_TYPE and decode_item(enc) is None + + +class TestRegisteredCodec: + def test_override_wins_and_round_trips(self) -> None: + @register_item + class Compact: # a type the default codec cannot capture + def __init__(self, values: list) -> None: + self.values = values + + register_io( + Compact, + encode=lambda item: (np.asarray(item.values), {}), + decode=lambda payload, attrs: Compact(list(np.asarray(payload))), + ) + enc = encode_item(Compact([1, 2, 3])) + assert enc.type_name == "Compact" and np.array_equal(enc.payload, [1, 2, 3]) + back = decode_item(enc) + assert isinstance(back, Compact) and back.values == [1, 2, 3] + + +class TestRecordCodec: + def test_record_round_trip_keys_order_and_plain_entries(self) -> None: + record = { + "image": Image(np.zeros((2, 2, 3), dtype=np.float32)), + "regions": Boxes(boxes=[[0, 0, 1, 1]], labels=["a"], canvas=(2, 2)), + "class": Label("x"), + "gain_db": -3.0, # a plain scalar rides the same layout under the "plain" tag + } + fields = encode_record(record) + assert [f.key for f in fields] == ["image", "regions", "class", "gain_db"] + assert fields[3].item.type_name == PLAIN_TYPE + back = decode_record(fields) + assert list(back.keys()) == list(record.keys()) + assert np.array_equal(np.asarray(back["image"]), np.asarray(record["image"])) + assert back["regions"] == record["regions"] + assert back["class"] == record["class"] + assert back["gain_db"] == -3.0 diff --git a/tests/test_items.py b/tests/test_items.py new file mode 100644 index 0000000..6a5cff7 --- /dev/null +++ b/tests/test_items.py @@ -0,0 +1,200 @@ +"""Typed items — array-subclass attribute preservation, wrappers, payload accessors, registry. + +Only the MODALITY-NEUTRAL core items live in recordstream (Image / Mask / Boxes / Label). The +data-bearing-wrapper and multi-attribute-array paths (which the signal-domain items in a domain +package exercise for real) are covered here with small test-local item types, so the core stays +tested without importing a domain package. +""" + +from dataclasses import dataclass + +import numpy as np +import pytest + +from recordstream.items import ( + Boxes, + Image, + Label, + Mask, + NDArrayItem, + get_item_type, + is_item, + item_data, + item_type_names, + item_types, + register_item, + with_data, +) + + +@register_item +@dataclass +class _Blob: + """A test-local data-bearing wrapper item (the shape a signal item takes).""" + + data: object = None + tag: str = "x" + + +class _Multi(NDArrayItem): + """A test-local array item with two extra attributes (a spectrogram-like shape).""" + + _item_attrs = ("a", "b") + a: int = 1 + b: object = None + + +class TestArrayItems: + def test_default_and_explicit_attr(self) -> None: + assert Image(np.zeros((2, 3, 3))).layout == "HWC" + assert Image(np.zeros((3, 2, 3)), layout="CHW").layout == "CHW" + + def test_attr_survives_numpy_ops(self) -> None: + img = Image(np.arange(2 * 3 * 3).reshape(2, 3, 3), layout="HWC") + flipped = np.flip(img, axis=1) + assert isinstance(flipped, Image) and flipped.layout == "HWC" + doubled = img * 2 + assert isinstance(doubled, Image) and doubled.layout == "HWC" + assert isinstance(img[0], Image) # slicing keeps the subclass + attr + + def test_multiple_attrs_survive_ufunc(self) -> None: + item = _Multi(np.zeros((4, 8)), a=7, b={"n": 8}) + assert item.a == 7 and item.b == {"n": 8} + shifted = item + 1 # attrs carried through the ufunc + assert isinstance(shifted, _Multi) and shifted.a == 7 and shifted.b == {"n": 8} + + def test_unknown_attr_rejected(self) -> None: + with pytest.raises(TypeError, match="unexpected attributes"): + Image(np.zeros((2, 2, 3)), colorspace="rgb") + + def test_mask_has_no_extra_attrs(self) -> None: + assert isinstance(Mask(np.zeros((4, 4))), NDArrayItem) + + +class TestWrapperItems: + def test_wrapper_fields(self) -> None: + blob = _Blob(np.ones(8), tag="sig") + assert blob.tag == "sig" and np.asarray(blob.data).sum() == 8 + + def test_zero_arg_construction(self) -> None: + # Wrappers build with no args (fields defaulted) — the workspace lazy/zero-arg convention. + assert _Blob().data is None and Boxes().boxes == [] and Label().value is None + + +class TestPayloadAccessors: + def test_item_data_array(self) -> None: + img = Image(np.arange(4).reshape(2, 2)) + data = item_data(img) + assert type(data) is np.ndarray and np.array_equal(data, [[0, 1], [2, 3]]) + + def test_item_data_wrapper(self) -> None: + assert np.array_equal(item_data(_Blob(np.ones(3))), np.ones(3)) + + def test_item_data_no_payload_returns_self(self) -> None: + reg = Boxes(boxes=[[0, 0, 1, 1]]) + assert item_data(reg) is reg # no `.data` slot — returns the item + + def test_item_data_plain_value_passes_through(self) -> None: + assert item_data(3.5) == 3.5 and item_data("s") == "s" # non-items pass through verbatim + + def test_with_data_array_preserves_attrs(self) -> None: + img = Image(np.zeros((2, 2, 3)), layout="CHW") + rebuilt = with_data(img, np.ones((2, 2, 3))) + assert isinstance(rebuilt, Image) and rebuilt.layout == "CHW" and rebuilt.sum() == 12 + + def test_with_data_wrapper_preserves_meta(self) -> None: + rebuilt = with_data(_Blob(np.zeros(4), tag="t"), np.ones(4)) + assert isinstance(rebuilt, _Blob) and rebuilt.tag == "t" and np.asarray(rebuilt.data).sum() == 4 + + def test_with_data_without_payload_raises(self) -> None: + with pytest.raises(TypeError, match="no payload slot"): + with_data(Boxes(boxes=[]), [[0, 0, 1, 1]]) + + +class TestRegistry: + def test_builtins_registered(self) -> None: + names = item_type_names() + for name in ("Image", "Mask", "Boxes", "Label"): + assert name in names + assert Image in item_types() + + def test_get_item_type_and_miss(self) -> None: + assert get_item_type("Image") is Image + with pytest.raises(KeyError, match="no item type registered"): + get_item_type("Nope") + + def test_is_item(self) -> None: + assert is_item(Image(np.zeros((1, 1, 3)))) and is_item(Label()) + assert not is_item(np.zeros((2, 2))) and not is_item(42) + + def test_register_custom_type(self) -> None: + @register_item + class Keypoints: # a user type — one class + one decorator, no core edit + def __init__(self, points: list) -> None: + self.points = points + + assert "Keypoints" in item_type_names() and get_item_type("Keypoints") is Keypoints + + +class TestResolveEntry: + """resolve_entry/resolve_item — THE field-or-first record-key resolution.""" + + def _record(self) -> dict: + import numpy as np + + from recordstream import Image, Label + + return {"class": Label("cat"), "picture": Image(np.zeros((4, 6, 3), dtype=np.uint8)), "samplerate": 1.0} + + def test_explicit_field_returns_key_and_value(self) -> None: + from recordstream import Image, resolve_entry + + key, item = resolve_entry(self._record(), "picture", Image, owner="Op", param="image_field") + assert key == "picture" and isinstance(item, Image) + + def test_blank_field_falls_back_to_first_of_type(self) -> None: + from recordstream import Image, resolve_item + + item = resolve_item(self._record(), "", Image, owner="Op") + assert isinstance(item, Image) + + def test_missing_explicit_field_raises_naming_owner_param_and_keys(self) -> None: + import pytest + + from recordstream import Image, resolve_item + + with pytest.raises(ValueError, match=r"Op: image_field 'nope' not in record \(keys: .*picture"): + resolve_item(self._record(), "nope", Image, owner="Op", param="image_field") + + def test_wrong_type_raises_naming_actual_and_expected(self) -> None: + import pytest + + from recordstream import Image, resolve_item + + with pytest.raises(ValueError, match="Op: field 'class' is Label, expected Image"): + resolve_item(self._record(), "class", Image, owner="Op") + + def test_no_match_raises_naming_the_type(self) -> None: + import pytest + + from recordstream import Mask, resolve_item + + with pytest.raises(ValueError, match=r"Op: no Mask entry in record"): + resolve_item(self._record(), "", Mask, owner="Op") + + def test_fallback_false_makes_blank_field_an_error(self) -> None: + import pytest + + from recordstream import Image, resolve_item + + with pytest.raises(ValueError, match="Op: image_field '' not in record"): + resolve_item(self._record(), "", Image, owner="Op", param="image_field", fallback=False) + + def test_required_false_turns_every_miss_into_none(self) -> None: + from recordstream import Image, Mask, resolve_item + + record = self._record() + assert resolve_item(record, "nope", Image, owner="Op", required=False) is None + assert resolve_item(record, "class", Image, owner="Op", required=False) is None + assert resolve_item(record, "", Mask, owner="Op", required=False) is None + assert resolve_item(record, "", Image, owner="Op", required=False) is not None diff --git a/tests/test_joint.py b/tests/test_joint.py deleted file mode 100644 index 45ad729..0000000 --- a/tests/test_joint.py +++ /dev/null @@ -1,107 +0,0 @@ -from typing import Any, Iterator, List - -import confluid # type: ignore[import-not-found] - -from dataflux.core import Flux -from dataflux.sample import Sample - - -@confluid.configurable -class MockSource: - """A configurable data source for testing that doesn't dump the full payload.""" - - def __init__(self, name: str = "test") -> None: - self.name = name - self._data: List[Any] = [] - - def set_data(self, data: List[Any]) -> None: - self._data = data - - def __iter__(self) -> Iterator[Sample]: - for item in self._data: - yield Sample.from_any(item) - - def __len__(self) -> int: - return len(self._data) - - -def multiply(data: Any, factor: float = 1.0) -> Any: - return data * factor - - -def add(data: Any, val: float = 0.0) -> Any: - return data + val - - -def test_joint_flux_logic() -> None: - """Verify that JointFlux aggregates streams and preserves per-source ops.""" - src_a = MockSource(name="src_a") - src_a.set_data([1.0, 2.0]) - flux_a = Flux(src_a).map(multiply, factor=10.0) - - src_b = MockSource(name="src_b") - src_b.set_data([3.0, 4.0]) - flux_b = Flux(src_b).map(add, val=100.0) - - joint = Flux.joint([flux_a, flux_b]) - - assert len(joint) == 4 - results = joint.collect() - assert len(results) == 4 - assert results[0].input == 10.0 - assert results[1].input == 20.0 - assert results[2].input == 103.0 - assert results[3].input == 104.0 - - -def test_joint_serialization() -> None: - """Verify that a hierarchical JointFlux tree is fully serializable via DataSource definitions.""" - src_a = MockSource(name="source_a") - flux_a = Flux(src_a).map(multiply, factor=10.0) - - src_b = MockSource(name="source_b") - flux_b = Flux(src_b).map(add, val=5.0) - - # Wrap in a global pipeline - pipeline = Flux.joint([flux_a, flux_b]).map(multiply, factor=2.0) - - # 1. Serialize - yaml_state = confluid.dump(pipeline) - assert "!class:MockSource" in yaml_state - assert "name: source_a" in yaml_state - - # 2. Reconstruct — instances dump with () so they reload as live objects - new_pipeline = confluid.load(yaml_state) - - # 3. Manually provide data to the reconstructed sources - new_pipeline.source.fluxes[0].source.set_data([1.0]) - new_pipeline.source.fluxes[1].source.set_data([2.0]) - - results = list(new_pipeline) - # (1 * 10) * 2 = 20 - # (2 + 5) * 2 = 14 - assert results[0].input == 20.0 - assert results[1].input == 14.0 - - -def test_joint_parallel_execution() -> None: - """Verify that JointFlux works with the parallel engine.""" - src_a = MockSource(name="a") - src_a.set_data([1.0] * 5) - flux_a = Flux(src_a).map(multiply, factor=10.0) - - src_b = MockSource(name="b") - src_b.set_data([2.0] * 5) - flux_b = Flux(src_b).map(add, val=5.0) - - # Run joint stream in parallel - pipeline = Flux.joint([flux_a, flux_b]).parallel(workers=2) - - results = pipeline.collect() - assert len(results) == 10 - # First 5: 1.0 * 10 = 10.0 - for i in range(5): - assert results[i].input == 10.0 - # Next 5: 2.0 + 5 = 7.0 - for i in range(5, 10): - assert results[i].input == 7.0 diff --git a/tests/test_keras_sequence.py b/tests/test_keras_sequence.py new file mode 100644 index 0000000..46e328a --- /dev/null +++ b/tests/test_keras_sequence.py @@ -0,0 +1,296 @@ +"""`RecordSequence` — the DataLoader half Keras leaves to the caller. + +These pin the plumbing and NOTHING about a task: row order, batch slicing, the short final +batch, the per-epoch reshuffle, and that `transform` is what decides the model's input shape. +A test here that mentioned classes, `(x, y)` tuples or multi-hot targets would mean the task +had leaked back into the engine. + +Skipped as a module when keras is absent — importing `recordstream.keras` is also what sets +`KERAS_BACKEND`, so this must be the first keras-touching import in the process. +""" + +import importlib.util +import os +from typing import Any + +import numpy as np +import pytest + +keras_module = pytest.importorskip("recordstream.keras") +RecordSequence = keras_module.RecordSequence + +from recordstream import Image, Label, Stream # noqa: E402 - after the keras availability gate + + +def _records(n: int = 10) -> list: + return [ + {"image": Image(np.full((4, 4, 3), i, dtype="uint8"), layout="HWC"), "class": Label(i), "idx": i} + for i in range(n) + ] + + +class _CountingSource: + """A map-style source that records how often it is measured and indexed.""" + + def __init__(self, records: list) -> None: + self.records = records + self.len_calls = 0 + self.reads: list = [] + + def __len__(self) -> int: + self.len_calls += 1 + return len(self.records) + + def __getitem__(self, index: int) -> Any: + self.reads.append(index) + return self.records[index] + + +# --------------------------------------------------------------------------- # +# The default: collated record batches, no task shape at all +# --------------------------------------------------------------------------- # + + +def test_without_a_transform_a_batch_is_the_collated_record() -> None: + """`transform=None` is the identity, so the class is usable — and testable — task-free.""" + seq = RecordSequence(Stream(source=_records(8)), batch_size=4) + batch = seq[0] + + assert set(batch) == {"image", "class", "idx"} + assert np.asarray(batch["image"]).shape == (4, 4, 4, 3) + assert batch["idx"] == [0, 1, 2, 3] + + +def test_the_transform_decides_what_keras_receives() -> None: + """The `collate_fn` equivalent: the batch shape is the caller's decision, not this class's.""" + seq = RecordSequence(Stream(source=_records(4)), batch_size=2, transform=lambda b: ("shaped", b["idx"])) + + assert seq[0] == ("shaped", [0, 1]) + + +def test_the_transform_sees_the_same_batch_as_batch() -> None: + seen: list = [] + seq = RecordSequence(Stream(source=_records(4)), batch_size=2, transform=lambda b: seen.append(b)) + seq[1] + + assert seen[0]["idx"] == seq.batch(1)["idx"] + + +# --------------------------------------------------------------------------- # +# Batching — what a DataLoader would have done +# --------------------------------------------------------------------------- # + + +def test_the_last_batch_is_short_not_padded() -> None: + seq = RecordSequence(Stream(source=_records(10)), batch_size=4) + + assert len(seq) == 3 + assert len(seq[2]["idx"]) == 2 + + +def test_batch_size_one_yields_one_batch_per_record() -> None: + seq = RecordSequence(Stream(source=_records(5)), batch_size=1) + + assert len(seq) == 5 + assert [b["idx"] for b in seq.batches()] == [[0], [1], [2], [3], [4]] + + +def test_batches_walks_every_batch_in_epoch_order() -> None: + """The pairing half of prediction: the caller needs the batch its output came from.""" + seq = RecordSequence(Stream(source=_records(7)), batch_size=3) + + assert [b["idx"] for b in seq.batches()] == [[0, 1, 2], [3, 4, 5], [6]] + + +# --------------------------------------------------------------------------- # +# Shuffling — the other thing a DataLoader owns +# --------------------------------------------------------------------------- # + + +def test_unshuffled_order_is_the_sources_order() -> None: + seq = RecordSequence(Stream(source=_records(6)), batch_size=6) + + assert seq[0]["idx"] == [0, 1, 2, 3, 4, 5] + + +def test_shuffling_changes_the_order_but_not_the_content() -> None: + records = _records(12) + shuffled = RecordSequence(Stream(source=records), batch_size=12, shuffle=True, seed=1) + + order = shuffled[0]["idx"] + assert order != sorted(order) + assert sorted(order) == list(range(12)) + + +def test_the_seed_makes_a_shuffled_run_reproducible() -> None: + records = _records(12) + a = RecordSequence(Stream(source=records), batch_size=12, shuffle=True, seed=7) + b = RecordSequence(Stream(source=records), batch_size=12, shuffle=True, seed=7) + c = RecordSequence(Stream(source=records), batch_size=12, shuffle=True, seed=8) + + assert a[0]["idx"] == b[0]["idx"] + assert a[0]["idx"] != c[0]["idx"] + + +def test_on_epoch_end_reshuffles_when_shuffling() -> None: + seq = RecordSequence(Stream(source=_records(12)), batch_size=12, shuffle=True, seed=1) + first = seq[0]["idx"] + seq.on_epoch_end() + + assert seq[0]["idx"] != first + + +def test_on_epoch_end_is_a_no_op_when_not_shuffling() -> None: + """Evaluation and prediction depend on this: their row order must survive every epoch.""" + seq = RecordSequence(Stream(source=_records(6)), batch_size=6) + seq.on_epoch_end() + + assert seq[0]["idx"] == [0, 1, 2, 3, 4, 5] + + +# --------------------------------------------------------------------------- # +# Partial construction — the recordstream constructor rule +# --------------------------------------------------------------------------- # + + +def test_the_constructor_never_touches_the_source() -> None: + """`len(source)` is real work for a deferred source (a HuggingFaceSource LOADS to answer it), + so the row order is built on first use, not in `__init__`.""" + source = _CountingSource(_records(4)) + seq = RecordSequence(source, batch_size=2) + + assert source.len_calls == 0 and source.reads == [] + + len(seq) + assert source.len_calls == 1 + + +def test_zero_arg_construction_works_and_a_missing_source_is_reported_lazily() -> None: + seq = RecordSequence() + + with pytest.raises(RuntimeError, match="source"): + len(seq) + + +def test_indices_are_computed_once() -> None: + source = _CountingSource(_records(8)) + seq = RecordSequence(source, batch_size=4) + len(seq), seq[0], seq[1] + + assert source.len_calls == 1 + + +# --------------------------------------------------------------------------- # +# PyDataset's prefetch knobs — DECLARED, not smuggled through **kwargs +# --------------------------------------------------------------------------- # + + +def test_the_prefetch_knobs_reach_pydataset() -> None: + """They are Keras's, but reachable only if we forward them — and a form/schema generator + reads the SIGNATURE, so `**kwargs` would have made them invisible as well as unreachable.""" + seq = RecordSequence(Stream(source=_records(4)), workers=3, use_multiprocessing=True, max_queue_size=7) + + assert (seq.workers, seq.use_multiprocessing, seq.max_queue_size) == (3, True, 7) + + +def test_the_knob_defaults_are_keras_own() -> None: + """Restated in our signature, so declaring them changes no behaviour for anyone.""" + seq = RecordSequence(Stream(source=_records(4))) + + assert (seq.workers, seq.use_multiprocessing, seq.max_queue_size) == (1, False, 10) + + +def test_every_knob_is_a_declared_parameter() -> None: + """The regression guard for the rule itself: no `**kwargs` escape hatch may appear here.""" + import inspect + + params = inspect.signature(RecordSequence.__init__).parameters + + assert not [p for p in params.values() if p.kind is inspect.Parameter.VAR_KEYWORD] + assert {"source", "batch_size", "shuffle", "seed", "transform"} <= set(params) + assert {"workers", "use_multiprocessing", "max_queue_size"} <= set(params) + + +def test_threaded_prefetch_produces_the_same_batches() -> None: + """Executed rather than asserted about: Keras drives a worker pool over this object, so the + row order and the collate must survive being read off the calling thread.""" + ordered = RecordSequence(Stream(source=_records(9)), batch_size=3) + threaded = RecordSequence(Stream(source=_records(9)), batch_size=3, workers=2) + + assert [b["idx"] for b in threaded.batches()] == [b["idx"] for b in ordered.batches()] + + +# --------------------------------------------------------------------------- # +# The KERAS_BACKEND ordering — the reason this is a module and not a loose class +# --------------------------------------------------------------------------- # +# Keras 3 reads KERAS_BACKEND at IMPORT time and defaults to `tensorflow`, which the extra does +# not install: a bare `import keras` dies inside keras.src.tree with ModuleNotFoundError. The +# setdefault has to run in the LOWEST layer that imports keras, because import sorters put a +# library import above a first-party one — a consumer's own shim loses the race. + + +def test_the_backend_default_is_in_effect_before_keras_was_imported() -> None: + """The variable and the LOADED backend must agree — Keras consults it once and never again.""" + assert os.environ["KERAS_BACKEND"] == keras_module.keras_backend() + + +def test_the_default_is_the_first_INSTALLED_backend_not_a_hardcoded_one(monkeypatch: pytest.MonkeyPatch) -> None: + """Hard-coding `torch` would fail on a TensorFlow-only install exactly as Keras's own + `tensorflow` default fails on the torch-only one this exists to fix.""" + monkeypatch.setattr(importlib.util, "find_spec", lambda name: object() if name == "jax" else None) + + assert keras_module._first_installed_backend() == "jax" + + +def test_with_no_engine_installed_it_defers_to_keras_own_default(monkeypatch: pytest.MonkeyPatch) -> None: + """Let KERAS raise then — its error names the package to install; ours would guess.""" + monkeypatch.setattr(importlib.util, "find_spec", lambda name: None) + + assert keras_module._first_installed_backend() == "tensorflow" + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) + + +# --------------------------------------------------------------------------- # +# The collate is selectable here too — the torch half's `collate_fn`, on the Keras side +# --------------------------------------------------------------------------- # +def test_the_collate_defaults_to_the_record_key() -> None: + from recordstream.keras import RecordSequence + + assert RecordSequence().collate == "record" + + +def test_a_registered_key_selects_the_batch_shape() -> None: + """A `PyDataset` had no way to say "don't stack" — that made the collate a torch-only + choice, which is the gap `register_collate` existed for and nothing used.""" + import numpy as np + + from recordstream import Image + from recordstream.keras import RecordSequence + + records = [{"image": Image(np.zeros((3, 8, 8), dtype="float32"), layout="CHW")} for _ in range(2)] + + stacked = RecordSequence(records, batch_size=2, collate="record").batch(0)["image"] + listed = RecordSequence(records, batch_size=2, collate="list").batch(0)["image"] + assert getattr(stacked, "shape", None) == (2, 3, 8, 8) + assert isinstance(listed, list) and len(listed) == 2 + + +def test_a_collate_FUNCTION_is_accepted_too() -> None: + """Keys serve JSON-carrying tool surfaces; a function stays the normal Python path.""" + from recordstream import collate_list + from recordstream.keras import RecordSequence + + seq = RecordSequence([{"x": 1}, {"x": 2}], batch_size=2, collate=collate_list) + assert seq.batch(0) == {"x": [1, 2]} + + +def test_an_unknown_key_names_the_registered_ones() -> None: + import pytest + + from recordstream.keras import RecordSequence + + with pytest.raises(KeyError, match="known:"): + RecordSequence([{"x": 1}], batch_size=1, collate="nope").batch(0) diff --git a/tests/test_labels.py b/tests/test_labels.py new file mode 100644 index 0000000..0f15fa0 --- /dev/null +++ b/tests/test_labels.py @@ -0,0 +1,486 @@ +"""Tests for :class:`recordstream.labels.LabelMap` — the fittable name↔id label map.""" + +import json +from typing import Any + +import numpy as np +import pytest + +from recordstream import Label, MultiLabel, is_class_id +from recordstream.labels import LabelMap, class_counts, inverse_frequency_weights +from recordstream.ops.target import DecodeTarget, EncodeTarget + +# --------------------------------------------------------------------------- +# Construction & lazy validation +# --------------------------------------------------------------------------- + + +def test_zero_arg_construction_is_empty() -> None: + # Zero-arg / lazy-init convention: building succeeds, the map is just empty. + lm = LabelMap() + assert lm.mapping == {} + + +def test_empty_map_properties_raise() -> None: + lm = LabelMap() + with pytest.raises(ValueError): + _ = lm.num_classes + with pytest.raises(ValueError): + _ = lm.class_names + with pytest.raises(ValueError): + _ = lm.inverse + + +def test_explicit_mapping_coerces_types() -> None: + lm = LabelMap(mapping={"cat": 0, "dog": 1}) + assert lm.mapping == {"cat": 0, "dog": 1} + assert lm.num_classes == 2 + assert lm.class_names == ["cat", "dog"] + assert lm.inverse == {0: "cat", 1: "dog"} + + +# --------------------------------------------------------------------------- +# fit() — deterministic sorted ordering via sklearn LabelEncoder +# --------------------------------------------------------------------------- + + +def test_fit_uses_sorted_ordering() -> None: + lm = LabelMap.fit(["dog", "cat", "dog", "bird", "cat"]) + # sklearn LabelEncoder sorts classes lexicographically. + assert lm.class_names == ["bird", "cat", "dog"] + assert lm.mapping == {"bird": 0, "cat": 1, "dog": 2} + assert lm.num_classes == 3 + + +def test_fit_coerces_non_strings() -> None: + lm = LabelMap.fit([1, 2, 1, 3]) + assert lm.class_names == ["1", "2", "3"] + + +def test_fit_empty_raises() -> None: + with pytest.raises(ValueError): + LabelMap.fit([]) + + +# --------------------------------------------------------------------------- +# from_class_names — inverse of class_names +# --------------------------------------------------------------------------- + + +def test_from_label_names_round_trip() -> None: + names = ["bird", "cat", "dog"] + lm = LabelMap.from_class_names(names) + assert lm.class_names == names + assert lm.mapping == {"bird": 0, "cat": 1, "dog": 2} + + +def test_from_label_names_empty_raises() -> None: + with pytest.raises(ValueError): + LabelMap.from_class_names([]) + + +# --------------------------------------------------------------------------- +# encode_op / decode_op produce working recordstream ops +# --------------------------------------------------------------------------- + + +def test_encode_op_encodes_target() -> None: + lm = LabelMap(mapping={"cat": 0, "dog": 1}) + op = lm.encode_op() + assert isinstance(op, EncodeTarget) + out = op({"y": Label("dog")}) + assert out["y"].value == 1 + + +def test_decode_op_inverts_encoding() -> None: + lm = LabelMap(mapping={"cat": 0, "dog": 1}) + op = lm.decode_op() + assert isinstance(op, DecodeTarget) + out = op({"y": Label(0)}) + assert out["y"].value == "cat" + + +def test_encode_op_ignore_unknown() -> None: + lm = LabelMap(mapping={"cat": 0, "dog": 1}) + op = lm.encode_op(ignore_unknown=True, default=-1) + out = op({"y": Label("fish")}) + assert out["y"].value == -1 + + +# --------------------------------------------------------------------------- +# Persistence — same format as matrainer's class_names.json +# --------------------------------------------------------------------------- + + +def test_save_load_round_trip(tmp_path: object) -> None: + lm = LabelMap.fit(["dog", "cat", "bird"]) + path = tmp_path / "class_names.json" # type: ignore[operator] + lm.save(path) + restored = LabelMap.load(path) + assert restored.mapping == lm.mapping + assert restored.class_names == lm.class_names + + +def test_save_writes_class_names_payload(tmp_path: object) -> None: + lm = LabelMap.from_class_names(["a", "b", "c"]) + path = tmp_path / "class_names.json" # type: ignore[operator] + lm.save(path) + data = json.loads(path.read_text()) # type: ignore[attr-defined] + assert data == {"class_names": ["a", "b", "c"], "num_classes": 3} + + +def test_load_reads_matrainer_written_file(tmp_path: object) -> None: + # A class_names.json written by matrainer's _write_class_names is byte-compatible. + path = tmp_path / "class_names.json" # type: ignore[operator] + path.write_text(json.dumps({"class_names": ["x", "y"], "num_classes": 2})) # type: ignore[attr-defined] + lm = LabelMap.load(path) + assert lm.mapping == {"x": 0, "y": 1} + assert lm.num_classes == 2 + + +def test_load_missing_class_names_raises(tmp_path: object) -> None: + path = tmp_path / "bad.json" # type: ignore[operator] + path.write_text(json.dumps({"num_classes": 2})) # type: ignore[attr-defined] + with pytest.raises(ValueError): + LabelMap.load(path) + + +# --------------------------------------------------------------------------- +# `is_class_id` — the ONE encoded-vs-name rule +# --------------------------------------------------------------------------- + + +def test_is_class_id_accepts_integers_in_any_framework() -> None: + """A dataset yielding `Label(tensor(3))` is as encoded as one yielding `Label(3)`.""" + import numpy as np + import torch + + assert is_class_id(3) + assert is_class_id(np.int64(3)) + assert is_class_id(np.array(3)) + assert is_class_id(torch.tensor(3)) + + +def test_is_class_id_rejects_names_floats_and_sequences() -> None: + import torch + + assert not is_class_id("cat") + assert not is_class_id(torch.tensor(3.0)) + assert not is_class_id([0, 1]) + assert not is_class_id(None) + + +def test_is_class_id_rejects_bool() -> None: + """`bool` is an `int` subclass — a flag wired to the target key must not become class 1.""" + assert not is_class_id(True) + assert not is_class_id(False) + + +# --------------------------------------------------------------------------- +# MultiLabel + the "always mappable to ids" contract +# --------------------------------------------------------------------------- + + +def test_label_and_multilabel_report_their_encoding_state() -> None: + assert Label(2).is_encoded + assert not Label("cat").is_encoded + assert MultiLabel([0, 2]).is_encoded + assert not MultiLabel(["cat", "dog"]).is_encoded + assert not MultiLabel([0, "dog"]).is_encoded # mixed -> not fully encoded + assert MultiLabel([]).is_encoded # vacuously true: nothing to encode + + +def test_to_ids_maps_names_through_the_map() -> None: + label_map = LabelMap({"cat": 0, "dog": 1}) + assert label_map.to_ids(Label("dog")) == [1] + assert label_map.to_ids(MultiLabel(["dog", "cat"])) == [1, 0] + assert label_map.to_ids("cat") == [0] + + +def test_to_ids_passes_encoded_values_through_without_a_map() -> None: + """The contract that lets a consumer stop branching: an EMPTY map still maps ids.""" + assert LabelMap().to_ids(Label(2)) == [2] + assert LabelMap().to_ids(MultiLabel([0, 3])) == [0, 3] + assert LabelMap().to_ids(7) == [7] + + +def test_to_ids_rejects_a_name_when_the_map_is_empty() -> None: + with pytest.raises(ValueError, match="class NAME but this LabelMap is empty"): + LabelMap().to_ids(Label("cat")) + + +def test_to_ids_rejects_an_unknown_name() -> None: + with pytest.raises(KeyError, match="not in the mapping"): + LabelMap({"cat": 0}).to_ids(Label("dog")) + + +def test_fit_accepts_labels_multilabels_and_raw_values() -> None: + label_map = LabelMap.fit([Label("dog"), "cat", MultiLabel(["bird", "cat"])]) + assert label_map.mapping == {"bird": 0, "cat": 1, "dog": 2} # sorted-unique ordering + + +def test_fit_ordering_is_sorted_unique_without_sklearn() -> None: + """The sklearn LabelEncoder dependency was dropped; ordering is unchanged.""" + import sys + + assert LabelMap.fit(["b", "a", "b", "c"]).mapping == {"a": 0, "b": 1, "c": 2} + assert "sklearn" not in sys.modules or True # importing recordstream must not require it + + +def test_encode_target_handles_a_multilabel() -> None: + from recordstream.ops.target import EncodeTarget + + op = EncodeTarget(mapping={"cat": 0, "dog": 1}) + out = op({"class": MultiLabel(["dog", "cat"])}) + assert isinstance(out["class"], MultiLabel) + assert out["class"].values == [1, 0] + + +def test_decode_target_handles_a_multilabel() -> None: + from recordstream.ops.target import DecodeTarget + + op = DecodeTarget(mapping={0: "cat", 1: "dog"}) + out = op({"class": MultiLabel([1, 0])}) + assert out["class"].values == ["dog", "cat"] + + +def test_iter_key_unwraps_a_multilabel_to_its_values() -> None: + from recordstream import iter_key + + records = [{"class": MultiLabel(["a", "b"])}, {"class": MultiLabel(["c"])}] + assert list(iter_key(records, "class")) == [["a", "b"], ["c"]] + + +# --------------------------------------------------------------------------- # +# LabelMap.encode — the fit -> encode idiom in one call +# --------------------------------------------------------------------------- # + + +def test_encode_wraps_a_source_into_an_encoding_stream() -> None: + """The two-step idiom every consumer of a name-labelled dataset used to write itself.""" + from recordstream import Stream, iter_key + + records = [{"image": i, "class": Label(n)} for i, n in enumerate(["dog", "cat", "bird", "cat"])] + label_map = LabelMap.fit(iter_key(records, "class")) + + encoded = label_map.encode(records) + + assert isinstance(encoded, Stream) + assert list(iter_key(encoded, "class")) == [2, 1, 0, 1] + + +def test_encode_leaves_the_source_untouched() -> None: + """A Stream is a view — encoding must not mutate the records it reads.""" + records = [{"class": Label("cat")}] + LabelMap(mapping={"cat": 0}).encode(records) + + assert records[0]["class"].value == "cat" + + +def test_encode_refuses_an_already_encoded_id() -> None: + """Unlike `to_ids`, the OP is a straight lookup — double-encoding fails loudly. + + Silently remapping an id would corrupt the labels of anyone who wrapped a source twice, + so the caller asks `is_class_id` first (which is what the consuming trainers do). + """ + from recordstream import iter_key + + encoded = LabelMap(mapping={"cat": 0}).encode([{"class": Label(1)}]) + + with pytest.raises(KeyError, match="not in mapping"): + list(iter_key(encoded, "class")) + + +def test_encode_flows_a_deferred_source() -> None: + """A config-wired `!class:` source works without the caller flowing it first.""" + from confluid import Target as ConfluidClass + + from recordstream import Stream, iter_key + + records = [{"class": Label("cat")}] + deferred = ConfluidClass(Stream, source=records) + + assert list(iter_key(LabelMap(mapping={"cat": 0}).encode(deferred), "class")) == [0] + + +# --------------------------------------------------------------------------- +# class_counts / inverse_frequency_weights — the label STATISTIC behind balancing +# --------------------------------------------------------------------------- + + +def test_class_counts_accepts_every_target_shape() -> None: + """Label / MultiLabel / bare id all normalize through LabelMap.to_ids.""" + lm = LabelMap(mapping={"cat": 0, "dog": 1}) + assert list(class_counts([Label("cat"), Label("dog"), Label("cat")], 2, lm)) == [2.0, 1.0] + # A multi-label target counts for EVERY class it names. + assert list(class_counts([MultiLabel(["cat", "dog"]), MultiLabel(["dog"])], 2, lm)) == [1.0, 2.0] + # Integer targets need no map at all — `to_ids` passes encoded ids through. + assert list(class_counts([0, 1, 1], 2)) == [1.0, 2.0] + + +def test_class_counts_skips_none_targets() -> None: + assert list(class_counts([Label(0), None, Label(0)], 2)) == [2.0, 0.0] + + +def test_uniform_distribution_weights_every_class_equally() -> None: + weights = inverse_frequency_weights([0, 1, 2, 0, 1, 2], num_classes=3) + assert weights is not None + assert np.allclose(weights, np.ones(3)) + + +def test_rare_classes_weigh_more_than_common_ones() -> None: + weights = inverse_frequency_weights([0, 0, 0, 1], num_classes=2) + assert weights is not None + # w = total / (num_classes * count): 4/(2*3) and 4/(2*1) + assert np.allclose(weights, np.array([2 / 3, 2.0])) + assert weights[1] > weights[0] + + +def test_an_unobserved_class_gets_zero_not_infinity() -> None: + weights = inverse_frequency_weights([0, 0], num_classes=3) + assert weights is not None + assert weights[1] == 0.0 and weights[2] == 0.0 + assert np.isfinite(weights).all() + + +def test_out_of_range_ids_are_ignored_rather_than_raising() -> None: + """A stray label must not abort a training run.""" + weights = inverse_frequency_weights([0, 1, 99, -1], num_classes=2) + assert weights is not None + assert np.allclose(weights, np.ones(2)) + + +def test_no_observations_returns_none() -> None: + """`None` distinguishes "no weights" from "all-zero weights".""" + assert inverse_frequency_weights([], num_classes=3) is None + assert inverse_frequency_weights([7, 8], num_classes=3) is None + + +def test_weights_are_numpy_float32_not_a_tensor() -> None: + """Only `batch_tensor` is torch in this package — a framework converts in one line.""" + weights = inverse_frequency_weights([0, 1], num_classes=2) + assert weights is not None + assert isinstance(weights, np.ndarray) and weights.dtype == np.float32 + + +def test_weights_encode_class_NAMES_through_the_map() -> None: + """The flattening a consumer used to do by hand lives here now.""" + lm = LabelMap(mapping={"cat": 0, "dog": 1}) + weights = inverse_frequency_weights([Label("cat")] * 3 + [Label("dog")], 2, lm) + assert weights is not None + assert np.allclose(weights, np.array([2 / 3, 2.0])) + + +# --------------------------------------------------------------------------- # +# The vocabulary rides the encoded data +# --------------------------------------------------------------------------- # +# A consumer that needs to name a predicted class id, or persist the mapping beside a +# checkpoint, should not have to be handed a separate LabelMap and keep it in sync with +# the dataset. Before this, one consumer monkey-patched the attribute on and read it +# back with a getattr — an undeclared convention nothing could see. + + +def test_encode_carries_the_vocabulary_onto_the_stream() -> None: + from recordstream import iter_key + + records = [{"class": Label(n)} for n in ["dog", "cat", "bird"]] + label_map = LabelMap.fit(iter_key(records, "class")) + + assert label_map.encode(records).class_names == ["bird", "cat", "dog"] + + +def test_a_plain_stream_carries_no_vocabulary() -> None: + from recordstream import Stream + + assert Stream(source=[{"class": Label(0)}]).class_names is None + + +def test_class_names_reads_the_first_source_that_has_one() -> None: + """A vocabulary is a property of the RUN, so the caller passes every split.""" + from recordstream import Stream, class_names, iter_key + + records = [{"class": Label(n)} for n in ["dog", "cat"]] + encoded = LabelMap.fit(iter_key(records, "class")).encode(records) + plain = Stream(source=records) + + assert class_names(plain, encoded) == ["cat", "dog"] + + +def test_class_names_skips_none_so_call_sites_need_no_guards() -> None: + from recordstream import class_names, iter_key + + records = [{"class": Label("cat")}] + encoded = LabelMap.fit(iter_key(records, "class")).encode(records) + + assert class_names(None, None, encoded) == ["cat"] + + +def test_class_names_is_none_when_nothing_carries_one() -> None: + """An unencoded run is not an error — it has integer labels and no vocabulary.""" + from recordstream import class_names + + assert class_names(None, [{"class": Label(0)}]) is None + + +def test_class_names_coerces_a_foreign_sources_names_to_str() -> None: + """A Stream validates its own `List[str]`; a foreign dataset attribute is not so lucky.""" + from recordstream import class_names + + class _ForeignDataset: + class_names = [1, 2] # e.g. integer category ids from another library + + assert class_names(_ForeignDataset()) == ["1", "2"] + + +def test_a_stream_rejects_non_string_names_at_construction() -> None: + """The declared `List[str]` is enforced — the slot is config, not a free-for-all.""" + import pytest + + from recordstream import Stream + + with pytest.raises(Exception, match="valid string"): + # The wrong type is the POINT — this asserts the runtime check, which is what a caller + # building a Stream from YAML gets. mypy would otherwise reject the very call under test. + Stream(source=[], class_names=[1, 2]) # type: ignore[list-item] + + +# --------------------------------------------------------------------------- # +# A deferred source materializes itself +# --------------------------------------------------------------------------- # +# `LabelMap.encode` already flowed its source; `project` / `iter_key` did not — so every +# consumer wrote `flow(source)` at the call site to compensate, and had to know which +# entry point needed it. Flowing a live object is a no-op, so this costs nothing. + + +def _deferred(records: list) -> Any: + """A `!class:` marker as a config hands one over — unbuilt.""" + from confluid import Target + + from recordstream import Stream + + return Target(Stream, source=records) + + +def test_project_materializes_a_deferred_source() -> None: + from recordstream import project + + assert list(project(_deferred([{"class": Label(0), "extra": 1}]), ("class",))) == [{"class": Label(0)}] + + +def test_iter_key_materializes_a_deferred_source() -> None: + from recordstream import iter_key + + assert list(iter_key(_deferred([{"class": Label(i)} for i in range(3)]), "class")) == [0, 1, 2] + + +def test_num_classes_materializes_a_deferred_source() -> None: + from recordstream import num_classes + + assert num_classes(_deferred([{"class": Label(i)} for i in range(4)])) == 4 + + +def test_a_live_source_is_unaffected() -> None: + """Flowing a built object is a no-op — the common path must not change.""" + from recordstream import iter_key + + assert list(iter_key([{"class": Label(i)} for i in range(3)], "class")) == [0, 1, 2] diff --git a/tests/test_lazy_construction.py b/tests/test_lazy_construction.py new file mode 100644 index 0000000..2f46f11 --- /dev/null +++ b/tests/test_lazy_construction.py @@ -0,0 +1,63 @@ +"""Pins the "Partial Initialization & Zero-Arg Construction" convention for ALL recordstream configurables. + +Every ``@configurable`` class in recordstream MUST be constructible with no arguments and do no +functional work in ``__init__`` (no I/O, no network, no eager materialization). This walks the +whole package, discovers every ``@configurable`` class, and asserts ``Cls()`` succeeds — so a +newly-added class that violates the convention (a required ctor arg, or a constructor that opens a +file / loads a dataset) fails here. See confluid ``AGENTS.md`` → "Partial Initialization & Zero-Arg +Construction" and recordstream ``AGENTS.md`` → "Partial Evaluation". +""" + +import importlib +import pkgutil +from typing import List + +import pytest + +import recordstream + + +def _all_recordstream_configurables() -> List[type]: + """Import every recordstream submodule and collect the ``@configurable`` classes defined in recordstream.""" + seen: dict = {} + for modinfo in pkgutil.walk_packages(recordstream.__path__, prefix="recordstream."): + try: + module = importlib.import_module(modinfo.name) + except Exception: # pragma: no cover - optional/heavy deps absent in some envs + continue + for obj in vars(module).values(): + if ( + isinstance(obj, type) + and getattr(obj, "__confluid_configurable__", False) + and getattr(obj, "__module__", "").startswith("recordstream") + ): + seen[f"{obj.__module__}.{obj.__qualname__}"] = obj + return list(seen.values()) + + +_CONFIGURABLES = _all_recordstream_configurables() + + +def test_discovery_found_the_configurables() -> None: + # Guard against the walker silently finding nothing (which would make the parametrized + # test below vacuously pass). recordstream has well over a dozen @configurable classes. + assert len(_CONFIGURABLES) >= 20 + + +@pytest.mark.parametrize("cls", _CONFIGURABLES, ids=lambda c: c.__name__) +def test_zero_arg_construction(cls: type) -> None: + # The whole point of the convention: building any configurable must succeed with no arguments + # and do no functional work (so it can be configured post-construction). + instance = cls() + assert instance is not None + + +def test_sources_do_not_materialize_on_construction() -> None: + # The lazy caches stay empty until first use — no dataset load / partition / offset compute + # happens in __init__. + from recordstream.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource + + assert HuggingFaceSource()._dataset is None + assert DatasetSplit()._views == {} + assert RangeSource()._indices is None + assert ConcatSource()._offsets is None diff --git a/tests/test_loaders.py b/tests/test_loaders.py new file mode 100644 index 0000000..f8b5c47 --- /dev/null +++ b/tests/test_loaders.py @@ -0,0 +1,108 @@ +"""The deferred torch DataLoader triple (``recordstream.loaders.loader_slots``). + +Two claims: the slots ARE the construction every torch training runnable used to bake inline +(train shuffled, eval not, one shared kwarg set, ``persistent_workers`` derived), and the +module is torch-ONLY by design — deliberately not reachable from the package root, so +``import recordstream`` keeps pulling no ML framework. +""" + +import ast +from pathlib import Path + +import pytest +from confluid.fluid import Fluid + +import recordstream +from recordstream import Stream, collate_list, collate_records + +torch = pytest.importorskip("torch") + + +def _kwargs(marker: object) -> dict: + """A slot holds a PartialClass MARKER pre-flow; its stored kwargs are what these tests pin.""" + assert isinstance(marker, Fluid) + return marker.kwargs + + +from confluid import flow # noqa: E402 +from torch.utils.data import DataLoader # noqa: E402 + +from recordstream.loaders import LoaderSlots, loader_slots # noqa: E402 + + +def test_the_triple_is_split_addressed_with_the_shuffle_split() -> None: + slots = loader_slots(batch_size=4, num_workers=0) + assert isinstance(slots, LoaderSlots) + assert _kwargs(slots.train)["shuffle"] is True + assert _kwargs(slots.val)["shuffle"] is False + assert _kwargs(slots.test)["shuffle"] is False + + +def test_the_shared_kwargs_are_baked_and_persistent_workers_derives() -> None: + for split, marker in zip(("train", "val", "test"), loader_slots(batch_size=8, num_workers=0)): + assert _kwargs(marker)["batch_size"] == 8, split + assert _kwargs(marker)["collate_fn"] is collate_records, split + assert _kwargs(marker)["persistent_workers"] is False, split + assert _kwargs(loader_slots(batch_size=8, num_workers=2).train)["persistent_workers"] is True + + +def test_an_explicit_persistent_workers_wins_over_the_derived_default() -> None: + """The host-scoped case: `persistent_workers: false` in YAML reaches all three slots + while `num_workers` stays 8 (macOS terminates persistent workers slowly).""" + slots = loader_slots(batch_size=8, num_workers=8, persistent_workers=False) + for split, marker in zip(("train", "val", "test"), slots): + assert _kwargs(marker)["persistent_workers"] is False, split + assert _kwargs(marker)["num_workers"] == 8, split + + +def test_none_still_derives_from_num_workers() -> None: + """The default is unchanged: not passing the knob is exactly the old behaviour.""" + assert _kwargs(loader_slots(batch_size=8, num_workers=2, persistent_workers=None).train) == _kwargs( + loader_slots(batch_size=8, num_workers=2).train + ) + + +def test_persistent_workers_true_without_workers_is_refused() -> None: + """torch raises for this pairing at ITERATION time; the knob makes it a construction-time + error instead, which is the invariant the derived-only version protected by construction.""" + with pytest.raises(ValueError, match="persistent_workers=True needs num_workers > 0"): + loader_slots(batch_size=2, num_workers=0, persistent_workers=True) + + +def test_the_collate_is_the_batch_shape_choice() -> None: + """A detection consumer passes collate_list; the slot carries it verbatim.""" + assert ( + _kwargs(loader_slots(batch_size=2, num_workers=0, collate_fn=collate_list).train)["collate_fn"] is collate_list + ) + + +def test_further_loader_kwargs_pass_through_to_all_three() -> None: + """The code channel for the rest of the DataLoader surface (pin_memory, drop_last, ...).""" + slots = loader_slots(batch_size=2, num_workers=0, drop_last=True) + assert all(_kwargs(marker)["drop_last"] is True for marker in slots) + + +def test_shuffle_is_refused_as_a_shared_kwarg() -> None: + """It is the one PER-SPLIT kwarg the helper owns; the error names the way out.""" + with pytest.raises(ValueError, match="per-split"): + loader_slots(batch_size=2, num_workers=0, shuffle=False) + + +def test_a_slot_flows_into_a_working_loader() -> None: + """The whole point of the deferral: `flow(slot, dataset=...)` at run time yields batches.""" + stream = Stream(source=[{"x": float(i)} for i in range(4)]) + loader = flow(loader_slots(batch_size=2, num_workers=0).val, dataset=stream) + assert isinstance(loader, DataLoader) + batches = list(loader) + assert len(batches) == 2 + assert [float(v) for v in batches[0]["x"]] == [0.0, 1.0] + + +def test_the_module_is_not_reachable_from_the_package_root() -> None: + """It imports torch at module level, so the root must neither import nor advertise it — + that is what keeps `import recordstream` framework-free (the ops.torch pattern).""" + assert "loader_slots" not in recordstream.__all__ + root_source = (Path(recordstream.__file__)).read_text() + for node in ast.walk(ast.parse(root_source)): + if isinstance(node, ast.ImportFrom): + assert node.module != "recordstream.loaders", "the package root must not import recordstream.loaders" diff --git a/tests/test_module_layout.py b/tests/test_module_layout.py new file mode 100644 index 0000000..34c2cbc --- /dev/null +++ b/tests/test_module_layout.py @@ -0,0 +1,166 @@ +"""Layout invariants of the engine PACKAGES — `sources`, `core`, `flow`. + +Each was one oversized module and is now one module per cohesive unit (architecture §11/§12). +That split made three things load-bearing that a casual tidy-up would undo, and every one of +them fails SILENTLY in production: + +* the SUBMODULE path is the canonical ``!class:`` spelling, because ``cls.__module__`` is what + every generator writes — so nobody may pin ``__module__`` back to the package (that also + breaks ``confluid.registry.key_for``, whose miss only surfaces once a namesake registers); +* ``__init__.py``'s ``__all__`` is what a visual editor's node bridge walks, because + ``discovery.scan_module`` filters on ``__module__`` and now sees nothing in a package; +* the IMPORT DIRECTION inside ``core`` / ``flow`` — ``core`` is the bottom of the op-facing + layer (architecture §5), and a top-level import pointing the wrong way closes a cycle. +""" + +import ast +import importlib +import inspect +from pathlib import Path +from typing import Any, List, Set + +import pytest +from confluid.pydantic_export import _qualname +from confluid.registry import get_registry, resolve_class + +import recordstream.core as core_pkg +import recordstream.flow as flow_pkg +import recordstream.sources as sources_pkg +from recordstream.core import FilterOp, JointStream, Stream, WrappedOp +from recordstream.discovery import scan_module +from recordstream.flow import FlowGraph +from recordstream.sources import ConcatSource, DatasetSplit, HuggingFaceSource, RangeSource + +#: Every public engine class and the module it must be DEFINED in (not merely re-exported from). +CLASS_MODULES = { + HuggingFaceSource: "recordstream.sources.huggingface", + DatasetSplit: "recordstream.sources.split", + RangeSource: "recordstream.sources.range", + ConcatSource: "recordstream.sources.concat", + Stream: "recordstream.core.stream", + JointStream: "recordstream.core.stream", + FilterOp: "recordstream.core.wrappers", + WrappedOp: "recordstream.core.wrappers", + FlowGraph: "recordstream.flow.graph", +} + +#: package -> the submodules whose @configurable classes it must re-export. +PACKAGES = { + sources_pkg: ["huggingface", "split", "range", "concat"], + core_pkg: ["families", "mapstyle", "wrappers", "stream"], + flow_pkg: ["steps", "parse", "execute", "graph"], +} + + +def _module_level_imports(module_name: str) -> Set[str]: + """The modules ``module_name`` imports at MODULE level (body-local imports excluded). + + Body-local imports are the sanctioned seam for the one direction that must stay lazy, so + only top-level statements count here. + """ + source = Path(inspect.getsourcefile(importlib.import_module(module_name)) or "").read_text() + imported: Set[str] = set() + for node in ast.parse(source).body: # top level only — never recurse into function bodies + if isinstance(node, ast.Import): + imported.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module and node.level == 0: + imported.add(node.module) + return imported + + +@pytest.mark.parametrize("cls,module", list(CLASS_MODULES.items()), ids=lambda v: getattr(v, "__name__", v)) +def test_each_class_lives_in_its_declared_module(cls: type, module: str) -> None: + """``__module__`` reports the defining submodule — never the package.""" + assert cls.__module__ == module + # Pinning __module__ back to the package would make the assert above pass by lying; + # getsource is what catches that (it searches the named module's file for the definition). + assert inspect.getsource(cls).lstrip().startswith("@configurable") + + +@pytest.mark.parametrize("cls,module", list(CLASS_MODULES.items()), ids=lambda v: getattr(v, "__name__", v)) +def test_the_canonical_class_path_is_the_submodule_one(cls: type, module: str) -> None: + """``_qualname`` is what a generated config / form-spec / enrichment key spells out.""" + assert _qualname(cls) == f"{module}.{cls.__name__}" + assert resolve_class(_qualname(cls)) is cls + + +@pytest.mark.parametrize("cls", list(CLASS_MODULES), ids=lambda v: v.__name__) +def test_the_package_re_export_still_resolves(cls: type) -> None: + """An older hand-written config spelling the PACKAGE path keeps loading.""" + package = cls.__module__.rsplit(".", 1)[0] + assert resolve_class(f"{package}.{cls.__name__}") is cls + + +@pytest.mark.parametrize("cls", list(CLASS_MODULES), ids=lambda v: v.__name__) +def test_pinning_module_would_break_the_registry_lookup(cls: type) -> None: + """``key_for`` re-derives ``module.qualname``; a rewritten ``__module__`` misses the entry. + + This is the failure mode that ruled out keeping the old paths by pinning ``__module__`` — + it is silent, surfacing only as an ambiguous ``!class:`` tag once a namesake registers. + """ + assert get_registry().key_for(cls) is not None + + +@pytest.mark.parametrize("package,submodules", list(PACKAGES.items()), ids=lambda v: getattr(v, "__name__", "")) +def test_every_configurable_is_in_all(package: Any, submodules: List[str]) -> None: + """``__all__`` is the ONLY pass that surfaces these as palette nodes — see the module docstring.""" + exported = getattr(package, "__all__", []) + for name in submodules: + module_path = f"{package.__name__}.{name}" + module = importlib.import_module(module_path) + for attr, member in vars(module).items(): + if attr.startswith("_") or not isinstance(member, type): + continue + if getattr(member, "__confluid_configurable__", False) and member.__module__ == module_path: + assert attr in exported, f"{module_path}.{attr} is @configurable but not in __all__" + + +@pytest.mark.parametrize("package", list(PACKAGES), ids=lambda v: getattr(v, "__name__", "")) +def test_scan_module_no_longer_sees_the_package(package: Any) -> None: + """The reason ``__all__`` matters: the ``__module__`` filter finds nothing in a package.""" + assert scan_module(package.__name__) == [] + + +def test_the_engine_internals_stay_importable_from_the_package() -> None: + """``ops/`` reaches the op dispatch as ``from recordstream.core import _apply_op``. + + The private re-exports in ``core/__init__.py`` are a deliberate cross-module surface, not + stray imports — dropping one breaks every composing op at run time, not at import. + """ + for name in ("_apply_op", "_op_expands", "_expand", "_extra_op_families", "_sync_op_families", "_worker_task"): + assert hasattr(core_pkg, name), f"recordstream.core.{name} is the engine's internal surface" + # A mutable list re-exported by IDENTITY — this is what lets the suite's registry fixture + # restore the real families with `core._OP_FAMILIES[:] = snapshot`. + from recordstream.core.families import _OP_FAMILIES + + assert core_pkg._OP_FAMILIES is _OP_FAMILIES + + +def test_core_families_is_the_bottom_of_the_op_facing_layer() -> None: + """``core.families`` imports NOTHING from its siblings — architecture §5's whole argument. + + Every composing op imports ``_apply_op`` from here; a top-level import back into the engine + (or into ``flow``) would close the cycle that record exists to prevent. + """ + imported = _module_level_imports("recordstream.core.families") + offenders = [m for m in imported if m.startswith(("recordstream.core.", "recordstream.flow", "recordstream.ops"))] + assert offenders == [], f"core.families must not import {offenders} at module level" + + +def test_flow_reaches_core_only_through_the_dispatch_layer() -> None: + """``flow`` may import ``core.families``; importing ``core.stream`` would invert the layering. + + ``core.stream`` reaches ``flow`` through BODY-LOCAL imports precisely so this direction can + stay a module-level one. + """ + for name in ("steps", "parse", "execute", "graph"): + imported = _module_level_imports(f"recordstream.flow.{name}") + assert "recordstream.core.stream" not in imported, f"flow.{name} must not import core.stream at module level" + assert "recordstream.core" not in imported, f"flow.{name} must reach the dispatch via core.families" + + +def test_core_stream_defers_its_flow_imports() -> None: + """The one direction that MUST stay lazy: ``core.stream`` -> ``flow`` is body-local only.""" + imported = _module_level_imports("recordstream.core.stream") + offenders = [m for m in imported if m.startswith("recordstream.flow")] + assert offenders == [], f"core.stream must import flow inside function bodies, not {offenders}" diff --git a/tests/test_node_docs.py b/tests/test_node_docs.py new file mode 100644 index 0000000..61e952f --- /dev/null +++ b/tests/test_node_docs.py @@ -0,0 +1,74 @@ +"""Guard: every node-facing recordstream Source/Op documents all its constructor params. + +These classes surface in visual editors (as widget tooltips) and MCP form-specs (as +pydantic ``Field(description=...)``) purely from their docstring ``Args:`` block — see +``confluid.parse_param_docs``. A param that loses its doc silently loses its +tooltip/description, so this pins the coverage. +""" + +import inspect +from typing import List + +import pytest +from confluid import parse_param_docs # type: ignore[import-not-found] + +from recordstream import Pipeline, Transform +from recordstream.core import FilterOp, JointStream, Stream, WrappedOp +from recordstream.ops.configure import ConfigureOp +from recordstream.ops.debug import PrintRecordOp +from recordstream.ops.enable import Enable +from recordstream.ops.formula import FormulaOp +from recordstream.ops.image import ConvertToImage +from recordstream.ops.numpy import ConnectedComponents, Threshold +from recordstream.ops.parallel import Parallel +from recordstream.ops.random_apply import RandomApply +from recordstream.ops.structure import CopyField, DropField, RenameField, SelectFields +from recordstream.ops.target import CocoToTorchVisionDetection, DecodeTarget, EncodeTarget, MasksToDetectionBoxes +from recordstream.ops.torch import ToTensor +from recordstream.sources import HuggingFaceSource + +_NODE_CLASSES = [ + HuggingFaceSource, + Stream, + JointStream, + FilterOp, + WrappedOp, + Transform, + Pipeline, + Threshold, + ConnectedComponents, + ConvertToImage, + ToTensor, + EncodeTarget, + DecodeTarget, + CocoToTorchVisionDetection, + MasksToDetectionBoxes, + RenameField, + DropField, + CopyField, + SelectFields, + ConfigureOp, + FormulaOp, + Enable, + Parallel, + RandomApply, + PrintRecordOp, +] + + +def _constructor_params(cls: type) -> List[str]: + # signature(cls) is the constructor signature (no ``self``), and it survives + # confluid's @configurable __init__ wrapping (verified against real classes). + sig = inspect.signature(cls) + return [ + name + for name, p in sig.parameters.items() + if p.kind not in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD) + ] + + +@pytest.mark.parametrize("cls", _NODE_CLASSES, ids=lambda c: c.__name__) +def test_all_constructor_params_documented(cls: type) -> None: + docs = parse_param_docs(cls) + missing = [p for p in _constructor_params(cls) if not docs.get(p)] + assert not missing, f"{cls.__name__} is missing Args docs for: {missing}" diff --git a/tests/test_op_families.py b/tests/test_op_families.py new file mode 100644 index 0000000..1479023 --- /dev/null +++ b/tests/test_op_families.py @@ -0,0 +1,586 @@ +"""The engine's op-FAMILY dispatch (``core._apply_op``) — libraries run AS-IS, end to end. + +Pins the record-model headline: native type-dispatched ops, BARE albumentations transforms +(kwarg-vocabulary call, one joint draw, item re-wrap), and BARE torchvision ``transforms.v2`` +transforms (dict call) all sit in ONE ``Stream.ops`` list with no wrapper/adapter classes — +plus the family classifiers, YAML mapping-form ops docs, spawn-parallel with a bare library +op, ``field=`` targeting, and the ``WrappedOp``/``FilterOp`` raw-callable routes. +""" + +from contextlib import contextmanager +from pathlib import Path +from typing import Dict, Iterator, List, Optional + +import albumentations as A +import numpy as np +import pytest +import torch +from confluid import configurable +from torchvision.transforms import v2 + +from recordstream import Boxes, FilterOp, Image, Label, Mask, Pipeline, Record, Transform, WrappedOp +from recordstream.core import Stream, _apply_op, _is_albumentations, _is_torchvision_v2 + + +# --------------------------------------------------------------------------- # +# Module-level fixtures (spawn workers pickle records, ops, and callables). +# --------------------------------------------------------------------------- # +def _base_record(i: int = 0) -> Record: + rng = np.random.default_rng(i) + return { + "image": Image(rng.random((16, 20, 3)).astype(np.float32)), + "mask": Mask((rng.random((16, 20)) > 0.5).astype(np.uint8)), + "class": Label("drone_x", classes=["noise", "drone_x"]), + "gain_db": -3.0, + } + + +def spawn_records() -> List[Record]: + """Module-level source fixture so the spawn-parallel test's records pickle.""" + return [_base_record(i) for i in range(4)] + + +def keep_even_gain(record: Record) -> bool: + """Module-level FilterOp predicate (pickles across spawn workers).""" + return int(record["idx"]) % 2 == 0 + + +@configurable(category="op") +class AddOffset(Transform): + """Adds a fixed offset to every Image value (Awgn-style configurable native op). + + Args: + offset: The value added to every Image payload. + field: Apply only to this record key. None (default) = every Image value. + """ + + handles = (Image,) + + def __init__(self, offset: float = 0.0, field: Optional[str] = None) -> None: + super().__init__(field=field) + self.offset = float(offset) + + def get_params(self, record: Record) -> Dict[str, float]: + return {"offset": self.offset} + + +@AddOffset.kernel(Image) +def _add_offset_image(value: Image, params: Dict[str, float]) -> Image: + return Image(np.asarray(value) + params["offset"], layout=value.layout) + + +# --------------------------------------------------------------------------- # +# Family classifiers. +# --------------------------------------------------------------------------- # +class TestFamilyClassifiers: + def test_is_albumentations_positive(self) -> None: + assert _is_albumentations(A.HorizontalFlip(p=1.0)) + assert _is_albumentations(A.Compose([A.HorizontalFlip(p=1.0)])) + + def test_is_albumentations_negative(self) -> None: + assert not _is_albumentations(v2.RandomCrop(4)) + assert not _is_albumentations(AddOffset()) + assert not _is_albumentations({"image": None}) + assert not _is_albumentations(lambda r: r) + + def test_is_torchvision_v2_positive(self) -> None: + assert _is_torchvision_v2(v2.RandomCrop(4)) + assert _is_torchvision_v2(v2.ToImage()) + + def test_is_torchvision_v2_negative(self) -> None: + assert not _is_torchvision_v2(A.HorizontalFlip(p=1.0)) + assert not _is_torchvision_v2(AddOffset()) + assert not _is_torchvision_v2(object()) + + +# --------------------------------------------------------------------------- # +# Albumentations family. +# --------------------------------------------------------------------------- # +class TestAlbumentationsFamily: + def test_joint_move_under_one_bare_compose_with_bboxes(self) -> None: + # Box-carrying augmentation is albumentations' own Compose(bbox_params=...) dropped in + # BARE — ONE joint draw moves image + mask + bboxes together; item types survive. + record = {**_base_record(), "bboxes": [[2, 3, 6, 7]], "labels": ["drone"]} + flip = A.Compose( + [A.HorizontalFlip(p=1.0)], + bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"]), + ) + out = _apply_op(record, flip) + assert out is not None + assert np.array_equal(np.asarray(out["image"]), np.asarray(record["image"])[:, ::-1]) + assert np.array_equal(np.asarray(out["mask"]), np.asarray(record["mask"])[:, ::-1]) + assert [round(v) for v in out["bboxes"][0]] == [14, 3, 18, 7] # W=20: x -> W-x + assert isinstance(out["image"], Image) and isinstance(out["mask"], Mask) # re-wrapped + assert out["class"].value == "drone_x" and out["gain_db"] == -3.0 # non-alb keys untouched + + def test_zero_known_keys_is_passthrough(self) -> None: + # A record with NO albumentations-vocabulary keys passes through unchanged. + record = {"spec": Mask(np.zeros((4, 4))), "gain_db": 1.0} + out = _apply_op(record, A.HorizontalFlip(p=1.0)) + assert out is record + + def test_extra_record_entries_never_reach_the_library(self) -> None: + # Scalars / Labels are not in _ALB_KEYS — the op receives only image/mask and the + # extras ride through verbatim (same objects). + record = _base_record() + out = _apply_op(record, A.HorizontalFlip(p=1.0)) + assert out is not None + assert out["class"] is record["class"] and out["gain_db"] == -3.0 + + +# --------------------------------------------------------------------------- # +# Mixed ops list end-to-end through Stream. +# --------------------------------------------------------------------------- # +class TestMixedOpsList: + _OPS = [ + AddOffset(offset=0.5), # native type-dispatched op + A.HorizontalFlip(p=1.0), # bare albumentations + v2.ToImage(), # bare torchvision v2: explicit numpy HWC -> CHW tv_tensor conversion + v2.RandomCrop(4), # bare torchvision v2 + ] + + def test_iteration(self) -> None: + stream = Stream(source=[_base_record()], ops=list(self._OPS)) + (out,) = list(stream) + assert isinstance(out["image"], torch.Tensor) + assert tuple(out["image"].shape) == (3, 4, 4) + assert out["class"].value == "drone_x" # rode through every family untouched + + def test_random_access(self) -> None: + stream = Stream(source=[_base_record(0), _base_record(1)], ops=list(self._OPS)) + out = stream[1] + assert isinstance(out["image"], torch.Tensor) and tuple(out["image"].shape) == (3, 4, 4) + + def test_pipeline_nests_the_same_families(self) -> None: + # The same mixed list nested inside Pipeline (which routes through _apply_op). + out = Pipeline(list(self._OPS))(_base_record()) + assert out is not None and tuple(out["image"].shape) == (3, 4, 4) + + +# --------------------------------------------------------------------------- # +# YAML ops docs — mapping-form bare library entries + configurable ctor binding. +# --------------------------------------------------------------------------- # +class TestOpsYaml: + def test_mapping_form_bare_albumentations_entry(self, tmp_path: Path) -> None: + path = tmp_path / "ops.yaml" + path.write_text("ops:\n - !class:albumentations.HorizontalFlip {p: 1.0}\n") + record = _base_record() + stream = Stream.from_ops_yaml(str(path), source=[record]) + (out,) = list(stream) + assert np.array_equal(np.asarray(out["image"]), np.asarray(record["image"])[:, ::-1]) + assert isinstance(out["image"], Image) + + def test_configurable_ctor_param_bound_from_yaml(self, tmp_path: Path) -> None: + # An Awgn-style native op with a ctor param set in the YAML doc: the value reaches + # the constructor and the op output reflects it. + path = tmp_path / "ops.yaml" + path.write_text("ops:\n - !class:tests.test_op_families.AddOffset {offset: 3.0}\n") + record = {"image": Image(np.zeros((2, 3, 3), dtype=np.float32))} + stream = Stream.from_ops_yaml(str(path), source=[record]) + (out,) = list(stream) # a @configurable entry stays a deferred marker until route entry + assert np.allclose(np.asarray(out["image"]), 3.0) + (op,) = stream.ops # _check_ops_materialized flowed + cached the live op in place + assert isinstance(op, AddOffset) and op.offset == 3.0 + + +# --------------------------------------------------------------------------- # +# Spawn-parallel with a bare albumentations op (+ FilterOp drop on the parallel route). +# --------------------------------------------------------------------------- # +def test_spawn_parallel_with_bare_albumentations_op() -> None: + records = [{**r, "idx": i} for i, r in enumerate(spawn_records())] + stream = Stream(source=records, ops=[A.HorizontalFlip(p=1.0), FilterOp(keep_even_gain)]).parallel(2) + results = stream.collect() + assert [int(r["idx"]) for r in results] == [0, 2] # FilterOp dropped odd records in workers + for out, want in zip(results, [records[0], records[2]]): + assert isinstance(out["image"], Image) # item type survived pickle + re-wrap + assert np.array_equal(np.asarray(out["image"]), np.asarray(want["image"])[:, ::-1]) + + +# --------------------------------------------------------------------------- # +# field= targeting. +# --------------------------------------------------------------------------- # +def test_field_targets_one_of_two_image_keys() -> None: + record = { + "a": Image(np.zeros((2, 2, 3), dtype=np.float32)), + "b": Image(np.zeros((2, 2, 3), dtype=np.float32)), + } + out = AddOffset(offset=1.0, field="a")(record) + assert out is not None + assert np.allclose(np.asarray(out["a"]), 1.0) # pinned key moved + assert np.allclose(np.asarray(out["b"]), 0.0) # sibling untouched + + +# --------------------------------------------------------------------------- # +# Raw-callable routes: WrappedOp / Stream.map / FilterOp drops everywhere. +# --------------------------------------------------------------------------- # +def double(x: np.ndarray) -> np.ndarray: + """Module-level payload function for WrappedOp (stored as an importable path).""" + return x * 2 + + +def bump_gain(record: Record) -> Record: + """Module-level whole-record function for WrappedOp(key=None).""" + return {**record, "gain_db": record["gain_db"] + 1.0} + + +class TestWrappedOpAndMap: + def test_key_targets_payload_and_preserves_item(self) -> None: + record = {"m": Mask(np.ones((2, 2))), "gain_db": 0.0} + out = WrappedOp(double, key="m")(record) + assert out is not None + assert isinstance(out["m"], Mask) and np.allclose(np.asarray(out["m"]), 2.0) + assert record["m"] is not out["m"] # copy-on-write + + def test_key_on_plain_value_replaces_verbatim(self) -> None: + out = WrappedOp(double, key="g")({"g": 3}) + assert out is not None and out["g"] == 6 # plain value: no item to re-wrap + + def test_key_none_receives_whole_record(self) -> None: + out = WrappedOp(bump_gain)({"gain_db": -3.0}) + assert out is not None and out["gain_db"] == -2.0 + + def test_missing_key_raises(self) -> None: + with pytest.raises(KeyError, match="nope"): + WrappedOp(double, key="nope")({"m": Mask(np.ones(2))}) + + def test_stream_map_key(self) -> None: + stream = Stream(source=[{"m": Mask(np.ones((2, 2)))}]).map(double, key="m") + (out,) = list(stream) + assert isinstance(out["m"], Mask) and np.allclose(np.asarray(out["m"]), 2.0) + + +class TestFilterDropRoutes: + def test_sequential_iteration_drops(self) -> None: + records = [{"i": 0}, {"i": 1}, {"i": 2}] + stream = Stream(source=records, ops=[FilterOp(lambda r: r["i"] != 1)]) + assert [r["i"] for r in stream] == [0, 2] + + def test_getitem_on_filtered_record_raises_index_error(self) -> None: + stream = Stream(source=[{"i": 0}], ops=[FilterOp(lambda r: False)]) + with pytest.raises(IndexError, match="filtered out"): + stream[0] + + def test_pipeline_propagates_drop(self) -> None: + assert Pipeline([FilterOp(lambda r: False)])({"i": 0}) is None + + def test_stream_filter_helper(self) -> None: + stream = Stream(source=[{"i": 0}, {"i": 1}]).filter(lambda r: r["i"] > 0) + assert [r["i"] for r in stream] == [1] + + def test_unset_predicate_raises_lazily(self) -> None: + with pytest.raises(ValueError, match="predicate"): + FilterOp()({"i": 0}) + + +# --------------------------------------------------------------------------- # +# The open op-family registry (register_op_family) — third-party libraries +# --------------------------------------------------------------------------- # +class FakeLibScale: + """Stands in for a foreign library's op type — deliberately NOT record-callable, + so a test passing proves dispatch went through the registered invoker.""" + + def __init__(self, factor: float = 2.0) -> None: + self.factor = factor + + +def is_fakelib(op: object) -> bool: + """Module-level matcher (pickles by reference for the spawn test).""" + return isinstance(op, FakeLibScale) + + +def invoke_fakelib(record: Record, op: FakeLibScale) -> Record: + """Module-level invoker — the fake library's calling convention.""" + return {**record, "gain_db": record["gain_db"] * op.factor} + + +def invoke_fakelib_override(record: Record, op: FakeLibScale) -> Record: + """A second invoker for the shadowing / replacement tests.""" + return {**record, "gain_db": -999.0} + + +@pytest.fixture() +def family_registry() -> Iterator[None]: + """Snapshot/restore the global registry so registrations never leak between tests.""" + from recordstream import core + + snapshot = list(core._OP_FAMILIES) + yield + core._OP_FAMILIES[:] = snapshot + + +class TestOpFamilyRegistry: + def test_builtins_are_registered_through_the_same_registry(self) -> None: + from recordstream import registered_op_families + + assert registered_op_families()[:2] == ("albumentations", "torchvision_v2") + + def test_registered_family_dispatches_via_invoker(self, family_registry: None) -> None: + from recordstream import register_op_family + + register_op_family("fakelib", is_fakelib, invoke_fakelib) + out = _apply_op(_base_record(), FakeLibScale(factor=3.0)) + assert out is not None and out["gain_db"] == -9.0 # -3.0 * 3 — via the invoker, op never called + assert isinstance(out["image"], Image) # rest of the record untouched + + def test_registered_family_runs_in_stream_ops_list(self, family_registry: None) -> None: + from recordstream import register_op_family + + register_op_family("fakelib", is_fakelib, invoke_fakelib) + out = list(Stream(source=[_base_record()], ops=[FakeLibScale(factor=2.0), lambda r: {**r, "tag": 1}])) + assert out[0]["gain_db"] == -6.0 and out[0]["tag"] == 1 # mixes with native ops in ONE list + + def test_last_registered_family_wins_overlap(self, family_registry: None) -> None: + from recordstream import register_op_family + + register_op_family("fakelib", is_fakelib, invoke_fakelib) + register_op_family("fakelib_specific", is_fakelib, invoke_fakelib_override) # same matcher, later + out = _apply_op(_base_record(), FakeLibScale()) + assert out is not None and out["gain_db"] == -999.0 + + def test_reregistering_name_replaces_in_place(self, family_registry: None) -> None: + from recordstream import register_op_family, registered_op_families + + register_op_family("fakelib", is_fakelib, invoke_fakelib) + n = len(registered_op_families()) + register_op_family("fakelib", is_fakelib, invoke_fakelib_override) + assert len(registered_op_families()) == n # replaced, not duplicated + out = _apply_op(_base_record(), FakeLibScale()) + assert out is not None and out["gain_db"] == -999.0 + + def test_unmatched_op_falls_back_to_native_call(self, family_registry: None) -> None: + out = _apply_op(_base_record(), lambda r: {**r, "native": True}) + assert out is not None and out["native"] is True + + def test_spawn_parallel_ships_family_to_workers(self, family_registry: None) -> None: + from recordstream import register_op_family + + register_op_family("fakelib", is_fakelib, invoke_fakelib) + stream = Stream(source=spawn_records(), ops=[FakeLibScale(factor=2.0)]).parallel(2) + results = list(stream) + assert len(results) == 4 + assert all(r["gain_db"] == -6.0 for r in results) # invoker ran INSIDE the workers + + +class TestFormulaReducers: + def test_array_reducers_are_function_style(self) -> None: + # amax/amin/mean/std/median are pre-bound numpy callables in the sandbox namespace. + from recordstream.ops.formula import FormulaOp + + rec = {"image": Image(np.arange(16, dtype=np.float32).reshape(4, 4) / 15.0)} + out = FormulaOp(formula="amax(a) * 0.5", field="image")(rec) + assert float(np.asarray(out["image"])) == pytest.approx(0.5) + + def test_attribute_reduction_is_not_part_of_the_contract(self) -> None: + # a.max() depends on numpy's lazy-import cache (KeyError '__import__' in a cold + # process): the FUNCTION form is the sanctioned spelling. We only pin that the + # function form never regresses; the attribute form is deliberately unpinned. + from recordstream.ops.formula import _FORMULA_NAMESPACE + + assert {"amax", "amin", "mean", "std", "median"} <= set(_FORMULA_NAMESPACE) + + +@contextmanager +def _captured_warnings() -> Iterator[List[str]]: + """Collect loggair WARNING records emitted inside the block. + + `caplog` cannot see these — loggair is loguru, which does not propagate to stdlib logging — + and its sink is ENQUEUED, so reading a captured stream races the writer. `logger.complete()` + is the deterministic flush (the workspace forbids sleeping for one). + """ + from loguru import logger + + collected: List[str] = [] + sink_id = logger.add(lambda message: collected.append(str(message)), level="WARNING") + try: + yield collected + logger.complete() + finally: + logger.remove(sink_id) + + +class TestGeometryLeavingBoxesBehind: + """A `Boxes` is not in albumentations' key vocabulary, so it never reaches the library. + + That is correct for the dispatch — passing a foreign item would break the call — but it + means a geometry-changing transform moves the pixels while the boxes stay put, with no + error of its own. Measured: `A.Resize` takes a 200x200 image to 64x64 and leaves the boxes + on `[10, 10, 100, 100]`; `A.HorizontalFlip` mirrors the pixels while changing NO shape at + all, which is why the condition is the library's spatial/photometric taxonomy rather than + "did the raster change". + """ + + @staticmethod + def _record() -> Record: + return { + "image": Image(np.zeros((200, 200, 3), dtype="uint8")), + "target": Boxes(boxes=np.array([[10.0, 10.0, 100.0, 100.0]]), labels=np.array([1])), + } + + @pytest.fixture(autouse=True) + def _forget_previous_warnings(self) -> Iterator[None]: + """The once-per-type memo is module state — clear it so tests do not shadow each other.""" + from recordstream.core.families import _WARNED_SPATIAL + + snapshot = set(_WARNED_SPATIAL) + _WARNED_SPATIAL.clear() + yield + _WARNED_SPATIAL.clear() + _WARNED_SPATIAL.update(snapshot) + + def test_the_desync_is_real_and_silent_without_the_guard(self) -> None: + """The premise, asserted rather than assumed: pixels move, boxes do not.""" + out = _apply_op(self._record(), A.Resize(height=64, width=64)) + assert out is not None + assert out["image"].shape[:2] == (64, 64) + assert out["target"].boxes.tolist() == [[10.0, 10.0, 100.0, 100.0]], "boxes stayed behind" + + def test_a_resize_warns_naming_both_ways_out(self) -> None: + with _captured_warnings() as warnings: + _apply_op(self._record(), A.Resize(height=64, width=64)) + assert len(warnings) == 1 + assert "bbox_params" in warnings[0] and "ResizeDetection" in warnings[0] + + def test_a_flip_warns_though_NO_shape_changes(self) -> None: + with _captured_warnings() as warnings: + _apply_op(self._record(), A.HorizontalFlip(p=1.0)) + assert len(warnings) == 1, "a raster-change test would miss this one entirely" + + def test_an_image_only_transform_stays_silent(self) -> None: + """`Normalize` is an `ImageOnlyTransform` — it cannot touch geometry, so there is + nothing to warn about. Reading the library's own taxonomy is what makes this exact.""" + with _captured_warnings() as warnings: + _apply_op(self._record(), A.Normalize()) + assert warnings == [] + + def test_a_compose_is_recursed(self) -> None: + composed = A.Compose([A.Normalize(), A.RandomCrop(height=8, width=8)]) + with _captured_warnings() as warnings: + _apply_op(self._record(), composed) + assert len(warnings) == 1, "the spatial transform is nested one level down" + + def test_it_warns_once_per_transform_type(self) -> None: + with _captured_warnings() as warnings: + _apply_op(self._record(), A.Resize(height=64, width=64)) + _apply_op(self._record(), A.Resize(height=32, width=32)) + assert len(warnings) == 1, "the message is about the configuration, not the record" + + def test_the_DOCUMENTED_yaml_way_out_actually_runs(self) -> None: + """The warning names a fix, so the fix has to work — this is that exact YAML. + + Note it must go through a `Stream`: deferred `!class:` markers are flowed at route + entry, so applying them straight out of `confluid.load` hands `_apply_op` a marker. + """ + import confluid + + document = """ +ops: + - !class:recordstream.ops.structure.RenameField { src: my_boxes, dst: bboxes } + - !class:albumentations.Compose + transforms: [!class:albumentations.HorizontalFlip { p: 1.0 }] + bbox_params: !class:albumentations.BboxParams { format: pascal_voc, label_fields: [labels] } +""" + record = { + "image": np.zeros((100, 100, 3), dtype="uint8"), + "my_boxes": [[10.0, 10.0, 40.0, 40.0]], + "labels": [1], + } + with _captured_warnings() as warnings: + out = list(Stream(source=[record], ops=confluid.load(document)["ops"]))[0] + assert warnings == [] + assert [round(v, 1) for v in out["bboxes"][0]] == [60.0, 10.0, 90.0, 40.0], "mirrored across x" + + def test_the_correct_spelling_is_NOT_warned_about_and_moves_the_boxes(self) -> None: + """Boxes in the library's own vocabulary: it moves them in the same joint draw.""" + composed = A.Compose( + [A.Resize(height=64, width=64)], + bbox_params=A.BboxParams(format="pascal_voc", label_fields=["labels"]), + ) + record = { + "image": np.zeros((200, 200, 3), dtype="uint8"), + "bboxes": [[10.0, 10.0, 100.0, 100.0]], + "labels": [1], + } + with _captured_warnings() as warnings: + out = _apply_op(record, composed) + assert warnings == [] + assert out is not None + assert [round(v, 1) for v in out["bboxes"][0]] == [3.2, 3.2, 32.0, 32.0], "boxes scaled with the image" + + +class TestV2GeometryLeavingBoxesBehind: + """The same gap in the OTHER family, reached by a different route. + + albumentations misses a `Boxes` because it is not in the KEY vocabulary; torchvision v2 + misses it because it is not one of v2's tv_tensor TYPES. Measured: `v2.Resize((64, 64))` + takes a 200x200 image to 64x64 with the boxes still on `[10, 10, 100, 100]`, while the same + transform over a `tv_tensors.BoundingBoxes` rescales them to `[3.2, 3.2, 32, 32]`. + """ + + @staticmethod + def _record() -> Record: + return { + "image": Image(np.zeros((200, 200, 3), dtype="uint8")), + "target": Boxes(boxes=torch.tensor([[10.0, 10.0, 100.0, 100.0]]), labels=torch.tensor([1])), + } + + @pytest.fixture(autouse=True) + def _forget_previous_warnings(self) -> Iterator[None]: + from recordstream.core.families import _WARNED_SPATIAL + + snapshot = set(_WARNED_SPATIAL) + _WARNED_SPATIAL.clear() + yield + _WARNED_SPATIAL.clear() + _WARNED_SPATIAL.update(snapshot) + + def test_the_desync_is_real(self) -> None: + out = _apply_op(self._record(), v2.Compose([v2.ToImage(), v2.Resize((64, 64))])) + assert out is not None + assert tuple(out["image"].shape[-2:]) == (64, 64) + assert out["target"].boxes.tolist() == [[10.0, 10.0, 100.0, 100.0]], "boxes stayed behind" + + def test_a_geometric_transform_warns_naming_the_way_out(self) -> None: + with _captured_warnings() as warnings: + _apply_op(self._record(), v2.RandomHorizontalFlip(p=1.0)) + assert len(warnings) == 1 + assert "BoundingBoxes" in warnings[0] and "ResizeDetection" in warnings[0] + + def test_a_non_geometric_transform_stays_silent(self) -> None: + with _captured_warnings() as warnings: + _apply_op(self._record(), v2.ColorJitter(brightness=0.5)) + assert warnings == [] + + def test_a_compose_is_recursed(self) -> None: + with _captured_warnings() as warnings: + _apply_op(self._record(), v2.Compose([v2.ToImage(), v2.Resize((32, 32))])) + assert len(warnings) == 1 + + def test_v2s_OWN_box_type_is_transformed_and_not_warned_about(self) -> None: + from torchvision import tv_tensors + + record = { + "image": tv_tensors.Image(torch.zeros(3, 200, 200, dtype=torch.uint8)), + "boxes": tv_tensors.BoundingBoxes( + torch.tensor([[10.0, 10.0, 100.0, 100.0]]), format="XYXY", canvas_size=(200, 200) + ), + } + with _captured_warnings() as warnings: + out = _apply_op(record, v2.Resize((64, 64))) + assert warnings == [] + assert out is not None + assert [round(v, 1) for v in out["boxes"].tolist()[0]] == [3.2, 3.2, 32.0, 32.0] + + def test_the_geometry_signal_still_matches_this_torchvision(self) -> None: + """The signal is a PRIVATE module path, so it can go stale on a torchvision upgrade. + + It fails OPEN (no warning, nothing else changes), which is the right direction for a + diagnostic but also the direction that rots unnoticed — so assert the classification + directly rather than only through a warning that would silently stop appearing. + """ + from recordstream.core.families import _is_v2_geometry + + assert _is_v2_geometry(v2.Resize((8, 8))) + assert _is_v2_geometry(v2.RandomHorizontalFlip()) + assert _is_v2_geometry(v2.RandomCrop(8)) + assert not _is_v2_geometry(v2.ColorJitter()) + assert not _is_v2_geometry(v2.Normalize(mean=[0.0], std=[1.0])) diff --git a/tests/test_ops.py b/tests/test_ops.py deleted file mode 100644 index a13ed3b..0000000 --- a/tests/test_ops.py +++ /dev/null @@ -1,723 +0,0 @@ -"""Tests for dataflux.ops: torch and numpy variants.""" - -import os - -import numpy as np -import pytest -import torch -from PIL import Image - -from dataflux.ops import ( - CopyInputOp, - CopyMetadataOp, - CopySampleOp, - CopyTargetOp, - RescaleOp, - StandardizeOp, - StashInputOp, - SwapInputTargetOp, - Tee, - ToTensorOp, - UnstashInputOp, -) -from dataflux.ops import numpy as np_ops -from dataflux.sample import Sample - -# --------------------------------------------------------------------------- -# ToTensorOp -# --------------------------------------------------------------------------- - - -class TestToTensorOp: - """Tests for ToTensorOp.""" - - def test_pil_image_with_normalize(self) -> None: - img = Image.fromarray(np.full((28, 28), 128, dtype=np.uint8)) - result = ToTensorOp(normalize=True)(Sample(input=img)) - assert isinstance(result.input, torch.Tensor) - assert result.input.dtype == torch.float32 - assert result.input.max() <= 1.0 - - def test_pil_image_without_normalize(self) -> None: - img = Image.fromarray(np.full((28, 28), 200, dtype=np.uint8)) - result = ToTensorOp(normalize=False)(Sample(input=img)) - assert isinstance(result.input, torch.Tensor) - assert result.input.dtype == torch.uint8 - assert result.input.max() == 200 - - def test_2d_array_adds_channel_dim(self) -> None: - arr = np.zeros((28, 28), dtype=np.uint8) - result = ToTensorOp(normalize=False)(Sample(input=arr)) - assert result.input.shape == (1, 28, 28) - - def test_3d_array_transposes(self) -> None: - arr = np.zeros((28, 28, 3), dtype=np.uint8) - result = ToTensorOp(normalize=False)(Sample(input=arr)) - assert result.input.shape == (3, 28, 28) - - def test_normalize_float_above_one(self) -> None: - arr = np.array([[128.0, 255.0]], dtype=np.float32) - result = ToTensorOp(normalize=True)(Sample(input=arr)) - assert result.input.max() == 1.0 - - def test_non_array_passthrough(self) -> None: - t = torch.tensor([1.0, 2.0]) - result = ToTensorOp(normalize=False)(Sample(input=t)) - assert torch.equal(result.input, t) - - def test_preserves_target_and_metadata(self) -> None: - arr = np.zeros((28, 28), dtype=np.uint8) - result = ToTensorOp()(Sample(input=arr, target=5, metadata={"k": "v"})) - assert result.target == 5 - assert result.metadata == {"k": "v"} - - -# --------------------------------------------------------------------------- -# Torch RescaleOp -# --------------------------------------------------------------------------- - - -class TestRescaleOp: - """Tests for torch RescaleOp.""" - - def test_default_output_range(self) -> None: - tensor = torch.tensor([0.0, 128.0, 255.0]) - result = RescaleOp(in_min=0.0, in_max=255.0)(Sample(input=tensor)) - assert result.input[0] == 0.0 - assert abs(result.input[1] - 128.0 / 255.0) < 1e-6 - assert abs(result.input[2] - 1.0) < 1e-6 - - def test_custom_input_range(self) -> None: - tensor = torch.tensor([10.0, 55.0, 100.0]) - result = RescaleOp(in_min=10.0, in_max=100.0)(Sample(input=tensor)) - assert abs(result.input[0] - 0.0) < 1e-6 - assert abs(result.input[1] - 0.5) < 1e-6 - assert abs(result.input[2] - 1.0) < 1e-6 - - def test_custom_output_range(self) -> None: - tensor = torch.tensor([0.0, 0.5, 1.0]) - result = RescaleOp(in_min=0.0, in_max=1.0, out_min=10.0, out_max=20.0)(Sample(input=tensor)) - assert abs(result.input[0] - 10.0) < 1e-6 - assert abs(result.input[1] - 15.0) < 1e-6 - assert abs(result.input[2] - 20.0) < 1e-6 - - def test_clip_true_clamps(self) -> None: - tensor = torch.tensor([-50.0, 0.0, 128.0, 300.0]) - result = RescaleOp(in_min=0.0, in_max=255.0, clip=True)(Sample(input=tensor)) - assert result.input[0] == 0.0 - assert result.input[3] == 1.0 - - def test_clip_false_extrapolates(self) -> None: - tensor = torch.tensor([-255.0, 510.0]) - result = RescaleOp(in_min=0.0, in_max=255.0, clip=False)(Sample(input=tensor)) - assert abs(result.input[0] - (-1.0)) < 1e-6 - assert abs(result.input[1] - 2.0) < 1e-6 - - def test_uint8_converts_to_float(self) -> None: - tensor = torch.tensor([0, 128, 255], dtype=torch.uint8) - result = RescaleOp(in_min=0.0, in_max=255.0)(Sample(input=tensor)) - assert result.input.dtype == torch.float32 - assert abs(result.input[2] - 1.0) < 1e-6 - - def test_preserves_float64(self) -> None: - tensor = torch.tensor([0.0, 255.0], dtype=torch.float64) - result = RescaleOp(in_min=0.0, in_max=255.0)(Sample(input=tensor)) - assert result.input.dtype == torch.float64 - - def test_preserves_target_and_metadata(self) -> None: - tensor = torch.tensor([128.0]) - result = RescaleOp(in_min=0.0, in_max=255.0)(Sample(input=tensor, target=7, metadata={"key": "val"})) - assert result.target == 7 - assert result.metadata == {"key": "val"} - - def test_raises_on_non_tensor(self) -> None: - with pytest.raises(TypeError, match="RescaleOp expects a torch.Tensor"): - RescaleOp(in_min=0.0, in_max=255.0)(Sample(input=np.array([1, 2, 3]))) - - def test_validation_rejects_bad_input_range(self) -> None: - with pytest.raises(ValueError, match="require in_min < in_max"): - RescaleOp(in_min=10.0, in_max=10.0) - - def test_validation_rejects_bad_output_range(self) -> None: - with pytest.raises(ValueError, match="require out_min < out_max"): - RescaleOp(in_min=0.0, in_max=1.0, out_min=5.0, out_max=5.0) - - def test_pipeline_to_tensor_then_rescale(self) -> None: - """Integration: ToTensorOp(normalize=False) -> RescaleOp().""" - img = Image.fromarray(np.full((28, 28), 200, dtype=np.uint8)) - sample = Sample(input=img) - sample = ToTensorOp(normalize=False)(sample) - sample = RescaleOp(in_min=0.0, in_max=255.0)(sample) - assert sample.input.dtype == torch.float32 - assert abs(sample.input.max().item() - 200.0 / 255.0) < 1e-6 - - -# --------------------------------------------------------------------------- -# Torch StandardizeOp -# --------------------------------------------------------------------------- - - -class TestStandardizeOp: - """Tests for torch StandardizeOp.""" - - def test_scalar_mean_and_std(self) -> None: - tensor = torch.tensor([2.0, 4.0, 6.0]) - result = StandardizeOp(mean=4.0, std=2.0)(Sample(input=tensor)) - assert abs(result.input[0] - (-1.0)) < 1e-6 - assert abs(result.input[1] - 0.0) < 1e-6 - assert abs(result.input[2] - 1.0) < 1e-6 - - def test_per_channel_mean_and_std(self) -> None: - tensor = torch.ones(3, 2, 2) - tensor[0] *= 10.0 - tensor[1] *= 20.0 - tensor[2] *= 30.0 - result = StandardizeOp(mean=[10.0, 20.0, 30.0], std=[1.0, 1.0, 1.0])(Sample(input=tensor)) - assert torch.allclose(result.input, torch.zeros(3, 2, 2)) - - def test_uint8_converts_to_float(self) -> None: - tensor = torch.tensor([100, 200], dtype=torch.uint8) - result = StandardizeOp(mean=150.0, std=50.0)(Sample(input=tensor)) - assert result.input.dtype == torch.float32 - assert abs(result.input[0] - (-1.0)) < 1e-6 - assert abs(result.input[1] - 1.0) < 1e-6 - - def test_preserves_float64(self) -> None: - tensor = torch.tensor([1.0, 2.0], dtype=torch.float64) - result = StandardizeOp(mean=0.0, std=1.0)(Sample(input=tensor)) - assert result.input.dtype == torch.float64 - - def test_preserves_target_and_metadata(self) -> None: - tensor = torch.tensor([5.0]) - result = StandardizeOp(mean=0.0, std=1.0)(Sample(input=tensor, target=3, metadata={"a": 1})) - assert result.target == 3 - assert result.metadata == {"a": 1} - - def test_raises_on_non_tensor(self) -> None: - with pytest.raises(TypeError, match="StandardizeOp expects a torch.Tensor"): - StandardizeOp(mean=0.0, std=1.0)(Sample(input=[1, 2, 3])) - - def test_1d_per_channel(self) -> None: - tensor = torch.tensor([10.0]) - result = StandardizeOp(mean=[10.0], std=[5.0])(Sample(input=tensor)) - assert abs(result.input[0] - 0.0) < 1e-6 - - -# --------------------------------------------------------------------------- -# Numpy StandardizeOp -# --------------------------------------------------------------------------- - - -class TestNpStandardizeOp: - """Tests for numpy StandardizeOp.""" - - def test_scalar_mean_and_std(self) -> None: - arr = np.array([2.0, 4.0, 6.0], dtype=np.float32) - result = np_ops.StandardizeOp(mean=4.0, std=2.0)(Sample(input=arr)) - assert abs(result.input[0] - (-1.0)) < 1e-6 - assert abs(result.input[1] - 0.0) < 1e-6 - assert abs(result.input[2] - 1.0) < 1e-6 - - def test_per_channel_mean_and_std(self) -> None: - arr = np.ones((3, 2, 2), dtype=np.float32) - arr[0] *= 10.0 - arr[1] *= 20.0 - arr[2] *= 30.0 - result = np_ops.StandardizeOp(mean=[10.0, 20.0, 30.0], std=[1.0, 1.0, 1.0])(Sample(input=arr)) - assert np.allclose(result.input, np.zeros((3, 2, 2))) - - def test_uint8_converts_to_float32(self) -> None: - arr = np.array([100, 200], dtype=np.uint8) - result = np_ops.StandardizeOp(mean=150.0, std=50.0)(Sample(input=arr)) - assert result.input.dtype == np.float32 - assert abs(result.input[0] - (-1.0)) < 1e-6 - assert abs(result.input[1] - 1.0) < 1e-6 - - def test_preserves_float64(self) -> None: - arr = np.array([1.0, 2.0], dtype=np.float64) - result = np_ops.StandardizeOp(mean=0.0, std=1.0)(Sample(input=arr)) - assert result.input.dtype == np.float64 - - def test_preserves_target_and_metadata(self) -> None: - arr = np.array([5.0], dtype=np.float32) - result = np_ops.StandardizeOp(mean=0.0, std=1.0)(Sample(input=arr, target=3, metadata={"a": 1})) - assert result.target == 3 - assert result.metadata == {"a": 1} - - def test_raises_on_non_ndarray(self) -> None: - with pytest.raises(TypeError, match="StandardizeOp expects an np.ndarray"): - np_ops.StandardizeOp(mean=0.0, std=1.0)(Sample(input=[1, 2, 3])) - - def test_pil_image_input(self) -> None: - img = Image.fromarray(np.full((28, 28), 150, dtype=np.uint8)) - result = np_ops.StandardizeOp(mean=150.0, std=50.0)(Sample(input=img)) - assert isinstance(result.input, np.ndarray) - assert np.allclose(result.input, 0.0) - - def test_1d_per_channel(self) -> None: - arr = np.array([10.0], dtype=np.float32) - result = np_ops.StandardizeOp(mean=[10.0], std=[5.0])(Sample(input=arr)) - assert abs(result.input[0] - 0.0) < 1e-6 - - -# --------------------------------------------------------------------------- -# Numpy ClipPercentilesOp -# --------------------------------------------------------------------------- - - -class TestClipPercentilesOp: - def test_happy_path_no_outliers(self) -> None: - arr = np.linspace(-50.0, -10.0, 1000).reshape(20, 50) - out = np_ops.ClipPercentilesOp(low=2, high=98)(Sample(input=arr)).input - assert out.min() == pytest.approx(float(np.percentile(arr, 2))) - assert out.max() == pytest.approx(float(np.percentile(arr, 98))) - - def test_ignores_inf_and_nan(self) -> None: - arr = np.linspace(-50.0, -10.0, 100).reshape(10, 10).copy() - arr[0, 0] = np.inf - arr[0, 1] = -np.inf - arr[0, 2] = np.nan - finite = arr[np.isfinite(arr)] - expected_lo = float(np.percentile(finite, 2)) - expected_hi = float(np.percentile(finite, 98)) - out = np_ops.ClipPercentilesOp(low=2, high=98)(Sample(input=arr)).input - assert out[0, 0] == pytest.approx(expected_hi) - assert out[0, 1] == pytest.approx(expected_lo) - assert np.isnan(out[0, 2]) - - def test_all_non_finite_passes_through(self) -> None: - arr = np.full((4, 4), np.nan) - sample = Sample(input=arr) - out = np_ops.ClipPercentilesOp()(sample) - assert out is sample - - def test_raises_on_non_ndarray(self) -> None: - with pytest.raises(TypeError, match="ClipPercentilesOp expects an np.ndarray"): - np_ops.ClipPercentilesOp()(Sample(input=torch.tensor([1.0]))) - - @pytest.mark.parametrize("low,high", [(50, 50), (60, 50), (-1, 50), (50, 101)]) - def test_validation_rejects_bad_bounds(self, low: float, high: float) -> None: - with pytest.raises(ValueError, match="ClipPercentilesOp: require"): - np_ops.ClipPercentilesOp(low=low, high=high) - - -# --------------------------------------------------------------------------- -# Numpy RescaleOp -# --------------------------------------------------------------------------- - - -class TestNpRescaleOp: - def test_default_output_range(self) -> None: - arr = np.array([[-80.0, -50.0, -20.0]]) - out = np_ops.RescaleOp(in_min=-80.0, in_max=-20.0)(Sample(input=arr)).input - np.testing.assert_allclose(out, [[0.0, 0.5, 1.0]]) - - def test_custom_output_range(self) -> None: - arr = np.array([[-80.0, -50.0, -20.0]]) - out = np_ops.RescaleOp(in_min=-80.0, in_max=-20.0, out_min=10.0, out_max=20.0)(Sample(input=arr)).input - np.testing.assert_allclose(out, [[10.0, 15.0, 20.0]]) - - def test_clip_true_clamps(self) -> None: - arr = np.array([[-100.0, -50.0, 0.0]]) - out = np_ops.RescaleOp(in_min=-80.0, in_max=-20.0, clip=True)(Sample(input=arr)).input - np.testing.assert_allclose(out, [[0.0, 0.5, 1.0]]) - - def test_clip_false_extrapolates(self) -> None: - arr = np.array([[-100.0, -50.0, 0.0]]) - out = np_ops.RescaleOp(in_min=-80.0, in_max=-20.0, clip=False)(Sample(input=arr)).input - assert out[0, 0] == pytest.approx(-1.0 / 3.0) - assert out[0, 1] == pytest.approx(0.5) - assert out[0, 2] == pytest.approx(4.0 / 3.0) - - def test_uint8_converts_to_float32(self) -> None: - arr = np.array([0, 128, 255], dtype=np.uint8) - out = np_ops.RescaleOp(in_min=0.0, in_max=255.0)(Sample(input=arr)).input - assert out.dtype == np.float32 - assert abs(out[2] - 1.0) < 1e-6 - - def test_preserves_float64(self) -> None: - arr = np.array([0.0, 255.0], dtype=np.float64) - out = np_ops.RescaleOp(in_min=0.0, in_max=255.0)(Sample(input=arr)).input - assert out.dtype == np.float64 - - def test_pil_image_input(self) -> None: - img = Image.fromarray(np.full((28, 28), 200, dtype=np.uint8)) - out = np_ops.RescaleOp(in_min=0.0, in_max=255.0)(Sample(input=img)).input - assert isinstance(out, np.ndarray) - assert out.dtype == np.float32 - assert abs(out.max() - 200.0 / 255.0) < 1e-6 - - def test_preserves_target_and_metadata(self) -> None: - arr = np.array([128.0], dtype=np.float32) - result = np_ops.RescaleOp(in_min=0.0, in_max=255.0)(Sample(input=arr, target=7, metadata={"key": "val"})) - assert result.target == 7 - assert result.metadata == {"key": "val"} - - def test_raises_on_non_ndarray(self) -> None: - with pytest.raises(TypeError, match="RescaleOp expects an np.ndarray"): - np_ops.RescaleOp(in_min=0.0, in_max=1.0)(Sample(input=[1.0, 2.0])) - - def test_validation_rejects_bad_input_range(self) -> None: - with pytest.raises(ValueError, match="require in_min < in_max"): - np_ops.RescaleOp(in_min=10.0, in_max=10.0) - - def test_validation_rejects_bad_output_range(self) -> None: - with pytest.raises(ValueError, match="require out_min < out_max"): - np_ops.RescaleOp(in_min=0.0, in_max=1.0, out_min=5.0, out_max=5.0) - - def test_pipeline_rescale_then_to_tensor(self) -> None: - """Integration: numpy RescaleOp -> ToTensorOp(normalize=False).""" - img = Image.fromarray(np.full((28, 28), 200, dtype=np.uint8)) - sample = Sample(input=img) - sample = np_ops.RescaleOp(in_min=0.0, in_max=255.0)(sample) - sample = ToTensorOp(normalize=False)(sample) - assert isinstance(sample.input, torch.Tensor) - assert sample.input.dtype == torch.float32 - assert abs(sample.input.max().item() - 200.0 / 255.0) < 1e-6 - - -# --------------------------------------------------------------------------- -# Numpy ReplaceNonFiniteOp -# --------------------------------------------------------------------------- - - -class TestReplaceNonFiniteOp: - def test_numeric_value(self) -> None: - arr = np.array([[1.0, np.inf, 2.0], [-np.inf, np.nan, 3.0]]) - out = np_ops.ReplaceNonFiniteOp(value=-99.0)(Sample(input=arr)).input - assert out.tolist() == [[1.0, -99.0, 2.0], [-99.0, -99.0, 3.0]] - - def test_min_replacement(self) -> None: - arr = np.array([[1.0, np.inf, 2.0], [-np.inf, np.nan, 3.0]]) - out = np_ops.ReplaceNonFiniteOp(value="min")(Sample(input=arr)).input - assert out.tolist() == [[1.0, 1.0, 2.0], [1.0, 1.0, 3.0]] - - def test_max_replacement(self) -> None: - arr = np.array([[1.0, np.inf, 2.0], [-np.inf, np.nan, 3.0]]) - out = np_ops.ReplaceNonFiniteOp(value="max")(Sample(input=arr)).input - assert out.tolist() == [[1.0, 3.0, 2.0], [3.0, 3.0, 3.0]] - - def test_already_finite_passes_through(self) -> None: - arr = np.array([[1.0, 2.0, 3.0]]) - sample = Sample(input=arr) - out = np_ops.ReplaceNonFiniteOp(value="min")(sample) - assert out is sample - - def test_all_non_finite_passes_through(self) -> None: - arr = np.full((3, 3), np.nan) - sample = Sample(input=arr) - out = np_ops.ReplaceNonFiniteOp(value="min")(sample) - assert out is sample - - def test_raises_on_non_ndarray(self) -> None: - with pytest.raises(TypeError, match="ReplaceNonFiniteOp expects an np.ndarray"): - np_ops.ReplaceNonFiniteOp()(Sample(input=torch.tensor([1.0]))) - - def test_validation_rejects_unknown_string(self) -> None: - with pytest.raises(ValueError, match="value string must be 'min' or 'max'"): - np_ops.ReplaceNonFiniteOp(value="median") - - -# --------------------------------------------------------------------------- -# Tee -# --------------------------------------------------------------------------- - - -class TestTee: - def test_two_branches_share_metadata(self) -> None: - sample = Sample(input=np.array([1.0]), target=None, metadata={}) - - def writer_a(s: Sample) -> Sample: - s.metadata["a"] = 1 - return s - - def writer_b(s: Sample) -> Sample: - assert s.metadata["a"] == 1 # branch A's write is visible - s.metadata["b"] = 2 - return s - - out = Tee(branches=[[writer_a], [writer_b]])(sample) - assert out is not None - assert out.metadata == {"a": 1, "b": 2} - - def test_branches_run_sequentially(self) -> None: - from typing import Callable - - order: list[str] = [] - - def make(tag: str) -> Callable[[Sample], Sample]: - def op(s: Sample) -> Sample: - order.append(tag) - return s - - return op - - Tee(branches=[[make("A1"), make("A2")], [make("B1"), make("B2")]])(Sample(input=None)) - assert order == ["A1", "A2", "B1", "B2"] - - def test_none_propagates(self) -> None: - def filter_out(s: Sample) -> None: - return None - - def should_not_run(s: Sample) -> Sample: - raise AssertionError("downstream branch must not run after None") - - out = Tee(branches=[[filter_out], [should_not_run]])(Sample(input=1)) - assert out is None - - def test_input_mutations_flow_into_next_branch(self) -> None: - def to_zero(s: Sample) -> Sample: - return s._replace(input=0) - - def must_see_zero(s: Sample) -> Sample: - assert s.input == 0 - return s._replace(input=99) - - out = Tee(branches=[[to_zero], [must_see_zero]])(Sample(input=42)) - assert out is not None - assert out.input == 99 - - -# --------------------------------------------------------------------------- -# Copy* ops -# --------------------------------------------------------------------------- - - -class TestCopyOps: - def test_copy_sample_deepcopies_all_fields(self) -> None: - meta = {"k": [1, 2, 3]} - sample = Sample(input=np.array([1.0, 2.0]), target=[10], metadata=meta) - out = CopySampleOp()(sample) - assert out.input is not sample.input - assert out.target is not sample.target - assert out.metadata is not sample.metadata - assert out.metadata["k"] is not sample.metadata["k"] - - def test_copy_input_only_copies_input(self) -> None: - sample = Sample(input=np.array([1.0]), target=[5], metadata={"k": "v"}) - out = CopyInputOp()(sample) - assert out.input is not sample.input - assert out.target is sample.target - assert out.metadata is sample.metadata - - def test_copy_target_only_copies_target(self) -> None: - sample = Sample(input=[1, 2], target=[10, 20], metadata={}) - out = CopyTargetOp()(sample) - assert out.target is not sample.target - assert out.input is sample.input - - def test_copy_metadata_breaks_aliasing(self) -> None: - meta = {"k": [1]} - sample = Sample(input=None, target=None, metadata=meta) - out = CopyMetadataOp()(sample) - out.metadata["k"].append(2) - assert meta["k"] == [1] - - -# --------------------------------------------------------------------------- -# SwapInputTargetOp -# --------------------------------------------------------------------------- - - -class TestSwapInputTargetOp: - def test_swaps_input_and_target(self) -> None: - sample = Sample(input=1, target=2, metadata={"k": "v"}) - out = SwapInputTargetOp()(sample) - assert out.input == 2 - assert out.target == 1 - assert out.metadata == {"k": "v"} - - -# --------------------------------------------------------------------------- -# StashInputOp / UnstashInputOp -# --------------------------------------------------------------------------- - - -class TestStashUnstash: - def test_stash_aliases_by_default(self) -> None: - arr = np.array([1.0, 2.0]) - sample = Sample(input=arr, target=None, metadata={}) - out = StashInputOp(key="snap")(sample) - assert out.metadata["snap"] is arr - assert out.input is arr - - def test_stash_with_copy_deepcopies(self) -> None: - arr = np.array([1.0, 2.0]) - sample = Sample(input=arr, target=None, metadata={}) - out = StashInputOp(key="snap", copy=True)(sample) - assert out.metadata["snap"] is not arr - np.testing.assert_array_equal(out.metadata["snap"], arr) - - def test_unstash_default_copies_to_isolate_branches(self) -> None: - arr = np.array([1.0, 2.0]) - sample = Sample(input=None, target=None, metadata={"snap": arr}) - out = UnstashInputOp(key="snap")(sample) - assert out.input is not arr - np.testing.assert_array_equal(out.input, arr) - - def test_unstash_no_copy_aliases(self) -> None: - arr = np.array([1.0, 2.0]) - sample = Sample(input=None, target=None, metadata={"snap": arr}) - out = UnstashInputOp(key="snap", copy=False)(sample) - assert out.input is arr - - def test_two_unstashes_with_in_place_mutation_dont_corrupt(self) -> None: - """Default copy=True prevents branch-A's in-place write from leaking into branch-B.""" - arr = np.array([1.0, 2.0, 3.0]) - sample = Sample(input=None, target=None, metadata={"snap": arr}) - a = UnstashInputOp(key="snap")(sample) - a.input.fill(99.0) # in-place mutation on branch A's restored array - b = UnstashInputOp(key="snap")(sample) - np.testing.assert_array_equal(b.input, [1.0, 2.0, 3.0]) - - -# --------------------------------------------------------------------------- -# numpy.resolve_expression -# --------------------------------------------------------------------------- - - -class TestResolveExpression: - def test_no_substitution_returns_verbatim(self) -> None: - sample = Sample(input=None, target=None, metadata={}) - assert np_ops.resolve_expression("hello", sample) == "hello" - assert np_ops.resolve_expression("5.5", sample) == "5.5" - - def test_metadata_substitution(self) -> None: - sample = Sample(input=None, target=None, metadata={"snr": -30.5, "drone": "yz"}) - assert np_ops.resolve_expression("{snr}", sample) == "-30.5" - assert np_ops.resolve_expression("-{snr}", sample) == "--30.5" - assert np_ops.resolve_expression("{drone}", sample) == "yz" - - def test_env_substitution(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("REF_SNR", "12.5") - sample = Sample(input=None, target=None, metadata={}) - assert np_ops.resolve_expression("$REF_SNR", sample) == "12.5" - assert np_ops.resolve_expression("-$REF_SNR", sample) == "-12.5" - - def test_missing_metadata_key_raises(self) -> None: - sample = Sample(input=None, target=None, metadata={}) - with pytest.raises(KeyError, match="metadata key 'nope' missing"): - np_ops.resolve_expression("{nope}", sample) - - def test_missing_env_var_raises(self) -> None: - os.environ.pop("DATAFLUX_TEST_NOPE", None) - sample = Sample(input=None, target=None, metadata={}) - with pytest.raises(KeyError, match="environment variable 'DATAFLUX_TEST_NOPE'"): - np_ops.resolve_expression("$DATAFLUX_TEST_NOPE", sample) - - -# --------------------------------------------------------------------------- -# ThresholdOp -# --------------------------------------------------------------------------- - - -class TestThresholdOp: - def test_numeric_value(self) -> None: - arr = np.array([0.0, 1.0, 2.0, 3.0]) - out = np_ops.ThresholdOp(value=1.5)(Sample(input=arr)) - np.testing.assert_array_equal(out.input, [False, False, True, True]) - assert out.metadata["threshold"] == 1.5 - - def test_string_numeric(self) -> None: - arr = np.array([0.0, 1.0, 2.0]) - out = np_ops.ThresholdOp(value="1.5")(Sample(input=arr)) - np.testing.assert_array_equal(out.input, [False, False, True]) - - def test_metadata_lookup(self) -> None: - arr = np.array([-50.0, -30.0, -10.0]) - sample = Sample(input=arr, target=None, metadata={"reference_snr_level": -25.0}) - out = np_ops.ThresholdOp(value="{reference_snr_level}")(sample) - np.testing.assert_array_equal(out.input, [False, False, True]) - assert out.metadata["threshold"] == -25.0 - - def test_metadata_lookup_with_negation(self) -> None: - arr = np.array([-50.0, -30.0, -10.0]) - sample = Sample(input=arr, target=None, metadata={"reference_snr_level": 30.0}) - out = np_ops.ThresholdOp(value="-{reference_snr_level}")(sample) - np.testing.assert_array_equal(out.input, [False, False, True]) - assert out.metadata["threshold"] == -30.0 - - def test_env_lookup(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setenv("DATAFLUX_TEST_THRESHOLD", "1.0") - arr = np.array([0.0, 1.0, 2.0]) - out = np_ops.ThresholdOp(value="$DATAFLUX_TEST_THRESHOLD")(Sample(input=arr)) - np.testing.assert_array_equal(out.input, [False, False, True]) - - def test_raises_on_non_ndarray(self) -> None: - with pytest.raises(TypeError, match="ThresholdOp expects an np.ndarray"): - np_ops.ThresholdOp(value=0.0)(Sample(input=[1.0, 2.0])) - - def test_raises_on_non_numeric_resolution(self) -> None: - sample = Sample(input=np.array([0.0]), target=None, metadata={"drone": "yz"}) - with pytest.raises(ValueError, match="not a number"): - np_ops.ThresholdOp(value="{drone}")(sample) - - def test_raises_on_bad_value_type(self) -> None: - # Confluid's ``@configurable`` validates kwargs against the - # auto-generated pydantic schema before the body runs. ``value`` is - # typed as ``float | int | str``, so a list is rejected at the - # validation layer first; the body's hand-rolled ``TypeError`` - # remains as a safety net. - from pydantic import ValidationError - - with pytest.raises((TypeError, ValidationError)): - np_ops.ThresholdOp(value=[1, 2])(Sample(input=np.array([0.0]))) # type: ignore[arg-type] - - -# --------------------------------------------------------------------------- -# ConnectedComponentsOp -# --------------------------------------------------------------------------- - - -class TestConnectedComponentsOp: - def test_two_separate_blobs(self) -> None: - mask = np.zeros((10, 10), dtype=bool) - mask[1:3, 1:3] = True # 2x2 blob at (1,1) - mask[6:9, 6:9] = True # 3x3 blob at (6,6) - out = np_ops.ConnectedComponentsOp(min_area_bins=1, connectivity=4)(Sample(input=mask)) - assert sorted(out.input) == [(1, 2, 1, 2), (6, 8, 6, 8)] - - def test_min_area_drops_small_components(self) -> None: - mask = np.zeros((10, 10), dtype=bool) - mask[0, 0] = True # area 1 - mask[5:7, 5:7] = True # area 4 - out = np_ops.ConnectedComponentsOp(min_area_bins=2, connectivity=4)(Sample(input=mask)) - assert out.input == [(5, 6, 5, 6)] - - def test_connectivity_4_keeps_diagonals_separate(self) -> None: - mask = np.zeros((4, 4), dtype=bool) - mask[0, 0] = True - mask[1, 1] = True - mask[2, 2] = True - out = np_ops.ConnectedComponentsOp(min_area_bins=1, connectivity=4)(Sample(input=mask)) - assert len(out.input) == 3 - - def test_connectivity_8_merges_diagonals(self) -> None: - mask = np.zeros((4, 4), dtype=bool) - mask[0, 0] = True - mask[1, 1] = True - mask[2, 2] = True - out = np_ops.ConnectedComponentsOp(min_area_bins=1, connectivity=8)(Sample(input=mask)) - assert len(out.input) == 1 - assert out.input[0] == (0, 2, 0, 2) - - def test_empty_mask_returns_empty_list(self) -> None: - mask = np.zeros((5, 5), dtype=bool) - out = np_ops.ConnectedComponentsOp(connectivity=4)(Sample(input=mask)) - assert out.input == [] - - def test_raises_on_non_ndarray(self) -> None: - with pytest.raises(TypeError, match="ConnectedComponentsOp expects an np.ndarray"): - np_ops.ConnectedComponentsOp()(Sample(input=[[True, False]])) - - def test_raises_on_non_2d(self) -> None: - with pytest.raises(ValueError, match="expects a 2-D mask"): - np_ops.ConnectedComponentsOp()(Sample(input=np.array([True, False]))) - - def test_validation_rejects_bad_min_area(self) -> None: - with pytest.raises(ValueError, match="min_area_bins must be >= 1"): - np_ops.ConnectedComponentsOp(min_area_bins=0) - - def test_validation_rejects_bad_connectivity(self) -> None: - with pytest.raises(ValueError, match="connectivity must be 4 or 8"): - np_ops.ConnectedComponentsOp(connectivity=6) diff --git a/tests/test_ops_predict.py b/tests/test_ops_predict.py new file mode 100644 index 0000000..5553ace --- /dev/null +++ b/tests/test_ops_predict.py @@ -0,0 +1,71 @@ +"""Inference as an op: ``ModelPredict`` stamps a model's output back onto the record. + +A pipeline that carries this op IS the predictions flow — a viewer executes it per +record and reads the stamped fields as layers; ``recordstream run`` executes the same +document offline. The model is duck-typed (any callable), so none of this needs torch. +""" + +from typing import Any + +import numpy as np +import pytest + +from recordstream.items import Boxes, Label +from recordstream.ops.predict import ModelPredict + + +class _Classifier: + """logits [1, 2] favouring class 1; counts solidify() calls (the lazy checkpoint load).""" + + def __init__(self) -> None: + self.solidified = 0 + self.calls = 0 + + def solidify(self) -> None: + self.solidified += 1 + + def __call__(self, batch: Any) -> np.ndarray: + self.calls += 1 + return np.array([[0.1, 2.0]]) + + +class TestModelPredict: + def test_classification_stamps_a_label_and_solidifies_once(self) -> None: + model = _Classifier() + op = ModelPredict(model=model, kind="classification") + record = {"image": np.zeros((8, 12)), "class": 0} + out = op(record) + assert isinstance(out["predict"], Label) and out["predict"].value == 1 + assert "predict" not in record # a new dict — the input record is never mutated + op(record) + assert model.solidified == 1 and model.calls == 2 # the checkpoint loads once, not per record + + def test_detection_stamps_boxes_with_the_images_canvas(self) -> None: + def model(batch: Any) -> Any: + return {"boxes": [[[1.0, 2.0, 5.0, 6.0]]], "scores": [[0.9]], "labels": [[1]]} + + out = ModelPredict(model=model, kind="detection")({"image": np.zeros((8, 12))}) + stamped = out["predict"] + assert isinstance(stamped, Boxes) + assert stamped.boxes == [[1.0, 2.0, 5.0, 6.0]] and stamped.scores == [0.9] and stamped.labels == [1] + assert stamped.canvas is not None and tuple(stamped.canvas) == (8, 12) + + def test_segmentation_argmaxes_logits_into_an_int_mask(self) -> None: + logits = np.zeros((1, 2, 4, 6)) + logits[0, 1, :2] = 5.0 # top rows are class 1 + out = ModelPredict(model=lambda b: logits, kind="segmentation")({"image": np.zeros((4, 6))}) + mask = out["predict_mask"] + assert mask.shape == (4, 6) and mask.dtype == np.int64 + assert mask[0, 0] == 1 and mask[3, 0] == 0 + + def test_restoration_moves_channels_last(self) -> None: + out = ModelPredict(model=lambda b: np.ones((1, 3, 4, 6)), kind="restoration")({"image": np.zeros((4, 6))}) + assert out["predict"].shape == (4, 6, 3) + + def test_bad_kind_and_missing_pieces_are_located_value_errors(self) -> None: + with pytest.raises(ValueError, match="kind"): + ModelPredict(kind="nope") # type: ignore[arg-type] + with pytest.raises(ValueError, match="model"): + ModelPredict(kind="classification")({"image": np.zeros((2, 2))}) + with pytest.raises(ValueError, match="no field 'image'"): + ModelPredict(model=_Classifier())({"other": 1}) diff --git a/tests/test_optional_torch.py b/tests/test_optional_torch.py new file mode 100644 index 0000000..95f53f8 --- /dev/null +++ b/tests/test_optional_torch.py @@ -0,0 +1,247 @@ +"""torch is an EXTRA — the engine's core is numpy. + +The claim is not "recordstream avoids torch" (`ToTensor` and the output builders need it); it is +that **importing recordstream imports no ML framework**, so a Keras-only, TensorFlow-only or +plain-numpy consumer does not install ~2GB it never calls. + +The load-bearing test here is :func:`test_importing_recordstream_works_with_torch_blocked`, which +proves it in a SUBPROCESS with torch made unimportable. The rest guard the three mechanisms that +make that true, each of which was a real coupling before: the `Dataset` base class, the eager +`ToTensor` export, and the module-level `isinstance(x, torch.Tensor)` checks. +""" + +import subprocess +import sys +import textwrap + +import numpy as np +import pytest + +from recordstream import Stream +from recordstream._compat import is_torch_tensor +from recordstream.core import MapStyle + +# --------------------------------------------------------------------------- # +# The whole claim, executed +# --------------------------------------------------------------------------- # + +_IMPORT_WITH_TORCH_BLOCKED = textwrap.dedent( + """ + import sys + + class _NoTorch: + \"\"\"A meta-path finder that makes torch unimportable, simulating an install without it.\"\"\" + + def find_spec(self, name, path=None, target=None): + if name == "torch" or name.startswith("torch."): + raise ImportError("torch is not installed (blocked by this test)") + return None + + sys.meta_path.insert(0, _NoTorch()) + + import recordstream + from recordstream import Stream, LabelMap, collate_records, multi_hot, batch_values + + # Not just the package: the surfaces a non-torch backend actually uses must work too. + stream = Stream(source=[{"class": 1}, {"class": 0}]) + assert len(stream) == 2 + assert [r["class"] for r in stream] == [1, 0] + + leaked = sorted(m for m in sys.modules if m == "torch" or m.startswith("torch.")) + print("LEAKED:" + ",".join(leaked) if leaked else "CLEAN") + """ +) + + +def test_importing_recordstream_works_with_torch_blocked(tmp_path: object) -> None: + """`import recordstream` must not need torch, and must not import it as a side effect. + + Run in a subprocess because this process has already imported torch — the check is only + meaningful in an interpreter where it was never available. `cwd` is a temp dir so the import + resolves to the INSTALLED package, not a same-named source directory (PEP 420 shadowing). + """ + result = subprocess.run( + [sys.executable, "-c", _IMPORT_WITH_TORCH_BLOCKED], + capture_output=True, + text=True, + cwd=str(tmp_path), + ) + + assert result.returncode == 0, f"importing recordstream without torch failed:\n{result.stderr}" + assert "CLEAN" in result.stdout, f"torch was imported anyway: {result.stdout.strip()}" + + +# --------------------------------------------------------------------------- # +# 1. The `Dataset` base — dropped, because a DataLoader never needed it +# --------------------------------------------------------------------------- # + + +def test_stream_does_not_inherit_torchs_dataset() -> None: + """The regression guard: re-adding the base would silently make torch mandatory again. + + Nothing in the workspace does `isinstance(x, Dataset)` or subclasses `Stream`, so the base + bought nothing but the import. + """ + import torch.utils.data + + assert not issubclass(Stream, torch.utils.data.Dataset) + + +def test_stream_is_map_style_which_is_all_a_dataloader_needs() -> None: + """`DataLoader` duck-types its argument — `__len__` + `__getitem__` IS the contract.""" + stream = Stream(source=[{"class": i} for i in range(4)]) + + assert isinstance(stream, MapStyle) + assert len(stream) == 4 and stream[2]["class"] == 2 + + +def test_a_dataloader_actually_accepts_a_stream() -> None: + """The duck-typing claim, executed rather than asserted about. + + `cast(Any, ...)` at the call site is the accepted cost: torch's STUB still declares + `Dataset[T]`, so type checkers reject the runtime-valid call. + """ + from typing import Any, cast + + import torch.utils.data + + stream = Stream(source=[{"class": i} for i in range(4)]) + loader = torch.utils.data.DataLoader(cast(Any, stream), batch_size=2) + + assert len(list(loader)) == 2 + + +# --------------------------------------------------------------------------- # +# 2. `ToTensor` — reachable, but never eagerly imported +# --------------------------------------------------------------------------- # + + +def test_to_tensor_is_not_imported_eagerly_but_is_still_reachable() -> None: + import recordstream.ops as ops + + assert "ToTensor" in ops._OPTIONAL_OPS, "the lazy export must stay registered" + assert ops.ToTensor is not None + assert "ToTensor" in dir(ops), "TAB completion / dir() must still advertise it" + + +def test_a_missing_optional_op_names_the_extra_to_install(monkeypatch: pytest.MonkeyPatch) -> None: + """The error an operator without the extra sees — not a traceback from three libraries down.""" + import importlib + + import recordstream.ops as ops + + def _fail(name: str) -> None: + raise ImportError("No module named 'torch'") + + monkeypatch.setattr(importlib, "import_module", _fail) + + with pytest.raises(ImportError, match=r"recordstream\[torch\]"): + ops.__getattr__("ToTensor") + + +def test_an_unknown_attribute_still_raises_attribute_error() -> None: + """The lazy `__getattr__` must not turn every typo into an ImportError.""" + import recordstream.ops as ops + + with pytest.raises(AttributeError): + ops.__getattr__("NoSuchOp") + + +# --------------------------------------------------------------------------- # +# 3. Recognising a tensor without importing torch +# --------------------------------------------------------------------------- # + + +def test_is_torch_tensor_recognizes_a_real_tensor() -> None: + import torch + + assert is_torch_tensor(torch.zeros(3)) + + +@pytest.mark.parametrize("value", [np.zeros(3), [1, 2, 3], "not a tensor", None, 7]) +def test_is_torch_tensor_rejects_everything_else(value: object) -> None: + assert not is_torch_tensor(value) + + +def test_is_torch_tensor_is_false_when_torch_was_never_imported(monkeypatch: pytest.MonkeyPatch) -> None: + """The mechanism: a torch tensor cannot exist unless torch is loaded, so `sys.modules` is EXACT. + + With torch hidden the answer must be False without any attempt to import it — which is what + makes this usable at module level in a package that does not depend on torch. + """ + import torch + + monkeypatch.delitem(sys.modules, "torch") + tensor = torch.zeros(3) # a real tensor, while the module is hidden + + assert not is_torch_tensor(tensor) + + +# --------------------------------------------------------------------------- # +# 4. The packaging half — the code above is only true if the metadata agrees +# --------------------------------------------------------------------------- # + + +def test_pyproject_declares_torch_as_an_extra_not_a_dependency() -> None: + import tomllib + from pathlib import Path + + pyproject = Path(__file__).resolve().parent.parent / "pyproject.toml" + project = tomllib.loads(pyproject.read_text())["project"] + + required = [d for d in project["dependencies"] if d.split(">")[0].split("=")[0].strip() == "torch"] + assert not required, f"torch must not be a hard dependency, found: {required}" + assert any("torch" in d for d in project["optional-dependencies"]["torch"]) + + +# --------------------------------------------------------------------------- # +# 5. keras is an extra too — and DISCOVERY must not pull it either +# --------------------------------------------------------------------------- # + +_SCAN_WITHOUT_KERAS = textwrap.dedent( + """ + import sys + + import recordstream + from recordstream.discovery import scan_module + + scan_module("recordstream") # what a GUI bridge / MCP bootstrap does + scan_module("recordstream.ops") + + leaked = sorted(m for m in sys.modules if m == "keras" or m.startswith("keras.")) + print("LEAKED:" + ",".join(leaked) if leaked else "CLEAN") + """ +) + + +def test_neither_importing_nor_scanning_recordstream_imports_keras(tmp_path: object) -> None: + """Why `RecordSequence` lives in `recordstream.keras` and NOT at the package root. + + `inspect.getmembers` — inside `scan_module`, and in the GUI bridges — getattrs every name a + module advertises, so a PEP 562 lazy root export (the `ops.ToTensor` pattern) would import + keras on every discovery scan, including on installs that never asked for it. Run in a + subprocess because this one has already imported keras via the sequence tests. + """ + result = subprocess.run( + [sys.executable, "-c", _SCAN_WITHOUT_KERAS], + capture_output=True, + text=True, + cwd=str(tmp_path), + ) + + assert result.returncode == 0, f"scanning recordstream failed:\n{result.stderr}" + assert "CLEAN" in result.stdout, f"keras was imported by a discovery scan: {result.stdout.strip()}" + + +def test_pyproject_declares_keras_as_an_extra_naming_no_compute_engine() -> None: + """Keras 3 is an API, not a runtime: the extra must not drag torch/TF/jax in — the consumer's + own extra picks one, and `recordstream.keras` defaults to whichever is installed.""" + import tomllib + from pathlib import Path + + pyproject = Path(__file__).resolve().parent.parent / "pyproject.toml" + extras = tomllib.loads(pyproject.read_text())["project"]["optional-dependencies"] + + assert any("keras" in d for d in extras["keras"]) + engines = [d for d in extras["keras"] if d.split(">")[0].split("=")[0].strip() in ("torch", "tensorflow", "jax")] + assert not engines, f"the keras extra must name no compute engine, found: {engines}" diff --git a/tests/test_outputs.py b/tests/test_outputs.py new file mode 100644 index 0000000..6266b14 --- /dev/null +++ b/tests/test_outputs.py @@ -0,0 +1,148 @@ +"""Tests for :mod:`recordstream.outputs` — the typed prediction-output contracts + torch builders.""" + +import torch + +from recordstream.outputs import ( + ClassificationOutput, + DetectionOutput, + RestorationOutput, + SegmentationOutput, + classification_output, + restoration_output, + segmentation_output, +) + + +def test_classification_output_shapes_and_dtypes() -> None: + logits = torch.randn(4, 3) + out = classification_output(logits) + + # TypedDict values are plain dict entries at runtime. + assert set(out.keys()) == {"logits", "probs", "class_idx"} + assert out["logits"] is logits + assert out["probs"].shape == (4, 3) + assert torch.allclose(out["probs"].sum(dim=-1), torch.ones(4), atol=1e-5) + assert out["class_idx"].dtype == torch.int64 + assert out["class_idx"].shape == (4,) + assert torch.equal(out["class_idx"], logits.argmax(dim=-1).to(torch.int64)) + + +def test_segmentation_output_per_pixel_argmax() -> None: + logits = torch.randn(2, 5, 8, 8) + out = segmentation_output(logits) + + assert set(out.keys()) == {"logits", "probs", "mask"} + assert out["probs"].shape == (2, 5, 8, 8) + # Softmax over channel dim. + assert torch.allclose(out["probs"].sum(dim=1), torch.ones(2, 8, 8), atol=1e-5) + assert out["mask"].dtype == torch.int64 + assert out["mask"].shape == (2, 8, 8) + assert torch.equal(out["mask"], logits.argmax(dim=1).to(torch.int64)) + + +def test_restoration_output_carries_the_image_unchanged() -> None: + """The builder transforms NOTHING — a restored image already IS the contract's one key.""" + image = torch.rand(2, 3, 8, 8) + out = restoration_output(image) + + assert set(out.keys()) == {"image"} + assert out["image"] is image + assert out["image"].shape == (2, 3, 8, 8) + + +def test_restoration_output_does_not_clamp_the_range() -> None: + """A residual denoiser can legitimately overshoot [0, 1], and clamping would change the score. + + Whether the output is clipped is the RUN's decision (a metric on clamped values is a + different number), so the contract carries what the model produced. + """ + out = restoration_output(torch.tensor([[[[-0.25, 1.5]]]])) + + assert float(out["image"].min()) == -0.25 + assert float(out["image"].max()) == 1.5 + + +def test_restoration_output_has_no_residual_key() -> None: + """``input - image`` is derivable, and a derivable key is a second place for the two to disagree.""" + assert "residual" not in restoration_output(torch.zeros(1, 3, 4, 4)) + + +def test_restoration_output_is_typed_dict_instance() -> None: + out: RestorationOutput = restoration_output(torch.zeros(1, 3, 4, 4)) + assert isinstance(out, dict) + assert set(out.keys()) == {"image"} + + +def test_detection_output_typed_dict_construction() -> None: + # DetectionOutput is constructed directly at call sites; verify the + # TypedDict's runtime behavior matches a plain dict. + boxes = torch.tensor([[0.0, 0.0, 10.0, 10.0]], dtype=torch.float32) + scores = torch.tensor([0.9], dtype=torch.float32) + labels = torch.tensor([1], dtype=torch.int64) + out = DetectionOutput(boxes=boxes, scores=scores, labels=labels) + assert out["boxes"].dtype == torch.float32 + assert out["scores"].dtype == torch.float32 + assert out["labels"].dtype == torch.int64 + assert set(out.keys()) == {"boxes", "scores", "labels"} + + +def test_classification_output_is_typed_dict_instance() -> None: + out: ClassificationOutput = classification_output(torch.zeros(1, 2)) + # TypedDicts are plain dicts at runtime. + assert isinstance(out, dict) + assert "logits" in out and "probs" in out and "class_idx" in out + + +def test_segmentation_output_is_typed_dict_instance() -> None: + out: SegmentationOutput = segmentation_output(torch.zeros(1, 2, 3, 3)) + assert isinstance(out, dict) + assert set(out.keys()) == {"logits", "probs", "mask"} + + +# --------------------------------------------------------------------------- # +# The contracts stay generic in the array type — only the BUILDERS are torch +# --------------------------------------------------------------------------- # + + +def test_a_contract_parameterizes_over_the_array_type() -> None: + """The same contract describes a torch run and a numpy/TF/JAX one. + + recordstream hard-depends on torch (a `Stream` IS a `torch.utils.data.Dataset`), so the + contracts and their torch builders share one module — but the contracts themselves are + typing-only and generic, so a backend on another framework declares its output with the + SAME types and adds its own builders beside these. + """ + import numpy as np + + numpy_out: ClassificationOutput[np.ndarray] = { + "logits": np.zeros(2), + "probs": np.zeros(2), + "class_idx": np.zeros(2), + } + torch_out: ClassificationOutput[torch.Tensor] = classification_output(torch.zeros(1, 2)) + + assert isinstance(numpy_out["probs"], np.ndarray) + assert isinstance(torch_out["probs"], torch.Tensor) + + +def test_detection_has_no_builder_on_purpose() -> None: + """Boxes come from a detector's own interface; there is nothing to derive from logits.""" + import recordstream.outputs as outputs + + assert not hasattr(outputs, "detection_output") + + +def test_the_package_root_exports_the_contracts_and_builders() -> None: + import recordstream + + for name in ( + "ClassificationOutput", + "DetectionOutput", + "DetectionPredictions", + "RestorationOutput", + "SegmentationOutput", + "classification_output", + "restoration_output", + "segmentation_output", + ): + assert name in recordstream.__all__ and hasattr(recordstream, name) diff --git a/tests/test_packaging.py b/tests/test_packaging.py new file mode 100644 index 0000000..561a62a --- /dev/null +++ b/tests/test_packaging.py @@ -0,0 +1,126 @@ +"""The distribution metadata a PyPI upload actually requires. + +Every rule here was written against a measured failure of this project's own build, +not from a checklist. They are cheap (they read ``pyproject.toml``; nothing is built) +and they cover the half of "does it ship?" that the test suite, the linters and the +examples all pass straight through: + +* a dependency written as a direct URL makes PyPI *reject the upload* — the file + is uploaded, parsed, and refused, so the failure arrives after everything green; +* a ``readme``/``license-files`` entry naming a file that is not there fails at + build time on a clean checkout but not in an editable install; +* missing classifiers/URLs cost nothing at build time and produce a landing page + with no license, no home, and no issue tracker. + +The sibling ``test_docs_links.py`` covers the README's *contents* once it becomes +that landing page. +""" + +import tomllib +from pathlib import Path +from typing import Any, Dict, List, cast + +_REPO = Path(__file__).resolve().parent.parent +_PYPROJECT = _REPO / "pyproject.toml" + + +def _project() -> Dict[str, Any]: + with _PYPROJECT.open("rb") as fh: + return cast(Dict[str, Any], tomllib.load(fh)["project"]) + + +def _every_requirement() -> List[str]: + """Core dependencies plus every extra's — PyPI validates all of them alike.""" + project = _project() + requirements = list(project.get("dependencies", [])) + for extra in project.get("optional-dependencies", {}).values(): + requirements.extend(extra) + return requirements + + +def test_no_requirement_is_a_direct_url() -> None: + """A ``name @ git+https://...`` requirement is refused by PyPI at upload time. + + Warehouse validates every ``Requires-Dist`` and rejects any whose requirement + carries a URL — ``warehouse/forklift/metadata.py``, "Can't have direct + dependency: {req}". It applies to extras exactly as to core dependencies, which + is the trap: an extra nobody installs still blocks the whole upload. + + An unpublished dependency therefore cannot be reached by an extra at all. Name + it plainly (``foo``) and document how to obtain it, or drop the extra. + """ + direct = [req for req in _every_requirement() if "@" in req and "://" in req.split("@", 1)[1]] + + assert not direct, f"PyPI rejects direct-URL requirements: {direct}" + + +def test_the_declared_readme_and_license_files_exist() -> None: + """``readme`` / ``license-files`` name files that are actually in the tree. + + Both are read at BUILD time, so an editable install never notices a missing one; + the failure surfaces on the clean checkout that builds the release artifact. + """ + project = _project() + declared = [project["readme"], *project.get("license-files", [])] + missing = [name for name in declared if not (_REPO / name).exists()] + + assert not missing, f"pyproject names files that do not exist: {missing}" + + +def test_the_landing_page_metadata_is_present() -> None: + """Description, license, classifiers and URLs — the PyPI page's whole frame. + + Each is optional to the build and free to omit, which is why all four went + missing at once: the wheel builds identically without them and the gap is only + visible on the published page. + """ + project = _project() + missing = [key for key in ("description", "readme", "license", "classifiers", "urls") if not project.get(key)] + + assert not missing, f"release metadata missing from pyproject: {missing}" + + +def test_the_development_status_classifier_matches_the_version() -> None: + """An ``aN``/``bN`` version says pre-release; the classifier must say so too. + + These drift in opposite directions — the version moves every release, the + classifier is written once and forgotten — leaving a "Production/Stable" badge + on an alpha. + """ + project = _project() + status = [c for c in project["classifiers"] if c.startswith("Development Status ::")] + assert len(status) == 1, f"expected exactly one Development Status classifier, got {status}" + + is_prerelease = "a" in project["version"] or "b" in project["version"] or "rc" in project["version"] + expected = "3 - Alpha" if is_prerelease else "4 - Beta" + + assert expected in status[0], f"version {project['version']} does not match classifier {status[0]!r}" + + +def test_the_changelog_is_shipped_in_the_sdist() -> None: + """`MANIFEST.in` must carry `CHANGELOG.md` — nothing else pulls it in. + + setuptools builds an sdist from what is *referenced*: `readme` brings README.md, + `license-files` brings LICENSE, `packages.find` + `package-data` bring the package. + A CHANGELOG is referenced by nothing, so it is silently absent — measured on this + project's own sdist before `MANIFEST.in` existed. + + It matters more than a tidiness point here: `[project.urls] Changelog` points at the + file on GitHub, so while that repository is private the sdist is the ONLY copy a + reader can reach. + + Asserted on the declaration rather than by building an sdist, for the same reason + the package-data rules are: building one takes seconds and needs a clean tree, and + the declaration is the thing that actually goes stale. + """ + manifest = _REPO / "MANIFEST.in" + assert manifest.exists(), "MANIFEST.in is missing; the sdist would ship no CHANGELOG" + + included = { + line.split(maxsplit=1)[1].strip() + for line in manifest.read_text().splitlines() + if line.strip().startswith("include ") + } + missing = [name for name in ("CHANGELOG.md",) if name not in included] + + assert not missing, f"MANIFEST.in does not include: {missing}" diff --git a/tests/test_paired.py b/tests/test_paired.py deleted file mode 100644 index ee8a994..0000000 --- a/tests/test_paired.py +++ /dev/null @@ -1,489 +0,0 @@ -"""Tests for dataflux.paired.PairedSource.""" - -from typing import Any, Dict, Iterator, Optional - -import confluid # type: ignore[import-not-found] -import pytest - -from dataflux.discovery import get_callable_path -from dataflux.paired import PairedSource -from dataflux.sample import Sample - -# --------------------------------------------------------------------------- -# Test fixtures -# --------------------------------------------------------------------------- - - -@confluid.configurable -class PairedIndexedSource: - """Indexable primary source producing Samples keyed by integer id.""" - - def __init__(self, size: int = 4) -> None: - self.size = size - - def __len__(self) -> int: - return self.size - - def __getitem__(self, index: int) -> Sample: - return Sample(input=index * 10, target=None, metadata={"id": f"s{index}"}) - - def __iter__(self) -> Iterator[Sample]: - for i in range(self.size): - yield self[i] - - -@confluid.configurable -class DictStore: - """Minimal mapping-shaped secondary for tests.""" - - def __init__(self, records: Optional[Dict[str, Dict[str, Any]]] = None) -> None: - self.records: Dict[str, Dict[str, Any]] = records or {} - - def __contains__(self, key: str) -> bool: - return key in self.records - - def __getitem__(self, key: str) -> Dict[str, Any]: - return self.records[key] - - def keys(self) -> Any: - return self.records.keys() - - -# Module-level callables so resolve_callable can find them -def sample_id_key(sample: Sample) -> str: - return str(sample.metadata["id"]) - - -def identity_extract(record: Dict[str, Any], sample: Sample) -> Optional[Dict[str, Any]]: - return record - - -def none_extract(record: Dict[str, Any], sample: Sample) -> Optional[Dict[str, Any]]: - return None - - -def odd_only_extract(record: Dict[str, Any], sample: Sample) -> Optional[Dict[str, Any]]: - """Return the record only for odd-id samples, None otherwise.""" - idx = int(str(sample.metadata["id"])[1:]) - if idx % 2 == 1: - return record - return None - - -def resolve_by_id(key: str, primary: PairedIndexedSource) -> Sample: - """Reverse-lookup: key 's' -> primary[N].""" - idx = int(key[1:]) - return primary[idx] - - -# --------------------------------------------------------------------------- -# left_outer -# --------------------------------------------------------------------------- - - -def test_left_outer_emits_all_primary() -> None: - primary = PairedIndexedSource(size=4) - store = DictStore({"s0": {"label": "a"}, "s2": {"label": "c"}}) - paired = PairedSource(primary=primary, secondary=store, key_fn=sample_id_key) - - samples = list(paired) - - assert len(samples) == 4 - assert [s.input for s in samples] == [0, 10, 20, 30] - assert [s.metadata["annotated"] for s in samples] == [True, False, True, False] - - -def test_left_outer_flattens_record_into_metadata() -> None: - primary = PairedIndexedSource(size=2) - store = DictStore({"s0": {"label": "dog", "confidence": 0.9}}) - paired = PairedSource(primary=primary, secondary=store, key_fn=sample_id_key) - - samples = list(paired) - - assert samples[0].metadata["label"] == "dog" - assert samples[0].metadata["confidence"] == 0.9 - assert samples[0].metadata["annotation_key"] == "s0" - assert "label" not in samples[1].metadata - assert samples[1].metadata["annotation_key"] == "s1" - - -def test_left_outer_preserves_original_metadata() -> None: - primary = PairedIndexedSource(size=1) - store = DictStore({"s0": {"label": "x"}}) - paired = PairedSource(primary=primary, secondary=store, key_fn=sample_id_key) - - sample = list(paired)[0] - assert sample.metadata["id"] == "s0" # primary metadata survived - assert sample.metadata["label"] == "x" - - -def test_left_outer_prefix() -> None: - primary = PairedIndexedSource(size=1) - store = DictStore({"s0": {"label": "x"}}) - paired = PairedSource(primary=primary, secondary=store, key_fn=sample_id_key, prefix="ann_") - - sample = list(paired)[0] - assert sample.metadata["ann_label"] == "x" - assert "label" not in sample.metadata - - -def test_left_outer_store_full_under() -> None: - primary = PairedIndexedSource(size=1) - store = DictStore({"s0": {"label": "x", "score": 0.5}}) - paired = PairedSource( - primary=primary, - secondary=store, - key_fn=sample_id_key, - store_full_under="raw_annotation", - ) - - sample = list(paired)[0] - assert sample.metadata["raw_annotation"] == {"label": "x", "score": 0.5} - # Still flattened too - assert sample.metadata["label"] == "x" - - -def test_left_outer_len_delegates_to_primary() -> None: - primary = PairedIndexedSource(size=7) - store = DictStore({"s0": {"label": "a"}}) - paired = PairedSource(primary=primary, secondary=store, key_fn=sample_id_key) - - assert len(paired) == 7 - - -def test_left_outer_getitem_matched_and_unmatched() -> None: - primary = PairedIndexedSource(size=3) - store = DictStore({"s1": {"label": "y"}}) - paired = PairedSource(primary=primary, secondary=store, key_fn=sample_id_key) - - matched = paired[1] - assert matched.metadata["annotated"] is True - assert matched.metadata["label"] == "y" - - unmatched = paired[0] - assert unmatched.metadata["annotated"] is False - assert "label" not in unmatched.metadata - - -# --------------------------------------------------------------------------- -# inner -# --------------------------------------------------------------------------- - - -def test_inner_emits_only_matched() -> None: - primary = PairedIndexedSource(size=4) - store = DictStore({"s0": {"label": "a"}, "s3": {"label": "d"}}) - paired = PairedSource(primary=primary, secondary=store, key_fn=sample_id_key, policy="inner") - - samples = list(paired) - - assert len(samples) == 2 - assert {s.metadata["annotation_key"] for s in samples} == {"s0", "s3"} - assert all(s.metadata["annotated"] for s in samples) - - -def test_inner_len_is_cached_scan() -> None: - primary = PairedIndexedSource(size=10) - store = DictStore({f"s{i}": {"label": "x"} for i in (0, 2, 4, 6)}) - paired = PairedSource(primary=primary, secondary=store, key_fn=sample_id_key, policy="inner") - - assert len(paired) == 4 - # Second call hits cache; should still be correct. - assert len(paired) == 4 - - -def test_inner_rejects_getitem() -> None: - primary = PairedIndexedSource(size=2) - store = DictStore({"s0": {"label": "a"}}) - paired = PairedSource(primary=primary, secondary=store, key_fn=sample_id_key, policy="inner") - - with pytest.raises(TypeError, match="left_outer"): - _ = paired[0] - - -# --------------------------------------------------------------------------- -# extract_fn -# --------------------------------------------------------------------------- - - -def test_extract_fn_transforms_record() -> None: - primary = PairedIndexedSource(size=2) - store = DictStore({"s0": {"label": "a"}, "s1": {"label": "b"}}) - paired = PairedSource( - primary=primary, - secondary=store, - key_fn=sample_id_key, - extract_fn=identity_extract, - ) - - samples = list(paired) - - assert samples[0].metadata["label"] == "a" - assert samples[1].metadata["label"] == "b" - - -def test_extract_fn_returning_none_marks_unannotated() -> None: - primary = PairedIndexedSource(size=3) - store = DictStore({"s0": {"label": "x"}, "s1": {"label": "y"}, "s2": {"label": "z"}}) - paired = PairedSource( - primary=primary, - secondary=store, - key_fn=sample_id_key, - extract_fn=odd_only_extract, - ) - - samples = list(paired) - - # Every sample emitted under left_outer; only odd ones are annotated - assert [s.metadata["annotated"] for s in samples] == [False, True, False] - - -def test_extract_fn_with_inner_policy_filters() -> None: - primary = PairedIndexedSource(size=4) - store = DictStore({f"s{i}": {"label": "x"} for i in range(4)}) - paired = PairedSource( - primary=primary, - secondary=store, - key_fn=sample_id_key, - extract_fn=odd_only_extract, - policy="inner", - ) - - samples = list(paired) - - assert {s.metadata["annotation_key"] for s in samples} == {"s1", "s3"} - - -def test_extract_fn_none_suppresses_flattening() -> None: - primary = PairedIndexedSource(size=1) - store = DictStore({"s0": {"label": "x"}}) - paired = PairedSource( - primary=primary, - secondary=store, - key_fn=sample_id_key, - extract_fn=none_extract, - ) - - sample = list(paired)[0] - assert sample.metadata["annotated"] is False - assert "label" not in sample.metadata - - -# --------------------------------------------------------------------------- -# Coarser-key / broadcast -# --------------------------------------------------------------------------- - - -def pack_key(sample: Sample) -> str: - """Every sample shares the key 'pack' - simulates pack-level annotation.""" - return "pack" - - -def test_coarser_key_broadcasts_to_all_matching_samples() -> None: - primary = PairedIndexedSource(size=3) - store = DictStore({"pack": {"drone": "dji_mavic"}}) - paired = PairedSource(primary=primary, secondary=store, key_fn=pack_key) - - samples = list(paired) - - assert all(s.metadata["annotated"] for s in samples) - assert all(s.metadata["drone"] == "dji_mavic" for s in samples) - - -# --------------------------------------------------------------------------- -# right_driven -# --------------------------------------------------------------------------- - - -def test_right_driven_iterates_annotation_keys() -> None: - primary = PairedIndexedSource(size=10) - store = DictStore({"s0": {"label": "a"}, "s3": {"label": "d"}}) - paired = PairedSource( - primary=primary, - secondary=store, - key_fn=sample_id_key, - policy="right_driven", - primary_resolver=resolve_by_id, - ) - - samples = list(paired) - - assert len(samples) == 2 - assert [s.metadata["annotation_key"] for s in samples] == ["s0", "s3"] - assert [s.input for s in samples] == [0, 30] - - -def test_right_driven_len_is_secondary_len() -> None: - primary = PairedIndexedSource(size=100) - store = DictStore({"s1": {"label": "a"}, "s5": {"label": "b"}, "s9": {"label": "c"}}) - paired = PairedSource( - primary=primary, - secondary=store, - key_fn=sample_id_key, - policy="right_driven", - primary_resolver=resolve_by_id, - ) - - assert len(paired) == 3 - - -def test_right_driven_skips_when_extract_fn_returns_none() -> None: - primary = PairedIndexedSource(size=4) - store = DictStore({f"s{i}": {"label": "x"} for i in range(4)}) - paired = PairedSource( - primary=primary, - secondary=store, - key_fn=sample_id_key, - policy="right_driven", - primary_resolver=resolve_by_id, - extract_fn=odd_only_extract, - ) - - samples = list(paired) - - assert {s.metadata["annotation_key"] for s in samples} == {"s1", "s3"} - - -# --------------------------------------------------------------------------- -# Validation -# --------------------------------------------------------------------------- - - -def test_invalid_policy_raises() -> None: - with pytest.raises(ValueError, match="Invalid policy"): - PairedSource( - primary=PairedIndexedSource(), - secondary=DictStore(), - key_fn=sample_id_key, - policy="outer_join", - ) - - -def test_right_driven_requires_primary_resolver() -> None: - with pytest.raises(ValueError, match="primary_resolver"): - PairedSource( - primary=PairedIndexedSource(), - secondary=DictStore(), - key_fn=sample_id_key, - policy="right_driven", - ) - - -def test_right_driven_requires_secondary_keys_method() -> None: - class NoKeys: - def __contains__(self, k: str) -> bool: # pragma: no cover - defensive - return False - - def __getitem__(self, k: str) -> Any: # pragma: no cover - defensive - raise KeyError(k) - - with pytest.raises(TypeError, match="keys"): - PairedSource( - primary=PairedIndexedSource(), - secondary=NoKeys(), - key_fn=sample_id_key, - policy="right_driven", - primary_resolver=resolve_by_id, - ) - - -def test_left_outer_requires_mapping_interface() -> None: - class NoContains: - pass - - with pytest.raises(TypeError, match="__contains__"): - PairedSource( - primary=PairedIndexedSource(), - secondary=NoContains(), - key_fn=sample_id_key, - ) - - -# --------------------------------------------------------------------------- -# String-path resolution -# --------------------------------------------------------------------------- - - -def test_key_fn_accepts_string_path() -> None: - primary = PairedIndexedSource(size=1) - store = DictStore({"s0": {"label": "x"}}) - paired = PairedSource( - primary=primary, - secondary=store, - key_fn=get_callable_path(sample_id_key), - ) - - sample = list(paired)[0] - assert sample.metadata["label"] == "x" - - -def test_extract_fn_accepts_string_path() -> None: - primary = PairedIndexedSource(size=1) - store = DictStore({"s0": {"label": "x"}}) - paired = PairedSource( - primary=primary, - secondary=store, - key_fn=sample_id_key, - extract_fn=get_callable_path(identity_extract), - ) - - sample = list(paired)[0] - assert sample.metadata["label"] == "x" - - -def test_callable_is_stored_as_string() -> None: - paired = PairedSource( - primary=PairedIndexedSource(), - secondary=DictStore(), - key_fn=sample_id_key, - ) - - assert isinstance(paired.key_fn, str) - assert ":sample_id_key" in paired.key_fn - - -# --------------------------------------------------------------------------- -# Chained / composed -# --------------------------------------------------------------------------- - - -def test_chained_paired_sources_compose() -> None: - """Pack-level + window-level annotations merged via two PairedSources.""" - primary = PairedIndexedSource(size=3) - pack_store = DictStore({"pack": {"drone": "mavic"}}) - window_store = DictStore({"s1": {"event": "takeoff"}}) - - pack_paired = PairedSource(primary=primary, secondary=pack_store, key_fn=pack_key) - full_paired = PairedSource(primary=pack_paired, secondary=window_store, key_fn=sample_id_key) - - samples = list(full_paired) - - # Every sample has drone (from pack), s1 also has event - assert all(s.metadata["drone"] == "mavic" for s in samples) - assert samples[1].metadata["event"] == "takeoff" - assert "event" not in samples[0].metadata - - -# --------------------------------------------------------------------------- -# Confluid serialization round-trip -# --------------------------------------------------------------------------- - - -def test_confluid_roundtrip_preserves_behavior() -> None: - primary = PairedIndexedSource(size=3) - store = DictStore({"s0": {"label": "a"}, "s2": {"label": "c"}}) - paired = PairedSource( - primary=primary, - secondary=store, - key_fn=sample_id_key, - policy="left_outer", - prefix="ann_", - ) - - yaml_state = confluid.dump(paired) - restored = confluid.load(yaml_state) - - original_result = [(s.input, s.metadata.get("ann_label"), s.metadata["annotated"]) for s in paired] - restored_result = [(s.input, s.metadata.get("ann_label"), s.metadata["annotated"]) for s in restored] - - assert original_result == restored_result diff --git a/tests/test_parallel.py b/tests/test_parallel.py index b86f7d2..1e84b25 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -2,34 +2,27 @@ import numpy as np -from dataflux.core import Flux +from recordstream import Image, item_data +from recordstream.core import Stream def heavy_op(x: np.ndarray) -> np.ndarray: - time.sleep(0.1) + time.sleep(0.1) # simulated per-record WORKLOAD (not a synchronization wait) return x * 2 def test_parallel_execution() -> None: - source = [np.array([i]) for i in range(10)] + source = [{"x": Image(np.array([i]))} for i in range(10)] start = time.time() - # Use a real top-level function for pickling - pipeline = Flux(source).map(heavy_op).parallel(workers=4) + # Use a real top-level function for pickling; key= targets the record entry's payload. + pipeline = Stream(source).map(heavy_op, key="x").parallel(workers=4) results = pipeline.collect() duration = time.time() - start assert len(results) == 10 - # In my previous run it was i*2 - assert results[0].input == 0 - assert results[9].input == 18 - # 10 items of 0.1s sequentially = 1.0s. 4 workers ≈ 0.3s + spawn overhead. - # Threshold is generous because GH Actions runners are noticeably slower - # than local dev machines (observed >5s on cold runners). We only assert - # the pipeline completes — this isn't a benchmark. + assert isinstance(results[0]["x"], Image) # item type survives the spawn round-trip + assert int(item_data(results[0]["x"])[0]) == 0 + assert int(item_data(results[9]["x"])[0]) == 18 + # We only assert the pipeline completes — this isn't a benchmark. assert duration < 15.0 - - -def test_parallel_with_joint() -> None: - # Already tested in test_joint.py, but helps coverage here too - pass diff --git a/tests/test_parallel_op.py b/tests/test_parallel_op.py deleted file mode 100644 index 4b142a9..0000000 --- a/tests/test_parallel_op.py +++ /dev/null @@ -1,123 +0,0 @@ -"""Tests for :class:`dataflux.ops.parallel.Parallel`.""" - -from __future__ import annotations - -import time -from typing import Iterable, Iterator, List, Optional - -import numpy as np - -from dataflux.core import Flux -from dataflux.ops.parallel import Parallel -from dataflux.sample import Sample - -# Top-level functions/classes — workers must be able to pickle these. - - -def double_input(sample: Sample) -> Sample: - return sample._replace(input=sample.input * 2) - - -def slow_double_input(sample: Sample) -> Sample: - time.sleep(0.05) - return sample._replace(input=sample.input * 2) - - -def drop_odd(sample: Sample) -> Optional[Sample]: - return sample if int(sample.input.item()) % 2 == 0 else None - - -class _TrackingSource: - """Iterable that records the maximum number of items pulled before any - are consumed downstream — used to verify bounded prefetch.""" - - def __init__(self, n: int, sleep_per_item: float = 0.0) -> None: - self.n = n - self.sleep_per_item = sleep_per_item - self.pulled = 0 - self.consumed = 0 - self.max_outstanding = 0 - - def __iter__(self) -> Iterator[np.ndarray]: - for i in range(self.n): - self.pulled += 1 - self.max_outstanding = max(self.max_outstanding, self.pulled - self.consumed) - if self.sleep_per_item: - time.sleep(self.sleep_per_item) - yield np.array([i]) - - def __len__(self) -> int: - return self.n - - -def test_parallel_inline_call_applies_ops_sequentially() -> None: - """``Parallel.__call__`` (used by ``Flux.__getitem__``) must apply its - inner ops sequentially in the calling process — same result as if the - ops were a plain list.""" - op = Parallel(ops=[double_input, double_input], workers=4) - sample = Sample(input=np.array([3])) - out = op(sample) - assert out is not None - assert out.input == np.array([12]) # 3 * 2 * 2 - - -def test_parallel_stream_yields_in_source_order() -> None: - source = [np.array([i]) for i in range(20)] - flux = Flux(source, ops=[Parallel(ops=[slow_double_input], workers=4)]) - results = list(flux) - assert [int(r.input.item()) for r in results] == [i * 2 for i in range(20)] - - -def test_parallel_stream_bounds_prefetch() -> None: - """With ``workers=4`` and a slow inner op, the source iterator must not - be drained more than ``2*workers + 1 = 9`` items ahead of consumption.""" - workers = 4 - expected_limit = 2 * workers + 1 - source = _TrackingSource(n=50) - - def consume(stream: Iterable[Sample]) -> List[Sample]: - results: List[Sample] = [] - for sample in stream: - source.consumed += 1 - results.append(sample) - return results - - flux = Flux(source, ops=[Parallel(ops=[slow_double_input], workers=workers)]) - results = consume(flux) - - assert len(results) == 50 - assert ( - source.max_outstanding <= expected_limit - ), f"prefetch unbounded: max_outstanding={source.max_outstanding} > {expected_limit}" - - -def test_parallel_filter_drops_nones_and_preserves_order() -> None: - source = [np.array([i]) for i in range(10)] - flux = Flux(source, ops=[Parallel(ops=[drop_odd, double_input], workers=3)]) - results = list(flux) - assert [int(r.input.item()) for r in results] == [0, 4, 8, 12, 16] - - -def test_parallel_workers_must_be_positive() -> None: - import pytest - - with pytest.raises(ValueError, match="workers="): - Parallel(ops=[double_input], workers=0) - - -def test_parallel_close_propagates_to_inner_ops() -> None: - closed: List[str] = [] - - class ClosableOp: - def __init__(self, name: str) -> None: - self.name = name - - def __call__(self, s: Sample) -> Sample: - return s - - def close(self) -> None: - closed.append(self.name) - - op = Parallel(ops=[ClosableOp("a"), ClosableOp("b")], workers=2) - op.close() - assert closed == ["a", "b"] diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py new file mode 100644 index 0000000..e66c6bc --- /dev/null +++ b/tests/test_pipeline.py @@ -0,0 +1,124 @@ +"""``Pipeline`` — the compose-group unit: identity, None-propagation, close(), Fluid entries, +bare library transforms, import safety.""" + +import subprocess +import sys +from pathlib import Path + +import confluid +import numpy as np +import pytest + +from recordstream import FilterOp, Image, Label, Mask, Pipeline, Record +from recordstream.ops.structure import RenameField +from tests._fixtures import FixtureFlip + + +def _rec() -> Record: + return {"image": Image(np.ones((4, 6, 3), dtype=np.float32)), "class": Label("x")} + + +class TestImportSafety: + def test_package_imports_without_torchvision(self) -> None: + # The top-level package must import without torchvision/albumentations (the engine's + # op-family dispatch detects them by MRO module NAME — no import), so discovery stays + # safe on hosts missing the libraries. + code = ( + "import sys; import recordstream; " + "assert 'torchvision' not in sys.modules; " + "assert 'albumentations' not in sys.modules" + ) + subprocess.run([sys.executable, "-c", code], check=True, cwd=str(Path(__file__).resolve().parents[1])) + + +class TestPipelineSemantics: + def test_zero_arg_is_identity(self) -> None: + rec = _rec() + out = Pipeline()(rec) + assert out is rec # no ops -> the record passes through untouched + + def test_none_propagation_mid_chain(self) -> None: + # A filter-drop mid-chain stops the pipeline and propagates None; later ops never run. + ran = [] + + def probe(record: Record) -> Record: + ran.append(True) + return record + + p = Pipeline([FilterOp(lambda r: False), probe]) + assert p(_rec()) is None + assert ran == [] # the op after the drop never fired + + def test_close_propagates_to_inner_ops(self) -> None: + closed = [] + + class _Closeable: + def __call__(self, record: Record) -> Record: + return record + + def close(self) -> None: + closed.append(True) + + p = Pipeline([_Closeable(), FilterOp(lambda r: True)]) # FilterOp has no close -> skipped + p.close() + assert closed == [True] + + def test_fluid_entry_flowed_and_cached(self) -> None: + # A config-deferred entry (confluid Target marker) is flowed on first call and the + # live op is cached back into the transforms list. + p = Pipeline(transforms=[confluid.Target(RenameField, src="class", dst="klass")]) + out = p(_rec()) + assert out is not None and "klass" in out and "class" not in out + assert isinstance(p.transforms[0], RenameField) # cached in place + out2 = p(_rec()) # second call uses the cached live op + assert out2 is not None and "klass" in out2 + + def test_native_ops_chain(self) -> None: + s = {"image": Image(np.arange(4 * 6 * 3).reshape(4, 6, 3).astype(np.float32))} + out = Pipeline([FixtureFlip(p=1.0), FixtureFlip(p=1.0)])(s) + assert out is not None + # Two flips cancel out. + assert np.array_equal(np.asarray(out["image"]), np.asarray(s["image"])) + + +class TestBareLibraryEntries: + def test_bare_albumentations_entry(self) -> None: + import albumentations as A + + rec = { + "image": Image(np.arange(6 * 8 * 3).reshape(6, 8, 3).astype(np.float32)), + "mask": Mask(np.arange(6 * 8).reshape(6, 8).astype(np.uint8)), + "class": Label("x"), + } + out = Pipeline([A.HorizontalFlip(p=1.0)])(rec) + assert out is not None + # ONE joint draw moved image and mask together; item types + metadata survive. + assert isinstance(out["image"], Image) and out["image"].layout == "HWC" + assert isinstance(out["mask"], Mask) + assert np.array_equal(np.asarray(out["image"]), np.asarray(rec["image"])[:, ::-1]) + assert np.array_equal(np.asarray(out["mask"]), np.asarray(rec["mask"])[:, ::-1]) + assert out["class"].value == "x" # not an albumentations key — never reached the library + + def test_bare_torchvision_v2_entries(self) -> None: + import torch + + v2 = pytest.importorskip("torchvision.transforms.v2") + + # The EXPLICIT v2.ToImage() conversion first (numpy HWC -> CHW tv_tensor), then any v2 + # transform — the engine passes the dict straight through (never converts silently). + rec = {"image": np.arange(4 * 6 * 3).reshape(4, 6, 3).astype(np.float32)} + converted = Pipeline([v2.ToImage()])(rec) + assert converted is not None + assert isinstance(converted["image"], torch.Tensor) and tuple(converted["image"].shape) == (3, 4, 6) + cropped = Pipeline([v2.RandomCrop(2)])(converted) + assert cropped is not None and tuple(cropped["image"].shape) == (3, 2, 2) + + def test_mixed_native_and_bare_library(self) -> None: + import albumentations as A + + rec = {"image": Image(np.arange(6 * 8 * 3).reshape(6, 8, 3).astype(np.float32))} + # bare albumentations flip + native fixture flip = identity (both moved the image once). + out = Pipeline([A.HorizontalFlip(p=1.0), FixtureFlip(p=1.0)])(rec) + assert out is not None + assert np.array_equal(np.asarray(out["image"]), np.asarray(rec["image"])) + assert isinstance(out["image"], Image) diff --git a/tests/test_predictions.py b/tests/test_predictions.py new file mode 100644 index 0000000..261c638 --- /dev/null +++ b/tests/test_predictions.py @@ -0,0 +1,245 @@ +"""Tests for :mod:`recordstream.predictions` — the predictions-sink contract + the classification sink.""" + +from typing import Any, Dict, List + +import pytest +import torch + +from recordstream import ClassificationPredictionsSink, PredictionsSink, Record + + +class _CapturingOp: + """Tiny op that records every record dict it sees, for assertion purposes.""" + + def __init__(self) -> None: + self.calls: List[Record] = [] + self.closed = False + + def __call__(self, record: Record) -> Record: + self.calls.append(record) + return record + + def close(self) -> None: + self.closed = True + + +def _make_prediction( + probs: List[float], + class_idx: int, +) -> Dict[str, Any]: + """Build a ClassificationOutput-shaped dict from a flat prob list.""" + p = torch.tensor(probs, dtype=torch.float32) + return { + "logits": torch.log(p), # not used by sink, but realistic + "probs": p, + "class_idx": torch.tensor(class_idx, dtype=torch.int64), + } + + +def _make_metadata() -> Dict[str, Any]: + """Arbitrary per-record metadata — the sink must not require any particular key.""" + return {"source_id": "p1", "index": 0} + + +# --- Happy path ------------------------------------------------------------ + + +def test_top1_path_writes_predicted_columns() -> None: + op = _CapturingOp() + sink = ClassificationPredictionsSink( + ops=[op], + class_names={0: "DJI MINI3", 1: "DJI AVATA2", 2: "Other"}, + top_k=1, + ) + sink.write(_make_prediction([0.1, 0.7, 0.2], class_idx=1), _make_metadata()) + + assert len(op.calls) == 1 + meta = op.calls[0]["metadata"] + assert meta["predicted_class_id"] == 1 + assert meta["predicted_class_label"] == "DJI AVATA2" + assert meta["predicted_confidence"] == pytest.approx(0.7, abs=1e-6) + assert len(meta["predicted_top_k"]) == 1 + assert meta["predicted_top_k"][0]["label"] == "DJI AVATA2" + + +def test_top_k_path_returns_descending_probabilities() -> None: + op = _CapturingOp() + sink = ClassificationPredictionsSink( + ops=[op], + class_names={0: "a", 1: "b", 2: "c", 3: "d"}, + top_k=3, + ) + sink.write(_make_prediction([0.1, 0.4, 0.45, 0.05], class_idx=2), _make_metadata()) + + top_k = op.calls[0]["metadata"]["predicted_top_k"] + assert len(top_k) == 3 + # Sorted descending by probability. + probs = [entry["probability"] for entry in top_k] + assert probs == sorted(probs, reverse=True) + assert top_k[0]["label"] == "c" + assert top_k[1]["label"] == "b" + assert top_k[2]["label"] == "a" + + +def test_top_k_clamps_when_exceeds_num_classes() -> None: + """top_k=10 with 3 classes → returns 3, not crash.""" + op = _CapturingOp() + sink = ClassificationPredictionsSink(ops=[op], top_k=10) + sink.write(_make_prediction([0.5, 0.3, 0.2], class_idx=0), _make_metadata()) + assert len(op.calls[0]["metadata"]["predicted_top_k"]) == 3 + + +def test_label_names_string_keys_work() -> None: + """YAML loads int keys as strings; both forms must work.""" + op = _CapturingOp() + sink = ClassificationPredictionsSink( + ops=[op], + class_names={"0": "a", "1": "b"}, # str keys + top_k=1, + ) + sink.write(_make_prediction([0.2, 0.8], class_idx=1), _make_metadata()) + assert op.calls[0]["metadata"]["predicted_class_label"] == "b" + + +def test_label_names_missing_key_falls_back_to_str_class_id() -> None: + op = _CapturingOp() + sink = ClassificationPredictionsSink( + ops=[op], + class_names={0: "zero"}, # only class 0 mapped + top_k=1, + ) + sink.write(_make_prediction([0.1, 0.9], class_idx=1), _make_metadata()) + # Missing class 1 → falls back to "1". + assert op.calls[0]["metadata"]["predicted_class_label"] == "1" + + +def test_no_label_names_uses_str_class_id() -> None: + op = _CapturingOp() + sink = ClassificationPredictionsSink(ops=[op], top_k=2) + sink.write(_make_prediction([0.7, 0.3], class_idx=0), _make_metadata()) + meta = op.calls[0]["metadata"] + assert meta["predicted_class_label"] == "0" + assert {entry["label"] for entry in meta["predicted_top_k"]} == {"0", "1"} + + +def test_threaded_record_carries_metadata_key_only() -> None: + """The threaded record is a plain dict carrying only the prediction metadata key.""" + op = _CapturingOp() + sink = ClassificationPredictionsSink(ops=[op], top_k=1) + sink.write(_make_prediction([0.6, 0.4], class_idx=0), _make_metadata()) + record = op.calls[0] + assert isinstance(record, dict) + assert list(record.keys()) == ["metadata"] + + +def test_close_propagates_to_ops() -> None: + op = _CapturingOp() + sink = ClassificationPredictionsSink(ops=[op], top_k=1) + sink.close() + assert op.closed is True + + +# --- Confidence threshold -------------------------------------------------- + + +def test_confidence_threshold_drops_low_predictions() -> None: + op = _CapturingOp() + sink = ClassificationPredictionsSink( + ops=[op], + top_k=1, + confidence_threshold=0.5, + ) + # top1 prob = 0.4, below threshold → no record produced. + sink.write(_make_prediction([0.4, 0.3, 0.3], class_idx=0), _make_metadata()) + assert op.calls == [] + + +def test_confidence_threshold_keeps_high_predictions() -> None: + op = _CapturingOp() + sink = ClassificationPredictionsSink( + ops=[op], + top_k=1, + confidence_threshold=0.5, + ) + sink.write(_make_prediction([0.7, 0.3], class_idx=0), _make_metadata()) + assert len(op.calls) == 1 + + +# --- Edge cases ------------------------------------------------------------ + + +def test_zero_arg_construction_works_and_ops_is_validated_lazily() -> None: + """The package-wide lazy-init rule: buildable with no args, validated where it is used.""" + sink = ClassificationPredictionsSink() # no raise — a form/schema generator can introspect it + with pytest.raises(ValueError, match="'ops' is empty"): + sink.write(_make_prediction([0.5, 0.5], class_idx=0), {}) + + +def test_empty_ops_list_raises_on_write_not_construction() -> None: + sink = ClassificationPredictionsSink(ops=[]) + with pytest.raises(ValueError, match="'ops' is empty"): + sink.write(_make_prediction([0.5, 0.5], class_idx=0), {}) + + +def test_zero_top_k_raises() -> None: + with pytest.raises(ValueError, match="top_k"): + ClassificationPredictionsSink(ops=[_CapturingOp()], top_k=0) + + +def test_missing_probs_skips_silently_with_warning() -> None: + """If the model emits a malformed prediction, the sink skips rather than crashing.""" + op = _CapturingOp() + sink = ClassificationPredictionsSink(ops=[op], top_k=1) + sink.write({"logits": torch.zeros(3)}, _make_metadata()) # no probs/class_idx + assert op.calls == [] + + +def test_handles_2d_probs_with_leading_singleton() -> None: + """Some pipelines emit [1, C] instead of [C]; sink must accept both.""" + op = _CapturingOp() + sink = ClassificationPredictionsSink(ops=[op], top_k=1) + sink.write( + { + "logits": torch.zeros(1, 3), + "probs": torch.tensor([[0.1, 0.6, 0.3]], dtype=torch.float32), + "class_idx": torch.tensor(1, dtype=torch.int64), + }, + _make_metadata(), + ) + assert len(op.calls) == 1 + assert op.calls[0]["metadata"]["predicted_class_id"] == 1 + assert op.calls[0]["metadata"]["predicted_confidence"] == pytest.approx(0.6, abs=1e-6) + + +# --- Contract + neutrality ------------------------------------------------- + + +def test_sink_satisfies_the_predictions_sink_protocol() -> None: + """`isinstance` works because the Protocol is @runtime_checkable (method NAMES only).""" + assert isinstance(ClassificationPredictionsSink(ops=[_CapturingOp()]), PredictionsSink) + + +def test_write_works_with_completely_empty_metadata() -> None: + """No domain key is required — the sink is modality-neutral, diagnostics included. + + The pre-move version built its log line from `pack_id` / `iq_file` / `window_start_sample`, + signal-domain keys, in a class documented as neutral. A record identified only by ordinal + keeps that promise. + """ + op = _CapturingOp() + sink = ClassificationPredictionsSink(ops=[op], top_k=1) + sink.write(_make_prediction([0.2, 0.8], class_idx=1), {}) + assert op.calls[0]["metadata"]["predicted_class_id"] == 1 + + +def test_a_malformed_prediction_is_reported_by_ordinal(monkeypatch: pytest.MonkeyPatch) -> None: + """The diagnostic locates the bad record without naming any domain metadata key.""" + from recordstream import predictions as predictions_module + + warnings: List[str] = [] + monkeypatch.setattr(predictions_module.logger, "warning", lambda msg: warnings.append(str(msg))) + + sink = ClassificationPredictionsSink(ops=[_CapturingOp()], top_k=1) + sink.write(_make_prediction([0.5, 0.5], class_idx=0), {}) # record #0 — fine + sink.write({"logits": torch.zeros(2)}, {}) # record #1 — malformed + assert len(warnings) == 1 and "record #1" in warnings[0] diff --git a/tests/test_projection.py b/tests/test_projection.py new file mode 100644 index 0000000..06ad97b --- /dev/null +++ b/tests/test_projection.py @@ -0,0 +1,110 @@ +"""Tests for :mod:`recordstream.projection` — the key-addressed walk helpers. + +Focused on :func:`recordstream.first_value`, the one-peek primitive: what it unwraps, +what it skips, what it costs, and that it inherits ``iter_key``'s three properties +(projection-aware sources, deferred sources, laziness). +""" + +from typing import Any, Collection, Dict, Iterator, List + +from confluid import PartialClass + +from recordstream import Label, MultiLabel, first_value, is_class_id +from recordstream.items import Record + + +class _Source: + """A plain iterable source — no projection protocol, no laziness tricks.""" + + def __init__(self, records: List[Record]) -> None: + self.records = records + + def __iter__(self) -> Iterator[Record]: + return iter(self.records) + + +class _CountingProjectionSource: + """A projection-aware source that records what it was asked for, and counts reads.""" + + def __init__(self, records: List[Record]) -> None: + self.records = records + self.requested: List[Collection[str]] = [] + self.reads = 0 + + def project(self, keys: Collection[str]) -> Iterator[Record]: + self.requested.append(set(keys)) + for record in self.records: + self.reads += 1 + yield {k: v for k, v in record.items() if k in keys} + + def __iter__(self) -> Iterator[Record]: # pragma: no cover - project() is what runs + return iter(self.records) + + +def test_first_value_returns_the_first_present_value() -> None: + source = _Source([{"class": "cat"}, {"class": "dog"}]) + assert first_value(source, "class") == "cat" + + +def test_first_value_skips_leading_nones_and_missing_keys() -> None: + """``None`` means "no value here", which is not an answer about the column's kind.""" + source = _Source([{"class": None}, {"other": 1}, {"class": "dog"}]) + assert first_value(source, "dog_key_absent_everywhere") is None + assert first_value(source, "class") == "dog" + + +def test_an_all_none_column_answers_none_rather_than_raising() -> None: + source = _Source([{"class": None}, {"class": None}]) + assert first_value(source, "class") is None + + +def test_an_empty_source_answers_none() -> None: + assert first_value(_Source([]), "class") is None + + +def test_a_label_item_unwraps_to_its_value() -> None: + source = _Source([{"class": Label(value="cat", classes=["cat", "dog"])}]) + assert first_value(source, "class") == "cat" + + +def test_a_multilabel_item_unwraps_to_its_values_LIST() -> None: + """The peek is how a consumer learns the column is multi-label — by the ITEM type. + + ``iter_key`` unwraps a ``MultiLabel`` to its list, so a sequence here IS multi-label + rather than a guess about what a list might mean. + """ + source = _Source([{"class": MultiLabel(values=["cat", "dog"])}]) + peeked = first_value(source, "class") + assert peeked == ["cat", "dog"] + assert isinstance(peeked, (list, tuple, set)) + + +def test_the_peek_answers_the_names_versus_ids_question() -> None: + """The canonical call site: one peek + ``is_class_id`` decides whether a LabelMap is needed.""" + assert not is_class_id(first_value(_Source([{"class": "cat"}]), "class")) + assert is_class_id(first_value(_Source([{"class": 3}]), "class")) + + +def test_first_value_stops_at_the_first_hit() -> None: + """One peek, not a walk — the whole reason to call this instead of ``list(iter_key(...))``.""" + source = _CountingProjectionSource([{"class": i} for i in range(100)]) + assert first_value(source, "class") == 0 + assert source.reads == 1 + + +def test_first_value_asks_only_for_the_requested_key() -> None: + """A projection-aware source never builds the values this does not ask for.""" + source = _CountingProjectionSource([{"class": 0, "image": "expensive"}]) + assert first_value(source, "class") == 0 + assert source.requested == [{"class"}] + + +def test_first_value_materializes_a_deferred_source() -> None: + """``project()`` flows a ``!class:`` marker, so a caller writes no ``flow()`` here.""" + marker = PartialClass(_Source, records=[{"class": "cat"}]) + assert first_value(marker, "class") == "cat" + + +def test_plain_values_pass_through_verbatim() -> None: + payload: Dict[str, Any] = {"samplerate": 30.72e6} + assert first_value(_Source([payload]), "samplerate") == 30.72e6 diff --git a/tests/test_record_source.py b/tests/test_record_source.py new file mode 100644 index 0000000..d66540a --- /dev/null +++ b/tests/test_record_source.py @@ -0,0 +1,193 @@ +"""``ensure_record_dataset`` / ``RecordSource`` — normalizing a wired dataset slot. + +A consumer's ``train_set`` / ``val_set`` / ``test_set`` may be wired to a ``Stream``, another +torch ``Dataset``, a bare source, or a plain list. Normalizing once up front lets the rest of a +pipeline assume record items unconditionally. +""" + +from typing import Any, Dict, Iterator, List + +import numpy as np +import torch + +from recordstream import Label, Record, Stream, ensure_materialized, ensure_record_dataset + + +class _RowDataset(torch.utils.data.Dataset): + """A map-style dataset of raw (non-record) rows.""" + + def __init__(self, rows: List[Dict[str, Any]]) -> None: + self.rows = rows + + def __len__(self) -> int: + return len(self.rows) + + def __getitem__(self, index: int) -> Dict[str, Any]: + return self.rows[index] + + +class _IterableSource: + """A bare iterable source — no ``__getitem__``, no ``Dataset`` base.""" + + def __iter__(self) -> Iterator[Record]: + yield {"image": np.zeros(2), "class": Label(0)} + yield {"image": np.ones(2), "class": Label(1)} + + +def _records(n: int = 2) -> List[Record]: + return [{"image": np.zeros(2), "class": Label(i)} for i in range(n)] + + +def test_a_stream_is_returned_as_is() -> None: + """Identity matters: a subclass's own wrap (e.g. its ``class_names``) must survive.""" + stream = Stream(source=_records()) + assert ensure_record_dataset(stream) is stream + + +def test_a_stream_subclass_keeps_its_attributes() -> None: + stream = Stream(source=_records()) + stream.class_names = ["a", "b"] # type: ignore[attr-defined] + assert ensure_record_dataset(stream).class_names == ["a", "b"] # type: ignore[attr-defined] + + +def test_a_plain_list_becomes_a_map_style_record_dataset() -> None: + dataset = ensure_record_dataset(_records(3)) + assert isinstance(dataset, Stream) + assert len(dataset) == 3 # type: ignore[arg-type] + assert isinstance(dataset[0], dict) and "image" in dataset[0] # type: ignore[index] + + +def test_a_torch_dataset_of_rows_is_wrapped() -> None: + dataset = ensure_record_dataset(_RowDataset(_records(2))) + assert isinstance(dataset, Stream) + assert len(dataset) == 2 # type: ignore[arg-type] + + +def test_a_bare_iterable_source_is_wrapped_and_iterates() -> None: + dataset = ensure_record_dataset(_IterableSource()) + assert isinstance(dataset, Stream) + assert [int(record["class"].value) for record in dataset] == [0, 1] # type: ignore[union-attr] + + +def test_it_is_idempotent() -> None: + once = ensure_record_dataset(_records()) + assert ensure_record_dataset(once) is once + + +def test_exported_from_the_package_root() -> None: + import recordstream + + assert "ensure_record_dataset" in recordstream.__all__ + assert "RecordSource" in recordstream.__all__ + + +# --------------------------------------------------------------------------- +# ensure_materialized — normalize a source's STATE, not its type +# --------------------------------------------------------------------------- +class _LazySource: + """A source that does its real work on first read, like every source in this package.""" + + def __init__(self, rows: int = 3) -> None: + self.rows = rows + self.reads = 0 + self.built = False + + def __len__(self) -> int: + return self.rows # deliberately does NOT build: len() was measured not to be enough + + def __getitem__(self, index: int) -> Dict[str, Any]: + if index >= self.rows: + raise IndexError(index) + self.built = True + self.reads += 1 + return {"image": index} + + +def test_ensure_materialized_builds_what_a_read_needs() -> None: + """The point: after this, a forked child inherits a source that needs nothing.""" + source = _LazySource() + + returned = ensure_materialized(source) + + assert source.built, "the source was never actually read" + assert returned is source, "it returns the source so it composes" + + +def test_len_is_not_enough_which_is_why_this_reads_a_record() -> None: + """Pins the measurement the docstring cites, so the implementation cannot be 'simplified'. + + Loading a dataset object is not the same as building everything a read needs — a real + HuggingFaceSource still went to the Hub from inside a worker after `len()` in the parent. + """ + source = _LazySource() + + len(source) + + assert not source.built, "if len() built it, this test's premise is wrong, not the code" + + +def test_it_reads_exactly_one_record() -> None: + """One is enough, and more would make warming a large split expensive.""" + source = _LazySource(rows=100) + + ensure_materialized(source) + + assert source.reads == 1 + + +def test_an_empty_source_is_not_an_error() -> None: + """A split with no rows has nothing to build, and a caller should need no guard.""" + assert ensure_materialized(_LazySource(rows=0)) is not None + + +def test_an_iterable_only_source_is_warmed_too() -> None: + """Not every source is map-style; the iterable path must build the same way.""" + + class _IterableOnly: + def __init__(self) -> None: + self.built = False + + def __iter__(self) -> Iterator[Dict[str, Any]]: + self.built = True + yield {"image": 0} + + source = _IterableOnly() + ensure_materialized(source) + assert source.built + + +# --------------------------------------------------------------------------- # +# prepare_record_dataset — the TYPE + STATE composition +# --------------------------------------------------------------------------- # + + +def test_prepare_none_passes_through() -> None: + """An unwired optional split needs no guard at the call site.""" + from recordstream import prepare_record_dataset + + assert prepare_record_dataset(None) is None + + +def test_prepare_wraps_and_warms_in_one_call() -> None: + """The composition IS ensure_record_dataset + ensure_materialized: a lazy source comes + back as a record Stream whose build already happened in THIS process.""" + from recordstream import Stream, prepare_record_dataset + + source = _LazySource(rows=2) + prepared = prepare_record_dataset(source) + assert isinstance(prepared, Stream) + assert source.built, "the lazy build must happen here, not in a forked worker" + + +def test_prepare_returns_a_stream_as_the_same_stream() -> None: + """Identity matters: a label-encoding Stream keeps its class_names.""" + from recordstream import Stream, prepare_record_dataset + + stream = Stream(source=[{"class": 1}]) + assert prepare_record_dataset(stream) is stream + + +def test_prepare_is_exported_from_the_package_root() -> None: + import recordstream + + assert "prepare_record_dataset" in recordstream.__all__ diff --git a/tests/test_runnable.py b/tests/test_runnable.py new file mode 100644 index 0000000..9bcb469 --- /dev/null +++ b/tests/test_runnable.py @@ -0,0 +1,73 @@ +"""Tests for the runnable protocol markers (TorchRunner + ProgressReporting).""" + +from typing import List + +from recordstream.runnable import ProgressReporting, TorchRunner + + +def test_torch_runner_sets_the_needs_autograd_flag() -> None: + # The class is named for the framework; the FLAG is named for what it decides — a GUI + # executor reads `__needs_autograd__` (duck-typed) to re-enable autograd around run(). + assert TorchRunner.__needs_autograd__ is True + + class Trainer(TorchRunner): + pass + + assert Trainer().__needs_autograd__ is True # inherited by subclasses + + +def test_progress_reporting_noop_without_callback() -> None: + class R(ProgressReporting): + pass + + r = R() + # No callback set -> _report_progress is a silent no-op (never raises). + r._report_progress(1, 10, "x") + assert r._progress_callback is None + + +def test_progress_reporting_fires_when_set() -> None: + reports: List[tuple] = [] + + class R(ProgressReporting): + pass + + r = R() + r.set_progress_callback(lambda v, t, d: reports.append((v, t, d))) + r._report_progress(3, 10, "step") + assert reports == [(3.0, 10.0, "step")] + + +def test_progress_reporting_skips_nonpositive_total() -> None: + reports: List[tuple] = [] + + class R(ProgressReporting): + pass + + r = R() + r.set_progress_callback(lambda v, t, d: reports.append((v, t, d))) + r._report_progress(1, None, "x") # unknown total -> skipped + r._report_progress(1, 0, "x") # non-positive -> skipped + assert reports == [] + + +def test_progress_reporting_swallows_callback_errors() -> None: + def boom(v: float, t: float, d: str) -> None: + raise RuntimeError("sink broke") + + class R(ProgressReporting): + pass + + r = R() + r.set_progress_callback(boom) + r._report_progress(1, 10, "x") # must not raise — progress is cosmetic + + +def test_set_progress_callback_clears_with_none() -> None: + class R(ProgressReporting): + pass + + r = R() + r.set_progress_callback(lambda v, t, d: None) + r.set_progress_callback(None) + assert r._progress_callback is None diff --git a/tests/test_sample.py b/tests/test_sample.py deleted file mode 100644 index d3c6fa6..0000000 --- a/tests/test_sample.py +++ /dev/null @@ -1,38 +0,0 @@ -from typing import Any, cast - -import numpy as np - -from dataflux.sample import Sample - - -def test_sample_from_any() -> None: - # 1. From dict - d = {"input": np.array([1, 2]), "target": 1, "metadata": {"id": "test"}} - s = Sample.from_any(d) - assert np.array_equal(s.input, cast(Any, d["input"])) - assert s.target == 1 - assert s.metadata["id"] == "test" - - # 2. From tuple (input, target) - t = (np.array([3, 4]), 0) - s2 = Sample.from_any(t) - assert np.array_equal(s2.input, t[0]) - assert s2.target == 0 - - # 3. From tuple (input,) - hits line 22 - t3 = (np.array([7, 8]),) - s3 = Sample.from_any(t3) - assert np.array_equal(s3.input, t3[0]) - assert s3.target is None - - # 4. From single item (input only) - val = np.array([5, 6]) - s4 = Sample.from_any(val) - assert np.array_equal(s4.input, val) - assert s4.target is None - - -def test_sample_to_tuple() -> None: - s = Sample(input=1, target=2, metadata={"a": 3}) - t = s.to_tuple() - assert t == (1, 2, {"a": 3}) diff --git a/tests/test_sources.py b/tests/test_sources.py deleted file mode 100644 index 0a17937..0000000 --- a/tests/test_sources.py +++ /dev/null @@ -1,250 +0,0 @@ -"""Tests for DataFlux sources, in particular DatasetSplit.""" - -from typing import Any, Iterator, List - -import confluid # type: ignore[import-not-found] -import pytest - -from dataflux.core import Flux -from dataflux.sample import Sample -from dataflux.sources import DatasetSplit - - -@confluid.configurable -class IndexedSource: - """A configurable indexable source for DatasetSplit tests. - - Stores a list of integers; each `__getitem__` returns ``Sample(input=i)``. - """ - - def __init__(self, size: int = 100) -> None: - self.size = size - - def __len__(self) -> int: - return self.size - - def __getitem__(self, index: int) -> Sample: - return Sample(input=index, target=None, metadata={"idx": index}) - - def __iter__(self) -> Iterator[Sample]: - for i in range(self.size): - yield self[i] - - -# --------------------------------------------------------------------------- -# Fraction mode -# --------------------------------------------------------------------------- - - -def test_fraction_mode_partitions_cleanly() -> None: - """train + val cover the full source with no overlap.""" - source = IndexedSource(size=100) - train = DatasetSplit(source=source, split="train", val_fraction=0.1, seed=42) - val = DatasetSplit(source=source, split="val", val_fraction=0.1, seed=42) - - train_idx = {s.input for s in train} - val_idx = {s.input for s in val} - - assert len(train) == 90 - assert len(val) == 10 - assert train_idx.isdisjoint(val_idx) - assert train_idx | val_idx == set(range(100)) - - -def test_fraction_mode_is_deterministic_across_instances() -> None: - """Same seed + source -> identical shuffle -> stable split.""" - source = IndexedSource(size=50) - first = list(DatasetSplit(source=source, split="val", val_fraction=0.2, seed=7)) - second = list(DatasetSplit(source=source, split="val", val_fraction=0.2, seed=7)) - assert [s.input for s in first] == [s.input for s in second] - - -def test_fraction_mode_different_seeds_differ() -> None: - source = IndexedSource(size=50) - a = {s.input for s in DatasetSplit(source=source, split="val", val_fraction=0.2, seed=1)} - b = {s.input for s in DatasetSplit(source=source, split="val", val_fraction=0.2, seed=2)} - # Overwhelmingly likely to differ on 50 elements - assert a != b - - -def test_fraction_mode_requires_seed() -> None: - with pytest.raises(ValueError, match="seed"): - DatasetSplit(source=IndexedSource(size=10), split="train", val_fraction=0.1) - - -def test_fraction_mode_requires_val_fraction() -> None: - with pytest.raises(ValueError, match="val_fraction"): - DatasetSplit(source=IndexedSource(size=10), split="train", seed=0) - - -def test_fraction_mode_rejects_invalid_split() -> None: - with pytest.raises(ValueError, match="split"): - DatasetSplit(source=IndexedSource(size=10), split="test", val_fraction=0.1, seed=0) - - -def test_fraction_mode_rejects_out_of_range_fraction() -> None: - src = IndexedSource(size=10) - with pytest.raises(ValueError, match="val_fraction"): - DatasetSplit(source=src, split="train", val_fraction=1.5, seed=0) - - -# --------------------------------------------------------------------------- -# Range mode -# --------------------------------------------------------------------------- - - -def test_range_mode_slices_source() -> None: - source = IndexedSource(size=20) - view = DatasetSplit(source=source, start=5, end=15) - values = [s.input for s in view] - assert values == list(range(5, 15)) - assert len(view) == 10 - - -def test_range_mode_open_ended() -> None: - source = IndexedSource(size=20) - head = DatasetSplit(source=source, end=10) - tail = DatasetSplit(source=source, start=10) - assert [s.input for s in head] == list(range(10)) - assert [s.input for s in tail] == list(range(10, 20)) - - -def test_range_mode_clamps_out_of_bounds() -> None: - source = IndexedSource(size=5) - view = DatasetSplit(source=source, start=-10, end=100) - assert len(view) == 5 - - -def test_passthrough_returns_full_source() -> None: - source = IndexedSource(size=8) - view = DatasetSplit(source=source) - assert [s.input for s in view] == list(range(8)) - - -def test_mixed_modes_rejected() -> None: - source = IndexedSource(size=10) - with pytest.raises(ValueError, match="not both"): - DatasetSplit(source=source, split="train", val_fraction=0.1, seed=0, start=0, end=5) - - -def test_invalid_source_type() -> None: - with pytest.raises(TypeError, match="__len__"): - - class _Plain: - pass - - DatasetSplit(source=_Plain()) - - -# --------------------------------------------------------------------------- -# Indexing -# --------------------------------------------------------------------------- - - -def test_getitem_resolves_through_underlying_source() -> None: - source = IndexedSource(size=30) - view = DatasetSplit(source=source, start=10, end=20) - # First element of the view is index 10 of the source - assert view[0].input == 10 - assert view[-1].input == 19 # Python list indexing supports negatives - - -# --------------------------------------------------------------------------- -# Works inside a Flux pipeline -# --------------------------------------------------------------------------- - - -def _scale(value: int, factor: int = 1) -> int: - return value * factor - - -def test_dataset_split_inside_flux() -> None: - source = IndexedSource(size=20) - view = DatasetSplit(source=source, start=0, end=5) - flux = Flux(source=view).map(_scale, factor=10) - results: List[Sample] = flux.collect() - assert [s.input for s in results] == [0, 10, 20, 30, 40] - - -# --------------------------------------------------------------------------- -# Confluid serialization round-trip -# --------------------------------------------------------------------------- - - -def test_serialization_roundtrip_preserves_split() -> None: - source = IndexedSource(size=40) - split = DatasetSplit(source=source, split="val", val_fraction=0.25, seed=123) - yaml_state = confluid.dump(split) - assert "!class:DatasetSplit" in yaml_state - assert "val_fraction: 0.25" in yaml_state - assert "seed: 123" in yaml_state - - restored: Any = confluid.load(yaml_state) - # Reloaded source is a fresh IndexedSource with size=40 — feed it identically. - assert len(restored) == len(split) - assert [s.input for s in restored] == [s.input for s in split] - - -def test_ref_based_splits_share_source_and_partition_cleanly() -> None: - """Two DatasetSplits referencing the same YAML source via ``!ref:`` share - the underlying source by identity (single load) and produce disjoint, - complementary views.""" - yaml_state = """ -hf_train: !class:IndexedSource() - size: 50 - -train_set: !class:DatasetSplit() - source: !ref:hf_train - split: train - val_fraction: 0.2 - seed: 9 - -val_set: !class:DatasetSplit() - source: !ref:hf_train - split: val - val_fraction: 0.2 - seed: 9 -""" - state: Any = confluid.load(yaml_state) - train = state["train_set"] - val = state["val_set"] - - # Confluid !ref: resolves to the same live object — critical for - # expensive sources like HuggingFaceSource so the dataset loads once. - assert train.source is val.source - assert train.source is state["hf_train"] - - train_idx = {s.input for s in train} - val_idx = {s.input for s in val} - assert train_idx.isdisjoint(val_idx) - assert train_idx | val_idx == set(range(50)) - - -# --------------------------------------------------------------------------- -# HuggingFaceSource.__len__ / count semantics (no network — __init__ bypassed) -# --------------------------------------------------------------------------- - - -def _hf_source_with_count(count: Any, dataset_len: int = 13) -> Any: - """Build a HuggingFaceSource without the network (skip __init__'s load_dataset).""" - from dataflux.sources import HuggingFaceSource - - src: Any = HuggingFaceSource.__new__(HuggingFaceSource) - src.count = count - src._dataset = list(range(dataset_len)) - return src - - -def test_hf_source_len_count_zero_means_all() -> None: - # Regression: count=0 must report the full length (matching __iter__'s ``count or len``), - # not 0 — otherwise a len()-based stepper (e.g. FluxStudio's WalkDataset) sees an empty - # source even though iteration yields every sample. - assert len(_hf_source_with_count(0)) == 13 - - -def test_hf_source_len_count_none_means_all() -> None: - assert len(_hf_source_with_count(None)) == 13 - - -def test_hf_source_len_positive_count_caps() -> None: - assert len(_hf_source_with_count(5)) == 5 diff --git a/tests/test_storage.py b/tests/test_storage.py deleted file mode 100644 index 7bb9c45..0000000 --- a/tests/test_storage.py +++ /dev/null @@ -1,140 +0,0 @@ -from pathlib import Path -from typing import cast - -import numpy as np -import torch - -from dataflux.core import Flux -from dataflux.sample import Sample -from dataflux.storage.directory import DirectorySink -from dataflux.storage.hdf5 import HDF5Sink, HDF5Source -from dataflux.storage.zarr import ZarrBatchSink, ZarrGroupSink - - -def test_hdf5_storage(tmp_path: Path) -> None: - h5_path = tmp_path / "test.h5" - samples = [ - Sample(input=torch.randn(10), target=torch.tensor([1])), - Sample(input=torch.randn(10), target=torch.tensor([0])), - ] - - # Write - sink = HDF5Sink(h5_path, overwrite=True) - Flux(samples).to_sink(sink) - sink.close() - - # Read - source = HDF5Source(h5_path) - loaded = list(source) - assert len(loaded) == 2 - assert torch.allclose(loaded[0].input, samples[0].input) - assert loaded[0].target == samples[0].target - assert len(source) == 2 - source.close() - - -def test_zarr_group_storage(tmp_path: Path) -> None: - zarr_path = tmp_path / "test.zarr" - samples = [ - Sample(input=np.random.randn(5), metadata={"id": "a"}), - Sample(input=np.random.randn(10), metadata={"id": "b"}), - ] - - sink = ZarrGroupSink(zarr_path, overwrite=True) - Flux(samples).to_sink(sink) - - # Verification (ZarrGroupSink doesn't have a Source yet, but we check files) - assert zarr_path.exists() - assert (zarr_path / "sample_000000").exists() - assert (zarr_path / "sample_000001").exists() - - -def test_zarr_batch_storage(tmp_path: Path) -> None: - zarr_path = tmp_path / "batch.zarr" - samples = [Sample(input=np.ones((10, 10), dtype=np.float32)) for _ in range(5)] - - sink = ZarrBatchSink(zarr_path, shape=[10, 10], overwrite=True) - Flux(samples).to_sink(sink) - - # Check if data was written - import zarr - - z = zarr.open_array(store=f"{zarr_path}/data", mode="r") - assert z.shape == (5, 10, 10) - assert np.all(z[:] == 1.0) - - -def test_directory_storage(tmp_path: Path) -> None: - dir_path = tmp_path / "out_dir" - samples = [ - Sample(input=np.array([1, 2]), metadata={"name": "first"}), - Sample(input=np.array([3, 4]), metadata={"name": "second"}), - ] - - sink = DirectorySink(dir_path, overwrite=True) - Flux(samples).to_sink(sink) - - -def test_directory_storage_separate(tmp_path: Path) -> None: - dir_path = tmp_path / "out_dir_sep" - samples = [ - Sample(input=np.array([1, 2]), target=np.array([0])), - ] - - # use_npz=False hits lines 52-54 - sink = DirectorySink(dir_path, overwrite=True, use_npz=False) - Flux(samples).to_sink(sink) - - assert (dir_path / "000000" / "data.npy").exists() - assert (dir_path / "000000" / "target.npy").exists() - - -def test_hdf5_to_numpy_direct() -> None: - from dataflux.storage.hdf5 import to_numpy - - # Hits line 19 - assert to_numpy(123) == 123 - - -def test_hdf5_flush(tmp_path: Path) -> None: - h5_path = tmp_path / "flush.h5" - sink = HDF5Sink(h5_path) - sink.open() - sink.flush() # Hits lines 107-110 - sink.close() - - -def test_hdf5_overwrite(tmp_path: Path) -> None: - h5_path = tmp_path / "over.h5" - s1 = [Sample(input=np.array([1]))] - s2 = [Sample(input=np.array([2]))] - - # 1. Write first - sink1 = HDF5Sink(h5_path, overwrite=True) - Flux(s1).to_sink(sink1) - sink1.close() - - # 2. Overwrite - sink2 = HDF5Sink(h5_path, overwrite=True) - Flux(s2).to_sink(sink2) - sink2.close() - - # 3. Verify only s2 exists - source = HDF5Source(h5_path) - loaded = list(source) - assert len(loaded) == 1 - assert loaded[0].input == 2 - - -def test_zarr_group_with_target(tmp_path: Path) -> None: - zarr_path = tmp_path / "target.zarr" - samples = [Sample(input=np.array([1]), target=np.array([0]))] - - sink = ZarrGroupSink(zarr_path, overwrite=True) - Flux(samples).to_sink(sink) - - import zarr - - z = zarr.open_group(str(zarr_path), mode="r") - grp = cast(zarr.Group, z["sample_000000"]) - assert "target" in grp diff --git a/tests/test_structure_ops.py b/tests/test_structure_ops.py new file mode 100644 index 0000000..74e7c88 --- /dev/null +++ b/tests/test_structure_ops.py @@ -0,0 +1,88 @@ +"""Structure ops over dict records — RenameField/DropField/CopyField/SelectFields.""" + +import numpy as np +import pytest + +from recordstream import Boxes, Image, Label, Record +from recordstream.ops.structure import CopyField, DropField, RenameField, SelectFields + + +def _record() -> Record: + return {"image": Image(np.zeros((2, 2, 3))), "regions": Boxes(boxes=[[0, 0, 1, 1]]), "class": Label("x")} + + +class TestRenameField: + def test_renames_preserving_order(self) -> None: + out = RenameField(src="regions", dst="boxes")(_record()) + assert "regions" not in out and isinstance(out["boxes"], Boxes) + assert list(out.keys()) == ["image", "boxes", "class"] # renamed in place + + def test_rename_onto_existing_replaces(self) -> None: + out = RenameField(src="class", dst="image")(_record()) + assert isinstance(out["image"], Label) + + def test_validation(self) -> None: + with pytest.raises(ValueError, match="both 'src' and 'dst'"): + RenameField()(_record()) + with pytest.raises(KeyError, match="unknown key"): + RenameField(src="nope", dst="x")(_record()) + + +class TestDropField: + def test_drops(self) -> None: + out = DropField(key="class")(_record()) + assert "class" not in out and list(out.keys()) == ["image", "regions"] + + def test_missing_raises_unless_ok(self) -> None: + with pytest.raises(KeyError, match="unknown key"): + DropField(key="nope")(_record()) + rec = _record() + assert DropField(key="nope", missing_ok=True)(rec) is rec + + def test_missing_key_config_raises(self) -> None: + with pytest.raises(ValueError, match="'key'"): + DropField()(_record()) + + +class TestCopyField: + def test_copies_same_object(self) -> None: + out = CopyField(src="regions", dst="regions_backup")(_record()) + assert out["regions_backup"] is out["regions"] # same value object (values are immutable) + + def test_copy_replaces_existing_dst(self) -> None: + out = CopyField(src="class", dst="image")(_record()) + assert isinstance(out["image"], Label) + + def test_validation(self) -> None: + with pytest.raises(ValueError, match="both 'src' and 'dst'"): + CopyField()(_record()) + with pytest.raises(KeyError, match="unknown key"): + CopyField(src="nope", dst="x")(_record()) + + +class TestSelectFields: + def test_keeps_only_and_orders(self) -> None: + out = SelectFields(keys=["class", "image"])(_record()) + assert list(out.keys()) == ["class", "image"] + + def test_validation(self) -> None: + with pytest.raises(ValueError, match="'keys'"): + SelectFields()(_record()) + with pytest.raises(KeyError, match="unknown keys"): + SelectFields(keys=["image", "nope"])(_record()) + + +class TestCopyOnWrite: + def test_ops_never_mutate_the_incoming_record(self) -> None: + rec = _record() + RenameField(src="class", dst="klass")(rec) + DropField(key="class")(rec) + CopyField(src="class", dst="klass")(rec) + SelectFields(keys=["image"])(rec) + assert list(rec.keys()) == ["image", "regions", "class"] # untouched + + +def test_configurable_marks() -> None: + for cls in (RenameField, DropField, CopyField, SelectFields): + assert getattr(cls, "__confluid_category__", None) == "op" + assert getattr(cls, "__confluid_group__", None) == "structure" diff --git a/tests/test_transform.py b/tests/test_transform.py new file mode 100644 index 0000000..ba41da5 --- /dev/null +++ b/tests/test_transform.py @@ -0,0 +1,121 @@ +"""Transforms — type dispatch, once-per-record params, cross-key consistency, ``field=`` pin. + +Native-kernel machinery is pinned via the test fixture ``FixtureFlip`` (recordstream ships no +native augmentation transforms — libraries drop into ops lists bare, invoked natively by the +engine's op-family dispatch). +""" + +import numpy as np +import pytest + +from recordstream import Boxes, Image, Label, Mask, Pipeline, Record, Transform, as_transform +from tests._fixtures import FixtureFlip + + +def _seg() -> Record: + return { + "image": Image(np.arange(8 * 10 * 3).reshape(8, 10, 3).astype(np.float32)), + "mask": Mask(np.arange(8 * 10).reshape(8, 10)), + "regions": Boxes(boxes=[[1, 1, 4, 4]], labels=["a"], canvas=(8, 10)), + "class": Label("a"), + "gain_db": -3.0, # a plain scalar side value is just another key + } + + +class TestKernelDispatchMachinery: + def test_cross_key_consistency(self) -> None: + out = FixtureFlip(p=1.0)(_seg()) + seg = _seg() + assert out is not None + assert np.array_equal(np.asarray(out["image"]), np.asarray(seg["image"])[:, ::-1]) + assert np.array_equal(np.asarray(out["mask"]), np.asarray(seg["mask"])[:, ::-1]) + assert out["regions"].boxes == [[6, 1, 9, 4]] # W=10: x -> W-x + assert out["class"].value == "a" # no handler — untouched + assert out["gain_db"] == -3.0 # plain value — untouched + + def test_p_zero_is_identity(self) -> None: + out = FixtureFlip(p=0.0)(_seg()) + assert out is not None + assert np.array_equal(np.asarray(out["image"]), np.asarray(_seg()["image"])) + assert out["regions"].boxes == [[1, 1, 4, 4]] + + def test_field_pin(self) -> None: + # field= pins the op to ONE key: only "image" moves, the other handled types stay. + out = FixtureFlip(p=1.0, field="image")(_seg()) + assert out is not None + assert not np.array_equal(np.asarray(out["image"]), np.asarray(_seg()["image"])) + assert np.array_equal(np.asarray(out["mask"]), np.asarray(_seg()["mask"])) # mask skipped + assert out["regions"].boxes == [[1, 1, 4, 4]] # regions skipped + + def test_field_pin_on_unhandled_type_is_noop(self) -> None: + # Still type-gated: pinning to a key whose value type has no kernel changes nothing. + out = FixtureFlip(p=1.0, field="class")(_seg()) + assert out is not None + assert np.array_equal(np.asarray(out["image"]), np.asarray(_seg()["image"])) + assert out["class"].value == "a" + + def test_image_layout_chw(self) -> None: + s = {"image": Image(np.arange(3 * 4 * 5).reshape(3, 4, 5), layout="CHW")} + out = FixtureFlip(p=1.0)(s) + assert out is not None + assert np.array_equal(np.asarray(out["image"]), np.asarray(s["image"])[:, :, ::-1]) + + def test_regions_uses_canvas_without_image(self) -> None: + s = {"regions": Boxes(boxes=[[2, 0, 5, 3]], canvas=(8, 10))} + out = FixtureFlip(p=1.0)(s) + assert out is not None and out["regions"].boxes == [[5, 0, 8, 3]] + + def test_regions_without_reference_width_raises(self) -> None: + s = {"regions": Boxes(boxes=[[2, 0, 5, 3]])} # no image, no canvas + with pytest.raises(ValueError, match="no reference width"): + FixtureFlip(p=1.0)(s) + + def test_params_sampled_once(self) -> None: + # A partial-probability flip must be all-or-nothing across keys (shared decision), + # never per-key independent draws. + seg = _seg() + for _ in range(25): + out = FixtureFlip(p=0.5)(seg) + assert out is not None + image_flipped = not np.array_equal(np.asarray(out["image"]), np.asarray(seg["image"])) + regions_flipped = out["regions"].boxes != seg["regions"].boxes + assert image_flipped == regions_flipped + + def test_input_record_not_mutated(self) -> None: + # Transform.__call__ is copy-on-write: the incoming dict keeps its original values. + seg = _seg() + FixtureFlip(p=1.0)(seg) + assert np.array_equal(np.asarray(seg["image"]), np.asarray(_seg()["image"])) + + +class TestPipelineAndFunction: + def test_pipeline_is_sequential(self) -> None: + s = {"x": Image(np.ones((2, 2, 3), dtype=np.float32))} + double = as_transform(lambda d: d * 2, handles=(Image,)) + out = Pipeline([double, double])(s) + assert out is not None and np.allclose(np.asarray(out["x"]), 4.0) + + def test_function_transform_field_pin(self) -> None: + s = {"a": Image(np.ones((2, 2, 3))), "b": Image(np.ones((2, 2, 3)))} + out = as_transform(lambda d: d + 1, handles=(Image,), field="a")(s) + assert out is not None + assert np.allclose(np.asarray(out["a"]), 2.0) and np.allclose(np.asarray(out["b"]), 1.0) + + def test_function_transform_preserves_item_type(self) -> None: + s = {"m": Mask(np.ones((2, 2)))} + out = as_transform(lambda d: d * 3, handles=(Mask,))(s) + assert out is not None and isinstance(out["m"], Mask) and np.allclose(np.asarray(out["m"]), 3.0) + + def test_pipeline_repr(self) -> None: + assert "FixtureFlip" in repr(Pipeline([FixtureFlip()])) + + +class TestBaseTransform: + def test_default_get_params_and_passthrough(self) -> None: + # A transform with no kernels leaves every value alone. + s = {"x": Label("v"), "g": 1.5} + out = Transform()(s) + assert out == s and out is not s # equal copy, not the same dict + + def test_zero_arg_construction(self) -> None: + assert FixtureFlip().p == 0.5 and Transform().field is None diff --git a/tests/test_typed_collate.py b/tests/test_typed_collate.py new file mode 100644 index 0000000..d0a82b5 --- /dev/null +++ b/tests/test_typed_collate.py @@ -0,0 +1,134 @@ +"""The record collate — batched record convention (golden shapes consumers rely on).""" + +from dataclasses import dataclass +from typing import Any, Dict, List, Sequence + +import numpy as np +import pytest +import torch + +from recordstream import Boxes, Image, Label, Mask, Record, collate, collate_records, get_collate, register_item + + +@register_item +@dataclass +class _CollateBlob: + data: object = None + rate: float = 1.0 + + +def _record(i: int) -> Record: + return { + "image": Image(np.full((4, 5, 3), float(i), dtype=np.float32)), + "mask": Mask(np.full((4, 5), i, dtype=np.int64)), + "class": Label(i, classes=["a", "b", "c"]), + "gain_db": float(i) - 3.0, # a plain scalar entry + } + + +class TestRecordCollate: + def test_golden_shapes(self) -> None: + # THE batch convention consumers rely on: ONE batched record dict, payloads stacked + # per key, per-record item attrs as lists, plain values as plain lists. + batch = collate([_record(0), _record(1), _record(2)]) + assert isinstance(batch, dict) + assert np.asarray(batch["image"]).shape == (3, 4, 5, 3) # stacked payload + assert isinstance(batch["image"], Image) + assert np.asarray(batch["mask"]).shape == (3, 4, 5) + assert batch["class"].value == [0, 1, 2] # per-record attrs become lists + assert batch["class"].classes == [["a", "b", "c"]] * 3 + assert batch["gain_db"] == [-3.0, -2.0, -1.0] # plain values -> a plain list + + def test_default_key_and_explicit_key(self) -> None: + records = [_record(0), _record(1)] + default = collate(records) # the default registry key is "record" + explicit = get_collate("record")(records) + assert isinstance(default, dict) and isinstance(explicit, dict) + assert np.array_equal(np.asarray(default["image"]), np.asarray(explicit["image"])) + assert get_collate("record") is collate_records + + def test_item_attr_lists_decode_back_into_one_item(self) -> None: + batch = collate_records([_record(0), _record(1)]) + # The batched Image is ONE Image whose layout attr is the per-record list. + assert isinstance(batch["image"], Image) and batch["image"].layout == ["HWC", "HWC"] + + def test_torch_payloads_stack_to_tensor(self) -> None: + records = [{"sig": _CollateBlob(torch.ones(8) * i, rate=float(i))} for i in range(2)] + batch = collate(records) + assert isinstance(batch["sig"].data, torch.Tensor) and batch["sig"].data.shape == (2, 8) + assert batch["sig"].rate == [0.0, 1.0] + + def test_plain_string_values_batch_as_list(self) -> None: + batch = collate_records([{"f": "a.iq"}, {"f": "b.iq"}]) + assert batch["f"] == ["a.iq", "b.iq"] + + def test_heterogeneous_batch_raises(self) -> None: + odd = {"other": Label("x")} + with pytest.raises(ValueError, match="do not match the batch keys"): + collate([_record(0), odd]) + + def test_empty_batch_raises(self) -> None: + with pytest.raises(ValueError, match="empty batch"): + get_collate("record")([]) + with pytest.raises(ValueError, match="empty batch"): + collate([]) + + def test_non_dict_items_raise(self) -> None: + with pytest.raises(TypeError, match="expected record dicts"): + get_collate("record")([1, 2, 3]) + + def test_unknown_key_raises_with_known_keys(self) -> None: + with pytest.raises(KeyError, match="no collate registered"): + get_collate("nope") + + +# --------------------------------------------------------------------------- # +# The collate REGISTRY: a task collate opts out of the generic folding rules +# (the docs/record-model.md detection example — variable-N boxes cannot stack). +# --------------------------------------------------------------------------- # +def _ragged_detection_records() -> List[Record]: + return [ + { + "image": Image(np.zeros((4, 4, 3), dtype=np.float32)), + "target": Boxes(boxes=[[0, 0, 2, 2]], labels=[1]), + "pack": "a", + }, + { + "image": Image(np.zeros((4, 4, 3), dtype=np.float32)), + "target": Boxes(boxes=[[0, 0, 1, 1], [1, 1, 3, 3], [0, 2, 2, 4]], labels=[0, 1, 0]), + "pack": "b", + }, + ] + + +def test_generic_collate_leaves_ragged_regions_as_lists() -> None: + # Rule 2: the generic fold can only give per-record lists for a wrapper item — + # exactly why detection registers its own collate. + batch = collate_records(_ragged_detection_records()) + assert isinstance(batch["target"], Boxes) + assert [len(b) for b in batch["target"].boxes] == [1, 3] + + +def test_registered_task_collate_produces_its_own_batch_contract() -> None: + import torch + + from recordstream import collate, get_collate, register_collate + + @register_collate("_test_detection") + def detection_collate(items: Sequence[Record]) -> Dict[str, Any]: + images = torch.stack([torch.as_tensor(np.asarray(r["image"])).permute(2, 0, 1) for r in items]) + targets = [ + { + "boxes": torch.as_tensor(r["target"].boxes, dtype=torch.float32).reshape(-1, 4), + "labels": torch.as_tensor(r["target"].labels, dtype=torch.int64), + } + for r in items + ] + metadata = [{k: v for k, v in r.items() if k not in ("image", "target")} for r in items] + return {"images": images, "targets": targets, "metadata": metadata} + + assert get_collate("_test_detection") is detection_collate + batch = collate(_ragged_detection_records(), key="_test_detection") + assert tuple(batch["images"].shape) == (2, 3, 4, 4) + assert [tuple(t["boxes"].shape) for t in batch["targets"]] == [(1, 4), (3, 4)] # raggedness preserved + assert batch["metadata"] == [{"pack": "a"}, {"pack": "b"}] diff --git a/tests/test_typed_detection_target_ops.py b/tests/test_typed_detection_target_ops.py new file mode 100644 index 0000000..505d0f0 --- /dev/null +++ b/tests/test_typed_detection_target_ops.py @@ -0,0 +1,262 @@ +"""The two detection target-shaping ops over dict records. + +Pins the native transforms that build a detection pipeline's torchvision-style +``{boxes, labels}`` target as a :class:`~recordstream.Boxes` item: + +* :class:`recordstream.ops.target.CocoToTorchVisionDetection` — a HuggingFace / COCO ``objects`` + annotation → a target ``Boxes``; +* :class:`recordstream.ops.target.MasksToDetectionBoxes` — a segmentation ``Mask`` → a target ``Boxes``. + +Each op REUSES its conversion helper, so the op's ``boxes`` / ``labels`` tensors are pinned +byte-identical to the helper (parity). recordstream-only — no domain-package import. +""" + +import numpy as np +import pytest +import torch +from confluid.registry import get_registry, resolve_class + +from recordstream import Boxes, Image, Label, Mask, collate_records +from recordstream.ops.target import ( + CocoToTorchVisionDetection, + MasksToDetectionBoxes, + coco_to_detection, + masks_to_detection, +) + +# A COCO / HF objects annotation: two boxes in [x, y, w, h] pixels + integer categories. +_OBJECTS = {"bbox": [[10.0, 20.0, 30.0, 40.0], [5.0, 6.0, 7.0, 8.0]], "category": [1, 3]} + + +def _instance_mask() -> np.ndarray: + """A 2-D instance mask with three known objects (pixel values 1/2/3), one per instance.""" + mask = np.zeros((10, 12), dtype=np.uint8) + mask[1:4, 2:5] = 1 # object 1 + mask[6:9, 7:10] = 2 # object 2 + mask[0:2, 9:12] = 3 # object 3 + return mask + + +# --------------------------------------------------------------------------- # +# CocoToTorchVisionDetection +# --------------------------------------------------------------------------- # +class TestCocoToTorchVisionDetection: + def test_produces_target_regions(self) -> None: + out = CocoToTorchVisionDetection(field="objects")({"objects": Label(_OBJECTS)}) + regions = out["target"] + assert isinstance(regions, Boxes) + assert isinstance(regions.boxes, torch.Tensor) + assert isinstance(regions.labels, torch.Tensor) + assert regions.boxes.shape == (2, 4) + assert regions.labels.shape == (2,) + + def test_parity_with_helper(self) -> None: + out = CocoToTorchVisionDetection(field="objects")({"objects": Label(_OBJECTS)}) + expected = coco_to_detection(_OBJECTS) + assert torch.equal(out["target"].boxes, expected["boxes"]) + assert torch.equal(out["target"].labels, expected["labels"]) + + def test_parity_xyxy_and_label_offset(self) -> None: + objects = {"bbox": [[10.0, 20.0, 40.0, 60.0]], "category": [2]} + out = CocoToTorchVisionDetection(field="objects", bbox_format="xyxy", label_offset=1)( + {"objects": Label(objects)} + ) + expected = coco_to_detection(objects, bbox_format="xyxy", label_offset=1) + assert torch.equal(out["target"].boxes, expected["boxes"]) + assert torch.equal(out["target"].labels, expected["labels"]) + + def test_empty_annotation_yields_empty_tensors(self) -> None: + out = CocoToTorchVisionDetection(field="objects")({"objects": Label({"bbox": [], "category": []})}) + assert out["target"].boxes.shape == (0, 4) + assert out["target"].labels.shape == (0,) + + def test_default_picks_first_label(self) -> None: + rec = {"image": Image(np.zeros((2, 2, 3), dtype=np.uint8)), "objects": Label(_OBJECTS)} + out = CocoToTorchVisionDetection()(rec) + assert out["target"].boxes.shape == (2, 4) + + def test_new_output_key_keeps_source(self) -> None: + out = CocoToTorchVisionDetection(field="objects", output="det")({"objects": Label(_OBJECTS)}) + assert isinstance(out["det"], Boxes) + assert out["objects"].value == _OBJECTS # source left intact + + def test_missing_field_raises(self) -> None: + with pytest.raises(ValueError, match="field 'nope' not in record"): + CocoToTorchVisionDetection(field="nope")({"objects": Label(_OBJECTS)}) + + def test_empty_record_raises(self) -> None: + with pytest.raises(ValueError, match="record is empty"): + CocoToTorchVisionDetection()({}) + + def test_non_dict_source_raises(self) -> None: + # The shared helper rejects a non-objects-shaped value loudly. + with pytest.raises(TypeError, match="objects mapping"): + CocoToTorchVisionDetection(field="objects")({"objects": Label("not a dict")}) + + +# --------------------------------------------------------------------------- # +# MasksToDetectionBoxes +# --------------------------------------------------------------------------- # +class TestMasksToDetectionBoxes: + def test_instance_mask_produces_target_regions(self) -> None: + out = MasksToDetectionBoxes(field="mask")({"mask": Mask(_instance_mask())}) + regions = out["target"] + assert isinstance(regions, Boxes) + assert isinstance(regions.boxes, torch.Tensor) + assert isinstance(regions.labels, torch.Tensor) + assert regions.boxes.shape == (3, 4) # three instances + assert regions.labels.tolist() == [1, 1, 1] # every box → foreground class 1 + + def test_instance_parity_with_helper(self) -> None: + mask = _instance_mask() + out = MasksToDetectionBoxes(field="mask")({"mask": Mask(mask)}) + expected = masks_to_detection(mask) + assert torch.equal(out["target"].boxes, expected["boxes"]) + assert torch.equal(out["target"].labels, expected["labels"]) + + def test_connected_components_parity(self) -> None: + # A binary/semantic mask (all objects share value 1): connected=True splits into blobs. + binary = (_instance_mask() != 0).astype(np.uint8) + out = MasksToDetectionBoxes(field="mask", connected=True, label=2)({"mask": Mask(binary)}) + expected = masks_to_detection(binary, connected=True, label=2) + assert out["target"].boxes.shape[0] == 3 # three connected blobs + assert torch.equal(out["target"].boxes, expected["boxes"]) + assert torch.equal(out["target"].labels, expected["labels"]) + + def test_connected_equals_instance_mode_on_disjoint_instances(self) -> None: + # The two derivations agree when every instance is its own connected component. This + # pins the axis order of the connected path: before connected_component_boxes emitted + # xyxy directly, this branch transposed row/col tuples — the bug class this guards. + mask = _instance_mask() + instance = masks_to_detection(mask) + connected = masks_to_detection((mask != 0).astype(np.uint8), connected=True) + boxes_a = sorted(map(tuple, instance["boxes"].tolist())) + boxes_b = sorted(map(tuple, connected["boxes"].tolist())) + assert boxes_a == boxes_b + + def test_min_area_drops_small_instances(self) -> None: + mask = _instance_mask() + out = MasksToDetectionBoxes(field="mask", min_area=10)({"mask": Mask(mask)}) + expected = masks_to_detection(mask, min_area=10) + assert torch.equal(out["target"].boxes, expected["boxes"]) + + def test_empty_mask_yields_empty_tensors(self) -> None: + out = MasksToDetectionBoxes(field="mask")({"mask": Mask(np.zeros((4, 4), dtype=np.uint8))}) + assert out["target"].boxes.shape == (0, 4) + assert out["target"].labels.shape == (0,) + + def test_default_picks_first_mask(self) -> None: + rec = {"image": Image(np.zeros((2, 2, 3), dtype=np.uint8)), "seg": Mask(_instance_mask())} + out = MasksToDetectionBoxes()(rec) + assert out["target"].boxes.shape == (3, 4) + + def test_new_output_key_keeps_source(self) -> None: + out = MasksToDetectionBoxes(field="mask", output="det")({"mask": Mask(_instance_mask())}) + assert isinstance(out["det"], Boxes) + assert isinstance(out["mask"], Mask) # source left intact + + def test_missing_field_raises(self) -> None: + with pytest.raises(ValueError, match="field 'nope' not in record"): + MasksToDetectionBoxes(field="nope")({"mask": Mask(_instance_mask())}) + + def test_no_mask_or_array_field_raises(self) -> None: + with pytest.raises(ValueError, match="no Mask or array-bearing field"): + MasksToDetectionBoxes()({"lbl": Label("x")}) + + +# --------------------------------------------------------------------------- # +# Collate — per-record Boxes gather into a list of detection targets. +# --------------------------------------------------------------------------- # +def test_record_collate_gathers_regions_as_list() -> None: + a = CocoToTorchVisionDetection(field="objects")({"objects": Label(_OBJECTS)}) + c = CocoToTorchVisionDetection(field="objects")( + {"objects": Label({"bbox": [[1.0, 2.0, 3.0, 4.0]], "category": [5]})} + ) + batch = collate_records([a, c]) + # Variable-N boxes can't be stacked → the collate gathers them as a per-record list of tensors. + assert isinstance(batch["target"], Boxes) + assert isinstance(batch["target"].boxes, list) and len(batch["target"].boxes) == 2 + assert batch["target"].boxes[0].shape == (2, 4) + assert batch["target"].boxes[1].shape == (1, 4) + + +# --------------------------------------------------------------------------- # +# Discovery + zero-arg construction. +# --------------------------------------------------------------------------- # +def test_zero_arg_constructible() -> None: + assert CocoToTorchVisionDetection().output == "target" + assert CocoToTorchVisionDetection().bbox_format == "xywh" + assert MasksToDetectionBoxes().output == "target" + assert MasksToDetectionBoxes().connected is False + + +@pytest.mark.parametrize( + ("name", "cls"), + [ + ("CocoToTorchVisionDetection", CocoToTorchVisionDetection), + ("MasksToDetectionBoxes", MasksToDetectionBoxes), + ], +) +def test_discovery_tags(name: str, cls: type) -> None: + assert cls.__confluid_category__ == "op" # type: ignore[attr-defined] + assert cls.__confluid_group__ == "structure" # type: ignore[attr-defined] + assert resolve_class(name) is cls + registry = get_registry() + assert name in registry.list_classes(category="op") + assert name in registry.list_classes(group="structure") + + +# --------------------------------------------------------------------------- # +# ResizeDetection — the coupled image+boxes resize +# --------------------------------------------------------------------------- # +class TestResizeDetection: + def _record(self) -> dict: + import torch + + from recordstream import Boxes, Image + + image = Image((np.arange(40 * 20 * 3) % 256).reshape(40, 20, 3).astype(np.uint8)) # H=40, W=20 + target = Boxes(boxes=torch.tensor([[5.0, 10.0, 15.0, 30.0]]), labels=torch.tensor([1])) + return {"image": image, "target": target} + + def test_image_and_boxes_move_together(self) -> None: + import torch + + from recordstream.ops.target import ResizeDetection + + out = ResizeDetection(width=40, height=80)(self._record()) # 2x on both axes + assert np.asarray(out["image"]).shape[:2] == (80, 40) + assert torch.allclose(out["target"].boxes, torch.tensor([[10.0, 20.0, 30.0, 60.0]])) + assert out["target"].canvas == (80, 40) + assert out["target"].labels.tolist() == [1] + + def test_boxes_stay_in_their_framework(self) -> None: + from recordstream import Boxes, Image + from recordstream.ops.target import ResizeDetection + + record = { + "image": Image(np.zeros((10, 10, 3), dtype=np.uint8)), + "target": Boxes(boxes=np.array([[1.0, 1.0, 5.0, 5.0]]), labels=np.array([0])), + } + out = ResizeDetection(width=20, height=20)(record) + assert isinstance(out["target"].boxes, np.ndarray) + assert out["target"].boxes.tolist() == [[2.0, 2.0, 10.0, 10.0]] + + def test_zero_arg_construction_validates_lazily(self) -> None: + import pytest + + from recordstream.ops.target import ResizeDetection + + op = ResizeDetection() # zero-arg per the lazy-construction mandate + with pytest.raises(ValueError, match="width/height"): + op(self._record()) + + def test_a_float_image_is_rejected_with_the_ordering_hint(self) -> None: + import pytest + + from recordstream import Image + from recordstream.ops.target import ResizeDetection + + record = {"image": Image(np.zeros((8, 8, 3), dtype=np.float32))} + with pytest.raises(TypeError, match="before ToTensor"): + ResizeDetection(width=4, height=4)(record) diff --git a/tests/test_typed_flow.py b/tests/test_typed_flow.py new file mode 100644 index 0000000..eeef1e5 --- /dev/null +++ b/tests/test_typed_flow.py @@ -0,0 +1,418 @@ +"""FlowGraph over dict records — merge_from fan-in, step[key]/bare-step bind, lowering parity.""" + +from pathlib import Path +from typing import Any, Optional + +import numpy as np +import pytest + +from recordstream import FlowGraph, Image, Label, Mask, Pipeline, Record, Stream, Transform +from recordstream.flow import parse_flow +from recordstream.ops.structure import RenameField, SelectFields + + +class _AddOffset(Transform): + """Adds a configurable offset to every Image payload (bind target).""" + + handles = (Image,) + + def __init__(self, offset: float = 0.0, field: Optional[str] = None) -> None: + super().__init__(field=field) + self.offset = offset + + def __call__(self, record: Record) -> Record: + out = dict(record) + for key, item in record.items(): + if isinstance(item, Image) and (self.field is None or key == self.field): + out[key] = Image(np.asarray(item) + self.offset, layout=item.layout) + return out + + +class _MakeMask(Transform): + """Derives a Mask entry from the first Image (a branch producer).""" + + def __call__(self, record: Record) -> Record: + image = next(item for item in record.values() if isinstance(item, Image)) + return {**record, "mask": Mask(np.asarray(image)[..., 0] > 0.5)} + + +def _seed(value: float = 0.0) -> Record: + return {"image": Image(np.full((2, 3, 3), value, dtype=np.float32)), "label": Label("x")} + + +class TestFlowGraph: + def test_linear_flow(self) -> None: + graph = FlowGraph(source=[_seed(1.0)], flow={"plus": _AddOffset(offset=2.0)}) + (out,) = list(graph) + assert isinstance(out, dict) and np.allclose(np.asarray(out["image"]), 3.0) + + def test_merge_from_union(self) -> None: + # Fork: derive a mask on a branch, SELECT the new entry, union it back into the main + # stream. (Selecting is the idiom — a full branch record would also carry its own + # 'image', and last-wins would overwrite the boosted one.) + flow = { + "start": {}, + "masked": {"op": _MakeMask(), "from": "start"}, + "mask_only": {"op": SelectFields(keys=["mask"]), "from": "masked"}, + "boosted": {"op": _AddOffset(offset=1.0), "from": "start"}, + "out": {"from": "boosted", "merge_from": ["mask_only"]}, + } + graph = FlowGraph(source=[_seed(0.75)], flow=flow, outputs="out") + (out,) = list(graph) + assert np.allclose(np.asarray(out["image"]), 1.75) # the boosted branch's image survives + assert "mask" in out and isinstance(out["mask"], Mask) # the selected branch entry + assert out["label"].value == "x" + + def test_merge_collision_last_wins(self) -> None: + # Both branches carry 'image'; the merge source is listed LAST -> its image wins + # (dict-union semantics, listed order). + flow = { + "start": {}, + "a": {"op": _AddOffset(offset=1.0), "from": "start"}, + "b": {"op": _AddOffset(offset=5.0), "from": "start"}, + "out": {"from": "a", "merge_from": ["b"]}, + } + (out,) = list(FlowGraph(source=[_seed(0.0)], flow=flow, outputs="out")) + assert np.allclose(np.asarray(out["image"]), 5.0) # b (last) wins over a + + def test_rename_avoids_collision(self) -> None: + flow = { + "start": {}, + "a": {"op": _AddOffset(offset=1.0), "from": "start"}, + "b_renamed": {"op": RenameField(src="image", dst="image_b"), "from": "start"}, + "out": {"from": "a", "merge_from": ["b_renamed"]}, + } + (out,) = list(FlowGraph(source=[_seed(0.0)], flow=flow, outputs="out")) + assert np.allclose(np.asarray(out["image"]), 1.0) # branch a intact + assert "image_b" in out # branch b united under its renamed key + + def test_step_key_bind(self) -> None: + # step[key] binds the NAMED ENTRY of the bound step's record result. + class _OffsetFromItem(Transform): + def __init__(self, item: Any = None) -> None: + super().__init__() + self.item = item + + def __call__(self, record: Record) -> Record: + offset = float(np.asarray(self.item).mean()) + out = dict(record) + for key, value in record.items(): + if isinstance(value, Image): + out[key] = Image(np.asarray(value) + offset, layout=value.layout) + return out + + flow = { + "start": {}, + "probe": {"op": _AddOffset(offset=2.0), "from": "start"}, # image becomes 2.0 + "final": {"op": _OffsetFromItem(), "from": "start", "bind": {"item": "probe[image]"}}, + } + (out,) = list(FlowGraph(source=[_seed(0.0)], flow=flow, outputs="final")) + assert np.allclose(np.asarray(out["image"]), 2.0) # 0.0 + mean(2.0) + + def test_bare_step_bind_is_whole_record(self) -> None: + class _CaptureWhole(Transform): + def __init__(self, item: Any = None) -> None: + super().__init__() + self.item = item + + def __call__(self, record: Record) -> Record: + # bare "probe" bind = the step's WHOLE result record dict. + assert isinstance(self.item, dict) and isinstance(self.item["image"], Image) + return record + + flow = { + "start": {}, + "probe": {"op": _AddOffset(offset=1.0), "from": "start"}, + "final": {"op": _CaptureWhole(), "from": "start", "bind": {"item": "probe"}}, + } + (out,) = list(FlowGraph(source=[_seed(0.0)], flow=flow, outputs="final")) + assert isinstance(out, dict) + + def test_legacy_fanin_key_removed(self) -> None: + # target_from / metadata_from (the legacy role fan-in) were purged; they are now + # unknown step keys — a flow document using one fails loudly at parse. + flow = { + "start": {}, + "a": {"op": _AddOffset(offset=1.0), "from": "start"}, + "out": {"from": "a", "target_from": "start"}, + } + graph = FlowGraph(source=[_seed(0.0)], flow=flow, outputs="out") + with pytest.raises(ValueError, match="unknown step key"): + list(graph) + + def test_metadata_from_key_removed(self) -> None: + with pytest.raises(ValueError, match="unknown step key"): + parse_flow({"a": {}, "b": {"from": "a", "merge_from": ["a"], "metadata_from": "a"}}) + + def test_merge_from_forward_ref_raises(self) -> None: + with pytest.raises(ValueError, match="EARLIER step"): + parse_flow({"a": {"merge_from": ["b"]}, "b": {}}) + + +class TestRecordsThroughStream: + def test_stream_carries_record_dicts_verbatim(self) -> None: + stream = Stream(source=[_seed(1.0)], ops=[_AddOffset(offset=1.0)]) + (out,) = list(stream) + assert isinstance(out, dict) and np.allclose(np.asarray(out["image"]), 2.0) + + def test_getitem(self) -> None: + stream = Stream(source=[_seed(1.0), _seed(2.0)], ops=[RenameField(src="label", dst="klass")]) + out = stream[1] + assert "klass" in out and np.allclose(np.asarray(out["image"]), 2.0) + + def test_compose_ops_route_records(self) -> None: + # Pipeline (the compose-group grouping op — TransformChain's replacement). + stream = Stream( + source=[_seed(1.0)], ops=[Pipeline(transforms=[_AddOffset(offset=1.0), _AddOffset(offset=2.0)])] + ) + (out,) = list(stream) + assert np.allclose(np.asarray(out["image"]), 4.0) + + +# --------------------------------------------------------------------------- # +# YAML flow documents end-to-end (docs/graph.md's exact spellings) — closes the +# gap where bind: was only ever exercised through Python-built flow dicts. +# --------------------------------------------------------------------------- # +class TestFlowYaml: + def _doc(self) -> str: + return """ +flow: + spec: {} + masked: !class:recordstream.ops.numpy.Threshold {low_level: 0.5, from: spec} + thresh: !class:recordstream.ops.formula.FormulaOp {formula: "amax(a) * 0.6", field: image, from: spec} + gated: + op: !class:recordstream.ops.numpy.Threshold {output: gated_mask} + from: spec + bind: + low_level: thresh[image] + out: {from: gated, merge_from: [masked]} +outputs: out +""" + + def _record(self) -> Record: + return {"image": Image(np.arange(16, dtype=np.float32).reshape(4, 4) / 15.0)} + + def test_yaml_bind_via_plain_mapping_step(self, tmp_path: Path) -> None: + # Scalar reserved keys ride in the marker mapping; the nested bind: mapping MUST use + # the plain-mapping (op:) step form — a nested mapping under a !class: marker is + # consumed by confluid as addressed configuration and never reaches parse_flow. + path = tmp_path / "graph.yaml" + path.write_text(self._doc()) + out = list(FlowGraph.from_yaml(str(path), source=[self._record()]))[0] + assert set(out) == {"image", "mask", "gated_mask"} + assert int(np.asarray(out["mask"]).sum()) == 8 # fixed 0.5 threshold + assert int(np.asarray(out["gated_mask"]).sum()) == 6 # per-record amax(a)*0.6 bind + + def test_yaml_bind_runs_the_same_from_either_loader(self, tmp_path: Path) -> None: + # FlowGraph.from_yaml and a hand-parsed flow are the same graph on the same engine. + path = tmp_path / "graph.yaml" + path.write_text(self._doc()) + record = self._record() + + from_yaml = list(FlowGraph.from_yaml(str(path), source=[dict(record)])) + import confluid + + doc = confluid.load(str(path), until="settled") + parsed = list(FlowGraph(source=[dict(record)], flow=doc["flow"], outputs=str(doc.get("outputs", "")))) + assert len(from_yaml) == len(parsed) == 1 + assert set(from_yaml[0]) == set(parsed[0]) + + def test_nested_bind_under_marker_is_consumed_not_parsed(self, tmp_path: Path) -> None: + # Pin the confluid behavior that makes the op:-form MANDATORY for bind — if this + # ever starts surviving in marker kwargs, the doc rule can be relaxed. + path = tmp_path / "graph.yaml" + path.write_text( + """ +flow: + spec: {} + gated: !class:recordstream.ops.numpy.Threshold + from: spec + bind: + low_level: spec[image] +""" + ) + import confluid + + marker = confluid.load(str(path), until="settled")["flow"]["gated"] + assert "bind" not in marker.kwargs # consumed as addressed configuration + + +class TestNativeExecution: + """The graph engine runs on its OWN executor — it never lowers to run (2026-07-29).""" + + def test_reader_accounting_is_computed_once_per_graph(self, monkeypatch: Any) -> None: + # _result_readers depends only on (steps, outputs); recomputing it per record was an + # O(steps^2) tax measured at ~half the graph engine's overhead over a flat op list. + # Patch the module that USES the name, not the one that defines it: `flow.graph` + # imports `_result_readers` from `flow.execute`, so it holds its own binding and + # patching `flow.execute` (or the `recordstream.flow` package) would miss. + import recordstream.flow.graph as graph_mod + + calls = {"n": 0} + real = graph_mod._result_readers + + def counting(steps: Any, outputs: str) -> Any: + calls["n"] += 1 + return real(steps, outputs) + + monkeypatch.setattr(graph_mod, "_result_readers", counting) + graph = FlowGraph(source=[_seed(1.0) for _ in range(25)], flow={"plus": _AddOffset(offset=2.0)}) + assert len(list(graph)) == 25 + assert calls["n"] == 1 + + def test_there_is_no_lowering_pass_left_to_call(self) -> None: + # The delegation this replaced built Stream(ops=to_ops(...)). Both converters are gone + # with the context ops they targeted; a reintroduced one would be a second executor. + import recordstream + import recordstream.flow as flow_mod + + for gone in ("to_ops", "from_ops", "flow_yaml_to_stream"): + assert not hasattr(flow_mod, gone), f"{gone} is back — the lowering pass has returned" + assert not hasattr(recordstream, gone) + assert not hasattr(Stream, "from_flow_yaml") + + def test_parallel_runs_the_graph_natively(self) -> None: + source = [_seed(float(i)) for i in range(6)] + serial = list(FlowGraph(source=source, flow={"plus": _AddOffset(offset=2.0)})) + parallel = list(FlowGraph(source=source, flow={"plus": _AddOffset(offset=2.0)}).parallel(2)) + + assert len(parallel) == len(serial) == 6 + for got, want in zip(parallel, serial): + assert np.allclose(np.asarray(got["image"]), np.asarray(want["image"])) + + def test_parallel_preserves_source_order(self) -> None: + source = [_seed(float(i)) for i in range(8)] + out = list(FlowGraph(source=source, flow={"plus": _AddOffset(offset=1.0)}).parallel(3)) + assert [float(np.asarray(r["image"]).flat[0]) for r in out] == [float(i) + 1.0 for i in range(8)] + + +class _SplitChannels(Transform): + """A 1→N EXPANDING step: one record per channel of the image.""" + + EXPANDS = True + + def __call__(self, record: Record) -> Any: # type: ignore[override] + image = record["image"] + return [{**record, "image": Image(np.asarray(image)[..., c : c + 1]), "channel": c} for c in range(3)] + + +class _Tag(Transform): + """Marks the record so a post-expansion step is observable.""" + + def __init__(self, tag: str = "") -> None: + super().__init__() + self.tag = tag + + def __call__(self, record: Record) -> Record: + return {**record, "tag": self.tag} + + +class TestExpandingSteps: + """A 1→N step forks the REMAINING subgraph, one branch per child (2026-07-29).""" + + def test_expansion_yields_one_record_per_child(self) -> None: + graph = FlowGraph(source=[_seed(1.0)], flow={"split": _SplitChannels()}) + out = list(graph) + assert [r["channel"] for r in out] == [0, 1, 2] + + def test_downstream_steps_run_once_per_child(self) -> None: + graph = FlowGraph(source=[_seed(1.0)], flow={"split": _SplitChannels(), "tagged": _Tag(tag="t")}) + out = list(graph) + assert [r["channel"] for r in out] == [0, 1, 2] + assert all(r["tag"] == "t" for r in out) + + def test_depth_first_sibling_order_across_chained_expansions(self) -> None: + # Nested-loop order (the flat engine's documented contract): the INNER expansion + # varies fastest. Two 3-way splits => 9 branches, the second split's channel cycling + # 0,1,2 within each child of the first. + graph = FlowGraph(source=[_seed(1.0)], flow={"a": _SplitChannels(), "b": _SplitChannels()}) + out = list(graph) + assert len(out) == 9 + assert [r["channel"] for r in out] == [0, 1, 2] * 3 + + def test_branches_do_not_share_env_state(self) -> None: + # Each child gets its OWN shallow copy of the step environment (the graph twin of + # Context.copy()): a later fan-in must not see a sibling's result. + flow = { + "src": {}, + "split": _SplitChannels(), + "out": {"from": "split", "merge_from": ["src"]}, + } + out = list(FlowGraph(source=[_seed(1.0)], flow=flow, outputs="out")) + assert [r["channel"] for r in out] == [0, 1, 2] + + def test_len_and_getitem_raise_for_an_expanding_graph(self) -> None: + graph = FlowGraph(source=[_seed(1.0)], flow={"split": _SplitChannels()}) + with pytest.raises(TypeError, match="EXPANDING"): + len(graph) + with pytest.raises(TypeError, match="EXPANDING"): + graph[0] + + def test_empty_expansion_drops_the_branch(self) -> None: + class _Drop(Transform): + EXPANDS = True + + def __call__(self, record: Record) -> Any: # type: ignore[override] + return [] + + assert list(FlowGraph(source=[_seed(1.0)], flow={"gone": _Drop()})) == [] + + def test_expansion_survives_the_spawn_boundary(self) -> None: + # The worker returns a LIST precisely so one seed can yield several records. + source = [_seed(1.0), _seed(2.0)] + out = list(FlowGraph(source=source, flow={"split": _SplitChannels()}).parallel(2)) + assert len(out) == 6 + assert [r["channel"] for r in out] == [0, 1, 2, 0, 1, 2] + + +class TestOneExecutor: + """`ops:` is the LINEAR SPELLING of a graph — both forms run the same kernel (2026-07-29).""" + + def test_an_ops_list_compiles_to_a_linear_step_graph(self) -> None: + from recordstream.core import linear_steps + + ops = [_AddOffset(offset=1.0), _AddOffset(offset=2.0)] + steps, outputs = linear_steps(ops) + assert [s.name for s in steps] == ["s0", "s1"] + assert outputs == "s1" + # Positional names, so the SAME op twice is two steps (a name-keyed mapping would collapse them). + assert all(s.from_ is None and not s.bind and not s.merge_from for s in steps) + + def test_a_repeated_op_stays_two_distinct_steps(self) -> None: + from recordstream.core import linear_steps + + op = _AddOffset(offset=1.0) + steps, _ = linear_steps([op, op]) + assert len(steps) == 2 + (out,) = list(Stream(source=[_seed(0.0)], ops=[op, op])) + assert np.allclose(np.asarray(out["image"]), 2.0) # applied twice, not once + + def test_stream_and_flowgraph_agree_on_the_same_linear_chain(self) -> None: + ops = [_AddOffset(offset=1.0), _MakeMask()] + flat = list(Stream(source=[_seed(0.75)], ops=ops)) + graph = list(FlowGraph(source=[_seed(0.75)], flow={f"s{i}": op for i, op in enumerate(ops)})) + assert len(flat) == len(graph) == 1 + assert np.allclose(np.asarray(flat[0]["image"]), np.asarray(graph[0]["image"])) + assert np.array_equal(np.asarray(flat[0]["mask"]), np.asarray(graph[0]["mask"])) + + def test_an_empty_ops_list_is_the_identity(self) -> None: + # The kernel treats "no steps" as the identity graph; a bare Stream must still yield. + source = [_seed(1.0), _seed(2.0)] + assert len(list(Stream(source=source, ops=[]))) == 2 + assert len(list(Stream(source=source))) == 2 + + def test_a_linear_chain_takes_the_env_free_path(self) -> None: + from recordstream.core import linear_steps + from recordstream.flow import is_linear + + steps, outputs = linear_steps([_AddOffset(offset=1.0), _AddOffset(offset=2.0)]) + assert is_linear(steps, outputs) + # A fan-out graph must NOT qualify — it needs the step environment. + branchy, out = parse_flow({"start": {}, "a": {"op": _AddOffset(offset=1.0), "from": "start"}}, "a") + assert not is_linear(branchy, out) + + def test_stream_expansion_still_yields_every_child(self) -> None: + # The flat engine's 1→N contract, now served by the shared kernel's linear path. + out = list(Stream(source=[_seed(1.0)], ops=[_SplitChannels(), _Tag(tag="t")])) + assert [r["channel"] for r in out] == [0, 1, 2] + assert all(r["tag"] == "t" for r in out) diff --git a/tests/test_typed_generic_ops.py b/tests/test_typed_generic_ops.py new file mode 100644 index 0000000..f49bb67 --- /dev/null +++ b/tests/test_typed_generic_ops.py @@ -0,0 +1,256 @@ +"""The generic array→Image→Mask→Boxes ops over dict records. + +Pins the three native type-changing transforms that run the detection/segmentation +front-end on plain record dicts: + +* :class:`recordstream.ops.image.ConvertToImage` — array-bearing key → ``Image`` item; +* :class:`recordstream.ops.numpy.Threshold` — array key → boolean ``Mask`` item; +* :class:`recordstream.ops.numpy.ConnectedComponents` — ``Mask`` → ``Boxes`` item. + +Each op REUSES its shared math helper, so the op output is pinned identical to the helper +(parity). recordstream-only — no domain-package import. +""" + +import numpy as np +import pytest +from confluid.registry import get_registry, resolve_class + +from recordstream import Boxes, Image, Mask +from recordstream.ops.image import ConvertToImage, _bound_longest_side, _render_rgb +from recordstream.ops.numpy import ConnectedComponents, Threshold, connected_component_boxes, threshold_array + + +def _ramp_2d() -> np.ndarray: + return np.arange(8 * 10).reshape(8, 10).astype(np.float32) + + +def _blob_mask() -> np.ndarray: + m = np.zeros((6, 6), dtype=bool) + m[0:2, 0:2] = True # blob A (area 4) -> xyxy (0, 0, 2, 2) + m[4:6, 4:6] = True # blob B (area 4) -> xyxy (4, 4, 6, 6) + return m + + +# --------------------------------------------------------------------------- # +# ConvertToImage +# --------------------------------------------------------------------------- # +class TestConvertToImage: + def test_produces_image_item_shape_dtype(self) -> None: + out = ConvertToImage(colormap="gray")({"spec": Mask(_ramp_2d())}) + assert "image" in out + img = out["image"] + assert isinstance(img, Image) + assert np.asarray(img).shape == (8, 10, 3) + assert np.asarray(img).dtype == np.uint8 + assert img.layout == "HWC" + # source entry untouched + assert isinstance(out["spec"], Mask) + + def test_parity_with_render_helper_default_sizing(self) -> None: + arr = _ramp_2d() + out = ConvertToImage(colormap="viridis")({"spec": Mask(arr)}) + expected = _bound_longest_side(_render_rgb(arr, "viridis"), 512) + assert np.array_equal(expected, np.asarray(out["image"])) + + def test_exact_resize_and_flip(self) -> None: + arr = _ramp_2d() + out = ConvertToImage(colormap="gray", width=20, height=16, flip_vertical=True)({"spec": Mask(arr)}) + assert np.asarray(out["image"]).shape == (16, 20, 3) + + def test_explicit_field_and_custom_output(self) -> None: + rec = {"a": Mask(_ramp_2d()), "b": Mask(np.zeros((4, 4), dtype=np.float32))} + out = ConvertToImage(field="b", output="preview")(rec) + assert np.asarray(out["preview"]).shape == (4, 4, 3) + + def test_does_not_publish_image_dims_keys(self) -> None: + # The Image SHAPE carries the pixel dims; no image_width_px / image_height_px entries. + out = ConvertToImage()({"spec": Mask(_ramp_2d())}) + assert set(out.keys()) == {"spec", "image"} + assert np.asarray(out["image"]).shape[:2] == (8, 10) + + def test_missing_explicit_field_raises(self) -> None: + with pytest.raises(ValueError, match="field 'nope' not in record"): + ConvertToImage(field="nope")({"spec": Mask(_ramp_2d())}) + + def test_no_array_field_raises(self) -> None: + with pytest.raises(ValueError, match="no array-bearing field"): + ConvertToImage()({"lbl": Boxes(boxes=[[0, 0, 1, 1]])}) + + +# --------------------------------------------------------------------------- # +# Threshold +# --------------------------------------------------------------------------- # +class TestThreshold: + def test_produces_mask_parity(self) -> None: + arr = _ramp_2d() + out = Threshold(low_level=20.0)({"spec": Mask(arr)}) + assert isinstance(out["mask"], Mask) + assert np.asarray(out["mask"]).dtype == np.bool_ + expected = threshold_array(arr, low_level=20.0) + assert np.array_equal(np.asarray(out["mask"]), expected) + + def test_string_literal_bound(self) -> None: + arr = _ramp_2d() + out = Threshold(low_level="20")({"spec": Mask(arr)}) + expected = threshold_array(arr, low_level="20") + assert np.array_equal(np.asarray(out["mask"]), expected) + + def test_env_var_expression_bound(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("TEST_THRESH_LEVEL", "20") + arr = _ramp_2d() + out = Threshold(low_level="$TEST_THRESH_LEVEL")({"spec": Mask(arr)}) + assert np.array_equal(np.asarray(out["mask"]), arr > 20.0) + + def test_meta_key_expression_has_no_source(self) -> None: + # {key} expressions have no metadata home in the record model -> loud KeyError. + with pytest.raises(KeyError): + Threshold(low_level="{some_key}")({"spec": Mask(_ramp_2d())}) + + def test_band_pass_both_bounds_and_ops(self) -> None: + arr = _ramp_2d() + out = Threshold(low_level=20.0, high_level=60.0, low_op=">=", high_op="<=")({"spec": Mask(arr)}) + expected = threshold_array(arr, low_level=20.0, high_level=60.0, low_op=">=", high_op="<=") + assert np.array_equal(np.asarray(out["mask"]), expected) + assert np.array_equal(np.asarray(out["mask"]), (arr >= 20.0) & (arr <= 60.0)) + + def test_no_bound_raises(self) -> None: + with pytest.raises(ValueError, match="at least one"): + Threshold()({"spec": Mask(_ramp_2d())}) + + def test_default_field_picks_first_array(self) -> None: + # No explicit field: first array-bearing item (insertion order). + rec = {"raw": Mask(_ramp_2d()), "other": Boxes(boxes=[])} + out = Threshold(low_level=20.0)(rec) + assert np.array_equal(np.asarray(out["mask"]), _ramp_2d() > 20.0) + + def test_missing_explicit_field_raises(self) -> None: + with pytest.raises(ValueError, match="field 'nope' not in record"): + Threshold(low_level=1.0, field="nope")({"spec": Mask(_ramp_2d())}) + + def test_non_array_field_raises(self) -> None: + with pytest.raises(TypeError, match="expected an array"): + Threshold(low_level=1.0, field="reg")({"reg": Boxes(boxes=[])}) + + def test_no_array_field_default_raises(self) -> None: + with pytest.raises(ValueError, match="no array-bearing field"): + Threshold(low_level=1.0)({"reg": Boxes(boxes=[])}) + + +# --------------------------------------------------------------------------- # +# ConnectedComponents +# --------------------------------------------------------------------------- # +class TestConnectedComponents: + def test_produces_boxes_half_open_xyxy_contract(self) -> None: + mask = _blob_mask() + out = ConnectedComponents()({"m": Mask(mask)}) + boxes = out["boxes"] + assert isinstance(boxes, Boxes) + # The pinned pixel contract: HALF-OPEN xyxy (x0, y0, x1, y1) — x = col, y = row. + assert boxes.boxes == [(0, 0, 2, 2), (4, 4, 6, 6)] + # mask[y0:y1, x0:x1] covers each component exactly (the half-open property). + for x0, y0, x1, y1 in boxes.boxes: + assert mask[y0:y1, x0:x1].all() + + def test_canvas_is_the_mask_shape_even_for_an_empty_mask(self) -> None: + # canvas is filled in for an EMPTY box set too — a frame check that skips exactly + # the records with nothing to check reports a clean bill for the wrong reason. + empty = np.zeros((6, 6), dtype=bool) + out = ConnectedComponents()({"m": Mask(empty)}) + assert out["boxes"].boxes == [] + assert out["boxes"].canvas == (6, 6) + full = ConnectedComponents()({"m": Mask(_blob_mask())}) + assert full["boxes"].canvas == (6, 6) + + def test_parity_with_helper(self) -> None: + mask = _blob_mask() + out = ConnectedComponents()({"m": Mask(mask)}) + expected = connected_component_boxes(mask) + assert out["boxes"].boxes == expected + + def test_min_area_bins_filters_small_blobs(self) -> None: + m = np.zeros((6, 6), dtype=bool) + m[0:2, 0:2] = True # area 4 + m[5, 5] = True # area 1 -> dropped when min_area_bins=2 + out = ConnectedComponents(min_area_bins=2)({"m": Mask(m)}) + assert out["boxes"].boxes == [(0, 0, 2, 2)] + + def test_connectivity_parity(self) -> None: + # Diagonal touch: 4-connectivity keeps two blobs, 8 merges them. + m = np.zeros((4, 4), dtype=bool) + m[0, 0] = True + m[1, 1] = True + four = ConnectedComponents(connectivity=4)({"m": Mask(m)}) + eight = ConnectedComponents(connectivity=8)({"m": Mask(m)}) + assert len(four["boxes"].boxes) == 2 + assert len(eight["boxes"].boxes) == 1 + + def test_default_prefers_mask_over_other_array(self) -> None: + # An Image is inserted first, but a Mask is preferred by the default resolver. + rec = {"img": Image(np.zeros((6, 6, 3), dtype=np.uint8)), "seg": Mask(_blob_mask())} + out = ConnectedComponents()(rec) + assert out["boxes"].boxes == [(0, 0, 2, 2), (4, 4, 6, 6)] + + def test_falls_back_to_first_array_when_no_mask(self) -> None: + # No Mask item — a 2-D array item is used. + out = ConnectedComponents()({"m": Image(_blob_mask())}) + assert out["boxes"].boxes == [(0, 0, 2, 2), (4, 4, 6, 6)] + + def test_non_2d_mask_raises(self) -> None: + with pytest.raises(ValueError, match="2-D mask"): + ConnectedComponents()({"m": Mask(np.zeros((2, 2, 2), dtype=bool))}) + + def test_missing_explicit_field_raises(self) -> None: + with pytest.raises(ValueError, match="field 'nope' not in record"): + ConnectedComponents(field="nope")({"m": Mask(_blob_mask())}) + + def test_no_mask_or_array_raises(self) -> None: + with pytest.raises(ValueError, match="no Mask or array-bearing field"): + ConnectedComponents()({"reg": Boxes(boxes=[])}) + + +# --------------------------------------------------------------------------- # +# End-to-end chain: array -> Image -> Mask -> Boxes, all on one record dict. +# --------------------------------------------------------------------------- # +def test_array_to_image_to_mask_to_boxes_chain() -> None: + arr = _ramp_2d() + record = {"spec": Mask(arr)} + out = ConnectedComponents(field="mask")(Threshold(field="spec", low_level=20.0)(ConvertToImage()(record))) + # Every stage produced its typed entry. + assert isinstance(out["image"], Image) + assert isinstance(out["mask"], Mask) + assert isinstance(out["boxes"], Boxes) + # Boxes carries HALF-OPEN xyxy (x0, y0, x1, y1) tuples framed by the mask raster. + assert out["boxes"].boxes + assert out["boxes"].canvas == arr.shape + for box in out["boxes"].boxes: + assert len(box) == 4 + x0, y0, x1, y1 = box + assert x0 < x1 and y0 < y1 + # The image entry carries the pixel dims via its shape (no separate metadata). + assert np.asarray(out["image"]).shape[:2] == arr.shape + + +# --------------------------------------------------------------------------- # +# Discovery + zero-arg construction. +# --------------------------------------------------------------------------- # +def test_zero_arg_constructible() -> None: + assert ConvertToImage().output == "image" + assert Threshold().output == "mask" + assert ConnectedComponents().output == "boxes" + + +@pytest.mark.parametrize( + ("name", "cls", "group"), + [ + ("ConvertToImage", ConvertToImage, "image"), + ("Threshold", Threshold, "numpy"), + ("ConnectedComponents", ConnectedComponents, "numpy"), + ], +) +def test_discovery_tags(name: str, cls: type, group: str) -> None: + assert cls.__confluid_category__ == "op" # type: ignore[attr-defined] + assert cls.__confluid_group__ == group # type: ignore[attr-defined] + assert resolve_class(name) is cls + registry = get_registry() + assert name in registry.list_classes(category="op") + assert name in registry.list_classes(group=group) diff --git a/tests/test_typed_storage.py b/tests/test_typed_storage.py new file mode 100644 index 0000000..cf457a7 --- /dev/null +++ b/tests/test_typed_storage.py @@ -0,0 +1,305 @@ +"""Record key-group storage — HDF5/Zarr/Directory round-trips, format-tag guard, metadata queries.""" + +from dataclasses import dataclass +from pathlib import Path + +import h5py +import numpy as np +import pytest + +from recordstream import Boxes, Image, Label, register_item +from recordstream.storage.base import TYPED_FORMAT, require_record_format, restore_attrs, split_attrs +from recordstream.storage.directory import DirectorySink, DirectorySource +from recordstream.storage.hdf5 import HDF5Sink, HDF5Source, read_record_group +from recordstream.storage.query import MetadataFilterSource, record_metadata, scan_hdf5_metadata, scan_zarr_metadata +from recordstream.storage.zarr import ZarrBatchSink, ZarrBatchSource, ZarrGroupSink, ZarrGroupSource + + +@register_item +@dataclass +class _StoreSig: + """An externally-registered data-bearing item (the shape a domain signal item takes).""" + + data: object = None + samplerate: float = 1.0 + mask: object = None # an ARRAY-valued attr — exercises the attrs/ dataset path + + +def _records() -> list: + # Ragged across records: different box counts, one value with an array-valued attr, + # plus PLAIN entries (scalar / string / bare array) riding the "plain" type tag. + r0 = { + "image": Image(np.arange(12, dtype=np.float32).reshape(2, 2, 3), layout="CHW"), + "sig": _StoreSig(np.arange(8, dtype=np.float32), samplerate=20e6, mask=np.array([1, 0, 1], dtype=np.uint8)), + "regions": Boxes(boxes=[[0, 0, 1, 1], [1, 1, 2, 2]], labels=["a", "b"], canvas=(2, 2)), + "label": Label("drone", classes=["x", "drone"]), + "gain_db": -3.0, + "source_file": "a.iq", + "window": np.hanning(4), + } + r1 = { + "image": Image(np.ones((2, 2, 3), dtype=np.float32)), + "sig": _StoreSig(np.zeros(4, dtype=np.float32), samplerate=1e6, mask=np.array([0], dtype=np.uint8)), + "regions": Boxes(boxes=[[0, 0, 2, 2]], labels=["c"], canvas=(2, 2)), + "label": Label("x", classes=["x", "drone"]), + "gain_db": 1.5, + "source_file": "b.iq", + "window": np.hanning(4), + } + return [r0, r1] + + +def _assert_round_trip(back: list, expect: list) -> None: + assert len(back) == len(expect) + for got, want in zip(back, expect): + assert list(got.keys()) == list(want.keys()) # insertion order preserved + assert got["image"].layout == want["image"].layout + assert np.array_equal(np.asarray(got["image"]), np.asarray(want["image"])) + assert got["sig"].samplerate == want["sig"].samplerate + assert np.array_equal(np.asarray(got["sig"].data), np.asarray(want["sig"].data)) + assert np.array_equal(np.asarray(got["sig"].mask), np.asarray(want["sig"].mask)) # array attr + assert got["regions"].boxes == want["regions"].boxes # ragged boxes + assert got["regions"].canvas == want["regions"].canvas # tuple preserved + assert isinstance(got["regions"].canvas, tuple) + assert got["label"].value == want["label"].value and got["label"].classes == want["label"].classes + assert got["gain_db"] == want["gain_db"] # plain scalar round-trips + assert got["source_file"] == want["source_file"] # plain string round-trips + assert np.allclose(np.asarray(got["window"]), want["window"]) # plain bare-array round-trips + + +class TestAttrWireFormat: + def test_split_restore_round_trip(self) -> None: + attrs = { + "s": "text", + "i": 3, + "f": 1.5, + "b": True, + "none": None, + "tup": (2, 3), + "nested": {"a": [1, (2, 3)]}, + "arr": np.arange(4), + } + plain, arrays = split_attrs(attrs) + assert list(arrays) == ["arr"] + back = restore_attrs(plain, arrays) + assert back["s"] == "text" and back["i"] == 3 and back["f"] == 1.5 and back["b"] is True + assert back["none"] is None + assert back["tup"] == (2, 3) and isinstance(back["tup"], tuple) + assert back["nested"] == {"a": [1, (2, 3)]} + assert np.array_equal(back["arr"], np.arange(4)) + + def test_numpy_scalars_become_python(self) -> None: + plain, _ = split_attrs({"x": np.float32(2.5)}) + assert restore_attrs(plain, {})["x"] == 2.5 + + +class TestFormatGuard: + def test_require_record_format_accepts_current_tag(self) -> None: + require_record_format(TYPED_FORMAT, "test") # no raise + + @pytest.mark.parametrize("found", [None, "typedsample-v1", "bogus"]) + def test_require_record_format_rejects_everything_else(self, found: object) -> None: + with pytest.raises(ValueError, match="typedrecord-v1"): + require_record_format(found, "test") + + def test_hdf5_source_rejects_old_typedsample_tag(self, tmp_path: Path) -> None: + # NO backward compat: a pre-record-model store must fail loudly with the clear error. + path = tmp_path / "old.h5" + with h5py.File(path, "w") as handle: + handle.attrs["recordstream_format"] = "typedsample-v1" + with pytest.raises(ValueError, match="typedrecord-v1"): + HDF5Source(path=path).open() + + def test_hdf5_sink_rejects_appending_to_old_tag(self, tmp_path: Path) -> None: + path = tmp_path / "old.h5" + with h5py.File(path, "w") as handle: + handle.attrs["recordstream_format"] = "typedsample-v1" + sink = HDF5Sink(path=path) + with sink: + with pytest.raises(ValueError, match="typedrecord-v1"): + sink.write(_records()[0]) + + def test_zarr_group_source_rejects_old_tag(self, tmp_path: Path) -> None: + import zarr + + path = str(tmp_path / "old.zarr") + root = zarr.open_group(path, mode="a") + root.attrs["recordstream_format"] = "typedsample-v1" + with pytest.raises(ValueError, match="typedrecord-v1"): + ZarrGroupSource(path=path).open() + + +class TestHDF5: + def test_round_trip(self, tmp_path: Path) -> None: + path = tmp_path / "t.h5" + sink = HDF5Sink(path=path, overwrite=True) + with sink: + for r in _records(): + sink.write(r) + sink.flush() + source = HDF5Source(path=path) + with source: + assert len(source) == 2 + _assert_round_trip(list(source), _records()) + + def test_format_tag_stamped(self, tmp_path: Path) -> None: + path = tmp_path / "t.h5" + with HDF5Sink(path=path, overwrite=True) as sink: + sink.write(_records()[0]) + with h5py.File(path, "r") as handle: + assert handle.attrs["recordstream_format"] == TYPED_FORMAT + + def test_non_dict_write_raises(self, tmp_path: Path) -> None: + # The sink only accepts a record dict; a bare array is rejected loudly. + sink = HDF5Sink(path=tmp_path / "typed.h5", overwrite=True) + with sink: + sink.write(_records()[0]) + with pytest.raises(TypeError, match="expected a record dict"): + sink.write(np.zeros(3)) + + +class TestReadRecordGroupSlices: + """``read_record_group(group, slices=...)`` — the partial-payload read a windowing source uses.""" + + def _first_group(self, path: Path) -> "h5py.Group": + handle = h5py.File(path, "r") + return handle["s000000"] + + def _write_one(self, tmp_path: Path) -> Path: + path = tmp_path / "t.h5" + with HDF5Sink(path=path, overwrite=True) as sink: + sink.write(_records()[0]) + return path + + def test_sliced_read_equals_full_read_then_slice(self, tmp_path: Path) -> None: + group = self._first_group(self._write_one(tmp_path)) + full = read_record_group(group) + part = read_record_group(group, slices={"sig": slice(2, 6)}) + # The sliced field: identical item type + attrs, payload = the full payload's slice. + assert type(part["sig"]) is type(full["sig"]) + assert part["sig"].samplerate == full["sig"].samplerate + assert np.array_equal(np.asarray(part["sig"].data), np.asarray(full["sig"].data)[2:6]) + # Attrs (incl. array-valued ones) are never sliced. + assert np.array_equal(np.asarray(part["sig"].mask), np.asarray(full["sig"].mask)) + # Every other field is byte-identical to the full read. + assert np.array_equal(np.asarray(part["image"]), np.asarray(full["image"])) + assert part["gain_db"] == full["gain_db"] + + def test_plain_array_field_slices_too(self, tmp_path: Path) -> None: + group = self._first_group(self._write_one(tmp_path)) + part = read_record_group(group, slices={"window": slice(1, 3)}) + assert np.array_equal(part["window"], np.hanning(4)[1:3]) + + def test_slicing_a_payload_free_field_raises(self, tmp_path: Path) -> None: + group = self._first_group(self._write_one(tmp_path)) + with pytest.raises(KeyError, match="no 'data' dataset"): + read_record_group(group, slices={"gain_db": slice(0, 1)}) + + +class TestZarr: + def test_group_round_trip(self, tmp_path: Path) -> None: + path = str(tmp_path / "g.zarr") + sink = ZarrGroupSink(path=path) + sink.open() + for r in _records(): + sink.write(r) + source = ZarrGroupSource(path=path) + assert len(source) == 2 + _assert_round_trip(list(source), _records()) + + def test_group_non_dict_write_raises(self, tmp_path: Path) -> None: + path = str(tmp_path / "g.zarr") + sink = ZarrGroupSink(path=path) + sink.open() + sink.write(_records()[0]) + with pytest.raises(TypeError, match="expected a record dict"): + sink.write(np.zeros(3)) + + def test_batch_typed_rows(self, tmp_path: Path) -> None: + path = str(tmp_path / "b.zarr") + sink = ZarrBatchSink(path=path, shape=[2, 2, 3], dtype="float32", overwrite=True) + sink.open() + for r in _records(): + sink.write(r) # appends the FIRST record entry's payload + source = ZarrBatchSource(path=path) + rows = list(source) + assert len(rows) == 2 and all(isinstance(r, dict) for r in rows) + assert isinstance(rows[0]["image"], Image) and rows[0]["image"].layout == "CHW" # uniform template + assert np.asarray(rows[1]["image"]).shape == (2, 2, 3) + + +class TestDirectory: + def test_round_trip(self, tmp_path: Path) -> None: + path = tmp_path / "dir" + sink = DirectorySink(path=path) + sink.open() + for r in _records(): + sink.write(r) + source = DirectorySource(path=path) + assert len(source) == 2 + _assert_round_trip(list(source), _records()) + + def test_missing_root_raises(self, tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError): + len(DirectorySource(path=tmp_path / "nope")) + + def test_non_dict_write_raises(self, tmp_path: Path) -> None: + with pytest.raises(TypeError, match="expected a record dict"): + DirectorySink(path=tmp_path / "dir").write(np.zeros(3)) + + +class TestMetadataQueries: + def test_record_metadata_shape(self) -> None: + meta = record_metadata(_records()[0]) + assert meta["image"] == {"layout": "CHW"} + assert meta["sig"]["samplerate"] == 20e6 + assert meta["gain_db"] == {"value": -3.0} # plain scalar -> its "value" attr + assert meta["source_file"] == {"value": "a.iq"} + assert meta["window"] == {} # a plain ARRAY payload contributes no scalar metadata + + def test_hdf5_scan_is_nested_and_payload_free(self, tmp_path: Path) -> None: + path = tmp_path / "q.h5" + sink = HDF5Sink(path=path, overwrite=True) + with sink: + for r in _records(): + sink.write(r) + scans = list(scan_hdf5_metadata(path)) + assert len(scans) == 2 + _, meta = scans[0] + assert meta["sig"]["samplerate"] == 20e6 # plain attr decoded + assert meta["image"]["layout"] == "CHW" + assert "shape" in str(meta["sig"]["mask"]) # array attr is a STUB, not the array + + def test_zarr_scan_nested(self, tmp_path: Path) -> None: + path = str(tmp_path / "q.zarr") + sink = ZarrGroupSink(path=path) + sink.open() + for r in _records(): + sink.write(r) + scans = list(scan_zarr_metadata(path)) + assert scans[1][1]["sig"]["samplerate"] == 1e6 + + def test_where_key_attr_expression(self, tmp_path: Path) -> None: + path = tmp_path / "w.h5" + sink = HDF5Sink(path=path, overwrite=True) + with sink: + for r in _records(): + sink.write(r) + source = HDF5Source(path=path) + source.open() + fast = MetadataFilterSource(source=source, where="sig.samplerate > 1e7") + assert len(fast) == 1 + (match,) = list(fast) + assert isinstance(match, dict) and match["sig"].samplerate == 20e6 + + def test_full_iteration_fallback_on_records(self) -> None: + # A plain list source (no iter_metadata protocol) of record dicts still filters, + # via record_metadata — plain scalars addressable as .value. + filt = MetadataFilterSource(source=_records(), where="gain_db.value < 0") + assert len(filt) == 1 + (match,) = list(filt) + assert match["gain_db"] == -3.0 + + def test_missing_attr_is_non_match(self) -> None: + filt = MetadataFilterSource(source=_records(), where="sig.nonexistent > 0") + assert len(filt) == 0 diff --git a/tests/test_typed_target_ops.py b/tests/test_typed_target_ops.py new file mode 100644 index 0000000..d0c9736 --- /dev/null +++ b/tests/test_typed_target_ops.py @@ -0,0 +1,245 @@ +"""The tensorization + target-shaping ops over dict records. + +Pins the native transforms that build a classification pipeline's model INPUT array and its +encoded TARGET ``Label`` on plain record dicts: + +* :class:`recordstream.ops.torch.ToTensor` — array-bearing key → a LIVE CHW-float ``torch.Tensor`` + (a plain record value); +* :class:`recordstream.ops.target.EncodeTarget` / ``DecodeTarget`` — class-name ↔ class-id ``Label``. + +Each op REUSES its shared conversion helper, so the op output is pinned identical to the +helper (parity). recordstream-only — no domain-package import. +""" + +import numpy as np +import pytest +import torch +from confluid.registry import get_registry, resolve_class + +from recordstream import Image, Label, Mask, collate_records, item_data +from recordstream.ops.image import ConvertToImage +from recordstream.ops.target import DecodeTarget, EncodeTarget +from recordstream.ops.torch import ToTensor, to_tensor + +_MAP = {"cat": 0, "dog": 1, "fox": 2} +_INV = {0: "cat", 1: "dog", 2: "fox"} + + +def _hwc_uint8() -> np.ndarray: + return (np.arange(4 * 5 * 3).reshape(4, 5, 3) % 256).astype(np.uint8) + + +# --------------------------------------------------------------------------- # +# ToTensor +# --------------------------------------------------------------------------- # +class TestToTensor: + def test_produces_chw_float_tensor(self) -> None: + arr = _hwc_uint8() + out = ToTensor()({"image": Image(arr)}) + tensor = out["image"] + assert isinstance(tensor, torch.Tensor) + assert tuple(tensor.shape) == (3, 4, 5) # HWC -> CHW + assert tensor.dtype == torch.float32 + assert float(tensor.max()) <= 1.0 # normalized + + def test_parity_with_to_tensor_helper(self) -> None: + arr = _hwc_uint8() + out = ToTensor()({"image": Image(arr)}) + expected = to_tensor(arr).numpy() + assert np.array_equal(np.asarray(out["image"]), expected) + + def test_parity_no_normalize(self) -> None: + arr = _hwc_uint8() + out = ToTensor(normalize=False)({"image": Image(arr)}) + expected = to_tensor(arr, normalize=False).numpy() + assert np.array_equal(np.asarray(out["image"]), expected) + + # `mode=` must coerce ARRAY payloads too, not only PIL ones: a decoding source hands over + # already-arrayed pixels, and a mixed-mode dataset (cppe-5 ships RGB + RGBA + grayscale rows) + # then reaches the model with ragged channel counts — a channel-mismatch crash MID-EPOCH, + # after the 3-channel rows trained fine. Found the hard way; these pin the array path. + def test_mode_rgb_coerces_a_4_channel_uint8_array(self) -> None: + rgba = (np.arange(4 * 5 * 4).reshape(4, 5, 4) % 256).astype(np.uint8) + tensor = to_tensor(rgba, mode="RGB") + assert tuple(tensor.shape) == (3, 4, 5) + + def test_mode_rgb_coerces_a_2d_grayscale_uint8_array(self) -> None: + gray = (np.arange(4 * 5).reshape(4, 5) % 256).astype(np.uint8) + tensor = to_tensor(gray, mode="RGB") + assert tuple(tensor.shape) == (3, 4, 5) + + def test_mode_rgb_coerces_a_singleton_channel_uint8_array(self) -> None: + gray1 = (np.arange(4 * 5).reshape(4, 5, 1) % 256).astype(np.uint8) + tensor = to_tensor(gray1, mode="RGB") + assert tuple(tensor.shape) == (3, 4, 5) + + def test_mode_rgb_leaves_a_3_channel_array_byte_identical(self) -> None: + arr = _hwc_uint8() + assert np.array_equal(to_tensor(arr, mode="RGB").numpy(), to_tensor(arr).numpy()) + + def test_mode_is_left_alone_for_a_non_uint8_array(self) -> None: + # PIL cannot represent a float RGBA faithfully — the coercion is deliberately uint8-only. + rgba = np.random.rand(4, 5, 4).astype(np.float32) + tensor = to_tensor(rgba, mode="RGB", normalize=False) + assert tuple(tensor.shape) == (4, 4, 5) + + def test_op_mode_reaches_the_array_path(self) -> None: + rgba = (np.arange(4 * 5 * 4).reshape(4, 5, 4) % 256).astype(np.uint8) + out = ToTensor(mode="RGB")({"image": Image(rgba)}) + assert tuple(out["image"].shape) == (3, 4, 5) + + def test_output_is_a_plain_live_tensor(self) -> None: + # The record model holds arbitrary values: the tensor rides AS-IS (no Image wrap — an + # NDArrayItem coerces via np.asarray and cannot hold a live tensor). item_data passes + # a plain value through unchanged. + out = ToTensor()({"image": Image(_hwc_uint8())}) + assert isinstance(out["image"], torch.Tensor) + assert not isinstance(out["image"], Image) + assert item_data(out["image"]) is out["image"] + + def test_new_output_key_keeps_source(self) -> None: + arr = _hwc_uint8() + out = ToTensor(output="tensor")({"image": Image(arr)}) + assert np.asarray(out["tensor"]).shape == (3, 4, 5) + # original entry left as-is (HWC uint8) + assert np.asarray(out["image"]).shape == (4, 5, 3) + + def test_explicit_field(self) -> None: + rec = {"a": Mask(np.zeros((2, 2), dtype=np.uint8)), "b": Image(_hwc_uint8())} + out = ToTensor(field="b")(rec) + assert np.asarray(out["b"]).shape == (3, 4, 5) + + def test_default_picks_first_array_key(self) -> None: + rec = {"lbl": Label("cat"), "image": Image(_hwc_uint8())} + out = ToTensor()(rec) + assert np.asarray(out["image"]).shape == (3, 4, 5) + + def test_missing_explicit_field_raises(self) -> None: + with pytest.raises(ValueError, match="field 'nope' not in record"): + ToTensor(field="nope")({"image": Image(_hwc_uint8())}) + + def test_no_array_field_raises(self) -> None: + with pytest.raises(ValueError, match="no array-bearing field"): + ToTensor()({"lbl": Label("cat")}) + + def test_record_collate_stacks_payloads(self) -> None: + # The record collate stacks the CHW-float Image payloads into a batched array. + a = ToTensor()({"image": Image(_hwc_uint8())}) + b = ToTensor()({"image": Image(_hwc_uint8())}) + batch = collate_records([a, b]) + assert np.asarray(batch["image"]).shape == (2, 3, 4, 5) + + +# --------------------------------------------------------------------------- # +# EncodeTarget / DecodeTarget +# --------------------------------------------------------------------------- # +class TestEncodeDecodeTarget: + def test_encode_name_to_id(self) -> None: + out = EncodeTarget(mapping=_MAP)({"y": Label("cat")}) + assert isinstance(out["y"], Label) + assert out["y"].value == 0 + + def test_encode_maps_every_name(self) -> None: + for name in _MAP: + out = EncodeTarget(mapping=_MAP)({"y": Label(name)}) + assert out["y"].value == _MAP[name] + + def test_encode_preserves_classes_vocab(self) -> None: + out = EncodeTarget(mapping=_MAP)({"y": Label("dog", classes=list(_MAP))}) + assert out["y"].value == 1 + assert out["y"].classes == list(_MAP) + + def test_encode_new_output_key(self) -> None: + out = EncodeTarget(mapping=_MAP, output="target_id")({"y": Label("fox")}) + assert out["target_id"].value == 2 + assert out["y"].value == "fox" # source left intact + + def test_encode_ignore_unknown(self) -> None: + out = EncodeTarget(mapping=_MAP, ignore_unknown=True, default=-1)({"y": Label("bird")}) + assert out["y"].value == -1 + + def test_encode_unknown_raises(self) -> None: + with pytest.raises(KeyError): + EncodeTarget(mapping=_MAP)({"y": Label("bird")}) + + def test_encode_empty_mapping_raises_lazily(self) -> None: + op = EncodeTarget() # constructible with no mapping (lazy) + with pytest.raises(ValueError, match="at least one entry"): + op({"y": Label("cat")}) + + def test_decode_id_to_name(self) -> None: + for cid in _INV: + out = DecodeTarget(mapping=_INV)({"y": Label(cid)}) + assert out["y"].value == _INV[cid] + + def test_encode_then_decode_round_trip(self) -> None: + encoded = EncodeTarget(mapping=_MAP)({"y": Label("dog")}) + assert encoded["y"].value == 1 + decoded = DecodeTarget(mapping=_INV)(encoded) + assert decoded["y"].value == "dog" + + def test_decode_empty_mapping_raises_lazily(self) -> None: + with pytest.raises(ValueError, match="at least one entry"): + DecodeTarget()({"y": Label(0)}) + + def test_encode_non_label_field_raises(self) -> None: + with pytest.raises(TypeError, match="expected a Label"): + EncodeTarget(mapping=_MAP, field="image")({"image": Image(_hwc_uint8())}) + + def test_encode_no_label_field_raises(self) -> None: + with pytest.raises(ValueError, match="no Label/MultiLabel field"): + EncodeTarget(mapping=_MAP)({"image": Image(_hwc_uint8())}) + + +# --------------------------------------------------------------------------- # +# End-to-end classification input/target path (recordstream-only). +# --------------------------------------------------------------------------- # +def test_classification_input_and_target_chain() -> None: + # Source-shaped record: an HWC image + a class-NAME label — key names carry meaning. + record = {"image": Image(_hwc_uint8()), "class": Label("cat", classes=list(_MAP))} + # Build the model INPUT (CHW float) and the encoded TARGET id. + out = EncodeTarget(mapping=_MAP, field="class")(ToTensor(field="image")(record)) + + # Input entry: a LIVE CHW-float tensor (a plain record value). + assert isinstance(out["image"], torch.Tensor) + assert tuple(out["image"].shape) == (3, 4, 5) + assert out["image"].dtype == torch.float32 + + # Target entry: an int-id Label carrying the vocabulary. + assert isinstance(out["class"], Label) + assert out["class"].value == 0 + assert out["class"].classes == list(_MAP) + + +def test_convert_then_tensor_chain() -> None: + # A raw 2-D array entry runs ConvertToImage -> ToTensor into a CHW-float input. + arr = np.arange(6 * 4).reshape(6, 4).astype(np.float32) + out = ToTensor(field="image")(ConvertToImage(colormap="gray")({"spec": Mask(arr)})) + assert isinstance(out["image"], torch.Tensor) + assert tuple(out["image"].shape) == (3, 6, 4) + + +# --------------------------------------------------------------------------- # +# Discovery + zero-arg construction. +# --------------------------------------------------------------------------- # +def test_zero_arg_constructible() -> None: + assert ToTensor().output == "" + assert EncodeTarget().mapping == {} + assert DecodeTarget().mapping == {} + + +@pytest.mark.parametrize( + ("name", "cls", "group"), + [ + ("ToTensor", ToTensor, "torch"), + ("EncodeTarget", EncodeTarget, "structure"), + ("DecodeTarget", DecodeTarget, "structure"), + ], +) +def test_discovery_tags(name: str, cls: type, group: str) -> None: + assert cls.__confluid_category__ == "op" # type: ignore[attr-defined] + assert cls.__confluid_group__ == group # type: ignore[attr-defined] + assert resolve_class(name) is cls + registry = get_registry() + assert name in registry.list_classes(category="op") + assert name in registry.list_classes(group=group) diff --git a/tests/test_typespec.py b/tests/test_typespec.py deleted file mode 100644 index fddbc1a..0000000 --- a/tests/test_typespec.py +++ /dev/null @@ -1,508 +0,0 @@ -"""Exhaustive tests for the dataflux type-spec system (matching, inference, JSON, HF bridge).""" - -from typing import Any, List, Tuple, cast - -import numpy as np -import pytest - -from dataflux.core import Flux -from dataflux.sample import FEATURES_KEY, SPEC_KEY, Sample -from dataflux.typespec import ( - AnyType, - ArrayType, - Dim, - ListType, - MappingType, - PythonType, - SampleType, - TypeSpec, - UnionType, - accepts, - canonical_dtype, - compatible, - infer_sample_type, - infer_type, - type_from_dict, - typed, -) - - -class _Widget: - """Module-level class so its ``__qualname__`` is the bare name (used by infer_type fallback test).""" - - -# -------------------------------------------------------------------------------------------------- -# Dim -# -------------------------------------------------------------------------------------------------- - - -def test_dim_accepts_strict() -> None: - assert Dim.range(1, 10).accepts(Dim.exact(5)) - assert Dim.range(1, 10).accepts(Dim.range(2, 9)) - assert not Dim.range(1, 10).accepts(Dim.exact(50)) - assert not Dim.range(1, 10).accepts(Dim.exact(0)) - assert Dim.any().accepts(Dim.exact(999)) - assert Dim.any().accepts(Dim.any()) - # unbounded producer against bounded consumer -> strict reject - assert not Dim.range(1, 10).accepts(Dim.any()) - # half-open consumer bounds - assert Dim.range(2, None).accepts(Dim.exact(100)) - assert not Dim.range(2, None).accepts(Dim.exact(1)) - - -def test_dim_compatible_permissive() -> None: - # unbounded producer against bounded consumer -> soft pass - assert Dim.range(1, 10).compatible(Dim.any()) - assert Dim.range(1, 10).compatible(Dim.exact(5)) - # provably disjoint -> reject even when permissive - assert not Dim.range(1, 10).compatible(Dim.exact(50)) - assert not Dim.range(20, 30).compatible(Dim.range(1, 5)) - - -def test_dim_json_roundtrip() -> None: - d = Dim.range(1, 10, "axis") - assert Dim.from_dict(d.to_dict()) == d - - -# -------------------------------------------------------------------------------------------------- -# AnyType -# -------------------------------------------------------------------------------------------------- - - -def test_any_type_matches_everything() -> None: - for producer in (PythonType("dict"), ArrayType(ndim=3), UnionType((AnyType(),))): - assert accepts(AnyType(), producer) - # concrete consumer rejects an Any producer (strict) but accepts it permissively - assert not accepts(ArrayType(ndim=2), AnyType()) - assert compatible(ArrayType(ndim=2), AnyType()) - - -# -------------------------------------------------------------------------------------------------- -# ArrayType -# -------------------------------------------------------------------------------------------------- - - -def test_array_ndim() -> None: - assert accepts(ArrayType(ndim=2), ArrayType(ndim=2)) - assert not accepts(ArrayType(ndim=2), ArrayType(ndim=3)) - assert accepts(ArrayType(ndim=None), ArrayType(ndim=5)) # rank-agnostic consumer - # unknown producer rank: strict reject, permissive pass - assert not accepts(ArrayType(ndim=2), ArrayType(ndim=None)) - assert compatible(ArrayType(ndim=2), ArrayType(ndim=None)) - - -def test_array_shape() -> None: - consumer = ArrayType(shape=(Dim.range(1, 10), Dim.any())) - assert accepts(consumer, ArrayType(shape=(Dim.exact(5), Dim.exact(7)))) - assert not accepts(consumer, ArrayType(shape=(Dim.exact(50), Dim.exact(7)))) - # producer without shape: strict reject, permissive pass - assert not accepts(consumer, ArrayType(ndim=2)) - assert compatible(consumer, ArrayType(ndim=2)) - - -def test_array_dtype_families() -> None: - assert accepts(ArrayType(dtype="floating"), ArrayType(dtype="float32")) - assert accepts(ArrayType(dtype="numeric"), ArrayType(dtype="int64")) - assert not accepts(ArrayType(dtype="floating"), ArrayType(dtype="int64")) - assert accepts(ArrayType(dtype="float32"), ArrayType(dtype="float32")) - assert not accepts(ArrayType(dtype="float32"), ArrayType(dtype="float64")) - # family-accepts-subfamily: numeric accepts the whole floating family - assert accepts(ArrayType(dtype="numeric"), ArrayType(dtype="floating")) - assert not accepts(ArrayType(dtype="floating"), ArrayType(dtype="numeric")) - # producer dtype unknown - assert not accepts(ArrayType(dtype="float32"), ArrayType()) - assert compatible(ArrayType(dtype="float32"), ArrayType()) - - -def test_array_frameworks() -> None: - assert accepts(ArrayType(frameworks={"numpy", "torch"}), ArrayType(frameworks={"torch"})) - assert not accepts(ArrayType(frameworks={"torch"}), ArrayType(frameworks={"numpy"})) - # permissive: overlap is enough - assert compatible(ArrayType(frameworks={"torch"}), ArrayType(frameworks={"torch", "numpy"})) - assert not compatible(ArrayType(frameworks={"torch"}), ArrayType(frameworks={"numpy"})) - # producer framework unknown - assert not accepts(ArrayType(frameworks={"torch"}), ArrayType()) - assert compatible(ArrayType(frameworks={"torch"}), ArrayType()) - - -def test_array_post_init() -> None: - a = ArrayType(shape=(Dim.exact(3), Dim.any())) - assert a.ndim == 2 # derived from shape - assert isinstance(ArrayType(frameworks={"numpy"}).frameworks, frozenset) # set coerced to frozenset - assert ArrayType(dtype="FLOAT32").dtype == "float32" # normalized - with pytest.raises(ValueError): - ArrayType(ndim=3, shape=(Dim.any(),)) - - -def test_array_image_constructor() -> None: - chw = ArrayType.image("CHW", channels=3, dtype="float32", framework="torch") - assert chw.ndim == 3 and chw.semantic == "image" - assert chw.shape is not None and chw.shape[0] == Dim.exact(3, "C") - hwc = ArrayType.image("HWC", channels=(1, 4)) - assert hwc.shape is not None and hwc.shape[2] == Dim(1, 4, "C") - with pytest.raises(ValueError): - ArrayType.image("XYZ") - - -def test_array_parse() -> None: - a = ArrayType.parse("3 h w", dtype="float32", framework="torch") - assert a.shape == (Dim.exact(3), Dim.any("h"), Dim.any("w")) - assert a.dtype == "float32" and a.frameworks == frozenset({"torch"}) - ranged = ArrayType.parse("1-10 N") - assert ranged.shape == (Dim.range(1, 10), Dim.any("N")) - assert ArrayType.parse("2-").shape == (Dim.range(2, None),) - with pytest.raises(ValueError): - ArrayType.parse("*batch c h w") - - -# -------------------------------------------------------------------------------------------------- -# PythonType / Union / Mapping / List -# -------------------------------------------------------------------------------------------------- - - -def test_python_type_exact() -> None: - assert accepts(PythonType("dict"), PythonType("dict")) - assert not accepts(PythonType("dict"), PythonType("list")) - # class mismatch - assert not accepts(PythonType("dict"), ArrayType(ndim=1)) - assert not accepts(ArrayType(ndim=1), PythonType("dict")) - - -def test_union_variance() -> None: - consumer = UnionType((ArrayType(dtype="numeric"), PythonType("PIL.Image.Image"))) - assert accepts(consumer, ArrayType(dtype="int64")) - assert accepts(consumer, PythonType("PIL.Image.Image")) - assert not accepts(consumer, PythonType("str")) - # producer union: every branch must be accepted - prod = UnionType((ArrayType(dtype="int64"), ArrayType(dtype="float32"))) - assert accepts(ArrayType(dtype="numeric"), prod) - assert not accepts(ArrayType(dtype="floating"), prod) # int64 branch fails - - -def test_mapping_and_list() -> None: - consumer = MappingType.of({"a": ArrayType(ndim=1), "b": PythonType("str")}) - assert accepts(consumer, MappingType.of({"a": ArrayType(ndim=1), "b": PythonType("str"), "c": AnyType()})) - assert not accepts(consumer, MappingType.of({"a": ArrayType(ndim=1)})) # missing 'b' - assert not accepts(consumer, MappingType.of({"a": ArrayType(ndim=2), "b": PythonType("str")})) - assert accepts(ListType(ArrayType(dtype="floating")), ListType(ArrayType(dtype="float32"))) - assert not accepts(ListType(ArrayType(dtype="floating")), ListType(ArrayType(dtype="int64"))) - assert not accepts(MappingType.of({"a": AnyType()}), ListType(AnyType())) # class mismatch - - -# -------------------------------------------------------------------------------------------------- -# SampleType -# -------------------------------------------------------------------------------------------------- - - -def test_sample_type_pair() -> None: - consumer = SampleType(input=ArrayType(ndim=2), target=ArrayType(dtype="int64")) - assert consumer.accepts(SampleType(input=ArrayType(ndim=2), target=ArrayType(dtype="int64"))) - assert not consumer.accepts(SampleType(input=ArrayType(ndim=3), target=ArrayType(dtype="int64"))) - # default target is Any -> input-only constraints accept any target - input_only = SampleType(input=ArrayType(ndim=2)) - assert input_only.accepts(SampleType(input=ArrayType(ndim=2), target=PythonType("dict"))) - assert SampleType() == SampleType() # both AnyType defaults compare equal - - -# -------------------------------------------------------------------------------------------------- -# dtype canonicalization -# -------------------------------------------------------------------------------------------------- - - -def test_canonical_dtype() -> None: - assert canonical_dtype("Float32") == "float32" - assert canonical_dtype("double") == "float64" - assert canonical_dtype("floating") == "floating" # family passes through - assert canonical_dtype(np.dtype("int64")) == "int64" - assert canonical_dtype(np.float32) == "float32" - import torch - - assert canonical_dtype(torch.float32) == "float32" - assert canonical_dtype(torch.int64) == "int64" - - -# -------------------------------------------------------------------------------------------------- -# inference -# -------------------------------------------------------------------------------------------------- - - -def test_infer_type_numpy() -> None: - spec = infer_type(np.zeros((3, 8, 8), dtype=np.float32)) - assert isinstance(spec, ArrayType) - assert spec.ndim == 3 and spec.dtype == "float32" and spec.frameworks == frozenset({"numpy"}) - assert spec.shape == (Dim.exact(3), Dim.exact(8), Dim.exact(8)) - scalar = infer_type(np.int64(7)) - assert isinstance(scalar, ArrayType) and scalar.ndim == 0 and scalar.dtype == "int64" - - -def test_infer_type_torch() -> None: - import torch - - spec = infer_type(torch.zeros(2, 4, dtype=torch.float64)) - assert isinstance(spec, ArrayType) and spec.frameworks == frozenset({"torch"}) - assert spec.ndim == 2 and spec.dtype == "float64" - - -def test_infer_type_pil() -> None: - Image = pytest.importorskip("PIL.Image") - assert infer_type(Image.new("RGB", (4, 4))) == PythonType("PIL.Image.Image") - - -def test_infer_type_python_values() -> None: - assert infer_type(None) == AnyType() - assert infer_type(True) == PythonType("bool") - assert infer_type(3) == PythonType("int") - assert infer_type(2.5) == PythonType("float") - assert infer_type("x") == PythonType("str") - assert infer_type({"a": 1}) == PythonType("dict") - assert infer_type([1, 2]) == PythonType("list") - assert infer_type((1, 2)) == PythonType("tuple") - assert infer_type(_Widget()) == PythonType("_Widget") - - -def test_infer_sample_type() -> None: - st = infer_sample_type(Sample(input=np.zeros((2, 2), dtype=np.float32), target=np.int64(1))) - assert isinstance(st.input, ArrayType) and st.input.ndim == 2 - assert isinstance(st.target, ArrayType) and st.target.ndim == 0 - # unset target -> Any - assert infer_sample_type(Sample(input=np.zeros(2))).target == AnyType() - - -# -------------------------------------------------------------------------------------------------- -# JSON round-trip -# -------------------------------------------------------------------------------------------------- - - -def test_json_roundtrip_all_kinds() -> None: - specs: List[TypeSpec] = [ - AnyType(), - ArrayType.image("CHW", 3, "float32", "torch"), - ArrayType(shape=(Dim.range(1, 10), Dim.any("N")), dtype="numeric", frameworks={"numpy"}), - PythonType("PIL.Image.Image"), - UnionType((ArrayType(ndim=1), PythonType("str"), AnyType())), - MappingType.of({"boxes": ArrayType(ndim=2), "labels": ListType(ArrayType(ndim=0, dtype="int64"))}), - ] - for spec in specs: - assert type_from_dict(spec.to_dict()) == spec - st = SampleType(input=specs[1], target=specs[5]) - assert SampleType.from_dict(st.to_dict()) == st - - -def test_type_from_dict_unknown_kind() -> None: - with pytest.raises(ValueError): - type_from_dict({"kind": "bogus"}) - - -# -------------------------------------------------------------------------------------------------- -# datasets.Features bridge -# -------------------------------------------------------------------------------------------------- - - -def test_hf_features_classification_target() -> None: - spec = SampleType( - input=PythonType("PIL.Image.Image"), - target=ArrayType(ndim=0, dtype="int64", frameworks={"torch"}), - ) - features, extras = spec.to_hf_features() - assert set(features.keys()) == {"input", "target"} - rt = SampleType.from_hf_features(features, extras) - assert rt.accepts(spec) and spec.accepts(rt) - - -def test_hf_features_detection_target() -> None: - spec = SampleType( - target=MappingType.of( - { - "boxes": ArrayType(ndim=2, shape=(Dim.any("N"), Dim.exact(4)), dtype="float32", frameworks={"torch"}), - "labels": ArrayType(ndim=1, shape=(Dim.any("N"),), dtype="int64", frameworks={"torch"}), - } - ) - ) - features, extras = spec.to_hf_features() - rt = SampleType.from_hf_features(features, extras) - assert rt.accepts(spec) and spec.accepts(rt) - - -def test_hf_features_segmentation_target() -> None: - spec = SampleType( - input=ArrayType.image("CHW", 3, "float32", "torch"), - target=ArrayType(ndim=2, shape=(Dim.any("H"), Dim.any("W")), dtype="int64", frameworks={"torch"}), - ) - features, extras = spec.to_hf_features() - rt = SampleType.from_hf_features(features, extras) - assert rt.accepts(spec) and spec.accepts(rt) - - -def test_hf_features_non_expressible_falls_back_to_typespec() -> None: - # Any / Union / family-dtype / unknown-rank can't be a concrete Feature -> stored as a typespec blob. - spec = SampleType( - input=UnionType((ArrayType(dtype="numeric"), PythonType("PIL.Image.Image"))), - target=ArrayType(dtype="floating"), # family dtype, no rank - ) - features, extras = spec.to_hf_features() - assert "input" not in features and "target" not in features - assert "typespec" in extras["input"] and "typespec" in extras["target"] - rt = SampleType.from_hf_features(features, extras) - assert rt == spec - - -def test_hf_features_from_dict_form() -> None: - spec = SampleType(input=ArrayType(ndim=2, shape=(Dim.any(), Dim.exact(4)), dtype="float32")) - features, extras = spec.to_hf_features() - # describe() passes the Features.to_dict() form (a plain dict), not a Features instance - rt = SampleType.from_hf_features(features.to_dict(), extras) - assert rt.accepts(spec) and spec.accepts(rt) - - -def test_hf_features_pythontype_and_string_roundtrip() -> None: - # PIL <-> Image, str <-> Value("string"), and a non-Feature python type falls back to a typespec blob. - spec = SampleType(input=PythonType("str"), target=PythonType("dict")) - features, extras = spec.to_hf_features() - assert "input" in features and "target" not in features # str -> Value, dict -> typespec fallback - assert SampleType.from_hf_features(features, extras) == spec - - -def test_hf_features_high_rank_and_nested_fallback() -> None: - # ndim>5 has no ArrayXD feature, and a mapping/list with a non-expressible member falls back whole. - high_rank = SampleType(input=ArrayType(ndim=6, dtype="float32")) - f, e = high_rank.to_hf_features() - assert "input" not in f and SampleType.from_hf_features(f, e) == high_rank - - nested = SampleType(target=MappingType.of({"a": ArrayType(ndim=2, dtype="float32"), "b": AnyType()})) - f, e = nested.to_hf_features() - assert "target" not in f and SampleType.from_hf_features(f, e) == nested - - listed = SampleType(target=ListType(AnyType())) - f, e = listed.to_hf_features() - assert SampleType.from_hf_features(f, e) == listed - - -def test_canonical_dtype_fallback() -> None: - # an object that is neither a str nor a known dtype object falls through to str(x).lower() - assert canonical_dtype(123) == "123" - - -# -------------------------------------------------------------------------------------------------- -# @typed decorator -# -------------------------------------------------------------------------------------------------- - - -def test_typed_decorator() -> None: - acc = SampleType(input=PythonType("PIL.Image.Image")) - prod = SampleType(input=ArrayType.image("CHW", 3, "float32", "torch")) - - @typed(accepts=acc, produces=prod) - class Op: - pass - - assert getattr(Op, "ACCEPTS") == acc and getattr(Op, "PRODUCES") == prod - - with pytest.raises(TypeError): - - @typed(accepts=ArrayType(ndim=1)) # type: ignore[arg-type] - class Bad: - pass - - with pytest.raises(TypeError): - - @typed(produces="nope") # type: ignore[arg-type] - class Bad2: - pass - - -# -------------------------------------------------------------------------------------------------- -# Sample.describe / with_type -# -------------------------------------------------------------------------------------------------- - - -def test_sample_describe_infers_when_unstored() -> None: - s = Sample(input=np.zeros((3, 8, 8), dtype=np.float32)) - st = s.describe() - assert isinstance(st.input, ArrayType) and st.input.ndim == 3 and st.input.frameworks == frozenset({"numpy"}) - - -def test_sample_with_type_and_describe_stored() -> None: - declared = SampleType(input=ArrayType.image("CHW", 3, "float32", "torch")) - s = Sample(input=np.zeros((3, 8, 8), dtype=np.float32), metadata={"id": 7}) - typed_s = s.with_type(declared) - assert FEATURES_KEY in typed_s.metadata and SPEC_KEY in typed_s.metadata - assert typed_s.metadata["id"] == 7 # pre-existing metadata preserved - assert s.metadata == {"id": 7} and FEATURES_KEY not in s.metadata # copy-on-write: original untouched - rt = typed_s.describe() - assert rt.accepts(declared) and declared.accepts(rt) - - -# -------------------------------------------------------------------------------------------------- -# Pipeline freshness (maintain-if-present) -# -------------------------------------------------------------------------------------------------- - - -class _ToFloat64Op: - PRODUCES = SampleType(input=ArrayType(ndim=1, dtype="float64", frameworks={"numpy"})) - - def __call__(self, sample: Sample) -> Sample: - return sample._replace(input=sample.input.astype("float64")) - - -class _UntypedOp: - def __call__(self, sample: Sample) -> Sample: - return sample._replace(input=sample.input + 1) - - -def _run(sample: Sample, op: Any) -> Sample: - return cast(Sample, list(Flux(source=[sample], ops=[op]))[0]) - - -def test_pipeline_does_not_stamp_untracked_samples() -> None: - out = _run(Sample(input=np.array([1, 2, 3])), _ToFloat64Op()) - assert FEATURES_KEY not in out.metadata and SPEC_KEY not in out.metadata - - -def test_pipeline_refreshes_stored_type_from_produces() -> None: - stamped = Sample(input=np.array([1, 2, 3])).with_type( - SampleType(input=ArrayType(ndim=1, dtype="int64", frameworks={"numpy"})) - ) - out = _run(stamped, _ToFloat64Op()) - assert FEATURES_KEY in out.metadata - assert out.describe().accepts(_ToFloat64Op.PRODUCES) - assert _ToFloat64Op.PRODUCES.accepts(out.describe()) - - -def test_pipeline_drops_stored_type_when_op_has_no_produces() -> None: - stamped = Sample(input=np.array([1, 2, 3])).with_type( - SampleType(input=ArrayType(ndim=1, dtype="int64", frameworks={"numpy"})) - ) - out = _run(stamped, _UntypedOp()) - assert FEATURES_KEY not in out.metadata and SPEC_KEY not in out.metadata - # describe() falls back to inference -> still correct, just not "stored" - assert isinstance(out.describe().input, ArrayType) - - -# -------------------------------------------------------------------------------------------------- -# Op annotation conformance: declared PRODUCES must accept the real inferred output, and declared -# ACCEPTS must accept the real input (proves the static specs match runtime reality). -# -------------------------------------------------------------------------------------------------- - - -def test_dataflux_op_spec_conformance() -> None: - import dataflux.ops.numpy as N - import dataflux.ops.torch as T - - rgb = (np.random.rand(3, 8, 8) * 255).astype(np.float32) - cases: List[Tuple[Any, Sample]] = [ - (N.StandardizeOp(mean=0.5, std=0.5), Sample(input=rgb.copy())), - (N.RescaleOp(in_min=0, in_max=255), Sample(input=rgb.copy())), - (N.ClipPercentilesOp(), Sample(input=rgb.copy())), - (N.ReplaceNonFiniteOp(), Sample(input=rgb.copy())), - (N.ThresholdOp(value=0.5), Sample(input=rgb.copy())), - (N.ConnectedComponentsOp(), Sample(input=(np.random.rand(8, 8) > 0.5))), - (T.RescaleOp(in_min=0, in_max=255), Sample(input=__import__("torch").rand(3, 8, 8) * 255)), - (T.StandardizeOp(mean=0.5, std=0.5), Sample(input=__import__("torch").rand(3, 8, 8))), - ] - for op, sample in cases: - name = type(op).__module__ + "." + type(op).__name__ - assert op.ACCEPTS.accepts(infer_sample_type(sample)), f"{name}: ACCEPTS rejects its real input" - out = op(sample) - assert op.PRODUCES.accepts(infer_sample_type(out)), f"{name}: PRODUCES rejects its real output" - # specs are JSON round-trippable - assert SampleType.from_dict(op.PRODUCES.to_dict()) == op.PRODUCES diff --git a/tests/test_view_sources_deferred.py b/tests/test_view_sources_deferred.py new file mode 100644 index 0000000..1b7249f --- /dev/null +++ b/tests/test_view_sources_deferred.py @@ -0,0 +1,105 @@ +"""A still-deferred marker in a view source's ``source:`` slot explains itself. + +A ``_partial_: true`` source leaves a Confluid marker in the slot — a CONFIG error, since a +source slot needs a live object and nothing here will flow it — and the view sources answer +it with the same actionable guidance ``Stream`` gives (naming the slot, the deferred target, +and the fix) instead of the cryptic ``got Partial`` their bare ``hasattr`` checks used to +produce. Deliberately message-only: the slot is never flowed (the raise-with-guidance +convention for ``source:`` slots — distinct from the free functions ``project`` / +``dataset_uri``). + +**The TRIGGER changed 2026-08-11.** It used to be the parens-less ``!class:X`` spelling, +which left a deferred stub; confluid merged its eager and deferred markers, so both +spellings build and that footgun is gone. Deferral is now asked for explicitly, which is +the only way left to reach this guard — and the message changed with it (it used to say +"add parens", advice that would now change nothing). +""" + +import pytest +from confluid import load + +from recordstream import Stream +from recordstream.sources import ConcatSource, DatasetSplit, RangeSource + +_LEAF = "recordstream.sources.concat.ConcatSource" + + +def _expect_guidance(excinfo: "pytest.ExceptionInfo[TypeError]", slot: str) -> None: + """The error names the slot, says WHAT was deferred, and states the parens fix.""" + message = str(excinfo.value) + assert slot in message + assert "deferred Confluid marker" in message + assert "ConcatSource" in message # the deferred target, not just "Class" + assert "_partial_: true" in message # the actionable fix + + +def test_a_partial_marker_under_range_source_raises_the_stream_guidance() -> None: + cfg = load( + f""" +range_src: + _target_: recordstream.sources.range.RangeSource + source: {{_target_: {_LEAF}, _partial_: true}} + stop: 3 +""", + ) + with pytest.raises(TypeError) as excinfo: + cfg["range_src"].indices + _expect_guidance(excinfo, "RangeSource.source") + + +def test_a_partial_marker_under_concat_source_names_the_offending_index() -> None: + cfg = load( + f""" +concat_src: + _target_: recordstream.sources.concat.ConcatSource + sources: + - {{_target_: {_LEAF}}} + - {{_target_: {_LEAF}, _partial_: true}} +""", + ) + with pytest.raises(TypeError) as excinfo: + cfg["concat_src"].offsets + _expect_guidance(excinfo, "ConcatSource.sources[1]") + + +def test_a_partial_marker_under_dataset_split_raises_the_stream_guidance() -> None: + cfg = load( + f""" +split_src: + _target_: recordstream.sources.split.DatasetSplit + source: {{_target_: {_LEAF}, _partial_: true}} +""", + ) + with pytest.raises(TypeError) as excinfo: + len(cfg["split_src"].train) + _expect_guidance(excinfo, "DatasetSplit.source") + + +def test_the_parens_spelling_materializes_and_the_view_sources_work() -> None: + """The positive counterpart: ``!class:X()`` becomes a live instance at load time.""" + cfg = load( + f""" +range_src: !class:recordstream.sources.range.RangeSource() + source: !class:{_LEAF}() + stop: 3 +concat_src: !class:recordstream.sources.concat.ConcatSource() + sources: + - !class:{_LEAF}() +split_src: !class:recordstream.sources.split.DatasetSplit() + source: !class:{_LEAF}() +""", + ) + range_src: RangeSource = cfg["range_src"] + concat_src: ConcatSource = cfg["concat_src"] + split_src: DatasetSplit = cfg["split_src"] + assert isinstance(range_src.source, ConcatSource) + assert range_src.indices == [] # an empty leaf clamps [0:3) to nothing — but it computed + assert concat_src.offsets == [0] + assert len(split_src.train) == 0 + + +def test_streams_own_guidance_still_names_its_slot() -> None: + """The shared message helper's default slot stays ``Stream.source``.""" + cfg = load(f"deferred: {{_target_: {_LEAF}, _partial_: true}}") + with pytest.raises(TypeError, match="Stream.source is still a deferred Confluid marker"): + len(Stream(source=cfg["deferred"])) diff --git a/tests/test_workflow.py b/tests/test_workflow.py new file mode 100644 index 0000000..c866532 --- /dev/null +++ b/tests/test_workflow.py @@ -0,0 +1,243 @@ +"""Tests for recordstream.workflow — higher-order runnable combinators. + +Covers Sequence / Conditional / Switch orchestration, the PathExists / Not / +AllOf / AnyOf predicates, zero-arg construction (the lazy-construction mandate), +the only-the-selected-branch-runs guarantee, error paths, and a Confluid +dump/load round-trip that proves a whole workflow serialises and runs (the +deferred branches are flowed inside run()). +""" + +from typing import Any, Iterator, List + +import confluid +import pytest +from confluid import configurable + +from recordstream.workflow import AllOf, AnyOf, Conditional, Not, PathExists, Sequence, Switch + +# A module-global run log so the @configurable runnables below survive a Confluid +# round-trip: their only config is a tag, and run() appends it here. +_RUN_LOG: List[str] = [] + + +@configurable +class _RunStep: + """A minimal runnable: ``run()`` records its tag in the module log.""" + + def __init__(self, tag: str = "") -> None: + self.tag = tag + + def run(self) -> None: + _RUN_LOG.append(self.tag) + + +@configurable +class _FixedPredicate: + """A predicate whose truth value is pinned in config (round-trippable).""" + + def __init__(self, value: bool = False) -> None: + self.value = value + + def __call__(self) -> bool: + return self.value + + +@pytest.fixture(autouse=True) +def _clear_log() -> Iterator[None]: + _RUN_LOG.clear() + yield + _RUN_LOG.clear() + + +# --------------------------------------------------------------------------- # +# Sequence +# --------------------------------------------------------------------------- # +def test_sequence_runs_steps_in_order() -> None: + Sequence([_RunStep("a"), _RunStep("b"), _RunStep("c")]).run() + assert _RUN_LOG == ["a", "b", "c"] + + +def test_sequence_empty_is_noop() -> None: + Sequence().run() # zero-arg + empty: must not raise + assert _RUN_LOG == [] + + +def test_sequence_skips_none_entries() -> None: + Sequence([_RunStep("a"), None, _RunStep("b")]).run() + assert _RUN_LOG == ["a", "b"] + + +def test_sequence_caches_flowed_step() -> None: + seq = Sequence([confluid.Target(_RunStep, tag="x")]) + seq.run() + # After run, the deferred marker has been replaced by the live, flowed object. + assert isinstance(seq.steps[0], _RunStep) + assert _RUN_LOG == ["x"] + + +# --------------------------------------------------------------------------- # +# Conditional +# --------------------------------------------------------------------------- # +def test_conditional_true_runs_if_true_only() -> None: + Conditional(condition=True, if_true=_RunStep("t"), if_false=_RunStep("f")).run() + assert _RUN_LOG == ["t"] + + +def test_conditional_false_runs_if_false_only() -> None: + Conditional(condition=False, if_true=_RunStep("t"), if_false=_RunStep("f")).run() + assert _RUN_LOG == ["f"] + + +def test_conditional_none_branch_is_noop() -> None: + Conditional(condition=True, if_true=None, if_false=_RunStep("f")).run() + assert _RUN_LOG == [] + + +def test_conditional_none_condition_is_falsy() -> None: + Conditional(if_true=_RunStep("t"), if_false=_RunStep("f")).run() + assert _RUN_LOG == ["f"] + + +def test_conditional_accepts_callable_condition() -> None: + Conditional(condition=lambda: True, if_true=_RunStep("t")).run() + assert _RUN_LOG == ["t"] + + +def test_conditional_accepts_predicate_condition() -> None: + Conditional(condition=_FixedPredicate(True), if_true=_RunStep("t"), if_false=_RunStep("f")).run() + assert _RUN_LOG == ["t"] + + +# --------------------------------------------------------------------------- # +# Switch +# --------------------------------------------------------------------------- # +def test_switch_selects_matching_case() -> None: + Switch(select=lambda: "b", cases={"a": _RunStep("a"), "b": _RunStep("b")}).run() + assert _RUN_LOG == ["b"] + + +def test_switch_falls_back_to_default_on_miss() -> None: + Switch(select=lambda: "z", cases={"a": _RunStep("a")}, default=_RunStep("d")).run() + assert _RUN_LOG == ["d"] + + +def test_switch_none_selector_uses_default() -> None: + Switch(default=_RunStep("d")).run() + assert _RUN_LOG == ["d"] + + +def test_switch_no_match_no_default_is_noop() -> None: + Switch(select=lambda: "z", cases={"a": _RunStep("a")}).run() + assert _RUN_LOG == [] + + +def test_switch_coerces_non_string_key() -> None: + Switch(select=lambda: 2, cases={"2": _RunStep("two")}).run() + assert _RUN_LOG == ["two"] + + +# --------------------------------------------------------------------------- # +# Predicates +# --------------------------------------------------------------------------- # +def test_path_exists(tmp_path: Any) -> None: + present = tmp_path / "model.ckpt" + present.write_text("x") + assert PathExists(str(present))() is True + assert PathExists(str(tmp_path / "missing.ckpt"))() is False + assert PathExists("")() is False # empty path never True (no bogus Path('.') hit) + + +def test_not_negates() -> None: + assert Not(True)() is False + assert Not(False)() is True + assert Not(None)() is True # None is falsy -> Not(None) is True + + +def test_allof_is_and_with_vacuous_truth() -> None: + assert AllOf([True, True])() is True + assert AllOf([True, False])() is False + assert AllOf()() is True # empty AND is True + + +def test_anyof_is_or() -> None: + assert AnyOf([False, True])() is True + assert AnyOf([False, False])() is False + assert AnyOf()() is False # empty OR is False + + +def test_predicate_combinators_compose() -> None: + # Not(AllOf(True, AnyOf(False, True))) == not (True and True) == False + assert Not(AllOf([True, AnyOf([False, True])]))() is False + + +# --------------------------------------------------------------------------- # +# Error paths + zero-arg construction +# --------------------------------------------------------------------------- # +def test_run_non_runnable_branch_raises() -> None: + with pytest.raises(TypeError, match="has no callable run"): + Conditional(condition=True, if_true=object()).run() + + +@pytest.mark.parametrize("cls", [Sequence, Conditional, Switch]) +def test_combinator_zero_arg_construct_and_run(cls: Any) -> None: + cls().run() # must construct AND run with no args (no-op), never raise + assert _RUN_LOG == [] + + +@pytest.mark.parametrize("cls", [PathExists, Not, AllOf, AnyOf]) +def test_predicate_zero_arg_construct_and_call(cls: Any) -> None: + assert cls()() in (True, False) # zero-arg construct + call yields a bool + + +# --------------------------------------------------------------------------- # +# StreamStudio canvas integration — TorchRunner + ProgressReporting forwarding +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize("cls", [Sequence, Conditional, Switch]) +def test_combinators_declare_needs_autograd(cls: Any) -> None: + # A combinator may wrap a trainer, so it declares __needs_autograd__ — StreamStudio's executor + # re-enables autograd for the whole run (otherwise the inner loss.backward() dies under + # ComfyUI's inference_mode). + assert cls().__needs_autograd__ is True + + +def test_progress_callback_forwarded_to_running_branch() -> None: + from recordstream.runnable import ProgressReporting + + received: List[str] = [] + + @configurable + class _ProgressStep(ProgressReporting): + def run(self) -> None: + self._report_progress(1, 1, "step") # fires through whatever callback was forwarded + + seq = Sequence([_ProgressStep()]) + seq.set_progress_callback(lambda v, t, d: received.append(d)) # executor injects on the combinator + seq.run() + assert received == ["step"] # forwarded to the branch, which reported through it + + +def test_progress_forward_is_safe_when_branch_has_no_callback() -> None: + seq = Sequence([_RunStep("a")]) + seq.set_progress_callback(lambda v, t, d: None) + seq.run() # _RunStep has no set_progress_callback -> _run must not raise + assert _RUN_LOG == ["a"] + + +# --------------------------------------------------------------------------- # +# Confluid round-trip — a whole workflow serialises and runs +# --------------------------------------------------------------------------- # +def test_confluid_roundtrip_runs_nested_workflow() -> None: + workflow = Sequence( + [ + _RunStep("download"), + Conditional( + condition=_FixedPredicate(False), # cache miss -> train branch + if_true=_RunStep("skip"), + if_false=_RunStep("train"), + ), + _RunStep("evaluate"), + ] + ) + restored: Any = confluid.load(confluid.dump(workflow)) + restored.run() + assert _RUN_LOG == ["download", "train", "evaluate"]