From e0f6e3de3bd59095cceaf651f0a80d2e53686873 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 29 Aug 2026 12:04:52 +0200 Subject: [PATCH 1/3] feat(zarr-metadata): add optional msgspec integration module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit zarr_metadata.msgspec is the msgspec sibling of zarr_metadata.pydantic: field types over the core metadata models plus a dec_hook (and a make_dec_hook composer for applications with hooks of their own) so the models can be used as field types in msgspec Structs and with msgspec.json.decode / msgspec.convert. msgspec consults its hooks only for types it does not support natively, and it supports dataclasses natively — its field-by-field coercion would bypass from_json — so each field type is a runtime marker class msgspec treats as a custom type, with the core model class registered as a virtual subclass; decoded instances ARE the core classes, and static type checkers see the field types as the core classes. Raw documents route through from_json, existing instances pass through unchanged, and MetadataValidationError surfaces as msgspec.ValidationError with the loc-annotated problem messages. Serialization cannot be delegated (msgspec's value-driven encoders never consult enc_hook for dataclasses), so the canonical path stays an explicit to_json(); the module documents this and a test pins the msgspec behavior. Assisted-by: ClaudeCode:claude-fable-5 --- .../zarr-metadata/changes/+msgspec.feature.md | 8 + packages/zarr-metadata/docs/api/index.md | 2 + packages/zarr-metadata/docs/api/msgspec.md | 5 + packages/zarr-metadata/docs/index.md | 12 ++ packages/zarr-metadata/mkdocs.yml | 1 + packages/zarr-metadata/pyproject.toml | 4 +- .../src/zarr_metadata/msgspec.py | 186 +++++++++++++++++ .../tests/model/test_msgspec_module.py | 193 ++++++++++++++++++ 8 files changed, 410 insertions(+), 1 deletion(-) create mode 100644 packages/zarr-metadata/changes/+msgspec.feature.md create mode 100644 packages/zarr-metadata/docs/api/msgspec.md create mode 100644 packages/zarr-metadata/src/zarr_metadata/msgspec.py create mode 100644 packages/zarr-metadata/tests/model/test_msgspec_module.py diff --git a/packages/zarr-metadata/changes/+msgspec.feature.md b/packages/zarr-metadata/changes/+msgspec.feature.md new file mode 100644 index 0000000000..8dc3b6688a --- /dev/null +++ b/packages/zarr-metadata/changes/+msgspec.feature.md @@ -0,0 +1,8 @@ +Added `zarr_metadata.msgspec`, an optional msgspec integration module: field +types over the core metadata models plus a `dec_hook` (and a `make_dec_hook` +composer for applications with hooks of their own) that route raw documents +through the models' strict `from_json` parser, so the models can be used as +field types in `msgspec.Struct` classes and with `msgspec.json.decode` / +`msgspec.convert`. Invalid documents surface as `msgspec.ValidationError` +with the loc-annotated problem messages. msgspec stays an optional +dependency of `zarr-metadata`; the module requires msgspec 0.19 or newer. diff --git a/packages/zarr-metadata/docs/api/index.md b/packages/zarr-metadata/docs/api/index.md index 5e230c7aa2..c6ab7169c9 100644 --- a/packages/zarr-metadata/docs/api/index.md +++ b/packages/zarr-metadata/docs/api/index.md @@ -10,6 +10,8 @@ The package is organized to mirror the structure of the Zarr specifications: structural validators, loc-aware parsers, and the `UNSET` sentinel - [`zarr_metadata.pydantic`](pydantic.md) — optional Pydantic field types over the models +- [`zarr_metadata.msgspec`](msgspec.md) — optional msgspec field types and + decode hook over the models - [`zarr_metadata.v2`](v2.md) — `TypedDict` shapes for Zarr v2 documents (`.zarray`, `.zgroup`, `.zattrs`, `.zmetadata`) - [`zarr_metadata.v3`](v3/index.md) — `TypedDict` shapes for Zarr v3 diff --git a/packages/zarr-metadata/docs/api/msgspec.md b/packages/zarr-metadata/docs/api/msgspec.md new file mode 100644 index 0000000000..ff6aae5783 --- /dev/null +++ b/packages/zarr-metadata/docs/api/msgspec.md @@ -0,0 +1,5 @@ +--- +title: msgspec +--- + +::: zarr_metadata.msgspec diff --git a/packages/zarr-metadata/docs/index.md b/packages/zarr-metadata/docs/index.md index 58e4f290c6..911ad22122 100644 --- a/packages/zarr-metadata/docs/index.md +++ b/packages/zarr-metadata/docs/index.md @@ -35,6 +35,9 @@ closely model the content of the Zarr specifications, such as: - **Optional Pydantic integration** ([`zarr_metadata.pydantic`](api/pydantic.md), requires Pydantic 2.13 or newer): each model as a Pydantic field type that validates raw documents through the same strict parser. +- **Optional msgspec integration** ([`zarr_metadata.msgspec`](api/msgspec.md), + requires msgspec 0.19 or newer): field types and a decode hook that route + raw documents through the same strict parser. ## What this is for @@ -67,6 +70,15 @@ A bare `TypeAdapter` over a public document `TypedDict` is a coercive shape adapter, not a Zarr conformance validator; it may coerce values or discard members that the strict model parser rejects. +The optional msgspec integration does the same through msgspec's decode hook: + +```python +import msgspec +import zarr_metadata.msgspec as zmm + +metadata = msgspec.convert(raw, zmm.ZarrV3ArrayMetadata, dec_hook=zmm.dec_hook) +``` + ## Validation boundary The model validators enforce the declared document structure and a small set diff --git a/packages/zarr-metadata/mkdocs.yml b/packages/zarr-metadata/mkdocs.yml index 18e1fc8c35..77a4fe6349 100644 --- a/packages/zarr-metadata/mkdocs.yml +++ b/packages/zarr-metadata/mkdocs.yml @@ -17,6 +17,7 @@ nav: - api/index.md - ' zarr_metadata.model': api/model.md - ' zarr_metadata.pydantic': api/pydantic.md + - ' zarr_metadata.msgspec': api/msgspec.md - ' zarr_metadata.v2': api/v2.md - ' zarr_metadata.v3': - api/v3/index.md diff --git a/packages/zarr-metadata/pyproject.toml b/packages/zarr-metadata/pyproject.toml index a58d3579a1..e4a93eac53 100644 --- a/packages/zarr-metadata/pyproject.toml +++ b/packages/zarr-metadata/pyproject.toml @@ -47,7 +47,9 @@ Changelog = "https://github.com/zarr-developers/zarr-python/blob/main/packages/z Documentation = "https://zarr-metadata.readthedocs.io/" [dependency-groups] -test = ["pytest", "pydantic>=2.13", "jsonschema"] +# The msgspec floor matches the requirement of the zarr package, the +# integration's motivating consumer. +test = ["pytest", "pydantic>=2.13", "jsonschema", "msgspec>=0.19"] docs = [ # Pins match the zarr-python docs environment in the repo-root # pyproject.toml so the two sites render with the same toolchain. diff --git a/packages/zarr-metadata/src/zarr_metadata/msgspec.py b/packages/zarr-metadata/src/zarr_metadata/msgspec.py new file mode 100644 index 0000000000..dc334af1e1 --- /dev/null +++ b/packages/zarr-metadata/src/zarr_metadata/msgspec.py @@ -0,0 +1,186 @@ +"""Optional msgspec integration: field types and a decode hook over the core models. + +Importing this module requires msgspec; the core package deliberately does +not depend on it, so this module is never imported by `zarr_metadata` itself. + +msgspec's extension point is a decode hook consulted only for annotated +types msgspec does not support natively — and the core models are +dataclasses, a kind msgspec supports natively, so annotating a field with a +core model class directly would engage msgspec's own field-by-field +dataclass coercion and bypass `from_json` (the single source of truth for +structural validation and normalization). Each name this module exports is +therefore a runtime marker class that msgspec treats as a custom type, +forcing every value for the field through this module's `dec_hook`. Each +marker registers its core model class as a virtual subclass, so the values +`dec_hook` produces satisfy msgspec's type check while the instances ARE +the core classes — they interoperate freely with non-msgspec code +(equality, isinstance, nesting). Static type checkers see each field type +as its core model class, so `manifest.metadata` below is a +`zarr_metadata.model.ZarrV3ArrayMetadata`. + +`dec_hook` routes a raw document through `from_json` and passes an existing +model instance through unchanged. `MetadataValidationError` subclasses +`ValueError`, so a failed parse surfaces as a +`msgspec.ValidationError` carrying the +loc-annotated problem messages, with msgspec appending the path of the +failing field (``- at `$.metadata` ``). + +Usage: + + import msgspec + + import zarr_metadata.msgspec as zmm + + class ArrayManifest(msgspec.Struct): + path: str + metadata: zmm.ZarrV3ArrayMetadata + + manifest = msgspec.json.decode(data, type=ArrayManifest, dec_hook=zmm.dec_hook) + +The same hook serves `msgspec.convert` — e.g. +`msgspec.convert(doc, zmm.ZarrV3ArrayMetadata, dec_hook=zmm.dec_hook)` — +and a prebuilt `msgspec.json.Decoder`. An application that already has a +decode hook of its own composes via `make_dec_hook(wrapped=...)`. + +Two msgspec limits shape what this module can offer: + +- Serialization cannot be delegated: msgspec's encoders are value-driven + and consult their `enc_hook` only for objects msgspec cannot encode + natively, which a dataclass never is, so no hook can route a model + instance through `to_json`. Encoding a model directly either raises (the + `UNSET` sentinel is unencodable) or silently emits a raw field dump that + is NOT the canonical document (no shorthand collapse, no omit-empty + conventions). Serialize explicitly: put `model.to_json()` — the + canonical document — wherever msgspec-encodable output is needed. +- Unions may contain at most one hook-handled type, so a field cannot be + typed as, say, array-or-group metadata. Decode such a field as an + untyped mapping and dispatch on its content. +""" + +from __future__ import annotations + +import abc +from typing import TYPE_CHECKING, Any, Final + +from zarr_metadata import model as _model + +if TYPE_CHECKING: + from collections.abc import Callable + + # For static type checkers the field types ARE the core model classes. + ZarrV3ArrayMetadata = _model.ZarrV3ArrayMetadata + ZarrV2ArrayMetadata = _model.ZarrV2ArrayMetadata + ZarrV3GroupMetadata = _model.ZarrV3GroupMetadata + ZarrV2GroupMetadata = _model.ZarrV2GroupMetadata + ZarrV3ConsolidatedMetadata = _model.ZarrV3ConsolidatedMetadata + ZarrV2ConsolidatedMetadata = _model.ZarrV2ConsolidatedMetadata + ZarrV3MetadataField = _model.ZarrV3NamedConfig +else: + # At runtime each field type is a marker: an empty ABC msgspec treats as + # a custom type (so `dec_hook` is consulted) with the core model class + # registered as a virtual subclass (so the core instances `dec_hook` + # returns satisfy msgspec's isinstance check on hook results). + + class ZarrV3ArrayMetadata(abc.ABC): # noqa: B024 + """Field type for a v3 array metadata document (`zarr.json` content).""" + + ZarrV3ArrayMetadata.register(_model.ZarrV3ArrayMetadata) + + class ZarrV2ArrayMetadata(abc.ABC): # noqa: B024 + """Field type for a v2 array metadata document (merged `.zarray` + `.zattrs` form).""" + + ZarrV2ArrayMetadata.register(_model.ZarrV2ArrayMetadata) + + class ZarrV3GroupMetadata(abc.ABC): # noqa: B024 + """Field type for a v3 group metadata document (`zarr.json` content).""" + + ZarrV3GroupMetadata.register(_model.ZarrV3GroupMetadata) + + class ZarrV2GroupMetadata(abc.ABC): # noqa: B024 + """Field type for a v2 group metadata document (merged `.zgroup` + `.zattrs` form).""" + + ZarrV2GroupMetadata.register(_model.ZarrV2GroupMetadata) + + class ZarrV3ConsolidatedMetadata(abc.ABC): # noqa: B024 + """Field type for v3 inline consolidated metadata.""" + + ZarrV3ConsolidatedMetadata.register(_model.ZarrV3ConsolidatedMetadata) + + class ZarrV2ConsolidatedMetadata(abc.ABC): # noqa: B024 + """Field type for a v2 `.zmetadata` document.""" + + ZarrV2ConsolidatedMetadata.register(_model.ZarrV2ConsolidatedMetadata) + + class ZarrV3MetadataField(abc.ABC): # noqa: B024 + """Field type for one normalized v3 metadata extension envelope.""" + + ZarrV3MetadataField.register(_model.ZarrV3NamedConfig) + + +_DECODERS: Final[dict[type, tuple[type, Callable[[object], object]]]] = { + ZarrV3ArrayMetadata: (_model.ZarrV3ArrayMetadata, _model.ZarrV3ArrayMetadata.from_json), + ZarrV2ArrayMetadata: (_model.ZarrV2ArrayMetadata, _model.ZarrV2ArrayMetadata.from_json), + ZarrV3GroupMetadata: (_model.ZarrV3GroupMetadata, _model.ZarrV3GroupMetadata.from_json), + ZarrV2GroupMetadata: (_model.ZarrV2GroupMetadata, _model.ZarrV2GroupMetadata.from_json), + ZarrV3ConsolidatedMetadata: ( + _model.ZarrV3ConsolidatedMetadata, + _model.ZarrV3ConsolidatedMetadata.from_json, + ), + ZarrV2ConsolidatedMetadata: ( + _model.ZarrV2ConsolidatedMetadata, + _model.ZarrV2ConsolidatedMetadata.from_json, + ), + ZarrV3MetadataField: (_model.ZarrV3NamedConfig, _model.ZarrV3NamedConfig.from_json), +} +"""Marker class -> (pass-through core class, document parser).""" + + +def dec_hook(type: type, obj: Any) -> Any: + """Decode `obj` for a field annotated with one of this module's field types. + + Pass as `dec_hook=` to `msgspec.json.decode`, `msgspec.convert`, or a + `msgspec.json.Decoder`. An existing core model instance passes through + unchanged; anything else is parsed by the core model's `from_json`. A + type this module does not cover raises `NotImplementedError`, msgspec's + convention for "still unsupported"; to keep decoding custom types of + your own alongside these, chain your hook with `make_dec_hook`. + """ + try: + core_cls, parse = _DECODERS[type] + except KeyError: + raise NotImplementedError(f"Objects of type {type} are not supported") from None + if isinstance(obj, core_cls): + return obj + return parse(obj) + + +def make_dec_hook(wrapped: Callable[[type, Any], Any] | None = None) -> Callable[[type, Any], Any]: + """Return a decode hook that also delegates unknown types to `wrapped`. + + The returned hook handles this module's field types exactly like + `dec_hook` and hands every other type to `wrapped`, so an application's + existing custom-type decoding keeps working alongside the model field + types. With no `wrapped` hook this returns `dec_hook` itself. + """ + if wrapped is None: + return dec_hook + + def hook(type: type, obj: Any) -> Any: + if type in _DECODERS: + return dec_hook(type, obj) + return wrapped(type, obj) + + return hook + + +__all__ = [ + "ZarrV2ArrayMetadata", + "ZarrV2ConsolidatedMetadata", + "ZarrV2GroupMetadata", + "ZarrV3ArrayMetadata", + "ZarrV3ConsolidatedMetadata", + "ZarrV3GroupMetadata", + "ZarrV3MetadataField", + "dec_hook", + "make_dec_hook", +] diff --git a/packages/zarr-metadata/tests/model/test_msgspec_module.py b/packages/zarr-metadata/tests/model/test_msgspec_module.py new file mode 100644 index 0000000000..5d55fd4e90 --- /dev/null +++ b/packages/zarr-metadata/tests/model/test_msgspec_module.py @@ -0,0 +1,193 @@ +"""Tests for `zarr_metadata.msgspec`, the optional msgspec integration module. + +Unlike pydantic, msgspec consults its extension hooks only for types it does +not support natively — and it supports dataclasses natively, so the module's +field types are runtime markers rather than the core model classes (see the +module docstring). Instances are still the CORE model classes (no parallel +hierarchy), so decoded values interoperate freely with non-msgspec code. +""" + +import msgspec +import msgspec.json +import pytest + +import zarr_metadata.msgspec as zmm +from zarr_metadata.model import ( + ZarrV2ArrayMetadata, + ZarrV2ConsolidatedMetadata, + ZarrV2GroupMetadata, + ZarrV3ArrayMetadata, + ZarrV3ConsolidatedMetadata, + ZarrV3GroupMetadata, + ZarrV3NamedConfig, +) + +V3_ARRAY_DOC = dict(ZarrV3ArrayMetadata.create_default(shape=(4,)).to_json()) +V2_ARRAY_DOC = dict(ZarrV2ArrayMetadata.create_default(shape=(4,), chunks=(2,)).to_json()) +V3_GROUP_DOC = {"zarr_format": 3, "node_type": "group", "attributes": {"a": 1}} +V2_GROUP_DOC = {"zarr_format": 2, "attributes": {"a": 1}} +V3_CONSOLIDATED_DOC = { + "kind": "inline", + "must_understand": False, + "metadata": {"a": dict(V3_ARRAY_DOC)}, +} +V2_CONSOLIDATED_DOC = { + "zarr_consolidated_format": 1, + "metadata": {".zgroup": {"zarr_format": 2}}, +} + +FIELD_CASES = [ + pytest.param(zmm.ZarrV3ArrayMetadata, ZarrV3ArrayMetadata, V3_ARRAY_DOC, id="array-v3"), + pytest.param(zmm.ZarrV2ArrayMetadata, ZarrV2ArrayMetadata, V2_ARRAY_DOC, id="array-v2"), + pytest.param(zmm.ZarrV3GroupMetadata, ZarrV3GroupMetadata, V3_GROUP_DOC, id="group-v3"), + pytest.param(zmm.ZarrV2GroupMetadata, ZarrV2GroupMetadata, V2_GROUP_DOC, id="group-v2"), + pytest.param( + zmm.ZarrV3ConsolidatedMetadata, + ZarrV3ConsolidatedMetadata, + V3_CONSOLIDATED_DOC, + id="consolidated-v3", + ), + pytest.param( + zmm.ZarrV2ConsolidatedMetadata, + ZarrV2ConsolidatedMetadata, + V2_CONSOLIDATED_DOC, + id="consolidated-v2", + ), + pytest.param(zmm.ZarrV3MetadataField, ZarrV3NamedConfig, {"name": "bytes"}, id="field-v3"), +] + + +@pytest.mark.parametrize(("field_type", "model_cls", "doc"), FIELD_CASES) +def test_field_type_decodes_and_passes_through( + field_type: object, model_cls: type, doc: dict[str, object] +) -> None: + """Each field type parses its raw document into the CORE model class via + from_json, passes existing instances through unchanged, and works for + `msgspec.convert` and `msgspec.json.decode` alike.""" + model = msgspec.convert(doc, field_type, dec_hook=zmm.dec_hook) + assert type(model) is model_cls + assert msgspec.convert(model, field_type, dec_hook=zmm.dec_hook) is model + decoded = msgspec.json.decode(msgspec.json.encode(doc), type=field_type, dec_hook=zmm.dec_hook) + assert decoded == model + + +def test_struct_field_round_trip() -> None: + """A Struct field decodes a raw document through from_json (the library's + normalization applies), and re-encoding the canonical document emitted by + to_json revalidates to an equal manifest — serialization routes through + to_json explicitly because msgspec's encoders cannot delegate dataclasses + (see the module docstring).""" + + class ArrayManifest(msgspec.Struct): + path: str + metadata: zmm.ZarrV3ArrayMetadata + + data = msgspec.json.encode({"path": "a/b", "metadata": V3_ARRAY_DOC}) + manifest = msgspec.json.decode(data, type=ArrayManifest, dec_hook=zmm.dec_hook) + assert isinstance(manifest.metadata, ZarrV3ArrayMetadata) + assert manifest.metadata.shape == (4,) + + out = msgspec.json.encode({"path": manifest.path, "metadata": manifest.metadata.to_json()}) + assert msgspec.json.decode(out, type=ArrayManifest, dec_hook=zmm.dec_hook) == manifest + + +def test_wrapped_hook_composes() -> None: + """make_dec_hook keeps an application's own custom-type decoding working + alongside the model field types; with no wrapped hook it is dec_hook.""" + + class Fraction: + def __init__(self, value: float) -> None: + self.value = value + + def consumer_hook(type: type, obj: object) -> object: + if type is Fraction: + return Fraction(obj) + raise NotImplementedError(type) + + class Manifest(msgspec.Struct): + scale: Fraction + metadata: zmm.ZarrV3GroupMetadata + + hook = zmm.make_dec_hook(consumer_hook) + manifest = msgspec.convert({"scale": 0.5, "metadata": V3_GROUP_DOC}, Manifest, dec_hook=hook) + assert isinstance(manifest.scale, Fraction) + assert manifest.scale.value == 0.5 + assert type(manifest.metadata) is ZarrV3GroupMetadata + assert zmm.make_dec_hook() is zmm.dec_hook + + +def test_decoded_models_share_no_mutable_state() -> None: + """Decoding the same document twice yields equal but independent models, + and mutating the source document does not reach into a decoded model.""" + doc = {"zarr_format": 3, "node_type": "group", "attributes": {"a": [1]}} + first = msgspec.convert(doc, zmm.ZarrV3GroupMetadata, dec_hook=zmm.dec_hook) + second = msgspec.convert(doc, zmm.ZarrV3GroupMetadata, dec_hook=zmm.dec_hook) + assert first == second + assert first is not second + doc["attributes"]["a"].append(2) + assert first.attributes == {"a": (1,)} + + +def test_invalid_document_surfaces_problems_in_validation_error() -> None: + """A defective document fails as msgspec.ValidationError carrying the + loc-annotated problem messages from MetadataValidationError, plus + msgspec's own path annotation for the failing field.""" + + class ArrayManifest(msgspec.Struct): + metadata: zmm.ZarrV3ArrayMetadata + + doc = dict(V3_ARRAY_DOC) + del doc["chunk_key_encoding"] + with pytest.raises( + msgspec.ValidationError, match=r"chunk_key_encoding: missing required key.*\$\.metadata" + ): + msgspec.convert({"metadata": doc}, ArrayManifest, dec_hook=zmm.dec_hook) + + +def test_dec_hook_rejects_uncovered_types() -> None: + """A type the module does not cover raises NotImplementedError, msgspec's + convention for a still-unsupported custom type.""" + with pytest.raises(NotImplementedError, match="int"): + zmm.dec_hook(int, 1) + + +def test_core_class_annotations_bypass_dec_hook() -> None: + """Why the field types are markers: msgspec supports dataclasses natively + and never consults dec_hook for them, so annotating with a core model + class would engage msgspec's field-by-field coercion instead of from_json + (today it fails to resolve the models' TYPE_CHECKING-only annotations). + If this test starts failing because decoding succeeded or the hook was + called, msgspec's dataclass handling changed — revisit the markers.""" + calls: list[type] = [] + + def hook(type: type, obj: object) -> object: + calls.append(type) + raise NotImplementedError(type) + + with pytest.raises((NameError, TypeError)): + msgspec.convert(V3_ARRAY_DOC, ZarrV3ArrayMetadata, dec_hook=hook) + assert calls == [] + + +def test_native_encode_cannot_emit_canonical_documents() -> None: + """Why serialization must route through to_json explicitly: msgspec's + encoders never consult enc_hook for dataclasses, so a model instance + either fails to encode (the UNSET sentinel is unencodable) or encodes as + a raw field dump that is not the canonical document. If either half + starts failing, msgspec grew an encode override point — revisit the + module's serialization guidance.""" + with pytest.raises(TypeError, match="sentinel"): + msgspec.json.encode(ZarrV3ArrayMetadata.from_json(V3_ARRAY_DOC)) + + config = ZarrV3NamedConfig.from_json({"name": "bytes"}) + assert msgspec.json.decode(msgspec.json.encode(config)) != config.to_json() + + +def test_core_package_does_not_import_msgspec() -> None: + """Importing zarr_metadata (in a fresh interpreter) must not import + msgspec: the integration is opt-in via zarr_metadata.msgspec.""" + import subprocess + import sys + + code = "import sys, zarr_metadata; assert 'msgspec' not in sys.modules, 'leaked'" + subprocess.run([sys.executable, "-c", code], check=True) From 1463b6055de9599a8fb87cfc2aecc5e06d2cc0b2 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 29 Aug 2026 13:33:21 +0200 Subject: [PATCH 2/3] fix(zarr-metadata): harden the msgspec integration after review Findings from a multi-angle review of the new module, applied: - Import msgspec at module scope so the integration fails fast where msgspec is absent, mirroring zarr_metadata.pydantic; the docstring's "requires msgspec" claim was previously unenforced. - Treat unhashable annotation objects as ordinary lookup misses: they previously raised TypeError past the KeyError guard, breaking the NotImplementedError contract and preventing a composed hook from ever delegating to the wrapped hook. - Make the runtime markers non-instantiable (shared _FieldType base): a hollow marker instance would have passed msgspec's result check while having none of the core class's fields. The module docstring now also states the exported names are for annotations only. - Derive the marker registrations from the decode table and add a parity test, so a marker cannot pair with one core class in the registration and another in the decoder, and the table provably covers the exported surface. - Document the third msgspec limit (no JSON Schema for custom types without a schema_hook) and show the explicit to_json serialization path in the docs index snippet. - Share the canonical test documents between the pydantic and msgspec integration tests via tests/model/_cases.py instead of a verbatim copy, correct the no-mutable-state test to claim only what from_json guarantees, and pin the native-encode test to msgspec's message rather than the sentinel type name. Assisted-by: ClaudeCode:claude-fable-5 --- packages/zarr-metadata/docs/index.md | 5 + .../src/zarr_metadata/msgspec.py | 110 ++++++++++++------ packages/zarr-metadata/tests/model/_cases.py | 19 +++ .../tests/model/test_msgspec_module.py | 88 ++++++++++---- .../tests/model/test_pydantic_module.py | 22 ++-- 5 files changed, 172 insertions(+), 72 deletions(-) diff --git a/packages/zarr-metadata/docs/index.md b/packages/zarr-metadata/docs/index.md index 911ad22122..04a184c025 100644 --- a/packages/zarr-metadata/docs/index.md +++ b/packages/zarr-metadata/docs/index.md @@ -77,8 +77,13 @@ import msgspec import zarr_metadata.msgspec as zmm metadata = msgspec.convert(raw, zmm.ZarrV3ArrayMetadata, dec_hook=zmm.dec_hook) +encoded = metadata.to_key_value()["zarr.json"] ``` +Serialization stays explicit (`to_json` / `to_key_value`): msgspec encodes +dataclasses natively, so no hook can make `msgspec.json.encode` emit the +canonical document — see [`zarr_metadata.msgspec`](api/msgspec.md). + ## Validation boundary The model validators enforce the declared document structure and a small set diff --git a/packages/zarr-metadata/src/zarr_metadata/msgspec.py b/packages/zarr-metadata/src/zarr_metadata/msgspec.py index dc334af1e1..8161a29fe2 100644 --- a/packages/zarr-metadata/src/zarr_metadata/msgspec.py +++ b/packages/zarr-metadata/src/zarr_metadata/msgspec.py @@ -16,14 +16,17 @@ the core classes — they interoperate freely with non-msgspec code (equality, isinstance, nesting). Static type checkers see each field type as its core model class, so `manifest.metadata` below is a -`zarr_metadata.model.ZarrV3ArrayMetadata`. +`zarr_metadata.model.ZarrV3ArrayMetadata`. That alias holds for +annotations only: at runtime the exported names are empty markers with +none of the core classes' methods, so construct, parse, and serialize +through `zarr_metadata.model` — `zmm.ZarrV3ArrayMetadata.from_json(...)` +type-checks but fails. `dec_hook` routes a raw document through `from_json` and passes an existing model instance through unchanged. `MetadataValidationError` subclasses -`ValueError`, so a failed parse surfaces as a -`msgspec.ValidationError` carrying the -loc-annotated problem messages, with msgspec appending the path of the -failing field (``- at `$.metadata` ``). +`ValueError`, so a failed parse surfaces as a `msgspec.ValidationError` +carrying the loc-annotated problem messages, with msgspec appending the +path of the failing field (``- at `$.metadata` ``). Usage: @@ -42,7 +45,7 @@ class ArrayManifest(msgspec.Struct): and a prebuilt `msgspec.json.Decoder`. An application that already has a decode hook of its own composes via `make_dec_hook(wrapped=...)`. -Two msgspec limits shape what this module can offer: +Three msgspec limits shape what this module can offer: - Serialization cannot be delegated: msgspec's encoders are value-driven and consult their `enc_hook` only for objects msgspec cannot encode @@ -55,6 +58,11 @@ class ArrayManifest(msgspec.Struct): - Unions may contain at most one hook-handled type, so a field cannot be typed as, say, array-or-group metadata. Decode such a field as an untyped mapping and dispatch on its content. +- JSON Schema generation (`msgspec.json.schema`) rejects custom types + unless the caller supplies a `schema_hook`, so a Struct using these + field types cannot produce a schema out of the box. When a JSON Schema + for the document forms is what you need, `zarr_metadata.pydantic` is + the schema-capable integration. """ from __future__ import annotations @@ -62,6 +70,11 @@ class ArrayManifest(msgspec.Struct): import abc from typing import TYPE_CHECKING, Any, Final +# The import is unused by name: it makes the module fail fast where msgspec +# is absent (everything exported here is inert without it), mirroring how +# `zarr_metadata.pydantic` fails at import when pydantic is absent. +import msgspec # noqa: F401 # pyright: ignore[reportUnusedImport] + from zarr_metadata import model as _model if TYPE_CHECKING: @@ -79,43 +92,42 @@ class ArrayManifest(msgspec.Struct): # At runtime each field type is a marker: an empty ABC msgspec treats as # a custom type (so `dec_hook` is consulted) with the core model class # registered as a virtual subclass (so the core instances `dec_hook` - # returns satisfy msgspec's isinstance check on hook results). + # returns satisfy msgspec's isinstance check on hook results). The + # registrations are derived from `_DECODERS` below, keeping one table + # that pairs each marker with its core class. - class ZarrV3ArrayMetadata(abc.ABC): # noqa: B024 - """Field type for a v3 array metadata document (`zarr.json` content).""" + class _FieldType(abc.ABC): # noqa: B024 + """Base of the runtime field-type markers; never instantiated.""" - ZarrV3ArrayMetadata.register(_model.ZarrV3ArrayMetadata) + __slots__ = () - class ZarrV2ArrayMetadata(abc.ABC): # noqa: B024 - """Field type for a v2 array metadata document (merged `.zarray` + `.zattrs` form).""" + def __new__(cls) -> None: + raise TypeError( + f"{cls.__name__} is a field-type marker for msgspec annotations only; " + "construct instances via the corresponding zarr_metadata.model class" + ) - ZarrV2ArrayMetadata.register(_model.ZarrV2ArrayMetadata) + class ZarrV3ArrayMetadata(_FieldType): + """Field type for a v3 array metadata document (`zarr.json` content).""" - class ZarrV3GroupMetadata(abc.ABC): # noqa: B024 - """Field type for a v3 group metadata document (`zarr.json` content).""" + class ZarrV2ArrayMetadata(_FieldType): + """Field type for a v2 array metadata document (merged `.zarray` + `.zattrs` form).""" - ZarrV3GroupMetadata.register(_model.ZarrV3GroupMetadata) + class ZarrV3GroupMetadata(_FieldType): + """Field type for a v3 group metadata document (`zarr.json` content).""" - class ZarrV2GroupMetadata(abc.ABC): # noqa: B024 + class ZarrV2GroupMetadata(_FieldType): """Field type for a v2 group metadata document (merged `.zgroup` + `.zattrs` form).""" - ZarrV2GroupMetadata.register(_model.ZarrV2GroupMetadata) - - class ZarrV3ConsolidatedMetadata(abc.ABC): # noqa: B024 + class ZarrV3ConsolidatedMetadata(_FieldType): """Field type for v3 inline consolidated metadata.""" - ZarrV3ConsolidatedMetadata.register(_model.ZarrV3ConsolidatedMetadata) - - class ZarrV2ConsolidatedMetadata(abc.ABC): # noqa: B024 + class ZarrV2ConsolidatedMetadata(_FieldType): """Field type for a v2 `.zmetadata` document.""" - ZarrV2ConsolidatedMetadata.register(_model.ZarrV2ConsolidatedMetadata) - - class ZarrV3MetadataField(abc.ABC): # noqa: B024 + class ZarrV3MetadataField(_FieldType): """Field type for one normalized v3 metadata extension envelope.""" - ZarrV3MetadataField.register(_model.ZarrV3NamedConfig) - _DECODERS: Final[dict[type, tuple[type, Callable[[object], object]]]] = { ZarrV3ArrayMetadata: (_model.ZarrV3ArrayMetadata, _model.ZarrV3ArrayMetadata.from_json), @@ -134,6 +146,30 @@ class ZarrV3MetadataField(abc.ABC): # noqa: B024 } """Marker class -> (pass-through core class, document parser).""" +if not TYPE_CHECKING: + for _marker, (_core_cls, _) in _DECODERS.items(): + _marker.register(_core_cls) + + +def _lookup(type: type) -> tuple[type, Callable[[object], object]] | None: + """Return the decode entry for `type`, or None for types not covered here. + + msgspec can hand a hook parametrized annotation objects, which may be + unhashable; those are never this module's markers, so a failed hash is + an ordinary miss rather than an error. + """ + try: + return _DECODERS.get(type) + except TypeError: + return None + + +def _decode(entry: tuple[type, Callable[[object], object]], obj: Any) -> Any: + core_cls, parse = entry + if isinstance(obj, core_cls): + return obj + return parse(obj) + def dec_hook(type: type, obj: Any) -> Any: """Decode `obj` for a field annotated with one of this module's field types. @@ -145,13 +181,10 @@ def dec_hook(type: type, obj: Any) -> Any: convention for "still unsupported"; to keep decoding custom types of your own alongside these, chain your hook with `make_dec_hook`. """ - try: - core_cls, parse = _DECODERS[type] - except KeyError: - raise NotImplementedError(f"Objects of type {type} are not supported") from None - if isinstance(obj, core_cls): - return obj - return parse(obj) + entry = _lookup(type) + if entry is None: + raise NotImplementedError(f"Objects of type {type} are not supported") + return _decode(entry, obj) def make_dec_hook(wrapped: Callable[[type, Any], Any] | None = None) -> Callable[[type, Any], Any]: @@ -166,9 +199,10 @@ def make_dec_hook(wrapped: Callable[[type, Any], Any] | None = None) -> Callable return dec_hook def hook(type: type, obj: Any) -> Any: - if type in _DECODERS: - return dec_hook(type, obj) - return wrapped(type, obj) + entry = _lookup(type) + if entry is None: + return wrapped(type, obj) + return _decode(entry, obj) return hook diff --git a/packages/zarr-metadata/tests/model/_cases.py b/packages/zarr-metadata/tests/model/_cases.py index 15faa65539..2c55f7439d 100644 --- a/packages/zarr-metadata/tests/model/_cases.py +++ b/packages/zarr-metadata/tests/model/_cases.py @@ -6,9 +6,28 @@ import pytest +from zarr_metadata.model import ZarrV2ArrayMetadata, ZarrV3ArrayMetadata + if TYPE_CHECKING: from contextlib import AbstractContextManager +# Canonical documents shared by the integration test modules +# (test_pydantic_module.py, test_msgspec_module.py), so a model change that +# alters a canonical document is corrected in one place. +V3_ARRAY_DOC = dict(ZarrV3ArrayMetadata.create_default(shape=(4,)).to_json()) +V2_ARRAY_DOC = dict(ZarrV2ArrayMetadata.create_default(shape=(4,), chunks=(2,)).to_json()) +V3_GROUP_DOC = {"zarr_format": 3, "node_type": "group", "attributes": {"a": 1}} +V2_GROUP_DOC = {"zarr_format": 2, "attributes": {"a": 1}} +V3_CONSOLIDATED_DOC = { + "kind": "inline", + "must_understand": False, + "metadata": {"a": dict(V3_ARRAY_DOC)}, +} +V2_CONSOLIDATED_DOC = { + "zarr_consolidated_format": 1, + "metadata": {".zgroup": {"zarr_format": 2}}, +} + TIn = TypeVar("TIn") TOut = TypeVar("TOut") diff --git a/packages/zarr-metadata/tests/model/test_msgspec_module.py b/packages/zarr-metadata/tests/model/test_msgspec_module.py index 5d55fd4e90..e89dbf2b96 100644 --- a/packages/zarr-metadata/tests/model/test_msgspec_module.py +++ b/packages/zarr-metadata/tests/model/test_msgspec_module.py @@ -12,6 +12,14 @@ import pytest import zarr_metadata.msgspec as zmm +from tests.model._cases import ( + V2_ARRAY_DOC, + V2_CONSOLIDATED_DOC, + V2_GROUP_DOC, + V3_ARRAY_DOC, + V3_CONSOLIDATED_DOC, + V3_GROUP_DOC, +) from zarr_metadata.model import ( ZarrV2ArrayMetadata, ZarrV2ConsolidatedMetadata, @@ -22,20 +30,6 @@ ZarrV3NamedConfig, ) -V3_ARRAY_DOC = dict(ZarrV3ArrayMetadata.create_default(shape=(4,)).to_json()) -V2_ARRAY_DOC = dict(ZarrV2ArrayMetadata.create_default(shape=(4,), chunks=(2,)).to_json()) -V3_GROUP_DOC = {"zarr_format": 3, "node_type": "group", "attributes": {"a": 1}} -V2_GROUP_DOC = {"zarr_format": 2, "attributes": {"a": 1}} -V3_CONSOLIDATED_DOC = { - "kind": "inline", - "must_understand": False, - "metadata": {"a": dict(V3_ARRAY_DOC)}, -} -V2_CONSOLIDATED_DOC = { - "zarr_consolidated_format": 1, - "metadata": {".zgroup": {"zarr_format": 2}}, -} - FIELD_CASES = [ pytest.param(zmm.ZarrV3ArrayMetadata, ZarrV3ArrayMetadata, V3_ARRAY_DOC, id="array-v3"), pytest.param(zmm.ZarrV2ArrayMetadata, ZarrV2ArrayMetadata, V2_ARRAY_DOC, id="array-v2"), @@ -71,6 +65,18 @@ def test_field_type_decodes_and_passes_through( assert decoded == model +def test_registry_is_consistent() -> None: + """One decode table drives everything: it covers exactly the exported + field types, each parser is its core class's from_json, and each core + class is registered as a virtual subclass of its marker (so instances + dec_hook returns satisfy msgspec's check on hook results).""" + field_types = {getattr(zmm, name) for name in zmm.__all__ if name[0] == "Z"} + assert field_types == set(zmm._DECODERS) + for marker, (core_cls, parse) in zmm._DECODERS.items(): + assert parse == core_cls.from_json + assert issubclass(core_cls, marker) + + def test_struct_field_round_trip() -> None: """A Struct field decodes a raw document through from_json (the library's normalization applies), and re-encoding the canonical document emitted by @@ -93,7 +99,8 @@ class ArrayManifest(msgspec.Struct): def test_wrapped_hook_composes() -> None: """make_dec_hook keeps an application's own custom-type decoding working - alongside the model field types; with no wrapped hook it is dec_hook.""" + alongside the model field types — including custom types whose annotation + objects are unhashable — and with no wrapped hook it is dec_hook.""" class Fraction: def __init__(self, value: float) -> None: @@ -113,12 +120,30 @@ class Manifest(msgspec.Struct): assert isinstance(manifest.scale, Fraction) assert manifest.scale.value == 0.5 assert type(manifest.metadata) is ZarrV3GroupMetadata + + # An unhashable annotation object is an ordinary miss, delegated onward. + class Unhashable: + __hash__ = None # type: ignore[assignment] + + unhashable = Unhashable() + seen: list[object] = [] + + def recording_hook(type: type, obj: object) -> object: + seen.append(type) + return obj + + assert zmm.make_dec_hook(recording_hook)(unhashable, 1) == 1 + assert seen == [unhashable] + assert zmm.make_dec_hook() is zmm.dec_hook -def test_decoded_models_share_no_mutable_state() -> None: - """Decoding the same document twice yields equal but independent models, - and mutating the source document does not reach into a decoded model.""" +def test_decoding_is_stateless_and_normalizing() -> None: + """Decoding the same document twice yields equal but independent model + instances, and from_json's normalization converts JSON arrays to tuples, + so mutating a list in the source document cannot reach a decoded model. + (from_json copies mappings shallowly; only to_json guarantees a document + that shares no mutable state.)""" doc = {"zarr_format": 3, "node_type": "group", "attributes": {"a": [1]}} first = msgspec.convert(doc, zmm.ZarrV3GroupMetadata, dec_hook=zmm.dec_hook) second = msgspec.convert(doc, zmm.ZarrV3GroupMetadata, dec_hook=zmm.dec_hook) @@ -146,9 +171,20 @@ class ArrayManifest(msgspec.Struct): def test_dec_hook_rejects_uncovered_types() -> None: """A type the module does not cover raises NotImplementedError, msgspec's - convention for a still-unsupported custom type.""" + convention for a still-unsupported custom type — including annotation + objects that are not hashable.""" with pytest.raises(NotImplementedError, match="int"): zmm.dec_hook(int, 1) + with pytest.raises(NotImplementedError, match="not supported"): + zmm.dec_hook([], 1) + + +def test_field_type_markers_cannot_be_instantiated() -> None: + """The exported names are for annotations only: instantiating a runtime + marker raises instead of minting a hollow object that would pass + msgspec's result check while having none of the core class's fields.""" + with pytest.raises(TypeError, match="field-type marker"): + zmm.ZarrV3ArrayMetadata() def test_core_class_annotations_bypass_dec_hook() -> None: @@ -176,7 +212,7 @@ def test_native_encode_cannot_emit_canonical_documents() -> None: a raw field dump that is not the canonical document. If either half starts failing, msgspec grew an encode override point — revisit the module's serialization guidance.""" - with pytest.raises(TypeError, match="sentinel"): + with pytest.raises(TypeError, match="unsupported"): msgspec.json.encode(ZarrV3ArrayMetadata.from_json(V3_ARRAY_DOC)) config = ZarrV3NamedConfig.from_json({"name": "bytes"}) @@ -191,3 +227,15 @@ def test_core_package_does_not_import_msgspec() -> None: code = "import sys, zarr_metadata; assert 'msgspec' not in sys.modules, 'leaked'" subprocess.run([sys.executable, "-c", code], check=True) + + +def test_module_requires_msgspec() -> None: + """Importing the integration module without msgspec fails fast at import + (its exports are inert without msgspec), mirroring zarr_metadata.pydantic.""" + import subprocess + import sys + + code = "import sys; sys.modules['msgspec'] = None; import zarr_metadata.msgspec" + proc = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=False) + assert proc.returncode != 0 + assert "msgspec" in proc.stderr diff --git a/packages/zarr-metadata/tests/model/test_pydantic_module.py b/packages/zarr-metadata/tests/model/test_pydantic_module.py index d15b3f118c..be9165c448 100644 --- a/packages/zarr-metadata/tests/model/test_pydantic_module.py +++ b/packages/zarr-metadata/tests/model/test_pydantic_module.py @@ -13,6 +13,14 @@ from pydantic import BaseModel, TypeAdapter, ValidationError import zarr_metadata.pydantic as zmp +from tests.model._cases import ( + V2_ARRAY_DOC, + V2_CONSOLIDATED_DOC, + V2_GROUP_DOC, + V3_ARRAY_DOC, + V3_CONSOLIDATED_DOC, + V3_GROUP_DOC, +) from zarr_metadata.model import ( ZarrV2ArrayMetadata, ZarrV2ConsolidatedMetadata, @@ -23,20 +31,6 @@ ZarrV3NamedConfig, ) -V3_ARRAY_DOC = dict(ZarrV3ArrayMetadata.create_default(shape=(4,)).to_json()) -V2_ARRAY_DOC = dict(ZarrV2ArrayMetadata.create_default(shape=(4,), chunks=(2,)).to_json()) -V3_GROUP_DOC = {"zarr_format": 3, "node_type": "group", "attributes": {"a": 1}} -V2_GROUP_DOC = {"zarr_format": 2, "attributes": {"a": 1}} -V3_CONSOLIDATED_DOC = { - "kind": "inline", - "must_understand": False, - "metadata": {"a": dict(V3_ARRAY_DOC)}, -} -V2_CONSOLIDATED_DOC = { - "zarr_consolidated_format": 1, - "metadata": {".zgroup": {"zarr_format": 2}}, -} - FIELD_CASES = [ pytest.param(zmp.ZarrV3ArrayMetadata, ZarrV3ArrayMetadata, V3_ARRAY_DOC, id="array-v3"), pytest.param(zmp.ZarrV2ArrayMetadata, ZarrV2ArrayMetadata, V2_ARRAY_DOC, id="array-v2"), From 948099f73e673ae7881546d25f7a7b0dcaf093dd Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Sat, 29 Aug 2026 15:44:16 +0200 Subject: [PATCH 3/3] chore(zarr-metadata): name the msgspec changelog fragment after the PR ci/check_changelog_entries.py requires an integer PR-number stem, which the towncrier orphan name used until the PR existed. Assisted-by: ClaudeCode:claude-fable-5 --- .../zarr-metadata/changes/{+msgspec.feature.md => 312.feature.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/zarr-metadata/changes/{+msgspec.feature.md => 312.feature.md} (100%) diff --git a/packages/zarr-metadata/changes/+msgspec.feature.md b/packages/zarr-metadata/changes/312.feature.md similarity index 100% rename from packages/zarr-metadata/changes/+msgspec.feature.md rename to packages/zarr-metadata/changes/312.feature.md