Preserve storage metadata and limit Python support to 3.12-3.14

.github/workflows/compatibility.yml

@@ -18,13 +18,12 @@ jobs:
   compatibility:
     name: ${{ matrix.os }} / Python ${{ matrix.python-version }}
     runs-on: ${{ matrix.os }}
-    continue-on-error: ${{ matrix.python-version == '3.15' }}
     timeout-minutes: 45
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ["3.12", "3.13", "3.14", "3.15"]
+        python-version: ["3.12", "3.13", "3.14"]
     env:
       UV_PYTHON: ${{ matrix.python-version }}
       UV_NO_MANAGED_PYTHON: "1"
@@ -39,22 +38,16 @@ jobs:
         uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: ${{ matrix.python-version }}
-          allow-prereleases: true
 
       - name: Install uv
         uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
         with:
           version: ${{ env.UV_VERSION }}
           enable-cache: true
 
-      - name: Install the complete stable environment
-        if: matrix.python-version != '3.15'
+      - name: Install the complete supported environment
         run: uv sync --locked --all-extras --dev
 
-      - name: Install the core prerelease environment
-        if: matrix.python-version == '3.15'
-        run: uv sync --locked --no-default-groups --group prerelease
-
       - name: Verify import
         run: uv run --no-sync python -c "import ml4t.data as package; print(package.__version__)"
 

README.md

@@ -34,10 +34,10 @@ The goal is to support an ongoing research workflow rather than one-off download
 ## Installation
 
 ML4T Data supports stable CPython 3.12 through 3.14 on Linux, macOS, and Windows.
-Python 3.15 prereleases are tested as informational compatibility checks but are not supported
-until the required dependencies publish compatible distributions. The Databento SDK currently
-has no CPython 3.15 distribution, so the 3.15 prerelease lane tests the core package without the
-Databento extra.
+Python 3.15 is not supported until the core dependency stack passes the complete compatibility
+suite on all three operating systems. Releases 0.1.0 and 0.1.1 predate this upper bound, so an
+unpinned installation on Python 3.15 may select one of those older releases. Use Python 3.12
+through 3.14 instead.
 
 ```bash
 pip install ml4t-data

docs/getting-started/installation.md

@@ -67,8 +67,9 @@ print(f"Fetched {len(df)} rows")
 ## Requirements
 
 - Stable CPython 3.12 through 3.14 on Linux, macOS, or Windows
-- Python 3.15 prereleases are tested as informational checks but are not supported
-- The Databento SDK does not yet publish a Python 3.15 distribution, so that optional extra is
-  excluded from prerelease testing until an upstream build is available
+- Python 3.15 is not supported until the core dependency stack passes the complete compatibility
+  suite on Linux, macOS, and Windows
+- Releases 0.1.0 and 0.1.1 predate the Python 3.15 upper bound, so an unpinned installation on
+  Python 3.15 may select one of those older releases
 - Polars (automatically installed)
 - Provider-specific SDKs (optional)

pyproject.toml

@@ -70,7 +70,7 @@ classifiers = [
     "Topic :: Scientific/Engineering :: Information Analysis",
     "Typing :: Typed",
 ]
-requires-python = ">=3.12"
+requires-python = ">=3.12,<3.15"
 
 # Core dependencies
 dependencies = [
@@ -88,7 +88,6 @@ dependencies = [
     "python-dotenv>=1.0.0",
     "pydantic-settings>=2.0.0",
     "pydantic>=2.12,<3",
-    "pydantic>=2.14.0b1; python_version >= '3.15'",
     # Utilities
     "structlog>=23.0.0",
     "platformdirs>=4.0.0",
@@ -173,13 +172,6 @@ test = [
     "ty",
     "xlsxwriter>=3.1.0",
 ]
-prerelease = [
-    { include-group = "test" },
-    "yfinance>=0.2.0",
-    "oandapyV20>=0.7.0",
-    "requests>=2.32.0",
-    "cot-reports>=0.1.0",
-]
 dev = [
     { include-group = "test" },
     "ruff>=0.8.0",
@@ -191,6 +183,9 @@ dev = [
     "oandapyV20>=0.7.0",
 ]
 
+[tool.uv]
+prerelease = "disallow"
+
 [tool.pytest.ini_options]
 testpaths = ["tests"]
 python_files = ["test_*.py"]

src/ml4t/data/managers/storage_manager.py

@@ -15,9 +15,11 @@
 
 import polars as pl
 import structlog
+from pydantic import ValidationError
 from tenacity import RetryError
 
 from ml4t.data.core.schemas import align_frames_for_concat, timestamp_bounds
+from ml4t.data.storage.backend import normalize_storage_metadata
 from ml4t.data.utils.conversion import pandas_to_polars
 
 if TYPE_CHECKING:
@@ -493,6 +495,20 @@ def update(
 
             logger.info("Found existing data", rows=len(existing_df))
 
+            existing_metadata = (
+                normalize_storage_metadata(self.storage.get_metadata(key), key) or {}
+            )
+            existing_provider = existing_metadata.get("provider")
+            if (
+                not provider
+                and existing_provider is not None
+                and not isinstance(existing_provider, str)
+            ):
+                raise ValueError(
+                    f"Stored metadata provider for '{key}' must be a string, "
+                    f"got {type(existing_provider).__name__}"
+                )
+
             # Get the date range from existing data
             _, last_timestamp = timestamp_bounds(existing_df)
             if last_timestamp.tzinfo is None:
@@ -588,29 +604,78 @@ def update(
             from ml4t.data.core.models import DataObject, Metadata
 
             min_ts, max_ts = timestamp_bounds(merged_df)
-
-            updated_metadata = Metadata(
-                provider=provider or "auto",
-                symbol=symbol,
-                asset_class=asset_class,
-                bar_type="time",
-                bar_params={"frequency": frequency},
-                data_range={
-                    "start": str(min_ts),
-                    "end": str(max_ts),
-                },
-                attributes={
-                    "last_update": datetime.now().isoformat(),
+            existing_attributes = existing_metadata.get("attributes")
+            updated_attributes = (
+                existing_attributes.copy() if isinstance(existing_attributes, dict) else {}
+            )
+            updated_at = datetime.now(UTC)
+            updated_attributes.update(
+                {
+                    # BulkManager compares this legacy field with naive local cutoffs.
+                    "last_update": updated_at.astimezone().replace(tzinfo=None).isoformat(),
                     "update_type": "incremental",
                     "gaps_filled": fill_gaps and len(gaps) > 0,
                     "provider_history_limited": provider_history_limited,
-                },
+                }
+            )
+
+            existing_model_values = {
+                name: value
+                for name in Metadata.model_fields
+                if (value := existing_metadata.get(name)) is not None
+            }
+            resolved_provider = (
+                provider
+                or (existing_provider if isinstance(existing_provider, str) else None)
+                or "auto"
             )
+            existing_bar_type = existing_metadata.get("bar_type")
+            existing_bar_params = existing_metadata.get("bar_params")
+            metadata_values = {
+                **existing_model_values,
+                "provider": resolved_provider,
+                "symbol": symbol,
+                "asset_class": asset_class,
+                "bar_type": existing_bar_type if isinstance(existing_bar_type, str) else "time",
+                "bar_params": (
+                    existing_bar_params
+                    if isinstance(existing_bar_params, dict)
+                    else {"frequency": frequency}
+                ),
+                "start_date": min_ts,
+                "end_date": max_ts,
+                "last_updated": updated_at,
+                "data_range": {
+                    "start": str(min_ts),
+                    "end": str(max_ts),
+                },
+                "attributes": updated_attributes,
+            }
+
+            try:
+                updated_metadata = Metadata.model_validate(metadata_values)
+            except ValidationError as error:
+                invalid_fields = {
+                    location[0]
+                    for detail in error.errors()
+                    if (location := detail.get("loc")) and isinstance(location[0], str)
+                }
+                removable_fields = invalid_fields - {"provider", "symbol", "asset_class"}
+                if not removable_fields:
+                    raise
+                logger.warning(
+                    "Ignoring invalid optional fields in stored metadata",
+                    key=key,
+                    fields=sorted(removable_fields),
+                )
+                for field in removable_fields:
+                    metadata_values.pop(field, None)
+                updated_metadata = Metadata.model_validate(metadata_values)
 
             updated_obj = DataObject(data=merged_df, metadata=updated_metadata)
 
             metadata_dict = updated_obj.metadata.model_dump() if updated_obj.metadata else None
-            self.storage.write(updated_obj.data, key, metadata_dict)
+            self.storage.write(updated_obj.data, key, metadata_dict, preserve_metadata=True)
 
             logger.info(
                 "Incremental update completed",

src/ml4t/data/storage/backend.py

@@ -28,12 +28,20 @@
 
 
 def normalize_storage_metadata(metadata: Any, key: str | None = None) -> dict[str, Any] | None:
-    """Return domain metadata from a canonical or legacy storage record."""
+    """Return flattened domain metadata from a canonical or legacy storage record.
+
+    Non-null values in ``custom`` override committed fields. A null custom value does not clear a
+    non-null committed value, but custom-only keys remain present even when their value is null.
+    """
     if not isinstance(metadata, dict) or not metadata:
         return None
 
     custom = metadata.get("custom")
-    normalized = {**metadata, **custom} if isinstance(custom, dict) else metadata.copy()
+    normalized = metadata.copy()
+    if isinstance(custom, dict):
+        for name, value in custom.items():
+            if value is not None or normalized.get(name) is None:
+                normalized[name] = value
 
     if key is not None:
         parts = key.split("/", 2)
@@ -93,19 +101,58 @@ def _recover_key_staging(key_path: Path) -> None:
             elif staging_path.is_dir():
                 shutil.rmtree(staging_path)
 
+    def _effective_metadata(
+        self,
+        key: str,
+        metadata: dict[str, Any] | None,
+        preserve_metadata: bool,
+    ) -> dict[str, Any]:
+        """Resolve metadata for a write while holding its key lock."""
+        effective_metadata = metadata.copy() if metadata else {}
+        if not preserve_metadata:
+            return effective_metadata
+        try:
+            current_record = self._current_commit(key).metadata
+        except KeyError:
+            return effective_metadata
+        except RuntimeError as error:
+            logger.warning(
+                "Ignoring unreadable metadata while replacing stored data",
+                key=key,
+                error=str(error),
+            )
+            return effective_metadata
+
+        current_custom = current_record.get("custom")
+        if not isinstance(current_custom, dict):
+            return effective_metadata
+
+        existing_attributes = current_custom.get("attributes")
+        updated_attributes = effective_metadata.get("attributes")
+        effective_metadata = {**current_custom, **effective_metadata}
+        if isinstance(existing_attributes, dict) and isinstance(updated_attributes, dict):
+            effective_metadata["attributes"] = {
+                **existing_attributes,
+                **updated_attributes,
+            }
+        return effective_metadata
+
     @abstractmethod
     def write(
         self,
         data: pl.LazyFrame | pl.DataFrame,
         key: str,
         metadata: dict[str, Any] | None = None,
+        *,
+        preserve_metadata: bool = False,
     ) -> Path:
         """Write data to storage.
 
         Args:
             data: Polars LazyFrame to write
             key: Storage key (e.g., "BTC-USD", "SPY")
             metadata: Optional metadata to store alongside data
+            preserve_metadata: Merge metadata into the current custom block under the write lock
 
         Returns:
             Path to written file

src/ml4t/data/storage/flat.py

@@ -39,14 +39,20 @@ def __init__(self, config: StorageConfig):
         super().__init__(config)
 
     def write(
-        self, data: pl.LazyFrame | pl.DataFrame, key: str, metadata: dict[str, Any] | None = None
+        self,
+        data: pl.LazyFrame | pl.DataFrame,
+        key: str,
+        metadata: dict[str, Any] | None = None,
+        *,
+        preserve_metadata: bool = False,
     ) -> Path:
         """Write data as a single file.
 
         Args:
             data: Data to write
             key: Storage key (e.g., "BTC-USD")
             metadata: Optional metadata
+            preserve_metadata: Merge metadata into the current custom block under the write lock
 
         Returns:
             Path to written file
@@ -56,18 +62,19 @@ def write(
 
         df = lazy_data.collect()
         with self._key_lock(key):
+            effective_metadata = self._effective_metadata(key, metadata, preserve_metadata)
             staging_path, generation_id = self._prepare_generation(key)
             try:
                 staged_file = staging_path / "data.parquet"
                 self._atomic_write(df, staged_file)
                 commit_metadata = (
                     {
-                        "last_updated": datetime.now().isoformat(),
+                        "last_updated": datetime.now(UTC).isoformat(),
                         "file_path": "data.parquet",
                         "row_count": len(df),
                         "schema": list(df.columns),
                         "file_size_mb": staged_file.stat().st_size / (1024 * 1024),
-                        "custom": metadata or {},
+                        "custom": effective_metadata,
                     }
                     if self.config.metadata_tracking
                     else {}