Enable predictions on dataset partitions and remove refitting on validation for forecasting

DashAI/alembic/versions/a5f2c71e9d40_add_split_to_prediction.py

@@ -0,0 +1,42 @@
+"""add the split column to prediction
+
+Revision ID: a5f2c71e9d40
+Revises: b7c1d4e9f206
+Create Date: 2026-09-01 10:00:00.000000
+
+Predictions made before this column existed covered the whole dataset, which
+is what a null value means, so no backfill is needed.
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+
+from alembic import op
+
+revision: str = "a5f2c71e9d40"
+down_revision: Union[str, None] = "b7c1d4e9f206"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    bind = op.get_bind()
+    inspector = sa.inspect(bind)
+    if "prediction" not in inspector.get_table_names():
+        return
+    columns = {column["name"] for column in inspector.get_columns("prediction")}
+    if "split" in columns:
+        return
+    op.add_column("prediction", sa.Column("split", sa.String(), nullable=True))
+
+
+def downgrade() -> None:
+    bind = op.get_bind()
+    inspector = sa.inspect(bind)
+    if "prediction" not in inspector.get_table_names():
+        return
+    columns = {column["name"] for column in inspector.get_columns("prediction")}
+    if "split" not in columns:
+        return
+    op.drop_column("prediction", "split")

DashAI/back/api/api_v1/endpoints/explainers.py

@@ -28,7 +28,7 @@
     ModelSession,
     Run,
 )
-from DashAI.back.splitters.splits_payload import splitter_class_for
+from DashAI.back.splitters.splits_payload import run_splits
 
 if TYPE_CHECKING:
     from sqlalchemy.orm import sessionmaker
@@ -1107,8 +1107,6 @@ async def get_explainable_splits(
     HTTPException
         If the run does not exist in the database.
     """
-    import json
-
     with session_factory() as db:
         try:
             run: Run = db.get(Run, run_id)
@@ -1123,10 +1121,8 @@ async def get_explainable_splits(
                     status_code=status.HTTP_404_NOT_FOUND,
                     detail="Model session not found",
                 )
-            split_indexes = json.loads(run.split_indexes) if run.split_indexes else {}
+            split_indexes = run.split_indexes
             session_splits = model_session.splits
-            if isinstance(session_splits, str):
-                session_splits = json.loads(session_splits)
         except exc.SQLAlchemyError as e:
             log.exception(e)
             raise HTTPException(
@@ -1135,14 +1131,12 @@ async def get_explainable_splits(
             ) from e
 
     try:
-        splitter_class = splitter_class_for(session_splits, component_registry)
+        return {"splits": run_splits(session_splits, split_indexes, component_registry)}
     except ValueError as e:
         raise HTTPException(
             status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e)
         ) from e
 
-    return {"splits": splitter_class.explainable_splits(split_indexes)}
-
 
 @router.post("/local/valid-datasets")
 @inject
@@ -1167,9 +1161,6 @@ async def valid_datasets(
     dict
         ``{"valid_dataset_ids": [...]}`` with the ids of the valid datasets.
     """
-    # get_columns_spec reads only the Arrow schema metadata (column names +
-    # types), never the rows, so validating every dataset stays cheap even with
-    # many/large (e.g. image) datasets on the platform.
     from DashAI.back.dataloaders.classes.dashai_dataset import get_columns_spec
 
     with session_factory() as db:

DashAI/back/api/api_v1/endpoints/predict.py

@@ -15,6 +15,7 @@
     Run,
 )
 from DashAI.back.job.predict_job import run_manual_prediction
+from DashAI.back.splitters.splits_payload import predictable_splits
 
 if TYPE_CHECKING:
     from sqlalchemy.orm import Session, sessionmaker
@@ -44,6 +45,10 @@ async def create_prediction(
         The ID of the trained model/run.
     dataset_id : int | None
         The ID of the dataset to use for prediction (optional).
+    split : str | None
+        The partition of that dataset to predict on (optional). Only applies
+        when the dataset is the one the model was trained on; ``None`` and
+        ``"all"`` both mean every row.
     session_factory : Callable[..., ContextManager[Session]]
         A factory that creates a context manager that handles a SQLAlchemy session.
         The generated session can be used to access and query the database.
@@ -69,6 +74,7 @@ async def create_prediction(
         prediction = Prediction(
             run_id=run.id,
             dataset_id=params.dataset_id,
+            split=params.split,
         )
         db.add(prediction)
         db.commit()
@@ -127,6 +133,78 @@ async def get_all_predictions(
         return predictions
 
 
+@router.get("/splits/{run_id}")
+@inject
+async def get_prediction_splits(
+    run_id: int,
+    session_factory: "sessionmaker" = Depends(lambda: di["session_factory"]),
+    component_registry: "ComponentRegistry" = Depends(lambda: di["component_registry"]),
+):
+    """Return the dataset partitions a prediction of this run may target.
+
+    The partitions describe the dataset the model was trained on, so they only
+    apply to a prediction run on that same dataset; any other dataset is
+    predicted whole. Which partitions exist depends on how the run was
+    evaluated, so the splitter that produced it decides the list and its names,
+    and a task whose models only predict forward keeps just the partitions
+    outside the window its saved model was fitted through.
+
+    Parameters
+    ----------
+    run_id : int
+        The ID of the trained model/run.
+    session_factory : Callable[..., ContextManager[Session]]
+        A factory that creates a context manager that handles a SQLAlchemy session.
+    component_registry : ComponentRegistry
+        Registry used to resolve the splitter that produced the run.
+
+    Returns
+    -------
+    dict
+        A ``splits`` list of ``{"name", "rows"}`` entries, empty when the run
+        has no partition worth offering.
+
+    Raises
+    ------
+    HTTPException
+        If the run or its model session does not exist, or if the run was
+        produced by a splitter that is no longer registered.
+    """
+    db: Session
+    with session_factory() as db:
+        run: Run = db.get(Run, run_id)
+        if not run:
+            raise HTTPException(
+                status_code=status.HTTP_404_NOT_FOUND, detail="Run not found"
+            )
+        model_session: ModelSession = db.get(ModelSession, run.model_session_id)
+        if not model_session:
+            raise HTTPException(
+                status_code=status.HTTP_404_NOT_FOUND,
+                detail="Model session not found",
+            )
+        session_splits = model_session.splits
+        split_indexes = run.split_indexes
+        training_dataset_id = model_session.dataset_id
+        task_name = model_session.task_name
+        evaluation_strategy = model_session.evaluation_strategy
+
+    try:
+        splits = predictable_splits(
+            session_splits,
+            split_indexes,
+            component_registry,
+            task_name=task_name,
+            evaluation_strategy=evaluation_strategy,
+        )
+    except ValueError as e:
+        raise HTTPException(
+            status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(e)
+        ) from e
+
+    return {"splits": splits, "training_dataset_id": training_dataset_id}
+
+
 @router.get("/filter_datasets")
 async def filter_datasets_endpoint(
     run_id: int = Query(..., description="The ID of the trained model/run"),

DashAI/back/api/api_v1/schemas/prediction_params.py

@@ -6,3 +6,4 @@
 class PredictionCreationParams(BaseModel):
     run_id: int
     dataset_id: Optional[int] = None
+    split: Optional[str] = None

DashAI/back/dependencies/database/models.py

@@ -233,6 +233,7 @@ class Prediction(Base):
     id: Mapped[int] = mapped_column(primary_key=True)
     run_id: Mapped[int] = mapped_column(ForeignKey("run.id", ondelete="CASCADE"))
     dataset_id: Mapped[int] = mapped_column(ForeignKey("dataset.id"), nullable=True)
+    split: Mapped[str] = mapped_column(String, nullable=True)
     huey_id: Mapped[str] = mapped_column(String, nullable=True)
     created: Mapped[DateTime] = mapped_column(DateTime, default=datetime.now)
     last_modified: Mapped[DateTime] = mapped_column(

DashAI/back/evaluation/base_evaluation_strategy.py

@@ -23,16 +23,11 @@ class BaseEvaluationStrategy(metaclass=ABCMeta):
 
     TYPE: Final[str] = "EvaluationStrategy"
 
-    # How this strategy divides the dataset. The frontend renders holdout
-    # controls or fold controls from this rather than comparing class names,
-    # which is what previously made a new strategy unreachable from the UI.
     KIND: str = "holdout"
-
-    # Which partitions this strategy records metrics for. Scoring the training
-    # partition means predicting on rows the model was fitted on, which is a
-    # fit statistic; a forecaster has no such thing to report.
     SCORED_SPLITS: tuple = (SplitEnum.TRAIN, SplitEnum.VALIDATION, SplitEnum.TEST)
 
+    FINAL_FIT_PARTITIONS: tuple = ("train",)
+
     @classmethod
     def get_metadata(cls) -> dict:
         """Describe the strategy for the frontend.

DashAI/back/evaluation/forecasting_holdout.py

@@ -5,107 +5,44 @@
 
 
 class ForecastingHoldoutEvaluationStrategy(SinglePartitionEvaluationStrategy):
-    """Holdout evaluation that treats validation as history rather than a sample.
+    """Holdout evaluation that records no in-sample metrics.
 
-    Two things the ordinary holdout strategy assumes are wrong for a
-    forecaster, and both of them are decisions about evaluation rather than
-    about any model.
+    One thing the ordinary holdout strategy assumes is wrong for a forecaster,
+    and it is a decision about evaluation rather than about any model.
 
     **The training partition is not scored.** Scoring it would mean asking the
     model about dates it was fitted on. That is an in-sample fit statistic,
     which is a real diagnostic but is not comparable with a forecast made
     several steps out; showing the two side by side in one results table
     invites exactly that comparison. Only validation and test are recorded.
 
-    **The kept model is fitted through validation.** For most tasks the
-    validation partition is a held out sample that has to stay out of the fit.
-    For a forecaster it is simply the most recent stretch of the series, and
-    the stretch nearest to whatever comes next. Leaving it out makes the model
-    reach across the whole validation window before arriving at the first test
-    row, so the test metrics describe a longer horizon than the one being
-    asked about.
+    **The kept model is fitted on the training partition alone**, like every
+    other holdout run, and nothing is fed to it afterwards. Two approaches that
+    would have changed that were tried and dropped, both because they hand the
+    model data from a partition it was meant to be held out from:
 
-    The validation metrics are still measured on a model fitted on training
-    data alone, which is what makes them honest: they are recorded before the
-    refit. So the two columns in the results table answer different questions,
-    and both answer them fairly.
+        refitting through validation before scoring test, which overwrote the
+        fit the validation metrics came from, so the saved model could not
+        reproduce its own results table;
 
-        validation metrics  <- model fitted on train
-        test metrics        <- model fitted on train + validation
+        advancing the model through the observed validation rows at predict
+        time, which re-estimates nothing but still lets a held out partition
+        reach the model, which no other task in DashAI does.
 
-    Hyperparameter search is untouched. Its trials are scored on validation,
-    so they must not be fitted on it.
-    """
-
-    COMPATIBLE_COMPONENTS = ["ForecastingTask"]
-    SCORED_SPLITS: tuple = (SplitEnum.VALIDATION, SplitEnum.TEST)
-
-    def execute(self, x, y, run, db):
-        """Score validation on a trial fit, then refit and score test.
-
-        Parameters
-        ----------
-        x : DatasetDict
-            Input partitions, keyed by split name.
-        y : DatasetDict
-            Target partitions, keyed by split name.
-        run : Run
-            Database model representing the current run.
-        db : Session
-            SQLAlchemy session used to persist metrics.
-
-        Returns
-        -------
-        tuple
-            The trained model and the paths of any HPO plots.
-        """
-        plot_paths = []
-        model = self.model
-
-        model.x_data = x
-        model.y_data = y
-
-        if self.optimizer and self.run_optimizable_parameters:
-            self._report_progress(0.2, "Hyperparameter optimization")
-            model = self._do_hpo(model, x, y, run, db)
-            plot_paths = self._generate_hpo_plots(run)
+    So the two columns describe different horizons, and deliberately:
 
-        # Fitted on training data only, so the validation score below measures
-        # a model that has not seen the rows it is being scored on.
-        self._report_progress(0.5, "Training")
-        model.train(x["train"], y["train"])
+        validation metrics  <- forecasting 1..len(val) past the fit
+        test metrics        <- forecasting len(val)+1..len(val)+len(test),
+                               its own forecasts standing in for validation
 
-        self._report_progress(0.8, "Computing validation metrics")
-        self._calculate_metrics_if_missing(model, run, db, SplitEnum.VALIDATION)
+    The test column is therefore the harder question, not the same one further
+    along. Comparing like with like over a chosen horizon is what
+    ``RollingOriginSplitter`` is for, since its ``horizon`` says outright how
+    many steps ahead each refit is scored on.
 
-        # Now the model that gets kept: the same configuration, refitted with
-        # the validation rows included, since for a series they are history.
-        self._report_progress(0.9, "Refitting on train and validation")
-        self._fit_final_model(model, x, y)
-
-        self._report_progress(0.95, "Computing test metrics")
-        self._calculate_metrics_if_missing(model, run, db, SplitEnum.TEST)
-
-        return model, plot_paths
-
-    def _fit_final_model(self, model, x, y):
-        """Fit the kept model on the training and validation rows together.
-
-        Parameters
-        ----------
-        model : BaseModel
-            The model to fit.
-        x : DatasetDict
-            Input partitions.
-        y : DatasetDict
-            Target partitions.
-        """
-        validation_x = x.get("validation")
-        validation_y = y.get("validation")
-
-        if validation_x is None or validation_y is None or len(validation_x) == 0:
-            model.train(x["train"], y["train"])
-            return
+    Hyperparameter search is untouched. Its trials are scored on validation, so
+    they must not be fitted on it.
+    """
 
-        extend = type(model)._extend
-        model.train(extend(x["train"], validation_x), extend(y["train"], validation_y))
+    COMPATIBLE_COMPONENTS = ["ForecastingTask"]
+    SCORED_SPLITS: tuple = (SplitEnum.VALIDATION, SplitEnum.TEST)

DashAI/back/evaluation/holdout.py

@@ -156,10 +156,7 @@ def evaluate(self, model, input_dataset, output_dataset, metric):
             output_dataset["validation"], is_fit=False
         )
 
-        # Calculate metric for train and validation data each trial. The
-        # training partition is skipped for a strategy that does not score it,
-        # which for a forecaster is not a preference: predicting on dates it
-        # was fitted on is refused, so asking would fail every trial.
+        # Calculate metric for train and validation data each trial.
         if SplitEnum.TRAIN in self.SCORED_SPLITS:
             model.calculate_metrics(split=SplitEnum.TRAIN, level=LevelEnum.TRIAL)
         model.calculate_metrics(split=SplitEnum.VALIDATION, level=LevelEnum.TRIAL)