From 8edbf8af9eb70cf8f965fd4acb6977cbea040307 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 21 Nov 2025 15:41:08 +0100 Subject: [PATCH 01/72] feat: start test cases for deserializing time series data to BeliefsDataFrames Signed-off-by: F.N. Claessen --- .../schemas/tests/test_sensor_data_schema.py | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py index b1590937a8..6294e749ae 100644 --- a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py +++ b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py @@ -5,6 +5,7 @@ from marshmallow import ValidationError import pandas as pd +from timely_beliefs import BeliefsDataFrame from unittest import mock from flexmeasures.api.common.schemas.sensor_data import ( @@ -481,3 +482,70 @@ def test_build_asset_jobs_data(db, app, add_battery_assets): app.queues["forecasting"].empty() assert app.queues["scheduling"].count == 0 assert app.queues["forecasting"].count == 0 + + +@pytest.mark.parametrize( + "deserialization_input, exp_length, exp_deserialization_output", + [ + # A single 2-hour event (spanning 3 wall-clock hours) is deserialized to a BeliefsDataFrame, to be recorded on a 1-hour sensor + ( + { + "sensor": "epex_da", # name is used to look up the corresponding sensor ID + "start": "2025-11-21T14:40+01", + "duration": "PT2H", + "values": [20.5], + "unit": "EUR/kWh", + }, + 2, + {}, + ), + # Six 2-minute events (spanning 2 wall-clock hours) are deserialized to a BeliefsDataFrame, to be recorded on a 1-hour sensor + ( + { + "sensor": "epex_da", # name is used to look up the corresponding sensor ID + "start": "2025-11-21T10:50:40+01", + "duration": "PT12M", + "values": [1, 2, 3, 4, 5, 6], + "unit": "EUR/kWh", + }, + 2, + {}, + ), + ], +) +@pytest.mark.parametrize( + "requesting_user", ["test_supplier_user_4@seita.nl"], indirect=True +) +def test_time_series_deserialization( + setup_markets, + setup_roles_users, + deserialization_input, + exp_length, + exp_deserialization_output, + requesting_user, +): + """Test loading of posted sensor data.""" + schema = PostSensorDataSchema() + + # Look up sensor by name and replace it with the sensor ID + sensor = setup_markets[deserialization_input["sensor"]] + deserialization_input["sensor"] = sensor.id + + deser = schema.load(deserialization_input) + bdf = deser["bdf"] + + assert isinstance(bdf, BeliefsDataFrame) + assert bdf.event_resolution == sensor.event_resolution + assert len(bdf) == exp_length + + resolution = pd.Timedelta(deserialization_input["duration"]) / len( + deserialization_input["values"] + ) + if resolution < sensor.event_resolution: + assert bdf.event_starts[0] == pd.Timestamp( + deserialization_input["start"] + ).floor(sensor.event_resolution) + else: + assert bdf.event_starts[0] == pd.Timestamp(deserialization_input["start"]) + assert len(bdf.lineage.belief_times) == 1, "expected unique belief time" + # assert deser == exp_deserialization_output From 3b2770f94d87164d1bfd92f142ead694971b3493 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 21 Nov 2025 15:41:49 +0100 Subject: [PATCH 02/72] fix: update type annotation Signed-off-by: F.N. Claessen --- flexmeasures/api/common/schemas/sensor_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index 7f6b0f337a..7d70299daa 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -303,7 +303,7 @@ def check_multiple_instantenous_values(self, data, **kwargs): ) @post_load() - def post_load_sequence(self, data: dict, **kwargs) -> BeliefsDataFrame: + def post_load_sequence(self, data: dict, **kwargs) -> dict[str, BeliefsDataFrame]: """ If needed, upsample and convert units, then deserialize to a BeliefsDataFrame. Returns a dict with the BDF in it, as that is expected by webargs when used with as_kwargs=True. From 34a2dc50fdeca42e2088c6d035373efd362f0409 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 21 Nov 2025 15:42:25 +0100 Subject: [PATCH 03/72] feat: handle resampling after converting to a BeliefsDataFrame Signed-off-by: F.N. Claessen --- .../api/common/schemas/sensor_data.py | 29 ++++--------------- 1 file changed, 5 insertions(+), 24 deletions(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index 7d70299daa..66b8eaf94a 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -267,28 +267,6 @@ def check_schema_unit_against_type(self, data, **kwargs): f"The unit required for this message type should be convertible to an energy price unit, got incompatible unit: {posted_unit}" ) - @validates_schema - def check_resolution_compatibility_of_sensor_data(self, data, **kwargs): - """Ensure event frequency is compatible with the sensor's event resolution. - - For a sensor recording instantaneous values, any event frequency is compatible. - For a sensor recording non-instantaneous values, the event frequency must fit the sensor's event resolution. - Currently, only upsampling is supported (e.g. converting hourly events to 15-minute events). - """ - required_resolution = data["sensor"].event_resolution - - if required_resolution == timedelta(hours=0): - # For instantaneous sensors, any event frequency is compatible - return - - # The event frequency is inferred by assuming sequential, equidistant values within a time interval. - # The event resolution is assumed to be equal to the event frequency. - inferred_resolution = data["duration"] / len(data["values"]) - if inferred_resolution % required_resolution != timedelta(hours=0): - raise ValidationError( - f"Resolution of {inferred_resolution} is incompatible with the sensor's required resolution of {required_resolution}." - ) - @validates_schema def check_multiple_instantenous_values(self, data, **kwargs): """Ensure that we are not getting multiple instantaneous values that overlap. @@ -308,7 +286,6 @@ def post_load_sequence(self, data: dict, **kwargs) -> dict[str, BeliefsDataFrame If needed, upsample and convert units, then deserialize to a BeliefsDataFrame. Returns a dict with the BDF in it, as that is expected by webargs when used with as_kwargs=True. """ - data = self.possibly_upsample_values(data) data = self.possibly_convert_units(data) bdf = self.load_bdf(data) @@ -371,6 +348,7 @@ def load_bdf(sensor_data: dict) -> BeliefsDataFrame: event_resolution = sensor_data["duration"] / num_values start = sensor_data["start"] sensor = sensor_data["sensor"] + inferred_resolution = sensor_data["duration"] / len(sensor_data["values"]) if frequency := sensor.get_attribute("frequency"): start = pd.Timestamp(start).round(frequency) @@ -396,12 +374,15 @@ def load_bdf(sensor_data: dict) -> BeliefsDataFrame: belief_timing["belief_horizon"] = sensor_data["horizon"] else: belief_timing["belief_time"] = server_now() - return BeliefsDataFrame( + bdf = BeliefsDataFrame( s, source=source, sensor=sensor_data["sensor"], + event_resolution=inferred_resolution, **belief_timing, ) + resampled_bdf = bdf.resample_events(sensor.event_resolution) + return resampled_bdf class GetSensorDataSchemaEntityAddress(GetSensorDataSchema): From 8d2d2d3bad0afa96e88653551ab9409937d40164 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 21 Nov 2025 15:48:57 +0100 Subject: [PATCH 04/72] fix: catch NotImplementedError to fix failing test Signed-off-by: F.N. Claessen --- flexmeasures/api/common/schemas/sensor_data.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index 66b8eaf94a..754f219d1d 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -381,7 +381,12 @@ def load_bdf(sensor_data: dict) -> BeliefsDataFrame: event_resolution=inferred_resolution, **belief_timing, ) - resampled_bdf = bdf.resample_events(sensor.event_resolution) + try: + resampled_bdf = bdf.resample_events(sensor.event_resolution) + except NotImplementedError: + raise ValidationError( + f"Resolution of {inferred_resolution} is incompatible with the sensor's required resolution of {sensor.event_resolution}." + ) return resampled_bdf From 86ad7745107624e5a6e5e63ccd63e301b438d22c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 21 Nov 2025 16:21:41 +0100 Subject: [PATCH 05/72] feat: two more test cases Signed-off-by: F.N. Claessen --- .../schemas/tests/test_sensor_data_schema.py | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py index 6294e749ae..f31e9c9072 100644 --- a/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py +++ b/flexmeasures/api/common/schemas/tests/test_sensor_data_schema.py @@ -487,7 +487,7 @@ def test_build_asset_jobs_data(db, app, add_battery_assets): @pytest.mark.parametrize( "deserialization_input, exp_length, exp_deserialization_output", [ - # A single 2-hour event (spanning 3 wall-clock hours) is deserialized to a BeliefsDataFrame, to be recorded on a 1-hour sensor + # # A single 2-hour event (spanning 3 wall-clock hours) is deserialized to a BeliefsDataFrame, to be recorded on a 1-hour sensor ( { "sensor": "epex_da", # name is used to look up the corresponding sensor ID @@ -499,6 +499,30 @@ def test_build_asset_jobs_data(db, app, add_battery_assets): 2, {}, ), + # One 5-minute event (spanning 2 wall-clock hours) is deserialized to a BeliefsDataFrame, to be recorded on a 1-hour sensor + ( + { + "sensor": "epex_da", # name is used to look up the corresponding sensor ID + "start": "2025-11-21T10:58:40+01", + "duration": "PT5M", + "values": [1], + "unit": "EUR/kWh", + }, + 2, + {}, + ), + # One 2-minute event (spanning 1 wall-clock hour) is deserialized to a BeliefsDataFrame, to be recorded on a 1-hour sensor + ( + { + "sensor": "epex_da", # name is used to look up the corresponding sensor ID + "start": "2025-11-21T10:58:00+01", + "duration": "PT2M", + "values": [1], + "unit": "EUR/kWh", + }, + 1, + {}, + ), # Six 2-minute events (spanning 2 wall-clock hours) are deserialized to a BeliefsDataFrame, to be recorded on a 1-hour sensor ( { From 30fb2ad412da6edc4ea7687cfac837a3666635f5 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 21 Nov 2025 17:01:35 +0100 Subject: [PATCH 06/72] fix: update test expectations Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/test_sensor_data.py | 6 ------ flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py | 6 +++--- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data.py b/flexmeasures/api/v3_0/tests/test_sensor_data.py index c9c1822c7c..49bc4ef574 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data.py @@ -145,12 +145,6 @@ def test_post_sensor_data_bad_auth( "request_field, new_value, error_field, error_text", [ ("start", "2021-06-07T00:00:00", "start", "Not a valid aware datetime"), - ( - "duration", - "PT30M", - "_schema", - "Resolution of 0:05:00 is incompatible", - ), # downsampling not supported ("unit", "m", "_schema", "Required unit"), ("type", "GetSensorDataRequest", "type", "Must be one of"), ], diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py b/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py index 3912bc0cca..aafa84b94d 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data_fresh_db.py @@ -31,12 +31,12 @@ ), # upsample from single value for 1-hour interval, sent as float rather than list of floats ( 4, - 0, + 6, "m³/h", False, - None, + -11.28, False, - 422, + 200, ), # failed to resample from 15-min intervals to 10-min intervals ( 10, From 6c08284715af42b8edd63aa15108c9daf3fd64de Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sat, 2 May 2026 13:29:35 +0100 Subject: [PATCH 07/72] fix: floor direct POST sensor data starts Signed-off-by: Mohamed Belhsan Hmida --- .../api/common/schemas/sensor_data.py | 6 ++- .../api/v3_0/tests/test_sensor_data.py | 38 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index 644d51412f..889a542e2d 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -446,7 +446,11 @@ def load_bdf(sensor_data: dict) -> BeliefsDataFrame: sensor = sensor_data["sensor"] inferred_resolution = sensor_data["duration"] / len(sensor_data["values"]) - if frequency := sensor.get_attribute("frequency"): + if sensor.event_resolution != timedelta(0) and sensor.get_attribute( + "round_datetimes_on_ingestion", True + ): + start = pd.Timestamp(start).floor(sensor.event_resolution) + elif frequency := sensor.get_attribute("frequency"): start = pd.Timestamp(start).round(frequency) if event_resolution == timedelta(hours=0): diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data.py b/flexmeasures/api/v3_0/tests/test_sensor_data.py index ae0a698294..a9e705b538 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data.py @@ -2,6 +2,7 @@ from datetime import timedelta from flask import url_for +import pandas as pd import pytest from sqlalchemy import event from sqlalchemy.engine import Engine @@ -233,6 +234,43 @@ def test_post_invalid_sensor_data( ) +@pytest.mark.parametrize( + "requesting_user", ["test_supplier_user_4@seita.nl"], indirect=True +) +def test_post_non_instantaneous_sensor_data_floor( + client, setup_api_test_data, requesting_user +): + post_data = make_sensor_data_request_for_gas_sensor(unit="m³/h") + post_data["start"] = "2021-06-08T00:00:40+02:00" + sensor = setup_api_test_data["some gas sensor"] + + rows = len(sensor.search_beliefs()) + + response = client.post( + url_for("SensorAPI:post_data", id=sensor.id), + json=post_data, + ) + + assert response.status_code == 200 + + data = sensor.search_beliefs().reset_index() + new_data = data[ + data["event_start"].between( + pd.Timestamp("2021-06-07 22:00:00+0000", tz="UTC"), + pd.Timestamp("2021-06-07 22:50:00+0000", tz="UTC"), + ) + ] + assert len(sensor.search_beliefs()) - rows == 6 + assert list(new_data["event_start"]) == [ + pd.Timestamp("2021-06-07 22:00:00+0000", tz="UTC"), + pd.Timestamp("2021-06-07 22:10:00+0000", tz="UTC"), + pd.Timestamp("2021-06-07 22:20:00+0000", tz="UTC"), + pd.Timestamp("2021-06-07 22:30:00+0000", tz="UTC"), + pd.Timestamp("2021-06-07 22:40:00+0000", tz="UTC"), + pd.Timestamp("2021-06-07 22:50:00+0000", tz="UTC"), + ] + + @pytest.mark.parametrize( "requesting_user", ["test_supplier_user_4@seita.nl"], indirect=True ) From f590e583b53778a38e263d863068ea049fde23d1 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sat, 2 May 2026 13:30:20 +0100 Subject: [PATCH 08/72] fix: floor uploaded sensor data datetimes Signed-off-by: Mohamed Belhsan Hmida --- .../v3_0/tests/test_sensors_api_freshdb.py | 79 +++++++++++++++++++ flexmeasures/data/schemas/sensors.py | 35 ++++++++ 2 files changed, 114 insertions(+) diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py b/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py index f0c7e12e0c..749ca8d503 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py @@ -1,6 +1,7 @@ import io import pytest from datetime import timedelta +import pandas as pd from flask import url_for from sqlalchemy import select @@ -232,6 +233,84 @@ def test_upload_sensor_data_with_unit_conversion_success( assert [b.event_value for b in beliefs] == expected_event_values +@pytest.mark.parametrize( + "requesting_user, sensor_index, start_date, data_resolution, data_values, expected_event_starts, expected_event_values", + [ + ( + "test_prosumer_user_2@seita.nl", + 1, + "2025-01-01T10:00:40+00:00", + timedelta(minutes=15), + [2, 3, 4], + [ + pd.Timestamp("2025-01-01T10:00:00+00:00"), + pd.Timestamp("2025-01-01T10:15:00+00:00"), + pd.Timestamp("2025-01-01T10:30:00+00:00"), + ], + [2, 3, 4], + ), + ( + "test_prosumer_user_2@seita.nl", + 2, + "2025-01-01T10:00:40+00:00", + timedelta(minutes=30), + [10, 20, 20, 40], + [ + pd.Timestamp("2025-01-01T10:00:00+00:00"), + pd.Timestamp("2025-01-01T11:00:00+00:00"), + ], + [30, 60], + ), + ], + indirect=["requesting_user"], +) +def test_upload_sensor_data_floors_offclock_datetimes( + fresh_db, + client, + add_battery_assets_fresh_db, + requesting_user, + sensor_index, + start_date, + data_resolution, + data_values, + expected_event_starts, + expected_event_values, +): + test_battery = add_battery_assets_fresh_db["Test battery"] + sensor = test_battery.sensors[sensor_index] + + csv_content = generate_csv_content( + start_time_str=start_date, + interval=data_resolution, + values=data_values, + ) + file_obj = io.BytesIO(csv_content.encode("utf-8")) + + response = client.post( + url_for("SensorAPI:upload_data", id=sensor.id), + data={"uploaded-files": (file_obj, "data.csv"), "unit": sensor.unit}, + content_type="multipart/form-data", + ) + assert response.status_code == 200 + + beliefs = ( + fresh_db.session.execute( + select(TimedBelief) + .filter(TimedBelief.sensor_id == sensor.id) + .filter(TimedBelief.event_start >= expected_event_starts[0]) + .filter( + TimedBelief.event_start + < expected_event_starts[-1] + sensor.event_resolution + ) + .order_by(TimedBelief.event_start) + ) + .scalars() + .all() + ) + assert [b.event_start for b in beliefs] == expected_event_starts + assert [b.event_value for b in beliefs] == expected_event_values + + @pytest.mark.parametrize( "requesting_user, sensor_index, data_unit, data_resolution, data_values, expected_err_msg, expected_status", [ diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index af73d93ae1..8775531371 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -67,6 +67,7 @@ class TimedEventSchema(Schema): def __init__( self, timezone: str | None = None, + event_resolution: timedelta | None = None, value_validator: Validator | None = None, to_unit: str | None = None, default_src_unit: str | None = None, @@ -79,6 +80,7 @@ def __init__( :param timezone: Optionally, set a timezone to be able to interpret nominal durations. """ self.timezone = timezone + self.event_resolution = event_resolution self.value_validator = value_validator super().__init__(*args, **kwargs) if to_unit is not None: @@ -97,12 +99,23 @@ def validate_value(self, _value, **kwargs): if self.value_validator is not None: self.value_validator(_value) + def floor_timing_fields(self, data: dict) -> None: + if self.event_resolution in (None, timedelta(0)): + return + + for key in ("datetime", "start", "end"): + if data.get(key) is not None: + data[key] = ( + pd.Timestamp(data[key]).floor(self.event_resolution).to_pydatetime() + ) + @validates_schema def check_time_window(self, data, **kwargs): """Checks whether a complete time interval can be derived from the timing fields. The data is updated in-place, guaranteeing that the 'start' and 'end' fields are filled out. """ + self.floor_timing_fields(data) dt = data.get("datetime") start = data.get("start") end = data.get("end") @@ -336,6 +349,7 @@ def __init__( default_src_unit: str | None = None, return_magnitude: bool = False, timezone: str | None = None, + event_resolution: timedelta | None = None, value_validator: Validator | None = None, additional_sensor_units: list[str] | None = None, **kwargs, @@ -380,6 +394,7 @@ def __init__( value_validator = RepurposeValidatorToIgnoreSensorsAndLists(value_validator) self.validators.insert(0, value_validator) self.timezone = timezone + self.event_resolution = event_resolution self.value_validator = value_validator if to_unit.startswith("/") and len(to_unit) < 2: raise ValueError( @@ -443,6 +458,7 @@ def _deserialize_list(self, value: list[dict]) -> list[dict]: fields.Nested( TimedEventSchema( timezone=self.timezone, + event_resolution=self.event_resolution, value_validator=self.value_validator, to_unit=self.to_unit, default_src_unit=self.default_src_unit, @@ -709,6 +725,11 @@ def post_load(self, fields, **kwargs): # Reraise the error if an event frequency could not be inferred pd.infer_freq(bdf.index.unique("event_start")) + if sensor.event_resolution != timedelta(0) and sensor.get_attribute( + "round_datetimes_on_ingestion", True + ): + bdf = floor_bdf_event_starts(bdf, bdf.event_resolution) + bdf["event_value"] = convert_units( bdf["event_value"], from_unit, @@ -747,6 +768,20 @@ def post_load(self, fields, **kwargs): return fields +def floor_bdf_event_starts( + bdf: tb.BeliefsDataFrame, event_resolution: timedelta +) -> tb.BeliefsDataFrame: + floored_bdf = pd.DataFrame(bdf.reset_index()) + floored_bdf["event_start"] = pd.DatetimeIndex(bdf.event_starts).floor( + event_resolution + ) + return tb.BeliefsDataFrame( + floored_bdf, + sensor=bdf.sensor, + event_resolution=bdf.event_resolution, + ) + + class QuantitySchema(Schema): """Represents a quantity string like '1 EUR/MWh'.""" From 0566ceb4d3107c56b00a3db56f8b5e9be13e606a Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sat, 2 May 2026 13:35:51 +0100 Subject: [PATCH 09/72] fix: floor flex-model scheduling datetimes Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/api/v3_0/sensors.py | 26 +++++++++++++ .../api/v3_0/tests/test_sensor_schedules.py | 38 +++++++++++++++++++ .../data/schemas/scheduling/storage.py | 9 +++++ 3 files changed, 73 insertions(+) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 3d0e407453..23ab7e2e79 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -1,7 +1,9 @@ from __future__ import annotations import isodate +from copy import deepcopy from datetime import datetime, timedelta +import pandas as pd from flexmeasures.data.services.sensors import ( serialize_sensor_status_data, @@ -221,6 +223,27 @@ class TriggerScheduleKwargsSchema(Schema): ) +def floor_timed_event_datetimes(flex_model: dict, sensor: Sensor) -> dict: + if sensor.event_resolution == timedelta(0) or not sensor.get_attribute( + "round_datetimes_on_ingestion", True + ): + return flex_model + + floored_flex_model = deepcopy(flex_model) + for value in floored_flex_model.values(): + if not isinstance(value, list): + continue + for timed_event in value: + if not isinstance(timed_event, dict): + continue + for key in ("datetime", "start", "end"): + if key in timed_event: + timed_event[key] = isodate.datetime_isoformat( + pd.Timestamp(timed_event[key]).floor(sensor.event_resolution) + ) + return floored_flex_model + + class SensorAPI(FlaskView): route_base = "/sensors" trailing_slash = False @@ -886,6 +909,9 @@ def trigger_schedule( f"Resolution of {resolution} is incompatible with the sensor's required resolution of {sensor.event_resolution}." ) + if flex_model is not None: + flex_model = floor_timed_event_datetimes(flex_model, sensor) + end_of_schedule = start_of_schedule + duration scheduler_kwargs = dict( asset_or_sensor=sensor, diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index 4490d5d996..a77a2959ec 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -98,6 +98,44 @@ def test_trigger_schedule_with_invalid_flexmodel( ) +@pytest.mark.parametrize( + "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True +) +def test_trigger_schedule_floors_flex_model_datetimes( + app, + add_market_prices, + add_battery_assets, + battery_soc_sensor, + add_charging_station_assets, + keep_scheduling_queue_empty, + requesting_user, +): + message = message_for_trigger_schedule(with_targets=True) + offclock_target = "2015-01-02T23:00:40+01:00" + expected_target = "2015-01-02T23:00:00+01:00" + for field in ("soc-targets", "soc-minima", "soc-maxima"): + message["flex-model"][field][0]["datetime"] = offclock_target + + sensor = add_charging_station_assets["Test charging station"].sensors[0] + with app.test_client() as client: + trigger_schedule_response = client.post( + url_for("SensorAPI:trigger_schedule", id=sensor.id), + json=message, + ) + + assert trigger_schedule_response.status_code == 200 + assert len(app.queues["scheduling"]) == 1 + + job = app.queues["scheduling"].jobs[0] + for field in ("soc-targets", "soc-minima", "soc-maxima"): + target = job.kwargs["flex_model"][field][0] + assert target["datetime"] == expected_target + if "start" in target: + assert parse_datetime(target["start"]) == parse_datetime(expected_target) + if "end" in target: + assert parse_datetime(target["end"]) == parse_datetime(expected_target) + + @pytest.mark.parametrize("message", [message_for_trigger_schedule(unknown_prices=True)]) @pytest.mark.parametrize( "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index 807774fe5a..e5b2bf9ad1 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -236,6 +236,12 @@ def __init__( self.start = start self.sensor = sensor self.timezone = sensor.timezone if sensor is not None else None + self.rounding_resolution = ( + sensor.event_resolution + if sensor is not None + and sensor.get_attribute("round_datetimes_on_ingestion", True) + else None + ) # guess default soc-unit if default_soc_unit is None: @@ -250,6 +256,7 @@ def __init__( to_unit="MWh", default_src_unit=default_soc_unit, timezone=self.timezone, + event_resolution=self.rounding_resolution, data_key="soc-maxima", ) @@ -257,6 +264,7 @@ def __init__( to_unit="MWh", default_src_unit=default_soc_unit, timezone=self.timezone, + event_resolution=self.rounding_resolution, data_key="soc-minima", value_validator=validate.Range(min=0), ) @@ -264,6 +272,7 @@ def __init__( to_unit="MWh", default_src_unit=default_soc_unit, timezone=self.timezone, + event_resolution=self.rounding_resolution, data_key="soc-targets", ) From a857806055d66ca94c8d28c3aa3581db7aee0600 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida <149331360+BelhsanHmida@users.noreply.github.com> Date: Sat, 9 May 2026 14:04:15 +0100 Subject: [PATCH 10/72] fix: preserve 422 validation for invalid flex-model datetimes Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> Signed-off-by: Mohamed Belhsan Hmida <149331360+BelhsanHmida@users.noreply.github.com> --- flexmeasures/api/v3_0/sensors.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index dc37fffb35..64c4831f83 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -233,17 +233,30 @@ def floor_timed_event_datetimes(flex_model: dict, sensor: Sensor) -> dict: return flex_model floored_flex_model = deepcopy(flex_model) - for value in floored_flex_model.values(): + for value_name, value in floored_flex_model.items(): if not isinstance(value, list): continue - for timed_event in value: + for index, timed_event in enumerate(value): if not isinstance(timed_event, dict): continue for key in ("datetime", "start", "end"): if key in timed_event: - timed_event[key] = isodate.datetime_isoformat( - pd.Timestamp(timed_event[key]).floor(sensor.event_resolution) - ) + try: + timed_event[key] = isodate.datetime_isoformat( + pd.Timestamp(timed_event[key]).floor(sensor.event_resolution) + ) + except (TypeError, ValueError) as exc: + raise ValidationError( + { + value_name: { + index: { + key: [ + f"Not a valid datetime: {timed_event[key]!r}." + ] + } + } + } + ) from exc return floored_flex_model From f30429cc560ad2e6de59c142335924c57dfb0b4e Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sat, 9 May 2026 14:13:00 +0100 Subject: [PATCH 11/72] style: apply pre-commit Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/api/v3_0/sensors.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 64c4831f83..c678c14e66 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -243,7 +243,9 @@ def floor_timed_event_datetimes(flex_model: dict, sensor: Sensor) -> dict: if key in timed_event: try: timed_event[key] = isodate.datetime_isoformat( - pd.Timestamp(timed_event[key]).floor(sensor.event_resolution) + pd.Timestamp(timed_event[key]).floor( + sensor.event_resolution + ) ) except (TypeError, ValueError) as exc: raise ValidationError( From 3a56e0c40d03a6c1cf8715f8fd88cf36f7516ea8 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 9 May 2026 13:25:02 +0000 Subject: [PATCH 12/72] fix: return 422 for invalid flex-model timed-event datetimes Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/be65737b-e479-49f5-bd8b-5c62a1b8b340 Co-authored-by: BelhsanHmida <149331360+BelhsanHmida@users.noreply.github.com> --- flexmeasures/api/v3_0/sensors.py | 5 +++- .../api/v3_0/tests/test_sensor_schedules.py | 29 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index c678c14e66..dc258ac201 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -940,7 +940,10 @@ def trigger_schedule( ) if flex_model is not None: - flex_model = floor_timed_event_datetimes(flex_model, sensor) + try: + flex_model = floor_timed_event_datetimes(flex_model, sensor) + except ValidationError as err: + return unprocessable_entity(err.messages) end_of_schedule = start_of_schedule + duration scheduler_kwargs = dict( diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index de3deb7ef7..7497b24971 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -152,6 +152,35 @@ def test_trigger_schedule_floors_flex_model_datetimes( assert parse_datetime(target["end"]) == parse_datetime(expected_target) +@pytest.mark.parametrize( + "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True +) +def test_trigger_schedule_invalid_flex_model_datetime_returns_422( + app, + add_battery_assets, + keep_scheduling_queue_empty, + requesting_user, +): + message = message_for_trigger_schedule(with_targets=True) + message["flex-model"]["soc-minima"][0]["datetime"] = "not-a-datetime" + + sensor = add_battery_assets["Test battery"].sensors[0] + with app.test_client() as client: + trigger_schedule_response = client.post( + url_for("SensorAPI:trigger_schedule", id=sensor.id), + json=message, + ) + + assert trigger_schedule_response.status_code == 422 + assert "soc-minima" in trigger_schedule_response.json["message"]["json"] + assert ( + "Not a valid datetime" + in trigger_schedule_response.json["message"]["json"]["soc-minima"]["0"][ + "datetime" + ][0] + ) + + @pytest.mark.parametrize( "flex_config, field", [ From dd2fcc4ec813863a3986693fe5e784198fb451ad Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 9 May 2026 13:26:42 +0000 Subject: [PATCH 13/72] test: refine invalid flex-model datetime regression assertion Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/be65737b-e479-49f5-bd8b-5c62a1b8b340 Co-authored-by: BelhsanHmida <149331360+BelhsanHmida@users.noreply.github.com> --- flexmeasures/api/v3_0/tests/test_sensor_schedules.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index 7497b24971..92e53b3ac1 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -155,7 +155,7 @@ def test_trigger_schedule_floors_flex_model_datetimes( @pytest.mark.parametrize( "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True ) -def test_trigger_schedule_invalid_flex_model_datetime_returns_422( +def test_trigger_schedule_with_invalid_flex_model_datetime( app, add_battery_assets, keep_scheduling_queue_empty, @@ -173,12 +173,10 @@ def test_trigger_schedule_invalid_flex_model_datetime_returns_422( assert trigger_schedule_response.status_code == 422 assert "soc-minima" in trigger_schedule_response.json["message"]["json"] - assert ( - "Not a valid datetime" - in trigger_schedule_response.json["message"]["json"]["soc-minima"]["0"][ - "datetime" - ][0] - ) + datetime_error = trigger_schedule_response.json["message"]["json"]["soc-minima"][ + "0" + ]["datetime"][0] + assert "Not a valid datetime" in datetime_error @pytest.mark.parametrize( From aa93fe11f4b65ea95843d3e1609fe62a4056e1b9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 9 May 2026 13:28:05 +0000 Subject: [PATCH 14/72] test: make flex-model datetime error assertion less brittle Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/be65737b-e479-49f5-bd8b-5c62a1b8b340 Co-authored-by: BelhsanHmida <149331360+BelhsanHmida@users.noreply.github.com> --- flexmeasures/api/v3_0/tests/test_sensor_schedules.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index 92e53b3ac1..0dfb656b9c 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -172,11 +172,12 @@ def test_trigger_schedule_with_invalid_flex_model_datetime( ) assert trigger_schedule_response.status_code == 422 - assert "soc-minima" in trigger_schedule_response.json["message"]["json"] - datetime_error = trigger_schedule_response.json["message"]["json"]["soc-minima"][ - "0" - ]["datetime"][0] - assert "Not a valid datetime" in datetime_error + errors = trigger_schedule_response.json["message"]["json"] + minima_errors = errors.get("soc-minima", {}) + timed_event_errors = minima_errors.get("0", {}) + datetime_errors = timed_event_errors.get("datetime", []) + assert datetime_errors + assert "Not a valid datetime" in datetime_errors[0] @pytest.mark.parametrize( From e0f88a81045b53eb1ac1a7077e4f876c6879cf20 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 9 May 2026 13:29:27 +0000 Subject: [PATCH 15/72] test: assert error payload shape for invalid flex-model datetime Agent-Logs-Url: https://github.com/FlexMeasures/flexmeasures/sessions/be65737b-e479-49f5-bd8b-5c62a1b8b340 Co-authored-by: BelhsanHmida <149331360+BelhsanHmida@users.noreply.github.com> --- flexmeasures/api/v3_0/tests/test_sensor_schedules.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index 0dfb656b9c..6aad9bd4cf 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -172,9 +172,14 @@ def test_trigger_schedule_with_invalid_flex_model_datetime( ) assert trigger_schedule_response.status_code == 422 + assert "message" in trigger_schedule_response.json + assert "json" in trigger_schedule_response.json["message"] errors = trigger_schedule_response.json["message"]["json"] + assert "soc-minima" in errors minima_errors = errors.get("soc-minima", {}) + assert "0" in minima_errors timed_event_errors = minima_errors.get("0", {}) + assert "datetime" in timed_event_errors datetime_errors = timed_event_errors.get("datetime", []) assert datetime_errors assert "Not a valid datetime" in datetime_errors[0] From aaebfaa1bac6f4e3807381705779e335adabd240 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 12 May 2026 08:58:29 +0200 Subject: [PATCH 16/72] refactor: simplify test, incl. by not mixing timezone offsets Signed-off-by: F.N. Claessen --- .../api/v3_0/tests/test_sensor_data.py | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data.py b/flexmeasures/api/v3_0/tests/test_sensor_data.py index 92b23fdde9..4ec1e9834f 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data.py @@ -246,11 +246,16 @@ def test_post_invalid_sensor_data( def test_post_non_instantaneous_sensor_data_floor( client, setup_api_test_data, requesting_user ): + imprecise_start = "2021-06-08T00:00:40+02:00" + precise_start = "2021-06-08T00:00:00+02:00" + precise_end = "2021-06-08T01:00:00+02:00" post_data = make_sensor_data_request_for_gas_sensor(unit="m³/h") - post_data["start"] = "2021-06-08T00:00:40+02:00" + post_data["start"] = imprecise_start sensor = setup_api_test_data["some gas sensor"] - rows = len(sensor.search_beliefs()) + assert ( + len(sensor.search_beliefs(precise_start, precise_end)) == 0 + ), "No beliefs were expected before we post our test data." response = client.post( url_for("SensorAPI:post_data", id=sensor.id), @@ -259,21 +264,15 @@ def test_post_non_instantaneous_sensor_data_floor( assert response.status_code == 200 - data = sensor.search_beliefs().reset_index() - new_data = data[ - data["event_start"].between( - pd.Timestamp("2021-06-07 22:00:00+0000", tz="UTC"), - pd.Timestamp("2021-06-07 22:50:00+0000", tz="UTC"), - ) - ] - assert len(sensor.search_beliefs()) - rows == 6 + new_data = sensor.search_beliefs(precise_start, precise_end).reset_index() + assert len(new_data) == 6 assert list(new_data["event_start"]) == [ - pd.Timestamp("2021-06-07 22:00:00+0000", tz="UTC"), - pd.Timestamp("2021-06-07 22:10:00+0000", tz="UTC"), - pd.Timestamp("2021-06-07 22:20:00+0000", tz="UTC"), - pd.Timestamp("2021-06-07 22:30:00+0000", tz="UTC"), - pd.Timestamp("2021-06-07 22:40:00+0000", tz="UTC"), - pd.Timestamp("2021-06-07 22:50:00+0000", tz="UTC"), + pd.Timestamp("2021-06-08 00:00:00+02"), + pd.Timestamp("2021-06-08 00:10:00+02"), + pd.Timestamp("2021-06-08 00:20:00+02"), + pd.Timestamp("2021-06-08 00:30:00+02"), + pd.Timestamp("2021-06-08 00:40:00+02"), + pd.Timestamp("2021-06-08 00:50:00+02"), ] From 43fe0b5b95ea1bb1785b72bc2f55c063cf2e922c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 12 May 2026 09:06:12 +0200 Subject: [PATCH 17/72] refactor: rename variable to match variable name in test_trigger_schedule_floors_flex_model_datetimes Signed-off-by: F.N. Claessen --- flexmeasures/api/v3_0/tests/test_sensor_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data.py b/flexmeasures/api/v3_0/tests/test_sensor_data.py index 4ec1e9834f..880103de27 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data.py @@ -246,11 +246,11 @@ def test_post_invalid_sensor_data( def test_post_non_instantaneous_sensor_data_floor( client, setup_api_test_data, requesting_user ): - imprecise_start = "2021-06-08T00:00:40+02:00" + offclock_start = "2021-06-08T00:00:40+02:00" precise_start = "2021-06-08T00:00:00+02:00" precise_end = "2021-06-08T01:00:00+02:00" post_data = make_sensor_data_request_for_gas_sensor(unit="m³/h") - post_data["start"] = imprecise_start + post_data["start"] = offclock_start sensor = setup_api_test_data["some gas sensor"] assert ( From 9624f09f6bc69353ca2ad49bd98b68d34de2f58d Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 12 May 2026 09:24:29 +0200 Subject: [PATCH 18/72] refactor: use search method over writing custom query Signed-off-by: F.N. Claessen --- .../v3_0/tests/test_sensors_api_freshdb.py | 21 ++++++------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py b/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py index 749ca8d503..0d85011457 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py @@ -293,22 +293,13 @@ def test_upload_sensor_data_floors_offclock_datetimes( ) assert response.status_code == 200 - beliefs = ( - fresh_db.session.execute( - select(TimedBelief) - .filter(TimedBelief.sensor_id == sensor.id) - .filter(TimedBelief.event_start >= expected_event_starts[0]) - .filter( - TimedBelief.event_start - < expected_event_starts[-1] + sensor.event_resolution - ) - .order_by(TimedBelief.event_start) - ) - .scalars() - .all() + bdf = sensor.search_beliefs( + expected_event_starts[0], expected_event_starts[-1] + sensor.event_resolution + ) + pd.testing.assert_index_equal( + bdf.event_starts, pd.DatetimeIndex(expected_event_starts, name="event_start") ) - assert [b.event_start for b in beliefs] == expected_event_starts - assert [b.event_value for b in beliefs] == expected_event_values + assert bdf["event_value"].to_list() == expected_event_values @pytest.mark.parametrize( From 860c700391075af1e564b287f4fa7d243e9dd66a Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Tue, 12 May 2026 09:33:18 +0200 Subject: [PATCH 19/72] feat: test resampling from 5 to 15 minutes Signed-off-by: F.N. Claessen --- .../api/v3_0/tests/test_sensors_api_freshdb.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py b/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py index 0d85011457..c5370c4d3d 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py @@ -261,6 +261,18 @@ def test_upload_sensor_data_with_unit_conversion_success( ], [30, 60], ), + ( + "test_prosumer_user_2@seita.nl", + 1, + "2025-01-01T10:00:40+00:00", + timedelta(minutes=5), + [10, 20, 30, 40], + [ + pd.Timestamp("2025-01-01T10:00:00+00:00"), + pd.Timestamp("2025-01-01T10:15:00+00:00"), + ], + [20, 40], + ), ], indirect=["requesting_user"], ) From 222841089bf0e5e440f1d1b643d7d7d3d95dc1b0 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 15 May 2026 16:49:57 +0100 Subject: [PATCH 20/72] docs: document flex-model datetime flooring helper Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/api/v3_0/sensors.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index dc258ac201..df5aea3059 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -227,6 +227,11 @@ class TriggerScheduleKwargsSchema(Schema): def floor_timed_event_datetimes(flex_model: dict, sensor: Sensor) -> dict: + """Floor timed-event datetimes in list-valued flex-model fields. + + This only touches list entries that look like timed-event dictionaries, + such as ``soc-minima``, ``soc-maxima`` and ``soc-targets``. + """ if sensor.event_resolution == timedelta(0) or not sensor.get_attribute( "round_datetimes_on_ingestion", True ): From 0d92e52e2d3ab941fb3aab8b541b6d1177447437 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 15 May 2026 16:50:17 +0100 Subject: [PATCH 21/72] test: clarify sensor_index assumptions in upload tests Signed-off-by: Mohamed Belhsan Hmida --- .../api/v3_0/tests/test_sensors_api_freshdb.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py b/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py index c5370c4d3d..01cb7e95c6 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py @@ -11,6 +11,20 @@ from flexmeasures.api.v3_0.tests.utils import generate_csv_content +TEST_BATTERY_SENSOR_NAMES = ( + "power", + "power (kW)", + "energy (kWh)", + "state of charge", + "consumption sensor", + "cost sensor", +) + + +def assert_test_battery_sensor(sensor, sensor_index: int) -> None: + assert sensor.name == TEST_BATTERY_SENSOR_NAMES[sensor_index] + + @pytest.mark.parametrize( "requesting_user, sensor_index, data_unit, data_resolution, data_values, expected_event_values, expected_status", [ @@ -187,6 +201,7 @@ def test_upload_sensor_data_with_unit_conversion_success( ) test_battery = add_battery_assets_fresh_db["Test battery"] sensor = test_battery.sensors[sensor_index] + assert_test_battery_sensor(sensor, sensor_index) num_test_intervals = len(data_values) print( f"Uploading data to sensor '{sensor.name}' with unit={sensor.unit} and resolution={sensor.event_resolution}." @@ -290,6 +305,7 @@ def test_upload_sensor_data_floors_offclock_datetimes( ): test_battery = add_battery_assets_fresh_db["Test battery"] sensor = test_battery.sensors[sensor_index] + assert_test_battery_sensor(sensor, sensor_index) csv_content = generate_csv_content( start_time_str=start_date, @@ -371,6 +387,7 @@ def test_upload_sensor_data_with_unit_conversion_failure( ) test_battery = add_battery_assets_fresh_db["Test battery"] sensor = test_battery.sensors[sensor_index] + assert_test_battery_sensor(sensor, sensor_index) print( f"Uploading data to sensor '{sensor.name}' with unit={sensor.unit} and resolution={sensor.event_resolution}." ) From b7d3cf0251851202eef73baa01a309f7c157cf3f Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sun, 17 May 2026 02:00:23 +0100 Subject: [PATCH 22/72] feat: handle floored downsampling posts Signed-off-by: Mohamed Belhsan Hmida --- .../api/common/schemas/sensor_data.py | 27 +++++++-- .../api/v3_0/tests/test_sensor_data.py | 55 ++++++++++++++----- 2 files changed, 64 insertions(+), 18 deletions(-) diff --git a/flexmeasures/api/common/schemas/sensor_data.py b/flexmeasures/api/common/schemas/sensor_data.py index 3701b0b198..b78e895682 100644 --- a/flexmeasures/api/common/schemas/sensor_data.py +++ b/flexmeasures/api/common/schemas/sensor_data.py @@ -369,7 +369,8 @@ def check_resolution_compatibility_of_sensor_data(self, data, **kwargs): For a sensor recording instantaneous values, any event frequency is compatible. For a sensor recording non-instantaneous values, the event frequency must fit the sensor's event resolution. - Currently, only upsampling is supported (e.g. converting hourly events to 15-minute events). + Upsampling and downsampling are supported when the inferred resolution and + the sensor resolution are multiples of each other. """ required_resolution = data["sensor"].event_resolution @@ -380,7 +381,9 @@ def check_resolution_compatibility_of_sensor_data(self, data, **kwargs): # The event frequency is inferred by assuming sequential, equidistant values within a time interval. # The event resolution is assumed to be equal to the event frequency. inferred_resolution = data["duration"] / len(data["values"]) - if inferred_resolution % required_resolution != timedelta(hours=0): + if inferred_resolution % required_resolution != timedelta( + hours=0 + ) and required_resolution % inferred_resolution != timedelta(hours=0): raise ValidationError( f"Resolution of {inferred_resolution} is incompatible with the sensor's required resolution of {required_resolution}." ) @@ -407,6 +410,7 @@ def post_load_sequence(self, data: dict, **kwargs) -> dict[str, BeliefsDataFrame data = self.possibly_upsample_values(data) data = self.possibly_convert_units(data) bdf = self.load_bdf(data) + bdf = self.possibly_downsample_bdf(bdf, data["sensor"].event_resolution) # Post-load validation against message type _type = data.get("type", None) @@ -429,7 +433,7 @@ def possibly_convert_units(data): data["values"], from_unit=data["unit"], to_unit=data["sensor"].unit, - event_resolution=data["sensor"].event_resolution, + event_resolution=data["duration"] / len(data["values"]), ) return data @@ -457,6 +461,21 @@ def possibly_upsample_values(data): ) return data + @staticmethod + def possibly_downsample_bdf( + bdf: BeliefsDataFrame, required_resolution: timedelta + ) -> BeliefsDataFrame: + """ + Downsample the data if needed, to fit the sensor's resolution. + Marshmallow runs this after validation. + """ + if required_resolution == timedelta(hours=0): + return bdf + + if bdf.event_resolution < required_resolution: + bdf = bdf.resample_events(required_resolution) + return bdf + @staticmethod def load_bdf(sensor_data: dict) -> BeliefsDataFrame: """ @@ -469,7 +488,7 @@ def load_bdf(sensor_data: dict) -> BeliefsDataFrame: sensor = sensor_data["sensor"] if sensor.event_resolution != timedelta(0) and sensor.get_attribute( - "round_datetimes_on_ingestion", True + "floor_datetimes_to_resolution", True ): start = pd.Timestamp(start).floor(sensor.event_resolution) elif frequency := sensor.get_attribute("frequency"): diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data.py b/flexmeasures/api/v3_0/tests/test_sensor_data.py index 880103de27..271ecfb830 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data.py @@ -200,10 +200,10 @@ def test_post_sensor_data_bad_auth( ("start", "2021-06-07T00:00:00", "start", "Not a valid aware datetime"), ( "duration", - "PT30M", + "PT25M", "_schema", - "Resolution of 0:05:00 is incompatible", - ), # downsampling not supported + "Resolution of 0:04:10 is incompatible", + ), ("unit", "m", "_schema", "Required unit"), ("type", "GetSensorDataRequest", "type", "Must be one of"), ], @@ -243,14 +243,40 @@ def test_post_invalid_sensor_data( @pytest.mark.parametrize( "requesting_user", ["test_supplier_user_4@seita.nl"], indirect=True ) +@pytest.mark.parametrize( + "offclock_start, precise_start, precise_end, values, expected_values", + [ + ( + "2021-06-08T00:00:40+02:00", + "2021-06-08T00:00:00+02:00", + "2021-06-08T01:00:00+02:00", + [-11.28] * 6, + [-11.28] * 6, + ), + ( + "2021-06-09T00:00:40+02:00", + "2021-06-09T00:00:00+02:00", + "2021-06-09T01:00:00+02:00", + [100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 1100, 1200], + [150, 350, 550, 750, 950, 1150], + ), + ], +) def test_post_non_instantaneous_sensor_data_floor( - client, setup_api_test_data, requesting_user + client, + setup_api_test_data, + requesting_user, + offclock_start, + precise_start, + precise_end, + values, + expected_values, ): - offclock_start = "2021-06-08T00:00:40+02:00" - precise_start = "2021-06-08T00:00:00+02:00" - precise_end = "2021-06-08T01:00:00+02:00" - post_data = make_sensor_data_request_for_gas_sensor(unit="m³/h") + post_data = make_sensor_data_request_for_gas_sensor( + num_values=len(values), unit="m³/h" + ) post_data["start"] = offclock_start + post_data["values"] = values sensor = setup_api_test_data["some gas sensor"] assert ( @@ -267,13 +293,14 @@ def test_post_non_instantaneous_sensor_data_floor( new_data = sensor.search_beliefs(precise_start, precise_end).reset_index() assert len(new_data) == 6 assert list(new_data["event_start"]) == [ - pd.Timestamp("2021-06-08 00:00:00+02"), - pd.Timestamp("2021-06-08 00:10:00+02"), - pd.Timestamp("2021-06-08 00:20:00+02"), - pd.Timestamp("2021-06-08 00:30:00+02"), - pd.Timestamp("2021-06-08 00:40:00+02"), - pd.Timestamp("2021-06-08 00:50:00+02"), + pd.Timestamp(precise_start), + pd.Timestamp(precise_start) + pd.Timedelta(minutes=10), + pd.Timestamp(precise_start) + pd.Timedelta(minutes=20), + pd.Timestamp(precise_start) + pd.Timedelta(minutes=30), + pd.Timestamp(precise_start) + pd.Timedelta(minutes=40), + pd.Timestamp(precise_start) + pd.Timedelta(minutes=50), ] + assert new_data["event_value"].to_list() == expected_values @pytest.mark.parametrize( From 531d2dbabe1518335083bcddf552b07af94d4f6f Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sun, 17 May 2026 02:00:28 +0100 Subject: [PATCH 23/72] test: cover datetime flooring opt-out Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/api/v3_0/sensors.py | 6 +- .../api/v3_0/tests/test_sensor_schedules.py | 57 ++++++++++++++++++- .../data/schemas/scheduling/storage.py | 10 ++-- 3 files changed, 64 insertions(+), 9 deletions(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index df5aea3059..de040e0842 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -233,7 +233,7 @@ def floor_timed_event_datetimes(flex_model: dict, sensor: Sensor) -> dict: such as ``soc-minima``, ``soc-maxima`` and ``soc-targets``. """ if sensor.event_resolution == timedelta(0) or not sensor.get_attribute( - "round_datetimes_on_ingestion", True + "floor_datetimes_to_resolution", True ): return flex_model @@ -1251,6 +1251,8 @@ def fetch_one(self, id, sensor: Sensor): timezone: Europe/Amsterdam event_resolution: PT15M entity_address: ea1.2021-01.io.flexmeasures:fm1.14 + attributes: + floor_datetimes_to_resolution: true generic_asset_id: 1 power_sensor: summary: A power sensor recording average consumption every 5 minutes. @@ -1314,6 +1316,7 @@ def post(self, sensor_data: dict): "event_resolution": "PT1H" "unit": "kWh" "generic_asset_id": 1 + "attributes": '{"floor_datetimes_to_resolution": false}' responses: 201: description: New Sensor @@ -1330,6 +1333,7 @@ def post(self, sensor_data: dict): "entity_address": "ea1.2023-08.localhost:fm1.1" "event_resolution": "PT1H" "generic_asset_id": 1 + "attributes": '{"floor_datetimes_to_resolution": false}' "timezone": "UTC" "id": 2 400: diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index 6aad9bd4cf..1e71006fff 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -32,6 +32,10 @@ def setup_capacity_sensor_on_asset_in_supplier_account(db, setup_generic_assets) return sensor +def get_sensor_by_name(asset, name: str) -> Sensor: + return next(sensor for sensor in asset.sensors if sensor.name == name) + + @pytest.mark.parametrize( "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True ) @@ -132,7 +136,9 @@ def test_trigger_schedule_floors_flex_model_datetimes( for field in ("soc-targets", "soc-minima", "soc-maxima"): message["flex-model"][field][0]["datetime"] = offclock_target - sensor = add_charging_station_assets["Test charging station"].sensors[0] + sensor = get_sensor_by_name( + add_charging_station_assets["Test charging station"], "power" + ) with app.test_client() as client: trigger_schedule_response = client.post( url_for("SensorAPI:trigger_schedule", id=sensor.id), @@ -152,6 +158,47 @@ def test_trigger_schedule_floors_flex_model_datetimes( assert parse_datetime(target["end"]) == parse_datetime(expected_target) +@pytest.mark.parametrize( + "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True +) +def test_trigger_schedule_keeps_flex_model_datetimes_when_flooring_disabled( + app, + add_market_prices, + add_battery_assets, + battery_soc_sensor, + add_charging_station_assets, + keep_scheduling_queue_empty, + requesting_user, +): + message = message_for_trigger_schedule(with_targets=True) + offclock_target = "2015-01-02T23:00:40+01:00" + for field in ("soc-targets", "soc-minima", "soc-maxima"): + message["flex-model"][field][0]["datetime"] = offclock_target + + sensor = get_sensor_by_name( + add_charging_station_assets["Test charging station"], "power" + ) + original_attributes = dict(sensor.attributes or {}) + sensor.attributes = { + **original_attributes, + "floor_datetimes_to_resolution": False, + } + with app.test_client() as client: + trigger_schedule_response = client.post( + url_for("SensorAPI:trigger_schedule", id=sensor.id), + json=message, + ) + sensor.attributes = original_attributes + + assert trigger_schedule_response.status_code == 200 + assert len(app.queues["scheduling"]) == 1 + + job = app.queues["scheduling"].jobs[0] + for field in ("soc-targets", "soc-minima", "soc-maxima"): + target = job.kwargs["flex_model"][field][0] + assert target["datetime"] == offclock_target + + @pytest.mark.parametrize( "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True ) @@ -321,7 +368,9 @@ def test_get_schedule_fallback( start = "2015-01-02T00:00:00+01:00" epex_da = get_test_sensor(db) - charging_station = add_charging_station_assets[charging_station_name].sensors[0] + charging_station = get_sensor_by_name( + add_charging_station_assets[charging_station_name], "power" + ) capacity = charging_station.get_attribute( "capacity_in_mw", @@ -479,7 +528,9 @@ def test_get_schedule_fallback_not_redirect( start = "2015-01-02T00:00:00+01:00" epex_da = get_test_sensor(db) - charging_station = add_charging_station_assets[charging_station_name].sensors[0] + charging_station = get_sensor_by_name( + add_charging_station_assets[charging_station_name], "power" + ) capacity = charging_station.get_attribute( "capacity_in_mw", diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index e5b2bf9ad1..a41e609e18 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -236,10 +236,10 @@ def __init__( self.start = start self.sensor = sensor self.timezone = sensor.timezone if sensor is not None else None - self.rounding_resolution = ( + self.flooring_resolution = ( sensor.event_resolution if sensor is not None - and sensor.get_attribute("round_datetimes_on_ingestion", True) + and sensor.get_attribute("floor_datetimes_to_resolution", True) else None ) @@ -256,7 +256,7 @@ def __init__( to_unit="MWh", default_src_unit=default_soc_unit, timezone=self.timezone, - event_resolution=self.rounding_resolution, + event_resolution=self.flooring_resolution, data_key="soc-maxima", ) @@ -264,7 +264,7 @@ def __init__( to_unit="MWh", default_src_unit=default_soc_unit, timezone=self.timezone, - event_resolution=self.rounding_resolution, + event_resolution=self.flooring_resolution, data_key="soc-minima", value_validator=validate.Range(min=0), ) @@ -272,7 +272,7 @@ def __init__( to_unit="MWh", default_src_unit=default_soc_unit, timezone=self.timezone, - event_resolution=self.rounding_resolution, + event_resolution=self.flooring_resolution, data_key="soc-targets", ) From 067bd92145e637fe0ef20bb99af2f4a2b79c7c21 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sun, 17 May 2026 02:00:35 +0100 Subject: [PATCH 24/72] fix: reject flooring event collisions Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/schemas/sensors.py | 44 ++++++++++--- .../data/schemas/tests/test_sensor.py | 61 +++++++++++++++++++ 2 files changed, 96 insertions(+), 9 deletions(-) diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 8775531371..6437b539e3 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -235,8 +235,18 @@ def timezone_validator(value: str): attributes = JSON( required=False, metadata=dict( - description="JSON serializable attributes to store arbitrary information on the sensor. A few attributes lead to special behaviour, such as `consumption_is_positive`, which informs the platform whether consumption values should be saved (and shown in charts) as positive or negative values.", - example="{consumption_is_positive: True}", + description=( + "JSON serializable attributes to store arbitrary information on " + "the sensor. A few attributes lead to special behaviour, such as " + "`consumption_is_positive`, which informs the platform whether " + "consumption values should be saved (and shown in charts) as " + "positive or negative values, `floor_datetimes_to_resolution`, " + "which controls whether off-clock datetimes are floored to a " + "non-instantaneous sensor's resolution, and `frequency`, which " + "rounds incoming instantaneous measurements to a configured " + "Pandas frequency." + ), + example='{"consumption_is_positive": true, "floor_datetimes_to_resolution": true}', ), ) @@ -726,7 +736,7 @@ def post_load(self, fields, **kwargs): pd.infer_freq(bdf.index.unique("event_start")) if sensor.event_resolution != timedelta(0) and sensor.get_attribute( - "round_datetimes_on_ingestion", True + "floor_datetimes_to_resolution", True ): bdf = floor_bdf_event_starts(bdf, bdf.event_resolution) @@ -771,15 +781,31 @@ def post_load(self, fields, **kwargs): def floor_bdf_event_starts( bdf: tb.BeliefsDataFrame, event_resolution: timedelta ) -> tb.BeliefsDataFrame: - floored_bdf = pd.DataFrame(bdf.reset_index()) - floored_bdf["event_start"] = pd.DatetimeIndex(bdf.event_starts).floor( + floored_event_starts = bdf.index.get_level_values("event_start").floor( event_resolution ) - return tb.BeliefsDataFrame( - floored_bdf, - sensor=bdf.sensor, - event_resolution=bdf.event_resolution, + + new_index = pd.MultiIndex.from_arrays( + [ + ( + floored_event_starts + if name == "event_start" + else bdf.index.get_level_values(name) + ) + for name in bdf.index.names + ], + names=bdf.index.names, ) + if new_index.duplicated().any(): + raise ValidationError( + "Flooring event_start would merge multiple beliefs with the same " + "source, belief_time and event_start. Please provide data already " + "aligned to the event resolution or use distinct belief/source metadata." + ) + + floored_bdf = bdf.copy() + floored_bdf.index = new_index + return floored_bdf class QuantitySchema(Schema): diff --git a/flexmeasures/data/schemas/tests/test_sensor.py b/flexmeasures/data/schemas/tests/test_sensor.py index a547eaa66a..50606a2bc4 100644 --- a/flexmeasures/data/schemas/tests/test_sensor.py +++ b/flexmeasures/data/schemas/tests/test_sensor.py @@ -1,8 +1,11 @@ import pytest +import pandas as pd +import timely_beliefs as tb from flexmeasures import Sensor from flexmeasures.data.schemas.sensors import ( QuantityOrSensor, VariableQuantityField, + floor_bdf_event_starts, ) from flexmeasures.utils.unit_utils import ur from marshmallow import ValidationError @@ -175,3 +178,61 @@ def test_time_series_field(input_param, dst_unit, fails, db): assert not fails except Exception as e: assert fails, e + + +def test_floor_bdf_event_starts(setup_dummy_sensors, setup_sources): + sensor1, _, _, _ = setup_dummy_sensors + belief_time = pd.Timestamp("2025-01-01T09:00:00+00:00") + bdf = tb.BeliefsDataFrame( + pd.DataFrame( + { + "event_start": pd.to_datetime( + [ + "2025-01-01T10:00:40+00:00", + "2025-01-01T10:15:40+00:00", + ] + ), + "belief_time": [belief_time, belief_time], + "source": [setup_sources["Seita"], setup_sources["Seita"]], + "event_value": [1.0, 2.0], + } + ), + sensor=sensor1, + event_resolution=pd.Timedelta(minutes=15), + ) + + floored_bdf = floor_bdf_event_starts(bdf, pd.Timedelta(minutes=15)) + + pd.testing.assert_index_equal( + floored_bdf.event_starts, + pd.DatetimeIndex( + ["2025-01-01T10:00:00+00:00", "2025-01-01T10:15:00+00:00"], + name="event_start", + ), + ) + assert floored_bdf["event_value"].to_list() == [1.0, 2.0] + + +def test_floor_bdf_event_starts_rejects_collisions(setup_dummy_sensors, setup_sources): + sensor1, _, _, _ = setup_dummy_sensors + belief_time = pd.Timestamp("2025-01-01T09:00:00+00:00") + bdf = tb.BeliefsDataFrame( + pd.DataFrame( + { + "event_start": pd.to_datetime( + [ + "2025-01-01T10:00:40+00:00", + "2025-01-01T10:08:10+00:00", + ] + ), + "belief_time": [belief_time, belief_time], + "source": [setup_sources["Seita"], setup_sources["Seita"]], + "event_value": [1.0, 2.0], + } + ), + sensor=sensor1, + event_resolution=pd.Timedelta(minutes=7, seconds=30), + ) + + with pytest.raises(ValidationError, match="would merge multiple beliefs"): + floor_bdf_event_starts(bdf, pd.Timedelta(minutes=15)) From fa31e0e3302b14cf9baa65e0620bdd9fd088f955 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sun, 17 May 2026 02:06:24 +0100 Subject: [PATCH 25/72] docs: document datetime flooring attribute Signed-off-by: Mohamed Belhsan Hmida --- documentation/api/notation.rst | 11 ++++++++++- flexmeasures/ui/static/openapi-specs.json | 11 ++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/documentation/api/notation.rst b/documentation/api/notation.rst index dd956c8c23..ddc6aaa2e7 100644 --- a/documentation/api/notation.rst +++ b/documentation/api/notation.rst @@ -120,6 +120,12 @@ In all current versions of the FlexMeasures API, only equidistant timeseries dat - "start" should be a timestamp on the hour or a multiple of the sensor resolution thereafter (e.g. "16:10" works if the resolution is 5 minutes), and - "duration" should also be a multiple of the sensor resolution. +For non-instantaneous sensors, FlexMeasures floors off-clock datetimes to the +sensor's resolution by default when ingesting sensor data. For example, data +posted with ``"start": "2026-05-12T08:29:58+02:00"`` to a 15-minute sensor is +saved from ``2026-05-12T08:15:00+02:00``. Set the sensor attribute +``"floor_datetimes_to_resolution": false`` to disable this behaviour. + .. _beliefs: @@ -249,7 +255,10 @@ FlexMeasures handles two types of time series, which can be distinguished by def Specifying a frequency and resolution is redundant for POST requests that contain both "values" and a "duration" ― FlexMeasures computes the frequency by dividing the duration by the number of values, and, for sensors that record non-instantaneous events, assumes the resolution of the data is equal to the frequency. When POSTing data, FlexMeasures checks this inferred resolution against the required resolution of the sensors that are posted to. -If these can't be matched (through upsampling), an error will occur. +If these can't be matched through upsampling or downsampling, an error will occur. +Off-clock event starts for non-instantaneous sensors are floored to the sensor's resolution by default. +The sensor attribute ``floor_datetimes_to_resolution`` can be set to ``false`` to keep incoming datetimes unchanged. +This flooring behaviour is distinct from the existing ``frequency`` sensor attribute, which rounds incoming instantaneous measurements to a configured Pandas frequency. GET requests (such as */sensors/data*) return data with a frequency either equal to the resolution that the sensor is configured for (for non-instantaneous sensors), or a default frequency befitting (in our opinion) the requested time interval. A "resolution" may be specified explicitly to obtain the data in downsampled form, which can be very beneficial for download speed. diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index a11c8b84e7..a8e6579eae 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -177,6 +177,9 @@ "timezone": "Europe/Amsterdam", "event_resolution": "PT15M", "entity_address": "ea1.2021-01.io.flexmeasures:fm1.14", + "attributes": { + "floor_datetimes_to_resolution": true + }, "generic_asset_id": 1 } }, @@ -1189,7 +1192,8 @@ "name": "power", "event_resolution": "PT1H", "unit": "kWh", - "generic_asset_id": 1 + "generic_asset_id": 1, + "attributes": "{\"floor_datetimes_to_resolution\": false}" } } } @@ -1214,6 +1218,7 @@ "entity_address": "ea1.2023-08.localhost:fm1.1", "event_resolution": "PT1H", "generic_asset_id": 1, + "attributes": "{\"floor_datetimes_to_resolution\": false}", "timezone": "UTC", "id": 2 } @@ -5699,8 +5704,8 @@ "description": "Obsolete identifier from [USEF](https://www.usef.energy/)." }, "attributes": { - "description": "JSON serializable attributes to store arbitrary information on the sensor. A few attributes lead to special behaviour, such as `consumption_is_positive`, which informs the platform whether consumption values should be saved (and shown in charts) as positive or negative values.", - "example": "{consumption_is_positive: True}" + "description": "JSON serializable attributes to store arbitrary information on the sensor. A few attributes lead to special behaviour, such as `consumption_is_positive`, which informs the platform whether consumption values should be saved (and shown in charts) as positive or negative values, `floor_datetimes_to_resolution`, which controls whether off-clock datetimes are floored to a non-instantaneous sensor's resolution, and `frequency`, which rounds incoming instantaneous measurements to a configured Pandas frequency.", + "example": "{\"consumption_is_positive\": true, \"floor_datetimes_to_resolution\": true}" }, "generic_asset_id": { "type": "integer", From cd4cfd2dd810bf50b65368526ea63e4cdfc48651 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Sun, 17 May 2026 02:06:56 +0100 Subject: [PATCH 26/72] docs: add changelog entry Signed-off-by: Mohamed Belhsan Hmida --- documentation/changelog.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 81de9d392a..30b4aebe20 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -13,6 +13,7 @@ New features * Improve source filtering in the sensor data GET endpoint by exposing the documented query parameters in Swagger and allowing filtering by the account linked to data sources [see `PR #2083 `_] * Added a unified job status endpoint ``GET /api/v3_0/jobs/`` to retrieve the current execution status and result message for any background job [see `PR #2141 `_] * New ``GET /api/v3_0/sources`` endpoint to list accessible data sources and defined types, with ``only_latest=true`` by default to return only the most recent version per source [see `PR #2126 `_] +* Floor off-clock API datetimes to a non-instantaneous sensor's resolution by default when ingesting sensor data, uploading sensor data, and handling scheduler flex-model timed events; configurable with the ``floor_datetimes_to_resolution`` sensor attribute [see `PR #2146 `_] Infrastructure / Support ---------------------- From d4ee9454ac2f6ec1532f1729026e149b29ee28b8 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Thu, 21 May 2026 09:48:19 +0100 Subject: [PATCH 27/72] docs: align sensor data resampling docs Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/api/v3_0/sensors.py | 4 ++-- flexmeasures/ui/static/openapi-specs.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 85febd182f..98ac717f74 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -582,7 +582,7 @@ def upload_data( The unit has to be convertible to the sensor's unit. The resolution of the data has to match the sensor's required resolution, but - FlexMeasures will attempt to upsample lower resolutions. + FlexMeasures will attempt to resample compatible resolutions. The list of values may include null values. The request body is limited by FLEXMEASURES_MAX_SENSOR_DATA_INGESTION_BYTES (3 MiB by default). @@ -718,7 +718,7 @@ def post_data(self, id: int, sensor: Sensor, sensor_data: dict): The sensor is the one with ID=1. The unit has to be convertible to the sensor's unit. The resolution of the data has to match the sensor's required resolution, but - FlexMeasures will attempt to upsample lower resolutions. + FlexMeasures will attempt to resample compatible resolutions. The list of values may include null values. The request body is limited by FLEXMEASURES_MAX_SENSOR_DATA_INGESTION_BYTES (3 MiB by default). diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index b2c6ca38f7..3f24818074 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -487,7 +487,7 @@ }, "post": { "summary": "Post sensor data", - "description": "Send data values via JSON, where the duration and number of values determine the resolution.\n\nThe example request posts four values for a duration of one hour, where the first\nevent start is at the given start time, and subsequent events start in 15 minute intervals throughout the one hour duration.\n\nThe sensor is the one with ID=1.\nThe unit has to be convertible to the sensor's unit.\nThe resolution of the data has to match the sensor's required resolution, but\nFlexMeasures will attempt to upsample lower resolutions.\nThe list of values may include null values.\nThe request body is limited by FLEXMEASURES_MAX_SENSOR_DATA_INGESTION_BYTES\n(3 MiB by default).\n", + "description": "Send data values via JSON, where the duration and number of values determine the resolution.\n\nThe example request posts four values for a duration of one hour, where the first\nevent start is at the given start time, and subsequent events start in 15 minute intervals throughout the one hour duration.\n\nThe sensor is the one with ID=1.\nThe unit has to be convertible to the sensor's unit.\nThe resolution of the data has to match the sensor's required resolution, but\nFlexMeasures will attempt to resample compatible resolutions.\nThe list of values may include null values.\nThe request body is limited by FLEXMEASURES_MAX_SENSOR_DATA_INGESTION_BYTES\n(3 MiB by default).\n", "security": [ { "ApiAuthKey": [] @@ -1532,7 +1532,7 @@ "/api/v3_0/sensors/{id}/data/upload": { "post": { "summary": "Upload sensor data by file", - "description": "The file should have columns for a timestamp (event_start) and a value (event_value).\nThe timestamp should be in ISO 8601 format.\nThe value should be a numeric value.\n\nThe unit has to be convertible to the sensor's unit.\nThe resolution of the data has to match the sensor's required resolution, but\nFlexMeasures will attempt to upsample lower resolutions.\nThe list of values may include null values.\nThe request body is limited by FLEXMEASURES_MAX_SENSOR_DATA_INGESTION_BYTES\n(3 MiB by default).\n", + "description": "The file should have columns for a timestamp (event_start) and a value (event_value).\nThe timestamp should be in ISO 8601 format.\nThe value should be a numeric value.\n\nThe unit has to be convertible to the sensor's unit.\nThe resolution of the data has to match the sensor's required resolution, but\nFlexMeasures will attempt to resample compatible resolutions.\nThe list of values may include null values.\nThe request body is limited by FLEXMEASURES_MAX_SENSOR_DATA_INGESTION_BYTES\n(3 MiB by default).\n", "security": [ { "ApiKeyAuth": [] From 61583e328161bef3b107af67dd226dfe4a90ad79 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 22 May 2026 05:23:31 +0100 Subject: [PATCH 28/72] fix: preserve submitted flex-model in schedule jobs Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/api/v3_0/sensors.py | 49 ------------------- .../api/v3_0/tests/test_sensor_schedules.py | 11 ++--- 2 files changed, 3 insertions(+), 57 deletions(-) diff --git a/flexmeasures/api/v3_0/sensors.py b/flexmeasures/api/v3_0/sensors.py index 98ac717f74..eab474f488 100644 --- a/flexmeasures/api/v3_0/sensors.py +++ b/flexmeasures/api/v3_0/sensors.py @@ -1,9 +1,7 @@ from __future__ import annotations import isodate -from copy import deepcopy from datetime import datetime, timedelta -import pandas as pd from flexmeasures.data.services.sensors import ( serialize_sensor_status_data, @@ -289,47 +287,6 @@ def support_legacy_field_name(self, data, **kwargs): return data -def floor_timed_event_datetimes(flex_model: dict, sensor: Sensor) -> dict: - """Floor timed-event datetimes in list-valued flex-model fields. - - This only touches list entries that look like timed-event dictionaries, - such as ``soc-minima``, ``soc-maxima`` and ``soc-targets``. - """ - if sensor.event_resolution == timedelta(0) or not sensor.get_attribute( - "floor_datetimes_to_resolution", True - ): - return flex_model - - floored_flex_model = deepcopy(flex_model) - for value_name, value in floored_flex_model.items(): - if not isinstance(value, list): - continue - for index, timed_event in enumerate(value): - if not isinstance(timed_event, dict): - continue - for key in ("datetime", "start", "end"): - if key in timed_event: - try: - timed_event[key] = isodate.datetime_isoformat( - pd.Timestamp(timed_event[key]).floor( - sensor.event_resolution - ) - ) - except (TypeError, ValueError) as exc: - raise ValidationError( - { - value_name: { - index: { - key: [ - f"Not a valid datetime: {timed_event[key]!r}." - ] - } - } - } - ) from exc - return floored_flex_model - - class SensorAPI(FlaskView): route_base = "/sensors" trailing_slash = False @@ -1045,12 +1002,6 @@ def trigger_schedule( f"Resolution of {resolution} is incompatible with the sensor's required resolution of {sensor.event_resolution}." ) - if flex_model is not None: - try: - flex_model = floor_timed_event_datetimes(flex_model, sensor) - except ValidationError as err: - return unprocessable_entity(err.messages) - end_of_schedule = start_of_schedule + duration scheduler_kwargs = dict( asset_or_sensor=sensor, diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index 1e71006fff..2c5081a49c 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -121,7 +121,7 @@ def test_trigger_schedule_with_invalid_flexmodel( @pytest.mark.parametrize( "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True ) -def test_trigger_schedule_floors_flex_model_datetimes( +def test_trigger_schedule_preserves_flex_model_datetimes_in_job_kwargs( app, add_market_prices, add_battery_assets, @@ -132,7 +132,6 @@ def test_trigger_schedule_floors_flex_model_datetimes( ): message = message_for_trigger_schedule(with_targets=True) offclock_target = "2015-01-02T23:00:40+01:00" - expected_target = "2015-01-02T23:00:00+01:00" for field in ("soc-targets", "soc-minima", "soc-maxima"): message["flex-model"][field][0]["datetime"] = offclock_target @@ -151,17 +150,13 @@ def test_trigger_schedule_floors_flex_model_datetimes( job = app.queues["scheduling"].jobs[0] for field in ("soc-targets", "soc-minima", "soc-maxima"): target = job.kwargs["flex_model"][field][0] - assert target["datetime"] == expected_target - if "start" in target: - assert parse_datetime(target["start"]) == parse_datetime(expected_target) - if "end" in target: - assert parse_datetime(target["end"]) == parse_datetime(expected_target) + assert target["datetime"] == offclock_target @pytest.mark.parametrize( "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True ) -def test_trigger_schedule_keeps_flex_model_datetimes_when_flooring_disabled( +def test_trigger_schedule_preserves_flex_model_datetimes_when_flooring_disabled( app, add_market_prices, add_battery_assets, From 2eceadf7d1ba18cf0a0dd126c68b8d130294fa0a Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 22 May 2026 06:00:59 +0100 Subject: [PATCH 29/72] feat: keep storage soc event times in schema Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/schemas/scheduling/storage.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index 9dd32c2ed9..e1c60c7143 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -237,12 +237,6 @@ def __init__( self.start = start self.sensor = sensor self.timezone = sensor.timezone if sensor is not None else None - self.flooring_resolution = ( - sensor.event_resolution - if sensor is not None - and sensor.get_attribute("floor_datetimes_to_resolution", True) - else None - ) # guess default soc-unit if default_soc_unit is None: @@ -257,7 +251,6 @@ def __init__( to_unit="MWh", default_src_unit=default_soc_unit, timezone=self.timezone, - event_resolution=self.flooring_resolution, data_key="soc-maxima", ) @@ -265,7 +258,6 @@ def __init__( to_unit="MWh", default_src_unit=default_soc_unit, timezone=self.timezone, - event_resolution=self.flooring_resolution, data_key="soc-minima", value_validator=validate.Range(min=0), ) @@ -273,7 +265,6 @@ def __init__( to_unit="MWh", default_src_unit=default_soc_unit, timezone=self.timezone, - event_resolution=self.flooring_resolution, data_key="soc-targets", ) From 1d37e4f5a8e933cf0a2b95b37d0f15e4f70f7e2e Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 22 May 2026 06:01:04 +0100 Subject: [PATCH 30/72] feat: project off-tick soc constraints to grid Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/models/planning/storage.py | 289 ++++++++++++++++++- 1 file changed, 273 insertions(+), 16 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 93a67a3e96..3c1ddb1477 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -648,22 +648,6 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 # soc-maxima will become a soft constraint (modelled as stock commitments), so remove hard constraint soc_maxima[d] = None - if soc_at_start[d] is not None: - device_constraints[d] = add_storage_constraints( - start, - end, - resolution, - soc_at_start[d], - soc_targets[d], - soc_maxima[d], - soc_minima[d], - soc_max[d], - soc_min[d], - ) - else: - # No need to validate non-existing storage constraints - skip_validation = True - power_capacity_in_mw[d] = get_continuous_series_sensor_or_quantity( variable_quantity=power_capacity_in_mw[d], unit="MW", @@ -679,6 +663,9 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 if sensor_d is not None and sensor_d.get_attribute( "is_strictly_non_positive" ): + production_capacity_d = pd.Series( + 0, index=power_capacity_in_mw[d].index + ) device_constraints[d]["derivative min"] = 0 else: production_capacity_d = get_continuous_series_sensor_or_quantity( @@ -747,6 +734,9 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 if sensor_d is not None and sensor_d.get_attribute( "is_strictly_non_negative" ): + consumption_capacity_d = pd.Series( + 0, index=power_capacity_in_mw[d].index + ) device_constraints[d]["derivative max"] = 0 else: consumption_capacity_d = get_continuous_series_sensor_or_quantity( @@ -811,6 +801,44 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 # consumption-capacity will become a hard constraint device_constraints[d]["derivative max"] = consumption_capacity_d + if soc_at_start[d] is not None: + if ( + sensor_d is not None + and sensor_d.event_resolution != timedelta(0) + and sensor_d.get_attribute("floor_datetimes_to_resolution", True) + ): + ( + soc_targets[d], + soc_maxima[d], + soc_minima[d], + ) = normalize_off_tick_soc_constraints( + soc_targets[d], + soc_maxima[d], + soc_minima[d], + consumption_capacity_d, + production_capacity_d, + resolution, + soc_min[d], + soc_max[d], + ) + + storage_constraints = add_storage_constraints( + start, + end, + resolution, + soc_at_start[d], + soc_targets[d], + soc_maxima[d], + soc_minima[d], + soc_max[d], + soc_min[d], + ) + for column in ("equals", "min", "max"): + device_constraints[d][column] = storage_constraints[column] + else: + # No need to validate non-existing storage constraints + skip_validation = True + all_stock_delta = [] for is_usage, soc_delta in zip([False, True], [soc_gain[d], soc_usage[d]]): @@ -1716,6 +1744,235 @@ def create_constraint_violations_message(constraint_violations: list) -> str: return message +def _is_on_schedule_tick(dt: datetime, resolution: timedelta) -> bool: + timestamp = pd.Timestamp(dt) + return timestamp == timestamp.floor(resolution) + + +def _soc_value_in_mwh(value: ur.Quantity | float | int) -> float: + if isinstance(value, ur.Quantity): + return value.to("MWh").magnitude + return float(value) + + +def _soc_event_at( + soc_event: dict[str, datetime | float], + dt: pd.Timestamp, + value: float, +) -> dict[str, datetime | float]: + shifted_event = copy.copy(soc_event) + shifted_event["value"] = value + shifted_event["start"] = dt.to_pydatetime() + shifted_event["end"] = dt.to_pydatetime() + if "datetime" in shifted_event: + shifted_event["datetime"] = dt.to_pydatetime() + shifted_event.pop("duration", None) + return shifted_event + + +def _energy_capacity_between( + capacity: pd.Series, + start: pd.Timestamp, + end: pd.Timestamp, + resolution: timedelta, +) -> float: + if end <= start: + return 0 + + if capacity.index.tz is not None: + start = start.tz_convert(capacity.index.tz) + end = end.tz_convert(capacity.index.tz) + + capacity = capacity.astype(float).fillna(0) + tick = start.floor(resolution) + energy = 0.0 + while tick < end: + next_tick = tick + resolution + overlap_start = max(start, tick) + overlap_end = min(end, next_tick) + if overlap_end > overlap_start and tick in capacity.index: + energy += float(capacity.loc[tick]) * ( + (overlap_end - overlap_start) / pd.Timedelta(hours=1) + ) + tick = next_tick + return energy + + +def _add_soc_bound( + soc_events: list[dict[str, datetime | float]], + soc_event: dict[str, datetime | float], + bound_type: str, +) -> None: + for existing_event in soc_events: + if existing_event.get("start") == soc_event.get("start") and existing_event.get( + "end" + ) == soc_event.get("end"): + existing_value = _soc_value_in_mwh(existing_event["value"]) + soc_value = _soc_value_in_mwh(soc_event["value"]) + existing_event["value"] = ( + max(existing_value, soc_value) + if bound_type == "min" + else min(existing_value, soc_value) + ) + return + soc_events.append(soc_event) + + +def normalize_off_tick_soc_constraints( + soc_targets: list[dict[str, datetime | float]] | Sensor | None, + soc_maxima: list[dict[str, datetime | float]] | Sensor | None, + soc_minima: list[dict[str, datetime | float]] | Sensor | None, + consumption_capacity: pd.Series, + production_capacity: pd.Series, + resolution: timedelta, + soc_min: ur.Quantity | float, + soc_max: ur.Quantity | float, +) -> tuple[ + list[dict[str, datetime | float]] | Sensor | None, + list[dict[str, datetime | float]] | Sensor | None, + list[dict[str, datetime | float]] | Sensor | None, +]: + """Project off-tick point-like SoC constraints onto the scheduling grid.""" + + if isinstance(soc_minima, Sensor) or isinstance(soc_maxima, Sensor): + return soc_targets, soc_maxima, soc_minima + + normalized_minima = copy.deepcopy(soc_minima or []) + normalized_maxima = copy.deepcopy(soc_maxima or []) + + soc_min_value = _soc_value_in_mwh(soc_min) + soc_max_value = _soc_value_in_mwh(soc_max) + + if not isinstance(soc_targets, Sensor) and soc_targets is not None: + normalized_targets = [] + for soc_target in soc_targets: + if soc_target["start"] != soc_target["end"] or _is_on_schedule_tick( + soc_target["end"], resolution + ): + normalized_targets.append(copy.copy(soc_target)) + continue + + target_time = pd.Timestamp(soc_target["end"]) + previous_tick = target_time.floor(resolution) + next_tick = target_time.ceil(resolution) + target_value = _soc_value_in_mwh(soc_target["value"]) + + charge_before_target = _energy_capacity_between( + consumption_capacity, previous_tick, target_time, resolution + ) + discharge_before_target = _energy_capacity_between( + production_capacity, previous_tick, target_time, resolution + ) + + normalized_targets.append( + _soc_event_at(soc_target, next_tick, target_value) + ) + _add_soc_bound( + normalized_minima, + _soc_event_at( + soc_target, + previous_tick, + max(soc_min_value, target_value - charge_before_target), + ), + bound_type="min", + ) + _add_soc_bound( + normalized_maxima, + _soc_event_at( + soc_target, + previous_tick, + min(soc_max_value, target_value + discharge_before_target), + ), + bound_type="max", + ) + else: + normalized_targets = soc_targets + + for soc_minimum in copy.deepcopy(soc_minima or []): + if soc_minimum["start"] != soc_minimum["end"] or _is_on_schedule_tick( + soc_minimum["end"], resolution + ): + continue + + minimum_time = pd.Timestamp(soc_minimum["end"]) + previous_tick = minimum_time.floor(resolution) + next_tick = minimum_time.ceil(resolution) + minimum_value = _soc_value_in_mwh(soc_minimum["value"]) + _add_soc_bound( + normalized_minima, + _soc_event_at( + soc_minimum, + previous_tick, + max( + soc_min_value, + minimum_value + - _energy_capacity_between( + consumption_capacity, previous_tick, minimum_time, resolution + ), + ), + ), + bound_type="min", + ) + _add_soc_bound( + normalized_minima, + _soc_event_at( + soc_minimum, + next_tick, + max( + soc_min_value, + minimum_value + - _energy_capacity_between( + production_capacity, minimum_time, next_tick, resolution + ), + ), + ), + bound_type="min", + ) + + for soc_maximum in copy.deepcopy(soc_maxima or []): + if soc_maximum["start"] != soc_maximum["end"] or _is_on_schedule_tick( + soc_maximum["end"], resolution + ): + continue + + maximum_time = pd.Timestamp(soc_maximum["end"]) + previous_tick = maximum_time.floor(resolution) + next_tick = maximum_time.ceil(resolution) + maximum_value = _soc_value_in_mwh(soc_maximum["value"]) + _add_soc_bound( + normalized_maxima, + _soc_event_at( + soc_maximum, + previous_tick, + min( + soc_max_value, + maximum_value + + _energy_capacity_between( + production_capacity, previous_tick, maximum_time, resolution + ), + ), + ), + bound_type="max", + ) + _add_soc_bound( + normalized_maxima, + _soc_event_at( + soc_maximum, + next_tick, + min( + soc_max_value, + maximum_value + + _energy_capacity_between( + consumption_capacity, maximum_time, next_tick, resolution + ), + ), + ), + bound_type="max", + ) + + return normalized_targets, normalized_maxima or None, normalized_minima or None + + def build_device_soc_values( soc_values: ur.Quantity | list[dict[str, datetime | float]] | pd.Series | None, soc_at_start: float, From a7b01a6a711b4436499c5e33e53c0de4a6c061bc Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 22 May 2026 06:01:08 +0100 Subject: [PATCH 31/72] test: cover off-tick soc target projection Signed-off-by: Mohamed Belhsan Hmida --- .../models/planning/tests/test_storage.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py index dbb5d792f7..4804593fbb 100644 --- a/flexmeasures/data/models/planning/tests/test_storage.py +++ b/flexmeasures/data/models/planning/tests/test_storage.py @@ -252,6 +252,51 @@ def test_battery_relaxation(add_battery_assets, db): ) # 100 EUR/(kW*h) * 0.025 MW * 1000 kW/MW * 4 hours +def test_off_tick_soc_target_is_projected_to_storage_grid(add_battery_assets, db): + _, battery = get_sensors_from_db( + db, add_battery_assets, battery_name="Test battery" + ) + tz = pytz.timezone("Europe/Amsterdam") + start = tz.localize(datetime(2015, 1, 1, 16, 45)) + end = tz.localize(datetime(2015, 1, 1, 17, 15)) + resolution = timedelta(minutes=15) + + scheduler = StorageScheduler( + battery, + start, + end, + resolution, + flex_model={ + "soc-at-start": "0 MWh", + "soc-min": "0 MWh", + "soc-max": "1 MWh", + "power-capacity": "0.04 MW", + "consumption-capacity": "0.04 MW", + "production-capacity": "0 MW", + "roundtrip-efficiency": 1, + "storage-efficiency": 1, + "soc-targets": [ + { + "datetime": "2015-01-01T17:12:00+01:00", + "value": "1 MWh", + } + ], + }, + flex_context={ + "consumption-price": "0 EUR/MWh", + "production-price": "0 EUR/MWh", + "site-power-capacity": "1 MW", + }, + ) + + _, _, _, _, _, device_constraints, _, _ = scheduler._prepare(skip_validation=True) + storage_constraints = device_constraints[0].tz_convert(tz) + + assert pd.isna(storage_constraints.loc[start, "equals"]) + assert storage_constraints.loc[start, "min"] == pytest.approx(0.992 * 4) + assert storage_constraints.loc[start + resolution, "equals"] == pytest.approx(4) + + def test_deserialize_storage_soc_at_start_from_state_of_charge_sensor( add_charging_station_assets, setup_markets, setup_sources, db ): From 7dea2a7a1cd26293376a5ad82090067a63a7db7d Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 22 May 2026 06:01:12 +0100 Subject: [PATCH 32/72] fix: relax off-tick soc constraints by default Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/models/planning/storage.py | 57 ++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 3c1ddb1477..c4a30d3f57 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -44,6 +44,7 @@ storage_asset_types = ["one-way_evse", "two-way_evse", "battery", "heat-storage"] +SOC_TIMED_EVENT_FIELDS = ("soc-targets", "soc-minima", "soc-maxima") class MetaStorageScheduler(Scheduler): @@ -1043,6 +1044,7 @@ def deserialize_flex_config(self): self.flex_model = {} self.collect_flex_config() + self.enable_relax_soc_constraints_for_off_tick_soc_constraints() self.flex_context = FlexContextSchema().load(self.flex_context) if isinstance(self.flex_model, dict): @@ -1096,6 +1098,33 @@ def deserialize_flex_config(self): return self.flex_model + def enable_relax_soc_constraints_for_off_tick_soc_constraints(self) -> None: + """Relax SoC constraints when off-tick SoC events require grid projection.""" + if self.flex_context is None: + self.flex_context = {} + if not self.flex_model_has_off_tick_soc_constraints(): + return + self.flex_context["relax-soc-constraints"] = True + + def flex_model_has_off_tick_soc_constraints(self) -> bool: + if isinstance(self.flex_model, dict): + resolution = self.resolution + if self.sensor is not None: + if not self.sensor.get_attribute("floor_datetimes_to_resolution", True): + return False + resolution = resolution or self.sensor.event_resolution + return flex_model_has_off_tick_soc_constraints( + self.flex_model, resolution=resolution + ) + if isinstance(self.flex_model, list): + return any( + flex_model_has_off_tick_soc_constraints( + flex_model, resolution=self.resolution + ) + for flex_model in self.flex_model + ) + return False + def has_soc_at_start(self) -> bool: return ( "soc-at-start" in self.flex_model @@ -1749,6 +1778,34 @@ def _is_on_schedule_tick(dt: datetime, resolution: timedelta) -> bool: return timestamp == timestamp.floor(resolution) +def flex_model_has_off_tick_soc_constraints( + flex_model: dict, + resolution: timedelta | None, +) -> bool: + if resolution in (None, timedelta(0)): + return False + + for field_name in SOC_TIMED_EVENT_FIELDS: + field_value = flex_model.get(field_name) + if not isinstance(field_value, list): + continue + for soc_event in field_value: + if not isinstance(soc_event, dict): + continue + for timing_field in ("datetime", "start", "end"): + if soc_event.get(timing_field) is None: + continue + try: + is_on_tick = _is_on_schedule_tick( + soc_event[timing_field], resolution + ) + except (TypeError, ValueError): + continue + if not is_on_tick: + return True + return False + + def _soc_value_in_mwh(value: ur.Quantity | float | int) -> float: if isinstance(value, ur.Quantity): return value.to("MWh").magnitude From e7b9ee1a8112d2c1d035864a6a7b804b261ae418 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 22 May 2026 06:01:16 +0100 Subject: [PATCH 33/72] test: cover off-tick soc relaxation Signed-off-by: Mohamed Belhsan Hmida --- .../models/planning/tests/test_storage.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py index 4804593fbb..eaac9de055 100644 --- a/flexmeasures/data/models/planning/tests/test_storage.py +++ b/flexmeasures/data/models/planning/tests/test_storage.py @@ -297,6 +297,46 @@ def test_off_tick_soc_target_is_projected_to_storage_grid(add_battery_assets, db assert storage_constraints.loc[start + resolution, "equals"] == pytest.approx(4) +def test_off_tick_soc_constraints_enable_relax_soc_constraints(add_battery_assets, db): + _, battery = get_sensors_from_db( + db, add_battery_assets, battery_name="Test battery" + ) + tz = pytz.timezone("Europe/Amsterdam") + start = tz.localize(datetime(2015, 1, 1, 16, 45)) + end = tz.localize(datetime(2015, 1, 1, 17, 15)) + resolution = timedelta(minutes=15) + + scheduler = StorageScheduler( + battery, + start, + end, + resolution, + flex_model={ + "soc-at-start": "0 MWh", + "soc-min": "0 MWh", + "soc-max": "1 MWh", + "power-capacity": "0.04 MW", + "soc-targets": [ + { + "datetime": "2015-01-01T17:12:00+01:00", + "value": "1 MWh", + } + ], + }, + flex_context={ + "consumption-price": "0 EUR/MWh", + "production-price": "0 EUR/MWh", + "relax-soc-constraints": False, + }, + ) + + scheduler.deserialize_config() + + assert scheduler.flex_context["relax_soc_constraints"] is True + assert scheduler.flex_context["soc_minima_breach_price"] is not None + assert scheduler.flex_context["soc_maxima_breach_price"] is not None + + def test_deserialize_storage_soc_at_start_from_state_of_charge_sensor( add_charging_station_assets, setup_markets, setup_sources, db ): From 03cf1e20fd518bf412e38c9ab0c4d7e3c3cdfcc5 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 22 May 2026 13:05:50 +0100 Subject: [PATCH 34/72] fix: preserve non-event soc constraints Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/models/planning/storage.py | 64 ++++++++++++++++---- 1 file changed, 51 insertions(+), 13 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index c4a30d3f57..05880fc8cf 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -1875,32 +1875,58 @@ def _add_soc_bound( soc_events.append(soc_event) +def _normalized_soc_events_or_original( + original_soc_events: ( + list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None + ), + normalized_soc_events: list[dict[str, datetime | float]], +) -> list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None: + if isinstance(original_soc_events, list): + return normalized_soc_events + if original_soc_events is None and normalized_soc_events: + return normalized_soc_events + return original_soc_events + + def normalize_off_tick_soc_constraints( - soc_targets: list[dict[str, datetime | float]] | Sensor | None, - soc_maxima: list[dict[str, datetime | float]] | Sensor | None, - soc_minima: list[dict[str, datetime | float]] | Sensor | None, + soc_targets: ( + list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None + ), + soc_maxima: ( + list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None + ), + soc_minima: ( + list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None + ), consumption_capacity: pd.Series, production_capacity: pd.Series, resolution: timedelta, soc_min: ur.Quantity | float, soc_max: ur.Quantity | float, ) -> tuple[ - list[dict[str, datetime | float]] | Sensor | None, - list[dict[str, datetime | float]] | Sensor | None, - list[dict[str, datetime | float]] | Sensor | None, + list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, + list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, + list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, ]: """Project off-tick point-like SoC constraints onto the scheduling grid.""" - if isinstance(soc_minima, Sensor) or isinstance(soc_maxima, Sensor): + if not any( + isinstance(soc_events, list) and soc_events + for soc_events in (soc_targets, soc_maxima, soc_minima) + ): return soc_targets, soc_maxima, soc_minima - normalized_minima = copy.deepcopy(soc_minima or []) - normalized_maxima = copy.deepcopy(soc_maxima or []) + normalized_minima = ( + copy.deepcopy(soc_minima) if isinstance(soc_minima, list) else [] + ) + normalized_maxima = ( + copy.deepcopy(soc_maxima) if isinstance(soc_maxima, list) else [] + ) soc_min_value = _soc_value_in_mwh(soc_min) soc_max_value = _soc_value_in_mwh(soc_max) - if not isinstance(soc_targets, Sensor) and soc_targets is not None: + if isinstance(soc_targets, list): normalized_targets = [] for soc_target in soc_targets: if soc_target["start"] != soc_target["end"] or _is_on_schedule_tick( @@ -1945,7 +1971,11 @@ def normalize_off_tick_soc_constraints( else: normalized_targets = soc_targets - for soc_minimum in copy.deepcopy(soc_minima or []): + if isinstance(soc_minima, list): + soc_minimum_events = copy.deepcopy(soc_minima) + else: + soc_minimum_events = [] + for soc_minimum in soc_minimum_events: if soc_minimum["start"] != soc_minimum["end"] or _is_on_schedule_tick( soc_minimum["end"], resolution ): @@ -1986,7 +2016,11 @@ def normalize_off_tick_soc_constraints( bound_type="min", ) - for soc_maximum in copy.deepcopy(soc_maxima or []): + if isinstance(soc_maxima, list): + soc_maximum_events = copy.deepcopy(soc_maxima) + else: + soc_maximum_events = [] + for soc_maximum in soc_maximum_events: if soc_maximum["start"] != soc_maximum["end"] or _is_on_schedule_tick( soc_maximum["end"], resolution ): @@ -2027,7 +2061,11 @@ def normalize_off_tick_soc_constraints( bound_type="max", ) - return normalized_targets, normalized_maxima or None, normalized_minima or None + return ( + normalized_targets, + _normalized_soc_events_or_original(soc_maxima, normalized_maxima), + _normalized_soc_events_or_original(soc_minima, normalized_minima), + ) def build_device_soc_values( From 9e26e034495341ffb34be954e111d5e1f6992424 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 22 May 2026 13:51:40 +0100 Subject: [PATCH 35/72] fix: normalize off-tick soc on schedule grid Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/models/planning/storage.py | 36 ++++++++++++++------ 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 05880fc8cf..5d4b4777a0 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -803,11 +803,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 device_constraints[d]["derivative max"] = consumption_capacity_d if soc_at_start[d] is not None: - if ( - sensor_d is not None - and sensor_d.event_resolution != timedelta(0) - and sensor_d.get_attribute("floor_datetimes_to_resolution", True) - ): + if _should_normalize_off_tick_soc_constraints(sensor_d): ( soc_targets[d], soc_maxima[d], @@ -1108,18 +1104,20 @@ def enable_relax_soc_constraints_for_off_tick_soc_constraints(self) -> None: def flex_model_has_off_tick_soc_constraints(self) -> bool: if isinstance(self.flex_model, dict): - resolution = self.resolution - if self.sensor is not None: - if not self.sensor.get_attribute("floor_datetimes_to_resolution", True): - return False - resolution = resolution or self.sensor.event_resolution + if not _should_normalize_off_tick_soc_constraints(self.sensor): + return False + resolution = _get_soc_constraint_resolution( + self.resolution, self.sensor, self.default_resolution + ) return flex_model_has_off_tick_soc_constraints( self.flex_model, resolution=resolution ) if isinstance(self.flex_model, list): + resolution = self.resolution or self.default_resolution return any( flex_model_has_off_tick_soc_constraints( - flex_model, resolution=self.resolution + flex_model.get("sensor-flex-model", flex_model), + resolution=resolution, ) for flex_model in self.flex_model ) @@ -1778,6 +1776,22 @@ def _is_on_schedule_tick(dt: datetime, resolution: timedelta) -> bool: return timestamp == timestamp.floor(resolution) +def _get_soc_constraint_resolution( + schedule_resolution: timedelta | None, + sensor: Sensor | None, + default_resolution: timedelta, +) -> timedelta | None: + if schedule_resolution not in (None, timedelta(0)): + return schedule_resolution + if sensor is not None and sensor.event_resolution != timedelta(0): + return sensor.event_resolution + return default_resolution + + +def _should_normalize_off_tick_soc_constraints(sensor: Sensor | None) -> bool: + return sensor is None or sensor.get_attribute("floor_datetimes_to_resolution", True) + + def flex_model_has_off_tick_soc_constraints( flex_model: dict, resolution: timedelta | None, From 9e4e7093f2e12814c3f20430fbab40e55addf454 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 22 May 2026 13:52:05 +0100 Subject: [PATCH 36/72] test: cover off-tick soc bounds Signed-off-by: Mohamed Belhsan Hmida --- .../models/planning/tests/test_storage.py | 187 +++++++++++++++++- 1 file changed, 186 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py index eaac9de055..5927f0655c 100644 --- a/flexmeasures/data/models/planning/tests/test_storage.py +++ b/flexmeasures/data/models/planning/tests/test_storage.py @@ -7,7 +7,10 @@ import pandas as pd from flexmeasures.data.models.planning import Scheduler -from flexmeasures.data.models.planning.storage import StorageScheduler +from flexmeasures.data.models.planning.storage import ( + StorageScheduler, + normalize_off_tick_soc_constraints, +) from flexmeasures.data.models.planning.utils import initialize_index from flexmeasures.data.models.time_series import Sensor, TimedBelief from flexmeasures.data.models.planning.tests.utils import ( @@ -297,6 +300,188 @@ def test_off_tick_soc_target_is_projected_to_storage_grid(add_battery_assets, db assert storage_constraints.loc[start + resolution, "equals"] == pytest.approx(4) +def test_off_tick_soc_target_is_projected_for_instantaneous_sensor( + add_battery_assets, db +): + _, battery = get_sensors_from_db( + db, add_battery_assets, battery_name="Test battery" + ) + instantaneous_power_sensor = Sensor( + name="instantaneous-power", + generic_asset=battery.generic_asset, + event_resolution=timedelta(0), + unit="MW", + ) + db.session.add(instantaneous_power_sensor) + db.session.flush() + tz = pytz.timezone("Europe/Amsterdam") + start = tz.localize(datetime(2015, 1, 1, 16, 45)) + end = tz.localize(datetime(2015, 1, 1, 17, 15)) + resolution = timedelta(minutes=15) + + scheduler = StorageScheduler( + instantaneous_power_sensor, + start, + end, + resolution, + flex_model={ + "soc-at-start": "0 MWh", + "soc-min": "0 MWh", + "soc-max": "1 MWh", + "power-capacity": "0.04 MW", + "consumption-capacity": "0.04 MW", + "production-capacity": "0 MW", + "roundtrip-efficiency": 1, + "storage-efficiency": 1, + "soc-targets": [ + { + "datetime": "2015-01-01T17:12:00+01:00", + "value": "1 MWh", + } + ], + }, + flex_context={ + "consumption-price": "0 EUR/MWh", + "production-price": "0 EUR/MWh", + "site-power-capacity": "1 MW", + }, + ) + + _, _, _, _, _, device_constraints, _, _ = scheduler._prepare(skip_validation=True) + storage_constraints = device_constraints[0].tz_convert(tz) + + assert storage_constraints.loc[start, "min"] == pytest.approx(0.992 * 4) + assert storage_constraints.loc[start + resolution, "equals"] == pytest.approx(4) + + +@pytest.mark.parametrize( + "soc_minima, soc_maxima, expected_previous_value, expected_next_value", + [ + ( + [ + { + "datetime": "2015-01-01T17:12:00+01:00", + "start": "2015-01-01T17:12:00+01:00", + "end": "2015-01-01T17:12:00+01:00", + "value": 1, + } + ], + None, + 0.992, + 1, + ), + ( + None, + [ + { + "datetime": "2015-01-01T17:12:00+01:00", + "start": "2015-01-01T17:12:00+01:00", + "end": "2015-01-01T17:12:00+01:00", + "value": 0.5, + } + ], + 0.5, + 0.502, + ), + ], +) +def test_off_tick_soc_bounds_are_projected_to_storage_grid( + soc_minima, + soc_maxima, + expected_previous_value, + expected_next_value, +): + tz = pytz.timezone("Europe/Amsterdam") + resolution = timedelta(minutes=15) + previous_tick = pd.Timestamp(tz.localize(datetime(2015, 1, 1, 17))) + next_tick = previous_tick + resolution + capacity = pd.Series( + 0.04, index=pd.date_range(previous_tick, next_tick, freq=resolution) + ) + + _, normalized_maxima, normalized_minima = normalize_off_tick_soc_constraints( + soc_targets=None, + soc_maxima=soc_maxima, + soc_minima=soc_minima, + consumption_capacity=capacity, + production_capacity=pd.Series(0, index=capacity.index), + resolution=resolution, + soc_min=0, + soc_max=1, + ) + + normalized_events = normalized_minima or normalized_maxima + + assert _soc_event_value_at(normalized_events, previous_tick) == pytest.approx( + expected_previous_value + ) + assert _soc_event_value_at(normalized_events, next_tick) == pytest.approx( + expected_next_value + ) + + +def _soc_event_value_at(events, dt): + matches = [ + event + for event in events + if pd.Timestamp(event["start"]) == dt and pd.Timestamp(event["end"]) == dt + ] + assert len(matches) == 1 + return matches[0]["value"] + + +def test_off_tick_soc_bounds_are_merged_on_the_same_storage_grid_tick(): + tz = pytz.timezone("Europe/Amsterdam") + resolution = timedelta(minutes=15) + previous_tick = pd.Timestamp(tz.localize(datetime(2015, 1, 1, 17))) + next_tick = previous_tick + resolution + capacity = pd.Series( + 0, index=pd.date_range(previous_tick, next_tick, freq=resolution) + ) + + _, normalized_maxima, normalized_minima = normalize_off_tick_soc_constraints( + soc_targets=None, + soc_maxima=[ + { + "datetime": tz.localize(datetime(2015, 1, 1, 17, 4)), + "start": tz.localize(datetime(2015, 1, 1, 17, 4)), + "end": tz.localize(datetime(2015, 1, 1, 17, 4)), + "value": 0.8, + }, + { + "datetime": tz.localize(datetime(2015, 1, 1, 17, 8)), + "start": tz.localize(datetime(2015, 1, 1, 17, 8)), + "end": tz.localize(datetime(2015, 1, 1, 17, 8)), + "value": 0.6, + }, + ], + soc_minima=[ + { + "datetime": tz.localize(datetime(2015, 1, 1, 17, 4)), + "start": tz.localize(datetime(2015, 1, 1, 17, 4)), + "end": tz.localize(datetime(2015, 1, 1, 17, 4)), + "value": 0.4, + }, + { + "datetime": tz.localize(datetime(2015, 1, 1, 17, 8)), + "start": tz.localize(datetime(2015, 1, 1, 17, 8)), + "end": tz.localize(datetime(2015, 1, 1, 17, 8)), + "value": 0.7, + }, + ], + consumption_capacity=capacity, + production_capacity=capacity, + resolution=resolution, + soc_min=0, + soc_max=1, + ) + + assert _soc_event_value_at(normalized_minima, previous_tick) == pytest.approx(0.7) + assert _soc_event_value_at(normalized_minima, next_tick) == pytest.approx(0.7) + assert _soc_event_value_at(normalized_maxima, previous_tick) == pytest.approx(0.6) + assert _soc_event_value_at(normalized_maxima, next_tick) == pytest.approx(0.6) + + def test_off_tick_soc_constraints_enable_relax_soc_constraints(add_battery_assets, db): _, battery = get_sensors_from_db( db, add_battery_assets, battery_name="Test battery" From 32a3d2b5e93460981beedd752dd57f6d1d29c217 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 22 May 2026 13:55:51 +0100 Subject: [PATCH 37/72] test: tolerate upload conversion precision Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py b/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py index 01cb7e95c6..43cbb06b5f 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py @@ -245,7 +245,7 @@ def test_upload_sensor_data_with_unit_conversion_success( len(beliefs) == expected_num_beliefs ), f"Fetched {len(beliefs)} beliefs from the database, expecting {expected_num_beliefs}." - assert [b.event_value for b in beliefs] == expected_event_values + assert [b.event_value for b in beliefs] == pytest.approx(expected_event_values) @pytest.mark.parametrize( From 10fc632192785b28caf225590bfbb43b90501fbc Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 22 May 2026 14:42:53 +0100 Subject: [PATCH 38/72] test: compare quantities by value Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/schemas/tests/test_sensor.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/flexmeasures/data/schemas/tests/test_sensor.py b/flexmeasures/data/schemas/tests/test_sensor.py index 50606a2bc4..a712922227 100644 --- a/flexmeasures/data/schemas/tests/test_sensor.py +++ b/flexmeasures/data/schemas/tests/test_sensor.py @@ -11,6 +11,11 @@ from marshmallow import ValidationError +def assert_quantity_matches(actual_quantity, expected_quantity) -> None: + assert str(actual_quantity.units) == str(expected_quantity.units) + assert actual_quantity.magnitude == pytest.approx(expected_quantity.magnitude) + + @pytest.mark.parametrize( "src_quantity, dst_unit, fails, exp_dst_quantity", [ @@ -84,10 +89,12 @@ def test_quantity_or_sensor_deserialize( try: dst_quantity = schema.deserialize(src_quantity) if isinstance(src_quantity, (ur.Quantity, int, float)): - assert dst_quantity == ur.Quantity(exp_dst_quantity) + assert_quantity_matches(dst_quantity, ur.Quantity(exp_dst_quantity)) assert str(dst_quantity) == exp_dst_quantity elif isinstance(src_quantity, list): - assert dst_quantity[0]["value"] == ur.Quantity(exp_dst_quantity) + assert_quantity_matches( + dst_quantity[0]["value"], ur.Quantity(exp_dst_quantity) + ) assert str(dst_quantity[0]["value"]) == exp_dst_quantity assert not fails except ValidationError as e: From 56ad4ebe71586392fbce31cae1397217ca550287 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 22 May 2026 18:30:08 +0100 Subject: [PATCH 39/72] test: avoid exact quantity string comparison Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/schemas/tests/test_sensor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/flexmeasures/data/schemas/tests/test_sensor.py b/flexmeasures/data/schemas/tests/test_sensor.py index a712922227..d07a6160d3 100644 --- a/flexmeasures/data/schemas/tests/test_sensor.py +++ b/flexmeasures/data/schemas/tests/test_sensor.py @@ -95,7 +95,6 @@ def test_quantity_or_sensor_deserialize( assert_quantity_matches( dst_quantity[0]["value"], ur.Quantity(exp_dst_quantity) ) - assert str(dst_quantity[0]["value"]) == exp_dst_quantity assert not fails except ValidationError as e: assert fails, e From 9268de88b25733c6b3e5608bbe00bfe44d50094e Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Tue, 26 May 2026 08:30:46 +0100 Subject: [PATCH 40/72] fix: skip flooring for instantaneous sensors Signed-off-by: Mohamed Belhsan Hmida --- .../data/schemas/scheduling/storage.py | 1 + .../data/schemas/tests/test_scheduling.py | 23 ++++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index 9dd32c2ed9..c6aaaca2db 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -240,6 +240,7 @@ def __init__( self.flooring_resolution = ( sensor.event_resolution if sensor is not None + and sensor.event_resolution != timedelta(0) and sensor.get_attribute("floor_datetimes_to_resolution", True) else None ) diff --git a/flexmeasures/data/schemas/tests/test_scheduling.py b/flexmeasures/data/schemas/tests/test_scheduling.py index 4fb1bf67af..cf2c7c52a7 100644 --- a/flexmeasures/data/schemas/tests/test_scheduling.py +++ b/flexmeasures/data/schemas/tests/test_scheduling.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timedelta import pytz import pytest @@ -14,6 +14,7 @@ StorageFlexModelSchema, DBStorageFlexModelSchema, ) +from flexmeasures.data.models.time_series import Sensor from flexmeasures.data.schemas.sensors import TimedEventSchema, VariableQuantityField @@ -162,6 +163,26 @@ def test_process_scheduler_flex_model_process_type(db, app, setup_dummy_sensors) assert process_scheduler_flex_model["process_type"] == ProcessType.SHIFTABLE +def test_storage_flex_model_schema_does_not_floor_instantaneous_sensor( + db, app, dummy_asset +): + sensor = Sensor( + "instantaneous power sensor", + generic_asset=dummy_asset, + event_resolution=timedelta(0), + unit="MW", + ) + db.session.add(sensor) + db.session.flush() + + schema = StorageFlexModelSchema( + sensor=sensor, + start=datetime(2023, 1, 1, tzinfo=pytz.UTC), + ) + + assert schema.flooring_resolution is None + + @pytest.mark.parametrize( "fields, fails", [ From 7c0c9b42f6a0e99c330bbd8ffe1fb220104f1db0 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Tue, 26 May 2026 08:30:52 +0100 Subject: [PATCH 41/72] test: fetch upload sensors by name Signed-off-by: Mohamed Belhsan Hmida --- .../api/v3_0/tests/test_sensor_schedules.py | 9 +-- .../v3_0/tests/test_sensors_api_freshdb.py | 75 +++++++------------ flexmeasures/api/v3_0/tests/utils.py | 5 ++ 3 files changed, 38 insertions(+), 51 deletions(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py index 2c5081a49c..6928c452ab 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_schedules.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_schedules.py @@ -10,7 +10,10 @@ from flexmeasures.api.common.responses import unknown_schedule, unrecognized_event from flexmeasures.api.tests.utils import check_deprecation -from flexmeasures.api.v3_0.tests.utils import message_for_trigger_schedule +from flexmeasures.api.v3_0.tests.utils import ( + get_sensor_by_name, + message_for_trigger_schedule, +) from flexmeasures.data.models.data_sources import DataSource from flexmeasures.data.models.time_series import Sensor from flexmeasures.utils.job_utils import work_on_rq @@ -32,10 +35,6 @@ def setup_capacity_sensor_on_asset_in_supplier_account(db, setup_generic_assets) return sensor -def get_sensor_by_name(asset, name: str) -> Sensor: - return next(sensor for sensor in asset.sensors if sensor.name == name) - - @pytest.mark.parametrize( "requesting_user", ["test_prosumer_user@seita.nl"], indirect=True ) diff --git a/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py b/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py index 01cb7e95c6..8c695a17e4 100644 --- a/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py +++ b/flexmeasures/api/v3_0/tests/test_sensors_api_freshdb.py @@ -8,29 +8,15 @@ from timely_beliefs import BeliefsDataFrame from flexmeasures.data.models.time_series import TimedBelief -from flexmeasures.api.v3_0.tests.utils import generate_csv_content - - -TEST_BATTERY_SENSOR_NAMES = ( - "power", - "power (kW)", - "energy (kWh)", - "state of charge", - "consumption sensor", - "cost sensor", -) - - -def assert_test_battery_sensor(sensor, sensor_index: int) -> None: - assert sensor.name == TEST_BATTERY_SENSOR_NAMES[sensor_index] +from flexmeasures.api.v3_0.tests.utils import generate_csv_content, get_sensor_by_name @pytest.mark.parametrize( - "requesting_user, sensor_index, data_unit, data_resolution, data_values, expected_event_values, expected_status", + "requesting_user, sensor_name, data_unit, data_resolution, data_values, expected_event_values, expected_status", [ ( "test_prosumer_user_2@seita.nl", - 2, # this sensor has unit=kWh, res=01:00 + "energy (kWh)", # this sensor has unit=kWh, res=01:00 "kWh", # No conversion needed - kWh to kWh timedelta(hours=1), # No resampling - 1 hour to 1 hour [45.3] * 4, @@ -39,7 +25,7 @@ def assert_test_battery_sensor(sensor, sensor_index: int) -> None: ), ( "test_prosumer_user_2@seita.nl", - 0, # this sensor has unit=MW, res=00:15 + "power", # this sensor has unit=MW, res=00:15 "kWh", # Conversion needed - kWh to MW timedelta(hours=1), # Upsampling - 1 hour to 15 minutes [45.3] * 4, @@ -50,7 +36,7 @@ def assert_test_battery_sensor(sensor, sensor_index: int) -> None: ), ( "test_prosumer_user_2@seita.nl", - 1, # this sensor has unit=kW, res=00:15 + "power (kW)", # this sensor has unit=kW, res=00:15 "MW", # Conversion needed - MW to kW timedelta(hours=1), # Upsampling - 1 hour to 15 minutes [2] * 6, @@ -61,7 +47,7 @@ def assert_test_battery_sensor(sensor, sensor_index: int) -> None: ), ( "test_prosumer_user_2@seita.nl", - 1, # this sensor has unit=kW, res=00:15 + "power (kW)", # this sensor has unit=kW, res=00:15 "kWh", # Conversion needed - kWh to kW # Upsampling - 30 minutes to 15 minutes timedelta(minutes=30), @@ -73,7 +59,7 @@ def assert_test_battery_sensor(sensor, sensor_index: int) -> None: ), ( "test_prosumer_user_2@seita.nl", - 2, # this sensor has unit=kWh, res=01:00 + "energy (kWh)", # this sensor has unit=kWh, res=01:00 "kWh", # No conversion needed - kWh to kWh timedelta(minutes=30), # Downsampling - 30 minutes to 1 hour [10, 20, 20, 40], @@ -85,7 +71,7 @@ def assert_test_battery_sensor(sensor, sensor_index: int) -> None: ), ( "test_prosumer_user_2@seita.nl", - 1, # this sensor has unit=kW, res=00:15 + "power (kW)", # this sensor has unit=kW, res=00:15 "kWh", # Conversion needed - kWh to kW # Downsampling - 7.5 minutes to 15 minutes timedelta(minutes=7, seconds=30), @@ -98,7 +84,7 @@ def assert_test_battery_sensor(sensor, sensor_index: int) -> None: ), ( "test_prosumer_user_2@seita.nl", - 1, # this sensor has unit=kW, res=00:15 + "power (kW)", # this sensor has unit=kW, res=00:15 "MW", # Conversion needed - MW to kW # Downsampling - 7.5 minutes to 15 minutes timedelta(minutes=7, seconds=30), @@ -112,7 +98,7 @@ def assert_test_battery_sensor(sensor, sensor_index: int) -> None: ), ( "test_prosumer_user_2@seita.nl", - 3, # this sensor has unit=kWh, res=00:00 + "state of charge", # this sensor has unit=kWh, res=00:00 "MWh", # Conversion needed - MWh to kWh # No resampling - 7.5 minutes to instantaneous timedelta(minutes=7, seconds=30), @@ -122,7 +108,7 @@ def assert_test_battery_sensor(sensor, sensor_index: int) -> None: ), ( "test_prosumer_user_2@seita.nl", - 4, # this sensor has unit=EUR/kWh, res=01:00 + "consumption sensor", # this sensor has unit=EUR/kWh, res=01:00 "EUR/MWh", # Conversion needed - EUR/MWh to EUR/kWh timedelta(minutes=30), # Downsampling - 30 minutes to 1 hour [200, 300, 400, 500], @@ -131,7 +117,7 @@ def assert_test_battery_sensor(sensor, sensor_index: int) -> None: ), ( "test_prosumer_user_2@seita.nl", - 4, # this sensor has unit=EUR/kWh, res=01:00 + "consumption sensor", # this sensor has unit=EUR/kWh, res=01:00 "EUR/kWh", # Conversion needed - EUR/kWh to EUR/kWh timedelta(hours=2), # Upsampling - 2 hours to 1 hour [200, 300, 400], @@ -140,7 +126,7 @@ def assert_test_battery_sensor(sensor, sensor_index: int) -> None: ), ( "test_prosumer_user_2@seita.nl", - 5, # this sensor has unit=EUR, res=01:00 + "cost sensor", # this sensor has unit=EUR, res=01:00 "kEUR", # Conversion needed - kEUR to EUR timedelta(minutes=30), # Downsampling - 30 minutes to 1 hour [2, 3, 4, 2], @@ -150,7 +136,7 @@ def assert_test_battery_sensor(sensor, sensor_index: int) -> None: ), ( "test_prosumer_user_2@seita.nl", - 5, # this sensor has unit=EUR, res=01:00 + "cost sensor", # this sensor has unit=EUR, res=01:00 "kEUR", # Conversion needed - kEUR to EUR timedelta(hours=2), # Upsampling - 2 hours to 1 hour [5, 6], @@ -163,7 +149,7 @@ def assert_test_battery_sensor(sensor, sensor_index: int) -> None: ), ( "test_prosumer_user_2@seita.nl", - 5, # this sensor has unit=EUR, res=01:00 + "cost sensor", # this sensor has unit=EUR, res=01:00 "kEUR", # Conversion needed - kEUR to EUR timedelta(hours=1), # No resampling - 1 hour (!) to 1 hour # Note that this test case could also define 2 hours between rows, but since there is only 1 row of data, @@ -181,7 +167,7 @@ def test_upload_sensor_data_with_unit_conversion_success( client, add_battery_assets_fresh_db, requesting_user, - sensor_index, + sensor_name, data_unit, data_resolution, data_values, @@ -200,8 +186,7 @@ def test_upload_sensor_data_with_unit_conversion_success( "2025-01-01T10:00:00+00:00" # This date would be used to generate CSV content ) test_battery = add_battery_assets_fresh_db["Test battery"] - sensor = test_battery.sensors[sensor_index] - assert_test_battery_sensor(sensor, sensor_index) + sensor = get_sensor_by_name(test_battery, sensor_name) num_test_intervals = len(data_values) print( f"Uploading data to sensor '{sensor.name}' with unit={sensor.unit} and resolution={sensor.event_resolution}." @@ -249,11 +234,11 @@ def test_upload_sensor_data_with_unit_conversion_success( @pytest.mark.parametrize( - "requesting_user, sensor_index, start_date, data_resolution, data_values, expected_event_starts, expected_event_values", + "requesting_user, sensor_name, start_date, data_resolution, data_values, expected_event_starts, expected_event_values", [ ( "test_prosumer_user_2@seita.nl", - 1, + "power (kW)", "2025-01-01T10:00:40+00:00", timedelta(minutes=15), [2, 3, 4], @@ -266,7 +251,7 @@ def test_upload_sensor_data_with_unit_conversion_success( ), ( "test_prosumer_user_2@seita.nl", - 2, + "energy (kWh)", "2025-01-01T10:00:40+00:00", timedelta(minutes=30), [10, 20, 20, 40], @@ -278,7 +263,7 @@ def test_upload_sensor_data_with_unit_conversion_success( ), ( "test_prosumer_user_2@seita.nl", - 1, + "power (kW)", "2025-01-01T10:00:40+00:00", timedelta(minutes=5), [10, 20, 30, 40], @@ -296,7 +281,7 @@ def test_upload_sensor_data_floors_offclock_datetimes( client, add_battery_assets_fresh_db, requesting_user, - sensor_index, + sensor_name, start_date, data_resolution, data_values, @@ -304,8 +289,7 @@ def test_upload_sensor_data_floors_offclock_datetimes( expected_event_values, ): test_battery = add_battery_assets_fresh_db["Test battery"] - sensor = test_battery.sensors[sensor_index] - assert_test_battery_sensor(sensor, sensor_index) + sensor = get_sensor_by_name(test_battery, sensor_name) csv_content = generate_csv_content( start_time_str=start_date, @@ -331,11 +315,11 @@ def test_upload_sensor_data_floors_offclock_datetimes( @pytest.mark.parametrize( - "requesting_user, sensor_index, data_unit, data_resolution, data_values, expected_err_msg, expected_status", + "requesting_user, sensor_name, data_unit, data_resolution, data_values, expected_err_msg, expected_status", [ ( "test_prosumer_user_2@seita.nl", - 1, # this sensor has unit=kW, res=00:15 + "power (kW)", # this sensor has unit=kW, res=00:15 "m/s", # Invalid conversion - m/s to kW timedelta(hours=1), # Upsampling - 1 hour to 15 minutes [45.3, 45.3], @@ -344,7 +328,7 @@ def test_upload_sensor_data_floors_offclock_datetimes( ), ( "test_prosumer_user_2@seita.nl", - 2, # this sensor has unit=kWh, res=01:00 + "energy (kWh)", # this sensor has unit=kWh, res=01:00 "kW", # Conversion needed - kW to kWh timedelta(minutes=30), # Downsampling - 30 minutes to 1 hour [20, 40, 40, 80], @@ -353,7 +337,7 @@ def test_upload_sensor_data_floors_offclock_datetimes( ), ( "test_prosumer_user_2@seita.nl", - 3, # this sensor has unit=kWh, res=00:00 + "state of charge", # this sensor has unit=kWh, res=00:00 "kW", # Conversion needed - kW to kWh # No resampling - 7.5 minutes to instantaneous timedelta(minutes=7, seconds=30), @@ -369,7 +353,7 @@ def test_upload_sensor_data_with_unit_conversion_failure( client, add_battery_assets_fresh_db, requesting_user, - sensor_index, + sensor_name, data_unit, data_resolution, data_values, @@ -386,8 +370,7 @@ def test_upload_sensor_data_with_unit_conversion_failure( "2025-01-01T10:00:00+00:00" # This date would be used to generate CSV content ) test_battery = add_battery_assets_fresh_db["Test battery"] - sensor = test_battery.sensors[sensor_index] - assert_test_battery_sensor(sensor, sensor_index) + sensor = get_sensor_by_name(test_battery, sensor_name) print( f"Uploading data to sensor '{sensor.name}' with unit={sensor.unit} and resolution={sensor.event_resolution}." ) diff --git a/flexmeasures/api/v3_0/tests/utils.py b/flexmeasures/api/v3_0/tests/utils.py index 5409174393..79683e868c 100644 --- a/flexmeasures/api/v3_0/tests/utils.py +++ b/flexmeasures/api/v3_0/tests/utils.py @@ -6,6 +6,11 @@ from flexmeasures import Asset, User from flexmeasures.data.models.audit_log import AssetAuditLog +from flexmeasures.data.models.time_series import Sensor + + +def get_sensor_by_name(asset: Asset, name: str) -> Sensor: + return next(sensor for sensor in asset.sensors if sensor.name == name) def make_sensor_data_request_for_gas_sensor( From baf55433bd4c95f6b05ea7518c450a98392822f0 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida <149331360+BelhsanHmida@users.noreply.github.com> Date: Tue, 26 May 2026 08:36:37 +0100 Subject: [PATCH 42/72] Update flexmeasures/api/v3_0/tests/test_sensor_data.py Co-authored-by: Felix Claessen <30658763+Flix6x@users.noreply.github.com> Signed-off-by: Mohamed Belhsan Hmida <149331360+BelhsanHmida@users.noreply.github.com> --- flexmeasures/api/v3_0/tests/test_sensor_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data.py b/flexmeasures/api/v3_0/tests/test_sensor_data.py index 797eb9f1e6..40e8455acc 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data.py @@ -442,7 +442,7 @@ def test_post_sensor_data_rejects_unknown_sensor_before_queueing( [-11.28] * 6, ), ( - "2021-06-09T00:00:40+02:00", + "2021-06-09T00:04:40+02:00", # floor rather than ceil to 5-min "2021-06-09T00:00:00+02:00", "2021-06-09T01:00:00+02:00", [100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 1100, 1200], From 8b2bb7eb853e803e373e2760d0dc7b3f1bc17425 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida <149331360+BelhsanHmida@users.noreply.github.com> Date: Wed, 27 May 2026 02:44:25 +0100 Subject: [PATCH 43/72] Update flexmeasures/api/v3_0/tests/test_sensor_data.py Co-authored-by: Felix Claessen <30658763+Flix6x@users.noreply.github.com> Signed-off-by: Mohamed Belhsan Hmida <149331360+BelhsanHmida@users.noreply.github.com> --- flexmeasures/api/v3_0/tests/test_sensor_data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flexmeasures/api/v3_0/tests/test_sensor_data.py b/flexmeasures/api/v3_0/tests/test_sensor_data.py index 40e8455acc..78b895523e 100644 --- a/flexmeasures/api/v3_0/tests/test_sensor_data.py +++ b/flexmeasures/api/v3_0/tests/test_sensor_data.py @@ -360,6 +360,7 @@ def test_post_sensor_data_rejects_large_json( "duration", "PT25M", "_schema", + # PT25M / 6 values = 4m10s "Resolution of 0:04:10 is incompatible", ), ("unit", "m", "_schema", "Required unit"), From 26a8e22bb0e1c3223531cb702e22732d76f474a6 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 5 Jun 2026 04:05:57 +0100 Subject: [PATCH 44/72] docs: update changelog for soc projection Signed-off-by: Mohamed Belhsan Hmida --- documentation/changelog.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/documentation/changelog.rst b/documentation/changelog.rst index f4fdd79045..c014c30490 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -9,7 +9,7 @@ v1.0.0 | July XX, 2026 New features ------------- -* Floor off-clock API datetimes to a non-instantaneous sensor's resolution by default when ingesting sensor data, uploading sensor data, and handling scheduler flex-model timed events; configurable with the ``floor_datetimes_to_resolution`` sensor attribute [see `PR #2146 `_] +* Floor off-clock API datetimes to a non-instantaneous sensor's resolution by default when ingesting sensor data, uploading sensor data, and handling scheduler flex-model timed events; configurable with the ``floor_datetimes_to_resolution`` sensor attribute [see `PR #2146 `_ and `PR #2194 `_] * Sensor references in flex-model and flex-context support various ways of filtering by source [see `PR #2209 `_] @@ -48,7 +48,6 @@ New features * Added a unified job status endpoint ``GET /api/v3_0/jobs/`` to retrieve the current execution status and result message for any background job [see `PR #2141 `_] * Add ``flexmeasures jobs inspect-job`` CLI command to show job status and metadata information (similar to the job status endpoint in the API) [see `PR #2202 `_] * New ``GET /api/v3_0/sources`` endpoint to list accessible data sources and defined types, with ``only_latest=true`` by default to return only the most recent version per source [see `PR #2126 `_] -* Floor off-clock API datetimes to a non-instantaneous sensor's resolution by default when ingesting sensor data, uploading sensor data, and handling scheduler flex-model timed events; configurable with the ``floor_datetimes_to_resolution`` sensor attribute [see `PR #2146 `_] * Add support for filtering sensor data GET requests by ``source-type`` on ``/api/v3_0/sensors//data`` [see `PR #2127 `_] * Making monitoring alerts more flexible: allow ``flexmeasures monitor`` alerts to target one or more user IDs or email addresses with ``--recipient``; ``flexmeasures monitor last-seen`` can now narrow monitored users to one or more accounts with ``--account`` or to client accounts with ``--consultancy`` [see `PR #2158 `_] * Improve LightGBM daily seasonal lag handling for sub-hourly forecasting sensors [see `PR #2157 `_] From 95e592258d8c2f73881e9cb17002f454fcfba5bb Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 5 Jun 2026 04:07:50 +0100 Subject: [PATCH 45/72] docs: clarify soc projection terminology Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/models/planning/storage.py | 4 ++-- flexmeasures/data/models/planning/tests/test_storage.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 0a81697d4e..38cf4abd29 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -1128,7 +1128,7 @@ def deserialize_flex_config(self): return self.flex_model def enable_relax_soc_constraints_for_off_tick_soc_constraints(self) -> None: - """Relax SoC constraints when off-tick SoC events require grid projection.""" + """Relax SoC constraints when off-tick SoC events require scheduling-tick projection.""" if self.flex_context is None: self.flex_context = {} if not self.flex_model_has_off_tick_soc_constraints(): @@ -2125,7 +2125,7 @@ def normalize_off_tick_soc_constraints( list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, ]: - """Project off-tick point-like SoC constraints onto the scheduling grid.""" + """Project off-tick point-like SoC constraints onto the scheduling ticks.""" if not any( isinstance(soc_events, list) and soc_events diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py index 3a3609e4d0..4a8c190396 100644 --- a/flexmeasures/data/models/planning/tests/test_storage.py +++ b/flexmeasures/data/models/planning/tests/test_storage.py @@ -293,7 +293,7 @@ def test_battery_relaxation(add_battery_assets, db): ) # 100 EUR/(kW*h) * 0.025 MW * 1000 kW/MW * 4 hours -def test_off_tick_soc_target_is_projected_to_storage_grid(add_battery_assets, db): +def test_off_tick_soc_target_is_projected_to_scheduling_ticks(add_battery_assets, db): _, battery = get_sensors_from_db( db, add_battery_assets, battery_name="Test battery" ) @@ -423,7 +423,7 @@ def test_off_tick_soc_target_is_projected_for_instantaneous_sensor( ), ], ) -def test_off_tick_soc_bounds_are_projected_to_storage_grid( +def test_off_tick_soc_bounds_are_projected_to_scheduling_ticks( soc_minima, soc_maxima, expected_previous_value, @@ -468,7 +468,7 @@ def _soc_event_value_at(events, dt): return matches[0]["value"] -def test_off_tick_soc_bounds_are_merged_on_the_same_storage_grid_tick(): +def test_off_tick_soc_bounds_are_merged_on_the_same_scheduling_tick(): tz = pytz.timezone("Europe/Amsterdam") resolution = timedelta(minutes=15) previous_tick = pd.Timestamp(tz.localize(datetime(2015, 1, 1, 17))) From 8b07cfa2e96846ff0d1177de507587825ba53b16 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 5 Jun 2026 04:15:57 +0100 Subject: [PATCH 46/72] refactor: rename soc normalization to projection Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/models/planning/storage.py | 56 +++++++++---------- .../models/planning/tests/test_storage.py | 20 +++---- 2 files changed, 35 insertions(+), 41 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 38cf4abd29..fcf0f24191 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -806,12 +806,12 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 device_constraints[d]["derivative max"] = consumption_capacity_d if soc_at_start[d] is not None: - if _should_normalize_off_tick_soc_constraints(sensor_d): + if _should_project_off_tick_soc_constraints(sensor_d): ( soc_targets[d], soc_maxima[d], soc_minima[d], - ) = normalize_off_tick_soc_constraints( + ) = project_off_tick_soc_constraints( soc_targets[d], soc_maxima[d], soc_minima[d], @@ -1137,7 +1137,7 @@ def enable_relax_soc_constraints_for_off_tick_soc_constraints(self) -> None: def flex_model_has_off_tick_soc_constraints(self) -> bool: if isinstance(self.flex_model, dict): - if not _should_normalize_off_tick_soc_constraints(self.sensor): + if not _should_project_off_tick_soc_constraints(self.sensor): return False resolution = _get_soc_constraint_resolution( self.resolution, self.sensor, self.default_resolution @@ -1991,7 +1991,7 @@ def _get_soc_constraint_resolution( return default_resolution -def _should_normalize_off_tick_soc_constraints(sensor: Sensor | None) -> bool: +def _should_project_off_tick_soc_constraints(sensor: Sensor | None) -> bool: return sensor is None or sensor.get_attribute("floor_datetimes_to_resolution", True) @@ -2092,20 +2092,20 @@ def _add_soc_bound( soc_events.append(soc_event) -def _normalized_soc_events_or_original( +def _projected_soc_events_or_original( original_soc_events: ( list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None ), - normalized_soc_events: list[dict[str, datetime | float]], + projected_soc_events: list[dict[str, datetime | float]], ) -> list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None: if isinstance(original_soc_events, list): - return normalized_soc_events - if original_soc_events is None and normalized_soc_events: - return normalized_soc_events + return projected_soc_events + if original_soc_events is None and projected_soc_events: + return projected_soc_events return original_soc_events -def normalize_off_tick_soc_constraints( +def project_off_tick_soc_constraints( soc_targets: ( list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None ), @@ -2133,23 +2133,19 @@ def normalize_off_tick_soc_constraints( ): return soc_targets, soc_maxima, soc_minima - normalized_minima = ( - copy.deepcopy(soc_minima) if isinstance(soc_minima, list) else [] - ) - normalized_maxima = ( - copy.deepcopy(soc_maxima) if isinstance(soc_maxima, list) else [] - ) + projected_minima = copy.deepcopy(soc_minima) if isinstance(soc_minima, list) else [] + projected_maxima = copy.deepcopy(soc_maxima) if isinstance(soc_maxima, list) else [] soc_min_value = _soc_value_in_mwh(soc_min) soc_max_value = _soc_value_in_mwh(soc_max) if isinstance(soc_targets, list): - normalized_targets = [] + projected_targets = [] for soc_target in soc_targets: if soc_target["start"] != soc_target["end"] or _is_on_schedule_tick( soc_target["end"], resolution ): - normalized_targets.append(copy.copy(soc_target)) + projected_targets.append(copy.copy(soc_target)) continue target_time = pd.Timestamp(soc_target["end"]) @@ -2164,11 +2160,9 @@ def normalize_off_tick_soc_constraints( production_capacity, previous_tick, target_time, resolution ) - normalized_targets.append( - _soc_event_at(soc_target, next_tick, target_value) - ) + projected_targets.append(_soc_event_at(soc_target, next_tick, target_value)) _add_soc_bound( - normalized_minima, + projected_minima, _soc_event_at( soc_target, previous_tick, @@ -2177,7 +2171,7 @@ def normalize_off_tick_soc_constraints( bound_type="min", ) _add_soc_bound( - normalized_maxima, + projected_maxima, _soc_event_at( soc_target, previous_tick, @@ -2186,7 +2180,7 @@ def normalize_off_tick_soc_constraints( bound_type="max", ) else: - normalized_targets = soc_targets + projected_targets = soc_targets if isinstance(soc_minima, list): soc_minimum_events = copy.deepcopy(soc_minima) @@ -2203,7 +2197,7 @@ def normalize_off_tick_soc_constraints( next_tick = minimum_time.ceil(resolution) minimum_value = _soc_value_in_mwh(soc_minimum["value"]) _add_soc_bound( - normalized_minima, + projected_minima, _soc_event_at( soc_minimum, previous_tick, @@ -2218,7 +2212,7 @@ def normalize_off_tick_soc_constraints( bound_type="min", ) _add_soc_bound( - normalized_minima, + projected_minima, _soc_event_at( soc_minimum, next_tick, @@ -2248,7 +2242,7 @@ def normalize_off_tick_soc_constraints( next_tick = maximum_time.ceil(resolution) maximum_value = _soc_value_in_mwh(soc_maximum["value"]) _add_soc_bound( - normalized_maxima, + projected_maxima, _soc_event_at( soc_maximum, previous_tick, @@ -2263,7 +2257,7 @@ def normalize_off_tick_soc_constraints( bound_type="max", ) _add_soc_bound( - normalized_maxima, + projected_maxima, _soc_event_at( soc_maximum, next_tick, @@ -2279,9 +2273,9 @@ def normalize_off_tick_soc_constraints( ) return ( - normalized_targets, - _normalized_soc_events_or_original(soc_maxima, normalized_maxima), - _normalized_soc_events_or_original(soc_minima, normalized_minima), + projected_targets, + _projected_soc_events_or_original(soc_maxima, projected_maxima), + _projected_soc_events_or_original(soc_minima, projected_minima), ) diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py index 4a8c190396..f2ec05eedd 100644 --- a/flexmeasures/data/models/planning/tests/test_storage.py +++ b/flexmeasures/data/models/planning/tests/test_storage.py @@ -9,7 +9,7 @@ from flexmeasures.data.models.planning import Scheduler from flexmeasures.data.models.planning.storage import ( StorageScheduler, - normalize_off_tick_soc_constraints, + project_off_tick_soc_constraints, ) from flexmeasures.data.models.planning.utils import initialize_index from flexmeasures.data.models.time_series import Sensor, TimedBelief @@ -437,7 +437,7 @@ def test_off_tick_soc_bounds_are_projected_to_scheduling_ticks( 0.04, index=pd.date_range(previous_tick, next_tick, freq=resolution) ) - _, normalized_maxima, normalized_minima = normalize_off_tick_soc_constraints( + _, projected_maxima, projected_minima = project_off_tick_soc_constraints( soc_targets=None, soc_maxima=soc_maxima, soc_minima=soc_minima, @@ -448,12 +448,12 @@ def test_off_tick_soc_bounds_are_projected_to_scheduling_ticks( soc_max=1, ) - normalized_events = normalized_minima or normalized_maxima + projected_events = projected_minima or projected_maxima - assert _soc_event_value_at(normalized_events, previous_tick) == pytest.approx( + assert _soc_event_value_at(projected_events, previous_tick) == pytest.approx( expected_previous_value ) - assert _soc_event_value_at(normalized_events, next_tick) == pytest.approx( + assert _soc_event_value_at(projected_events, next_tick) == pytest.approx( expected_next_value ) @@ -477,7 +477,7 @@ def test_off_tick_soc_bounds_are_merged_on_the_same_scheduling_tick(): 0, index=pd.date_range(previous_tick, next_tick, freq=resolution) ) - _, normalized_maxima, normalized_minima = normalize_off_tick_soc_constraints( + _, projected_maxima, projected_minima = project_off_tick_soc_constraints( soc_targets=None, soc_maxima=[ { @@ -514,10 +514,10 @@ def test_off_tick_soc_bounds_are_merged_on_the_same_scheduling_tick(): soc_max=1, ) - assert _soc_event_value_at(normalized_minima, previous_tick) == pytest.approx(0.7) - assert _soc_event_value_at(normalized_minima, next_tick) == pytest.approx(0.7) - assert _soc_event_value_at(normalized_maxima, previous_tick) == pytest.approx(0.6) - assert _soc_event_value_at(normalized_maxima, next_tick) == pytest.approx(0.6) + assert _soc_event_value_at(projected_minima, previous_tick) == pytest.approx(0.7) + assert _soc_event_value_at(projected_minima, next_tick) == pytest.approx(0.7) + assert _soc_event_value_at(projected_maxima, previous_tick) == pytest.approx(0.6) + assert _soc_event_value_at(projected_maxima, next_tick) == pytest.approx(0.6) def test_off_tick_soc_constraints_enable_relax_soc_constraints(add_battery_assets, db): From 6568284f233c6752774d480a723add65c92ee11c Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 5 Jun 2026 04:23:05 +0100 Subject: [PATCH 47/72] test: explain soc projection assertions Signed-off-by: Mohamed Belhsan Hmida --- .../models/planning/tests/test_storage.py | 54 +++++++++++++------ 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py index f2ec05eedd..82799c3b19 100644 --- a/flexmeasures/data/models/planning/tests/test_storage.py +++ b/flexmeasures/data/models/planning/tests/test_storage.py @@ -333,9 +333,15 @@ def test_off_tick_soc_target_is_projected_to_scheduling_ticks(add_battery_assets _, _, _, _, _, device_constraints, _, _ = scheduler._prepare(skip_validation=True) storage_constraints = device_constraints[0].tz_convert(tz) - assert pd.isna(storage_constraints.loc[start, "equals"]) - assert storage_constraints.loc[start, "min"] == pytest.approx(0.992 * 4) - assert storage_constraints.loc[start + resolution, "equals"] == pytest.approx(4) + assert pd.isna( + storage_constraints.loc[start, "equals"] + ), "off-tick targets should not become exact constraints on the previous tick" + assert storage_constraints.loc[start, "min"] == pytest.approx( + 0.992 * 4 + ), "previous tick should allow charging the missing 0.008 MWh before the target time" + assert storage_constraints.loc[start + resolution, "equals"] == pytest.approx( + 4 + ), "next tick should carry the projected exact target" def test_off_tick_soc_target_is_projected_for_instantaneous_sensor( @@ -388,8 +394,12 @@ def test_off_tick_soc_target_is_projected_for_instantaneous_sensor( _, _, _, _, _, device_constraints, _, _ = scheduler._prepare(skip_validation=True) storage_constraints = device_constraints[0].tz_convert(tz) - assert storage_constraints.loc[start, "min"] == pytest.approx(0.992 * 4) - assert storage_constraints.loc[start + resolution, "equals"] == pytest.approx(4) + assert storage_constraints.loc[start, "min"] == pytest.approx( + 0.992 * 4 + ), "instantaneous sensors should still project the previous-tick minimum" + assert storage_constraints.loc[start + resolution, "equals"] == pytest.approx( + 4 + ), "instantaneous sensors should still project the exact target to the next tick" @pytest.mark.parametrize( @@ -452,10 +462,10 @@ def test_off_tick_soc_bounds_are_projected_to_scheduling_ticks( assert _soc_event_value_at(projected_events, previous_tick) == pytest.approx( expected_previous_value - ) + ), "previous tick should use the capacity-adjusted projected SoC bound" assert _soc_event_value_at(projected_events, next_tick) == pytest.approx( expected_next_value - ) + ), "next tick should use the projected SoC bound implied by reachability" def _soc_event_value_at(events, dt): @@ -464,7 +474,7 @@ def _soc_event_value_at(events, dt): for event in events if pd.Timestamp(event["start"]) == dt and pd.Timestamp(event["end"]) == dt ] - assert len(matches) == 1 + assert len(matches) == 1, "projection should create exactly one event per tick" return matches[0]["value"] @@ -514,10 +524,18 @@ def test_off_tick_soc_bounds_are_merged_on_the_same_scheduling_tick(): soc_max=1, ) - assert _soc_event_value_at(projected_minima, previous_tick) == pytest.approx(0.7) - assert _soc_event_value_at(projected_minima, next_tick) == pytest.approx(0.7) - assert _soc_event_value_at(projected_maxima, previous_tick) == pytest.approx(0.6) - assert _soc_event_value_at(projected_maxima, next_tick) == pytest.approx(0.6) + assert _soc_event_value_at(projected_minima, previous_tick) == pytest.approx( + 0.7 + ), "merged minima should keep the stricter lower bound on the previous tick" + assert _soc_event_value_at(projected_minima, next_tick) == pytest.approx( + 0.7 + ), "merged minima should keep the stricter lower bound on the next tick" + assert _soc_event_value_at(projected_maxima, previous_tick) == pytest.approx( + 0.6 + ), "merged maxima should keep the stricter upper bound on the previous tick" + assert _soc_event_value_at(projected_maxima, next_tick) == pytest.approx( + 0.6 + ), "merged maxima should keep the stricter upper bound on the next tick" def test_off_tick_soc_constraints_enable_relax_soc_constraints(add_battery_assets, db): @@ -555,9 +573,15 @@ def test_off_tick_soc_constraints_enable_relax_soc_constraints(add_battery_asset scheduler.deserialize_config() - assert scheduler.flex_context["relax_soc_constraints"] is True - assert scheduler.flex_context["soc_minima_breach_price"] is not None - assert scheduler.flex_context["soc_maxima_breach_price"] is not None + assert ( + scheduler.flex_context["relax_soc_constraints"] is True + ), "off-tick SoC constraints should automatically enable SoC relaxation" + assert ( + scheduler.flex_context["soc_minima_breach_price"] is not None + ), "auto-enabled SoC relaxation should include a minima breach price" + assert ( + scheduler.flex_context["soc_maxima_breach_price"] is not None + ), "auto-enabled SoC relaxation should include a maxima breach price" def test_deserialize_storage_soc_at_start_from_state_of_charge_sensor( From 2a3309be89c07f1a7e9222cd5e22aa0193999860 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida <149331360+BelhsanHmida@users.noreply.github.com> Date: Fri, 5 Jun 2026 04:24:55 +0100 Subject: [PATCH 48/72] Update flexmeasures/data/models/planning/storage.py Co-authored-by: Felix Claessen <30658763+Flix6x@users.noreply.github.com> Signed-off-by: Mohamed Belhsan Hmida <149331360+BelhsanHmida@users.noreply.github.com> --- flexmeasures/data/models/planning/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index fcf0f24191..bef345322e 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -1983,7 +1983,7 @@ def _get_soc_constraint_resolution( schedule_resolution: timedelta | None, sensor: Sensor | None, default_resolution: timedelta, -) -> timedelta | None: +) -> timedelta: if schedule_resolution not in (None, timedelta(0)): return schedule_resolution if sensor is not None and sensor.event_resolution != timedelta(0): From 0ab3ccdfe24459532343151281e120da12ebd8aa Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 5 Jun 2026 04:32:39 +0100 Subject: [PATCH 49/72] docs: explain soc constraint projection policy Signed-off-by: Mohamed Belhsan Hmida --- documentation/concepts/data-model.rst | 20 +++++++++++++++++++ documentation/features/scheduling.rst | 2 ++ flexmeasures/data/models/planning/storage.py | 20 ++++++++++++++++++- .../models/planning/tests/test_storage.py | 5 +++++ .../data/schemas/scheduling/metadata.py | 6 +++--- 5 files changed, 49 insertions(+), 4 deletions(-) diff --git a/documentation/concepts/data-model.rst b/documentation/concepts/data-model.rst index 9b2549cc78..cc91497e78 100644 --- a/documentation/concepts/data-model.rst +++ b/documentation/concepts/data-model.rst @@ -100,6 +100,26 @@ Each belief links to a sensor and a data source. Here are two examples: See also :ref:`one_or_multiple_sensors` for guidance on when such beliefs are best recorded on one shared sensor and when separate sensors are preferable. +.. _projecting_scheduling_constraints: + +Projecting scheduling constraints to a fixed resolution +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Schedulers operate on a fixed scheduling resolution, such as 15 minutes. +This means the optimization problem can enforce state-of-charge constraints only on the scheduling ticks implied by that resolution. +When a storage flex-model contains point-like ``soc-targets``, ``soc-minima`` or ``soc-maxima`` between two scheduling ticks, FlexMeasures projects those constraints onto the surrounding ticks instead of simply flooring them. + +For an off-tick ``soc-target``, the target value is moved to the next scheduling tick as an exact target. +The previous tick receives lower and upper bounds that reflect how much the asset could still charge or discharge between the previous tick and the original target time. +For off-tick ``soc-minima``, both surrounding ticks receive lower bounds that preserve whether the requested minimum can still be reached. +For off-tick ``soc-maxima``, both surrounding ticks receive upper bounds with the same reachability logic. + +The projection uses the ``consumption-capacity`` and ``production-capacity`` active at the relevant ticks. +If multiple projected lower bounds land on the same tick, the highest lower bound is kept. +If multiple projected upper bounds land on the same tick, the lowest upper bound is kept. +Because projection can introduce additional bounds and more complex combinations can become infeasible, FlexMeasures enables ``relax-soc-constraints`` automatically when off-tick SoC constraints are submitted. + + .. _signs_of_power_beliefs: About signs of power & energy values diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst index 1643330a6d..22e5190b53 100644 --- a/documentation/features/scheduling.rst +++ b/documentation/features/scheduling.rst @@ -253,6 +253,8 @@ For more details on the possible formats for field values, see :ref:`variable_qu .. [#minimum_overlap] In case this field defines partially overlapping time periods, the minimum value is selected. See :ref:`variable_quantities`. +.. [#projecting_scheduling_constraints] Off-tick ``soc-targets``, ``soc-minima`` and ``soc-maxima`` are projected to the surrounding scheduling ticks. See :ref:`projecting_scheduling_constraints`. + For more details on the possible formats for field values, see :ref:`variable_quantities`. Usually, not the whole flexibility model is needed. diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index bef345322e..7da3e7a194 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -2125,7 +2125,25 @@ def project_off_tick_soc_constraints( list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, ]: - """Project off-tick point-like SoC constraints onto the scheduling ticks.""" + """Project off-tick point-like SoC constraints onto scheduling ticks. + + The scheduler can only enforce constraints at its fixed scheduling resolution. + Point-like ``soc-targets``, ``soc-minima`` and ``soc-maxima`` that fall between + two scheduling ticks are therefore replaced by constraints on the previous and + next tick that preserve reachability using the available charge and discharge + capacity between the original event time and those ticks. + + ``soc-targets`` are projected to an exact target on the next tick, plus + capacity-adjusted lower and upper bounds on the previous tick. ``soc-minima`` + become lower bounds on both surrounding ticks, and ``soc-maxima`` become upper + bounds on both surrounding ticks. If multiple projected bounds land on the same + tick, the stricter lower or upper bound is kept. + + Returns ``(soc_targets, soc_maxima, soc_minima)`` with projected list-based + timed events. Non-list specifications such as sensors, series, fixed + quantities, or ``None`` are returned unchanged unless projected bounds need to + be added to a missing list. + """ if not any( isinstance(soc_events, list) and soc_events diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py index 82799c3b19..d81e1d967f 100644 --- a/flexmeasures/data/models/planning/tests/test_storage.py +++ b/flexmeasures/data/models/planning/tests/test_storage.py @@ -294,6 +294,7 @@ def test_battery_relaxation(add_battery_assets, db): def test_off_tick_soc_target_is_projected_to_scheduling_ticks(add_battery_assets, db): + """Off-tick targets become a next-tick target and a reachable previous-tick bound.""" _, battery = get_sensors_from_db( db, add_battery_assets, battery_name="Test battery" ) @@ -347,6 +348,7 @@ def test_off_tick_soc_target_is_projected_to_scheduling_ticks(add_battery_assets def test_off_tick_soc_target_is_projected_for_instantaneous_sensor( add_battery_assets, db ): + """Off-tick projection also applies when the scheduled sensor is instantaneous.""" _, battery = get_sensors_from_db( db, add_battery_assets, battery_name="Test battery" ) @@ -439,6 +441,7 @@ def test_off_tick_soc_bounds_are_projected_to_scheduling_ticks( expected_previous_value, expected_next_value, ): + """Off-tick minima and maxima are projected as reachable bounds on surrounding ticks.""" tz = pytz.timezone("Europe/Amsterdam") resolution = timedelta(minutes=15) previous_tick = pd.Timestamp(tz.localize(datetime(2015, 1, 1, 17))) @@ -479,6 +482,7 @@ def _soc_event_value_at(events, dt): def test_off_tick_soc_bounds_are_merged_on_the_same_scheduling_tick(): + """Projected bounds sharing a tick keep the stricter minimum or maximum.""" tz = pytz.timezone("Europe/Amsterdam") resolution = timedelta(minutes=15) previous_tick = pd.Timestamp(tz.localize(datetime(2015, 1, 1, 17))) @@ -539,6 +543,7 @@ def test_off_tick_soc_bounds_are_merged_on_the_same_scheduling_tick(): def test_off_tick_soc_constraints_enable_relax_soc_constraints(add_battery_assets, db): + """Off-tick SoC constraints enable relaxation because projection can add bounds.""" _, battery = get_sensors_from_db( db, add_battery_assets, battery_name="Test battery" ) diff --git a/flexmeasures/data/schemas/scheduling/metadata.py b/flexmeasures/data/schemas/scheduling/metadata.py index 2aaa23b9a1..57ef666dee 100644 --- a/flexmeasures/data/schemas/scheduling/metadata.py +++ b/flexmeasures/data/schemas/scheduling/metadata.py @@ -245,7 +245,7 @@ def to_dict(self): SOC_MINIMA = MetaData( description="""Set points that form lower boundaries, e.g. to target a full car battery in the morning. If a ``soc-minima-breach-price`` is defined, the ``soc-minima`` become soft constraints in the optimization problem. -Otherwise, they become hard constraints. [#maximum_overlap]_. Both single points in time and ranges are possible, see example.""", +Otherwise, they become hard constraints. [#maximum_overlap]_. Both single points in time and ranges are possible, see example. [#projecting_scheduling_constraints]_""", example=[ {"datetime": "2024-02-05T08:00:00+01:00", "value": "8.2 kWh"}, { @@ -258,7 +258,7 @@ def to_dict(self): SOC_MAXIMA = MetaData( description="""Set points that form upper boundaries at certain times, e.g. to target an empty heat buffer before a maintenance window. If a ``soc-maxima-breach-price`` is defined, the ``soc-maxima`` become soft constraints in the optimization problem. -Otherwise, they become hard constraints. [#minimum_overlap]_""", +Otherwise, they become hard constraints. [#minimum_overlap]_ [#projecting_scheduling_constraints]_""", example=[ { "value": "51 kWh", @@ -270,7 +270,7 @@ def to_dict(self): SOC_TARGETS = MetaData( description=""" Exact set point(s) of the storage's state of charge that the scheduler needs to realize. -These are hard constraints, which means that any infeasible state-of-charge targets would prevent a complete schedule from being computed. +These are hard constraints, which means that any infeasible state-of-charge targets would prevent a complete schedule from being computed. [#projecting_scheduling_constraints]_ """, example=[{"datetime": "2024-02-05T08:00:00+01:00", "value": "3.2 kWh"}], ) From 26c85a6cd53499e95c327b9e3cb6805281621dc6 Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 5 Jun 2026 12:42:04 +0100 Subject: [PATCH 50/72] fix: support missing soc bounds in projection Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/models/planning/storage.py | 44 +++++++++++++------ .../models/planning/tests/test_storage.py | 36 +++++++++++++++ 2 files changed, 66 insertions(+), 14 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 7da3e7a194..4b4cf94442 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -2029,6 +2029,20 @@ def _soc_value_in_mwh(value: ur.Quantity | float | int) -> float: return float(value) +def _optional_soc_value_in_mwh(value: ur.Quantity | float | int | None) -> float | None: + if value is None: + return None + return _soc_value_in_mwh(value) + + +def _clamp_soc_min(value: float, soc_min: float | None) -> float: + return value if soc_min is None else max(soc_min, value) + + +def _clamp_soc_max(value: float, soc_max: float | None) -> float: + return value if soc_max is None else min(soc_max, value) + + def _soc_event_at( soc_event: dict[str, datetime | float], dt: pd.Timestamp, @@ -2118,8 +2132,8 @@ def project_off_tick_soc_constraints( consumption_capacity: pd.Series, production_capacity: pd.Series, resolution: timedelta, - soc_min: ur.Quantity | float, - soc_max: ur.Quantity | float, + soc_min: ur.Quantity | float | None, + soc_max: ur.Quantity | float | None, ) -> tuple[ list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, @@ -2154,8 +2168,8 @@ def project_off_tick_soc_constraints( projected_minima = copy.deepcopy(soc_minima) if isinstance(soc_minima, list) else [] projected_maxima = copy.deepcopy(soc_maxima) if isinstance(soc_maxima, list) else [] - soc_min_value = _soc_value_in_mwh(soc_min) - soc_max_value = _soc_value_in_mwh(soc_max) + soc_min_value = _optional_soc_value_in_mwh(soc_min) + soc_max_value = _optional_soc_value_in_mwh(soc_max) if isinstance(soc_targets, list): projected_targets = [] @@ -2184,7 +2198,7 @@ def project_off_tick_soc_constraints( _soc_event_at( soc_target, previous_tick, - max(soc_min_value, target_value - charge_before_target), + _clamp_soc_min(target_value - charge_before_target, soc_min_value), ), bound_type="min", ) @@ -2193,7 +2207,9 @@ def project_off_tick_soc_constraints( _soc_event_at( soc_target, previous_tick, - min(soc_max_value, target_value + discharge_before_target), + _clamp_soc_max( + target_value + discharge_before_target, soc_max_value + ), ), bound_type="max", ) @@ -2219,12 +2235,12 @@ def project_off_tick_soc_constraints( _soc_event_at( soc_minimum, previous_tick, - max( - soc_min_value, + _clamp_soc_min( minimum_value - _energy_capacity_between( consumption_capacity, previous_tick, minimum_time, resolution ), + soc_min_value, ), ), bound_type="min", @@ -2234,12 +2250,12 @@ def project_off_tick_soc_constraints( _soc_event_at( soc_minimum, next_tick, - max( - soc_min_value, + _clamp_soc_min( minimum_value - _energy_capacity_between( production_capacity, minimum_time, next_tick, resolution ), + soc_min_value, ), ), bound_type="min", @@ -2264,12 +2280,12 @@ def project_off_tick_soc_constraints( _soc_event_at( soc_maximum, previous_tick, - min( - soc_max_value, + _clamp_soc_max( maximum_value + _energy_capacity_between( production_capacity, previous_tick, maximum_time, resolution ), + soc_max_value, ), ), bound_type="max", @@ -2279,12 +2295,12 @@ def project_off_tick_soc_constraints( _soc_event_at( soc_maximum, next_tick, - min( - soc_max_value, + _clamp_soc_max( maximum_value + _energy_capacity_between( consumption_capacity, maximum_time, next_tick, resolution ), + soc_max_value, ), ), bound_type="max", diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py index d81e1d967f..a68a35f6d8 100644 --- a/flexmeasures/data/models/planning/tests/test_storage.py +++ b/flexmeasures/data/models/planning/tests/test_storage.py @@ -471,6 +471,42 @@ def test_off_tick_soc_bounds_are_projected_to_scheduling_ticks( ), "next tick should use the projected SoC bound implied by reachability" +def test_off_tick_soc_projection_accepts_missing_global_bounds(): + """Missing global SoC bounds leave projected off-tick bounds unclamped.""" + tz = pytz.timezone("Europe/Amsterdam") + resolution = timedelta(minutes=15) + previous_tick = pd.Timestamp(tz.localize(datetime(2015, 1, 1, 17))) + next_tick = previous_tick + resolution + capacity = pd.Series( + 0.04, index=pd.date_range(previous_tick, next_tick, freq=resolution) + ) + + _, projected_maxima, projected_minima = project_off_tick_soc_constraints( + soc_targets=[ + { + "datetime": "2015-01-01T17:12:00+01:00", + "start": "2015-01-01T17:12:00+01:00", + "end": "2015-01-01T17:12:00+01:00", + "value": 0.5, + } + ], + soc_maxima=None, + soc_minima=None, + consumption_capacity=capacity, + production_capacity=capacity, + resolution=resolution, + soc_min=None, + soc_max=None, + ) + + assert _soc_event_value_at(projected_minima, previous_tick) == pytest.approx( + 0.492 + ), "missing global soc-min should not clamp the projected previous-tick minimum" + assert _soc_event_value_at(projected_maxima, previous_tick) == pytest.approx( + 0.508 + ), "missing global soc-max should not clamp the projected previous-tick maximum" + + def _soc_event_value_at(events, dt): matches = [ event From dc2bf9a81d483b0efe60ea7f6b993c2a570ae0fe Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 5 Jun 2026 12:56:50 +0100 Subject: [PATCH 51/72] refactor: move off-tick soc detection to scheduling schema Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/models/planning/storage.py | 113 +++++------------- .../data/schemas/scheduling/storage.py | 23 ++++ flexmeasures/data/schemas/scheduling/utils.py | 61 ++++++++++ 3 files changed, 111 insertions(+), 86 deletions(-) create mode 100644 flexmeasures/data/schemas/scheduling/utils.py diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 4b4cf94442..66fe9ea3b2 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -35,6 +35,10 @@ FlexContextSchema, MultiSensorFlexModelSchema, ) +from flexmeasures.data.schemas.scheduling.utils import ( + is_on_schedule_tick, + should_project_off_tick_soc_constraints, +) from flexmeasures.data.schemas.sensors import SensorReference, VariableQuantityField from flexmeasures.utils.calculations import ( integrate_time_series, @@ -45,7 +49,6 @@ storage_asset_types = ["one-way_evse", "two-way_evse", "battery", "heat-storage"] -SOC_TIMED_EVENT_FIELDS = ("soc-targets", "soc-minima", "soc-maxima") class MetaStorageScheduler(Scheduler): @@ -806,7 +809,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 device_constraints[d]["derivative max"] = consumption_capacity_d if soc_at_start[d] is not None: - if _should_project_off_tick_soc_constraints(sensor_d): + if should_project_off_tick_soc_constraints(sensor_d): ( soc_targets[d], soc_maxima[d], @@ -1073,8 +1076,8 @@ def deserialize_flex_config(self): self.flex_model = {} self.collect_flex_config() - self.enable_relax_soc_constraints_for_off_tick_soc_constraints() - self.flex_context = FlexContextSchema().load(self.flex_context) + if self.flex_context is None: + self.flex_context = {} if isinstance(self.flex_model, dict): if self.sensor.generic_asset.asset_type.name in storage_asset_types: @@ -1086,11 +1089,16 @@ def deserialize_flex_config(self): self.ensure_soc_min_max() # Now it's time to check if our flex configuration holds up to schemas - self.flex_model = StorageFlexModelSchema( + schema = StorageFlexModelSchema( start=self.start, sensor=self.sensor, default_soc_unit=self.flex_model.get("soc-unit"), - ).load(self.flex_model) + schedule_resolution=self.resolution, + default_resolution=self.default_resolution, + ) + self.flex_model = schema.load(self.flex_model) + if schema.has_off_tick_soc_constraints: + self.enable_relax_soc_constraints() # Extend schedule period in case a target exceeds its end self.possibly_extend_end(soc_targets=self.flex_model.get("soc_targets")) @@ -1104,13 +1112,18 @@ def deserialize_flex_config(self): flex_model=sensor_flex_model["sensor_flex_model"], sensor=sensor_flex_model.get("sensor"), ) - self.flex_model[d] = StorageFlexModelSchema( + schema = StorageFlexModelSchema( start=self.start, sensor=sensor_flex_model.get("sensor"), default_soc_unit=sensor_flex_model["sensor_flex_model"].get( "soc-unit" ), - ).load(sensor_flex_model["sensor_flex_model"]) + schedule_resolution=self.resolution, + default_resolution=self.default_resolution, + ) + self.flex_model[d] = schema.load(sensor_flex_model["sensor_flex_model"]) + if schema.has_off_tick_soc_constraints: + self.enable_relax_soc_constraints() self.flex_model[d]["sensor"] = sensor_flex_model.get("sensor") self.flex_model[d]["asset"] = sensor_flex_model.get("asset") @@ -1125,37 +1138,14 @@ def deserialize_flex_config(self): f"Unsupported type of flex-model: '{type(self.flex_model)}'" ) + self.flex_context = FlexContextSchema().load(self.flex_context) + return self.flex_model - def enable_relax_soc_constraints_for_off_tick_soc_constraints(self) -> None: + def enable_relax_soc_constraints(self) -> None: """Relax SoC constraints when off-tick SoC events require scheduling-tick projection.""" - if self.flex_context is None: - self.flex_context = {} - if not self.flex_model_has_off_tick_soc_constraints(): - return self.flex_context["relax-soc-constraints"] = True - def flex_model_has_off_tick_soc_constraints(self) -> bool: - if isinstance(self.flex_model, dict): - if not _should_project_off_tick_soc_constraints(self.sensor): - return False - resolution = _get_soc_constraint_resolution( - self.resolution, self.sensor, self.default_resolution - ) - return flex_model_has_off_tick_soc_constraints( - self.flex_model, resolution=resolution - ) - if isinstance(self.flex_model, list): - resolution = self.resolution or self.default_resolution - return any( - flex_model_has_off_tick_soc_constraints( - flex_model.get("sensor-flex-model", flex_model), - resolution=resolution, - ) - for flex_model in self.flex_model - ) - return False - def has_soc_at_start(self) -> bool: return ( "soc-at-start" in self.flex_model @@ -1974,55 +1964,6 @@ def create_constraint_violations_message(constraint_violations: list) -> str: return message -def _is_on_schedule_tick(dt: datetime, resolution: timedelta) -> bool: - timestamp = pd.Timestamp(dt) - return timestamp == timestamp.floor(resolution) - - -def _get_soc_constraint_resolution( - schedule_resolution: timedelta | None, - sensor: Sensor | None, - default_resolution: timedelta, -) -> timedelta: - if schedule_resolution not in (None, timedelta(0)): - return schedule_resolution - if sensor is not None and sensor.event_resolution != timedelta(0): - return sensor.event_resolution - return default_resolution - - -def _should_project_off_tick_soc_constraints(sensor: Sensor | None) -> bool: - return sensor is None or sensor.get_attribute("floor_datetimes_to_resolution", True) - - -def flex_model_has_off_tick_soc_constraints( - flex_model: dict, - resolution: timedelta | None, -) -> bool: - if resolution in (None, timedelta(0)): - return False - - for field_name in SOC_TIMED_EVENT_FIELDS: - field_value = flex_model.get(field_name) - if not isinstance(field_value, list): - continue - for soc_event in field_value: - if not isinstance(soc_event, dict): - continue - for timing_field in ("datetime", "start", "end"): - if soc_event.get(timing_field) is None: - continue - try: - is_on_tick = _is_on_schedule_tick( - soc_event[timing_field], resolution - ) - except (TypeError, ValueError): - continue - if not is_on_tick: - return True - return False - - def _soc_value_in_mwh(value: ur.Quantity | float | int) -> float: if isinstance(value, ur.Quantity): return value.to("MWh").magnitude @@ -2174,7 +2115,7 @@ def project_off_tick_soc_constraints( if isinstance(soc_targets, list): projected_targets = [] for soc_target in soc_targets: - if soc_target["start"] != soc_target["end"] or _is_on_schedule_tick( + if soc_target["start"] != soc_target["end"] or is_on_schedule_tick( soc_target["end"], resolution ): projected_targets.append(copy.copy(soc_target)) @@ -2221,7 +2162,7 @@ def project_off_tick_soc_constraints( else: soc_minimum_events = [] for soc_minimum in soc_minimum_events: - if soc_minimum["start"] != soc_minimum["end"] or _is_on_schedule_tick( + if soc_minimum["start"] != soc_minimum["end"] or is_on_schedule_tick( soc_minimum["end"], resolution ): continue @@ -2266,7 +2207,7 @@ def project_off_tick_soc_constraints( else: soc_maximum_events = [] for soc_maximum in soc_maximum_events: - if soc_maximum["start"] != soc_maximum["end"] or _is_on_schedule_tick( + if soc_maximum["start"] != soc_maximum["end"] or is_on_schedule_tick( soc_maximum["end"], resolution ): continue diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index 8453c2149d..463762f059 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -6,6 +6,7 @@ from flask import current_app from marshmallow import ( Schema, + pre_load, post_load, validate, validates_schema, @@ -18,6 +19,11 @@ from flexmeasures.data.schemas.generic_assets import GenericAssetIdField from flexmeasures.data.schemas.units import QuantityField from flexmeasures.data.schemas.scheduling import metadata +from flexmeasures.data.schemas.scheduling.utils import ( + flex_model_has_off_tick_soc_constraints, + get_soc_constraint_resolution, + should_project_off_tick_soc_constraints, +) from flexmeasures.data.schemas.sensors import ( SensorReference, SensorReferenceSchema, @@ -253,12 +259,18 @@ def __init__( sensor: Sensor | None, *args, default_soc_unit: str | None = None, + schedule_resolution: timedelta | None = None, + default_resolution: timedelta = timedelta(minutes=15), **kwargs, ): """Pass the schedule's start, so we can use it to validate soc-target datetimes.""" self.start = start self.sensor = sensor self.timezone = sensor.timezone if sensor is not None else None + self.has_off_tick_soc_constraints = False + self.soc_constraint_resolution = get_soc_constraint_resolution( + schedule_resolution, sensor, default_resolution + ) self.flooring_resolution = ( sensor.event_resolution if sensor is not None @@ -306,6 +318,17 @@ def __init__( if field.startswith("soc_"): setattr(self.fields[field], "default_src_unit", default_soc_unit) + @pre_load + def detect_off_tick_soc_constraints(self, data: dict, **kwargs) -> dict: + self.has_off_tick_soc_constraints = ( + isinstance(data, dict) + and should_project_off_tick_soc_constraints(self.sensor) + and flex_model_has_off_tick_soc_constraints( + data, resolution=self.soc_constraint_resolution + ) + ) + return data + @validates_schema def check_whether_targets_exceed_max_planning_horizon(self, data: dict, **kwargs): # skip check if the flex-model does not define a sensor: the StorageScheduler will not base its resolution on this flex-model diff --git a/flexmeasures/data/schemas/scheduling/utils.py b/flexmeasures/data/schemas/scheduling/utils.py new file mode 100644 index 0000000000..07d15a07ca --- /dev/null +++ b/flexmeasures/data/schemas/scheduling/utils.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from datetime import datetime, timedelta + +import pandas as pd + +from flexmeasures import Sensor + + +SOC_TIMED_EVENT_FIELDS = ("soc-targets", "soc-minima", "soc-maxima") + + +def is_on_schedule_tick(dt: datetime, resolution: timedelta) -> bool: + timestamp = pd.Timestamp(dt) + return timestamp == timestamp.floor(resolution) + + +def get_soc_constraint_resolution( + schedule_resolution: timedelta | None, + sensor: Sensor | None, + default_resolution: timedelta, +) -> timedelta: + if schedule_resolution not in (None, timedelta(0)): + return schedule_resolution + if sensor is not None and sensor.event_resolution != timedelta(0): + return sensor.event_resolution + return default_resolution + + +def should_project_off_tick_soc_constraints(sensor: Sensor | None) -> bool: + return sensor is None or sensor.get_attribute("floor_datetimes_to_resolution", True) + + +def flex_model_has_off_tick_soc_constraints( + flex_model: dict, + resolution: timedelta | None, +) -> bool: + if resolution in (None, timedelta(0)): + return False + + for field_name in SOC_TIMED_EVENT_FIELDS: + field_value = flex_model.get( + field_name, flex_model.get(field_name.replace("-", "_")) + ) + if not isinstance(field_value, list): + continue + for soc_event in field_value: + if not isinstance(soc_event, dict): + continue + for timing_field in ("datetime", "start", "end"): + if soc_event.get(timing_field) is None: + continue + try: + is_on_tick = is_on_schedule_tick( + soc_event[timing_field], resolution + ) + except (TypeError, ValueError): + continue + if not is_on_tick: + return True + return False From b3b2d2b18604e3cd29801f5f3e5977ad6eb46c8d Mon Sep 17 00:00:00 2001 From: Mohamed Belhsan Hmida Date: Fri, 5 Jun 2026 12:59:40 +0100 Subject: [PATCH 52/72] refactor: make soc projection policy explicit Signed-off-by: Mohamed Belhsan Hmida --- flexmeasures/data/models/planning/storage.py | 259 +++++++++++-------- 1 file changed, 148 insertions(+), 111 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 66fe9ea3b2..b109943a60 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -2,8 +2,9 @@ import re import copy +from dataclasses import dataclass from datetime import datetime, timedelta -from typing import Type +from typing import Literal, Type import pandas as pd import numpy as np @@ -50,6 +51,36 @@ storage_asset_types = ["one-way_evse", "two-way_evse", "battery", "heat-storage"] +SocBoundType = Literal["min", "max"] +SocProjectionTick = Literal["previous", "next"] +SocCapacityType = Literal["consumption", "production"] +SocCapacityPeriod = Literal["before", "after"] + + +@dataclass(frozen=True) +class SocProjectionRule: + bound_type: SocBoundType + tick: SocProjectionTick + capacity: SocCapacityType + period: SocCapacityPeriod + sign: int + + +SOC_PROJECTION_POLICIES = { + "soc-targets": ( + SocProjectionRule("min", "previous", "consumption", "before", -1), + SocProjectionRule("max", "previous", "production", "before", +1), + ), + "soc-minima": ( + SocProjectionRule("min", "previous", "consumption", "before", -1), + SocProjectionRule("min", "next", "production", "after", -1), + ), + "soc-maxima": ( + SocProjectionRule("max", "previous", "production", "before", +1), + SocProjectionRule("max", "next", "consumption", "after", +1), + ), +} + class MetaStorageScheduler(Scheduler): """This class defines the constraints of a schedule for a storage device from the @@ -2027,6 +2058,83 @@ def _energy_capacity_between( return energy +def _tick_for_projection_rule( + rule: SocProjectionRule, + previous_tick: pd.Timestamp, + next_tick: pd.Timestamp, +) -> pd.Timestamp: + return previous_tick if rule.tick == "previous" else next_tick + + +def _capacity_for_projection_rule( + rule: SocProjectionRule, + consumption_capacity: pd.Series, + production_capacity: pd.Series, +) -> pd.Series: + return ( + consumption_capacity if rule.capacity == "consumption" else production_capacity + ) + + +def _capacity_period_for_projection_rule( + rule: SocProjectionRule, + previous_tick: pd.Timestamp, + event_time: pd.Timestamp, + next_tick: pd.Timestamp, +) -> tuple[pd.Timestamp, pd.Timestamp]: + if rule.period == "before": + return previous_tick, event_time + return event_time, next_tick + + +def _clamp_projected_soc_value( + value: float, + rule: SocProjectionRule, + soc_min: float | None, + soc_max: float | None, +) -> float: + if rule.bound_type == "min": + return _clamp_soc_min(value, soc_min) + return _clamp_soc_max(value, soc_max) + + +def _add_projected_soc_bound( + projected_minima: list[dict[str, datetime | float]], + projected_maxima: list[dict[str, datetime | float]], + soc_event: dict[str, datetime | float], + rule: SocProjectionRule, + event_time: pd.Timestamp, + previous_tick: pd.Timestamp, + next_tick: pd.Timestamp, + consumption_capacity: pd.Series, + production_capacity: pd.Series, + resolution: timedelta, + soc_min: float | None, + soc_max: float | None, +) -> None: + capacity_start, capacity_end = _capacity_period_for_projection_rule( + rule, previous_tick, event_time, next_tick + ) + capacity = _capacity_for_projection_rule( + rule, consumption_capacity, production_capacity + ) + projected_value = _soc_value_in_mwh(soc_event["value"]) + rule.sign * ( + _energy_capacity_between(capacity, capacity_start, capacity_end, resolution) + ) + projected_soc_events = ( + projected_minima if rule.bound_type == "min" else projected_maxima + ) + _add_soc_bound( + projected_soc_events, + _soc_event_at( + soc_event, + _tick_for_projection_rule(rule, previous_tick, next_tick), + _clamp_projected_soc_value(projected_value, rule, soc_min, soc_max), + ), + bound_type=rule.bound_type, + ) + + def _add_soc_bound( soc_events: list[dict[str, datetime | float]], soc_event: dict[str, datetime | float], @@ -2126,126 +2234,55 @@ def project_off_tick_soc_constraints( next_tick = target_time.ceil(resolution) target_value = _soc_value_in_mwh(soc_target["value"]) - charge_before_target = _energy_capacity_between( - consumption_capacity, previous_tick, target_time, resolution - ) - discharge_before_target = _energy_capacity_between( - production_capacity, previous_tick, target_time, resolution - ) - projected_targets.append(_soc_event_at(soc_target, next_tick, target_value)) - _add_soc_bound( - projected_minima, - _soc_event_at( + for rule in SOC_PROJECTION_POLICIES["soc-targets"]: + _add_projected_soc_bound( + projected_minima, + projected_maxima, soc_target, + rule, + target_time, previous_tick, - _clamp_soc_min(target_value - charge_before_target, soc_min_value), - ), - bound_type="min", - ) - _add_soc_bound( - projected_maxima, - _soc_event_at( - soc_target, - previous_tick, - _clamp_soc_max( - target_value + discharge_before_target, soc_max_value - ), - ), - bound_type="max", - ) + next_tick, + consumption_capacity, + production_capacity, + resolution, + soc_min_value, + soc_max_value, + ) else: projected_targets = soc_targets - if isinstance(soc_minima, list): - soc_minimum_events = copy.deepcopy(soc_minima) - else: - soc_minimum_events = [] - for soc_minimum in soc_minimum_events: - if soc_minimum["start"] != soc_minimum["end"] or is_on_schedule_tick( - soc_minimum["end"], resolution - ): + for field_name, soc_events in ( + ("soc-minima", soc_minima), + ("soc-maxima", soc_maxima), + ): + if not isinstance(soc_events, list): continue + for soc_event in copy.deepcopy(soc_events): + if soc_event["start"] != soc_event["end"] or is_on_schedule_tick( + soc_event["end"], resolution + ): + continue - minimum_time = pd.Timestamp(soc_minimum["end"]) - previous_tick = minimum_time.floor(resolution) - next_tick = minimum_time.ceil(resolution) - minimum_value = _soc_value_in_mwh(soc_minimum["value"]) - _add_soc_bound( - projected_minima, - _soc_event_at( - soc_minimum, - previous_tick, - _clamp_soc_min( - minimum_value - - _energy_capacity_between( - consumption_capacity, previous_tick, minimum_time, resolution - ), - soc_min_value, - ), - ), - bound_type="min", - ) - _add_soc_bound( - projected_minima, - _soc_event_at( - soc_minimum, - next_tick, - _clamp_soc_min( - minimum_value - - _energy_capacity_between( - production_capacity, minimum_time, next_tick, resolution - ), + event_time = pd.Timestamp(soc_event["end"]) + previous_tick = event_time.floor(resolution) + next_tick = event_time.ceil(resolution) + for rule in SOC_PROJECTION_POLICIES[field_name]: + _add_projected_soc_bound( + projected_minima, + projected_maxima, + soc_event, + rule, + event_time, + previous_tick, + next_tick, + consumption_capacity, + production_capacity, + resolution, soc_min_value, - ), - ), - bound_type="min", - ) - - if isinstance(soc_maxima, list): - soc_maximum_events = copy.deepcopy(soc_maxima) - else: - soc_maximum_events = [] - for soc_maximum in soc_maximum_events: - if soc_maximum["start"] != soc_maximum["end"] or is_on_schedule_tick( - soc_maximum["end"], resolution - ): - continue - - maximum_time = pd.Timestamp(soc_maximum["end"]) - previous_tick = maximum_time.floor(resolution) - next_tick = maximum_time.ceil(resolution) - maximum_value = _soc_value_in_mwh(soc_maximum["value"]) - _add_soc_bound( - projected_maxima, - _soc_event_at( - soc_maximum, - previous_tick, - _clamp_soc_max( - maximum_value - + _energy_capacity_between( - production_capacity, previous_tick, maximum_time, resolution - ), soc_max_value, - ), - ), - bound_type="max", - ) - _add_soc_bound( - projected_maxima, - _soc_event_at( - soc_maximum, - next_tick, - _clamp_soc_max( - maximum_value - + _energy_capacity_between( - consumption_capacity, maximum_time, next_tick, resolution - ), - soc_max_value, - ), - ), - bound_type="max", - ) + ) return ( projected_targets, From d255bd3c591cabafec24c084d8119585c843e7e6 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 Jul 2026 14:09:38 +0200 Subject: [PATCH 53/72] refactor: move off-tick SoC projection logic to a dedicated module Pure code move: extract the projection helpers, rule table and project_off_tick_soc_constraints() from planning/storage.py into planning/soc_projection.py, and update imports. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01YNPESudGKciPZq88nGJn3C Signed-off-by: F.N. Claessen --- .../data/models/planning/soc_projection.py | 341 ++++++++++++++++++ flexmeasures/data/models/planning/storage.py | 332 +---------------- .../models/planning/tests/test_storage.py | 4 +- 3 files changed, 347 insertions(+), 330 deletions(-) create mode 100644 flexmeasures/data/models/planning/soc_projection.py diff --git a/flexmeasures/data/models/planning/soc_projection.py b/flexmeasures/data/models/planning/soc_projection.py new file mode 100644 index 0000000000..86bf981f03 --- /dev/null +++ b/flexmeasures/data/models/planning/soc_projection.py @@ -0,0 +1,341 @@ +"""Projection of off-tick point-like SoC constraints onto scheduling ticks.""" + +from __future__ import annotations + +import copy +from dataclasses import dataclass +from datetime import datetime, timedelta +from typing import Literal + +import pandas as pd + +from flexmeasures.data.models.time_series import Sensor +from flexmeasures.data.schemas.scheduling.utils import is_on_schedule_tick +from flexmeasures.utils.unit_utils import ur + + +SocBoundType = Literal["min", "max"] +SocProjectionTick = Literal["previous", "next"] +SocCapacityType = Literal["consumption", "production"] +SocCapacityPeriod = Literal["before", "after"] + + +@dataclass(frozen=True) +class SocProjectionRule: + bound_type: SocBoundType + tick: SocProjectionTick + capacity: SocCapacityType + period: SocCapacityPeriod + sign: int + + +SOC_PROJECTION_POLICIES = { + "soc-targets": ( + SocProjectionRule("min", "previous", "consumption", "before", -1), + SocProjectionRule("max", "previous", "production", "before", +1), + ), + "soc-minima": ( + SocProjectionRule("min", "previous", "consumption", "before", -1), + SocProjectionRule("min", "next", "production", "after", -1), + ), + "soc-maxima": ( + SocProjectionRule("max", "previous", "production", "before", +1), + SocProjectionRule("max", "next", "consumption", "after", +1), + ), +} + + +def _soc_value_in_mwh(value: ur.Quantity | float | int) -> float: + if isinstance(value, ur.Quantity): + return value.to("MWh").magnitude + return float(value) + + +def _optional_soc_value_in_mwh(value: ur.Quantity | float | int | None) -> float | None: + if value is None: + return None + return _soc_value_in_mwh(value) + + +def _clamp_soc_min(value: float, soc_min: float | None) -> float: + return value if soc_min is None else max(soc_min, value) + + +def _clamp_soc_max(value: float, soc_max: float | None) -> float: + return value if soc_max is None else min(soc_max, value) + + +def _soc_event_at( + soc_event: dict[str, datetime | float], + dt: pd.Timestamp, + value: float, +) -> dict[str, datetime | float]: + shifted_event = copy.copy(soc_event) + shifted_event["value"] = value + shifted_event["start"] = dt.to_pydatetime() + shifted_event["end"] = dt.to_pydatetime() + if "datetime" in shifted_event: + shifted_event["datetime"] = dt.to_pydatetime() + shifted_event.pop("duration", None) + return shifted_event + + +def _energy_capacity_between( + capacity: pd.Series, + start: pd.Timestamp, + end: pd.Timestamp, + resolution: timedelta, +) -> float: + if end <= start: + return 0 + + if capacity.index.tz is not None: + start = start.tz_convert(capacity.index.tz) + end = end.tz_convert(capacity.index.tz) + + capacity = capacity.astype(float).fillna(0) + tick = start.floor(resolution) + energy = 0.0 + while tick < end: + next_tick = tick + resolution + overlap_start = max(start, tick) + overlap_end = min(end, next_tick) + if overlap_end > overlap_start and tick in capacity.index: + energy += float(capacity.loc[tick]) * ( + (overlap_end - overlap_start) / pd.Timedelta(hours=1) + ) + tick = next_tick + return energy + + +def _tick_for_projection_rule( + rule: SocProjectionRule, + previous_tick: pd.Timestamp, + next_tick: pd.Timestamp, +) -> pd.Timestamp: + return previous_tick if rule.tick == "previous" else next_tick + + +def _capacity_for_projection_rule( + rule: SocProjectionRule, + consumption_capacity: pd.Series, + production_capacity: pd.Series, +) -> pd.Series: + return ( + consumption_capacity if rule.capacity == "consumption" else production_capacity + ) + + +def _capacity_period_for_projection_rule( + rule: SocProjectionRule, + previous_tick: pd.Timestamp, + event_time: pd.Timestamp, + next_tick: pd.Timestamp, +) -> tuple[pd.Timestamp, pd.Timestamp]: + if rule.period == "before": + return previous_tick, event_time + return event_time, next_tick + + +def _clamp_projected_soc_value( + value: float, + rule: SocProjectionRule, + soc_min: float | None, + soc_max: float | None, +) -> float: + if rule.bound_type == "min": + return _clamp_soc_min(value, soc_min) + return _clamp_soc_max(value, soc_max) + + +def _add_projected_soc_bound( + projected_minima: list[dict[str, datetime | float]], + projected_maxima: list[dict[str, datetime | float]], + soc_event: dict[str, datetime | float], + rule: SocProjectionRule, + event_time: pd.Timestamp, + previous_tick: pd.Timestamp, + next_tick: pd.Timestamp, + consumption_capacity: pd.Series, + production_capacity: pd.Series, + resolution: timedelta, + soc_min: float | None, + soc_max: float | None, +) -> None: + capacity_start, capacity_end = _capacity_period_for_projection_rule( + rule, previous_tick, event_time, next_tick + ) + capacity = _capacity_for_projection_rule( + rule, consumption_capacity, production_capacity + ) + projected_value = _soc_value_in_mwh(soc_event["value"]) + rule.sign * ( + _energy_capacity_between(capacity, capacity_start, capacity_end, resolution) + ) + projected_soc_events = ( + projected_minima if rule.bound_type == "min" else projected_maxima + ) + _add_soc_bound( + projected_soc_events, + _soc_event_at( + soc_event, + _tick_for_projection_rule(rule, previous_tick, next_tick), + _clamp_projected_soc_value(projected_value, rule, soc_min, soc_max), + ), + bound_type=rule.bound_type, + ) + + +def _add_soc_bound( + soc_events: list[dict[str, datetime | float]], + soc_event: dict[str, datetime | float], + bound_type: str, +) -> None: + for existing_event in soc_events: + if existing_event.get("start") == soc_event.get("start") and existing_event.get( + "end" + ) == soc_event.get("end"): + existing_value = _soc_value_in_mwh(existing_event["value"]) + soc_value = _soc_value_in_mwh(soc_event["value"]) + existing_event["value"] = ( + max(existing_value, soc_value) + if bound_type == "min" + else min(existing_value, soc_value) + ) + return + soc_events.append(soc_event) + + +def _projected_soc_events_or_original( + original_soc_events: ( + list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None + ), + projected_soc_events: list[dict[str, datetime | float]], +) -> list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None: + if isinstance(original_soc_events, list): + return projected_soc_events + if original_soc_events is None and projected_soc_events: + return projected_soc_events + return original_soc_events + + +def project_off_tick_soc_constraints( + soc_targets: ( + list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None + ), + soc_maxima: ( + list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None + ), + soc_minima: ( + list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None + ), + consumption_capacity: pd.Series, + production_capacity: pd.Series, + resolution: timedelta, + soc_min: ur.Quantity | float | None, + soc_max: ur.Quantity | float | None, +) -> tuple[ + list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, + list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, + list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, +]: + """Project off-tick point-like SoC constraints onto scheduling ticks. + + The scheduler can only enforce constraints at its fixed scheduling resolution. + Point-like ``soc-targets``, ``soc-minima`` and ``soc-maxima`` that fall between + two scheduling ticks are therefore replaced by constraints on the previous and + next tick that preserve reachability using the available charge and discharge + capacity between the original event time and those ticks. + + ``soc-targets`` are projected to an exact target on the next tick, plus + capacity-adjusted lower and upper bounds on the previous tick. ``soc-minima`` + become lower bounds on both surrounding ticks, and ``soc-maxima`` become upper + bounds on both surrounding ticks. If multiple projected bounds land on the same + tick, the stricter lower or upper bound is kept. + + Returns ``(soc_targets, soc_maxima, soc_minima)`` with projected list-based + timed events. Non-list specifications such as sensors, series, fixed + quantities, or ``None`` are returned unchanged unless projected bounds need to + be added to a missing list. + """ + + if not any( + isinstance(soc_events, list) and soc_events + for soc_events in (soc_targets, soc_maxima, soc_minima) + ): + return soc_targets, soc_maxima, soc_minima + + projected_minima = copy.deepcopy(soc_minima) if isinstance(soc_minima, list) else [] + projected_maxima = copy.deepcopy(soc_maxima) if isinstance(soc_maxima, list) else [] + + soc_min_value = _optional_soc_value_in_mwh(soc_min) + soc_max_value = _optional_soc_value_in_mwh(soc_max) + + if isinstance(soc_targets, list): + projected_targets = [] + for soc_target in soc_targets: + if soc_target["start"] != soc_target["end"] or is_on_schedule_tick( + soc_target["end"], resolution + ): + projected_targets.append(copy.copy(soc_target)) + continue + + target_time = pd.Timestamp(soc_target["end"]) + previous_tick = target_time.floor(resolution) + next_tick = target_time.ceil(resolution) + target_value = _soc_value_in_mwh(soc_target["value"]) + + projected_targets.append(_soc_event_at(soc_target, next_tick, target_value)) + for rule in SOC_PROJECTION_POLICIES["soc-targets"]: + _add_projected_soc_bound( + projected_minima, + projected_maxima, + soc_target, + rule, + target_time, + previous_tick, + next_tick, + consumption_capacity, + production_capacity, + resolution, + soc_min_value, + soc_max_value, + ) + else: + projected_targets = soc_targets + + for field_name, soc_events in ( + ("soc-minima", soc_minima), + ("soc-maxima", soc_maxima), + ): + if not isinstance(soc_events, list): + continue + for soc_event in copy.deepcopy(soc_events): + if soc_event["start"] != soc_event["end"] or is_on_schedule_tick( + soc_event["end"], resolution + ): + continue + + event_time = pd.Timestamp(soc_event["end"]) + previous_tick = event_time.floor(resolution) + next_tick = event_time.ceil(resolution) + for rule in SOC_PROJECTION_POLICIES[field_name]: + _add_projected_soc_bound( + projected_minima, + projected_maxima, + soc_event, + rule, + event_time, + previous_tick, + next_tick, + consumption_capacity, + production_capacity, + resolution, + soc_min_value, + soc_max_value, + ) + + return ( + projected_targets, + _projected_soc_events_or_original(soc_maxima, projected_maxima), + _projected_soc_events_or_original(soc_minima, projected_minima), + ) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index a210d604f0..4e4a6508a1 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -2,9 +2,8 @@ import re import copy -from dataclasses import dataclass from datetime import datetime, timedelta -from typing import Literal, Type +from typing import Type import pandas as pd import numpy as np @@ -37,8 +36,10 @@ FlexContextSchema, MultiSensorFlexModelSchema, ) +from flexmeasures.data.models.planning.soc_projection import ( + project_off_tick_soc_constraints, +) from flexmeasures.data.schemas.scheduling.utils import ( - is_on_schedule_tick, should_project_off_tick_soc_constraints, ) from flexmeasures.data.schemas.sensors import SensorReference, VariableQuantityField @@ -53,35 +54,6 @@ storage_asset_types = ["one-way_evse", "two-way_evse", "battery", "heat-storage"] -SocBoundType = Literal["min", "max"] -SocProjectionTick = Literal["previous", "next"] -SocCapacityType = Literal["consumption", "production"] -SocCapacityPeriod = Literal["before", "after"] - - -@dataclass(frozen=True) -class SocProjectionRule: - bound_type: SocBoundType - tick: SocProjectionTick - capacity: SocCapacityType - period: SocCapacityPeriod - sign: int - - -SOC_PROJECTION_POLICIES = { - "soc-targets": ( - SocProjectionRule("min", "previous", "consumption", "before", -1), - SocProjectionRule("max", "previous", "production", "before", +1), - ), - "soc-minima": ( - SocProjectionRule("min", "previous", "consumption", "before", -1), - SocProjectionRule("min", "next", "production", "after", -1), - ), - "soc-maxima": ( - SocProjectionRule("max", "previous", "production", "before", +1), - SocProjectionRule("max", "next", "consumption", "after", +1), - ), -} #: Key used to store and retrieve the ``SchedulingJobResult`` in RQ job metadata #: and in the multi-result list returned by ``StorageScheduler.compute()``. @@ -2930,302 +2902,6 @@ def create_constraint_violations_message(constraint_violations: list) -> str: return message -def _soc_value_in_mwh(value: ur.Quantity | float | int) -> float: - if isinstance(value, ur.Quantity): - return value.to("MWh").magnitude - return float(value) - - -def _optional_soc_value_in_mwh(value: ur.Quantity | float | int | None) -> float | None: - if value is None: - return None - return _soc_value_in_mwh(value) - - -def _clamp_soc_min(value: float, soc_min: float | None) -> float: - return value if soc_min is None else max(soc_min, value) - - -def _clamp_soc_max(value: float, soc_max: float | None) -> float: - return value if soc_max is None else min(soc_max, value) - - -def _soc_event_at( - soc_event: dict[str, datetime | float], - dt: pd.Timestamp, - value: float, -) -> dict[str, datetime | float]: - shifted_event = copy.copy(soc_event) - shifted_event["value"] = value - shifted_event["start"] = dt.to_pydatetime() - shifted_event["end"] = dt.to_pydatetime() - if "datetime" in shifted_event: - shifted_event["datetime"] = dt.to_pydatetime() - shifted_event.pop("duration", None) - return shifted_event - - -def _energy_capacity_between( - capacity: pd.Series, - start: pd.Timestamp, - end: pd.Timestamp, - resolution: timedelta, -) -> float: - if end <= start: - return 0 - - if capacity.index.tz is not None: - start = start.tz_convert(capacity.index.tz) - end = end.tz_convert(capacity.index.tz) - - capacity = capacity.astype(float).fillna(0) - tick = start.floor(resolution) - energy = 0.0 - while tick < end: - next_tick = tick + resolution - overlap_start = max(start, tick) - overlap_end = min(end, next_tick) - if overlap_end > overlap_start and tick in capacity.index: - energy += float(capacity.loc[tick]) * ( - (overlap_end - overlap_start) / pd.Timedelta(hours=1) - ) - tick = next_tick - return energy - - -def _tick_for_projection_rule( - rule: SocProjectionRule, - previous_tick: pd.Timestamp, - next_tick: pd.Timestamp, -) -> pd.Timestamp: - return previous_tick if rule.tick == "previous" else next_tick - - -def _capacity_for_projection_rule( - rule: SocProjectionRule, - consumption_capacity: pd.Series, - production_capacity: pd.Series, -) -> pd.Series: - return ( - consumption_capacity if rule.capacity == "consumption" else production_capacity - ) - - -def _capacity_period_for_projection_rule( - rule: SocProjectionRule, - previous_tick: pd.Timestamp, - event_time: pd.Timestamp, - next_tick: pd.Timestamp, -) -> tuple[pd.Timestamp, pd.Timestamp]: - if rule.period == "before": - return previous_tick, event_time - return event_time, next_tick - - -def _clamp_projected_soc_value( - value: float, - rule: SocProjectionRule, - soc_min: float | None, - soc_max: float | None, -) -> float: - if rule.bound_type == "min": - return _clamp_soc_min(value, soc_min) - return _clamp_soc_max(value, soc_max) - - -def _add_projected_soc_bound( - projected_minima: list[dict[str, datetime | float]], - projected_maxima: list[dict[str, datetime | float]], - soc_event: dict[str, datetime | float], - rule: SocProjectionRule, - event_time: pd.Timestamp, - previous_tick: pd.Timestamp, - next_tick: pd.Timestamp, - consumption_capacity: pd.Series, - production_capacity: pd.Series, - resolution: timedelta, - soc_min: float | None, - soc_max: float | None, -) -> None: - capacity_start, capacity_end = _capacity_period_for_projection_rule( - rule, previous_tick, event_time, next_tick - ) - capacity = _capacity_for_projection_rule( - rule, consumption_capacity, production_capacity - ) - projected_value = _soc_value_in_mwh(soc_event["value"]) + rule.sign * ( - _energy_capacity_between(capacity, capacity_start, capacity_end, resolution) - ) - projected_soc_events = ( - projected_minima if rule.bound_type == "min" else projected_maxima - ) - _add_soc_bound( - projected_soc_events, - _soc_event_at( - soc_event, - _tick_for_projection_rule(rule, previous_tick, next_tick), - _clamp_projected_soc_value(projected_value, rule, soc_min, soc_max), - ), - bound_type=rule.bound_type, - ) - - -def _add_soc_bound( - soc_events: list[dict[str, datetime | float]], - soc_event: dict[str, datetime | float], - bound_type: str, -) -> None: - for existing_event in soc_events: - if existing_event.get("start") == soc_event.get("start") and existing_event.get( - "end" - ) == soc_event.get("end"): - existing_value = _soc_value_in_mwh(existing_event["value"]) - soc_value = _soc_value_in_mwh(soc_event["value"]) - existing_event["value"] = ( - max(existing_value, soc_value) - if bound_type == "min" - else min(existing_value, soc_value) - ) - return - soc_events.append(soc_event) - - -def _projected_soc_events_or_original( - original_soc_events: ( - list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None - ), - projected_soc_events: list[dict[str, datetime | float]], -) -> list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None: - if isinstance(original_soc_events, list): - return projected_soc_events - if original_soc_events is None and projected_soc_events: - return projected_soc_events - return original_soc_events - - -def project_off_tick_soc_constraints( - soc_targets: ( - list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None - ), - soc_maxima: ( - list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None - ), - soc_minima: ( - list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None - ), - consumption_capacity: pd.Series, - production_capacity: pd.Series, - resolution: timedelta, - soc_min: ur.Quantity | float | None, - soc_max: ur.Quantity | float | None, -) -> tuple[ - list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, - list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, - list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, -]: - """Project off-tick point-like SoC constraints onto scheduling ticks. - - The scheduler can only enforce constraints at its fixed scheduling resolution. - Point-like ``soc-targets``, ``soc-minima`` and ``soc-maxima`` that fall between - two scheduling ticks are therefore replaced by constraints on the previous and - next tick that preserve reachability using the available charge and discharge - capacity between the original event time and those ticks. - - ``soc-targets`` are projected to an exact target on the next tick, plus - capacity-adjusted lower and upper bounds on the previous tick. ``soc-minima`` - become lower bounds on both surrounding ticks, and ``soc-maxima`` become upper - bounds on both surrounding ticks. If multiple projected bounds land on the same - tick, the stricter lower or upper bound is kept. - - Returns ``(soc_targets, soc_maxima, soc_minima)`` with projected list-based - timed events. Non-list specifications such as sensors, series, fixed - quantities, or ``None`` are returned unchanged unless projected bounds need to - be added to a missing list. - """ - - if not any( - isinstance(soc_events, list) and soc_events - for soc_events in (soc_targets, soc_maxima, soc_minima) - ): - return soc_targets, soc_maxima, soc_minima - - projected_minima = copy.deepcopy(soc_minima) if isinstance(soc_minima, list) else [] - projected_maxima = copy.deepcopy(soc_maxima) if isinstance(soc_maxima, list) else [] - - soc_min_value = _optional_soc_value_in_mwh(soc_min) - soc_max_value = _optional_soc_value_in_mwh(soc_max) - - if isinstance(soc_targets, list): - projected_targets = [] - for soc_target in soc_targets: - if soc_target["start"] != soc_target["end"] or is_on_schedule_tick( - soc_target["end"], resolution - ): - projected_targets.append(copy.copy(soc_target)) - continue - - target_time = pd.Timestamp(soc_target["end"]) - previous_tick = target_time.floor(resolution) - next_tick = target_time.ceil(resolution) - target_value = _soc_value_in_mwh(soc_target["value"]) - - projected_targets.append(_soc_event_at(soc_target, next_tick, target_value)) - for rule in SOC_PROJECTION_POLICIES["soc-targets"]: - _add_projected_soc_bound( - projected_minima, - projected_maxima, - soc_target, - rule, - target_time, - previous_tick, - next_tick, - consumption_capacity, - production_capacity, - resolution, - soc_min_value, - soc_max_value, - ) - else: - projected_targets = soc_targets - - for field_name, soc_events in ( - ("soc-minima", soc_minima), - ("soc-maxima", soc_maxima), - ): - if not isinstance(soc_events, list): - continue - for soc_event in copy.deepcopy(soc_events): - if soc_event["start"] != soc_event["end"] or is_on_schedule_tick( - soc_event["end"], resolution - ): - continue - - event_time = pd.Timestamp(soc_event["end"]) - previous_tick = event_time.floor(resolution) - next_tick = event_time.ceil(resolution) - for rule in SOC_PROJECTION_POLICIES[field_name]: - _add_projected_soc_bound( - projected_minima, - projected_maxima, - soc_event, - rule, - event_time, - previous_tick, - next_tick, - consumption_capacity, - production_capacity, - resolution, - soc_min_value, - soc_max_value, - ) - - return ( - projected_targets, - _projected_soc_events_or_original(soc_maxima, projected_maxima), - _projected_soc_events_or_original(soc_minima, projected_minima), - ) - - def build_device_soc_values( soc_values: ur.Quantity | list[dict[str, datetime | float]] | pd.Series | None, soc_at_start: float, diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py index 92d7615d02..b9384ea289 100644 --- a/flexmeasures/data/models/planning/tests/test_storage.py +++ b/flexmeasures/data/models/planning/tests/test_storage.py @@ -8,10 +8,10 @@ import pandas as pd from flexmeasures.data.models.planning import Scheduler -from flexmeasures.data.models.planning.storage import ( - StorageScheduler, +from flexmeasures.data.models.planning.soc_projection import ( project_off_tick_soc_constraints, ) +from flexmeasures.data.models.planning.storage import StorageScheduler from flexmeasures.data.models.planning.utils import initialize_index from flexmeasures.data.models.time_series import Sensor, TimedBelief from flexmeasures.data.models.planning.tests.utils import ( From 289a5ccf29a58fc7d4cca20751edc0dfd86c9cab Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 Jul 2026 14:10:32 +0200 Subject: [PATCH 54/72] docs: add docstrings to the SoC projection helpers Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01YNPESudGKciPZq88nGJn3C Signed-off-by: F.N. Claessen --- .../data/models/planning/soc_projection.py | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/flexmeasures/data/models/planning/soc_projection.py b/flexmeasures/data/models/planning/soc_projection.py index 86bf981f03..d06dbc44fe 100644 --- a/flexmeasures/data/models/planning/soc_projection.py +++ b/flexmeasures/data/models/planning/soc_projection.py @@ -22,6 +22,20 @@ @dataclass(frozen=True) class SocProjectionRule: + """One projection rule for an off-tick SoC event. + + Each rule creates a single projected bound: + + - ``bound_type``: whether the projected bound is a lower ("min") or upper ("max") bound. + - ``tick``: whether the bound lands on the scheduling tick just before or just after the event. + - ``capacity``: which device capacity limits how fast the SoC can move towards or away from + the event value ("consumption" for charging, "production" for discharging). + - ``period``: over which period that capacity applies: from the previous tick up to the + event time ("before"), or from the event time up to the next tick ("after"). + - ``sign``: -1 to lower the event value by the reachable energy (loosening a lower bound), + +1 to raise it (loosening an upper bound). + """ + bound_type: SocBoundType tick: SocProjectionTick capacity: SocCapacityType @@ -29,6 +43,10 @@ class SocProjectionRule: sign: int +#: How each type of off-tick point-like SoC constraint is projected onto the +#: surrounding scheduling ticks (see :class:`SocProjectionRule`). +#: Off-tick ``soc-targets`` additionally become an exact target on the next tick, +#: which is handled directly in :func:`project_off_tick_soc_constraints`. SOC_PROJECTION_POLICIES = { "soc-targets": ( SocProjectionRule("min", "previous", "consumption", "before", -1), @@ -46,22 +64,26 @@ class SocProjectionRule: def _soc_value_in_mwh(value: ur.Quantity | float | int) -> float: + """Return the SoC value as a plain float in MWh.""" if isinstance(value, ur.Quantity): return value.to("MWh").magnitude return float(value) def _optional_soc_value_in_mwh(value: ur.Quantity | float | int | None) -> float | None: + """Return the SoC value as a plain float in MWh, passing through None.""" if value is None: return None return _soc_value_in_mwh(value) def _clamp_soc_min(value: float, soc_min: float | None) -> float: + """Raise a projected lower bound to the storage's global soc-min, if defined.""" return value if soc_min is None else max(soc_min, value) def _clamp_soc_max(value: float, soc_max: float | None) -> float: + """Lower a projected upper bound to the storage's global soc-max, if defined.""" return value if soc_max is None else min(soc_max, value) @@ -70,6 +92,11 @@ def _soc_event_at( dt: pd.Timestamp, value: float, ) -> dict[str, datetime | float]: + """Return a copy of a point-like SoC event, moved to the given tick with the given value. + + All timing fields are set to the tick (any 'duration' is dropped), so the result + remains a point-like (instantaneous) event. + """ shifted_event = copy.copy(soc_event) shifted_event["value"] = value shifted_event["start"] = dt.to_pydatetime() @@ -86,6 +113,13 @@ def _energy_capacity_between( end: pd.Timestamp, resolution: timedelta, ) -> float: + """Compute how much energy (in MWh) the device can move between two times. + + Integrates the given power capacity series (in MW, indexed by tick start at the + given resolution) over the period from ``start`` to ``end``, weighing partial + tick overlaps. Missing capacity values count as zero (conservative: the + projected bound then sticks close to the original event value). + """ if end <= start: return 0 @@ -113,6 +147,7 @@ def _tick_for_projection_rule( previous_tick: pd.Timestamp, next_tick: pd.Timestamp, ) -> pd.Timestamp: + """Return the scheduling tick on which the rule's projected bound lands.""" return previous_tick if rule.tick == "previous" else next_tick @@ -121,6 +156,7 @@ def _capacity_for_projection_rule( consumption_capacity: pd.Series, production_capacity: pd.Series, ) -> pd.Series: + """Return the capacity series that limits SoC movement for this rule.""" return ( consumption_capacity if rule.capacity == "consumption" else production_capacity ) @@ -132,6 +168,7 @@ def _capacity_period_for_projection_rule( event_time: pd.Timestamp, next_tick: pd.Timestamp, ) -> tuple[pd.Timestamp, pd.Timestamp]: + """Return the period over which the rule's capacity applies.""" if rule.period == "before": return previous_tick, event_time return event_time, next_tick @@ -143,6 +180,7 @@ def _clamp_projected_soc_value( soc_min: float | None, soc_max: float | None, ) -> float: + """Clamp a projected bound to the storage's global soc-min/soc-max limits.""" if rule.bound_type == "min": return _clamp_soc_min(value, soc_min) return _clamp_soc_max(value, soc_max) @@ -162,6 +200,13 @@ def _add_projected_soc_bound( soc_min: float | None, soc_max: float | None, ) -> None: + """Apply one projection rule to one off-tick SoC event. + + Computes the capacity-adjusted bound value (event value +/- the energy the + device can move over the rule's capacity period), clamps it to the global SoC + limits, and merges it into the projected minima or maxima (keeping the + stricter bound if one already exists on the same tick). + """ capacity_start, capacity_end = _capacity_period_for_projection_rule( rule, previous_tick, event_time, next_tick ) @@ -190,6 +235,11 @@ def _add_soc_bound( soc_event: dict[str, datetime | float], bound_type: str, ) -> None: + """Add a SoC bound to a list of timed events, merging bounds on the same period. + + If an event with the same start and end already exists, the stricter bound wins: + the maximum of two lower bounds, or the minimum of two upper bounds. + """ for existing_event in soc_events: if existing_event.get("start") == soc_event.get("start") and existing_event.get( "end" @@ -211,6 +261,11 @@ def _projected_soc_events_or_original( ), projected_soc_events: list[dict[str, datetime | float]], ) -> list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None: + """Choose between the projected event list and the original specification. + + Only list-based (or missing) specifications can absorb projected bounds; + sensors, series and fixed quantities are returned unchanged. + """ if isinstance(original_soc_events, list): return projected_soc_events if original_soc_events is None and projected_soc_events: From ca83e9f8217aff731472dfe3b547d3801faed7e3 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 Jul 2026 14:12:58 +0200 Subject: [PATCH 55/72] refactor: replace the SoC projection rule table with explicit projections The SocProjectionRule dataclass, policy table and five decoder helpers abstracted just six fixed projection rules; writing the projections out explicitly is shorter and easier to audit. Also simplify the reachable-energy computation: the projection periods always lie within a single scheduling tick, so no tick-walking loop is needed. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01YNPESudGKciPZq88nGJn3C Signed-off-by: F.N. Claessen --- .../data/models/planning/soc_projection.py | 309 ++++++------------ 1 file changed, 106 insertions(+), 203 deletions(-) diff --git a/flexmeasures/data/models/planning/soc_projection.py b/flexmeasures/data/models/planning/soc_projection.py index d06dbc44fe..15b58d75a9 100644 --- a/flexmeasures/data/models/planning/soc_projection.py +++ b/flexmeasures/data/models/planning/soc_projection.py @@ -3,9 +3,7 @@ from __future__ import annotations import copy -from dataclasses import dataclass from datetime import datetime, timedelta -from typing import Literal import pandas as pd @@ -14,55 +12,6 @@ from flexmeasures.utils.unit_utils import ur -SocBoundType = Literal["min", "max"] -SocProjectionTick = Literal["previous", "next"] -SocCapacityType = Literal["consumption", "production"] -SocCapacityPeriod = Literal["before", "after"] - - -@dataclass(frozen=True) -class SocProjectionRule: - """One projection rule for an off-tick SoC event. - - Each rule creates a single projected bound: - - - ``bound_type``: whether the projected bound is a lower ("min") or upper ("max") bound. - - ``tick``: whether the bound lands on the scheduling tick just before or just after the event. - - ``capacity``: which device capacity limits how fast the SoC can move towards or away from - the event value ("consumption" for charging, "production" for discharging). - - ``period``: over which period that capacity applies: from the previous tick up to the - event time ("before"), or from the event time up to the next tick ("after"). - - ``sign``: -1 to lower the event value by the reachable energy (loosening a lower bound), - +1 to raise it (loosening an upper bound). - """ - - bound_type: SocBoundType - tick: SocProjectionTick - capacity: SocCapacityType - period: SocCapacityPeriod - sign: int - - -#: How each type of off-tick point-like SoC constraint is projected onto the -#: surrounding scheduling ticks (see :class:`SocProjectionRule`). -#: Off-tick ``soc-targets`` additionally become an exact target on the next tick, -#: which is handled directly in :func:`project_off_tick_soc_constraints`. -SOC_PROJECTION_POLICIES = { - "soc-targets": ( - SocProjectionRule("min", "previous", "consumption", "before", -1), - SocProjectionRule("max", "previous", "production", "before", +1), - ), - "soc-minima": ( - SocProjectionRule("min", "previous", "consumption", "before", -1), - SocProjectionRule("min", "next", "production", "after", -1), - ), - "soc-maxima": ( - SocProjectionRule("max", "previous", "production", "before", +1), - SocProjectionRule("max", "next", "consumption", "after", +1), - ), -} - - def _soc_value_in_mwh(value: ur.Quantity | float | int) -> float: """Return the SoC value as a plain float in MWh.""" if isinstance(value, ur.Quantity): @@ -107,7 +56,7 @@ def _soc_event_at( return shifted_event -def _energy_capacity_between( +def _reachable_energy( capacity: pd.Series, start: pd.Timestamp, end: pd.Timestamp, @@ -115,119 +64,22 @@ def _energy_capacity_between( ) -> float: """Compute how much energy (in MWh) the device can move between two times. - Integrates the given power capacity series (in MW, indexed by tick start at the - given resolution) over the period from ``start`` to ``end``, weighing partial - tick overlaps. Missing capacity values count as zero (conservative: the - projected bound then sticks close to the original event value). + The period from ``start`` to ``end`` always lies within a single scheduling + tick (from the tick just before an off-tick event to the event, or from the + event to the tick just after it), so the given power capacity series (in MW, + indexed by tick start at the given resolution) applies at a single constant + value. A missing capacity value counts as zero (conservative: the projected + bound then sticks close to the original event value). """ if end <= start: return 0 - - if capacity.index.tz is not None: - start = start.tz_convert(capacity.index.tz) - end = end.tz_convert(capacity.index.tz) - - capacity = capacity.astype(float).fillna(0) tick = start.floor(resolution) - energy = 0.0 - while tick < end: - next_tick = tick + resolution - overlap_start = max(start, tick) - overlap_end = min(end, next_tick) - if overlap_end > overlap_start and tick in capacity.index: - energy += float(capacity.loc[tick]) * ( - (overlap_end - overlap_start) / pd.Timedelta(hours=1) - ) - tick = next_tick - return energy - - -def _tick_for_projection_rule( - rule: SocProjectionRule, - previous_tick: pd.Timestamp, - next_tick: pd.Timestamp, -) -> pd.Timestamp: - """Return the scheduling tick on which the rule's projected bound lands.""" - return previous_tick if rule.tick == "previous" else next_tick - - -def _capacity_for_projection_rule( - rule: SocProjectionRule, - consumption_capacity: pd.Series, - production_capacity: pd.Series, -) -> pd.Series: - """Return the capacity series that limits SoC movement for this rule.""" - return ( - consumption_capacity if rule.capacity == "consumption" else production_capacity - ) - - -def _capacity_period_for_projection_rule( - rule: SocProjectionRule, - previous_tick: pd.Timestamp, - event_time: pd.Timestamp, - next_tick: pd.Timestamp, -) -> tuple[pd.Timestamp, pd.Timestamp]: - """Return the period over which the rule's capacity applies.""" - if rule.period == "before": - return previous_tick, event_time - return event_time, next_tick - - -def _clamp_projected_soc_value( - value: float, - rule: SocProjectionRule, - soc_min: float | None, - soc_max: float | None, -) -> float: - """Clamp a projected bound to the storage's global soc-min/soc-max limits.""" - if rule.bound_type == "min": - return _clamp_soc_min(value, soc_min) - return _clamp_soc_max(value, soc_max) - - -def _add_projected_soc_bound( - projected_minima: list[dict[str, datetime | float]], - projected_maxima: list[dict[str, datetime | float]], - soc_event: dict[str, datetime | float], - rule: SocProjectionRule, - event_time: pd.Timestamp, - previous_tick: pd.Timestamp, - next_tick: pd.Timestamp, - consumption_capacity: pd.Series, - production_capacity: pd.Series, - resolution: timedelta, - soc_min: float | None, - soc_max: float | None, -) -> None: - """Apply one projection rule to one off-tick SoC event. - - Computes the capacity-adjusted bound value (event value +/- the energy the - device can move over the rule's capacity period), clamps it to the global SoC - limits, and merges it into the projected minima or maxima (keeping the - stricter bound if one already exists on the same tick). - """ - capacity_start, capacity_end = _capacity_period_for_projection_rule( - rule, previous_tick, event_time, next_tick - ) - capacity = _capacity_for_projection_rule( - rule, consumption_capacity, production_capacity - ) - projected_value = _soc_value_in_mwh(soc_event["value"]) + rule.sign * ( - _energy_capacity_between(capacity, capacity_start, capacity_end, resolution) - ) - projected_soc_events = ( - projected_minima if rule.bound_type == "min" else projected_maxima - ) - _add_soc_bound( - projected_soc_events, - _soc_event_at( - soc_event, - _tick_for_projection_rule(rule, previous_tick, next_tick), - _clamp_projected_soc_value(projected_value, rule, soc_min, soc_max), - ), - bound_type=rule.bound_type, - ) + if capacity.index.tz is not None: + tick = tick.tz_convert(capacity.index.tz) + capacity_in_mw = capacity.get(tick) + if capacity_in_mw is None or pd.isna(capacity_in_mw): + return 0 + return float(capacity_in_mw) * ((end - start) / pd.Timedelta(hours=1)) def _add_soc_bound( @@ -301,11 +153,22 @@ def project_off_tick_soc_constraints( next tick that preserve reachability using the available charge and discharge capacity between the original event time and those ticks. - ``soc-targets`` are projected to an exact target on the next tick, plus - capacity-adjusted lower and upper bounds on the previous tick. ``soc-minima`` - become lower bounds on both surrounding ticks, and ``soc-maxima`` become upper - bounds on both surrounding ticks. If multiple projected bounds land on the same - tick, the stricter lower or upper bound is kept. + For an off-tick event with value ``v`` at time ``t``, between previous tick ``p`` + and next tick ``n``: + + - ``soc-targets`` become an exact target ``v`` on ``n``, plus bounds on ``p`` that + keep the target reachable at ``t``: a lower bound of ``v`` minus the energy that + can still be charged between ``p`` and ``t``, and an upper bound of ``v`` plus + the energy that can still be discharged between ``p`` and ``t``. + - ``soc-minima`` become lower bounds on both surrounding ticks: on ``p``, ``v`` + minus the energy that can be charged between ``p`` and ``t``; on ``n``, ``v`` + minus the energy that can be discharged between ``t`` and ``n``. + - ``soc-maxima`` become upper bounds on both surrounding ticks: on ``p``, ``v`` + plus the energy that can be discharged between ``p`` and ``t``; on ``n``, ``v`` + plus the energy that can be charged between ``t`` and ``n``. + + If multiple projected bounds land on the same tick, the stricter lower or upper + bound is kept. Projected bounds are clamped to the global ``soc-min``/``soc-max``. Returns ``(soc_targets, soc_maxima, soc_minima)`` with projected list-based timed events. Non-list specifications such as sensors, series, fixed @@ -325,6 +188,20 @@ def project_off_tick_soc_constraints( soc_min_value = _optional_soc_value_in_mwh(soc_min) soc_max_value = _optional_soc_value_in_mwh(soc_max) + def add_min_bound(soc_event: dict, tick: pd.Timestamp, value: float) -> None: + _add_soc_bound( + projected_minima, + _soc_event_at(soc_event, tick, _clamp_soc_min(value, soc_min_value)), + bound_type="min", + ) + + def add_max_bound(soc_event: dict, tick: pd.Timestamp, value: float) -> None: + _add_soc_bound( + projected_maxima, + _soc_event_at(soc_event, tick, _clamp_soc_max(value, soc_max_value)), + bound_type="max", + ) + if isinstance(soc_targets, list): projected_targets = [] for soc_target in soc_targets: @@ -338,33 +215,54 @@ def project_off_tick_soc_constraints( previous_tick = target_time.floor(resolution) next_tick = target_time.ceil(resolution) target_value = _soc_value_in_mwh(soc_target["value"]) + chargeable = _reachable_energy( + consumption_capacity, previous_tick, target_time, resolution + ) + dischargeable = _reachable_energy( + production_capacity, previous_tick, target_time, resolution + ) + # Exact target on the next tick, plus previous-tick bounds that keep + # the target reachable at the original (off-tick) target time. projected_targets.append(_soc_event_at(soc_target, next_tick, target_value)) - for rule in SOC_PROJECTION_POLICIES["soc-targets"]: - _add_projected_soc_bound( - projected_minima, - projected_maxima, - soc_target, - rule, - target_time, - previous_tick, - next_tick, - consumption_capacity, - production_capacity, - resolution, - soc_min_value, - soc_max_value, - ) + add_min_bound(soc_target, previous_tick, target_value - chargeable) + add_max_bound(soc_target, previous_tick, target_value + dischargeable) else: projected_targets = soc_targets - for field_name, soc_events in ( - ("soc-minima", soc_minima), - ("soc-maxima", soc_maxima), - ): - if not isinstance(soc_events, list): - continue - for soc_event in copy.deepcopy(soc_events): + if isinstance(soc_minima, list): + for soc_event in copy.deepcopy(soc_minima): + if soc_event["start"] != soc_event["end"] or is_on_schedule_tick( + soc_event["end"], resolution + ): + continue + + event_time = pd.Timestamp(soc_event["end"]) + previous_tick = event_time.floor(resolution) + next_tick = event_time.ceil(resolution) + minimum = _soc_value_in_mwh(soc_event["value"]) + + # Lower bounds on both surrounding ticks that preserve whether the + # requested minimum can still be reached at the original event time. + add_min_bound( + soc_event, + previous_tick, + minimum + - _reachable_energy( + consumption_capacity, previous_tick, event_time, resolution + ), + ) + add_min_bound( + soc_event, + next_tick, + minimum + - _reachable_energy( + production_capacity, event_time, next_tick, resolution + ), + ) + + if isinstance(soc_maxima, list): + for soc_event in copy.deepcopy(soc_maxima): if soc_event["start"] != soc_event["end"] or is_on_schedule_tick( soc_event["end"], resolution ): @@ -373,21 +271,26 @@ def project_off_tick_soc_constraints( event_time = pd.Timestamp(soc_event["end"]) previous_tick = event_time.floor(resolution) next_tick = event_time.ceil(resolution) - for rule in SOC_PROJECTION_POLICIES[field_name]: - _add_projected_soc_bound( - projected_minima, - projected_maxima, - soc_event, - rule, - event_time, - previous_tick, - next_tick, - consumption_capacity, - production_capacity, - resolution, - soc_min_value, - soc_max_value, - ) + maximum = _soc_value_in_mwh(soc_event["value"]) + + # Upper bounds on both surrounding ticks that preserve whether the + # requested maximum can still be respected at the original event time. + add_max_bound( + soc_event, + previous_tick, + maximum + + _reachable_energy( + production_capacity, previous_tick, event_time, resolution + ), + ) + add_max_bound( + soc_event, + next_tick, + maximum + + _reachable_energy( + consumption_capacity, event_time, next_tick, resolution + ), + ) return ( projected_targets, From 8ba7a524ca77889913a15228c2ef7ee1b4257811 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 Jul 2026 14:15:34 +0200 Subject: [PATCH 56/72] fix: project off-tick SoC constraints before they are softened into commitments Previously the projection ran only in the hard-constraint path, after soc-minima/soc-maxima with breach prices had already been converted to soft StockCommitments (and nulled). Since off-tick SoC constraints auto-enable relax-soc-constraints, off-tick minima/maxima would in practice reach the commitment builder unprojected, while bounds projected from off-tick targets were applied as hard constraints despite relaxation being enabled. Now the projection runs right after the device capacities are known and before the soft/hard split, so both paths consume on-tick events: projected bounds are softened by breach prices exactly like user-given bounds, and only fall back to hard constraints when no breach price is set. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01YNPESudGKciPZq88nGJn3C Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 290 ++++++++++--------- 1 file changed, 147 insertions(+), 143 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 4e4a6508a1..d08f3ba1f9 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -811,70 +811,6 @@ def device_list_series( as_instantaneous_events=True, resolve_overlaps="max", ) - if ( - self.flex_context.get("soc_minima_breach_price") is not None - and soc_minima[d] is not None - ): - soc_minima_breach_price = self.flex_context["soc_minima_breach_price"] - any_soc_minima_breach_price = get_continuous_series_sensor_or_quantity( - variable_quantity=soc_minima_breach_price, - unit=self.flex_context["shared_currency_unit"] + "/MWh", - query_window=(start + resolution, end + resolution), - resolution=resolution, - beliefs_before=belief_time, - fill_sides=True, - ).shift(-1, freq=resolution) - all_soc_minima_breach_price = get_continuous_series_sensor_or_quantity( - variable_quantity=soc_minima_breach_price, - unit=self.flex_context["shared_currency_unit"] - + "/MWh*h", # from EUR/MWh² to EUR/MWh/resolution - query_window=(start + resolution, end + resolution), - resolution=resolution, - beliefs_before=belief_time, - fill_sides=True, - ).shift(-1, freq=resolution) - # Set up commitments DataFrame - # soc_minima_d is a temp variable because add_storage_constraints can't deal with Series yet - soc_minima_d = get_continuous_series_sensor_or_quantity( - variable_quantity=soc_minima[d], - unit="MWh", - query_window=(start + resolution, end + resolution), - resolution=resolution, - beliefs_before=belief_time, - as_instantaneous_events=True, - resolve_overlaps="max", - ) - # shift soc minima by one resolution (they define a state at a certain time, - # while the commitment defines what the total stock should be at the end of a time slot, - # where the time slot is indexed by its starting time) - soc_minima_d = soc_minima_d.shift(-1, freq=resolution) * ( - timedelta(hours=1) / resolution - ) - soc_at_start[d] * (timedelta(hours=1) / resolution) - - commitment = StockCommitment( - name="any soc minima", - quantity=soc_minima_d, - # negative price because breaching in the downwards (shortage) direction is penalized - downwards_deviation_price=-any_soc_minima_breach_price, - index=index, - _type="any", - device=d, - ) - commitments.append(commitment) - - commitment = StockCommitment( - name="all soc minima", - quantity=soc_minima_d, - # negative price because breaching in the downwards (shortage) direction is penalized - downwards_deviation_price=-all_soc_minima_breach_price, - index=index, - device=d, - ) - commitments.append(commitment) - - # soc-minima will become a soft constraint (modelled as stock commitments), so remove hard constraint - soc_minima[d] = None - if isinstance(soc_maxima[d], (Sensor, SensorReference)): soc_maxima[d] = get_continuous_series_sensor_or_quantity( variable_quantity=soc_maxima[d], @@ -885,69 +821,6 @@ def device_list_series( as_instantaneous_events=True, resolve_overlaps="min", ) - if ( - self.flex_context.get("soc_maxima_breach_price") is not None - and soc_maxima[d] is not None - ): - soc_maxima_breach_price = self.flex_context["soc_maxima_breach_price"] - any_soc_maxima_breach_price = get_continuous_series_sensor_or_quantity( - variable_quantity=soc_maxima_breach_price, - unit=self.flex_context["shared_currency_unit"] + "/MWh", - query_window=(start + resolution, end + resolution), - resolution=resolution, - beliefs_before=belief_time, - fill_sides=True, - ).shift(-1, freq=resolution) - all_soc_maxima_breach_price = get_continuous_series_sensor_or_quantity( - variable_quantity=soc_maxima_breach_price, - unit=self.flex_context["shared_currency_unit"] - + "/MWh*h", # from EUR/MWh² to EUR/MWh/resolution - query_window=(start + resolution, end + resolution), - resolution=resolution, - beliefs_before=belief_time, - fill_sides=True, - ).shift(-1, freq=resolution) - # Set up commitments DataFrame - # soc_maxima_d is a temp variable because add_storage_constraints can't deal with Series yet - soc_maxima_d = get_continuous_series_sensor_or_quantity( - variable_quantity=soc_maxima[d], - unit="MWh", - query_window=(start + resolution, end + resolution), - resolution=resolution, - beliefs_before=belief_time, - as_instantaneous_events=True, - resolve_overlaps="min", - ) - # shift soc maxima by one resolution (they define a state at a certain time, - # while the commitment defines what the total stock should be at the end of a time slot, - # where the time slot is indexed by its starting time) - soc_maxima_d = soc_maxima_d.shift(-1, freq=resolution) * ( - timedelta(hours=1) / resolution - ) - soc_at_start[d] * (timedelta(hours=1) / resolution) - - commitment = StockCommitment( - name="any soc maxima", - quantity=soc_maxima_d, - # positive price because breaching in the upwards (surplus) direction is penalized - upwards_deviation_price=any_soc_maxima_breach_price, - index=index, - _type="any", - device=d, - ) - commitments.append(commitment) - - commitment = StockCommitment( - name="all soc maxima", - quantity=soc_maxima_d, - # positive price because breaching in the upwards (surplus) direction is penalized - upwards_deviation_price=all_soc_maxima_breach_price, - index=index, - device=d, - ) - commitments.append(commitment) - - # soc-maxima will become a soft constraint (modelled as stock commitments), so remove hard constraint - soc_maxima[d] = None power_capacity_in_mw[d] = get_continuous_series_sensor_or_quantity( variable_quantity=power_capacity_in_mw[d], @@ -1102,6 +975,153 @@ def device_list_series( # consumption-capacity will become a hard constraint device_constraints[d]["derivative max"] = consumption_capacity_d + # Project off-tick point-like SoC constraints onto the scheduling ticks + # before they are turned into soft commitments or hard constraints, + # so that both paths consume on-tick events. + if should_project_off_tick_soc_constraints(sensor_d): + ( + soc_targets[d], + soc_maxima[d], + soc_minima[d], + ) = project_off_tick_soc_constraints( + soc_targets[d], + soc_maxima[d], + soc_minima[d], + consumption_capacity_d, + production_capacity_d, + resolution, + soc_min[d], + soc_max[d], + ) + + if ( + self.flex_context.get("soc_minima_breach_price") is not None + and soc_minima[d] is not None + ): + soc_minima_breach_price = self.flex_context["soc_minima_breach_price"] + any_soc_minima_breach_price = get_continuous_series_sensor_or_quantity( + variable_quantity=soc_minima_breach_price, + unit=self.flex_context["shared_currency_unit"] + "/MWh", + query_window=(start + resolution, end + resolution), + resolution=resolution, + beliefs_before=belief_time, + fill_sides=True, + ).shift(-1, freq=resolution) + all_soc_minima_breach_price = get_continuous_series_sensor_or_quantity( + variable_quantity=soc_minima_breach_price, + unit=self.flex_context["shared_currency_unit"] + + "/MWh*h", # from EUR/MWh² to EUR/MWh/resolution + query_window=(start + resolution, end + resolution), + resolution=resolution, + beliefs_before=belief_time, + fill_sides=True, + ).shift(-1, freq=resolution) + # Set up commitments DataFrame + # soc_minima_d is a temp variable because add_storage_constraints can't deal with Series yet + soc_minima_d = get_continuous_series_sensor_or_quantity( + variable_quantity=soc_minima[d], + unit="MWh", + query_window=(start + resolution, end + resolution), + resolution=resolution, + beliefs_before=belief_time, + as_instantaneous_events=True, + resolve_overlaps="max", + ) + # shift soc minima by one resolution (they define a state at a certain time, + # while the commitment defines what the total stock should be at the end of a time slot, + # where the time slot is indexed by its starting time) + soc_minima_d = soc_minima_d.shift(-1, freq=resolution) * ( + timedelta(hours=1) / resolution + ) - soc_at_start[d] * (timedelta(hours=1) / resolution) + + commitment = StockCommitment( + name="any soc minima", + quantity=soc_minima_d, + # negative price because breaching in the downwards (shortage) direction is penalized + downwards_deviation_price=-any_soc_minima_breach_price, + index=index, + _type="any", + device=d, + ) + commitments.append(commitment) + + commitment = StockCommitment( + name="all soc minima", + quantity=soc_minima_d, + # negative price because breaching in the downwards (shortage) direction is penalized + downwards_deviation_price=-all_soc_minima_breach_price, + index=index, + device=d, + ) + commitments.append(commitment) + + # soc-minima will become a soft constraint (modelled as stock commitments), so remove hard constraint + soc_minima[d] = None + + if ( + self.flex_context.get("soc_maxima_breach_price") is not None + and soc_maxima[d] is not None + ): + soc_maxima_breach_price = self.flex_context["soc_maxima_breach_price"] + any_soc_maxima_breach_price = get_continuous_series_sensor_or_quantity( + variable_quantity=soc_maxima_breach_price, + unit=self.flex_context["shared_currency_unit"] + "/MWh", + query_window=(start + resolution, end + resolution), + resolution=resolution, + beliefs_before=belief_time, + fill_sides=True, + ).shift(-1, freq=resolution) + all_soc_maxima_breach_price = get_continuous_series_sensor_or_quantity( + variable_quantity=soc_maxima_breach_price, + unit=self.flex_context["shared_currency_unit"] + + "/MWh*h", # from EUR/MWh² to EUR/MWh/resolution + query_window=(start + resolution, end + resolution), + resolution=resolution, + beliefs_before=belief_time, + fill_sides=True, + ).shift(-1, freq=resolution) + # Set up commitments DataFrame + # soc_maxima_d is a temp variable because add_storage_constraints can't deal with Series yet + soc_maxima_d = get_continuous_series_sensor_or_quantity( + variable_quantity=soc_maxima[d], + unit="MWh", + query_window=(start + resolution, end + resolution), + resolution=resolution, + beliefs_before=belief_time, + as_instantaneous_events=True, + resolve_overlaps="min", + ) + # shift soc maxima by one resolution (they define a state at a certain time, + # while the commitment defines what the total stock should be at the end of a time slot, + # where the time slot is indexed by its starting time) + soc_maxima_d = soc_maxima_d.shift(-1, freq=resolution) * ( + timedelta(hours=1) / resolution + ) - soc_at_start[d] * (timedelta(hours=1) / resolution) + + commitment = StockCommitment( + name="any soc maxima", + quantity=soc_maxima_d, + # positive price because breaching in the upwards (surplus) direction is penalized + upwards_deviation_price=any_soc_maxima_breach_price, + index=index, + _type="any", + device=d, + ) + commitments.append(commitment) + + commitment = StockCommitment( + name="all soc maxima", + quantity=soc_maxima_d, + # positive price because breaching in the upwards (surplus) direction is penalized + upwards_deviation_price=all_soc_maxima_breach_price, + index=index, + device=d, + ) + commitments.append(commitment) + + # soc-maxima will become a soft constraint (modelled as stock commitments), so remove hard constraint + soc_maxima[d] = None + # only apply SOC constraints to the first device of a shared stock apply_soc_constraints = True for stock_id, devices in self.stock_groups.items(): @@ -1110,22 +1130,6 @@ def device_list_series( break if soc_at_start[d] is not None and apply_soc_constraints: - if should_project_off_tick_soc_constraints(sensor_d): - ( - soc_targets[d], - soc_maxima[d], - soc_minima[d], - ) = project_off_tick_soc_constraints( - soc_targets[d], - soc_maxima[d], - soc_minima[d], - consumption_capacity_d, - production_capacity_d, - resolution, - soc_min[d], - soc_max[d], - ) - storage_constraints = add_storage_constraints( start, end, From cd0b69ffb3c98ba969ddcf5fbdf95feb0fed40c8 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 Jul 2026 14:17:57 +0200 Subject: [PATCH 57/72] feat: account for (dis)charging efficiencies in off-tick SoC reachability The reachable energy used to project off-tick SoC constraints now converts grid power to stock change: charging at power P moves the stock at rate P * charging_efficiency (which can exceed 1, e.g. a heat pump's COP), and discharging at power P moves the stock at rate P / discharging_efficiency. To that end, compute the (dis)charging efficiency series before the SoC constraint handling in the device loop and pass them to the projection. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01YNPESudGKciPZq88nGJn3C Signed-off-by: F.N. Claessen --- .../data/models/planning/soc_projection.py | 95 +++++++++++++++++-- flexmeasures/data/models/planning/storage.py | 76 +++++++-------- 2 files changed, 126 insertions(+), 45 deletions(-) diff --git a/flexmeasures/data/models/planning/soc_projection.py b/flexmeasures/data/models/planning/soc_projection.py index 15b58d75a9..6a0c63d83d 100644 --- a/flexmeasures/data/models/planning/soc_projection.py +++ b/flexmeasures/data/models/planning/soc_projection.py @@ -56,13 +56,38 @@ def _soc_event_at( return shifted_event +def _efficiency_at( + efficiency: pd.Series | ur.Quantity | float | None, + tick: pd.Timestamp, +) -> float: + """Return the (dis)charging efficiency at the given tick as a plain float. + + Efficiencies may be given as a series (indexed by tick start), a fixed + dimensionless quantity, or a plain number. Missing values default to 1. + """ + if efficiency is None: + return 1 + if isinstance(efficiency, pd.Series): + if efficiency.index.tz is not None: + tick = tick.tz_convert(efficiency.index.tz) + value = efficiency.get(tick) + if value is None or pd.isna(value): + return 1 + return float(value) + if isinstance(efficiency, ur.Quantity): + return float(efficiency.to("dimensionless").magnitude) + return float(efficiency) + + def _reachable_energy( capacity: pd.Series, start: pd.Timestamp, end: pd.Timestamp, resolution: timedelta, + efficiency: pd.Series | ur.Quantity | float | None = None, + efficiency_affects_stock: str = "multiply", ) -> float: - """Compute how much energy (in MWh) the device can move between two times. + """Compute by how much energy (in MWh) the device can move its stock between two times. The period from ``start`` to ``end`` always lies within a single scheduling tick (from the tick just before an off-tick event to the event, or from the @@ -70,6 +95,13 @@ def _reachable_energy( indexed by tick start at the given resolution) applies at a single constant value. A missing capacity value counts as zero (conservative: the projected bound then sticks close to the original event value). + + The capacity limits the power exchanged with the grid, while the projected + bounds concern the stock (SoC). Charging at power P raises the stock at rate + P * charging_efficiency (``efficiency_affects_stock="multiply"``; note that a + charging efficiency can exceed 1, e.g. a heat pump's COP), while discharging + at power P lowers the stock at rate P / discharging_efficiency + (``efficiency_affects_stock="divide"``). """ if end <= start: return 0 @@ -79,7 +111,17 @@ def _reachable_energy( capacity_in_mw = capacity.get(tick) if capacity_in_mw is None or pd.isna(capacity_in_mw): return 0 - return float(capacity_in_mw) * ((end - start) / pd.Timedelta(hours=1)) + efficiency_value = _efficiency_at(efficiency, tick) + stock_rate_in_mw = float(capacity_in_mw) + if efficiency_affects_stock == "multiply": + stock_rate_in_mw *= efficiency_value + elif efficiency_value != 0: + stock_rate_in_mw /= efficiency_value + else: + # A zero discharging efficiency means the stock can drop arbitrarily + # fast without producing any power, so the bound becomes unbounded. + stock_rate_in_mw = float("inf") + return stock_rate_in_mw * ((end - start) / pd.Timedelta(hours=1)) def _add_soc_bound( @@ -140,6 +182,8 @@ def project_off_tick_soc_constraints( resolution: timedelta, soc_min: ur.Quantity | float | None, soc_max: ur.Quantity | float | None, + charging_efficiency: pd.Series | ur.Quantity | float | None = None, + discharging_efficiency: pd.Series | ur.Quantity | float | None = None, ) -> tuple[ list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, @@ -167,6 +211,11 @@ def project_off_tick_soc_constraints( plus the energy that can be discharged between ``p`` and ``t``; on ``n``, ``v`` plus the energy that can be charged between ``t`` and ``n``. + The reachable energy accounts for the (dis)charging efficiencies: charging at + grid power P moves the stock at rate P * charging_efficiency (which can exceed + 1, e.g. a heat pump's COP), and discharging at grid power P moves the stock at + rate P / discharging_efficiency. + If multiple projected bounds land on the same tick, the stricter lower or upper bound is kept. Projected bounds are clamped to the global ``soc-min``/``soc-max``. @@ -216,10 +265,20 @@ def add_max_bound(soc_event: dict, tick: pd.Timestamp, value: float) -> None: next_tick = target_time.ceil(resolution) target_value = _soc_value_in_mwh(soc_target["value"]) chargeable = _reachable_energy( - consumption_capacity, previous_tick, target_time, resolution + consumption_capacity, + previous_tick, + target_time, + resolution, + efficiency=charging_efficiency, + efficiency_affects_stock="multiply", ) dischargeable = _reachable_energy( - production_capacity, previous_tick, target_time, resolution + production_capacity, + previous_tick, + target_time, + resolution, + efficiency=discharging_efficiency, + efficiency_affects_stock="divide", ) # Exact target on the next tick, plus previous-tick bounds that keep @@ -249,7 +308,12 @@ def add_max_bound(soc_event: dict, tick: pd.Timestamp, value: float) -> None: previous_tick, minimum - _reachable_energy( - consumption_capacity, previous_tick, event_time, resolution + consumption_capacity, + previous_tick, + event_time, + resolution, + efficiency=charging_efficiency, + efficiency_affects_stock="multiply", ), ) add_min_bound( @@ -257,7 +321,12 @@ def add_max_bound(soc_event: dict, tick: pd.Timestamp, value: float) -> None: next_tick, minimum - _reachable_energy( - production_capacity, event_time, next_tick, resolution + production_capacity, + event_time, + next_tick, + resolution, + efficiency=discharging_efficiency, + efficiency_affects_stock="divide", ), ) @@ -280,7 +349,12 @@ def add_max_bound(soc_event: dict, tick: pd.Timestamp, value: float) -> None: previous_tick, maximum + _reachable_energy( - production_capacity, previous_tick, event_time, resolution + production_capacity, + previous_tick, + event_time, + resolution, + efficiency=discharging_efficiency, + efficiency_affects_stock="divide", ), ) add_max_bound( @@ -288,7 +362,12 @@ def add_max_bound(soc_event: dict, tick: pd.Timestamp, value: float) -> None: next_tick, maximum + _reachable_energy( - consumption_capacity, event_time, next_tick, resolution + consumption_capacity, + event_time, + next_tick, + resolution, + efficiency=charging_efficiency, + efficiency_affects_stock="multiply", ), ) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index d08f3ba1f9..707bc0162f 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -975,6 +975,43 @@ def device_list_series( # consumption-capacity will become a hard constraint device_constraints[d]["derivative max"] = consumption_capacity_d + # Apply round-trip efficiency evenly to charging and discharging + charging_efficiency[d] = ( + get_continuous_series_sensor_or_quantity( + variable_quantity=charging_efficiency[d], + unit="dimensionless", + query_window=(start, end), + resolution=resolution, + beliefs_before=belief_time, + ) + .astype(float) + .fillna(1) + ) + discharging_efficiency[d] = ( + get_continuous_series_sensor_or_quantity( + variable_quantity=discharging_efficiency[d], + unit="dimensionless", + query_window=(start, end), + resolution=resolution, + beliefs_before=belief_time, + ) + .astype(float) + .fillna(1) + ) + + roundtrip_efficiency = flex_model[d].get( + "roundtrip_efficiency", + asset_d.flex_model.get("roundtrip-efficiency", 1), + ) + + # if roundtrip efficiency is provided in the flex-model or defined as an asset attribute + if ( + "roundtrip_efficiency" in flex_model[d] + or asset_d.flex_model.get("roundtrip-efficiency") is not None + ): + charging_efficiency[d] = roundtrip_efficiency**0.5 + discharging_efficiency[d] = roundtrip_efficiency**0.5 + # Project off-tick point-like SoC constraints onto the scheduling ticks # before they are turned into soft commitments or hard constraints, # so that both paths consume on-tick events. @@ -992,6 +1029,8 @@ def device_list_series( resolution, soc_min[d], soc_max[d], + charging_efficiency=charging_efficiency[d], + discharging_efficiency=discharging_efficiency[d], ) if ( @@ -1179,43 +1218,6 @@ def device_list_series( device_constraints[d]["stock delta"] = all_stock_delta.sum(1) device_constraints[d]["stock delta"] *= timedelta(hours=1) / resolution - # Apply round-trip efficiency evenly to charging and discharging - charging_efficiency[d] = ( - get_continuous_series_sensor_or_quantity( - variable_quantity=charging_efficiency[d], - unit="dimensionless", - query_window=(start, end), - resolution=resolution, - beliefs_before=belief_time, - ) - .astype(float) - .fillna(1) - ) - discharging_efficiency[d] = ( - get_continuous_series_sensor_or_quantity( - variable_quantity=discharging_efficiency[d], - unit="dimensionless", - query_window=(start, end), - resolution=resolution, - beliefs_before=belief_time, - ) - .astype(float) - .fillna(1) - ) - - roundtrip_efficiency = flex_model[d].get( - "roundtrip_efficiency", - asset_d.flex_model.get("roundtrip-efficiency", 1), - ) - - # if roundtrip efficiency is provided in the flex-model or defined as an asset attribute - if ( - "roundtrip_efficiency" in flex_model[d] - or asset_d.flex_model.get("roundtrip-efficiency") is not None - ): - charging_efficiency[d] = roundtrip_efficiency**0.5 - discharging_efficiency[d] = roundtrip_efficiency**0.5 - device_constraints[d]["derivative down efficiency"] = ( discharging_efficiency[d] ) From 54c576bc491d60f24822f255ce4fe67e2dd2e531 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 Jul 2026 14:18:39 +0200 Subject: [PATCH 58/72] fix: ceil the schedule-end extension to the scheduling resolution Off-tick soc-target datetimes are no longer floored during deserialization, so a target that drives the schedule extension could set an off-tick schedule end. Its projection onto the next scheduling tick would then fall beyond the schedule end and be silently disregarded. Ceiling the extension keeps the projected target within the schedule. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01YNPESudGKciPZq88nGJn3C Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 707bc0162f..546d9a7632 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -1798,6 +1798,15 @@ def possibly_extend_end(self, soc_targets, sensor: Sensor = None): if soc_targets and not isinstance(soc_targets, (Sensor, SensorReference)): max_target_datetime = max([soc_target["end"] for soc_target in soc_targets]) + # Off-tick target times are preserved during deserialization, and their + # projection moves the target to the next scheduling tick. Ceil to the + # scheduling resolution, so the projected target still falls within the + # schedule (instead of being disregarded as beyond its end). + resolution = self.resolution or sensor.event_resolution + if resolution not in (None, timedelta(0)): + max_target_datetime = ( + pd.Timestamp(max_target_datetime).ceil(resolution).to_pydatetime() + ) if max_target_datetime > self.end: max_server_horizon = get_max_planning_horizon(sensor.event_resolution) if max_server_horizon: From d2360b51520eba5a5c0aaf4bab96886b77091263 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 Jul 2026 14:19:30 +0200 Subject: [PATCH 59/72] fix: respect an explicit relax-soc-constraints: False on off-tick SoC constraints Auto-enabling SoC constraint relaxation used to silently override a user's explicit choice to keep SoC constraints hard. Now the explicit False is respected (with a logged warning that projection may add infeasible bounds), and auto-enabling is logged as well. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01YNPESudGKciPZq88nGJn3C Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 31 +++++++++++++-- .../models/planning/tests/test_storage.py | 39 +++++++++++++------ 2 files changed, 54 insertions(+), 16 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 546d9a7632..76cef9564e 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -1535,17 +1535,40 @@ def _deserialize_flex_model(self): return self.flex_model def enable_relax_soc_constraints(self) -> None: - """Relax SoC constraints when off-tick SoC events require scheduling-tick projection.""" + """Relax SoC constraints when off-tick SoC events require scheduling-tick projection. + + Projection can add bounds (and stricter combinations of bounds), which could + render the problem infeasible if they remain hard constraints. Therefore, + ``relax-soc-constraints`` is enabled unless the user explicitly disabled it, + in which case we respect that choice and only log a warning. + """ + + def _enable(context: dict) -> None: + if context.get("relax-soc-constraints") is False: + current_app.logger.warning( + "Off-tick SoC constraints are projected onto the scheduling ticks, " + "which can add bounds that render the scheduling problem infeasible, " + "but 'relax-soc-constraints' is explicitly disabled. " + "Keeping SoC constraints hard." + ) + return + if context.get("relax-soc-constraints") is not True: + current_app.logger.info( + "Enabling 'relax-soc-constraints' because off-tick SoC constraints " + "are projected onto the scheduling ticks." + ) + context["relax-soc-constraints"] = True + if self.flex_context is None: self.flex_context = {} if isinstance(self.flex_context, dict): - self.flex_context["relax-soc-constraints"] = True + _enable(self.flex_context) for commodity_context in self.flex_context.get("commodities", []): - commodity_context["relax-soc-constraints"] = True + _enable(commodity_context) return if isinstance(self.flex_context, list): for commodity_context in self.flex_context: - commodity_context["relax-soc-constraints"] = True + _enable(commodity_context) def has_soc_at_start(self) -> bool: return ( diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py index b9384ea289..e25146ab0f 100644 --- a/flexmeasures/data/models/planning/tests/test_storage.py +++ b/flexmeasures/data/models/planning/tests/test_storage.py @@ -1222,8 +1222,14 @@ def test_off_tick_soc_bounds_are_merged_on_the_same_scheduling_tick(): ), "merged maxima should keep the stricter upper bound on the next tick" -def test_off_tick_soc_constraints_enable_relax_soc_constraints(add_battery_assets, db): - """Off-tick SoC constraints enable relaxation because projection can add bounds.""" +@pytest.mark.parametrize("explicit_relax_setting", [None, False]) +def test_off_tick_soc_constraints_enable_relax_soc_constraints( + add_battery_assets, db, explicit_relax_setting +): + """Off-tick SoC constraints enable relaxation because projection can add bounds. + + An explicit ``relax-soc-constraints: False`` is respected, though. + """ _, battery = get_sensors_from_db( db, add_battery_assets, battery_name="Test battery" ) @@ -1252,21 +1258,30 @@ def test_off_tick_soc_constraints_enable_relax_soc_constraints(add_battery_asset flex_context={ "consumption-price": "0 EUR/MWh", "production-price": "0 EUR/MWh", - "relax-soc-constraints": False, + **( + {} + if explicit_relax_setting is None + else {"relax-soc-constraints": explicit_relax_setting} + ), }, ) scheduler.deserialize_config() - assert ( - scheduler.flex_context["relax_soc_constraints"] is True - ), "off-tick SoC constraints should automatically enable SoC relaxation" - assert ( - scheduler.flex_context["soc_minima_breach_price"] is not None - ), "auto-enabled SoC relaxation should include a minima breach price" - assert ( - scheduler.flex_context["soc_maxima_breach_price"] is not None - ), "auto-enabled SoC relaxation should include a maxima breach price" + if explicit_relax_setting is False: + assert ( + scheduler.flex_context["relax_soc_constraints"] is False + ), "an explicit relax-soc-constraints: False should be respected" + else: + assert ( + scheduler.flex_context["relax_soc_constraints"] is True + ), "off-tick SoC constraints should automatically enable SoC relaxation" + assert ( + scheduler.flex_context["soc_minima_breach_price"] is not None + ), "auto-enabled SoC relaxation should include a minima breach price" + assert ( + scheduler.flex_context["soc_maxima_breach_price"] is not None + ), "auto-enabled SoC relaxation should include a maxima breach price" def test_deserialize_storage_soc_at_start_from_state_of_charge_sensor( From dc57d1563d8828016690dff67e401c6066327758 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 Jul 2026 14:21:07 +0200 Subject: [PATCH 60/72] fix: log a warning when projected SoC bounds cannot be merged and are dropped Bounds projected from off-tick SoC events can only be merged into list-based (or missing) soc-minima/soc-maxima specifications. When the counterpart field is given as a sensor, series or fixed quantity, the projected bounds are dropped; that is now logged instead of silent. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01YNPESudGKciPZq88nGJn3C Signed-off-by: F.N. Claessen --- .../data/models/planning/soc_projection.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/flexmeasures/data/models/planning/soc_projection.py b/flexmeasures/data/models/planning/soc_projection.py index 6a0c63d83d..fa1e3bb03d 100644 --- a/flexmeasures/data/models/planning/soc_projection.py +++ b/flexmeasures/data/models/planning/soc_projection.py @@ -3,6 +3,7 @@ from __future__ import annotations import copy +import logging from datetime import datetime, timedelta import pandas as pd @@ -11,6 +12,8 @@ from flexmeasures.data.schemas.scheduling.utils import is_on_schedule_tick from flexmeasures.utils.unit_utils import ur +logger = logging.getLogger(__name__) + def _soc_value_in_mwh(value: ur.Quantity | float | int) -> float: """Return the SoC value as a plain float in MWh.""" @@ -154,16 +157,24 @@ def _projected_soc_events_or_original( list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None ), projected_soc_events: list[dict[str, datetime | float]], + field_name: str, ) -> list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None: """Choose between the projected event list and the original specification. Only list-based (or missing) specifications can absorb projected bounds; - sensors, series and fixed quantities are returned unchanged. + sensors, series and fixed quantities are returned unchanged. If projected + bounds had to be dropped as a result, a warning is logged. """ if isinstance(original_soc_events, list): return projected_soc_events if original_soc_events is None and projected_soc_events: return projected_soc_events + if projected_soc_events: + logger.warning( + f"Dropping {len(projected_soc_events)} projected SoC bound(s): " + f"the '{field_name}' field is not given as a list of timed events, " + f"so projected bounds cannot be merged into it." + ) return original_soc_events @@ -373,6 +384,6 @@ def add_max_bound(soc_event: dict, tick: pd.Timestamp, value: float) -> None: return ( projected_targets, - _projected_soc_events_or_original(soc_maxima, projected_maxima), - _projected_soc_events_or_original(soc_minima, projected_minima), + _projected_soc_events_or_original(soc_maxima, projected_maxima, "soc-maxima"), + _projected_soc_events_or_original(soc_minima, projected_minima, "soc-minima"), ) From 2e49f87eba7da4338ebafda4eb7616376c69b6cc Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 Jul 2026 14:25:28 +0200 Subject: [PATCH 61/72] refactor: remove inert flooring machinery from StorageFlexModelSchema Assigning new instance-level soc fields in __init__ never had any effect: marshmallow resolves fields from the class-level declared fields, so the flooring resolution (and timezone) passed to those instance fields was dead code, contradicting this PR's preservation of off-tick SoC event datetimes. Remove it, and instead realize the intended per-instance overrides (timezone and default soc-unit) by setting attributes on the bound fields. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01YNPESudGKciPZq88nGJn3C Signed-off-by: F.N. Claessen --- .../data/schemas/scheduling/storage.py | 45 +++++-------------- 1 file changed, 10 insertions(+), 35 deletions(-) diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index f94114642a..11c92be2db 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -270,14 +270,6 @@ def __init__( self.soc_constraint_resolution = get_soc_constraint_resolution( schedule_resolution, sensor, default_resolution ) - self.flooring_resolution = ( - sensor.event_resolution - if sensor is not None - and sensor.event_resolution != timedelta(0) - and sensor.get_attribute("floor_datetimes_to_resolution", True) - else None - ) - # guess default soc-unit if default_soc_unit is None: if self.sensor is not None and self.sensor.unit in ("MWh", "kWh"): @@ -287,34 +279,17 @@ def __init__( else: default_soc_unit = "MWh" - self.soc_maxima = VariableQuantityField( - to_unit="MWh", - default_src_unit=default_soc_unit, - timezone=self.timezone, - event_resolution=self.flooring_resolution, - data_key="soc-maxima", - ) - - self.soc_minima = VariableQuantityField( - to_unit="MWh", - default_src_unit=default_soc_unit, - timezone=self.timezone, - event_resolution=self.flooring_resolution, - data_key="soc-minima", - value_validator=validate.Range(min=0), - ) - self.soc_targets = VariableQuantityField( - to_unit="MWh", - default_src_unit=default_soc_unit, - timezone=self.timezone, - event_resolution=self.flooring_resolution, - data_key="soc-targets", - ) - super().__init__(*args, **kwargs) - if default_soc_unit is not None: - for field in self.fields.keys(): - if field.startswith("soc_"): + for field in self.fields.keys(): + if field.startswith("soc_"): + # Override the class-level placeholders. Note that assigning new + # instance-level fields would be inert (marshmallow resolves fields + # from the class-level declared fields), so we set attributes on + # the bound fields instead. SoC event datetimes are deliberately + # not floored (no event_resolution is set): off-tick events are + # preserved and later projected onto the scheduling ticks. + setattr(self.fields[field], "timezone", self.timezone) + if default_soc_unit is not None: setattr(self.fields[field], "default_src_unit", default_soc_unit) @pre_load From 3576a1ff3fb32a95623816cabd268388308f36b2 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 Jul 2026 14:26:13 +0200 Subject: [PATCH 62/72] fix: warn when off-tick SoC constraints are disregarded without projection When off-tick projection is disabled (floor_datetimes_to_resolution=False), point-like SoC events between scheduling ticks match no entry in the constraint series and were silently dropped. Log a warning instead, and document the attribute's role as the projection opt-out, as well as the fact that an explicit relax-soc-constraints: False is respected. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01YNPESudGKciPZq88nGJn3C Signed-off-by: F.N. Claessen --- documentation/concepts/data-model.rst | 5 ++++- flexmeasures/data/models/planning/storage.py | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/documentation/concepts/data-model.rst b/documentation/concepts/data-model.rst index 6456e3720d..1d0c85ae5b 100644 --- a/documentation/concepts/data-model.rst +++ b/documentation/concepts/data-model.rst @@ -117,7 +117,10 @@ For off-tick ``soc-maxima``, both surrounding ticks receive upper bounds with th The projection uses the ``consumption-capacity`` and ``production-capacity`` active at the relevant ticks. If multiple projected lower bounds land on the same tick, the highest lower bound is kept. If multiple projected upper bounds land on the same tick, the lowest upper bound is kept. -Because projection can introduce additional bounds and more complex combinations can become infeasible, FlexMeasures enables ``relax-soc-constraints`` automatically when off-tick SoC constraints are submitted. +Because projection can introduce additional bounds and more complex combinations can become infeasible, FlexMeasures enables ``relax-soc-constraints`` automatically when off-tick SoC constraints are submitted (unless it was explicitly set to ``False``, which is respected with a logged warning). + +Projection can be disabled per sensor by setting the ``floor_datetimes_to_resolution`` sensor attribute to ``False``. +In that case, off-tick point-like SoC constraints cannot be enforced on the scheduling ticks and are disregarded (with a logged warning). .. _signs_of_power_beliefs: diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index b178fdcc76..cb9d53bf00 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -2901,6 +2901,20 @@ def build_device_soc_values( device_values.loc[soc_constraint_start:end_of_schedule] = soc continue + if ( + soc_constraint_start == soc_constraint_end + and soc_constraint_start not in device_values.index + ): + # Point-like events between scheduling ticks match no index entry. + # This can happen when off-tick projection is disabled through the + # sensor's floor_datetimes_to_resolution attribute. + current_app.logger.warning( + f"Disregarding off-tick SoC constraint at {soc_constraint_start} " + f"(value: {soc}): it does not fall on the scheduling ticks and " + f"off-tick projection is disabled for this sensor." + ) + continue + device_values.loc[soc_constraint_start:soc_constraint_end] = soc if not disregarded_periods: From eafa169cfea38e830f87ab14b63b8556cb38606e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 Jul 2026 14:27:22 +0200 Subject: [PATCH 63/72] refactor: detect off-tick SoC constraints in the scheduler, not the schema The schema's pre_load detection used a guessed resolution (falling back to the sensor's event resolution or 15 minutes even when the scheduler would resolve a different scheduling resolution) and kept mutable per-load state on the schema instance. Detect off-tick SoC events on the serialized flex-model in the scheduler instead, using the scheduler's actual resolution, so the relax decision matches what the projection will actually do. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01YNPESudGKciPZq88nGJn3C Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 45 +++++++++++++------ .../data/schemas/scheduling/storage.py | 23 ---------- 2 files changed, 32 insertions(+), 36 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index cb9d53bf00..8ed9cdd34a 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -41,6 +41,8 @@ project_off_tick_soc_constraints, ) from flexmeasures.data.schemas.scheduling.utils import ( + flex_model_has_off_tick_soc_constraints, + get_soc_constraint_resolution, should_project_off_tick_soc_constraints, ) from flexmeasures.data.schemas.sensors import SensorReference, VariableQuantityField @@ -1373,17 +1375,17 @@ def _deserialize_flex_model(self): if self.sensor.generic_asset.asset_type.name in storage_asset_types: self.ensure_soc_at_start() + self._possibly_relax_off_tick_soc_constraints( + self.flex_model, sensor=self.sensor + ) + # Now it's time to check if our flex configuration holds up to schemas schema = StorageFlexModelSchema( start=self.start, sensor=self.sensor, default_soc_unit=self.flex_model.get("soc-unit"), - schedule_resolution=self.resolution, - default_resolution=self.default_resolution, ) self.flex_model = schema.load(self.flex_model) - if schema.has_off_tick_soc_constraints: - self.enable_relax_soc_constraints() # Extend schedule period in case a target exceeds its end self.possibly_extend_end(soc_targets=self.flex_model.get("soc_targets")) @@ -1400,22 +1402,22 @@ def _deserialize_flex_model(self): soc_sensor = None if soc_sensor_id is not None: soc_sensor = Sensor.query.filter_by(id=soc_sensor_id).first() + sensor_d = ( + sensor_flex_model.get("sensor") + if sensor_flex_model.get("sensor") is not None + else soc_sensor + ) + self._possibly_relax_off_tick_soc_constraints( + sensor_flex_model["sensor_flex_model"], sensor=sensor_d + ) schema = StorageFlexModelSchema( start=self.start, - sensor=( - sensor_flex_model.get("sensor") - if sensor_flex_model.get("sensor") is not None - else soc_sensor - ), + sensor=sensor_d, default_soc_unit=sensor_flex_model["sensor_flex_model"].get( "soc-unit" ), - schedule_resolution=self.resolution, - default_resolution=self.default_resolution, ) self.flex_model[d] = schema.load(sensor_flex_model["sensor_flex_model"]) - if schema.has_off_tick_soc_constraints: - self.enable_relax_soc_constraints() self.flex_model[d]["sensor"] = sensor_flex_model.get("sensor") self.flex_model[d]["asset"] = sensor_flex_model.get("asset") @@ -1431,6 +1433,23 @@ def _deserialize_flex_model(self): return self.flex_model + def _possibly_relax_off_tick_soc_constraints( + self, flex_model: dict, sensor: Sensor | None + ) -> None: + """Enable SoC constraint relaxation if the (serialized) flex-model contains off-tick SoC events. + + The detection uses the scheduler's actual resolution (falling back to the + sensor's event resolution), matching the resolution later used to project + off-tick SoC constraints onto the scheduling ticks. + """ + if not should_project_off_tick_soc_constraints(sensor): + return + resolution = get_soc_constraint_resolution( + self.resolution, sensor, self.default_resolution + ) + if flex_model_has_off_tick_soc_constraints(flex_model, resolution=resolution): + self.enable_relax_soc_constraints() + def enable_relax_soc_constraints(self) -> None: """Relax SoC constraints when off-tick SoC events require scheduling-tick projection. diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index 11c92be2db..e39de7164b 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -6,7 +6,6 @@ from flask import current_app from marshmallow import ( Schema, - pre_load, post_load, validate, validates_schema, @@ -19,11 +18,6 @@ from flexmeasures.data.schemas.generic_assets import GenericAssetIdField from flexmeasures.data.schemas.units import QuantityField from flexmeasures.data.schemas.scheduling import metadata -from flexmeasures.data.schemas.scheduling.utils import ( - flex_model_has_off_tick_soc_constraints, - get_soc_constraint_resolution, - should_project_off_tick_soc_constraints, -) from flexmeasures.data.schemas.sensors import ( SensorReference, OutputSensorReferenceSchema, @@ -258,18 +252,12 @@ def __init__( sensor: Sensor | None, *args, default_soc_unit: str | None = None, - schedule_resolution: timedelta | None = None, - default_resolution: timedelta = timedelta(minutes=15), **kwargs, ): """Pass the schedule's start, so we can use it to validate soc-target datetimes.""" self.start = start self.sensor = sensor self.timezone = sensor.timezone if sensor is not None else None - self.has_off_tick_soc_constraints = False - self.soc_constraint_resolution = get_soc_constraint_resolution( - schedule_resolution, sensor, default_resolution - ) # guess default soc-unit if default_soc_unit is None: if self.sensor is not None and self.sensor.unit in ("MWh", "kWh"): @@ -292,17 +280,6 @@ def __init__( if default_soc_unit is not None: setattr(self.fields[field], "default_src_unit", default_soc_unit) - @pre_load - def detect_off_tick_soc_constraints(self, data: dict, **kwargs) -> dict: - self.has_off_tick_soc_constraints = ( - isinstance(data, dict) - and should_project_off_tick_soc_constraints(self.sensor) - and flex_model_has_off_tick_soc_constraints( - data, resolution=self.soc_constraint_resolution - ) - ) - return data - @validates_schema def check_whether_targets_exceed_max_planning_horizon(self, data: dict, **kwargs): # skip check if the flex-model does not define a sensor: the StorageScheduler will not base its resolution on this flex-model From dd9ba6029d2686ae87211fa90a79b120a96cce8a Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 Jul 2026 14:27:51 +0200 Subject: [PATCH 64/72] docs: explain why the flex-model is deserialized before the flex-context Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01YNPESudGKciPZq88nGJn3C Signed-off-by: F.N. Claessen --- flexmeasures/data/models/planning/storage.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 8ed9cdd34a..4cfcdc7d55 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -1326,6 +1326,8 @@ def deserialize_flex_config(self): self.collect_flex_config() if self.flex_context is None: self.flex_context = {} + # The flex-model is deserialized first, because off-tick SoC constraints + # may enable relax-soc-constraints on the still-serialized flex-context. self._deserialize_flex_model() self._deserialize_flex_context() From d13e37b264b4c9481602f690e4416725b6fc0a75 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 Jul 2026 14:35:15 +0200 Subject: [PATCH 65/72] test: cover efficiency-aware projection, schedule-end extension and the soft path - Pure test that reachable energy uses charging efficiency (incl. COP > 1) and discharging efficiency. - A target beyond the schedule end extends it to the tick carrying the projected target (ceiled), instead of the target being disregarded. - With a soc-minima breach price, projected off-tick minima feed the soft stock commitments rather than becoming hard constraints. - Pin relax-constraints/relax-soc-constraints to False in the hard-path projection tests, since projected bounds are now softened by default. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01YNPESudGKciPZq88nGJn3C Signed-off-by: F.N. Claessen --- .../models/planning/tests/test_storage.py | 168 ++++++++++++++++++ 1 file changed, 168 insertions(+) diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py index e25146ab0f..d16b523bd2 100644 --- a/flexmeasures/data/models/planning/tests/test_storage.py +++ b/flexmeasures/data/models/planning/tests/test_storage.py @@ -972,6 +972,9 @@ def test_off_tick_soc_target_is_projected_to_scheduling_ticks(add_battery_assets "consumption-price": "0 EUR/MWh", "production-price": "0 EUR/MWh", "site-power-capacity": "1 MW", + # keep SoC constraints hard, so we can assert the projected bounds directly + "relax-constraints": False, + "relax-soc-constraints": False, }, ) @@ -1034,6 +1037,9 @@ def test_off_tick_soc_target_is_projected_for_instantaneous_sensor( "consumption-price": "0 EUR/MWh", "production-price": "0 EUR/MWh", "site-power-capacity": "1 MW", + # keep SoC constraints hard, so we can assert the projected bounds directly + "relax-constraints": False, + "relax-soc-constraints": False, }, ) @@ -1284,6 +1290,168 @@ def test_off_tick_soc_constraints_enable_relax_soc_constraints( ), "auto-enabled SoC relaxation should include a maxima breach price" +def test_off_tick_soc_projection_accounts_for_efficiencies(): + """Reachable energy converts grid power to stock change using the (dis)charging efficiencies.""" + tz = pytz.timezone("Europe/Amsterdam") + resolution = timedelta(minutes=15) + previous_tick = pd.Timestamp(tz.localize(datetime(2015, 1, 1, 17))) + next_tick = previous_tick + resolution + capacity = pd.Series( + 0.04, index=pd.date_range(previous_tick, next_tick, freq=resolution) + ) + + _, _, projected_minima = project_off_tick_soc_constraints( + soc_targets=None, + soc_maxima=None, + soc_minima=[ + { + "datetime": tz.localize(datetime(2015, 1, 1, 17, 12)), + "start": tz.localize(datetime(2015, 1, 1, 17, 12)), + "end": tz.localize(datetime(2015, 1, 1, 17, 12)), + "value": 1, + } + ], + consumption_capacity=capacity, + production_capacity=capacity, + resolution=resolution, + soc_min=0, + soc_max=None, + charging_efficiency=4, # e.g. a heat pump's COP + discharging_efficiency=0.8, + ) + + # Charging between 17:00 and 17:12 moves the stock by 0.04 MW * 4 * 0.2 h. + assert _soc_event_value_at(projected_minima, previous_tick) == pytest.approx( + 1 - 0.04 * 4 * 0.2 + ), "the previous-tick minimum should account for the charging efficiency" + # Discharging between 17:12 and 17:15 moves the stock by 0.04 MW / 0.8 * 0.05 h. + assert _soc_event_value_at(projected_minima, next_tick) == pytest.approx( + 1 - 0.04 / 0.8 * 0.05 + ), "the next-tick minimum should account for the discharging efficiency" + + +def test_off_tick_soc_target_extends_schedule_end_to_next_tick(add_battery_assets, db): + """A target beyond the schedule end extends it to a scheduling tick covering the projection.""" + _, battery = get_sensors_from_db( + db, add_battery_assets, battery_name="Test battery" + ) + tz = pytz.timezone("Europe/Amsterdam") + start = tz.localize(datetime(2015, 1, 1, 16, 45)) + end = tz.localize(datetime(2015, 1, 1, 17, 0)) # before the off-tick target + resolution = timedelta(minutes=15) + + scheduler = StorageScheduler( + battery, + start, + end, + resolution, + flex_model={ + "soc-at-start": "0 MWh", + "soc-min": "0 MWh", + "soc-max": "1 MWh", + "power-capacity": "0.04 MW", + "consumption-capacity": "0.04 MW", + "production-capacity": "0 MW", + "roundtrip-efficiency": 1, + "storage-efficiency": 1, + "soc-targets": [ + { + "datetime": "2015-01-01T17:12:00+01:00", + "value": "1 MWh", + } + ], + }, + flex_context={ + "consumption-price": "0 EUR/MWh", + "production-price": "0 EUR/MWh", + "site-power-capacity": "1 MW", + # keep SoC constraints hard, so we can assert the projected target directly + "relax-soc-constraints": False, + }, + ) + + _, _, schedule_end, _, _, device_constraints, _, _ = scheduler._prepare( + skip_validation=True + ) + + assert schedule_end == tz.localize( + datetime(2015, 1, 1, 17, 15) + ), "the schedule end should be ceiled to the tick carrying the projected target" + storage_constraints = device_constraints[0].tz_convert(tz) + assert storage_constraints.loc[start + resolution, "equals"] == pytest.approx( + 4 + ), "the projected target should fall within the (extended) schedule" + + +def test_off_tick_soc_minima_are_projected_into_soft_commitments( + add_battery_assets, db +): + """With a breach price, projected off-tick minima feed the soft commitments, not hard bounds.""" + _, battery = get_sensors_from_db( + db, add_battery_assets, battery_name="Test battery" + ) + tz = pytz.timezone("Europe/Amsterdam") + start = tz.localize(datetime(2015, 1, 1, 16, 45)) + end = tz.localize(datetime(2015, 1, 1, 17, 15)) + resolution = timedelta(minutes=15) + + scheduler = StorageScheduler( + battery, + start, + end, + resolution, + flex_model={ + "soc-at-start": "0 MWh", + "soc-min": "0 MWh", + "soc-max": "1 MWh", + "power-capacity": "0.04 MW", + "consumption-capacity": "0.04 MW", + "production-capacity": "0.04 MW", + "roundtrip-efficiency": 1, + "storage-efficiency": 1, + "soc-minima": [ + { + "datetime": "2015-01-01T17:12:00+01:00", + "value": "1 MWh", + } + ], + }, + flex_context={ + "consumption-price": "0 EUR/MWh", + "production-price": "0 EUR/MWh", + "site-power-capacity": "1 MW", + "soc-minima-breach-price": "1000 EUR/MWh", + }, + ) + + _, _, _, _, _, device_constraints, _, commitments = scheduler._prepare( + skip_validation=True + ) + + storage_constraints = device_constraints[0].tz_convert(tz) + assert ( + storage_constraints["min"] == 0 + ).all(), ( + "with a breach price, only the global soc-min should remain a hard constraint" + ) + + soc_minima_commitments = [ + c for c in commitments if getattr(c, "name", "") == "any soc minima" + ] + assert len(soc_minima_commitments) == 1 + quantity = soc_minima_commitments[0].quantity.tz_convert(tz) + # The projected previous-tick minimum (1 MWh - 0.04 MW * 0.2 h = 0.992 MWh) + # constrains the stock at the end of the slot starting at 16:45. + assert quantity.loc[start] == pytest.approx( + 0.992 * 4 + ), "the soft commitment should use the projected previous-tick minimum" + # The projected next-tick minimum (1 MWh - 0.04 MW * 0.05 h = 0.998 MWh) + # constrains the stock at the end of the slot starting at 17:00. + assert quantity.loc[start + resolution] == pytest.approx( + 0.998 * 4 + ), "the soft commitment should use the projected next-tick minimum" + + def test_deserialize_storage_soc_at_start_from_state_of_charge_sensor( add_charging_station_assets, setup_markets, setup_sources, db ): From f0a5cc203493060fcda65ffb6eff6909356744ab Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 Jul 2026 14:52:59 +0200 Subject: [PATCH 66/72] feat: scope automatic SoC relaxation to devices with off-tick constraints When relaxation is enabled purely because off-tick SoC constraints require projection (i.e. the user's flex-context did not already soften SoC constraints via relax flags or explicit breach prices), the softening now applies only to the devices that actually use off-tick SoC constraints, tracked by their power sensor. Other devices in a multi-device flex-model keep their hard SoC constraints. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01YNPESudGKciPZq88nGJn3C Signed-off-by: F.N. Claessen --- documentation/concepts/data-model.rst | 1 + flexmeasures/data/models/planning/storage.py | 74 +++++++++++++++++++- 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/documentation/concepts/data-model.rst b/documentation/concepts/data-model.rst index 1d0c85ae5b..8fc5a53b95 100644 --- a/documentation/concepts/data-model.rst +++ b/documentation/concepts/data-model.rst @@ -118,6 +118,7 @@ The projection uses the ``consumption-capacity`` and ``production-capacity`` act If multiple projected lower bounds land on the same tick, the highest lower bound is kept. If multiple projected upper bounds land on the same tick, the lowest upper bound is kept. Because projection can introduce additional bounds and more complex combinations can become infeasible, FlexMeasures enables ``relax-soc-constraints`` automatically when off-tick SoC constraints are submitted (unless it was explicitly set to ``False``, which is respected with a logged warning). +When relaxation is enabled purely because of off-tick projection (rather than by the flex-context settings), the softening is scoped to the devices that actually use off-tick SoC constraints; other devices keep their hard SoC constraints. Projection can be disabled per sensor by setting the ``floor_datetimes_to_resolution`` sensor attribute to ``False``. In that case, off-tick point-like SoC constraints cannot be enforced on the scheduling ticks and are disregarded (with a logged warning). diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 4cfcdc7d55..8142114004 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -930,6 +930,7 @@ def device_list_series( if ( self.flex_context.get("soc_minima_breach_price") is not None and soc_minima[d] is not None + and self._soc_relaxation_applies_to(sensor_d) ): soc_minima_breach_price = self.flex_context["soc_minima_breach_price"] any_soc_minima_breach_price = get_continuous_series_sensor_or_quantity( @@ -994,6 +995,7 @@ def device_list_series( if ( self.flex_context.get("soc_maxima_breach_price") is not None and soc_maxima[d] is not None + and self._soc_relaxation_applies_to(sensor_d) ): soc_maxima_breach_price = self.flex_context["soc_maxima_breach_price"] any_soc_maxima_breach_price = get_continuous_series_sensor_or_quantity( @@ -1326,6 +1328,12 @@ def deserialize_flex_config(self): self.collect_flex_config() if self.flex_context is None: self.flex_context = {} + #: Power sensor ids of devices whose off-tick SoC constraints triggered + #: automatic relaxation (None marks an entry without a power sensor). + self.off_tick_soc_sensor_ids: set[int | None] = set() + #: Whether SoC constraint softening should apply only to those devices + #: (True when relaxation was enabled purely because of off-tick projection). + self.scope_soc_relaxation_to_off_tick_devices: bool = False # The flex-model is deserialized first, because off-tick SoC constraints # may enable relax-soc-constraints on the still-serialized flex-context. self._deserialize_flex_model() @@ -1378,7 +1386,7 @@ def _deserialize_flex_model(self): self.ensure_soc_at_start() self._possibly_relax_off_tick_soc_constraints( - self.flex_model, sensor=self.sensor + self.flex_model, sensor=self.sensor, power_sensor=self.sensor ) # Now it's time to check if our flex configuration holds up to schemas @@ -1410,7 +1418,9 @@ def _deserialize_flex_model(self): else soc_sensor ) self._possibly_relax_off_tick_soc_constraints( - sensor_flex_model["sensor_flex_model"], sensor=sensor_d + sensor_flex_model["sensor_flex_model"], + sensor=sensor_d, + power_sensor=sensor_flex_model.get("sensor"), ) schema = StorageFlexModelSchema( start=self.start, @@ -1436,13 +1446,21 @@ def _deserialize_flex_model(self): return self.flex_model def _possibly_relax_off_tick_soc_constraints( - self, flex_model: dict, sensor: Sensor | None + self, + flex_model: dict, + sensor: Sensor | None, + power_sensor: Sensor | None = None, ) -> None: """Enable SoC constraint relaxation if the (serialized) flex-model contains off-tick SoC events. The detection uses the scheduler's actual resolution (falling back to the sensor's event resolution), matching the resolution later used to project off-tick SoC constraints onto the scheduling ticks. + + When relaxation is enabled purely because of off-tick projection (rather + than by the user's own flex-context settings), softening is scoped to the + devices that actually use off-tick SoC constraints (tracked here by their + power sensor). """ if not should_project_off_tick_soc_constraints(sensor): return @@ -1450,8 +1468,58 @@ def _possibly_relax_off_tick_soc_constraints( self.resolution, sensor, self.default_resolution ) if flex_model_has_off_tick_soc_constraints(flex_model, resolution=resolution): + self.off_tick_soc_sensor_ids.add( + power_sensor.id if power_sensor is not None else None + ) + self.scope_soc_relaxation_to_off_tick_devices = ( + not self._soc_relaxation_user_enabled() + ) self.enable_relax_soc_constraints() + def _soc_relaxation_user_enabled(self) -> bool: + """Whether the user's own (serialized) flex-context already softens SoC constraints. + + That is the case when any context defines a SoC breach price explicitly, + sets ``relax-soc-constraints`` to ``True``, or leaves relaxation to the + general ``relax-constraints`` flag (which defaults to ``True``). + """ + if isinstance(self.flex_context, dict): + contexts = [self.flex_context] + list( + self.flex_context.get("commodities", []) + ) + elif isinstance(self.flex_context, list): + contexts = self.flex_context + else: + return True + for context in contexts: + if ( + context.get("soc-minima-breach-price") is not None + or context.get("soc-maxima-breach-price") is not None + ): + return True + if context.get("relax-soc-constraints") is True: + return True + if context.get("relax-soc-constraints") is None and context.get( + "relax-constraints", True + ): + return True + return False + + def _soc_relaxation_applies_to(self, sensor_d: Sensor | None) -> bool: + """Whether SoC constraint softening applies to the device with this power sensor. + + Softening applies to all devices, unless relaxation was auto-enabled purely + for off-tick SoC constraint projection, in which case it is scoped to the + devices that use off-tick SoC constraints. + """ + if not getattr(self, "scope_soc_relaxation_to_off_tick_devices", False): + return True + off_tick_sensor_ids = getattr(self, "off_tick_soc_sensor_ids", set()) + if None in off_tick_sensor_ids: + # An entry without a power sensor cannot be matched to a device. + return True + return sensor_d is not None and sensor_d.id in off_tick_sensor_ids + def enable_relax_soc_constraints(self) -> None: """Relax SoC constraints when off-tick SoC events require scheduling-tick projection. From f5967625fa422c32aad60dbe3acd9b6d53db8cd0 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 Jul 2026 14:53:02 +0200 Subject: [PATCH 67/72] test: cover off-tick SoC constraints in a multi-device flex-model Two devices via the serialized multi-sensor flex-model path: the off-tick device's soc-minima are projected and softened into stock commitments, while the on-tick device's soc-minima remain hard constraints. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01YNPESudGKciPZq88nGJn3C Signed-off-by: F.N. Claessen --- .../models/planning/tests/test_storage.py | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py index d16b523bd2..1a71c2d640 100644 --- a/flexmeasures/data/models/planning/tests/test_storage.py +++ b/flexmeasures/data/models/planning/tests/test_storage.py @@ -11,6 +11,7 @@ from flexmeasures.data.models.planning.soc_projection import ( project_off_tick_soc_constraints, ) +from flexmeasures.data.models.generic_assets import GenericAsset from flexmeasures.data.models.planning.storage import StorageScheduler from flexmeasures.data.models.planning.utils import initialize_index from flexmeasures.data.models.time_series import Sensor, TimedBelief @@ -1452,6 +1453,121 @@ def test_off_tick_soc_minima_are_projected_into_soft_commitments( ), "the soft commitment should use the projected next-tick minimum" +def test_off_tick_soc_relaxation_is_scoped_to_the_off_tick_device( + add_battery_assets, db +): + """In a multi-device flex-model, auto-relaxation softens only the off-tick device. + + Device 0 uses an off-tick soc-minima (triggering automatic relaxation and + projection), while device 1 uses an on-tick soc-minima. With relaxation + otherwise disabled, device 0's minima should become soft commitments and + device 1's minima should remain hard constraints. + """ + template = add_battery_assets["Test battery"] + asset = GenericAsset( + name="Test multi-device battery site", + generic_asset_type=template.generic_asset_type, + owner=template.owner, + ) + sensor_0 = Sensor( + name="multi-device power 0", + generic_asset=asset, + event_resolution=timedelta(minutes=15), + unit="MW", + ) + sensor_1 = Sensor( + name="multi-device power 1", + generic_asset=asset, + event_resolution=timedelta(minutes=15), + unit="MW", + ) + db.session.add_all([asset, sensor_0, sensor_1]) + db.session.flush() + tz = pytz.timezone("Europe/Amsterdam") + start = tz.localize(datetime(2015, 1, 1, 16, 45)) + end = tz.localize(datetime(2015, 1, 1, 17, 15)) + resolution = timedelta(minutes=15) + + common_flex_model = { + "soc-at-start": "0 MWh", + "soc-min": "0 MWh", + "soc-max": "1 MWh", + "power-capacity": "0.04 MW", + "consumption-capacity": "0.04 MW", + "production-capacity": "0.04 MW", + "roundtrip-efficiency": 1, + "storage-efficiency": 1, + } + scheduler = StorageScheduler( + asset, + start, + end, + resolution, + flex_model=[ + { + "sensor": sensor_0.id, + **common_flex_model, + "soc-minima": [ + { + "datetime": "2015-01-01T17:12:00+01:00", + "value": "1 MWh", + } + ], + }, + { + "sensor": sensor_1.id, + **common_flex_model, + "soc-minima": [ + { + "datetime": "2015-01-01T17:00:00+01:00", + "value": "1 MWh", + } + ], + }, + ], + flex_context={ + "consumption-price": "0 EUR/MWh", + "production-price": "0 EUR/MWh", + "site-power-capacity": "1 MW", + # relaxation is otherwise off, so any softening is due to off-tick projection + "relax-constraints": False, + }, + ) + + _, _, _, _, _, device_constraints, _, commitments = scheduler._prepare( + skip_validation=True + ) + + assert ( + scheduler.flex_context["relax_soc_constraints"] is True + ), "off-tick SoC constraints should automatically enable SoC relaxation" + + soc_minima_commitments = [ + c for c in commitments if getattr(c, "name", "") == "any soc minima" + ] + assert ( + len(soc_minima_commitments) == 1 + and (soc_minima_commitments[0].device == 0).all() + ), "only the off-tick device should have its soc-minima softened into commitments" + quantity = soc_minima_commitments[0].quantity.tz_convert(tz) + assert quantity.loc[start] == pytest.approx( + 0.992 * 4 + ), "the soft commitment should use the projected previous-tick minimum" + assert quantity.loc[start + resolution] == pytest.approx( + 0.998 * 4 + ), "the soft commitment should use the projected next-tick minimum" + + constraints_0 = device_constraints[0].tz_convert(tz) + assert ( + constraints_0["min"] == 0 + ).all(), "the off-tick device should keep only the global soc-min as a hard bound" + + constraints_1 = device_constraints[1].tz_convert(tz) + assert constraints_1.loc[start, "min"] == pytest.approx( + 4 + ), "the on-tick device's soc-minima should remain a hard constraint" + + def test_deserialize_storage_soc_at_start_from_state_of_charge_sensor( add_charging_station_assets, setup_markets, setup_sources, db ): From db98b260db05d6528a5aece62430bb72d0143abc Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 Jul 2026 15:37:24 +0200 Subject: [PATCH 68/72] feat: project an off-tick starting state of charge onto the next tick Following the original recipe in issue #10: when soc-at-start is resolved from the state-of-charge field (an instantaneous sensor or time series) and the underlying measurement was taken at an off-tick time within the first scheduling interval, the SoC is assumed to hold from the schedule start until that time, and the next scheduling tick receives an upper bound (value plus chargeable energy since then) and a lower bound (value minus dischargeable energy), efficiency-aware and clamped to the global soc-min/soc-max. The measurement time is recorded (per power sensor) when resolving the state-of-charge field, in both single-sensor and multi-device (shared stock) modes. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01YNPESudGKciPZq88nGJn3C Signed-off-by: F.N. Claessen --- documentation/concepts/data-model.rst | 2 + .../data/models/planning/soc_projection.py | 97 +++++++++++++++++++ flexmeasures/data/models/planning/storage.py | 38 ++++++++ 3 files changed, 137 insertions(+) diff --git a/documentation/concepts/data-model.rst b/documentation/concepts/data-model.rst index 8fc5a53b95..ac6a3fa20c 100644 --- a/documentation/concepts/data-model.rst +++ b/documentation/concepts/data-model.rst @@ -120,6 +120,8 @@ If multiple projected upper bounds land on the same tick, the lowest upper bound Because projection can introduce additional bounds and more complex combinations can become infeasible, FlexMeasures enables ``relax-soc-constraints`` automatically when off-tick SoC constraints are submitted (unless it was explicitly set to ``False``, which is respected with a logged warning). When relaxation is enabled purely because of off-tick projection (rather than by the flex-context settings), the softening is scoped to the devices that actually use off-tick SoC constraints; other devices keep their hard SoC constraints. +The starting state of charge is projected as well: when ``soc-at-start`` is resolved from the ``state-of-charge`` field and the underlying measurement was taken at an off-tick time within the first scheduling interval, the SoC is assumed to hold until that time, and the next scheduling tick receives bounds reflecting how much the device can still (dis)charge after it. + Projection can be disabled per sensor by setting the ``floor_datetimes_to_resolution`` sensor attribute to ``False``. In that case, off-tick point-like SoC constraints cannot be enforced on the scheduling ticks and are disregarded (with a logged warning). diff --git a/flexmeasures/data/models/planning/soc_projection.py b/flexmeasures/data/models/planning/soc_projection.py index fa1e3bb03d..ab96c9c680 100644 --- a/flexmeasures/data/models/planning/soc_projection.py +++ b/flexmeasures/data/models/planning/soc_projection.py @@ -387,3 +387,100 @@ def add_max_bound(soc_event: dict, tick: pd.Timestamp, value: float) -> None: _projected_soc_events_or_original(soc_maxima, projected_maxima, "soc-maxima"), _projected_soc_events_or_original(soc_minima, projected_minima, "soc-minima"), ) + + +def project_off_tick_soc_at_start( + soc_at_start_time: datetime, + soc_at_start: ur.Quantity | float, + soc_maxima: ( + list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None + ), + soc_minima: ( + list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None + ), + schedule_start: datetime, + consumption_capacity: pd.Series, + production_capacity: pd.Series, + resolution: timedelta, + soc_min: ur.Quantity | float | None, + soc_max: ur.Quantity | float | None, + charging_efficiency: pd.Series | ur.Quantity | float | None = None, + discharging_efficiency: pd.Series | ur.Quantity | float | None = None, +) -> tuple[ + list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, + list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, +]: + """Project an off-tick starting state of charge onto the next scheduling tick. + + When the starting SoC is known at a time ``t`` between the schedule start and + the next scheduling tick ``n`` (e.g. because the ``state-of-charge`` field + resolved to a measurement taken at ``t``), the SoC is assumed to hold from the + schedule start until ``t`` (the device is not moving its stock before then), and + the SoC at ``n`` is bounded by how much the device can (dis)charge between ``t`` + and ``n``: + + - an upper bound of ``soc_at_start`` plus the energy chargeable between ``t`` and ``n``, + - a lower bound of ``soc_at_start`` minus the energy dischargeable between ``t`` and ``n``. + + Both bounds are clamped to the global ``soc-min``/``soc-max`` and merged into + the given ``soc-maxima``/``soc-minima`` (the stricter bound wins on collisions). + Known SoC times on a scheduling tick, or outside the first scheduling interval, + leave the bounds unchanged. + + Returns ``(soc_maxima, soc_minima)``. + """ + event_time = pd.Timestamp(soc_at_start_time) + start = pd.Timestamp(schedule_start) + if is_on_schedule_tick(event_time, resolution) or not ( + start < event_time < start + resolution + ): + return soc_maxima, soc_minima + + next_tick = event_time.ceil(resolution) + value = _soc_value_in_mwh(soc_at_start) + chargeable = _reachable_energy( + consumption_capacity, + event_time, + next_tick, + resolution, + efficiency=charging_efficiency, + efficiency_affects_stock="multiply", + ) + dischargeable = _reachable_energy( + production_capacity, + event_time, + next_tick, + resolution, + efficiency=discharging_efficiency, + efficiency_affects_stock="divide", + ) + + projected_minima = copy.deepcopy(soc_minima) if isinstance(soc_minima, list) else [] + projected_maxima = copy.deepcopy(soc_maxima) if isinstance(soc_maxima, list) else [] + tick_datetime = next_tick.to_pydatetime() + _add_soc_bound( + projected_minima, + { + "start": tick_datetime, + "end": tick_datetime, + "value": _clamp_soc_min( + value - dischargeable, _optional_soc_value_in_mwh(soc_min) + ), + }, + bound_type="min", + ) + _add_soc_bound( + projected_maxima, + { + "start": tick_datetime, + "end": tick_datetime, + "value": _clamp_soc_max( + value + chargeable, _optional_soc_value_in_mwh(soc_max) + ), + }, + bound_type="max", + ) + return ( + _projected_soc_events_or_original(soc_maxima, projected_maxima, "soc-maxima"), + _projected_soc_events_or_original(soc_minima, projected_minima, "soc-minima"), + ) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 8142114004..9a91824993 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -38,6 +38,7 @@ MultiSensorFlexModelSchema, ) from flexmeasures.data.models.planning.soc_projection import ( + project_off_tick_soc_at_start, project_off_tick_soc_constraints, ) from flexmeasures.data.schemas.scheduling.utils import ( @@ -910,6 +911,26 @@ def device_list_series( # before they are turned into soft commitments or hard constraints, # so that both paths consume on-tick events. if should_project_off_tick_soc_constraints(sensor_d): + # A starting SoC known at an off-tick time within the first + # scheduling interval bounds the SoC on the next tick. + soc_at_start_time = getattr(self, "soc_at_start_datetimes", {}).get( + getattr(sensor_d, "id", None) + ) + if soc_at_start_time is not None and soc_at_start[d] is not None: + soc_maxima[d], soc_minima[d] = project_off_tick_soc_at_start( + soc_at_start_time, + soc_at_start[d], + soc_maxima[d], + soc_minima[d], + start, + consumption_capacity_d, + production_capacity_d, + resolution, + soc_min[d], + soc_max[d], + charging_efficiency=charging_efficiency[d], + discharging_efficiency=discharging_efficiency[d], + ) ( soc_targets[d], soc_maxima[d], @@ -1566,6 +1587,21 @@ def has_soc_at_start(self) -> bool: def has_soc_at_start_in(flex_model: dict) -> bool: return "soc-at-start" in flex_model and flex_model["soc-at-start"] is not None + def _record_soc_at_start_datetime( + self, sensor: Sensor | None, soc_datetime: datetime | None + ) -> None: + """Remember at which time the starting state of charge is actually known. + + Keyed by the device's power sensor id. Used to project an off-tick + starting SoC onto the next scheduling tick (see + :func:`flexmeasures.data.models.planning.soc_projection.project_off_tick_soc_at_start`). + """ + if sensor is None or soc_datetime is None: + return + if not hasattr(self, "soc_at_start_datetimes"): + self.soc_at_start_datetimes: dict[int, datetime] = {} + self.soc_at_start_datetimes[sensor.id] = soc_datetime + def _get_soc_lookup_radius( self, sensor: Sensor | None = None, slack_steps: int = 4 ) -> timedelta: @@ -1691,6 +1727,7 @@ def _resolve_soc_at_start_from_sensor( beliefs_df["time_distance"] == beliefs_df["time_distance"].min() ] nearest_belief = nearest_beliefs.loc[nearest_beliefs["event_start"].idxmax()] + self._record_soc_at_start_datetime(sensor, nearest_belief["event_start"]) return self._convert_soc_value_to_mwh( value=nearest_belief["event_value"], @@ -1743,6 +1780,7 @@ def _resolve_soc_at_start_from_time_series( ) _, nearest_segment = min(candidate_segments, key=lambda item: item[0]) + self._record_soc_at_start_datetime(sensor, nearest_segment["start"]) return (nearest_segment["value"] / ur.Quantity("MWh")).magnitude def _resolve_soc_at_start_from_state_of_charge( From 03fd551b034fdbceba79b81b0fb2083d36ecefb3 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 Jul 2026 15:37:26 +0200 Subject: [PATCH 69/72] test: cover off-tick soc-at-start projection A pure test for the next-tick bounds (efficiency-aware, no-op on ticks), and an end-to-end test resolving the starting SoC from an off-tick state-of-charge time series entry. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01YNPESudGKciPZq88nGJn3C Signed-off-by: F.N. Claessen --- .../models/planning/tests/test_storage.py | 106 ++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py index 1a71c2d640..ddce607501 100644 --- a/flexmeasures/data/models/planning/tests/test_storage.py +++ b/flexmeasures/data/models/planning/tests/test_storage.py @@ -9,6 +9,7 @@ from flexmeasures.data.models.planning import Scheduler from flexmeasures.data.models.planning.soc_projection import ( + project_off_tick_soc_at_start, project_off_tick_soc_constraints, ) from flexmeasures.data.models.generic_assets import GenericAsset @@ -1568,6 +1569,111 @@ def test_off_tick_soc_relaxation_is_scoped_to_the_off_tick_device( ), "the on-tick device's soc-minima should remain a hard constraint" +def test_project_off_tick_soc_at_start_bounds_the_next_tick(): + """An off-tick starting SoC bounds the next tick by reachable (dis)charge energy.""" + tz = pytz.timezone("Europe/Amsterdam") + resolution = timedelta(minutes=15) + start = pd.Timestamp(tz.localize(datetime(2015, 1, 1, 16, 45))) + next_tick = start + resolution + capacity = pd.Series(0.04, index=pd.date_range(start, next_tick, freq=resolution)) + + soc_maxima, soc_minima = project_off_tick_soc_at_start( + soc_at_start_time=tz.localize(datetime(2015, 1, 1, 16, 47)), + soc_at_start=0.5, + soc_maxima=None, + soc_minima=None, + schedule_start=start, + consumption_capacity=capacity, + production_capacity=capacity, + resolution=resolution, + soc_min=0, + soc_max=1, + charging_efficiency=0.9, + discharging_efficiency=0.8, + ) + + # 13 minutes remain between the known SoC (16:47) and the next tick (17:00). + assert _soc_event_value_at(soc_maxima, next_tick) == pytest.approx( + 0.5 + 0.04 * 0.9 * (13 / 60) + ), "the next tick's upper bound should reflect the chargeable energy since 16:47" + assert _soc_event_value_at(soc_minima, next_tick) == pytest.approx( + 0.5 - 0.04 / 0.8 * (13 / 60) + ), "the next tick's lower bound should reflect the dischargeable energy since 16:47" + + # A known SoC time on a scheduling tick (or outside the first interval) is a no-op. + assert project_off_tick_soc_at_start( + soc_at_start_time=start.to_pydatetime(), + soc_at_start=0.5, + soc_maxima=None, + soc_minima=None, + schedule_start=start, + consumption_capacity=capacity, + production_capacity=capacity, + resolution=resolution, + soc_min=0, + soc_max=1, + ) == (None, None) + + +def test_off_tick_state_of_charge_bounds_first_scheduling_interval( + add_battery_assets, db +): + """A state-of-charge measurement at an off-tick time caps the SoC at the next tick.""" + _, battery = get_sensors_from_db( + db, add_battery_assets, battery_name="Test battery" + ) + tz = pytz.timezone("Europe/Amsterdam") + start = tz.localize(datetime(2015, 1, 1, 16, 45)) + end = tz.localize(datetime(2015, 1, 1, 17, 15)) + resolution = timedelta(minutes=15) + + scheduler = StorageScheduler( + battery, + start, + end, + resolution, + flex_model={ + "soc-min": "0 MWh", + "soc-max": "1 MWh", + "power-capacity": "0.04 MW", + "consumption-capacity": "0.04 MW", + "production-capacity": "0.04 MW", + "roundtrip-efficiency": 1, + "storage-efficiency": 1, + # the starting SoC is known at 16:47, not at the schedule start + "state-of-charge": [ + { + "start": "2015-01-01T16:47:00+01:00", + "end": "2015-01-01T16:47:00+01:00", + "value": "0 MWh", + } + ], + }, + flex_context={ + "consumption-price": "0 EUR/MWh", + "production-price": "0 EUR/MWh", + "site-power-capacity": "1 MW", + # keep SoC constraints hard, so we can assert the projected bounds directly + "relax-constraints": False, + "relax-soc-constraints": False, + }, + ) + + _, _, _, _, soc_at_start, device_constraints, _, _ = scheduler._prepare( + skip_validation=True + ) + + assert soc_at_start[0] == pytest.approx(0), "the starting SoC should be resolved" + storage_constraints = device_constraints[0].tz_convert(tz) + # Charging can only start at 16:47, so by 17:00 at most 13 minutes of charging fit. + assert storage_constraints.loc[start, "max"] == pytest.approx( + 0.04 * (13 / 60) * 4 + ), "the first interval should be capped by the chargeable energy since 16:47" + assert storage_constraints.loc[start, "min"] == pytest.approx( + 0 + ), "the lower bound should be clamped to soc-min" + + def test_deserialize_storage_soc_at_start_from_state_of_charge_sensor( add_charging_station_assets, setup_markets, setup_sources, db ): From 1a0b144eb489a84dc45718e63a3a53ed3fd0342c Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 Jul 2026 16:21:03 +0200 Subject: [PATCH 70/72] Track off-tick SoC relaxation scoping and starting-SoC timing per stock Both trackers previously keyed on the device's power sensor id and relied on the positional convention that the first device of a stock group applies the stock's constraints: - off_tick_soc_sensor_ids becomes off_tick_stock_keys: entries are tracked by the stock key resolved from their serialized state-of-charge field, so scoping covers all devices sharing the stock (entries without a resolvable stock key fall back to a namespaced power-sensor key). - soc_at_start_datetimes is keyed by stock key, threaded through the soc-at-start resolution chain (exact stock key in multi-device mode; a None key covers the single-sensor time-series case). Adds a regression test for the shared-stock case: off-tick soc-minima on a stock-only entry soften the whole group's constraints while an unrelated device keeps its hard on-tick minima. Co-Authored-By: Claude Fable 5 --- flexmeasures/data/models/planning/storage.py | 101 ++++++++++------ .../models/planning/tests/test_storage.py | 113 ++++++++++++++++++ 2 files changed, 178 insertions(+), 36 deletions(-) diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 9a91824993..4c9bc4ce83 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -18,7 +18,10 @@ SchedulerOutputType, StockCommitment, ) -from flexmeasures.data.models.planning.devices import DeviceInventory +from flexmeasures.data.models.planning.devices import ( + DeviceInventory, + _resolve_stock_key, +) from flexmeasures.data.models.planning.linear_optimization import device_scheduler from flexmeasures.data.models.planning.utils import ( add_tiny_price_slope, @@ -157,6 +160,13 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 self.stock_models = inventory.stock_entries # The stock groups' device indices align with the device models self.stock_groups = inventory.stock_groups + # Off-tick SoC relaxation scoping and starting-SoC projection track stocks + # (not devices), so look up each device's stock key. + device_stock_key = { + d: stock_key + for stock_key, group_devices in self.stock_groups.items() + for d in group_devices + } # List the asset(s) and sensor(s) being scheduled sensors: list[Sensor | None] = inventory.power_sensors @@ -209,7 +219,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 # so the device could discharge more energy than its store holds. if soc_at_start[d0] is None: resolved_soc_at_start = self._resolve_stock_soc_at_start( - stock_model, sensor=sensors[d0] + stock_model, sensor=sensors[d0], stock_key=stock_id ) if resolved_soc_at_start is not None: soc_at_start[d0] = resolved_soc_at_start @@ -912,10 +922,13 @@ def device_list_series( # so that both paths consume on-tick events. if should_project_off_tick_soc_constraints(sensor_d): # A starting SoC known at an off-tick time within the first - # scheduling interval bounds the SoC on the next tick. - soc_at_start_time = getattr(self, "soc_at_start_datetimes", {}).get( - getattr(sensor_d, "id", None) - ) + # scheduling interval bounds the SoC on the next tick. The timing + # is tracked per stock; a None key covers the single-sensor case + # where the stock key cannot be resolved at record time. + soc_at_start_datetimes = getattr(self, "soc_at_start_datetimes", {}) + soc_at_start_time = soc_at_start_datetimes.get(device_stock_key.get(d)) + if soc_at_start_time is None: + soc_at_start_time = soc_at_start_datetimes.get(None) if soc_at_start_time is not None and soc_at_start[d] is not None: soc_maxima[d], soc_minima[d] = project_off_tick_soc_at_start( soc_at_start_time, @@ -951,7 +964,7 @@ def device_list_series( if ( self.flex_context.get("soc_minima_breach_price") is not None and soc_minima[d] is not None - and self._soc_relaxation_applies_to(sensor_d) + and self._soc_relaxation_applies_to(device_stock_key.get(d), sensor_d) ): soc_minima_breach_price = self.flex_context["soc_minima_breach_price"] any_soc_minima_breach_price = get_continuous_series_sensor_or_quantity( @@ -1016,7 +1029,7 @@ def device_list_series( if ( self.flex_context.get("soc_maxima_breach_price") is not None and soc_maxima[d] is not None - and self._soc_relaxation_applies_to(sensor_d) + and self._soc_relaxation_applies_to(device_stock_key.get(d), sensor_d) ): soc_maxima_breach_price = self.flex_context["soc_maxima_breach_price"] any_soc_maxima_breach_price = get_continuous_series_sensor_or_quantity( @@ -1349,9 +1362,10 @@ def deserialize_flex_config(self): self.collect_flex_config() if self.flex_context is None: self.flex_context = {} - #: Power sensor ids of devices whose off-tick SoC constraints triggered - #: automatic relaxation (None marks an entry without a power sensor). - self.off_tick_soc_sensor_ids: set[int | None] = set() + #: Stock keys of stocks whose off-tick SoC constraints triggered automatic + #: relaxation (None marks an entry whose stock key cannot be resolved from + #: its serialized form, e.g. a state-of-charge time series). + self.off_tick_stock_keys: set = set() #: Whether SoC constraint softening should apply only to those devices #: (True when relaxation was enabled purely because of off-tick projection). self.scope_soc_relaxation_to_off_tick_devices: bool = False @@ -1480,8 +1494,10 @@ def _possibly_relax_off_tick_soc_constraints( When relaxation is enabled purely because of off-tick projection (rather than by the user's own flex-context settings), softening is scoped to the - devices that actually use off-tick SoC constraints (tracked here by their - power sensor). + stocks that actually use off-tick SoC constraints (tracked here by their + stock key, so all devices sharing the stock are covered). An entry without + a resolvable stock key is tracked by its power sensor instead (its stock + gets a synthetic key only later, when the device inventory is built). """ if not should_project_off_tick_soc_constraints(sensor): return @@ -1489,9 +1505,10 @@ def _possibly_relax_off_tick_soc_constraints( self.resolution, sensor, self.default_resolution ) if flex_model_has_off_tick_soc_constraints(flex_model, resolution=resolution): - self.off_tick_soc_sensor_ids.add( - power_sensor.id if power_sensor is not None else None - ) + stock_key = _resolve_stock_key(flex_model.get("state-of-charge")) + if stock_key is None and power_sensor is not None: + stock_key = ("sensor", power_sensor.id) + self.off_tick_stock_keys.add(stock_key) self.scope_soc_relaxation_to_off_tick_devices = ( not self._soc_relaxation_user_enabled() ) @@ -1526,20 +1543,27 @@ def _soc_relaxation_user_enabled(self) -> bool: return True return False - def _soc_relaxation_applies_to(self, sensor_d: Sensor | None) -> bool: - """Whether SoC constraint softening applies to the device with this power sensor. + def _soc_relaxation_applies_to( + self, stock_key, sensor_d: Sensor | None = None + ) -> bool: + """Whether SoC constraint softening applies to the stock with this key. - Softening applies to all devices, unless relaxation was auto-enabled purely + Softening applies to all stocks, unless relaxation was auto-enabled purely for off-tick SoC constraint projection, in which case it is scoped to the - devices that use off-tick SoC constraints. + stocks that use off-tick SoC constraints (covering all devices sharing them). + A stock tracked by power sensor (for lack of a resolvable stock key) is + matched via the device's power sensor. """ if not getattr(self, "scope_soc_relaxation_to_off_tick_devices", False): return True - off_tick_sensor_ids = getattr(self, "off_tick_soc_sensor_ids", set()) - if None in off_tick_sensor_ids: - # An entry without a power sensor cannot be matched to a device. + off_tick_stock_keys = getattr(self, "off_tick_stock_keys", set()) + if None in off_tick_stock_keys: + # An entry with neither a resolvable stock key nor a power sensor + # cannot be matched to a stock. + return True + if stock_key is not None and stock_key in off_tick_stock_keys: return True - return sensor_d is not None and sensor_d.id in off_tick_sensor_ids + return sensor_d is not None and ("sensor", sensor_d.id) in off_tick_stock_keys def enable_relax_soc_constraints(self) -> None: """Relax SoC constraints when off-tick SoC events require scheduling-tick projection. @@ -1588,19 +1612,20 @@ def has_soc_at_start_in(flex_model: dict) -> bool: return "soc-at-start" in flex_model and flex_model["soc-at-start"] is not None def _record_soc_at_start_datetime( - self, sensor: Sensor | None, soc_datetime: datetime | None + self, stock_key, soc_datetime: datetime | None ) -> None: - """Remember at which time the starting state of charge is actually known. + """Remember at which time a stock's starting state of charge is actually known. - Keyed by the device's power sensor id. Used to project an off-tick - starting SoC onto the next scheduling tick (see + Keyed by the stock key (a None key covers the single-sensor case where the + stock key cannot be resolved, e.g. a state-of-charge time series). Used to + project an off-tick starting SoC onto the next scheduling tick (see :func:`flexmeasures.data.models.planning.soc_projection.project_off_tick_soc_at_start`). """ - if sensor is None or soc_datetime is None: + if soc_datetime is None: return if not hasattr(self, "soc_at_start_datetimes"): - self.soc_at_start_datetimes: dict[int, datetime] = {} - self.soc_at_start_datetimes[sensor.id] = soc_datetime + self.soc_at_start_datetimes: dict = {} + self.soc_at_start_datetimes[stock_key] = soc_datetime def _get_soc_lookup_radius( self, sensor: Sensor | None = None, slack_steps: int = 4 @@ -1727,7 +1752,7 @@ def _resolve_soc_at_start_from_sensor( beliefs_df["time_distance"] == beliefs_df["time_distance"].min() ] nearest_belief = nearest_beliefs.loc[nearest_beliefs["event_start"].idxmax()] - self._record_soc_at_start_datetime(sensor, nearest_belief["event_start"]) + self._record_soc_at_start_datetime(soc_sensor.id, nearest_belief["event_start"]) return self._convert_soc_value_to_mwh( value=nearest_belief["event_value"], @@ -1737,12 +1762,14 @@ def _resolve_soc_at_start_from_sensor( ) def _resolve_soc_at_start_from_time_series( - self, soc_time_series: list[dict], sensor: Sensor | None = None + self, soc_time_series: list[dict], sensor: Sensor | None = None, stock_key=None ) -> float: """Resolve ``soc-at-start`` from a ``state-of-charge`` time series. :param soc_time_series: SoC time series specification. :param sensor: Optional scheduled power sensor. + :param stock_key: Key of the stock whose SoC is being resolved, if known + (a time series does not resolve to a stock key by itself). :returns: Starting SoC in MWh. """ lookup_radius = self._get_soc_lookup_radius(sensor) @@ -1780,7 +1807,7 @@ def _resolve_soc_at_start_from_time_series( ) _, nearest_segment = min(candidate_segments, key=lambda item: item[0]) - self._record_soc_at_start_datetime(sensor, nearest_segment["start"]) + self._record_soc_at_start_datetime(stock_key, nearest_segment["start"]) return (nearest_segment["value"] / ur.Quantity("MWh")).magnitude def _resolve_soc_at_start_from_state_of_charge( @@ -1832,7 +1859,7 @@ def _resolve_soc_at_start_from_state_of_charge( return None def _resolve_stock_soc_at_start( - self, stock_model: dict, sensor: Sensor | None = None + self, stock_model: dict, sensor: Sensor | None = None, stock_key=None ) -> float | None: """Resolve a stock's soc-at-start (in MWh) from its (deserialized) state-of-charge. @@ -1860,7 +1887,9 @@ def _resolve_stock_soc_at_start( state_of_charge, percent_conversion_model, sensor ) if isinstance(state_of_charge, list): - return self._resolve_soc_at_start_from_time_series(state_of_charge, sensor) + return self._resolve_soc_at_start_from_time_series( + state_of_charge, sensor, stock_key=stock_key + ) return None def possibly_extend_end(self, soc_targets, sensor: Sensor = None): diff --git a/flexmeasures/data/models/planning/tests/test_storage.py b/flexmeasures/data/models/planning/tests/test_storage.py index ddce607501..9435e2c3db 100644 --- a/flexmeasures/data/models/planning/tests/test_storage.py +++ b/flexmeasures/data/models/planning/tests/test_storage.py @@ -1963,3 +1963,116 @@ def test_resolve_soc_at_start_from_percent_sensor_uses_device_sensor_fallback( ) == 2.5 ) + + +def test_off_tick_soc_relaxation_covers_all_devices_of_a_shared_stock( + add_battery_assets, db +): + """Auto-relaxation scoped by stock covers the whole stock group. + + Devices 0 and 1 share a stock whose SoC parameters - including an off-tick + soc-minima - live on a stock-only entry, while device 2 uses an on-tick + soc-minima of its own. The shared stock's minima should be softened into + commitments (landing on the group's first device), while device 2's minima + should remain hard constraints. + """ + template = add_battery_assets["Test battery"] + asset = GenericAsset( + name="Test shared-stock battery site", + generic_asset_type=template.generic_asset_type, + owner=template.owner, + ) + power_sensors = [ + Sensor( + name=f"shared-stock power {i}", + generic_asset=asset, + event_resolution=timedelta(minutes=15), + unit="MW", + ) + for i in range(3) + ] + soc_sensor = Sensor( + name="shared-stock state of charge", + generic_asset=asset, + event_resolution=timedelta(0), + unit="MWh", + ) + db.session.add_all([asset, soc_sensor, *power_sensors]) + db.session.flush() + tz = pytz.timezone("Europe/Amsterdam") + start = tz.localize(datetime(2015, 1, 1, 16, 45)) + end = tz.localize(datetime(2015, 1, 1, 17, 15)) + resolution = timedelta(minutes=15) + + device_properties = { + "power-capacity": "0.04 MW", + "consumption-capacity": "0.04 MW", + "production-capacity": "0.04 MW", + "roundtrip-efficiency": 1, + } + soc_parameters = { + "soc-at-start": "0 MWh", + "soc-min": "0 MWh", + "soc-max": "1 MWh", + } + off_tick_minima = [{"datetime": "2015-01-01T17:12:00+01:00", "value": "1 MWh"}] + on_tick_minima = [{"datetime": "2015-01-01T17:00:00+01:00", "value": "1 MWh"}] + + scheduler = StorageScheduler( + asset, + start, + end, + resolution, + flex_model=[ + { + "sensor": power_sensors[0].id, + "state-of-charge": {"sensor": soc_sensor.id}, + **device_properties, + }, + { + "sensor": power_sensors[1].id, + "state-of-charge": {"sensor": soc_sensor.id}, + **device_properties, + }, + { + # Stock-only entry holding the shared stock's SoC parameters + "state-of-charge": {"sensor": soc_sensor.id}, + **soc_parameters, + "soc-minima": off_tick_minima, + }, + { + "sensor": power_sensors[2].id, + **device_properties, + **soc_parameters, + "soc-minima": on_tick_minima, + }, + ], + flex_context={ + "consumption-price": "0 EUR/MWh", + "production-price": "0 EUR/MWh", + "site-power-capacity": "1 MW", + # relaxation is otherwise off, so any softening is due to off-tick projection + "relax-constraints": False, + }, + ) + + _, _, _, _, _, device_constraints, _, commitments = scheduler._prepare( + skip_validation=True + ) + + assert ( + scheduler.flex_context["relax_soc_constraints"] is True + ), "off-tick SoC constraints should automatically enable SoC relaxation" + + soc_minima_commitments = [ + c for c in commitments if getattr(c, "name", "") == "any soc minima" + ] + assert ( + len(soc_minima_commitments) == 1 + and (soc_minima_commitments[0].device == 0).all() + ), "the shared stock's soc-minima should be softened once, on the group's first device" + + constraints_2 = device_constraints[2].tz_convert(tz) + assert constraints_2.loc[start, "min"] == pytest.approx( + 4 + ), "the on-tick device's soc-minima should remain a hard constraint" From dd8608c770610b07c6f0f181c36d17fbe85cca4e Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 Jul 2026 16:27:12 +0200 Subject: [PATCH 71/72] refactor: express the SoC projection policies as a declarative rule table Reinstate a policy table, now minimal: with soc-at-start added there are eight bound rules following one symmetric pattern, and each rule only needs to state which bound lands on which surrounding tick. The capacity period, charge-vs-discharge direction, efficiency treatment and loosening sign all derive from those two choices (SocProjectionRule.uses_charging / .sign). A small _SocProjection working state bundles the device's capacities, efficiencies and SoC limits, and accumulates the projected bounds, removing the four near-identical projection blocks and the long parameter threading. Public signatures and behavior are unchanged. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01YNPESudGKciPZq88nGJn3C Signed-off-by: F.N. Claessen --- .../data/models/planning/soc_projection.py | 431 +++++++++--------- 1 file changed, 217 insertions(+), 214 deletions(-) diff --git a/flexmeasures/data/models/planning/soc_projection.py b/flexmeasures/data/models/planning/soc_projection.py index ab96c9c680..266a7b0099 100644 --- a/flexmeasures/data/models/planning/soc_projection.py +++ b/flexmeasures/data/models/planning/soc_projection.py @@ -4,7 +4,9 @@ import copy import logging +from dataclasses import dataclass, field from datetime import datetime, timedelta +from typing import Literal import pandas as pd @@ -14,6 +16,68 @@ logger = logging.getLogger(__name__) +SocBoundType = Literal["min", "max"] +SocProjectionTick = Literal["previous", "next"] + +TimedEventList = list[dict[str, datetime | float]] +SocSpecification = TimedEventList | pd.Series | Sensor | ur.Quantity | None + + +@dataclass(frozen=True) +class SocProjectionRule: + """One projected bound for an off-tick point-like SoC event. + + A rule only needs to state *which* bound lands on *which* surrounding + scheduling tick; everything else follows from preserving reachability: + + - The capacity period runs from the tick to the event time (``previous``) + or from the event time to the tick (``next``). + - A bound is loosened by the energy the device can move through that period + towards satisfying the original event: lower bounds by charging up to it + (on the previous tick) or by discharging away from it (on the next tick), + and upper bounds vice versa. + - Lower bounds are loosened downwards, upper bounds upwards. + """ + + bound_type: SocBoundType + tick: SocProjectionTick + + @property + def uses_charging(self) -> bool: + """Whether the bound is loosened by chargeable (rather than dischargeable) energy.""" + return (self.bound_type == "min") == (self.tick == "previous") + + @property + def sign(self) -> int: + """Lower bounds are loosened downwards (-1), upper bounds upwards (+1).""" + return -1 if self.bound_type == "min" else +1 + + +#: How each type of off-tick point-like SoC event is projected onto the +#: surrounding scheduling ticks (see :class:`SocProjectionRule`). +#: Off-tick ``soc-targets`` additionally become an exact target on the next +#: tick, and ``soc-at-start`` denotes a starting SoC known at an off-tick time +#: within the first scheduling interval; both are handled by their respective +#: entry points below. +SOC_PROJECTION_POLICIES: dict[str, tuple[SocProjectionRule, ...]] = { + "soc-targets": ( + SocProjectionRule("min", "previous"), + SocProjectionRule("max", "previous"), + ), + "soc-minima": ( + SocProjectionRule("min", "previous"), + SocProjectionRule("min", "next"), + ), + "soc-maxima": ( + SocProjectionRule("max", "previous"), + SocProjectionRule("max", "next"), + ), + "soc-at-start": ( + SocProjectionRule("min", "next"), + SocProjectionRule("max", "next"), + ), +} + def _soc_value_in_mwh(value: ur.Quantity | float | int) -> float: """Return the SoC value as a plain float in MWh.""" @@ -29,16 +93,6 @@ def _optional_soc_value_in_mwh(value: ur.Quantity | float | int | None) -> float return _soc_value_in_mwh(value) -def _clamp_soc_min(value: float, soc_min: float | None) -> float: - """Raise a projected lower bound to the storage's global soc-min, if defined.""" - return value if soc_min is None else max(soc_min, value) - - -def _clamp_soc_max(value: float, soc_max: float | None) -> float: - """Lower a projected upper bound to the storage's global soc-max, if defined.""" - return value if soc_max is None else min(soc_max, value) - - def _soc_event_at( soc_event: dict[str, datetime | float], dt: pd.Timestamp, @@ -153,12 +207,10 @@ def _add_soc_bound( def _projected_soc_events_or_original( - original_soc_events: ( - list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None - ), - projected_soc_events: list[dict[str, datetime | float]], + original_soc_events: SocSpecification, + projected_soc_events: TimedEventList, field_name: str, -) -> list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None: +) -> SocSpecification: """Choose between the projected event list and the original specification. Only list-based (or missing) specifications can absorb projected bounds; @@ -178,16 +230,100 @@ def _projected_soc_events_or_original( return original_soc_events +@dataclass +class _SocProjection: + """Working state for projecting the off-tick SoC events of one device. + + Bundles the device's capacities, efficiencies and global SoC limits, and + accumulates the projected lower and upper bounds while rules are applied. + """ + + consumption_capacity: pd.Series + production_capacity: pd.Series + resolution: timedelta + soc_min: float | None + soc_max: float | None + charging_efficiency: pd.Series | ur.Quantity | float | None = None + discharging_efficiency: pd.Series | ur.Quantity | float | None = None + minima: TimedEventList = field(default_factory=list) + maxima: TimedEventList = field(default_factory=list) + + def reachable_energy( + self, charging: bool, start: pd.Timestamp, end: pd.Timestamp + ) -> float: + """The energy (in MWh) the stock can move up (charging) or down (discharging).""" + if charging: + return _reachable_energy( + self.consumption_capacity, + start, + end, + self.resolution, + efficiency=self.charging_efficiency, + efficiency_affects_stock="multiply", + ) + return _reachable_energy( + self.production_capacity, + start, + end, + self.resolution, + efficiency=self.discharging_efficiency, + efficiency_affects_stock="divide", + ) + + def apply_rule( + self, + rule: SocProjectionRule, + soc_event: dict[str, datetime | float], + event_time: pd.Timestamp, + ) -> None: + """Apply one projection rule to one off-tick SoC event. + + Computes the reachability-adjusted bound value, clamps it to the global + SoC limits, and merges it into the projected minima or maxima (keeping + the stricter bound if one already exists on the same tick). + """ + previous_tick = event_time.floor(self.resolution) + next_tick = event_time.ceil(self.resolution) + if rule.tick == "previous": + tick, period = previous_tick, (previous_tick, event_time) + else: + tick, period = next_tick, (event_time, next_tick) + value = _soc_value_in_mwh( + soc_event["value"] + ) + rule.sign * self.reachable_energy(rule.uses_charging, *period) + if rule.bound_type == "min": + if self.soc_min is not None: + value = max(self.soc_min, value) + _add_soc_bound(self.minima, _soc_event_at(soc_event, tick, value), "min") + else: + if self.soc_max is not None: + value = min(self.soc_max, value) + _add_soc_bound(self.maxima, _soc_event_at(soc_event, tick, value), "max") + + def apply_policy( + self, + field_name: str, + soc_event: dict[str, datetime | float], + event_time: pd.Timestamp, + ) -> None: + """Apply all projection rules of the given policy to one off-tick SoC event.""" + for rule in SOC_PROJECTION_POLICIES[field_name]: + self.apply_rule(rule, soc_event, event_time) + + +def _is_projectable( + soc_event: dict[str, datetime | float], resolution: timedelta +) -> bool: + """Whether the SoC event is point-like and falls between two scheduling ticks.""" + return soc_event["start"] == soc_event["end"] and not is_on_schedule_tick( + soc_event["end"], resolution + ) + + def project_off_tick_soc_constraints( - soc_targets: ( - list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None - ), - soc_maxima: ( - list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None - ), - soc_minima: ( - list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None - ), + soc_targets: SocSpecification, + soc_maxima: SocSpecification, + soc_minima: SocSpecification, consumption_capacity: pd.Series, production_capacity: pd.Series, resolution: timedelta, @@ -195,18 +331,15 @@ def project_off_tick_soc_constraints( soc_max: ur.Quantity | float | None, charging_efficiency: pd.Series | ur.Quantity | float | None = None, discharging_efficiency: pd.Series | ur.Quantity | float | None = None, -) -> tuple[ - list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, - list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, - list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, -]: +) -> tuple[SocSpecification, SocSpecification, SocSpecification]: """Project off-tick point-like SoC constraints onto scheduling ticks. The scheduler can only enforce constraints at its fixed scheduling resolution. Point-like ``soc-targets``, ``soc-minima`` and ``soc-maxima`` that fall between two scheduling ticks are therefore replaced by constraints on the previous and next tick that preserve reachability using the available charge and discharge - capacity between the original event time and those ticks. + capacity between the original event time and those ticks + (see :data:`SOC_PROJECTION_POLICIES`). For an off-tick event with value ``v`` at time ``t``, between previous tick ``p`` and next tick ``n``: @@ -242,162 +375,62 @@ def project_off_tick_soc_constraints( ): return soc_targets, soc_maxima, soc_minima - projected_minima = copy.deepcopy(soc_minima) if isinstance(soc_minima, list) else [] - projected_maxima = copy.deepcopy(soc_maxima) if isinstance(soc_maxima, list) else [] - - soc_min_value = _optional_soc_value_in_mwh(soc_min) - soc_max_value = _optional_soc_value_in_mwh(soc_max) - - def add_min_bound(soc_event: dict, tick: pd.Timestamp, value: float) -> None: - _add_soc_bound( - projected_minima, - _soc_event_at(soc_event, tick, _clamp_soc_min(value, soc_min_value)), - bound_type="min", - ) - - def add_max_bound(soc_event: dict, tick: pd.Timestamp, value: float) -> None: - _add_soc_bound( - projected_maxima, - _soc_event_at(soc_event, tick, _clamp_soc_max(value, soc_max_value)), - bound_type="max", - ) + projection = _SocProjection( + consumption_capacity=consumption_capacity, + production_capacity=production_capacity, + resolution=resolution, + soc_min=_optional_soc_value_in_mwh(soc_min), + soc_max=_optional_soc_value_in_mwh(soc_max), + charging_efficiency=charging_efficiency, + discharging_efficiency=discharging_efficiency, + minima=copy.deepcopy(soc_minima) if isinstance(soc_minima, list) else [], + maxima=copy.deepcopy(soc_maxima) if isinstance(soc_maxima, list) else [], + ) if isinstance(soc_targets, list): projected_targets = [] for soc_target in soc_targets: - if soc_target["start"] != soc_target["end"] or is_on_schedule_tick( - soc_target["end"], resolution - ): + if not _is_projectable(soc_target, resolution): projected_targets.append(copy.copy(soc_target)) continue - target_time = pd.Timestamp(soc_target["end"]) - previous_tick = target_time.floor(resolution) - next_tick = target_time.ceil(resolution) - target_value = _soc_value_in_mwh(soc_target["value"]) - chargeable = _reachable_energy( - consumption_capacity, - previous_tick, - target_time, - resolution, - efficiency=charging_efficiency, - efficiency_affects_stock="multiply", - ) - dischargeable = _reachable_energy( - production_capacity, - previous_tick, - target_time, - resolution, - efficiency=discharging_efficiency, - efficiency_affects_stock="divide", - ) - # Exact target on the next tick, plus previous-tick bounds that keep # the target reachable at the original (off-tick) target time. - projected_targets.append(_soc_event_at(soc_target, next_tick, target_value)) - add_min_bound(soc_target, previous_tick, target_value - chargeable) - add_max_bound(soc_target, previous_tick, target_value + dischargeable) + projected_targets.append( + _soc_event_at( + soc_target, + target_time.ceil(resolution), + _soc_value_in_mwh(soc_target["value"]), + ) + ) + projection.apply_policy("soc-targets", soc_target, target_time) else: projected_targets = soc_targets - if isinstance(soc_minima, list): - for soc_event in copy.deepcopy(soc_minima): - if soc_event["start"] != soc_event["end"] or is_on_schedule_tick( - soc_event["end"], resolution - ): - continue - - event_time = pd.Timestamp(soc_event["end"]) - previous_tick = event_time.floor(resolution) - next_tick = event_time.ceil(resolution) - minimum = _soc_value_in_mwh(soc_event["value"]) - - # Lower bounds on both surrounding ticks that preserve whether the - # requested minimum can still be reached at the original event time. - add_min_bound( - soc_event, - previous_tick, - minimum - - _reachable_energy( - consumption_capacity, - previous_tick, - event_time, - resolution, - efficiency=charging_efficiency, - efficiency_affects_stock="multiply", - ), - ) - add_min_bound( - soc_event, - next_tick, - minimum - - _reachable_energy( - production_capacity, - event_time, - next_tick, - resolution, - efficiency=discharging_efficiency, - efficiency_affects_stock="divide", - ), - ) - - if isinstance(soc_maxima, list): - for soc_event in copy.deepcopy(soc_maxima): - if soc_event["start"] != soc_event["end"] or is_on_schedule_tick( - soc_event["end"], resolution - ): - continue - - event_time = pd.Timestamp(soc_event["end"]) - previous_tick = event_time.floor(resolution) - next_tick = event_time.ceil(resolution) - maximum = _soc_value_in_mwh(soc_event["value"]) - - # Upper bounds on both surrounding ticks that preserve whether the - # requested maximum can still be respected at the original event time. - add_max_bound( - soc_event, - previous_tick, - maximum - + _reachable_energy( - production_capacity, - previous_tick, - event_time, - resolution, - efficiency=discharging_efficiency, - efficiency_affects_stock="divide", - ), - ) - add_max_bound( - soc_event, - next_tick, - maximum - + _reachable_energy( - consumption_capacity, - event_time, - next_tick, - resolution, - efficiency=charging_efficiency, - efficiency_affects_stock="multiply", - ), - ) + for field_name, soc_events in ( + ("soc-minima", soc_minima), + ("soc-maxima", soc_maxima), + ): + if not isinstance(soc_events, list): + continue + for soc_event in copy.deepcopy(soc_events): + if _is_projectable(soc_event, resolution): + projection.apply_policy( + field_name, soc_event, pd.Timestamp(soc_event["end"]) + ) return ( projected_targets, - _projected_soc_events_or_original(soc_maxima, projected_maxima, "soc-maxima"), - _projected_soc_events_or_original(soc_minima, projected_minima, "soc-minima"), + _projected_soc_events_or_original(soc_maxima, projection.maxima, "soc-maxima"), + _projected_soc_events_or_original(soc_minima, projection.minima, "soc-minima"), ) def project_off_tick_soc_at_start( soc_at_start_time: datetime, soc_at_start: ur.Quantity | float, - soc_maxima: ( - list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None - ), - soc_minima: ( - list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None - ), + soc_maxima: SocSpecification, + soc_minima: SocSpecification, schedule_start: datetime, consumption_capacity: pd.Series, production_capacity: pd.Series, @@ -406,10 +439,7 @@ def project_off_tick_soc_at_start( soc_max: ur.Quantity | float | None, charging_efficiency: pd.Series | ur.Quantity | float | None = None, discharging_efficiency: pd.Series | ur.Quantity | float | None = None, -) -> tuple[ - list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, - list[dict[str, datetime | float]] | pd.Series | Sensor | ur.Quantity | None, -]: +) -> tuple[SocSpecification, SocSpecification]: """Project an off-tick starting state of charge onto the next scheduling tick. When the starting SoC is known at a time ``t`` between the schedule start and @@ -436,51 +466,24 @@ def project_off_tick_soc_at_start( ): return soc_maxima, soc_minima - next_tick = event_time.ceil(resolution) - value = _soc_value_in_mwh(soc_at_start) - chargeable = _reachable_energy( - consumption_capacity, - event_time, - next_tick, - resolution, - efficiency=charging_efficiency, - efficiency_affects_stock="multiply", - ) - dischargeable = _reachable_energy( - production_capacity, - event_time, - next_tick, - resolution, - efficiency=discharging_efficiency, - efficiency_affects_stock="divide", - ) - - projected_minima = copy.deepcopy(soc_minima) if isinstance(soc_minima, list) else [] - projected_maxima = copy.deepcopy(soc_maxima) if isinstance(soc_maxima, list) else [] - tick_datetime = next_tick.to_pydatetime() - _add_soc_bound( - projected_minima, - { - "start": tick_datetime, - "end": tick_datetime, - "value": _clamp_soc_min( - value - dischargeable, _optional_soc_value_in_mwh(soc_min) - ), - }, - bound_type="min", - ) - _add_soc_bound( - projected_maxima, - { - "start": tick_datetime, - "end": tick_datetime, - "value": _clamp_soc_max( - value + chargeable, _optional_soc_value_in_mwh(soc_max) - ), - }, - bound_type="max", + projection = _SocProjection( + consumption_capacity=consumption_capacity, + production_capacity=production_capacity, + resolution=resolution, + soc_min=_optional_soc_value_in_mwh(soc_min), + soc_max=_optional_soc_value_in_mwh(soc_max), + charging_efficiency=charging_efficiency, + discharging_efficiency=discharging_efficiency, + minima=copy.deepcopy(soc_minima) if isinstance(soc_minima, list) else [], + maxima=copy.deepcopy(soc_maxima) if isinstance(soc_maxima, list) else [], ) + soc_event = { + "start": event_time.to_pydatetime(), + "end": event_time.to_pydatetime(), + "value": _soc_value_in_mwh(soc_at_start), + } + projection.apply_policy("soc-at-start", soc_event, event_time) return ( - _projected_soc_events_or_original(soc_maxima, projected_maxima, "soc-maxima"), - _projected_soc_events_or_original(soc_minima, projected_minima, "soc-minima"), + _projected_soc_events_or_original(soc_maxima, projection.maxima, "soc-maxima"), + _projected_soc_events_or_original(soc_minima, projection.minima, "soc-minima"), ) From b5c1668dad450c07806e881dbbc29e7645844541 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 17 Jul 2026 16:45:43 +0200 Subject: [PATCH 72/72] fix(test): tolerate generic aliases when scanning for Field subclasses On Python 3.10, module-level generic aliases such as list[dict[...]] pass inspect.isclass (bpo-46080 changed this in 3.11) and then make issubclass raise a TypeError in test_all_custom_fields_call_super_deserialize. The new type aliases in soc_projection.py triggered exactly that. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01YNPESudGKciPZq88nGJn3C Signed-off-by: F.N. Claessen --- flexmeasures/tests/test_schemas.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/flexmeasures/tests/test_schemas.py b/flexmeasures/tests/test_schemas.py index 66e76a3ef9..2ad823d992 100644 --- a/flexmeasures/tests/test_schemas.py +++ b/flexmeasures/tests/test_schemas.py @@ -18,12 +18,18 @@ def iter_marshmallow_field_subclasses(): """Yield (class, module) for Marshmallow Field subclasses.""" for module in iter_flexmeasures_modules(): for obj in vars(module).values(): - if ( - inspect.isclass(obj) - and issubclass(obj, ma_fields.Field) - and obj is not ma_fields.Field - and obj.__module__ == module.__name__ - ): + try: + is_field_subclass = ( + inspect.isclass(obj) + and issubclass(obj, ma_fields.Field) + and obj is not ma_fields.Field + and obj.__module__ == module.__name__ + ) + except TypeError: + # On Python 3.10, generic aliases like list[dict] pass + # inspect.isclass but make issubclass raise (bpo-46080). + continue + if is_field_subclass: yield obj, module