diff --git a/documentation/changelog.rst b/documentation/changelog.rst index 0eb355d858..a5fe9c722c 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -25,6 +25,7 @@ New features * CLI support for adding/editing account attributes [see `PR #2242 `_] * Improve chart axis domain for event values not around zero, with a per-sub-chart ``y-axis`` option in ``sensors_to_show`` (default ``zero``, which pads the axis out to include zero) that can be set to ``data`` to fit a sub-chart's y-axis to the values shown, to an explicit ``[min, max]`` domain that the axis will cover at least (expanding to fit data beyond it), or to a strict ``{"min": min, "max": max}`` domain that the axis will never exceed (clamping data beyond it, with a warning when that happens), editable from the graph editor [see `PR #2244 `_] * Extended ``GET /api/v3_0/jobs/`` with a ``result`` field containing ``unresolved`` and ``resolved`` soft state-of-charge constraint analysis (``soc-minima``/``soc-maxima`` violations or satisfied constraints, keyed by asset ID) for scheduling jobs; both arrays are empty when no SoC constraints were defined [see `PR #2072 `_] +* New ``soc-value-at-end`` flex-model field for storage devices: assigns a marginal value (fixed quantity or sensor reference) to energy left in storage at the end of the scheduling horizon, countering myopic depletion of the storage [see `PR #2310 `_] * Extended the scheduling job ``result`` field with a ``num-beliefs`` field reporting the total number of beliefs (scheduled values) saved to the database [see `PR #2280 `_] * Migrate the asset tree in the UI's Structure tab from Vega to ECharts, adding interactive pan/zoom navigation and refreshed node styling [see `PR #2025 `_] diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst index aec2ce3397..62fc8e8390 100644 --- a/documentation/features/scheduling.rst +++ b/documentation/features/scheduling.rst @@ -232,6 +232,9 @@ For more details on the possible formats for field values, see :ref:`variable_qu * - ``soc-usage`` - |SOC_USAGE.example| - .. include:: ../_autodoc/SOC_USAGE.rst + * - ``soc-value-at-end`` + - |SOC_VALUE_AT_END.example| + - .. include:: ../_autodoc/SOC_VALUE_AT_END.rst * - ``roundtrip-efficiency`` - |ROUNDTRIP_EFFICIENCY.example| - .. include:: ../_autodoc/ROUNDTRIP_EFFICIENCY.rst diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index a6686748b8..3eda69adab 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -37,7 +37,11 @@ FlexContextSchema, MultiSensorFlexModelSchema, ) -from flexmeasures.data.schemas.sensors import SensorReference, VariableQuantityField +from flexmeasures.data.schemas.sensors import ( + SensorReference, + VariableQuantityField, + PriceField, +) from flexmeasures.data.services.scheduling_result import SchedulingJobResult from flexmeasures.utils.calculations import ( integrate_time_series, @@ -177,6 +181,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 soc_maxima = [None] * num_flexible_devices soc_gain = [None] * num_flexible_devices soc_usage = [None] * num_flexible_devices + soc_value_at_end = [None] * num_flexible_devices prefer_charging_sooner = [None] * num_flexible_devices prefer_curtailing_later = [None] * num_flexible_devices @@ -210,6 +215,7 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 soc_maxima[d0] = stock_model.get("soc_maxima") soc_gain[d0] = stock_model.get("soc_gain") soc_usage[d0] = stock_model.get("soc_usage") + soc_value_at_end[d0] = stock_model.get("soc_value_at_end") prefer_charging_sooner[d0] = stock_model.get("prefer_charging_sooner") prefer_curtailing_later[d0] = stock_model.get("prefer_curtailing_later") @@ -834,6 +840,34 @@ def device_list_series( # soc-maxima will become a soft constraint (modelled as stock commitments), so remove hard constraint soc_maxima[d] = None + if soc_value_at_end[d] is not None and soc_at_start[d] is not None: + # Assign a marginal value to energy left in storage at the end of the + # planning window, to counter myopic depletion of the storage. + soc_value_at_end_d = get_continuous_series_sensor_or_quantity( + variable_quantity=soc_value_at_end[d], + unit=self.flex_context["shared_currency_unit"] + + "/MWh*h", # from EUR/MWh² to EUR/MWh/resolution + query_window=(start + resolution, end + resolution), + resolution=resolution, + beliefs_before=belief_time, + fill_sides=True, + ).shift(-1, freq=resolution) + # Only the state of charge at the end of the planning window is valued + soc_value_at_end_d.iloc[:-1] = 0 + + commitment = StockCommitment( + name="value of soc at end", + # baseline is an (absolute) zero state of charge, so the upwards + # deviation in the final time slot is the final state of charge + quantity=-soc_at_start[d] * (timedelta(hours=1) / resolution), + # negative prices reward a higher state of charge at the end + upwards_deviation_price=-soc_value_at_end_d, + downwards_deviation_price=-soc_value_at_end_d, + index=index, + device=d, + ) + commitments.append(commitment) + # only apply SOC constraints to the first device of a shared stock apply_soc_constraints = True @@ -1288,6 +1322,7 @@ def deserialize_flex_config(self): self.collect_flex_config() self._deserialize_flex_context() self._deserialize_flex_model() + self._validate_flex_model_price_units() # Classify all flex-model entries (and the flex-context's inflexible devices) # once; scheduling and result mapping rely on this inventory for device @@ -1330,6 +1365,37 @@ def _deserialize_flex_context(self): f"Unsupported type of flex-context: '{type(self.flex_context)}'" ) + def _validate_flex_model_price_units(self): + """Check that price fields in the flex-model use the flex-context's shared currency. + + The flex-context validates that all its price fields share one currency + (its ``shared_currency_unit``); here we hold the flex-model's price fields + (declared as PriceField) to that same currency, so that unit conversion + cannot fail later, when the scheduler runs. + """ + shared_currency_unit = self.flex_context.get("shared_currency_unit") + if shared_currency_unit is None: + return + flex_model = ( + self.flex_model if isinstance(self.flex_model, list) else [self.flex_model] + ) + for flex_model_d in flex_model: + for field_name, field in StorageFlexModelSchema._declared_fields.items(): + if ( + not isinstance(field, PriceField) + or flex_model_d.get(field_name) is None + ): + continue + price_unit = field._get_unit(flex_model_d[field_name]) + currency_unit = str( + (ur.Quantity(price_unit) / ur.Quantity(f"1{field.to_unit}")).units + ) + if not units_are_convertible(currency_unit, shared_currency_unit): + raise ValidationError( + f"Invalid unit. A valid unit would be, for example, '{shared_currency_unit + field.to_unit}', because the flex-context uses '{shared_currency_unit}' as its currency. However, the '{field.data_key}' field in the flex-model uses an incompatible price unit ('{price_unit}').", + field_name=field.data_key, + ) + def _deserialize_flex_model(self): if isinstance(self.flex_model, dict): if self.sensor.generic_asset.asset_type.name in storage_asset_types: diff --git a/flexmeasures/data/models/planning/tests/test_solver.py b/flexmeasures/data/models/planning/tests/test_solver.py index 9acf49a6f7..68ac40b866 100644 --- a/flexmeasures/data/models/planning/tests/test_solver.py +++ b/flexmeasures/data/models/planning/tests/test_solver.py @@ -215,6 +215,92 @@ def test_battery_solver_day_2( ) +@pytest.mark.parametrize( + "soc_value_at_end, expect_full_at_end", + [ + ("0 EUR/MWh", False), + ("0.001 EUR/kWh", False), + ("1000 EUR/MWh", True), + ], +) +def test_battery_solver_day_2_with_soc_value_at_end( + setup_planning_test_data, + add_battery_assets, + soc_value_at_end: str, + expect_full_at_end: bool, + db, +): + """Check that valuing the SoC at the end of the scheduling horizon counters myopic depletion. + + Day 2 is set up with 8 expensive, then 8 cheap, then again 8 expensive hours. + Without (or with only a tiny) soc-value-at-end, the battery sells out towards the end of the horizon. + With a soc-value-at-end that exceeds the highest consumption price, the battery ends the horizon full instead. + """ + _epex_da, battery = get_sensors_from_db(db, add_battery_assets) + tz = pytz.timezone("Europe/Amsterdam") + start = tz.localize(datetime(2015, 1, 2)) + end = tz.localize(datetime(2015, 1, 3)) + resolution = timedelta(minutes=15) + soc_at_start = battery.get_attribute("soc_in_mwh") + soc_min = 0.5 + soc_max = 4.5 + scheduler = StorageScheduler( + battery, + start, + end, + resolution, + flex_model={ + "soc-at-start": soc_at_start, + "soc-min": soc_min, + "soc-max": soc_max, + "roundtrip-efficiency": 1, + "storage-efficiency": 1, + "prefer-curtailing-later": False, + "soc-value-at-end": soc_value_at_end, + }, + ) + schedule = scheduler.compute() + + # Check if constraints were met + soc_schedule = check_constraints(battery, schedule, soc_at_start) + + if expect_full_at_end: + np.testing.assert_approx_equal( + soc_schedule.iloc[-1], soc_max, significant=3 + ) # The value of a full battery at the end beats the energy prices + else: + np.testing.assert_approx_equal( + soc_schedule.iloc[-1], soc_min, significant=3 + ) # Battery still sold out at the end of its planning horizon + + +def test_soc_value_at_end_currency_mismatch( + setup_planning_test_data, + add_battery_assets, + db, +): + """A soc-value-at-end in a different currency than the flex-context's shared currency is rejected upon deserialization.""" + from marshmallow import ValidationError + + _epex_da, battery = get_sensors_from_db(db, add_battery_assets) + tz = pytz.timezone("Europe/Amsterdam") + start = tz.localize(datetime(2015, 1, 2)) + end = tz.localize(datetime(2015, 1, 3)) + resolution = timedelta(minutes=15) + scheduler = StorageScheduler( + battery, + start, + end, + resolution, + flex_model={ + "soc-at-start": battery.get_attribute("soc_in_mwh"), + "soc-value-at-end": "1000 USD/MWh", + }, + ) + with pytest.raises(ValidationError, match="soc-value-at-end"): + scheduler.compute() + + def run_test_charge_discharge_sign( battery, roundtrip_efficiency, diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py index 93c86ac555..e016d065b4 100644 --- a/flexmeasures/data/schemas/scheduling/__init__.py +++ b/flexmeasures/data/schemas/scheduling/__init__.py @@ -24,6 +24,7 @@ SensorIdField, SensorReference, OutputSensorReferenceSchema, + PriceField, ) from flexmeasures.data.schemas.scheduling import metadata from flexmeasures.data.schemas.units import UnitField @@ -72,8 +73,8 @@ def forbid_time_series_specs(self, data: dict, **kwargs): class CommitmentSchema(Schema): name = fields.Str(required=True, data_key="name") baseline = VariableQuantityField("MW", required=False, data_key="baseline") - up_price = VariableQuantityField("/MW", required=False, data_key="up-price") - down_price = VariableQuantityField( + up_price = PriceField("/MW", required=False, data_key="up-price") + down_price = PriceField( "/MW", required=False, data_key="down-price", @@ -147,7 +148,7 @@ class DBCommitmentSchema(CommitmentSchema, NoTimeSeriesSpecs): class SharedSchema(Schema): """Shared schema for fields common across commodities in flex-context and commodity-context.""" - consumption_price = VariableQuantityField( + consumption_price = PriceField( "/MWh", required=False, data_key="consumption-price", @@ -155,7 +156,7 @@ class SharedSchema(Schema): metadata=metadata.CONSUMPTION_PRICE.to_dict(), ) - production_price = VariableQuantityField( + production_price = PriceField( "/MWh", required=False, data_key="production-price", @@ -187,7 +188,7 @@ class SharedSchema(Schema): metadata=metadata.SITE_PRODUCTION_CAPACITY.to_dict(), ) - ems_consumption_breach_price = VariableQuantityField( + ems_consumption_breach_price = PriceField( "/MW", data_key="site-consumption-breach-price", required=False, @@ -195,7 +196,7 @@ class SharedSchema(Schema): metadata=metadata.SITE_CONSUMPTION_BREACH_PRICE.to_dict(), ) - ems_production_breach_price = VariableQuantityField( + ems_production_breach_price = PriceField( "/MW", data_key="site-production-breach-price", required=False, @@ -212,7 +213,7 @@ class SharedSchema(Schema): metadata=metadata.SITE_PEAK_CONSUMPTION.to_dict(), ) - ems_peak_consumption_price = VariableQuantityField( + ems_peak_consumption_price = PriceField( "/MW", data_key="site-peak-consumption-price", required=False, @@ -229,7 +230,7 @@ class SharedSchema(Schema): metadata=metadata.SITE_PEAK_PRODUCTION.to_dict(), ) - ems_peak_production_price = VariableQuantityField( + ems_peak_production_price = PriceField( "/MW", data_key="site-peak-production-price", required=False, @@ -238,28 +239,28 @@ class SharedSchema(Schema): ) # Breach prices for device capacity constraints - consumption_breach_price = VariableQuantityField( + consumption_breach_price = PriceField( "/MW", data_key="consumption-breach-price", required=False, value_validator=validate.Range(min=0), metadata=metadata.CONSUMPTION_BREACH_PRICE.to_dict(), ) - production_breach_price = VariableQuantityField( + production_breach_price = PriceField( "/MW", data_key="production-breach-price", required=False, value_validator=validate.Range(min=0), metadata=metadata.PRODUCTION_BREACH_PRICE.to_dict(), ) - soc_minima_breach_price = VariableQuantityField( + soc_minima_breach_price = PriceField( "/MWh", data_key="soc-minima-breach-price", required=False, value_validator=validate.Range(min=0), metadata=metadata.SOC_MINIMA_BREACH_PRICE.to_dict(), ) - soc_maxima_breach_price = VariableQuantityField( + soc_maxima_breach_price = PriceField( "/MWh", data_key="soc-maxima-breach-price", required=False, @@ -340,7 +341,7 @@ def _try_to_convert_price_units(self, data: dict, original_data: dict, **kwargs) shared_currency_unit = None previous_field_name = None for field in self.declared_fields: - if field[-5:] == "price" and field in data: + if isinstance(self.declared_fields[field], PriceField) and field in data: price_field = self.declared_fields[field] price_unit = price_field._get_unit(data[field]) currency_unit = str( @@ -363,6 +364,13 @@ def _try_to_convert_price_units(self, data: dict, original_data: dict, **kwargs) if shared_currency_unit not in price_unit: error_message += f" Also note that all prices in the flex-context must share the same currency unit (in this case: '{shared_currency_unit}')." raise ValidationError(error_message, field_name=field_name) + # Also hold the nested commitment prices to the shared currency + shared_currency_unit, previous_field_name = ( + self._validate_commitment_price_units( + data, shared_currency_unit, previous_field_name + ) + ) + if shared_currency_unit is not None: data["shared_currency_unit"] = shared_currency_unit elif sensor := data.get("consumption_price_sensor"): @@ -373,6 +381,46 @@ def _try_to_convert_price_units(self, data: dict, original_data: dict, **kwargs) data["shared_currency_unit"] = "EUR" return data + def _validate_commitment_price_units( + self, + data: dict, + shared_currency_unit: str | None, + previous_field_name: str | None, + ) -> tuple[str | None, str | None]: + """Hold the nested commitment prices to the shared currency.""" + for commitment in data.get("commitments", []): + for field, price_field in CommitmentSchema._declared_fields.items(): + if not isinstance(price_field, PriceField) or field not in commitment: + continue + price_unit = price_field._get_unit(commitment[field]) + currency_unit = self._extract_currency_unit(price_unit) + if shared_currency_unit is None: + shared_currency_unit = str( + ur.Quantity(currency_unit).to_base_units().units + ) + previous_field_name = price_field.data_key + if not units_are_convertible(currency_unit, shared_currency_unit): + field_name = price_field.data_key + error_message = f"Invalid unit. A valid unit would be, for example, '{shared_currency_unit}/MWh' (this example uses '{shared_currency_unit}', because '{previous_field_name}' used that currency). However, you passed an incompatible price ('{price_unit}') for the '{field_name}' field of commitment '{commitment.get('name')}'. Also note that all prices in the flex-context must share the same currency unit (in this case: '{shared_currency_unit}')." + raise ValidationError(error_message, field_name="commitments") + return shared_currency_unit, previous_field_name + + @staticmethod + def _extract_currency_unit(price_unit: str) -> str: + """Obtain the currency part of a price unit, whose denominator may vary. + + >>> FlexContextSchema()._extract_currency_unit("EUR/MWh") + 'EUR' + >>> FlexContextSchema()._extract_currency_unit("USD/MW") + 'USD' + """ + q = ur.Quantity(f"1 {price_unit}") + for denominator in ("MWh", "MW"): + candidate = str((q * ur.Quantity(f"1 {denominator}")).to_base_units().units) + if is_currency_unit(candidate): + return candidate + return str(q.units) + @staticmethod def _to_currency_per_mwh(price_unit: str) -> str: """Convert a price unit to a base currency used to express that price per MWh. @@ -862,6 +910,15 @@ def check_prices(self, data: dict, original_data: dict, **kwargs): }, "example-units": EXAMPLE_UNIT_TYPES["power"], }, + "soc-value-at-end": { + "default": None, + "description": rst_to_openapi(metadata.SOC_VALUE_AT_END.description), + "types": { + "backend": "typeThree", + "ui": "One fixed value or a dynamic signal (via a sensor).", + }, + "example-units": EXAMPLE_UNIT_TYPES["energy-price"], + }, "roundtrip-efficiency": { "default": None, "description": rst_to_openapi(metadata.ROUNDTRIP_EFFICIENCY.description), diff --git a/flexmeasures/data/schemas/scheduling/metadata.py b/flexmeasures/data/schemas/scheduling/metadata.py index 4f6f9d4298..1771bd9d69 100644 --- a/flexmeasures/data/schemas/scheduling/metadata.py +++ b/flexmeasures/data/schemas/scheduling/metadata.py @@ -314,6 +314,14 @@ def to_dict(self): """, example=[{"datetime": "2024-02-05T08:00:00+01:00", "value": "3.2 kWh"}], ) +SOC_VALUE_AT_END = MetaData( + description="""Marginal value assigned to energy left in storage at the end of the scheduling horizon. +Without it, the scheduler sees no benefit in ending the horizon with a non-zero state of charge, which can lead to myopic behaviour such as fully depleting the storage towards the end of the horizon. +It must use the same currency as the other price settings and cannot be negative. +Set it per device (for example, lower for a heat pump's thermal buffer than for a battery, due to the difference between the COP and the battery's charging efficiency), either as a fixed quantity or as a sensor reference. +""", + example="60 EUR/MWh", +) SOC_GAIN = MetaData( description="""SoC gain per time step, e.g. from a secondary energy source. Useful if energy is inserted by an external process (in-flow). This field allows setting multiple components, either fixed or dynamic, which add up to an aggregated gain. diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index 60d9da3448..ffcf61025f 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -22,6 +22,7 @@ SensorReference, OutputSensorReferenceSchema, VariableQuantityField, + PriceField, ) from flexmeasures.utils.unit_utils import ( ur, @@ -246,6 +247,14 @@ class StorageFlexModelSchema(Schema): metadata=metadata.SOC_USAGE.to_dict(), ) + soc_value_at_end = PriceField( + "/MWh", + data_key="soc-value-at-end", + required=False, + value_validator=validate.Range(min=0), + metadata=metadata.SOC_VALUE_AT_END.to_dict(), + ) + def __init__( self, start: datetime, @@ -489,6 +498,13 @@ class DBStorageFlexModelSchema(Schema): metadata={"deprecated field": "soc-usage"}, ) + soc_value_at_end = PriceField( + "/MWh", + data_key="soc-value-at-end", + required=False, + value_validator=validate.Range(min=0), + ) + roundtrip_efficiency = EfficiencyField( data_key="roundtrip-efficiency", required=False, diff --git a/flexmeasures/data/schemas/sensors.py b/flexmeasures/data/schemas/sensors.py index 72694534a6..58975ded70 100644 --- a/flexmeasures/data/schemas/sensors.py +++ b/flexmeasures/data/schemas/sensors.py @@ -678,6 +678,18 @@ def _get_unit( return unit +class PriceField(VariableQuantityField): + """VariableQuantityField for monetary values. + + Price fields participate in currency validation: all price fields in the + flex-context must share one currency (recorded as the flex-context's + ``shared_currency_unit``), and price fields in a flex-model must use a + currency that is convertible to the flex-context's shared currency. + """ + + pass + + class RepurposeValidatorToIgnoreSensorsAndLists(validate.Validator): """Validator that executes another validator (the one you initialize it with) only on non-Sensor and non-list values.""" diff --git a/flexmeasures/data/schemas/tests/test_scheduling.py b/flexmeasures/data/schemas/tests/test_scheduling.py index 90842d2cf9..1441cb1a28 100644 --- a/flexmeasures/data/schemas/tests/test_scheduling.py +++ b/flexmeasures/data/schemas/tests/test_scheduling.py @@ -341,6 +341,53 @@ def load_schema(): }, {"commitments.0.baseline": "Cannot convert value `10 kWh` to 'MW'"}, ), + # Commitment prices must share the flex-context's currency + ( + { + "consumption-price": "100 EUR/MWh", + "commitments": [ + { + "name": "a sample commitment", + "baseline": "10 kW", + "up-price": "100 USD/MWh", + } + ], + }, + { + "commitments": "all prices in the flex-context must share the same currency unit" + }, + ), + # Commitment prices sharing the flex-context's currency are fine + ( + { + "consumption-price": "100 EUR/MWh", + "commitments": [ + { + "name": "a sample commitment", + "baseline": "10 kW", + "up-price": "100 EUR/MWh", + "down-price": "0.12 EUR/kWh", + } + ], + }, + False, + ), + # Commitments can also set the shared currency (mixed currencies still fail) + ( + { + "commitments": [ + { + "name": "a sample commitment", + "baseline": "10 kW", + "up-price": "100 USD/MWh", + "down-price": "120 EUR/MWh", + } + ] + }, + { + "commitments": "all prices in the flex-context must share the same currency unit" + }, + ), # Energy price units with a power baseline ( { diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index 0eab864eef..ac87ca2b73 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -6369,6 +6369,11 @@ ], "items": {} }, + "soc-value-at-end": { + "description": "Marginal value assigned to energy left in storage at the end of the scheduling horizon.\nWithout it, the scheduler sees no benefit in ending the horizon with a non-zero state of charge, which can lead to myopic behaviour such as fully depleting the storage towards the end of the horizon.\nIt must use the same currency as the other price settings and cannot be negative.\nSet it per device (for example, lower for a heat pump's thermal buffer than for a battery, due to the difference between the COP and the battery's charging efficiency), either as a fixed quantity or as a sensor reference.\n", + "example": "60 EUR/MWh", + "$ref": "#/components/schemas/VariableQuantityOpenAPI" + }, "sensor": { "type": "integer", "description": "ID of the device's power sensor."