From 3b539775ea95b654938303046f893b5655351d83 Mon Sep 17 00:00:00 2001 From: "F.N. Claessen" Date: Fri, 10 Jul 2026 11:54:14 +0200 Subject: [PATCH] feat: define intermediate power constraints via DB flex-models on the asset tree Extend the storage flex-model's group field to also accept {"asset": }, so a group can be identified by an asset-keyed flex-model entry - the form that DB-stored flex-models naturally produce. This makes intermediate power constraints fully definable via an asset tree with stored flex-models, with an empty flex-model in the scheduling trigger. - Asset-referenced group entries define no power sensor of their own; the group's scheduled aggregate power is saved via the entry's consumption and/or production output sensors, following the usual output-sensor conventions (full profile on a single sensor; clip-split when both given). - Sensor-referenced groups keep saving the aggregate to the group power sensor, and may now also define output sensors. - The UI asset flex-model editor recommends the parent asset when editing a child's group field (one-click suggestion), and hints in both the parent's and children's editors when the parent's flex-model defines power-capacity. - New tutorial: toy example for intermediate power constraints, driven entirely by DB-stored flex-models. Bug fixes uncovered along the way: - AssetTriggerSchema.check_flex_model_sensors raised KeyError on asset-only flex-model entries. - The freeze_server_now test fixture leaked its server_now monkeypatch into all subsequently run tests in the same process, causing order-dependent duplicate-key failures when scheduling jobs saved beliefs with identical (frozen) belief times. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01TAad46Ayg86DHpL6nY54sX Signed-off-by: F.N. Claessen --- documentation/api/change_log.rst | 1 + documentation/changelog.rst | 1 + documentation/features/scheduling.rst | 16 +- documentation/index.rst | 1 + .../tut/toy-example-group-constraints.rst | 158 ++++++++ .../tests/test_asset_schedules_fresh_db.py | 11 +- flexmeasures/conftest.py | 63 +-- flexmeasures/data/models/planning/storage.py | 378 +++++++++++------ .../planning/tests/test_group_constraints.py | 382 ++++++++++++++++++ .../data/schemas/scheduling/__init__.py | 8 +- .../data/schemas/scheduling/metadata.py | 7 +- .../data/schemas/scheduling/storage.py | 32 +- .../data/schemas/tests/test_scheduling.py | 14 +- flexmeasures/ui/static/openapi-specs.json | 62 ++- .../ui/templates/assets/asset_properties.html | 88 +++- flexmeasures/ui/tests/test_asset_crud.py | 72 ++++ flexmeasures/ui/views/assets/views.py | 20 + 17 files changed, 1136 insertions(+), 178 deletions(-) create mode 100644 documentation/tut/toy-example-group-constraints.rst diff --git a/documentation/api/change_log.rst b/documentation/api/change_log.rst index 351a16ac62..7a8306e100 100644 --- a/documentation/api/change_log.rst +++ b/documentation/api/change_log.rst @@ -10,6 +10,7 @@ v3.0-32 | July XX, 2026 - Extended ``GET /api/v3_0/jobs/`` with a ``result`` field containing ``unresolved`` and ``resolved`` arrays, each keyed by asset ID. For scheduling jobs, this surfaces soft state-of-charge constraint analysis: ``soc-minima`` and ``soc-maxima`` violations (with a ``violation`` magnitude) or satisfied constraints (with a ``margin`` headroom). Both arrays are empty when no SoC constraints were defined. - Added a ``group`` field to the storage flex-model, accepted by the `/assets/(id)/schedules/trigger <../api/v3_0.html#post--api-v3_0-assets-id-schedules-trigger>`_ (POST) endpoint, referencing a power sensor representing a group of devices (e.g. a shared inverter or feeder). The group's ``power-capacity`` is enforced as a hard constraint on the group's aggregate power, while its ``consumption-capacity``/``production-capacity`` are enforced as soft constraints with default breach prices; the group's scheduled aggregate power is saved to the group sensor. +- The ``group`` field also accepts a ``{"asset": }`` reference (besides ``{"sensor": }``), pointing at an asset whose own (DB-stored) flex-model defines the group's constraints. Such a group defines no power sensor of its own; its aggregate schedule is instead saved via its ``consumption``/``production`` output sensor references, following the same conventions as any other asset-only flex-model entry. This lets the entire flex-model for a device tree (including groups) live in the DB, with ``flex-model`` omitted or empty on the trigger request. v3.0-31 | 2026-06-01 """""""""""""""""""" diff --git a/documentation/changelog.rst b/documentation/changelog.rst index b507e06338..9c3ecc482a 100644 --- a/documentation/changelog.rst +++ b/documentation/changelog.rst @@ -22,6 +22,7 @@ New features * CLI support for adding/editing account attributes [see `PR #2242 `_] * Extended ``GET /api/v3_0/jobs/`` with a ``result`` field containing ``unresolved`` and ``resolved`` soft state-of-charge constraint analysis (``soc-minima``/``soc-maxima`` violations or satisfied constraints, keyed by asset ID) for scheduling jobs; both arrays are empty when no SoC constraints were defined [see `PR #2072 `_] * Add support for intermediate power constraints on groups of devices, via a new ``group`` field in the storage flex-model [see `PR #2276 `_ and `issue #2092 `_] +* The ``group`` field now also accepts a ``{"asset": }`` reference (in addition to ``{"sensor": }``), allowing intermediate power constraints to be defined entirely from flex-models stored on the asset tree, with results saved via the group's ``consumption``/``production`` output sensors, without needing any flex-model in the scheduling trigger [see `issue #2092 `_] Infrastructure / Support ---------------------- diff --git a/documentation/features/scheduling.rst b/documentation/features/scheduling.rst index 4f66e2d07f..3873d0d30a 100644 --- a/documentation/features/scheduling.rst +++ b/documentation/features/scheduling.rst @@ -275,14 +275,22 @@ For more details on the possible formats for field values, see :ref:`variable_qu Intermediate power constraints """"""""""""""""""""""""""""""" -In a multi-device flex-model list, a device entry may declare a ``group`` field referencing a power sensor that represents a group of devices, for example a hybrid inverter shared by a battery and PV installation, or a feeder shared by several devices. This lets you model an intermediate power constraint that sits between the individual devices and the site as a whole. +In a multi-device flex-model list, a device entry may declare a ``group`` field referencing a group of devices, for example a hybrid inverter shared by a battery and PV installation, or a feeder shared by several devices. This lets you model an intermediate power constraint that sits between the individual devices and the site as a whole. The ``group`` field accepts exactly one of two references: -The group sensor gets its own flex-model entry, defining constraints on the group's aggregate (summed) power: +- ``{"sensor": }``: the group is identified by a power sensor, which itself gets its own flex-model entry (typically passed alongside the device entries; mainly useful for API-passed flex-models). +- ``{"asset": }``: the group is identified by the flex-model entry stored on that asset (typically a sub-EMS/asset in the asset tree, such as the inverter in the example below). Such a group entry defines no power sensor of its own; instead, like any other asset-only entry, it may define ``consumption`` and/or ``production`` output sensor references (see below) on which the group's aggregate power gets saved. + +Either way, the group reference's target (sensor or asset) gets its own flex-model entry, defining constraints on the group's aggregate (summed) power: - ``power-capacity`` on the group is a **hard** constraint (applied in both directions). - ``consumption-capacity`` and ``production-capacity`` on the group are **soft** constraints, enforced with the same default breach prices used at the site level (10000 currency/kW); users cannot configure custom breach prices for groups. -The group's scheduled aggregate power is saved to the group sensor as a schedule output. Groups can be nested (a group entry may itself reference a parent group), but cyclic references are rejected. Groups require a multi-device flex-model; they are rejected when scheduling a single sensor. +The group's scheduled aggregate power is saved as a schedule output, following the same conventions used for any device's schedule output: + +- If the group's flex-model entry has a ``sensor`` field, the aggregate power is saved directly to that sensor. +- Otherwise (an asset-only entry), the aggregate power is saved via its ``consumption`` and/or ``production`` output sensor references: with only ``consumption`` set, the full profile is saved consumption-positive; with only ``production`` set, the full profile is saved production-positive (i.e. sign-flipped before saving); with both set, the profile is split into its non-negative part (saved to ``consumption``) and its non-positive part (saved, as a positive magnitude, to ``production``). + +Groups can be nested (a group entry may itself reference a parent group), but cyclic references are rejected. Groups require a multi-device flex-model; they are rejected when scheduling a single sensor. Example, for a 2.5 kW hybrid inverter (sensor 5) shared by a battery (sensor 1) and PV installation (sensor 2), taken from `issue #2092 `_: @@ -296,6 +304,8 @@ Example, for a 2.5 kW hybrid inverter (sensor 5) shared by a battery (sensor 1) Here, the battery and PV installation may each individually schedule up to 2 kW, but their combined power flowing through the shared inverter is hard-limited to 2.5 kW. +The ``{"asset": }`` variant lets you define the entire flex-model on the asset tree in the DB, with no flex-model needed in the scheduling trigger at all: each device asset carries its own (partial) flex-model, including a ``group`` field pointing at the parent asset that represents the shared equipment, and that parent asset's own flex-model defines the group's constraints and output sensor(s). Triggering a schedule for the top-level site asset with an empty (or omitted) ``flex-model`` then collects the full configuration from the tree. For a hands-on walkthrough (including how to store flex-models on assets, and where the resulting schedules end up), see :ref:`tut_toy_schedule_group_constraints`. + Usually, not the whole flexibility model is needed. FlexMeasures can infer missing values in the flex model, and even get them (as default) from the sensor's attributes. diff --git a/documentation/index.rst b/documentation/index.rst index b2c91d487e..987717cedf 100644 --- a/documentation/index.rst +++ b/documentation/index.rst @@ -184,6 +184,7 @@ In :ref:`getting_started`, we have some helpful tips how to dive into this docum tut/toy-example-from-scratch tut/toy-example-expanded tut/toy-example-multiasset-curtailment + tut/toy-example-group-constraints tut/flex-model-v2g tut/multi-feed-storage tut/multi-commodity diff --git a/documentation/tut/toy-example-group-constraints.rst b/documentation/tut/toy-example-group-constraints.rst new file mode 100644 index 0000000000..0b8221be42 --- /dev/null +++ b/documentation/tut/toy-example-group-constraints.rst @@ -0,0 +1,158 @@ +.. _tut_toy_schedule_group_constraints: + + +Toy example IV: Intermediate power constraints (groups) +================================================================ + +So far, our flexible devices (the battery and the PV inverter) have only ever been constrained directly by the building's own grid connection capacity. +But in reality, several devices are often physically wired together behind a shared piece of equipment before they reach the site's connection, and that piece of equipment has its own power limit. + +The classic example is a **hybrid inverter**: a battery and a PV installation share one inverter, and while each device could individually push a lot of power, the inverter itself caps their *combined* power flow. +This is what the ``group`` field in the storage flex-model is for (see :ref:`storage_device_scheduler` for the general explanation). This tutorial shows a fully DB-driven setup, where the entire flex-model lives on the asset tree, and you trigger a schedule for the site with an empty flex-model. + +We'll build the following little asset tree: + +.. code-block:: text + + site (building) + └── inverter (hybrid inverter, hard power-capacity 2.5 kW) + ├── battery (device, group member) + └── PV (device, group member) + +Setting up the asset tree +--------------------------------------- + +We create the site, the inverter (the group) and the two devices as assets, with the inverter and devices being children of the site. +Each device also needs output sensors to record its schedule (since these devices won't have a dedicated power sensor of their own — they are "asset-only" flex-model entries), and the inverter needs an output sensor for the group's aggregate schedule. + +.. code-block:: bash + + $ flexmeasures add asset --name "toy site" --asset-type-id 5 --account-id 1 + Successfully created asset with ID 10. + + $ flexmeasures add asset --name "hybrid inverter" --asset-type-id 5 --account-id 1 --parent-asset 10 + Successfully created asset with ID 11. + + $ flexmeasures add asset --name "toy battery" --asset-type-id 5 --account-id 1 --parent-asset 10 + Successfully created asset with ID 12. + + $ flexmeasures add asset --name "toy PV" --asset-type-id 5 --account-id 1 --parent-asset 10 + Successfully created asset with ID 13. + + $ flexmeasures add sensor --name "inverter aggregate power" --unit MW --event-resolution PT15M --asset-id 11 + Successfully created sensor with ID 21. + + $ flexmeasures add sensor --name "battery consumption" --unit MW --event-resolution PT15M --asset-id 12 + Successfully created sensor with ID 22. + $ flexmeasures add sensor --name "battery production" --unit MW --event-resolution PT15M --asset-id 12 + Successfully created sensor with ID 23. + + $ flexmeasures add sensor --name "PV production" --unit MW --event-resolution PT15M --asset-id 13 + Successfully created sensor with ID 24. + +.. note:: Asset type IDs and IDs returned above will differ in your own setup — substitute your own. + +Storing the flex-models on the assets +--------------------------------------- + +Rather than sending a flex-model with the trigger request, we store each asset's (partial) flex-model directly on the asset. FlexMeasures will walk the tree and collect these into one combined flex-model when scheduling the site (see ``GenericAsset.get_flex_model`` and ``Scheduler.collect_flex_config``). + +You can set an asset's flex-model with ``PATCH /api/v3_0/assets/``, sending a ``flex_model`` field with the JSON below. (The FlexMeasures UI's flex-model editor on the asset's properties page supports this too, and even suggests the parent asset as a candidate for the ``group`` field.) + +The battery is a device with both a consumption and production output sensor (it can charge and discharge), belonging to the inverter's group: + +.. code-block:: json + + { + "flex_model": { + "power-capacity": "2 kW", + "consumption-capacity": "2 kW", + "production-capacity": "2 kW", + "group": {"asset": 11}, + "consumption": {"sensor": 22}, + "production": {"sensor": 23} + } + } + +Sent as ``PATCH /api/v3_0/assets/12``. + +The PV installation only produces, so it only needs a production output sensor: + +.. code-block:: json + + { + "flex_model": { + "power-capacity": "2 kW", + "consumption-capacity": "0 kW", + "production-capacity": "2 kW", + "group": {"asset": 11}, + "production": {"sensor": 24} + } + } + +Sent as ``PATCH /api/v3_0/assets/13``. + +Finally, the inverter's own flex-model defines the group's hard power-capacity and where to save the group's aggregate schedule (as it has no power sensor of its own, either): + +.. code-block:: json + + { + "flex_model": { + "power-capacity": "2.5 kW", + "consumption": {"sensor": 21} + } + } + +Sent as ``PATCH /api/v3_0/assets/11``. + +Note that neither the battery, the PV installation, nor the inverter reference a ``sensor`` field of their own for scheduling purposes — this is what makes them "asset-only" entries. Instead, results are always saved via their ``consumption``/``production`` output sensor references. + +Triggering the schedule +--------------------------------------- + +We now trigger a schedule for the site (asset 10) with an empty (or omitted) flex-model. Everything the scheduler needs is picked up from the DB-stored flex-models on the asset tree. + +.. tabs:: + + .. tab:: CLI + + .. code-block:: bash + + $ flexmeasures add schedule \ + --asset 10 \ + --start ${TOMORROW}T00:00+01:00 --duration PT4H \ + --flex-model '[]' + New schedule is stored. + + .. tab:: API + + Example call: `[POST] http://localhost:5000/api/v3_0/assets/10/schedules/trigger <../api/v3_0.html#post--api-v3_0-assets-id-schedules-trigger>`_ (update the start date to tomorrow): + + .. code-block:: json + + { + "start": "2026-07-11T00:00+01:00", + "duration": "PT4H", + "flex-model": [] + } + +Inspecting the results +--------------------------------------- + +Once the job has finished, three schedules were computed and saved: + +- The battery's schedule, split (as it can both charge and discharge) between sensor 22 (``battery consumption``, holding the non-negative, consumption-positive part) and sensor 23 (``battery production``, holding the non-positive part, stored as a positive magnitude). +- The PV installation's schedule, saved entirely to sensor 24 (``PV production``), sign-flipped to be stored as a positive magnitude (since PV only produces). +- The inverter group's aggregate schedule, saved to sensor 21 (``inverter aggregate power``), equal to the (consumption-positive) sum of the battery's and PV's schedules. + +You can inspect any of these with: + +.. code-block:: bash + + $ flexmeasures show beliefs --sensor 21 --start ${TOMORROW}T00:00:00+01:00 --duration PT4H + +The group's aggregate power never exceeds 2.5 kW in either direction — even though the battery and PV could individually reach 2 kW each (4 kW combined) — because the hybrid inverter's hard ``power-capacity`` caps their sum. This mirrors the scenario from `issue #2092 `_, and is exercised end-to-end (at the planning level, bypassing the API/CLI layer but exercising the same DB-tree flex-model collection) by the test ``test_pure_db_tree_group_constraint`` in ``flexmeasures/data/models/planning/tests/test_group_constraints.py``. + +.. note:: If a device only ever consumes or only ever produces, you only need to define the corresponding single output sensor (as we did for the PV installation above). Only devices (or groups) that can go both ways need both a ``consumption`` and a ``production`` output sensor. + +This concludes our tour of intermediate power constraints. For the full field reference, see :ref:`storage_device_scheduler` and the "Intermediate power constraints" section of :ref:`scheduling`. diff --git a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py index 9a14f016d9..c928c6a701 100644 --- a/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py +++ b/flexmeasures/api/v3_0/tests/test_asset_schedules_fresh_db.py @@ -959,11 +959,12 @@ def get_beliefs_as_series(sensor: Sensor) -> pd.Series: control_job_id = control_response.json["schedule"] work_on_rq(app.queues["scheduling"], exc_handler=handle_scheduling_exception) - assert ( - Job.fetch( - control_job_id, connection=app.queues["scheduling"].connection - ).is_finished - is True + control_job = Job.fetch( + control_job_id, connection=app.queues["scheduling"].connection + ) + assert control_job.is_finished is True, ( + f"control job ended as '{control_job.get_status()}': " + f"{control_job.latest_result().exc_string if control_job.latest_result() else None}" ) control_scheduler_source = get_data_source_for_job( Job.fetch(control_job_id, connection=app.queues["scheduling"].connection) diff --git a/flexmeasures/conftest.py b/flexmeasures/conftest.py index 212d8f3140..119dc8a81a 100644 --- a/flexmeasures/conftest.py +++ b/flexmeasures/conftest.py @@ -2059,6 +2059,27 @@ def add_test_sensor_with_anomalous_beliefs( return {"anomaly-sensor": sensor} +def _patch_server_now_in_module(module, module_name: str, value, originals: dict): + """Patch server_now in a single module, remembering the original only the first + time we patch it, so repeated freeze calls still restore the true original.""" + try: + originals.setdefault(module_name, module.server_now) + setattr(module, "server_now", lambda: value) + except Exception: + # skip modules that cannot be inspected or modified + pass + + +def _patch_server_now_in_loaded_modules(value, originals: dict): + """Patch server_now in all currently loaded FlexMeasures modules.""" + for module in list(sys.modules.values()): # copy to avoid RuntimeError + if not isinstance(module, type(sys)): # skip placeholders + continue + name = getattr(module, "__name__", "") + if name.startswith("flexmeasures") and hasattr(module, "server_now"): + _patch_server_now_in_module(module, name, value, originals) + + @pytest.fixture def freeze_server_now(): """ @@ -2068,40 +2089,24 @@ def freeze_server_now(): def test_x(freeze_server_now): freeze_server_now(pd.Timestamp("2025-01-15T12:23:58+01")) """ - patched_modules = set() + original_server_nows: dict = {} # module name -> original server_now function + original_import = builtins.__import__ def _freeze(value: datetime | pd.Timestamp): if isinstance(value, pd.Timestamp): value = value.to_pydatetime() # Patch currently loaded FlexMeasures modules - for module in list(sys.modules.values()): # copy to avoid RuntimeError - try: - if not isinstance(module, type(sys)): # skip placeholders - continue - name = getattr(module, "__name__", "") - if not name.startswith("flexmeasures"): - continue - if hasattr(module, "server_now"): - setattr(module, "server_now", lambda: value) - patched_modules.add(module.__name__) - except Exception: - # skip modules that cannot be inspected or modified - pass + _patch_server_now_in_loaded_modules(value, original_server_nows) # Optionally, warn if new modules are imported later - original_import = builtins.__import__ - def import_hook(name, *args, **kwargs): mod = original_import(name, *args, **kwargs) - if hasattr(mod, "server_now") and mod not in patched_modules: + mod_name = getattr(mod, "__name__", name) + if hasattr(mod, "server_now") and mod_name not in original_server_nows: warnings.warn( f"Module {name} imported after server_now was frozen; patching it now." ) - try: - setattr(mod, "server_now", lambda: value) - patched_modules.add(name) - except Exception: - pass + _patch_server_now_in_module(mod, mod_name, value, original_server_nows) return mod builtins.__import__ = import_hook @@ -2110,5 +2115,15 @@ def import_hook(name, *args, **kwargs): yield _freeze - # cleanup: restore the original import function - builtins.__import__ = builtins.__import__ + # Cleanup: restore the original import function and unfreeze server_now in all + # patched modules. Without this, the frozen clock leaks into every test that runs + # afterwards in the same process (e.g. scheduling jobs then reuse the exact same + # belief_time, causing unique-key violations on saving beliefs). + builtins.__import__ = original_import + for module_name, original_server_now in original_server_nows.items(): + module = sys.modules.get(module_name) + if module is not None: + try: + setattr(module, "server_now", original_server_now) + except Exception: + pass diff --git a/flexmeasures/data/models/planning/storage.py b/flexmeasures/data/models/planning/storage.py index 344654bd57..d971287ade 100644 --- a/flexmeasures/data/models/planning/storage.py +++ b/flexmeasures/data/models/planning/storage.py @@ -143,43 +143,72 @@ def _prepare(self, skip_validation: bool = False) -> tuple: # noqa: C901 # Identify group entries: entries carrying intermediate power constraints on a # group of devices (e.g. a sub-EMS). A group entry is a flex-model entry whose - # own `sensor` matches the sensor referenced by another entry's `group` field. - def _group_sensor_id(fm: dict) -> int | None: + # own `sensor` matches the sensor referenced by another entry's `group` field, or + # whose own `asset` matches the asset referenced by another entry's `group` + # field (in which case the entry defines no power sensor of its own). + def _ref_id(value) -> int | None: + if value is None: + return None + return value.id if hasattr(value, "id") else value + + def _group_key(fm: dict) -> tuple[str, int] | None: + """Return a normalized ('sensor', id) or ('asset', id) key for the group + a flex-model entry's `group` field references, or None if it has none.""" group = fm.get("group") if not group: return None - group_sensor = group.get("sensor") if isinstance(group, dict) else group - if group_sensor is None: - return None - return group_sensor.id if isinstance(group_sensor, Sensor) else group_sensor + if isinstance(group, dict): + group_sensor_id = _ref_id(group.get("sensor")) + group_asset_id = _ref_id(group.get("asset")) + else: + # backwards-compat: a raw sensor id/object + group_sensor_id = _ref_id(group) + group_asset_id = None + if group_sensor_id is not None: + return ("sensor", group_sensor_id) + if group_asset_id is not None: + return ("asset", group_asset_id) + return None - group_sensor_ids: set[int] = set() + def _group_key_label(gkey: tuple[str, int]) -> str: + kind, gid = gkey + return f"{kind} {gid}" + + group_keys: set[tuple[str, int]] = set() for fm in flex_model: - gid = _group_sensor_id(fm) - if gid is not None: - group_sensor_ids.add(gid) + gkey = _group_key(fm) + if gkey is not None: + group_keys.add(gkey) # Identify stock models: entries not defining a power sensor, but only a (state-of-charge) sensor self.stock_models = {} device_models = [] # everything except stock models and group entries stock_models = {} # stock models only - group_models: dict[int, dict] = {} # group sensor id -> group entry + group_models: dict[tuple[str, int], dict] = {} # group key -> group entry missing_soc_sensor_i = -len(flex_model) for fm in flex_model: - # group entry: this entry's own sensor is the aggregate sensor referenced by - # another entry's `group` field. Group entries are not schedulable devices; - # they carry constraints on the summed power of their member devices. - fm_sensor = fm.get("sensor") - if fm_sensor is not None: - fm_sensor_id = ( - fm_sensor.id if isinstance(fm_sensor, Sensor) else fm_sensor - ) - if fm_sensor_id in group_sensor_ids: - group_models[fm_sensor_id] = fm - continue + # group entry: this entry's own sensor/asset is the aggregate sensor/asset + # referenced by another entry's `group` field. Group entries are not + # schedulable devices; they carry constraints on the summed power of their + # member devices. + fm_sensor_id = _ref_id(fm.get("sensor")) + if fm_sensor_id is not None and ("sensor", fm_sensor_id) in group_keys: + group_models[("sensor", fm_sensor_id)] = fm + continue + fm_asset_id = _ref_id(fm.get("asset")) + if fm_asset_id is not None and ("asset", fm_asset_id) in group_keys: + if fm_sensor_id is not None: + raise ValueError( + f"Group entry for asset {fm_asset_id} is referenced by " + "asset, but also carries a 'sensor' field; an asset-" + "referenced group entry must not define its own power " + "sensor." + ) + group_models[("asset", fm_asset_id)] = fm + continue # stock model: entry in the flex-model list where the sensor key is the state-of-charge sensor of the device (e.g. a stock) # Only apply this detection in multi-device mode; in single-sensor mode the power sensor is self.sensor (not in the fm dict) @@ -273,74 +302,75 @@ def _group_sensor_id(fm: dict) -> int | None: "charging_efficiency", "discharging_efficiency", "roundtrip_efficiency", - "consumption", - "production", ) self._group_models = group_models - self._group_to_devices: dict[int, list[int]] = {} + self._group_to_devices: dict[tuple[str, int], list[int]] = {} def _resolve_group_leaf_devices( - gid: int, path: tuple[int, ...] = () + gkey: tuple[str, int], path: tuple[tuple[str, int], ...] = () ) -> list[int]: - if gid in path: + if gkey in path: raise ValueError( - f"Cyclic 'group' reference detected involving group sensor {gid}." + f"Cyclic 'group' reference detected involving group " + f"{_group_key_label(gkey)}." ) - if gid in self._group_to_devices: - return self._group_to_devices[gid] - group_entry = group_models.get(gid) + if gkey in self._group_to_devices: + return self._group_to_devices[gkey] + group_entry = group_models.get(gkey) if group_entry is None: raise ValueError( - f"The 'group' field references sensor {gid}, but no flex-model " - f"entry was found for that sensor. Add a flex-model entry for " - f"the group sensor {gid}." + f"The 'group' field references {_group_key_label(gkey)}, but no " + f"flex-model entry was found for it. Add a flex-model entry for " + f"the group {_group_key_label(gkey)}." ) - direct_member_gid = _group_sensor_id(group_entry) leaves: list[int] = [] seen: set[int] = set() for d, fm in enumerate(device_models): - member_gid = _group_sensor_id(fm) - if member_gid == gid: + member_gkey = _group_key(fm) + if member_gkey == gkey: if d not in seen: leaves.append(d) seen.add(d) # Also resolve members that are themselves groups pointing at this group - for other_gid, other_entry in group_models.items(): - if other_gid == gid: + for other_gkey, other_entry in group_models.items(): + if other_gkey == gkey: continue - if _group_sensor_id(other_entry) == gid: - for leaf in _resolve_group_leaf_devices(other_gid, path + (gid,)): + if _group_key(other_entry) == gkey: + for leaf in _resolve_group_leaf_devices(other_gkey, path + (gkey,)): if leaf not in seen: leaves.append(leaf) seen.add(leaf) - if direct_member_gid is not None: - # This group entry is itself a member of another group; that is - # resolved from the other side (the outer group's own resolution), - # so nothing more to do here. - pass - self._group_to_devices[gid] = leaves + self._group_to_devices[gkey] = leaves return leaves if not skip_validation: - dangling = group_sensor_ids - set(group_models.keys()) + dangling = group_keys - set(group_models.keys()) if dangling: raise ValueError( - "The 'group' field references sensor(s) " - f"{sorted(dangling)}, but no flex-model entry was found for " - "the group sensor(s). Add a flex-model entry for the group " - "sensor(s), carrying the group's power-capacity, " + "The 'group' field references " + f"{sorted(_group_key_label(g) for g in dangling)}, but no " + "flex-model entry was found for it. Add a flex-model entry for " + "the group, carrying the group's power-capacity, " "consumption-capacity and/or production-capacity." ) - for gid, group_entry in group_models.items(): + for gkey, group_entry in group_models.items(): offending = [ field for field in device_only_fields if group_entry.get(field) ] if offending: raise ValueError( - f"Group entry for sensor {gid} carries device-only field(s) " - f"{offending}, which is not allowed: group entries only " - "describe constraints on the group's aggregate power." + f"Group entry for {_group_key_label(gkey)} carries " + f"device-only field(s) {offending}, which is not allowed: " + "group entries only describe constraints on the group's " + "aggregate power." + ) + if gkey[0] == "asset" and group_entry.get("sensor") is not None: + raise ValueError( + f"Group entry for {_group_key_label(gkey)} is referenced by " + "asset, but also carries a 'sensor' field; an asset-" + "referenced group entry must not define its own power " + "sensor." ) if not any( group_entry.get(field) is not None @@ -351,18 +381,18 @@ def _resolve_group_leaf_devices( ) ): raise ValueError( - f"Group entry for sensor {gid} defines none of " + f"Group entry for {_group_key_label(gkey)} defines none of " "'power-capacity', 'consumption-capacity' or " "'production-capacity'; such an entry has no effect." ) - for gid in list(group_models.keys()): - leaves = _resolve_group_leaf_devices(gid) + for gkey in list(group_models.keys()): + leaves = _resolve_group_leaf_devices(gkey) if not leaves: if not skip_validation: raise ValueError( - f"The 'group' field references sensor {gid}, but no device " - f"in the flex-model belongs to that group." + f"The 'group' field references {_group_key_label(gkey)}, " + "but no device in the flex-model belongs to that group." ) if not skip_validation and leaves: commodities = { @@ -370,8 +400,8 @@ def _resolve_group_leaf_devices( } if len(commodities) > 1: raise ValueError( - f"All member devices of group {gid} must share the same " - f"commodity; found {sorted(commodities)}." + f"All member devices of group {_group_key_label(gkey)} must " + f"share the same commodity; found {sorted(commodities)}." ) # Rebuild stock_groups using only device_models (which have sensors) @@ -880,10 +910,11 @@ def device_list_series( default_group_breach_price = ur.Quantity( f"10000 {self.flex_context['shared_currency_unit']}/kW" ) - for group_sensor_id, leaf_members in self._group_to_devices.items(): + for group_key, leaf_members in self._group_to_devices.items(): if not leaf_members: continue - group_entry = self._group_models[group_sensor_id] + group_entry = self._group_models[group_key] + group_label = f"{group_key[0]}:{group_key[1]}" group_commodity = device_models[leaf_members[0]].get( "commodity", "electricity" ) @@ -955,24 +986,24 @@ def device_list_series( ) commitments.append( FlowCommitment( - name=f"group {group_sensor_id} any consumption breach", + name=f"group {group_label} any consumption breach", quantity=group_consumption_capacity, upwards_deviation_price=any_group_consumption_breach_price, _type="any", index=index, device=group_devices, - device_group=f"group:{group_sensor_id}", + device_group=f"group:{group_label}", commodity=group_commodity, ) ) commitments.append( FlowCommitment( - name=f"group {group_sensor_id} all consumption breaches", + name=f"group {group_label} all consumption breaches", quantity=group_consumption_capacity, upwards_deviation_price=all_group_consumption_breach_price, index=index, device=group_devices, - device_group=f"group:{group_sensor_id}", + device_group=f"group:{group_label}", commodity=group_commodity, ) ) @@ -1000,24 +1031,24 @@ def device_list_series( ) commitments.append( FlowCommitment( - name=f"group {group_sensor_id} any production breach", + name=f"group {group_label} any production breach", quantity=group_production_capacity, downwards_deviation_price=-any_group_production_breach_price, _type="any", index=index, device=group_devices, - device_group=f"group:{group_sensor_id}", + device_group=f"group:{group_label}", commodity=group_commodity, ) ) commitments.append( FlowCommitment( - name=f"group {group_sensor_id} all production breaches", + name=f"group {group_label} all production breaches", quantity=group_production_capacity, downwards_deviation_price=-all_group_production_breach_price, index=index, device=group_devices, - device_group=f"group:{group_sensor_id}", + device_group=f"group:{group_label}", commodity=group_commodity, ) ) @@ -2714,55 +2745,86 @@ def _build_consumption_production_schedules( """ schedules: dict = {} for d, flex_model_d in enumerate(flex_model): - consumption_field = flex_model_d.get("consumption") - production_field = flex_model_d.get("production") - consumption_sensor = ( - consumption_field["sensor"] - if isinstance(consumption_field, dict) and "sensor" in consumption_field - else None - ) - production_sensor = ( - production_field["sensor"] - if isinstance(production_field, dict) and "sensor" in production_field - else None - ) - if consumption_sensor is None and production_sensor is None: - continue power_series = ems_schedule[d] # in MW; consumption is positive - if consumption_sensor is not None and production_sensor is None: - # Full power profile on the consumption sensor (consumption positive, production negative). - schedules[consumption_sensor] = convert_units( - power_series, - "MW", - consumption_sensor.unit, - event_resolution=consumption_sensor.event_resolution, - ) - elif production_sensor is not None and consumption_sensor is None: - # Full power profile on the production sensor in native scheduler convention. - # make_schedule inverts the sign via consumption_is_positive=False on the sensor. - schedules[production_sensor] = convert_units( - power_series, - "MW", - production_sensor.unit, - event_resolution=production_sensor.event_resolution, - ) - else: - # Both sensors defined: clip to non-negative (consumption) and non-positive (production) parts. - # make_schedule inverts the sign for the production sensor via consumption_is_positive=False. - schedules[consumption_sensor] = convert_units( - power_series.clip(lower=0), - "MW", - consumption_sensor.unit, - event_resolution=consumption_sensor.event_resolution, - ) - schedules[production_sensor] = convert_units( - power_series.clip(upper=0), - "MW", - production_sensor.unit, - event_resolution=production_sensor.event_resolution, - ) + StorageScheduler._split_schedule_over_output_sensors( + flex_model_d, power_series, schedules + ) return schedules + @staticmethod + def _split_schedule_over_output_sensors( + flex_model_d: dict, + power_series: pd.Series, + schedules: dict, + ) -> None: + """Save a power schedule (in MW, consumption positive) to the output sensor(s) + (``consumption``/``production``) defined on a single flex-model entry, in-place + on ``schedules`` (mapping output sensor -> power schedule). + + Follows the same conventions as :func:`_build_consumption_production_schedules`: + + - **Only** ``consumption`` **sensor defined**: the full power schedule is written to that + sensor using the scheduler's native sign convention (consumption positive, production + negative). ``make_schedule`` applies no further sign change because the sensor already + has ``consumption_is_positive=True``. + - **Only** ``production`` **sensor defined**: the full power schedule is written to that + sensor in the scheduler's native sign convention (consumption positive, production + negative). ``make_schedule`` inverts the sign based on the sensor's + ``consumption_is_positive=False`` attribute so that production is stored as positive values. + - **Both** ``consumption`` **and** ``production`` **sensors defined**: only the non-negative + part of the schedule (charging / consuming) is written to the consumption sensor, and only + the non-positive part (discharging / producing, still as negative values) is written to + the production sensor. ``make_schedule`` inverts the sign for the production sensor. + + Unit conversion from MW to each sensor's unit is applied. + """ + consumption_field = flex_model_d.get("consumption") + production_field = flex_model_d.get("production") + consumption_sensor = ( + consumption_field["sensor"] + if isinstance(consumption_field, dict) and "sensor" in consumption_field + else None + ) + production_sensor = ( + production_field["sensor"] + if isinstance(production_field, dict) and "sensor" in production_field + else None + ) + if consumption_sensor is None and production_sensor is None: + return + if consumption_sensor is not None and production_sensor is None: + # Full power profile on the consumption sensor (consumption positive, production negative). + schedules[consumption_sensor] = convert_units( + power_series, + "MW", + consumption_sensor.unit, + event_resolution=consumption_sensor.event_resolution, + ) + elif production_sensor is not None and consumption_sensor is None: + # Full power profile on the production sensor in native scheduler convention. + # make_schedule inverts the sign via consumption_is_positive=False on the sensor. + schedules[production_sensor] = convert_units( + power_series, + "MW", + production_sensor.unit, + event_resolution=production_sensor.event_resolution, + ) + else: + # Both sensors defined: clip to non-negative (consumption) and non-positive (production) parts. + # make_schedule inverts the sign for the production sensor via consumption_is_positive=False. + schedules[consumption_sensor] = convert_units( + power_series.clip(lower=0), + "MW", + consumption_sensor.unit, + event_resolution=consumption_sensor.event_resolution, + ) + schedules[production_sensor] = convert_units( + power_series.clip(upper=0), + "MW", + production_sensor.unit, + event_resolution=production_sensor.event_resolution, + ) + def _reconstruct_commodity_to_devices(self) -> dict[str, list[int]]: """Reconstruct the mapping of commodity -> device indices as enumerated by `_prepare()`. @@ -2928,29 +2990,70 @@ def _compute_commodity_aggregate_schedules( commodity_aggregate.clip(upper=0) ) + @staticmethod + def _merge_group_output_schedules( + consumption_production_schedule: dict, group_output_schedules: dict + ) -> None: + """Merge group output schedules (already unit-converted) into + ``consumption_production_schedule`` in-place, avoiding overwrite of a device's + own output sensor with a group's.""" + for out_sensor, out_schedule in group_output_schedules.items(): + if out_sensor in consumption_production_schedule: + raise ValueError( + f"Sensor {out_sensor.id} is used as an output sensor both by a " + "device and by a group; a sensor cannot be both." + ) + consumption_production_schedule[out_sensor] = out_schedule + def _add_group_schedules( self, storage_schedule: dict, ems_schedule: list[pd.Series] - ) -> None: - """Save each group's aggregate power schedule under the group's own sensor, - summed over its leaf member devices (in-place on storage_schedule). + ) -> dict: + """Save each group's aggregate power schedule. + + - Sensor-referenced groups: the aggregate is saved under the group's own power + sensor (as before), added in-place to ``storage_schedule`` (still in MW, + native scheduler convention; unit conversion happens later, alongside other + device sensors). + - Asset-referenced groups: the group entry defines no power sensor of its own; + instead, the aggregate is saved via the group entry's own ``consumption``/ + ``production`` output sensors, if defined. + - Either kind of group may additionally define ``consumption``/``production`` + output sensors, in which case the aggregate is also split/saved there. + + The ``consumption``/``production`` output schedules are returned separately + (already unit-converted, like ``_build_consumption_production_schedules``) + rather than added to ``storage_schedule``, to avoid double unit conversion. + + :returns: Dict mapping each group output sensor to its power schedule. """ group_models = getattr(self, "_group_models", None) or {} group_to_devices = getattr(self, "_group_to_devices", None) or {} - for group_sensor_id, leaf_members in group_to_devices.items(): - group_entry = group_models.get(group_sensor_id) - group_sensor = group_entry.get("sensor") if group_entry else None - if group_sensor is None or not leaf_members: + group_output_schedules: dict = {} + for group_key, leaf_members in group_to_devices.items(): + group_entry = group_models.get(group_key) + if group_entry is None or not leaf_members: continue - if group_sensor in storage_schedule: - raise ValueError( - f"Sensor {group_sensor.id} is used both as a device sensor and " - "as a group sensor; a sensor cannot be both." - ) - storage_schedule[group_sensor] = pd.concat( + group_aggregate = pd.concat( [ems_schedule[d] for d in leaf_members], axis=1, ).sum(axis=1) + group_sensor = group_entry.get("sensor") + if group_sensor is not None: + if group_sensor in storage_schedule: + raise ValueError( + f"Sensor {group_sensor.id} is used both as a device sensor " + "and as a group sensor; a sensor cannot be both." + ) + storage_schedule[group_sensor] = group_aggregate + + # For both sensor-ref and asset-ref groups, also honor any consumption/ + # production output sensors defined on the group entry itself. + self._split_schedule_over_output_sensors( + group_entry, group_aggregate, group_output_schedules + ) + return group_output_schedules + def compute(self, skip_validation: bool = False) -> SchedulerOutputType: """Schedule a battery or Charge Point based directly on the latest beliefs regarding market prices within the specified time window. For the resulting consumption schedule, consumption is defined as positive values. @@ -3001,7 +3104,9 @@ def compute(self, skip_validation: bool = False) -> SchedulerOutputType: elif sensor is not None and sensor in storage_schedule: storage_schedule[sensor] += ems_schedule[d] - self._add_group_schedules(storage_schedule, ems_schedule) + group_output_schedules = self._add_group_schedules( + storage_schedule, ems_schedule + ) # Obtain the aggregate power schedule, too, if the flex-context states the associated sensor. Fill with the sum of schedules made here. # Restricted to electricity devices (flexible and inflexible), per decision. @@ -3053,6 +3158,9 @@ def compute(self, skip_validation: bool = False) -> SchedulerOutputType: consumption_production_schedule = self._build_consumption_production_schedules( flex_model_for_soc, ems_schedule ) + self._merge_group_output_schedules( + consumption_production_schedule, group_output_schedules + ) # Resample each device schedule to the resolution of the device's power sensor if self.resolution is None: @@ -3130,9 +3238,11 @@ def compute(self, skip_validation: bool = False) -> SchedulerOutputType: for sensor, soc in soc_schedule.items() ] # Determine which sensors are consumption vs. production output sensors + group_models_for_output = getattr(self, "_group_models", None) or {} consumption_output_sensors = { flex_model_d["consumption"]["sensor"] - for flex_model_d in flex_model_for_soc + for flex_model_d in list(flex_model_for_soc) + + list(group_models_for_output.values()) if isinstance(flex_model_d.get("consumption"), dict) and "sensor" in flex_model_d["consumption"] } diff --git a/flexmeasures/data/models/planning/tests/test_group_constraints.py b/flexmeasures/data/models/planning/tests/test_group_constraints.py index ff35fe7a0a..4a08ee0342 100644 --- a/flexmeasures/data/models/planning/tests/test_group_constraints.py +++ b/flexmeasures/data/models/planning/tests/test_group_constraints.py @@ -4,6 +4,7 @@ import pytest import pandas as pd +from flexmeasures.data.models.generic_assets import GenericAsset, GenericAssetType from flexmeasures.data.models.time_series import Sensor from flexmeasures.data.models.planning.storage import StorageScheduler from flexmeasures.utils.unit_utils import ur @@ -40,6 +41,33 @@ def make_group_sensor(db, building, unit="MW"): return s +def make_sub_asset(db, building): + """Create a child asset (e.g. an inverter/sub-EMS) under `building`.""" + asset_type = GenericAssetType(name=_unique_name("group test asset type")) + db.session.add(asset_type) + asset = GenericAsset( + name=_unique_name("group test sub-asset"), + generic_asset_type=asset_type, + parent_asset=building, + owner=building.owner, + ) + db.session.add(asset) + db.session.commit() + return asset + + +def make_output_sensor(db, asset, unit="MW"): + s = Sensor( + name=_unique_name("output sensor"), + generic_asset=asset, + event_resolution=timedelta(hours=1), + unit=unit, + ) + db.session.add(s) + db.session.commit() + return s + + def base_flex_context(): return { "consumption_price": ur.Quantity("100 EUR/MWh"), @@ -405,3 +433,357 @@ def test_group_cycle_raises(db, building): scheduler = run_scheduler(building, flex_model, base_flex_context()) with pytest.raises(ValueError, match="Cyclic"): scheduler.compute() + + +def test_group_asset_ref_hard_cap(db, building): + """An asset-referenced group entry (no power sensor of its own) still caps the + aggregate power of its members, and saves the aggregate on its consumption output + sensor (consumption-only case: full profile, consumption positive).""" + battery, pv = make_sensors(db, building, n=2) + inverter = make_sub_asset(db, building) + consumption_sensor = make_output_sensor(db, inverter) + + flex_model = [ + { + "sensor": battery, + "soc_at_start": 1.0, + "soc_min": 0.0, + "soc_max": 2.0, + "power_capacity_in_mw": ur.Quantity("2 MW"), + "consumption_capacity": ur.Quantity("2 MW"), + "production_capacity": ur.Quantity("2 MW"), + "group": {"asset": inverter}, + }, + { + "sensor": pv, + "power_capacity_in_mw": ur.Quantity("2 MW"), + "consumption_capacity": ur.Quantity("0 MW"), + "production_capacity": ur.Quantity("2 MW"), + "group": {"asset": inverter}, + }, + { + "asset": inverter, + "power_capacity_in_mw": ur.Quantity("2.5 MW"), + "consumption": {"sensor": consumption_sensor}, + }, + ] + scheduler = run_scheduler( + building, flex_model, base_flex_context(), return_multiple=True + ) + results = scheduler.compute() + + storage_schedules = { + r["sensor"]: r["data"] for r in results if r.get("name") == "storage_schedule" + } + assert battery in storage_schedules + assert pv in storage_schedules + # The group entry has no power sensor of its own, so it isn't in storage_schedules. + assert not any( + getattr(sensor, "asset", None) == inverter for sensor in storage_schedules + ) + + consumption_schedules = { + r["sensor"]: r["data"] + for r in results + if r.get("name") == "consumption_schedule" + } + assert consumption_sensor in consumption_schedules + aggregate = consumption_schedules[consumption_sensor] + assert (aggregate.abs() <= 2.5 + 1e-6).all() + assert ( + (storage_schedules[battery] + storage_schedules[pv] - aggregate).abs() < 1e-6 + ).all() + + +def test_group_asset_ref_production_only_output(db, building): + """An asset-referenced group entry with only a production output sensor gets the + full aggregate profile in native (consumption-positive) convention; sign inversion + to production-positive happens downstream in make_schedule.""" + battery, pv = make_sensors(db, building, n=2) + inverter = make_sub_asset(db, building) + production_sensor = make_output_sensor(db, inverter) + + flex_model = [ + { + "sensor": battery, + "soc_at_start": 1.0, + "soc_min": 0.0, + "soc_max": 2.0, + "power_capacity_in_mw": ur.Quantity("2 MW"), + "group": {"asset": inverter}, + }, + { + "sensor": pv, + "power_capacity_in_mw": ur.Quantity("2 MW"), + "consumption_capacity": ur.Quantity("0 MW"), + "group": {"asset": inverter}, + }, + { + "asset": inverter, + "power_capacity_in_mw": ur.Quantity("2.5 MW"), + "production": {"sensor": production_sensor}, + }, + ] + scheduler = run_scheduler( + building, flex_model, base_flex_context(), return_multiple=True + ) + results = scheduler.compute() + + storage_schedules = { + r["sensor"]: r["data"] for r in results if r.get("name") == "storage_schedule" + } + production_schedules = { + r["sensor"]: r["data"] + for r in results + if r.get("name") == "production_schedule" + } + assert production_sensor in production_schedules + expected = storage_schedules[battery] + storage_schedules[pv] + assert ((production_schedules[production_sensor] - expected).abs() < 1e-6).all() + + +def test_group_asset_ref_both_outputs_split(db, building): + """An asset-referenced group entry with both consumption and production output + sensors gets the clip-split of the aggregate: non-negative to consumption, + non-positive to production.""" + battery, pv = make_sensors(db, building, n=2) + inverter = make_sub_asset(db, building) + consumption_sensor = make_output_sensor(db, inverter) + production_sensor = make_output_sensor(db, inverter) + + flex_model = [ + { + "sensor": battery, + "soc_at_start": 1.0, + "soc_min": 0.0, + "soc_max": 2.0, + "power_capacity_in_mw": ur.Quantity("2 MW"), + "group": {"asset": inverter}, + }, + { + "sensor": pv, + "power_capacity_in_mw": ur.Quantity("2 MW"), + "consumption_capacity": ur.Quantity("0 MW"), + "group": {"asset": inverter}, + }, + { + "asset": inverter, + "power_capacity_in_mw": ur.Quantity("2.5 MW"), + "consumption": {"sensor": consumption_sensor}, + "production": {"sensor": production_sensor}, + }, + ] + scheduler = run_scheduler( + building, flex_model, base_flex_context(), return_multiple=True + ) + results = scheduler.compute() + + storage_schedules = { + r["sensor"]: r["data"] for r in results if r.get("name") == "storage_schedule" + } + consumption_schedules = { + r["sensor"]: r["data"] + for r in results + if r.get("name") == "consumption_schedule" + } + production_schedules = { + r["sensor"]: r["data"] + for r in results + if r.get("name") == "production_schedule" + } + aggregate = storage_schedules[battery] + storage_schedules[pv] + assert ( + (consumption_schedules[consumption_sensor] - aggregate.clip(lower=0)).abs() + < 1e-6 + ).all() + assert ( + (production_schedules[production_sensor] - aggregate.clip(upper=0)).abs() < 1e-6 + ).all() + # Consistency: consumption plus production reconstructs the full aggregate. + assert ( + ( + consumption_schedules[consumption_sensor] + + production_schedules[production_sensor] + - aggregate + ).abs() + < 1e-6 + ).all() + + +def test_group_asset_ref_dangling_raises(db, building): + battery = make_sensors(db, building, n=1)[0] + other_asset = make_sub_asset(db, building) + + flex_model = [ + { + "sensor": battery, + "power_capacity_in_mw": ur.Quantity("2 MW"), + "group": {"asset": other_asset}, + }, + ] + scheduler = run_scheduler(building, flex_model, base_flex_context()) + with pytest.raises(ValueError, match="group"): + scheduler.compute() + + +def test_group_asset_ref_with_sensor_raises(db, building): + """An asset-referenced group entry must not also carry a `sensor` field.""" + battery, pv = make_sensors(db, building, n=2) + inverter = make_sub_asset(db, building) + bogus_sensor = make_group_sensor(db, building) + + flex_model = [ + { + "sensor": battery, + "power_capacity_in_mw": ur.Quantity("2 MW"), + "group": {"asset": inverter}, + }, + { + "sensor": pv, + "power_capacity_in_mw": ur.Quantity("2 MW"), + "group": {"asset": inverter}, + }, + { + "asset": inverter, + "sensor": bogus_sensor, + "power_capacity_in_mw": ur.Quantity("2.5 MW"), + }, + ] + scheduler = run_scheduler(building, flex_model, base_flex_context()) + with pytest.raises(ValueError, match="asset-"): + scheduler.compute() + + +def test_nested_group_mixed_ref_kinds(db, building): + """Inner group referenced by asset, outer group referenced by sensor: leaf + resolution must work transitively across both kinds.""" + battery = make_sensors(db, building, n=1)[0] + inverter = make_sub_asset(db, building) + outer_group_sensor = make_group_sensor(db, building) + + flex_model = [ + { + "sensor": battery, + "soc_at_start": 1.0, + "soc_min": 0.0, + "soc_max": 2.0, + "power_capacity_in_mw": ur.Quantity("2 MW"), + "group": {"asset": inverter}, + }, + { + "asset": inverter, + "power_capacity_in_mw": ur.Quantity("3 MW"), + "group": {"sensor": outer_group_sensor}, + }, + { + "sensor": outer_group_sensor, + "power_capacity_in_mw": ur.Quantity("3.5 MW"), + }, + ] + scheduler = run_scheduler( + building, flex_model, base_flex_context(), return_multiple=True + ) + results = scheduler.compute() + storage_schedules = { + r["sensor"]: r["data"] for r in results if r.get("name") == "storage_schedule" + } + assert outer_group_sensor in storage_schedules + assert ( + (storage_schedules[outer_group_sensor] - storage_schedules[battery]).abs() + < 1e-6 + ).all() + + +def test_pure_db_tree_group_constraint(db, building): + """End-to-end (planning-level): the entire flex-model lives on the asset tree in + the DB (no flex-model entries are passed to the scheduler at all). A site asset + (``building``) has two child assets (battery-like and PV-like) that are both + asset-only device entries (no power sensor of their own; results are saved via + consumption/production output sensors) belonging to a group referenced by a third + child asset (an "inverter"), which itself defines the group's hard power-capacity + and saves the group's aggregate via its own consumption output sensor. + + Triggering the site asset with an empty flex-model list should still produce a + correctly constrained schedule, entirely from `GenericAsset.flex_model` attributes, + via `Scheduler.collect_flex_config`. + """ + inverter = make_sub_asset(db, building) + battery_asset = make_sub_asset(db, building) + pv_asset = make_sub_asset(db, building) + + inverter_consumption_sensor = make_output_sensor(db, inverter) + battery_consumption_sensor = make_output_sensor(db, battery_asset) + battery_production_sensor = make_output_sensor(db, battery_asset) + pv_production_sensor = make_output_sensor(db, pv_asset) + + # Store the flex-model entirely on the assets in the DB (asset-only entries: no + # "sensor" key, so results are saved via consumption/production output sensors). + battery_asset.flex_model = { + "power-capacity": "2 MW", + "consumption-capacity": "2 MW", + "production-capacity": "2 MW", + "group": {"asset": inverter.id}, + "consumption": {"sensor": battery_consumption_sensor.id}, + "production": {"sensor": battery_production_sensor.id}, + } + pv_asset.flex_model = { + "power-capacity": "2 MW", + "consumption-capacity": "0 MW", + "production-capacity": "2 MW", + "group": {"asset": inverter.id}, + "production": {"sensor": pv_production_sensor.id}, + } + inverter.flex_model = { + "power-capacity": "2.5 MW", + "consumption": {"sensor": inverter_consumption_sensor.id}, + } + db.session.add_all([battery_asset, pv_asset, inverter]) + db.session.commit() + + scheduler = StorageScheduler( + asset_or_sensor=building, + start=pd.Timestamp("2023-01-01T00:00:00", tz="Europe/Amsterdam"), + end=pd.Timestamp("2023-01-01T04:00:00", tz="Europe/Amsterdam"), + resolution=timedelta(hours=1), + flex_model=[], # entirely DB-driven + # `building`'s own flex-context (in the DB) already sets a large + # site-power-capacity; override the (unpopulated) sensor-based + # consumption-price with fixed quantities here. Real deserialization + # (collect_flex_config + schema loading) is exercised, unlike in the other + # tests in this file (which bypass it). + flex_context={ + "consumption-price": "100 EUR/MWh", + "production-price": "100 EUR/MWh", + }, + return_multiple=True, + ) + results = scheduler.compute() + + consumption_schedules = { + r["sensor"]: r["data"] + for r in results + if r.get("name") == "consumption_schedule" + } + production_schedules = { + r["sensor"]: r["data"] + for r in results + if r.get("name") == "production_schedule" + } + + assert inverter_consumption_sensor in consumption_schedules + aggregate = consumption_schedules[inverter_consumption_sensor] + # Hard cap on the group's aggregate power respected. + assert (aggregate.abs() <= 2.5 + 1e-6).all() + + # The aggregate equals the sum of the (signed, consumption-positive) member + # device schedules, reconstructed from their consumption/production outputs. + battery_signed = consumption_schedules.get( + battery_consumption_sensor, + pd.Series(0.0, index=aggregate.index), + ) + production_schedules.get( + battery_production_sensor, pd.Series(0.0, index=aggregate.index) + ) + pv_signed = production_schedules.get( + pv_production_sensor, pd.Series(0.0, index=aggregate.index) + ) + assert ((battery_signed + pv_signed - aggregate).abs() < 1e-6).all() diff --git a/flexmeasures/data/schemas/scheduling/__init__.py b/flexmeasures/data/schemas/scheduling/__init__.py index 1062e7568d..da91718107 100644 --- a/flexmeasures/data/schemas/scheduling/__init__.py +++ b/flexmeasures/data/schemas/scheduling/__init__.py @@ -948,7 +948,7 @@ def check_prices(self, data: dict, original_data: dict, **kwargs): "description": rst_to_openapi(metadata.GROUP.description), "types": { "backend": "typeTwo", - "ui": "A power sensor representing a group of devices; also records the group's scheduled aggregate power.", + "ui": "A power sensor or an asset representing a group of devices; a sensor-referenced group also records the group's scheduled aggregate power, while an asset-referenced group records it via its own consumption/production output sensors.", }, "example-units": EXAMPLE_UNIT_TYPES["power"], }, @@ -1265,7 +1265,11 @@ def check_flex_model_sensors(self, data, **kwargs): asset = data["asset"] sensors = [] for sensor_flex_model in data.get("flex_model", []): - sensor = sensor_flex_model["sensor"] + sensor = sensor_flex_model.get("sensor") + if sensor is None: + # Asset-only entries (e.g. an asset-referenced `group` entry) carry no + # sensor of their own; nothing to check here. + continue if sensor in sensors: raise FMValidationError( f"Sensor {sensor_flex_model['sensor'].id} should not occur more than once in the flex-model" diff --git a/flexmeasures/data/schemas/scheduling/metadata.py b/flexmeasures/data/schemas/scheduling/metadata.py index 55a5c79177..c6340194a0 100644 --- a/flexmeasures/data/schemas/scheduling/metadata.py +++ b/flexmeasures/data/schemas/scheduling/metadata.py @@ -393,9 +393,10 @@ def to_dict(self): example="0 kW", ) GROUP = MetaData( - description="""Reference to a power sensor (``{"sensor": }``) representing a group of devices whose aggregate power is constrained. -The group sensor itself should get its own flex-model entry defining the group's ``power-capacity`` (hard constraint) and/or ``consumption-capacity``/``production-capacity`` (soft constraints with default breach prices). -The group's scheduled aggregate power is saved to the group sensor. + description="""Reference to a group of devices whose aggregate power is constrained, given as either a power sensor (``{"sensor": }``) or an asset (``{"asset": }``) - exactly one of the two. +The referenced sensor or asset should itself get its own flex-model entry defining the group's ``power-capacity`` (hard constraint) and/or ``consumption-capacity``/``production-capacity`` (soft constraints with default breach prices). +When the group is referenced by ``sensor``, the group's scheduled aggregate power is saved to that group sensor. +When the group is referenced by ``asset`` (e.g. a sub-EMS asset in the tree), the group entry defines no power sensor of its own; the group's aggregate power is instead saved via that entry's own ``consumption`` and/or ``production`` output sensors, following the usual output-sensor conventions. """, example={"sensor": 5}, ) diff --git a/flexmeasures/data/schemas/scheduling/storage.py b/flexmeasures/data/schemas/scheduling/storage.py index 2d29adc4b2..0c5df11492 100644 --- a/flexmeasures/data/schemas/scheduling/storage.py +++ b/flexmeasures/data/schemas/scheduling/storage.py @@ -19,6 +19,7 @@ from flexmeasures.data.schemas.units import QuantityField from flexmeasures.data.schemas.scheduling import metadata from flexmeasures.data.schemas.sensors import ( + SensorIdField, SensorReference, SensorReferenceSchema, OutputSensorReferenceSchema, @@ -43,6 +44,33 @@ def _validate_group_sensor_is_power_sensor(group: dict): ) +class GroupReferenceSchema(SensorReferenceSchema): + """Reference to a group of devices whose aggregate power is constrained. + + Accepts exactly one of: + - ``{"sensor": }``: the group's aggregate power is stored on this power sensor + (the sensor must itself carry a flex-model entry defining the group's + constraints). + - ``{"asset": }``: the group is identified by the flex-model entry on this + asset (typically a sub-EMS/asset in the tree). Such a group entry defines no + power sensor of its own; instead it may define ``consumption`` and/or + ``production`` output sensors on which the group's aggregate power gets saved, + following the usual output-sensor conventions. + """ + + sensor = SensorIdField(required=False) + asset = GenericAssetIdField(required=False) + + @validates_schema + def validate_exactly_one_reference(self, data: dict, **kwargs): + has_sensor = "sensor" in data + has_asset = "asset" in data + if has_sensor == has_asset: # both or neither + raise ValidationError( + "The `group` field must reference exactly one of 'sensor' or 'asset'." + ) + + # Telling type hints what to expect after schema parsing SoCTarget = TypedDict( "SoCTarget", @@ -161,7 +189,7 @@ class StorageFlexModelSchema(Schema): ) group = fields.Nested( - SensorReferenceSchema, + GroupReferenceSchema, data_key="group", required=False, metadata=metadata.GROUP.to_dict(), @@ -452,7 +480,7 @@ class DBStorageFlexModelSchema(Schema): production = fields.Nested(OutputSensorReferenceSchema) group = fields.Nested( - SensorReferenceSchema, + GroupReferenceSchema, data_key="group", required=False, metadata=metadata.GROUP.to_dict(), diff --git a/flexmeasures/data/schemas/tests/test_scheduling.py b/flexmeasures/data/schemas/tests/test_scheduling.py index 8027f39a72..88bf86ce9c 100644 --- a/flexmeasures/data/schemas/tests/test_scheduling.py +++ b/flexmeasures/data/schemas/tests/test_scheduling.py @@ -920,7 +920,7 @@ def test_flex_model_schemas( schema.load(flex_model) -def test_storage_flex_model_group_field(db, app, setup_dummy_sensors): +def test_storage_flex_model_group_field(db, app, setup_dummy_sensors, dummy_asset): """The `group` field should load a `{"sensor": }` reference to a power Sensor, reject non-power sensors, and reject unknown sensor IDs.""" energy_sensor, _, _, power_sensor = setup_dummy_sensors @@ -943,6 +943,18 @@ def test_storage_flex_model_group_field(db, app, setup_dummy_sensors): with pytest.raises(ValidationError, match="No sensor found"): schema.load({"group": {"sensor": -1}}) + # A valid asset reference loads to a GenericAsset + flex_model = schema.load({"group": {"asset": dummy_asset.id}}) + assert flex_model["group"]["asset"] == dummy_asset + + # Both sensor and asset given: rejected + with pytest.raises(ValidationError, match="exactly one"): + schema.load({"group": {"sensor": power_sensor.id, "asset": dummy_asset.id}}) + + # Neither sensor nor asset given: rejected + with pytest.raises(ValidationError, match="exactly one"): + schema.load({"group": {}}) + @pytest.mark.parametrize( ["flex_context", "fails"], diff --git a/flexmeasures/ui/static/openapi-specs.json b/flexmeasures/ui/static/openapi-specs.json index a751c5f665..44e712d077 100644 --- a/flexmeasures/ui/static/openapi-specs.json +++ b/flexmeasures/ui/static/openapi-specs.json @@ -7,7 +7,7 @@ }, "termsOfService": null, "title": "FlexMeasures", - "version": "1.0.0" + "version": "0.33.2" }, "externalDocs": { "description": "FlexMeasures runs on the open source FlexMeasures technology. Read the docs here.", @@ -6119,6 +6119,62 @@ ], "additionalProperties": false }, + "GroupReference": { + "type": "object", + "properties": { + "sensor": { + "type": "integer" + }, + "source-types": { + "type": [ + "array", + "null" + ], + "default": null, + "description": "Only use beliefs from sources with these source types (e.g. 'user', 'script', 'forecaster', 'scheduler').", + "items": { + "type": "string" + } + }, + "exclude-source-types": { + "type": [ + "array", + "null" + ], + "default": null, + "description": "Exclude beliefs from sources with these source types.", + "items": { + "type": "string" + } + }, + "sources": { + "type": [ + "array", + "null" + ], + "default": null, + "description": "Only use beliefs from these data source IDs.", + "items": { + "type": "integer" + } + }, + "source-account": { + "type": [ + "array", + "null" + ], + "default": null, + "description": "Only use beliefs from data sources linked to these account IDs.", + "items": { + "type": "integer" + } + }, + "asset": { + "type": "integer" + } + }, + "description": "Sensor reference from which to look up a variable quantity." + }, "StorageFlexModelSchemaOpenAPI": { "type": "object", "properties": { @@ -6179,11 +6235,11 @@ "$ref": "#/components/schemas/VariableQuantityOpenAPI" }, "group": { - "description": "Reference to a power sensor ({\"sensor\": }) representing a group of devices whose aggregate power is constrained.\nThe group sensor itself should get its own flex-model entry defining the group's power-capacity (hard constraint) and/or consumption-capacity/production-capacity (soft constraints with default breach prices).\nThe group's scheduled aggregate power is saved to the group sensor.\n", + "description": "Reference to a group of devices whose aggregate power is constrained, given as either a power sensor ({\"sensor\": }) or an asset ({\"asset\": }) - exactly one of the two.\nThe referenced sensor or asset should itself get its own flex-model entry defining the group's power-capacity (hard constraint) and/or consumption-capacity/production-capacity (soft constraints with default breach prices).\nWhen the group is referenced by sensor, the group's scheduled aggregate power is saved to that group sensor.\nWhen the group is referenced by asset (e.g. a sub-EMS asset in the tree), the group entry defines no power sensor of its own; the group's aggregate power is instead saved via that entry's own consumption and/or production output sensors, following the usual output-sensor conventions.\n", "example": { "sensor": 5 }, - "$ref": "#/components/schemas/SensorReference" + "$ref": "#/components/schemas/GroupReference" }, "prefer-curtailing-later": { "type": "boolean", diff --git a/flexmeasures/ui/templates/assets/asset_properties.html b/flexmeasures/ui/templates/assets/asset_properties.html index 4564e7ef45..d6e1736621 100644 --- a/flexmeasures/ui/templates/assets/asset_properties.html +++ b/flexmeasures/ui/templates/assets/asset_properties.html @@ -381,6 +381,22 @@
Edit {{ asset.name }}'s flex-model
+ {% if parent_has_power_capacity %} + + {% elif own_has_power_capacity and has_child_assets %} + + {% endif %} +
@@ -436,6 +452,15 @@
Edit {{ asset.name }}'s flex-model
// This variable is used to prevent reRenderForm from running on initial load let hasInitialized = false; + // Info used to hint the user towards using the parent asset as the `group` + // for the flex-model's `group` field (and vice versa, hinting the parent + // that its children can join its power-capacity group). + const parentAssetId = {{ parent_asset_id | tojson | safe }}; + const parentAssetName = {{ parent_asset_name | tojson | safe }}; + const parentHasPowerCapacity = {{ parent_has_power_capacity | tojson | safe }}; + const ownHasPowerCapacity = {{ own_has_power_capacity | tojson | safe }}; + const hasChildAssets = {{ has_child_assets | tojson | safe }}; + const schemaSpecs = {{ flex_model_schema | tojson | safe }}; const assetFlexModelSchema = {}; const FlexModelFieldValidTypes = {}; @@ -586,6 +611,27 @@
Edit {{ asset.name }}'s flex-model
senSearchResEle.style.display = 'none'; } + function setCardGroupAsset(assetId) { + if (!assetId) { + return; + } + + const flexModel = getFlexModel(); + flexModel["group"] = { "asset": assetId }; + setFlexModel(flexModel); + + setTimeout(() => { + const card = document.getElementById('group-control'); + if (!card) { + return; + } + setActiveCard(card); + card.classList.add('border-on-click'); + renderSelectInfoCards('group'); + renderFlexInputOptions(); + }, 500); + } + async function searchSensors() { const searchValue = sensorSearchBar.value.toLowerCase(); spinnerElement.style.display = 'flex'; @@ -684,6 +730,37 @@
Edit {{ asset.name }}'s flex-model
` : ""; + // Hint towards using the parent asset as the value for `group`, + // especially when the parent already limits the group's power. + let groupParentHint = ""; + if (modelKey === "group" && parentAssetId) { + const powerCapacityNote = parentHasPowerCapacity + ? `The parent asset ${parentAssetName} defines a power-capacity. Consider joining that group so this device counts toward the parent's limit.` + : `Consider using the parent asset ${parentAssetName} as the group.`; + groupParentHint = ` +
+ ${powerCapacityNote} +
+ +
+
+ `; + } + + // Hint towards the fact that children can reference this asset + // as their `group`, when this asset itself has a power-capacity. + let powerCapacityChildrenHint = ""; + if (modelKey === "power-capacity" && ownHasPowerCapacity && hasChildAssets) { + powerCapacityChildrenHint = ` +
+ + This asset has child assets. They can set group: {"asset": {{ asset.id }} } in their own flex-model's group field, to share this power-capacity limit. +
+ `; + } + flexInfoContainer.innerHTML = `
${storageEfficiencyWarning} + ${groupParentHint} + ${powerCapacityChildrenHint} -
+
Possible types: ${schemaSpecs[modelKey]["types"]["ui"] || "No types available."}
Example units: ${schemaSpecs[modelKey]["example-units"].join(", ")}
`; + + const useParentAsGroupBtn = document.getElementById('useParentAsGroupBtn'); + if (useParentAsGroupBtn) { + useParentAsGroupBtn.onclick = function () { + setCardGroupAsset(parentAssetId); + }; + } } else { flexInfoContainer.innerHTML = ""; } diff --git a/flexmeasures/ui/tests/test_asset_crud.py b/flexmeasures/ui/tests/test_asset_crud.py index 3306f60498..f5ad1df2f9 100644 --- a/flexmeasures/ui/tests/test_asset_crud.py +++ b/flexmeasures/ui/tests/test_asset_crud.py @@ -466,3 +466,75 @@ def test_admin_only_buttons_on_properties_page( assert b"Create asset" in page.data else: assert b"Create asset" not in page.data + + +def test_group_field_hints_on_properties_page( + db, client, as_admin, setup_accounts, setup_generic_asset_types +): + """The properties page should hint at using the parent asset as `group`, + depending on whether the (child's) parent or the asset itself defines a + power-capacity in its flex-model.""" + + # Parent with a power-capacity in its flex-model, and a child without a group set. + parent_with_capacity = GenericAsset( + name="parent-with-power-capacity", + generic_asset_type=setup_generic_asset_types["battery"], + owner=setup_accounts["Prosumer"], + latitude=10, + longitude=100, + flex_model={"power-capacity": "400kW"}, + ) + db.session.add(parent_with_capacity) + db.session.flush() + + child = GenericAsset( + name="child-of-parent-with-power-capacity", + generic_asset_type=setup_generic_asset_types["battery"], + owner=setup_accounts["Prosumer"], + latitude=10, + longitude=100, + parent_asset_id=parent_with_capacity.id, + ) + db.session.add(child) + + # Parent without children with power-capacity and without children (no hints). + lone_asset = GenericAsset( + name="lone-asset-no-hints", + generic_asset_type=setup_generic_asset_types["battery"], + owner=setup_accounts["Prosumer"], + latitude=10, + longitude=100, + ) + db.session.add(lone_asset) + db.session.commit() + + # (a) child whose parent has power-capacity: expect the "join parent group" hint. + child_page = client.get( + url_for("AssetCrudUI:properties", id=child.id), + follow_redirects=True, + ) + assert child_page.status_code == 200 + assert b"parent-with-power-capacity" in child_page.data + assert b"power-capacity" in child_page.data + assert f'group: {{"asset": {parent_with_capacity.id} }}'.encode() in child_page.data + + # (b) parent with power-capacity and children: expect the "children can join" hint. + parent_page = client.get( + url_for("AssetCrudUI:properties", id=parent_with_capacity.id), + follow_redirects=True, + ) + assert parent_page.status_code == 200 + assert b"Child assets can" in parent_page.data + assert ( + f'group: {{"asset": {parent_with_capacity.id} }}'.encode() in parent_page.data + ) + + # (c) an asset with neither a parent with power-capacity, nor children with + # power-capacity of its own: expect no hints. + lone_page = client.get( + url_for("AssetCrudUI:properties", id=lone_asset.id), + follow_redirects=True, + ) + assert lone_page.status_code == 200 + assert b"Consider setting" not in lone_page.data + assert b"Child assets can" not in lone_page.data diff --git a/flexmeasures/ui/views/assets/views.py b/flexmeasures/ui/views/assets/views.py index f580a064e7..e74cdf0ef5 100644 --- a/flexmeasures/ui/views/assets/views.py +++ b/flexmeasures/ui/views/assets/views.py @@ -434,6 +434,21 @@ def properties(self, id: str): .all() ) + # Info to help the user set the `group` field of the flex-model: + # - if this asset has a parent, suggest referencing the parent as the group + # - if the parent already defines a power-capacity, hint that this asset + # should join that group to count toward the parent's limit + # - if this asset itself defines a power-capacity and has children, hint + # that those children can reference this asset as their group + parent_asset_id = asset.parent_asset.id if asset.parent_asset else None + parent_asset_name = asset.parent_asset.name if asset.parent_asset else None + parent_has_power_capacity = bool( + asset.parent_asset + and "power-capacity" in (asset.parent_asset.flex_model or {}) + ) + own_has_power_capacity = "power-capacity" in (asset.flex_model or {}) + has_child_assets = bool(asset.child_assets) + # Can the user create a sibling of this asset? # - Has a parent → check create-children on that parent asset. # - No parent, owned account → check create-children on that account. @@ -484,4 +499,9 @@ def properties(self, id: str): attributes_label=ATTRIBUTES_FIELD_LABEL, attributes_description=ATTRIBUTES_FIELD_DESCRIPTION, stored_secrets=get_secret_overview(asset.secrets), + parent_asset_id=parent_asset_id, + parent_asset_name=parent_asset_name, + parent_has_power_capacity=parent_has_power_capacity, + own_has_power_capacity=own_has_power_capacity, + has_child_assets=has_child_assets, )