From 510b235e096039b86d24874067c89b4f29e34a59 Mon Sep 17 00:00:00 2001 From: Louis Parkes-Talbot Date: Tue, 18 Aug 2026 14:36:02 +0200 Subject: [PATCH] Provide the shape 2025-01 expects for saved scenario actions --- docs/changelog.md | 13 ++ src/pyetm/config/api_compat.py | 36 +++ src/pyetm/models/scenario_loader.py | 101 ++++---- src/pyetm/models/scenarios.py | 217 ++++++++---------- .../scenario_runners/create_saved_scenario.py | 24 +- .../scenario_runners/update_saved_scenario.py | 25 +- tests/config/test_api_compat.py | 58 +++++ tests/models/test_scenario_loader.py | 58 +++++ tests/models/test_scenarios_collections.py | 41 +++- tests/services/conftest.py | 10 +- .../test_create_saved_scenario.py | 41 +++- .../test_update_saved_scenario.py | 35 ++- 12 files changed, 425 insertions(+), 234 deletions(-) create mode 100644 src/pyetm/config/api_compat.py create mode 100644 tests/config/test_api_compat.py create mode 100644 tests/models/test_scenario_loader.py diff --git a/docs/changelog.md b/docs/changelog.md index ae0fa1b7..89d89390 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,19 @@ All notable changes to pyetm are documented here. +## [2.0.3] - 26-08-2026 + +### Fixed + +- Saving scenarios to MyETM works again on the stable `2025-01` version. pyetm now + sends the request shape its target expects: `2025-01` reads the saved scenario + attributes from the top level, newer engines from a `saved_scenario` root key. + Affected `Scenario.new()`, `Session.save()` and any `Scenarios.from_excel()` + run whose MAIN rows are not marked `session=TRUE`, which previously failed with + `422: title: is missing, scenario_id: is missing`. +- A scenario that cannot be saved to MyETM no longer passes silently. The failure is + recorded on the scenario and reported by `Scenarios.from_excel()`. + ## [2.0.2] - 23-07-2026 ### Added diff --git a/src/pyetm/config/api_compat.py b/src/pyetm/config/api_compat.py new file mode 100644 index 00000000..b3d91c32 --- /dev/null +++ b/src/pyetm/config/api_compat.py @@ -0,0 +1,36 @@ +"""Request shapes that work across every engine version pyetm targets. + +Sibling of :mod:`pyetm.config.curve_registry`, which does the same for curve names. +""" + +from __future__ import annotations + +from typing import Any +from urllib.parse import urlparse + +STABLE_2025_01_LABEL = "2025-01" + + +def _targets_stable_2025_01(base_url: str | None) -> bool: + """Is this client pointed at the 2025-01 stable engine?""" + if not base_url: + return False + + host = urlparse(base_url).hostname or "" + return host.split(".")[0] == STABLE_2025_01_LABEL + + +def saved_scenario_payload( + attributes: dict[str, Any], base_url: str | None = None +) -> dict[str, Any]: + """Build a saved scenario body the target engine can read. + + Stable 2025-01 reads the attributes from the top level of the request; every + engine from 2026-01 onwards reads them from a ``saved_scenario`` root key. The + root key doubles as the signal that keeps Rails' ParamsWrapper from choosing the + shape for us, so it is what anything other than 2025-01 gets. + """ + if _targets_stable_2025_01(base_url): + return dict(attributes) + + return {"saved_scenario": attributes} diff --git a/src/pyetm/models/scenario_loader.py b/src/pyetm/models/scenario_loader.py index e338f012..22307c95 100644 --- a/src/pyetm/models/scenario_loader.py +++ b/src/pyetm/models/scenario_loader.py @@ -1,15 +1,23 @@ """Utilities for loading scenarios from various sources.""" import logging -from typing import Protocol, Optional, Dict, Any, cast +from typing import Any, Protocol, cast + from pyetm.models.session import Session logger = logging.getLogger(__name__) +def _warn_save_failed(session: Session, row_label: str, error: Exception) -> Session: + """Record a failed MyETM save on the session itself.""" + message = f"Row '{row_label}' was not saved to MyETM: {error}" + logger.warning("%s Returning session instead.", message) + session.add_warning("save", message) + return session + + class ScenarioLoader(Protocol): - """ - Protocol for loading, copying, and creating scenarios. + """Protocol for loading, copying, and creating scenarios. Different implementations interpret scenario IDs differently: - SessionLoader: IDs refer to ETEngine Sessions @@ -20,10 +28,10 @@ def load( self, scenario_id: int, area_code: Any, - end_year: Optional[int], + end_year: int | None, row_label: str, - metadata_updates: Dict[str, Any], - ) -> Optional[Session]: + metadata_updates: dict[str, Any], + ) -> Session | None: """Load an existing scenario by ID and apply metadata updates.""" ... @@ -31,33 +39,31 @@ def copy( self, scenario_id: int, row_label: str, - metadata_updates: Dict[str, Any], - ) -> Optional[Session]: + metadata_updates: dict[str, Any], + ) -> Session | None: """Create a deep copy of a scenario (no template link).""" ... def create_new( self, area_code: Any, - end_year: Optional[int], + end_year: int | None, row_label: str, - metadata_updates: Dict[str, Any], - ) -> Optional[Session]: + metadata_updates: dict[str, Any], + ) -> Session | None: """Create a brand new scenario.""" ... class SessionLoader: - """ - Loader for ETEngine Sessions. + """Loader for ETEngine Sessions. Interprets IDs as ETEngine scenario/session IDs. """ def __init__(self, packer_helper: Any) -> None: - """ - Args: - packer_helper: Reference to ScenarioPacker instance for helper methods + """Args: + packer_helper: Reference to ScenarioPacker instance for helper methods """ self._helper = packer_helper @@ -65,10 +71,10 @@ def load( self, scenario_id: int, area_code: Any, - end_year: Optional[int], + end_year: int | None, row_label: str, - metadata_updates: Dict[str, Any], - ) -> Optional[Session]: + metadata_updates: dict[str, Any], + ) -> Session | None: """Load an ETEngine Session by ID.""" scenario = self._helper._load_or_create_scenario( scenario_id, area_code, end_year, row_label, **metadata_updates @@ -82,8 +88,8 @@ def copy( self, scenario_id: int, row_label: str, - metadata_updates: Dict[str, Any], - ) -> Optional[Session]: + metadata_updates: dict[str, Any], + ) -> Session | None: """Deep copy an ETEngine Session.""" try: source_scenario = Session.load(scenario_id) @@ -100,10 +106,10 @@ def copy( def create_new( self, area_code: Any, - end_year: Optional[int], + end_year: int | None, row_label: str, - metadata_updates: Dict[str, Any], - ) -> Optional[Session]: + metadata_updates: dict[str, Any], + ) -> Session | None: """Create a new ETEngine Session.""" scenario = self._helper._load_or_create_scenario( None, area_code, end_year, row_label, **metadata_updates @@ -115,22 +121,19 @@ def create_new( class SavedScenarioLoader: - """ - Loader for MyETM SavedScenarios. + """Loader for MyETM SavedScenarios. Interprets IDs as MyETM SavedScenario IDs and automatically saves new scenarios. """ def __init__(self, packer_helper: Any) -> None: - """ - Args: - packer_helper: Reference to ScenarioPacker instance for helper methods + """Args: + packer_helper: Reference to ScenarioPacker instance for helper methods """ self._helper = packer_helper def _require_authentication(self) -> None: - """ - Validate that authentication token is available for SavedScenario operations. + """Validate that authentication token is available for SavedScenario operations. Raises: PermissionError: If no ETM_API_TOKEN is configured @@ -150,10 +153,10 @@ def load( self, scenario_id: int, area_code: Any, - end_year: Optional[int], + end_year: int | None, row_label: str, - metadata_updates: Dict[str, Any], - ) -> Optional[Session]: + metadata_updates: dict[str, Any], + ) -> Session | None: """Load a SavedScenario from MyETM.""" from pyetm.models.scenario import Scenario @@ -181,8 +184,8 @@ def copy( self, scenario_id: int, row_label: str, - metadata_updates: Dict[str, Any], - ) -> Optional[Session]: + metadata_updates: dict[str, Any], + ) -> Session | None: """Copy a SavedScenario and save the copy to MyETM.""" from pyetm.models.scenario import Scenario @@ -197,7 +200,7 @@ def copy( try: saved_copy = copied_session.save(title=title) # Validate that we got a proper SavedScenario back with required attributes - if hasattr(saved_copy, 'id') and hasattr(saved_copy, 'scenario_id'): + if hasattr(saved_copy, "id") and hasattr(saved_copy, "scenario_id"): logger.info( "Automatically saved copy to MyETM with ID %s (session ID: %s)", saved_copy.id, @@ -211,12 +214,7 @@ def copy( ) return copied_session except Exception as save_error: - logger.warning( - "Failed to save copy to MyETM for row '%s': %s. Returning session instead.", - row_label, - save_error, - ) - return copied_session + return _warn_save_failed(copied_session, row_label, save_error) except Exception as e: error_msg = str(e) if "does not exist" in error_msg or "not found" in error_msg.lower(): @@ -237,13 +235,11 @@ def copy( def create_new( self, area_code: Any, - end_year: Optional[int], + end_year: int | None, row_label: str, - metadata_updates: Dict[str, Any], - ) -> Optional[Session]: + metadata_updates: dict[str, Any], + ) -> Session | None: """Create a new scenario and save it to MyETM.""" - from pyetm.models.scenario import Scenario - # Validate authentication before attempting to save self._require_authentication() @@ -259,7 +255,7 @@ def create_new( try: saved_scenario = scenario.save(title=title) # Validate that we got a proper SavedScenario back with required attributes - if hasattr(saved_scenario, 'id') and hasattr(saved_scenario, 'scenario_id'): + if hasattr(saved_scenario, "id") and hasattr(saved_scenario, "scenario_id"): logger.info( "Saved new scenario to MyETM with ID %s (session ID: %s)", saved_scenario.id, @@ -273,9 +269,4 @@ def create_new( ) return cast(Session, scenario) except Exception as e: - logger.warning( - "Failed to save new scenario to MyETM for row '%s': %s. Returning session instead.", - row_label, - e, - ) - return cast(Session, scenario) + return _warn_save_failed(cast(Session, scenario), row_label, e) diff --git a/src/pyetm/models/scenarios.py b/src/pyetm/models/scenarios.py index 7d29d267..a7f03162 100644 --- a/src/pyetm/models/scenarios.py +++ b/src/pyetm/models/scenarios.py @@ -1,30 +1,28 @@ """Collection and bulk operations for scenarios.""" from __future__ import annotations + import logging +from collections.abc import Iterable, Iterator, Sequence from os import PathLike from pathlib import Path from typing import ( - Dict, - Iterable, - Iterator, - List, - Union, - TypedDict, - Optional, - Any, - Sequence, TYPE_CHECKING, + Any, + TypedDict, cast, ) + +import pandas as pd from pydantic import Field, PrivateAttr -from pyetm.models.session import Session -from pyetm.models.base import Base + from pyetm.clients import BaseClient, get_client -from pyetm.clients.base_client import AsyncBatchRunner, MAX_CONCURRENT -from pyetm.types import AnnualExportType, HourlyCurveType, CarrierType -from .scenario import Scenario, SavedScenarioError -import pandas as pd +from pyetm.clients.base_client import MAX_CONCURRENT, AsyncBatchRunner +from pyetm.models.base import Base +from pyetm.models.session import Session +from pyetm.types import AnnualExportType, CarrierType + +from .scenario import SavedScenarioError, Scenario logger = logging.getLogger(__name__) @@ -33,26 +31,24 @@ class ScenarioCreationParams(TypedDict, total=False): - """ - Type definition for create_many parameter dicts. + """Type definition for create_many parameter dicts. Note: template_id must be a Session ID (ETEngine), not a SavedScenario ID (MyETM). """ - title: Optional[str] - scenario_id: Optional[int] - template_id: Optional[int] - area_code: Optional[str] - end_year: Optional[int] - private: Optional[bool] - user_values: Optional[dict[str, Any]] - custom_curves: Optional[dict[str, Any]] - sortables: Optional[dict[str, list[Any]]] + title: str | None + scenario_id: int | None + template_id: int | None + area_code: str | None + end_year: int | None + private: bool | None + user_values: dict[str, Any] | None + custom_curves: dict[str, Any] | None + sortables: dict[str, list[Any]] | None class Scenarios(Base): - """ - A collection of SavedScenario and/or Session objects. + """A collection of SavedScenario and/or Session objects. Can hold both Scenario and Session objects to support mixed collections loaded from Excel or other sources. @@ -60,37 +56,32 @@ class Scenarios(Base): Warnings from bulk operations are collected in the inherited _warning_collector. """ - items: List[Union[Scenario, Session]] = Field(default_factory=list) - _packer: Optional["ScenarioPacker"] = PrivateAttr(default=None) + items: list[Scenario | Session] = Field(default_factory=list) + _packer: ScenarioPacker | None = PrivateAttr(default=None) - def __iter__(self) -> Iterator[Union[Scenario, Session]]: # type: ignore[override] + def __iter__(self) -> Iterator[Scenario | Session]: # type: ignore[override] return iter(self.items) def __len__(self) -> int: return len(self.items) - def __getitem__(self, index: int) -> Union[Scenario, Session]: + def __getitem__(self, index: int) -> Scenario | Session: return self.items[index] - def add(self, *scenarios: Union[Scenario, Session]) -> None: + def add(self, *scenarios: Scenario | Session) -> None: self.items.extend(scenarios) - def extend(self, scenarios: Iterable[Union[Scenario, Session]]) -> None: + def extend(self, scenarios: Iterable[Scenario | Session]) -> None: self.items.extend(list(scenarios)) @property - def sessions(self) -> List["Session"]: - """ - Get the underlying ETEngine Session objects from all items. - """ - return [ - item.session if isinstance(item, Scenario) else item for item in self.items - ] + def sessions(self) -> list[Session]: + """Get the underlying ETEngine Session objects from all items.""" + return [item.session if isinstance(item, Scenario) else item for item in self.items] @property - def combine(self) -> "ScenarioPacker": - """ - Helps users with quick access to a packer. The combine keyword makes + def combine(self) -> ScenarioPacker: + """Helps users with quick access to a packer. The combine keyword makes sense when spelling out method calls to scenarios. E.g. scenarios.combine.inputs.to_dataframe() @@ -110,8 +101,7 @@ def get_hourly_output_curves( self, carrier_type: CarrierType, ) -> dict[str, dict[str, pd.DataFrame]]: - """ - Get hourly output curves for all scenarios by carrier type. + """Get hourly output curves for all scenarios by carrier type. Args: carrier_type: Carrier type alias (electricity, heat, hydrogen, methane) @@ -126,8 +116,8 @@ def get_hourly_output_curves( return self.combine.hourly_output_curves(carrier_type) def _ensure_hourly_curves_fetched(self, carrier_type: str) -> None: - """ - Ensure all scenarios have fetched their hourly output curves. + """Ensure all scenarios have fetched their hourly output curves. + Args: carrier_type: The carrier type to fetch curves for """ @@ -146,10 +136,9 @@ def _ensure_hourly_curves_fetched(self, carrier_type: str) -> None: def get_annual_exports( self, - exports: Optional[AnnualExportType | Sequence[AnnualExportType]] = None, + exports: AnnualExportType | Sequence[AnnualExportType] | None = None, ) -> dict[str, dict[str, pd.DataFrame]]: - """ - Get annual exports for all scenarios, organized by export type. + """Get annual exports for all scenarios, organized by export type. Returns: Dict mapping export names to dicts of {scenario_title: DataFrame} @@ -159,8 +148,8 @@ def get_annual_exports( @classmethod def load_all( cls, - client: Optional[BaseClient] = None, - ) -> "Scenarios": + client: BaseClient | None = None, + ) -> Scenarios: """Load all saved scenarios belonging to the authenticated user. Fetches all MyETM saved scenarios for the authenticated user in a single request. @@ -185,21 +174,20 @@ def load_all( result = FetchUserSavedScenariosRunner.run(client=client) if not result.success: - raise ValueError( - f"Failed to fetch user saved scenarios: {'; '.join(result.errors)}" - ) + raise ValueError(f"Failed to fetch user saved scenarios: {'; '.join(result.errors)}") if result.data is None: raise ValueError("No data returned from API") # Use model_validate to avoid N+1 API calls saved_scenarios = [Scenario.model_validate(data) for data in result.data] - return cls(items=cast(List[Union[Scenario, Session]], saved_scenarios)) + return cls(items=cast(list[Scenario | Session], saved_scenarios)) @classmethod - def load_many(cls, saved_scenario_ids: Iterable[int], client: Optional[BaseClient] = None) -> "Scenarios": - """ - Load multiple SavedScenario objects by their MyETM saved scenario IDs. + def load_many( + cls, saved_scenario_ids: Iterable[int], client: BaseClient | None = None + ) -> Scenarios: + """Load multiple SavedScenario objects by their MyETM saved scenario IDs. This is a bulk operation - individual failures are collected as warnings to allow partial success. Use PYETM_ERROR_MODE=safe to raise on first error. @@ -240,9 +228,8 @@ def create_many( area_code: str | None = None, end_year: int | None = None, client: BaseClient | None = None, - ) -> "Scenarios": - """ - Create multiple SavedScenario objects from parameter dicts. + ) -> Scenarios: + """Create multiple SavedScenario objects from parameter dicts. If scenario_id is not provided in params, creates a new Session first. @@ -273,20 +260,18 @@ def create_many( client = get_client() # Create the collection with bulk context enabled - scenarios_list: List[Union[Scenario, Session]] = [] + scenarios_list: list[Scenario | Session] = [] scenarios = cls(items=scenarios_list) scenarios.set_bulk_context(True) # Separate data parameters from creation parameters DATA_PARAMS = ["user_values", "custom_curves", "sortables"] - creation_params_list: List[Dict[str, Any]] = [] - data_to_apply: List[tuple[int, Dict[str, Any]]] = ( - [] - ) # List of (scenario_index, data_dict) + creation_params_list: list[dict[str, Any]] = [] + data_to_apply: list[tuple[int, dict[str, Any]]] = [] # List of (scenario_index, data_dict) for idx, params in enumerate(scenario_params): # Make a copy to avoid modifying original - params_copy: Dict[str, Any] = dict(params) + params_copy: dict[str, Any] = dict(params) # Extract all data params declaratively data = {key: params_copy.pop(key, None) for key in DATA_PARAMS} @@ -388,9 +373,7 @@ def create_many( # Apply data parameters concurrently after all scenarios are created if data_to_apply and saved_scenarios: - failure_warnings = cls._apply_data_concurrently( - saved_scenarios, data_to_apply, client - ) + failure_warnings = cls._apply_data_concurrently(saved_scenarios, data_to_apply, client) # Add data application failures as warnings to the collection for warning in failure_warnings: @@ -405,22 +388,18 @@ def create_many( scenarios._merge_submodel_warnings(scenario) # Display summary if there were any warnings - if len(scenarios.warnings) > 0 or any( - len(s.warnings) > 0 for s in saved_scenarios - ): + if len(scenarios.warnings) > 0 or any(len(s.warnings) > 0 for s in saved_scenarios): print("\n=== Batch Creation Summary ===") scenarios.show_warnings() return scenarios def to_excel(self, path: PathLike[str] | str, **export_options: Any) -> None: - """ - Export all scenarios to Excel. + """Export all scenarios to Excel. Note: This exports the underlying session data from each SavedScenario. The scenario_id column will contain Scenario IDs (MyETM). """ - if not self.items: raise ValueError("No scenarios to export") @@ -432,10 +411,9 @@ def to_excel(self, path: PathLike[str] | str, **export_options: Any) -> None: @classmethod def from_excel( - cls, xlsx_path: PathLike[str] | str, update: bool | List[str] = False - ) -> "Scenarios": - """ - Import all scenarios from Excel file. + cls, xlsx_path: PathLike[str] | str, update: bool | list[str] = False + ) -> Scenarios: + """Import all scenarios from Excel file. Loads all scenarios from the Excel file, including both: - SavedScenarios (where 'session' column is False or missing) @@ -460,11 +438,13 @@ def from_excel( ) else: all_scenarios.sort(key=lambda s: s.id if hasattr(s, "id") else 0) - scenarios_list: List[Union[Scenario, Session]] = all_scenarios # type: ignore[assignment] + scenarios_list: list[Scenario | Session] = all_scenarios # type: ignore[assignment] scenarios.items = scenarios_list scenarios._packer = packer + scenarios._merge_submodel_warnings(*scenarios.items, key_attr="id") + # Auto-display warnings if any if len(scenarios.warnings) > 0: scenarios.show_warnings() @@ -472,16 +452,13 @@ def from_excel( return scenarios @staticmethod - def _get_session(item: Union[Scenario, Session]) -> Session: - """ - Safely extract Session from either a Scenario or Session object. - """ + def _get_session(item: Scenario | Session) -> Session: + """Safely extract Session from either a Scenario or Session object.""" return item.session if isinstance(item, Scenario) else item @staticmethod def _format_data_error(metadata: tuple[Any, ...], result: Any) -> str: - """ - Format error message based on request metadata and result. + """Format error message based on request metadata and result. Args: metadata: Tuple of (req_type, scenario_idx, scenario_title[, curve_key]) @@ -496,18 +473,19 @@ def _format_data_error(metadata: tuple[Any, ...], result: Any) -> str: if req_type == "custom_curve" and len(metadata) > 3: curve_key = metadata[3] - return f"Failed to upload curve '{curve_key}' for scenario '{scenario_title}': {error_msg}" + return ( + f"Failed to upload curve '{curve_key}' for scenario '{scenario_title}': {error_msg}" + ) else: return f"Failed to update {req_type} for scenario '{scenario_title}': {error_msg}" @staticmethod def _apply_data_concurrently( - scenarios: List[Scenario], - data_to_apply: List[tuple[Any, ...]], + scenarios: list[Scenario], + data_to_apply: list[tuple[Any, ...]], client: BaseClient, - ) -> List[str]: - """ - Apply user_values/curves/sortables to scenarios concurrently using runners. + ) -> list[str]: + """Apply user_values/curves/sortables to scenarios concurrently using runners. Args: scenarios: List of scenarios @@ -517,18 +495,18 @@ def _apply_data_concurrently( Returns: List of warning messages for failed data applications """ - from pyetm.services.scenario_runners.update_inputs import UpdateInputsRunner from pyetm.services.scenario_runners.update_custom_curves import ( UpdateCustomCurvesRunner, ) + from pyetm.services.scenario_runners.update_inputs import UpdateInputsRunner from pyetm.services.scenario_runners.update_sortables import ( UpdateSortablesRunner, ) requests = [] - request_metadata: list[Any] = ( - [] - ) # Track what each request is for - can be 3 or 4 element tuples + request_metadata: list[ + Any + ] = [] # Track what each request is for - can be 3 or 4 element tuples warnings = [] # Collect warning messages for scenario_idx, data in data_to_apply: @@ -541,15 +519,13 @@ def _apply_data_concurrently( # Use UpdateInputsRunner to build user_values requests if data.get("user_values"): try: - request = UpdateInputsRunner.build_request( - session, data["user_values"] - ) + request = UpdateInputsRunner.build_request(session, data["user_values"]) requests.append(request) - request_metadata.append( - ("user_values", scenario_idx, scenario.title) - ) + request_metadata.append(("user_values", scenario_idx, scenario.title)) except Exception as e: - warning_msg = f"Failed to build user_values request for scenario '{scenario.title}': {e}" + warning_msg = ( + f"Failed to build user_values request for scenario '{scenario.title}': {e}" + ) warnings.append(warning_msg) # Use UpdateCustomCurvesRunner to build curve requests @@ -569,20 +545,18 @@ def _apply_data_concurrently( ) ) except Exception as e: - warning_msg = f"Failed to build curve requests for scenario '{scenario.title}': {e}" + warning_msg = ( + f"Failed to build curve requests for scenario '{scenario.title}': {e}" + ) warnings.append(warning_msg) # Use UpdateSortablesRunner to build sortables requests if data.get("sortables"): for sortable_type, order in data["sortables"].items(): try: - request = UpdateSortablesRunner.build_request( - session, sortable_type, order - ) + request = UpdateSortablesRunner.build_request(session, sortable_type, order) requests.append(request) - request_metadata.append( - ("sortables", scenario_idx, scenario.title) - ) + request_metadata.append(("sortables", scenario_idx, scenario.title)) except Exception as e: warning_msg = f"Failed to build sortables request for scenario '{scenario.title}': {e}" warnings.append(warning_msg) @@ -615,11 +589,10 @@ def _apply_data_concurrently( @classmethod def discard_many( cls, - saved_scenario_ids: List[int], - client: Optional[BaseClient] = None, - ) -> Dict[str, Any]: - """ - Discard multiple saved scenarios in bulk (soft-delete). + saved_scenario_ids: list[int], + client: BaseClient | None = None, + ) -> dict[str, Any]: + """Discard multiple saved scenarios in bulk (soft-delete). The scenarios are marked as discarded and hidden from listings, but can be recovered through the MyETM web interface within 60 days. After 60 days, @@ -650,9 +623,7 @@ def discard_many( # Build discard requests for all scenarios requests = [] for scenario_id in saved_scenario_ids: - request = DiscardSavedScenarioRunner.build_request( - saved_scenario_id=scenario_id - ) + request = DiscardSavedScenarioRunner.build_request(saved_scenario_id=scenario_id) requests.append(request) # Format requests for AsyncBatchRunner @@ -681,8 +652,6 @@ def discard_many( else: failed.append(scenario_id) error_msg = "; ".join(result.errors) if result.errors else "Unknown error" - logger.warning( - f"Failed to discard saved scenario {scenario_id}: {error_msg}" - ) + logger.warning(f"Failed to discard saved scenario {scenario_id}: {error_msg}") return {"successful": successful, "failed": failed} diff --git a/src/pyetm/services/scenario_runners/create_saved_scenario.py b/src/pyetm/services/scenario_runners/create_saved_scenario.py index f696ef2d..3690ae81 100644 --- a/src/pyetm/services/scenario_runners/create_saved_scenario.py +++ b/src/pyetm/services/scenario_runners/create_saved_scenario.py @@ -1,14 +1,16 @@ """Service for creating saved scenarios.""" -from typing import Any, Dict +from typing import Any + +from pyetm.clients.base_client import BaseClient +from pyetm.config.api_compat import saved_scenario_payload from pyetm.services.scenario_runners.base_runner import BaseRunner + from ..service_result import ServiceResult -from pyetm.clients.base_client import BaseClient -class CreateSavedScenarioRunner(BaseRunner[Dict[str, Any]]): - """ - Runner for creating a SavedScenario in MyETM from a SessionID scenario. +class CreateSavedScenarioRunner(BaseRunner[dict[str, Any]]): + """Runner for creating a SavedScenario in MyETM from a SessionID scenario. POST /api/v3/saved_scenarios """ @@ -18,10 +20,9 @@ class CreateSavedScenarioRunner(BaseRunner[Dict[str, Any]]): @staticmethod def run( - client: BaseClient, saved_scenario_data: Dict[str, Any], **kwargs: Any - ) -> ServiceResult[Dict[str, Any]]: - """ - Create a new SavedScenario in MyETM. + client: BaseClient, saved_scenario_data: dict[str, Any], **kwargs: Any + ) -> ServiceResult[dict[str, Any]]: + """Create a new SavedScenario in MyETM. Args: client: The HTTP client to use @@ -46,8 +47,7 @@ def run( return ServiceResult.fail(errors) all_allowed = ( - CreateSavedScenarioRunner.REQUIRED_KEYS - + CreateSavedScenarioRunner.OPTIONAL_KEYS + CreateSavedScenarioRunner.REQUIRED_KEYS + CreateSavedScenarioRunner.OPTIONAL_KEYS ) filtered_data, warnings = CreateSavedScenarioRunner._filter_allowed_fields( saved_scenario_data, @@ -55,7 +55,7 @@ def run( "create saved scenario", ) - payload = {"saved_scenario": filtered_data} + payload = saved_scenario_payload(filtered_data, client.session.base_url) result = CreateSavedScenarioRunner._make_request( client=client, diff --git a/src/pyetm/services/scenario_runners/update_saved_scenario.py b/src/pyetm/services/scenario_runners/update_saved_scenario.py index 64ec041b..96ed6598 100644 --- a/src/pyetm/services/scenario_runners/update_saved_scenario.py +++ b/src/pyetm/services/scenario_runners/update_saved_scenario.py @@ -1,14 +1,16 @@ """Service for updating an existing saved scenario.""" -from typing import Any, Dict +from typing import Any + +from pyetm.clients.base_client import BaseClient +from pyetm.config.api_compat import saved_scenario_payload from pyetm.services.scenario_runners.base_runner import BaseRunner + from ..service_result import ServiceResult -from pyetm.clients.base_client import BaseClient -class UpdateSavedScenarioRunner(BaseRunner[Dict[str, Any]]): - """ - Runner for updating a SavedScenario in MyETM. +class UpdateSavedScenarioRunner(BaseRunner[dict[str, Any]]): + """Runner for updating a SavedScenario in MyETM. PUT /api/v3/saved_scenarios/:id """ @@ -19,11 +21,10 @@ class UpdateSavedScenarioRunner(BaseRunner[Dict[str, Any]]): def run( client: BaseClient, saved_scenario_id: int, - update_data: Dict[str, Any], + update_data: dict[str, Any], **kwargs: Any, - ) -> ServiceResult[Dict[str, Any]]: - """ - Update an existing SavedScenario in MyETM. + ) -> ServiceResult[dict[str, Any]]: + """Update an existing SavedScenario in MyETM. Args: client: The HTTP client to use @@ -51,11 +52,9 @@ def run( ) if not filtered_data: - return ServiceResult.fail( - ["No valid fields provided for update"] + warnings - ) + return ServiceResult.fail(["No valid fields provided for update"] + warnings) - payload = {"saved_scenario": filtered_data} + payload = saved_scenario_payload(filtered_data, client.session.base_url) result = UpdateSavedScenarioRunner._make_request( client=client, diff --git a/tests/config/test_api_compat.py b/tests/config/test_api_compat.py new file mode 100644 index 00000000..3af10f8d --- /dev/null +++ b/tests/config/test_api_compat.py @@ -0,0 +1,58 @@ +"""The saved scenario body differs between the stable engine and everything newer. + +Each reader below mirrors how one engine version pulls attributes out of the request, +so a payload change that would break either deployment fails here. +""" + +from typing import Any + +import pytest + +from pyetm.config.api_compat import saved_scenario_payload + +ATTRIBUTES = {"scenario_id": 123, "title": "Fix check", "private": False} +PERMITTED = ("scenario_id", "title", "description", "private") + +STABLE = "https://2025-01.engine.energytransitionmodel.com/api/v3" +PRO = "https://engine.energytransitionmodel.com/api/v3" + + +def read_as_2025_01(body: dict[str, Any]) -> dict[str, Any]: + """CreateSavedScenario::Contract at tag stable.2025.01 reads top-level keys.""" + return {key: body[key] for key in PERMITTED if key in body} + + +def read_as_2026_01(body: dict[str, Any]) -> dict[str, Any]: + """params.require(:saved_scenario) at tag 2026-01 and later reads the root key. + + The KeyError stands in for ActionController::ParameterMissing. + """ + nested = body["saved_scenario"] + return {key: nested[key] for key in PERMITTED if key in nested} + + +def test_stable_engine_receives_the_attributes_at_the_top_level(): + body = saved_scenario_payload(ATTRIBUTES, STABLE) + + assert read_as_2025_01(body) == ATTRIBUTES + + +@pytest.mark.parametrize( + "base_url", + [ + PRO, + "https://beta.engine.energytransitionmodel.com/api/v3", + "http://localhost:3000/api/v3", + "https://2025-01-collections.energytransitionmodel.com/api/v3", + None, + ], +) +def test_every_other_engine_receives_the_root_key(base_url): + body = saved_scenario_payload(ATTRIBUTES, base_url) + + assert read_as_2026_01(body) == ATTRIBUTES + + +def test_the_stable_body_carries_no_root_key(): + """A root key would stop the 2025-01 contract seeing the attributes at all.""" + assert "saved_scenario" not in saved_scenario_payload(ATTRIBUTES, STABLE) diff --git a/tests/models/test_scenario_loader.py b/tests/models/test_scenario_loader.py new file mode 100644 index 00000000..0804ca6d --- /dev/null +++ b/tests/models/test_scenario_loader.py @@ -0,0 +1,58 @@ +"""A saved scenario that cannot be persisted must not look like a success.""" + +import pytest + +from pyetm.models.scenario import Scenario +from pyetm.models.scenario_loader import SavedScenarioLoader +from pyetm.models.session import Session + + +class StubPackerHelper: + """Minimal stand-in for ScenarioPacker's loader callbacks.""" + + def __init__(self, session: Session) -> None: + self.session = session + + def _load_or_create_scenario(self, *args, **kwargs) -> Session: + return self.session + + def _apply_metadata_to_scenario(self, *args, **kwargs) -> None: + return None + + +@pytest.fixture +def session() -> Session: + return Session(id=1, area_code="nl2023", end_year=2050) + + +@pytest.fixture +def loader(monkeypatch, session) -> SavedScenarioLoader: + monkeypatch.setattr(SavedScenarioLoader, "_require_authentication", lambda self: None) + return SavedScenarioLoader(StubPackerHelper(session)) + + +def test_failed_save_warns_and_still_returns_the_session(loader, session, monkeypatch): + """The warning must not escalate into an exception, or the row is lost.""" + + def raise_422(**kwargs): + raise RuntimeError("422: title: is missing, scenario_id: is missing") + + monkeypatch.setattr(session, "save", raise_422) + + result = loader.create_new("nl2023", 2050, "SESSION_A", {}) + + assert result is session + messages = [str(warning) for warning in result.warnings] + assert len(messages) == 1 + assert "SESSION_A" in messages[0] + assert "422" in messages[0] + + +def test_successful_save_records_no_warning(loader, session, monkeypatch): + saved = Scenario(id=2, scenario_id=1, title="Saved") + monkeypatch.setattr(session, "save", lambda **kwargs: saved) + + result = loader.create_new("nl2023", 2050, "SESSION_A", {}) + + assert result is saved + assert len(session.warnings) == 0 diff --git a/tests/models/test_scenarios_collections.py b/tests/models/test_scenarios_collections.py index a66ee2b2..464384d3 100644 --- a/tests/models/test_scenarios_collections.py +++ b/tests/models/test_scenarios_collections.py @@ -1,13 +1,12 @@ -import pytest +from unittest.mock import Mock, patch + import pandas as pd -import tempfile -from pathlib import Path -from unittest.mock import Mock, patch, MagicMock -from pyetm.models.sessions import Sessions -from pyetm.models.scenarios import Scenarios -from pyetm.models.session import Session + from pyetm.models.scenario import Scenario from pyetm.models.scenario_packer import ScenarioPacker +from pyetm.models.scenarios import Scenarios +from pyetm.models.session import Session +from pyetm.models.sessions import Sessions class TestScenariosFromExcel: @@ -521,3 +520,31 @@ def test_get_methods_work_through_combine_property(self): assert collection._packer is not None # Verify fetch was triggered with new API scenario1.get_hourly_curves.assert_called_once_with(["electricity"]) + + +class TestScenariosFromExcelWarnings: + """Per-row failures recorded by the loaders must reach the caller.""" + + def test_from_excel_lifts_warnings_off_the_scenarios(self): + session = Session(id=100, area_code="nl2023", end_year=2050) + session.add_warning("save", "Row 'SESSION_A' was not saved to MyETM: 422") + + mock_packer = Mock(spec=ScenarioPacker) + mock_packer._scenarios.return_value = [session] + + with patch.object(ScenarioPacker, "from_excel", return_value=mock_packer): + result = Scenarios.from_excel("test.xlsx") + + assert len(result.warnings) == 1 + assert "not saved to MyETM" in str(list(result.warnings)[0]) + + def test_from_excel_stays_quiet_when_every_row_succeeded(self): + session = Session(id=100, area_code="nl2023", end_year=2050) + + mock_packer = Mock(spec=ScenarioPacker) + mock_packer._scenarios.return_value = [session] + + with patch.object(ScenarioPacker, "from_excel", return_value=mock_packer): + result = Scenarios.from_excel("test.xlsx") + + assert len(result.warnings) == 0 diff --git a/tests/services/conftest.py b/tests/services/conftest.py index 416c0003..2d328060 100644 --- a/tests/services/conftest.py +++ b/tests/services/conftest.py @@ -30,15 +30,16 @@ def dummy_client(fake_response): from types import SimpleNamespace class DummyClient: - def __init__(self, response, supported_methods=None): + def __init__(self, response, supported_methods=None, base_url=None): self._response = response self.calls = [] self.supported_methods = supported_methods or ["get"] + self.base_url = base_url or "https://engine.energytransitionmodel.com/api/v3" @property def session(self): # Create a session with all supported HTTP methods - session_methods = {} + session_methods = {"base_url": self.base_url} for method in self.supported_methods: session_methods[method] = self._create_mock_method(method) return SimpleNamespace(**session_methods) @@ -63,14 +64,15 @@ def mock_method(url, params=None, json=None, **kwargs): return mock_method - def _make_client(response, method="get"): + def _make_client(response, method="get", base_url=None): if isinstance(response, dict): # If dict is provided, create a successful response return DummyClient( fake_response(ok=True, status_code=200, json_data=response), supported_methods=[method], + base_url=base_url, ) - return DummyClient(response, supported_methods=[method]) + return DummyClient(response, supported_methods=[method], base_url=base_url) return _make_client diff --git a/tests/services/scenario_runners/test_create_saved_scenario.py b/tests/services/scenario_runners/test_create_saved_scenario.py index 73ea7b6f..a0910bd6 100644 --- a/tests/services/scenario_runners/test_create_saved_scenario.py +++ b/tests/services/scenario_runners/test_create_saved_scenario.py @@ -1,3 +1,4 @@ +from pyetm.config.api_compat import saved_scenario_payload from pyetm.services.scenario_runners.create_saved_scenario import ( CreateSavedScenarioRunner, ) @@ -20,7 +21,9 @@ def test_create_saved_scenario_success_minimal(dummy_client, fake_response): assert result.success is True assert result.data == body assert result.errors == [] - assert client.calls == [("/saved_scenarios", {"json": {"saved_scenario": saved_scenario_data}})] + assert client.calls == [ + ("/saved_scenarios", {"json": saved_scenario_payload(saved_scenario_data)}) + ] def test_create_saved_scenario_success_with_optional_fields(dummy_client, fake_response): @@ -44,7 +47,9 @@ def test_create_saved_scenario_success_with_optional_fields(dummy_client, fake_r assert result.success is True assert result.data == body assert result.errors == [] - assert client.calls == [("/saved_scenarios", {"json": {"saved_scenario": saved_scenario_data}})] + assert client.calls == [ + ("/saved_scenarios", {"json": saved_scenario_payload(saved_scenario_data)}) + ] def test_create_saved_scenario_missing_required_field_scenario_id(dummy_client, fake_response): @@ -122,13 +127,9 @@ def test_create_saved_scenario_filters_invalid_fields(dummy_client, fake_respons assert warning in result.errors # Should only send valid fields - expected_payload = { - "saved_scenario": { - "scenario_id": 123, - "title": "My Saved Scenario", - "private": True, - } - } + expected_payload = saved_scenario_payload( + {"scenario_id": 123, "title": "My Saved Scenario", "private": True} + ) assert client.calls == [("/saved_scenarios", {"json": expected_payload})] @@ -198,7 +199,7 @@ def test_create_saved_scenario_with_kwargs(dummy_client, fake_response): # Verify basic structure assert len(client.calls) == 1 assert client.calls[0][0] == "/saved_scenarios" - assert client.calls[0][1]["json"] == {"saved_scenario": saved_scenario_data} + assert client.calls[0][1]["json"] == saved_scenario_payload(saved_scenario_data) def test_create_saved_scenario_payload_structure(dummy_client, fake_response): @@ -218,7 +219,7 @@ def test_create_saved_scenario_payload_structure(dummy_client, fake_response): # Verify the exact payload structure expected_call = ( "/saved_scenarios", - {"json": {"saved_scenario": saved_scenario_data}}, + {"json": saved_scenario_payload(saved_scenario_data)}, ) assert client.calls == [expected_call] @@ -233,3 +234,21 @@ def test_create_saved_scenario_empty_data(dummy_client, fake_response): assert result.success is False assert result.data is None assert len(client.calls) == 0 # Should not make API call + + +def test_create_saved_scenario_flat_payload_for_stable_engine(dummy_client, fake_response): + """2025-01 reads the attributes from the top level of the request.""" + body = {"id": 461, "scenario_id": 123, "title": "Stable"} + response = fake_response(ok=True, status_code=201, json_data=body) + client = dummy_client( + response, + method="post", + base_url="https://2025-01.engine.energytransitionmodel.com/api/v3", + ) + + saved_scenario_data = {"scenario_id": 123, "title": "Stable"} + + result = CreateSavedScenarioRunner.run(client, saved_scenario_data) + + assert result.success is True + assert client.calls == [("/saved_scenarios", {"json": saved_scenario_data})] diff --git a/tests/services/scenario_runners/test_update_saved_scenario.py b/tests/services/scenario_runners/test_update_saved_scenario.py index 5ecbf8e4..ce05dd37 100644 --- a/tests/services/scenario_runners/test_update_saved_scenario.py +++ b/tests/services/scenario_runners/test_update_saved_scenario.py @@ -1,3 +1,4 @@ +from pyetm.config.api_compat import saved_scenario_payload from pyetm.services.scenario_runners.update_saved_scenario import ( UpdateSavedScenarioRunner, ) @@ -20,7 +21,7 @@ def test_update_saved_scenario_success_single_field(dummy_client, fake_response) assert result.success is True assert result.data == body assert result.errors == [] - assert client.calls == [("/saved_scenarios/456", {"json": {"saved_scenario": update_data}})] + assert client.calls == [("/saved_scenarios/456", {"json": saved_scenario_payload(update_data)})] def test_update_saved_scenario_success_multiple_fields(dummy_client, fake_response): @@ -43,7 +44,7 @@ def test_update_saved_scenario_success_multiple_fields(dummy_client, fake_respon assert result.success is True assert result.data == body assert result.errors == [] - assert client.calls == [("/saved_scenarios/456", {"json": {"saved_scenario": update_data}})] + assert client.calls == [("/saved_scenarios/456", {"json": saved_scenario_payload(update_data)})] def test_update_saved_scenario_success_all_allowed_fields(dummy_client, fake_response): @@ -68,7 +69,7 @@ def test_update_saved_scenario_success_all_allowed_fields(dummy_client, fake_res assert result.success is True assert result.data == body assert result.errors == [] - assert client.calls == [("/saved_scenarios/456", {"json": {"saved_scenario": update_data}})] + assert client.calls == [("/saved_scenarios/456", {"json": saved_scenario_payload(update_data)})] def test_update_saved_scenario_empty_update_data(dummy_client, fake_response): @@ -115,7 +116,7 @@ def test_update_saved_scenario_filters_invalid_fields(dummy_client, fake_respons assert warning in result.errors # Should only send valid fields - expected_payload = {"saved_scenario": {"title": "Updated Title", "scenario_id": 123}} + expected_payload = saved_scenario_payload({"title": "Updated Title", "scenario_id": 123}) assert client.calls == [("/saved_scenarios/456", {"json": expected_payload})] @@ -218,7 +219,7 @@ def test_update_saved_scenario_with_kwargs(dummy_client, fake_response): # Verify basic structure assert len(client.calls) == 1 assert client.calls[0][0] == "/saved_scenarios/456" - assert client.calls[0][1]["json"] == {"saved_scenario": update_data} + assert client.calls[0][1]["json"] == saved_scenario_payload(update_data) def test_update_saved_scenario_payload_structure(dummy_client, fake_response): @@ -236,7 +237,7 @@ def test_update_saved_scenario_payload_structure(dummy_client, fake_response): # Verify the exact payload structure expected_call = ( "/saved_scenarios/456", - {"json": {"saved_scenario": update_data}}, + {"json": saved_scenario_payload(update_data)}, ) assert client.calls == [expected_call] @@ -256,7 +257,7 @@ def test_update_saved_scenario_discard(dummy_client, fake_response): assert result.success is True assert result.data == body assert result.errors == [] - assert client.calls == [("/saved_scenarios/456", {"json": {"saved_scenario": update_data}})] + assert client.calls == [("/saved_scenarios/456", {"json": saved_scenario_payload(update_data)})] def test_update_saved_scenario_change_privacy(dummy_client, fake_response): @@ -274,4 +275,22 @@ def test_update_saved_scenario_change_privacy(dummy_client, fake_response): assert result.success is True assert result.data == body assert result.errors == [] - assert client.calls == [("/saved_scenarios/456", {"json": {"saved_scenario": update_data}})] + assert client.calls == [("/saved_scenarios/456", {"json": saved_scenario_payload(update_data)})] + + +def test_update_saved_scenario_flat_payload_for_stable_engine(dummy_client, fake_response): + """2025-01 reads the attributes from the top level of the request.""" + body = {"id": 456, "title": "Stable"} + response = fake_response(ok=True, status_code=200, json_data=body) + client = dummy_client( + response, + method="put", + base_url="https://2025-01.engine.energytransitionmodel.com/api/v3", + ) + + update_data = {"title": "Stable"} + + result = UpdateSavedScenarioRunner.run(client, saved_scenario_id=456, update_data=update_data) + + assert result.success is True + assert client.calls == [("/saved_scenarios/456", {"json": update_data})]