diff --git a/changelog/800.fix.md b/changelog/800.fix.md new file mode 100644 index 000000000..ac703ef4c --- /dev/null +++ b/changelog/800.fix.md @@ -0,0 +1,2 @@ +`default_ignore_datasets.yaml` is now shipped inside the `climate_ref` package and resolved with `importlib.resources`. +The `config` and `solve_config` pytest fixtures previously located it relative to the monorepo source tree, so they raised `ValueError` for any project that installs `climate-ref` from a wheel. diff --git a/packages/climate-ref/src/climate_ref/config.py b/packages/climate-ref/src/climate_ref/config.py index cb122b801..e21124e62 100644 --- a/packages/climate-ref/src/climate_ref/config.py +++ b/packages/climate-ref/src/climate_ref/config.py @@ -406,9 +406,8 @@ def _load_config(config_file: str | Path, doc: dict[str, Any]) -> "Config": DEFAULT_IGNORE_DATASETS_MAX_AGE = datetime.timedelta(hours=6) -DEFAULT_IGNORE_DATASETS_URL = ( - "https://raw.githubusercontent.com/Climate-REF/climate-ref/refs/heads/main/default_ignore_datasets.yaml" -) +DEFAULT_IGNORE_DATASETS_FILENAME = "default_ignore_datasets.yaml" +DEFAULT_IGNORE_DATASETS_URL = f"https://raw.githubusercontent.com/Climate-REF/climate-ref/refs/heads/main/{DEFAULT_IGNORE_DATASETS_FILENAME}" def _get_default_ignore_datasets_file() -> Path: @@ -423,7 +422,7 @@ def _get_default_ignore_datasets_file() -> Path: : Path to the ignore datasets file under the user cache directory. """ - return platformdirs.user_cache_path("climate_ref") / "default_ignore_datasets.yaml" + return platformdirs.user_cache_path("climate_ref") / DEFAULT_IGNORE_DATASETS_FILENAME def refresh_ignore_datasets_file(config: "Config") -> None: diff --git a/packages/climate-ref/src/climate_ref/conftest_plugin.py b/packages/climate-ref/src/climate_ref/conftest_plugin.py index dc5813a75..e3675eed7 100644 --- a/packages/climate-ref/src/climate_ref/conftest_plugin.py +++ b/packages/climate-ref/src/climate_ref/conftest_plugin.py @@ -26,6 +26,7 @@ from __future__ import annotations +import importlib.resources import os import re import tempfile @@ -41,7 +42,7 @@ from typer.testing import CliRunner from climate_ref import cli -from climate_ref.config import Config, DiagnosticProviderConfig +from climate_ref.config import DEFAULT_IGNORE_DATASETS_FILENAME, Config, DiagnosticProviderConfig from climate_ref.datasets.cmip6 import CMIP6DatasetAdapter from climate_ref.datasets.obs4mips import Obs4MIPsDatasetAdapter from climate_ref.models import Execution @@ -231,17 +232,38 @@ def data_catalog( } +def packaged_ignore_datasets_file() -> Path: + """ + Locate the ``default_ignore_datasets.yaml`` shipped inside the ``climate_ref`` package. + + Returns + ------- + : + Path to the packaged ignore datasets file. + + Raises + ------ + ValueError + If the file is missing from the installed package. + """ + resource = importlib.resources.files("climate_ref") / DEFAULT_IGNORE_DATASETS_FILENAME + local_ignore_file = Path(str(resource)) + if not local_ignore_file.is_file(): # pragma: no cover - indicates a packaging error + raise ValueError(f"Could not find ignore file at {local_ignore_file}") + return local_ignore_file + + def _use_local_ignore_datasets_file(cfg: Config) -> None: """ - Point the config at the in-tree default_ignore_datasets.yaml and disable fetching. + Point the config at the packaged default_ignore_datasets.yaml and disable fetching. This keeps tests deterministic and offline: they exercise the shipped default file rather than whatever copy happens to be cached on the host. + + The file is resolved from the installed package rather than the source tree, + so the fixtures also work for provider packages that depend on a released ``climate-ref`` wheel. """ - local_ignore_file = Path(__file__).parents[4] / "default_ignore_datasets.yaml" - if not local_ignore_file.is_file(): - raise ValueError(f"Could not find ignore file at {local_ignore_file}") - cfg.ignore_datasets_file = local_ignore_file + cfg.ignore_datasets_file = packaged_ignore_datasets_file() cfg.ignore_datasets_url = "" diff --git a/packages/climate-ref/src/climate_ref/default_ignore_datasets.yaml b/packages/climate-ref/src/climate_ref/default_ignore_datasets.yaml new file mode 100644 index 000000000..441b131d5 --- /dev/null +++ b/packages/climate-ref/src/climate_ref/default_ignore_datasets.yaml @@ -0,0 +1,62 @@ +# This file is used to specify datasets that should be ignored by default. +# +# It can be used to ignore datasets that are known to have issues and keep +# diagnostics that use multiple datasets from running. +# +# The format used in this configuration file is: +# provider: +# diagnostic: +# source_type: +# - facet: value +# - other_facet: [other_value1, other_value2] +# +# The facets listed here will be used to create `climate_ref_core.constraints.IgnoreFacets` +# instances that will be prepended to the constraints of the data requirements +# of the diagnostics that the facets are listed under. +# +# If no `ignore_datasets_file` is specified in the REF configuration, this file +# will be downloaded from GitHub and used. If the local copy of this file is +# older than 6 hours, it will be updated. +# +esmvaltool: + sea-ice-sensitivity: + cmip6: + - experiment_id: historical + grid_label: gn + member_id: r1i1p1f1 + source_id: CAS-ESM2-0 + table_id: SImon + variable_id: siconc + version: v20201225 + - experiment_id: historical + source_id: BCC-ESM1 + table_id: SImon + variable_id: siconc + - experiment_id: historical + source_id: CMCC-CM2-HR4 + table_id: SImon + variable_id: siconc + - experiment_id: historical + source_id: CMCC-CM2-SR5 + table_id: SImon + variable_id: siconc + - experiment_id: historical + source_id: CMCC-ESM2 + table_id: SImon + variable_id: siconc + - experiment_id: historical + source_id: ICON-ESM-LR + table_id: SImon + variable_id: siconc + - experiment_id: historical + source_id: NorESM2-LM + table_id: SImon + variable_id: siconc + - experiment_id: historical + source_id: NorESM2-MM + table_id: SImon + variable_id: siconc + - experiment_id: historical + source_id: SAM0-UNICON + table_id: SImon + variable_id: siconc diff --git a/packages/climate-ref/tests/unit/test_conftest_plugin.py b/packages/climate-ref/tests/unit/test_conftest_plugin.py new file mode 100644 index 000000000..c114ee13d --- /dev/null +++ b/packages/climate-ref/tests/unit/test_conftest_plugin.py @@ -0,0 +1,30 @@ +"""Tests for the shared pytest plugin shipped with ``climate_ref``.""" + +from pathlib import Path + +from climate_ref.config import DEFAULT_IGNORE_DATASETS_FILENAME, Config +from climate_ref.conftest_plugin import _use_local_ignore_datasets_file, packaged_ignore_datasets_file + +# The canonical copy, served over `DEFAULT_IGNORE_DATASETS_URL` from the default branch. +REPO_ROOT_IGNORE_FILE = Path(__file__).parents[4] / DEFAULT_IGNORE_DATASETS_FILENAME + + +def test_packaged_ignore_datasets_file_is_resolvable(): + # Resolved from the installed package, so this also holds for a wheel install. + assert packaged_ignore_datasets_file().is_file() + + +def test_packaged_ignore_datasets_file_matches_repo_root(): + # The root copy is what `DEFAULT_IGNORE_DATASETS_URL` serves, so the two must not drift apart. + assert REPO_ROOT_IGNORE_FILE.is_file() + assert packaged_ignore_datasets_file().read_bytes() == REPO_ROOT_IGNORE_FILE.read_bytes() + + +def test_use_local_ignore_datasets_file_disables_fetching(): + cfg = Config.default() + + _use_local_ignore_datasets_file(cfg) + + assert cfg.ignore_datasets_file == packaged_ignore_datasets_file() + # An empty URL short-circuits `refresh_ignore_datasets_file`, keeping tests offline. + assert cfg.ignore_datasets_url == ""