From 9deb4e0648d851fd0537aa4d917f75d91eece2ca Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 7 Sep 2026 18:15:23 +0200 Subject: [PATCH 01/31] refactor; deleted module resources --- .../_cdf_tk/data_classes/__init__.py | 2 - .../_cdf_tk/data_classes/_module_resources.py | 215 ------------------ 2 files changed, 217 deletions(-) delete mode 100644 cognite_toolkit/_cdf_tk/data_classes/_module_resources.py diff --git a/cognite_toolkit/_cdf_tk/data_classes/__init__.py b/cognite_toolkit/_cdf_tk/data_classes/__init__.py index bfad012da3..cab85eb80e 100644 --- a/cognite_toolkit/_cdf_tk/data_classes/__init__.py +++ b/cognite_toolkit/_cdf_tk/data_classes/__init__.py @@ -31,7 +31,6 @@ ) from ._issues import Issue, IssueList from ._module_directories import ModuleDirectories, ModuleLocation -from ._module_resources import ModuleResources from ._packages import Package, Packages from ._tracking_info import CommandTracking, DeploymentTracking, TrackingEvent from ._yaml_comments import YAMLComments @@ -62,7 +61,6 @@ "IssueList", "ModuleDirectories", "ModuleLocation", - "ModuleResources", "Package", "Packages", "ResourceContainerDeployResult", diff --git a/cognite_toolkit/_cdf_tk/data_classes/_module_resources.py b/cognite_toolkit/_cdf_tk/data_classes/_module_resources.py deleted file mode 100644 index d596cd676c..0000000000 --- a/cognite_toolkit/_cdf_tk/data_classes/_module_resources.py +++ /dev/null @@ -1,215 +0,0 @@ -import sys -from dataclasses import dataclass -from functools import cached_property -from pathlib import Path -from typing import Any, ClassVar - -import yaml - -from cognite_toolkit import _version -from cognite_toolkit._cdf_tk.cdf_toml import CDFToml -from cognite_toolkit._cdf_tk.constants import DEFAULT_ENV -from cognite_toolkit._cdf_tk.resource_ios import ResourceTypes -from cognite_toolkit._cdf_tk.utils import ( - safe_write, - tmp_build_directory, -) -from cognite_toolkit._cdf_tk.utils.file import yaml_safe_dump -from cognite_toolkit._cdf_tk.utils.useful_types import T_ID - -from ._base import ConfigCore -from ._build_variables import BuildVariables -from ._built_modules import BuiltModule, BuiltModuleList -from ._built_resources import BuiltFullResourceList -from ._config_yaml import BuildConfigYAML -from ._module_directories import ModuleDirectories - -if sys.version_info >= (3, 11): - from typing import Self -else: - from typing_extensions import Self - - -@dataclass -class ModulesInfo: - version: str - modules: BuiltModuleList - - @classmethod - def load(cls, data: dict[str, Any]) -> Self: - return cls( - version=data["version"], - modules=BuiltModuleList([BuiltModule.load(module_data) for module_data in data["modules"]]), - ) - - def dump(self) -> dict[str, Any]: - return { - "version": self.version, - "modules": [module.dump() for module in self.modules], - } - - -@dataclass -class BuildInfo(ConfigCore): - filename: ClassVar[str] = "build_info.{build_env}.yaml" - top_warning: ClassVar[str] = "# DO NOT MODIFY THIS FILE MANUALLY. IT IS AUTO-GENERATED BY THE COGNITE TOOLKIT." - modules: ModulesInfo - - @classmethod - def load(cls, data: dict[str, Any], build_env: str, filepath: Path) -> Self: - return cls(filepath, ModulesInfo.load(data["modules"])) - - @classmethod - def rebuild(cls, organization_dir: Path, build_env: str | None, needs_rebuild: set[Path] | None = None) -> Self: - # To avoid circular imports - # Ideally, this class should be in a separate module - from cognite_toolkit._cdf_tk.commands.build_cmd import BuildCommand - - with tmp_build_directory() as build_dir: - cdf_toml = CDFToml.load() - if build_env is None: - config = BuildConfigYAML.load_default(organization_dir) - else: - config = BuildConfigYAML.load_from_directory(organization_dir, build_env) - if needs_rebuild is None: - # Use path syntax to select all modules in the source directory - config.environment.selected = [Path("")] - else: - # Use path syntax to select only the modules that need to be rebuilt - config.environment.selected = list(needs_rebuild) - build = BuildCommand(silent=True, skip_tracking=True).build_config( - build_dir=build_dir, - organization_dir=organization_dir, - config=config, - packages=cdf_toml.modules.packages, - clean=True, - verbose=False, - progress_bar=True, - ) - - new_build = cls( - filepath=organization_dir / cls.get_filename(build_env or DEFAULT_ENV), - modules=ModulesInfo(version=_version.__version__, modules=build), - ) - if needs_rebuild is not None and (existing := cls._get_existing(organization_dir, build_env or DEFAULT_ENV)): - # Merge the existing modules with the new modules - new_modules_by_path = {module.location.path: module for module in new_build.modules.modules} - existing_modules_by_path = {module.location.path: module for module in existing.modules.modules} - all_module_paths = set(new_modules_by_path) | set(existing_modules_by_path) - - module_list = BuiltModuleList( - [ - new_modules_by_path[path] if path in new_modules_by_path else existing_modules_by_path[path] - for path in all_module_paths - ] - ) - new_build.modules.modules = module_list - - new_build.dump_to_file() - return new_build - - @classmethod - def _get_existing(cls, organization_dir: Path, build_env: str) -> Self | None: - try: - existing = cls.load_from_directory(organization_dir, build_env) - except FileNotFoundError: - return None - if existing.modules.version != _version.__version__: - return None - return existing - - def dump(self) -> dict[str, Any]: - return { - "modules": self.modules.dump(), - } - - def dump_to_file(self) -> None: - dumped = self.dump() - # Avoid dumping pointer references: https://stackoverflow.com/questions/51272814/python-yaml-dumping-pointer-references - yaml.Dumper.ignore_aliases = lambda *args: True # type: ignore[method-assign] - content = yaml_safe_dump(dumped) - content = f"{self.top_warning}\n{content}" - safe_write(self.filepath, content) - - def compare_modules( - self, - current_modules: ModuleDirectories, - current_variables: BuildVariables, - resource_dirs: set[str] | None = None, - ) -> set[Path]: - current_module_by_path = {module.relative_path: module for module in current_modules} - cached_module_by_path_and_iteration = { - (module.location.path, module.iteration): module for module in self.modules.modules - } - needs_rebuild = set() - for path, current_module in current_module_by_path.items(): - if resource_dirs is not None and all( - resource_dir not in current_module.resource_directories for resource_dir in resource_dirs - ): - # The module does not contain any of the specified resources, so it does not need to be rebuilt. - continue - - if (path, None) not in cached_module_by_path_and_iteration and ( - path, - 1, - ) not in cached_module_by_path_and_iteration: - needs_rebuild.add(path) - continue - for no, current_module_variables in enumerate(current_variables.get_module_variables(current_module), 1): - cached_module = cached_module_by_path_and_iteration[(path, no)] - if current_module.hash != cached_module.location.hash: - needs_rebuild.add(path) - continue - if set(current_module_variables) != set(cached_module.build_variables): - needs_rebuild.add(path) - return needs_rebuild - - -class ModuleResources: - """This class is used to retrieve resource information from the build info. - - It is responsible for ensuring that the build info is up-to-date with the - latest changes in the source directory. - """ - - def __init__(self, organization_dir: Path, build_env: str | None) -> None: - self._organization_dir = organization_dir - self._build_env = build_env or DEFAULT_ENV - self._build_info: BuildInfo - try: - self._build_info = BuildInfo.load_from_directory(organization_dir, build_env or DEFAULT_ENV) - self._has_rebuilt = False - except (FileNotFoundError, KeyError): - # FileNotFound = Not run before. - # KeyError = Version mismatch/Changed format - self._build_info = BuildInfo.rebuild(organization_dir, build_env) - self._has_rebuilt = True - - @cached_property - def _current_modules(self) -> ModuleDirectories: - return ModuleDirectories.load(self._organization_dir, {Path("")}) - - @cached_property - def _current_variables(self) -> BuildVariables: - config_yaml = BuildConfigYAML.load_from_directory(self._organization_dir, self._build_env) - return BuildVariables.load_raw( - config_yaml.variables, self._current_modules.available_paths, self._current_modules.selected.available_paths - ) - - def list_resources( - self, id_type: type[T_ID], resource_dir: ResourceTypes, kind: str - ) -> BuiltFullResourceList[T_ID]: - if not self._has_rebuilt: - if needs_rebuild := self._build_info.compare_modules( - self._current_modules, self._current_variables, {resource_dir} - ): - self._build_info = BuildInfo.rebuild(self._organization_dir, self._build_env, needs_rebuild) - return self._build_info.modules.modules.get_resources(id_type, resource_dir, kind) - - def list(self) -> BuiltModuleList: - # Check if the build info is up to date - if not self._has_rebuilt: - if needs_rebuild := self._build_info.compare_modules(self._current_modules, self._current_variables): - self._build_info = BuildInfo.rebuild(self._organization_dir, self._build_env, needs_rebuild) - self._has_rebuilt = True - return self._build_info.modules.modules From b4f0eca801bc85d9692c845aeff9d2a72719153b Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 7 Sep 2026 18:20:00 +0200 Subject: [PATCH 02/31] tests: deleted outdated tests --- tests/test_unit/conftest.py | 25 +- .../test_cdf_tk/test_commands/test_build.py | 219 ------------------ .../test_cdf_tk/test_commands/test_clean.py | 16 -- .../test_cdf_tk/test_cruds/test_base.py | 68 +----- .../test_cruds/test_hosted_extractor.py | 28 --- 5 files changed, 4 insertions(+), 352 deletions(-) delete mode 100644 tests/test_unit/test_cdf_tk/test_commands/test_build.py delete mode 100644 tests/test_unit/test_cdf_tk/test_commands/test_clean.py delete mode 100644 tests/test_unit/test_cdf_tk/test_cruds/test_hosted_extractor.py diff --git a/tests/test_unit/conftest.py b/tests/test_unit/conftest.py index 8d34303d48..89c2793701 100644 --- a/tests/test_unit/conftest.py +++ b/tests/test_unit/conftest.py @@ -23,9 +23,8 @@ from cognite_toolkit._cdf_tk.client.resource_classes.data_modeling._data_model import DataModelResponseWithViews from cognite_toolkit._cdf_tk.client.resource_classes.migration import InstanceSource from cognite_toolkit._cdf_tk.client.testing import monkeypatch_toolkit_client -from cognite_toolkit._cdf_tk.commands import BuildCommand, ModulesCommand, RepoCommand -from cognite_toolkit._cdf_tk.constants import BUILD_ENVIRONMENT_FILE, MODULES -from cognite_toolkit._cdf_tk.data_classes._config_yaml import BuildEnvironment +from cognite_toolkit._cdf_tk.commands import ModulesCommand, RepoCommand +from cognite_toolkit._cdf_tk.constants import MODULES from cognite_toolkit._cdf_tk.utils import read_yaml_file from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables from tests.constants import BASE_URL, CDF_PROJECT, REPO_ROOT @@ -164,26 +163,6 @@ def buildable_modules_mutable( return organization_dir -@pytest.fixture -def build_environment( - build_tmp_path: Path, - complete_org_dir: Path, - env_vars_with_client: EnvironmentVariables, -) -> BuildEnvironment: - """Fixture that builds modules and returns the BuildEnvironment.""" - BuildCommand(silent=True, skip_tracking=True).execute( - verbose=False, - organization_dir=complete_org_dir, - build_dir=build_tmp_path, - selected=None, - no_clean=False, - client=env_vars_with_client.get_client(), - build_env_name="dev", - on_error="raise", - ) - return BuildEnvironment.load(read_yaml_file(build_tmp_path / BUILD_ENVIRONMENT_FILE), "dev", "build") - - @pytest.fixture def capture_print(monkeypatch: MonkeyPatch) -> PrintCapture: capture = PrintCapture() diff --git a/tests/test_unit/test_cdf_tk/test_commands/test_build.py b/tests/test_unit/test_cdf_tk/test_commands/test_build.py deleted file mode 100644 index ce804fda0a..0000000000 --- a/tests/test_unit/test_cdf_tk/test_commands/test_build.py +++ /dev/null @@ -1,219 +0,0 @@ -from __future__ import annotations - -import os -from pathlib import Path -from unittest.mock import MagicMock, patch - -import pytest -from _pytest.monkeypatch import MonkeyPatch -from cognite.client import data_modeling as dm - -from cognite_toolkit._cdf_tk.client.identifiers import RawDatabaseId -from cognite_toolkit._cdf_tk.client.resource_classes.data_modeling import SpaceResponse -from cognite_toolkit._cdf_tk.commands.build_cmd import BuildCommand -from cognite_toolkit._cdf_tk.data_classes import BuildVariables, Environment -from cognite_toolkit._cdf_tk.exceptions import ( - ToolkitMissingModuleError, -) -from cognite_toolkit._cdf_tk.feature_flags import Flags -from cognite_toolkit._cdf_tk.hints import ModuleDefinition -from cognite_toolkit._cdf_tk.resource_ios import RawDatabaseCRUD, TransformationIO -from cognite_toolkit._cdf_tk.tk_warnings import LowSeverityWarning, MissingDependencyWarning -from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables -from tests import data -from tests.test_unit.approval_client import ApprovalToolkitClient - - -@pytest.fixture(scope="session") -def dummy_environment() -> Environment: - return Environment( - name="dev", - project="my_project", - validation_type="dev", - selected=["none"], - ) - - -class TestBuildCommand: - def test_module_not_found_error(self, tmp_path: Path) -> None: - with pytest.raises(ToolkitMissingModuleError): - BuildCommand(print_warning=False).execute( - verbose=False, - build_dir=tmp_path, - organization_dir=data.PROJECT_WITH_BAD_MODULES, - selected=None, - build_env_name="no_module", - no_clean=False, - ) - - def test_module_with_non_resource_directories(self, tmp_path: Path) -> None: - cmd = BuildCommand(print_warning=False) - cmd.execute( - verbose=False, - build_dir=tmp_path, - organization_dir=data.PROJECT_WITH_BAD_MODULES, - selected=None, - build_env_name="ill_module", - no_clean=False, - ) - - assert len(cmd.warning_list) >= 1 - assert ( - LowSeverityWarning( - f"Module 'ill_made_module' has non-resource directories: ['spaces']. {ModuleDefinition.short()}" - ) - in cmd.warning_list - ) - - @pytest.mark.skipif(not Flags.GRAPHQL.is_enabled(), reason="GraphQL schema files will give warnings") - def test_custom_project_no_warnings(self, tmp_path: Path, monkeypatch: MonkeyPatch) -> None: - cmd = BuildCommand(print_warning=False) - monkeypatch.setenv("CDF_PROJECT", "some-project") - cmd.execute( - verbose=False, - build_dir=tmp_path, - organization_dir=data.PROJECT_NO_COGNITE_MODULES, - selected=None, - build_env_name="dev", - no_clean=False, - ) - - assert not cmd.warning_list, f"No warnings should be raised. Got warnings: {cmd.warning_list}" - # There are two transformations in the project, expect two transformation files - transformation_files = [ - f for f in (tmp_path / "transformations").iterdir() if f.is_file() and TransformationIO.is_supported_file(f) - ] - assert len(transformation_files) == 2 - - def test_build_complete_org_without_warnings( - self, - tmp_path: Path, - env_vars_with_client: EnvironmentVariables, - ) -> None: - cmd = BuildCommand(silent=True, skip_tracking=True) - with patch.dict( - os.environ, - {"CDF_PROJECT": env_vars_with_client.CDF_PROJECT, "CDF_CLUSTER": env_vars_with_client.CDF_CLUSTER}, - ): - cmd.execute( - verbose=False, - build_dir=tmp_path / "build", - organization_dir=data.COMPLETE_ORG, - selected=None, - build_env_name="dev", - no_clean=False, - ) - - assert not cmd.warning_list, ( - f"No warnings should be raised. Got {len(cmd.warning_list)} warnings: {cmd.warning_list}" - ) - - def test_build_no_warnings_when_space_exists_in_cdf( - self, env_vars_with_client: EnvironmentVariables, toolkit_client_approval: ApprovalToolkitClient, tmp_path: Path - ) -> None: - my_group = """name: gp_trigger_issue -sourceId: '1234567890123456789' -capabilities: -- dataModelInstancesAcl: - actions: - - READ - scope: - spaceIdScope: - spaceIds: - - existing-space -""" - filepath = tmp_path / "my_org" / "modules" / "my_module" / "auth" / "my.Group.yaml" - filepath.parent.mkdir(parents=True, exist_ok=True) - filepath.write_text(my_group) - - # Simulate that the space exists in CDF - toolkit_client_approval.append( - SpaceResponse, SpaceResponse(space="existing-space", is_global=False, created_time=1, last_updated_time=1) - ) - cmd = BuildCommand(silent=True, skip_tracking=True) - with patch.dict( - os.environ, - {"CDF_PROJECT": env_vars_with_client.CDF_PROJECT, "CDF_CLUSTER": env_vars_with_client.CDF_CLUSTER}, - ): - cmd.execute( - verbose=False, - organization_dir=tmp_path / "my_org", - build_dir=tmp_path / "build", - selected=None, - build_env_name=None, - no_clean=False, - client=toolkit_client_approval.mock_client, - on_error="raise", - ) - assert len(cmd.warning_list) == 0 - - def test_build_missing_raw_db_gives_warning( - self, env_vars_with_client: EnvironmentVariables, toolkit_client_approval: ApprovalToolkitClient, tmp_path: Path - ) -> None: - raw_group_yaml = """name: tmp_group_raw_acl_issue -sourceId: '1234567890123456789' -metadata: - origin: cognite-toolkit -capabilities: -- rawAcl: - actions: - - LIST - - READ - - WRITE - scope: - tableScope: - dbsToTables: - not_existing: [] -""" - filepath = tmp_path / "my_org" / "modules" / "my_module" / "auth" / "my.Group.yaml" - filepath.parent.mkdir(parents=True, exist_ok=True) - filepath.write_text(raw_group_yaml) - - cmd = BuildCommand(silent=True) - with patch.dict( - os.environ, - {"CDF_PROJECT": env_vars_with_client.CDF_PROJECT, "CDF_CLUSTER": env_vars_with_client.CDF_CLUSTER}, - ): - cmd.execute( - verbose=False, - organization_dir=tmp_path / "my_org", - build_dir=tmp_path / "build", - selected=None, - build_env_name=None, - no_clean=False, - client=toolkit_client_approval.mock_client, - on_error="raise", - ) - assert len(cmd.warning_list) == 1 - missing_dep = cmd.warning_list[0] - assert isinstance(missing_dep, MissingDependencyWarning) - assert missing_dep.dependency_type == RawDatabaseCRUD.resource_cls.__name__ - assert missing_dep.identifier == RawDatabaseId(name="not_existing") - - -class TestCheckYamlSemantics: - def test_build_valid_read_int_version(self) -> None: - cmd = BuildCommand(silent=True) - raw_yaml = """destination: - dataModel: - destinationType: CogniteFile - externalId: MyModel - space: my_space - version: 1_0_0 - instanceSpace: my_space - type: instances -externalId: some_external_id - """ - source_filepath = MagicMock(spec=Path) - source_filepath.read_text.return_value = raw_yaml - source_filepath.suffix = ".yaml" - source_filepath.read_bytes.return_value = raw_yaml.encode("utf-8") - - source_files = cmd._replace_variables( - [source_filepath], BuildVariables([]), TransformationIO.folder_name, Path("my_module"), verbose=False - ) - assert len(source_files) == 1 - source_file = source_files[0] - assert isinstance(source_file.loaded, dict) - actual = dm.DataModelId.load(source_file.loaded["destination"]["dataModel"]) - assert actual == dm.DataModelId("my_space", "MyModel", "1_0_0") diff --git a/tests/test_unit/test_cdf_tk/test_commands/test_clean.py b/tests/test_unit/test_cdf_tk/test_commands/test_clean.py deleted file mode 100644 index 75dac2d66a..0000000000 --- a/tests/test_unit/test_cdf_tk/test_commands/test_clean.py +++ /dev/null @@ -1,16 +0,0 @@ -from cognite_toolkit._cdf_tk.commands import CleanCommand -from cognite_toolkit._cdf_tk.data_classes._config_yaml import BuildEnvironment - - -class TestCleanCommandSelectModules: - """Test the CleanCommand._select_modules method.""" - - def test_select_specific_module( - self, - build_environment: BuildEnvironment, - ) -> None: - """Test that when a specific module is specified, it is returned.""" - cmd = CleanCommand(silent=True, skip_tracking=True) - result = cmd._select_modules(build_environment, module_str="my_example_module") - assert len(result) == 1 - assert result[0].dir.name == "my_example_module", f"Expected 'my_example_module', got {result[0].dir.name}" diff --git a/tests/test_unit/test_cdf_tk/test_cruds/test_base.py b/tests/test_unit/test_cdf_tk/test_cruds/test_base.py index 238aa94b21..2bf1414863 100644 --- a/tests/test_unit/test_cdf_tk/test_cruds/test_base.py +++ b/tests/test_unit/test_cdf_tk/test_cruds/test_base.py @@ -1,12 +1,8 @@ -import os -import shutil -import tempfile import typing from collections import Counter, defaultdict -from collections.abc import Iterable, Iterator -from contextlib import contextmanager +from collections.abc import Iterable from pathlib import Path -from typing import cast, get_args, get_origin +from typing import get_args, get_origin from unittest.mock import MagicMock import pytest @@ -20,7 +16,6 @@ from cognite.client.data_classes.hosted_extractors import Destination from pytest import MonkeyPatch -from cognite_toolkit._cdf_tk.cdf_toml import CDFToml from cognite_toolkit._cdf_tk.client.resource_classes.app_version import AppVersionResponse from cognite_toolkit._cdf_tk.client.resource_classes.cognite_file import CogniteFileResponse from cognite_toolkit._cdf_tk.client.resource_classes.filemetadata import FileMetadataResponse @@ -28,14 +23,6 @@ from cognite_toolkit._cdf_tk.client.resource_classes.streamlit_ import StreamlitResponse from cognite_toolkit._cdf_tk.client.resource_classes.transformation import TransformationResponse from cognite_toolkit._cdf_tk.client.testing import monkeypatch_toolkit_client -from cognite_toolkit._cdf_tk.commands import ( - BuildCommand, - ModulesCommand, -) -from cognite_toolkit._cdf_tk.constants import MODULES -from cognite_toolkit._cdf_tk.data_classes import ( - BuildConfigYAML, -) from cognite_toolkit._cdf_tk.feature_flags import FeatureFlag, Flags from cognite_toolkit._cdf_tk.resource_ios import ( CRUD_LIST, @@ -53,12 +40,8 @@ ResourceTypes, TransformationIO, WorkflowTriggerIO, - get_crud, ) -from cognite_toolkit._cdf_tk.utils import tmp_build_directory from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables -from tests.constants import REPO_ROOT -from tests.data import COMPLETE_ORG from tests.test_unit.approval_client import ApprovalToolkitClient from tests.test_unit.test_cdf_tk.constants import SNAPSHOTS_DIR_ALL from tests.test_unit.utils import FakeCogniteResourceGenerator @@ -214,53 +197,6 @@ def test_resource_types_is_up_to_date() -> None: assert not extra, f"Extra {extra=}" -@contextmanager -def tmp_org_directory() -> Iterator[Path]: - # Include worker ID to ensure each pytest-xdist worker has its own temp directory - worker_id = os.environ.get("PYTEST_XDIST_WORKER", "master") - org_dir = Path(tempfile.mkdtemp(prefix=f"orgdir.{worker_id}.", suffix=".tmp", dir=Path.cwd())) - try: - yield org_dir - finally: - shutil.rmtree(org_dir) - - -def cognite_module_files_with_loader() -> Iterable[tuple]: - with tmp_org_directory() as organization_dir, tmp_build_directory() as build_dir: - ModulesCommand(module_source_dir=COMPLETE_ORG / MODULES).init(organization_dir, select_all=True, clean=True) - cdf_toml = CDFToml.load(REPO_ROOT) - config = BuildConfigYAML.load_from_directory(organization_dir, "dev") - config.set_environment_variables() - # Use path syntax to select all modules in the source directory - config.environment.selected = [Path()] - - built_modules = BuildCommand().build_config( - build_dir=build_dir, - organization_dir=organization_dir, - config=config, - packages=cdf_toml.modules.packages, - clean=True, - verbose=False, - ) - for module in built_modules: - for resource_folder, resources in module.resources.items(): - for resource in resources: - try: - loader = get_crud(resource_folder, resource.kind) - except ValueError: - # Cannot find loader for resource kind - continue - filepath = cast(Path, resource.destination) - if issubclass(loader, ResourceIO): - raw = yaml.CSafeLoader(filepath.read_text()).get_data() - - if isinstance(raw, dict): - yield pytest.param(loader, raw, id=f"{module.name} - {filepath.stem} - dict") - elif isinstance(raw, list): - for no, item in enumerate(raw): - yield pytest.param(loader, item, id=f"{module.name} - {filepath.stem} - list {no}") - - def sensitive_strings_test_cases() -> Iterable[tuple]: yield pytest.param( WorkflowTriggerIO, diff --git a/tests/test_unit/test_cdf_tk/test_cruds/test_hosted_extractor.py b/tests/test_unit/test_cdf_tk/test_cruds/test_hosted_extractor.py deleted file mode 100644 index 65516f9697..0000000000 --- a/tests/test_unit/test_cdf_tk/test_cruds/test_hosted_extractor.py +++ /dev/null @@ -1,28 +0,0 @@ -from pathlib import Path - -from cognite_toolkit._cdf_tk.commands.build_cmd import BuildCommand -from cognite_toolkit._cdf_tk.resource_ios import HostedExtractorSourceIO -from cognite_toolkit._cdf_tk.tk_warnings import MissingRequiredParameterWarning -from tests.data import COMPLETE_ORG - - -class TestHostedExtractorDestinationLoader: - def test_hosted_extractor_destination_spec(self, tmp_path: Path) -> None: - cmd = BuildCommand(print_warning=False) - cmd.execute( - verbose=False, - build_dir=tmp_path, - organization_dir=COMPLETE_ORG, - selected=None, - build_env_name="dev", - no_clean=False, - ) - - warns = [ - w - for w in cmd.warning_list - if isinstance(w, MissingRequiredParameterWarning) - and HostedExtractorSourceIO.folder_name in w.filepath.parts - ] - - assert len(warns) == 0, warns From c116b3d6299650ca7d90ea3ff51255b47bdb822a Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 7 Sep 2026 18:35:15 +0200 Subject: [PATCH 03/31] tests: updated build v1 to v2 tests --- 0.8.192 | 0 remove-build-v1-from-run-commands | 0 .../test_build_config_yaml.py | 24 ++-- tests/test_unit/test_cli/test_behavior.py | 110 +++++++----------- .../test_cli/test_command_sequences.py | 68 +++++------ 5 files changed, 80 insertions(+), 122 deletions(-) create mode 100644 0.8.192 create mode 100644 remove-build-v1-from-run-commands diff --git a/0.8.192 b/0.8.192 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/remove-build-v1-from-run-commands b/remove-build-v1-from-run-commands new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_unit/test_cdf_tk/test_data_classes/test_build_config_yaml.py b/tests/test_unit/test_cdf_tk/test_data_classes/test_build_config_yaml.py index d13fe946b2..614f785a40 100644 --- a/tests/test_unit/test_cdf_tk/test_data_classes/test_build_config_yaml.py +++ b/tests/test_unit/test_cdf_tk/test_data_classes/test_build_config_yaml.py @@ -1,24 +1,22 @@ from __future__ import annotations -from cognite_toolkit._cdf_tk.cdf_toml import CDFToml -from cognite_toolkit._cdf_tk.commands import BuildCommand -from cognite_toolkit._cdf_tk.data_classes import BuildConfigYAML +from cognite_toolkit._cdf_tk.commands import BuildV2Command +from cognite_toolkit._cdf_tk.commands.build_v2.data_classes import BuildParameters +from cognite_toolkit._cdf_tk.constants import MODULES from cognite_toolkit._cdf_tk.resource_ios import CRUDS_BY_FOLDER_NAME -from cognite_toolkit._cdf_tk.utils import iterate_modules from tests.data import PROJECT_FOR_TEST from tests.test_unit.test_cdf_tk.constants import BUILD_DIR class TestBuildConfigYAML: - def test_build_config_create_valid_build_folder(self, project_for_test_config_dev_yaml: str) -> None: - build_env_name = "dev" - cdf_toml = CDFToml.load(PROJECT_FOR_TEST) - config = BuildConfigYAML.load_from_directory(PROJECT_FOR_TEST, build_env_name) - available_modules = {module.name for module, _ in iterate_modules(PROJECT_FOR_TEST)} - config.environment.selected = list(available_modules) - - BuildCommand(silent=True).build_config( - BUILD_DIR, PROJECT_FOR_TEST, config=config, packages=cdf_toml.modules.packages, clean=True, verbose=False + def test_build_config_create_valid_build_folder(self) -> None: + BuildV2Command(silent=True, skip_tracking=True).build( + parameters=BuildParameters( + organization_dir=PROJECT_FOR_TEST, + build_dir=BUILD_DIR, + config_yaml=PROJECT_FOR_TEST / "config.dev.yaml", + user_selected_modules=[f"{MODULES}/"], + ), ) # The resulting build folder should only have subfolders that are matching the folder name diff --git a/tests/test_unit/test_cli/test_behavior.py b/tests/test_unit/test_cli/test_behavior.py index d919a1eddf..131626bdc5 100644 --- a/tests/test_unit/test_cli/test_behavior.py +++ b/tests/test_unit/test_cli/test_behavior.py @@ -1,8 +1,6 @@ -import os import sys from pathlib import Path from typing import cast -from unittest.mock import MagicMock, patch import pytest import yaml @@ -13,7 +11,7 @@ from cognite_toolkit._cdf_tk import cdf_toml from cognite_toolkit._cdf_tk.client import ToolkitClient -from cognite_toolkit._cdf_tk.client.identifiers import NameId, WorkflowVersionId +from cognite_toolkit._cdf_tk.client.identifiers import WorkflowVersionId from cognite_toolkit._cdf_tk.client.resource_classes.agent import AgentResponse from cognite_toolkit._cdf_tk.client.resource_classes.data_modeling import ( ContainerId, @@ -22,7 +20,6 @@ DataModelId, DataModelResponse, Float64Property, - SpaceId, SpaceResponse, TextProperty, ViewCorePropertyResponse, @@ -51,20 +48,17 @@ WorkflowVersionResponse, ) from cognite_toolkit._cdf_tk.commands import ( - BuildCommand, BuildV2Command, DeployOptions, DeployV2Command, DumpResourceCommand, PullV2Command, ) -from cognite_toolkit._cdf_tk.commands.build_v2.data_classes import BuildParameters +from cognite_toolkit._cdf_tk.commands.build_v2.data_classes import BuildParameters, ConsistencyError from cognite_toolkit._cdf_tk.commands.dump_resource import DataModelFinder, WorkflowFinder from cognite_toolkit._cdf_tk.constants import MODULES -from cognite_toolkit._cdf_tk.data_classes import BuildConfigYAML, Environment -from cognite_toolkit._cdf_tk.exceptions import ToolkitDuplicatedModuleError +from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError from cognite_toolkit._cdf_tk.resource_ios import AgentIO, LocationFilterIO, WorkflowVersionIO -from cognite_toolkit._cdf_tk.tk_warnings import MissingDependencyWarning from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables from cognite_toolkit._cdf_tk.utils.file import yaml_safe_dump from tests.constants import CDF_PROJECT, chdir @@ -123,23 +117,14 @@ def test_inject_custom_environmental_variables( def test_duplicated_modules(build_tmp_path: Path) -> None: - config = MagicMock(spec=BuildConfigYAML) - config.environment = MagicMock(spec=Environment) - config.environment.name = "dev" - config.environment.selected = ["module1"] - with pytest.raises(ToolkitDuplicatedModuleError) as err: - BuildCommand().build_config( - build_dir=build_tmp_path, - organization_dir=PROJECT_WITH_DUPLICATES, - config=config, - packages={}, + with pytest.raises(ToolkitValueError, match="ambiguous selected"): + BuildV2Command(silent=True, skip_tracking=True).build( + parameters=BuildParameters( + organization_dir=PROJECT_WITH_DUPLICATES, + build_dir=build_tmp_path, + user_selected_modules=["module1"], + ), ) - l1, l2, l3, l4, l5 = map(str.strip, str(err.value).splitlines()) - assert l1 == "Ambiguous module selected in config.dev.yaml:" - assert l2 == "module1 exists in:" - assert l3 == "modules/examples/module1" - assert l4 == "modules/models/module1" - assert l5.startswith("You can use the path syntax to disambiguate between modules with the same name") def test_pull_dataset( @@ -579,23 +564,18 @@ def test_build_custom_project( build_tmp_path: Path, ) -> None: expected_resources = { - "timeseries", "data_modeling", "data_sets", - "raw", "extraction_pipelines", "transformations", "robotics", } - BuildCommand(silent=True).execute( - organization_dir=PROJECT_NO_COGNITE_MODULES, - build_dir=build_tmp_path, - selected=None, - build_env_name="dev", - no_clean=False, - client=None, - on_error="raise", - verbose=False, + BuildV2Command(silent=True, skip_tracking=True).build( + parameters=BuildParameters( + organization_dir=PROJECT_NO_COGNITE_MODULES, + build_dir=build_tmp_path, + config_yaml=PROJECT_NO_COGNITE_MODULES / "config.dev.yaml", + ), ) actual_resources = {path.name for path in build_tmp_path.iterdir() if path.is_dir()} @@ -611,15 +591,12 @@ def test_build_project_selecting_parent_path( build_tmp_path: Path, ) -> None: expected_resources = {"auth", "data_modeling", "files", "transformations", "data_sets"} - BuildCommand(silent=True).execute( - organization_dir=PROJECT_FOR_TEST, - build_dir=build_tmp_path, - selected=None, - build_env_name="dev", - no_clean=False, - client=None, - on_error="raise", - verbose=False, + BuildV2Command(silent=True, skip_tracking=True).build( + parameters=BuildParameters( + organization_dir=PROJECT_FOR_TEST, + build_dir=build_tmp_path, + config_yaml=PROJECT_FOR_TEST / "config.dev.yaml", + ), ) actual_resources = {path.name for path in build_tmp_path.iterdir() if path.is_dir()} @@ -666,15 +643,12 @@ def test_deploy_group_with_unknown_acl( def test_build_project_with_only_top_level_variables( build_tmp_path: Path, ) -> None: - BuildCommand(silent=True).execute( - organization_dir=PROJECT_NO_COGNITE_MODULES, - build_dir=build_tmp_path, - selected=None, - build_env_name="top_level_variables", - no_clean=False, - client=None, - on_error="raise", - verbose=False, + BuildV2Command(silent=True, skip_tracking=True).build( + parameters=BuildParameters( + organization_dir=PROJECT_NO_COGNITE_MODULES, + build_dir=build_tmp_path, + config_yaml=PROJECT_NO_COGNITE_MODULES / "config.top_level_variables.yaml", + ), ) assert build_tmp_path.exists() @@ -1071,21 +1045,17 @@ def test_warning_missing_dependency( (my_org / "config.dev.yaml").write_text(default_config_dev_yaml, encoding="utf-8") - cmd = BuildCommand(silent=True, skip_tracking=True) - with patch.dict(os.environ, {"CDF_PROJECT": CDF_PROJECT}): - cmd.execute( - verbose=False, + folder = BuildV2Command(silent=True, skip_tracking=True).build( + client=env_vars_with_client.get_client(), + parameters=BuildParameters( organization_dir=my_org, build_dir=tmp_path / "build", - selected=None, - build_env_name="dev", - no_clean=False, - client=env_vars_with_client.get_client(), - on_error="raise", - ) - assert len(cmd.warning_list) == 1 - warning = cmd.warning_list[0] - assert isinstance(warning, MissingDependencyWarning) - assert warning.identifier == SpaceId(space="my_non_existent_space") - - assert warning.required_by == {(NameId(name="scoped_group"), yaml_filepath.relative_to(my_org))} + config_yaml=my_org / "config.dev.yaml", + ), + ) + insights = [insight for insight in folder.all_insights if insight.code == "UNKNOWN-REFERENCE"] + assert len(insights) == 1 + insight = insights[0] + assert isinstance(insight, ConsistencyError) + assert insight.message == "Unknown reference to spaces with id 'my_non_existent_space'" + assert insight.source_file == "modules/my_module/auth/scoped_group.Group.yaml" diff --git a/tests/test_unit/test_cli/test_command_sequences.py b/tests/test_unit/test_cli/test_command_sequences.py index 86badf208a..cfa8ced3b9 100644 --- a/tests/test_unit/test_cli/test_command_sequences.py +++ b/tests/test_unit/test_cli/test_command_sequences.py @@ -16,9 +16,7 @@ from pytest import MonkeyPatch from cognite_toolkit._cdf_tk.commands import ( - BuildCommand, BuildV2Command, - CleanCommand, DeployOptions, DeployV2Command, ) @@ -30,7 +28,6 @@ from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables from tests.data import BUILDABLE_PACKAGE, COMPLETE_ORG, COMPLETE_ORG_ALPHA_FLAGS from tests.test_unit.approval_client import ApprovalToolkitClient -from tests.test_unit.utils import mock_read_yaml_file THIS_DIR = Path(__file__).resolve().parent SNAPSHOTS_DIR = THIS_DIR / "test_build_deploy_snapshots" @@ -49,23 +46,6 @@ def find_all_modules() -> Iterator[Path]: yield pytest.param(module, id=f"{module.parent.name}/{module.name}") -def mock_environments_yaml_file(module_path: Path, monkeypatch: MonkeyPatch) -> None: - return mock_read_yaml_file( - { - "config.dev.yaml": { - "environment": { - "name": "dev", - "project": "pytest-project", - "type": "dev", - "selected": [module_path.name], - } - } - }, - monkeypatch, - modify=True, - ) - - @pytest.mark.parametrize("module_path", list(find_all_modules())) def test_build_deploy_module( module_path: Path, @@ -182,28 +162,38 @@ def test_init_build_clean( buildable_modules: Path, data_regression, ) -> None: - mock_environments_yaml_file(module_path, monkeypatch) - - BuildCommand(silent=True, skip_tracking=True).execute( - verbose=False, - organization_dir=buildable_modules, - build_dir=build_tmp_path, - selected=None, - no_clean=False, + BuildV2Command(silent=True, skip_tracking=True).build( client=env_vars_with_client.get_client(), - build_env_name="dev", - on_error="raise", + parameters=BuildParameters( + organization_dir=buildable_modules, + build_dir=build_tmp_path, + config_yaml=buildable_modules / "config.dev.yaml", + user_selected_modules=[module_path.name], + ), ) - CleanCommand(silent=True, skip_tracking=True).execute( - env_vars=env_vars_with_client, - build_dir=build_tmp_path, - build_env_name="dev", - dry_run=False, - include=None, - module_str=module_path.name, - all_modules=True, - verbose=False, + monkeypatch.setattr( + "cognite_toolkit._cdf_tk.commands.deploy_v2.command.confirm_by_typing_project_name", + lambda msg, client: True, ) + with patch.dict( + os.environ, + {"CDF_ENVIRON": "pytest", "CDF_BUILD_TYPE": "dev"}, + ): + DeployV2Command(silent=True, skip_tracking=True).deploy( + user_build_dir=build_tmp_path, + env_vars=env_vars_with_client, + options=DeployOptions( + operation="clean", + cdf_project=env_vars_with_client.CDF_PROJECT, + drop=True, + dry_run=False, + include=None, + verbose=False, + drop_data=True, + force_update=False, + environment_variables=env_vars_with_client.dump(), + ), + ) not_mocked = toolkit_client_approval.not_mocked_calls() assert not not_mocked, ( From 45e30e860c6253b7e991b396d1f7c93e80547f1a Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 7 Sep 2026 18:35:58 +0200 Subject: [PATCH 04/31] refactor: deleted build v1 commands --- 0.8.193 | 0 cognite_toolkit/_cdf_tk/commands/__init__.py | 2 - cognite_toolkit/_cdf_tk/commands/build_cmd.py | 623 ------------------ 3 files changed, 625 deletions(-) create mode 100644 0.8.193 delete mode 100644 cognite_toolkit/_cdf_tk/commands/build_cmd.py diff --git a/0.8.193 b/0.8.193 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cognite_toolkit/_cdf_tk/commands/__init__.py b/cognite_toolkit/_cdf_tk/commands/__init__.py index b1f6ba43c2..a460d4d31a 100644 --- a/cognite_toolkit/_cdf_tk/commands/__init__.py +++ b/cognite_toolkit/_cdf_tk/commands/__init__.py @@ -8,7 +8,6 @@ from ._upload import UploadCommand from .about import AboutCommand from .auth import AuthCommand -from .build_cmd import BuildCommand from .build_v2.build_v2 import BuildV2Command from .clean import CleanCommand from .deploy_v2.command import DeploymentStep, DeployOptions, DeployV2Command @@ -25,7 +24,6 @@ __all__ = [ "AboutCommand", "AuthCommand", - "BuildCommand", "BuildV2Command", "CleanCommand", "DeployOptions", diff --git a/cognite_toolkit/_cdf_tk/commands/build_cmd.py b/cognite_toolkit/_cdf_tk/commands/build_cmd.py deleted file mode 100644 index 92f4afd9e0..0000000000 --- a/cognite_toolkit/_cdf_tk/commands/build_cmd.py +++ /dev/null @@ -1,623 +0,0 @@ -import contextlib -import re -from collections import defaultdict -from collections.abc import Hashable, Iterable, Sequence -from pathlib import Path -from typing import Any, Literal, cast - -import yaml -from cognite.client.exceptions import CogniteAPIError -from rich import print -from rich.panel import Panel -from rich.progress import track - -from cognite_toolkit._cdf_tk.builders import Builder, create_builder -from cognite_toolkit._cdf_tk.cdf_toml import CDFToml -from cognite_toolkit._cdf_tk.client import ToolkitClient -from cognite_toolkit._cdf_tk.client.identifiers import RawDatabaseId -from cognite_toolkit._cdf_tk.commands._base import ToolkitCommand -from cognite_toolkit._cdf_tk.constants import ( - _RUNNING_IN_BROWSER, - BUILD_FOLDER_ENCODING, - DEFAULT_ENV, - HINT_LEAD_TEXT, - ROOT_MODULES, - TEMPLATE_VARS_FILE_SUFFIXES, - URL, - YAML_SUFFIX, -) -from cognite_toolkit._cdf_tk.data_classes import ( - BuildConfigYAML, - BuildDestinationFile, - BuildSourceFile, - BuildVariables, - BuiltModule, - BuiltModuleList, - BuiltResource, - BuiltResourceList, - ModuleDirectories, - ModuleLocation, - SourceLocation, - SourceLocationEager, - SourceLocationLazy, -) -from cognite_toolkit._cdf_tk.exceptions import ( - ToolkitError, - ToolkitYAMLFormatError, -) -from cognite_toolkit._cdf_tk.hints import Hint, ModuleDefinition, verify_module_directory -from cognite_toolkit._cdf_tk.resource_ios import ( - ContainerCRUD, - DataCRUD, - DataModelIO, - DataSetsIO, - ExtractionPipelineConfigIO, - FileCRUD, - LocationFilterIO, - NodeCRUD, - RawDatabaseCRUD, - RawTableCRUD, - ResourceIO, - SpaceCRUD, - TransformationIO, - ViewIO, -) -from cognite_toolkit._cdf_tk.tk_warnings import ( - DuplicatedItemWarning, - FileReadWarning, - LowSeverityWarning, - MediumSeverityWarning, - MissingDependencyWarning, - UnresolvedVariableWarning, - WarningList, -) -from cognite_toolkit._cdf_tk.tk_warnings.fileread import MissingRequiredIdentifierWarning -from cognite_toolkit._cdf_tk.utils import ( - calculate_hash, - humanize_collection, - quote_int_value_by_key_in_yaml, - read_yaml_content, - safe_read, - safe_write, - stringify_value_by_key_in_yaml, -) -from cognite_toolkit._cdf_tk.utils.file import safe_rmtree -from cognite_toolkit._cdf_tk.utils.modules import parse_user_selected_modules -from cognite_toolkit._cdf_tk.validation import ( - validate_data_set_is_set, - validate_module_selection, - validate_modules_variables, - validate_resource_yaml_pydantic, -) -from cognite_toolkit._version import __version__ - - -class BuildCommand(ToolkitCommand): - def __init__( - self, - print_warning: bool = True, - skip_tracking: bool = False, - silent: bool = False, - client: ToolkitClient | None = None, - ) -> None: - super().__init__(print_warning, skip_tracking, silent, client) - self.existing_resources_by_loader: dict[type[ResourceIO], set[Hashable]] = defaultdict(set) - self.instantiated_loaders: dict[type[ResourceIO], ResourceIO] = {} - - # Built State - self._module_names_by_variable_key: dict[str, list[str]] = defaultdict(list) - self._builder_by_resource_folder: dict[str, Builder] = {} - self._ids_by_resource_type: dict[type[ResourceIO], dict[Hashable, SourceLocation]] = defaultdict(dict) - self._dependencies_by_required: dict[tuple[type[ResourceIO], Hashable], list[tuple[Hashable, Path]]] = ( - defaultdict(list) - ) - self._has_built = False - self._printed_variable_tree_structure_hint = False - - def execute( - self, - verbose: bool, - organization_dir: Path, - build_dir: Path, - selected: list[str | Path] | None, - build_env_name: str | None, - no_clean: bool, - client: ToolkitClient | None = None, - on_error: Literal["continue", "raise"] = "continue", - ) -> BuiltModuleList: - if organization_dir in {Path("."), Path("./")}: - organization_dir = Path.cwd() - verify_module_directory(organization_dir, build_env_name) - - cdf_toml = CDFToml.load() - - if (organization_dir / BuildConfigYAML.get_filename(build_env_name or DEFAULT_ENV)).exists(): - config = BuildConfigYAML.load_from_directory(organization_dir, build_env_name or DEFAULT_ENV) - else: - # Loads the default environment - config = BuildConfigYAML.load_default(organization_dir) - - if selected: - config.environment.selected = parse_user_selected_modules(selected, organization_dir) - - directory_name = "current directory" if organization_dir == Path(".") else f"project '{organization_dir!s}'" - root_modules = [ - module_dir for root_module in ROOT_MODULES if (module_dir := organization_dir / root_module).exists() - ] - module_locations = "\n".join(f" - Module directory '{root_module!s}'" for root_module in root_modules) - print( - Panel( - f"Building {directory_name}:\n - Toolkit Version '{__version__!s}'\n" - f" - Environment name {build_env_name!r}, validation-type {config.environment.validation_type!r}.\n" - f" - Config '{config.filepath!s}'" - f"\n{module_locations}", - expand=False, - ) - ) - - return self.build_config( - build_dir=build_dir, - organization_dir=organization_dir, - config=config, - packages=cdf_toml.modules.packages, - clean=not no_clean, - verbose=verbose, - client=client, - on_error=on_error, - ) - - def build_config( - self, - build_dir: Path, - organization_dir: Path, - config: BuildConfigYAML, - packages: dict[str, list[str]], - clean: bool = False, - verbose: bool = False, - client: ToolkitClient | None = None, - progress_bar: bool = False, - on_error: Literal["continue", "raise"] = "continue", - ) -> BuiltModuleList: - is_populated = build_dir.exists() and any(build_dir.iterdir()) - if is_populated and clean: - safe_rmtree(build_dir) - build_dir.mkdir() - if not _RUNNING_IN_BROWSER: - self.console(f"Cleaned existing build directory {build_dir!s}.") - elif is_populated and not _RUNNING_IN_BROWSER: - self.warn( - LowSeverityWarning("Build directory is not empty. Run without --no-clean to remove existing files.") - ) - elif build_dir.exists() and not _RUNNING_IN_BROWSER: - self.console("Build directory does already exist and is empty. No need to create it.") - else: - build_dir.mkdir(exist_ok=True) - - if issue := config.validate_environment(): - self.warn(issue) - - user_selected_modules = config.environment.get_selected_modules(packages) - modules = ModuleDirectories.load(organization_dir, user_selected_modules) - module_warnings = validate_module_selection(modules, config, packages, user_selected_modules, organization_dir) - if module_warnings: - self.warning_list.extend(module_warnings) - if self.print_warning: - print(str(module_warnings)) - - if verbose: - self.console("Selected packages:") - selected_packages = [package for package in packages if package in config.environment.selected] - if len(selected_packages) == 0: - self.console(" None", prefix="") - for package in selected_packages: - self.console(f" {package}", prefix="") - self.console("Selected modules:") - for module in [module.name for module in modules.selected]: - self.console(f" {module}", prefix="") - - variables = BuildVariables.load_raw( - config.variables, modules.available_paths, modules.selected.available_paths, config.filepath - ) - warnings = validate_modules_variables(variables.selected, config.filepath) - if warnings: - self.console( - f"Found the following warnings in config.{config.environment.name}.yaml:", - prefix="[bold red]Warning:[/]", - ) - for warning in warnings: - if self.print_warning: - print(f" {warning.get_message()}") - - # Setup state before building modules - self._module_names_by_variable_key.clear() - self._builder_by_resource_folder.clear() - for variable in variables: - for module_location in modules: - if variable.location in module_location.relative_path.parts: - self._module_names_by_variable_key[variable.key].append(module_location.name) - if self._has_built: - # Todo: Reset of state?? - raise RuntimeError("In the build command, the `build_config` method should only be called once.") - else: - self._has_built = True - - built_modules = self.build_modules(modules.selected, build_dir, variables, verbose, progress_bar, on_error) - - self._check_missing_dependencies(organization_dir, client) - - build_environment = config.create_build_environment(built_modules, modules.selected) - build_environment.dump_to_file(build_dir) - if not _RUNNING_IN_BROWSER: - self.console(f"Build complete. Files are located in {build_dir!s}/") - return built_modules - - def build_modules( - self, - modules: ModuleDirectories, - build_dir: Path, - variables: BuildVariables, - verbose: bool = False, - progress_bar: bool = False, - on_error: Literal["continue", "raise"] = "continue", - ) -> BuiltModuleList: - build = BuiltModuleList() - warning_count = len(self.warning_list) - if progress_bar: - modules_iter = cast( - Iterable[ModuleLocation], track(modules, description="Building modules", transient=True) - ) - else: - modules_iter = modules - for module in modules_iter: - if verbose: - self.console(f"Processing module {module.name}") - module_variable_sets = variables.get_module_variables(module) - last_identifiers: set[tuple[Hashable, Path]] = set() - for iteration, module_variables in enumerate(module_variable_sets, 1): - try: - built_module_resources = self._build_module_resources(module, build_dir, module_variables, verbose) - except ToolkitError as err: - if on_error == "raise": - raise - - suffix = "" if len(module_variable_sets) == 1 else f" ({iteration} of {len(module_variable_sets)})" - - print(f" [bold red]Failed Building:([/][red]: {module.name}{suffix}") - print(f" [bold red]ERROR ([/][red]{type(err).__name__}[/][bold red]):[/] {err}") - built_status = type(err).__name__ - built_module_resources = {} - else: - built_status = "Success" - - # Check for duplicates - identifiers = { - (resource.identifier, resource.source.path) - for resources in built_module_resources.values() - for resource in resources - } - if duplicates := (identifiers & last_identifiers): - duplicate_warnings = WarningList[FileReadWarning]() - for identifier, path in duplicates: - duplicate_warnings.append(DuplicatedItemWarning(path, identifier, path)) - self.warning_list.extend(duplicate_warnings) - print(str(duplicate_warnings)) - print( - f" {HINT_LEAD_TEXT}This is likely due to missing variable in the " - f"identifier when using the module {module.name!r} as a template." - ) - last_identifiers = identifiers - - module_warnings = len(self.warning_list) - warning_count - warning_count = len(self.warning_list) - - name = module.name if len(module_variable_sets) == 1 else f"{module.name} (iteration {iteration})" - built_module = BuiltModule( - name=name, - location=SourceLocationLazy( - path=module.relative_path, - absolute_path=module.dir, - ), - build_variables=module_variables, - resources=built_module_resources, - warning_count=module_warnings, - status=built_status, - iteration=iteration, - module_id=module.module_id, - package_id=module.package_id, - ) - build.append(built_module) - - return build - - def _build_module_resources( - self, - module: ModuleLocation, - build_dir: Path, - module_variables: BuildVariables, - verbose: bool, - ) -> dict[str, BuiltResourceList]: - build_resources_by_folder: dict[str, BuiltResourceList] = defaultdict(BuiltResourceList) - if not_resource_directory := module.not_resource_directories: - self.warn( - LowSeverityWarning( - f"Module {module.dir.name!r} has non-resource directories: {sorted(not_resource_directory)}. {ModuleDefinition.short()}" - ) - ) - - for resource_name, resource_files in module.source_paths_by_resource_folder.items(): - source_files = self._replace_variables(resource_files, module_variables, resource_name, module.dir, verbose) - - if resource_name == "data_models": - resource_name = "data_modeling" - self.warn( - MediumSeverityWarning( - "The resource folder 'data_models' is deprecated and will be removed in v1.0. " - "Please rename the folder to 'data_modeling'." - ) - ) - - builder = self._get_builder(build_dir, resource_name) - - built_resources = BuiltResourceList[Hashable]() - for destination in builder.build(source_files, module): - if not isinstance(destination, BuildDestinationFile): - for warning in destination: - self.warn(warning) - continue - if destination.loader is FileCRUD: - # This is a content file that we should not copy to the build directory. - continue - - safe_write(destination.path, destination.content, encoding=BUILD_FOLDER_ENCODING) - if issubclass(destination.loader, DataCRUD): - continue - - file_warnings, identifiers_kind_pairs = self.check_built_resource( - destination.loaded, - destination.loader, - destination.source, - ) - file_warnings.extend(destination.warnings) - - if file_warnings: - self.warning_list.extend(file_warnings) - # Here we do not use the self.warn method as we want to print the warnings as a group. - if self.print_warning: - print(str(file_warnings)) - - built_source = BuiltResourceList( - [ - BuiltResource( - identifier, - destination.source, - kind, - destination.path, - extra_sources=destination.extra_sources, - ) - for identifier, kind in identifiers_kind_pairs - ] - ) - built_resources.extend(built_source) - - builder.validate_directory(built_resources, module) - - build_resources_by_folder[resource_name].extend(built_resources) - - return build_resources_by_folder - - def _get_builder(self, build_dir: Path, resource_name: str) -> Builder: - if resource_name not in self._builder_by_resource_folder: - self._builder_by_resource_folder[resource_name] = create_builder(resource_name, build_dir, self.warn) - builder = self._builder_by_resource_folder[resource_name] - return builder - - def _replace_variables( - self, - resource_files: Sequence[Path], - variables: BuildVariables, - resource_name: str, - module_dir: Path, - verbose: bool, - ) -> list[BuildSourceFile]: - source_files: list[BuildSourceFile] = [] - - for source_path in resource_files: - if source_path.suffix.lower() not in TEMPLATE_VARS_FILE_SUFFIXES: - continue - - if verbose: - self.console(f"Processing file {source_path.name}...") - - content = safe_read(source_path) - # We cannot use the content as the basis for hash as this have been encoded. - # Instead, we use the source path, which will hash the bytes of the file directly, - # which is what we do in the deploy step to verify that the source file has not changed. - source = SourceLocationEager(source_path, calculate_hash(source_path, shorten=True)) - - content = variables.replace(content, source_path) - - replace_warnings = self._check_variables_replaced(content, module_dir, source_path) - - if source_path.suffix not in YAML_SUFFIX: - source_files.append(BuildSourceFile(source, content, None)) - continue - - if resource_name in { - TransformationIO.folder_name, - DataModelIO.folder_name, - LocationFilterIO.folder_name, - }: - # Ensure that all keys that are version gets read as strings. - # This is required by DataModels, Views, and Transformations that reference DataModels and Views. - content = quote_int_value_by_key_in_yaml(content, key="version") - - if resource_name in ExtractionPipelineConfigIO.folder_name: - # Ensure that the config variables are stings. - # This is required by ExtractionPipelineConfig - content = stringify_value_by_key_in_yaml(content, key="config") - try: - loaded = read_yaml_content(content) - except yaml.YAMLError as e: - message = ( - f"YAML validation error for {source_path.as_posix()!r} after substituting config variables:\n{e}" - ) - if unresolved_variables := [ - w.variable.removesuffix("}}").removeprefix("{{").strip() - for w in replace_warnings - if isinstance(w, UnresolvedVariableWarning) - ]: - variable_str = humanize_collection(set(unresolved_variables)) - source_str = variables.source_path.as_posix() if variables.source_path else "config.[ENV].yaml" - suffix = "s" if len(unresolved_variables) > 1 else "" - message += f"\n{HINT_LEAD_TEXT}Add the following variable{suffix} to the {source_str!r} file: {variable_str!r}." - message += ( - f"\nRead more about build variables: {Hint.link(URL.build_variables, URL.build_variables)}." - ) - - raise ToolkitYAMLFormatError(message) - - source_files.append(BuildSourceFile(source, content, loaded)) - - return source_files - - def _check_variables_replaced(self, content: str, module: Path, source_path: Path) -> WarningList[FileReadWarning]: - all_unmatched = re.findall(pattern=r"\{\{.*?\}\}", string=content) - warning_list = WarningList[FileReadWarning]() - for unmatched in all_unmatched: - warning_list.append(UnresolvedVariableWarning(source_path, unmatched)) - variable = unmatched[2:-2] - if module_names := self._module_names_by_variable_key.get(variable): - module_str = ( - f"{module_names[0]!r}" - if len(module_names) == 1 - else (", ".join(module_names[:-1]) + f" or {module_names[-1]}") - ) - if not self._printed_variable_tree_structure_hint: - self._printed_variable_tree_structure_hint = True - self.console( - f"The variables in 'config.[ENV].yaml' need to be organised in a tree structure following" - f"\n the folder structure of the modules, but can also be moved up the config hierarchy to be shared between modules." - f"\n The variable {variable!r} is defined in the variable section{'s' if len(module_names) > 1 else ''} {module_str}." - f"\n Check that {'these paths reflect' if len(module_names) > 1 else 'this path reflects'} " - f"the location of {module.as_posix()}.", - prefix=" [bold green]Hint:[/] ", - ) - self.warning_list.extend(warning_list) - if self.print_warning and warning_list: - print(str(warning_list)) - return warning_list - - def _check_missing_dependencies(self, project_config_dir: Path, client: ToolkitClient | None = None) -> None: - existing = {(resource_cls, id_) for resource_cls, ids in self._ids_by_resource_type.items() for id_ in ids} - missing_dependencies = set(self._dependencies_by_required.keys()) - existing - for loader_cls, id_ in missing_dependencies: - if self._is_system_resource(loader_cls, id_): - continue - elif loader_cls is DataSetsIO and id_ == "": - # Special case used by the location filter to indicate filter out all classical resources. - continue - elif client and self._check_resource_exists_in_cdf(client, loader_cls, id_): - continue - required_by: set[tuple[Hashable, Path]] = set() - for required, path in self._dependencies_by_required[(loader_cls, id_)]: - if path.name.endswith(RawTableCRUD.kind) and loader_cls is RawDatabaseCRUD: - # RAW Databases are automatically created when a Raw Table is created. - continue - relative_path = path.relative_to(project_config_dir) - - required_by.add((required, relative_path)) - has_checked_cdf = client is not None - self.warn(MissingDependencyWarning(loader_cls.resource_cls.__name__, id_, required_by, has_checked_cdf)) - - def _check_resource_exists_in_cdf(self, client: ToolkitClient, loader_cls: type[ResourceIO], id_: Hashable) -> bool: - """Check is the resource exists in the CDF project. If there are any issues assume it does not exist.""" - if id_ in self.existing_resources_by_loader[loader_cls]: - return True - - if loader_cls not in self.instantiated_loaders: - self.instantiated_loaders[loader_cls] = loader_cls.create_loader(client) - loader = self.instantiated_loaders[loader_cls] - with contextlib.suppress(CogniteAPIError): - retrieved = loader.retrieve([id_]) - if retrieved: - self.existing_resources_by_loader[loader_cls].add(id_) - return True - return False - - def check_built_resource( - self, - parsed: dict[str, Any] | list[dict[str, Any]], - loader: type[ResourceIO], - source: SourceLocation, - ) -> tuple[WarningList[FileReadWarning], list[tuple[Hashable, str]]]: - warning_list = WarningList[FileReadWarning]() - - warning_list.extend(validate_resource_yaml_pydantic(parsed, loader.yaml_cls, source.path)) - - is_dict_item = isinstance(parsed, dict) - items = [parsed] if isinstance(parsed, dict) else parsed - - identifier_kind_pairs: list[tuple[Hashable, str]] = [] - for no, item in enumerate(items, 1): - element_no = None if is_dict_item else no - - identifier: Any | None = None - # Raw Tables and Raw Databases can have different loaders in the same file. - item_loader = loader - try: - identifier = item_loader.get_id(item) - except KeyError as error: - if loader is RawTableCRUD: - try: - identifier = RawDatabaseCRUD.get_id(item) - item_loader = RawDatabaseCRUD - except KeyError: - warning_list.append( - MissingRequiredIdentifierWarning(source.path, element_no, tuple(), error.args) - ) - else: - warning_list.append(MissingRequiredIdentifierWarning(source.path, element_no, tuple(), error.args)) - - if identifier: - identifier_kind_pairs.append((identifier, item_loader.kind)) - if first_seen := self._ids_by_resource_type[item_loader].get(identifier): - if isinstance(identifier, RawDatabaseId): - # RawDatabases are picked up from both RawTables and RawDatabases files. Note it is not possible - # to define a raw table without also defining the raw database. Thus, it is impossible to - # avoid duplicated RawDatabase warnings if you have multiple RawTables files. - continue - if first_seen.hash != source.hash: - warning_list.append(DuplicatedItemWarning(source.path, identifier, first_seen.path)) - else: - self._ids_by_resource_type[item_loader][identifier] = source - - try: - dependencies = list(item_loader.get_dependent_items(item)) - except (AttributeError, ValueError, TypeError, KeyError): - if element_no: - location = f" at element {element_no}" - else: - location = "" - raise ToolkitYAMLFormatError( - f"Error in {source.path.as_posix()}{location}. " - f"Failed to extract dependencies from {item_loader.kind}." - ) - for dependency in dependencies: - self._dependencies_by_required[dependency].append((identifier, source.path)) - - data_set_warnings = validate_data_set_is_set(items, loader.resource_cls, source.path) - warning_list.extend(data_set_warnings) - - item_warnings = item_loader.check_item(item, filepath=source.path, element_no=element_no) - warning_list.extend(item_warnings) - - return warning_list, identifier_kind_pairs - - @staticmethod - def _is_system_resource(resource_cls: type[ResourceIO], id_: Hashable) -> bool: - """System resources are deployed to all CDF project and should not be checked for dependencies.""" - return ( - resource_cls in {ContainerCRUD, ViewIO, DataModelIO, NodeCRUD, SpaceCRUD} - and hasattr(id_, "space") - and id_.space.startswith("cdf_") - ) From 85fce5e3e838f2901d9c9a87614b07087c8ff585 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 7 Sep 2026 18:42:08 +0200 Subject: [PATCH 05/31] refactor: one less cognite-sdk import --- remove-build-v1 | 0 tests/test_unit/test_toolkit_package.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 remove-build-v1 diff --git a/remove-build-v1 b/remove-build-v1 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_unit/test_toolkit_package.py b/tests/test_unit/test_toolkit_package.py index 382d0bb3a7..04f3b2fa9b 100644 --- a/tests/test_unit/test_toolkit_package.py +++ b/tests/test_unit/test_toolkit_package.py @@ -94,7 +94,7 @@ def test_no_cognite_sdk_imports() -> None: The goal is to fully remove the cognite-sdk dependency from the toolkit (with the exception of Auth and protobuf files). This test tracks progress toward that goal. """ - _assert_import_violations(_extract_cognite_sdk_imports, "cognite.client imports", 95) + _assert_import_violations(_extract_cognite_sdk_imports, "cognite.client imports", 94) def _parse_package_name(dependency: str) -> str: From f8cd74e532c0a58581c584074f06dd478995c5cd Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 7 Sep 2026 18:47:36 +0200 Subject: [PATCH 06/31] tests: updated clean tests --- .../test_cdf_tk/test_commands/test_clean.py | 16 ------ .../test_cruds/test_extraction_pipeline.py | 29 ----------- .../test_cli/test_command_sequences.py | 50 +++++++++++-------- 3 files changed, 29 insertions(+), 66 deletions(-) delete mode 100644 tests/test_unit/test_cdf_tk/test_commands/test_clean.py diff --git a/tests/test_unit/test_cdf_tk/test_commands/test_clean.py b/tests/test_unit/test_cdf_tk/test_commands/test_clean.py deleted file mode 100644 index 75dac2d66a..0000000000 --- a/tests/test_unit/test_cdf_tk/test_commands/test_clean.py +++ /dev/null @@ -1,16 +0,0 @@ -from cognite_toolkit._cdf_tk.commands import CleanCommand -from cognite_toolkit._cdf_tk.data_classes._config_yaml import BuildEnvironment - - -class TestCleanCommandSelectModules: - """Test the CleanCommand._select_modules method.""" - - def test_select_specific_module( - self, - build_environment: BuildEnvironment, - ) -> None: - """Test that when a specific module is specified, it is returned.""" - cmd = CleanCommand(silent=True, skip_tracking=True) - result = cmd._select_modules(build_environment, module_str="my_example_module") - assert len(result) == 1 - assert result[0].dir.name == "my_example_module", f"Expected 'my_example_module', got {result[0].dir.name}" diff --git a/tests/test_unit/test_cdf_tk/test_cruds/test_extraction_pipeline.py b/tests/test_unit/test_cdf_tk/test_cruds/test_extraction_pipeline.py index a124a9caf9..45cd033a26 100644 --- a/tests/test_unit/test_cdf_tk/test_cruds/test_extraction_pipeline.py +++ b/tests/test_unit/test_cdf_tk/test_cruds/test_extraction_pipeline.py @@ -13,7 +13,6 @@ ExtractionPipelineConfigRequest, ExtractionPipelineConfigResponse, ) -from cognite_toolkit._cdf_tk.commands import CleanCommand from cognite_toolkit._cdf_tk.resource_ios import ( DataSetsIO, ExtractionPipelineConfigIO, @@ -66,34 +65,6 @@ def test_load_extraction_pipeline_upsert_update_one( "unchanged": len(resources.unchanged), } == {"create": 1, "changed": 0, "delete": 1, "unchanged": 0} - def test_load_extraction_pipeline_delete_one( - self, - toolkit_client_approval: ApprovalToolkitClient, - env_vars_with_client: EnvironmentVariables, - monkeypatch: MonkeyPatch, - ) -> None: - toolkit_client_approval.append( - ExtractionPipelineConfigResponse, - ExtractionPipelineConfigResponse( - external_id="ep_src_asset", - description="DB extractor config reading data from Springfield SAP", - config="\n logger: \n {level: WARN}", - revision=1, - created_time=0, - ), - ) - - local_file = MagicMock(spec=Path) - local_file.read_text.return_value = self.config_yaml - local_file.stem = "ep_src_asset" - - cmd = CleanCommand(print_warning=False) - loader = ExtractionPipelineConfigIO.create_loader(env_vars_with_client.get_client()) - with patch.object(ExtractionPipelineConfigIO, "find_files", return_value=[local_file]): - res = cmd.clean_resources(loader, env_vars_with_client, [], dry_run=True, drop=True) - assert res is not None - assert res.deleted == 1 - class TestExtractionPipelineLoader: @pytest.mark.parametrize( diff --git a/tests/test_unit/test_cli/test_command_sequences.py b/tests/test_unit/test_cli/test_command_sequences.py index 86badf208a..9a35cf4250 100644 --- a/tests/test_unit/test_cli/test_command_sequences.py +++ b/tests/test_unit/test_cli/test_command_sequences.py @@ -16,9 +16,7 @@ from pytest import MonkeyPatch from cognite_toolkit._cdf_tk.commands import ( - BuildCommand, BuildV2Command, - CleanCommand, DeployOptions, DeployV2Command, ) @@ -182,28 +180,38 @@ def test_init_build_clean( buildable_modules: Path, data_regression, ) -> None: - mock_environments_yaml_file(module_path, monkeypatch) - - BuildCommand(silent=True, skip_tracking=True).execute( - verbose=False, - organization_dir=buildable_modules, - build_dir=build_tmp_path, - selected=None, - no_clean=False, + BuildV2Command(silent=True, skip_tracking=True).build( client=env_vars_with_client.get_client(), - build_env_name="dev", - on_error="raise", + parameters=BuildParameters( + organization_dir=buildable_modules, + build_dir=build_tmp_path, + config_yaml=buildable_modules / "config.dev.yaml", + user_selected_modules=[module_path.name], + ), ) - CleanCommand(silent=True, skip_tracking=True).execute( - env_vars=env_vars_with_client, - build_dir=build_tmp_path, - build_env_name="dev", - dry_run=False, - include=None, - module_str=module_path.name, - all_modules=True, - verbose=False, + monkeypatch.setattr( + "cognite_toolkit._cdf_tk.commands.deploy_v2.command.confirm_by_typing_project_name", + lambda msg, client: True, ) + with patch.dict( + os.environ, + {"CDF_ENVIRON": "pytest", "CDF_BUILD_TYPE": "dev"}, + ): + DeployV2Command(silent=True, skip_tracking=True).deploy( + user_build_dir=build_tmp_path, + env_vars=env_vars_with_client, + options=DeployOptions( + operation="clean", + cdf_project=env_vars_with_client.CDF_PROJECT, + drop=True, + dry_run=False, + include=None, + verbose=False, + drop_data=True, + force_update=False, + environment_variables=env_vars_with_client.dump(), + ), + ) not_mocked = toolkit_client_approval.not_mocked_calls() assert not not_mocked, ( From 138e63ff03f9be7e6e4c8d0f1e2e672b3d5b4494 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 7 Sep 2026 18:48:54 +0200 Subject: [PATCH 07/31] refactor; reorder --- cognite_toolkit/_cdf_tk/apps/_core_app.py | 2 +- cognite_toolkit/_cdf_tk/commands/__init__.py | 2 -- cognite_toolkit/_cdf_tk/commands/_base.py | 4 ++++ cognite_toolkit/_cdf_tk/commands/clean.py | 4 +--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cognite_toolkit/_cdf_tk/apps/_core_app.py b/cognite_toolkit/_cdf_tk/apps/_core_app.py index 940b26572d..7d4e47e59d 100644 --- a/cognite_toolkit/_cdf_tk/apps/_core_app.py +++ b/cognite_toolkit/_cdf_tk/apps/_core_app.py @@ -22,8 +22,8 @@ DeployOptions, DeployV2Command, ) +from cognite_toolkit._cdf_tk.commands._base import AVAILABLE_DATA_TYPES from cognite_toolkit._cdf_tk.commands.build_v2.data_classes import BuildParameters, ConfigYAML -from cognite_toolkit._cdf_tk.commands.clean import AVAILABLE_DATA_TYPES from cognite_toolkit._cdf_tk.exceptions import ToolkitFileNotFoundError from cognite_toolkit._cdf_tk.tk_warnings import ToolkitDeprecationWarning from cognite_toolkit._cdf_tk.utils import humanize_collection diff --git a/cognite_toolkit/_cdf_tk/commands/__init__.py b/cognite_toolkit/_cdf_tk/commands/__init__.py index b1f6ba43c2..5971b63b5b 100644 --- a/cognite_toolkit/_cdf_tk/commands/__init__.py +++ b/cognite_toolkit/_cdf_tk/commands/__init__.py @@ -10,7 +10,6 @@ from .auth import AuthCommand from .build_cmd import BuildCommand from .build_v2.build_v2 import BuildV2Command -from .clean import CleanCommand from .deploy_v2.command import DeploymentStep, DeployOptions, DeployV2Command from .dump_resource import DumpResourceCommand from .entity_matching import EntityMatchingCommand @@ -27,7 +26,6 @@ "AuthCommand", "BuildCommand", "BuildV2Command", - "CleanCommand", "DeployOptions", "DeployV2Command", "DeploymentStep", diff --git a/cognite_toolkit/_cdf_tk/commands/_base.py b/cognite_toolkit/_cdf_tk/commands/_base.py index eaeacd3c2a..372cc5309d 100644 --- a/cognite_toolkit/_cdf_tk/commands/_base.py +++ b/cognite_toolkit/_cdf_tk/commands/_base.py @@ -9,6 +9,7 @@ from cognite_toolkit._cdf_tk.cdf_toml import CDFToml from cognite_toolkit._cdf_tk.client import ToolkitClient from cognite_toolkit._cdf_tk.data_classes import CommandTracking +from cognite_toolkit._cdf_tk.resource_ios import CRUDS_BY_FOLDER_NAME from cognite_toolkit._cdf_tk.tk_warnings import ( ToolkitWarning, WarningList, @@ -106,3 +107,6 @@ def _collect_click_command_names(group: Any, names: set[str]) -> None: for name, cmd in group.commands.items(): names.add(name) _collect_click_command_names(cmd, names) + + +AVAILABLE_DATA_TYPES: tuple[str, ...] = tuple(CRUDS_BY_FOLDER_NAME) diff --git a/cognite_toolkit/_cdf_tk/commands/clean.py b/cognite_toolkit/_cdf_tk/commands/clean.py index be421a1646..21aaff2170 100644 --- a/cognite_toolkit/_cdf_tk/commands/clean.py +++ b/cognite_toolkit/_cdf_tk/commands/clean.py @@ -9,7 +9,7 @@ from rich.panel import Panel from cognite_toolkit._cdf_tk.client._resource_base import T_Identifier, T_RequestResource, T_ResponseResource -from cognite_toolkit._cdf_tk.commands._base import ToolkitCommand +from cognite_toolkit._cdf_tk.commands._base import AVAILABLE_DATA_TYPES, ToolkitCommand from cognite_toolkit._cdf_tk.constants import ( _RUNNING_IN_BROWSER, BUILD_ENVIRONMENT_FILE, @@ -56,8 +56,6 @@ from ._utils import _print_ids_or_length -AVAILABLE_DATA_TYPES: tuple[str, ...] = tuple(CRUDS_BY_FOLDER_NAME) - class CleanCommand(ToolkitCommand): def clean_resources( From 98c4e99f686421f17424508c6aee7ddedb891a88 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 7 Sep 2026 18:49:31 +0200 Subject: [PATCH 08/31] refactor: delete clean v1 --- cognite_toolkit/_cdf_tk/commands/clean.py | 395 ---------------------- 1 file changed, 395 deletions(-) delete mode 100644 cognite_toolkit/_cdf_tk/commands/clean.py diff --git a/cognite_toolkit/_cdf_tk/commands/clean.py b/cognite_toolkit/_cdf_tk/commands/clean.py deleted file mode 100644 index 21aaff2170..0000000000 --- a/cognite_toolkit/_cdf_tk/commands/clean.py +++ /dev/null @@ -1,395 +0,0 @@ -import traceback -from collections.abc import Sequence -from graphlib import TopologicalSorter -from pathlib import Path - -import questionary -from cognite.client.exceptions import CogniteAPIError, CogniteNotFoundError -from rich import print -from rich.panel import Panel - -from cognite_toolkit._cdf_tk.client._resource_base import T_Identifier, T_RequestResource, T_ResponseResource -from cognite_toolkit._cdf_tk.commands._base import AVAILABLE_DATA_TYPES, ToolkitCommand -from cognite_toolkit._cdf_tk.constants import ( - _RUNNING_IN_BROWSER, - BUILD_ENVIRONMENT_FILE, - HINT_LEAD_TEXT, - HINT_LEAD_TEXT_LEN, -) -from cognite_toolkit._cdf_tk.data_classes import ( - BuildEnvironment, - DeployResults, - ResourceContainerDeployResult, - ResourceDeployResult, -) -from cognite_toolkit._cdf_tk.data_classes._module_directories import ReadModule -from cognite_toolkit._cdf_tk.exceptions import ( - ToolkitCleanResourceError, - ToolkitMissingModuleError, - ToolkitNotADirectoryError, - ToolkitValidationError, - ToolkitValueError, -) -from cognite_toolkit._cdf_tk.resource_ios import ( - CRUDS_BY_FOLDER_NAME, - DataCRUD, - DataSetsIO, - FileCRUD, - RawDatabaseCRUD, - ResourceContainerIO, - ResourceIO, - ResourceWorker, -) -from cognite_toolkit._cdf_tk.resource_ios._base_ios import Loader -from cognite_toolkit._cdf_tk.resource_ios._resource_ios import SimulatorModelRevisionIO, SimulatorRoutineRevisionIO -from cognite_toolkit._cdf_tk.tk_warnings import ( - LowSeverityWarning, - MediumSeverityWarning, - ToolkitDependenciesIncludedWarning, - ToolkitNotSupportedWarning, -) -from cognite_toolkit._cdf_tk.utils import ( - humanize_collection, - read_yaml_file, -) -from cognite_toolkit._cdf_tk.utils.auth import EnvironmentVariables - -from ._utils import _print_ids_or_length - - -class CleanCommand(ToolkitCommand): - def clean_resources( - self, - loader: ResourceIO[T_Identifier, T_RequestResource, T_ResponseResource], - env_vars: EnvironmentVariables, - read_modules: list[ReadModule], - dry_run: bool = False, - drop: bool = True, - drop_data: bool = False, - verbose: bool = False, - ) -> ResourceDeployResult | None: - if not isinstance(loader, ResourceContainerIO) and not drop: - # Skipping silently as this, we will not drop data or delete this resource - return ResourceDeployResult(name=loader.display_name) - if not loader.support_drop: - print(f" [bold green]INFO:[/] {loader.display_name!r} cleaning is not supported, skipping...") - return ResourceDeployResult(name=loader.display_name) - elif isinstance(loader, ResourceContainerIO) and not drop_data: - print( - f" [bold]INFO:[/] Skipping cleaning of {loader.display_name!r}. This is a data resource (it contains " - f"data and is not only configuration/metadata) and therefore " - "requires the --drop-data flag to be set to perform cleaning..." - ) - return ResourceContainerDeployResult(name=loader.display_name, item_name=loader.item_name) - - worker = ResourceWorker(loader, "clean") - files = worker.load_files(read_modules=read_modules) - if not files: - return None - # Since we do a clean, we do not want to verify that everything exists wrt data sets, spaces etc. - local_resources = worker.load_resources( - filepaths=files, - environment_variables=env_vars.dump(include_os=True), - is_dry_run=True, - ) - existing_resources = loader.retrieve(list(local_resources.keys())) - nr_of_existing = len(existing_resources) - - if drop: - prefix = "Would clean" if dry_run else "Cleaning" - with_data = "with data " if isinstance(loader, ResourceContainerIO) else "" - else: - prefix = "Would drop data from" if dry_run else "Dropping data from" - with_data = "" - print(f"[bold]{prefix} {nr_of_existing} {loader.display_name} {with_data}from CDF...[/]") - if not isinstance(loader, RawDatabaseCRUD): - for duplicate in worker.duplicates: - self.warn(LowSeverityWarning(f"Duplicate {loader.display_name} {duplicate}.")) - - # Deleting resources. - if isinstance(loader, ResourceContainerIO) and drop_data: - nr_of_dropped_datapoints = self._drop_data(existing_resources, loader, dry_run, verbose) - if drop: - nr_of_deleted = self._delete_resources(existing_resources, loader, dry_run, verbose) - else: - nr_of_deleted = 0 - if verbose: - print("") - return ResourceContainerDeployResult( - name=loader.display_name, - deleted=nr_of_deleted, - total=nr_of_existing, - dropped_datapoints=nr_of_dropped_datapoints, - item_name=loader.item_name, - ) - elif not isinstance(self, ResourceContainerIO) and drop: - nr_of_deleted = self._delete_resources(existing_resources, loader, dry_run, verbose) - if verbose: - print("") - return ResourceDeployResult(name=loader.display_name, deleted=nr_of_deleted, total=nr_of_existing) - else: - return ResourceDeployResult(name=loader.display_name) - - def _delete_resources( - self, loaded_resources: Sequence[T_ResponseResource], loader: ResourceIO, dry_run: bool, verbose: bool - ) -> int: - nr_of_deleted = 0 - resource_ids = loader.get_ids(loaded_resources) - if dry_run: - nr_of_deleted += len(resource_ids) - if verbose: - print(f" Would have deleted {_print_ids_or_length(resource_ids)}.") - return nr_of_deleted - - if resource_ids and loader.drop_confirmation_message: - if not questionary.confirm(loader.drop_confirmation_message, default=False).ask(): - return 0 - - try: - nr_of_deleted += loader.delete(resource_ids) - except CogniteAPIError as e: - self.warn(MediumSeverityWarning(f"Failed to delete {_print_ids_or_length(resource_ids)}. Error {e}.")) - if verbose: - print(Panel(traceback.format_exc())) - except CogniteNotFoundError: - if verbose: - print(f" [bold]INFO:[/] {_print_ids_or_length(resource_ids)} do(es) not exist.") - else: # Delete succeeded - if verbose: - print(f" Deleted {_print_ids_or_length(resource_ids)}.") - return nr_of_deleted - - def _drop_data( - self, - loaded_resources: Sequence[T_ResponseResource], - loader: ResourceContainerIO, - dry_run: bool, - verbose: bool, - ) -> int: - nr_of_dropped = 0 - resource_ids = loader.get_ids(loaded_resources) - if dry_run: - resource_drop_count = loader.count(resource_ids) - nr_of_dropped += resource_drop_count - if verbose: - self._verbose_print_drop(resource_drop_count, resource_ids, loader, dry_run) - return nr_of_dropped - - try: - resource_drop_count = loader.drop_data(resource_ids) - nr_of_dropped += resource_drop_count - except CogniteAPIError as e: - if e.code == 404 and verbose: - print(f" [bold]INFO:[/] {len(resource_ids)} {loader.display_name} do(es) not exist.") - except CogniteNotFoundError: - return nr_of_dropped - else: # Delete succeeded - if verbose: - self._verbose_print_drop(resource_drop_count, resource_ids, loader, dry_run) - return nr_of_dropped - - def _interactive_module_selection(self, built_modules: list[ReadModule] | None) -> list[ReadModule] | None: - if not built_modules: - return None - choices = [ - questionary.Choice(title=built_module.dir.name, value=built_module) for built_module in built_modules - ] - - selected_modules = questionary.checkbox( - "Which modules would you like to clean?", - instruction="Use arrow up/down, press space to select item(s) and enter to save", - choices=choices, - validate=lambda choice: "You must select at least one module." if len(choice) == 0 else True, - ).unsafe_ask() - - if not selected_modules: - return None - return selected_modules - - def _verbose_print_drop( - self, drop_count: int, resource_ids: Sequence[T_Identifier], loader: ResourceContainerIO, dry_run: bool - ) -> None: - prefix = "Would have dropped" if dry_run else "Dropped" - if drop_count > 0: - print( - f" {prefix} {drop_count:,} {loader.item_name} from {loader.display_name}: " - f"{_print_ids_or_length(resource_ids)}." - ) - elif drop_count == 0: - verb = "is" if len(resource_ids) == 1 else "are" - print( - f" The {loader.display_name}: {_print_ids_or_length(resource_ids)} {verb} empty, " - f"thus no {loader.item_name} will be {'touched' if dry_run else 'dropped'}." - ) - else: - # Count is not supported - print(f" {prefix} all {loader.item_name} from {loader.display_name}: {_print_ids_or_length(resource_ids)}.") - - def _select_modules(self, clean_state: BuildEnvironment, module_str: str | None) -> list[ReadModule] | None: - if module_str: - return [module for module in clean_state.read_modules if module.dir.name == module_str] - return self._interactive_module_selection(clean_state.read_modules) - - def execute( - self, - env_vars: EnvironmentVariables, - build_dir: Path, - build_env_name: str | None, - dry_run: bool, - include: list[str] | None, - module_str: str | None, - verbose: bool, - all_modules: bool = False, - ) -> None: - if not build_dir.exists(): - raise ToolkitNotADirectoryError( - "The build directory does not exists. Did you forget to run `cdf-tk build` first?" - ) - clean_state = BuildEnvironment.load(read_yaml_file(build_dir / BUILD_ENVIRONMENT_FILE), build_env_name, "clean") - clean_state.set_environment_variables() - errors = clean_state.check_source_files_changed() - for error in errors: - self.warn(error) - if errors: - raise ToolkitCleanResourceError( - "One or more source files have been modified since the last build. Please rebuild the project." - ) - client = env_vars.get_client(clean_state.is_strict_validation) - environment_vars = "" - if not _RUNNING_IN_BROWSER: - environment_vars = f"\n\nConnected to {env_vars.as_string()}" - - action = "" - if dry_run: - action = "(dry-run) " - - print( - Panel( - f"[bold]Cleaning {action}[/]resource from CDF project {client.config.project} based " - f"on resource files in {build_dir} directory." - f"{environment_vars}", - expand=False, - ) - ) - - if not build_dir.is_dir(): - raise ToolkitNotADirectoryError(f"'{build_dir}'. Did you forget to run `cdf build` first?") - - selected_modules: list[ReadModule] - if all_modules: - selected_modules = clean_state.read_modules or [] - if not selected_modules: - raise ToolkitValueError("No modules available to clean.") - elif module_str: - selected_modules = [module for module in clean_state.read_modules if module.dir.name == module_str] - if not selected_modules: - available_module_names = {module.dir.name for module in clean_state.read_modules} - raise ToolkitMissingModuleError( - f"No modules matched the selection: {module_str}. Available modules: {sorted(available_module_names)}" - ) - else: - selected_modules = self._interactive_module_selection(clean_state.read_modules) or [] - if not selected_modules: - raise ToolkitValueError( - "No module specified with the --module option and no modules selected interactively." - ) - - selected_resource_folders = { - resource_folder for module in selected_modules for resource_folder in module.resource_directories - } - selected_loaders = self.get_selected_loaders(build_dir, selected_resource_folders, include) - - results = DeployResults([], "clean", dry_run=dry_run) - - resolved_list: list[type[Loader]] = [] - should_include: list[type[Loader]] = [] - # The topological sort can include loaders that are not selected, so we need to check for that. - for loader_cls in TopologicalSorter(selected_loaders).static_order(): - if loader_cls in selected_loaders: - resolved_list.append(loader_cls) - elif (build_dir / loader_cls.folder_name).is_dir(): - should_include.append(loader_cls) - # Otherwise, it is not in the build directory and not selected, so we skip it. - # There should be a warning in the build step if it is missing. - if should_include: - self.warn(ToolkitDependenciesIncludedWarning([item.folder_name for item in should_include])) - - for loader_cls in reversed(resolved_list): - if not issubclass(loader_cls, ResourceIO): - continue - loader = loader_cls.create_loader(client, build_dir) - if isinstance(loader, DataSetsIO | SimulatorModelRevisionIO | SimulatorRoutineRevisionIO): - self.warn(ToolkitNotSupportedWarning(feature=f"{loader.display_name} clean.")) - continue - result = self.clean_resources( - loader, - env_vars=env_vars, - read_modules=selected_modules, - drop=True, - dry_run=dry_run, - drop_data=True, - verbose=verbose, - ) - if result: - results[result.name] = result - if results.has_counts: - print(results.counts_table()) - if results.has_uploads: - print(results.uploads_table()) - - def get_selected_loaders( - self, build_dir: Path, read_resource_folders: set[str], include: list[str] | None - ) -> dict[type[Loader], frozenset[type[Loader]]]: - selected_loaders: dict[type[Loader], frozenset[type[Loader]]] = {} - for folder_name, loader_classes in CRUDS_BY_FOLDER_NAME.items(): - if include is not None and folder_name not in include: - continue - if folder_name in read_resource_folders: - selected_loaders.update( - { - loader_cls: (loader_cls.dependencies | {FileCRUD}) - if issubclass(loader_cls, SimulatorModelRevisionIO) - else loader_cls.dependencies - for loader_cls in loader_classes - } - ) - continue - if not (build_dir / folder_name).is_dir(): - continue - folder_has_supported_files = False - for loader_cls in loader_classes: - if loader_cls.any_supported_files(build_dir / folder_name): - folder_has_supported_files = True - if issubclass(loader_cls, SimulatorModelRevisionIO): - # Special case, we need to ensure that the file is uploaded before we try to deploy. - selected_loaders[loader_cls] = loader_cls.dependencies | {FileCRUD} - else: - selected_loaders[loader_cls] = loader_cls.dependencies - elif issubclass(loader_cls, DataCRUD): - # Data Loaders are always included, as they will have - # the files in the module folder and not the build folder. - selected_loaders[loader_cls] = loader_cls.dependencies - - if not folder_has_supported_files: - kinds = [loader_cls.kind for loader_cls in loader_classes] - yaml_file = next((build_dir / folder_name).glob("*.yaml"), None) - suggestion = "" - if yaml_file: - suggestion = f"\n{' ' * HINT_LEAD_TEXT_LEN}For example: '{yaml_file.stem}.{kinds[0]}.yaml'." - self.warn( - MediumSeverityWarning( - f"No supported files found in {folder_name!r} folder. Skipping...\n" - f"{HINT_LEAD_TEXT}All resource in the {folder_name!r} folder are expected to have suffix: " - f"{humanize_collection(kinds)!r}.{suggestion}" - ) - ) - return selected_loaders - - @staticmethod - def validate_include(include: list[str] | None) -> list[str]: - if include and (invalid_types := set(include).difference(AVAILABLE_DATA_TYPES)): - raise ToolkitValidationError( - f"Invalid resource types specified: {invalid_types}, available types: {AVAILABLE_DATA_TYPES}" - ) - include = include or list(AVAILABLE_DATA_TYPES) - return include From 8052ad5514e103429ad32b92e3edabefff9644c2 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 7 Sep 2026 19:42:56 +0200 Subject: [PATCH 09/31] tests; update --- tests/test_unit/test_toolkit_package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_unit/test_toolkit_package.py b/tests/test_unit/test_toolkit_package.py index 382d0bb3a7..04f3b2fa9b 100644 --- a/tests/test_unit/test_toolkit_package.py +++ b/tests/test_unit/test_toolkit_package.py @@ -94,7 +94,7 @@ def test_no_cognite_sdk_imports() -> None: The goal is to fully remove the cognite-sdk dependency from the toolkit (with the exception of Auth and protobuf files). This test tracks progress toward that goal. """ - _assert_import_violations(_extract_cognite_sdk_imports, "cognite.client imports", 95) + _assert_import_violations(_extract_cognite_sdk_imports, "cognite.client imports", 94) def _parse_package_name(dependency: str) -> str: From d783d5048f5707946a17228e976de860f713c5c8 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 7 Sep 2026 20:08:30 +0200 Subject: [PATCH 10/31] tests: remove tests and start init cleanup --- cognite_toolkit/_cdf_tk/builders/__init__.py | 43 +----------- .../test_cdf_tk/test_builders/__init__.py | 0 .../test_cdf_tk/test_builders/test_builder.py | 66 ------------------- .../test_builders/test_location_builder.py | 53 --------------- 4 files changed, 2 insertions(+), 160 deletions(-) delete mode 100644 tests/test_unit/test_cdf_tk/test_builders/__init__.py delete mode 100644 tests/test_unit/test_cdf_tk/test_builders/test_builder.py delete mode 100644 tests/test_unit/test_cdf_tk/test_builders/test_location_builder.py diff --git a/cognite_toolkit/_cdf_tk/builders/__init__.py b/cognite_toolkit/_cdf_tk/builders/__init__.py index 30fea31d5c..2bec72ce54 100644 --- a/cognite_toolkit/_cdf_tk/builders/__init__.py +++ b/cognite_toolkit/_cdf_tk/builders/__init__.py @@ -1,42 +1,3 @@ -from collections.abc import Callable -from pathlib import Path +from ._base import Builder -from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning - -from ._base import Builder, DefaultBuilder, get_resource_crud -from ._datamodels import DataModelBuilder -from ._file import FileBuilder -from ._function import FunctionBuilder -from ._location import LocationBuilder -from ._raw import RawBuilder -from ._ruleset import RuleSetBuilder -from ._streamlit import StreamlitBuilder -from ._transformation import TransformationBuilder - - -def create_builder( - resource_folder: str, - build_dir: Path | None, - warn: Callable[[ToolkitWarning], None] | None = None, -) -> Builder: - if builder_cls := _BUILDER_BY_RESOURCE_FOLDER.get(resource_folder): - return builder_cls(build_dir, warn=warn) # type: ignore[abstract] - - return DefaultBuilder(build_dir, resource_folder, warn) - - -_BUILDER_BY_RESOURCE_FOLDER = {_builder._resource_folder: _builder for _builder in Builder.__subclasses__()} -__all__ = [ - "Builder", - "DataModelBuilder", - "DefaultBuilder", - "FileBuilder", - "FunctionBuilder", - "LocationBuilder", - "RawBuilder", - "RuleSetBuilder", - "StreamlitBuilder", - "TransformationBuilder", - "create_builder", - "get_resource_crud", -] +__all__ = ["Builder"] diff --git a/tests/test_unit/test_cdf_tk/test_builders/__init__.py b/tests/test_unit/test_cdf_tk/test_builders/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/test_unit/test_cdf_tk/test_builders/test_builder.py b/tests/test_unit/test_cdf_tk/test_builders/test_builder.py deleted file mode 100644 index 00e94052b2..0000000000 --- a/tests/test_unit/test_cdf_tk/test_builders/test_builder.py +++ /dev/null @@ -1,66 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -import pytest - -from cognite_toolkit._cdf_tk.builders import get_resource_crud -from cognite_toolkit._cdf_tk.resource_ios import ( - RESOURCE_CRUD_LIST, - GroupAllScopedCRUD, - GroupIO, - GroupResourceScopedCRUD, - ResourceIO, -) -from cognite_toolkit._cdf_tk.tk_warnings import ToolkitNotSupportedWarning, ToolkitWarning -from cognite_toolkit._cdf_tk.tk_warnings.fileread import UnknownResourceTypeWarning - - -class TestGetCRUD: - @pytest.mark.parametrize( - "source_path, resource_folder, expected_loader_cls", - [ - pytest.param( - Path(f"some_path/{crud_cls.folder_name}/my.{crud_cls.kind}.yaml"), - crud_cls.folder_name, - { - GroupResourceScopedCRUD: GroupIO, - GroupAllScopedCRUD: GroupIO, - }.get(crud_cls, crud_cls), - id=crud_cls.__name__, - ) - for crud_cls in RESOURCE_CRUD_LIST - ], - ) - def test_get_crud_no_warning( - self, source_path: Path, resource_folder: str, expected_loader_cls: type[ResourceIO] - ) -> None: - crud_cls, warning = get_resource_crud(source_path, resource_folder) - - assert warning is None - assert crud_cls is expected_loader_cls - - @pytest.mark.parametrize( - "source_path, resource_folder, expected_warning_cls", - [ - pytest.param( - Path(f"some_path/unknown_folder/my.{GroupIO.kind}.yaml"), - "unknown_folder", - ToolkitNotSupportedWarning, - id="Unknown folder, known kind", - ), - pytest.param( - Path("some_path/group/my.UnknownKind.yaml"), - GroupIO.folder_name, - UnknownResourceTypeWarning, - id="Known folder, unknown kind", - ), - ], - ) - def test_get_crud_warning( - self, source_path: Path, resource_folder: str, expected_warning_cls: type[ToolkitWarning] - ) -> None: - crud_cls, warning = get_resource_crud(source_path, resource_folder) - - assert crud_cls is None - assert type(warning) is expected_warning_cls diff --git a/tests/test_unit/test_cdf_tk/test_builders/test_location_builder.py b/tests/test_unit/test_cdf_tk/test_builders/test_location_builder.py deleted file mode 100644 index 15a5b88b8f..0000000000 --- a/tests/test_unit/test_cdf_tk/test_builders/test_location_builder.py +++ /dev/null @@ -1,53 +0,0 @@ -from pathlib import Path -from unittest.mock import MagicMock - -import pytest - -from cognite_toolkit._cdf_tk.builders._location import LocationBuilder -from cognite_toolkit._cdf_tk.data_classes._build_files import BuildSourceFile -from cognite_toolkit._cdf_tk.data_classes._module_directories import ModuleLocation -from cognite_toolkit._cdf_tk.exceptions import ToolkitError - - -def test_location_builder_detect_cyclic_references(build_tmp_path): - # Setup - - module_location = MagicMock(spec=ModuleLocation) - - source_file = MagicMock(spec=BuildSourceFile) - source_file.source = MagicMock() - source_file.source.path = Path("cyclic.LocationFilter.yaml") - source_file.loaded = [ - {"externalId": "location1", "description": "Location 1", "parentExternalId": "location2"}, - {"externalId": "location2", "description": "Location 2", "parentExternalId": "location3"}, - { - "externalId": "location3", - "description": "Location 3", - "parentExternalId": "location1", # This creates a cycle - }, - ] - - location_builder = LocationBuilder(build_dir=build_tmp_path) - with pytest.raises(ToolkitError, match=r"Circular dependency found in Locations*"): - list(location_builder.build(source_files=[source_file], module=module_location)) - - -def test_location_builder_detect_self_reference(build_tmp_path): - # Setup - - module_location = MagicMock(spec=ModuleLocation) - - source_file = MagicMock(spec=BuildSourceFile) - source_file.source = MagicMock() - source_file.source.path = Path("self.LocationFilter.yaml") - source_file.loaded = [ - { - "externalId": "location1", - "description": "Location 1", - "parentExternalId": "location1", - }, # This creates a self-reference - ] - - location_builder = LocationBuilder(build_dir=build_tmp_path) - with pytest.raises(ToolkitError, match=r"Circular dependency found in Locations*"): - list(location_builder.build(source_files=[source_file], module=module_location)) From 57a830bc8c7db74ae6c26e760a0ba8525bad5406 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 7 Sep 2026 20:09:47 +0200 Subject: [PATCH 11/31] refactor: deleted builder module --- cognite_toolkit/_cdf_tk/builders/__init__.py | 3 - cognite_toolkit/_cdf_tk/builders/_base.py | 178 ----------------- .../_cdf_tk/builders/_datamodels.py | 90 --------- cognite_toolkit/_cdf_tk/builders/_file.py | 88 --------- cognite_toolkit/_cdf_tk/builders/_function.py | 187 ------------------ cognite_toolkit/_cdf_tk/builders/_location.py | 84 -------- cognite_toolkit/_cdf_tk/builders/_raw.py | 72 ------- cognite_toolkit/_cdf_tk/builders/_ruleset.py | 105 ---------- .../_cdf_tk/builders/_streamlit.py | 110 ----------- .../_cdf_tk/builders/_transformation.py | 111 ----------- .../test_cdf_tk/test_cruds/test_data_model.py | 46 ----- 11 files changed, 1074 deletions(-) delete mode 100644 cognite_toolkit/_cdf_tk/builders/__init__.py delete mode 100644 cognite_toolkit/_cdf_tk/builders/_base.py delete mode 100644 cognite_toolkit/_cdf_tk/builders/_datamodels.py delete mode 100644 cognite_toolkit/_cdf_tk/builders/_file.py delete mode 100644 cognite_toolkit/_cdf_tk/builders/_function.py delete mode 100644 cognite_toolkit/_cdf_tk/builders/_location.py delete mode 100644 cognite_toolkit/_cdf_tk/builders/_raw.py delete mode 100644 cognite_toolkit/_cdf_tk/builders/_ruleset.py delete mode 100644 cognite_toolkit/_cdf_tk/builders/_streamlit.py delete mode 100644 cognite_toolkit/_cdf_tk/builders/_transformation.py diff --git a/cognite_toolkit/_cdf_tk/builders/__init__.py b/cognite_toolkit/_cdf_tk/builders/__init__.py deleted file mode 100644 index 2bec72ce54..0000000000 --- a/cognite_toolkit/_cdf_tk/builders/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from ._base import Builder - -__all__ = ["Builder"] diff --git a/cognite_toolkit/_cdf_tk/builders/_base.py b/cognite_toolkit/_cdf_tk/builders/_base.py deleted file mode 100644 index f48555098c..0000000000 --- a/cognite_toolkit/_cdf_tk/builders/_base.py +++ /dev/null @@ -1,178 +0,0 @@ -import difflib -from abc import ABC, abstractmethod -from collections.abc import Callable, Iterable, Sequence -from pathlib import Path -from typing import Any, ClassVar - -from cognite_toolkit._cdf_tk.constants import INDEX_PATTERN -from cognite_toolkit._cdf_tk.data_classes import ( - BuildDestinationFile, - BuildSourceFile, - BuiltResourceList, - ModuleLocation, -) -from cognite_toolkit._cdf_tk.exceptions import ( - AmbiguousResourceFileError, -) -from cognite_toolkit._cdf_tk.resource_ios import ( - RESOURCE_CRUD_BY_FOLDER_NAME, - GroupIO, - ResourceIO, -) -from cognite_toolkit._cdf_tk.tk_warnings import ( - ToolkitNotSupportedWarning, - ToolkitWarning, - WarningList, -) -from cognite_toolkit._cdf_tk.tk_warnings.fileread import ( - UnknownResourceTypeWarning, -) -from cognite_toolkit._cdf_tk.utils import ( - humanize_collection, -) - - -class Builder(ABC): - _resource_folder: ClassVar[str | None] = None - - def __init__( - self, - build_dir: Path | None, - resource_folder: str | None = None, - warn: Callable[[ToolkitWarning], None] | None = None, - ): - self._build_dir = build_dir - self.warn = warn - self.resource_counter = 0 - if self._resource_folder is not None: - self.resource_folder = self._resource_folder - elif resource_folder is not None: - self.resource_folder = resource_folder - else: - raise ValueError("Either _resource_folder or resource_folder must be set.") - - @property - def build_dir(self) -> Path: - if self._build_dir is None: - raise ValueError("build_dir must be set for this operation.") - return self._build_dir - - @abstractmethod - def build( - self, source_files: list[BuildSourceFile], module: ModuleLocation, console: Callable[[str], None] | None = None - ) -> Iterable[BuildDestinationFile | Sequence[ToolkitWarning]]: - raise NotImplementedError() - - def load_extra_field(self, extra: str) -> tuple[str, Any]: - """Overload in subclass to load extra fields from a file.""" - raise NotImplementedError( - f"Extra field {extra!r} by {type(self).__name__} - {self.resource_folder} is not supported." - ) - - def validate_directory( - self, built_resources: BuiltResourceList, module: ModuleLocation - ) -> WarningList[ToolkitWarning]: - """This can be overridden to add additional validation for the built resources.""" - return WarningList[ToolkitWarning]() - - # Helper methods - def _create_destination_path(self, source_path: Path, kind: str) -> Path: - """Creates the filepath in the build directory for the given source path. - - Note that this is a complex operation as the modules in the source are nested while the build directory is flat. - This means that we lose information and risk having duplicate filenames. To avoid this, we prefix the filename - with a number to ensure uniqueness. - """ - filestem = source_path.stem - # Get rid of the local index - filestem = INDEX_PATTERN.sub("", filestem) - - # Increment to ensure we do not get duplicate filenames when we flatten the file - # structure from the module to the build directory. - self.resource_counter += 1 - - filename = f"{self.resource_counter}.{filestem}" - if not filename.casefold().endswith(kind.casefold()): - filename = f"{filename}.{kind}" - filename = f"{filename}{source_path.suffix}" - destination_path = self.build_dir / self.resource_folder / filename - destination_path.parent.mkdir(parents=True, exist_ok=True) - return destination_path - - def _get_loader(self, source_path: Path) -> tuple[None, ToolkitWarning] | tuple[type[ResourceIO], None]: - return get_resource_crud(source_path, self.resource_folder) - - -def get_resource_crud( - source_path: Path, resource_folder: str -) -> tuple[None, ToolkitWarning] | tuple[type[ResourceIO], None]: - """Get the appropriate CRUD class for the given source file and resource folder.""" - folder_cruds = RESOURCE_CRUD_BY_FOLDER_NAME.get(resource_folder, []) - if not folder_cruds: - return None, ToolkitNotSupportedWarning( - f"resource of type {resource_folder!r} in {source_path.name}.", - details=f"Available resources are: {humanize_collection(RESOURCE_CRUD_BY_FOLDER_NAME.keys())}", - ) - - crud_candidates = [crud_cls for crud_cls in folder_cruds if crud_cls.is_supported_file(source_path)] - if len(crud_candidates) == 0: - suggestion: str | None = None - if "." in source_path.stem: - core, kind = source_path.stem.rsplit(".", 1) - match = difflib.get_close_matches(kind, [crud_cls.kind for crud_cls in folder_cruds]) - if match: - suggested_name = f"{core}.{match[0]}{source_path.suffix}" - suggestion = f"Did you mean to call the file {suggested_name!r}?" - else: - kinds = [crud.kind for crud in folder_cruds] - if len(kinds) == 1: - suggestion = f"Did you mean to call the file '{source_path.stem}.{kinds[0]}{source_path.suffix}'?" - else: - suggestion = ( - f"All files in the {resource_folder!r} folder must have a file extension that matches " - f"the resource type. Supported types are: {humanize_collection(kinds)}." - ) - return None, UnknownResourceTypeWarning(source_path, suggestion) - elif len(crud_candidates) > 1 and all(issubclass(loader, GroupIO) for loader in crud_candidates): - # There are two group cruds, one for resource scoped and one for all scoped. - return GroupIO, None - elif len(crud_candidates) == 1: - return crud_candidates[0], None - - # This is unreachable with our current ResourceCRUD classes. We have tests that is exhaustive over - # all ResourceCRUDs to ensure this. - names = humanize_collection( - [f"'{source_path.stem}.{loader.kind}{source_path.suffix}'" for loader in crud_candidates], bind_word="or" - ) - raise AmbiguousResourceFileError( - f"Ambiguous resource file {source_path.name} in {resource_folder} folder. " - f"Unclear whether it is {humanize_collection([crud_cls.kind for crud_cls in crud_candidates], bind_word='or')}." - f"\nPlease name the file {names}." - ) - - -class DefaultBuilder(Builder): - """This is used to build resources that do not have a specific builder.""" - - def build( - self, source_files: list[BuildSourceFile], module: ModuleLocation, console: Callable[[str], None] | None = None - ) -> Iterable[BuildDestinationFile | list[ToolkitWarning]]: - for source_file in source_files: - if source_file.loaded is None: - # Not a YAML file - continue - loader, warning = self._get_loader(source_file.source.path) - if loader is None: - if warning is not None: - yield [warning] - continue - destination_path = self._create_destination_path(source_file.source.path, loader.kind) - - destination = BuildDestinationFile( - path=destination_path, - loaded=source_file.loaded, - loader=loader, - source=source_file.source, - extra_sources=None, - ) - yield destination diff --git a/cognite_toolkit/_cdf_tk/builders/_datamodels.py b/cognite_toolkit/_cdf_tk/builders/_datamodels.py deleted file mode 100644 index 7b0f7e4b75..0000000000 --- a/cognite_toolkit/_cdf_tk/builders/_datamodels.py +++ /dev/null @@ -1,90 +0,0 @@ -import shutil -from collections.abc import Callable, Iterable -from pathlib import Path -from typing import Any - -from cognite_toolkit._cdf_tk.builders import Builder -from cognite_toolkit._cdf_tk.constants import INDEX_PATTERN -from cognite_toolkit._cdf_tk.data_classes import ( - BuildDestinationFile, - BuildSourceFile, - ModuleLocation, - SourceLocation, -) -from cognite_toolkit._cdf_tk.exceptions import ToolkitFileNotFoundError -from cognite_toolkit._cdf_tk.resource_ios import GraphQLCRUD -from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning - - -class DataModelBuilder(Builder): - _resource_folder = GraphQLCRUD.folder_name - - def build( - self, source_files: list[BuildSourceFile], module: ModuleLocation, console: Callable[[str], None] | None = None - ) -> Iterable[BuildDestinationFile | list[ToolkitWarning]]: - graphql_files = { - source_file.source.path: source_file - for source_file in source_files - if source_file.source.path.suffix == ".graphql" - } - - for source_file in source_files: - loaded = source_file.loaded - if loaded is None: - # Not a YAML file - continue - loader, warning = self._get_loader(source_file.source.path) - if loader is None: - if warning is not None: - yield [warning] - continue - - destination_path = self._create_destination_path(source_file.source.path, loader.kind) - - extra_sources: list[SourceLocation] | None = None - if loader is GraphQLCRUD: - # The GraphQL must be copied over instead of added to the DML field as - # it is hashed in the deployment step and used to determine if the DML has changed. - extra_sources = self._copy_graphql_to_build(source_file, destination_path, graphql_files) - - destination = BuildDestinationFile( - path=destination_path, - loaded=loaded, - loader=loader, - source=source_file.source, - extra_sources=extra_sources, - ) - yield destination - - def _copy_graphql_to_build( - self, - source_file: BuildSourceFile, - destination_path: Path, - graphql_files: dict[Path, BuildSourceFile], - ) -> list[SourceLocation]: - extra_sources: list[SourceLocation] = [] - loaded_list: list[dict[str, Any]] = ( - source_file.loaded if isinstance(source_file.loaded, list) else [source_file.loaded] # type: ignore[list-item] - ) - - for entry in loaded_list: - if "dml" in entry: - expected_filename = entry["dml"] - else: - expected_filename = f"{INDEX_PATTERN.sub('', source_file.source.path.stem.removesuffix(GraphQLCRUD.kind).removesuffix('.'))}.graphql" - expected_path = source_file.source.path.parent / Path(expected_filename) - - if expected_path in graphql_files: - dest_graphql = destination_path.with_suffix(".graphql") - shutil.copy(graphql_files[expected_path].source.path, dest_graphql) - extra_sources.append(graphql_files[expected_path].source) - # The build renames the .graphql file; update dml so deploy can locate it. - entry["dml"] = dest_graphql.name - else: - raise ToolkitFileNotFoundError( - f"Failed to find GraphQL file. Expected {expected_filename} adjacent to {source_file.source.path.as_posix()}" - ) - return extra_sources - - def load_extra_field(self, extra: str) -> tuple[str, Any]: - return "dml", extra diff --git a/cognite_toolkit/_cdf_tk/builders/_file.py b/cognite_toolkit/_cdf_tk/builders/_file.py deleted file mode 100644 index a074ec9b6a..0000000000 --- a/cognite_toolkit/_cdf_tk/builders/_file.py +++ /dev/null @@ -1,88 +0,0 @@ -import copy -from collections.abc import Callable, Iterable -from typing import Any - -from cognite_toolkit._cdf_tk.builders import Builder -from cognite_toolkit._cdf_tk.data_classes import ( - BuildDestinationFile, - BuildSourceFile, - ModuleLocation, -) -from cognite_toolkit._cdf_tk.exceptions import ToolkitYAMLFormatError -from cognite_toolkit._cdf_tk.resource_ios import CogniteFileCRUD, FileCRUD, FileMetadataCRUD -from cognite_toolkit._cdf_tk.tk_warnings import LowSeverityWarning, ToolkitWarning - - -class FileBuilder(Builder): - _resource_folder = FileMetadataCRUD.folder_name - template_pattern = "$FILENAME" - - def build( - self, source_files: list[BuildSourceFile], module: ModuleLocation, console: Callable[[str], None] | None = None - ) -> Iterable[BuildDestinationFile | list[ToolkitWarning]]: - for source_file in source_files: - loaded = source_file.loaded - if loaded is None: - continue - - loader, warning = self._get_loader(source_file.source.path) - if loader is None: - if warning is not None: - yield [warning] - continue - if loader in {FileMetadataCRUD, CogniteFileCRUD}: - loaded = self._expand_file_metadata(loaded, module, console) - destination_path = self._create_destination_path(source_file.source.path, loader.kind) - - yield BuildDestinationFile( - path=destination_path, - loaded=loaded, - loader=loader, - source=source_file.source, - extra_sources=None, - ) - - @classmethod - def _expand_file_metadata( - cls, - raw_list: list[dict[str, Any]] | dict[str, Any], - module: ModuleLocation, - console: Callable[[str], None] | None = None, - ) -> list[dict[str, Any]] | dict[str, Any]: - is_file_template = ( - isinstance(raw_list, list) - and len(raw_list) == 1 - and cls.template_pattern in raw_list[0].get("externalId", "") - ) - if not is_file_template: - if (isinstance(raw_list, dict) and cls.template_pattern in raw_list.get("externalId", "")) or ( - isinstance(raw_list, list) - and any(cls.template_pattern in entry.get("externalId", "") for entry in raw_list) - ): - raw_type = "dictionary" if isinstance(raw_list, dict) else "list with multiple entries" - LowSeverityWarning( - f"Invalid file template {cls.template_pattern!r} usage detected in {module.relative_path.as_posix()!r}.\n" - f"The file template is expected in a list with a single entry, but got {raw_type}." - ).print_warning() - - return raw_list - if not (isinstance(raw_list, list) and raw_list and isinstance(raw_list[0], dict)): - raise ToolkitYAMLFormatError( - f"Expected a list with a single dictionary in the file metadata file {module.dir}, " - f"but got {type(raw_list).__name__}" - ) - template = raw_list[0] - if console: - console( - f"Detected file template name {cls.template_pattern!r} in {module.relative_path.as_posix()!r}" - f"Expanding file metadata..." - ) - expanded_metadata: list[dict[str, Any]] = [] - for filepath in module.source_paths_by_resource_folder[FileCRUD.folder_name]: - if not FileCRUD.is_supported_file(filepath): - continue - new_entry = copy.deepcopy(template) - new_entry["externalId"] = new_entry["externalId"].replace(cls.template_pattern, filepath.name) - new_entry["name"] = filepath.name - expanded_metadata.append(new_entry) - return expanded_metadata diff --git a/cognite_toolkit/_cdf_tk/builders/_function.py b/cognite_toolkit/_cdf_tk/builders/_function.py deleted file mode 100644 index cf1ee86576..0000000000 --- a/cognite_toolkit/_cdf_tk/builders/_function.py +++ /dev/null @@ -1,187 +0,0 @@ -import shutil -import time -from collections.abc import Callable, Iterable, Sequence -from pathlib import Path -from typing import Any - -from cognite_toolkit._cdf_tk.builders import Builder -from cognite_toolkit._cdf_tk.data_classes import ( - BuildDestinationFile, - BuildSourceFile, - BuiltResourceList, - ModuleLocation, -) -from cognite_toolkit._cdf_tk.exceptions import ToolkitFileExistsError, ToolkitNotADirectoryError, ToolkitValueError -from cognite_toolkit._cdf_tk.feature_flags import Flags -from cognite_toolkit._cdf_tk.resource_ios import FunctionIO -from cognite_toolkit._cdf_tk.tk_warnings import ( - FileReadWarning, - HighSeverityWarning, - LowSeverityWarning, - MediumSeverityWarning, - RequirementsTXTValidationWarning, - ToolkitWarning, - WarningList, -) -from cognite_toolkit._cdf_tk.utils import validate_requirements_with_pip - - -class FunctionBuilder(Builder): - _resource_folder = FunctionIO.folder_name - - def __init__(self, build_dir: Path, warn: Callable[[ToolkitWarning], None]) -> None: - super().__init__(build_dir, warn=warn) - # Metrics for telemetry - self.validation_count = 0 - self.validation_failures = 0 - self.validation_credential_errors = 0 - self.validation_time_ms = 0 - - def _validate_function_requirements( - self, - requirements_txt: Path, - raw_function: dict[str, Any], - filepath: Path, - external_id: str, - ) -> RequirementsTXTValidationWarning | None: - """Validate function requirements.txt using pip dry-run.""" - start_time = time.time() - validation_result = validate_requirements_with_pip( - requirements_txt_path=requirements_txt, - index_url=raw_function.get("indexUrl"), - extra_index_urls=raw_function.get("extraIndexUrls"), - ) - elapsed_ms = int((time.time() - start_time) * 1000) - self.validation_count += 1 - self.validation_time_ms += elapsed_ms - - if validation_result.success: - return None - - self.validation_failures += 1 - if validation_result.is_credential_error: - self.validation_credential_errors += 1 - - return RequirementsTXTValidationWarning( - filepath=filepath, - external_id=external_id, - error_details=validation_result.short_error, - is_credential_error=validation_result.is_credential_error, - resource="function", - ) - - def build( - self, - source_files: list[BuildSourceFile], - module: ModuleLocation, - console: Callable[[str], None] | None = None, - ) -> Iterable[BuildDestinationFile | Sequence[ToolkitWarning]]: - for source_file in source_files: - if source_file.loaded is None: - continue - if source_file.source.path.parent.parent != module.dir: - # Function YAML files must be in the resource folder. - continue - - loader, warning = self._get_loader(source_file.source.path) - if loader is None: - if warning is not None: - yield [warning] - continue - - warnings = WarningList[FileReadWarning]() - if loader is FunctionIO: - warnings = self.copy_function_directory_to_build(source_file) - - destination_path = self._create_destination_path(source_file.source.path, loader.kind) - - yield BuildDestinationFile( - path=destination_path, - loaded=source_file.loaded, - loader=loader, - source=source_file.source, - extra_sources=None, - warnings=warnings, - ) - - def validate_directory( - self, - built_resources: BuiltResourceList, - module: ModuleLocation, - ) -> WarningList[ToolkitWarning]: - warnings = WarningList[ToolkitWarning]() - has_config_files = any(resource.kind == FunctionIO.kind for resource in built_resources) - if has_config_files: - return warnings - config_files_misplaced = [ - file - for file in module.source_paths_by_resource_folder[FunctionIO.folder_name] - if FunctionIO.is_supported_file(file) - ] - if config_files_misplaced: # and not has_config_files: - for yaml_source_path in config_files_misplaced: - required_location = module.dir / FunctionIO.folder_name / yaml_source_path.name - warning = LowSeverityWarning( - f"The required Function resource configuration file " - f"was not found in {required_location.as_posix()!r}. " - f"The file {yaml_source_path.as_posix()!r} is currently " - f"considered part of the Function's artifacts and " - f"will not be processed by the Toolkit.", - ) - warnings.append(warning) - return warnings - - def copy_function_directory_to_build(self, source_file: BuildSourceFile) -> WarningList[FileReadWarning]: - raw_content = source_file.loaded - if raw_content is None: - # This should already be checked before calling this method. - raise ToolkitValueError("Function source file should be a YAML file.") - raw_functions = raw_content if isinstance(raw_content, list) else [raw_content] - warnings = WarningList[FileReadWarning]() - for raw_function in raw_functions: - external_id = raw_function.get("externalId") - function_path = raw_function.get("functionPath") - if not external_id: - warnings.append( - HighSeverityWarning( - f"Function in {source_file.source.path.as_posix()!r} has no externalId defined. " - f"This is used to match the function to the function directory.", - ), - ) - continue - if not function_path: - warnings.append( - MediumSeverityWarning( - f"Function {external_id} in {source_file.source.path.as_posix()!r} has no function_path defined.", - ), - ) - - function_directory = source_file.source.path.with_name(external_id) - - if not function_directory.is_dir(): - raise ToolkitNotADirectoryError( - f"Function directory not found for externalId {external_id} defined in {source_file.source.path.as_posix()!r}.", - ) - - # Validate requirements.txt if present and feature is enabled - if ( - Flags.FUNCTION_REQUIREMENTS_VALIDATION.is_enabled() - and (requirements_txt := function_directory / "requirements.txt").exists() - ): - warning = self._validate_function_requirements( - requirements_txt, - raw_function, - source_file.source.path, - external_id, - ) - if warning: - warnings.append(warning) - - destination = self.build_dir / self.resource_folder / external_id - if destination.exists(): - raise ToolkitFileExistsError( - f"Function {external_id!r} is duplicated. If this is unexpected, ensure you have a clean build directory.", - ) - shutil.copytree(function_directory, destination, ignore=shutil.ignore_patterns("__pycache__")) - - return warnings diff --git a/cognite_toolkit/_cdf_tk/builders/_location.py b/cognite_toolkit/_cdf_tk/builders/_location.py deleted file mode 100644 index ca4c7bbf1a..0000000000 --- a/cognite_toolkit/_cdf_tk/builders/_location.py +++ /dev/null @@ -1,84 +0,0 @@ -from collections.abc import Callable, Iterable, Sequence -from graphlib import CycleError, TopologicalSorter -from typing import Any - -from cognite_toolkit._cdf_tk.builders._base import Builder -from cognite_toolkit._cdf_tk.data_classes._build_files import BuildDestinationFile, BuildSourceFile -from cognite_toolkit._cdf_tk.data_classes._module_directories import ModuleLocation -from cognite_toolkit._cdf_tk.exceptions import ToolkitError -from cognite_toolkit._cdf_tk.resource_ios._resource_ios.location import LocationFilterIO -from cognite_toolkit._cdf_tk.tk_warnings.base import ToolkitWarning, WarningList -from cognite_toolkit._cdf_tk.tk_warnings.fileread import FileReadWarning - - -class LocationBuilder(Builder): - _resource_folder = LocationFilterIO.folder_name - - def build( - self, source_files: list[BuildSourceFile], module: ModuleLocation, console: Callable[[str], None] | None = None - ) -> Iterable[BuildDestinationFile | Sequence[ToolkitWarning]]: - location_by_external_id: dict[str, tuple[dict[str, Any], BuildSourceFile]] = {} - location_hierarchy_graph: dict[str, list[Any]] = {} - - # Ordering all location filters in to ensure correct hierarchy dependency - # within the module. This is required by the Location API. - # Doing this in three stages: - # 1. collect all locations across source files, - # 2. sort them in a topological order, - # 3. create a new file for each location where the prefix index ensures deployment order - # ... while also maintaining reference to source file - - for source_file in source_files: - loader, warning = self._get_loader(source_file.source.path) - if isinstance(loader, LocationFilterIO): - if warning is not None: - yield [warning] - continue - - loaded_locations = ( - source_file.loaded - if isinstance(source_file.loaded, list) - else [source_file.loaded] - if source_file.loaded - else [] - ) - for loaded_location in loaded_locations: - ext_id = loaded_location.get("externalId") - parent_external_id = loaded_location.get("parentExternalId") - - if ext_id: - location_by_external_id[ext_id] = loaded_location, source_file - location_hierarchy_graph.setdefault(ext_id, []) - - if parent_external_id: - location_hierarchy_graph.setdefault(parent_external_id, []) - location_hierarchy_graph[ext_id].append(parent_external_id) - - warnings = WarningList[FileReadWarning]() - - ordered_locations: list[dict] = [] - try: - for external_id in TopologicalSorter(location_hierarchy_graph).static_order(): - if external_id not in location_by_external_id: - # The dependency is not in the module, so we skip it. - continue - location, _ = location_by_external_id[external_id] - ordered_locations.append(location) - except CycleError: - raise ToolkitError( - "Circular dependency found in Locations. Locations must be hierarchical. Please check the externalId and parentExternalId fields." - ) - - for item in ordered_locations: - external_id = item["externalId"] - (location, build_source_file) = location_by_external_id[external_id] - destination_path = self._create_destination_path(build_source_file.source.path, loader.kind) # type: ignore[union-attr] - - yield BuildDestinationFile( - path=destination_path, - loaded=location, - loader=loader, # type: ignore[arg-type] - source=build_source_file.source, - extra_sources=None, - warnings=warnings, - ) diff --git a/cognite_toolkit/_cdf_tk/builders/_raw.py b/cognite_toolkit/_cdf_tk/builders/_raw.py deleted file mode 100644 index 012cc248b2..0000000000 --- a/cognite_toolkit/_cdf_tk/builders/_raw.py +++ /dev/null @@ -1,72 +0,0 @@ -from collections import defaultdict -from collections.abc import Callable, Iterable, Sequence -from typing import Any - -from cognite_toolkit._cdf_tk.builders import Builder -from cognite_toolkit._cdf_tk.client.identifiers import RawDatabaseId -from cognite_toolkit._cdf_tk.data_classes import ( - BuildDestinationFile, - BuildSourceFile, - ModuleLocation, - SourceLocation, - SourceLocationEager, -) -from cognite_toolkit._cdf_tk.resource_ios import RawDatabaseCRUD, RawTableCRUD, ResourceIO -from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning -from cognite_toolkit._cdf_tk.utils import calculate_hash -from cognite_toolkit._cdf_tk.utils.file import yaml_safe_dump - - -class RawBuilder(Builder): - _resource_folder = RawDatabaseCRUD.folder_name - - def build( - self, source_files: list[BuildSourceFile], module: ModuleLocation, console: Callable[[str], None] | None = None - ) -> Iterable[BuildDestinationFile | Sequence[ToolkitWarning]]: - for source_file in source_files: - loaded = source_file.loaded - if loaded is None: - continue - loaded_list = loaded if isinstance(loaded, list) else [loaded] - seen_databases: set[tuple] = set() - entry_by_loader: dict[type[ResourceIO], list[dict[str, Any]]] = defaultdict(list) - has_split_table_and_database = False - - for item in loaded_list: - try: - table_id = RawTableCRUD.get_id(item) - except KeyError: - seen_databases.add(tuple(item.items())) - entry_by_loader[RawDatabaseCRUD].append(item) - else: - entry_by_loader[RawTableCRUD].append(item) - db_item = RawDatabaseCRUD.dump_id(RawDatabaseId(name=table_id.db_name)) - hashable_db_item = tuple(db_item.items()) - if hashable_db_item not in seen_databases: - seen_databases.add(hashable_db_item) - entry_by_loader[RawDatabaseCRUD].append(db_item) - has_split_table_and_database = True - - for loader, entries in entry_by_loader.items(): - if not entries: - continue - destination_path = self._create_destination_path(source_file.source.path, loader.kind) - - if loader is RawDatabaseCRUD and has_split_table_and_database: - # We have inferred the database from a Table file, so we need to recalculate the hash - # in case we also inferred the database from another Table file - new_hash = calculate_hash( - yaml_safe_dump(sorted(entries, key=lambda entry: entry["dbName"])), - shorten=True, - ) - source: SourceLocation = SourceLocationEager(path=source_file.source.path, _hash=new_hash) - else: - source = source_file.source - - yield BuildDestinationFile( - path=destination_path, - loaded=entries, - loader=loader, - source=source, - extra_sources=None, - ) diff --git a/cognite_toolkit/_cdf_tk/builders/_ruleset.py b/cognite_toolkit/_cdf_tk/builders/_ruleset.py deleted file mode 100644 index 8572c34bd2..0000000000 --- a/cognite_toolkit/_cdf_tk/builders/_ruleset.py +++ /dev/null @@ -1,105 +0,0 @@ -from collections.abc import Callable, Iterable -from pathlib import Path -from typing import Any - -from cognite_toolkit._cdf_tk.builders import Builder -from cognite_toolkit._cdf_tk.constants import BUILD_FOLDER_ENCODING -from cognite_toolkit._cdf_tk.data_classes import ( - BuildDestinationFile, - BuildSourceFile, - ModuleLocation, - SourceLocation, -) -from cognite_toolkit._cdf_tk.exceptions import ToolkitFileNotFoundError, ToolkitYAMLFormatError -from cognite_toolkit._cdf_tk.resource_ios import RuleSetVersionIO -from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning -from cognite_toolkit._cdf_tk.utils import safe_write - - -class RuleSetBuilder(Builder): - _resource_folder = RuleSetVersionIO.folder_name - - def build( - self, source_files: list[BuildSourceFile], module: ModuleLocation, console: Callable[[str], None] | None = None - ) -> Iterable[BuildDestinationFile | list[ToolkitWarning]]: - ttl_files = { - source_file.source.path: source_file - for source_file in source_files - if source_file.source.path.suffix == ".ttl" - } - - for source_file in source_files: - loaded = source_file.loaded - if loaded is None: - continue - loader, warning = self._get_loader(source_file.source.path) - if loader is None: - if warning is not None: - yield [warning] - continue - - destination_path = self._create_destination_path(source_file.source.path, loader.kind) - - extra_sources: list[SourceLocation] | None = None - if loader is RuleSetVersionIO: - extra_sources = self._add_rules(loaded, source_file, ttl_files, destination_path) - - destination = BuildDestinationFile( - path=destination_path, - loaded=loaded, - loader=loader, - source=source_file.source, - extra_sources=extra_sources, - ) - yield destination - - def load_extra_field(self, extra: str) -> tuple[str, Any]: - return "rules", [extra] - - def _add_rules( - self, - loaded: dict[str, Any] | list[dict[str, Any]], - source_file: BuildSourceFile, - ttl_files: dict[Path, BuildSourceFile], - ruleset_destination_path: Path, - ) -> list[SourceLocation]: - loaded_list = loaded if isinstance(loaded, list) else [loaded] - extra_sources: list[SourceLocation] = [] - for entry in loaded_list: - try: - id_ = RuleSetVersionIO.get_id(entry) - except KeyError: - continue - filepath = source_file.source.path - ttl_file = self._get_ttl_file(filepath, id_.rule_set_external_id, ttl_files) - - if "rules" in entry and ttl_file is not None: - raise ToolkitYAMLFormatError( - f"'rules' is defined in both the YAML and a separate file named {ttl_file.source.path}\n" - f"Please remove one: either the inline 'rules' in {filepath} or the file {ttl_file.source.path}", - ) - if "rules" not in entry and ttl_file is None: - raise ToolkitFileNotFoundError( - f"'rules' is missing and no .ttl file found. Expected {filepath.stem}.ttl or {id_.rule_set_external_id}.ttl next to {filepath}", - filepath, - ) - if ttl_file is not None: - destination_path = self._create_destination_path(ttl_file.source.path, "Rules") - safe_write(destination_path, ttl_file.content, encoding=BUILD_FOLDER_ENCODING) - entry["rules"] = [ttl_file.content] - extra_sources.append(ttl_file.source) - - return extra_sources - - @staticmethod - def _get_ttl_file( - source_file: Path, rule_set_external_id: str | None, ttl_files: dict[Path, BuildSourceFile] - ) -> BuildSourceFile | None: - ttl_path = source_file.parent / f"{source_file.stem}.ttl" - if ttl_path in ttl_files: - return ttl_files[ttl_path] - if rule_set_external_id: - ttl_path = source_file.parent / f"{rule_set_external_id}.ttl" - if ttl_path in ttl_files: - return ttl_files[ttl_path] - return None diff --git a/cognite_toolkit/_cdf_tk/builders/_streamlit.py b/cognite_toolkit/_cdf_tk/builders/_streamlit.py deleted file mode 100644 index 66571aab5f..0000000000 --- a/cognite_toolkit/_cdf_tk/builders/_streamlit.py +++ /dev/null @@ -1,110 +0,0 @@ -import shutil -from collections.abc import Callable, Iterable, Sequence - -from cognite_toolkit._cdf_tk.builders import Builder -from cognite_toolkit._cdf_tk.data_classes import ( - BuildDestinationFile, - BuildSourceFile, - ModuleLocation, -) -from cognite_toolkit._cdf_tk.exceptions import ToolkitFileExistsError, ToolkitNotADirectoryError, ToolkitValueError -from cognite_toolkit._cdf_tk.feature_flags import Flags -from cognite_toolkit._cdf_tk.resource_ios import StreamlitIO -from cognite_toolkit._cdf_tk.tk_warnings import ( - FileReadWarning, - HighSeverityWarning, - RequirementsTXTValidationWarning, - StreamlitRequirementsWarning, - ToolkitWarning, - WarningList, -) -from cognite_toolkit._cdf_tk.utils import validate_requirements_with_pip -from cognite_toolkit._cdf_tk.utils.file import safe_read - - -class StreamlitBuilder(Builder): - _resource_folder = StreamlitIO.folder_name - - def build( - self, source_files: list[BuildSourceFile], module: ModuleLocation, console: Callable[[str], None] | None = None - ) -> Iterable[BuildDestinationFile | Sequence[ToolkitWarning]]: - for source_file in source_files: - if source_file.loaded is None: - continue - if source_file.source.path.parent.parent != module.dir: - # Streamlit YAML files must be in the resource folder top level - continue - - loader, warning = self._get_loader(source_file.source.path) - if loader is None: - if warning is not None: - yield [warning] - continue - - warnings = WarningList[FileReadWarning]() - if loader is StreamlitIO: - warnings = self.copy_app_directory_to_build(source_file) - - destination_path = self._create_destination_path(source_file.source.path, loader.kind) - - yield BuildDestinationFile( - path=destination_path, - loaded=source_file.loaded, - loader=loader, - source=source_file.source, - extra_sources=None, - warnings=warnings, - ) - - def copy_app_directory_to_build(self, source_file: BuildSourceFile) -> WarningList[FileReadWarning]: - raw_content = source_file.loaded - if raw_content is None: - # This should already be checked before calling this method. - raise ToolkitValueError("Streamlit source file should be a YAML file.") - raw_apps = raw_content if isinstance(raw_content, list) else [raw_content] - warnings = WarningList[FileReadWarning]() - for raw_app in raw_apps: - external_id = raw_app.get("externalId") - if not external_id: - warnings.append( - HighSeverityWarning( - f"StreamlitApp in {source_file.source.path.as_posix()!r} has no externalId defined. " - f"This is used to match the Streamlit App to directory." - ) - ) - continue - app_directory = source_file.source.path.with_name(external_id) - - if not app_directory.is_dir(): - raise ToolkitNotADirectoryError( - f"StreamlitApp directory not found in {app_directory}(based on externalId {external_id} defined in {source_file.source.path.as_posix()!r}.)" - ) - - if (requirements_txt := app_directory / "requirements.txt").exists() and ( - Flags.FUNCTION_REQUIREMENTS_VALIDATION.is_enabled() - ): - validation_result = validate_requirements_with_pip(requirements_txt_path=requirements_txt) - if not validation_result.success: - warnings.append( - RequirementsTXTValidationWarning( - filepath=source_file.source.path, - error_details=validation_result.short_error, - is_credential_error=validation_result.is_credential_error, - external_id=external_id, - resource="streamlit", - ) - ) - - requirements_file_content = safe_read(app_directory / "requirements.txt").splitlines() - missing_packages = StreamlitIO._missing_recommended_requirements(requirements_file_content) - if len(missing_packages) > 0: - warnings.append(StreamlitRequirementsWarning(app_directory / "requirements.txt", missing_packages)) - - destination = self.build_dir / self.resource_folder / external_id - if destination.exists(): - raise ToolkitFileExistsError( - f"StreamlitApp {external_id!r} is duplicated. If this is unexpected, ensure you have a clean build directory." - ) - shutil.copytree(app_directory, destination, ignore=shutil.ignore_patterns("__pycache__")) - - return warnings diff --git a/cognite_toolkit/_cdf_tk/builders/_transformation.py b/cognite_toolkit/_cdf_tk/builders/_transformation.py deleted file mode 100644 index 1132bd7e36..0000000000 --- a/cognite_toolkit/_cdf_tk/builders/_transformation.py +++ /dev/null @@ -1,111 +0,0 @@ -from collections.abc import Callable, Iterable -from pathlib import Path -from typing import Any - -from cognite_toolkit._cdf_tk.builders import Builder -from cognite_toolkit._cdf_tk.constants import BUILD_FOLDER_ENCODING -from cognite_toolkit._cdf_tk.data_classes import ( - BuildDestinationFile, - BuildSourceFile, - ModuleLocation, - SourceLocation, -) -from cognite_toolkit._cdf_tk.exceptions import ToolkitYAMLFormatError -from cognite_toolkit._cdf_tk.resource_ios import TransformationIO -from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, ToolkitWarning -from cognite_toolkit._cdf_tk.utils import safe_write - - -class TransformationBuilder(Builder): - _resource_folder = TransformationIO.folder_name - - def build( - self, source_files: list[BuildSourceFile], module: ModuleLocation, console: Callable[[str], None] | None = None - ) -> Iterable[BuildDestinationFile | list[ToolkitWarning]]: - query_files = { - source_file.source.path: source_file - for source_file in source_files - if source_file.source.path.suffix == ".sql" - } - - for source_file in source_files: - loaded = source_file.loaded - if loaded is None: - # Not a YAML file - continue - loader, warning = self._get_loader(source_file.source.path) - if loader is None: - if warning is not None: - yield [warning] - continue - - destination_path = self._create_destination_path(source_file.source.path, loader.kind) - - extra_sources: list[SourceLocation] | None = None - if loader is TransformationIO: - extra_sources = self._add_query(loaded, source_file, query_files, destination_path) - - destination = BuildDestinationFile( - path=destination_path, - loaded=loaded, - loader=loader, - source=source_file.source, - extra_sources=extra_sources, - ) - yield destination - - def load_extra_field(self, extra: str) -> tuple[str, Any]: - return "query", extra - - def _add_query( - self, - loaded: dict[str, Any] | list[dict[str, Any]], - source_file: BuildSourceFile, - query_files: dict[Path, BuildSourceFile], - transformation_destination_path: Path, - ) -> list[SourceLocation]: - loaded_list = loaded if isinstance(loaded, list) else [loaded] - extra_sources: list[SourceLocation] = [] - for entry in loaded_list: - try: - id_ = TransformationIO.get_id(entry) - except KeyError: - # This will be validated later - continue - filepath = source_file.source.path - query_file = self._get_query_file(filepath, id_.external_id, query_files) - - if "query" in entry and query_file is not None: - raise ToolkitYAMLFormatError( - f"query property is ambiguously defined in both the yaml file and a separate file named {query_file}\n" - f"Please remove one of the definitions, either the query property in {filepath} or the file {query_file}", - ) - elif "query" not in entry and query_file is None: - warning = HighSeverityWarning( - f"query property or is missing in {filepath.as_posix()!r}. It can be inline or a separate file named {filepath.stem}.sql or {id_}.sql", - ) - if self.warn: - self.warn(warning) - else: - warning.print_warning() - elif query_file is not None: - destination_path = self._create_destination_path(query_file.source.path, "Query") - safe_write(destination_path, query_file.content, encoding=BUILD_FOLDER_ENCODING) - relative = destination_path.relative_to(transformation_destination_path.parent) - entry["queryFile"] = relative.as_posix() - extra_sources.append(query_file.source) - - return extra_sources - - @staticmethod - def _get_query_file( - source_file: Path, transformation_external_id: str | None, query_files: dict[Path, BuildSourceFile] - ) -> BuildSourceFile | None: - query_file = source_file.parent / f"{source_file.stem}.sql" - if query_file in query_files: - return query_files[query_file] - if transformation_external_id: - query_file = source_file.parent / f"{transformation_external_id}.sql" - if query_file in query_files: - return query_files[query_file] - return None diff --git a/tests/test_unit/test_cdf_tk/test_cruds/test_data_model.py b/tests/test_unit/test_cdf_tk/test_cruds/test_data_model.py index 71f4e99517..eba0cc76dc 100644 --- a/tests/test_unit/test_cdf_tk/test_cruds/test_data_model.py +++ b/tests/test_unit/test_cdf_tk/test_cruds/test_data_model.py @@ -711,49 +711,3 @@ def test_dml_compile_error_surfaced_as_actionable_message(self) -> None: ) with pytest.raises(ToolkitAPIError, match="Type 'Foo' not found"): self._make_api(body)._post_graphql({"query": "...", "variables": {}}) - - -class TestDataModelBuilder: - """Regression tests for DataModelBuilder (build v1).""" - - def test_dml_updated_to_renamed_graphql_in_build(self, tmp_path: Path) -> None: - # Regression test: build renames .graphql files with a long prefix, but deploy - # looks up the file via entry["dml"]. _copy_graphql_to_build must update "dml" - # so that deploy finds the renamed file instead of the original source name. - from cognite_toolkit._cdf_tk.builders._datamodels import DataModelBuilder - from cognite_toolkit._cdf_tk.data_classes._build_files import BuildSourceFile - from cognite_toolkit._cdf_tk.data_classes._built_resources import SourceLocationEager - - source_dir = tmp_path / "source" / "data_modeling" - source_dir.mkdir(parents=True) - build_dir = tmp_path / "build" - build_dir.mkdir() - - yaml_path = source_dir / "my_model.GraphQLSchema.yaml" - graphql_path = source_dir / "original_schema.graphql" - yaml_path.write_text("space: my_space\nexternalId: MyModel\nversion: v1\ndml: original_schema.graphql\n") - graphql_path.write_text("type Foo { name: String }") - - entry: dict = {"space": "my_space", "externalId": "MyModel", "version": "v1", "dml": "original_schema.graphql"} - source_file = BuildSourceFile( - source=SourceLocationEager(path=yaml_path, _hash="abc"), - content=yaml_path.read_text(), - loaded=entry, - ) - graphql_source = BuildSourceFile( - source=SourceLocationEager(path=graphql_path, _hash="def"), - content=graphql_path.read_text(), - loaded=None, - ) - - builder = DataModelBuilder(build_dir=build_dir) - destination_path = build_dir / "data_modeling" / "1-my_model-SPP-COR.my_model.GraphQLSchema.yaml" - destination_path.parent.mkdir(parents=True, exist_ok=True) - - builder._copy_graphql_to_build(source_file, destination_path, {graphql_path: graphql_source}) - - # The "dml" field in the entry dict must be updated to the renamed build filename. - renamed_graphql = destination_path.with_suffix(".graphql").name - assert entry["dml"] == renamed_graphql, ( - f"entry['dml'] was not updated after build rename: got {entry['dml']!r}, expected {renamed_graphql!r}" - ) From 7ab1a8e22e120295ebecc9eaca2d3d6b894b730d Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 7 Sep 2026 20:12:34 +0200 Subject: [PATCH 12/31] tests: update python-sdk imports --- tests/test_unit/test_toolkit_package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_unit/test_toolkit_package.py b/tests/test_unit/test_toolkit_package.py index 04f3b2fa9b..5cfb51eb07 100644 --- a/tests/test_unit/test_toolkit_package.py +++ b/tests/test_unit/test_toolkit_package.py @@ -94,7 +94,7 @@ def test_no_cognite_sdk_imports() -> None: The goal is to fully remove the cognite-sdk dependency from the toolkit (with the exception of Auth and protobuf files). This test tracks progress toward that goal. """ - _assert_import_violations(_extract_cognite_sdk_imports, "cognite.client imports", 94) + _assert_import_violations(_extract_cognite_sdk_imports, "cognite.client imports", 93) def _parse_package_name(dependency: str) -> str: From f52f50128a5dc3f888d65bf48d89ae479b8ca0bc Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 7 Sep 2026 20:16:38 +0200 Subject: [PATCH 13/31] refactor: delete data crud --- .../_cdf_tk/resource_ios/__init__.py | 4 +- .../_cdf_tk/resource_ios/_data_cruds.py | 139 +----------------- tests/test_unit/approval_client/client.py | 4 +- 3 files changed, 5 insertions(+), 142 deletions(-) diff --git a/cognite_toolkit/_cdf_tk/resource_ios/__init__.py b/cognite_toolkit/_cdf_tk/resource_ios/__init__.py index ea081a7c2b..cd7d16e135 100644 --- a/cognite_toolkit/_cdf_tk/resource_ios/__init__.py +++ b/cognite_toolkit/_cdf_tk/resource_ios/__init__.py @@ -18,7 +18,7 @@ from cognite_toolkit._cdf_tk.feature_flags import FeatureFlag, Flags from ._base_ios import DataCRUD, Loader, ResourceContainerIO, ResourceIO -from ._data_cruds import DatapointsCRUD, FileCRUD, RawFileCRUD +from ._data_cruds import RawFileCRUD from ._resource_ios import ( AgentIO, AppIO, @@ -222,13 +222,11 @@ def get_crud(resource_dir: str, kind: str) -> type[Loader]: "DataProductVersionIO", "DataSetsIO", "DatapointSubscriptionIO", - "DatapointsCRUD", "EdgeCRUD", "EventIO", "ExternalDataSourceIO", "ExtractionPipelineConfigIO", "ExtractionPipelineIO", - "FileCRUD", "FileMetadataCRUD", "FunctionIO", "FunctionScheduleIO", diff --git a/cognite_toolkit/_cdf_tk/resource_ios/_data_cruds.py b/cognite_toolkit/_cdf_tk/resource_ios/_data_cruds.py index 0605aa1e41..49955a45bb 100644 --- a/cognite_toolkit/_cdf_tk/resource_ios/_data_cruds.py +++ b/cognite_toolkit/_cdf_tk/resource_ios/_data_cruds.py @@ -1,154 +1,19 @@ import io from collections.abc import Iterable -from pathlib import Path from typing import TYPE_CHECKING, cast, final import pandas as pd -from cognite.client import data_modeling as dm -from cognite_toolkit._cdf_tk.client._resource_base import T_Identifier, T_RequestResource, T_ResponseResource -from cognite_toolkit._cdf_tk.client.identifiers import ExternalId -from cognite_toolkit._cdf_tk.client.resource_classes.cognite_file import CogniteFileRequest -from cognite_toolkit._cdf_tk.client.resource_classes.data_modeling import NodeId -from cognite_toolkit._cdf_tk.client.resource_classes.filemetadata import FileMetadataRequest from cognite_toolkit._cdf_tk.client.resource_classes.raw import RAWTableResponse -from cognite_toolkit._cdf_tk.constants import BUILD_FOLDER_ENCODING -from cognite_toolkit._cdf_tk.utils import read_yaml_content, safe_read from cognite_toolkit._cdf_tk.utils.file import read_csv -from ._base_ios import DataCRUD, ResourceIO -from ._resource_ios import CogniteFileCRUD, FileMetadataCRUD, RawTableCRUD, TimeSeriesCRUD +from ._base_ios import DataCRUD +from ._resource_ios import RawTableCRUD if TYPE_CHECKING: from cognite_toolkit._cdf_tk.data_classes import BuildEnvironment -@final -class DatapointsCRUD(DataCRUD): - item_name = "datapoints" - folder_name = "timeseries" - kind = "Datapoints" - dependencies = frozenset({TimeSeriesCRUD}) - _doc_url = "Time-series/operation/postMultiTimeSeriesDatapoints" - - @property - def display_name(self) -> str: - return "timeseries datapoints" - - def upload(self, state: "BuildEnvironment", dry_run: bool) -> Iterable[tuple[str, int]]: - if self.folder_name not in state.built_resources: - return - - resource_directories = state.built_resources[self.folder_name].get_resource_directories(self.folder_name) - - for resource_dir in resource_directories: - for datafile in resource_dir.rglob("*"): - if not datafile.stem.casefold().endswith(self.kind.casefold()): - continue - if datafile.suffix == ".csv": - # The replacement is used to ensure that we read exactly the same file on Windows and Linux - file_content = datafile.read_bytes().replace(b"\r\n", b"\n").decode("utf-8") - data = read_csv(io.StringIO(file_content), parse_dates=True, index_col=0) - data.index = pd.DatetimeIndex(data.index) - elif datafile.suffix == ".parquet": - data = pd.read_parquet(datafile, engine="pyarrow") - else: - continue - timeseries_ids = list(data.columns) - if len(timeseries_ids) == 1: - ts_str = timeseries_ids[0] - elif len(timeseries_ids) <= 10: - ts_str = str(timeseries_ids) - else: - ts_str = f"{len(timeseries_ids):,} timeseries" - - if data.empty: - yield ( - f"Empty file {datafile.as_posix()!r}. No datapoints to inserted.", - 0, - ) - continue - - if dry_run: - yield ( - f" Would insert '{len(data):,}x{len(data.columns):,}' datapoints from '{datafile!s}' into {ts_str}", - len(data) * len(data.columns), - ) - else: - self.client.time_series.data.insert_dataframe(data) - - yield ( - f" Inserted '{len(data):,}x{len(data.columns):,}' datapoints from '{datafile!s}' into {ts_str}", - len(data) * len(data.columns), - ) - - -@final -class FileCRUD(DataCRUD): - item_name = "file contents" - folder_name = "files" - kind = "File" - dependencies = frozenset({FileMetadataCRUD, CogniteFileCRUD}) - _doc_url = "Files/operation/initFileUpload" - - @property - def display_name(self) -> str: - return "file content" - - def upload(self, state: "BuildEnvironment", dry_run: bool) -> Iterable[tuple[str, int]]: - if self.folder_name not in state.built_resources: - return - - for resource in state.built_resources[self.folder_name]: - if resource.destination is None: - continue - - if result := { - FileMetadataCRUD.kind: (FileMetadataCRUD, "external_id"), - CogniteFileCRUD.kind: (CogniteFileCRUD, "instance_id"), - }.get(resource.kind): - loader_cls, id_name = result - meta: FileMetadataRequest | CogniteFileRequest = self._read_metadata( - resource.destination, - loader_cls, # type: ignore[arg-type] - resource.identifier, - ) - if meta.name is None: - continue - datafile = resource.source.path.parent / meta.name - if not datafile.exists(): - continue - - identifier = resource.identifier - if isinstance(identifier, ExternalId): - identifier = identifier.external_id - elif isinstance(identifier, NodeId): - identifier = dm.NodeId(identifier.space, identifier.external_id) - if dry_run: - yield f" Would upload file '{datafile!s}' to file with {id_name}={identifier!r}", 1 - else: - self.client.files.upload_content(path=str(datafile), **{id_name: identifier}) - yield f" Uploaded file '{datafile!s}' to file with {id_name}={identifier!r}", 1 - - @staticmethod - def _read_metadata( - destination: Path, - loader: type[ResourceIO[T_Identifier, T_RequestResource, T_ResponseResource]], - identifier: T_Identifier, - ) -> T_RequestResource: - built_content = read_yaml_content(safe_read(destination, encoding=BUILD_FOLDER_ENCODING)) - if isinstance(built_content, dict): - return loader.resource_write_cls._load(built_content) - elif isinstance(built_content, list): - write_resources = (loader.resource_write_cls._load(content) for content in built_content) - try: - return next(m for m in write_resources if loader.get_id(m) == identifier) - except StopIteration: - raise RuntimeError(f"Missing metadata for {destination.as_posix()}") - - raise RuntimeError(f"Unexpected content type {type(built_content)} in {destination.as_posix()}") - - @final class RawFileCRUD(DataCRUD): item_name = "rows" diff --git a/tests/test_unit/approval_client/client.py b/tests/test_unit/approval_client/client.py index a846fb52cd..b6f84fb1ba 100644 --- a/tests/test_unit/approval_client/client.py +++ b/tests/test_unit/approval_client/client.py @@ -76,7 +76,7 @@ from cognite_toolkit._cdf_tk.client.resource_classes.streams import StreamResponse from cognite_toolkit._cdf_tk.client.testing import ToolkitClientMock from cognite_toolkit._cdf_tk.constants import INDEX_PATTERN, STREAM_IMMUTABLE_TEMPLATE_NAME -from cognite_toolkit._cdf_tk.resource_ios import FileCRUD +from cognite_toolkit._cdf_tk.dataio import FileMetadataDataIO from cognite_toolkit._cdf_tk.utils import calculate_hash from cognite_toolkit._cdf_tk.utils.auth import CLIENT_NAME from tests.constants import CDF_PROJECT @@ -739,7 +739,7 @@ def _upload_file_content_files_api( entry = instance_id.dump() entry["filehash"] = filehash - created_resources[FileCRUD.__name__].append(entry) + created_resources[FileMetadataDataIO.__name__].append(entry) return FileMetadata( external_id=external_id, From c8e2d95af9d6a95d6d7e3ea2a1239d7bc7de64d1 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 7 Sep 2026 20:20:27 +0200 Subject: [PATCH 14/31] refactor: remove remaining part of crud --- .../_cdf_tk/resource_ios/__init__.py | 9 +- .../_cdf_tk/resource_ios/_base_ios.py | 24 ---- .../_cdf_tk/resource_ios/_data_cruds.py | 84 -------------- .../test_cdf_tk/test_cruds/test_raw_file.py | 107 ------------------ 4 files changed, 2 insertions(+), 222 deletions(-) delete mode 100644 cognite_toolkit/_cdf_tk/resource_ios/_data_cruds.py delete mode 100644 tests/test_unit/test_cdf_tk/test_cruds/test_raw_file.py diff --git a/cognite_toolkit/_cdf_tk/resource_ios/__init__.py b/cognite_toolkit/_cdf_tk/resource_ios/__init__.py index cd7d16e135..5b5caa5f4c 100644 --- a/cognite_toolkit/_cdf_tk/resource_ios/__init__.py +++ b/cognite_toolkit/_cdf_tk/resource_ios/__init__.py @@ -17,8 +17,7 @@ from cognite_toolkit._cdf_tk.feature_flags import FeatureFlag, Flags -from ._base_ios import DataCRUD, Loader, ResourceContainerIO, ResourceIO -from ._data_cruds import RawFileCRUD +from ._base_ios import Loader, ResourceContainerIO, ResourceIO from ._resource_ios import ( AgentIO, AppIO, @@ -119,10 +118,9 @@ for _loader in itertools.chain( ResourceIO.__subclasses__(), ResourceContainerIO.__subclasses__(), - DataCRUD.__subclasses__(), GroupIO.__subclasses__(), ): - if _loader in [ResourceIO, ResourceContainerIO, DataCRUD, GroupIO]: + if _loader in [ResourceIO, ResourceContainerIO, GroupIO]: # Skipping base classes continue # MyPy bug: https://github.com/python/mypy/issues/4717 @@ -150,7 +148,6 @@ CRUD_LIST = list(itertools.chain.from_iterable(CRUDS_BY_FOLDER_NAME.values())) RESOURCE_CRUD_LIST = [loader for loader in CRUD_LIST if issubclass(loader, ResourceIO)] RESOURCE_CRUD_CONTAINER_LIST = [loader for loader in CRUD_LIST if issubclass(loader, ResourceContainerIO)] -RESOURCE_DATA_CRUD_LIST = [loader for loader in CRUD_LIST if issubclass(loader, DataCRUD)] KINDS_BY_FOLDER_NAME: dict[str, set[str]] = {} for crud in CRUD_LIST: if crud.folder_name not in KINDS_BY_FOLDER_NAME: @@ -216,7 +213,6 @@ def get_crud(resource_dir: str, kind: str) -> type[Loader]: "AssetIO", "CogniteFileCRUD", "ContainerCRUD", - "DataCRUD", "DataModelIO", "DataProductIO", "DataProductVersionIO", @@ -243,7 +239,6 @@ def get_crud(resource_dir: str, kind: str) -> type[Loader]: "LocationFilterIO", "NodeCRUD", "RawDatabaseCRUD", - "RawFileCRUD", "RawTableCRUD", "RelationshipIO", "ResourceContainerIO", diff --git a/cognite_toolkit/_cdf_tk/resource_ios/_base_ios.py b/cognite_toolkit/_cdf_tk/resource_ios/_base_ios.py index a58b7a0c49..4592c5c977 100644 --- a/cognite_toolkit/_cdf_tk/resource_ios/_base_ios.py +++ b/cognite_toolkit/_cdf_tk/resource_ios/_base_ios.py @@ -24,7 +24,6 @@ if TYPE_CHECKING: from cognite_toolkit._cdf_tk.commands.build_v2.data_classes import BuildVariable, ResourceType - from cognite_toolkit._cdf_tk.data_classes import BuildEnvironment if sys.version_info >= (3, 11): from typing import Self @@ -502,26 +501,3 @@ def count(self, ids: Sequence[T_Identifier]) -> int: @abstractmethod def drop_data(self, ids: Sequence[T_Identifier]) -> int: raise NotImplementedError - - -class DataCRUD(Loader, ABC): - """This is the base class for all data CRUD.. - - A data CRUD is a CRUD that uploads data to CDF. It will typically depend on a - resource container that stores the data. For example, the datapoints crud depends - on the timeseries CRUD. - - It has only one required method: - - upload: Uploads the data to CDF. - - class attributes: - item_name: The name of the item that is stored in the resource container. This should be set in the subclass. - It is used to display messages when running operations. - - """ - - item_name: str - - @abstractmethod - def upload(self, state: "BuildEnvironment", dry_run: bool) -> Iterable[tuple[str, int]]: - raise NotImplementedError diff --git a/cognite_toolkit/_cdf_tk/resource_ios/_data_cruds.py b/cognite_toolkit/_cdf_tk/resource_ios/_data_cruds.py deleted file mode 100644 index 49955a45bb..0000000000 --- a/cognite_toolkit/_cdf_tk/resource_ios/_data_cruds.py +++ /dev/null @@ -1,84 +0,0 @@ -import io -from collections.abc import Iterable -from typing import TYPE_CHECKING, cast, final - -import pandas as pd - -from cognite_toolkit._cdf_tk.client.resource_classes.raw import RAWTableResponse -from cognite_toolkit._cdf_tk.utils.file import read_csv - -from ._base_ios import DataCRUD -from ._resource_ios import RawTableCRUD - -if TYPE_CHECKING: - from cognite_toolkit._cdf_tk.data_classes import BuildEnvironment - - -@final -class RawFileCRUD(DataCRUD): - item_name = "rows" - folder_name = "raw" - kind = "Raw" - dependencies = frozenset({RawTableCRUD}) - _doc_url = "Raw/operation/postRows" - - @property - def display_name(self) -> str: - return "raw rows" - - def upload(self, state: "BuildEnvironment", dry_run: bool) -> Iterable[tuple[str, int]]: - if self.folder_name not in state.built_resources: - return - - for resource in state.built_resources[self.folder_name]: - if resource.kind != RawTableCRUD.kind: - continue - table = cast(RAWTableResponse, resource.identifier) - datafile = next( - ( - resource.source.path.with_suffix(f".{file_type}") - for file_type in ["csv", "parquet"] - if (resource.source.path.with_suffix(f".{file_type}").exists()) - ), - None, - ) - if datafile is None: - # No adjacent data file found - continue - - if datafile.suffix == ".csv": - # The replacement is used to ensure that we read exactly the same file on Windows and Linux - file_content = datafile.read_bytes().replace(b"\r\n", b"\n").decode("utf-8") - data = read_csv(io.StringIO(file_content)).astype(object).fillna("") - if not data.columns.empty and data.columns[0] == "key": - print(f"Setting index to 'key' for {datafile.name}") - data.set_index("key", inplace=True) - elif datafile.suffix == ".parquet": - data = pd.read_parquet(datafile, engine="pyarrow") - else: - raise ValueError(f"Unsupported file type {datafile.suffix} for {datafile.name}") - - if data.empty: - yield ( - f"Empty file {datafile.as_posix()!r}. No rows to insert into {table!r}.", - 0, - ) - continue - - if dry_run: - yield ( - ( - f" Would insert {len(data):,} rows of {len(data.columns):,} columns from '{datafile!s}' " - f"into {table!r}." - ), - len(data), - ) - continue - - self.client.raw.rows.insert_dataframe( - db_name=table.db_name, table_name=table.name, dataframe=data, ensure_parent=False - ) - yield ( - (f" Inserted {len(data):,} rows of {len(data.columns):,} columns from '{datafile!s}' into {table!r}."), - len(data), - ) diff --git a/tests/test_unit/test_cdf_tk/test_cruds/test_raw_file.py b/tests/test_unit/test_cdf_tk/test_cruds/test_raw_file.py deleted file mode 100644 index d970530497..0000000000 --- a/tests/test_unit/test_cdf_tk/test_cruds/test_raw_file.py +++ /dev/null @@ -1,107 +0,0 @@ -from pathlib import Path -from typing import Any -from unittest.mock import MagicMock - -import pytest - -from cognite_toolkit._cdf_tk.client.resource_classes.raw import RAWTableRequest -from cognite_toolkit._cdf_tk.client.testing import monkeypatch_toolkit_client -from cognite_toolkit._cdf_tk.data_classes import BuildEnvironment, BuiltResource, BuiltResourceList, SourceLocationEager -from cognite_toolkit._cdf_tk.resource_ios import RawFileCRUD, RawTableCRUD - - -class TestRawFileLoader: - @pytest.mark.parametrize( - "csv_content, expected_write", - [ - ( - """myFloat,myInt,myString,myBool -0.1,1,hello,True -0.2,2,world,False -""", - { - 0: {"myFloat": 0.1, "myInt": 1, "myString": "hello", "myBool": True}, - 1: {"myFloat": 0.2, "myInt": 2, "myString": "world", "myBool": False}, - }, - ) - ], - ) - def test_upload_dtypes(self, csv_content: str, expected_write: dict[int, Any]) -> None: - with monkeypatch_toolkit_client() as client: - loader = RawFileCRUD.create_loader(client) - csv_file = MagicMock(spec=Path) - csv_file.read_bytes.return_value = csv_content.encode("utf-8") - csv_file.exists.return_value = True - csv_file.suffix = ".csv" - source_file = MagicMock(spec=Path) - source_file.with_suffix.return_value = csv_file - - state = BuildEnvironment() - state.built_resources[RawFileCRUD.folder_name] = BuiltResourceList( - [ - BuiltResource( - RAWTableRequest(db_name="myDB", name="myTable"), - SourceLocationEager(source_file, "1z234"), - RawTableCRUD.kind, - None, - None, - ) - ] - ) - - list(loader.upload(state, dry_run=False)) - - # Verify one upload call was made - assert client.raw.rows.insert_dataframe.call_count == 1 - _, kwargs = client.raw.rows.insert_dataframe.call_args - written_to_cdf = kwargs["dataframe"].to_dict(orient="index") - # All values and types should match the expected ingestion payload - assert written_to_cdf == expected_write - - def test_upload_preserves_numeric_types_and_sets_empty_strings_for_nulls(self) -> None: - with monkeypatch_toolkit_client() as client: - loader = RawFileCRUD.create_loader(client) - csv_file = MagicMock(spec=Path) - csv_content = """myFloat,myInt,myString,myBool -,1,hello,True -0.2,,world,False -""" - csv_file.read_bytes.return_value = csv_content.encode("utf-8") - csv_file.exists.return_value = True - csv_file.suffix = ".csv" - source_file = MagicMock(spec=Path) - source_file.with_suffix.return_value = csv_file - - state = BuildEnvironment() - state.built_resources[RawFileCRUD.folder_name] = BuiltResourceList( - [ - BuiltResource( - RAWTableRequest(db_name="myDB", name="myTable"), - SourceLocationEager(source_file, "1z234"), - RawTableCRUD.kind, - None, - None, - ) - ] - ) - - list(loader.upload(state, dry_run=False)) - - # Capture the DataFrame uploaded to RAW - - # Verify one upload call was made - assert client.raw.rows.insert_dataframe.call_count == 1 - _, kwargs = client.raw.rows.insert_dataframe.call_args - df = kwargs["dataframe"] - # Verify dtypes are object after astype(object).fillna("") - assert str(df.dtypes["myFloat"]) == "object" - assert str(df.dtypes["myInt"]) == "object" - - # Non-null float value remains numeric - assert isinstance(df.iloc[1]["myFloat"], float) - # CSV with nulls coerces an integer-like column to floats; value becomes 1.0 - assert isinstance(df.iloc[0]["myInt"], float) - # Null in float column becomes empty string - assert df.iloc[0]["myFloat"] == "" and isinstance(df.iloc[0]["myFloat"], str) - # Null in int column becomes empty string - assert df.iloc[1]["myInt"] == "" and isinstance(df.iloc[1]["myInt"], str) From e8eb3d496eaea5290320dbe4a7ac092cf7f3b3d7 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 7 Sep 2026 20:20:58 +0200 Subject: [PATCH 15/31] refactor: delete last pandas dependencies --- .../_cdf_tk/resource_ios/__init__.py | 1 - cognite_toolkit/_cdf_tk/utils/file.py | 41 +------------------ 2 files changed, 1 insertion(+), 41 deletions(-) diff --git a/cognite_toolkit/_cdf_tk/resource_ios/__init__.py b/cognite_toolkit/_cdf_tk/resource_ios/__init__.py index 5b5caa5f4c..4a46a54367 100644 --- a/cognite_toolkit/_cdf_tk/resource_ios/__init__.py +++ b/cognite_toolkit/_cdf_tk/resource_ios/__init__.py @@ -205,7 +205,6 @@ def get_crud(resource_dir: str, kind: str) -> type[Loader]: "RESOURCE_CRUD_BY_FOLDER_NAME", "RESOURCE_CRUD_CONTAINER_LIST", "RESOURCE_CRUD_LIST", - "RESOURCE_DATA_CRUD_LIST", "_EXCLUDED_CRUDS", "AgentIO", "AppIO", diff --git a/cognite_toolkit/_cdf_tk/utils/file.py b/cognite_toolkit/_cdf_tk/utils/file.py index b32bcd2bce..5733e31c89 100644 --- a/cognite_toolkit/_cdf_tk/utils/file.py +++ b/cognite_toolkit/_cdf_tk/utils/file.py @@ -9,7 +9,7 @@ import warnings from abc import abstractmethod from collections import UserDict, defaultdict -from collections.abc import Hashable, ItemsView, KeysView, ValuesView +from collections.abc import ItemsView, KeysView, ValuesView from contextlib import contextmanager from dataclasses import dataclass, field from io import BytesIO @@ -17,14 +17,12 @@ from typing import Any, Literal, TypeVar, overload from zipfile import ZipFile -import pandas as pd import yaml from rich import print from cognite_toolkit._cdf_tk.cdf_toml import CDFToml from cognite_toolkit._cdf_tk.constants import ENV_VAR_PATTERN, HINT_LEAD_TEXT, MODULES, URL from cognite_toolkit._cdf_tk.exceptions import ( - ToolkitValueError, ToolkitYAMLFormatError, ) from cognite_toolkit._cdf_tk.tk_warnings import EnvironmentVariableMissingWarning, MediumSeverityWarning @@ -426,26 +424,6 @@ def remove_trailing_newline(content: str) -> str: return content -def read_csv( - path: Path | typing.TextIO, - parse_dates: bool | None = None, - index_col: Hashable | None = None, - dtype: Any | None = None, -) -> pd.DataFrame: - """Reads CSV - - Args: - path (Path): Path to the CSV file. - parse_dates (bool, optional): Whether to parse dates. Defaults to None. - index_col (Hashable, optional): Index column. Defaults to None. - dtype (Any, optional): Data types. Defaults to None - - Returns: - pd.DataFrame: DataFrame with the CSV data. - """ - return pd.read_csv(path, parse_dates=parse_dates, index_col=index_col, dtype=dtype) - - def _handle_remove_readonly(func: Any, path: Any, exc: Any) -> None: excvalue = exc[1] if func in (os.rmdir, os.remove) and excvalue.errno == errno.EACCES: @@ -472,23 +450,6 @@ def safe_rmtree(path: Path) -> None: ).print_warning() -def get_table_columns(table: Path) -> list[str]: - """Get the columns of a table - - Args: - table (Path): Path to the table - - Returns: - list[str]: List of columns - """ - if table.suffix == ".csv": - return read_csv(table).columns.tolist() - elif table.suffix == ".parquet": - return pd.read_parquet(table).columns.tolist() - else: - raise ToolkitValueError(f"The file {table.name} is not a supported table format (csv, parquet)") - - @contextmanager def create_temporary_zip(directory: Path, zipname: str) -> typing.Generator[Path, None, None]: """ From f555a200500b665c401845d3f079e04306a9a858 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 7 Sep 2026 20:22:35 +0200 Subject: [PATCH 16/31] refactor: drop pandas as a core dependency --- pyproject.toml | 1 - uv.lock | 2 -- 2 files changed, 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index aa8e7f9e1d..738755b72e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,6 @@ dependencies = [ "python-dotenv >=1.0.0", "cognite-sdk>=8.6.0,<9.0.0", "httpx>=0.28.1", - "pandas >=1.5.3, <3.0.0", "pyyaml >=6.0.1", "typer >=0.12.0, <1.0.0", "rich >=13.9.4", diff --git a/uv.lock b/uv.lock index 437b123cc6..7d47095622 100644 --- a/uv.lock +++ b/uv.lock @@ -379,7 +379,6 @@ dependencies = [ { name = "httpx" }, { name = "mixpanel" }, { name = "packaging" }, - { name = "pandas" }, { name = "pip" }, { name = "pydantic" }, { name = "python-dateutil" }, @@ -440,7 +439,6 @@ requires-dist = [ { name = "mixpanel", specifier = ">=4.10.1" }, { name = "openpyxl", marker = "extra == 'table'", specifier = ">=3.1.5" }, { name = "packaging", specifier = ">=25" }, - { name = "pandas", specifier = ">=1.5.3,<3.0.0" }, { name = "pip", specifier = ">=25.0.1" }, { name = "pyarrow", marker = "extra == 'table'", specifier = ">=20.0.0" }, { name = "pydantic", specifier = ">=2.12.0" }, From 90520862f80f4fb5907910b5e18b63d782d5de20 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 7 Sep 2026 20:31:34 +0200 Subject: [PATCH 17/31] tests: update sdk imports --- tests/test_unit/test_toolkit_package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_unit/test_toolkit_package.py b/tests/test_unit/test_toolkit_package.py index 5cfb51eb07..1414fe099b 100644 --- a/tests/test_unit/test_toolkit_package.py +++ b/tests/test_unit/test_toolkit_package.py @@ -94,7 +94,7 @@ def test_no_cognite_sdk_imports() -> None: The goal is to fully remove the cognite-sdk dependency from the toolkit (with the exception of Auth and protobuf files). This test tracks progress toward that goal. """ - _assert_import_violations(_extract_cognite_sdk_imports, "cognite.client imports", 93) + _assert_import_violations(_extract_cognite_sdk_imports, "cognite.client imports", 92) def _parse_package_name(dependency: str) -> str: From c2845da011d93dcd2ab9325c23e3b833138bb8e1 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 7 Sep 2026 20:26:34 +0200 Subject: [PATCH 18/31] refactor: delete unused issues --- .../_cdf_tk/data_classes/__init__.py | 3 -- .../_cdf_tk/data_classes/_issues.py | 36 ------------------- 2 files changed, 39 deletions(-) delete mode 100644 cognite_toolkit/_cdf_tk/data_classes/_issues.py diff --git a/cognite_toolkit/_cdf_tk/data_classes/__init__.py b/cognite_toolkit/_cdf_tk/data_classes/__init__.py index cab85eb80e..65dfef8777 100644 --- a/cognite_toolkit/_cdf_tk/data_classes/__init__.py +++ b/cognite_toolkit/_cdf_tk/data_classes/__init__.py @@ -29,7 +29,6 @@ ResourceDeployResult, UploadDeployResult, ) -from ._issues import Issue, IssueList from ._module_directories import ModuleDirectories, ModuleLocation from ._packages import Package, Packages from ._tracking_info import CommandTracking, DeploymentTracking, TrackingEvent @@ -57,8 +56,6 @@ "DeploymentTracking", "Environment", "InitConfigYAML", - "Issue", - "IssueList", "ModuleDirectories", "ModuleLocation", "Package", diff --git a/cognite_toolkit/_cdf_tk/data_classes/_issues.py b/cognite_toolkit/_cdf_tk/data_classes/_issues.py deleted file mode 100644 index cf7d9eb736..0000000000 --- a/cognite_toolkit/_cdf_tk/data_classes/_issues.py +++ /dev/null @@ -1,36 +0,0 @@ -import sys -from collections import UserList - -if sys.version_info >= (3, 11): - from typing import Self -else: - from typing_extensions import Self - -from pydantic import BaseModel, Field - -from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning, WarningList - -MODULE_ISSUE_CODE = "MOD" - - -class Issue(BaseModel): - """Base class for all issues""" - - code: str - message: str | None = Field(default=None) - - -# temporary adapter to manage existing warnings -class IssueList(UserList[Issue]): - """List of build issues.""" - - @classmethod - def from_warning_list(cls, warning_list: WarningList[ToolkitWarning]) -> Self: - """Create a IssueList from a WarningList.""" - return cls([Issue(code="WARN", message=warning.get_message()) for warning in warning_list]) - - -class ModuleLoadingIssue(Issue): - """Issue with the loading of a module folder.""" - - code: str = "MOD_001" From fb1cfef641d32797e8b4d48fac793a1b2c980d0d Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 7 Sep 2026 20:27:16 +0200 Subject: [PATCH 19/31] refactor: delete more data classes --- .../_cdf_tk/data_classes/__init__.py | 3 -- .../_cdf_tk/data_classes/_build_files.py | 37 ------------------- 2 files changed, 40 deletions(-) delete mode 100644 cognite_toolkit/_cdf_tk/data_classes/_build_files.py diff --git a/cognite_toolkit/_cdf_tk/data_classes/__init__.py b/cognite_toolkit/_cdf_tk/data_classes/__init__.py index 65dfef8777..1dabdca587 100644 --- a/cognite_toolkit/_cdf_tk/data_classes/__init__.py +++ b/cognite_toolkit/_cdf_tk/data_classes/__init__.py @@ -1,4 +1,3 @@ -from ._build_files import BuildDestinationFile, BuildSourceFile from ._build_variables import BuildVariable, BuildVariables from ._built_modules import ( BuiltModule, @@ -36,9 +35,7 @@ __all__ = [ "BuildConfigYAML", - "BuildDestinationFile", "BuildEnvironment", - "BuildSourceFile", "BuildVariable", "BuildVariables", "BuiltFullResourceList", diff --git a/cognite_toolkit/_cdf_tk/data_classes/_build_files.py b/cognite_toolkit/_cdf_tk/data_classes/_build_files.py deleted file mode 100644 index aece141679..0000000000 --- a/cognite_toolkit/_cdf_tk/data_classes/_build_files.py +++ /dev/null @@ -1,37 +0,0 @@ -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -from cognite_toolkit._cdf_tk.resource_ios import ( - ResourceIO, -) -from cognite_toolkit._cdf_tk.tk_warnings import ( - WarningList, -) -from cognite_toolkit._cdf_tk.tk_warnings.fileread import ( - FileReadWarning, -) -from cognite_toolkit._cdf_tk.utils.file import yaml_safe_dump - -from ._built_resources import SourceLocation - - -@dataclass -class BuildSourceFile: - source: SourceLocation - content: str - loaded: list[dict[str, Any]] | dict[str, Any] | None = None - - -@dataclass -class BuildDestinationFile: - path: Path - loaded: list[dict[str, Any]] | dict[str, Any] - loader: type[ResourceIO] - source: SourceLocation - extra_sources: list[SourceLocation] | None - warnings: WarningList[FileReadWarning] = field(default_factory=WarningList[FileReadWarning]) - - @property - def content(self) -> str: - return yaml_safe_dump(self.loaded) From 3dee6a96038dbbda81120aa59822cd657b2ef960 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 7 Sep 2026 20:30:41 +0200 Subject: [PATCH 20/31] refactor: more to delete --- .../_cdf_tk/data_classes/__init__.py | 2 - .../_cdf_tk/data_classes/_config_yaml.py | 96 +------------------ 2 files changed, 2 insertions(+), 96 deletions(-) diff --git a/cognite_toolkit/_cdf_tk/data_classes/__init__.py b/cognite_toolkit/_cdf_tk/data_classes/__init__.py index 1dabdca587..dd655fc466 100644 --- a/cognite_toolkit/_cdf_tk/data_classes/__init__.py +++ b/cognite_toolkit/_cdf_tk/data_classes/__init__.py @@ -14,7 +14,6 @@ ) from ._config_yaml import ( BuildConfigYAML, - BuildEnvironment, ConfigEntry, ConfigYAMLs, Environment, @@ -35,7 +34,6 @@ __all__ = [ "BuildConfigYAML", - "BuildEnvironment", "BuildVariable", "BuildVariables", "BuiltFullResourceList", diff --git a/cognite_toolkit/_cdf_tk/data_classes/_config_yaml.py b/cognite_toolkit/_cdf_tk/data_classes/_config_yaml.py index 71076cf185..0df0897871 100644 --- a/cognite_toolkit/_cdf_tk/data_classes/_config_yaml.py +++ b/cognite_toolkit/_cdf_tk/data_classes/_config_yaml.py @@ -13,7 +13,6 @@ from rich import print from cognite_toolkit._cdf_tk.constants import ( - _RUNNING_IN_BROWSER, BUILD_ENVIRONMENT_FILE, DEFAULT_CONFIG_FILE, DEFAULT_ENV, @@ -22,15 +21,13 @@ SEARCH_VARIABLES_SUFFIX, EnvType, ) -from cognite_toolkit._cdf_tk.exceptions import ToolkitEnvError, ToolkitMissingModuleError -from cognite_toolkit._cdf_tk.hints import ModuleDefinition +from cognite_toolkit._cdf_tk.exceptions import ToolkitEnvError from cognite_toolkit._cdf_tk.resource_ios import CRUDS_BY_FOLDER_NAME, RawDatabaseCRUD from cognite_toolkit._cdf_tk.tk_warnings import ( FileReadWarning, MediumSeverityWarning, MissingFileWarning, SourceFileModifiedWarning, - ToolkitWarning, WarningList, ) from cognite_toolkit._cdf_tk.utils import ( @@ -44,10 +41,9 @@ from cognite_toolkit._cdf_tk.utils.modules import parse_user_selected_modules from cognite_toolkit._version import __version__ -from . import BuiltModuleList from ._base import ConfigCore, _load_version_variable from ._built_resources import BuiltResourceList -from ._module_directories import ModuleDirectories, ReadModule +from ._module_directories import ReadModule if sys.version_info >= (3, 11): from typing import Self @@ -116,38 +112,6 @@ class BuildConfigYAML(ConfigYAMLCore, ConfigCore): filename: ClassVar[str] = "config.{build_env}.yaml" variables: dict[str, Any] = field(default_factory=dict) - def validate_environment(self) -> ToolkitWarning | None: - if _RUNNING_IN_BROWSER: - return None - project = self.environment.project - project_env = os.environ.get("CDF_PROJECT") - if project_env == project: - return None - - is_strict_validation = self.environment.is_strict_validation - env_name = self.environment.name - file_name = self.get_filename(env_name) - missing_message = ( - "No 'CDF_PROJECT' environment variable set. This is expected to match the project " - f"set in environment section of {file_name!r}.\nThis is required for " - "building configurations for staging and prod environments to ensure that you do " - "not accidentally deploy to the wrong project." - ) - mismatch_message = ( - f"Project name mismatch between project set in the environment section of {file_name!r} and the " - f"environment variable 'CDF_PROJECT', {project} ≠ {project_env}.\nThis is required for " - "building configurations for staging and prod environments to ensure that you do not " - "accidentally deploy to the wrong project." - ) - if is_strict_validation and project_env is None: - raise ToolkitEnvError(missing_message) - elif is_strict_validation: - raise ToolkitEnvError(mismatch_message) - elif not is_strict_validation and project_env is None: - return MediumSeverityWarning(missing_message) - else: - return MediumSeverityWarning(mismatch_message) - @classmethod def load(cls, data: dict[str, Any], build_env_name: str, filepath: Path) -> Self: if "environment" not in data: @@ -164,62 +128,6 @@ def load(cls, data: dict[str, Any], build_env_name: str, filepath: Path) -> Self variables = data.get("variables", {}) return cls(environment=environment, variables=variables, filepath=filepath) - def create_build_environment( - self, built_modules: BuiltModuleList, selected_modules: ModuleDirectories - ) -> "BuildEnvironment": - return BuildEnvironment( - name=self.environment.name, - project=self.environment.project, - validation_type=self.environment.validation_type, - selected=self.environment.selected, - cdf_toolkit_version=__version__, - built_resources=built_modules.as_resources_by_folder(), - read_modules=[module.as_read_module() for module in selected_modules], - ) - - def get_selected_modules( - self, - modules_by_package: dict[str, list[str | Path]], - available_modules: set[str | Path], - organization_dir: Path, - verbose: bool, - ) -> list[str | Path]: - selected_packages = [ - package - for package in self.environment.selected - if package in modules_by_package and isinstance(package, str) - ] - if verbose: - print(" [bold green]INFO:[/] Selected packages:") - if len(selected_packages) == 0: - print(" None") - for package in selected_packages: - print(f" {package}") - - selected_modules = [module for module in self.environment.selected if module not in modules_by_package] - if missing := set(selected_modules) - available_modules: - hint = ModuleDefinition.long(missing, organization_dir) - raise ToolkitMissingModuleError( - f"The following selected modules are missing, please check path: {missing}.\n{hint}" - ) - - selected_modules.extend( - itertools.chain.from_iterable(modules_by_package[package] for package in selected_packages) - ) - if not selected_modules: - raise ToolkitEnvError( - f"No selected modules specified in {self.filepath!s}, have you configured " - f"the environment ({self.environment.name})?" - ) - if verbose: - print(" [bold green]INFO:[/] Selected modules:") - for module in selected_modules: - if isinstance(module, Path): - print(f" {module.as_posix()}") - else: - print(f" {module}") - return selected_modules - @classmethod def load_default(cls, organization_dir: Path) -> Self: return cls(filepath=organization_dir / BuildConfigYAML.get_filename(DEFAULT_ENV)) From 2440d9f1ad8e181bf2e8bf972b799d6c8155724b Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 7 Sep 2026 20:36:12 +0200 Subject: [PATCH 21/31] refactor: delete more --- .../_cdf_tk/data_classes/__init__.py | 22 -- .../_cdf_tk/data_classes/_built_modules.py | 127 --------- .../_cdf_tk/data_classes/_built_resources.py | 244 ----------------- .../_cdf_tk/data_classes/_config_yaml.py | 111 +------- .../test_data_classes/test_built_modules.py | 249 ------------------ 5 files changed, 3 insertions(+), 750 deletions(-) delete mode 100644 cognite_toolkit/_cdf_tk/data_classes/_built_modules.py delete mode 100644 cognite_toolkit/_cdf_tk/data_classes/_built_resources.py delete mode 100644 tests/test_unit/test_cdf_tk/test_data_classes/test_built_modules.py diff --git a/cognite_toolkit/_cdf_tk/data_classes/__init__.py b/cognite_toolkit/_cdf_tk/data_classes/__init__.py index dd655fc466..e49e5177ce 100644 --- a/cognite_toolkit/_cdf_tk/data_classes/__init__.py +++ b/cognite_toolkit/_cdf_tk/data_classes/__init__.py @@ -1,17 +1,4 @@ from ._build_variables import BuildVariable, BuildVariables -from ._built_modules import ( - BuiltModule, - BuiltModuleList, -) -from ._built_resources import ( - BuiltFullResourceList, - BuiltResource, - BuiltResourceFull, - BuiltResourceList, - SourceLocation, - SourceLocationEager, - SourceLocationLazy, -) from ._config_yaml import ( BuildConfigYAML, ConfigEntry, @@ -36,12 +23,6 @@ "BuildConfigYAML", "BuildVariable", "BuildVariables", - "BuiltFullResourceList", - "BuiltModule", - "BuiltModuleList", - "BuiltResource", - "BuiltResourceFull", - "BuiltResourceList", "CommandTracking", "ConfigEntry", "ConfigYAMLs", @@ -57,9 +38,6 @@ "Packages", "ResourceContainerDeployResult", "ResourceDeployResult", - "SourceLocation", - "SourceLocationEager", - "SourceLocationLazy", "TrackingEvent", "UploadDeployResult", "YAMLComments", diff --git a/cognite_toolkit/_cdf_tk/data_classes/_built_modules.py b/cognite_toolkit/_cdf_tk/data_classes/_built_modules.py deleted file mode 100644 index 1beba15606..0000000000 --- a/cognite_toolkit/_cdf_tk/data_classes/_built_modules.py +++ /dev/null @@ -1,127 +0,0 @@ -import sys -from collections.abc import Callable, Collection, Iterator, MutableSequence -from dataclasses import dataclass -from pathlib import Path -from typing import Any, SupportsIndex, overload - -from cognite_toolkit._cdf_tk.resource_ios import ResourceTypes -from cognite_toolkit._cdf_tk.utils.useful_types import T_ID - -from ._build_variables import BuildVariables -from ._built_resources import ( - BuiltFullResourceList, - BuiltResource, - BuiltResourceList, - SourceLocation, -) - -if sys.version_info >= (3, 11): - from typing import Self -else: - from typing_extensions import Self - - -@dataclass -class BuiltModule: - name: str - location: SourceLocation - build_variables: BuildVariables - resources: dict[str, BuiltResourceList] - warning_count: int - status: str - iteration: int - package_id: str | None = None - module_id: str | None = None - - @classmethod - def load(cls, data: dict[str, Any]) -> Self: - return cls( - name=data["name"], - location=SourceLocation.load(data["location"]), - build_variables=BuildVariables.load(data["build_variables"]), - resources={ - key: BuiltResourceList([BuiltResource.load(resource_data, key) for resource_data in resources_data]) - for key, resources_data in data["resources"].items() - }, - warning_count=data.get("warning_count", 0), - status=data.get("status", "Success"), - iteration=data.get("iteration", 1), - package_id=data.get("package_id"), - module_id=data.get("module_id"), - ) - - def dump(self) -> dict[str, Any]: - return { - "name": self.name, - "location": self.location.dump(), - "build_variables": self.build_variables.dump(), - "resources": { - key: [resource.dump(key) for resource in resources] for key, resources in self.resources.items() - }, - "warning_count": self.warning_count, - "status": self.status, - "iteration": self.iteration, - "package_id": self.package_id, - "module_id": self.module_id, - } - - -@dataclass -class BuiltModuleList(list, MutableSequence[BuiltModule]): - # Implemented to get correct type hints - def __init__(self, collection: Collection[BuiltModule] | None = None) -> None: - super().__init__(collection or []) - - def __iter__(self) -> Iterator[BuiltModule]: - return super().__iter__() - - @overload - def __getitem__(self, index: SupportsIndex) -> BuiltModule: ... - - @overload - def __getitem__(self, index: slice) -> Self: ... - - def __getitem__(self, index: SupportsIndex | slice, /) -> "BuiltModule | BuiltModuleList": - if isinstance(index, slice): - return BuiltModuleList(super().__getitem__(index)) - return super().__getitem__(index) - - def get_resources( - self, - id_type: type[T_ID] | None, - resource_dir: ResourceTypes, - kind: str | None = None, - selected: Path | str | None = None, - is_supported_file: Callable[[Path], bool] | None = None, - ) -> "BuiltFullResourceList[T_ID]": - resources = ( - resource.create_full(module, resource_dir) - for module in self - for resource in module.resources.get(resource_dir, []) - if kind is None or resource.kind == kind - ) - if isinstance(selected, str): - resources = (resource for resource in resources if resource.module_name == selected) - elif isinstance(selected, Path): - resources = ( - resource - for resource in resources - if (resource.source.path == selected or self._are_relative(resource.source.path, selected)) - ) - if is_supported_file: - # This is necessary as the destination file can be created from a source file that is not supported. - # This happens for RAW table files which produces a Database file. - resources = (resource for resource in resources if is_supported_file(resource.source.path)) - - return BuiltFullResourceList[T_ID](list(resources)) - - def as_resources_by_folder(self) -> dict[str, BuiltResourceList[T_ID]]: - resources_by_folder: dict[str, BuiltResourceList[T_ID]] = {} - for module in self: - for resource_dir, resources in module.resources.items(): - resources_by_folder.setdefault(resource_dir, BuiltResourceList()).extend(resources) - return resources_by_folder - - @staticmethod - def _are_relative(filepath: Path, select_path: Path) -> bool: - return filepath.resolve().is_relative_to(select_path.resolve()) diff --git a/cognite_toolkit/_cdf_tk/data_classes/_built_resources.py b/cognite_toolkit/_cdf_tk/data_classes/_built_resources.py deleted file mode 100644 index ae93bf16cb..0000000000 --- a/cognite_toolkit/_cdf_tk/data_classes/_built_resources.py +++ /dev/null @@ -1,244 +0,0 @@ -import sys -from abc import abstractmethod -from collections import defaultdict -from collections.abc import Collection, Iterator, MutableSequence -from dataclasses import dataclass -from functools import cached_property -from pathlib import Path -from typing import TYPE_CHECKING, Any, Generic, SupportsIndex, TypeVar, cast, overload - -from cognite_toolkit._cdf_tk.exceptions import ToolkitMissingResourceError -from cognite_toolkit._cdf_tk.resource_ios import get_crud -from cognite_toolkit._cdf_tk.resource_ios._base_ios import ResourceIO -from cognite_toolkit._cdf_tk.utils import ( - calculate_directory_hash, - calculate_hash, - load_yaml_inject_variables, - safe_read, -) -from cognite_toolkit._cdf_tk.utils.useful_types import T_ID - -from ._build_variables import BuildVariables - -if sys.version_info >= (3, 11): - from typing import Self -else: - from typing_extensions import Self -if TYPE_CHECKING: - from ._built_modules import BuiltModule - - -@dataclass -class SourceLocation: - """This represents the location of a built resource in a module structure. - - Args: - path: The relative path to the resource from the project directory. - """ - - path: Path - - @property - @abstractmethod - def hash(self) -> str: - """The hash of the resource file.""" - raise NotImplementedError() - - def dump(self) -> dict[str, Any]: - return { - "path": self.path.as_posix(), - "hash": str(self.hash), - } - - @classmethod - def load(cls, data: dict[str, Any]) -> "SourceLocationEager": - return SourceLocationEager( - path=Path(data["path"]), - _hash=str(data["hash"]), - ) - - -@dataclass -class SourceLocationLazy(SourceLocation): - absolute_path: Path - - @cached_property - def hash(self) -> str: - if self.absolute_path.is_dir(): - return calculate_directory_hash(self.absolute_path, shorten=True) - else: - return calculate_hash(self.absolute_path, shorten=True) - - -@dataclass -class SourceLocationEager(SourceLocation): - _hash: str - - @property - def hash(self) -> str: - return self._hash - - -@dataclass -class BuiltResource(Generic[T_ID]): - """This represents a built resource. - - Args: - identifier: The unique identifier of the resource. - source: The source location of the resource. - kind: The kind of resource. - destination: The destination of the resource. - extra_sources: Extra source locations of the resource, for example, Transformations might have - .sql files that are used to build the final resource. - - """ - - identifier: T_ID - source: SourceLocation - kind: str - destination: Path | None - extra_sources: list[SourceLocation] | None - - @classmethod - def load(cls, data: dict[str, Any], resource_folder: str) -> Self: - from cognite_toolkit._cdf_tk.resource_ios import ResourceIO, get_crud - - kind = data["kind"] - loader = cast(ResourceIO, get_crud(resource_folder, kind)) - identifier = loader.get_id(data["identifier"]) - - return cls( - source=SourceLocation.load(data["source"]), - kind=kind, - identifier=identifier, - destination=Path(data["destination"]) if "destination" in data else None, - extra_sources=[SourceLocation.load(source) for source in data.get("extra_sources", [])] or None, - ) - - def dump(self, resource_folder: str, include_destination: bool = False) -> dict[str, Any]: - from cognite_toolkit._cdf_tk.resource_ios import ResourceIO, get_crud - - loader = cast(ResourceIO, get_crud(resource_folder, self.kind)) - dumped = loader.dump_id(self.identifier) - - output: dict[str, Any] = { - "identifier": dumped, - "source": self.source.dump(), - "kind": self.kind, - } - if include_destination and self.destination: - output["destination"] = self.destination.as_posix() - if self.extra_sources: - output["extra_sources"] = [source.dump() for source in self.extra_sources] - return output - - def create_full(self, module: "BuiltModule", resource_dir: str) -> "BuiltResourceFull": - return BuiltResourceFull( - identifier=self.identifier, - source=self.source, - kind=self.kind, - destination=self.destination, - build_variables=module.build_variables, - module_name=module.name, - module_location=module.location.path, - resource_dir=resource_dir, - extra_sources=self.extra_sources, - ) - - -T_BuiltResource = TypeVar("T_BuiltResource", bound=BuiltResource) - - -@dataclass -class BuiltResourceFull(BuiltResource[T_ID]): - build_variables: BuildVariables - module_name: str - module_location: Path - resource_dir: str - - def load_resource_dict( - self, environment_variables: dict[str, str | None], validate: bool = False - ) -> dict[str, Any]: - content = self.build_variables.replace(safe_read(self.source.path), self.source.path) - loader = cast(ResourceIO, get_crud(self.resource_dir, self.kind)) - raw = load_yaml_inject_variables( - content, - environment_variables, - validate=validate, - original_filepath=self.source.path, - ) - if isinstance(raw, dict): - return raw - elif isinstance(raw, list): - for item in raw: - if loader.get_id(item) == self.identifier: - return item - raise ToolkitMissingResourceError(f"Resource {self.identifier} not found in {self.source.path}") - - -class BuiltResourceList(list, MutableSequence[BuiltResource[T_ID]], Generic[T_ID]): - # Implemented to get correct type hints - def __init__(self, collection: Collection[BuiltResource[T_ID]] | None = None) -> None: - super().__init__(collection or []) - - def __iter__(self) -> Iterator[BuiltResource[T_ID]]: - return super().__iter__() - - @overload - def __getitem__(self, index: SupportsIndex) -> BuiltResource[T_ID]: ... - - @overload - def __getitem__(self, index: slice) -> "BuiltResourceList[T_ID]": ... - - def __getitem__(self, index: SupportsIndex | slice, /) -> "BuiltResource[T_ID] | BuiltResourceList[T_ID]": - if isinstance(index, slice): - return BuiltResourceList[T_ID](super().__getitem__(index)) - return super().__getitem__(index) - - @property - def identifiers(self) -> list[T_ID]: - return [resource.identifier for resource in self] - - @classmethod - def load(cls, data: list[dict[str, Any]], resource_folder: str) -> "BuiltResourceList[T_ID]": - return cls([BuiltResource.load(resource_data, resource_folder) for resource_data in data]) - - def dump(self, resource_folder: str, include_destination: bool = False) -> list[dict[str, Any]]: - return [resource.dump(resource_folder, include_destination) for resource in self] - - def get_resource_directories(self, resource_folder: str) -> set[Path]: - output: set[Path] = set() - for resource in self: - index = next((i for i, part in enumerate(resource.source.path.parts) if part == resource_folder), None) - if index is None: - continue - path = Path("/".join(resource.source.path.parts[: index + 1])) - output.add(path) - - return output - - -class BuiltFullResourceList(BuiltResourceList[T_ID]): - # Implemented to get correct type hints - def __init__(self, collection: Collection[BuiltResourceFull[T_ID]] | None = None) -> None: - super().__init__(collection or []) - - def __iter__(self) -> Iterator[BuiltResourceFull[T_ID]]: - return cast(Iterator[BuiltResourceFull[T_ID]], super().__iter__()) - - @overload - def __getitem__(self, index: SupportsIndex) -> BuiltResourceFull[T_ID]: ... - - @overload - def __getitem__(self, index: slice) -> "BuiltFullResourceList[T_ID]": ... - - def __getitem__(self, index: SupportsIndex | slice, /) -> "BuiltResourceFull[T_ID] | BuiltFullResourceList[T_ID]": - if isinstance(index, slice): - return BuiltFullResourceList[T_ID](super().__getitem__(index)) - return cast(BuiltResourceFull[T_ID], super().__getitem__(index)) - - def by_file(self) -> "dict[Path, BuiltFullResourceList[T_ID]]": - resources_by_file: dict[Path, BuiltFullResourceList[T_ID]] = defaultdict(lambda: BuiltFullResourceList()) - for resource in self: - resources_by_file[resource.source.path].append(resource) - return resources_by_file diff --git a/cognite_toolkit/_cdf_tk/data_classes/_config_yaml.py b/cognite_toolkit/_cdf_tk/data_classes/_config_yaml.py index 0df0897871..98568649f0 100644 --- a/cognite_toolkit/_cdf_tk/data_classes/_config_yaml.py +++ b/cognite_toolkit/_cdf_tk/data_classes/_config_yaml.py @@ -7,13 +7,11 @@ from collections.abc import Hashable, Iterable, Sequence, Set from dataclasses import dataclass, field from pathlib import Path -from typing import Any, ClassVar, Literal, cast, get_args +from typing import Any, ClassVar, cast, get_args -import yaml from rich import print from cognite_toolkit._cdf_tk.constants import ( - BUILD_ENVIRONMENT_FILE, DEFAULT_CONFIG_FILE, DEFAULT_ENV, MODULES, @@ -22,28 +20,20 @@ EnvType, ) from cognite_toolkit._cdf_tk.exceptions import ToolkitEnvError -from cognite_toolkit._cdf_tk.resource_ios import CRUDS_BY_FOLDER_NAME, RawDatabaseCRUD +from cognite_toolkit._cdf_tk.resource_ios import CRUDS_BY_FOLDER_NAME from cognite_toolkit._cdf_tk.tk_warnings import ( - FileReadWarning, MediumSeverityWarning, - MissingFileWarning, - SourceFileModifiedWarning, - WarningList, ) from cognite_toolkit._cdf_tk.utils import ( YAMLComment, YAMLWithComments, - calculate_hash, flatten_dict, read_yaml_content, safe_read, ) from cognite_toolkit._cdf_tk.utils.modules import parse_user_selected_modules -from cognite_toolkit._version import __version__ -from ._base import ConfigCore, _load_version_variable -from ._built_resources import BuiltResourceList -from ._module_directories import ReadModule +from ._base import ConfigCore if sys.version_info >= (3, 11): from typing import Self @@ -136,101 +126,6 @@ def dump(self) -> dict[str, Any]: return {"environment": self.environment.dump(), "variables": self.variables} -@dataclass -class BuildEnvironment(Environment): - cdf_toolkit_version: str = __version__ - built_resources: dict[str, BuiltResourceList] = field(default_factory=dict) - read_modules: list[ReadModule] = field(default_factory=list) - - @property - def read_resource_folders(self) -> set[str]: - return {resource_folder for module in self.read_modules for resource_folder in module.resource_directories} - - def dump(self) -> dict[str, Any]: - output = super().dump() - output["cdf_toolkit_version"] = self.cdf_toolkit_version - if self.built_resources: - output["built_resources"] = { - resource_folder: resources.dump(resource_folder, include_destination=True) - for resource_folder, resources in self.built_resources.items() - } - if self.read_modules: - output["read_modules"] = [module.dump() for module in self.read_modules] - return output - - def dump_to_file(self, build_dir: Path) -> None: - (build_dir / BUILD_ENVIRONMENT_FILE).write_text( - "# DO NOT EDIT THIS FILE!\n" + yaml.dump(self.dump(), sort_keys=False, indent=2) - ) - - @classmethod - def load( - cls, data: dict[str, Any], build_name: str | None, action: Literal["build", "deploy", "clean", "pull"] = "build" - ) -> Self: - if "name" in data and build_name is not None and data["name"] != build_name: - raise ToolkitEnvError( - f"Expected to {action} for {build_name!r} environment, but the last " - f"build was created for the {data['name']!r} environment." - ) - build_name = build_name or data.get("name") - - version = _load_version_variable(data, BUILD_ENVIRONMENT_FILE) - _deprecation_selected(data) - built_resources: dict[str, BuiltResourceList] = {} - if "built_resources" in data: - # We expect to dump BuildEnvironment, and load DeployEnvironment - built_resources = { - resource_folder: BuiltResourceList.load(resources, resource_folder) - for resource_folder, resources in data["built_resources"].items() - } - read_modules: list[ReadModule] = [] - if "read_modules" in data: - read_modules = [ReadModule.load(module_data) for module_data in data["read_modules"]] - _deprecate_type(data, build_name or "dev") - try: - return cls( - name=data["name"], - project=data["project"], - validation_type=data["validation-type"], - selected=data["selected"], - cdf_toolkit_version=version, - built_resources=built_resources, - read_modules=read_modules, - ) - except KeyError: - raise ToolkitEnvError( - f" [bold red]ERROR:[/] Environment {build_name} is missing required fields 'name', 'project', 'validation-type', " - f"or 'selected' in {BUILD_ENVIRONMENT_FILE!s}" - ) - - def set_environment_variables(self) -> None: - os.environ["CDF_ENVIRON"] = self.name - os.environ["CDF_BUILD_TYPE"] = self.validation_type - - def check_source_files_changed(self) -> WarningList[FileReadWarning]: - warning_list = WarningList[FileReadWarning]() - for resource_folder, resources in self.built_resources.items(): - if resource_folder == RawDatabaseCRUD.folder_name: - # We modify the hash for RawDatabaseLoader, so we skip checking the hash for this folder. - continue - for resource in resources: - to_check = [resource.source, *(resource.extra_sources or [])] - for source in to_check: - source_filepath = source.path - if source_filepath.suffix in {".csv", ".parquet"}: - # When we copy over the source files we use utf-8 encoding, which can change the file hash. - # Thus, we skip checking the hash for these file types. - continue - - if not source_filepath.exists(): - warning_list.append( - MissingFileWarning(source_filepath, attempted_check="source file has changed") - ) - elif source.hash != calculate_hash(source_filepath, shorten=True): - warning_list.append(SourceFileModifiedWarning(source_filepath)) - return warning_list - - def _deprecation_selected(data: dict[str, Any]) -> None: if "selected_modules_and_packages" in data and "selected" not in data: print( diff --git a/tests/test_unit/test_cdf_tk/test_data_classes/test_built_modules.py b/tests/test_unit/test_cdf_tk/test_data_classes/test_built_modules.py deleted file mode 100644 index 567296e41f..0000000000 --- a/tests/test_unit/test_cdf_tk/test_data_classes/test_built_modules.py +++ /dev/null @@ -1,249 +0,0 @@ -from dataclasses import dataclass -from itertools import groupby -from pathlib import Path - -import pytest - -from cognite_toolkit._cdf_tk.data_classes import ( - BuildVariables, - BuiltModule, - BuiltModuleList, - BuiltResource, - BuiltResourceList, - SourceLocationEager, -) -from cognite_toolkit._cdf_tk.resource_ios import ResourceTypes - - -@dataclass -class GetResourcesArgs: - resource_dir: ResourceTypes - kind: str | None - selected: str | Path | None - - -class TestBuiltModuleList: - # Anchor for absolute paths in tests - anchor = f"{Path.cwd()}/" - - @pytest.mark.parametrize( - "module,args,expected", - [ - pytest.param( - { - Path(f"{anchor}modules/module1"): [ - Path(f"{anchor}modules/module1/transformations/my.Transformation.yaml"), - Path(f"{anchor}modules/module1/transformations/my.Schedule.yaml"), - ] - }, - GetResourcesArgs( - resource_dir="transformations", - kind="Transformation", - selected="module1", - ), - [Path(f"{anchor}modules/module1/transformations/my.Transformation.yaml")], - id="Select by module name", - ), - pytest.param( - { - Path(f"{anchor}modules/module1"): [ - Path(f"{anchor}modules/module1/transformations/my.Transformation.yaml"), - Path(f"{anchor}modules/module1/transformations/my.Schedule.yaml"), - ] - }, - GetResourcesArgs( - resource_dir="transformations", - kind="Transformation", - selected=Path("modules/module1"), - ), - [Path(f"{anchor}modules/module1/transformations/my.Transformation.yaml")], - id="Select with relative with module in absolute", - ), - pytest.param( - { - Path("modules/module1"): [ - Path("modules/module1/transformations/my.Transformation.yaml"), - Path("modules/module1/transformations/my.Schedule.yaml"), - ] - }, - GetResourcesArgs( - resource_dir="transformations", - kind="Transformation", - selected=Path(f"{anchor}modules/module1"), - ), - [Path("modules/module1/transformations/my.Transformation.yaml")], - id="Select with absolute with module in relative", - ), - pytest.param( - { - Path("modules/module1"): [ - Path(f"{anchor}modules/module1/transformations/my.Transformation.yaml"), - Path(f"{anchor}modules/module1/transformations/my.Schedule.yaml"), - ] - }, - GetResourcesArgs( - resource_dir="transformations", - kind="Transformation", - selected=Path(f"{anchor}modules/module1/transformations/my.Transformation.yaml"), - ), - [Path(f"{anchor}modules/module1/transformations/my.Transformation.yaml")], - id="Select file by absolute path", - ), - pytest.param( - { - Path("module1"): [ - Path("modules/module1/transformations/my.Transformation.yaml"), - Path("modules/module1/transformations/my.Schedule.yaml"), - ] - }, - GetResourcesArgs( - resource_dir="transformations", - kind="Transformation", - selected=Path("modules/module1/transformations/my.Transformation.yaml"), - ), - [Path("modules/module1/transformations/my.Transformation.yaml")], - id="Select file by relative path", - ), - pytest.param( - { - Path("modules/module1"): [ - Path(f"{anchor}modules/module1/transformations/my.Transformation.yaml"), - Path(f"{anchor}modules/module1/transformations/my.Schedule.yaml"), - ] - }, - GetResourcesArgs( - resource_dir="transformations", - kind="Transformation", - selected=Path(f"{anchor}modules/module1/transformations"), - ), - [Path(f"{anchor}modules/module1/transformations/my.Transformation.yaml")], - id="Select resource dir by absolute path", - ), - pytest.param( - { - Path("module1"): [ - Path("modules/module1/transformations/my.Transformation.yaml"), - Path("modules/module1/transformations/my.Schedule.yaml"), - ] - }, - GetResourcesArgs( - resource_dir="transformations", - kind="Transformation", - selected=Path("modules/module1/transformations/my.Transformation.yaml"), - ), - [Path("modules/module1/transformations/my.Transformation.yaml")], - id="Select resource dir by relative path", - ), - pytest.param( - { - Path(f"{anchor}module1"): [ - Path(f"{anchor}modules/module1/transformations/my.Transformation.yaml"), - Path(f"{anchor}modules/module1/transformations/my.Schedule.yaml"), - ], - Path(f"{anchor}module2"): [ - Path(f"{anchor}module2/transformations/other.Transformation.yaml"), - ], - }, - GetResourcesArgs( - resource_dir="transformations", - kind="Transformation", - selected=None, - ), - [ - Path(f"{anchor}modules/module1/transformations/my.Transformation.yaml"), - Path(f"{anchor}module2/transformations/other.Transformation.yaml"), - ], - id="Select all by kind", - ), - pytest.param( - { - Path(f"{anchor}module1"): [ - Path(f"{anchor}modules/module1/transformations/my.Transformation.yaml"), - ] - }, - GetResourcesArgs( - resource_dir="transformations", - kind="NonExistentKind", - selected=None, - ), - [], - id="Select with non-existent kind", - ), - pytest.param( - { - Path("modules/module1"): [ - Path(f"{anchor}modules/module1/transformations/my.Transformation.yaml"), - ] - }, - GetResourcesArgs( - resource_dir="transformations", - kind="Transformation", - selected=Path("/modules/module1/transformations/doesnotexist.yaml"), - ), - [], - id="Select with non-existent path", - ), - pytest.param( - { - Path("modules/module1"): [ - Path(f"{anchor}modules/module1/transformations/my.Transformation.yaml"), - ] - }, - GetResourcesArgs( - resource_dir="transformations", - kind="Transformation", - selected="notamodule", - ), - [], - id="Select with non-existent module name", - ), - ], - ) - def test_get_resources_selected( - self, - module: dict[Path, list[Path]], - args: GetResourcesArgs, - expected: list[Path], - ) -> None: - module_list = self._create_built_resource_list(module) - result = module_list.get_resources( - id_type=None, - resource_dir=args.resource_dir, - kind=args.kind, - selected=args.selected, - ) - actual = [item.source.path for item in result] - assert actual == expected - - @staticmethod - def _create_built_resource_list(module: dict[Path, list[Path]]) -> BuiltModuleList: - modules: list[BuiltModule] = [ - BuiltModule( - name=module_path.name, - location=SourceLocationEager(module_path, "hash1234"), - build_variables=BuildVariables([]), - resources={ - resource_dir: BuiltResourceList( - [ - BuiltResource( - identifier=f"resource_{i}", - source=SourceLocationEager(resource_path, "hash5678"), - kind=resource_path.stem.split(".")[-1], - destination=None, - extra_sources=None, - ) - for i, resource_path in enumerate(resource_paths) - ] - ) - for resource_dir, resource_paths in groupby( - sorted(resource_paths, key=lambda p: p.parent.name), key=lambda p: p.parent.name - ) - }, - warning_count=0, - status="success", - iteration=1, - ) - for module_path, resource_paths in module.items() - ] - module_list = BuiltModuleList(modules) - return module_list From dfba9b078561cbe4b6e4a5bae438e12b341dfd48 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 7 Sep 2026 20:41:13 +0200 Subject: [PATCH 22/31] refactor: continue as the code reaper --- .../_cdf_tk/data_classes/__init__.py | 3 - .../_cdf_tk/data_classes/_build_variables.py | 315 ------------------ cognite_toolkit/_cdf_tk/validation.py | 25 +- .../test_data_classes/test_build_variables.py | 216 ------------ .../test_cdf_tk/test_utils/test_utils.py | 35 +- 5 files changed, 2 insertions(+), 592 deletions(-) delete mode 100644 cognite_toolkit/_cdf_tk/data_classes/_build_variables.py delete mode 100644 tests/test_unit/test_cdf_tk/test_data_classes/test_build_variables.py diff --git a/cognite_toolkit/_cdf_tk/data_classes/__init__.py b/cognite_toolkit/_cdf_tk/data_classes/__init__.py index e49e5177ce..e77331c917 100644 --- a/cognite_toolkit/_cdf_tk/data_classes/__init__.py +++ b/cognite_toolkit/_cdf_tk/data_classes/__init__.py @@ -1,4 +1,3 @@ -from ._build_variables import BuildVariable, BuildVariables from ._config_yaml import ( BuildConfigYAML, ConfigEntry, @@ -21,8 +20,6 @@ __all__ = [ "BuildConfigYAML", - "BuildVariable", - "BuildVariables", "CommandTracking", "ConfigEntry", "ConfigYAMLs", diff --git a/cognite_toolkit/_cdf_tk/data_classes/_build_variables.py b/cognite_toolkit/_cdf_tk/data_classes/_build_variables.py deleted file mode 100644 index 70ec6ab6fb..0000000000 --- a/cognite_toolkit/_cdf_tk/data_classes/_build_variables.py +++ /dev/null @@ -1,315 +0,0 @@ -import re -import sys -import uuid -from collections import defaultdict -from collections.abc import Collection, Iterator, Sequence -from dataclasses import dataclass -from functools import cached_property -from pathlib import Path -from typing import Any, Literal, SupportsIndex, overload - -from cognite_toolkit._cdf_tk.data_classes._module_directories import ModuleLocation -from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError -from cognite_toolkit._cdf_tk.resource_ios._resource_ios.transformation import TransformationIO - -if sys.version_info >= (3, 11): - from typing import Self -else: - from typing_extensions import Self - - -@dataclass(frozen=True) -class BuildVariable: - """This is an internal representation of a build variable in a config.[env].file - - Args: - key: The name of the variable. - value: The value of the variable. - is_selected: Whether the variable is selected by the user through Environment.selected - location: The location for the variable which is used to determine the module(s) it belongs to - - """ - - key: str - value: str | int | float | bool | tuple[str | int | float | bool] - is_selected: bool - location: Path - iteration: int | None = None - - @property - def value_variable(self) -> str | int | float | bool | list[str | int | float | bool]: - """Returns the value of the variable as a variable.""" - if isinstance(self.value, tuple): - # Convert the tuple back to a list to make it JSON serializable - return list(self.value) - else: - return self.value - - def dump(self) -> dict[str, Any]: - return { - "key": self.key, - "value": self.value_variable, - "is_selected": self.is_selected, - "location": self.location.as_posix(), - } - - @classmethod - def load(cls, data: dict[str, Any]) -> Self: - if isinstance(data["value"], list): - # Convert the list to a tuple to make it hashable - value = tuple(data["value"]) - else: - value = data["value"] - return cls(data["key"], value, data["is_selected"], Path(data["location"])) - - -class BuildVariables(tuple, Sequence[BuildVariable]): - """This is an internal representation of the build variables in a config.[env].file - - The motivation for this class is to provide helper functions for the user to interact with the build variables. - """ - - # Subclassing tuple to make the class immutable. BuildVariables is expected to be initialized and - # then used as a read-only object. - def __new__(cls, collection: Collection[BuildVariable], source_path: Path | None = None) -> Self: - # Need to override __new__ to as we are subclassing a tuple: - # https://stackoverflow.com/questions/1565374/subclassing-tuple-with-multiple-init-arguments - return super().__new__(cls, tuple(collection)) - - def __init__(self, collection: Collection[BuildVariable], source_path: Path | None = None) -> None: - super().__init__() - self.source_path = source_path - - @cached_property - def selected(self) -> "BuildVariables": - return BuildVariables([variable for variable in self if variable.is_selected]) - - @classmethod - def load_raw( - cls, - raw_variable: dict[str, Any], - available_modules: set[Path], - selected_modules: set[Path] | None = None, - source_path: Path | None = None, - ) -> Self: - """Loads the variables from the user input.""" - variables = [] - to_check: list[tuple[Path, int | None, dict[str, Any]]] = [(Path(""), None, raw_variable)] - while to_check: - path, iteration, subdict = to_check.pop() - for key, value in subdict.items(): - subpath = path / key - if subpath in available_modules and isinstance(value, dict): - to_check.append((subpath, None, value)) - elif subpath in available_modules and isinstance(value, list): - for no, module_variables in enumerate(value, 1): - if not isinstance(module_variables, dict): - raise ToolkitValueError(f"Variables under a module must be a dictionary: {subpath}.") - to_check.append((subpath, no, module_variables)) - elif isinstance(value, dict): - # Remove this check to support variables with dictionary values. - continue - else: - hashable_values = tuple(value) if isinstance(value, list) else value - is_selected = selected_modules is None or path in selected_modules - variables.append(BuildVariable(key, hashable_values, is_selected, path, iteration)) - - return cls(variables, source_path=source_path) - - @classmethod - def load(cls, data: list[dict[str, Any]]) -> Self: - """Loads the variables from a dictionary.""" - return cls([BuildVariable.load(variable) for variable in data]) - - def get_module_variables(self, module: ModuleLocation) -> "list[BuildVariables]": - """Gets the variables for a specific module.""" - variables_by_key_by_iteration: dict[int | None, dict[str, list[BuildVariable]]] = defaultdict( - lambda: defaultdict(list) - ) - for variable in self: - if variable.location == module.relative_path or variable.location in module.parent_relative_paths: - variables_by_key_by_iteration[variable.iteration][variable.key].append(variable) - - base_variables: dict[str, list[BuildVariable]] = variables_by_key_by_iteration.pop(None, {}) - variable_sets: list[dict[str, list[BuildVariable]]] - if variables_by_key_by_iteration: - # Combine each with the base variables - variable_sets = [] - for _, variables_by_key in sorted(variables_by_key_by_iteration.items(), key=lambda x: x[0] or 0): - for key, build_variable in base_variables.items(): - variables_by_key[key].extend(build_variable) - variable_sets.append(variables_by_key) - else: - variable_sets = [base_variables] - - return [ - BuildVariables( - [ - # We select the variable with the longest path to ensure that the most specific variable is selected - max(variables, key=lambda v: len(v.location.parts)) - for variables in variable_set.values() - ], - source_path=self.source_path, - ) - for variable_set in variable_sets - ] - - @overload - def replace(self, content: str, file_path: Path | None = None, use_placeholder: Literal[False] = False) -> str: ... - - @overload - def replace( - self, content: str, file_path: Path | None = None, use_placeholder: Literal[True] = True - ) -> tuple[str, dict[str, BuildVariable]]: ... - - def replace( - self, content: str, file_path: Path | None = None, use_placeholder: bool = False - ) -> str | tuple[str, dict[str, BuildVariable]]: - # Extract file suffix from path, default to .yaml if not provided - file_suffix = file_path.suffix if file_path and file_path.suffix else ".yaml" - - variable_by_placeholder: dict[str, BuildVariable] = {} - for variable in self: - if not use_placeholder: - replace = variable.value_variable - else: - replace = f"VARIABLE_{uuid.uuid4().hex[:8]}" - variable_by_placeholder[replace] = variable - - _core_pattern = rf"{{{{\s*{variable.key}\s*}}}}" - if file_suffix == ".sql": - # For SQL files, convert lists to SQL-style tuples - if isinstance(replace, list): - replace = self._format_list_as_sql_tuple(replace) - content = re.sub(_core_pattern, str(replace), content) - elif file_suffix in {".yaml", ".yml", ".json"}: - # Check if this is a transformation file (ends with Transformation.yaml/yml) - is_transformation_file = file_path is not None and f".{TransformationIO.kind}." in file_path.name - # Check if variable is within a query field (SQL context) - is_in_query_field = self._is_in_query_field(content, variable.key) - - # For lists in query fields, use SQL-style tuples - # For transformation files, ensure SQL conversion is applied to query property variables - if is_transformation_file and is_in_query_field and isinstance(replace, list): - replace = self._format_list_as_sql_tuple(replace) - # Use simple pattern for SQL context (no YAML quoting needed) - content = re.sub(_core_pattern, str(replace), content) - else: - # Preserve data types for YAML - pattern = _core_pattern - if isinstance(replace, str) and (replace.isdigit() or replace.endswith(":")): - replace = f'"{replace}"' - pattern = rf"'{_core_pattern}'|{_core_pattern}|" + rf'"{_core_pattern}"' - elif replace is None: - replace = "null" - content = re.sub(pattern, str(replace), content) - else: - # For other file types, use simple string replacement - content = re.sub(_core_pattern, str(replace), content) - if use_placeholder: - return content, variable_by_placeholder - else: - return content - - @staticmethod - def _is_transformation_file(file_path: Path) -> bool: - """Check if the file path indicates a transformation YAML file. - - Transformation files are YAML files in the "transformations" folder. - - Args: - file_path: The file path to check - - Returns: - True if the file is a transformation YAML file - """ - # Check if path contains "transformations" folder and ends with .yaml/.yml - path_str = file_path.as_posix().lower() - return "transformations" in path_str and file_path.suffix.lower() in {".yaml", ".yml"} - - @staticmethod - def _format_list_as_sql_tuple(replace: list[Any]) -> str: - """Format a list as a SQL-style tuple string. - - Args: - replace: The list to format - - Returns: - SQL tuple string, e.g., "('A', 'B', 'C')" or "()" for empty lists - """ - if not replace: - # Empty list becomes empty SQL tuple - return "()" - else: - # Format list as SQL tuple: ('A', 'B', 'C') - formatted_items = [] - for item in replace: - if item is None: - formatted_items.append("NULL") - elif isinstance(item, str): - formatted_items.append(f"'{item}'") - else: - formatted_items.append(str(item)) - return f"({', '.join(formatted_items)})" - - @staticmethod - def _is_in_query_field(content: str, variable_key: str) -> bool: - """Check if a variable is within a query field in YAML. - - Assumes query is a top-level property. This detects various YAML formats: - - query: >- - - query: | - - query: "..." - - query: ... - """ - lines = content.split("\n") - variable_pattern = rf"{{{{\s*{re.escape(variable_key)}\s*}}}}" - in_query_field = False - - for line in lines: - # Check if this line starts a top-level query field - query_match = re.match(r"^query\s*:\s*(.*)$", line) - if query_match: - in_query_field = True - query_content_start = query_match.group(1).strip() - - # Check if variable is on the same line as query: declaration - if re.search(variable_pattern, line): - return True - - # If query content starts on same line (not a block scalar), check it - if query_content_start and not query_content_start.startswith(("|", ">", "|-", ">-", "|+", ">+")): - if re.search(variable_pattern, query_content_start): - return True - continue - - # Check if we're still in the query field - if in_query_field: - # If we hit another top-level property, we've exited the query field - if re.match(r"^\w+\s*:", line): - in_query_field = False - continue - - # We're still in the query field, check for variable - if re.search(variable_pattern, line): - return True - - return False - - # Implemented to get correct type hints - def __iter__(self) -> Iterator[BuildVariable]: - return super().__iter__() - - @overload - def __getitem__(self, index: SupportsIndex) -> BuildVariable: ... - - @overload - def __getitem__(self, index: slice) -> Self: ... - - def __getitem__(self, index: SupportsIndex | slice, /) -> "BuildVariable | BuildVariables": - if isinstance(index, slice): - return BuildVariables(super().__getitem__(index)) - return super().__getitem__(index) - - def dump(self) -> list[dict[str, Any]]: - return [variable.dump() for variable in self] diff --git a/cognite_toolkit/_cdf_tk/validation.py b/cognite_toolkit/_cdf_tk/validation.py index fd9b7721b6..c971e395a8 100644 --- a/cognite_toolkit/_cdf_tk/validation.py +++ b/cognite_toolkit/_cdf_tk/validation.py @@ -1,5 +1,4 @@ import inspect -import re from pathlib import Path from typing import Any, NamedTuple, TypeVar @@ -9,7 +8,7 @@ from cognite_toolkit._cdf_tk.cdf_toml import CDFToml from cognite_toolkit._cdf_tk.client._resource_base import ResponseResource from cognite_toolkit._cdf_tk.constants import DEV_ONLY_MODULES -from cognite_toolkit._cdf_tk.data_classes import BuildConfigYAML, BuildVariables, ModuleDirectories +from cognite_toolkit._cdf_tk.data_classes import BuildConfigYAML, ModuleDirectories from cognite_toolkit._cdf_tk.exceptions import ( ToolkitDuplicatedModuleError, ToolkitEnvError, @@ -19,7 +18,6 @@ from cognite_toolkit._cdf_tk.tk_warnings import ( DataSetMissingWarning, MediumSeverityWarning, - TemplateVariableWarning, WarningList, ) from cognite_toolkit._cdf_tk.tk_warnings.fileread import ResourceFormatWarning @@ -31,7 +29,6 @@ "humanize_validation_error_categorized", "validate_data_set_is_set", "validate_module_selection", - "validate_modules_variables", ] @@ -47,26 +44,6 @@ class _GroupEntry(NamedTuple): loc: tuple[str | int, ...] -def validate_modules_variables(variables: BuildVariables, filepath: Path) -> WarningList: - """Checks whether the config file has any issues. - - Currently, this checks for: - * Non-replaced template variables, such as . - - Args: - variables: The variables to check. - filepath: The filepath of the config.yaml. - """ - warning_list: WarningList = WarningList() - pattern = re.compile(r"<.*?>") - for variable in variables: - if isinstance(variable.value, str) and pattern.match(variable.value): - warning_list.append( - TemplateVariableWarning(filepath, variable.value, variable.key, ".".join(variable.location.parts)) - ) - return warning_list - - def validate_data_set_is_set( raw: dict[str, Any] | list[dict[str, Any]], resource_cls: type[ResponseResource], diff --git a/tests/test_unit/test_cdf_tk/test_data_classes/test_build_variables.py b/tests/test_unit/test_cdf_tk/test_data_classes/test_build_variables.py deleted file mode 100644 index b15969e4f7..0000000000 --- a/tests/test_unit/test_cdf_tk/test_data_classes/test_build_variables.py +++ /dev/null @@ -1,216 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -import yaml - -from cognite_toolkit._cdf_tk.data_classes import BuildVariables, ModuleLocation - - -class TestBuildVariables: - def test_replace_preserve_data_type(self) -> None: - source_yaml = """text: {{ my_text }} -bool: {{ my_bool }} -integer: {{ my_integer }} -float: {{ my_float }} -digit_string: {{ my_digit_string }} -quoted_string: "{{ my_quoted_string }}" -list: {{ my_list }} -null_value: {{ my_null_value }} -single_quoted_string: '{{ my_single_quoted_string }}' -composite: 'some_prefix_{{ my_composite }}' -prefix_text: {{ my_prefix_text }} -suffix_text: {{ my_suffix_text }} -""" - variables = BuildVariables.load_raw( - { - "my_text": "some text", - "my_bool": True, - "my_integer": 123, - "my_float": 123.456, - "my_digit_string": "123", - "my_quoted_string": "456", - "my_list": ["one", "two", "three"], - "my_null_value": None, - "my_single_quoted_string": "789", - "my_composite": "the suffix", - "my_prefix_text": "prefix:", - "my_suffix_text": ":suffix", - }, - available_modules=set(), - selected_modules=set(), - ) - - result = variables.replace(source_yaml) - - loaded = yaml.safe_load(result) - assert loaded == { - "text": "some text", - "bool": True, - "integer": 123, - "float": 123.456, - "digit_string": "123", - "quoted_string": "456", - "list": ["one", "two", "three"], - "null_value": None, - "single_quoted_string": "789", - "composite": "some_prefix_the suffix", - "prefix_text": "prefix:", - "suffix_text": ":suffix", - } - - def test_replace_not_preserve_type(self) -> None: - source_yaml = """dataset_id('{{dataset_external_id}}')""" - variables = BuildVariables.load_raw( - { - "dataset_external_id": "ds_external_id", - }, - available_modules=set(), - selected_modules=set(), - ) - - result = variables.replace(source_yaml, Path("test.sql")) - - assert result == "dataset_id('ds_external_id')" - - def test_replace_sql_list(self) -> None: - """Test that lists with mixed types in SQL files are formatted correctly.""" - source_sql = """SELECT * FROM table WHERE column IN {{ my_list }}""" - variables = BuildVariables.load_raw( - { - "my_list": ["A", 123, None, True], - }, - available_modules=set(), - selected_modules=set(), - ) - - result = variables.replace(source_sql, Path("test.sql")) - - assert result == "SELECT * FROM table WHERE column IN ('A', 123, NULL, True)" - - def test_replace_inline_sql_preserve_double_quotes(self) -> None: - source_yaml = """externalId: some_id -name: Some Transformation -destination: - type: nodes - view: - space: cdf_cdm - externalId: CogniteTimeSeries - version: v1 - instanceSpace: my_instance_space -query: >- - select "fpso_{{location_id}}" as externalId, "{{location_ID}}" as uid, "{{location_ID}}" as description -""" - variables = BuildVariables.load_raw( - { - "location_id": "uny", - "location_ID": "UNY", - }, - available_modules=set(), - selected_modules=set(), - ) - - result = variables.replace(source_yaml, Path("test.Transformation.yaml")) - - loaded = yaml.safe_load(result) - - assert loaded["query"] == 'select "fpso_uny" as externalId, "UNY" as uid, "UNY" as description' - - def test_replace_yaml_query_field_list(self) -> None: - """Test that lists in multiline query fields are converted to SQL-style tuples.""" - source_yaml = """externalId: some_id -name: Some Transformation -query: | - SELECT * FROM table - WHERE column IN {{ my_list }} - AND other_column = 'value' -""" - variables = BuildVariables.load_raw( - { - "my_list": ["X", "Y", "Z"], - }, - available_modules=set(), - selected_modules=set(), - ) - - result = variables.replace(source_yaml, Path("test.Transformation.yaml")) - - loaded = yaml.safe_load(result) - assert "('X', 'Y', 'Z')" in loaded["query"] - - def test_replace_yaml_query_field_list_mixed_types(self) -> None: - """Test that lists with mixed types in query fields are formatted correctly.""" - source_yaml = """externalId: some_id -name: Some Transformation -query: >- - SELECT * FROM table WHERE column IN {{ my_list }} -""" - variables = BuildVariables.load_raw( - { - "my_list": ["A", 123, None, True], - }, - available_modules=set(), - selected_modules=set(), - ) - - result = variables.replace(source_yaml, Path("test.Transformation.yaml")) - - loaded = yaml.safe_load(result) - assert loaded["query"] == "SELECT * FROM table WHERE column IN ('A', 123, NULL, True)" - - def test_replace_yaml_non_query_field_preserves_list(self) -> None: - """Test that lists outside query fields still use YAML list format.""" - source_yaml = """externalId: some_id -name: Some Transformation -tags: {{ my_list }} -query: >- - SELECT * FROM table -""" - variables = BuildVariables.load_raw( - { - "my_list": ["tag1", "tag2"], - }, - available_modules=set(), - selected_modules=set(), - ) - - result = variables.replace(source_yaml) - - loaded = yaml.safe_load(result) - # Tags should be a YAML list, not SQL tuple - assert loaded["tags"] == ["tag1", "tag2"] - - def test_get_module_variables_variable_preference_order(self) -> None: - source_yaml = """ -modules: - industry_apps: - module_version: '1' - pause_transformations: true - apm_datamodel_space: APM_SourceData - apm_sourcedata_model_version: '1.2.1' - - industry_apps_crna_common: - apm_sourcedata_model_version: '1' -""" - selected = { - Path("."), - Path("modules"), - Path("modules/industry_apps"), - Path("modules/industry_apps/industry_apps_crna_common"), - } - - variables = BuildVariables.load_raw( - yaml.safe_load(source_yaml), available_modules=selected, selected_modules=selected - ) - - assert len(variables) == 5 - location = ModuleLocation( - Path("modules/industry_apps/industry_apps_crna_common"), Path("."), source_paths=[], is_selected=True - ) - local_variables = variables.get_module_variables(location)[0] - - assert len(local_variables) == 4 - apm_sourcedata_model_version = next( - (variable for variable in local_variables if variable.key == "apm_sourcedata_model_version"), None - ) - assert apm_sourcedata_model_version.value == "1" diff --git a/tests/test_unit/test_cdf_tk/test_utils/test_utils.py b/tests/test_unit/test_cdf_tk/test_utils/test_utils.py index d1cd2181c0..441eceb6ca 100644 --- a/tests/test_unit/test_cdf_tk/test_utils/test_utils.py +++ b/tests/test_unit/test_cdf_tk/test_utils/test_utils.py @@ -10,12 +10,7 @@ import yaml from _pytest.mark import ParameterSet -from cognite_toolkit._cdf_tk.data_classes import BuildVariable, BuildVariables -from cognite_toolkit._cdf_tk.tk_warnings import ( - EnvironmentVariableMissingWarning, - TemplateVariableWarning, - catch_warnings, -) +from cognite_toolkit._cdf_tk.tk_warnings import EnvironmentVariableMissingWarning, catch_warnings from cognite_toolkit._cdf_tk.utils import ( calculate_directory_hash, flatten_dict, @@ -27,7 +22,6 @@ ) from cognite_toolkit._cdf_tk.utils.file import yaml_safe_dump from cognite_toolkit._cdf_tk.utils.modules import module_directory_from_path -from cognite_toolkit._cdf_tk.validation import validate_modules_variables from tests.data import CALC_HASH_DATA, PROJECT_FOR_TEST @@ -52,33 +46,6 @@ def test_warning_when_missing_env_variable(self) -> None: assert warning_list[0] == expected_warning -@pytest.mark.parametrize( - "variable, expected_warnings", - [ - pytest.param( - BuildVariable("sourceId", "", False, Path()), - [TemplateVariableWarning(Path("config.yaml"), "", "sourceId", "")], - id="Single warning", - ), - pytest.param( - BuildVariable("sourceId", "", False, Path("a_module")), - [TemplateVariableWarning(Path("config.yaml"), "", "sourceId", "a_module")], - id="Nested warning", - ), - pytest.param( - BuildVariable("sourceId", "", False, Path("a_super_module/a_module")), - [TemplateVariableWarning(Path("config.yaml"), "", "sourceId", "a_super_module.a_module")], - id="Deep nested warning", - ), - pytest.param(BuildVariable("sourceId", "123", False, Path("a_module")), [], id="No warning"), - ], -) -def test_validate_config_yaml(variable: BuildVariable, expected_warnings: list[TemplateVariableWarning]) -> None: - warnings = validate_modules_variables(BuildVariables([variable]), Path("config.yaml")) - - assert sorted(warnings) == sorted(expected_warnings) - - def test_calculate_hash_on_folder() -> None: folder = CALC_HASH_DATA hash1 = calculate_directory_hash(folder) From 5f6d8535046de9a2f483a0df7fd63b46ec7e28cf Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 7 Sep 2026 20:47:20 +0200 Subject: [PATCH 23/31] refactor: delete more --- .../_cdf_tk/data_classes/__init__.py | 6 - .../_cdf_tk/data_classes/_deploy_results.py | 112 +----------------- cognite_toolkit/_cdf_tk/validation.py | 78 ------------ 3 files changed, 2 insertions(+), 194 deletions(-) diff --git a/cognite_toolkit/_cdf_tk/data_classes/__init__.py b/cognite_toolkit/_cdf_tk/data_classes/__init__.py index e77331c917..91340f902d 100644 --- a/cognite_toolkit/_cdf_tk/data_classes/__init__.py +++ b/cognite_toolkit/_cdf_tk/data_classes/__init__.py @@ -6,12 +6,9 @@ InitConfigYAML, ) from ._deploy_results import ( - DatapointDeployResult, DeployResult, DeployResults, - ResourceContainerDeployResult, ResourceDeployResult, - UploadDeployResult, ) from ._module_directories import ModuleDirectories, ModuleLocation from ._packages import Package, Packages @@ -23,7 +20,6 @@ "CommandTracking", "ConfigEntry", "ConfigYAMLs", - "DatapointDeployResult", "DeployResult", "DeployResults", "DeploymentTracking", @@ -33,9 +29,7 @@ "ModuleLocation", "Package", "Packages", - "ResourceContainerDeployResult", "ResourceDeployResult", "TrackingEvent", - "UploadDeployResult", "YAMLComments", ] diff --git a/cognite_toolkit/_cdf_tk/data_classes/_deploy_results.py b/cognite_toolkit/_cdf_tk/data_classes/_deploy_results.py index e453a702e5..82f45609e1 100644 --- a/cognite_toolkit/_cdf_tk/data_classes/_deploy_results.py +++ b/cognite_toolkit/_cdf_tk/data_classes/_deploy_results.py @@ -9,9 +9,9 @@ from rich.table import Table if sys.version_info >= (3, 11): - from typing import Self + pass else: - from typing_extensions import Self + pass @total_ordering @@ -52,79 +52,6 @@ def __iadd__(self, other: "ResourceDeployResult") -> "ResourceDeployResult": self.changed += other.changed self.unchanged += other.unchanged self.total += other.total - - if isinstance(other, ResourceContainerDeployResult): - return ResourceContainerDeployResult( - name=self.name, - created=self.created, - deleted=self.deleted, - changed=self.changed, - unchanged=self.unchanged, - total=self.total, - item_name=other.item_name, - dropped_datapoints=other.dropped_datapoints, - ) - else: - return self - - -@dataclass -class ResourceContainerDeployResult(ResourceDeployResult): - item_name: str = "" - dropped_datapoints: int = 0 - - def __iadd__(self, other: ResourceDeployResult) -> "ResourceContainerDeployResult": - if self.name != other.name: - raise ValueError("Cannot add two ResourceContainerDeployResult objects with different names") - super().__iadd__(other) - if isinstance(other, ResourceContainerDeployResult): - self.dropped_datapoints += other.dropped_datapoints - return self - - @classmethod - def from_resource_deploy_result( - cls, result: ResourceDeployResult, item_name: str = "", dropped_datapoints: int = 0 - ) -> Self: - return cls( - name=result.name, - created=result.created, - deleted=result.deleted, - changed=result.changed, - unchanged=result.unchanged, - total=result.total, - item_name=item_name, - dropped_datapoints=dropped_datapoints, - ) - - -@dataclass -class UploadDeployResult(DeployResult): - uploaded: int = 0 - item_name: str = "" - - def __iadd__(self, other: "UploadDeployResult") -> "UploadDeployResult": - if self.name != other.name: - raise ValueError("Cannot add two DeployResult objects with different names") - self.uploaded += other.uploaded - - if isinstance(other, DatapointDeployResult): - return DatapointDeployResult( - name=self.name, uploaded=self.uploaded, item_name=other.item_name, points=other.points - ) - else: - return self - - -@dataclass -class DatapointDeployResult(UploadDeployResult): - points: int = 0 - - def __iadd__(self, other: UploadDeployResult) -> UploadDeployResult: - if self.name != other.name: - raise ValueError("Cannot add two DeployResult objects with different names") - super().__iadd__(other) - if isinstance(other, DatapointDeployResult): - self.points += other.points return self @@ -143,12 +70,6 @@ def __init__( def has_counts(self) -> bool: return any(isinstance(entry, ResourceDeployResult) for entry in self.data.values()) - @property - def has_uploads(self) -> bool: - return any( - isinstance(entry, UploadDeployResult | ResourceContainerDeployResult) for entry in self.data.values() - ) - def counts_table( self, exclude_columns: set[Literal["Created", "Deleted", "Changed", "Untouched", "Total"]] | None = None ) -> Table: @@ -193,35 +114,6 @@ def counts_table( return table - def uploads_table(self) -> Table: - table = Table(title=f"Summary of Data {self.action.title()} operation (data is always uploaded):") - prefix = "Would have " if self.dry_run else "" - table.add_column("Resource", justify="right") - table.add_column(f"{prefix}Uploaded Data", justify="right", style="cyan") - table.add_column("Item Type", justify="right") - table.add_column("From files", justify="right", style="green") - table.add_column(f"{prefix}Deleted Data", justify="right", style="red") - for item in sorted( - entry - for entry in self.data.values() - if isinstance(entry, UploadDeployResult | ResourceContainerDeployResult) - ): - if item.name == "raw.tables": - # We skip this as we cannot count the number of datapoints in a raw table - # and all we can do is to print a misleading 0 for deleted datapoints. - continue - - if isinstance(item, UploadDeployResult): - if isinstance(item, DatapointDeployResult): - datapoints = f"{item.points:,}" - else: - datapoints = "-" - table.add_row(item.name, datapoints, item.item_name, str(item.uploaded), "-") - elif isinstance(item, ResourceContainerDeployResult): - table.add_row(item.name, "-", item.item_name, "-", f"{item.dropped_datapoints:,}") - - return table - def __getitem__(self, item: str) -> DeployResult: return self.data[item] diff --git a/cognite_toolkit/_cdf_tk/validation.py b/cognite_toolkit/_cdf_tk/validation.py index c971e395a8..de1c22e730 100644 --- a/cognite_toolkit/_cdf_tk/validation.py +++ b/cognite_toolkit/_cdf_tk/validation.py @@ -5,19 +5,9 @@ from pydantic import BaseModel, TypeAdapter, ValidationError from pydantic_core import ErrorDetails -from cognite_toolkit._cdf_tk.cdf_toml import CDFToml from cognite_toolkit._cdf_tk.client._resource_base import ResponseResource -from cognite_toolkit._cdf_tk.constants import DEV_ONLY_MODULES -from cognite_toolkit._cdf_tk.data_classes import BuildConfigYAML, ModuleDirectories -from cognite_toolkit._cdf_tk.exceptions import ( - ToolkitDuplicatedModuleError, - ToolkitEnvError, - ToolkitMissingModuleError, -) -from cognite_toolkit._cdf_tk.hints import ModuleDefinition from cognite_toolkit._cdf_tk.tk_warnings import ( DataSetMissingWarning, - MediumSeverityWarning, WarningList, ) from cognite_toolkit._cdf_tk.tk_warnings.fileread import ResourceFormatWarning @@ -28,7 +18,6 @@ "humanize_validation_error", "humanize_validation_error_categorized", "validate_data_set_is_set", - "validate_module_selection", ] @@ -287,70 +276,3 @@ def as_json_path(loc: tuple[str | int, ...]) -> str: suffix = ".".join([str(x) if isinstance(x, str) else f"[{x + 1}]" for x in loc]).replace(".[", "[") return f"{prefix}{suffix}" - - -def validate_module_selection( - modules: ModuleDirectories, - config: BuildConfigYAML, - packages: dict[str, list[str]], - selected_modules: set[str | Path], - organization_dir: Path, -) -> WarningList: - """Validates module selection and returns warnings for non-critical issues. - - Critical errors (duplicate modules, missing modules, no modules selected) are still raised - as exceptions as they prevent the build from proceeding. - """ - warnings: WarningList = WarningList() - - # Validations: Ambiguous selection. - selected_names = {s for s in config.environment.selected if isinstance(s, str)} - if duplicate_modules := { - module_name: paths - for module_name, paths in modules.as_path_by_name().items() - if len(paths) > 1 and module_name in selected_names - }: - # If the user has selected a module by name, and there are multiple modules with that name, raise an error. - # Note, if the user uses a path to select a module, this error will not be raised. - raise ToolkitDuplicatedModuleError( - f"Ambiguous module selected in config.{config.environment.name}.yaml:", duplicate_modules - ) - - # Package Referenced Modules Exists - for package, package_modules in packages.items(): - if package not in selected_names: - # We do not check packages that are not selected. - # Typically, the user will delete the modules that are irrelevant for them; - # thus we only check the selected packages. - continue - if missing_packages := set(package_modules) - modules.available_names: - raise ToolkitMissingModuleError( - f"Package {package} defined in {CDFToml.file_name!s} is referring " - f"the following missing modules {missing_packages}." - ) - - # Selected modules does not exists - if missing_modules := set(selected_modules) - modules.available: - hint = ModuleDefinition.long(missing_modules, organization_dir) - raise ToolkitMissingModuleError( - f"The following selected modules are missing, please check path: {missing_modules}.\n{hint}" - ) - - # Nothing is Selected - if not modules.selected: - raise ToolkitEnvError( - f"No selected modules specified in {config.filepath!s}, have you configured " - f"the environment ({config.environment.name})?" - ) - - # Dev modules warning (non-critical) - dev_modules = modules.available_names & DEV_ONLY_MODULES - if dev_modules and config.environment.validation_type != "dev": - warnings.append( - MediumSeverityWarning( - "The following modules should [bold]only[/bold] be used a in CDF Projects designated as dev (development): " - f"{humanize_collection(dev_modules)!r}", - ) - ) - - return warnings From 023b4f4759ba6c2c4c681b9a7edc31ef544a1ede Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 7 Sep 2026 20:49:00 +0200 Subject: [PATCH 24/31] refactro: more cleanup --- cognite_toolkit/_cdf_tk/data_classes/_base.py | 31 ++----------------- 1 file changed, 2 insertions(+), 29 deletions(-) diff --git a/cognite_toolkit/_cdf_tk/data_classes/_base.py b/cognite_toolkit/_cdf_tk/data_classes/_base.py index 5de7fe116d..4ce9e818a8 100644 --- a/cognite_toolkit/_cdf_tk/data_classes/_base.py +++ b/cognite_toolkit/_cdf_tk/data_classes/_base.py @@ -2,11 +2,9 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from pathlib import Path -from typing import Any, ClassVar, TypeVar +from typing import Any, ClassVar -from cognite_toolkit import _version -from cognite_toolkit._cdf_tk.constants import BUILD_ENVIRONMENT_FILE -from cognite_toolkit._cdf_tk.exceptions import ToolkitFileNotFoundError, ToolkitRequiredValueError, ToolkitVersionError +from cognite_toolkit._cdf_tk.exceptions import ToolkitFileNotFoundError, ToolkitRequiredValueError from cognite_toolkit._cdf_tk.utils import read_yaml_file if sys.version_info >= (3, 11): @@ -43,28 +41,3 @@ def load_from_directory(cls, organization_dir: Path, build_env: str) -> Self: @abstractmethod def load(cls, data: dict[str, Any], build_env: str, filepath: Path) -> Self: raise NotImplementedError - - -T_BuildConfig = TypeVar("T_BuildConfig", bound=ConfigCore) - - -def _load_version_variable(data: dict[str, Any], file_name: str) -> str: - try: - cdf_tk_version: str = data["cdf_toolkit_version"] - except KeyError: - err_msg = f"System variables are missing required field 'cdf_toolkit_version' in {file_name!s}. {{}}" - if file_name == BUILD_ENVIRONMENT_FILE: - raise ToolkitVersionError( - err_msg.format("Rerun `cdf build` to build the modules again and create it correctly.") - ) - raise ToolkitVersionError( - err_msg.format("Run `cdf modules upgrade` to initialize the modules again to create a correct file.") - ) - - if cdf_tk_version != _version.__version__: - raise ToolkitVersionError( - f"The version of the modules ({cdf_tk_version}) does not match the version of the installed CLI " - f"({_version.__version__}). Please either run `cdf modules upgrade` to upgrade the modules OR " - f"run `pip install cognite-toolkit=={cdf_tk_version}` to downgrade cdf-tk CLI." - ) - return cdf_tk_version From fc417297d522829d7b6e0f9c3982ab8c5a2b5bde Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 7 Sep 2026 20:08:30 +0200 Subject: [PATCH 25/31] tests: remove tests and start init cleanup --- cognite_toolkit/_cdf_tk/builders/__init__.py | 43 +----------- .../test_cdf_tk/test_builders/__init__.py | 0 .../test_cdf_tk/test_builders/test_builder.py | 66 ------------------- .../test_builders/test_location_builder.py | 53 --------------- 4 files changed, 2 insertions(+), 160 deletions(-) delete mode 100644 tests/test_unit/test_cdf_tk/test_builders/__init__.py delete mode 100644 tests/test_unit/test_cdf_tk/test_builders/test_builder.py delete mode 100644 tests/test_unit/test_cdf_tk/test_builders/test_location_builder.py diff --git a/cognite_toolkit/_cdf_tk/builders/__init__.py b/cognite_toolkit/_cdf_tk/builders/__init__.py index 30fea31d5c..2bec72ce54 100644 --- a/cognite_toolkit/_cdf_tk/builders/__init__.py +++ b/cognite_toolkit/_cdf_tk/builders/__init__.py @@ -1,42 +1,3 @@ -from collections.abc import Callable -from pathlib import Path +from ._base import Builder -from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning - -from ._base import Builder, DefaultBuilder, get_resource_crud -from ._datamodels import DataModelBuilder -from ._file import FileBuilder -from ._function import FunctionBuilder -from ._location import LocationBuilder -from ._raw import RawBuilder -from ._ruleset import RuleSetBuilder -from ._streamlit import StreamlitBuilder -from ._transformation import TransformationBuilder - - -def create_builder( - resource_folder: str, - build_dir: Path | None, - warn: Callable[[ToolkitWarning], None] | None = None, -) -> Builder: - if builder_cls := _BUILDER_BY_RESOURCE_FOLDER.get(resource_folder): - return builder_cls(build_dir, warn=warn) # type: ignore[abstract] - - return DefaultBuilder(build_dir, resource_folder, warn) - - -_BUILDER_BY_RESOURCE_FOLDER = {_builder._resource_folder: _builder for _builder in Builder.__subclasses__()} -__all__ = [ - "Builder", - "DataModelBuilder", - "DefaultBuilder", - "FileBuilder", - "FunctionBuilder", - "LocationBuilder", - "RawBuilder", - "RuleSetBuilder", - "StreamlitBuilder", - "TransformationBuilder", - "create_builder", - "get_resource_crud", -] +__all__ = ["Builder"] diff --git a/tests/test_unit/test_cdf_tk/test_builders/__init__.py b/tests/test_unit/test_cdf_tk/test_builders/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/test_unit/test_cdf_tk/test_builders/test_builder.py b/tests/test_unit/test_cdf_tk/test_builders/test_builder.py deleted file mode 100644 index 00e94052b2..0000000000 --- a/tests/test_unit/test_cdf_tk/test_builders/test_builder.py +++ /dev/null @@ -1,66 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -import pytest - -from cognite_toolkit._cdf_tk.builders import get_resource_crud -from cognite_toolkit._cdf_tk.resource_ios import ( - RESOURCE_CRUD_LIST, - GroupAllScopedCRUD, - GroupIO, - GroupResourceScopedCRUD, - ResourceIO, -) -from cognite_toolkit._cdf_tk.tk_warnings import ToolkitNotSupportedWarning, ToolkitWarning -from cognite_toolkit._cdf_tk.tk_warnings.fileread import UnknownResourceTypeWarning - - -class TestGetCRUD: - @pytest.mark.parametrize( - "source_path, resource_folder, expected_loader_cls", - [ - pytest.param( - Path(f"some_path/{crud_cls.folder_name}/my.{crud_cls.kind}.yaml"), - crud_cls.folder_name, - { - GroupResourceScopedCRUD: GroupIO, - GroupAllScopedCRUD: GroupIO, - }.get(crud_cls, crud_cls), - id=crud_cls.__name__, - ) - for crud_cls in RESOURCE_CRUD_LIST - ], - ) - def test_get_crud_no_warning( - self, source_path: Path, resource_folder: str, expected_loader_cls: type[ResourceIO] - ) -> None: - crud_cls, warning = get_resource_crud(source_path, resource_folder) - - assert warning is None - assert crud_cls is expected_loader_cls - - @pytest.mark.parametrize( - "source_path, resource_folder, expected_warning_cls", - [ - pytest.param( - Path(f"some_path/unknown_folder/my.{GroupIO.kind}.yaml"), - "unknown_folder", - ToolkitNotSupportedWarning, - id="Unknown folder, known kind", - ), - pytest.param( - Path("some_path/group/my.UnknownKind.yaml"), - GroupIO.folder_name, - UnknownResourceTypeWarning, - id="Known folder, unknown kind", - ), - ], - ) - def test_get_crud_warning( - self, source_path: Path, resource_folder: str, expected_warning_cls: type[ToolkitWarning] - ) -> None: - crud_cls, warning = get_resource_crud(source_path, resource_folder) - - assert crud_cls is None - assert type(warning) is expected_warning_cls diff --git a/tests/test_unit/test_cdf_tk/test_builders/test_location_builder.py b/tests/test_unit/test_cdf_tk/test_builders/test_location_builder.py deleted file mode 100644 index 15a5b88b8f..0000000000 --- a/tests/test_unit/test_cdf_tk/test_builders/test_location_builder.py +++ /dev/null @@ -1,53 +0,0 @@ -from pathlib import Path -from unittest.mock import MagicMock - -import pytest - -from cognite_toolkit._cdf_tk.builders._location import LocationBuilder -from cognite_toolkit._cdf_tk.data_classes._build_files import BuildSourceFile -from cognite_toolkit._cdf_tk.data_classes._module_directories import ModuleLocation -from cognite_toolkit._cdf_tk.exceptions import ToolkitError - - -def test_location_builder_detect_cyclic_references(build_tmp_path): - # Setup - - module_location = MagicMock(spec=ModuleLocation) - - source_file = MagicMock(spec=BuildSourceFile) - source_file.source = MagicMock() - source_file.source.path = Path("cyclic.LocationFilter.yaml") - source_file.loaded = [ - {"externalId": "location1", "description": "Location 1", "parentExternalId": "location2"}, - {"externalId": "location2", "description": "Location 2", "parentExternalId": "location3"}, - { - "externalId": "location3", - "description": "Location 3", - "parentExternalId": "location1", # This creates a cycle - }, - ] - - location_builder = LocationBuilder(build_dir=build_tmp_path) - with pytest.raises(ToolkitError, match=r"Circular dependency found in Locations*"): - list(location_builder.build(source_files=[source_file], module=module_location)) - - -def test_location_builder_detect_self_reference(build_tmp_path): - # Setup - - module_location = MagicMock(spec=ModuleLocation) - - source_file = MagicMock(spec=BuildSourceFile) - source_file.source = MagicMock() - source_file.source.path = Path("self.LocationFilter.yaml") - source_file.loaded = [ - { - "externalId": "location1", - "description": "Location 1", - "parentExternalId": "location1", - }, # This creates a self-reference - ] - - location_builder = LocationBuilder(build_dir=build_tmp_path) - with pytest.raises(ToolkitError, match=r"Circular dependency found in Locations*"): - list(location_builder.build(source_files=[source_file], module=module_location)) From 0f522c4e44fe485e6dbd67680bdc5baa813c91ec Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 7 Sep 2026 20:09:47 +0200 Subject: [PATCH 26/31] refactor: deleted builder module --- cognite_toolkit/_cdf_tk/builders/__init__.py | 3 - cognite_toolkit/_cdf_tk/builders/_base.py | 178 ----------------- .../_cdf_tk/builders/_datamodels.py | 90 --------- cognite_toolkit/_cdf_tk/builders/_file.py | 88 --------- cognite_toolkit/_cdf_tk/builders/_function.py | 187 ------------------ cognite_toolkit/_cdf_tk/builders/_location.py | 84 -------- cognite_toolkit/_cdf_tk/builders/_raw.py | 72 ------- cognite_toolkit/_cdf_tk/builders/_ruleset.py | 105 ---------- .../_cdf_tk/builders/_streamlit.py | 110 ----------- .../_cdf_tk/builders/_transformation.py | 111 ----------- .../test_cdf_tk/test_cruds/test_data_model.py | 46 ----- 11 files changed, 1074 deletions(-) delete mode 100644 cognite_toolkit/_cdf_tk/builders/__init__.py delete mode 100644 cognite_toolkit/_cdf_tk/builders/_base.py delete mode 100644 cognite_toolkit/_cdf_tk/builders/_datamodels.py delete mode 100644 cognite_toolkit/_cdf_tk/builders/_file.py delete mode 100644 cognite_toolkit/_cdf_tk/builders/_function.py delete mode 100644 cognite_toolkit/_cdf_tk/builders/_location.py delete mode 100644 cognite_toolkit/_cdf_tk/builders/_raw.py delete mode 100644 cognite_toolkit/_cdf_tk/builders/_ruleset.py delete mode 100644 cognite_toolkit/_cdf_tk/builders/_streamlit.py delete mode 100644 cognite_toolkit/_cdf_tk/builders/_transformation.py diff --git a/cognite_toolkit/_cdf_tk/builders/__init__.py b/cognite_toolkit/_cdf_tk/builders/__init__.py deleted file mode 100644 index 2bec72ce54..0000000000 --- a/cognite_toolkit/_cdf_tk/builders/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from ._base import Builder - -__all__ = ["Builder"] diff --git a/cognite_toolkit/_cdf_tk/builders/_base.py b/cognite_toolkit/_cdf_tk/builders/_base.py deleted file mode 100644 index f48555098c..0000000000 --- a/cognite_toolkit/_cdf_tk/builders/_base.py +++ /dev/null @@ -1,178 +0,0 @@ -import difflib -from abc import ABC, abstractmethod -from collections.abc import Callable, Iterable, Sequence -from pathlib import Path -from typing import Any, ClassVar - -from cognite_toolkit._cdf_tk.constants import INDEX_PATTERN -from cognite_toolkit._cdf_tk.data_classes import ( - BuildDestinationFile, - BuildSourceFile, - BuiltResourceList, - ModuleLocation, -) -from cognite_toolkit._cdf_tk.exceptions import ( - AmbiguousResourceFileError, -) -from cognite_toolkit._cdf_tk.resource_ios import ( - RESOURCE_CRUD_BY_FOLDER_NAME, - GroupIO, - ResourceIO, -) -from cognite_toolkit._cdf_tk.tk_warnings import ( - ToolkitNotSupportedWarning, - ToolkitWarning, - WarningList, -) -from cognite_toolkit._cdf_tk.tk_warnings.fileread import ( - UnknownResourceTypeWarning, -) -from cognite_toolkit._cdf_tk.utils import ( - humanize_collection, -) - - -class Builder(ABC): - _resource_folder: ClassVar[str | None] = None - - def __init__( - self, - build_dir: Path | None, - resource_folder: str | None = None, - warn: Callable[[ToolkitWarning], None] | None = None, - ): - self._build_dir = build_dir - self.warn = warn - self.resource_counter = 0 - if self._resource_folder is not None: - self.resource_folder = self._resource_folder - elif resource_folder is not None: - self.resource_folder = resource_folder - else: - raise ValueError("Either _resource_folder or resource_folder must be set.") - - @property - def build_dir(self) -> Path: - if self._build_dir is None: - raise ValueError("build_dir must be set for this operation.") - return self._build_dir - - @abstractmethod - def build( - self, source_files: list[BuildSourceFile], module: ModuleLocation, console: Callable[[str], None] | None = None - ) -> Iterable[BuildDestinationFile | Sequence[ToolkitWarning]]: - raise NotImplementedError() - - def load_extra_field(self, extra: str) -> tuple[str, Any]: - """Overload in subclass to load extra fields from a file.""" - raise NotImplementedError( - f"Extra field {extra!r} by {type(self).__name__} - {self.resource_folder} is not supported." - ) - - def validate_directory( - self, built_resources: BuiltResourceList, module: ModuleLocation - ) -> WarningList[ToolkitWarning]: - """This can be overridden to add additional validation for the built resources.""" - return WarningList[ToolkitWarning]() - - # Helper methods - def _create_destination_path(self, source_path: Path, kind: str) -> Path: - """Creates the filepath in the build directory for the given source path. - - Note that this is a complex operation as the modules in the source are nested while the build directory is flat. - This means that we lose information and risk having duplicate filenames. To avoid this, we prefix the filename - with a number to ensure uniqueness. - """ - filestem = source_path.stem - # Get rid of the local index - filestem = INDEX_PATTERN.sub("", filestem) - - # Increment to ensure we do not get duplicate filenames when we flatten the file - # structure from the module to the build directory. - self.resource_counter += 1 - - filename = f"{self.resource_counter}.{filestem}" - if not filename.casefold().endswith(kind.casefold()): - filename = f"{filename}.{kind}" - filename = f"{filename}{source_path.suffix}" - destination_path = self.build_dir / self.resource_folder / filename - destination_path.parent.mkdir(parents=True, exist_ok=True) - return destination_path - - def _get_loader(self, source_path: Path) -> tuple[None, ToolkitWarning] | tuple[type[ResourceIO], None]: - return get_resource_crud(source_path, self.resource_folder) - - -def get_resource_crud( - source_path: Path, resource_folder: str -) -> tuple[None, ToolkitWarning] | tuple[type[ResourceIO], None]: - """Get the appropriate CRUD class for the given source file and resource folder.""" - folder_cruds = RESOURCE_CRUD_BY_FOLDER_NAME.get(resource_folder, []) - if not folder_cruds: - return None, ToolkitNotSupportedWarning( - f"resource of type {resource_folder!r} in {source_path.name}.", - details=f"Available resources are: {humanize_collection(RESOURCE_CRUD_BY_FOLDER_NAME.keys())}", - ) - - crud_candidates = [crud_cls for crud_cls in folder_cruds if crud_cls.is_supported_file(source_path)] - if len(crud_candidates) == 0: - suggestion: str | None = None - if "." in source_path.stem: - core, kind = source_path.stem.rsplit(".", 1) - match = difflib.get_close_matches(kind, [crud_cls.kind for crud_cls in folder_cruds]) - if match: - suggested_name = f"{core}.{match[0]}{source_path.suffix}" - suggestion = f"Did you mean to call the file {suggested_name!r}?" - else: - kinds = [crud.kind for crud in folder_cruds] - if len(kinds) == 1: - suggestion = f"Did you mean to call the file '{source_path.stem}.{kinds[0]}{source_path.suffix}'?" - else: - suggestion = ( - f"All files in the {resource_folder!r} folder must have a file extension that matches " - f"the resource type. Supported types are: {humanize_collection(kinds)}." - ) - return None, UnknownResourceTypeWarning(source_path, suggestion) - elif len(crud_candidates) > 1 and all(issubclass(loader, GroupIO) for loader in crud_candidates): - # There are two group cruds, one for resource scoped and one for all scoped. - return GroupIO, None - elif len(crud_candidates) == 1: - return crud_candidates[0], None - - # This is unreachable with our current ResourceCRUD classes. We have tests that is exhaustive over - # all ResourceCRUDs to ensure this. - names = humanize_collection( - [f"'{source_path.stem}.{loader.kind}{source_path.suffix}'" for loader in crud_candidates], bind_word="or" - ) - raise AmbiguousResourceFileError( - f"Ambiguous resource file {source_path.name} in {resource_folder} folder. " - f"Unclear whether it is {humanize_collection([crud_cls.kind for crud_cls in crud_candidates], bind_word='or')}." - f"\nPlease name the file {names}." - ) - - -class DefaultBuilder(Builder): - """This is used to build resources that do not have a specific builder.""" - - def build( - self, source_files: list[BuildSourceFile], module: ModuleLocation, console: Callable[[str], None] | None = None - ) -> Iterable[BuildDestinationFile | list[ToolkitWarning]]: - for source_file in source_files: - if source_file.loaded is None: - # Not a YAML file - continue - loader, warning = self._get_loader(source_file.source.path) - if loader is None: - if warning is not None: - yield [warning] - continue - destination_path = self._create_destination_path(source_file.source.path, loader.kind) - - destination = BuildDestinationFile( - path=destination_path, - loaded=source_file.loaded, - loader=loader, - source=source_file.source, - extra_sources=None, - ) - yield destination diff --git a/cognite_toolkit/_cdf_tk/builders/_datamodels.py b/cognite_toolkit/_cdf_tk/builders/_datamodels.py deleted file mode 100644 index 7b0f7e4b75..0000000000 --- a/cognite_toolkit/_cdf_tk/builders/_datamodels.py +++ /dev/null @@ -1,90 +0,0 @@ -import shutil -from collections.abc import Callable, Iterable -from pathlib import Path -from typing import Any - -from cognite_toolkit._cdf_tk.builders import Builder -from cognite_toolkit._cdf_tk.constants import INDEX_PATTERN -from cognite_toolkit._cdf_tk.data_classes import ( - BuildDestinationFile, - BuildSourceFile, - ModuleLocation, - SourceLocation, -) -from cognite_toolkit._cdf_tk.exceptions import ToolkitFileNotFoundError -from cognite_toolkit._cdf_tk.resource_ios import GraphQLCRUD -from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning - - -class DataModelBuilder(Builder): - _resource_folder = GraphQLCRUD.folder_name - - def build( - self, source_files: list[BuildSourceFile], module: ModuleLocation, console: Callable[[str], None] | None = None - ) -> Iterable[BuildDestinationFile | list[ToolkitWarning]]: - graphql_files = { - source_file.source.path: source_file - for source_file in source_files - if source_file.source.path.suffix == ".graphql" - } - - for source_file in source_files: - loaded = source_file.loaded - if loaded is None: - # Not a YAML file - continue - loader, warning = self._get_loader(source_file.source.path) - if loader is None: - if warning is not None: - yield [warning] - continue - - destination_path = self._create_destination_path(source_file.source.path, loader.kind) - - extra_sources: list[SourceLocation] | None = None - if loader is GraphQLCRUD: - # The GraphQL must be copied over instead of added to the DML field as - # it is hashed in the deployment step and used to determine if the DML has changed. - extra_sources = self._copy_graphql_to_build(source_file, destination_path, graphql_files) - - destination = BuildDestinationFile( - path=destination_path, - loaded=loaded, - loader=loader, - source=source_file.source, - extra_sources=extra_sources, - ) - yield destination - - def _copy_graphql_to_build( - self, - source_file: BuildSourceFile, - destination_path: Path, - graphql_files: dict[Path, BuildSourceFile], - ) -> list[SourceLocation]: - extra_sources: list[SourceLocation] = [] - loaded_list: list[dict[str, Any]] = ( - source_file.loaded if isinstance(source_file.loaded, list) else [source_file.loaded] # type: ignore[list-item] - ) - - for entry in loaded_list: - if "dml" in entry: - expected_filename = entry["dml"] - else: - expected_filename = f"{INDEX_PATTERN.sub('', source_file.source.path.stem.removesuffix(GraphQLCRUD.kind).removesuffix('.'))}.graphql" - expected_path = source_file.source.path.parent / Path(expected_filename) - - if expected_path in graphql_files: - dest_graphql = destination_path.with_suffix(".graphql") - shutil.copy(graphql_files[expected_path].source.path, dest_graphql) - extra_sources.append(graphql_files[expected_path].source) - # The build renames the .graphql file; update dml so deploy can locate it. - entry["dml"] = dest_graphql.name - else: - raise ToolkitFileNotFoundError( - f"Failed to find GraphQL file. Expected {expected_filename} adjacent to {source_file.source.path.as_posix()}" - ) - return extra_sources - - def load_extra_field(self, extra: str) -> tuple[str, Any]: - return "dml", extra diff --git a/cognite_toolkit/_cdf_tk/builders/_file.py b/cognite_toolkit/_cdf_tk/builders/_file.py deleted file mode 100644 index a074ec9b6a..0000000000 --- a/cognite_toolkit/_cdf_tk/builders/_file.py +++ /dev/null @@ -1,88 +0,0 @@ -import copy -from collections.abc import Callable, Iterable -from typing import Any - -from cognite_toolkit._cdf_tk.builders import Builder -from cognite_toolkit._cdf_tk.data_classes import ( - BuildDestinationFile, - BuildSourceFile, - ModuleLocation, -) -from cognite_toolkit._cdf_tk.exceptions import ToolkitYAMLFormatError -from cognite_toolkit._cdf_tk.resource_ios import CogniteFileCRUD, FileCRUD, FileMetadataCRUD -from cognite_toolkit._cdf_tk.tk_warnings import LowSeverityWarning, ToolkitWarning - - -class FileBuilder(Builder): - _resource_folder = FileMetadataCRUD.folder_name - template_pattern = "$FILENAME" - - def build( - self, source_files: list[BuildSourceFile], module: ModuleLocation, console: Callable[[str], None] | None = None - ) -> Iterable[BuildDestinationFile | list[ToolkitWarning]]: - for source_file in source_files: - loaded = source_file.loaded - if loaded is None: - continue - - loader, warning = self._get_loader(source_file.source.path) - if loader is None: - if warning is not None: - yield [warning] - continue - if loader in {FileMetadataCRUD, CogniteFileCRUD}: - loaded = self._expand_file_metadata(loaded, module, console) - destination_path = self._create_destination_path(source_file.source.path, loader.kind) - - yield BuildDestinationFile( - path=destination_path, - loaded=loaded, - loader=loader, - source=source_file.source, - extra_sources=None, - ) - - @classmethod - def _expand_file_metadata( - cls, - raw_list: list[dict[str, Any]] | dict[str, Any], - module: ModuleLocation, - console: Callable[[str], None] | None = None, - ) -> list[dict[str, Any]] | dict[str, Any]: - is_file_template = ( - isinstance(raw_list, list) - and len(raw_list) == 1 - and cls.template_pattern in raw_list[0].get("externalId", "") - ) - if not is_file_template: - if (isinstance(raw_list, dict) and cls.template_pattern in raw_list.get("externalId", "")) or ( - isinstance(raw_list, list) - and any(cls.template_pattern in entry.get("externalId", "") for entry in raw_list) - ): - raw_type = "dictionary" if isinstance(raw_list, dict) else "list with multiple entries" - LowSeverityWarning( - f"Invalid file template {cls.template_pattern!r} usage detected in {module.relative_path.as_posix()!r}.\n" - f"The file template is expected in a list with a single entry, but got {raw_type}." - ).print_warning() - - return raw_list - if not (isinstance(raw_list, list) and raw_list and isinstance(raw_list[0], dict)): - raise ToolkitYAMLFormatError( - f"Expected a list with a single dictionary in the file metadata file {module.dir}, " - f"but got {type(raw_list).__name__}" - ) - template = raw_list[0] - if console: - console( - f"Detected file template name {cls.template_pattern!r} in {module.relative_path.as_posix()!r}" - f"Expanding file metadata..." - ) - expanded_metadata: list[dict[str, Any]] = [] - for filepath in module.source_paths_by_resource_folder[FileCRUD.folder_name]: - if not FileCRUD.is_supported_file(filepath): - continue - new_entry = copy.deepcopy(template) - new_entry["externalId"] = new_entry["externalId"].replace(cls.template_pattern, filepath.name) - new_entry["name"] = filepath.name - expanded_metadata.append(new_entry) - return expanded_metadata diff --git a/cognite_toolkit/_cdf_tk/builders/_function.py b/cognite_toolkit/_cdf_tk/builders/_function.py deleted file mode 100644 index cf1ee86576..0000000000 --- a/cognite_toolkit/_cdf_tk/builders/_function.py +++ /dev/null @@ -1,187 +0,0 @@ -import shutil -import time -from collections.abc import Callable, Iterable, Sequence -from pathlib import Path -from typing import Any - -from cognite_toolkit._cdf_tk.builders import Builder -from cognite_toolkit._cdf_tk.data_classes import ( - BuildDestinationFile, - BuildSourceFile, - BuiltResourceList, - ModuleLocation, -) -from cognite_toolkit._cdf_tk.exceptions import ToolkitFileExistsError, ToolkitNotADirectoryError, ToolkitValueError -from cognite_toolkit._cdf_tk.feature_flags import Flags -from cognite_toolkit._cdf_tk.resource_ios import FunctionIO -from cognite_toolkit._cdf_tk.tk_warnings import ( - FileReadWarning, - HighSeverityWarning, - LowSeverityWarning, - MediumSeverityWarning, - RequirementsTXTValidationWarning, - ToolkitWarning, - WarningList, -) -from cognite_toolkit._cdf_tk.utils import validate_requirements_with_pip - - -class FunctionBuilder(Builder): - _resource_folder = FunctionIO.folder_name - - def __init__(self, build_dir: Path, warn: Callable[[ToolkitWarning], None]) -> None: - super().__init__(build_dir, warn=warn) - # Metrics for telemetry - self.validation_count = 0 - self.validation_failures = 0 - self.validation_credential_errors = 0 - self.validation_time_ms = 0 - - def _validate_function_requirements( - self, - requirements_txt: Path, - raw_function: dict[str, Any], - filepath: Path, - external_id: str, - ) -> RequirementsTXTValidationWarning | None: - """Validate function requirements.txt using pip dry-run.""" - start_time = time.time() - validation_result = validate_requirements_with_pip( - requirements_txt_path=requirements_txt, - index_url=raw_function.get("indexUrl"), - extra_index_urls=raw_function.get("extraIndexUrls"), - ) - elapsed_ms = int((time.time() - start_time) * 1000) - self.validation_count += 1 - self.validation_time_ms += elapsed_ms - - if validation_result.success: - return None - - self.validation_failures += 1 - if validation_result.is_credential_error: - self.validation_credential_errors += 1 - - return RequirementsTXTValidationWarning( - filepath=filepath, - external_id=external_id, - error_details=validation_result.short_error, - is_credential_error=validation_result.is_credential_error, - resource="function", - ) - - def build( - self, - source_files: list[BuildSourceFile], - module: ModuleLocation, - console: Callable[[str], None] | None = None, - ) -> Iterable[BuildDestinationFile | Sequence[ToolkitWarning]]: - for source_file in source_files: - if source_file.loaded is None: - continue - if source_file.source.path.parent.parent != module.dir: - # Function YAML files must be in the resource folder. - continue - - loader, warning = self._get_loader(source_file.source.path) - if loader is None: - if warning is not None: - yield [warning] - continue - - warnings = WarningList[FileReadWarning]() - if loader is FunctionIO: - warnings = self.copy_function_directory_to_build(source_file) - - destination_path = self._create_destination_path(source_file.source.path, loader.kind) - - yield BuildDestinationFile( - path=destination_path, - loaded=source_file.loaded, - loader=loader, - source=source_file.source, - extra_sources=None, - warnings=warnings, - ) - - def validate_directory( - self, - built_resources: BuiltResourceList, - module: ModuleLocation, - ) -> WarningList[ToolkitWarning]: - warnings = WarningList[ToolkitWarning]() - has_config_files = any(resource.kind == FunctionIO.kind for resource in built_resources) - if has_config_files: - return warnings - config_files_misplaced = [ - file - for file in module.source_paths_by_resource_folder[FunctionIO.folder_name] - if FunctionIO.is_supported_file(file) - ] - if config_files_misplaced: # and not has_config_files: - for yaml_source_path in config_files_misplaced: - required_location = module.dir / FunctionIO.folder_name / yaml_source_path.name - warning = LowSeverityWarning( - f"The required Function resource configuration file " - f"was not found in {required_location.as_posix()!r}. " - f"The file {yaml_source_path.as_posix()!r} is currently " - f"considered part of the Function's artifacts and " - f"will not be processed by the Toolkit.", - ) - warnings.append(warning) - return warnings - - def copy_function_directory_to_build(self, source_file: BuildSourceFile) -> WarningList[FileReadWarning]: - raw_content = source_file.loaded - if raw_content is None: - # This should already be checked before calling this method. - raise ToolkitValueError("Function source file should be a YAML file.") - raw_functions = raw_content if isinstance(raw_content, list) else [raw_content] - warnings = WarningList[FileReadWarning]() - for raw_function in raw_functions: - external_id = raw_function.get("externalId") - function_path = raw_function.get("functionPath") - if not external_id: - warnings.append( - HighSeverityWarning( - f"Function in {source_file.source.path.as_posix()!r} has no externalId defined. " - f"This is used to match the function to the function directory.", - ), - ) - continue - if not function_path: - warnings.append( - MediumSeverityWarning( - f"Function {external_id} in {source_file.source.path.as_posix()!r} has no function_path defined.", - ), - ) - - function_directory = source_file.source.path.with_name(external_id) - - if not function_directory.is_dir(): - raise ToolkitNotADirectoryError( - f"Function directory not found for externalId {external_id} defined in {source_file.source.path.as_posix()!r}.", - ) - - # Validate requirements.txt if present and feature is enabled - if ( - Flags.FUNCTION_REQUIREMENTS_VALIDATION.is_enabled() - and (requirements_txt := function_directory / "requirements.txt").exists() - ): - warning = self._validate_function_requirements( - requirements_txt, - raw_function, - source_file.source.path, - external_id, - ) - if warning: - warnings.append(warning) - - destination = self.build_dir / self.resource_folder / external_id - if destination.exists(): - raise ToolkitFileExistsError( - f"Function {external_id!r} is duplicated. If this is unexpected, ensure you have a clean build directory.", - ) - shutil.copytree(function_directory, destination, ignore=shutil.ignore_patterns("__pycache__")) - - return warnings diff --git a/cognite_toolkit/_cdf_tk/builders/_location.py b/cognite_toolkit/_cdf_tk/builders/_location.py deleted file mode 100644 index ca4c7bbf1a..0000000000 --- a/cognite_toolkit/_cdf_tk/builders/_location.py +++ /dev/null @@ -1,84 +0,0 @@ -from collections.abc import Callable, Iterable, Sequence -from graphlib import CycleError, TopologicalSorter -from typing import Any - -from cognite_toolkit._cdf_tk.builders._base import Builder -from cognite_toolkit._cdf_tk.data_classes._build_files import BuildDestinationFile, BuildSourceFile -from cognite_toolkit._cdf_tk.data_classes._module_directories import ModuleLocation -from cognite_toolkit._cdf_tk.exceptions import ToolkitError -from cognite_toolkit._cdf_tk.resource_ios._resource_ios.location import LocationFilterIO -from cognite_toolkit._cdf_tk.tk_warnings.base import ToolkitWarning, WarningList -from cognite_toolkit._cdf_tk.tk_warnings.fileread import FileReadWarning - - -class LocationBuilder(Builder): - _resource_folder = LocationFilterIO.folder_name - - def build( - self, source_files: list[BuildSourceFile], module: ModuleLocation, console: Callable[[str], None] | None = None - ) -> Iterable[BuildDestinationFile | Sequence[ToolkitWarning]]: - location_by_external_id: dict[str, tuple[dict[str, Any], BuildSourceFile]] = {} - location_hierarchy_graph: dict[str, list[Any]] = {} - - # Ordering all location filters in to ensure correct hierarchy dependency - # within the module. This is required by the Location API. - # Doing this in three stages: - # 1. collect all locations across source files, - # 2. sort them in a topological order, - # 3. create a new file for each location where the prefix index ensures deployment order - # ... while also maintaining reference to source file - - for source_file in source_files: - loader, warning = self._get_loader(source_file.source.path) - if isinstance(loader, LocationFilterIO): - if warning is not None: - yield [warning] - continue - - loaded_locations = ( - source_file.loaded - if isinstance(source_file.loaded, list) - else [source_file.loaded] - if source_file.loaded - else [] - ) - for loaded_location in loaded_locations: - ext_id = loaded_location.get("externalId") - parent_external_id = loaded_location.get("parentExternalId") - - if ext_id: - location_by_external_id[ext_id] = loaded_location, source_file - location_hierarchy_graph.setdefault(ext_id, []) - - if parent_external_id: - location_hierarchy_graph.setdefault(parent_external_id, []) - location_hierarchy_graph[ext_id].append(parent_external_id) - - warnings = WarningList[FileReadWarning]() - - ordered_locations: list[dict] = [] - try: - for external_id in TopologicalSorter(location_hierarchy_graph).static_order(): - if external_id not in location_by_external_id: - # The dependency is not in the module, so we skip it. - continue - location, _ = location_by_external_id[external_id] - ordered_locations.append(location) - except CycleError: - raise ToolkitError( - "Circular dependency found in Locations. Locations must be hierarchical. Please check the externalId and parentExternalId fields." - ) - - for item in ordered_locations: - external_id = item["externalId"] - (location, build_source_file) = location_by_external_id[external_id] - destination_path = self._create_destination_path(build_source_file.source.path, loader.kind) # type: ignore[union-attr] - - yield BuildDestinationFile( - path=destination_path, - loaded=location, - loader=loader, # type: ignore[arg-type] - source=build_source_file.source, - extra_sources=None, - warnings=warnings, - ) diff --git a/cognite_toolkit/_cdf_tk/builders/_raw.py b/cognite_toolkit/_cdf_tk/builders/_raw.py deleted file mode 100644 index 012cc248b2..0000000000 --- a/cognite_toolkit/_cdf_tk/builders/_raw.py +++ /dev/null @@ -1,72 +0,0 @@ -from collections import defaultdict -from collections.abc import Callable, Iterable, Sequence -from typing import Any - -from cognite_toolkit._cdf_tk.builders import Builder -from cognite_toolkit._cdf_tk.client.identifiers import RawDatabaseId -from cognite_toolkit._cdf_tk.data_classes import ( - BuildDestinationFile, - BuildSourceFile, - ModuleLocation, - SourceLocation, - SourceLocationEager, -) -from cognite_toolkit._cdf_tk.resource_ios import RawDatabaseCRUD, RawTableCRUD, ResourceIO -from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning -from cognite_toolkit._cdf_tk.utils import calculate_hash -from cognite_toolkit._cdf_tk.utils.file import yaml_safe_dump - - -class RawBuilder(Builder): - _resource_folder = RawDatabaseCRUD.folder_name - - def build( - self, source_files: list[BuildSourceFile], module: ModuleLocation, console: Callable[[str], None] | None = None - ) -> Iterable[BuildDestinationFile | Sequence[ToolkitWarning]]: - for source_file in source_files: - loaded = source_file.loaded - if loaded is None: - continue - loaded_list = loaded if isinstance(loaded, list) else [loaded] - seen_databases: set[tuple] = set() - entry_by_loader: dict[type[ResourceIO], list[dict[str, Any]]] = defaultdict(list) - has_split_table_and_database = False - - for item in loaded_list: - try: - table_id = RawTableCRUD.get_id(item) - except KeyError: - seen_databases.add(tuple(item.items())) - entry_by_loader[RawDatabaseCRUD].append(item) - else: - entry_by_loader[RawTableCRUD].append(item) - db_item = RawDatabaseCRUD.dump_id(RawDatabaseId(name=table_id.db_name)) - hashable_db_item = tuple(db_item.items()) - if hashable_db_item not in seen_databases: - seen_databases.add(hashable_db_item) - entry_by_loader[RawDatabaseCRUD].append(db_item) - has_split_table_and_database = True - - for loader, entries in entry_by_loader.items(): - if not entries: - continue - destination_path = self._create_destination_path(source_file.source.path, loader.kind) - - if loader is RawDatabaseCRUD and has_split_table_and_database: - # We have inferred the database from a Table file, so we need to recalculate the hash - # in case we also inferred the database from another Table file - new_hash = calculate_hash( - yaml_safe_dump(sorted(entries, key=lambda entry: entry["dbName"])), - shorten=True, - ) - source: SourceLocation = SourceLocationEager(path=source_file.source.path, _hash=new_hash) - else: - source = source_file.source - - yield BuildDestinationFile( - path=destination_path, - loaded=entries, - loader=loader, - source=source, - extra_sources=None, - ) diff --git a/cognite_toolkit/_cdf_tk/builders/_ruleset.py b/cognite_toolkit/_cdf_tk/builders/_ruleset.py deleted file mode 100644 index 8572c34bd2..0000000000 --- a/cognite_toolkit/_cdf_tk/builders/_ruleset.py +++ /dev/null @@ -1,105 +0,0 @@ -from collections.abc import Callable, Iterable -from pathlib import Path -from typing import Any - -from cognite_toolkit._cdf_tk.builders import Builder -from cognite_toolkit._cdf_tk.constants import BUILD_FOLDER_ENCODING -from cognite_toolkit._cdf_tk.data_classes import ( - BuildDestinationFile, - BuildSourceFile, - ModuleLocation, - SourceLocation, -) -from cognite_toolkit._cdf_tk.exceptions import ToolkitFileNotFoundError, ToolkitYAMLFormatError -from cognite_toolkit._cdf_tk.resource_ios import RuleSetVersionIO -from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning -from cognite_toolkit._cdf_tk.utils import safe_write - - -class RuleSetBuilder(Builder): - _resource_folder = RuleSetVersionIO.folder_name - - def build( - self, source_files: list[BuildSourceFile], module: ModuleLocation, console: Callable[[str], None] | None = None - ) -> Iterable[BuildDestinationFile | list[ToolkitWarning]]: - ttl_files = { - source_file.source.path: source_file - for source_file in source_files - if source_file.source.path.suffix == ".ttl" - } - - for source_file in source_files: - loaded = source_file.loaded - if loaded is None: - continue - loader, warning = self._get_loader(source_file.source.path) - if loader is None: - if warning is not None: - yield [warning] - continue - - destination_path = self._create_destination_path(source_file.source.path, loader.kind) - - extra_sources: list[SourceLocation] | None = None - if loader is RuleSetVersionIO: - extra_sources = self._add_rules(loaded, source_file, ttl_files, destination_path) - - destination = BuildDestinationFile( - path=destination_path, - loaded=loaded, - loader=loader, - source=source_file.source, - extra_sources=extra_sources, - ) - yield destination - - def load_extra_field(self, extra: str) -> tuple[str, Any]: - return "rules", [extra] - - def _add_rules( - self, - loaded: dict[str, Any] | list[dict[str, Any]], - source_file: BuildSourceFile, - ttl_files: dict[Path, BuildSourceFile], - ruleset_destination_path: Path, - ) -> list[SourceLocation]: - loaded_list = loaded if isinstance(loaded, list) else [loaded] - extra_sources: list[SourceLocation] = [] - for entry in loaded_list: - try: - id_ = RuleSetVersionIO.get_id(entry) - except KeyError: - continue - filepath = source_file.source.path - ttl_file = self._get_ttl_file(filepath, id_.rule_set_external_id, ttl_files) - - if "rules" in entry and ttl_file is not None: - raise ToolkitYAMLFormatError( - f"'rules' is defined in both the YAML and a separate file named {ttl_file.source.path}\n" - f"Please remove one: either the inline 'rules' in {filepath} or the file {ttl_file.source.path}", - ) - if "rules" not in entry and ttl_file is None: - raise ToolkitFileNotFoundError( - f"'rules' is missing and no .ttl file found. Expected {filepath.stem}.ttl or {id_.rule_set_external_id}.ttl next to {filepath}", - filepath, - ) - if ttl_file is not None: - destination_path = self._create_destination_path(ttl_file.source.path, "Rules") - safe_write(destination_path, ttl_file.content, encoding=BUILD_FOLDER_ENCODING) - entry["rules"] = [ttl_file.content] - extra_sources.append(ttl_file.source) - - return extra_sources - - @staticmethod - def _get_ttl_file( - source_file: Path, rule_set_external_id: str | None, ttl_files: dict[Path, BuildSourceFile] - ) -> BuildSourceFile | None: - ttl_path = source_file.parent / f"{source_file.stem}.ttl" - if ttl_path in ttl_files: - return ttl_files[ttl_path] - if rule_set_external_id: - ttl_path = source_file.parent / f"{rule_set_external_id}.ttl" - if ttl_path in ttl_files: - return ttl_files[ttl_path] - return None diff --git a/cognite_toolkit/_cdf_tk/builders/_streamlit.py b/cognite_toolkit/_cdf_tk/builders/_streamlit.py deleted file mode 100644 index 66571aab5f..0000000000 --- a/cognite_toolkit/_cdf_tk/builders/_streamlit.py +++ /dev/null @@ -1,110 +0,0 @@ -import shutil -from collections.abc import Callable, Iterable, Sequence - -from cognite_toolkit._cdf_tk.builders import Builder -from cognite_toolkit._cdf_tk.data_classes import ( - BuildDestinationFile, - BuildSourceFile, - ModuleLocation, -) -from cognite_toolkit._cdf_tk.exceptions import ToolkitFileExistsError, ToolkitNotADirectoryError, ToolkitValueError -from cognite_toolkit._cdf_tk.feature_flags import Flags -from cognite_toolkit._cdf_tk.resource_ios import StreamlitIO -from cognite_toolkit._cdf_tk.tk_warnings import ( - FileReadWarning, - HighSeverityWarning, - RequirementsTXTValidationWarning, - StreamlitRequirementsWarning, - ToolkitWarning, - WarningList, -) -from cognite_toolkit._cdf_tk.utils import validate_requirements_with_pip -from cognite_toolkit._cdf_tk.utils.file import safe_read - - -class StreamlitBuilder(Builder): - _resource_folder = StreamlitIO.folder_name - - def build( - self, source_files: list[BuildSourceFile], module: ModuleLocation, console: Callable[[str], None] | None = None - ) -> Iterable[BuildDestinationFile | Sequence[ToolkitWarning]]: - for source_file in source_files: - if source_file.loaded is None: - continue - if source_file.source.path.parent.parent != module.dir: - # Streamlit YAML files must be in the resource folder top level - continue - - loader, warning = self._get_loader(source_file.source.path) - if loader is None: - if warning is not None: - yield [warning] - continue - - warnings = WarningList[FileReadWarning]() - if loader is StreamlitIO: - warnings = self.copy_app_directory_to_build(source_file) - - destination_path = self._create_destination_path(source_file.source.path, loader.kind) - - yield BuildDestinationFile( - path=destination_path, - loaded=source_file.loaded, - loader=loader, - source=source_file.source, - extra_sources=None, - warnings=warnings, - ) - - def copy_app_directory_to_build(self, source_file: BuildSourceFile) -> WarningList[FileReadWarning]: - raw_content = source_file.loaded - if raw_content is None: - # This should already be checked before calling this method. - raise ToolkitValueError("Streamlit source file should be a YAML file.") - raw_apps = raw_content if isinstance(raw_content, list) else [raw_content] - warnings = WarningList[FileReadWarning]() - for raw_app in raw_apps: - external_id = raw_app.get("externalId") - if not external_id: - warnings.append( - HighSeverityWarning( - f"StreamlitApp in {source_file.source.path.as_posix()!r} has no externalId defined. " - f"This is used to match the Streamlit App to directory." - ) - ) - continue - app_directory = source_file.source.path.with_name(external_id) - - if not app_directory.is_dir(): - raise ToolkitNotADirectoryError( - f"StreamlitApp directory not found in {app_directory}(based on externalId {external_id} defined in {source_file.source.path.as_posix()!r}.)" - ) - - if (requirements_txt := app_directory / "requirements.txt").exists() and ( - Flags.FUNCTION_REQUIREMENTS_VALIDATION.is_enabled() - ): - validation_result = validate_requirements_with_pip(requirements_txt_path=requirements_txt) - if not validation_result.success: - warnings.append( - RequirementsTXTValidationWarning( - filepath=source_file.source.path, - error_details=validation_result.short_error, - is_credential_error=validation_result.is_credential_error, - external_id=external_id, - resource="streamlit", - ) - ) - - requirements_file_content = safe_read(app_directory / "requirements.txt").splitlines() - missing_packages = StreamlitIO._missing_recommended_requirements(requirements_file_content) - if len(missing_packages) > 0: - warnings.append(StreamlitRequirementsWarning(app_directory / "requirements.txt", missing_packages)) - - destination = self.build_dir / self.resource_folder / external_id - if destination.exists(): - raise ToolkitFileExistsError( - f"StreamlitApp {external_id!r} is duplicated. If this is unexpected, ensure you have a clean build directory." - ) - shutil.copytree(app_directory, destination, ignore=shutil.ignore_patterns("__pycache__")) - - return warnings diff --git a/cognite_toolkit/_cdf_tk/builders/_transformation.py b/cognite_toolkit/_cdf_tk/builders/_transformation.py deleted file mode 100644 index 1132bd7e36..0000000000 --- a/cognite_toolkit/_cdf_tk/builders/_transformation.py +++ /dev/null @@ -1,111 +0,0 @@ -from collections.abc import Callable, Iterable -from pathlib import Path -from typing import Any - -from cognite_toolkit._cdf_tk.builders import Builder -from cognite_toolkit._cdf_tk.constants import BUILD_FOLDER_ENCODING -from cognite_toolkit._cdf_tk.data_classes import ( - BuildDestinationFile, - BuildSourceFile, - ModuleLocation, - SourceLocation, -) -from cognite_toolkit._cdf_tk.exceptions import ToolkitYAMLFormatError -from cognite_toolkit._cdf_tk.resource_ios import TransformationIO -from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, ToolkitWarning -from cognite_toolkit._cdf_tk.utils import safe_write - - -class TransformationBuilder(Builder): - _resource_folder = TransformationIO.folder_name - - def build( - self, source_files: list[BuildSourceFile], module: ModuleLocation, console: Callable[[str], None] | None = None - ) -> Iterable[BuildDestinationFile | list[ToolkitWarning]]: - query_files = { - source_file.source.path: source_file - for source_file in source_files - if source_file.source.path.suffix == ".sql" - } - - for source_file in source_files: - loaded = source_file.loaded - if loaded is None: - # Not a YAML file - continue - loader, warning = self._get_loader(source_file.source.path) - if loader is None: - if warning is not None: - yield [warning] - continue - - destination_path = self._create_destination_path(source_file.source.path, loader.kind) - - extra_sources: list[SourceLocation] | None = None - if loader is TransformationIO: - extra_sources = self._add_query(loaded, source_file, query_files, destination_path) - - destination = BuildDestinationFile( - path=destination_path, - loaded=loaded, - loader=loader, - source=source_file.source, - extra_sources=extra_sources, - ) - yield destination - - def load_extra_field(self, extra: str) -> tuple[str, Any]: - return "query", extra - - def _add_query( - self, - loaded: dict[str, Any] | list[dict[str, Any]], - source_file: BuildSourceFile, - query_files: dict[Path, BuildSourceFile], - transformation_destination_path: Path, - ) -> list[SourceLocation]: - loaded_list = loaded if isinstance(loaded, list) else [loaded] - extra_sources: list[SourceLocation] = [] - for entry in loaded_list: - try: - id_ = TransformationIO.get_id(entry) - except KeyError: - # This will be validated later - continue - filepath = source_file.source.path - query_file = self._get_query_file(filepath, id_.external_id, query_files) - - if "query" in entry and query_file is not None: - raise ToolkitYAMLFormatError( - f"query property is ambiguously defined in both the yaml file and a separate file named {query_file}\n" - f"Please remove one of the definitions, either the query property in {filepath} or the file {query_file}", - ) - elif "query" not in entry and query_file is None: - warning = HighSeverityWarning( - f"query property or is missing in {filepath.as_posix()!r}. It can be inline or a separate file named {filepath.stem}.sql or {id_}.sql", - ) - if self.warn: - self.warn(warning) - else: - warning.print_warning() - elif query_file is not None: - destination_path = self._create_destination_path(query_file.source.path, "Query") - safe_write(destination_path, query_file.content, encoding=BUILD_FOLDER_ENCODING) - relative = destination_path.relative_to(transformation_destination_path.parent) - entry["queryFile"] = relative.as_posix() - extra_sources.append(query_file.source) - - return extra_sources - - @staticmethod - def _get_query_file( - source_file: Path, transformation_external_id: str | None, query_files: dict[Path, BuildSourceFile] - ) -> BuildSourceFile | None: - query_file = source_file.parent / f"{source_file.stem}.sql" - if query_file in query_files: - return query_files[query_file] - if transformation_external_id: - query_file = source_file.parent / f"{transformation_external_id}.sql" - if query_file in query_files: - return query_files[query_file] - return None diff --git a/tests/test_unit/test_cdf_tk/test_cruds/test_data_model.py b/tests/test_unit/test_cdf_tk/test_cruds/test_data_model.py index 71f4e99517..eba0cc76dc 100644 --- a/tests/test_unit/test_cdf_tk/test_cruds/test_data_model.py +++ b/tests/test_unit/test_cdf_tk/test_cruds/test_data_model.py @@ -711,49 +711,3 @@ def test_dml_compile_error_surfaced_as_actionable_message(self) -> None: ) with pytest.raises(ToolkitAPIError, match="Type 'Foo' not found"): self._make_api(body)._post_graphql({"query": "...", "variables": {}}) - - -class TestDataModelBuilder: - """Regression tests for DataModelBuilder (build v1).""" - - def test_dml_updated_to_renamed_graphql_in_build(self, tmp_path: Path) -> None: - # Regression test: build renames .graphql files with a long prefix, but deploy - # looks up the file via entry["dml"]. _copy_graphql_to_build must update "dml" - # so that deploy finds the renamed file instead of the original source name. - from cognite_toolkit._cdf_tk.builders._datamodels import DataModelBuilder - from cognite_toolkit._cdf_tk.data_classes._build_files import BuildSourceFile - from cognite_toolkit._cdf_tk.data_classes._built_resources import SourceLocationEager - - source_dir = tmp_path / "source" / "data_modeling" - source_dir.mkdir(parents=True) - build_dir = tmp_path / "build" - build_dir.mkdir() - - yaml_path = source_dir / "my_model.GraphQLSchema.yaml" - graphql_path = source_dir / "original_schema.graphql" - yaml_path.write_text("space: my_space\nexternalId: MyModel\nversion: v1\ndml: original_schema.graphql\n") - graphql_path.write_text("type Foo { name: String }") - - entry: dict = {"space": "my_space", "externalId": "MyModel", "version": "v1", "dml": "original_schema.graphql"} - source_file = BuildSourceFile( - source=SourceLocationEager(path=yaml_path, _hash="abc"), - content=yaml_path.read_text(), - loaded=entry, - ) - graphql_source = BuildSourceFile( - source=SourceLocationEager(path=graphql_path, _hash="def"), - content=graphql_path.read_text(), - loaded=None, - ) - - builder = DataModelBuilder(build_dir=build_dir) - destination_path = build_dir / "data_modeling" / "1-my_model-SPP-COR.my_model.GraphQLSchema.yaml" - destination_path.parent.mkdir(parents=True, exist_ok=True) - - builder._copy_graphql_to_build(source_file, destination_path, {graphql_path: graphql_source}) - - # The "dml" field in the entry dict must be updated to the renamed build filename. - renamed_graphql = destination_path.with_suffix(".graphql").name - assert entry["dml"] == renamed_graphql, ( - f"entry['dml'] was not updated after build rename: got {entry['dml']!r}, expected {renamed_graphql!r}" - ) From 62c2c291c69c916c59be988853d47ea852a8256d Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 7 Sep 2026 20:16:38 +0200 Subject: [PATCH 27/31] refactor: delete data crud --- .../_cdf_tk/resource_ios/__init__.py | 4 +- .../_cdf_tk/resource_ios/_data_cruds.py | 139 +----------------- tests/test_unit/approval_client/client.py | 4 +- 3 files changed, 5 insertions(+), 142 deletions(-) diff --git a/cognite_toolkit/_cdf_tk/resource_ios/__init__.py b/cognite_toolkit/_cdf_tk/resource_ios/__init__.py index ea081a7c2b..cd7d16e135 100644 --- a/cognite_toolkit/_cdf_tk/resource_ios/__init__.py +++ b/cognite_toolkit/_cdf_tk/resource_ios/__init__.py @@ -18,7 +18,7 @@ from cognite_toolkit._cdf_tk.feature_flags import FeatureFlag, Flags from ._base_ios import DataCRUD, Loader, ResourceContainerIO, ResourceIO -from ._data_cruds import DatapointsCRUD, FileCRUD, RawFileCRUD +from ._data_cruds import RawFileCRUD from ._resource_ios import ( AgentIO, AppIO, @@ -222,13 +222,11 @@ def get_crud(resource_dir: str, kind: str) -> type[Loader]: "DataProductVersionIO", "DataSetsIO", "DatapointSubscriptionIO", - "DatapointsCRUD", "EdgeCRUD", "EventIO", "ExternalDataSourceIO", "ExtractionPipelineConfigIO", "ExtractionPipelineIO", - "FileCRUD", "FileMetadataCRUD", "FunctionIO", "FunctionScheduleIO", diff --git a/cognite_toolkit/_cdf_tk/resource_ios/_data_cruds.py b/cognite_toolkit/_cdf_tk/resource_ios/_data_cruds.py index 0605aa1e41..49955a45bb 100644 --- a/cognite_toolkit/_cdf_tk/resource_ios/_data_cruds.py +++ b/cognite_toolkit/_cdf_tk/resource_ios/_data_cruds.py @@ -1,154 +1,19 @@ import io from collections.abc import Iterable -from pathlib import Path from typing import TYPE_CHECKING, cast, final import pandas as pd -from cognite.client import data_modeling as dm -from cognite_toolkit._cdf_tk.client._resource_base import T_Identifier, T_RequestResource, T_ResponseResource -from cognite_toolkit._cdf_tk.client.identifiers import ExternalId -from cognite_toolkit._cdf_tk.client.resource_classes.cognite_file import CogniteFileRequest -from cognite_toolkit._cdf_tk.client.resource_classes.data_modeling import NodeId -from cognite_toolkit._cdf_tk.client.resource_classes.filemetadata import FileMetadataRequest from cognite_toolkit._cdf_tk.client.resource_classes.raw import RAWTableResponse -from cognite_toolkit._cdf_tk.constants import BUILD_FOLDER_ENCODING -from cognite_toolkit._cdf_tk.utils import read_yaml_content, safe_read from cognite_toolkit._cdf_tk.utils.file import read_csv -from ._base_ios import DataCRUD, ResourceIO -from ._resource_ios import CogniteFileCRUD, FileMetadataCRUD, RawTableCRUD, TimeSeriesCRUD +from ._base_ios import DataCRUD +from ._resource_ios import RawTableCRUD if TYPE_CHECKING: from cognite_toolkit._cdf_tk.data_classes import BuildEnvironment -@final -class DatapointsCRUD(DataCRUD): - item_name = "datapoints" - folder_name = "timeseries" - kind = "Datapoints" - dependencies = frozenset({TimeSeriesCRUD}) - _doc_url = "Time-series/operation/postMultiTimeSeriesDatapoints" - - @property - def display_name(self) -> str: - return "timeseries datapoints" - - def upload(self, state: "BuildEnvironment", dry_run: bool) -> Iterable[tuple[str, int]]: - if self.folder_name not in state.built_resources: - return - - resource_directories = state.built_resources[self.folder_name].get_resource_directories(self.folder_name) - - for resource_dir in resource_directories: - for datafile in resource_dir.rglob("*"): - if not datafile.stem.casefold().endswith(self.kind.casefold()): - continue - if datafile.suffix == ".csv": - # The replacement is used to ensure that we read exactly the same file on Windows and Linux - file_content = datafile.read_bytes().replace(b"\r\n", b"\n").decode("utf-8") - data = read_csv(io.StringIO(file_content), parse_dates=True, index_col=0) - data.index = pd.DatetimeIndex(data.index) - elif datafile.suffix == ".parquet": - data = pd.read_parquet(datafile, engine="pyarrow") - else: - continue - timeseries_ids = list(data.columns) - if len(timeseries_ids) == 1: - ts_str = timeseries_ids[0] - elif len(timeseries_ids) <= 10: - ts_str = str(timeseries_ids) - else: - ts_str = f"{len(timeseries_ids):,} timeseries" - - if data.empty: - yield ( - f"Empty file {datafile.as_posix()!r}. No datapoints to inserted.", - 0, - ) - continue - - if dry_run: - yield ( - f" Would insert '{len(data):,}x{len(data.columns):,}' datapoints from '{datafile!s}' into {ts_str}", - len(data) * len(data.columns), - ) - else: - self.client.time_series.data.insert_dataframe(data) - - yield ( - f" Inserted '{len(data):,}x{len(data.columns):,}' datapoints from '{datafile!s}' into {ts_str}", - len(data) * len(data.columns), - ) - - -@final -class FileCRUD(DataCRUD): - item_name = "file contents" - folder_name = "files" - kind = "File" - dependencies = frozenset({FileMetadataCRUD, CogniteFileCRUD}) - _doc_url = "Files/operation/initFileUpload" - - @property - def display_name(self) -> str: - return "file content" - - def upload(self, state: "BuildEnvironment", dry_run: bool) -> Iterable[tuple[str, int]]: - if self.folder_name not in state.built_resources: - return - - for resource in state.built_resources[self.folder_name]: - if resource.destination is None: - continue - - if result := { - FileMetadataCRUD.kind: (FileMetadataCRUD, "external_id"), - CogniteFileCRUD.kind: (CogniteFileCRUD, "instance_id"), - }.get(resource.kind): - loader_cls, id_name = result - meta: FileMetadataRequest | CogniteFileRequest = self._read_metadata( - resource.destination, - loader_cls, # type: ignore[arg-type] - resource.identifier, - ) - if meta.name is None: - continue - datafile = resource.source.path.parent / meta.name - if not datafile.exists(): - continue - - identifier = resource.identifier - if isinstance(identifier, ExternalId): - identifier = identifier.external_id - elif isinstance(identifier, NodeId): - identifier = dm.NodeId(identifier.space, identifier.external_id) - if dry_run: - yield f" Would upload file '{datafile!s}' to file with {id_name}={identifier!r}", 1 - else: - self.client.files.upload_content(path=str(datafile), **{id_name: identifier}) - yield f" Uploaded file '{datafile!s}' to file with {id_name}={identifier!r}", 1 - - @staticmethod - def _read_metadata( - destination: Path, - loader: type[ResourceIO[T_Identifier, T_RequestResource, T_ResponseResource]], - identifier: T_Identifier, - ) -> T_RequestResource: - built_content = read_yaml_content(safe_read(destination, encoding=BUILD_FOLDER_ENCODING)) - if isinstance(built_content, dict): - return loader.resource_write_cls._load(built_content) - elif isinstance(built_content, list): - write_resources = (loader.resource_write_cls._load(content) for content in built_content) - try: - return next(m for m in write_resources if loader.get_id(m) == identifier) - except StopIteration: - raise RuntimeError(f"Missing metadata for {destination.as_posix()}") - - raise RuntimeError(f"Unexpected content type {type(built_content)} in {destination.as_posix()}") - - @final class RawFileCRUD(DataCRUD): item_name = "rows" diff --git a/tests/test_unit/approval_client/client.py b/tests/test_unit/approval_client/client.py index a846fb52cd..b6f84fb1ba 100644 --- a/tests/test_unit/approval_client/client.py +++ b/tests/test_unit/approval_client/client.py @@ -76,7 +76,7 @@ from cognite_toolkit._cdf_tk.client.resource_classes.streams import StreamResponse from cognite_toolkit._cdf_tk.client.testing import ToolkitClientMock from cognite_toolkit._cdf_tk.constants import INDEX_PATTERN, STREAM_IMMUTABLE_TEMPLATE_NAME -from cognite_toolkit._cdf_tk.resource_ios import FileCRUD +from cognite_toolkit._cdf_tk.dataio import FileMetadataDataIO from cognite_toolkit._cdf_tk.utils import calculate_hash from cognite_toolkit._cdf_tk.utils.auth import CLIENT_NAME from tests.constants import CDF_PROJECT @@ -739,7 +739,7 @@ def _upload_file_content_files_api( entry = instance_id.dump() entry["filehash"] = filehash - created_resources[FileCRUD.__name__].append(entry) + created_resources[FileMetadataDataIO.__name__].append(entry) return FileMetadata( external_id=external_id, From fd5066ce35581cd874fea67357d4f4b77711b779 Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 7 Sep 2026 20:20:27 +0200 Subject: [PATCH 28/31] refactor: remove remaining part of crud --- .../_cdf_tk/resource_ios/__init__.py | 9 +- .../_cdf_tk/resource_ios/_base_ios.py | 24 ---- .../_cdf_tk/resource_ios/_data_cruds.py | 84 -------------- .../test_cdf_tk/test_cruds/test_raw_file.py | 107 ------------------ 4 files changed, 2 insertions(+), 222 deletions(-) delete mode 100644 cognite_toolkit/_cdf_tk/resource_ios/_data_cruds.py delete mode 100644 tests/test_unit/test_cdf_tk/test_cruds/test_raw_file.py diff --git a/cognite_toolkit/_cdf_tk/resource_ios/__init__.py b/cognite_toolkit/_cdf_tk/resource_ios/__init__.py index cd7d16e135..5b5caa5f4c 100644 --- a/cognite_toolkit/_cdf_tk/resource_ios/__init__.py +++ b/cognite_toolkit/_cdf_tk/resource_ios/__init__.py @@ -17,8 +17,7 @@ from cognite_toolkit._cdf_tk.feature_flags import FeatureFlag, Flags -from ._base_ios import DataCRUD, Loader, ResourceContainerIO, ResourceIO -from ._data_cruds import RawFileCRUD +from ._base_ios import Loader, ResourceContainerIO, ResourceIO from ._resource_ios import ( AgentIO, AppIO, @@ -119,10 +118,9 @@ for _loader in itertools.chain( ResourceIO.__subclasses__(), ResourceContainerIO.__subclasses__(), - DataCRUD.__subclasses__(), GroupIO.__subclasses__(), ): - if _loader in [ResourceIO, ResourceContainerIO, DataCRUD, GroupIO]: + if _loader in [ResourceIO, ResourceContainerIO, GroupIO]: # Skipping base classes continue # MyPy bug: https://github.com/python/mypy/issues/4717 @@ -150,7 +148,6 @@ CRUD_LIST = list(itertools.chain.from_iterable(CRUDS_BY_FOLDER_NAME.values())) RESOURCE_CRUD_LIST = [loader for loader in CRUD_LIST if issubclass(loader, ResourceIO)] RESOURCE_CRUD_CONTAINER_LIST = [loader for loader in CRUD_LIST if issubclass(loader, ResourceContainerIO)] -RESOURCE_DATA_CRUD_LIST = [loader for loader in CRUD_LIST if issubclass(loader, DataCRUD)] KINDS_BY_FOLDER_NAME: dict[str, set[str]] = {} for crud in CRUD_LIST: if crud.folder_name not in KINDS_BY_FOLDER_NAME: @@ -216,7 +213,6 @@ def get_crud(resource_dir: str, kind: str) -> type[Loader]: "AssetIO", "CogniteFileCRUD", "ContainerCRUD", - "DataCRUD", "DataModelIO", "DataProductIO", "DataProductVersionIO", @@ -243,7 +239,6 @@ def get_crud(resource_dir: str, kind: str) -> type[Loader]: "LocationFilterIO", "NodeCRUD", "RawDatabaseCRUD", - "RawFileCRUD", "RawTableCRUD", "RelationshipIO", "ResourceContainerIO", diff --git a/cognite_toolkit/_cdf_tk/resource_ios/_base_ios.py b/cognite_toolkit/_cdf_tk/resource_ios/_base_ios.py index a58b7a0c49..4592c5c977 100644 --- a/cognite_toolkit/_cdf_tk/resource_ios/_base_ios.py +++ b/cognite_toolkit/_cdf_tk/resource_ios/_base_ios.py @@ -24,7 +24,6 @@ if TYPE_CHECKING: from cognite_toolkit._cdf_tk.commands.build_v2.data_classes import BuildVariable, ResourceType - from cognite_toolkit._cdf_tk.data_classes import BuildEnvironment if sys.version_info >= (3, 11): from typing import Self @@ -502,26 +501,3 @@ def count(self, ids: Sequence[T_Identifier]) -> int: @abstractmethod def drop_data(self, ids: Sequence[T_Identifier]) -> int: raise NotImplementedError - - -class DataCRUD(Loader, ABC): - """This is the base class for all data CRUD.. - - A data CRUD is a CRUD that uploads data to CDF. It will typically depend on a - resource container that stores the data. For example, the datapoints crud depends - on the timeseries CRUD. - - It has only one required method: - - upload: Uploads the data to CDF. - - class attributes: - item_name: The name of the item that is stored in the resource container. This should be set in the subclass. - It is used to display messages when running operations. - - """ - - item_name: str - - @abstractmethod - def upload(self, state: "BuildEnvironment", dry_run: bool) -> Iterable[tuple[str, int]]: - raise NotImplementedError diff --git a/cognite_toolkit/_cdf_tk/resource_ios/_data_cruds.py b/cognite_toolkit/_cdf_tk/resource_ios/_data_cruds.py deleted file mode 100644 index 49955a45bb..0000000000 --- a/cognite_toolkit/_cdf_tk/resource_ios/_data_cruds.py +++ /dev/null @@ -1,84 +0,0 @@ -import io -from collections.abc import Iterable -from typing import TYPE_CHECKING, cast, final - -import pandas as pd - -from cognite_toolkit._cdf_tk.client.resource_classes.raw import RAWTableResponse -from cognite_toolkit._cdf_tk.utils.file import read_csv - -from ._base_ios import DataCRUD -from ._resource_ios import RawTableCRUD - -if TYPE_CHECKING: - from cognite_toolkit._cdf_tk.data_classes import BuildEnvironment - - -@final -class RawFileCRUD(DataCRUD): - item_name = "rows" - folder_name = "raw" - kind = "Raw" - dependencies = frozenset({RawTableCRUD}) - _doc_url = "Raw/operation/postRows" - - @property - def display_name(self) -> str: - return "raw rows" - - def upload(self, state: "BuildEnvironment", dry_run: bool) -> Iterable[tuple[str, int]]: - if self.folder_name not in state.built_resources: - return - - for resource in state.built_resources[self.folder_name]: - if resource.kind != RawTableCRUD.kind: - continue - table = cast(RAWTableResponse, resource.identifier) - datafile = next( - ( - resource.source.path.with_suffix(f".{file_type}") - for file_type in ["csv", "parquet"] - if (resource.source.path.with_suffix(f".{file_type}").exists()) - ), - None, - ) - if datafile is None: - # No adjacent data file found - continue - - if datafile.suffix == ".csv": - # The replacement is used to ensure that we read exactly the same file on Windows and Linux - file_content = datafile.read_bytes().replace(b"\r\n", b"\n").decode("utf-8") - data = read_csv(io.StringIO(file_content)).astype(object).fillna("") - if not data.columns.empty and data.columns[0] == "key": - print(f"Setting index to 'key' for {datafile.name}") - data.set_index("key", inplace=True) - elif datafile.suffix == ".parquet": - data = pd.read_parquet(datafile, engine="pyarrow") - else: - raise ValueError(f"Unsupported file type {datafile.suffix} for {datafile.name}") - - if data.empty: - yield ( - f"Empty file {datafile.as_posix()!r}. No rows to insert into {table!r}.", - 0, - ) - continue - - if dry_run: - yield ( - ( - f" Would insert {len(data):,} rows of {len(data.columns):,} columns from '{datafile!s}' " - f"into {table!r}." - ), - len(data), - ) - continue - - self.client.raw.rows.insert_dataframe( - db_name=table.db_name, table_name=table.name, dataframe=data, ensure_parent=False - ) - yield ( - (f" Inserted {len(data):,} rows of {len(data.columns):,} columns from '{datafile!s}' into {table!r}."), - len(data), - ) diff --git a/tests/test_unit/test_cdf_tk/test_cruds/test_raw_file.py b/tests/test_unit/test_cdf_tk/test_cruds/test_raw_file.py deleted file mode 100644 index d970530497..0000000000 --- a/tests/test_unit/test_cdf_tk/test_cruds/test_raw_file.py +++ /dev/null @@ -1,107 +0,0 @@ -from pathlib import Path -from typing import Any -from unittest.mock import MagicMock - -import pytest - -from cognite_toolkit._cdf_tk.client.resource_classes.raw import RAWTableRequest -from cognite_toolkit._cdf_tk.client.testing import monkeypatch_toolkit_client -from cognite_toolkit._cdf_tk.data_classes import BuildEnvironment, BuiltResource, BuiltResourceList, SourceLocationEager -from cognite_toolkit._cdf_tk.resource_ios import RawFileCRUD, RawTableCRUD - - -class TestRawFileLoader: - @pytest.mark.parametrize( - "csv_content, expected_write", - [ - ( - """myFloat,myInt,myString,myBool -0.1,1,hello,True -0.2,2,world,False -""", - { - 0: {"myFloat": 0.1, "myInt": 1, "myString": "hello", "myBool": True}, - 1: {"myFloat": 0.2, "myInt": 2, "myString": "world", "myBool": False}, - }, - ) - ], - ) - def test_upload_dtypes(self, csv_content: str, expected_write: dict[int, Any]) -> None: - with monkeypatch_toolkit_client() as client: - loader = RawFileCRUD.create_loader(client) - csv_file = MagicMock(spec=Path) - csv_file.read_bytes.return_value = csv_content.encode("utf-8") - csv_file.exists.return_value = True - csv_file.suffix = ".csv" - source_file = MagicMock(spec=Path) - source_file.with_suffix.return_value = csv_file - - state = BuildEnvironment() - state.built_resources[RawFileCRUD.folder_name] = BuiltResourceList( - [ - BuiltResource( - RAWTableRequest(db_name="myDB", name="myTable"), - SourceLocationEager(source_file, "1z234"), - RawTableCRUD.kind, - None, - None, - ) - ] - ) - - list(loader.upload(state, dry_run=False)) - - # Verify one upload call was made - assert client.raw.rows.insert_dataframe.call_count == 1 - _, kwargs = client.raw.rows.insert_dataframe.call_args - written_to_cdf = kwargs["dataframe"].to_dict(orient="index") - # All values and types should match the expected ingestion payload - assert written_to_cdf == expected_write - - def test_upload_preserves_numeric_types_and_sets_empty_strings_for_nulls(self) -> None: - with monkeypatch_toolkit_client() as client: - loader = RawFileCRUD.create_loader(client) - csv_file = MagicMock(spec=Path) - csv_content = """myFloat,myInt,myString,myBool -,1,hello,True -0.2,,world,False -""" - csv_file.read_bytes.return_value = csv_content.encode("utf-8") - csv_file.exists.return_value = True - csv_file.suffix = ".csv" - source_file = MagicMock(spec=Path) - source_file.with_suffix.return_value = csv_file - - state = BuildEnvironment() - state.built_resources[RawFileCRUD.folder_name] = BuiltResourceList( - [ - BuiltResource( - RAWTableRequest(db_name="myDB", name="myTable"), - SourceLocationEager(source_file, "1z234"), - RawTableCRUD.kind, - None, - None, - ) - ] - ) - - list(loader.upload(state, dry_run=False)) - - # Capture the DataFrame uploaded to RAW - - # Verify one upload call was made - assert client.raw.rows.insert_dataframe.call_count == 1 - _, kwargs = client.raw.rows.insert_dataframe.call_args - df = kwargs["dataframe"] - # Verify dtypes are object after astype(object).fillna("") - assert str(df.dtypes["myFloat"]) == "object" - assert str(df.dtypes["myInt"]) == "object" - - # Non-null float value remains numeric - assert isinstance(df.iloc[1]["myFloat"], float) - # CSV with nulls coerces an integer-like column to floats; value becomes 1.0 - assert isinstance(df.iloc[0]["myInt"], float) - # Null in float column becomes empty string - assert df.iloc[0]["myFloat"] == "" and isinstance(df.iloc[0]["myFloat"], str) - # Null in int column becomes empty string - assert df.iloc[1]["myInt"] == "" and isinstance(df.iloc[1]["myInt"], str) From 4c449f9606527066851c1686a507389342f037ca Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 7 Sep 2026 20:20:58 +0200 Subject: [PATCH 29/31] refactor: delete last pandas dependencies --- .../_cdf_tk/resource_ios/__init__.py | 1 - cognite_toolkit/_cdf_tk/utils/file.py | 41 +------------------ 2 files changed, 1 insertion(+), 41 deletions(-) diff --git a/cognite_toolkit/_cdf_tk/resource_ios/__init__.py b/cognite_toolkit/_cdf_tk/resource_ios/__init__.py index 5b5caa5f4c..4a46a54367 100644 --- a/cognite_toolkit/_cdf_tk/resource_ios/__init__.py +++ b/cognite_toolkit/_cdf_tk/resource_ios/__init__.py @@ -205,7 +205,6 @@ def get_crud(resource_dir: str, kind: str) -> type[Loader]: "RESOURCE_CRUD_BY_FOLDER_NAME", "RESOURCE_CRUD_CONTAINER_LIST", "RESOURCE_CRUD_LIST", - "RESOURCE_DATA_CRUD_LIST", "_EXCLUDED_CRUDS", "AgentIO", "AppIO", diff --git a/cognite_toolkit/_cdf_tk/utils/file.py b/cognite_toolkit/_cdf_tk/utils/file.py index b32bcd2bce..5733e31c89 100644 --- a/cognite_toolkit/_cdf_tk/utils/file.py +++ b/cognite_toolkit/_cdf_tk/utils/file.py @@ -9,7 +9,7 @@ import warnings from abc import abstractmethod from collections import UserDict, defaultdict -from collections.abc import Hashable, ItemsView, KeysView, ValuesView +from collections.abc import ItemsView, KeysView, ValuesView from contextlib import contextmanager from dataclasses import dataclass, field from io import BytesIO @@ -17,14 +17,12 @@ from typing import Any, Literal, TypeVar, overload from zipfile import ZipFile -import pandas as pd import yaml from rich import print from cognite_toolkit._cdf_tk.cdf_toml import CDFToml from cognite_toolkit._cdf_tk.constants import ENV_VAR_PATTERN, HINT_LEAD_TEXT, MODULES, URL from cognite_toolkit._cdf_tk.exceptions import ( - ToolkitValueError, ToolkitYAMLFormatError, ) from cognite_toolkit._cdf_tk.tk_warnings import EnvironmentVariableMissingWarning, MediumSeverityWarning @@ -426,26 +424,6 @@ def remove_trailing_newline(content: str) -> str: return content -def read_csv( - path: Path | typing.TextIO, - parse_dates: bool | None = None, - index_col: Hashable | None = None, - dtype: Any | None = None, -) -> pd.DataFrame: - """Reads CSV - - Args: - path (Path): Path to the CSV file. - parse_dates (bool, optional): Whether to parse dates. Defaults to None. - index_col (Hashable, optional): Index column. Defaults to None. - dtype (Any, optional): Data types. Defaults to None - - Returns: - pd.DataFrame: DataFrame with the CSV data. - """ - return pd.read_csv(path, parse_dates=parse_dates, index_col=index_col, dtype=dtype) - - def _handle_remove_readonly(func: Any, path: Any, exc: Any) -> None: excvalue = exc[1] if func in (os.rmdir, os.remove) and excvalue.errno == errno.EACCES: @@ -472,23 +450,6 @@ def safe_rmtree(path: Path) -> None: ).print_warning() -def get_table_columns(table: Path) -> list[str]: - """Get the columns of a table - - Args: - table (Path): Path to the table - - Returns: - list[str]: List of columns - """ - if table.suffix == ".csv": - return read_csv(table).columns.tolist() - elif table.suffix == ".parquet": - return pd.read_parquet(table).columns.tolist() - else: - raise ToolkitValueError(f"The file {table.name} is not a supported table format (csv, parquet)") - - @contextmanager def create_temporary_zip(directory: Path, zipname: str) -> typing.Generator[Path, None, None]: """ From 1e836a56e385fd60ccb1d062775c8a634137633d Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 7 Sep 2026 20:22:35 +0200 Subject: [PATCH 30/31] refactor: drop pandas as a core dependency --- pyproject.toml | 1 - uv.lock | 2 -- 2 files changed, 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index aa8e7f9e1d..738755b72e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,6 @@ dependencies = [ "python-dotenv >=1.0.0", "cognite-sdk>=8.6.0,<9.0.0", "httpx>=0.28.1", - "pandas >=1.5.3, <3.0.0", "pyyaml >=6.0.1", "typer >=0.12.0, <1.0.0", "rich >=13.9.4", diff --git a/uv.lock b/uv.lock index 437b123cc6..7d47095622 100644 --- a/uv.lock +++ b/uv.lock @@ -379,7 +379,6 @@ dependencies = [ { name = "httpx" }, { name = "mixpanel" }, { name = "packaging" }, - { name = "pandas" }, { name = "pip" }, { name = "pydantic" }, { name = "python-dateutil" }, @@ -440,7 +439,6 @@ requires-dist = [ { name = "mixpanel", specifier = ">=4.10.1" }, { name = "openpyxl", marker = "extra == 'table'", specifier = ">=3.1.5" }, { name = "packaging", specifier = ">=25" }, - { name = "pandas", specifier = ">=1.5.3,<3.0.0" }, { name = "pip", specifier = ">=25.0.1" }, { name = "pyarrow", marker = "extra == 'table'", specifier = ">=20.0.0" }, { name = "pydantic", specifier = ">=2.12.0" }, From aa1591e976505aa176dc43dad5ed643d66b6742c Mon Sep 17 00:00:00 2001 From: anders-albert Date: Mon, 7 Sep 2026 20:31:34 +0200 Subject: [PATCH 31/31] tests: update sdk imports --- tests/test_unit/test_toolkit_package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_unit/test_toolkit_package.py b/tests/test_unit/test_toolkit_package.py index 5cfb51eb07..1414fe099b 100644 --- a/tests/test_unit/test_toolkit_package.py +++ b/tests/test_unit/test_toolkit_package.py @@ -94,7 +94,7 @@ def test_no_cognite_sdk_imports() -> None: The goal is to fully remove the cognite-sdk dependency from the toolkit (with the exception of Auth and protobuf files). This test tracks progress toward that goal. """ - _assert_import_violations(_extract_cognite_sdk_imports, "cognite.client imports", 93) + _assert_import_violations(_extract_cognite_sdk_imports, "cognite.client imports", 92) def _parse_package_name(dependency: str) -> str: