diff --git a/cognite_toolkit/_cdf_tk/data_classes/__init__.py b/cognite_toolkit/_cdf_tk/data_classes/__init__.py index cab85eb80e..91340f902d 100644 --- a/cognite_toolkit/_cdf_tk/data_classes/__init__.py +++ b/cognite_toolkit/_cdf_tk/data_classes/__init__.py @@ -1,35 +1,15 @@ -from ._build_files import BuildDestinationFile, BuildSourceFile -from ._build_variables import BuildVariable, BuildVariables -from ._built_modules import ( - BuiltModule, - BuiltModuleList, -) -from ._built_resources import ( - BuiltFullResourceList, - BuiltResource, - BuiltResourceFull, - BuiltResourceList, - SourceLocation, - SourceLocationEager, - SourceLocationLazy, -) from ._config_yaml import ( BuildConfigYAML, - BuildEnvironment, ConfigEntry, ConfigYAMLs, Environment, InitConfigYAML, ) from ._deploy_results import ( - DatapointDeployResult, DeployResult, DeployResults, - ResourceContainerDeployResult, ResourceDeployResult, - UploadDeployResult, ) -from ._issues import Issue, IssueList from ._module_directories import ModuleDirectories, ModuleLocation from ._packages import Package, Packages from ._tracking_info import CommandTracking, DeploymentTracking, TrackingEvent @@ -37,38 +17,19 @@ __all__ = [ "BuildConfigYAML", - "BuildDestinationFile", - "BuildEnvironment", - "BuildSourceFile", - "BuildVariable", - "BuildVariables", - "BuiltFullResourceList", - "BuiltModule", - "BuiltModuleList", - "BuiltResource", - "BuiltResourceFull", - "BuiltResourceList", "CommandTracking", "ConfigEntry", "ConfigYAMLs", - "DatapointDeployResult", "DeployResult", "DeployResults", "DeploymentTracking", "Environment", "InitConfigYAML", - "Issue", - "IssueList", "ModuleDirectories", "ModuleLocation", "Package", "Packages", - "ResourceContainerDeployResult", "ResourceDeployResult", - "SourceLocation", - "SourceLocationEager", - "SourceLocationLazy", "TrackingEvent", - "UploadDeployResult", "YAMLComments", ] diff --git a/cognite_toolkit/_cdf_tk/data_classes/_base.py b/cognite_toolkit/_cdf_tk/data_classes/_base.py index 5de7fe116d..4ce9e818a8 100644 --- a/cognite_toolkit/_cdf_tk/data_classes/_base.py +++ b/cognite_toolkit/_cdf_tk/data_classes/_base.py @@ -2,11 +2,9 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from pathlib import Path -from typing import Any, ClassVar, TypeVar +from typing import Any, ClassVar -from cognite_toolkit import _version -from cognite_toolkit._cdf_tk.constants import BUILD_ENVIRONMENT_FILE -from cognite_toolkit._cdf_tk.exceptions import ToolkitFileNotFoundError, ToolkitRequiredValueError, ToolkitVersionError +from cognite_toolkit._cdf_tk.exceptions import ToolkitFileNotFoundError, ToolkitRequiredValueError from cognite_toolkit._cdf_tk.utils import read_yaml_file if sys.version_info >= (3, 11): @@ -43,28 +41,3 @@ def load_from_directory(cls, organization_dir: Path, build_env: str) -> Self: @abstractmethod def load(cls, data: dict[str, Any], build_env: str, filepath: Path) -> Self: raise NotImplementedError - - -T_BuildConfig = TypeVar("T_BuildConfig", bound=ConfigCore) - - -def _load_version_variable(data: dict[str, Any], file_name: str) -> str: - try: - cdf_tk_version: str = data["cdf_toolkit_version"] - except KeyError: - err_msg = f"System variables are missing required field 'cdf_toolkit_version' in {file_name!s}. {{}}" - if file_name == BUILD_ENVIRONMENT_FILE: - raise ToolkitVersionError( - err_msg.format("Rerun `cdf build` to build the modules again and create it correctly.") - ) - raise ToolkitVersionError( - err_msg.format("Run `cdf modules upgrade` to initialize the modules again to create a correct file.") - ) - - if cdf_tk_version != _version.__version__: - raise ToolkitVersionError( - f"The version of the modules ({cdf_tk_version}) does not match the version of the installed CLI " - f"({_version.__version__}). Please either run `cdf modules upgrade` to upgrade the modules OR " - f"run `pip install cognite-toolkit=={cdf_tk_version}` to downgrade cdf-tk CLI." - ) - return cdf_tk_version diff --git a/cognite_toolkit/_cdf_tk/data_classes/_build_files.py b/cognite_toolkit/_cdf_tk/data_classes/_build_files.py deleted file mode 100644 index aece141679..0000000000 --- a/cognite_toolkit/_cdf_tk/data_classes/_build_files.py +++ /dev/null @@ -1,37 +0,0 @@ -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -from cognite_toolkit._cdf_tk.resource_ios import ( - ResourceIO, -) -from cognite_toolkit._cdf_tk.tk_warnings import ( - WarningList, -) -from cognite_toolkit._cdf_tk.tk_warnings.fileread import ( - FileReadWarning, -) -from cognite_toolkit._cdf_tk.utils.file import yaml_safe_dump - -from ._built_resources import SourceLocation - - -@dataclass -class BuildSourceFile: - source: SourceLocation - content: str - loaded: list[dict[str, Any]] | dict[str, Any] | None = None - - -@dataclass -class BuildDestinationFile: - path: Path - loaded: list[dict[str, Any]] | dict[str, Any] - loader: type[ResourceIO] - source: SourceLocation - extra_sources: list[SourceLocation] | None - warnings: WarningList[FileReadWarning] = field(default_factory=WarningList[FileReadWarning]) - - @property - def content(self) -> str: - return yaml_safe_dump(self.loaded) diff --git a/cognite_toolkit/_cdf_tk/data_classes/_build_variables.py b/cognite_toolkit/_cdf_tk/data_classes/_build_variables.py deleted file mode 100644 index 70ec6ab6fb..0000000000 --- a/cognite_toolkit/_cdf_tk/data_classes/_build_variables.py +++ /dev/null @@ -1,315 +0,0 @@ -import re -import sys -import uuid -from collections import defaultdict -from collections.abc import Collection, Iterator, Sequence -from dataclasses import dataclass -from functools import cached_property -from pathlib import Path -from typing import Any, Literal, SupportsIndex, overload - -from cognite_toolkit._cdf_tk.data_classes._module_directories import ModuleLocation -from cognite_toolkit._cdf_tk.exceptions import ToolkitValueError -from cognite_toolkit._cdf_tk.resource_ios._resource_ios.transformation import TransformationIO - -if sys.version_info >= (3, 11): - from typing import Self -else: - from typing_extensions import Self - - -@dataclass(frozen=True) -class BuildVariable: - """This is an internal representation of a build variable in a config.[env].file - - Args: - key: The name of the variable. - value: The value of the variable. - is_selected: Whether the variable is selected by the user through Environment.selected - location: The location for the variable which is used to determine the module(s) it belongs to - - """ - - key: str - value: str | int | float | bool | tuple[str | int | float | bool] - is_selected: bool - location: Path - iteration: int | None = None - - @property - def value_variable(self) -> str | int | float | bool | list[str | int | float | bool]: - """Returns the value of the variable as a variable.""" - if isinstance(self.value, tuple): - # Convert the tuple back to a list to make it JSON serializable - return list(self.value) - else: - return self.value - - def dump(self) -> dict[str, Any]: - return { - "key": self.key, - "value": self.value_variable, - "is_selected": self.is_selected, - "location": self.location.as_posix(), - } - - @classmethod - def load(cls, data: dict[str, Any]) -> Self: - if isinstance(data["value"], list): - # Convert the list to a tuple to make it hashable - value = tuple(data["value"]) - else: - value = data["value"] - return cls(data["key"], value, data["is_selected"], Path(data["location"])) - - -class BuildVariables(tuple, Sequence[BuildVariable]): - """This is an internal representation of the build variables in a config.[env].file - - The motivation for this class is to provide helper functions for the user to interact with the build variables. - """ - - # Subclassing tuple to make the class immutable. BuildVariables is expected to be initialized and - # then used as a read-only object. - def __new__(cls, collection: Collection[BuildVariable], source_path: Path | None = None) -> Self: - # Need to override __new__ to as we are subclassing a tuple: - # https://stackoverflow.com/questions/1565374/subclassing-tuple-with-multiple-init-arguments - return super().__new__(cls, tuple(collection)) - - def __init__(self, collection: Collection[BuildVariable], source_path: Path | None = None) -> None: - super().__init__() - self.source_path = source_path - - @cached_property - def selected(self) -> "BuildVariables": - return BuildVariables([variable for variable in self if variable.is_selected]) - - @classmethod - def load_raw( - cls, - raw_variable: dict[str, Any], - available_modules: set[Path], - selected_modules: set[Path] | None = None, - source_path: Path | None = None, - ) -> Self: - """Loads the variables from the user input.""" - variables = [] - to_check: list[tuple[Path, int | None, dict[str, Any]]] = [(Path(""), None, raw_variable)] - while to_check: - path, iteration, subdict = to_check.pop() - for key, value in subdict.items(): - subpath = path / key - if subpath in available_modules and isinstance(value, dict): - to_check.append((subpath, None, value)) - elif subpath in available_modules and isinstance(value, list): - for no, module_variables in enumerate(value, 1): - if not isinstance(module_variables, dict): - raise ToolkitValueError(f"Variables under a module must be a dictionary: {subpath}.") - to_check.append((subpath, no, module_variables)) - elif isinstance(value, dict): - # Remove this check to support variables with dictionary values. - continue - else: - hashable_values = tuple(value) if isinstance(value, list) else value - is_selected = selected_modules is None or path in selected_modules - variables.append(BuildVariable(key, hashable_values, is_selected, path, iteration)) - - return cls(variables, source_path=source_path) - - @classmethod - def load(cls, data: list[dict[str, Any]]) -> Self: - """Loads the variables from a dictionary.""" - return cls([BuildVariable.load(variable) for variable in data]) - - def get_module_variables(self, module: ModuleLocation) -> "list[BuildVariables]": - """Gets the variables for a specific module.""" - variables_by_key_by_iteration: dict[int | None, dict[str, list[BuildVariable]]] = defaultdict( - lambda: defaultdict(list) - ) - for variable in self: - if variable.location == module.relative_path or variable.location in module.parent_relative_paths: - variables_by_key_by_iteration[variable.iteration][variable.key].append(variable) - - base_variables: dict[str, list[BuildVariable]] = variables_by_key_by_iteration.pop(None, {}) - variable_sets: list[dict[str, list[BuildVariable]]] - if variables_by_key_by_iteration: - # Combine each with the base variables - variable_sets = [] - for _, variables_by_key in sorted(variables_by_key_by_iteration.items(), key=lambda x: x[0] or 0): - for key, build_variable in base_variables.items(): - variables_by_key[key].extend(build_variable) - variable_sets.append(variables_by_key) - else: - variable_sets = [base_variables] - - return [ - BuildVariables( - [ - # We select the variable with the longest path to ensure that the most specific variable is selected - max(variables, key=lambda v: len(v.location.parts)) - for variables in variable_set.values() - ], - source_path=self.source_path, - ) - for variable_set in variable_sets - ] - - @overload - def replace(self, content: str, file_path: Path | None = None, use_placeholder: Literal[False] = False) -> str: ... - - @overload - def replace( - self, content: str, file_path: Path | None = None, use_placeholder: Literal[True] = True - ) -> tuple[str, dict[str, BuildVariable]]: ... - - def replace( - self, content: str, file_path: Path | None = None, use_placeholder: bool = False - ) -> str | tuple[str, dict[str, BuildVariable]]: - # Extract file suffix from path, default to .yaml if not provided - file_suffix = file_path.suffix if file_path and file_path.suffix else ".yaml" - - variable_by_placeholder: dict[str, BuildVariable] = {} - for variable in self: - if not use_placeholder: - replace = variable.value_variable - else: - replace = f"VARIABLE_{uuid.uuid4().hex[:8]}" - variable_by_placeholder[replace] = variable - - _core_pattern = rf"{{{{\s*{variable.key}\s*}}}}" - if file_suffix == ".sql": - # For SQL files, convert lists to SQL-style tuples - if isinstance(replace, list): - replace = self._format_list_as_sql_tuple(replace) - content = re.sub(_core_pattern, str(replace), content) - elif file_suffix in {".yaml", ".yml", ".json"}: - # Check if this is a transformation file (ends with Transformation.yaml/yml) - is_transformation_file = file_path is not None and f".{TransformationIO.kind}." in file_path.name - # Check if variable is within a query field (SQL context) - is_in_query_field = self._is_in_query_field(content, variable.key) - - # For lists in query fields, use SQL-style tuples - # For transformation files, ensure SQL conversion is applied to query property variables - if is_transformation_file and is_in_query_field and isinstance(replace, list): - replace = self._format_list_as_sql_tuple(replace) - # Use simple pattern for SQL context (no YAML quoting needed) - content = re.sub(_core_pattern, str(replace), content) - else: - # Preserve data types for YAML - pattern = _core_pattern - if isinstance(replace, str) and (replace.isdigit() or replace.endswith(":")): - replace = f'"{replace}"' - pattern = rf"'{_core_pattern}'|{_core_pattern}|" + rf'"{_core_pattern}"' - elif replace is None: - replace = "null" - content = re.sub(pattern, str(replace), content) - else: - # For other file types, use simple string replacement - content = re.sub(_core_pattern, str(replace), content) - if use_placeholder: - return content, variable_by_placeholder - else: - return content - - @staticmethod - def _is_transformation_file(file_path: Path) -> bool: - """Check if the file path indicates a transformation YAML file. - - Transformation files are YAML files in the "transformations" folder. - - Args: - file_path: The file path to check - - Returns: - True if the file is a transformation YAML file - """ - # Check if path contains "transformations" folder and ends with .yaml/.yml - path_str = file_path.as_posix().lower() - return "transformations" in path_str and file_path.suffix.lower() in {".yaml", ".yml"} - - @staticmethod - def _format_list_as_sql_tuple(replace: list[Any]) -> str: - """Format a list as a SQL-style tuple string. - - Args: - replace: The list to format - - Returns: - SQL tuple string, e.g., "('A', 'B', 'C')" or "()" for empty lists - """ - if not replace: - # Empty list becomes empty SQL tuple - return "()" - else: - # Format list as SQL tuple: ('A', 'B', 'C') - formatted_items = [] - for item in replace: - if item is None: - formatted_items.append("NULL") - elif isinstance(item, str): - formatted_items.append(f"'{item}'") - else: - formatted_items.append(str(item)) - return f"({', '.join(formatted_items)})" - - @staticmethod - def _is_in_query_field(content: str, variable_key: str) -> bool: - """Check if a variable is within a query field in YAML. - - Assumes query is a top-level property. This detects various YAML formats: - - query: >- - - query: | - - query: "..." - - query: ... - """ - lines = content.split("\n") - variable_pattern = rf"{{{{\s*{re.escape(variable_key)}\s*}}}}" - in_query_field = False - - for line in lines: - # Check if this line starts a top-level query field - query_match = re.match(r"^query\s*:\s*(.*)$", line) - if query_match: - in_query_field = True - query_content_start = query_match.group(1).strip() - - # Check if variable is on the same line as query: declaration - if re.search(variable_pattern, line): - return True - - # If query content starts on same line (not a block scalar), check it - if query_content_start and not query_content_start.startswith(("|", ">", "|-", ">-", "|+", ">+")): - if re.search(variable_pattern, query_content_start): - return True - continue - - # Check if we're still in the query field - if in_query_field: - # If we hit another top-level property, we've exited the query field - if re.match(r"^\w+\s*:", line): - in_query_field = False - continue - - # We're still in the query field, check for variable - if re.search(variable_pattern, line): - return True - - return False - - # Implemented to get correct type hints - def __iter__(self) -> Iterator[BuildVariable]: - return super().__iter__() - - @overload - def __getitem__(self, index: SupportsIndex) -> BuildVariable: ... - - @overload - def __getitem__(self, index: slice) -> Self: ... - - def __getitem__(self, index: SupportsIndex | slice, /) -> "BuildVariable | BuildVariables": - if isinstance(index, slice): - return BuildVariables(super().__getitem__(index)) - return super().__getitem__(index) - - def dump(self) -> list[dict[str, Any]]: - return [variable.dump() for variable in self] diff --git a/cognite_toolkit/_cdf_tk/data_classes/_built_modules.py b/cognite_toolkit/_cdf_tk/data_classes/_built_modules.py deleted file mode 100644 index 1beba15606..0000000000 --- a/cognite_toolkit/_cdf_tk/data_classes/_built_modules.py +++ /dev/null @@ -1,127 +0,0 @@ -import sys -from collections.abc import Callable, Collection, Iterator, MutableSequence -from dataclasses import dataclass -from pathlib import Path -from typing import Any, SupportsIndex, overload - -from cognite_toolkit._cdf_tk.resource_ios import ResourceTypes -from cognite_toolkit._cdf_tk.utils.useful_types import T_ID - -from ._build_variables import BuildVariables -from ._built_resources import ( - BuiltFullResourceList, - BuiltResource, - BuiltResourceList, - SourceLocation, -) - -if sys.version_info >= (3, 11): - from typing import Self -else: - from typing_extensions import Self - - -@dataclass -class BuiltModule: - name: str - location: SourceLocation - build_variables: BuildVariables - resources: dict[str, BuiltResourceList] - warning_count: int - status: str - iteration: int - package_id: str | None = None - module_id: str | None = None - - @classmethod - def load(cls, data: dict[str, Any]) -> Self: - return cls( - name=data["name"], - location=SourceLocation.load(data["location"]), - build_variables=BuildVariables.load(data["build_variables"]), - resources={ - key: BuiltResourceList([BuiltResource.load(resource_data, key) for resource_data in resources_data]) - for key, resources_data in data["resources"].items() - }, - warning_count=data.get("warning_count", 0), - status=data.get("status", "Success"), - iteration=data.get("iteration", 1), - package_id=data.get("package_id"), - module_id=data.get("module_id"), - ) - - def dump(self) -> dict[str, Any]: - return { - "name": self.name, - "location": self.location.dump(), - "build_variables": self.build_variables.dump(), - "resources": { - key: [resource.dump(key) for resource in resources] for key, resources in self.resources.items() - }, - "warning_count": self.warning_count, - "status": self.status, - "iteration": self.iteration, - "package_id": self.package_id, - "module_id": self.module_id, - } - - -@dataclass -class BuiltModuleList(list, MutableSequence[BuiltModule]): - # Implemented to get correct type hints - def __init__(self, collection: Collection[BuiltModule] | None = None) -> None: - super().__init__(collection or []) - - def __iter__(self) -> Iterator[BuiltModule]: - return super().__iter__() - - @overload - def __getitem__(self, index: SupportsIndex) -> BuiltModule: ... - - @overload - def __getitem__(self, index: slice) -> Self: ... - - def __getitem__(self, index: SupportsIndex | slice, /) -> "BuiltModule | BuiltModuleList": - if isinstance(index, slice): - return BuiltModuleList(super().__getitem__(index)) - return super().__getitem__(index) - - def get_resources( - self, - id_type: type[T_ID] | None, - resource_dir: ResourceTypes, - kind: str | None = None, - selected: Path | str | None = None, - is_supported_file: Callable[[Path], bool] | None = None, - ) -> "BuiltFullResourceList[T_ID]": - resources = ( - resource.create_full(module, resource_dir) - for module in self - for resource in module.resources.get(resource_dir, []) - if kind is None or resource.kind == kind - ) - if isinstance(selected, str): - resources = (resource for resource in resources if resource.module_name == selected) - elif isinstance(selected, Path): - resources = ( - resource - for resource in resources - if (resource.source.path == selected or self._are_relative(resource.source.path, selected)) - ) - if is_supported_file: - # This is necessary as the destination file can be created from a source file that is not supported. - # This happens for RAW table files which produces a Database file. - resources = (resource for resource in resources if is_supported_file(resource.source.path)) - - return BuiltFullResourceList[T_ID](list(resources)) - - def as_resources_by_folder(self) -> dict[str, BuiltResourceList[T_ID]]: - resources_by_folder: dict[str, BuiltResourceList[T_ID]] = {} - for module in self: - for resource_dir, resources in module.resources.items(): - resources_by_folder.setdefault(resource_dir, BuiltResourceList()).extend(resources) - return resources_by_folder - - @staticmethod - def _are_relative(filepath: Path, select_path: Path) -> bool: - return filepath.resolve().is_relative_to(select_path.resolve()) diff --git a/cognite_toolkit/_cdf_tk/data_classes/_built_resources.py b/cognite_toolkit/_cdf_tk/data_classes/_built_resources.py deleted file mode 100644 index ae93bf16cb..0000000000 --- a/cognite_toolkit/_cdf_tk/data_classes/_built_resources.py +++ /dev/null @@ -1,244 +0,0 @@ -import sys -from abc import abstractmethod -from collections import defaultdict -from collections.abc import Collection, Iterator, MutableSequence -from dataclasses import dataclass -from functools import cached_property -from pathlib import Path -from typing import TYPE_CHECKING, Any, Generic, SupportsIndex, TypeVar, cast, overload - -from cognite_toolkit._cdf_tk.exceptions import ToolkitMissingResourceError -from cognite_toolkit._cdf_tk.resource_ios import get_crud -from cognite_toolkit._cdf_tk.resource_ios._base_ios import ResourceIO -from cognite_toolkit._cdf_tk.utils import ( - calculate_directory_hash, - calculate_hash, - load_yaml_inject_variables, - safe_read, -) -from cognite_toolkit._cdf_tk.utils.useful_types import T_ID - -from ._build_variables import BuildVariables - -if sys.version_info >= (3, 11): - from typing import Self -else: - from typing_extensions import Self -if TYPE_CHECKING: - from ._built_modules import BuiltModule - - -@dataclass -class SourceLocation: - """This represents the location of a built resource in a module structure. - - Args: - path: The relative path to the resource from the project directory. - """ - - path: Path - - @property - @abstractmethod - def hash(self) -> str: - """The hash of the resource file.""" - raise NotImplementedError() - - def dump(self) -> dict[str, Any]: - return { - "path": self.path.as_posix(), - "hash": str(self.hash), - } - - @classmethod - def load(cls, data: dict[str, Any]) -> "SourceLocationEager": - return SourceLocationEager( - path=Path(data["path"]), - _hash=str(data["hash"]), - ) - - -@dataclass -class SourceLocationLazy(SourceLocation): - absolute_path: Path - - @cached_property - def hash(self) -> str: - if self.absolute_path.is_dir(): - return calculate_directory_hash(self.absolute_path, shorten=True) - else: - return calculate_hash(self.absolute_path, shorten=True) - - -@dataclass -class SourceLocationEager(SourceLocation): - _hash: str - - @property - def hash(self) -> str: - return self._hash - - -@dataclass -class BuiltResource(Generic[T_ID]): - """This represents a built resource. - - Args: - identifier: The unique identifier of the resource. - source: The source location of the resource. - kind: The kind of resource. - destination: The destination of the resource. - extra_sources: Extra source locations of the resource, for example, Transformations might have - .sql files that are used to build the final resource. - - """ - - identifier: T_ID - source: SourceLocation - kind: str - destination: Path | None - extra_sources: list[SourceLocation] | None - - @classmethod - def load(cls, data: dict[str, Any], resource_folder: str) -> Self: - from cognite_toolkit._cdf_tk.resource_ios import ResourceIO, get_crud - - kind = data["kind"] - loader = cast(ResourceIO, get_crud(resource_folder, kind)) - identifier = loader.get_id(data["identifier"]) - - return cls( - source=SourceLocation.load(data["source"]), - kind=kind, - identifier=identifier, - destination=Path(data["destination"]) if "destination" in data else None, - extra_sources=[SourceLocation.load(source) for source in data.get("extra_sources", [])] or None, - ) - - def dump(self, resource_folder: str, include_destination: bool = False) -> dict[str, Any]: - from cognite_toolkit._cdf_tk.resource_ios import ResourceIO, get_crud - - loader = cast(ResourceIO, get_crud(resource_folder, self.kind)) - dumped = loader.dump_id(self.identifier) - - output: dict[str, Any] = { - "identifier": dumped, - "source": self.source.dump(), - "kind": self.kind, - } - if include_destination and self.destination: - output["destination"] = self.destination.as_posix() - if self.extra_sources: - output["extra_sources"] = [source.dump() for source in self.extra_sources] - return output - - def create_full(self, module: "BuiltModule", resource_dir: str) -> "BuiltResourceFull": - return BuiltResourceFull( - identifier=self.identifier, - source=self.source, - kind=self.kind, - destination=self.destination, - build_variables=module.build_variables, - module_name=module.name, - module_location=module.location.path, - resource_dir=resource_dir, - extra_sources=self.extra_sources, - ) - - -T_BuiltResource = TypeVar("T_BuiltResource", bound=BuiltResource) - - -@dataclass -class BuiltResourceFull(BuiltResource[T_ID]): - build_variables: BuildVariables - module_name: str - module_location: Path - resource_dir: str - - def load_resource_dict( - self, environment_variables: dict[str, str | None], validate: bool = False - ) -> dict[str, Any]: - content = self.build_variables.replace(safe_read(self.source.path), self.source.path) - loader = cast(ResourceIO, get_crud(self.resource_dir, self.kind)) - raw = load_yaml_inject_variables( - content, - environment_variables, - validate=validate, - original_filepath=self.source.path, - ) - if isinstance(raw, dict): - return raw - elif isinstance(raw, list): - for item in raw: - if loader.get_id(item) == self.identifier: - return item - raise ToolkitMissingResourceError(f"Resource {self.identifier} not found in {self.source.path}") - - -class BuiltResourceList(list, MutableSequence[BuiltResource[T_ID]], Generic[T_ID]): - # Implemented to get correct type hints - def __init__(self, collection: Collection[BuiltResource[T_ID]] | None = None) -> None: - super().__init__(collection or []) - - def __iter__(self) -> Iterator[BuiltResource[T_ID]]: - return super().__iter__() - - @overload - def __getitem__(self, index: SupportsIndex) -> BuiltResource[T_ID]: ... - - @overload - def __getitem__(self, index: slice) -> "BuiltResourceList[T_ID]": ... - - def __getitem__(self, index: SupportsIndex | slice, /) -> "BuiltResource[T_ID] | BuiltResourceList[T_ID]": - if isinstance(index, slice): - return BuiltResourceList[T_ID](super().__getitem__(index)) - return super().__getitem__(index) - - @property - def identifiers(self) -> list[T_ID]: - return [resource.identifier for resource in self] - - @classmethod - def load(cls, data: list[dict[str, Any]], resource_folder: str) -> "BuiltResourceList[T_ID]": - return cls([BuiltResource.load(resource_data, resource_folder) for resource_data in data]) - - def dump(self, resource_folder: str, include_destination: bool = False) -> list[dict[str, Any]]: - return [resource.dump(resource_folder, include_destination) for resource in self] - - def get_resource_directories(self, resource_folder: str) -> set[Path]: - output: set[Path] = set() - for resource in self: - index = next((i for i, part in enumerate(resource.source.path.parts) if part == resource_folder), None) - if index is None: - continue - path = Path("/".join(resource.source.path.parts[: index + 1])) - output.add(path) - - return output - - -class BuiltFullResourceList(BuiltResourceList[T_ID]): - # Implemented to get correct type hints - def __init__(self, collection: Collection[BuiltResourceFull[T_ID]] | None = None) -> None: - super().__init__(collection or []) - - def __iter__(self) -> Iterator[BuiltResourceFull[T_ID]]: - return cast(Iterator[BuiltResourceFull[T_ID]], super().__iter__()) - - @overload - def __getitem__(self, index: SupportsIndex) -> BuiltResourceFull[T_ID]: ... - - @overload - def __getitem__(self, index: slice) -> "BuiltFullResourceList[T_ID]": ... - - def __getitem__(self, index: SupportsIndex | slice, /) -> "BuiltResourceFull[T_ID] | BuiltFullResourceList[T_ID]": - if isinstance(index, slice): - return BuiltFullResourceList[T_ID](super().__getitem__(index)) - return cast(BuiltResourceFull[T_ID], super().__getitem__(index)) - - def by_file(self) -> "dict[Path, BuiltFullResourceList[T_ID]]": - resources_by_file: dict[Path, BuiltFullResourceList[T_ID]] = defaultdict(lambda: BuiltFullResourceList()) - for resource in self: - resources_by_file[resource.source.path].append(resource) - return resources_by_file diff --git a/cognite_toolkit/_cdf_tk/data_classes/_config_yaml.py b/cognite_toolkit/_cdf_tk/data_classes/_config_yaml.py index 71076cf185..98568649f0 100644 --- a/cognite_toolkit/_cdf_tk/data_classes/_config_yaml.py +++ b/cognite_toolkit/_cdf_tk/data_classes/_config_yaml.py @@ -7,14 +7,11 @@ from collections.abc import Hashable, Iterable, Sequence, Set from dataclasses import dataclass, field from pathlib import Path -from typing import Any, ClassVar, Literal, cast, get_args +from typing import Any, ClassVar, cast, get_args -import yaml from rich import print from cognite_toolkit._cdf_tk.constants import ( - _RUNNING_IN_BROWSER, - BUILD_ENVIRONMENT_FILE, DEFAULT_CONFIG_FILE, DEFAULT_ENV, MODULES, @@ -22,32 +19,21 @@ SEARCH_VARIABLES_SUFFIX, EnvType, ) -from cognite_toolkit._cdf_tk.exceptions import ToolkitEnvError, ToolkitMissingModuleError -from cognite_toolkit._cdf_tk.hints import ModuleDefinition -from cognite_toolkit._cdf_tk.resource_ios import CRUDS_BY_FOLDER_NAME, RawDatabaseCRUD +from cognite_toolkit._cdf_tk.exceptions import ToolkitEnvError +from cognite_toolkit._cdf_tk.resource_ios import CRUDS_BY_FOLDER_NAME from cognite_toolkit._cdf_tk.tk_warnings import ( - FileReadWarning, MediumSeverityWarning, - MissingFileWarning, - SourceFileModifiedWarning, - ToolkitWarning, - WarningList, ) from cognite_toolkit._cdf_tk.utils import ( YAMLComment, YAMLWithComments, - calculate_hash, flatten_dict, read_yaml_content, safe_read, ) from cognite_toolkit._cdf_tk.utils.modules import parse_user_selected_modules -from cognite_toolkit._version import __version__ -from . import BuiltModuleList -from ._base import ConfigCore, _load_version_variable -from ._built_resources import BuiltResourceList -from ._module_directories import ModuleDirectories, ReadModule +from ._base import ConfigCore if sys.version_info >= (3, 11): from typing import Self @@ -116,38 +102,6 @@ class BuildConfigYAML(ConfigYAMLCore, ConfigCore): filename: ClassVar[str] = "config.{build_env}.yaml" variables: dict[str, Any] = field(default_factory=dict) - def validate_environment(self) -> ToolkitWarning | None: - if _RUNNING_IN_BROWSER: - return None - project = self.environment.project - project_env = os.environ.get("CDF_PROJECT") - if project_env == project: - return None - - is_strict_validation = self.environment.is_strict_validation - env_name = self.environment.name - file_name = self.get_filename(env_name) - missing_message = ( - "No 'CDF_PROJECT' environment variable set. This is expected to match the project " - f"set in environment section of {file_name!r}.\nThis is required for " - "building configurations for staging and prod environments to ensure that you do " - "not accidentally deploy to the wrong project." - ) - mismatch_message = ( - f"Project name mismatch between project set in the environment section of {file_name!r} and the " - f"environment variable 'CDF_PROJECT', {project} ≠ {project_env}.\nThis is required for " - "building configurations for staging and prod environments to ensure that you do not " - "accidentally deploy to the wrong project." - ) - if is_strict_validation and project_env is None: - raise ToolkitEnvError(missing_message) - elif is_strict_validation: - raise ToolkitEnvError(mismatch_message) - elif not is_strict_validation and project_env is None: - return MediumSeverityWarning(missing_message) - else: - return MediumSeverityWarning(mismatch_message) - @classmethod def load(cls, data: dict[str, Any], build_env_name: str, filepath: Path) -> Self: if "environment" not in data: @@ -164,62 +118,6 @@ def load(cls, data: dict[str, Any], build_env_name: str, filepath: Path) -> Self variables = data.get("variables", {}) return cls(environment=environment, variables=variables, filepath=filepath) - def create_build_environment( - self, built_modules: BuiltModuleList, selected_modules: ModuleDirectories - ) -> "BuildEnvironment": - return BuildEnvironment( - name=self.environment.name, - project=self.environment.project, - validation_type=self.environment.validation_type, - selected=self.environment.selected, - cdf_toolkit_version=__version__, - built_resources=built_modules.as_resources_by_folder(), - read_modules=[module.as_read_module() for module in selected_modules], - ) - - def get_selected_modules( - self, - modules_by_package: dict[str, list[str | Path]], - available_modules: set[str | Path], - organization_dir: Path, - verbose: bool, - ) -> list[str | Path]: - selected_packages = [ - package - for package in self.environment.selected - if package in modules_by_package and isinstance(package, str) - ] - if verbose: - print(" [bold green]INFO:[/] Selected packages:") - if len(selected_packages) == 0: - print(" None") - for package in selected_packages: - print(f" {package}") - - selected_modules = [module for module in self.environment.selected if module not in modules_by_package] - if missing := set(selected_modules) - available_modules: - hint = ModuleDefinition.long(missing, organization_dir) - raise ToolkitMissingModuleError( - f"The following selected modules are missing, please check path: {missing}.\n{hint}" - ) - - selected_modules.extend( - itertools.chain.from_iterable(modules_by_package[package] for package in selected_packages) - ) - if not selected_modules: - raise ToolkitEnvError( - f"No selected modules specified in {self.filepath!s}, have you configured " - f"the environment ({self.environment.name})?" - ) - if verbose: - print(" [bold green]INFO:[/] Selected modules:") - for module in selected_modules: - if isinstance(module, Path): - print(f" {module.as_posix()}") - else: - print(f" {module}") - return selected_modules - @classmethod def load_default(cls, organization_dir: Path) -> Self: return cls(filepath=organization_dir / BuildConfigYAML.get_filename(DEFAULT_ENV)) @@ -228,101 +126,6 @@ def dump(self) -> dict[str, Any]: return {"environment": self.environment.dump(), "variables": self.variables} -@dataclass -class BuildEnvironment(Environment): - cdf_toolkit_version: str = __version__ - built_resources: dict[str, BuiltResourceList] = field(default_factory=dict) - read_modules: list[ReadModule] = field(default_factory=list) - - @property - def read_resource_folders(self) -> set[str]: - return {resource_folder for module in self.read_modules for resource_folder in module.resource_directories} - - def dump(self) -> dict[str, Any]: - output = super().dump() - output["cdf_toolkit_version"] = self.cdf_toolkit_version - if self.built_resources: - output["built_resources"] = { - resource_folder: resources.dump(resource_folder, include_destination=True) - for resource_folder, resources in self.built_resources.items() - } - if self.read_modules: - output["read_modules"] = [module.dump() for module in self.read_modules] - return output - - def dump_to_file(self, build_dir: Path) -> None: - (build_dir / BUILD_ENVIRONMENT_FILE).write_text( - "# DO NOT EDIT THIS FILE!\n" + yaml.dump(self.dump(), sort_keys=False, indent=2) - ) - - @classmethod - def load( - cls, data: dict[str, Any], build_name: str | None, action: Literal["build", "deploy", "clean", "pull"] = "build" - ) -> Self: - if "name" in data and build_name is not None and data["name"] != build_name: - raise ToolkitEnvError( - f"Expected to {action} for {build_name!r} environment, but the last " - f"build was created for the {data['name']!r} environment." - ) - build_name = build_name or data.get("name") - - version = _load_version_variable(data, BUILD_ENVIRONMENT_FILE) - _deprecation_selected(data) - built_resources: dict[str, BuiltResourceList] = {} - if "built_resources" in data: - # We expect to dump BuildEnvironment, and load DeployEnvironment - built_resources = { - resource_folder: BuiltResourceList.load(resources, resource_folder) - for resource_folder, resources in data["built_resources"].items() - } - read_modules: list[ReadModule] = [] - if "read_modules" in data: - read_modules = [ReadModule.load(module_data) for module_data in data["read_modules"]] - _deprecate_type(data, build_name or "dev") - try: - return cls( - name=data["name"], - project=data["project"], - validation_type=data["validation-type"], - selected=data["selected"], - cdf_toolkit_version=version, - built_resources=built_resources, - read_modules=read_modules, - ) - except KeyError: - raise ToolkitEnvError( - f" [bold red]ERROR:[/] Environment {build_name} is missing required fields 'name', 'project', 'validation-type', " - f"or 'selected' in {BUILD_ENVIRONMENT_FILE!s}" - ) - - def set_environment_variables(self) -> None: - os.environ["CDF_ENVIRON"] = self.name - os.environ["CDF_BUILD_TYPE"] = self.validation_type - - def check_source_files_changed(self) -> WarningList[FileReadWarning]: - warning_list = WarningList[FileReadWarning]() - for resource_folder, resources in self.built_resources.items(): - if resource_folder == RawDatabaseCRUD.folder_name: - # We modify the hash for RawDatabaseLoader, so we skip checking the hash for this folder. - continue - for resource in resources: - to_check = [resource.source, *(resource.extra_sources or [])] - for source in to_check: - source_filepath = source.path - if source_filepath.suffix in {".csv", ".parquet"}: - # When we copy over the source files we use utf-8 encoding, which can change the file hash. - # Thus, we skip checking the hash for these file types. - continue - - if not source_filepath.exists(): - warning_list.append( - MissingFileWarning(source_filepath, attempted_check="source file has changed") - ) - elif source.hash != calculate_hash(source_filepath, shorten=True): - warning_list.append(SourceFileModifiedWarning(source_filepath)) - return warning_list - - def _deprecation_selected(data: dict[str, Any]) -> None: if "selected_modules_and_packages" in data and "selected" not in data: print( diff --git a/cognite_toolkit/_cdf_tk/data_classes/_deploy_results.py b/cognite_toolkit/_cdf_tk/data_classes/_deploy_results.py index e453a702e5..82f45609e1 100644 --- a/cognite_toolkit/_cdf_tk/data_classes/_deploy_results.py +++ b/cognite_toolkit/_cdf_tk/data_classes/_deploy_results.py @@ -9,9 +9,9 @@ from rich.table import Table if sys.version_info >= (3, 11): - from typing import Self + pass else: - from typing_extensions import Self + pass @total_ordering @@ -52,79 +52,6 @@ def __iadd__(self, other: "ResourceDeployResult") -> "ResourceDeployResult": self.changed += other.changed self.unchanged += other.unchanged self.total += other.total - - if isinstance(other, ResourceContainerDeployResult): - return ResourceContainerDeployResult( - name=self.name, - created=self.created, - deleted=self.deleted, - changed=self.changed, - unchanged=self.unchanged, - total=self.total, - item_name=other.item_name, - dropped_datapoints=other.dropped_datapoints, - ) - else: - return self - - -@dataclass -class ResourceContainerDeployResult(ResourceDeployResult): - item_name: str = "" - dropped_datapoints: int = 0 - - def __iadd__(self, other: ResourceDeployResult) -> "ResourceContainerDeployResult": - if self.name != other.name: - raise ValueError("Cannot add two ResourceContainerDeployResult objects with different names") - super().__iadd__(other) - if isinstance(other, ResourceContainerDeployResult): - self.dropped_datapoints += other.dropped_datapoints - return self - - @classmethod - def from_resource_deploy_result( - cls, result: ResourceDeployResult, item_name: str = "", dropped_datapoints: int = 0 - ) -> Self: - return cls( - name=result.name, - created=result.created, - deleted=result.deleted, - changed=result.changed, - unchanged=result.unchanged, - total=result.total, - item_name=item_name, - dropped_datapoints=dropped_datapoints, - ) - - -@dataclass -class UploadDeployResult(DeployResult): - uploaded: int = 0 - item_name: str = "" - - def __iadd__(self, other: "UploadDeployResult") -> "UploadDeployResult": - if self.name != other.name: - raise ValueError("Cannot add two DeployResult objects with different names") - self.uploaded += other.uploaded - - if isinstance(other, DatapointDeployResult): - return DatapointDeployResult( - name=self.name, uploaded=self.uploaded, item_name=other.item_name, points=other.points - ) - else: - return self - - -@dataclass -class DatapointDeployResult(UploadDeployResult): - points: int = 0 - - def __iadd__(self, other: UploadDeployResult) -> UploadDeployResult: - if self.name != other.name: - raise ValueError("Cannot add two DeployResult objects with different names") - super().__iadd__(other) - if isinstance(other, DatapointDeployResult): - self.points += other.points return self @@ -143,12 +70,6 @@ def __init__( def has_counts(self) -> bool: return any(isinstance(entry, ResourceDeployResult) for entry in self.data.values()) - @property - def has_uploads(self) -> bool: - return any( - isinstance(entry, UploadDeployResult | ResourceContainerDeployResult) for entry in self.data.values() - ) - def counts_table( self, exclude_columns: set[Literal["Created", "Deleted", "Changed", "Untouched", "Total"]] | None = None ) -> Table: @@ -193,35 +114,6 @@ def counts_table( return table - def uploads_table(self) -> Table: - table = Table(title=f"Summary of Data {self.action.title()} operation (data is always uploaded):") - prefix = "Would have " if self.dry_run else "" - table.add_column("Resource", justify="right") - table.add_column(f"{prefix}Uploaded Data", justify="right", style="cyan") - table.add_column("Item Type", justify="right") - table.add_column("From files", justify="right", style="green") - table.add_column(f"{prefix}Deleted Data", justify="right", style="red") - for item in sorted( - entry - for entry in self.data.values() - if isinstance(entry, UploadDeployResult | ResourceContainerDeployResult) - ): - if item.name == "raw.tables": - # We skip this as we cannot count the number of datapoints in a raw table - # and all we can do is to print a misleading 0 for deleted datapoints. - continue - - if isinstance(item, UploadDeployResult): - if isinstance(item, DatapointDeployResult): - datapoints = f"{item.points:,}" - else: - datapoints = "-" - table.add_row(item.name, datapoints, item.item_name, str(item.uploaded), "-") - elif isinstance(item, ResourceContainerDeployResult): - table.add_row(item.name, "-", item.item_name, "-", f"{item.dropped_datapoints:,}") - - return table - def __getitem__(self, item: str) -> DeployResult: return self.data[item] diff --git a/cognite_toolkit/_cdf_tk/data_classes/_issues.py b/cognite_toolkit/_cdf_tk/data_classes/_issues.py deleted file mode 100644 index cf7d9eb736..0000000000 --- a/cognite_toolkit/_cdf_tk/data_classes/_issues.py +++ /dev/null @@ -1,36 +0,0 @@ -import sys -from collections import UserList - -if sys.version_info >= (3, 11): - from typing import Self -else: - from typing_extensions import Self - -from pydantic import BaseModel, Field - -from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning, WarningList - -MODULE_ISSUE_CODE = "MOD" - - -class Issue(BaseModel): - """Base class for all issues""" - - code: str - message: str | None = Field(default=None) - - -# temporary adapter to manage existing warnings -class IssueList(UserList[Issue]): - """List of build issues.""" - - @classmethod - def from_warning_list(cls, warning_list: WarningList[ToolkitWarning]) -> Self: - """Create a IssueList from a WarningList.""" - return cls([Issue(code="WARN", message=warning.get_message()) for warning in warning_list]) - - -class ModuleLoadingIssue(Issue): - """Issue with the loading of a module folder.""" - - code: str = "MOD_001" diff --git a/cognite_toolkit/_cdf_tk/validation.py b/cognite_toolkit/_cdf_tk/validation.py index fd9b7721b6..de1c22e730 100644 --- a/cognite_toolkit/_cdf_tk/validation.py +++ b/cognite_toolkit/_cdf_tk/validation.py @@ -1,25 +1,13 @@ import inspect -import re from pathlib import Path from typing import Any, NamedTuple, TypeVar from pydantic import BaseModel, TypeAdapter, ValidationError from pydantic_core import ErrorDetails -from cognite_toolkit._cdf_tk.cdf_toml import CDFToml from cognite_toolkit._cdf_tk.client._resource_base import ResponseResource -from cognite_toolkit._cdf_tk.constants import DEV_ONLY_MODULES -from cognite_toolkit._cdf_tk.data_classes import BuildConfigYAML, BuildVariables, ModuleDirectories -from cognite_toolkit._cdf_tk.exceptions import ( - ToolkitDuplicatedModuleError, - ToolkitEnvError, - ToolkitMissingModuleError, -) -from cognite_toolkit._cdf_tk.hints import ModuleDefinition from cognite_toolkit._cdf_tk.tk_warnings import ( DataSetMissingWarning, - MediumSeverityWarning, - TemplateVariableWarning, WarningList, ) from cognite_toolkit._cdf_tk.tk_warnings.fileread import ResourceFormatWarning @@ -30,8 +18,6 @@ "humanize_validation_error", "humanize_validation_error_categorized", "validate_data_set_is_set", - "validate_module_selection", - "validate_modules_variables", ] @@ -47,26 +33,6 @@ class _GroupEntry(NamedTuple): loc: tuple[str | int, ...] -def validate_modules_variables(variables: BuildVariables, filepath: Path) -> WarningList: - """Checks whether the config file has any issues. - - Currently, this checks for: - * Non-replaced template variables, such as . - - Args: - variables: The variables to check. - filepath: The filepath of the config.yaml. - """ - warning_list: WarningList = WarningList() - pattern = re.compile(r"<.*?>") - for variable in variables: - if isinstance(variable.value, str) and pattern.match(variable.value): - warning_list.append( - TemplateVariableWarning(filepath, variable.value, variable.key, ".".join(variable.location.parts)) - ) - return warning_list - - def validate_data_set_is_set( raw: dict[str, Any] | list[dict[str, Any]], resource_cls: type[ResponseResource], @@ -310,70 +276,3 @@ def as_json_path(loc: tuple[str | int, ...]) -> str: suffix = ".".join([str(x) if isinstance(x, str) else f"[{x + 1}]" for x in loc]).replace(".[", "[") return f"{prefix}{suffix}" - - -def validate_module_selection( - modules: ModuleDirectories, - config: BuildConfigYAML, - packages: dict[str, list[str]], - selected_modules: set[str | Path], - organization_dir: Path, -) -> WarningList: - """Validates module selection and returns warnings for non-critical issues. - - Critical errors (duplicate modules, missing modules, no modules selected) are still raised - as exceptions as they prevent the build from proceeding. - """ - warnings: WarningList = WarningList() - - # Validations: Ambiguous selection. - selected_names = {s for s in config.environment.selected if isinstance(s, str)} - if duplicate_modules := { - module_name: paths - for module_name, paths in modules.as_path_by_name().items() - if len(paths) > 1 and module_name in selected_names - }: - # If the user has selected a module by name, and there are multiple modules with that name, raise an error. - # Note, if the user uses a path to select a module, this error will not be raised. - raise ToolkitDuplicatedModuleError( - f"Ambiguous module selected in config.{config.environment.name}.yaml:", duplicate_modules - ) - - # Package Referenced Modules Exists - for package, package_modules in packages.items(): - if package not in selected_names: - # We do not check packages that are not selected. - # Typically, the user will delete the modules that are irrelevant for them; - # thus we only check the selected packages. - continue - if missing_packages := set(package_modules) - modules.available_names: - raise ToolkitMissingModuleError( - f"Package {package} defined in {CDFToml.file_name!s} is referring " - f"the following missing modules {missing_packages}." - ) - - # Selected modules does not exists - if missing_modules := set(selected_modules) - modules.available: - hint = ModuleDefinition.long(missing_modules, organization_dir) - raise ToolkitMissingModuleError( - f"The following selected modules are missing, please check path: {missing_modules}.\n{hint}" - ) - - # Nothing is Selected - if not modules.selected: - raise ToolkitEnvError( - f"No selected modules specified in {config.filepath!s}, have you configured " - f"the environment ({config.environment.name})?" - ) - - # Dev modules warning (non-critical) - dev_modules = modules.available_names & DEV_ONLY_MODULES - if dev_modules and config.environment.validation_type != "dev": - warnings.append( - MediumSeverityWarning( - "The following modules should [bold]only[/bold] be used a in CDF Projects designated as dev (development): " - f"{humanize_collection(dev_modules)!r}", - ) - ) - - return warnings diff --git a/tests/test_unit/test_cdf_tk/test_data_classes/test_build_variables.py b/tests/test_unit/test_cdf_tk/test_data_classes/test_build_variables.py deleted file mode 100644 index b15969e4f7..0000000000 --- a/tests/test_unit/test_cdf_tk/test_data_classes/test_build_variables.py +++ /dev/null @@ -1,216 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -import yaml - -from cognite_toolkit._cdf_tk.data_classes import BuildVariables, ModuleLocation - - -class TestBuildVariables: - def test_replace_preserve_data_type(self) -> None: - source_yaml = """text: {{ my_text }} -bool: {{ my_bool }} -integer: {{ my_integer }} -float: {{ my_float }} -digit_string: {{ my_digit_string }} -quoted_string: "{{ my_quoted_string }}" -list: {{ my_list }} -null_value: {{ my_null_value }} -single_quoted_string: '{{ my_single_quoted_string }}' -composite: 'some_prefix_{{ my_composite }}' -prefix_text: {{ my_prefix_text }} -suffix_text: {{ my_suffix_text }} -""" - variables = BuildVariables.load_raw( - { - "my_text": "some text", - "my_bool": True, - "my_integer": 123, - "my_float": 123.456, - "my_digit_string": "123", - "my_quoted_string": "456", - "my_list": ["one", "two", "three"], - "my_null_value": None, - "my_single_quoted_string": "789", - "my_composite": "the suffix", - "my_prefix_text": "prefix:", - "my_suffix_text": ":suffix", - }, - available_modules=set(), - selected_modules=set(), - ) - - result = variables.replace(source_yaml) - - loaded = yaml.safe_load(result) - assert loaded == { - "text": "some text", - "bool": True, - "integer": 123, - "float": 123.456, - "digit_string": "123", - "quoted_string": "456", - "list": ["one", "two", "three"], - "null_value": None, - "single_quoted_string": "789", - "composite": "some_prefix_the suffix", - "prefix_text": "prefix:", - "suffix_text": ":suffix", - } - - def test_replace_not_preserve_type(self) -> None: - source_yaml = """dataset_id('{{dataset_external_id}}')""" - variables = BuildVariables.load_raw( - { - "dataset_external_id": "ds_external_id", - }, - available_modules=set(), - selected_modules=set(), - ) - - result = variables.replace(source_yaml, Path("test.sql")) - - assert result == "dataset_id('ds_external_id')" - - def test_replace_sql_list(self) -> None: - """Test that lists with mixed types in SQL files are formatted correctly.""" - source_sql = """SELECT * FROM table WHERE column IN {{ my_list }}""" - variables = BuildVariables.load_raw( - { - "my_list": ["A", 123, None, True], - }, - available_modules=set(), - selected_modules=set(), - ) - - result = variables.replace(source_sql, Path("test.sql")) - - assert result == "SELECT * FROM table WHERE column IN ('A', 123, NULL, True)" - - def test_replace_inline_sql_preserve_double_quotes(self) -> None: - source_yaml = """externalId: some_id -name: Some Transformation -destination: - type: nodes - view: - space: cdf_cdm - externalId: CogniteTimeSeries - version: v1 - instanceSpace: my_instance_space -query: >- - select "fpso_{{location_id}}" as externalId, "{{location_ID}}" as uid, "{{location_ID}}" as description -""" - variables = BuildVariables.load_raw( - { - "location_id": "uny", - "location_ID": "UNY", - }, - available_modules=set(), - selected_modules=set(), - ) - - result = variables.replace(source_yaml, Path("test.Transformation.yaml")) - - loaded = yaml.safe_load(result) - - assert loaded["query"] == 'select "fpso_uny" as externalId, "UNY" as uid, "UNY" as description' - - def test_replace_yaml_query_field_list(self) -> None: - """Test that lists in multiline query fields are converted to SQL-style tuples.""" - source_yaml = """externalId: some_id -name: Some Transformation -query: | - SELECT * FROM table - WHERE column IN {{ my_list }} - AND other_column = 'value' -""" - variables = BuildVariables.load_raw( - { - "my_list": ["X", "Y", "Z"], - }, - available_modules=set(), - selected_modules=set(), - ) - - result = variables.replace(source_yaml, Path("test.Transformation.yaml")) - - loaded = yaml.safe_load(result) - assert "('X', 'Y', 'Z')" in loaded["query"] - - def test_replace_yaml_query_field_list_mixed_types(self) -> None: - """Test that lists with mixed types in query fields are formatted correctly.""" - source_yaml = """externalId: some_id -name: Some Transformation -query: >- - SELECT * FROM table WHERE column IN {{ my_list }} -""" - variables = BuildVariables.load_raw( - { - "my_list": ["A", 123, None, True], - }, - available_modules=set(), - selected_modules=set(), - ) - - result = variables.replace(source_yaml, Path("test.Transformation.yaml")) - - loaded = yaml.safe_load(result) - assert loaded["query"] == "SELECT * FROM table WHERE column IN ('A', 123, NULL, True)" - - def test_replace_yaml_non_query_field_preserves_list(self) -> None: - """Test that lists outside query fields still use YAML list format.""" - source_yaml = """externalId: some_id -name: Some Transformation -tags: {{ my_list }} -query: >- - SELECT * FROM table -""" - variables = BuildVariables.load_raw( - { - "my_list": ["tag1", "tag2"], - }, - available_modules=set(), - selected_modules=set(), - ) - - result = variables.replace(source_yaml) - - loaded = yaml.safe_load(result) - # Tags should be a YAML list, not SQL tuple - assert loaded["tags"] == ["tag1", "tag2"] - - def test_get_module_variables_variable_preference_order(self) -> None: - source_yaml = """ -modules: - industry_apps: - module_version: '1' - pause_transformations: true - apm_datamodel_space: APM_SourceData - apm_sourcedata_model_version: '1.2.1' - - industry_apps_crna_common: - apm_sourcedata_model_version: '1' -""" - selected = { - Path("."), - Path("modules"), - Path("modules/industry_apps"), - Path("modules/industry_apps/industry_apps_crna_common"), - } - - variables = BuildVariables.load_raw( - yaml.safe_load(source_yaml), available_modules=selected, selected_modules=selected - ) - - assert len(variables) == 5 - location = ModuleLocation( - Path("modules/industry_apps/industry_apps_crna_common"), Path("."), source_paths=[], is_selected=True - ) - local_variables = variables.get_module_variables(location)[0] - - assert len(local_variables) == 4 - apm_sourcedata_model_version = next( - (variable for variable in local_variables if variable.key == "apm_sourcedata_model_version"), None - ) - assert apm_sourcedata_model_version.value == "1" diff --git a/tests/test_unit/test_cdf_tk/test_data_classes/test_built_modules.py b/tests/test_unit/test_cdf_tk/test_data_classes/test_built_modules.py deleted file mode 100644 index 567296e41f..0000000000 --- a/tests/test_unit/test_cdf_tk/test_data_classes/test_built_modules.py +++ /dev/null @@ -1,249 +0,0 @@ -from dataclasses import dataclass -from itertools import groupby -from pathlib import Path - -import pytest - -from cognite_toolkit._cdf_tk.data_classes import ( - BuildVariables, - BuiltModule, - BuiltModuleList, - BuiltResource, - BuiltResourceList, - SourceLocationEager, -) -from cognite_toolkit._cdf_tk.resource_ios import ResourceTypes - - -@dataclass -class GetResourcesArgs: - resource_dir: ResourceTypes - kind: str | None - selected: str | Path | None - - -class TestBuiltModuleList: - # Anchor for absolute paths in tests - anchor = f"{Path.cwd()}/" - - @pytest.mark.parametrize( - "module,args,expected", - [ - pytest.param( - { - Path(f"{anchor}modules/module1"): [ - Path(f"{anchor}modules/module1/transformations/my.Transformation.yaml"), - Path(f"{anchor}modules/module1/transformations/my.Schedule.yaml"), - ] - }, - GetResourcesArgs( - resource_dir="transformations", - kind="Transformation", - selected="module1", - ), - [Path(f"{anchor}modules/module1/transformations/my.Transformation.yaml")], - id="Select by module name", - ), - pytest.param( - { - Path(f"{anchor}modules/module1"): [ - Path(f"{anchor}modules/module1/transformations/my.Transformation.yaml"), - Path(f"{anchor}modules/module1/transformations/my.Schedule.yaml"), - ] - }, - GetResourcesArgs( - resource_dir="transformations", - kind="Transformation", - selected=Path("modules/module1"), - ), - [Path(f"{anchor}modules/module1/transformations/my.Transformation.yaml")], - id="Select with relative with module in absolute", - ), - pytest.param( - { - Path("modules/module1"): [ - Path("modules/module1/transformations/my.Transformation.yaml"), - Path("modules/module1/transformations/my.Schedule.yaml"), - ] - }, - GetResourcesArgs( - resource_dir="transformations", - kind="Transformation", - selected=Path(f"{anchor}modules/module1"), - ), - [Path("modules/module1/transformations/my.Transformation.yaml")], - id="Select with absolute with module in relative", - ), - pytest.param( - { - Path("modules/module1"): [ - Path(f"{anchor}modules/module1/transformations/my.Transformation.yaml"), - Path(f"{anchor}modules/module1/transformations/my.Schedule.yaml"), - ] - }, - GetResourcesArgs( - resource_dir="transformations", - kind="Transformation", - selected=Path(f"{anchor}modules/module1/transformations/my.Transformation.yaml"), - ), - [Path(f"{anchor}modules/module1/transformations/my.Transformation.yaml")], - id="Select file by absolute path", - ), - pytest.param( - { - Path("module1"): [ - Path("modules/module1/transformations/my.Transformation.yaml"), - Path("modules/module1/transformations/my.Schedule.yaml"), - ] - }, - GetResourcesArgs( - resource_dir="transformations", - kind="Transformation", - selected=Path("modules/module1/transformations/my.Transformation.yaml"), - ), - [Path("modules/module1/transformations/my.Transformation.yaml")], - id="Select file by relative path", - ), - pytest.param( - { - Path("modules/module1"): [ - Path(f"{anchor}modules/module1/transformations/my.Transformation.yaml"), - Path(f"{anchor}modules/module1/transformations/my.Schedule.yaml"), - ] - }, - GetResourcesArgs( - resource_dir="transformations", - kind="Transformation", - selected=Path(f"{anchor}modules/module1/transformations"), - ), - [Path(f"{anchor}modules/module1/transformations/my.Transformation.yaml")], - id="Select resource dir by absolute path", - ), - pytest.param( - { - Path("module1"): [ - Path("modules/module1/transformations/my.Transformation.yaml"), - Path("modules/module1/transformations/my.Schedule.yaml"), - ] - }, - GetResourcesArgs( - resource_dir="transformations", - kind="Transformation", - selected=Path("modules/module1/transformations/my.Transformation.yaml"), - ), - [Path("modules/module1/transformations/my.Transformation.yaml")], - id="Select resource dir by relative path", - ), - pytest.param( - { - Path(f"{anchor}module1"): [ - Path(f"{anchor}modules/module1/transformations/my.Transformation.yaml"), - Path(f"{anchor}modules/module1/transformations/my.Schedule.yaml"), - ], - Path(f"{anchor}module2"): [ - Path(f"{anchor}module2/transformations/other.Transformation.yaml"), - ], - }, - GetResourcesArgs( - resource_dir="transformations", - kind="Transformation", - selected=None, - ), - [ - Path(f"{anchor}modules/module1/transformations/my.Transformation.yaml"), - Path(f"{anchor}module2/transformations/other.Transformation.yaml"), - ], - id="Select all by kind", - ), - pytest.param( - { - Path(f"{anchor}module1"): [ - Path(f"{anchor}modules/module1/transformations/my.Transformation.yaml"), - ] - }, - GetResourcesArgs( - resource_dir="transformations", - kind="NonExistentKind", - selected=None, - ), - [], - id="Select with non-existent kind", - ), - pytest.param( - { - Path("modules/module1"): [ - Path(f"{anchor}modules/module1/transformations/my.Transformation.yaml"), - ] - }, - GetResourcesArgs( - resource_dir="transformations", - kind="Transformation", - selected=Path("/modules/module1/transformations/doesnotexist.yaml"), - ), - [], - id="Select with non-existent path", - ), - pytest.param( - { - Path("modules/module1"): [ - Path(f"{anchor}modules/module1/transformations/my.Transformation.yaml"), - ] - }, - GetResourcesArgs( - resource_dir="transformations", - kind="Transformation", - selected="notamodule", - ), - [], - id="Select with non-existent module name", - ), - ], - ) - def test_get_resources_selected( - self, - module: dict[Path, list[Path]], - args: GetResourcesArgs, - expected: list[Path], - ) -> None: - module_list = self._create_built_resource_list(module) - result = module_list.get_resources( - id_type=None, - resource_dir=args.resource_dir, - kind=args.kind, - selected=args.selected, - ) - actual = [item.source.path for item in result] - assert actual == expected - - @staticmethod - def _create_built_resource_list(module: dict[Path, list[Path]]) -> BuiltModuleList: - modules: list[BuiltModule] = [ - BuiltModule( - name=module_path.name, - location=SourceLocationEager(module_path, "hash1234"), - build_variables=BuildVariables([]), - resources={ - resource_dir: BuiltResourceList( - [ - BuiltResource( - identifier=f"resource_{i}", - source=SourceLocationEager(resource_path, "hash5678"), - kind=resource_path.stem.split(".")[-1], - destination=None, - extra_sources=None, - ) - for i, resource_path in enumerate(resource_paths) - ] - ) - for resource_dir, resource_paths in groupby( - sorted(resource_paths, key=lambda p: p.parent.name), key=lambda p: p.parent.name - ) - }, - warning_count=0, - status="success", - iteration=1, - ) - for module_path, resource_paths in module.items() - ] - module_list = BuiltModuleList(modules) - return module_list diff --git a/tests/test_unit/test_cdf_tk/test_utils/test_utils.py b/tests/test_unit/test_cdf_tk/test_utils/test_utils.py index d1cd2181c0..441eceb6ca 100644 --- a/tests/test_unit/test_cdf_tk/test_utils/test_utils.py +++ b/tests/test_unit/test_cdf_tk/test_utils/test_utils.py @@ -10,12 +10,7 @@ import yaml from _pytest.mark import ParameterSet -from cognite_toolkit._cdf_tk.data_classes import BuildVariable, BuildVariables -from cognite_toolkit._cdf_tk.tk_warnings import ( - EnvironmentVariableMissingWarning, - TemplateVariableWarning, - catch_warnings, -) +from cognite_toolkit._cdf_tk.tk_warnings import EnvironmentVariableMissingWarning, catch_warnings from cognite_toolkit._cdf_tk.utils import ( calculate_directory_hash, flatten_dict, @@ -27,7 +22,6 @@ ) from cognite_toolkit._cdf_tk.utils.file import yaml_safe_dump from cognite_toolkit._cdf_tk.utils.modules import module_directory_from_path -from cognite_toolkit._cdf_tk.validation import validate_modules_variables from tests.data import CALC_HASH_DATA, PROJECT_FOR_TEST @@ -52,33 +46,6 @@ def test_warning_when_missing_env_variable(self) -> None: assert warning_list[0] == expected_warning -@pytest.mark.parametrize( - "variable, expected_warnings", - [ - pytest.param( - BuildVariable("sourceId", "", False, Path()), - [TemplateVariableWarning(Path("config.yaml"), "", "sourceId", "")], - id="Single warning", - ), - pytest.param( - BuildVariable("sourceId", "", False, Path("a_module")), - [TemplateVariableWarning(Path("config.yaml"), "", "sourceId", "a_module")], - id="Nested warning", - ), - pytest.param( - BuildVariable("sourceId", "", False, Path("a_super_module/a_module")), - [TemplateVariableWarning(Path("config.yaml"), "", "sourceId", "a_super_module.a_module")], - id="Deep nested warning", - ), - pytest.param(BuildVariable("sourceId", "123", False, Path("a_module")), [], id="No warning"), - ], -) -def test_validate_config_yaml(variable: BuildVariable, expected_warnings: list[TemplateVariableWarning]) -> None: - warnings = validate_modules_variables(BuildVariables([variable]), Path("config.yaml")) - - assert sorted(warnings) == sorted(expected_warnings) - - def test_calculate_hash_on_folder() -> None: folder = CALC_HASH_DATA hash1 = calculate_directory_hash(folder)