diff --git a/cognite_toolkit/_cdf_tk/builders/__init__.py b/cognite_toolkit/_cdf_tk/builders/__init__.py deleted file mode 100644 index 30fea31d5c..0000000000 --- a/cognite_toolkit/_cdf_tk/builders/__init__.py +++ /dev/null @@ -1,42 +0,0 @@ -from collections.abc import Callable -from pathlib import Path - -from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning - -from ._base import Builder, DefaultBuilder, get_resource_crud -from ._datamodels import DataModelBuilder -from ._file import FileBuilder -from ._function import FunctionBuilder -from ._location import LocationBuilder -from ._raw import RawBuilder -from ._ruleset import RuleSetBuilder -from ._streamlit import StreamlitBuilder -from ._transformation import TransformationBuilder - - -def create_builder( - resource_folder: str, - build_dir: Path | None, - warn: Callable[[ToolkitWarning], None] | None = None, -) -> Builder: - if builder_cls := _BUILDER_BY_RESOURCE_FOLDER.get(resource_folder): - return builder_cls(build_dir, warn=warn) # type: ignore[abstract] - - return DefaultBuilder(build_dir, resource_folder, warn) - - -_BUILDER_BY_RESOURCE_FOLDER = {_builder._resource_folder: _builder for _builder in Builder.__subclasses__()} -__all__ = [ - "Builder", - "DataModelBuilder", - "DefaultBuilder", - "FileBuilder", - "FunctionBuilder", - "LocationBuilder", - "RawBuilder", - "RuleSetBuilder", - "StreamlitBuilder", - "TransformationBuilder", - "create_builder", - "get_resource_crud", -] diff --git a/cognite_toolkit/_cdf_tk/builders/_base.py b/cognite_toolkit/_cdf_tk/builders/_base.py deleted file mode 100644 index f48555098c..0000000000 --- a/cognite_toolkit/_cdf_tk/builders/_base.py +++ /dev/null @@ -1,178 +0,0 @@ -import difflib -from abc import ABC, abstractmethod -from collections.abc import Callable, Iterable, Sequence -from pathlib import Path -from typing import Any, ClassVar - -from cognite_toolkit._cdf_tk.constants import INDEX_PATTERN -from cognite_toolkit._cdf_tk.data_classes import ( - BuildDestinationFile, - BuildSourceFile, - BuiltResourceList, - ModuleLocation, -) -from cognite_toolkit._cdf_tk.exceptions import ( - AmbiguousResourceFileError, -) -from cognite_toolkit._cdf_tk.resource_ios import ( - RESOURCE_CRUD_BY_FOLDER_NAME, - GroupIO, - ResourceIO, -) -from cognite_toolkit._cdf_tk.tk_warnings import ( - ToolkitNotSupportedWarning, - ToolkitWarning, - WarningList, -) -from cognite_toolkit._cdf_tk.tk_warnings.fileread import ( - UnknownResourceTypeWarning, -) -from cognite_toolkit._cdf_tk.utils import ( - humanize_collection, -) - - -class Builder(ABC): - _resource_folder: ClassVar[str | None] = None - - def __init__( - self, - build_dir: Path | None, - resource_folder: str | None = None, - warn: Callable[[ToolkitWarning], None] | None = None, - ): - self._build_dir = build_dir - self.warn = warn - self.resource_counter = 0 - if self._resource_folder is not None: - self.resource_folder = self._resource_folder - elif resource_folder is not None: - self.resource_folder = resource_folder - else: - raise ValueError("Either _resource_folder or resource_folder must be set.") - - @property - def build_dir(self) -> Path: - if self._build_dir is None: - raise ValueError("build_dir must be set for this operation.") - return self._build_dir - - @abstractmethod - def build( - self, source_files: list[BuildSourceFile], module: ModuleLocation, console: Callable[[str], None] | None = None - ) -> Iterable[BuildDestinationFile | Sequence[ToolkitWarning]]: - raise NotImplementedError() - - def load_extra_field(self, extra: str) -> tuple[str, Any]: - """Overload in subclass to load extra fields from a file.""" - raise NotImplementedError( - f"Extra field {extra!r} by {type(self).__name__} - {self.resource_folder} is not supported." - ) - - def validate_directory( - self, built_resources: BuiltResourceList, module: ModuleLocation - ) -> WarningList[ToolkitWarning]: - """This can be overridden to add additional validation for the built resources.""" - return WarningList[ToolkitWarning]() - - # Helper methods - def _create_destination_path(self, source_path: Path, kind: str) -> Path: - """Creates the filepath in the build directory for the given source path. - - Note that this is a complex operation as the modules in the source are nested while the build directory is flat. - This means that we lose information and risk having duplicate filenames. To avoid this, we prefix the filename - with a number to ensure uniqueness. - """ - filestem = source_path.stem - # Get rid of the local index - filestem = INDEX_PATTERN.sub("", filestem) - - # Increment to ensure we do not get duplicate filenames when we flatten the file - # structure from the module to the build directory. - self.resource_counter += 1 - - filename = f"{self.resource_counter}.{filestem}" - if not filename.casefold().endswith(kind.casefold()): - filename = f"{filename}.{kind}" - filename = f"{filename}{source_path.suffix}" - destination_path = self.build_dir / self.resource_folder / filename - destination_path.parent.mkdir(parents=True, exist_ok=True) - return destination_path - - def _get_loader(self, source_path: Path) -> tuple[None, ToolkitWarning] | tuple[type[ResourceIO], None]: - return get_resource_crud(source_path, self.resource_folder) - - -def get_resource_crud( - source_path: Path, resource_folder: str -) -> tuple[None, ToolkitWarning] | tuple[type[ResourceIO], None]: - """Get the appropriate CRUD class for the given source file and resource folder.""" - folder_cruds = RESOURCE_CRUD_BY_FOLDER_NAME.get(resource_folder, []) - if not folder_cruds: - return None, ToolkitNotSupportedWarning( - f"resource of type {resource_folder!r} in {source_path.name}.", - details=f"Available resources are: {humanize_collection(RESOURCE_CRUD_BY_FOLDER_NAME.keys())}", - ) - - crud_candidates = [crud_cls for crud_cls in folder_cruds if crud_cls.is_supported_file(source_path)] - if len(crud_candidates) == 0: - suggestion: str | None = None - if "." in source_path.stem: - core, kind = source_path.stem.rsplit(".", 1) - match = difflib.get_close_matches(kind, [crud_cls.kind for crud_cls in folder_cruds]) - if match: - suggested_name = f"{core}.{match[0]}{source_path.suffix}" - suggestion = f"Did you mean to call the file {suggested_name!r}?" - else: - kinds = [crud.kind for crud in folder_cruds] - if len(kinds) == 1: - suggestion = f"Did you mean to call the file '{source_path.stem}.{kinds[0]}{source_path.suffix}'?" - else: - suggestion = ( - f"All files in the {resource_folder!r} folder must have a file extension that matches " - f"the resource type. Supported types are: {humanize_collection(kinds)}." - ) - return None, UnknownResourceTypeWarning(source_path, suggestion) - elif len(crud_candidates) > 1 and all(issubclass(loader, GroupIO) for loader in crud_candidates): - # There are two group cruds, one for resource scoped and one for all scoped. - return GroupIO, None - elif len(crud_candidates) == 1: - return crud_candidates[0], None - - # This is unreachable with our current ResourceCRUD classes. We have tests that is exhaustive over - # all ResourceCRUDs to ensure this. - names = humanize_collection( - [f"'{source_path.stem}.{loader.kind}{source_path.suffix}'" for loader in crud_candidates], bind_word="or" - ) - raise AmbiguousResourceFileError( - f"Ambiguous resource file {source_path.name} in {resource_folder} folder. " - f"Unclear whether it is {humanize_collection([crud_cls.kind for crud_cls in crud_candidates], bind_word='or')}." - f"\nPlease name the file {names}." - ) - - -class DefaultBuilder(Builder): - """This is used to build resources that do not have a specific builder.""" - - def build( - self, source_files: list[BuildSourceFile], module: ModuleLocation, console: Callable[[str], None] | None = None - ) -> Iterable[BuildDestinationFile | list[ToolkitWarning]]: - for source_file in source_files: - if source_file.loaded is None: - # Not a YAML file - continue - loader, warning = self._get_loader(source_file.source.path) - if loader is None: - if warning is not None: - yield [warning] - continue - destination_path = self._create_destination_path(source_file.source.path, loader.kind) - - destination = BuildDestinationFile( - path=destination_path, - loaded=source_file.loaded, - loader=loader, - source=source_file.source, - extra_sources=None, - ) - yield destination diff --git a/cognite_toolkit/_cdf_tk/builders/_datamodels.py b/cognite_toolkit/_cdf_tk/builders/_datamodels.py deleted file mode 100644 index 7b0f7e4b75..0000000000 --- a/cognite_toolkit/_cdf_tk/builders/_datamodels.py +++ /dev/null @@ -1,90 +0,0 @@ -import shutil -from collections.abc import Callable, Iterable -from pathlib import Path -from typing import Any - -from cognite_toolkit._cdf_tk.builders import Builder -from cognite_toolkit._cdf_tk.constants import INDEX_PATTERN -from cognite_toolkit._cdf_tk.data_classes import ( - BuildDestinationFile, - BuildSourceFile, - ModuleLocation, - SourceLocation, -) -from cognite_toolkit._cdf_tk.exceptions import ToolkitFileNotFoundError -from cognite_toolkit._cdf_tk.resource_ios import GraphQLCRUD -from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning - - -class DataModelBuilder(Builder): - _resource_folder = GraphQLCRUD.folder_name - - def build( - self, source_files: list[BuildSourceFile], module: ModuleLocation, console: Callable[[str], None] | None = None - ) -> Iterable[BuildDestinationFile | list[ToolkitWarning]]: - graphql_files = { - source_file.source.path: source_file - for source_file in source_files - if source_file.source.path.suffix == ".graphql" - } - - for source_file in source_files: - loaded = source_file.loaded - if loaded is None: - # Not a YAML file - continue - loader, warning = self._get_loader(source_file.source.path) - if loader is None: - if warning is not None: - yield [warning] - continue - - destination_path = self._create_destination_path(source_file.source.path, loader.kind) - - extra_sources: list[SourceLocation] | None = None - if loader is GraphQLCRUD: - # The GraphQL must be copied over instead of added to the DML field as - # it is hashed in the deployment step and used to determine if the DML has changed. - extra_sources = self._copy_graphql_to_build(source_file, destination_path, graphql_files) - - destination = BuildDestinationFile( - path=destination_path, - loaded=loaded, - loader=loader, - source=source_file.source, - extra_sources=extra_sources, - ) - yield destination - - def _copy_graphql_to_build( - self, - source_file: BuildSourceFile, - destination_path: Path, - graphql_files: dict[Path, BuildSourceFile], - ) -> list[SourceLocation]: - extra_sources: list[SourceLocation] = [] - loaded_list: list[dict[str, Any]] = ( - source_file.loaded if isinstance(source_file.loaded, list) else [source_file.loaded] # type: ignore[list-item] - ) - - for entry in loaded_list: - if "dml" in entry: - expected_filename = entry["dml"] - else: - expected_filename = f"{INDEX_PATTERN.sub('', source_file.source.path.stem.removesuffix(GraphQLCRUD.kind).removesuffix('.'))}.graphql" - expected_path = source_file.source.path.parent / Path(expected_filename) - - if expected_path in graphql_files: - dest_graphql = destination_path.with_suffix(".graphql") - shutil.copy(graphql_files[expected_path].source.path, dest_graphql) - extra_sources.append(graphql_files[expected_path].source) - # The build renames the .graphql file; update dml so deploy can locate it. - entry["dml"] = dest_graphql.name - else: - raise ToolkitFileNotFoundError( - f"Failed to find GraphQL file. Expected {expected_filename} adjacent to {source_file.source.path.as_posix()}" - ) - return extra_sources - - def load_extra_field(self, extra: str) -> tuple[str, Any]: - return "dml", extra diff --git a/cognite_toolkit/_cdf_tk/builders/_file.py b/cognite_toolkit/_cdf_tk/builders/_file.py deleted file mode 100644 index a074ec9b6a..0000000000 --- a/cognite_toolkit/_cdf_tk/builders/_file.py +++ /dev/null @@ -1,88 +0,0 @@ -import copy -from collections.abc import Callable, Iterable -from typing import Any - -from cognite_toolkit._cdf_tk.builders import Builder -from cognite_toolkit._cdf_tk.data_classes import ( - BuildDestinationFile, - BuildSourceFile, - ModuleLocation, -) -from cognite_toolkit._cdf_tk.exceptions import ToolkitYAMLFormatError -from cognite_toolkit._cdf_tk.resource_ios import CogniteFileCRUD, FileCRUD, FileMetadataCRUD -from cognite_toolkit._cdf_tk.tk_warnings import LowSeverityWarning, ToolkitWarning - - -class FileBuilder(Builder): - _resource_folder = FileMetadataCRUD.folder_name - template_pattern = "$FILENAME" - - def build( - self, source_files: list[BuildSourceFile], module: ModuleLocation, console: Callable[[str], None] | None = None - ) -> Iterable[BuildDestinationFile | list[ToolkitWarning]]: - for source_file in source_files: - loaded = source_file.loaded - if loaded is None: - continue - - loader, warning = self._get_loader(source_file.source.path) - if loader is None: - if warning is not None: - yield [warning] - continue - if loader in {FileMetadataCRUD, CogniteFileCRUD}: - loaded = self._expand_file_metadata(loaded, module, console) - destination_path = self._create_destination_path(source_file.source.path, loader.kind) - - yield BuildDestinationFile( - path=destination_path, - loaded=loaded, - loader=loader, - source=source_file.source, - extra_sources=None, - ) - - @classmethod - def _expand_file_metadata( - cls, - raw_list: list[dict[str, Any]] | dict[str, Any], - module: ModuleLocation, - console: Callable[[str], None] | None = None, - ) -> list[dict[str, Any]] | dict[str, Any]: - is_file_template = ( - isinstance(raw_list, list) - and len(raw_list) == 1 - and cls.template_pattern in raw_list[0].get("externalId", "") - ) - if not is_file_template: - if (isinstance(raw_list, dict) and cls.template_pattern in raw_list.get("externalId", "")) or ( - isinstance(raw_list, list) - and any(cls.template_pattern in entry.get("externalId", "") for entry in raw_list) - ): - raw_type = "dictionary" if isinstance(raw_list, dict) else "list with multiple entries" - LowSeverityWarning( - f"Invalid file template {cls.template_pattern!r} usage detected in {module.relative_path.as_posix()!r}.\n" - f"The file template is expected in a list with a single entry, but got {raw_type}." - ).print_warning() - - return raw_list - if not (isinstance(raw_list, list) and raw_list and isinstance(raw_list[0], dict)): - raise ToolkitYAMLFormatError( - f"Expected a list with a single dictionary in the file metadata file {module.dir}, " - f"but got {type(raw_list).__name__}" - ) - template = raw_list[0] - if console: - console( - f"Detected file template name {cls.template_pattern!r} in {module.relative_path.as_posix()!r}" - f"Expanding file metadata..." - ) - expanded_metadata: list[dict[str, Any]] = [] - for filepath in module.source_paths_by_resource_folder[FileCRUD.folder_name]: - if not FileCRUD.is_supported_file(filepath): - continue - new_entry = copy.deepcopy(template) - new_entry["externalId"] = new_entry["externalId"].replace(cls.template_pattern, filepath.name) - new_entry["name"] = filepath.name - expanded_metadata.append(new_entry) - return expanded_metadata diff --git a/cognite_toolkit/_cdf_tk/builders/_function.py b/cognite_toolkit/_cdf_tk/builders/_function.py deleted file mode 100644 index cf1ee86576..0000000000 --- a/cognite_toolkit/_cdf_tk/builders/_function.py +++ /dev/null @@ -1,187 +0,0 @@ -import shutil -import time -from collections.abc import Callable, Iterable, Sequence -from pathlib import Path -from typing import Any - -from cognite_toolkit._cdf_tk.builders import Builder -from cognite_toolkit._cdf_tk.data_classes import ( - BuildDestinationFile, - BuildSourceFile, - BuiltResourceList, - ModuleLocation, -) -from cognite_toolkit._cdf_tk.exceptions import ToolkitFileExistsError, ToolkitNotADirectoryError, ToolkitValueError -from cognite_toolkit._cdf_tk.feature_flags import Flags -from cognite_toolkit._cdf_tk.resource_ios import FunctionIO -from cognite_toolkit._cdf_tk.tk_warnings import ( - FileReadWarning, - HighSeverityWarning, - LowSeverityWarning, - MediumSeverityWarning, - RequirementsTXTValidationWarning, - ToolkitWarning, - WarningList, -) -from cognite_toolkit._cdf_tk.utils import validate_requirements_with_pip - - -class FunctionBuilder(Builder): - _resource_folder = FunctionIO.folder_name - - def __init__(self, build_dir: Path, warn: Callable[[ToolkitWarning], None]) -> None: - super().__init__(build_dir, warn=warn) - # Metrics for telemetry - self.validation_count = 0 - self.validation_failures = 0 - self.validation_credential_errors = 0 - self.validation_time_ms = 0 - - def _validate_function_requirements( - self, - requirements_txt: Path, - raw_function: dict[str, Any], - filepath: Path, - external_id: str, - ) -> RequirementsTXTValidationWarning | None: - """Validate function requirements.txt using pip dry-run.""" - start_time = time.time() - validation_result = validate_requirements_with_pip( - requirements_txt_path=requirements_txt, - index_url=raw_function.get("indexUrl"), - extra_index_urls=raw_function.get("extraIndexUrls"), - ) - elapsed_ms = int((time.time() - start_time) * 1000) - self.validation_count += 1 - self.validation_time_ms += elapsed_ms - - if validation_result.success: - return None - - self.validation_failures += 1 - if validation_result.is_credential_error: - self.validation_credential_errors += 1 - - return RequirementsTXTValidationWarning( - filepath=filepath, - external_id=external_id, - error_details=validation_result.short_error, - is_credential_error=validation_result.is_credential_error, - resource="function", - ) - - def build( - self, - source_files: list[BuildSourceFile], - module: ModuleLocation, - console: Callable[[str], None] | None = None, - ) -> Iterable[BuildDestinationFile | Sequence[ToolkitWarning]]: - for source_file in source_files: - if source_file.loaded is None: - continue - if source_file.source.path.parent.parent != module.dir: - # Function YAML files must be in the resource folder. - continue - - loader, warning = self._get_loader(source_file.source.path) - if loader is None: - if warning is not None: - yield [warning] - continue - - warnings = WarningList[FileReadWarning]() - if loader is FunctionIO: - warnings = self.copy_function_directory_to_build(source_file) - - destination_path = self._create_destination_path(source_file.source.path, loader.kind) - - yield BuildDestinationFile( - path=destination_path, - loaded=source_file.loaded, - loader=loader, - source=source_file.source, - extra_sources=None, - warnings=warnings, - ) - - def validate_directory( - self, - built_resources: BuiltResourceList, - module: ModuleLocation, - ) -> WarningList[ToolkitWarning]: - warnings = WarningList[ToolkitWarning]() - has_config_files = any(resource.kind == FunctionIO.kind for resource in built_resources) - if has_config_files: - return warnings - config_files_misplaced = [ - file - for file in module.source_paths_by_resource_folder[FunctionIO.folder_name] - if FunctionIO.is_supported_file(file) - ] - if config_files_misplaced: # and not has_config_files: - for yaml_source_path in config_files_misplaced: - required_location = module.dir / FunctionIO.folder_name / yaml_source_path.name - warning = LowSeverityWarning( - f"The required Function resource configuration file " - f"was not found in {required_location.as_posix()!r}. " - f"The file {yaml_source_path.as_posix()!r} is currently " - f"considered part of the Function's artifacts and " - f"will not be processed by the Toolkit.", - ) - warnings.append(warning) - return warnings - - def copy_function_directory_to_build(self, source_file: BuildSourceFile) -> WarningList[FileReadWarning]: - raw_content = source_file.loaded - if raw_content is None: - # This should already be checked before calling this method. - raise ToolkitValueError("Function source file should be a YAML file.") - raw_functions = raw_content if isinstance(raw_content, list) else [raw_content] - warnings = WarningList[FileReadWarning]() - for raw_function in raw_functions: - external_id = raw_function.get("externalId") - function_path = raw_function.get("functionPath") - if not external_id: - warnings.append( - HighSeverityWarning( - f"Function in {source_file.source.path.as_posix()!r} has no externalId defined. " - f"This is used to match the function to the function directory.", - ), - ) - continue - if not function_path: - warnings.append( - MediumSeverityWarning( - f"Function {external_id} in {source_file.source.path.as_posix()!r} has no function_path defined.", - ), - ) - - function_directory = source_file.source.path.with_name(external_id) - - if not function_directory.is_dir(): - raise ToolkitNotADirectoryError( - f"Function directory not found for externalId {external_id} defined in {source_file.source.path.as_posix()!r}.", - ) - - # Validate requirements.txt if present and feature is enabled - if ( - Flags.FUNCTION_REQUIREMENTS_VALIDATION.is_enabled() - and (requirements_txt := function_directory / "requirements.txt").exists() - ): - warning = self._validate_function_requirements( - requirements_txt, - raw_function, - source_file.source.path, - external_id, - ) - if warning: - warnings.append(warning) - - destination = self.build_dir / self.resource_folder / external_id - if destination.exists(): - raise ToolkitFileExistsError( - f"Function {external_id!r} is duplicated. If this is unexpected, ensure you have a clean build directory.", - ) - shutil.copytree(function_directory, destination, ignore=shutil.ignore_patterns("__pycache__")) - - return warnings diff --git a/cognite_toolkit/_cdf_tk/builders/_location.py b/cognite_toolkit/_cdf_tk/builders/_location.py deleted file mode 100644 index ca4c7bbf1a..0000000000 --- a/cognite_toolkit/_cdf_tk/builders/_location.py +++ /dev/null @@ -1,84 +0,0 @@ -from collections.abc import Callable, Iterable, Sequence -from graphlib import CycleError, TopologicalSorter -from typing import Any - -from cognite_toolkit._cdf_tk.builders._base import Builder -from cognite_toolkit._cdf_tk.data_classes._build_files import BuildDestinationFile, BuildSourceFile -from cognite_toolkit._cdf_tk.data_classes._module_directories import ModuleLocation -from cognite_toolkit._cdf_tk.exceptions import ToolkitError -from cognite_toolkit._cdf_tk.resource_ios._resource_ios.location import LocationFilterIO -from cognite_toolkit._cdf_tk.tk_warnings.base import ToolkitWarning, WarningList -from cognite_toolkit._cdf_tk.tk_warnings.fileread import FileReadWarning - - -class LocationBuilder(Builder): - _resource_folder = LocationFilterIO.folder_name - - def build( - self, source_files: list[BuildSourceFile], module: ModuleLocation, console: Callable[[str], None] | None = None - ) -> Iterable[BuildDestinationFile | Sequence[ToolkitWarning]]: - location_by_external_id: dict[str, tuple[dict[str, Any], BuildSourceFile]] = {} - location_hierarchy_graph: dict[str, list[Any]] = {} - - # Ordering all location filters in to ensure correct hierarchy dependency - # within the module. This is required by the Location API. - # Doing this in three stages: - # 1. collect all locations across source files, - # 2. sort them in a topological order, - # 3. create a new file for each location where the prefix index ensures deployment order - # ... while also maintaining reference to source file - - for source_file in source_files: - loader, warning = self._get_loader(source_file.source.path) - if isinstance(loader, LocationFilterIO): - if warning is not None: - yield [warning] - continue - - loaded_locations = ( - source_file.loaded - if isinstance(source_file.loaded, list) - else [source_file.loaded] - if source_file.loaded - else [] - ) - for loaded_location in loaded_locations: - ext_id = loaded_location.get("externalId") - parent_external_id = loaded_location.get("parentExternalId") - - if ext_id: - location_by_external_id[ext_id] = loaded_location, source_file - location_hierarchy_graph.setdefault(ext_id, []) - - if parent_external_id: - location_hierarchy_graph.setdefault(parent_external_id, []) - location_hierarchy_graph[ext_id].append(parent_external_id) - - warnings = WarningList[FileReadWarning]() - - ordered_locations: list[dict] = [] - try: - for external_id in TopologicalSorter(location_hierarchy_graph).static_order(): - if external_id not in location_by_external_id: - # The dependency is not in the module, so we skip it. - continue - location, _ = location_by_external_id[external_id] - ordered_locations.append(location) - except CycleError: - raise ToolkitError( - "Circular dependency found in Locations. Locations must be hierarchical. Please check the externalId and parentExternalId fields." - ) - - for item in ordered_locations: - external_id = item["externalId"] - (location, build_source_file) = location_by_external_id[external_id] - destination_path = self._create_destination_path(build_source_file.source.path, loader.kind) # type: ignore[union-attr] - - yield BuildDestinationFile( - path=destination_path, - loaded=location, - loader=loader, # type: ignore[arg-type] - source=build_source_file.source, - extra_sources=None, - warnings=warnings, - ) diff --git a/cognite_toolkit/_cdf_tk/builders/_raw.py b/cognite_toolkit/_cdf_tk/builders/_raw.py deleted file mode 100644 index 012cc248b2..0000000000 --- a/cognite_toolkit/_cdf_tk/builders/_raw.py +++ /dev/null @@ -1,72 +0,0 @@ -from collections import defaultdict -from collections.abc import Callable, Iterable, Sequence -from typing import Any - -from cognite_toolkit._cdf_tk.builders import Builder -from cognite_toolkit._cdf_tk.client.identifiers import RawDatabaseId -from cognite_toolkit._cdf_tk.data_classes import ( - BuildDestinationFile, - BuildSourceFile, - ModuleLocation, - SourceLocation, - SourceLocationEager, -) -from cognite_toolkit._cdf_tk.resource_ios import RawDatabaseCRUD, RawTableCRUD, ResourceIO -from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning -from cognite_toolkit._cdf_tk.utils import calculate_hash -from cognite_toolkit._cdf_tk.utils.file import yaml_safe_dump - - -class RawBuilder(Builder): - _resource_folder = RawDatabaseCRUD.folder_name - - def build( - self, source_files: list[BuildSourceFile], module: ModuleLocation, console: Callable[[str], None] | None = None - ) -> Iterable[BuildDestinationFile | Sequence[ToolkitWarning]]: - for source_file in source_files: - loaded = source_file.loaded - if loaded is None: - continue - loaded_list = loaded if isinstance(loaded, list) else [loaded] - seen_databases: set[tuple] = set() - entry_by_loader: dict[type[ResourceIO], list[dict[str, Any]]] = defaultdict(list) - has_split_table_and_database = False - - for item in loaded_list: - try: - table_id = RawTableCRUD.get_id(item) - except KeyError: - seen_databases.add(tuple(item.items())) - entry_by_loader[RawDatabaseCRUD].append(item) - else: - entry_by_loader[RawTableCRUD].append(item) - db_item = RawDatabaseCRUD.dump_id(RawDatabaseId(name=table_id.db_name)) - hashable_db_item = tuple(db_item.items()) - if hashable_db_item not in seen_databases: - seen_databases.add(hashable_db_item) - entry_by_loader[RawDatabaseCRUD].append(db_item) - has_split_table_and_database = True - - for loader, entries in entry_by_loader.items(): - if not entries: - continue - destination_path = self._create_destination_path(source_file.source.path, loader.kind) - - if loader is RawDatabaseCRUD and has_split_table_and_database: - # We have inferred the database from a Table file, so we need to recalculate the hash - # in case we also inferred the database from another Table file - new_hash = calculate_hash( - yaml_safe_dump(sorted(entries, key=lambda entry: entry["dbName"])), - shorten=True, - ) - source: SourceLocation = SourceLocationEager(path=source_file.source.path, _hash=new_hash) - else: - source = source_file.source - - yield BuildDestinationFile( - path=destination_path, - loaded=entries, - loader=loader, - source=source, - extra_sources=None, - ) diff --git a/cognite_toolkit/_cdf_tk/builders/_ruleset.py b/cognite_toolkit/_cdf_tk/builders/_ruleset.py deleted file mode 100644 index 8572c34bd2..0000000000 --- a/cognite_toolkit/_cdf_tk/builders/_ruleset.py +++ /dev/null @@ -1,105 +0,0 @@ -from collections.abc import Callable, Iterable -from pathlib import Path -from typing import Any - -from cognite_toolkit._cdf_tk.builders import Builder -from cognite_toolkit._cdf_tk.constants import BUILD_FOLDER_ENCODING -from cognite_toolkit._cdf_tk.data_classes import ( - BuildDestinationFile, - BuildSourceFile, - ModuleLocation, - SourceLocation, -) -from cognite_toolkit._cdf_tk.exceptions import ToolkitFileNotFoundError, ToolkitYAMLFormatError -from cognite_toolkit._cdf_tk.resource_ios import RuleSetVersionIO -from cognite_toolkit._cdf_tk.tk_warnings import ToolkitWarning -from cognite_toolkit._cdf_tk.utils import safe_write - - -class RuleSetBuilder(Builder): - _resource_folder = RuleSetVersionIO.folder_name - - def build( - self, source_files: list[BuildSourceFile], module: ModuleLocation, console: Callable[[str], None] | None = None - ) -> Iterable[BuildDestinationFile | list[ToolkitWarning]]: - ttl_files = { - source_file.source.path: source_file - for source_file in source_files - if source_file.source.path.suffix == ".ttl" - } - - for source_file in source_files: - loaded = source_file.loaded - if loaded is None: - continue - loader, warning = self._get_loader(source_file.source.path) - if loader is None: - if warning is not None: - yield [warning] - continue - - destination_path = self._create_destination_path(source_file.source.path, loader.kind) - - extra_sources: list[SourceLocation] | None = None - if loader is RuleSetVersionIO: - extra_sources = self._add_rules(loaded, source_file, ttl_files, destination_path) - - destination = BuildDestinationFile( - path=destination_path, - loaded=loaded, - loader=loader, - source=source_file.source, - extra_sources=extra_sources, - ) - yield destination - - def load_extra_field(self, extra: str) -> tuple[str, Any]: - return "rules", [extra] - - def _add_rules( - self, - loaded: dict[str, Any] | list[dict[str, Any]], - source_file: BuildSourceFile, - ttl_files: dict[Path, BuildSourceFile], - ruleset_destination_path: Path, - ) -> list[SourceLocation]: - loaded_list = loaded if isinstance(loaded, list) else [loaded] - extra_sources: list[SourceLocation] = [] - for entry in loaded_list: - try: - id_ = RuleSetVersionIO.get_id(entry) - except KeyError: - continue - filepath = source_file.source.path - ttl_file = self._get_ttl_file(filepath, id_.rule_set_external_id, ttl_files) - - if "rules" in entry and ttl_file is not None: - raise ToolkitYAMLFormatError( - f"'rules' is defined in both the YAML and a separate file named {ttl_file.source.path}\n" - f"Please remove one: either the inline 'rules' in {filepath} or the file {ttl_file.source.path}", - ) - if "rules" not in entry and ttl_file is None: - raise ToolkitFileNotFoundError( - f"'rules' is missing and no .ttl file found. Expected {filepath.stem}.ttl or {id_.rule_set_external_id}.ttl next to {filepath}", - filepath, - ) - if ttl_file is not None: - destination_path = self._create_destination_path(ttl_file.source.path, "Rules") - safe_write(destination_path, ttl_file.content, encoding=BUILD_FOLDER_ENCODING) - entry["rules"] = [ttl_file.content] - extra_sources.append(ttl_file.source) - - return extra_sources - - @staticmethod - def _get_ttl_file( - source_file: Path, rule_set_external_id: str | None, ttl_files: dict[Path, BuildSourceFile] - ) -> BuildSourceFile | None: - ttl_path = source_file.parent / f"{source_file.stem}.ttl" - if ttl_path in ttl_files: - return ttl_files[ttl_path] - if rule_set_external_id: - ttl_path = source_file.parent / f"{rule_set_external_id}.ttl" - if ttl_path in ttl_files: - return ttl_files[ttl_path] - return None diff --git a/cognite_toolkit/_cdf_tk/builders/_streamlit.py b/cognite_toolkit/_cdf_tk/builders/_streamlit.py deleted file mode 100644 index 66571aab5f..0000000000 --- a/cognite_toolkit/_cdf_tk/builders/_streamlit.py +++ /dev/null @@ -1,110 +0,0 @@ -import shutil -from collections.abc import Callable, Iterable, Sequence - -from cognite_toolkit._cdf_tk.builders import Builder -from cognite_toolkit._cdf_tk.data_classes import ( - BuildDestinationFile, - BuildSourceFile, - ModuleLocation, -) -from cognite_toolkit._cdf_tk.exceptions import ToolkitFileExistsError, ToolkitNotADirectoryError, ToolkitValueError -from cognite_toolkit._cdf_tk.feature_flags import Flags -from cognite_toolkit._cdf_tk.resource_ios import StreamlitIO -from cognite_toolkit._cdf_tk.tk_warnings import ( - FileReadWarning, - HighSeverityWarning, - RequirementsTXTValidationWarning, - StreamlitRequirementsWarning, - ToolkitWarning, - WarningList, -) -from cognite_toolkit._cdf_tk.utils import validate_requirements_with_pip -from cognite_toolkit._cdf_tk.utils.file import safe_read - - -class StreamlitBuilder(Builder): - _resource_folder = StreamlitIO.folder_name - - def build( - self, source_files: list[BuildSourceFile], module: ModuleLocation, console: Callable[[str], None] | None = None - ) -> Iterable[BuildDestinationFile | Sequence[ToolkitWarning]]: - for source_file in source_files: - if source_file.loaded is None: - continue - if source_file.source.path.parent.parent != module.dir: - # Streamlit YAML files must be in the resource folder top level - continue - - loader, warning = self._get_loader(source_file.source.path) - if loader is None: - if warning is not None: - yield [warning] - continue - - warnings = WarningList[FileReadWarning]() - if loader is StreamlitIO: - warnings = self.copy_app_directory_to_build(source_file) - - destination_path = self._create_destination_path(source_file.source.path, loader.kind) - - yield BuildDestinationFile( - path=destination_path, - loaded=source_file.loaded, - loader=loader, - source=source_file.source, - extra_sources=None, - warnings=warnings, - ) - - def copy_app_directory_to_build(self, source_file: BuildSourceFile) -> WarningList[FileReadWarning]: - raw_content = source_file.loaded - if raw_content is None: - # This should already be checked before calling this method. - raise ToolkitValueError("Streamlit source file should be a YAML file.") - raw_apps = raw_content if isinstance(raw_content, list) else [raw_content] - warnings = WarningList[FileReadWarning]() - for raw_app in raw_apps: - external_id = raw_app.get("externalId") - if not external_id: - warnings.append( - HighSeverityWarning( - f"StreamlitApp in {source_file.source.path.as_posix()!r} has no externalId defined. " - f"This is used to match the Streamlit App to directory." - ) - ) - continue - app_directory = source_file.source.path.with_name(external_id) - - if not app_directory.is_dir(): - raise ToolkitNotADirectoryError( - f"StreamlitApp directory not found in {app_directory}(based on externalId {external_id} defined in {source_file.source.path.as_posix()!r}.)" - ) - - if (requirements_txt := app_directory / "requirements.txt").exists() and ( - Flags.FUNCTION_REQUIREMENTS_VALIDATION.is_enabled() - ): - validation_result = validate_requirements_with_pip(requirements_txt_path=requirements_txt) - if not validation_result.success: - warnings.append( - RequirementsTXTValidationWarning( - filepath=source_file.source.path, - error_details=validation_result.short_error, - is_credential_error=validation_result.is_credential_error, - external_id=external_id, - resource="streamlit", - ) - ) - - requirements_file_content = safe_read(app_directory / "requirements.txt").splitlines() - missing_packages = StreamlitIO._missing_recommended_requirements(requirements_file_content) - if len(missing_packages) > 0: - warnings.append(StreamlitRequirementsWarning(app_directory / "requirements.txt", missing_packages)) - - destination = self.build_dir / self.resource_folder / external_id - if destination.exists(): - raise ToolkitFileExistsError( - f"StreamlitApp {external_id!r} is duplicated. If this is unexpected, ensure you have a clean build directory." - ) - shutil.copytree(app_directory, destination, ignore=shutil.ignore_patterns("__pycache__")) - - return warnings diff --git a/cognite_toolkit/_cdf_tk/builders/_transformation.py b/cognite_toolkit/_cdf_tk/builders/_transformation.py deleted file mode 100644 index 1132bd7e36..0000000000 --- a/cognite_toolkit/_cdf_tk/builders/_transformation.py +++ /dev/null @@ -1,111 +0,0 @@ -from collections.abc import Callable, Iterable -from pathlib import Path -from typing import Any - -from cognite_toolkit._cdf_tk.builders import Builder -from cognite_toolkit._cdf_tk.constants import BUILD_FOLDER_ENCODING -from cognite_toolkit._cdf_tk.data_classes import ( - BuildDestinationFile, - BuildSourceFile, - ModuleLocation, - SourceLocation, -) -from cognite_toolkit._cdf_tk.exceptions import ToolkitYAMLFormatError -from cognite_toolkit._cdf_tk.resource_ios import TransformationIO -from cognite_toolkit._cdf_tk.tk_warnings import HighSeverityWarning, ToolkitWarning -from cognite_toolkit._cdf_tk.utils import safe_write - - -class TransformationBuilder(Builder): - _resource_folder = TransformationIO.folder_name - - def build( - self, source_files: list[BuildSourceFile], module: ModuleLocation, console: Callable[[str], None] | None = None - ) -> Iterable[BuildDestinationFile | list[ToolkitWarning]]: - query_files = { - source_file.source.path: source_file - for source_file in source_files - if source_file.source.path.suffix == ".sql" - } - - for source_file in source_files: - loaded = source_file.loaded - if loaded is None: - # Not a YAML file - continue - loader, warning = self._get_loader(source_file.source.path) - if loader is None: - if warning is not None: - yield [warning] - continue - - destination_path = self._create_destination_path(source_file.source.path, loader.kind) - - extra_sources: list[SourceLocation] | None = None - if loader is TransformationIO: - extra_sources = self._add_query(loaded, source_file, query_files, destination_path) - - destination = BuildDestinationFile( - path=destination_path, - loaded=loaded, - loader=loader, - source=source_file.source, - extra_sources=extra_sources, - ) - yield destination - - def load_extra_field(self, extra: str) -> tuple[str, Any]: - return "query", extra - - def _add_query( - self, - loaded: dict[str, Any] | list[dict[str, Any]], - source_file: BuildSourceFile, - query_files: dict[Path, BuildSourceFile], - transformation_destination_path: Path, - ) -> list[SourceLocation]: - loaded_list = loaded if isinstance(loaded, list) else [loaded] - extra_sources: list[SourceLocation] = [] - for entry in loaded_list: - try: - id_ = TransformationIO.get_id(entry) - except KeyError: - # This will be validated later - continue - filepath = source_file.source.path - query_file = self._get_query_file(filepath, id_.external_id, query_files) - - if "query" in entry and query_file is not None: - raise ToolkitYAMLFormatError( - f"query property is ambiguously defined in both the yaml file and a separate file named {query_file}\n" - f"Please remove one of the definitions, either the query property in {filepath} or the file {query_file}", - ) - elif "query" not in entry and query_file is None: - warning = HighSeverityWarning( - f"query property or is missing in {filepath.as_posix()!r}. It can be inline or a separate file named {filepath.stem}.sql or {id_}.sql", - ) - if self.warn: - self.warn(warning) - else: - warning.print_warning() - elif query_file is not None: - destination_path = self._create_destination_path(query_file.source.path, "Query") - safe_write(destination_path, query_file.content, encoding=BUILD_FOLDER_ENCODING) - relative = destination_path.relative_to(transformation_destination_path.parent) - entry["queryFile"] = relative.as_posix() - extra_sources.append(query_file.source) - - return extra_sources - - @staticmethod - def _get_query_file( - source_file: Path, transformation_external_id: str | None, query_files: dict[Path, BuildSourceFile] - ) -> BuildSourceFile | None: - query_file = source_file.parent / f"{source_file.stem}.sql" - if query_file in query_files: - return query_files[query_file] - if transformation_external_id: - query_file = source_file.parent / f"{transformation_external_id}.sql" - if query_file in query_files: - return query_files[query_file] - return None diff --git a/tests/test_unit/test_cdf_tk/test_builders/__init__.py b/tests/test_unit/test_cdf_tk/test_builders/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/test_unit/test_cdf_tk/test_builders/test_builder.py b/tests/test_unit/test_cdf_tk/test_builders/test_builder.py deleted file mode 100644 index 00e94052b2..0000000000 --- a/tests/test_unit/test_cdf_tk/test_builders/test_builder.py +++ /dev/null @@ -1,66 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -import pytest - -from cognite_toolkit._cdf_tk.builders import get_resource_crud -from cognite_toolkit._cdf_tk.resource_ios import ( - RESOURCE_CRUD_LIST, - GroupAllScopedCRUD, - GroupIO, - GroupResourceScopedCRUD, - ResourceIO, -) -from cognite_toolkit._cdf_tk.tk_warnings import ToolkitNotSupportedWarning, ToolkitWarning -from cognite_toolkit._cdf_tk.tk_warnings.fileread import UnknownResourceTypeWarning - - -class TestGetCRUD: - @pytest.mark.parametrize( - "source_path, resource_folder, expected_loader_cls", - [ - pytest.param( - Path(f"some_path/{crud_cls.folder_name}/my.{crud_cls.kind}.yaml"), - crud_cls.folder_name, - { - GroupResourceScopedCRUD: GroupIO, - GroupAllScopedCRUD: GroupIO, - }.get(crud_cls, crud_cls), - id=crud_cls.__name__, - ) - for crud_cls in RESOURCE_CRUD_LIST - ], - ) - def test_get_crud_no_warning( - self, source_path: Path, resource_folder: str, expected_loader_cls: type[ResourceIO] - ) -> None: - crud_cls, warning = get_resource_crud(source_path, resource_folder) - - assert warning is None - assert crud_cls is expected_loader_cls - - @pytest.mark.parametrize( - "source_path, resource_folder, expected_warning_cls", - [ - pytest.param( - Path(f"some_path/unknown_folder/my.{GroupIO.kind}.yaml"), - "unknown_folder", - ToolkitNotSupportedWarning, - id="Unknown folder, known kind", - ), - pytest.param( - Path("some_path/group/my.UnknownKind.yaml"), - GroupIO.folder_name, - UnknownResourceTypeWarning, - id="Known folder, unknown kind", - ), - ], - ) - def test_get_crud_warning( - self, source_path: Path, resource_folder: str, expected_warning_cls: type[ToolkitWarning] - ) -> None: - crud_cls, warning = get_resource_crud(source_path, resource_folder) - - assert crud_cls is None - assert type(warning) is expected_warning_cls diff --git a/tests/test_unit/test_cdf_tk/test_builders/test_location_builder.py b/tests/test_unit/test_cdf_tk/test_builders/test_location_builder.py deleted file mode 100644 index 15a5b88b8f..0000000000 --- a/tests/test_unit/test_cdf_tk/test_builders/test_location_builder.py +++ /dev/null @@ -1,53 +0,0 @@ -from pathlib import Path -from unittest.mock import MagicMock - -import pytest - -from cognite_toolkit._cdf_tk.builders._location import LocationBuilder -from cognite_toolkit._cdf_tk.data_classes._build_files import BuildSourceFile -from cognite_toolkit._cdf_tk.data_classes._module_directories import ModuleLocation -from cognite_toolkit._cdf_tk.exceptions import ToolkitError - - -def test_location_builder_detect_cyclic_references(build_tmp_path): - # Setup - - module_location = MagicMock(spec=ModuleLocation) - - source_file = MagicMock(spec=BuildSourceFile) - source_file.source = MagicMock() - source_file.source.path = Path("cyclic.LocationFilter.yaml") - source_file.loaded = [ - {"externalId": "location1", "description": "Location 1", "parentExternalId": "location2"}, - {"externalId": "location2", "description": "Location 2", "parentExternalId": "location3"}, - { - "externalId": "location3", - "description": "Location 3", - "parentExternalId": "location1", # This creates a cycle - }, - ] - - location_builder = LocationBuilder(build_dir=build_tmp_path) - with pytest.raises(ToolkitError, match=r"Circular dependency found in Locations*"): - list(location_builder.build(source_files=[source_file], module=module_location)) - - -def test_location_builder_detect_self_reference(build_tmp_path): - # Setup - - module_location = MagicMock(spec=ModuleLocation) - - source_file = MagicMock(spec=BuildSourceFile) - source_file.source = MagicMock() - source_file.source.path = Path("self.LocationFilter.yaml") - source_file.loaded = [ - { - "externalId": "location1", - "description": "Location 1", - "parentExternalId": "location1", - }, # This creates a self-reference - ] - - location_builder = LocationBuilder(build_dir=build_tmp_path) - with pytest.raises(ToolkitError, match=r"Circular dependency found in Locations*"): - list(location_builder.build(source_files=[source_file], module=module_location)) diff --git a/tests/test_unit/test_cdf_tk/test_cruds/test_data_model.py b/tests/test_unit/test_cdf_tk/test_cruds/test_data_model.py index 71f4e99517..eba0cc76dc 100644 --- a/tests/test_unit/test_cdf_tk/test_cruds/test_data_model.py +++ b/tests/test_unit/test_cdf_tk/test_cruds/test_data_model.py @@ -711,49 +711,3 @@ def test_dml_compile_error_surfaced_as_actionable_message(self) -> None: ) with pytest.raises(ToolkitAPIError, match="Type 'Foo' not found"): self._make_api(body)._post_graphql({"query": "...", "variables": {}}) - - -class TestDataModelBuilder: - """Regression tests for DataModelBuilder (build v1).""" - - def test_dml_updated_to_renamed_graphql_in_build(self, tmp_path: Path) -> None: - # Regression test: build renames .graphql files with a long prefix, but deploy - # looks up the file via entry["dml"]. _copy_graphql_to_build must update "dml" - # so that deploy finds the renamed file instead of the original source name. - from cognite_toolkit._cdf_tk.builders._datamodels import DataModelBuilder - from cognite_toolkit._cdf_tk.data_classes._build_files import BuildSourceFile - from cognite_toolkit._cdf_tk.data_classes._built_resources import SourceLocationEager - - source_dir = tmp_path / "source" / "data_modeling" - source_dir.mkdir(parents=True) - build_dir = tmp_path / "build" - build_dir.mkdir() - - yaml_path = source_dir / "my_model.GraphQLSchema.yaml" - graphql_path = source_dir / "original_schema.graphql" - yaml_path.write_text("space: my_space\nexternalId: MyModel\nversion: v1\ndml: original_schema.graphql\n") - graphql_path.write_text("type Foo { name: String }") - - entry: dict = {"space": "my_space", "externalId": "MyModel", "version": "v1", "dml": "original_schema.graphql"} - source_file = BuildSourceFile( - source=SourceLocationEager(path=yaml_path, _hash="abc"), - content=yaml_path.read_text(), - loaded=entry, - ) - graphql_source = BuildSourceFile( - source=SourceLocationEager(path=graphql_path, _hash="def"), - content=graphql_path.read_text(), - loaded=None, - ) - - builder = DataModelBuilder(build_dir=build_dir) - destination_path = build_dir / "data_modeling" / "1-my_model-SPP-COR.my_model.GraphQLSchema.yaml" - destination_path.parent.mkdir(parents=True, exist_ok=True) - - builder._copy_graphql_to_build(source_file, destination_path, {graphql_path: graphql_source}) - - # The "dml" field in the entry dict must be updated to the renamed build filename. - renamed_graphql = destination_path.with_suffix(".graphql").name - assert entry["dml"] == renamed_graphql, ( - f"entry['dml'] was not updated after build rename: got {entry['dml']!r}, expected {renamed_graphql!r}" - )