From b374f0f10903eaa981203e7e779c5532eb43abf8 Mon Sep 17 00:00:00 2001 From: vikramlc Date: Thu, 3 Sep 2026 16:43:05 +0530 Subject: [PATCH 1/5] feat(integrations): add Integrations API client --- cognite/client/_api/integrations/__init__.py | 238 ++++++++++++++ cognite/client/_api/integrations/actions.py | 193 +++++++++++ cognite/client/_api/integrations/config.py | 121 +++++++ cognite/client/_api/integrations/errors.py | 68 ++++ cognite/client/_api/integrations/tasks.py | 130 ++++++++ cognite/client/_cognite_client.py | 11 + .../client/_sync_api/integrations/__init__.py | 209 ++++++++++++ .../client/_sync_api/integrations/actions.py | 158 +++++++++ .../client/_sync_api/integrations/config.py | 106 ++++++ .../client/_sync_api/integrations/errors.py | 66 ++++ .../client/_sync_api/integrations/tasks.py | 119 +++++++ cognite/client/_sync_cognite_client.py | 2 + .../data_classes/integrations/__init__.py | 55 ++++ .../data_classes/integrations/actions.py | 127 ++++++++ .../data_classes/integrations/config.py | 133 ++++++++ .../data_classes/integrations/errors.py | 67 ++++ .../data_classes/integrations/integrations.py | 304 ++++++++++++++++++ .../client/data_classes/integrations/tasks.py | 113 +++++++ cognite/client/testing.py | 38 +++ cognite/client/utils/_url.py | 4 + .../test_api/test_integrations/__init__.py | 0 .../test_integrations/test_actions.py | 87 +++++ .../test_api/test_integrations/test_config.py | 78 +++++ .../test_api/test_integrations/test_errors.py | 36 +++ .../test_integrations/test_integrations.py | 118 +++++++ .../test_api/test_integrations/test_tasks.py | 56 ++++ tests/tests_unit/test_api_client.py | 9 + .../test_data_classes/test_integrations.py | 191 +++++++++++ 28 files changed, 2837 insertions(+) create mode 100644 cognite/client/_api/integrations/__init__.py create mode 100644 cognite/client/_api/integrations/actions.py create mode 100644 cognite/client/_api/integrations/config.py create mode 100644 cognite/client/_api/integrations/errors.py create mode 100644 cognite/client/_api/integrations/tasks.py create mode 100644 cognite/client/_sync_api/integrations/__init__.py create mode 100644 cognite/client/_sync_api/integrations/actions.py create mode 100644 cognite/client/_sync_api/integrations/config.py create mode 100644 cognite/client/_sync_api/integrations/errors.py create mode 100644 cognite/client/_sync_api/integrations/tasks.py create mode 100644 cognite/client/data_classes/integrations/__init__.py create mode 100644 cognite/client/data_classes/integrations/actions.py create mode 100644 cognite/client/data_classes/integrations/config.py create mode 100644 cognite/client/data_classes/integrations/errors.py create mode 100644 cognite/client/data_classes/integrations/integrations.py create mode 100644 cognite/client/data_classes/integrations/tasks.py create mode 100644 tests/tests_unit/test_api/test_integrations/__init__.py create mode 100644 tests/tests_unit/test_api/test_integrations/test_actions.py create mode 100644 tests/tests_unit/test_api/test_integrations/test_config.py create mode 100644 tests/tests_unit/test_api/test_integrations/test_errors.py create mode 100644 tests/tests_unit/test_api/test_integrations/test_integrations.py create mode 100644 tests/tests_unit/test_api/test_integrations/test_tasks.py create mode 100644 tests/tests_unit/test_data_classes/test_integrations.py diff --git a/cognite/client/_api/integrations/__init__.py b/cognite/client/_api/integrations/__init__.py new file mode 100644 index 0000000000..fe775981e5 --- /dev/null +++ b/cognite/client/_api/integrations/__init__.py @@ -0,0 +1,238 @@ +from __future__ import annotations + +from collections.abc import AsyncIterator, Sequence +from typing import TYPE_CHECKING, overload + +from cognite.client._api.integrations.actions import IntegrationActionsAPI +from cognite.client._api.integrations.config import IntegrationConfigAPI +from cognite.client._api.integrations.errors import IntegrationErrorsAPI +from cognite.client._api.integrations.tasks import IntegrationTasksAPI +from cognite.client._api_client import APIClient +from cognite.client._constants import DEFAULT_LIMIT_READ +from cognite.client.data_classes.integrations.integrations import ( + Integration, + IntegrationList, + IntegrationUpdate, + IntegrationWrite, +) +from cognite.client.utils._experimental import FeaturePreviewWarning +from cognite.client.utils._identifier import IdentifierSequence +from cognite.client.utils.useful_types import SequenceNotStr + +if TYPE_CHECKING: + from cognite.client import AsyncCogniteClient + from cognite.client.config import ClientConfig + + +class IntegrationsAPI(APIClient): + _RESOURCE_PATH = "/integrations" + + def __init__(self, config: ClientConfig, api_version: str | None, cognite_client: AsyncCogniteClient) -> None: + super().__init__(config, api_version, cognite_client) + self.tasks = IntegrationTasksAPI(config, api_version, cognite_client) + self.errors = IntegrationErrorsAPI(config, api_version, cognite_client) + self.config = IntegrationConfigAPI(config, api_version, cognite_client) + self.actions = IntegrationActionsAPI(config, api_version, cognite_client) + self._warning = FeaturePreviewWarning(api_maturity="alpha", sdk_maturity="alpha", feature_name="Integrations") + + @overload + def __call__(self, chunk_size: None = None, limit: int | None = None) -> AsyncIterator[Integration]: ... + + @overload + def __call__(self, chunk_size: int, limit: int | None = None) -> AsyncIterator[IntegrationList]: ... + + async def __call__( + self, chunk_size: int | None = None, limit: int | None = None + ) -> AsyncIterator[Integration] | AsyncIterator[IntegrationList]: + """Iterate over integrations + + Fetches integrations as they are iterated over, so you keep a limited number of integrations in memory. + + Args: + chunk_size (int | None): Number of integrations to return in each chunk. Defaults to yielding one integration a time. + limit (int | None): Maximum number of integrations to return. Defaults to return all items. + + Yields: + Integration | IntegrationList: yields Integration one by one if chunk_size is not specified, else IntegrationList objects. + """ # noqa: DOC404 + self._warning.warn() + async for item in self._list_generator( + method="GET", + list_cls=IntegrationList, + resource_cls=Integration, + chunk_size=chunk_size, + limit=limit, + headers=self._alpha_version_header(), + ): + yield item + + async def list(self, limit: int | None = DEFAULT_LIMIT_READ) -> IntegrationList: + """`List integrations `_ + + Args: + limit (int | None): Maximum number of integrations to return. Defaults to 25. Set to -1, float("inf") or None to return all items. + + Returns: + IntegrationList: List of integrations + + Examples: + + List integrations: + + >>> from cognite.client import CogniteClient, AsyncCogniteClient + >>> client = CogniteClient() + >>> # async_client = AsyncCogniteClient() # another option + >>> res = client.integrations.list(limit=10) + + Iterate over integrations, one-by-one: + + >>> for integration in client.integrations(): + ... integration # do something with the integration + """ + self._warning.warn() + return await self._list( + method="GET", + list_cls=IntegrationList, + resource_cls=Integration, + limit=limit, + headers=self._alpha_version_header(), + ) + + @overload + async def create(self, integration: IntegrationWrite) -> Integration: ... + + @overload + async def create(self, integration: Sequence[IntegrationWrite]) -> IntegrationList: ... + + async def create(self, integration: IntegrationWrite | Sequence[IntegrationWrite]) -> Integration | IntegrationList: + """`Create one or more integrations `_ + + Args: + integration (IntegrationWrite | Sequence[IntegrationWrite]): Integration or list of integrations to create. + + Returns: + Integration | IntegrationList: Created integration(s) + + Examples: + + Create a new integration: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes.integrations import Extractor, IntegrationWrite + >>> client = CogniteClient() + >>> integration = IntegrationWrite( + ... external_id="my-integration", + ... extractor=Extractor(external_id="cognite-simple-influxdb-extractor"), + ... ) + >>> res = client.integrations.create(integration) + """ + self._warning.warn() + return await self._create_multiple( + list_cls=IntegrationList, + resource_cls=Integration, + items=integration, + input_resource_cls=IntegrationWrite, + headers=self._alpha_version_header(), + ) + + @overload + async def retrieve(self, external_id: str, ignore_unknown_ids: bool = False) -> Integration | None: ... + + @overload + async def retrieve(self, external_id: SequenceNotStr[str], ignore_unknown_ids: bool = False) -> IntegrationList: ... + + async def retrieve( + self, external_id: str | SequenceNotStr[str], ignore_unknown_ids: bool = False + ) -> Integration | IntegrationList | None: + """`Retrieve one or more integrations by external id `_ + + Args: + external_id (str | SequenceNotStr[str]): External id or list of external ids to retrieve. + ignore_unknown_ids (bool): Ignore external ids that are not found rather than throw an exception. + + Returns: + Integration | IntegrationList | None: Requested integration(s), or None if a single requested external id is not found. + + Examples: + + Retrieve an integration by external id: + + >>> from cognite.client import CogniteClient, AsyncCogniteClient + >>> client = CogniteClient() + >>> # async_client = AsyncCogniteClient() # another option + >>> res = client.integrations.retrieve("my-integration") + """ + self._warning.warn() + identifiers = IdentifierSequence.load(external_ids=external_id) + return await self._retrieve_multiple( + list_cls=IntegrationList, + resource_cls=Integration, + identifiers=identifiers, + ignore_unknown_ids=ignore_unknown_ids, + headers=self._alpha_version_header(), + ) + + @overload + async def update(self, item: Integration | IntegrationWrite | IntegrationUpdate) -> Integration: ... + + @overload + async def update(self, item: Sequence[Integration | IntegrationWrite | IntegrationUpdate]) -> IntegrationList: ... + + async def update( + self, + item: Integration + | IntegrationWrite + | IntegrationUpdate + | Sequence[Integration | IntegrationWrite | IntegrationUpdate], + ) -> Integration | IntegrationList: + """`Update one or more integrations `_ + + Args: + item (Integration | IntegrationWrite | IntegrationUpdate | Sequence[Integration | IntegrationWrite | IntegrationUpdate]): Integration(s) to update. + + Returns: + Integration | IntegrationList: Updated integration(s) + + Examples: + + Update an integration that you have fetched. This will perform a full update of the integration: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes.integrations import IntegrationUpdate + >>> client = CogniteClient() + >>> update = IntegrationUpdate(external_id="my-integration") + >>> update.description.set("My new description") + >>> res = client.integrations.update(update) + """ + self._warning.warn() + return await self._update_multiple( + list_cls=IntegrationList, + resource_cls=Integration, + update_cls=IntegrationUpdate, + items=item, + headers=self._alpha_version_header(), + ) + + async def delete(self, external_id: str | SequenceNotStr[str], ignore_unknown_ids: bool = False) -> None: + """`Delete one or more integrations `_ + + Args: + external_id (str | SequenceNotStr[str]): External id or list of external ids to delete. + ignore_unknown_ids (bool): Ignore external ids that are not found rather than throw an exception. + + Examples: + + Delete integrations by external id: + + >>> from cognite.client import CogniteClient, AsyncCogniteClient + >>> client = CogniteClient() + >>> # async_client = AsyncCogniteClient() # another option + >>> client.integrations.delete(external_id=["my-integration"]) + """ + self._warning.warn() + await self._delete_multiple( + identifiers=IdentifierSequence.load(external_ids=external_id), + wrap_ids=True, + extra_body_fields={"ignoreUnknownIds": ignore_unknown_ids}, + headers=self._alpha_version_header(), + ) diff --git a/cognite/client/_api/integrations/actions.py b/cognite/client/_api/integrations/actions.py new file mode 100644 index 0000000000..001bd73219 --- /dev/null +++ b/cognite/client/_api/integrations/actions.py @@ -0,0 +1,193 @@ +from __future__ import annotations + +from collections.abc import Sequence +from typing import TYPE_CHECKING, Any, overload + +from cognite.client._api_client import APIClient +from cognite.client._constants import DEFAULT_LIMIT_READ +from cognite.client.data_classes.integrations.actions import Action, ActionList, ActionWrite +from cognite.client.utils._auxiliary import drop_none_values, split_into_chunks +from cognite.client.utils._experimental import FeaturePreviewWarning +from cognite.client.utils._identifier import IdentifierSequence +from cognite.client.utils.useful_types import SequenceNotStr + +if TYPE_CHECKING: + from cognite.client import AsyncCogniteClient + from cognite.client.config import ClientConfig + +_CREATE_LIMIT = 20 +_CANCEL_LIMIT = 100 + + +class IntegrationActionsAPI(APIClient): + _RESOURCE_PATH = "/integrations/actions" + + def __init__(self, config: ClientConfig, api_version: str | None, cognite_client: AsyncCogniteClient) -> None: + super().__init__(config, api_version, cognite_client) + self._warning = FeaturePreviewWarning(api_maturity="alpha", sdk_maturity="alpha", feature_name="Integrations") + + @overload + async def create(self, external_id: str, action: ActionWrite) -> Action: ... + + @overload + async def create(self, external_id: str, action: Sequence[ActionWrite]) -> ActionList: ... + + async def create(self, external_id: str, action: ActionWrite | Sequence[ActionWrite]) -> Action | ActionList: + """`Create one or more actions `_ + + Args: + external_id (str): External id of the integration to trigger the action(s) against. + action (ActionWrite | Sequence[ActionWrite]): Action or list of actions to create. + + Returns: + Action | ActionList: Created action(s) + + Examples: + + Trigger an action against an integration: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes.integrations import ActionWrite + >>> client = CogniteClient() + >>> action = ActionWrite(external_id="my-restart-action", action_name="restart") + >>> res = client.integrations.actions.create("my-integration", action) + """ + self._warning.warn() + single_item = isinstance(action, ActionWrite) + items: list[ActionWrite] = [action] if isinstance(action, ActionWrite) else list(action) + + created: list[dict[str, Any]] = [] + for chunk in split_into_chunks(items, _CREATE_LIMIT): + response = await self._post( + self._RESOURCE_PATH, + params={"externalId": external_id}, + json={"items": [item.dump(camel_case=True) for item in chunk]}, + headers=self._alpha_version_header(), + semaphore=self._get_semaphore("write"), + ) + created.extend(response.json()["items"]) + + if single_item: + return Action._load(created[0]) + return ActionList._load(created) + + async def list( + self, + external_id: str | None = None, + created_after: int | None = None, + include_completed: bool = True, + limit: int | None = DEFAULT_LIMIT_READ, + ) -> ActionList: + """`List actions `_ + + Args: + external_id (str | None): Only return actions for the integration with this external id. If not given, actions across all integrations you have access to are returned. + created_after (int | None): Only return actions created at or after this time, in milliseconds since epoch. + include_completed (bool): Whether to include actions in a terminal state (succeeded, failed, canceled). If False, only pending/running/cancel_pending actions are returned. + limit (int | None): Maximum number of actions to return. Defaults to 25. Set to -1, float("inf") or None to return all items. + + Returns: + ActionList: List of actions + + Examples: + + List pending actions for an integration: + + >>> from cognite.client import CogniteClient, AsyncCogniteClient + >>> client = CogniteClient() + >>> # async_client = AsyncCogniteClient() # another option + >>> res = client.integrations.actions.list( + ... external_id="my-integration", include_completed=False + ... ) + """ + self._warning.warn() + return await self._list( + method="GET", + list_cls=ActionList, + resource_cls=Action, + limit=limit, + filter=drop_none_values( + { + "externalId": external_id, + "createdAfter": created_after, + "includeCompleted": include_completed, + } + ), + headers=self._alpha_version_header(), + ) + + @overload + async def retrieve(self, external_id: str, ignore_unknown_ids: bool = False) -> Action | None: ... + + @overload + async def retrieve(self, external_id: SequenceNotStr[str], ignore_unknown_ids: bool = False) -> ActionList: ... + + async def retrieve( + self, external_id: str | SequenceNotStr[str], ignore_unknown_ids: bool = False + ) -> Action | ActionList | None: + """`Retrieve one or more actions by external id `_ + + Args: + external_id (str | SequenceNotStr[str]): External id or list of external ids of actions to retrieve. + ignore_unknown_ids (bool): Ignore external ids that are not found rather than throw an exception. + + Returns: + Action | ActionList | None: Requested action(s), or None if a single requested external id is not found. + + Examples: + + Retrieve an action by external id: + + >>> from cognite.client import CogniteClient, AsyncCogniteClient + >>> client = CogniteClient() + >>> # async_client = AsyncCogniteClient() # another option + >>> res = client.integrations.actions.retrieve("my-restart-action") + """ + self._warning.warn() + identifiers = IdentifierSequence.load(external_ids=external_id) + return await self._retrieve_multiple( + list_cls=ActionList, + resource_cls=Action, + identifiers=identifiers, + ignore_unknown_ids=ignore_unknown_ids, + headers=self._alpha_version_header(), + ) + + async def cancel(self, external_id: str | SequenceNotStr[str], ignore_unknown_ids: bool = False) -> ActionList: + """`Cancel one or more actions `_ + + Only actions in the `pending`, `running` or `cancel_pending` state can be cancelled. + + Args: + external_id (str | SequenceNotStr[str]): External id or list of external ids of actions to cancel. + ignore_unknown_ids (bool): Ignore external ids that are not found rather than throw an exception. + + Returns: + ActionList: The action(s), with their updated status. + + Examples: + + Cancel a pending action: + + >>> from cognite.client import CogniteClient, AsyncCogniteClient + >>> client = CogniteClient() + >>> # async_client = AsyncCogniteClient() # another option + >>> res = client.integrations.actions.cancel("my-restart-action") + """ + self._warning.warn() + identifiers = IdentifierSequence.load(external_ids=external_id) + + cancelled: list[dict[str, Any]] = [] + for chunk in identifiers.chunked(_CANCEL_LIMIT): + body: dict[str, Any] = {"items": chunk.as_dicts()} + if ignore_unknown_ids: + body["ignoreUnknownIds"] = True + response = await self._post( + f"{self._RESOURCE_PATH}/cancel", + json=body, + headers=self._alpha_version_header(), + semaphore=self._get_semaphore("write"), + ) + cancelled.extend(response.json()["items"]) + + return ActionList._load(cancelled) diff --git a/cognite/client/_api/integrations/config.py b/cognite/client/_api/integrations/config.py new file mode 100644 index 0000000000..e78f61dc55 --- /dev/null +++ b/cognite/client/_api/integrations/config.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from cognite.client._api_client import APIClient +from cognite.client._constants import DEFAULT_LIMIT_READ +from cognite.client.data_classes.integrations.config import ( + ConfigRevision, + ConfigRevisionMetadataList, + ConfigRevisionWrite, +) +from cognite.client.utils._auxiliary import drop_none_values +from cognite.client.utils._experimental import FeaturePreviewWarning + +if TYPE_CHECKING: + from cognite.client import AsyncCogniteClient + from cognite.client.config import ClientConfig + + +class IntegrationConfigAPI(APIClient): + _RESOURCE_PATH = "/integrations/config" + + def __init__(self, config: ClientConfig, api_version: str | None, cognite_client: AsyncCogniteClient) -> None: + super().__init__(config, api_version, cognite_client) + self._warning = FeaturePreviewWarning(api_maturity="alpha", sdk_maturity="alpha", feature_name="Integrations") + + async def create(self, config: ConfigRevision | ConfigRevisionWrite) -> ConfigRevision: + """`Create a new configuration revision `_ + + Args: + config (ConfigRevision | ConfigRevisionWrite): Configuration revision to create. + + Returns: + ConfigRevision: Created configuration revision + + Examples: + + Create a config revision: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes.integrations import ConfigRevisionWrite + >>> client = CogniteClient() + >>> res = client.integrations.config.create( + ... ConfigRevisionWrite(external_id="my-integration", config="my config contents") + ... ) + """ + self._warning.warn() + if isinstance(config, ConfigRevision): + config = config.as_write() + response = await self._post( + self._RESOURCE_PATH, + json=config.dump(camel_case=True), + headers=self._alpha_version_header(), + semaphore=self._get_semaphore("write"), + ) + return ConfigRevision._load(response.json()) + + async def retrieve(self, external_id: str, revision: int | None = None) -> ConfigRevision: + """`Retrieve a specific configuration revision, or the latest by default `_ + + Args: + external_id (str): External id of the integration to retrieve config from. + revision (int | None): Optionally specify a revision number to retrieve. Defaults to the latest revision. + + Returns: + ConfigRevision: Retrieved configuration revision + + Examples: + + Retrieve latest config revision: + + >>> from cognite.client import CogniteClient, AsyncCogniteClient + >>> client = CogniteClient() + >>> # async_client = AsyncCogniteClient() # another option + >>> res = client.integrations.config.retrieve("my-integration") + """ + self._warning.warn() + response = await self._get( + self._RESOURCE_PATH, + params=drop_none_values({"externalId": external_id, "revision": revision}), + headers=self._alpha_version_header(), + semaphore=self._get_semaphore("read"), + ) + return ConfigRevision._load(response.json()) + + async def list( + self, external_id: str | None = None, limit: int | None = DEFAULT_LIMIT_READ + ) -> ConfigRevisionMetadataList: + """`List configuration revisions `_ + + Lists metadata about configuration revisions (without the config contents itself), ordered by revision + number descending. + + Note: + This endpoint does not support cursor-based pagination: it always returns up to `limit` of the most + recent revisions in a single page (server-side capped at 100). + + Args: + external_id (str | None): Only return config revisions for the integration with this external id. + limit (int | None): Maximum number of config revisions to return. Defaults to 25. + + Returns: + ConfigRevisionMetadataList: List of configuration revision metadata + + Examples: + + List config revisions for an integration: + + >>> from cognite.client import CogniteClient, AsyncCogniteClient + >>> client = CogniteClient() + >>> # async_client = AsyncCogniteClient() # another option + >>> res = client.integrations.config.list(external_id="my-integration") + """ + self._warning.warn() + response = await self._get( + f"{self._RESOURCE_PATH}/revisions", + params=drop_none_values({"externalId": external_id, "limit": limit}), + headers=self._alpha_version_header(), + semaphore=self._get_semaphore("read"), + ) + return ConfigRevisionMetadataList._load(response.json()["items"]) diff --git a/cognite/client/_api/integrations/errors.py b/cognite/client/_api/integrations/errors.py new file mode 100644 index 0000000000..78d22cd482 --- /dev/null +++ b/cognite/client/_api/integrations/errors.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from cognite.client._api_client import APIClient +from cognite.client._constants import DEFAULT_LIMIT_READ +from cognite.client.data_classes.integrations.errors import IntegrationError, IntegrationErrorList +from cognite.client.utils._auxiliary import drop_none_values +from cognite.client.utils._experimental import FeaturePreviewWarning + +if TYPE_CHECKING: + from cognite.client import AsyncCogniteClient + from cognite.client.config import ClientConfig + + +class IntegrationErrorsAPI(APIClient): + _RESOURCE_PATH = "/integrations" + + def __init__(self, config: ClientConfig, api_version: str | None, cognite_client: AsyncCogniteClient) -> None: + super().__init__(config, api_version, cognite_client) + self._warning = FeaturePreviewWarning(api_maturity="alpha", sdk_maturity="alpha", feature_name="Integrations") + + async def list( + self, + external_id: str | None = None, + task: str | None = None, + min_start_time: int | None = None, + max_end_time: int | None = None, + limit: int | None = DEFAULT_LIMIT_READ, + ) -> IntegrationErrorList: + """`List errors `_ + + Args: + external_id (str | None): Only return errors for the integration with this external id. + task (str | None): Only return errors for the task with this name. Requires `external_id` to also be set. + min_start_time (int | None): Only return errors that started at or after this time, in milliseconds since epoch. + max_end_time (int | None): Only return errors that ended at or before this time, in milliseconds since epoch. + limit (int | None): Maximum number of errors to return. Defaults to 25. Set to -1, float("inf") or None to return all items. + + Returns: + IntegrationErrorList: List of errors + + Examples: + + List errors for a single integration: + + >>> from cognite.client import CogniteClient, AsyncCogniteClient + >>> client = CogniteClient() + >>> # async_client = AsyncCogniteClient() # another option + >>> res = client.integrations.errors.list(external_id="my-integration") + """ + self._warning.warn() + return await self._list( + method="GET", + url_path=f"{self._RESOURCE_PATH}/errors", + list_cls=IntegrationErrorList, + resource_cls=IntegrationError, + limit=limit, + filter=drop_none_values( + { + "externalId": external_id, + "task": task, + "minStartTime": min_start_time, + "maxEndTime": max_end_time, + } + ), + headers=self._alpha_version_header(), + ) diff --git a/cognite/client/_api/integrations/tasks.py b/cognite/client/_api/integrations/tasks.py new file mode 100644 index 0000000000..b28efccf1f --- /dev/null +++ b/cognite/client/_api/integrations/tasks.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from cognite.client._api_client import APIClient +from cognite.client._constants import DEFAULT_LIMIT_READ +from cognite.client.data_classes.integrations.tasks import SyncResult, TaskHistory, TaskHistoryList +from cognite.client.utils._auxiliary import drop_none_values +from cognite.client.utils._experimental import FeaturePreviewWarning + +if TYPE_CHECKING: + from cognite.client import AsyncCogniteClient + from cognite.client.config import ClientConfig + + +class IntegrationTasksAPI(APIClient): + _RESOURCE_PATH = "/integrations" + + def __init__(self, config: ClientConfig, api_version: str | None, cognite_client: AsyncCogniteClient) -> None: + super().__init__(config, api_version, cognite_client) + self._warning = FeaturePreviewWarning(api_maturity="alpha", sdk_maturity="alpha", feature_name="Integrations") + + async def list_history( + self, + external_id: str | None = None, + task_name: str | None = None, + last_per_task: bool = False, + limit: int | None = DEFAULT_LIMIT_READ, + ) -> TaskHistoryList: + """`List task history `_ + + Args: + external_id (str | None): Only return history for the integration with this external id. + task_name (str | None): Only return history for the task with this name. Requires `external_id` to also be set. + last_per_task (bool): Only return the latest history entry per task. + limit (int | None): Maximum number of history entries to return. Defaults to 25. Set to -1, float("inf") or None to return all items. + + Returns: + TaskHistoryList: List of task history entries + + Examples: + + List task history for a single integration: + + >>> from cognite.client import CogniteClient, AsyncCogniteClient + >>> client = CogniteClient() + >>> # async_client = AsyncCogniteClient() # another option + >>> res = client.integrations.tasks.list_history(external_id="my-integration") + """ + self._warning.warn() + return await self._list( + method="GET", + url_path=f"{self._RESOURCE_PATH}/history", + list_cls=TaskHistoryList, + resource_cls=TaskHistory, + limit=limit, + filter=drop_none_values( + { + "externalId": external_id, + "taskName": task_name, + "lastPerTask": last_per_task, + } + ), + headers=self._alpha_version_header(), + ) + + async def sync( + self, + external_id: str, + task_name: str | None = None, + include_errors: bool = False, + include_task_updates: bool = False, + start_time: int | None = None, + cursor: str | None = None, + limit: int | None = DEFAULT_LIMIT_READ, + ) -> SyncResult: + """`Sync integration history `_ + + Incrementally fetch task history and/or errors for an integration since a previous sync call. This is more + efficient than repeatedly listing task history/errors when polling for updates, e.g. for dashboards or + alerting. At least one of `include_errors` and `include_task_updates` must be True. + + Args: + external_id (str): Only return history for the integration with this external id. + task_name (str | None): Only return history for the task with this name. + include_errors (bool): Include errors reported since the last sync. + include_task_updates (bool): Include task history entries reported since the last sync. + start_time (int | None): Only return items reported at or after this time, in milliseconds since epoch. Only used on the first call, pass the returned cursor on subsequent calls instead. + cursor (str | None): Cursor returned from a previous call to this method, to continue syncing from where you left off. + limit (int | None): Maximum number of items to return in this page. Defaults to 25. + + Returns: + SyncResult: A single page of results. Inspect `more_data` to see whether you should immediately call this method again with the returned `next_cursor`, or back off before doing so. + + Examples: + + Sync task history and errors for an integration: + + >>> from cognite.client import CogniteClient, AsyncCogniteClient + >>> client = CogniteClient() + >>> # async_client = AsyncCogniteClient() # another option + >>> res = client.integrations.tasks.sync( + ... external_id="my-integration", include_errors=True, include_task_updates=True + ... ) + >>> while res.more_data: + ... res = client.integrations.tasks.sync( + ... external_id="my-integration", + ... include_errors=True, + ... include_task_updates=True, + ... cursor=res.next_cursor, + ... ) + """ + self._warning.warn() + response = await self._get( + url_path=f"{self._RESOURCE_PATH}/sync", + params=drop_none_values( + { + "externalId": external_id, + "taskName": task_name, + "includeErrors": include_errors, + "includeTaskUpdates": include_task_updates, + "startTime": start_time, + "cursor": cursor, + "limit": limit, + } + ), + headers=self._alpha_version_header(), + semaphore=self._get_semaphore("read"), + ) + return SyncResult._load(response.json()) diff --git a/cognite/client/_cognite_client.py b/cognite/client/_cognite_client.py index 4bf0769583..1929be3381 100644 --- a/cognite/client/_cognite_client.py +++ b/cognite/client/_cognite_client.py @@ -19,6 +19,7 @@ from cognite.client._api.geospatial import GeospatialAPI from cognite.client._api.hosted_extractors import HostedExtractorsAPI from cognite.client._api.iam import IAMAPI +from cognite.client._api.integrations import IntegrationsAPI from cognite.client._api.labels import LabelsAPI from cognite.client._api.limits import LimitsAPI from cognite.client._api.metering import MeteringAPI @@ -84,6 +85,10 @@ SessionsAPI, TokenAPI, ) + from cognite.client._api.integrations.actions import IntegrationActionsAPI + from cognite.client._api.integrations.config import IntegrationConfigAPI + from cognite.client._api.integrations.errors import IntegrationErrorsAPI + from cognite.client._api.integrations.tasks import IntegrationTasksAPI from cognite.client._api.postgres_gateway.tables import TablesAPI from cognite.client._api.postgres_gateway.users import UsersAPI from cognite.client._api.raw import RawDatabasesAPI, RawRowsAPI, RawTablesAPI # type: ignore[attr-defined] @@ -173,6 +178,7 @@ def __init__(self, config: ClientConfig | None = None) -> None: self.workflows = WorkflowAPI(self._config, self._API_VERSION, self) self.units = UnitAPI(self._config, self._API_VERSION, self) self.simulators = SimulatorsAPI(self._config, self._API_VERSION, self) + self.integrations = IntegrationsAPI(self._config, self._API_VERSION, self) # APIs just using base_url: self._api_client = APIClient(self._config, api_version=None, cognite_client=self) @@ -478,6 +484,11 @@ def _make_accessors_for_building_docs() -> None: AsyncCogniteClient.simulators.routines.revisions = SimulatorRoutineRevisionsAPI # type: ignore AsyncCogniteClient.simulators.runs = SimulatorRunsAPI # type: ignore AsyncCogniteClient.simulators.logs = SimulatorLogsAPI # type: ignore + AsyncCogniteClient.integrations = IntegrationsAPI # type: ignore + AsyncCogniteClient.integrations.tasks = IntegrationTasksAPI # type: ignore + AsyncCogniteClient.integrations.errors = IntegrationErrorsAPI # type: ignore + AsyncCogniteClient.integrations.config = IntegrationConfigAPI # type: ignore + AsyncCogniteClient.integrations.actions = IntegrationActionsAPI # type: ignore if _should_build_docs: diff --git a/cognite/client/_sync_api/integrations/__init__.py b/cognite/client/_sync_api/integrations/__init__.py new file mode 100644 index 0000000000..39fa9289d4 --- /dev/null +++ b/cognite/client/_sync_api/integrations/__init__.py @@ -0,0 +1,209 @@ +""" +=============================================================================== +05ec77fce9ac730a8d9873350250de4f +This file is auto-generated from the Async API modules, - do not edit manually! +=============================================================================== +""" + +from __future__ import annotations + +from collections.abc import Iterator, Sequence +from typing import TYPE_CHECKING, overload + +from cognite.client import AsyncCogniteClient +from cognite.client._constants import DEFAULT_LIMIT_READ +from cognite.client._sync_api.integrations.actions import SyncIntegrationActionsAPI +from cognite.client._sync_api.integrations.config import SyncIntegrationConfigAPI +from cognite.client._sync_api.integrations.errors import SyncIntegrationErrorsAPI +from cognite.client._sync_api.integrations.tasks import SyncIntegrationTasksAPI +from cognite.client._sync_api_client import SyncAPIClient +from cognite.client.data_classes.integrations.integrations import ( + Integration, + IntegrationList, + IntegrationUpdate, + IntegrationWrite, +) +from cognite.client.utils._async_helpers import SyncIterator, run_sync +from cognite.client.utils.useful_types import SequenceNotStr + +if TYPE_CHECKING: + from cognite.client import AsyncCogniteClient + + +class SyncIntegrationsAPI(SyncAPIClient): + """Auto-generated, do not modify manually.""" + + def __init__(self, async_client: AsyncCogniteClient) -> None: + self.__async_client = async_client + self.tasks = SyncIntegrationTasksAPI(async_client) + self.errors = SyncIntegrationErrorsAPI(async_client) + self.config = SyncIntegrationConfigAPI(async_client) + self.actions = SyncIntegrationActionsAPI(async_client) + + @overload + def __call__(self, chunk_size: None = None, limit: int | None = None) -> Iterator[Integration]: ... + + @overload + def __call__(self, chunk_size: int, limit: int | None = None) -> Iterator[IntegrationList]: ... + + def __call__( + self, chunk_size: int | None = None, limit: int | None = None + ) -> Iterator[Integration] | Iterator[IntegrationList]: + """ + Iterate over integrations + + Fetches integrations as they are iterated over, so you keep a limited number of integrations in memory. + + Args: + chunk_size (int | None): Number of integrations to return in each chunk. Defaults to yielding one integration a time. + limit (int | None): Maximum number of integrations to return. Defaults to return all items. + + Yields: + Integration | IntegrationList: yields Integration one by one if chunk_size is not specified, else IntegrationList objects. + """ # noqa: DOC404 + yield from SyncIterator(self.__async_client.integrations(chunk_size=chunk_size, limit=limit)) # type: ignore [misc] + + def list(self, limit: int | None = DEFAULT_LIMIT_READ) -> IntegrationList: + """ + `List integrations `_ + + Args: + limit (int | None): Maximum number of integrations to return. Defaults to 25. Set to -1, float("inf") or None to return all items. + + Returns: + IntegrationList: List of integrations + + Examples: + + List integrations: + + >>> from cognite.client import CogniteClient, AsyncCogniteClient + >>> client = CogniteClient() + >>> # async_client = AsyncCogniteClient() # another option + >>> res = client.integrations.list(limit=10) + + Iterate over integrations, one-by-one: + + >>> for integration in client.integrations(): + ... integration # do something with the integration + """ + return run_sync(self.__async_client.integrations.list(limit=limit)) + + @overload + def create(self, integration: IntegrationWrite) -> Integration: ... + + @overload + def create(self, integration: Sequence[IntegrationWrite]) -> IntegrationList: ... + + def create(self, integration: IntegrationWrite | Sequence[IntegrationWrite]) -> Integration | IntegrationList: + """ + `Create one or more integrations `_ + + Args: + integration (IntegrationWrite | Sequence[IntegrationWrite]): Integration or list of integrations to create. + + Returns: + Integration | IntegrationList: Created integration(s) + + Examples: + + Create a new integration: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes.integrations import Extractor, IntegrationWrite + >>> client = CogniteClient() + >>> integration = IntegrationWrite( + ... external_id="my-integration", + ... extractor=Extractor(external_id="cognite-simple-influxdb-extractor"), + ... ) + >>> res = client.integrations.create(integration) + """ + return run_sync(self.__async_client.integrations.create(integration=integration)) + + @overload + def retrieve(self, external_id: str, ignore_unknown_ids: bool = False) -> Integration | None: ... + + @overload + def retrieve(self, external_id: SequenceNotStr[str], ignore_unknown_ids: bool = False) -> IntegrationList: ... + + def retrieve( + self, external_id: str | SequenceNotStr[str], ignore_unknown_ids: bool = False + ) -> Integration | IntegrationList | None: + """ + `Retrieve one or more integrations by external id `_ + + Args: + external_id (str | SequenceNotStr[str]): External id or list of external ids to retrieve. + ignore_unknown_ids (bool): Ignore external ids that are not found rather than throw an exception. + + Returns: + Integration | IntegrationList | None: Requested integration(s), or None if a single requested external id is not found. + + Examples: + + Retrieve an integration by external id: + + >>> from cognite.client import CogniteClient, AsyncCogniteClient + >>> client = CogniteClient() + >>> # async_client = AsyncCogniteClient() # another option + >>> res = client.integrations.retrieve("my-integration") + """ + return run_sync( + self.__async_client.integrations.retrieve(external_id=external_id, ignore_unknown_ids=ignore_unknown_ids) + ) + + @overload + def update(self, item: Integration | IntegrationWrite | IntegrationUpdate) -> Integration: ... + + @overload + def update(self, item: Sequence[Integration | IntegrationWrite | IntegrationUpdate]) -> IntegrationList: ... + + def update( + self, + item: Integration + | IntegrationWrite + | IntegrationUpdate + | Sequence[Integration | IntegrationWrite | IntegrationUpdate], + ) -> Integration | IntegrationList: + """ + `Update one or more integrations `_ + + Args: + item (Integration | IntegrationWrite | IntegrationUpdate | Sequence[Integration | IntegrationWrite | IntegrationUpdate]): Integration(s) to update. + + Returns: + Integration | IntegrationList: Updated integration(s) + + Examples: + + Update an integration that you have fetched. This will perform a full update of the integration: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes.integrations import IntegrationUpdate + >>> client = CogniteClient() + >>> update = IntegrationUpdate(external_id="my-integration") + >>> update.description.set("My new description") + >>> res = client.integrations.update(update) + """ + return run_sync(self.__async_client.integrations.update(item=item)) + + def delete(self, external_id: str | SequenceNotStr[str], ignore_unknown_ids: bool = False) -> None: + """ + `Delete one or more integrations `_ + + Args: + external_id (str | SequenceNotStr[str]): External id or list of external ids to delete. + ignore_unknown_ids (bool): Ignore external ids that are not found rather than throw an exception. + + Examples: + + Delete integrations by external id: + + >>> from cognite.client import CogniteClient, AsyncCogniteClient + >>> client = CogniteClient() + >>> # async_client = AsyncCogniteClient() # another option + >>> client.integrations.delete(external_id=["my-integration"]) + """ + return run_sync( + self.__async_client.integrations.delete(external_id=external_id, ignore_unknown_ids=ignore_unknown_ids) + ) diff --git a/cognite/client/_sync_api/integrations/actions.py b/cognite/client/_sync_api/integrations/actions.py new file mode 100644 index 0000000000..e4b880e395 --- /dev/null +++ b/cognite/client/_sync_api/integrations/actions.py @@ -0,0 +1,158 @@ +""" +=============================================================================== +3f635530474c2e7d60de287a95002ad1 +This file is auto-generated from the Async API modules, - do not edit manually! +=============================================================================== +""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import TYPE_CHECKING, overload + +from cognite.client import AsyncCogniteClient +from cognite.client._constants import DEFAULT_LIMIT_READ +from cognite.client._sync_api_client import SyncAPIClient +from cognite.client.data_classes.integrations.actions import Action, ActionList, ActionWrite +from cognite.client.utils._async_helpers import run_sync +from cognite.client.utils.useful_types import SequenceNotStr + +if TYPE_CHECKING: + from cognite.client import AsyncCogniteClient + +_CREATE_LIMIT = 20 +_CANCEL_LIMIT = 100 + + +class SyncIntegrationActionsAPI(SyncAPIClient): + """Auto-generated, do not modify manually.""" + + def __init__(self, async_client: AsyncCogniteClient) -> None: + self.__async_client = async_client + + @overload + def create(self, external_id: str, action: ActionWrite) -> Action: ... + + @overload + def create(self, external_id: str, action: Sequence[ActionWrite]) -> ActionList: ... + + def create(self, external_id: str, action: ActionWrite | Sequence[ActionWrite]) -> Action | ActionList: + """ + `Create one or more actions `_ + + Args: + external_id (str): External id of the integration to trigger the action(s) against. + action (ActionWrite | Sequence[ActionWrite]): Action or list of actions to create. + + Returns: + Action | ActionList: Created action(s) + + Examples: + + Trigger an action against an integration: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes.integrations import ActionWrite + >>> client = CogniteClient() + >>> action = ActionWrite(external_id="my-restart-action", action_name="restart") + >>> res = client.integrations.actions.create("my-integration", action) + """ + return run_sync(self.__async_client.integrations.actions.create(external_id=external_id, action=action)) + + def list( + self, + external_id: str | None = None, + created_after: int | None = None, + include_completed: bool = True, + limit: int | None = DEFAULT_LIMIT_READ, + ) -> ActionList: + """ + `List actions `_ + + Args: + external_id (str | None): Only return actions for the integration with this external id. If not given, actions across all integrations you have access to are returned. + created_after (int | None): Only return actions created at or after this time, in milliseconds since epoch. + include_completed (bool): Whether to include actions in a terminal state (succeeded, failed, canceled). If False, only pending/running/cancel_pending actions are returned. + limit (int | None): Maximum number of actions to return. Defaults to 25. Set to -1, float("inf") or None to return all items. + + Returns: + ActionList: List of actions + + Examples: + + List pending actions for an integration: + + >>> from cognite.client import CogniteClient, AsyncCogniteClient + >>> client = CogniteClient() + >>> # async_client = AsyncCogniteClient() # another option + >>> res = client.integrations.actions.list( + ... external_id="my-integration", include_completed=False + ... ) + """ + return run_sync( + self.__async_client.integrations.actions.list( + external_id=external_id, created_after=created_after, include_completed=include_completed, limit=limit + ) + ) + + @overload + def retrieve(self, external_id: str, ignore_unknown_ids: bool = False) -> Action | None: ... + + @overload + def retrieve(self, external_id: SequenceNotStr[str], ignore_unknown_ids: bool = False) -> ActionList: ... + + def retrieve( + self, external_id: str | SequenceNotStr[str], ignore_unknown_ids: bool = False + ) -> Action | ActionList | None: + """ + `Retrieve one or more actions by external id `_ + + Args: + external_id (str | SequenceNotStr[str]): External id or list of external ids of actions to retrieve. + ignore_unknown_ids (bool): Ignore external ids that are not found rather than throw an exception. + + Returns: + Action | ActionList | None: Requested action(s), or None if a single requested external id is not found. + + Examples: + + Retrieve an action by external id: + + >>> from cognite.client import CogniteClient, AsyncCogniteClient + >>> client = CogniteClient() + >>> # async_client = AsyncCogniteClient() # another option + >>> res = client.integrations.actions.retrieve("my-restart-action") + """ + return run_sync( + self.__async_client.integrations.actions.retrieve( + external_id=external_id, ignore_unknown_ids=ignore_unknown_ids + ) + ) + + def cancel(self, external_id: str | SequenceNotStr[str], ignore_unknown_ids: bool = False) -> ActionList: + """ + `Cancel one or more actions `_ + + Only actions in the `pending`, `running` or `cancel_pending` state can be cancelled. + + Args: + external_id (str | SequenceNotStr[str]): External id or list of external ids of actions to cancel. + ignore_unknown_ids (bool): Ignore external ids that are not found rather than throw an exception. + + Returns: + ActionList: The action(s), with their updated status. + + Examples: + + Cancel a pending action: + + >>> from cognite.client import CogniteClient, AsyncCogniteClient + >>> client = CogniteClient() + >>> # async_client = AsyncCogniteClient() # another option + >>> res = client.integrations.actions.cancel("my-restart-action") + """ + return run_sync( + self.__async_client.integrations.actions.cancel( + external_id=external_id, ignore_unknown_ids=ignore_unknown_ids + ) + ) diff --git a/cognite/client/_sync_api/integrations/config.py b/cognite/client/_sync_api/integrations/config.py new file mode 100644 index 0000000000..9ab95373e1 --- /dev/null +++ b/cognite/client/_sync_api/integrations/config.py @@ -0,0 +1,106 @@ +""" +=============================================================================== +cce5eacd4949dd8e85347c3906460b9a +This file is auto-generated from the Async API modules, - do not edit manually! +=============================================================================== +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from cognite.client import AsyncCogniteClient +from cognite.client._constants import DEFAULT_LIMIT_READ +from cognite.client._sync_api_client import SyncAPIClient +from cognite.client.data_classes.integrations.config import ( + ConfigRevision, + ConfigRevisionMetadataList, + ConfigRevisionWrite, +) +from cognite.client.utils._async_helpers import run_sync + +if TYPE_CHECKING: + from cognite.client import AsyncCogniteClient + + +class SyncIntegrationConfigAPI(SyncAPIClient): + """Auto-generated, do not modify manually.""" + + def __init__(self, async_client: AsyncCogniteClient) -> None: + self.__async_client = async_client + + def create(self, config: ConfigRevision | ConfigRevisionWrite) -> ConfigRevision: + """ + `Create a new configuration revision `_ + + Args: + config (ConfigRevision | ConfigRevisionWrite): Configuration revision to create. + + Returns: + ConfigRevision: Created configuration revision + + Examples: + + Create a config revision: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes.integrations import ConfigRevisionWrite + >>> client = CogniteClient() + >>> res = client.integrations.config.create( + ... ConfigRevisionWrite(external_id="my-integration", config="my config contents") + ... ) + """ + return run_sync(self.__async_client.integrations.config.create(config=config)) + + def retrieve(self, external_id: str, revision: int | None = None) -> ConfigRevision: + """ + `Retrieve a specific configuration revision, or the latest by default `_ + + Args: + external_id (str): External id of the integration to retrieve config from. + revision (int | None): Optionally specify a revision number to retrieve. Defaults to the latest revision. + + Returns: + ConfigRevision: Retrieved configuration revision + + Examples: + + Retrieve latest config revision: + + >>> from cognite.client import CogniteClient, AsyncCogniteClient + >>> client = CogniteClient() + >>> # async_client = AsyncCogniteClient() # another option + >>> res = client.integrations.config.retrieve("my-integration") + """ + return run_sync(self.__async_client.integrations.config.retrieve(external_id=external_id, revision=revision)) + + def list( + self, external_id: str | None = None, limit: int | None = DEFAULT_LIMIT_READ + ) -> ConfigRevisionMetadataList: + """ + `List configuration revisions `_ + + Lists metadata about configuration revisions (without the config contents itself), ordered by revision + number descending. + + Note: + This endpoint does not support cursor-based pagination: it always returns up to `limit` of the most + recent revisions in a single page (server-side capped at 100). + + Args: + external_id (str | None): Only return config revisions for the integration with this external id. + limit (int | None): Maximum number of config revisions to return. Defaults to 25. + + Returns: + ConfigRevisionMetadataList: List of configuration revision metadata + + Examples: + + List config revisions for an integration: + + >>> from cognite.client import CogniteClient, AsyncCogniteClient + >>> client = CogniteClient() + >>> # async_client = AsyncCogniteClient() # another option + >>> res = client.integrations.config.list(external_id="my-integration") + """ + return run_sync(self.__async_client.integrations.config.list(external_id=external_id, limit=limit)) diff --git a/cognite/client/_sync_api/integrations/errors.py b/cognite/client/_sync_api/integrations/errors.py new file mode 100644 index 0000000000..cce4c66d70 --- /dev/null +++ b/cognite/client/_sync_api/integrations/errors.py @@ -0,0 +1,66 @@ +""" +=============================================================================== +d6864ed3490d3a8820c75aae29a7b5b3 +This file is auto-generated from the Async API modules, - do not edit manually! +=============================================================================== +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from cognite.client import AsyncCogniteClient +from cognite.client._constants import DEFAULT_LIMIT_READ +from cognite.client._sync_api_client import SyncAPIClient +from cognite.client.data_classes.integrations.errors import IntegrationErrorList +from cognite.client.utils._async_helpers import run_sync + +if TYPE_CHECKING: + from cognite.client import AsyncCogniteClient + + +class SyncIntegrationErrorsAPI(SyncAPIClient): + """Auto-generated, do not modify manually.""" + + def __init__(self, async_client: AsyncCogniteClient) -> None: + self.__async_client = async_client + + def list( + self, + external_id: str | None = None, + task: str | None = None, + min_start_time: int | None = None, + max_end_time: int | None = None, + limit: int | None = DEFAULT_LIMIT_READ, + ) -> IntegrationErrorList: + """ + `List errors `_ + + Args: + external_id (str | None): Only return errors for the integration with this external id. + task (str | None): Only return errors for the task with this name. Requires `external_id` to also be set. + min_start_time (int | None): Only return errors that started at or after this time, in milliseconds since epoch. + max_end_time (int | None): Only return errors that ended at or before this time, in milliseconds since epoch. + limit (int | None): Maximum number of errors to return. Defaults to 25. Set to -1, float("inf") or None to return all items. + + Returns: + IntegrationErrorList: List of errors + + Examples: + + List errors for a single integration: + + >>> from cognite.client import CogniteClient, AsyncCogniteClient + >>> client = CogniteClient() + >>> # async_client = AsyncCogniteClient() # another option + >>> res = client.integrations.errors.list(external_id="my-integration") + """ + return run_sync( + self.__async_client.integrations.errors.list( + external_id=external_id, + task=task, + min_start_time=min_start_time, + max_end_time=max_end_time, + limit=limit, + ) + ) diff --git a/cognite/client/_sync_api/integrations/tasks.py b/cognite/client/_sync_api/integrations/tasks.py new file mode 100644 index 0000000000..50b032d634 --- /dev/null +++ b/cognite/client/_sync_api/integrations/tasks.py @@ -0,0 +1,119 @@ +""" +=============================================================================== +231d30469457b599925a4f28e9de9a93 +This file is auto-generated from the Async API modules, - do not edit manually! +=============================================================================== +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from cognite.client import AsyncCogniteClient +from cognite.client._constants import DEFAULT_LIMIT_READ +from cognite.client._sync_api_client import SyncAPIClient +from cognite.client.data_classes.integrations.tasks import SyncResult, TaskHistoryList +from cognite.client.utils._async_helpers import run_sync + +if TYPE_CHECKING: + from cognite.client import AsyncCogniteClient + + +class SyncIntegrationTasksAPI(SyncAPIClient): + """Auto-generated, do not modify manually.""" + + def __init__(self, async_client: AsyncCogniteClient) -> None: + self.__async_client = async_client + + def list_history( + self, + external_id: str | None = None, + task_name: str | None = None, + last_per_task: bool = False, + limit: int | None = DEFAULT_LIMIT_READ, + ) -> TaskHistoryList: + """ + `List task history `_ + + Args: + external_id (str | None): Only return history for the integration with this external id. + task_name (str | None): Only return history for the task with this name. Requires `external_id` to also be set. + last_per_task (bool): Only return the latest history entry per task. + limit (int | None): Maximum number of history entries to return. Defaults to 25. Set to -1, float("inf") or None to return all items. + + Returns: + TaskHistoryList: List of task history entries + + Examples: + + List task history for a single integration: + + >>> from cognite.client import CogniteClient, AsyncCogniteClient + >>> client = CogniteClient() + >>> # async_client = AsyncCogniteClient() # another option + >>> res = client.integrations.tasks.list_history(external_id="my-integration") + """ + return run_sync( + self.__async_client.integrations.tasks.list_history( + external_id=external_id, task_name=task_name, last_per_task=last_per_task, limit=limit + ) + ) + + def sync( + self, + external_id: str, + task_name: str | None = None, + include_errors: bool = False, + include_task_updates: bool = False, + start_time: int | None = None, + cursor: str | None = None, + limit: int | None = DEFAULT_LIMIT_READ, + ) -> SyncResult: + """ + `Sync integration history `_ + + Incrementally fetch task history and/or errors for an integration since a previous sync call. This is more + efficient than repeatedly listing task history/errors when polling for updates, e.g. for dashboards or + alerting. At least one of `include_errors` and `include_task_updates` must be True. + + Args: + external_id (str): Only return history for the integration with this external id. + task_name (str | None): Only return history for the task with this name. + include_errors (bool): Include errors reported since the last sync. + include_task_updates (bool): Include task history entries reported since the last sync. + start_time (int | None): Only return items reported at or after this time, in milliseconds since epoch. Only used on the first call, pass the returned cursor on subsequent calls instead. + cursor (str | None): Cursor returned from a previous call to this method, to continue syncing from where you left off. + limit (int | None): Maximum number of items to return in this page. Defaults to 25. + + Returns: + SyncResult: A single page of results. Inspect `more_data` to see whether you should immediately call this method again with the returned `next_cursor`, or back off before doing so. + + Examples: + + Sync task history and errors for an integration: + + >>> from cognite.client import CogniteClient, AsyncCogniteClient + >>> client = CogniteClient() + >>> # async_client = AsyncCogniteClient() # another option + >>> res = client.integrations.tasks.sync( + ... external_id="my-integration", include_errors=True, include_task_updates=True + ... ) + >>> while res.more_data: + ... res = client.integrations.tasks.sync( + ... external_id="my-integration", + ... include_errors=True, + ... include_task_updates=True, + ... cursor=res.next_cursor, + ... ) + """ + return run_sync( + self.__async_client.integrations.tasks.sync( + external_id=external_id, + task_name=task_name, + include_errors=include_errors, + include_task_updates=include_task_updates, + start_time=start_time, + cursor=cursor, + limit=limit, + ) + ) diff --git a/cognite/client/_sync_cognite_client.py b/cognite/client/_sync_cognite_client.py index 79c7f51917..58d601117d 100644 --- a/cognite/client/_sync_cognite_client.py +++ b/cognite/client/_sync_cognite_client.py @@ -26,6 +26,7 @@ from cognite.client._sync_api.geospatial import SyncGeospatialAPI from cognite.client._sync_api.hosted_extractors import SyncHostedExtractorsAPI from cognite.client._sync_api.iam import SyncIAMAPI +from cognite.client._sync_api.integrations import SyncIntegrationsAPI from cognite.client._sync_api.labels import SyncLabelsAPI from cognite.client._sync_api.limits import SyncLimitsAPI from cognite.client._sync_api.metering import SyncMeteringAPI @@ -84,6 +85,7 @@ def __init__(self, config: ClientConfig | None = None) -> None: self.geospatial = SyncGeospatialAPI(async_client) self.hosted_extractors = SyncHostedExtractorsAPI(async_client) self.iam = SyncIAMAPI(async_client) + self.integrations = SyncIntegrationsAPI(async_client) self.labels = SyncLabelsAPI(async_client) self.limits = SyncLimitsAPI(async_client) self.metering = SyncMeteringAPI(async_client) diff --git a/cognite/client/data_classes/integrations/__init__.py b/cognite/client/data_classes/integrations/__init__.py new file mode 100644 index 0000000000..569d2c4767 --- /dev/null +++ b/cognite/client/data_classes/integrations/__init__.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from cognite.client.data_classes.integrations.actions import ( + Action, + ActionList, + ActionWrite, + ActionWriteList, +) +from cognite.client.data_classes.integrations.config import ( + ConfigRevision, + ConfigRevisionMetadata, + ConfigRevisionMetadataList, + ConfigRevisionWrite, +) +from cognite.client.data_classes.integrations.errors import ( + IntegrationError, + IntegrationErrorList, +) +from cognite.client.data_classes.integrations.integrations import ( + Extractor, + Integration, + IntegrationList, + IntegrationUpdate, + IntegrationWrite, + IntegrationWriteList, + Task, +) +from cognite.client.data_classes.integrations.tasks import ( + SyncResult, + TaskHistory, + TaskHistoryList, +) + +__all__ = [ + "Action", + "ActionList", + "ActionWrite", + "ActionWriteList", + "ConfigRevision", + "ConfigRevisionMetadata", + "ConfigRevisionMetadataList", + "ConfigRevisionWrite", + "Extractor", + "Integration", + "IntegrationError", + "IntegrationErrorList", + "IntegrationList", + "IntegrationUpdate", + "IntegrationWrite", + "IntegrationWriteList", + "SyncResult", + "Task", + "TaskHistory", + "TaskHistoryList", +] diff --git a/cognite/client/data_classes/integrations/actions.py b/cognite/client/data_classes/integrations/actions.py new file mode 100644 index 0000000000..b9d3bad508 --- /dev/null +++ b/cognite/client/data_classes/integrations/actions.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +from abc import ABC +from typing import Any, Literal, TypeAlias + +from typing_extensions import Self + +from cognite.client.data_classes._base import ( + CogniteResourceList, + ExternalIDTransformerMixin, + WriteableCogniteResource, + WriteableCogniteResourceList, +) + +ActionType: TypeAlias = Literal["start_task", "stop_task", "custom"] +ActionStatus: TypeAlias = Literal["pending", "running", "failed", "succeeded", "cancel_pending", "canceled"] + + +class ActionCore(WriteableCogniteResource["ActionWrite"], ABC): + """An action is a request for an integration to do something outside its normal task loop, + e.g. restart, reload config, or start/stop a task. + + The extractor polls for pending actions (through checkin) and reports the outcome back; no inbound + connectivity is required on the extractor side. + + Args: + external_id (str): External id of the action. Must be unique for the resource type. + action_name (str): Name of the action to trigger. Must match a name the extractor has registered as available. + call_metadata (dict[str, str] | None): Custom, application specific metadata passed to the extractor along with the action. + """ + + def __init__( + self, + external_id: str, + action_name: str, + call_metadata: dict[str, str] | None = None, + ) -> None: + self.external_id = external_id + self.action_name = action_name + self.call_metadata = call_metadata + + +class ActionWrite(ActionCore): + """An action is a request for an integration to do something outside its normal task loop. + This is the write/create format of the action. + + Args: + external_id (str): External id of the action. Must be unique for the resource type. + action_name (str): Name of the action to trigger. Must match a name the extractor has registered as available. + call_metadata (dict[str, str] | None): Custom, application specific metadata passed to the extractor along with the action. + """ + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + return cls( + external_id=resource["externalId"], + action_name=resource["actionName"], + call_metadata=resource.get("callMetadata"), + ) + + def as_write(self) -> ActionWrite: + return self + + +class Action(ActionCore): + """An action is a request for an integration to do something outside its normal task loop. + This is the read/response format of the action. + + Args: + external_id (str): External id of the action. Must be unique for the resource type. + action_name (str): Name of the action to trigger. Must match a name the extractor has registered as available. + status (ActionStatus): Current status of the action. + created_time (int): The time when this action was created, in milliseconds since epoch. + last_updated_time (int): The time when this action was last updated, in milliseconds since epoch. + call_metadata (dict[str, str] | None): Custom, application specific metadata passed to the extractor along with the action. + result_message (str | None): Message reported by the extractor when the action completed or failed. + result_metadata (dict[str, str] | None): Custom, application specific metadata reported by the extractor when the action completed or failed. + """ + + def __init__( + self, + external_id: str, + action_name: str, + status: ActionStatus, + created_time: int, + last_updated_time: int, + call_metadata: dict[str, str] | None = None, + result_message: str | None = None, + result_metadata: dict[str, str] | None = None, + ) -> None: + super().__init__(external_id=external_id, action_name=action_name, call_metadata=call_metadata) + self.status = status + self.created_time = created_time + self.last_updated_time = last_updated_time + self.result_message = result_message + self.result_metadata = result_metadata + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + return cls( + external_id=resource["externalId"], + action_name=resource["actionName"], + status=resource["status"], + created_time=resource["createdTime"], + last_updated_time=resource["lastUpdatedTime"], + call_metadata=resource.get("callMetadata"), + result_message=resource.get("resultMessage"), + result_metadata=resource.get("resultMetadata"), + ) + + def as_write(self) -> ActionWrite: + """Returns this Action as an ActionWrite""" + return ActionWrite(external_id=self.external_id, action_name=self.action_name, call_metadata=self.call_metadata) + + def __hash__(self) -> int: + return hash(self.external_id) + + +class ActionWriteList(CogniteResourceList[ActionWrite], ExternalIDTransformerMixin): + _RESOURCE = ActionWrite + + +class ActionList(WriteableCogniteResourceList[ActionWrite, Action], ExternalIDTransformerMixin): + _RESOURCE = Action + + def as_write(self) -> ActionWriteList: + return ActionWriteList([item.as_write() for item in self.data]) diff --git a/cognite/client/data_classes/integrations/config.py b/cognite/client/data_classes/integrations/config.py new file mode 100644 index 0000000000..bca89194ad --- /dev/null +++ b/cognite/client/data_classes/integrations/config.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +from abc import ABC +from typing import Any + +from typing_extensions import Self + +from cognite.client.data_classes._base import ( + CogniteResource, + CogniteResourceList, + ExternalIDTransformerMixin, + WriteableCogniteResource, +) + + +class ConfigRevisionCore(WriteableCogniteResource["ConfigRevisionWrite"], ABC): + """A versioned configuration document associated with an integration. + + Every write creates a new, immutable revision rather than overwriting the previous one. + + Args: + external_id (str): External id of the integration this config revision belongs to. + config (str | None): Contents of this configuration revision. + description (str | None): Short description of this configuration revision. + """ + + def __init__(self, external_id: str, config: str | None = None, description: str | None = None) -> None: + self.external_id = external_id + self.config = config + self.description = description + + +class ConfigRevisionWrite(ConfigRevisionCore): + """A new configuration revision to create for an integration. + + Args: + external_id (str): External id of the integration to create the config revision for. + config (str | None): Contents of this configuration revision. + description (str | None): Short description of this configuration revision. + """ + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + return cls( + external_id=resource["externalId"], + config=resource.get("config"), + description=resource.get("description"), + ) + + def as_write(self) -> ConfigRevisionWrite: + return self + + +class ConfigRevision(ConfigRevisionCore): + """A single configuration revision for an integration, including its contents. + + Args: + external_id (str): External id of the integration this config revision belongs to. + revision (int): The revision number of this config revision. + created_time (int): Time the config revision was created, in milliseconds since epoch. + last_updated_time (int): Time the config revision was last updated, in milliseconds since epoch. + config (str | None): Contents of this configuration revision. + description (str | None): Short description of this configuration revision. + """ + + def __init__( + self, + external_id: str, + revision: int, + created_time: int, + last_updated_time: int, + config: str | None = None, + description: str | None = None, + ) -> None: + super().__init__(external_id=external_id, config=config, description=description) + self.revision = revision + self.created_time = created_time + self.last_updated_time = last_updated_time + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + return cls( + external_id=resource["externalId"], + revision=resource["revision"], + created_time=resource["createdTime"], + last_updated_time=resource["lastUpdatedTime"], + config=resource.get("config"), + description=resource.get("description"), + ) + + def as_write(self) -> ConfigRevisionWrite: + """Returns this ConfigRevision as a ConfigRevisionWrite""" + return ConfigRevisionWrite(external_id=self.external_id, config=self.config, description=self.description) + + +class ConfigRevisionMetadata(CogniteResource): + """Metadata about a configuration revision, without the config contents itself. + + Args: + external_id (str): External id of the integration this config revision belongs to. + revision (int): The revision number of this config revision. + created_time (int): Time the config revision was created, in milliseconds since epoch. + last_updated_time (int): Time the config revision was last updated, in milliseconds since epoch. + description (str | None): Short description of this configuration revision. + """ + + def __init__( + self, + external_id: str, + revision: int, + created_time: int, + last_updated_time: int, + description: str | None = None, + ) -> None: + self.external_id = external_id + self.revision = revision + self.created_time = created_time + self.last_updated_time = last_updated_time + self.description = description + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + return cls( + external_id=resource["externalId"], + revision=resource["revision"], + created_time=resource["createdTime"], + last_updated_time=resource["lastUpdatedTime"], + description=resource.get("description"), + ) + + +class ConfigRevisionMetadataList(CogniteResourceList[ConfigRevisionMetadata], ExternalIDTransformerMixin): + _RESOURCE = ConfigRevisionMetadata diff --git a/cognite/client/data_classes/integrations/errors.py b/cognite/client/data_classes/integrations/errors.py new file mode 100644 index 0000000000..8132941540 --- /dev/null +++ b/cognite/client/data_classes/integrations/errors.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from typing import Any, Literal, TypeAlias + +from typing_extensions import Self + +from cognite.client.data_classes._base import CogniteResource, CogniteResourceList, ExternalIDTransformerMixin +from cognite.client.data_classes.integrations.integrations import ActiveConfigRevision + +ErrorLevel: TypeAlias = Literal["warning", "error", "fatal"] +IntegrationErrorType: TypeAlias = Literal["general", "config", "task_never_closed", "seen_deadline_missed"] + + +class IntegrationError(CogniteResource): + """A problem an extractor encountered while running a task, reported to CDF. + + Args: + external_id (str): External id of the integration the error belongs to. + level (ErrorLevel): Severity of the error. + description (str): Short description of the error. + start_time (int): Time the error started, in milliseconds since epoch. + details (str | None): Full details of the error, e.g. a stack trace. + end_time (int | None): Time the error was resolved, in milliseconds since epoch. Not set while unresolved. + task (str | None): Name of the task the error occurred in. Not set if the error applies to the extractor generally. + type (IntegrationErrorType | None): Category of the error. + active_config_revision (int | Literal["local"] | None): The config revision (or "local") active when the error occurred. + """ + + def __init__( + self, + external_id: str, + level: ErrorLevel, + description: str, + start_time: int, + details: str | None = None, + end_time: int | None = None, + task: str | None = None, + type: IntegrationErrorType | None = None, + active_config_revision: ActiveConfigRevision | None = None, + ) -> None: + self.external_id = external_id + self.level = level + self.description = description + self.details = details + self.start_time = start_time + self.end_time = end_time + self.task = task + self.type = type + self.active_config_revision = active_config_revision + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + return cls( + external_id=resource["externalId"], + level=resource["level"], + description=resource["description"], + details=resource.get("details"), + start_time=resource["startTime"], + end_time=resource.get("endTime"), + task=resource.get("task"), + type=resource.get("type"), + active_config_revision=resource.get("activeConfigRevision"), + ) + + +class IntegrationErrorList(CogniteResourceList[IntegrationError], ExternalIDTransformerMixin): + _RESOURCE = IntegrationError diff --git a/cognite/client/data_classes/integrations/integrations.py b/cognite/client/data_classes/integrations/integrations.py new file mode 100644 index 0000000000..cc6268eacd --- /dev/null +++ b/cognite/client/data_classes/integrations/integrations.py @@ -0,0 +1,304 @@ +from __future__ import annotations + +from abc import ABC +from typing import Any, Literal, TypeAlias + +from typing_extensions import Self + +from cognite.client.data_classes._base import ( + CogniteObjectUpdate, + CognitePrimitiveUpdate, + CogniteResource, + CogniteResourceList, + CogniteUpdate, + ExternalIDTransformerMixin, + PropertySpec, + WriteableCogniteResource, + WriteableCogniteResourceList, +) + +ActiveConfigRevision: TypeAlias = int | Literal["local"] + + +class Extractor(CogniteResource): + """The extractor (or other process) that reports as this integration. + + Args: + external_id (str): External id of the extractor, e.g. "cognite-simple-influxdb-extractor" for a Cognite-built extractor. + version (str | None): The version of the extractor. + """ + + def __init__(self, external_id: str, version: str | None = None) -> None: + self.external_id = external_id + self.version = version + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + return cls(external_id=resource["externalId"], version=resource.get("version")) + + +class Task(CogniteResource): + """A named unit of work in an integration, reported by the extractor. + + Args: + type (Literal["continuous", "batch"]): Whether the task runs for the lifetime of the extractor (continuous) or runs to completion and exits (batch). + name (str): Name of the task, unique within the integration. + action (bool): Whether this task can be triggered through an Action. Defaults to False. + description (str | None): Description of the task. + sources (list[str] | None): Lineage: URIs of the systems/resources this task reads from. + targets (list[str] | None): Lineage: URIs of the CDF (or other) resources this task writes to. + """ + + def __init__( + self, + type: Literal["continuous", "batch"], + name: str, + action: bool = False, + description: str | None = None, + sources: list[str] | None = None, + targets: list[str] | None = None, + ) -> None: + self.type = type + self.name = name + self.action = action + self.description = description + self.sources = sources + self.targets = targets + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + return cls( + type=resource["type"], + name=resource["name"], + action=resource.get("action", False), + description=resource.get("description"), + sources=resource.get("sources"), + targets=resource.get("targets"), + ) + + +class IntegrationCore(WriteableCogniteResource["IntegrationWrite"], ABC): + """An integration is a record of an extractor or other process that sends data to CDF. + + It identifies the extractor type and holds the configuration, task history, and error history for that data + pipeline. Note that an integration isn't an extraction pipeline: don't use both for the same external ID. + + Args: + external_id (str): The external ID provided by the client. Must be unique for the resource type. + extractor (Extractor): The extractor (or other process) that reports as this integration. + name (str | None): Name of the integration. + description (str | None): Description of the integration. + documentation (str | None): Documentation for the integration, formatted as markdown. + metadata (dict[str, str] | None): Custom, application specific metadata. String key -> String value. + allowed_not_seen_minutes (int | None): Number of minutes the integration is allowed to not report in before it's flagged as inactive. Defaults to 1440 (1 day) server-side. + """ + + def __init__( + self, + external_id: str, + extractor: Extractor, + name: str | None = None, + description: str | None = None, + documentation: str | None = None, + metadata: dict[str, str] | None = None, + allowed_not_seen_minutes: int | None = None, + ) -> None: + self.external_id = external_id + self.extractor = extractor + self.name = name + self.description = description + self.documentation = documentation + self.metadata = metadata + self.allowed_not_seen_minutes = allowed_not_seen_minutes + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + result = super().dump(camel_case) + result["extractor"] = self.extractor.dump(camel_case) + return result + + +class IntegrationWrite(IntegrationCore): + """An integration is a record of an extractor or other process that sends data to CDF. + This is the write/create format of the integration. + + Args: + external_id (str): The external ID provided by the client. Must be unique for the resource type. + extractor (Extractor): The extractor (or other process) that reports as this integration. + name (str | None): Name of the integration. + description (str | None): Description of the integration. + documentation (str | None): Documentation for the integration, formatted as markdown. + metadata (dict[str, str] | None): Custom, application specific metadata. String key -> String value. + allowed_not_seen_minutes (int | None): Number of minutes the integration is allowed to not report in before it's flagged as inactive. Defaults to 1440 (1 day) server-side. + """ + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + return cls( + external_id=resource["externalId"], + extractor=Extractor._load(resource["extractor"]), + name=resource.get("name"), + description=resource.get("description"), + documentation=resource.get("documentation"), + metadata=resource.get("metadata"), + allowed_not_seen_minutes=resource.get("allowedNotSeenMinutes"), + ) + + def as_write(self) -> IntegrationWrite: + return self + + +class Integration(IntegrationCore): + """An integration is a record of an extractor or other process that sends data to CDF. + This is the read/response format of the integration. + + Args: + external_id (str): The external ID provided by the client. Must be unique for the resource type. + extractor (Extractor): The extractor (or other process) that reports as this integration. + created_time (int): The time when this integration was created, in milliseconds since epoch. + last_updated_time (int): The time when this integration was last updated, in milliseconds since epoch. + name (str | None): Name of the integration. + description (str | None): Description of the integration. + documentation (str | None): Documentation for the integration, formatted as markdown. + metadata (dict[str, str] | None): Custom, application specific metadata. String key -> String value. + allowed_not_seen_minutes (int | None): Number of minutes the integration is allowed to not report in before it's flagged as inactive. + last_seen (int | None): The time this integration was last seen (checked in), in milliseconds since epoch. + last_config_revision (int | None): The revision number of the last config revision created for this integration. + active_config_revision (int | Literal["local"] | None): The config revision currently reported active by the extractor, or "local" if it's using a local config file instead of a revision managed through CDF. + tasks (list[Task] | None): The tasks the extractor has reported as part of this integration. + """ + + def __init__( + self, + external_id: str, + extractor: Extractor, + created_time: int, + last_updated_time: int, + name: str | None = None, + description: str | None = None, + documentation: str | None = None, + metadata: dict[str, str] | None = None, + allowed_not_seen_minutes: int | None = None, + last_seen: int | None = None, + last_config_revision: int | None = None, + active_config_revision: ActiveConfigRevision | None = None, + tasks: list[Task] | None = None, + ) -> None: + super().__init__( + external_id=external_id, + extractor=extractor, + name=name, + description=description, + documentation=documentation, + metadata=metadata, + allowed_not_seen_minutes=allowed_not_seen_minutes, + ) + self.created_time = created_time + self.last_updated_time = last_updated_time + self.last_seen = last_seen + self.last_config_revision = last_config_revision + self.active_config_revision = active_config_revision + self.tasks = tasks or [] + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + result = super().dump(camel_case) + result["tasks"] = [task.dump(camel_case) for task in self.tasks] + return result + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + return cls( + external_id=resource["externalId"], + extractor=Extractor._load(resource["extractor"]), + created_time=resource["createdTime"], + last_updated_time=resource["lastUpdatedTime"], + name=resource.get("name"), + description=resource.get("description"), + documentation=resource.get("documentation"), + metadata=resource.get("metadata"), + allowed_not_seen_minutes=resource.get("allowedNotSeenMinutes"), + last_seen=resource.get("lastSeen"), + last_config_revision=resource.get("lastConfigRevision"), + active_config_revision=resource.get("activeConfigRevision"), + tasks=[Task._load(task) for task in resource.get("tasks", [])], + ) + + def as_write(self) -> IntegrationWrite: + """Returns this Integration as an IntegrationWrite""" + return IntegrationWrite( + external_id=self.external_id, + extractor=self.extractor, + name=self.name, + description=self.description, + documentation=self.documentation, + metadata=self.metadata, + allowed_not_seen_minutes=self.allowed_not_seen_minutes, + ) + + def __hash__(self) -> int: + return hash(self.external_id) + + +class IntegrationWriteList(CogniteResourceList[IntegrationWrite], ExternalIDTransformerMixin): + _RESOURCE = IntegrationWrite + + +class IntegrationList(WriteableCogniteResourceList[IntegrationWrite, Integration], ExternalIDTransformerMixin): + _RESOURCE = Integration + + def as_write(self) -> IntegrationWriteList: + return IntegrationWriteList([item.as_write() for item in self.data]) + + +class IntegrationUpdate(CogniteUpdate): + """Changes applied to an integration + + Args: + external_id (str): The external ID provided by the client. Must be unique for the resource type. + """ + + def __init__(self, external_id: str) -> None: + super().__init__(external_id=external_id) + + class _PrimitiveIntegrationUpdate(CognitePrimitiveUpdate): + def set(self, value: Any) -> IntegrationUpdate: + return self._set(value) + + class _ObjectIntegrationUpdate(CogniteObjectUpdate): + def set(self, value: dict) -> IntegrationUpdate: + return self._set(value) + + def add(self, value: dict) -> IntegrationUpdate: + return self._add(value) + + def remove(self, value: list) -> IntegrationUpdate: + return self._remove(value) + + @property + def name(self) -> _PrimitiveIntegrationUpdate: + return IntegrationUpdate._PrimitiveIntegrationUpdate(self, "name") + + @property + def description(self) -> _PrimitiveIntegrationUpdate: + return IntegrationUpdate._PrimitiveIntegrationUpdate(self, "description") + + @property + def documentation(self) -> _PrimitiveIntegrationUpdate: + return IntegrationUpdate._PrimitiveIntegrationUpdate(self, "documentation") + + @property + def allowed_not_seen_minutes(self) -> _PrimitiveIntegrationUpdate: + return IntegrationUpdate._PrimitiveIntegrationUpdate(self, "allowedNotSeenMinutes") + + @property + def metadata(self) -> _ObjectIntegrationUpdate: + return IntegrationUpdate._ObjectIntegrationUpdate(self, "metadata") + + @classmethod + def _get_update_properties(cls, item: CogniteResource | None = None) -> list[PropertySpec]: + return [ + PropertySpec("name"), + PropertySpec("description"), + PropertySpec("documentation"), + PropertySpec("allowed_not_seen_minutes"), + PropertySpec("metadata", is_object=True), + ] diff --git a/cognite/client/data_classes/integrations/tasks.py b/cognite/client/data_classes/integrations/tasks.py new file mode 100644 index 0000000000..a850f5d3f7 --- /dev/null +++ b/cognite/client/data_classes/integrations/tasks.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +from typing import Any + +from typing_extensions import Self + +from cognite.client.data_classes._base import CogniteResource, CogniteResourceList, ExternalIDTransformerMixin +from cognite.client.data_classes.integrations.errors import IntegrationErrorList +from cognite.client.data_classes.integrations.integrations import ActiveConfigRevision + + +class TaskHistory(CogniteResource): + """A single start/stop event of a task, reported by the extractor. + + Args: + external_id (str): External id of the integration the task belongs to. + task_name (str): Name of the task. + start_time (int): Time the task started, in milliseconds since epoch. + end_time (int | None): Time the task ended, in milliseconds since epoch. Not set while the task is still running. + message (str | None): Optional message reported when the task started or ended. + error_count (int): Number of errors reported for this task run. + warning_count (int): Number of warnings reported for this task run. + fatal_count (int): Number of fatal errors reported for this task run. + active_config_revision (int | Literal["local"] | None): The config revision (or "local") active at the time of this task run. + sources (list[str] | None): Lineage: URIs of the systems/resources this task read from. + targets (list[str] | None): Lineage: URIs of the CDF (or other) resources this task wrote to. + """ + + def __init__( + self, + external_id: str, + task_name: str, + start_time: int, + end_time: int | None = None, + message: str | None = None, + error_count: int = 0, + warning_count: int = 0, + fatal_count: int = 0, + active_config_revision: ActiveConfigRevision | None = None, + sources: list[str] | None = None, + targets: list[str] | None = None, + ) -> None: + self.external_id = external_id + self.task_name = task_name + self.start_time = start_time + self.end_time = end_time + self.message = message + self.error_count = error_count + self.warning_count = warning_count + self.fatal_count = fatal_count + self.active_config_revision = active_config_revision + self.sources = sources + self.targets = targets + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + return cls( + external_id=resource["externalId"], + task_name=resource["taskName"], + start_time=resource["startTime"], + end_time=resource.get("endTime"), + message=resource.get("message"), + error_count=resource.get("errorCount", 0), + warning_count=resource.get("warningCount", 0), + fatal_count=resource.get("fatalCount", 0), + active_config_revision=resource.get("activeConfigRevision"), + sources=resource.get("sources"), + targets=resource.get("targets"), + ) + + +class TaskHistoryList(CogniteResourceList[TaskHistory], ExternalIDTransformerMixin): + _RESOURCE = TaskHistory + + +class SyncResult(CogniteResource): + """The result of a single call to the incremental integration sync endpoint. + + Args: + next_cursor (str): Cursor to pass into the next call to continue from where this page left off. + more_data (bool): Whether there is more data available immediately (True), or whether the caller should back off before polling again (False). + history (TaskHistoryList | None): Task history entries since the previous cursor, if requested. + errors (IntegrationErrorList | None): Errors reported since the previous cursor, if requested. + """ + + def __init__( + self, + next_cursor: str, + more_data: bool = False, + history: TaskHistoryList | None = None, + errors: IntegrationErrorList | None = None, + ) -> None: + self.next_cursor = next_cursor + self.more_data = more_data + self.history = history + self.errors = errors + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + result = super().dump(camel_case) + if self.history is not None: + result["history"] = self.history.dump(camel_case) + if self.errors is not None: + result["errors"] = self.errors.dump(camel_case) + return result + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + return cls( + next_cursor=resource["nextCursor"], + more_data=resource.get("moreData", False), + history=TaskHistoryList._load(resource["history"]) if "history" in resource else None, + errors=IntegrationErrorList._load(resource["errors"]) if "errors" in resource else None, + ) diff --git a/cognite/client/testing.py b/cognite/client/testing.py index 846641b01f..8a6f4c7dee 100644 --- a/cognite/client/testing.py +++ b/cognite/client/testing.py @@ -51,6 +51,11 @@ from cognite.client._api.iam.security_categories import SecurityCategoriesAPI from cognite.client._api.iam.sessions import SessionsAPI from cognite.client._api.iam.token import TokenAPI +from cognite.client._api.integrations import IntegrationsAPI +from cognite.client._api.integrations.actions import IntegrationActionsAPI +from cognite.client._api.integrations.config import IntegrationConfigAPI +from cognite.client._api.integrations.errors import IntegrationErrorsAPI +from cognite.client._api.integrations.tasks import IntegrationTasksAPI from cognite.client._api.labels import LabelsAPI from cognite.client._api.limits import LimitsAPI from cognite.client._api.metering import MeteringAPI @@ -140,6 +145,11 @@ from cognite.client._sync_api.iam.security_categories import SyncSecurityCategoriesAPI from cognite.client._sync_api.iam.sessions import SyncSessionsAPI from cognite.client._sync_api.iam.token import SyncTokenAPI +from cognite.client._sync_api.integrations import SyncIntegrationsAPI +from cognite.client._sync_api.integrations.actions import SyncIntegrationActionsAPI +from cognite.client._sync_api.integrations.config import SyncIntegrationConfigAPI +from cognite.client._sync_api.integrations.errors import SyncIntegrationErrorsAPI +from cognite.client._sync_api.integrations.tasks import SyncIntegrationTasksAPI from cognite.client._sync_api.labels import SyncLabelsAPI from cognite.client._sync_api.limits import SyncLimitsAPI from cognite.client._sync_api.metering import SyncMeteringAPI @@ -335,6 +345,20 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: ) flip_spec_set_on(self.iam) + integrations_tasks = create_autospec(IntegrationTasksAPI, instance=True, spec_set=True) + integrations_errors = create_autospec(IntegrationErrorsAPI, instance=True, spec_set=True) + integrations_config = create_autospec(IntegrationConfigAPI, instance=True, spec_set=True) + integrations_actions = create_autospec(IntegrationActionsAPI, instance=True, spec_set=True) + self.integrations = create_autospec( + IntegrationsAPI, + instance=True, + tasks=integrations_tasks, + errors=integrations_errors, + config=integrations_config, + actions=integrations_actions, + ) + flip_spec_set_on(self.integrations) + self.labels = create_autospec(LabelsAPI, instance=True, spec_set=True) self.limits = create_autospec(LimitsAPI, instance=True, spec_set=True) self.metering = create_autospec(MeteringAPI, instance=True, spec_set=True) @@ -546,6 +570,20 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: ) flip_spec_set_on(self.iam) + integrations_tasks = create_autospec(SyncIntegrationTasksAPI, instance=True, spec_set=True) + integrations_errors = create_autospec(SyncIntegrationErrorsAPI, instance=True, spec_set=True) + integrations_config = create_autospec(SyncIntegrationConfigAPI, instance=True, spec_set=True) + integrations_actions = create_autospec(SyncIntegrationActionsAPI, instance=True, spec_set=True) + self.integrations = create_autospec( + SyncIntegrationsAPI, + instance=True, + tasks=integrations_tasks, + errors=integrations_errors, + config=integrations_config, + actions=integrations_actions, + ) + flip_spec_set_on(self.integrations) + self.labels = create_autospec(SyncLabelsAPI, instance=True, spec_set=True) self.limits = create_autospec(SyncLimitsAPI, instance=True, spec_set=True) self.metering = create_autospec(SyncMeteringAPI, instance=True, spec_set=True) diff --git a/cognite/client/utils/_url.py b/cognite/client/utils/_url.py index 567a752959..ffff73943c 100644 --- a/cognite/client/utils/_url.py +++ b/cognite/client/utils/_url.py @@ -27,6 +27,9 @@ "geospatial/featuretypes", "geospatial/featuretypes/[^/]+/features", "hostedextractors", + "integrations", + "integrations/actions", + "integrations/config", "labels", "postgresgateway", "profiles", @@ -56,6 +59,7 @@ "ai/tools/documents/task", "annotations/suggest", "extpipes/config/revert", + "integrations/actions/cancel", "transformations/cancel", "transformations/notifications", "transformations/run", diff --git a/tests/tests_unit/test_api/test_integrations/__init__.py b/tests/tests_unit/test_api/test_integrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/tests_unit/test_api/test_integrations/test_actions.py b/tests/tests_unit/test_api/test_integrations/test_actions.py new file mode 100644 index 0000000000..68e00a1795 --- /dev/null +++ b/tests/tests_unit/test_api/test_integrations/test_actions.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import re + +from pytest_httpx import HTTPXMock + +from cognite.client import AsyncCogniteClient, CogniteClient +from cognite.client.data_classes.integrations import Action, ActionList, ActionWrite +from tests.utils import get_url, jsgz_load + +ACTION_RESPONSE = { + "externalId": "my-action", + "actionName": "restart", + "status": "pending", + "createdTime": 0, + "lastUpdatedTime": 0, +} + + +class TestIntegrationActions: + def test_create_single( + self, cognite_client: CogniteClient, async_client: AsyncCogniteClient, httpx_mock: HTTPXMock + ) -> None: + httpx_mock.add_response( + method="POST", + url=re.compile(re.escape(get_url(async_client.integrations.actions, "/integrations/actions")) + r"\?.*"), + json={"items": [ACTION_RESPONSE]}, + status_code=201, + ) + + res = cognite_client.integrations.actions.create( + "my-integration", ActionWrite(external_id="my-action", action_name="restart") + ) + + assert isinstance(res, Action) + assert res.status == "pending" + + request = httpx_mock.get_requests()[0] + assert "externalId=my-integration" in str(request.url) + body = jsgz_load(request.content) + assert body == {"items": [{"externalId": "my-action", "actionName": "restart"}]} + + def test_list(self, cognite_client: CogniteClient, async_client: AsyncCogniteClient, httpx_mock: HTTPXMock) -> None: + url_pattern = re.compile( + re.escape(get_url(async_client.integrations.actions, "/integrations/actions")) + r"(?:\?.*)?$" + ) + httpx_mock.add_response(method="GET", url=url_pattern, json={"items": [ACTION_RESPONSE]}) + + res = cognite_client.integrations.actions.list(external_id="my-integration", include_completed=False) + + assert isinstance(res, ActionList) + assert len(res) == 1 + + request = httpx_mock.get_requests()[0] + assert "includeCompleted=false" in str(request.url) + + def test_retrieve( + self, cognite_client: CogniteClient, async_client: AsyncCogniteClient, httpx_mock: HTTPXMock + ) -> None: + httpx_mock.add_response( + method="POST", + url=get_url(async_client.integrations.actions, "/integrations/actions/byids"), + json={"items": [ACTION_RESPONSE]}, + ) + + res = cognite_client.integrations.actions.retrieve("my-action") + + assert isinstance(res, Action) + assert res.external_id == "my-action" + + def test_cancel( + self, cognite_client: CogniteClient, async_client: AsyncCogniteClient, httpx_mock: HTTPXMock + ) -> None: + cancelled_response = {**ACTION_RESPONSE, "status": "cancel_pending"} + httpx_mock.add_response( + method="POST", + url=get_url(async_client.integrations.actions, "/integrations/actions/cancel"), + json={"items": [cancelled_response]}, + ) + + res = cognite_client.integrations.actions.cancel("my-action") + + assert isinstance(res, ActionList) + assert res[0].status == "cancel_pending" + + body = jsgz_load(httpx_mock.get_requests()[0].content) + assert body == {"items": [{"externalId": "my-action"}]} diff --git a/tests/tests_unit/test_api/test_integrations/test_config.py b/tests/tests_unit/test_api/test_integrations/test_config.py new file mode 100644 index 0000000000..9e85f70ead --- /dev/null +++ b/tests/tests_unit/test_api/test_integrations/test_config.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +import re + +from pytest_httpx import HTTPXMock + +from cognite.client import AsyncCogniteClient, CogniteClient +from cognite.client.data_classes.integrations import ConfigRevision, ConfigRevisionMetadataList, ConfigRevisionWrite +from tests.utils import get_url, jsgz_load + +CONFIG_REVISION_RESPONSE = { + "externalId": "my-integration", + "revision": 1, + "config": "my config contents", + "createdTime": 0, + "lastUpdatedTime": 0, +} +CONFIG_REVISION_METADATA_RESPONSE = { + "externalId": "my-integration", + "revision": 1, + "createdTime": 0, + "lastUpdatedTime": 0, +} + + +class TestIntegrationConfig: + def test_create( + self, cognite_client: CogniteClient, async_client: AsyncCogniteClient, httpx_mock: HTTPXMock + ) -> None: + httpx_mock.add_response( + method="POST", + url=get_url(async_client.integrations.config, "/integrations/config"), + json=CONFIG_REVISION_RESPONSE, + ) + + res = cognite_client.integrations.config.create( + ConfigRevisionWrite(external_id="my-integration", config="my config contents") + ) + + assert isinstance(res, ConfigRevision) + assert res.revision == 1 + assert res.config == "my config contents" + + body = jsgz_load(httpx_mock.get_requests()[0].content) + assert body == {"externalId": "my-integration", "config": "my config contents"} + + def test_retrieve( + self, cognite_client: CogniteClient, async_client: AsyncCogniteClient, httpx_mock: HTTPXMock + ) -> None: + url_pattern = re.compile( + re.escape(get_url(async_client.integrations.config, "/integrations/config")) + r"(?:\?.*)?$" + ) + httpx_mock.add_response(method="GET", url=url_pattern, json=CONFIG_REVISION_RESPONSE) + + res = cognite_client.integrations.config.retrieve("my-integration", revision=1) + + assert isinstance(res, ConfigRevision) + assert res.revision == 1 + + request = httpx_mock.get_requests()[0] + assert "externalId=my-integration" in str(request.url) + assert "revision=1" in str(request.url) + + def test_list(self, cognite_client: CogniteClient, async_client: AsyncCogniteClient, httpx_mock: HTTPXMock) -> None: + url_pattern = re.compile( + re.escape(get_url(async_client.integrations.config, "/integrations/config/revisions")) + r"(?:\?.*)?$" + ) + httpx_mock.add_response(method="GET", url=url_pattern, json={"items": [CONFIG_REVISION_METADATA_RESPONSE]}) + + res = cognite_client.integrations.config.list(external_id="my-integration") + + assert isinstance(res, ConfigRevisionMetadataList) + assert len(res) == 1 + assert res[0].revision == 1 + + request = httpx_mock.get_requests()[0] + assert "externalId=my-integration" in str(request.url) + assert "limit=25" in str(request.url) diff --git a/tests/tests_unit/test_api/test_integrations/test_errors.py b/tests/tests_unit/test_api/test_integrations/test_errors.py new file mode 100644 index 0000000000..83cc49e5a8 --- /dev/null +++ b/tests/tests_unit/test_api/test_integrations/test_errors.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import re + +from pytest_httpx import HTTPXMock + +from cognite.client import AsyncCogniteClient, CogniteClient +from cognite.client.data_classes.integrations.errors import IntegrationErrorList +from tests.utils import get_url + +ERROR_RESPONSE = { + "externalId": "my-integration", + "level": "error", + "description": "Something went wrong", + "startTime": 100, + "task": "poll", +} + + +class TestIntegrationErrors: + def test_list(self, cognite_client: CogniteClient, async_client: AsyncCogniteClient, httpx_mock: HTTPXMock) -> None: + url_pattern = re.compile( + re.escape(get_url(async_client.integrations.errors, "/integrations/errors")) + r"(?:\?.*)?$" + ) + httpx_mock.add_response(method="GET", url=url_pattern, json={"items": [ERROR_RESPONSE]}) + + res = cognite_client.integrations.errors.list(external_id="my-integration", task="poll") + + assert isinstance(res, IntegrationErrorList) + assert len(res) == 1 + assert res[0].level == "error" + assert res[0].description == "Something went wrong" + + request = httpx_mock.get_requests()[0] + assert "externalId=my-integration" in str(request.url) + assert "task=poll" in str(request.url) diff --git a/tests/tests_unit/test_api/test_integrations/test_integrations.py b/tests/tests_unit/test_api/test_integrations/test_integrations.py new file mode 100644 index 0000000000..7d66b79408 --- /dev/null +++ b/tests/tests_unit/test_api/test_integrations/test_integrations.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import re + +from pytest_httpx import HTTPXMock + +from cognite.client import AsyncCogniteClient, CogniteClient +from cognite.client.data_classes.integrations import ( + Extractor, + Integration, + IntegrationList, + IntegrationUpdate, + IntegrationWrite, +) +from tests.utils import get_url, jsgz_load + +INTEGRATION_RESPONSE = { + "externalId": "my-integration", + "extractor": {"externalId": "cognite-simple-influxdb-extractor", "version": "1.0.0"}, + "name": "My integration", + "createdTime": 0, + "lastUpdatedTime": 0, + "tasks": [{"type": "continuous", "name": "poll", "action": False}], +} + + +class TestIntegrations: + def test_list(self, cognite_client: CogniteClient, async_client: AsyncCogniteClient, httpx_mock: HTTPXMock) -> None: + url_pattern = re.compile(re.escape(get_url(async_client.integrations, "/integrations")) + r"(?:\?.*)?$") + httpx_mock.add_response(method="GET", url=url_pattern, json={"items": [INTEGRATION_RESPONSE]}) + + res = cognite_client.integrations.list(limit=10) + + assert isinstance(res, IntegrationList) + assert len(res) == 1 + assert res[0].external_id == "my-integration" + assert res[0].tasks[0].name == "poll" + + request = httpx_mock.get_requests()[0] + assert request.method == "GET" + assert request.headers["cdf-version"] == async_client.integrations._alpha_version_header()["cdf-version"] + + def test_create( + self, cognite_client: CogniteClient, async_client: AsyncCogniteClient, httpx_mock: HTTPXMock + ) -> None: + httpx_mock.add_response( + method="POST", + url=get_url(async_client.integrations, "/integrations"), + json={"items": [INTEGRATION_RESPONSE]}, + ) + integration = IntegrationWrite( + external_id="my-integration", + extractor=Extractor(external_id="cognite-simple-influxdb-extractor", version="1.0.0"), + name="My integration", + ) + + res = cognite_client.integrations.create(integration) + + assert isinstance(res, Integration) + assert res.external_id == "my-integration" + + body = jsgz_load(httpx_mock.get_requests()[0].content) + assert body == { + "items": [ + { + "externalId": "my-integration", + "name": "My integration", + "extractor": {"externalId": "cognite-simple-influxdb-extractor", "version": "1.0.0"}, + } + ] + } + + def test_retrieve( + self, cognite_client: CogniteClient, async_client: AsyncCogniteClient, httpx_mock: HTTPXMock + ) -> None: + httpx_mock.add_response( + method="POST", + url=get_url(async_client.integrations, "/integrations/byids"), + json={"items": [INTEGRATION_RESPONSE]}, + ) + + res = cognite_client.integrations.retrieve("my-integration") + + assert isinstance(res, Integration) + assert res.external_id == "my-integration" + + body = jsgz_load(httpx_mock.get_requests()[0].content) + assert body == {"items": [{"externalId": "my-integration"}], "ignoreUnknownIds": False} + + def test_update( + self, cognite_client: CogniteClient, async_client: AsyncCogniteClient, httpx_mock: HTTPXMock + ) -> None: + httpx_mock.add_response( + method="POST", + url=get_url(async_client.integrations, "/integrations/update"), + json={"items": [INTEGRATION_RESPONSE]}, + ) + update = IntegrationUpdate(external_id="my-integration") + update.description.set("My new description") + + res = cognite_client.integrations.update(update) + + assert isinstance(res, Integration) + + body = jsgz_load(httpx_mock.get_requests()[0].content) + assert body == { + "items": [{"externalId": "my-integration", "update": {"description": {"set": "My new description"}}}] + } + + def test_delete( + self, cognite_client: CogniteClient, async_client: AsyncCogniteClient, httpx_mock: HTTPXMock + ) -> None: + httpx_mock.add_response(method="POST", url=get_url(async_client.integrations, "/integrations/delete"), json={}) + + cognite_client.integrations.delete("my-integration") + + body = jsgz_load(httpx_mock.get_requests()[0].content) + assert body == {"items": [{"externalId": "my-integration"}], "ignoreUnknownIds": False} diff --git a/tests/tests_unit/test_api/test_integrations/test_tasks.py b/tests/tests_unit/test_api/test_integrations/test_tasks.py new file mode 100644 index 0000000000..af4136d2df --- /dev/null +++ b/tests/tests_unit/test_api/test_integrations/test_tasks.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import re + +from pytest_httpx import HTTPXMock + +from cognite.client import AsyncCogniteClient, CogniteClient +from cognite.client.data_classes.integrations.tasks import SyncResult, TaskHistoryList +from tests.utils import get_url + +TASK_HISTORY_RESPONSE = { + "externalId": "my-integration", + "taskName": "poll", + "startTime": 100, + "endTime": 200, + "errorCount": 0, +} + + +class TestIntegrationTasks: + def test_list_history( + self, cognite_client: CogniteClient, async_client: AsyncCogniteClient, httpx_mock: HTTPXMock + ) -> None: + url_pattern = re.compile( + re.escape(get_url(async_client.integrations.tasks, "/integrations/history")) + r"(?:\?.*)?$" + ) + httpx_mock.add_response(method="GET", url=url_pattern, json={"items": [TASK_HISTORY_RESPONSE]}) + + res = cognite_client.integrations.tasks.list_history(external_id="my-integration", task_name="poll") + + assert isinstance(res, TaskHistoryList) + assert len(res) == 1 + assert res[0].task_name == "poll" + + request = httpx_mock.get_requests()[0] + assert "externalId=my-integration" in str(request.url) + assert "taskName=poll" in str(request.url) + + def test_sync(self, cognite_client: CogniteClient, async_client: AsyncCogniteClient, httpx_mock: HTTPXMock) -> None: + url_pattern = re.compile( + re.escape(get_url(async_client.integrations.tasks, "/integrations/sync")) + r"(?:\?.*)?$" + ) + httpx_mock.add_response( + method="GET", + url=url_pattern, + json={"nextCursor": "abc", "moreData": False, "history": [TASK_HISTORY_RESPONSE]}, + ) + + res = cognite_client.integrations.tasks.sync(external_id="my-integration", include_task_updates=True) + + assert isinstance(res, SyncResult) + assert res.next_cursor == "abc" + assert res.more_data is False + assert res.history is not None + assert res.history[0].task_name == "poll" + assert res.errors is None diff --git a/tests/tests_unit/test_api_client.py b/tests/tests_unit/test_api_client.py index d6937df5d0..84faaa9b64 100644 --- a/tests/tests_unit/test_api_client.py +++ b/tests/tests_unit/test_api_client.py @@ -1781,6 +1781,15 @@ async def test_is_retryable_resource_api_endpoints(self, method: str, path: str, ("POST", "https://api.cognitedata.com/api/v1/projects/bla/extpipes/runs/list", True), ("POST", "https://api.cognitedata.com/api/v1/projects/bla/extpipes/config", False), ("POST", "https://api.cognitedata.com/api/v1/projects/bla/extpipes/config/revert", False), + # Integrations + ("POST", "https://api.cognitedata.com/api/v1/projects/bla/integrations", False), + ("POST", "https://api.cognitedata.com/api/v1/projects/bla/integrations/byids", True), + ("POST", "https://api.cognitedata.com/api/v1/projects/bla/integrations/delete", False), + ("POST", "https://api.cognitedata.com/api/v1/projects/bla/integrations/update", True), + ("POST", "https://api.cognitedata.com/api/v1/projects/bla/integrations/config", False), + ("POST", "https://api.cognitedata.com/api/v1/projects/bla/integrations/actions", False), + ("POST", "https://api.cognitedata.com/api/v1/projects/bla/integrations/actions/byids", True), + ("POST", "https://api.cognitedata.com/api/v1/projects/bla/integrations/actions/cancel", False), # Transformations ("POST", "https://api.cognitedata.com/api/v1/projects/bla/transformations", False), ("POST", "https://api.cognitedata.com/api/v1/projects/bla/transformations/filter", True), diff --git a/tests/tests_unit/test_data_classes/test_integrations.py b/tests/tests_unit/test_data_classes/test_integrations.py new file mode 100644 index 0000000000..8ec4d6bf12 --- /dev/null +++ b/tests/tests_unit/test_data_classes/test_integrations.py @@ -0,0 +1,191 @@ +from __future__ import annotations + +from cognite.client.data_classes.integrations import ( + Action, + ConfigRevision, + Extractor, + Integration, + IntegrationError, + IntegrationUpdate, + Task, +) +from cognite.client.data_classes.integrations.tasks import SyncResult, TaskHistory + +INTEGRATION_DUMPED = { + "externalId": "my-integration", + "extractor": {"externalId": "cognite-simple-influxdb-extractor", "version": "1.0.0"}, + "name": "My integration", + "description": "A test integration", + "metadata": {"key": "value"}, + "allowedNotSeenMinutes": 60, + "lastSeen": 123, + "lastConfigRevision": 2, + "activeConfigRevision": "local", + "tasks": [{"type": "continuous", "name": "poll", "action": True, "description": "Polls for data"}], + "createdTime": 1, + "lastUpdatedTime": 2, +} + + +class TestIntegration: + def test_load_dump_round_trip(self) -> None: + loaded = Integration._load(INTEGRATION_DUMPED) + + assert loaded.external_id == "my-integration" + assert loaded.extractor.external_id == "cognite-simple-influxdb-extractor" + assert loaded.tasks[0].name == "poll" + assert loaded.tasks[0].action is True + assert loaded.active_config_revision == "local" + + assert loaded.dump(camel_case=True) == INTEGRATION_DUMPED + + def test_as_write(self) -> None: + loaded = Integration._load(INTEGRATION_DUMPED) + write = loaded.as_write() + + assert write.external_id == loaded.external_id + assert write.extractor.external_id == loaded.extractor.external_id + assert write.dump(camel_case=True) == { + "externalId": "my-integration", + "extractor": {"externalId": "cognite-simple-influxdb-extractor", "version": "1.0.0"}, + "name": "My integration", + "description": "A test integration", + "metadata": {"key": "value"}, + "allowedNotSeenMinutes": 60, + } + + +class TestIntegrationUpdate: + def test_set_and_set_null(self) -> None: + update = IntegrationUpdate(external_id="my-integration") + update.name.set("New name") + update.description.set(None) + + assert update.dump() == { + "externalId": "my-integration", + "update": {"name": {"set": "New name"}, "description": {"setNull": True}}, + } + + def test_metadata_add_remove(self) -> None: + update = IntegrationUpdate(external_id="my-integration") + update.metadata.add({"key": "value"}) + + assert update.dump() == { + "externalId": "my-integration", + "update": {"metadata": {"add": {"key": "value"}}}, + } + + def test_metadata_set(self) -> None: + update = IntegrationUpdate(external_id="my-integration") + update.metadata.set({"key": "value"}) + + assert update.dump() == { + "externalId": "my-integration", + "update": {"metadata": {"set": {"key": "value"}}}, + } + + +class TestAction: + def test_load_dump_round_trip(self) -> None: + dumped = { + "externalId": "my-action", + "actionName": "restart", + "status": "succeeded", + "callMetadata": {"reason": "manual"}, + "resultMessage": "Done", + "resultMetadata": {"durationMs": "42"}, + "createdTime": 1, + "lastUpdatedTime": 2, + } + loaded = Action._load(dumped) + + assert loaded.status == "succeeded" + assert loaded.dump(camel_case=True) == dumped + + def test_as_write(self) -> None: + loaded = Action._load( + { + "externalId": "my-action", + "actionName": "restart", + "status": "pending", + "createdTime": 1, + "lastUpdatedTime": 2, + } + ) + write = loaded.as_write() + + assert write.dump(camel_case=True) == {"externalId": "my-action", "actionName": "restart"} + + +class TestSyncResult: + def test_load_dump_round_trip(self) -> None: + dumped = { + "nextCursor": "abc123", + "moreData": True, + "history": [ + { + "externalId": "my-integration", + "taskName": "poll", + "startTime": 100, + "errorCount": 0, + "warningCount": 0, + "fatalCount": 0, + } + ], + "errors": [ + { + "externalId": "my-integration", + "level": "warning", + "description": "Slow response", + "startTime": 100, + } + ], + } + loaded = SyncResult._load(dumped) + + assert loaded.next_cursor == "abc123" + assert loaded.more_data is True + assert isinstance(loaded.history[0], TaskHistory) + assert isinstance(loaded.errors[0], IntegrationError) + + assert loaded.dump(camel_case=True) == dumped + + +class TestConfigRevision: + def test_load_dump_round_trip(self) -> None: + dumped = { + "externalId": "my-integration", + "revision": 3, + "description": "A config revision", + "config": "key: value", + "createdTime": 1, + "lastUpdatedTime": 2, + } + loaded = ConfigRevision._load(dumped) + + assert loaded.revision == 3 + assert loaded.dump(camel_case=True) == dumped + + write = loaded.as_write() + assert write.dump(camel_case=True) == { + "externalId": "my-integration", + "config": "key: value", + "description": "A config revision", + } + + +def test_extractor_load_dump() -> None: + dumped = {"externalId": "cognite-simple-influxdb-extractor", "version": "1.0.0"} + assert Extractor._load(dumped).dump(camel_case=True) == dumped + + +def test_task_load_dump() -> None: + dumped = { + "type": "batch", + "name": "sync", + "action": True, + "description": "Syncs data", + "sources": ["cdf://cluster/project/service/resource"], + "targets": ["cdf://cluster/project/timeseries/my-ts"], + } + assert Task._load(dumped).dump(camel_case=True) == dumped From 1ee2f02466b1932caf2aed2136cccf0490d3f5b1 Mon Sep 17 00:00:00 2001 From: vikramlc Date: Thu, 3 Sep 2026 17:24:09 +0530 Subject: [PATCH 2/5] refactor(odin): Fix lint issue --- cognite/client/data_classes/integrations/actions.py | 2 +- cognite/client/data_classes/integrations/errors.py | 2 +- cognite/client/data_classes/integrations/integrations.py | 4 ++-- cognite/client/data_classes/integrations/tasks.py | 2 +- tests/tests_unit/test_data_classes/test_integrations.py | 2 ++ 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cognite/client/data_classes/integrations/actions.py b/cognite/client/data_classes/integrations/actions.py index b9d3bad508..ae7ff01ae3 100644 --- a/cognite/client/data_classes/integrations/actions.py +++ b/cognite/client/data_classes/integrations/actions.py @@ -20,7 +20,7 @@ class ActionCore(WriteableCogniteResource["ActionWrite"], ABC): """An action is a request for an integration to do something outside its normal task loop, e.g. restart, reload config, or start/stop a task. - The extractor polls for pending actions (through checkin) and reports the outcome back; no inbound + The extractor polls for pending actions (through check-in) and reports the outcome back; no inbound connectivity is required on the extractor side. Args: diff --git a/cognite/client/data_classes/integrations/errors.py b/cognite/client/data_classes/integrations/errors.py index 8132941540..de3a050dd8 100644 --- a/cognite/client/data_classes/integrations/errors.py +++ b/cognite/client/data_classes/integrations/errors.py @@ -23,7 +23,7 @@ class IntegrationError(CogniteResource): end_time (int | None): Time the error was resolved, in milliseconds since epoch. Not set while unresolved. task (str | None): Name of the task the error occurred in. Not set if the error applies to the extractor generally. type (IntegrationErrorType | None): Category of the error. - active_config_revision (int | Literal["local"] | None): The config revision (or "local") active when the error occurred. + active_config_revision (ActiveConfigRevision | None): The config revision (or "local") active when the error occurred. """ def __init__( diff --git a/cognite/client/data_classes/integrations/integrations.py b/cognite/client/data_classes/integrations/integrations.py index cc6268eacd..b3e11a4987 100644 --- a/cognite/client/data_classes/integrations/integrations.py +++ b/cognite/client/data_classes/integrations/integrations.py @@ -41,7 +41,7 @@ class Task(CogniteResource): """A named unit of work in an integration, reported by the extractor. Args: - type (Literal["continuous", "batch"]): Whether the task runs for the lifetime of the extractor (continuous) or runs to completion and exits (batch). + type (Literal['continuous', 'batch']): Whether the task runs for the lifetime of the extractor (continuous) or runs to completion and exits (batch). name (str): Name of the task, unique within the integration. action (bool): Whether this task can be triggered through an Action. Defaults to False. description (str | None): Description of the task. @@ -163,7 +163,7 @@ class Integration(IntegrationCore): allowed_not_seen_minutes (int | None): Number of minutes the integration is allowed to not report in before it's flagged as inactive. last_seen (int | None): The time this integration was last seen (checked in), in milliseconds since epoch. last_config_revision (int | None): The revision number of the last config revision created for this integration. - active_config_revision (int | Literal["local"] | None): The config revision currently reported active by the extractor, or "local" if it's using a local config file instead of a revision managed through CDF. + active_config_revision (ActiveConfigRevision | None): The config revision currently reported active by the extractor, or "local" if it's using a local config file instead of a revision managed through CDF. tasks (list[Task] | None): The tasks the extractor has reported as part of this integration. """ diff --git a/cognite/client/data_classes/integrations/tasks.py b/cognite/client/data_classes/integrations/tasks.py index a850f5d3f7..e4b9d32210 100644 --- a/cognite/client/data_classes/integrations/tasks.py +++ b/cognite/client/data_classes/integrations/tasks.py @@ -21,7 +21,7 @@ class TaskHistory(CogniteResource): error_count (int): Number of errors reported for this task run. warning_count (int): Number of warnings reported for this task run. fatal_count (int): Number of fatal errors reported for this task run. - active_config_revision (int | Literal["local"] | None): The config revision (or "local") active at the time of this task run. + active_config_revision (ActiveConfigRevision | None): The config revision (or "local") active at the time of this task run. sources (list[str] | None): Lineage: URIs of the systems/resources this task read from. targets (list[str] | None): Lineage: URIs of the CDF (or other) resources this task wrote to. """ diff --git a/tests/tests_unit/test_data_classes/test_integrations.py b/tests/tests_unit/test_data_classes/test_integrations.py index 8ec4d6bf12..a7e431bc0d 100644 --- a/tests/tests_unit/test_data_classes/test_integrations.py +++ b/tests/tests_unit/test_data_classes/test_integrations.py @@ -145,6 +145,8 @@ def test_load_dump_round_trip(self) -> None: assert loaded.next_cursor == "abc123" assert loaded.more_data is True + assert loaded.history is not None + assert loaded.errors is not None assert isinstance(loaded.history[0], TaskHistory) assert isinstance(loaded.errors[0], IntegrationError) From c27b8321425ebab68b3fcc300351dee025477354 Mon Sep 17 00:00:00 2001 From: vikramlc Date: Thu, 3 Sep 2026 19:46:50 +0530 Subject: [PATCH 3/5] fix(odin): Review comment fixes --- cognite/client/_api/integrations/errors.py | 6 ++++++ cognite/client/_api/integrations/tasks.py | 12 ++++++++++++ cognite/client/_sync_api/integrations/errors.py | 5 ++++- cognite/client/_sync_api/integrations/tasks.py | 8 +++++++- .../data_classes/integrations/integrations.py | 4 ++-- cognite/client/data_classes/integrations/tasks.py | 4 ++-- .../test_api/test_integrations/test_errors.py | 5 +++++ .../test_api/test_integrations/test_tasks.py | 9 +++++++++ .../test_data_classes/test_integrations.py | 13 +++++++++++++ 9 files changed, 60 insertions(+), 6 deletions(-) diff --git a/cognite/client/_api/integrations/errors.py b/cognite/client/_api/integrations/errors.py index 78d22cd482..8be1acccd4 100644 --- a/cognite/client/_api/integrations/errors.py +++ b/cognite/client/_api/integrations/errors.py @@ -37,6 +37,9 @@ async def list( max_end_time (int | None): Only return errors that ended at or before this time, in milliseconds since epoch. limit (int | None): Maximum number of errors to return. Defaults to 25. Set to -1, float("inf") or None to return all items. + Raises: + ValueError: If `task` is given without `external_id`. + Returns: IntegrationErrorList: List of errors @@ -49,6 +52,9 @@ async def list( >>> # async_client = AsyncCogniteClient() # another option >>> res = client.integrations.errors.list(external_id="my-integration") """ + if task is not None and external_id is None: + raise ValueError("Specifying 'task' requires 'external_id' to also be set.") + self._warning.warn() return await self._list( method="GET", diff --git a/cognite/client/_api/integrations/tasks.py b/cognite/client/_api/integrations/tasks.py index b28efccf1f..42c4202b00 100644 --- a/cognite/client/_api/integrations/tasks.py +++ b/cognite/client/_api/integrations/tasks.py @@ -35,6 +35,9 @@ async def list_history( last_per_task (bool): Only return the latest history entry per task. limit (int | None): Maximum number of history entries to return. Defaults to 25. Set to -1, float("inf") or None to return all items. + Raises: + ValueError: If `task_name` is given without `external_id`. + Returns: TaskHistoryList: List of task history entries @@ -47,6 +50,9 @@ async def list_history( >>> # async_client = AsyncCogniteClient() # another option >>> res = client.integrations.tasks.list_history(external_id="my-integration") """ + if task_name is not None and external_id is None: + raise ValueError("Specifying 'task_name' requires 'external_id' to also be set.") + self._warning.warn() return await self._list( method="GET", @@ -89,6 +95,9 @@ async def sync( cursor (str | None): Cursor returned from a previous call to this method, to continue syncing from where you left off. limit (int | None): Maximum number of items to return in this page. Defaults to 25. + Raises: + ValueError: If both `include_errors` and `include_task_updates` are False. + Returns: SyncResult: A single page of results. Inspect `more_data` to see whether you should immediately call this method again with the returned `next_cursor`, or back off before doing so. @@ -110,6 +119,9 @@ async def sync( ... cursor=res.next_cursor, ... ) """ + if not include_errors and not include_task_updates: + raise ValueError("At least one of 'include_errors' or 'include_task_updates' must be True.") + self._warning.warn() response = await self._get( url_path=f"{self._RESOURCE_PATH}/sync", diff --git a/cognite/client/_sync_api/integrations/errors.py b/cognite/client/_sync_api/integrations/errors.py index cce4c66d70..82136119ca 100644 --- a/cognite/client/_sync_api/integrations/errors.py +++ b/cognite/client/_sync_api/integrations/errors.py @@ -1,6 +1,6 @@ """ =============================================================================== -d6864ed3490d3a8820c75aae29a7b5b3 +bde27dc1b43cfa3eeb5745a5669fe8cc This file is auto-generated from the Async API modules, - do not edit manually! =============================================================================== """ @@ -43,6 +43,9 @@ def list( max_end_time (int | None): Only return errors that ended at or before this time, in milliseconds since epoch. limit (int | None): Maximum number of errors to return. Defaults to 25. Set to -1, float("inf") or None to return all items. + Raises: + ValueError: If `task` is given without `external_id`. + Returns: IntegrationErrorList: List of errors diff --git a/cognite/client/_sync_api/integrations/tasks.py b/cognite/client/_sync_api/integrations/tasks.py index 50b032d634..743250a44c 100644 --- a/cognite/client/_sync_api/integrations/tasks.py +++ b/cognite/client/_sync_api/integrations/tasks.py @@ -1,6 +1,6 @@ """ =============================================================================== -231d30469457b599925a4f28e9de9a93 +c13367430cb1ccec84ff53d416749174 This file is auto-generated from the Async API modules, - do not edit manually! =============================================================================== """ @@ -41,6 +41,9 @@ def list_history( last_per_task (bool): Only return the latest history entry per task. limit (int | None): Maximum number of history entries to return. Defaults to 25. Set to -1, float("inf") or None to return all items. + Raises: + ValueError: If `task_name` is given without `external_id`. + Returns: TaskHistoryList: List of task history entries @@ -85,6 +88,9 @@ def sync( cursor (str | None): Cursor returned from a previous call to this method, to continue syncing from where you left off. limit (int | None): Maximum number of items to return in this page. Defaults to 25. + Raises: + ValueError: If both `include_errors` and `include_task_updates` are False. + Returns: SyncResult: A single page of results. Inspect `more_data` to see whether you should immediately call this method again with the returned `next_cursor`, or back off before doing so. diff --git a/cognite/client/data_classes/integrations/integrations.py b/cognite/client/data_classes/integrations/integrations.py index b3e11a4987..1ca1d05f7a 100644 --- a/cognite/client/data_classes/integrations/integrations.py +++ b/cognite/client/data_classes/integrations/integrations.py @@ -201,7 +201,7 @@ def __init__( def dump(self, camel_case: bool = True) -> dict[str, Any]: result = super().dump(camel_case) - result["tasks"] = [task.dump(camel_case) for task in self.tasks] + result["tasks"] = [task.dump(camel_case) for task in self.tasks or []] return result @classmethod @@ -219,7 +219,7 @@ def _load(cls, resource: dict[str, Any]) -> Self: last_seen=resource.get("lastSeen"), last_config_revision=resource.get("lastConfigRevision"), active_config_revision=resource.get("activeConfigRevision"), - tasks=[Task._load(task) for task in resource.get("tasks", [])], + tasks=[Task._load(task) for task in resource.get("tasks") or []], ) def as_write(self) -> IntegrationWrite: diff --git a/cognite/client/data_classes/integrations/tasks.py b/cognite/client/data_classes/integrations/tasks.py index e4b9d32210..92ff34a667 100644 --- a/cognite/client/data_classes/integrations/tasks.py +++ b/cognite/client/data_classes/integrations/tasks.py @@ -108,6 +108,6 @@ def _load(cls, resource: dict[str, Any]) -> Self: return cls( next_cursor=resource["nextCursor"], more_data=resource.get("moreData", False), - history=TaskHistoryList._load(resource["history"]) if "history" in resource else None, - errors=IntegrationErrorList._load(resource["errors"]) if "errors" in resource else None, + history=TaskHistoryList._load(resource["history"]) if resource.get("history") is not None else None, + errors=IntegrationErrorList._load(resource["errors"]) if resource.get("errors") is not None else None, ) diff --git a/tests/tests_unit/test_api/test_integrations/test_errors.py b/tests/tests_unit/test_api/test_integrations/test_errors.py index 83cc49e5a8..cbc79b611e 100644 --- a/tests/tests_unit/test_api/test_integrations/test_errors.py +++ b/tests/tests_unit/test_api/test_integrations/test_errors.py @@ -2,6 +2,7 @@ import re +import pytest from pytest_httpx import HTTPXMock from cognite.client import AsyncCogniteClient, CogniteClient @@ -34,3 +35,7 @@ def test_list(self, cognite_client: CogniteClient, async_client: AsyncCogniteCli request = httpx_mock.get_requests()[0] assert "externalId=my-integration" in str(request.url) assert "task=poll" in str(request.url) + + def test_list_task_without_external_id_raises(self, cognite_client: CogniteClient) -> None: + with pytest.raises(ValueError, match="'task' requires 'external_id'"): + cognite_client.integrations.errors.list(task="poll") diff --git a/tests/tests_unit/test_api/test_integrations/test_tasks.py b/tests/tests_unit/test_api/test_integrations/test_tasks.py index af4136d2df..3b3faf235d 100644 --- a/tests/tests_unit/test_api/test_integrations/test_tasks.py +++ b/tests/tests_unit/test_api/test_integrations/test_tasks.py @@ -2,6 +2,7 @@ import re +import pytest from pytest_httpx import HTTPXMock from cognite.client import AsyncCogniteClient, CogniteClient @@ -36,6 +37,10 @@ def test_list_history( assert "externalId=my-integration" in str(request.url) assert "taskName=poll" in str(request.url) + def test_list_history_task_name_without_external_id_raises(self, cognite_client: CogniteClient) -> None: + with pytest.raises(ValueError, match="'task_name' requires 'external_id'"): + cognite_client.integrations.tasks.list_history(task_name="poll") + def test_sync(self, cognite_client: CogniteClient, async_client: AsyncCogniteClient, httpx_mock: HTTPXMock) -> None: url_pattern = re.compile( re.escape(get_url(async_client.integrations.tasks, "/integrations/sync")) + r"(?:\?.*)?$" @@ -54,3 +59,7 @@ def test_sync(self, cognite_client: CogniteClient, async_client: AsyncCogniteCli assert res.history is not None assert res.history[0].task_name == "poll" assert res.errors is None + + def test_sync_requires_include_errors_or_include_task_updates(self, cognite_client: CogniteClient) -> None: + with pytest.raises(ValueError, match="'include_errors' or 'include_task_updates'"): + cognite_client.integrations.tasks.sync(external_id="my-integration") diff --git a/tests/tests_unit/test_data_classes/test_integrations.py b/tests/tests_unit/test_data_classes/test_integrations.py index a7e431bc0d..9c1a4af29f 100644 --- a/tests/tests_unit/test_data_classes/test_integrations.py +++ b/tests/tests_unit/test_data_classes/test_integrations.py @@ -39,6 +39,13 @@ def test_load_dump_round_trip(self) -> None: assert loaded.dump(camel_case=True) == INTEGRATION_DUMPED + def test_load_with_explicit_null_tasks(self) -> None: + dumped = {**INTEGRATION_DUMPED, "tasks": None} + loaded = Integration._load(dumped) + + assert loaded.tasks == [] + assert loaded.dump(camel_case=True)["tasks"] == [] + def test_as_write(self) -> None: loaded = Integration._load(INTEGRATION_DUMPED) write = loaded.as_write() @@ -152,6 +159,12 @@ def test_load_dump_round_trip(self) -> None: assert loaded.dump(camel_case=True) == dumped + def test_load_with_explicit_null_history_and_errors(self) -> None: + loaded = SyncResult._load({"nextCursor": "abc123", "moreData": False, "history": None, "errors": None}) + + assert loaded.history is None + assert loaded.errors is None + class TestConfigRevision: def test_load_dump_round_trip(self) -> None: From 4439f51c362a25a621873caacab39dc809a865a3 Mon Sep 17 00:00:00 2001 From: vikramlc Date: Mon, 7 Sep 2026 13:35:37 +0530 Subject: [PATCH 4/5] feat(odin): Add startup and checkin support in the sdk --- cognite/client/_api/integrations/__init__.py | 87 +++++++- cognite/client/_api/integrations/actions.py | 10 +- cognite/client/_api/integrations/config.py | 8 +- cognite/client/_api/integrations/errors.py | 4 +- cognite/client/_api/integrations/tasks.py | 6 +- cognite/client/_basic_api_client.py | 10 + .../client/_sync_api/integrations/__init__.py | 63 +++++- .../client/_sync_api/integrations/actions.py | 2 +- .../client/_sync_api/integrations/config.py | 2 +- .../client/_sync_api/integrations/errors.py | 2 +- .../client/_sync_api/integrations/tasks.py | 2 +- .../data_classes/integrations/__init__.py | 12 ++ .../data_classes/integrations/checkin.py | 194 ++++++++++++++++++ cognite/client/utils/_url.py | 2 + pyproject.toml | 6 + .../test_integrations/test_integrations.py | 57 ++++- tests/tests_unit/test_api_client.py | 2 + .../test_data_classes/test_integrations.py | 57 +++++ 18 files changed, 499 insertions(+), 27 deletions(-) create mode 100644 cognite/client/data_classes/integrations/checkin.py diff --git a/cognite/client/_api/integrations/__init__.py b/cognite/client/_api/integrations/__init__.py index fe775981e5..709bd22364 100644 --- a/cognite/client/_api/integrations/__init__.py +++ b/cognite/client/_api/integrations/__init__.py @@ -9,6 +9,7 @@ from cognite.client._api.integrations.tasks import IntegrationTasksAPI from cognite.client._api_client import APIClient from cognite.client._constants import DEFAULT_LIMIT_READ +from cognite.client.data_classes.integrations.checkin import CheckinRequest, CheckinResponse, StartupRequest from cognite.client.data_classes.integrations.integrations import ( Integration, IntegrationList, @@ -33,7 +34,7 @@ def __init__(self, config: ClientConfig, api_version: str | None, cognite_client self.errors = IntegrationErrorsAPI(config, api_version, cognite_client) self.config = IntegrationConfigAPI(config, api_version, cognite_client) self.actions = IntegrationActionsAPI(config, api_version, cognite_client) - self._warning = FeaturePreviewWarning(api_maturity="alpha", sdk_maturity="alpha", feature_name="Integrations") + self._warning = FeaturePreviewWarning(api_maturity="beta", sdk_maturity="alpha", feature_name="Integrations") @overload def __call__(self, chunk_size: None = None, limit: int | None = None) -> AsyncIterator[Integration]: ... @@ -62,7 +63,7 @@ async def __call__( resource_cls=Integration, chunk_size=chunk_size, limit=limit, - headers=self._alpha_version_header(), + headers=self._beta_version_header(), ): yield item @@ -95,7 +96,7 @@ async def list(self, limit: int | None = DEFAULT_LIMIT_READ) -> IntegrationList: list_cls=IntegrationList, resource_cls=Integration, limit=limit, - headers=self._alpha_version_header(), + headers=self._beta_version_header(), ) @overload @@ -132,7 +133,7 @@ async def create(self, integration: IntegrationWrite | Sequence[IntegrationWrite resource_cls=Integration, items=integration, input_resource_cls=IntegrationWrite, - headers=self._alpha_version_header(), + headers=self._beta_version_header(), ) @overload @@ -169,7 +170,7 @@ async def retrieve( resource_cls=Integration, identifiers=identifiers, ignore_unknown_ids=ignore_unknown_ids, - headers=self._alpha_version_header(), + headers=self._beta_version_header(), ) @overload @@ -210,7 +211,7 @@ async def update( resource_cls=Integration, update_cls=IntegrationUpdate, items=item, - headers=self._alpha_version_header(), + headers=self._beta_version_header(), ) async def delete(self, external_id: str | SequenceNotStr[str], ignore_unknown_ids: bool = False) -> None: @@ -234,5 +235,77 @@ async def delete(self, external_id: str | SequenceNotStr[str], ignore_unknown_id identifiers=IdentifierSequence.load(external_ids=external_id), wrap_ids=True, extra_body_fields={"ignoreUnknownIds": ignore_unknown_ids}, - headers=self._alpha_version_header(), + headers=self._beta_version_header(), ) + + async def startup(self, request: StartupRequest) -> CheckinResponse: + """`Report extractor startup `_ + + Reports that the extractor has (re)started, along with its current task configuration. + This closes any currently running tasks with an error. + + Note: + This is normally only called by extractor implementations as part of the + integrations startup protocol, not by typical SDK consumers. + + Args: + request (StartupRequest): The startup event to report. + + Returns: + CheckinResponse: The integration's latest config revision. + + Examples: + + Report extractor startup: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes.integrations import Extractor, StartupRequest + >>> client = CogniteClient() + >>> req = StartupRequest( + ... external_id="my-integration", + ... extractor=Extractor(external_id="cognite-simple-influxdb-extractor"), + ... ) + >>> res = client.integrations.startup(req) + """ + self._warning.warn() + response = await self._post( + f"{self._RESOURCE_PATH}/startup", + json=request.dump(camel_case=True), + headers=self._beta_version_header(), + semaphore=self._get_semaphore("write"), + ) + return CheckinResponse._load(response.json()) + + async def checkin(self, request: CheckinRequest) -> CheckinResponse: + """`Check in with the integrations service `_ + + Called periodically by extractors to signal that they are still alive, and to report task + start/stop events and errors that have occurred since the last check-in. + + Note: + This is normally only called by extractor implementations as part of the + integrations check-in protocol, not by typical SDK consumers. + + Args: + request (CheckinRequest): The check-in event to report. + + Returns: + CheckinResponse: The integration's latest config revision. + + Examples: + + Check in with no updates: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes.integrations import CheckinRequest + >>> client = CogniteClient() + >>> res = client.integrations.checkin(CheckinRequest(external_id="my-integration")) + """ + self._warning.warn() + response = await self._post( + f"{self._RESOURCE_PATH}/checkin", + json=request.dump(camel_case=True), + headers=self._beta_version_header(), + semaphore=self._get_semaphore("write"), + ) + return CheckinResponse._load(response.json()) diff --git a/cognite/client/_api/integrations/actions.py b/cognite/client/_api/integrations/actions.py index 001bd73219..1b06bc9f5b 100644 --- a/cognite/client/_api/integrations/actions.py +++ b/cognite/client/_api/integrations/actions.py @@ -24,7 +24,7 @@ class IntegrationActionsAPI(APIClient): def __init__(self, config: ClientConfig, api_version: str | None, cognite_client: AsyncCogniteClient) -> None: super().__init__(config, api_version, cognite_client) - self._warning = FeaturePreviewWarning(api_maturity="alpha", sdk_maturity="alpha", feature_name="Integrations") + self._warning = FeaturePreviewWarning(api_maturity="beta", sdk_maturity="alpha", feature_name="Integrations") @overload async def create(self, external_id: str, action: ActionWrite) -> Action: ... @@ -62,7 +62,7 @@ async def create(self, external_id: str, action: ActionWrite | Sequence[ActionWr self._RESOURCE_PATH, params={"externalId": external_id}, json={"items": [item.dump(camel_case=True) for item in chunk]}, - headers=self._alpha_version_header(), + headers=self._beta_version_header(), semaphore=self._get_semaphore("write"), ) created.extend(response.json()["items"]) @@ -113,7 +113,7 @@ async def list( "includeCompleted": include_completed, } ), - headers=self._alpha_version_header(), + headers=self._beta_version_header(), ) @overload @@ -150,7 +150,7 @@ async def retrieve( resource_cls=Action, identifiers=identifiers, ignore_unknown_ids=ignore_unknown_ids, - headers=self._alpha_version_header(), + headers=self._beta_version_header(), ) async def cancel(self, external_id: str | SequenceNotStr[str], ignore_unknown_ids: bool = False) -> ActionList: @@ -185,7 +185,7 @@ async def cancel(self, external_id: str | SequenceNotStr[str], ignore_unknown_id response = await self._post( f"{self._RESOURCE_PATH}/cancel", json=body, - headers=self._alpha_version_header(), + headers=self._beta_version_header(), semaphore=self._get_semaphore("write"), ) cancelled.extend(response.json()["items"]) diff --git a/cognite/client/_api/integrations/config.py b/cognite/client/_api/integrations/config.py index e78f61dc55..2d8813c310 100644 --- a/cognite/client/_api/integrations/config.py +++ b/cognite/client/_api/integrations/config.py @@ -22,7 +22,7 @@ class IntegrationConfigAPI(APIClient): def __init__(self, config: ClientConfig, api_version: str | None, cognite_client: AsyncCogniteClient) -> None: super().__init__(config, api_version, cognite_client) - self._warning = FeaturePreviewWarning(api_maturity="alpha", sdk_maturity="alpha", feature_name="Integrations") + self._warning = FeaturePreviewWarning(api_maturity="beta", sdk_maturity="alpha", feature_name="Integrations") async def create(self, config: ConfigRevision | ConfigRevisionWrite) -> ConfigRevision: """`Create a new configuration revision `_ @@ -50,7 +50,7 @@ async def create(self, config: ConfigRevision | ConfigRevisionWrite) -> ConfigRe response = await self._post( self._RESOURCE_PATH, json=config.dump(camel_case=True), - headers=self._alpha_version_header(), + headers=self._beta_version_header(), semaphore=self._get_semaphore("write"), ) return ConfigRevision._load(response.json()) @@ -78,7 +78,7 @@ async def retrieve(self, external_id: str, revision: int | None = None) -> Confi response = await self._get( self._RESOURCE_PATH, params=drop_none_values({"externalId": external_id, "revision": revision}), - headers=self._alpha_version_header(), + headers=self._beta_version_header(), semaphore=self._get_semaphore("read"), ) return ConfigRevision._load(response.json()) @@ -115,7 +115,7 @@ async def list( response = await self._get( f"{self._RESOURCE_PATH}/revisions", params=drop_none_values({"externalId": external_id, "limit": limit}), - headers=self._alpha_version_header(), + headers=self._beta_version_header(), semaphore=self._get_semaphore("read"), ) return ConfigRevisionMetadataList._load(response.json()["items"]) diff --git a/cognite/client/_api/integrations/errors.py b/cognite/client/_api/integrations/errors.py index 8be1acccd4..3fdbc22154 100644 --- a/cognite/client/_api/integrations/errors.py +++ b/cognite/client/_api/integrations/errors.py @@ -18,7 +18,7 @@ class IntegrationErrorsAPI(APIClient): def __init__(self, config: ClientConfig, api_version: str | None, cognite_client: AsyncCogniteClient) -> None: super().__init__(config, api_version, cognite_client) - self._warning = FeaturePreviewWarning(api_maturity="alpha", sdk_maturity="alpha", feature_name="Integrations") + self._warning = FeaturePreviewWarning(api_maturity="beta", sdk_maturity="alpha", feature_name="Integrations") async def list( self, @@ -70,5 +70,5 @@ async def list( "maxEndTime": max_end_time, } ), - headers=self._alpha_version_header(), + headers=self._beta_version_header(), ) diff --git a/cognite/client/_api/integrations/tasks.py b/cognite/client/_api/integrations/tasks.py index 42c4202b00..50a81a05fd 100644 --- a/cognite/client/_api/integrations/tasks.py +++ b/cognite/client/_api/integrations/tasks.py @@ -18,7 +18,7 @@ class IntegrationTasksAPI(APIClient): def __init__(self, config: ClientConfig, api_version: str | None, cognite_client: AsyncCogniteClient) -> None: super().__init__(config, api_version, cognite_client) - self._warning = FeaturePreviewWarning(api_maturity="alpha", sdk_maturity="alpha", feature_name="Integrations") + self._warning = FeaturePreviewWarning(api_maturity="beta", sdk_maturity="alpha", feature_name="Integrations") async def list_history( self, @@ -67,7 +67,7 @@ async def list_history( "lastPerTask": last_per_task, } ), - headers=self._alpha_version_header(), + headers=self._beta_version_header(), ) async def sync( @@ -136,7 +136,7 @@ async def sync( "limit": limit, } ), - headers=self._alpha_version_header(), + headers=self._beta_version_header(), semaphore=self._get_semaphore("read"), ) return SyncResult._load(response.json()) diff --git a/cognite/client/_basic_api_client.py b/cognite/client/_basic_api_client.py index c5611693db..91bade0258 100644 --- a/cognite/client/_basic_api_client.py +++ b/cognite/client/_basic_api_client.py @@ -247,6 +247,16 @@ def _alpha_version_header(self) -> dict[str, str]: # Maybe the user has set "beta" or something else, whatever the case, we just return "alpha": return {"cdf-version": "alpha"} + def _beta_version_header(self) -> dict[str, str]: + sub = self._api_subversion + if "beta" in sub: + return {"cdf-version": sub} + elif sub.isdecimal(): # default is something like "20230101" (see __api_subversion__ in _version.py) + return {"cdf-version": f"{sub}-beta"} + else: + # Maybe the user has set "alpha" or something else, whatever the case, we just return "beta": + return {"cdf-version": "beta"} + @property def _base_url_with_base_path(self) -> str: if self._api_version: diff --git a/cognite/client/_sync_api/integrations/__init__.py b/cognite/client/_sync_api/integrations/__init__.py index 39fa9289d4..2e371c6b30 100644 --- a/cognite/client/_sync_api/integrations/__init__.py +++ b/cognite/client/_sync_api/integrations/__init__.py @@ -1,6 +1,6 @@ """ =============================================================================== -05ec77fce9ac730a8d9873350250de4f +b875794d0b27a94c97637678fc1ccd6a This file is auto-generated from the Async API modules, - do not edit manually! =============================================================================== """ @@ -17,6 +17,7 @@ from cognite.client._sync_api.integrations.errors import SyncIntegrationErrorsAPI from cognite.client._sync_api.integrations.tasks import SyncIntegrationTasksAPI from cognite.client._sync_api_client import SyncAPIClient +from cognite.client.data_classes.integrations.checkin import CheckinRequest, CheckinResponse, StartupRequest from cognite.client.data_classes.integrations.integrations import ( Integration, IntegrationList, @@ -207,3 +208,63 @@ def delete(self, external_id: str | SequenceNotStr[str], ignore_unknown_ids: boo return run_sync( self.__async_client.integrations.delete(external_id=external_id, ignore_unknown_ids=ignore_unknown_ids) ) + + def startup(self, request: StartupRequest) -> CheckinResponse: + """ + `Report extractor startup `_ + + Reports that the extractor has (re)started, along with its current task configuration. + This closes any currently running tasks with an error. + + Note: + This is normally only called by extractor implementations as part of the + integrations startup protocol, not by typical SDK consumers. + + Args: + request (StartupRequest): The startup event to report. + + Returns: + CheckinResponse: The integration's latest config revision. + + Examples: + + Report extractor startup: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes.integrations import Extractor, StartupRequest + >>> client = CogniteClient() + >>> req = StartupRequest( + ... external_id="my-integration", + ... extractor=Extractor(external_id="cognite-simple-influxdb-extractor"), + ... ) + >>> res = client.integrations.startup(req) + """ + return run_sync(self.__async_client.integrations.startup(request=request)) + + def checkin(self, request: CheckinRequest) -> CheckinResponse: + """ + `Check in with the integrations service `_ + + Called periodically by extractors to signal that they are still alive, and to report task + start/stop events and errors that have occurred since the last check-in. + + Note: + This is normally only called by extractor implementations as part of the + integrations check-in protocol, not by typical SDK consumers. + + Args: + request (CheckinRequest): The check-in event to report. + + Returns: + CheckinResponse: The integration's latest config revision. + + Examples: + + Check in with no updates: + + >>> from cognite.client import CogniteClient + >>> from cognite.client.data_classes.integrations import CheckinRequest + >>> client = CogniteClient() + >>> res = client.integrations.checkin(CheckinRequest(external_id="my-integration")) + """ + return run_sync(self.__async_client.integrations.checkin(request=request)) diff --git a/cognite/client/_sync_api/integrations/actions.py b/cognite/client/_sync_api/integrations/actions.py index e4b880e395..861d3aa1b1 100644 --- a/cognite/client/_sync_api/integrations/actions.py +++ b/cognite/client/_sync_api/integrations/actions.py @@ -1,6 +1,6 @@ """ =============================================================================== -3f635530474c2e7d60de287a95002ad1 +5bbe6d0bdc778cf3dc45d354081cd5ee This file is auto-generated from the Async API modules, - do not edit manually! =============================================================================== """ diff --git a/cognite/client/_sync_api/integrations/config.py b/cognite/client/_sync_api/integrations/config.py index 9ab95373e1..fc4f1c23a7 100644 --- a/cognite/client/_sync_api/integrations/config.py +++ b/cognite/client/_sync_api/integrations/config.py @@ -1,6 +1,6 @@ """ =============================================================================== -cce5eacd4949dd8e85347c3906460b9a +f9b7f6c0d48433914f7696ffef5ba7d8 This file is auto-generated from the Async API modules, - do not edit manually! =============================================================================== """ diff --git a/cognite/client/_sync_api/integrations/errors.py b/cognite/client/_sync_api/integrations/errors.py index 82136119ca..48d0419b78 100644 --- a/cognite/client/_sync_api/integrations/errors.py +++ b/cognite/client/_sync_api/integrations/errors.py @@ -1,6 +1,6 @@ """ =============================================================================== -bde27dc1b43cfa3eeb5745a5669fe8cc +dcf8cdd6009c9164489cb8ff05bba49a This file is auto-generated from the Async API modules, - do not edit manually! =============================================================================== """ diff --git a/cognite/client/_sync_api/integrations/tasks.py b/cognite/client/_sync_api/integrations/tasks.py index 743250a44c..37263ec523 100644 --- a/cognite/client/_sync_api/integrations/tasks.py +++ b/cognite/client/_sync_api/integrations/tasks.py @@ -1,6 +1,6 @@ """ =============================================================================== -c13367430cb1ccec84ff53d416749174 +b4dc1cf697909aad510c8700ebe58dc4 This file is auto-generated from the Async API modules, - do not edit manually! =============================================================================== """ diff --git a/cognite/client/data_classes/integrations/__init__.py b/cognite/client/data_classes/integrations/__init__.py index 569d2c4767..63f866fe9e 100644 --- a/cognite/client/data_classes/integrations/__init__.py +++ b/cognite/client/data_classes/integrations/__init__.py @@ -6,6 +6,13 @@ ActionWrite, ActionWriteList, ) +from cognite.client.data_classes.integrations.checkin import ( + CheckinRequest, + CheckinResponse, + ErrorWithTask, + StartupRequest, + TaskUpdate, +) from cognite.client.data_classes.integrations.config import ( ConfigRevision, ConfigRevisionMetadata, @@ -36,10 +43,13 @@ "ActionList", "ActionWrite", "ActionWriteList", + "CheckinRequest", + "CheckinResponse", "ConfigRevision", "ConfigRevisionMetadata", "ConfigRevisionMetadataList", "ConfigRevisionWrite", + "ErrorWithTask", "Extractor", "Integration", "IntegrationError", @@ -48,8 +58,10 @@ "IntegrationUpdate", "IntegrationWrite", "IntegrationWriteList", + "StartupRequest", "SyncResult", "Task", "TaskHistory", "TaskHistoryList", + "TaskUpdate", ] diff --git a/cognite/client/data_classes/integrations/checkin.py b/cognite/client/data_classes/integrations/checkin.py new file mode 100644 index 0000000000..28ed2919ff --- /dev/null +++ b/cognite/client/data_classes/integrations/checkin.py @@ -0,0 +1,194 @@ +from __future__ import annotations + +from typing import Any, Literal, TypeAlias + +from typing_extensions import Self + +from cognite.client.data_classes._base import CogniteResource +from cognite.client.data_classes.integrations.integrations import ActiveConfigRevision, Extractor, Task + +TaskUpdateType: TypeAlias = Literal["started", "ended"] + + +class TaskUpdate(CogniteResource): + """A single start/stop event for a task, reported by the extractor on check-in. + + Args: + type (TaskUpdateType): Whether the task started or ended. + name (str): Name of the task being updated. + timestamp (int): Time of the event, in milliseconds since epoch. + message (str | None): Optional message tied to the task run. + """ + + def __init__(self, type: TaskUpdateType, name: str, timestamp: int, message: str | None = None) -> None: + self.type = type + self.name = name + self.timestamp = timestamp + self.message = message + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + return cls( + type=resource["type"], + name=resource["name"], + timestamp=resource["timestamp"], + message=resource.get("message"), + ) + + +class ErrorWithTask(CogniteResource): + """An error reported by the extractor as part of a check-in. + + Args: + level (Literal["warning", "error", "fatal"]): Severity of the error. + description (str): Short description of the error. + start_time (int): Time the error started, in milliseconds since epoch. + details (str | None): Full details of the error, e.g. a stack trace. + task (str | None): Name of the task the error occurred in. Not set if the error applies to the extractor generally. + end_time (int | None): Time the error was resolved, in milliseconds since epoch. Not set while unresolved. + active_config_revision (ActiveConfigRevision | None): The config revision (or "local") active when the error occurred. + """ + + def __init__( + self, + level: Literal["warning", "error", "fatal"], + description: str, + start_time: int, + details: str | None = None, + task: str | None = None, + end_time: int | None = None, + active_config_revision: ActiveConfigRevision | None = None, + ) -> None: + self.level = level + self.description = description + self.start_time = start_time + self.details = details + self.task = task + self.end_time = end_time + self.active_config_revision = active_config_revision + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + return cls( + level=resource["level"], + description=resource["description"], + start_time=resource["startTime"], + details=resource.get("details"), + task=resource.get("task"), + end_time=resource.get("endTime"), + active_config_revision=resource.get("activeConfigRevision"), + ) + + +class StartupRequest(CogniteResource): + """Reported by an extractor on startup: general information about the extractor, and an + indication that it has (re)started. This closes any currently running tasks with an error. + + Note: + This is normally only sent by extractor implementations as part of the integrations + startup protocol, not by typical SDK consumers. + + Args: + external_id (str): External id of the integration. + extractor (Extractor): The extractor reporting the startup event. + tasks (list[Task] | None): The tasks configured for this extractor. + active_config_revision (ActiveConfigRevision | None): The config revision (or "local") currently active. + timestamp (int | None): Time of the startup event, in milliseconds since epoch. + """ + + def __init__( + self, + external_id: str, + extractor: Extractor, + tasks: list[Task] | None = None, + active_config_revision: ActiveConfigRevision | None = None, + timestamp: int | None = None, + ) -> None: + self.external_id = external_id + self.extractor = extractor + self.tasks = tasks + self.active_config_revision = active_config_revision + self.timestamp = timestamp + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + result = super().dump(camel_case) + result["extractor"] = self.extractor.dump(camel_case) + if self.tasks is not None: + result["tasks"] = [task.dump(camel_case) for task in self.tasks] + return result + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + return cls( + external_id=resource["externalId"], + extractor=Extractor._load(resource["extractor"]), + tasks=[Task._load(task) for task in resource["tasks"]] if resource.get("tasks") is not None else None, + active_config_revision=resource.get("activeConfigRevision"), + timestamp=resource.get("timestamp"), + ) + + +class CheckinRequest(CogniteResource): + """Reported periodically by an extractor to signal it is still alive, and to report task + start/stop events and errors that have occurred since the last check-in. + + Note: + This is normally only sent by extractor implementations as part of the integrations + check-in protocol, not by typical SDK consumers. + + Args: + external_id (str): External id of the integration. + task_events (list[TaskUpdate] | None): Task start/stop events since the last check-in. + errors (list[ErrorWithTask] | None): Errors reported since the last check-in. + """ + + def __init__( + self, + external_id: str, + task_events: list[TaskUpdate] | None = None, + errors: list[ErrorWithTask] | None = None, + ) -> None: + self.external_id = external_id + self.task_events = task_events + self.errors = errors + + def dump(self, camel_case: bool = True) -> dict[str, Any]: + result = super().dump(camel_case) + if self.task_events is not None: + key = "taskEvents" if camel_case else "task_events" + result[key] = [event.dump(camel_case) for event in self.task_events] + if self.errors is not None: + result["errors"] = [error.dump(camel_case) for error in self.errors] + return result + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + return cls( + external_id=resource["externalId"], + task_events=[TaskUpdate._load(event) for event in resource["taskEvents"]] + if resource.get("taskEvents") is not None + else None, + errors=[ErrorWithTask._load(error) for error in resource["errors"]] + if resource.get("errors") is not None + else None, + ) + + +class CheckinResponse(CogniteResource): + """Response returned from both startup and check-in, containing the latest config revision. + + Args: + external_id (str): External id of the integration. + last_config_revision (int | None): The latest stored configuration revision for this integration. + """ + + def __init__(self, external_id: str, last_config_revision: int | None = None) -> None: + self.external_id = external_id + self.last_config_revision = last_config_revision + + @classmethod + def _load(cls, resource: dict[str, Any]) -> Self: + return cls( + external_id=resource["externalId"], + last_config_revision=resource.get("lastConfigRevision"), + ) diff --git a/cognite/client/utils/_url.py b/cognite/client/utils/_url.py index ffff73943c..3dfbf7da05 100644 --- a/cognite/client/utils/_url.py +++ b/cognite/client/utils/_url.py @@ -60,6 +60,8 @@ "annotations/suggest", "extpipes/config/revert", "integrations/actions/cancel", + "integrations/checkin", + "integrations/startup", "transformations/cancel", "transformations/notifications", "transformations/run", diff --git a/pyproject.toml b/pyproject.toml index 533b44f715..fe2fcf4511 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -90,6 +90,12 @@ docs = [ [project.urls] Documentation = "https://cognite-sdk-python.readthedocs-hosted.com" +[tool.codespell] +# "checkin"/"checkin()" match the Integrations API's actual operationId (integration_checkin) and +# URL path (/integrations/checkin), so they can't be reworded to "check-in" without diverging from +# the real API surface. +ignore-words-list = "checkin" + [tool.ruff] line-length = 120 target-version = "py310" diff --git a/tests/tests_unit/test_api/test_integrations/test_integrations.py b/tests/tests_unit/test_api/test_integrations/test_integrations.py index 7d66b79408..aac668efd7 100644 --- a/tests/tests_unit/test_api/test_integrations/test_integrations.py +++ b/tests/tests_unit/test_api/test_integrations/test_integrations.py @@ -6,11 +6,16 @@ from cognite.client import AsyncCogniteClient, CogniteClient from cognite.client.data_classes.integrations import ( + CheckinRequest, + CheckinResponse, + ErrorWithTask, Extractor, Integration, IntegrationList, IntegrationUpdate, IntegrationWrite, + StartupRequest, + TaskUpdate, ) from tests.utils import get_url, jsgz_load @@ -38,7 +43,7 @@ def test_list(self, cognite_client: CogniteClient, async_client: AsyncCogniteCli request = httpx_mock.get_requests()[0] assert request.method == "GET" - assert request.headers["cdf-version"] == async_client.integrations._alpha_version_header()["cdf-version"] + assert request.headers["cdf-version"] == async_client.integrations._beta_version_header()["cdf-version"] def test_create( self, cognite_client: CogniteClient, async_client: AsyncCogniteClient, httpx_mock: HTTPXMock @@ -87,6 +92,56 @@ def test_retrieve( body = jsgz_load(httpx_mock.get_requests()[0].content) assert body == {"items": [{"externalId": "my-integration"}], "ignoreUnknownIds": False} + def test_startup( + self, cognite_client: CogniteClient, async_client: AsyncCogniteClient, httpx_mock: HTTPXMock + ) -> None: + httpx_mock.add_response( + method="POST", + url=get_url(async_client.integrations, "/integrations/startup"), + json={"externalId": "my-integration", "lastConfigRevision": 3}, + ) + request = StartupRequest( + external_id="my-integration", + extractor=Extractor(external_id="cognite-simple-influxdb-extractor", version="1.0.0"), + ) + + res = cognite_client.integrations.startup(request) + + assert isinstance(res, CheckinResponse) + assert res.last_config_revision == 3 + + body = jsgz_load(httpx_mock.get_requests()[0].content) + assert body == { + "externalId": "my-integration", + "extractor": {"externalId": "cognite-simple-influxdb-extractor", "version": "1.0.0"}, + } + + def test_checkin( + self, cognite_client: CogniteClient, async_client: AsyncCogniteClient, httpx_mock: HTTPXMock + ) -> None: + httpx_mock.add_response( + method="POST", + url=get_url(async_client.integrations, "/integrations/checkin"), + json={"externalId": "my-integration", "lastConfigRevision": 4}, + ) + request = CheckinRequest( + external_id="my-integration", + task_events=[TaskUpdate(type="started", name="poll", timestamp=100)], + errors=[ErrorWithTask(level="warning", description="Slow response", start_time=100)], + ) + + res = cognite_client.integrations.checkin(request) + + assert isinstance(res, CheckinResponse) + assert res.last_config_revision == 4 + + body = jsgz_load(httpx_mock.get_requests()[0].content) + assert body == { + "externalId": "my-integration", + "taskEvents": [{"type": "started", "name": "poll", "timestamp": 100}], + "errors": [{"level": "warning", "description": "Slow response", "startTime": 100}], + } + def test_update( self, cognite_client: CogniteClient, async_client: AsyncCogniteClient, httpx_mock: HTTPXMock ) -> None: diff --git a/tests/tests_unit/test_api_client.py b/tests/tests_unit/test_api_client.py index 84faaa9b64..37282bb810 100644 --- a/tests/tests_unit/test_api_client.py +++ b/tests/tests_unit/test_api_client.py @@ -1790,6 +1790,8 @@ async def test_is_retryable_resource_api_endpoints(self, method: str, path: str, ("POST", "https://api.cognitedata.com/api/v1/projects/bla/integrations/actions", False), ("POST", "https://api.cognitedata.com/api/v1/projects/bla/integrations/actions/byids", True), ("POST", "https://api.cognitedata.com/api/v1/projects/bla/integrations/actions/cancel", False), + ("POST", "https://api.cognitedata.com/api/v1/projects/bla/integrations/startup", False), + ("POST", "https://api.cognitedata.com/api/v1/projects/bla/integrations/checkin", False), # Transformations ("POST", "https://api.cognitedata.com/api/v1/projects/bla/transformations", False), ("POST", "https://api.cognitedata.com/api/v1/projects/bla/transformations/filter", True), diff --git a/tests/tests_unit/test_data_classes/test_integrations.py b/tests/tests_unit/test_data_classes/test_integrations.py index 9c1a4af29f..c464af3bc5 100644 --- a/tests/tests_unit/test_data_classes/test_integrations.py +++ b/tests/tests_unit/test_data_classes/test_integrations.py @@ -2,12 +2,17 @@ from cognite.client.data_classes.integrations import ( Action, + CheckinRequest, + CheckinResponse, ConfigRevision, + ErrorWithTask, Extractor, Integration, IntegrationError, IntegrationUpdate, + StartupRequest, Task, + TaskUpdate, ) from cognite.client.data_classes.integrations.tasks import SyncResult, TaskHistory @@ -204,3 +209,55 @@ def test_task_load_dump() -> None: "targets": ["cdf://cluster/project/timeseries/my-ts"], } assert Task._load(dumped).dump(camel_case=True) == dumped + + +def test_task_update_load_dump() -> None: + dumped = {"type": "started", "name": "poll", "timestamp": 100, "message": "Task started"} + assert TaskUpdate._load(dumped).dump(camel_case=True) == dumped + + +def test_error_with_task_load_dump() -> None: + dumped = { + "level": "fatal", + "description": "Something went very wrong", + "startTime": 100, + "details": "Traceback ...", + "task": "poll", + "endTime": 200, + "activeConfigRevision": 2, + } + assert ErrorWithTask._load(dumped).dump(camel_case=True) == dumped + + +def test_startup_request_load_dump() -> None: + dumped = { + "externalId": "my-integration", + "extractor": {"externalId": "cognite-simple-influxdb-extractor", "version": "1.0.0"}, + "tasks": [{"type": "continuous", "name": "poll", "action": False}], + "activeConfigRevision": "local", + "timestamp": 100, + } + loaded = StartupRequest._load(dumped) + + assert loaded.tasks is not None + assert loaded.tasks[0].name == "poll" + assert loaded.dump(camel_case=True) == dumped + + +def test_checkin_request_load_dump() -> None: + dumped = { + "externalId": "my-integration", + "taskEvents": [{"type": "ended", "name": "poll", "timestamp": 200}], + "errors": [{"level": "error", "description": "Oops", "startTime": 100}], + } + loaded = CheckinRequest._load(dumped) + + assert loaded.task_events is not None + assert loaded.errors is not None + assert loaded.task_events[0].type == "ended" + assert loaded.dump(camel_case=True) == dumped + + +def test_checkin_response_load_dump() -> None: + dumped = {"externalId": "my-integration", "lastConfigRevision": 5} + assert CheckinResponse._load(dumped).dump(camel_case=True) == dumped From ff8a44d2740ca9cc150c505781048324c0ecd828 Mon Sep 17 00:00:00 2001 From: vikramlc Date: Mon, 7 Sep 2026 13:47:44 +0530 Subject: [PATCH 5/5] fix(odin): Fix lint issue --- .pre-commit-config.yaml | 7 +++++++ cognite/client/data_classes/integrations/checkin.py | 2 +- pyproject.toml | 6 ------ 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e5aa630cd7..a8b8a7a094 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -71,3 +71,10 @@ repos: hooks: - id: codespell files: ^cognite/.* + # "checkin" matches the Integrations API's actual operationId (integration_checkin) and + # URL path (/integrations/checkin), so it can't be reworded to "check-in" without diverging + # from the real API surface. Passed as a hook arg (not pyproject.toml's [tool.codespell]) + # since reading TOML config requires tomli/tomllib, which isn't guaranteed to be available + # in the hook's isolated environment on Python <3.11. + args: + - --ignore-words-list=checkin diff --git a/cognite/client/data_classes/integrations/checkin.py b/cognite/client/data_classes/integrations/checkin.py index 28ed2919ff..b6caeda3d5 100644 --- a/cognite/client/data_classes/integrations/checkin.py +++ b/cognite/client/data_classes/integrations/checkin.py @@ -40,7 +40,7 @@ class ErrorWithTask(CogniteResource): """An error reported by the extractor as part of a check-in. Args: - level (Literal["warning", "error", "fatal"]): Severity of the error. + level (Literal['warning', 'error', 'fatal']): Severity of the error. description (str): Short description of the error. start_time (int): Time the error started, in milliseconds since epoch. details (str | None): Full details of the error, e.g. a stack trace. diff --git a/pyproject.toml b/pyproject.toml index fe2fcf4511..533b44f715 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -90,12 +90,6 @@ docs = [ [project.urls] Documentation = "https://cognite-sdk-python.readthedocs-hosted.com" -[tool.codespell] -# "checkin"/"checkin()" match the Integrations API's actual operationId (integration_checkin) and -# URL path (/integrations/checkin), so they can't be reworded to "check-in" without diverging from -# the real API surface. -ignore-words-list = "checkin" - [tool.ruff] line-length = 120 target-version = "py310"