diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..703ad03 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +src/unstract/api_deployments/sdk_docstudio/** linguist-generated=true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 304fbda..e47c5fa 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,14 +14,14 @@ jobs: matrix: python-version: ["3.11", "3.12"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: ${{ matrix.python-version }} - name: Install uv - uses: astral-sh/setup-uv@v6 + uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0 with: version: "0.6.14" enable-cache: true @@ -37,3 +37,31 @@ jobs: - name: Tests (pytest) run: uv run pytest tests/ -v + + sdk-drift: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install uv + uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0 + with: + version: "0.6.14" + enable-cache: true + + # The generated tree is committed, so an edit to it reviews like any + # other change and then disappears on the next regeneration. Same for a + # spec change that never had the generator run over it. + - name: Regenerate from the committed spec + run: ./tools/gen_sdk.sh + + # `git add -N` first: a diff alone cannot see a file the generator has + # newly created, which is exactly what a spec growing an endpoint does. + - name: Fail if the committed SDK is not what the spec generates + run: | + git add -N -- src/unstract/api_deployments/sdk_docstudio + git diff --exit-code -- src/unstract/api_deployments/sdk_docstudio diff --git a/README.md b/README.md index eedc645..b0d9b6a 100644 --- a/README.md +++ b/README.md @@ -100,16 +100,18 @@ client = APIDeploymentsClient( The retry logic uses exponential backoff with full jitter and respects the `Retry-After` header on 429 responses. -## Unstract CLI +## Cloning an organization -Installing `unstract-client` also provides the `unstract` command: +Installing `unstract-client` also provides a clone command. The `unstract` +console script this package used to install is gone — the name now belongs to +the `unstract-cli` package — so invoke it as a module: ```bash pip install unstract-client -unstract --help +python -m unstract.clone --help ``` -### `unstract clone` +### `python -m unstract.clone` Clones an organization's resources to another org, on the same or a different deployment (e.g. promote **dev** → **QA** → **prod**). Covers adapters, @@ -124,7 +126,7 @@ so keys never land in shell history: export UNSTRACT_SRC_PLATFORM_KEY="" export UNSTRACT_TGT_PLATFORM_KEY="" -unstract clone \ +python -m unstract.clone \ --source-url https://dev.example.com --source-org org_dev123 \ --target-url https://qa.example.com --target-org org_qa456 \ --dry-run @@ -152,14 +154,14 @@ failed run can be resumed by re-running the same command. #### Compatibility -`unstract clone` is capability-probed: each phase checks for its endpoint on the +Cloning is capability-probed: each phase checks for its endpoint on the source and target, and clones only what both orgs support. A capability missing on either side is reported and skipped — the run never fails because of a version difference. Cloning a newer source into an older target therefore drops the entity types the target lacks (listed in the end-of-run report). - Run the source and target on the same (or a newer-target) Unstract build. -- Use `unstract-client >= 1.4.0`, the first release that ships `unstract clone`. +- Use `unstract-client >= 1.4.0`, the first release that ships the clone command. ## Questions and Feedback diff --git a/pyproject.toml b/pyproject.toml index 0cb4dff..4bdc7e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,12 @@ authors = [ {name = "Zipstack Inc", email = "devsupport@zipstack.com"}, ] dependencies = [ + # The transport layer is generated against httpx; attrs backs its models. + # Upper-bounded because the generated code is written against one minor + # series: a bump has to be regenerated and re-tested, not resolved into. + "httpx>=0.27,<0.29", + "attrs>=23.2", + # Kept for its exception classes, which callers catch by name. "requests>=2.32.3", "tenacity>=8.2.0", "click>=8.1", @@ -27,9 +33,6 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Python Modules", ] -[project.scripts] -unstract = "unstract.cli:main" - [build-system] requires = ["hatchling"] build-backend = "hatchling.build" @@ -64,6 +67,9 @@ lint = [ [tool.ruff] line-length = 88 +# Generated and vendored code is overwritten wholesale by its refresh script, so +# a lint finding there can never be fixed in place. +extend-exclude = ["src/unstract/api_deployments/sdk_docstudio", "tests/baseline"] [tool.ruff.lint] select = ["E", "F", "W", "I"] diff --git a/specs/docstudio-oss.json b/specs/docstudio-oss.json new file mode 100644 index 0000000..edf3196 --- /dev/null +++ b/specs/docstudio-oss.json @@ -0,0 +1,479 @@ +{ + "components": { + "schemas": { + "ErrorResponse": { + "properties": { + "message": { + "nullable": true + }, + "status": { + "type": "string" + } + }, + "type": "object" + }, + "ExecuteRequest": { + "description": "The documents to run, and the options that shape the result.\n\nSupply `files`, `presigned_urls`, or both.", + "properties": { + "custom_data": { + "nullable": true + }, + "files": { + "items": { + "format": "binary", + "type": "string" + }, + "type": "array" + }, + "hitl_packet_id": { + "nullable": true, + "type": "string" + }, + "hitl_queue_name": { + "nullable": true, + "type": "string" + }, + "include_extracted_text": { + "default": false, + "type": "boolean" + }, + "include_metadata": { + "default": false, + "type": "boolean" + }, + "include_metrics": { + "default": false, + "type": "boolean" + }, + "llm_profile_id": { + "nullable": true, + "type": "string" + }, + "presigned_urls": { + "items": { + "format": "uri", + "type": "string" + }, + "type": "array" + }, + "tags": { + "default": "", + "description": "Comma-separated list of tag names (EX:'tag1,tag2-name,tag3_name')", + "type": "string" + }, + "timeout": { + "default": -1, + "maximum": 300, + "minimum": -1, + "type": "integer" + }, + "use_file_history": { + "default": false, + "type": "boolean" + } + }, + "type": "object" + }, + "ExecuteResponse": { + "properties": { + "message": { + "$ref": "#/components/schemas/ExecutionMessage" + } + }, + "required": [ + "message" + ], + "type": "object" + }, + "ExecutionMessage": { + "description": "The execution's identity and, once it has finished, its per-file\nresults.", + "properties": { + "error": { + "type": "string" + }, + "execution_id": { + "type": "string" + }, + "execution_status": { + "type": "string" + }, + "result": { + "items": { + "$ref": "#/components/schemas/FileResult" + }, + "nullable": true, + "type": "array" + }, + "status_api": { + "type": "string" + } + }, + "required": [ + "error", + "execution_id", + "execution_status", + "status_api" + ], + "type": "object" + }, + "FileResult": { + "properties": { + "error": { + "nullable": true, + "type": "string" + }, + "file": { + "type": "string" + }, + "file_execution_id": { + "type": "string" + }, + "metadata": {}, + "metrics": {}, + "result": {}, + "status": { + "type": "string" + } + }, + "required": [ + "file" + ], + "type": "object" + }, + "StatusResponse": { + "properties": { + "message": { + "items": { + "$ref": "#/components/schemas/FileResult" + }, + "nullable": true, + "type": "array" + }, + "status": { + "type": "string" + } + }, + "required": [ + "status" + ], + "type": "object" + } + }, + "securitySchemes": { + "deploymentKey": { + "description": "The API deployment's own key.", + "scheme": "bearer", + "type": "http" + } + } + }, + "info": { + "title": "Unstract API", + "version": "v1" + }, + "openapi": "3.0.3", + "paths": { + "/deployment/api/{org_name}/{api_name}/": { + "get": { + "description": "Read the result of a previously started execution.\n\nThis read is one-shot: the first call that observes a completed execution acknowledges it and the stored result is discarded, so every later call for that execution answers 406. Poll while the execution is pending, and keep the payload of the call that returns it \u2014 it cannot be fetched again.", + "operationId": "status", + "parameters": [ + { + "description": "API deployment name.", + "in": "path", + "name": "api_name", + "required": true, + "schema": { + "pattern": "^[\\w-]+$", + "type": "string" + } + }, + { + "in": "query", + "name": "execution_id", + "required": true, + "schema": { + "minLength": 1, + "type": "string" + } + }, + { + "in": "query", + "name": "include_extracted_text", + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "in": "query", + "name": "include_metadata", + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "in": "query", + "name": "include_metrics", + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "description": "Organization identifier.", + "in": "path", + "name": "org_name", + "required": true, + "schema": { + "pattern": "^[\\w-]+$", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/StatusResponse" + } + } + }, + "description": "" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "The request failed validation." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "The API key is not valid." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "No API key was supplied." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "No such active deployment." + }, + "406": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "The result was already consumed by an earlier call." + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/StatusResponse" + } + } + }, + "description": "" + }, + "429": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "Too many concurrent executions; retry later." + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "" + } + }, + "security": [ + { + "deploymentKey": [] + } + ], + "tags": [ + "deployment" + ] + }, + "post": { + "description": "Execute an API deployment against one or more documents.\n\nSupply the documents either as `files` (multipart upload) or as `presigned_urls` (HTTPS S3 URLs), or both \u2014 a request carrying neither is rejected, and the two together may not exceed 32 documents.\n\nWith the default `timeout` of -1 the call returns as soon as the execution is queued; read the outcome from the status endpoint.", + "operationId": "execute", + "parameters": [ + { + "description": "API deployment name.", + "in": "path", + "name": "api_name", + "required": true, + "schema": { + "pattern": "^[\\w-]+$", + "type": "string" + } + }, + { + "description": "Organization identifier.", + "in": "path", + "name": "org_name", + "required": true, + "schema": { + "pattern": "^[\\w-]+$", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "$ref": "#/components/schemas/ExecuteRequest" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ExecuteResponse" + } + } + }, + "description": "" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "The request failed validation." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "The API key is not valid." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "No API key was supplied." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "No such active deployment." + }, + "409": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "The deployment has no active API key." + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ExecuteResponse" + } + } + }, + "description": "" + }, + "429": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "Too many concurrent executions; retry later." + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "" + } + }, + "security": [ + { + "deploymentKey": [] + } + ], + "tags": [ + "deployment" + ] + } + } + }, + "tags": [ + { + "description": "Run an API deployment against one or more documents and poll the result.", + "name": "deployment" + } + ] +} diff --git a/src/unstract/api_deployments/client.py b/src/unstract/api_deployments/client.py index e39731d..9c201a2 100644 --- a/src/unstract/api_deployments/client.py +++ b/src/unstract/api_deployments/client.py @@ -11,10 +11,26 @@ import ntpath import os import time -from urllib.parse import urlparse - -import requests -from requests.exceptions import ConnectionError, JSONDecodeError, Timeout +from typing import Any +from urllib.parse import parse_qs, urljoin, urlparse + +import attrs +import httpx + +# `requests` remains a dependency for its exception classes. Downstream code +# catches ConnectionError and Timeout by name around these calls, and the httpx +# equivalents are not subclasses, so they are translated at the transport seam. +from requests.exceptions import ( + ConnectionError, + ConnectTimeout, + ContentDecodingError, + InvalidURL, + MissingSchema, + ProxyError, + ReadTimeout, + Timeout, + TooManyRedirects, +) from tenacity import ( RetryCallState, Retrying, @@ -25,9 +41,85 @@ ) from tenacity.wait import wait_base +from unstract.api_deployments.sdk_docstudio import AuthenticatedClient +from unstract.api_deployments.sdk_docstudio.api.deployment import execute, status +from unstract.api_deployments.sdk_docstudio.models import ExecuteRequest +from unstract.api_deployments.sdk_docstudio.types import UNSET, File, Unset from unstract.api_deployments.utils import UnstractUtils +def _translate_transport_errors(fn, *args, **kwargs): + """Re-raise httpx transport failures as their ``requests`` equivalents. + + Callers document and catch the ``requests`` classes, and the retry policy + keys off them too, so the class chosen here decides whether a failure is + retried. Every branch is ordered before the base class it derives from. + ``RequestError`` is the catch-all for the transport subtree, which is where + a novel failure appears. httpx puts three families outside it: ``InvalidURL``, + translated here because ``requests`` raised its own, and ``StreamError`` and + ``CookieConflict``, which propagate as themselves. + """ + try: + return fn(*args, **kwargs) + except httpx.ConnectTimeout as e: + # ConnectTimeout is both a ConnectionError and a Timeout; the plain + # Timeout httpx implies would stop matching half the callers. + raise ConnectTimeout(str(e)) from e + except httpx.ReadTimeout as e: + raise ReadTimeout(str(e)) from e + except (httpx.WriteTimeout, httpx.PoolTimeout) as e: + # Neither had a Timeout equivalent: a send that failed and a pool that + # could not hand out a connection both surfaced as ConnectionError. + raise ConnectionError(str(e)) from e + except httpx.TimeoutException as e: + raise Timeout(str(e)) from e + except httpx.UnsupportedProtocol as e: + # A URL rejected before any socket is opened. Deliberately not a + # ConnectionError: retrying a malformed URL cannot start working. + raise MissingSchema(str(e)) from e + except httpx.ProxyError as e: + raise ProxyError(str(e)) from e + except httpx.ConnectError as e: + raise ConnectionError(str(e)) from e + except httpx.TooManyRedirects as e: + raise TooManyRedirects(str(e)) from e + except httpx.DecodingError as e: + raise ContentDecodingError(str(e)) from e + except httpx.InvalidURL as e: + raise InvalidURL(str(e)) from e + except httpx.RequestError as e: + raise ConnectionError(str(e)) from e + + +def _query_value(url: str, key: str) -> str: + """Read one required query parameter out of a URL, absolute or relative. + + Empty is not a usable value here: it polls for an execution the service + cannot identify and reports whatever it makes of a blank id. + """ + value = parse_qs(urlparse(url).query).get(key, [""])[0] + if not value: + raise APIDeploymentsClientException( + f"No {key} in {url!r}. The status endpoint the service returned " + "carries it; pass that endpoint unmodified." + ) + return value + + +def _forwarded_query(url: str) -> dict[str, str]: + """Everything else the service put on the status endpoint. + + A region hint, a signature, a cursor: the endpoint is the service's + instruction for reaching this execution, and dropping part of it polls + somewhere the execution is not. + """ + return { + key: values[-1] + for key, values in parse_qs(urlparse(url).query, keep_blank_values=True).items() + if key != "execution_id" + } + + class APIDeploymentsClientException(Exception): """A class to handle exceptions raised by the APIClient class.""" @@ -76,6 +168,16 @@ def __call__(self, retry_state: RetryCallState) -> float: return self._exp_jitter(retry_state) +#: Request fields this client sets itself. Anything outside these sets is reset +#: to UNSET (body) or filtered out (query) before the request goes out, so no +#: parameter is sent that the caller did not ask for. A new parameter must be +#: added here to be sent at all. +_EXECUTE_SEND_ONLY = frozenset( + {"timeout", "include_metadata", "files", "additional_properties"} +) +_STATUS_SEND_ONLY = frozenset({"execution_id", "include_metadata"}) + + class APIDeploymentsClient: """A class to invoke APIs deployed on the Unstract platform.""" @@ -104,6 +206,8 @@ def __init__( max_delay: float = 60.0, backoff_factor: float = 2.0, jitter: float = 1.0, + *, + transport_timeout: float | None = None, ): """Initializes the APIClient class. @@ -116,6 +220,10 @@ def __init__( max_delay (float): Maximum delay in seconds between retries. backoff_factor (float): Multiplier applied to delay for each retry. jitter (float): Maximum additive jitter in seconds added to each delay. + transport_timeout (float | None): Socket timeout in seconds. Unset + means a stalled connection blocks forever, which is what the + released client did; ``api_timeout`` cannot serve here because + it is an execution mode, not a socket timeout. """ if logging_level == "": logging_level = os.getenv("UNSTRACT_API_CLIENT_LOGGING_LEVEL", "INFO") @@ -147,6 +255,7 @@ def __init__( self.max_delay = max_delay self.backoff_factor = backoff_factor self.jitter = jitter + self.transport_timeout = transport_timeout def _is_retryable_status(self, status_code: int) -> bool: """Checks whether a status code should trigger a retry. @@ -169,6 +278,98 @@ def __save_base_url(self, full_url: str): self.base_url = parsed_url.scheme + "://" + parsed_url.netloc self.logger.debug("Base URL: " + self.base_url) + @property + def _transport(self): + """The HTTP client, built on first use. + + Untimed by default, matching the previous behaviour. ``api_timeout`` is + a backend execution mode (0 selects async execution), never a socket + timeout; feeding it to the transport fails deep in the connection layer + for the negative values the API accepts. ``transport_timeout`` is the + way to bound a stalled connection. + """ + if getattr(self, "_transport_client", None) is None: + self._transport_client = AuthenticatedClient( + base_url=self.base_url, + token=self.api_key, + verify_ssl=self.verify, + timeout=httpx.Timeout(self.transport_timeout), + raise_on_unexpected_status=False, + # The previous transport followed redirects. Without this a 30x + # from a load balancer is read as a terminal result with no + # status, which a poll loop reports as a finished-and-empty job. + follow_redirects=True, + ) + return self._transport_client + + @property + def _deployment_route(self) -> tuple[str, str]: + """Organisation and API name, from the deployment URL's last two + segments.""" + segments = urlparse(self.api_url).path.strip("/").split("/") + if len(segments) < 2: + raise APIDeploymentsClientException( + f"Cannot derive organisation and API name from api_url: {self.api_url}" + ) + return segments[-2], segments[-1] + + def _status_url(self, endpoint: str, path: str) -> str: + """Absolute URL to poll, under the deployment's own path prefix. + + ``base_url`` is scheme and host only, so a deployment served under a path + prefix would execute -- the execute call sends the caller's URL verbatim + -- and then never poll. The prefix is whatever precedes the spec route + inside the deployment URL. Where the two do not line up there is no + prefix to derive, and the path the service returned is used as it came: + a guessed path polls nothing, and the execution behind it has already + been paid for. + + Only the path is taken. A scheme and host in the reply would otherwise + decide where the deployment key is sent, and the reply is not the thing + that gets to choose that. + """ + route = path.rstrip("/") + prefix = urlparse(self.api_url).path.rstrip("/") + if route and prefix.endswith(route): + return self.base_url + prefix[: -len(route)] + path + # Joined rather than concatenated: the query travels as params. + return urljoin( + self.base_url, + urlparse(endpoint)._replace(scheme="", netloc="", query="").geturl(), + ) + + def _send(self, method: str, url: str, **kwargs) -> httpx.Response: + """Issue one request, translating transport failures on the way out. + + Translation happens here rather than around the retry loop, so + the retry policy still sees the exception types it is configured + to retry. + + The credential is read per request rather than captured with the + transport, so assigning ``api_key`` takes effect on the next call the + way it did when every call built its own header. + """ + kwargs["headers"] = { + **(kwargs.get("headers") or {}), + "Authorization": f"Bearer {self.api_key}", + } + return _translate_transport_errors( + self._transport.get_httpx_client().request, method, url, **kwargs + ) + + @staticmethod + def _read_body(response): + """Read the JSON body directly, never the generated response model. + + A model is only built for the statuses the spec declares, and error + bodies are typed loosely, so an undeclared status or any error response + has no usable model. ``None`` means the body was not JSON. + """ + try: + return response.json() + except ValueError: + return None + @staticmethod def _rewind_files(files): """Rewinds file objects so they can be re-sent on retry.""" @@ -180,7 +381,7 @@ def _rewind_files(files): if hasattr(file_obj[1], "seek"): file_obj[1].seek(0) - def _request_with_retry(self, method: str, url: str, **kwargs) -> requests.Response: + def _request_with_retry(self, method: str, url: str, **kwargs) -> httpx.Response: """Makes an HTTP request with exponential backoff retry logic. Uses ``tenacity`` with additive jitter and Retry-After support. @@ -188,10 +389,10 @@ def _request_with_retry(self, method: str, url: str, **kwargs) -> requests.Respo Args: method (str): The HTTP method (e.g., "GET", "POST"). url (str): The request URL. - **kwargs: Additional keyword arguments passed to requests.request(). + **kwargs: Additional keyword arguments passed to the transport. Returns: - requests.Response: The response from the request. + The response from the request. Raises: ConnectionError: If a connection error persists after all retries. @@ -267,13 +468,44 @@ def _retry_error_callback(retry_state: RetryCallState): reraise=False, ) - return retrier(requests.request, method, url, **kwargs) + return retrier(self._send, method, url, **kwargs) - def structure_file(self, file_paths: list[str]) -> dict: + def structure_file( + self, + file_paths: list[str], + *, + timeout: int | Unset = UNSET, + include_metadata: bool | Unset = UNSET, + include_metrics: bool | Unset = UNSET, + include_extracted_text: bool | Unset = UNSET, + use_file_history: bool | Unset = UNSET, + tags: str | Unset = UNSET, + llm_profile_id: str | None | Unset = UNSET, + hitl_queue_name: str | None | Unset = UNSET, + hitl_packet_id: str | None | Unset = UNSET, + presigned_urls: list[str] | Unset = UNSET, + custom_data: Any | Unset = UNSET, + ) -> dict: """Invokes the API deployed on the Unstract platform. + The keyword arguments are the request parameters the deployment accepts, + named as the API names them. One left unset is not sent at all, so the + server picks its own default; ``timeout`` and ``include_metadata`` fall + back to the values given at construction. + Args: file_paths (list[str]): The file path to the file to be uploaded. + timeout (int): Execution mode — ``0`` or below runs asynchronously. + include_metadata (bool): Include metadata in the result. + include_metrics (bool): Include metrics in the result. + include_extracted_text (bool): Include the extracted text. + use_file_history (bool): Reuse a previous result for the same file. + tags (str): Comma-separated tag names. + llm_profile_id (str): LLM profile to override the deployment's. + hitl_queue_name (str): Human-in-the-loop queue to route the file to. + hitl_packet_id (str): Human-in-the-loop packet to attach the file to. + presigned_urls (list[str]): URLs to fetch the inputs from. + custom_data (Any): Arbitrary data echoed back with the result. Returns: dict: The response from the API. @@ -281,63 +513,95 @@ def structure_file(self, file_paths: list[str]) -> dict: self.logger.debug("Invoking API: " + self.api_url) self.logger.debug("File paths: " + str(file_paths)) - headers = { - "Authorization": "Bearer " + self.api_key, + requested = { + "timeout": timeout, + "include_metadata": include_metadata, + "include_metrics": include_metrics, + "include_extracted_text": include_extracted_text, + "use_file_history": use_file_history, + "tags": tags, + "llm_profile_id": llm_profile_id, + "hitl_queue_name": hitl_queue_name, + "hitl_packet_id": hitl_packet_id, + "presigned_urls": presigned_urls, + "custom_data": custom_data, } - - form_data = { + # ``None`` is dropped with ``UNSET``: these are optional overrides, and a + # form field carries no null, so one would go out as the string "None" + # for the service to look up. + requested = { + k: v + for k, v in requested.items() + if not isinstance(v, Unset) and v is not None + } + params = { "timeout": self.api_timeout, "include_metadata": self.include_metadata, + **requested, } + send_only = _EXECUTE_SEND_ONLY | requested.keys() - files = [] - + handles = [] try: for file_path in file_paths: - record = ( - "files", - ( - ntpath.basename(file_path), - open(file_path, "rb"), - "application/octet-stream", - ), - ) - files.append(record) + handles.append(open(file_path, "rb")) except FileNotFoundError as e: + for handle in handles: + handle.close() raise APIDeploymentsClientException("File not found: " + str(e)) - if self.api_timeout == 0: - # Async mode: server returns immediately after queuing. - # A 5xx means queuing failed — safe to retry. - response = self._request_with_retry( - "POST", - self.api_url, - headers=headers, - data=form_data, - files=files, - verify=self.verify, - ) - else: - # Sync mode: server blocks during processing. - # A 5xx may mean it processed but response was lost — don't retry - # to avoid duplicate executions. - response = requests.post( - self.api_url, - headers=headers, - data=form_data, - files=files, - verify=self.verify, - ) + body = ExecuteRequest( + files=[ + File( + payload=handle, + file_name=ntpath.basename(file_path), + mime_type="application/octet-stream", + ) + for file_path, handle in zip(file_paths, handles) + ], + **params, + ) + # Only the fields this client sets are sent. Every other field carries the + # spec's declared default, and sending a default is not the same as + # omitting it: it pins a value the server would otherwise choose, and the + # two diverge the moment the server's own default changes. + for field in attrs.fields(ExecuteRequest): + if field.name not in send_only: + setattr(body, field.name, UNSET) + + org_name, api_name = self._deployment_route + request_kwargs = execute._get_kwargs(org_name, api_name, body=body) + # The generated builder pins a fixed multipart boundary in the header. An + # uploaded file containing those bytes would break the encoding, so let + # the transport pick a random boundary instead. + request_kwargs.get("headers", {}).pop("Content-Type", None) + method = request_kwargs.pop("method") + request_kwargs.pop("url") + # The deployment URL is the caller's, sent back verbatim. Rebuilding it + # from the spec's path template drops any prefix the deployment is + # served under, which no route template can express. + url = self.api_url + + try: + if params["timeout"] == 0: + # The request only queues the execution, so a 5xx means queuing + # failed and retrying cannot duplicate work. + response = self._request_with_retry(method, url, **request_kwargs) + else: + # The request runs the execution, so a 5xx may mean it ran and + # the response was lost: a retry would execute it twice. + response = self._send(method, url, **request_kwargs) + finally: + for handle in handles: + handle.close() self.logger.debug(response.status_code) self.logger.debug(response.text) # The returned object is wrapped in a "message" key. # Let's simplify the response. obj_to_return = {} - try: - response_data = response.json() - response_message = response_data.get("message", {}) - except JSONDecodeError: + response_data = self._read_body(response) + if response_data is None: self.logger.error( "Failed to decode JSON response. Raw response: %s", response.text, @@ -351,6 +615,7 @@ def structure_file(self, file_paths: list[str]) -> dict: "extraction_result": "", } return obj_to_return + response_message = response_data.get("message", {}) if response.status_code == 401: obj_to_return = { "status_code": response.status_code, @@ -399,37 +664,76 @@ def structure_file(self, file_paths: list[str]) -> dict: return obj_to_return - def check_execution_status(self, status_check_api_endpoint: str) -> dict: + def check_execution_status( + self, + status_check_api_endpoint: str, + *, + include_metadata: bool | Unset = UNSET, + include_metrics: bool | Unset = UNSET, + include_extracted_text: bool | Unset = UNSET, + ) -> dict: """Checks the status of the execution. + The keyword arguments are the query parameters the endpoint accepts, + named as the API names them. One left unset is not sent at all, so the + server picks its own default; ``include_metadata`` falls back to the + value given at construction. + Args: status_check_api_endpoint (str): The API endpoint to check the status of the execution. + include_metadata (bool): Include metadata in the result. + include_metrics (bool): Include metrics in the result. + include_extracted_text (bool): Include the extracted text. Returns: dict: The response from the API. """ - headers = { - "Authorization": "Bearer " + self.api_key, + self.logger.debug( + "Checking execution status via endpoint: " + status_check_api_endpoint + ) + requested = { + "include_metadata": include_metadata, + "include_metrics": include_metrics, + "include_extracted_text": include_extracted_text, + } + requested = {k: v for k, v in requested.items() if not isinstance(v, Unset)} + params = {"include_metadata": self.include_metadata, **requested} + + org_name, api_name = self._deployment_route + request_kwargs = status._get_kwargs( + org_name, + api_name, + execution_id=_query_value(status_check_api_endpoint, "execution_id"), + **params, + ) + # The generated builder writes every declared query parameter, including + # ones this client has never sent. Keep only what was asked for. + send_only = _STATUS_SEND_ONLY | requested.keys() + # Booleans are spelled the way urlencoding a Python bool spells them, + # which is what the released client sent. The service reads either, but + # traffic diffed against the previous release should show no change. + request_kwargs["params"] = { + **_forwarded_query(status_check_api_endpoint), + **{ + k: str(v) if isinstance(v, bool) else v + for k, v in request_kwargs["params"].items() + if k in send_only + }, } - status_call_url = self.base_url + status_check_api_endpoint - self.logger.debug("Checking execution status via endpoint: " + status_call_url) response = self._request_with_retry( - "GET", - status_call_url, - headers=headers, - params={"include_metadata": self.include_metadata}, - verify=self.verify, + request_kwargs.pop("method"), + self._status_url(status_check_api_endpoint, request_kwargs.pop("url")), + **request_kwargs, ) self.logger.debug(response.status_code) self.logger.debug(response.text) obj_to_return = {} - try: - response_data = response.json() - except JSONDecodeError: + response_data = self._read_body(response) + if response_data is None: self.logger.error( "Failed to decode JSON response. Raw response: %s", response.text, diff --git a/src/unstract/api_deployments/sdk_docstudio/__init__.py b/src/unstract/api_deployments/sdk_docstudio/__init__.py new file mode 100644 index 0000000..ad2a46e --- /dev/null +++ b/src/unstract/api_deployments/sdk_docstudio/__init__.py @@ -0,0 +1,9 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +"""A client library for accessing Unstract API""" + +from .client import AuthenticatedClient, Client + +__all__ = ( + "AuthenticatedClient", + "Client", +) diff --git a/src/unstract/api_deployments/sdk_docstudio/api/__init__.py b/src/unstract/api_deployments/sdk_docstudio/api/__init__.py new file mode 100644 index 0000000..d8d42ef --- /dev/null +++ b/src/unstract/api_deployments/sdk_docstudio/api/__init__.py @@ -0,0 +1,2 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +"""Contains methods for accessing the API""" diff --git a/src/unstract/api_deployments/sdk_docstudio/api/deployment/__init__.py b/src/unstract/api_deployments/sdk_docstudio/api/deployment/__init__.py new file mode 100644 index 0000000..c7e8df6 --- /dev/null +++ b/src/unstract/api_deployments/sdk_docstudio/api/deployment/__init__.py @@ -0,0 +1,2 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +"""Contains endpoint functions for accessing the API""" diff --git a/src/unstract/api_deployments/sdk_docstudio/api/deployment/execute.py b/src/unstract/api_deployments/sdk_docstudio/api/deployment/execute.py new file mode 100644 index 0000000..c9cc983 --- /dev/null +++ b/src/unstract/api_deployments/sdk_docstudio/api/deployment/execute.py @@ -0,0 +1,269 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +from http import HTTPStatus +from typing import Any +from urllib.parse import quote + +import httpx + +from ... import errors +from ...client import AuthenticatedClient, Client +from ...models.error_response import ErrorResponse +from ...models.execute_request import ExecuteRequest +from ...models.execute_response import ExecuteResponse +from ...types import UNSET, Response, Unset + + +def _get_kwargs( + org_name: str, + api_name: str, + *, + body: ExecuteRequest | Unset = UNSET, +) -> dict[str, Any]: + headers: dict[str, Any] = {} + + _kwargs: dict[str, Any] = { + "method": "post", + "url": "/deployment/api/{org_name}/{api_name}/".format( + org_name=quote(str(org_name), safe=""), + api_name=quote(str(api_name), safe=""), + ), + } + + if not isinstance(body, Unset): + _kwargs["files"] = body.to_multipart() + + headers["Content-Type"] = "multipart/form-data; boundary=+++" + + _kwargs["headers"] = headers + return _kwargs + + +def _parse_response( + *, client: AuthenticatedClient | Client, response: httpx.Response +) -> ErrorResponse | ExecuteResponse | None: + if response.status_code == 200: + response_200 = ExecuteResponse.from_dict(response.json()) + + return response_200 + + if response.status_code == 400: + response_400 = ErrorResponse.from_dict(response.json()) + + return response_400 + + if response.status_code == 401: + response_401 = ErrorResponse.from_dict(response.json()) + + return response_401 + + if response.status_code == 403: + response_403 = ErrorResponse.from_dict(response.json()) + + return response_403 + + if response.status_code == 404: + response_404 = ErrorResponse.from_dict(response.json()) + + return response_404 + + if response.status_code == 409: + response_409 = ErrorResponse.from_dict(response.json()) + + return response_409 + + if response.status_code == 422: + response_422 = ExecuteResponse.from_dict(response.json()) + + return response_422 + + if response.status_code == 429: + response_429 = ErrorResponse.from_dict(response.json()) + + return response_429 + + if response.status_code == 500: + response_500 = ErrorResponse.from_dict(response.json()) + + return response_500 + + if client.raise_on_unexpected_status: + raise errors.UnexpectedStatus(response.status_code, response.content) + else: + return None + + +def _build_response( + *, client: AuthenticatedClient | Client, response: httpx.Response +) -> Response[ErrorResponse | ExecuteResponse]: + return Response( + status_code=HTTPStatus(response.status_code), + content=response.content, + headers=response.headers, + parsed=_parse_response(client=client, response=response), + ) + + +def sync_detailed( + org_name: str, + api_name: str, + *, + client: AuthenticatedClient, + body: ExecuteRequest | Unset = UNSET, +) -> Response[ErrorResponse | ExecuteResponse]: + """Execute an API deployment against one or more documents. + + Supply the documents either as `files` (multipart upload) or as `presigned_urls` (HTTPS S3 URLs), or + both — a request carrying neither is rejected, and the two together may not exceed 32 documents. + + With the default `timeout` of -1 the call returns as soon as the execution is queued; read the + outcome from the status endpoint. + + Args: + org_name (str): + api_name (str): + body (ExecuteRequest | Unset): The documents to run, and the options that shape the + result. + + Supply `files`, `presigned_urls`, or both. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[ErrorResponse | ExecuteResponse] + """ + + kwargs = _get_kwargs( + org_name=org_name, + api_name=api_name, + body=body, + ) + + response = client.get_httpx_client().request( + **kwargs, + ) + + return _build_response(client=client, response=response) + + +def sync( + org_name: str, + api_name: str, + *, + client: AuthenticatedClient, + body: ExecuteRequest | Unset = UNSET, +) -> ErrorResponse | ExecuteResponse | None: + """Execute an API deployment against one or more documents. + + Supply the documents either as `files` (multipart upload) or as `presigned_urls` (HTTPS S3 URLs), or + both — a request carrying neither is rejected, and the two together may not exceed 32 documents. + + With the default `timeout` of -1 the call returns as soon as the execution is queued; read the + outcome from the status endpoint. + + Args: + org_name (str): + api_name (str): + body (ExecuteRequest | Unset): The documents to run, and the options that shape the + result. + + Supply `files`, `presigned_urls`, or both. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + ErrorResponse | ExecuteResponse + """ + + return sync_detailed( + org_name=org_name, + api_name=api_name, + client=client, + body=body, + ).parsed + + +async def asyncio_detailed( + org_name: str, + api_name: str, + *, + client: AuthenticatedClient, + body: ExecuteRequest | Unset = UNSET, +) -> Response[ErrorResponse | ExecuteResponse]: + """Execute an API deployment against one or more documents. + + Supply the documents either as `files` (multipart upload) or as `presigned_urls` (HTTPS S3 URLs), or + both — a request carrying neither is rejected, and the two together may not exceed 32 documents. + + With the default `timeout` of -1 the call returns as soon as the execution is queued; read the + outcome from the status endpoint. + + Args: + org_name (str): + api_name (str): + body (ExecuteRequest | Unset): The documents to run, and the options that shape the + result. + + Supply `files`, `presigned_urls`, or both. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[ErrorResponse | ExecuteResponse] + """ + + kwargs = _get_kwargs( + org_name=org_name, + api_name=api_name, + body=body, + ) + + response = await client.get_async_httpx_client().request(**kwargs) + + return _build_response(client=client, response=response) + + +async def asyncio( + org_name: str, + api_name: str, + *, + client: AuthenticatedClient, + body: ExecuteRequest | Unset = UNSET, +) -> ErrorResponse | ExecuteResponse | None: + """Execute an API deployment against one or more documents. + + Supply the documents either as `files` (multipart upload) or as `presigned_urls` (HTTPS S3 URLs), or + both — a request carrying neither is rejected, and the two together may not exceed 32 documents. + + With the default `timeout` of -1 the call returns as soon as the execution is queued; read the + outcome from the status endpoint. + + Args: + org_name (str): + api_name (str): + body (ExecuteRequest | Unset): The documents to run, and the options that shape the + result. + + Supply `files`, `presigned_urls`, or both. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + ErrorResponse | ExecuteResponse + """ + + return ( + await asyncio_detailed( + org_name=org_name, + api_name=api_name, + client=client, + body=body, + ) + ).parsed diff --git a/src/unstract/api_deployments/sdk_docstudio/api/deployment/status.py b/src/unstract/api_deployments/sdk_docstudio/api/deployment/status.py new file mode 100644 index 0000000..dfe9cea --- /dev/null +++ b/src/unstract/api_deployments/sdk_docstudio/api/deployment/status.py @@ -0,0 +1,293 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +from http import HTTPStatus +from typing import Any +from urllib.parse import quote + +import httpx + +from ... import errors +from ...client import AuthenticatedClient, Client +from ...models.error_response import ErrorResponse +from ...models.status_response import StatusResponse +from ...types import UNSET, Response, Unset + + +def _get_kwargs( + org_name: str, + api_name: str, + *, + execution_id: str, + include_extracted_text: bool | Unset = False, + include_metadata: bool | Unset = False, + include_metrics: bool | Unset = False, +) -> dict[str, Any]: + + params: dict[str, Any] = {} + + params["execution_id"] = execution_id + + params["include_extracted_text"] = include_extracted_text + + params["include_metadata"] = include_metadata + + params["include_metrics"] = include_metrics + + params = {k: v for k, v in params.items() if v is not UNSET and v is not None} + + _kwargs: dict[str, Any] = { + "method": "get", + "url": "/deployment/api/{org_name}/{api_name}/".format( + org_name=quote(str(org_name), safe=""), + api_name=quote(str(api_name), safe=""), + ), + "params": params, + } + + return _kwargs + + +def _parse_response( + *, client: AuthenticatedClient | Client, response: httpx.Response +) -> ErrorResponse | StatusResponse | None: + if response.status_code == 200: + response_200 = StatusResponse.from_dict(response.json()) + + return response_200 + + if response.status_code == 400: + response_400 = ErrorResponse.from_dict(response.json()) + + return response_400 + + if response.status_code == 401: + response_401 = ErrorResponse.from_dict(response.json()) + + return response_401 + + if response.status_code == 403: + response_403 = ErrorResponse.from_dict(response.json()) + + return response_403 + + if response.status_code == 404: + response_404 = ErrorResponse.from_dict(response.json()) + + return response_404 + + if response.status_code == 406: + response_406 = ErrorResponse.from_dict(response.json()) + + return response_406 + + if response.status_code == 422: + response_422 = StatusResponse.from_dict(response.json()) + + return response_422 + + if response.status_code == 429: + response_429 = ErrorResponse.from_dict(response.json()) + + return response_429 + + if response.status_code == 500: + response_500 = ErrorResponse.from_dict(response.json()) + + return response_500 + + if client.raise_on_unexpected_status: + raise errors.UnexpectedStatus(response.status_code, response.content) + else: + return None + + +def _build_response( + *, client: AuthenticatedClient | Client, response: httpx.Response +) -> Response[ErrorResponse | StatusResponse]: + return Response( + status_code=HTTPStatus(response.status_code), + content=response.content, + headers=response.headers, + parsed=_parse_response(client=client, response=response), + ) + + +def sync_detailed( + org_name: str, + api_name: str, + *, + client: AuthenticatedClient, + execution_id: str, + include_extracted_text: bool | Unset = False, + include_metadata: bool | Unset = False, + include_metrics: bool | Unset = False, +) -> Response[ErrorResponse | StatusResponse]: + """Read the result of a previously started execution. + + This read is one-shot: the first call that observes a completed execution acknowledges it and the + stored result is discarded, so every later call for that execution answers 406. Poll while the + execution is pending, and keep the payload of the call that returns it — it cannot be fetched again. + + Args: + org_name (str): + api_name (str): + execution_id (str): + include_extracted_text (bool | Unset): Default: False. + include_metadata (bool | Unset): Default: False. + include_metrics (bool | Unset): Default: False. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[ErrorResponse | StatusResponse] + """ + + kwargs = _get_kwargs( + org_name=org_name, + api_name=api_name, + execution_id=execution_id, + include_extracted_text=include_extracted_text, + include_metadata=include_metadata, + include_metrics=include_metrics, + ) + + response = client.get_httpx_client().request( + **kwargs, + ) + + return _build_response(client=client, response=response) + + +def sync( + org_name: str, + api_name: str, + *, + client: AuthenticatedClient, + execution_id: str, + include_extracted_text: bool | Unset = False, + include_metadata: bool | Unset = False, + include_metrics: bool | Unset = False, +) -> ErrorResponse | StatusResponse | None: + """Read the result of a previously started execution. + + This read is one-shot: the first call that observes a completed execution acknowledges it and the + stored result is discarded, so every later call for that execution answers 406. Poll while the + execution is pending, and keep the payload of the call that returns it — it cannot be fetched again. + + Args: + org_name (str): + api_name (str): + execution_id (str): + include_extracted_text (bool | Unset): Default: False. + include_metadata (bool | Unset): Default: False. + include_metrics (bool | Unset): Default: False. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + ErrorResponse | StatusResponse + """ + + return sync_detailed( + org_name=org_name, + api_name=api_name, + client=client, + execution_id=execution_id, + include_extracted_text=include_extracted_text, + include_metadata=include_metadata, + include_metrics=include_metrics, + ).parsed + + +async def asyncio_detailed( + org_name: str, + api_name: str, + *, + client: AuthenticatedClient, + execution_id: str, + include_extracted_text: bool | Unset = False, + include_metadata: bool | Unset = False, + include_metrics: bool | Unset = False, +) -> Response[ErrorResponse | StatusResponse]: + """Read the result of a previously started execution. + + This read is one-shot: the first call that observes a completed execution acknowledges it and the + stored result is discarded, so every later call for that execution answers 406. Poll while the + execution is pending, and keep the payload of the call that returns it — it cannot be fetched again. + + Args: + org_name (str): + api_name (str): + execution_id (str): + include_extracted_text (bool | Unset): Default: False. + include_metadata (bool | Unset): Default: False. + include_metrics (bool | Unset): Default: False. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + Response[ErrorResponse | StatusResponse] + """ + + kwargs = _get_kwargs( + org_name=org_name, + api_name=api_name, + execution_id=execution_id, + include_extracted_text=include_extracted_text, + include_metadata=include_metadata, + include_metrics=include_metrics, + ) + + response = await client.get_async_httpx_client().request(**kwargs) + + return _build_response(client=client, response=response) + + +async def asyncio( + org_name: str, + api_name: str, + *, + client: AuthenticatedClient, + execution_id: str, + include_extracted_text: bool | Unset = False, + include_metadata: bool | Unset = False, + include_metrics: bool | Unset = False, +) -> ErrorResponse | StatusResponse | None: + """Read the result of a previously started execution. + + This read is one-shot: the first call that observes a completed execution acknowledges it and the + stored result is discarded, so every later call for that execution answers 406. Poll while the + execution is pending, and keep the payload of the call that returns it — it cannot be fetched again. + + Args: + org_name (str): + api_name (str): + execution_id (str): + include_extracted_text (bool | Unset): Default: False. + include_metadata (bool | Unset): Default: False. + include_metrics (bool | Unset): Default: False. + + Raises: + errors.UnexpectedStatus: If the server returns an undocumented status code and Client.raise_on_unexpected_status is True. + httpx.TimeoutException: If the request takes longer than Client.timeout. + + Returns: + ErrorResponse | StatusResponse + """ + + return ( + await asyncio_detailed( + org_name=org_name, + api_name=api_name, + client=client, + execution_id=execution_id, + include_extracted_text=include_extracted_text, + include_metadata=include_metadata, + include_metrics=include_metrics, + ) + ).parsed diff --git a/src/unstract/api_deployments/sdk_docstudio/client.py b/src/unstract/api_deployments/sdk_docstudio/client.py new file mode 100644 index 0000000..5b50486 --- /dev/null +++ b/src/unstract/api_deployments/sdk_docstudio/client.py @@ -0,0 +1,283 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +import ssl +from typing import Any + +import httpx +from attrs import define, evolve, field + + +@define +class Client: + """A class for keeping track of data related to the API + + The following are accepted as keyword arguments and will be used to construct httpx Clients internally: + + ``base_url``: The base URL for the API, all requests are made to a relative path to this URL + + ``cookies``: A dictionary of cookies to be sent with every request + + ``headers``: A dictionary of headers to be sent with every request + + ``timeout``: The maximum amount of a time a request can take. API functions will raise + httpx.TimeoutException if this is exceeded. + + ``verify_ssl``: Whether or not to verify the SSL certificate of the API server. This should be True in production, + but can be set to False for testing purposes. + + ``follow_redirects``: Whether or not to follow redirects. Default value is False. + + ``httpx_args``: A dictionary of additional arguments to be passed to the ``httpx.Client`` and ``httpx.AsyncClient`` constructor. + + + Attributes: + raise_on_unexpected_status: Whether or not to raise an errors.UnexpectedStatus if the API returns a + status code that was not documented in the source OpenAPI document. Can also be provided as a keyword + argument to the constructor. + """ + + raise_on_unexpected_status: bool = field(default=False, kw_only=True) + _base_url: str = field(alias="base_url") + _cookies: dict[str, str] = field(factory=dict, kw_only=True, alias="cookies") + _headers: dict[str, str] = field(factory=dict, kw_only=True, alias="headers") + _timeout: httpx.Timeout | None = field(default=None, kw_only=True, alias="timeout") + _verify_ssl: str | bool | ssl.SSLContext = field( + default=True, kw_only=True, alias="verify_ssl" + ) + _follow_redirects: bool = field( + default=False, kw_only=True, alias="follow_redirects" + ) + _httpx_args: dict[str, Any] = field(factory=dict, kw_only=True, alias="httpx_args") + _client: httpx.Client | None = field(default=None, init=False) + _async_client: httpx.AsyncClient | None = field(default=None, init=False) + + def with_headers(self, headers: dict[str, str]) -> "Client": + """Get a new client matching this one with additional headers""" + if self._client is not None: + self._client.headers.update(headers) + if self._async_client is not None: + self._async_client.headers.update(headers) + return evolve(self, headers={**self._headers, **headers}) + + def with_cookies(self, cookies: dict[str, str]) -> "Client": + """Get a new client matching this one with additional cookies""" + if self._client is not None: + self._client.cookies.update(cookies) + if self._async_client is not None: + self._async_client.cookies.update(cookies) + return evolve(self, cookies={**self._cookies, **cookies}) + + def with_timeout(self, timeout: httpx.Timeout) -> "Client": + """Get a new client matching this one with a new timeout configuration""" + if self._client is not None: + self._client.timeout = timeout + if self._async_client is not None: + self._async_client.timeout = timeout + return evolve(self, timeout=timeout) + + def set_httpx_client(self, client: httpx.Client) -> "Client": + """Manually set the underlying httpx.Client + + **NOTE**: This will override any other settings on the client, including cookies, headers, and timeout. + """ + self._client = client + return self + + def get_httpx_client(self) -> httpx.Client: + """Get the underlying httpx.Client, constructing a new one if not previously set""" + if self._client is None: + self._client = httpx.Client( + base_url=self._base_url, + cookies=self._cookies, + headers=self._headers, + timeout=self._timeout, + verify=self._verify_ssl, + follow_redirects=self._follow_redirects, + **self._httpx_args, + ) + return self._client + + def __enter__(self) -> "Client": + """Enter a context manager for self.client—you cannot enter twice (see httpx docs)""" + self.get_httpx_client().__enter__() + return self + + def __exit__(self, *args: Any, **kwargs: Any) -> None: + """Exit a context manager for internal httpx.Client (see httpx docs)""" + self.get_httpx_client().__exit__(*args, **kwargs) + + def set_async_httpx_client(self, async_client: httpx.AsyncClient) -> "Client": + """Manually set the underlying httpx.AsyncClient + + **NOTE**: This will override any other settings on the client, including cookies, headers, and timeout. + """ + self._async_client = async_client + return self + + def get_async_httpx_client(self) -> httpx.AsyncClient: + """Get the underlying httpx.AsyncClient, constructing a new one if not previously set""" + if self._async_client is None: + self._async_client = httpx.AsyncClient( + base_url=self._base_url, + cookies=self._cookies, + headers=self._headers, + timeout=self._timeout, + verify=self._verify_ssl, + follow_redirects=self._follow_redirects, + **self._httpx_args, + ) + return self._async_client + + async def __aenter__(self) -> "Client": + """Enter a context manager for underlying httpx.AsyncClient—you cannot enter twice (see httpx docs)""" + await self.get_async_httpx_client().__aenter__() + return self + + async def __aexit__(self, *args: Any, **kwargs: Any) -> None: + """Exit a context manager for underlying httpx.AsyncClient (see httpx docs)""" + await self.get_async_httpx_client().__aexit__(*args, **kwargs) + + +@define +class AuthenticatedClient: + """A Client which has been authenticated for use on secured endpoints + + The following are accepted as keyword arguments and will be used to construct httpx Clients internally: + + ``base_url``: The base URL for the API, all requests are made to a relative path to this URL + + ``cookies``: A dictionary of cookies to be sent with every request + + ``headers``: A dictionary of headers to be sent with every request + + ``timeout``: The maximum amount of a time a request can take. API functions will raise + httpx.TimeoutException if this is exceeded. + + ``verify_ssl``: Whether or not to verify the SSL certificate of the API server. This should be True in production, + but can be set to False for testing purposes. + + ``follow_redirects``: Whether or not to follow redirects. Default value is False. + + ``httpx_args``: A dictionary of additional arguments to be passed to the ``httpx.Client`` and ``httpx.AsyncClient`` constructor. + + + Attributes: + raise_on_unexpected_status: Whether or not to raise an errors.UnexpectedStatus if the API returns a + status code that was not documented in the source OpenAPI document. Can also be provided as a keyword + argument to the constructor. + token: The token to use for authentication + prefix: The prefix to use for the Authorization header + auth_header_name: The name of the Authorization header + """ + + raise_on_unexpected_status: bool = field(default=False, kw_only=True) + _base_url: str = field(alias="base_url") + _cookies: dict[str, str] = field(factory=dict, kw_only=True, alias="cookies") + _headers: dict[str, str] = field(factory=dict, kw_only=True, alias="headers") + _timeout: httpx.Timeout | None = field(default=None, kw_only=True, alias="timeout") + _verify_ssl: str | bool | ssl.SSLContext = field( + default=True, kw_only=True, alias="verify_ssl" + ) + _follow_redirects: bool = field( + default=False, kw_only=True, alias="follow_redirects" + ) + _httpx_args: dict[str, Any] = field(factory=dict, kw_only=True, alias="httpx_args") + _client: httpx.Client | None = field(default=None, init=False) + _async_client: httpx.AsyncClient | None = field(default=None, init=False) + + token: str + prefix: str = "Bearer" + auth_header_name: str = "Authorization" + + def with_headers(self, headers: dict[str, str]) -> "AuthenticatedClient": + """Get a new client matching this one with additional headers""" + if self._client is not None: + self._client.headers.update(headers) + if self._async_client is not None: + self._async_client.headers.update(headers) + return evolve(self, headers={**self._headers, **headers}) + + def with_cookies(self, cookies: dict[str, str]) -> "AuthenticatedClient": + """Get a new client matching this one with additional cookies""" + if self._client is not None: + self._client.cookies.update(cookies) + if self._async_client is not None: + self._async_client.cookies.update(cookies) + return evolve(self, cookies={**self._cookies, **cookies}) + + def with_timeout(self, timeout: httpx.Timeout) -> "AuthenticatedClient": + """Get a new client matching this one with a new timeout configuration""" + if self._client is not None: + self._client.timeout = timeout + if self._async_client is not None: + self._async_client.timeout = timeout + return evolve(self, timeout=timeout) + + def set_httpx_client(self, client: httpx.Client) -> "AuthenticatedClient": + """Manually set the underlying httpx.Client + + **NOTE**: This will override any other settings on the client, including cookies, headers, and timeout. + """ + self._client = client + return self + + def get_httpx_client(self) -> httpx.Client: + """Get the underlying httpx.Client, constructing a new one if not previously set""" + if self._client is None: + self._headers[self.auth_header_name] = ( + f"{self.prefix} {self.token}" if self.prefix else self.token + ) + self._client = httpx.Client( + base_url=self._base_url, + cookies=self._cookies, + headers=self._headers, + timeout=self._timeout, + verify=self._verify_ssl, + follow_redirects=self._follow_redirects, + **self._httpx_args, + ) + return self._client + + def __enter__(self) -> "AuthenticatedClient": + """Enter a context manager for self.client—you cannot enter twice (see httpx docs)""" + self.get_httpx_client().__enter__() + return self + + def __exit__(self, *args: Any, **kwargs: Any) -> None: + """Exit a context manager for internal httpx.Client (see httpx docs)""" + self.get_httpx_client().__exit__(*args, **kwargs) + + def set_async_httpx_client( + self, async_client: httpx.AsyncClient + ) -> "AuthenticatedClient": + """Manually set the underlying httpx.AsyncClient + + **NOTE**: This will override any other settings on the client, including cookies, headers, and timeout. + """ + self._async_client = async_client + return self + + def get_async_httpx_client(self) -> httpx.AsyncClient: + """Get the underlying httpx.AsyncClient, constructing a new one if not previously set""" + if self._async_client is None: + self._headers[self.auth_header_name] = ( + f"{self.prefix} {self.token}" if self.prefix else self.token + ) + self._async_client = httpx.AsyncClient( + base_url=self._base_url, + cookies=self._cookies, + headers=self._headers, + timeout=self._timeout, + verify=self._verify_ssl, + follow_redirects=self._follow_redirects, + **self._httpx_args, + ) + return self._async_client + + async def __aenter__(self) -> "AuthenticatedClient": + """Enter a context manager for underlying httpx.AsyncClient—you cannot enter twice (see httpx docs)""" + await self.get_async_httpx_client().__aenter__() + return self + + async def __aexit__(self, *args: Any, **kwargs: Any) -> None: + """Exit a context manager for underlying httpx.AsyncClient (see httpx docs)""" + await self.get_async_httpx_client().__aexit__(*args, **kwargs) diff --git a/src/unstract/api_deployments/sdk_docstudio/errors.py b/src/unstract/api_deployments/sdk_docstudio/errors.py new file mode 100644 index 0000000..0d1fb6b --- /dev/null +++ b/src/unstract/api_deployments/sdk_docstudio/errors.py @@ -0,0 +1,17 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +"""Contains shared errors types that can be raised from API functions""" + + +class UnexpectedStatus(Exception): + """Raised by api functions when the response status an undocumented status and Client.raise_on_unexpected_status is True""" + + def __init__(self, status_code: int, content: bytes): + self.status_code = status_code + self.content = content + + super().__init__( + f"Unexpected status code: {status_code}\n\nResponse content:\n{content.decode(errors='ignore')}" + ) + + +__all__ = ["UnexpectedStatus"] diff --git a/src/unstract/api_deployments/sdk_docstudio/models/__init__.py b/src/unstract/api_deployments/sdk_docstudio/models/__init__.py new file mode 100644 index 0000000..c9a9dfd --- /dev/null +++ b/src/unstract/api_deployments/sdk_docstudio/models/__init__.py @@ -0,0 +1,18 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +"""Contains all the data models used in inputs/outputs""" + +from .error_response import ErrorResponse +from .execute_request import ExecuteRequest +from .execute_response import ExecuteResponse +from .execution_message import ExecutionMessage +from .file_result import FileResult +from .status_response import StatusResponse + +__all__ = ( + "ErrorResponse", + "ExecuteRequest", + "ExecuteResponse", + "ExecutionMessage", + "FileResult", + "StatusResponse", +) diff --git a/src/unstract/api_deployments/sdk_docstudio/models/error_response.py b/src/unstract/api_deployments/sdk_docstudio/models/error_response.py new file mode 100644 index 0000000..6286b50 --- /dev/null +++ b/src/unstract/api_deployments/sdk_docstudio/models/error_response.py @@ -0,0 +1,71 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..types import UNSET, Unset + +T = TypeVar("T", bound="ErrorResponse") + + +@_attrs_define +class ErrorResponse: + """ + Attributes: + message (Any | Unset): + status (str | Unset): + """ + + message: Any | Unset = UNSET + status: str | Unset = UNSET + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + message = self.message + + status = self.status + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({}) + if message is not UNSET: + field_dict["message"] = message + if status is not UNSET: + field_dict["status"] = status + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + message = d.pop("message", UNSET) + + status = d.pop("status", UNSET) + + error_response = cls( + message=message, + status=status, + ) + + error_response.additional_properties = d + return error_response + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/src/unstract/api_deployments/sdk_docstudio/models/execute_request.py b/src/unstract/api_deployments/sdk_docstudio/models/execute_request.py new file mode 100644 index 0000000..e581f59 --- /dev/null +++ b/src/unstract/api_deployments/sdk_docstudio/models/execute_request.py @@ -0,0 +1,326 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +from __future__ import annotations + +from collections.abc import Mapping +from io import BytesIO +from typing import Any, TypeVar, cast + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from .. import types +from ..types import UNSET, File, FileTypes, Unset + +T = TypeVar("T", bound="ExecuteRequest") + + +@_attrs_define +class ExecuteRequest: + """The documents to run, and the options that shape the result. + + Supply `files`, `presigned_urls`, or both. + + Attributes: + custom_data (Any | Unset): + files (list[File] | Unset): + hitl_packet_id (None | str | Unset): + hitl_queue_name (None | str | Unset): + include_extracted_text (bool | Unset): Default: False. + include_metadata (bool | Unset): Default: False. + include_metrics (bool | Unset): Default: False. + llm_profile_id (None | str | Unset): + presigned_urls (list[str] | Unset): + tags (str | Unset): Comma-separated list of tag names (EX:'tag1,tag2-name,tag3_name') Default: ''. + timeout (int | Unset): Default: -1. + use_file_history (bool | Unset): Default: False. + """ + + custom_data: Any | Unset = UNSET + files: list[File] | Unset = UNSET + hitl_packet_id: None | str | Unset = UNSET + hitl_queue_name: None | str | Unset = UNSET + include_extracted_text: bool | Unset = False + include_metadata: bool | Unset = False + include_metrics: bool | Unset = False + llm_profile_id: None | str | Unset = UNSET + presigned_urls: list[str] | Unset = UNSET + tags: str | Unset = "" + timeout: int | Unset = -1 + use_file_history: bool | Unset = False + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + custom_data = self.custom_data + + files: list[FileTypes] | Unset = UNSET + if not isinstance(self.files, Unset): + files = [] + for files_item_data in self.files: + files_item = files_item_data.to_tuple() + + files.append(files_item) + + hitl_packet_id: None | str | Unset + if isinstance(self.hitl_packet_id, Unset): + hitl_packet_id = UNSET + else: + hitl_packet_id = self.hitl_packet_id + + hitl_queue_name: None | str | Unset + if isinstance(self.hitl_queue_name, Unset): + hitl_queue_name = UNSET + else: + hitl_queue_name = self.hitl_queue_name + + include_extracted_text = self.include_extracted_text + + include_metadata = self.include_metadata + + include_metrics = self.include_metrics + + llm_profile_id: None | str | Unset + if isinstance(self.llm_profile_id, Unset): + llm_profile_id = UNSET + else: + llm_profile_id = self.llm_profile_id + + presigned_urls: list[str] | Unset = UNSET + if not isinstance(self.presigned_urls, Unset): + presigned_urls = self.presigned_urls + + tags = self.tags + + timeout = self.timeout + + use_file_history = self.use_file_history + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update({}) + if custom_data is not UNSET: + field_dict["custom_data"] = custom_data + if files is not UNSET: + field_dict["files"] = files + if hitl_packet_id is not UNSET: + field_dict["hitl_packet_id"] = hitl_packet_id + if hitl_queue_name is not UNSET: + field_dict["hitl_queue_name"] = hitl_queue_name + if include_extracted_text is not UNSET: + field_dict["include_extracted_text"] = include_extracted_text + if include_metadata is not UNSET: + field_dict["include_metadata"] = include_metadata + if include_metrics is not UNSET: + field_dict["include_metrics"] = include_metrics + if llm_profile_id is not UNSET: + field_dict["llm_profile_id"] = llm_profile_id + if presigned_urls is not UNSET: + field_dict["presigned_urls"] = presigned_urls + if tags is not UNSET: + field_dict["tags"] = tags + if timeout is not UNSET: + field_dict["timeout"] = timeout + if use_file_history is not UNSET: + field_dict["use_file_history"] = use_file_history + + return field_dict + + def to_multipart(self) -> types.RequestFiles: + files: types.RequestFiles = [] + + if not isinstance(self.custom_data, Unset): + files.append( + ("custom_data", (None, str(self.custom_data).encode(), "text/plain")) + ) + + if not isinstance(self.files, Unset): + for files_item_element in self.files: + files.append(("files", files_item_element.to_tuple())) + + if not isinstance(self.hitl_packet_id, Unset): + if isinstance(self.hitl_packet_id, str): + files.append( + ( + "hitl_packet_id", + (None, str(self.hitl_packet_id).encode(), "text/plain"), + ) + ) + else: + files.append( + ( + "hitl_packet_id", + (None, str(self.hitl_packet_id).encode(), "text/plain"), + ) + ) + + if not isinstance(self.hitl_queue_name, Unset): + if isinstance(self.hitl_queue_name, str): + files.append( + ( + "hitl_queue_name", + (None, str(self.hitl_queue_name).encode(), "text/plain"), + ) + ) + else: + files.append( + ( + "hitl_queue_name", + (None, str(self.hitl_queue_name).encode(), "text/plain"), + ) + ) + + if not isinstance(self.include_extracted_text, Unset): + files.append( + ( + "include_extracted_text", + (None, str(self.include_extracted_text).encode(), "text/plain"), + ) + ) + + if not isinstance(self.include_metadata, Unset): + files.append( + ( + "include_metadata", + (None, str(self.include_metadata).encode(), "text/plain"), + ) + ) + + if not isinstance(self.include_metrics, Unset): + files.append( + ( + "include_metrics", + (None, str(self.include_metrics).encode(), "text/plain"), + ) + ) + + if not isinstance(self.llm_profile_id, Unset): + if isinstance(self.llm_profile_id, str): + files.append( + ( + "llm_profile_id", + (None, str(self.llm_profile_id).encode(), "text/plain"), + ) + ) + else: + files.append( + ( + "llm_profile_id", + (None, str(self.llm_profile_id).encode(), "text/plain"), + ) + ) + + if not isinstance(self.presigned_urls, Unset): + for presigned_urls_item_element in self.presigned_urls: + files.append( + ( + "presigned_urls", + (None, str(presigned_urls_item_element).encode(), "text/plain"), + ) + ) + + if not isinstance(self.tags, Unset): + files.append(("tags", (None, str(self.tags).encode(), "text/plain"))) + + if not isinstance(self.timeout, Unset): + files.append(("timeout", (None, str(self.timeout).encode(), "text/plain"))) + + if not isinstance(self.use_file_history, Unset): + files.append( + ( + "use_file_history", + (None, str(self.use_file_history).encode(), "text/plain"), + ) + ) + + for prop_name, prop in self.additional_properties.items(): + files.append((prop_name, (None, str(prop).encode(), "text/plain"))) + + return files + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + custom_data = d.pop("custom_data", UNSET) + + _files = d.pop("files", UNSET) + files: list[File] | Unset = UNSET + if _files is not UNSET: + files = [] + for files_item_data in _files: + files_item = File(payload=BytesIO(files_item_data)) + + files.append(files_item) + + def _parse_hitl_packet_id(data: object) -> None | str | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(None | str | Unset, data) + + hitl_packet_id = _parse_hitl_packet_id(d.pop("hitl_packet_id", UNSET)) + + def _parse_hitl_queue_name(data: object) -> None | str | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(None | str | Unset, data) + + hitl_queue_name = _parse_hitl_queue_name(d.pop("hitl_queue_name", UNSET)) + + include_extracted_text = d.pop("include_extracted_text", UNSET) + + include_metadata = d.pop("include_metadata", UNSET) + + include_metrics = d.pop("include_metrics", UNSET) + + def _parse_llm_profile_id(data: object) -> None | str | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(None | str | Unset, data) + + llm_profile_id = _parse_llm_profile_id(d.pop("llm_profile_id", UNSET)) + + presigned_urls = cast(list[str], d.pop("presigned_urls", UNSET)) + + tags = d.pop("tags", UNSET) + + timeout = d.pop("timeout", UNSET) + + use_file_history = d.pop("use_file_history", UNSET) + + execute_request = cls( + custom_data=custom_data, + files=files, + hitl_packet_id=hitl_packet_id, + hitl_queue_name=hitl_queue_name, + include_extracted_text=include_extracted_text, + include_metadata=include_metadata, + include_metrics=include_metrics, + llm_profile_id=llm_profile_id, + presigned_urls=presigned_urls, + tags=tags, + timeout=timeout, + use_file_history=use_file_history, + ) + + execute_request.additional_properties = d + return execute_request + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/src/unstract/api_deployments/sdk_docstudio/models/execute_response.py b/src/unstract/api_deployments/sdk_docstudio/models/execute_response.py new file mode 100644 index 0000000..035eb5a --- /dev/null +++ b/src/unstract/api_deployments/sdk_docstudio/models/execute_response.py @@ -0,0 +1,69 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, TypeVar + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +if TYPE_CHECKING: + from ..models.execution_message import ExecutionMessage + + +T = TypeVar("T", bound="ExecuteResponse") + + +@_attrs_define +class ExecuteResponse: + """ + Attributes: + message (ExecutionMessage): The execution's identity and, once it has finished, its per-file + results. + """ + + message: ExecutionMessage + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + message = self.message.to_dict() + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "message": message, + } + ) + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + from ..models.execution_message import ExecutionMessage + + d = dict(src_dict) + message = ExecutionMessage.from_dict(d.pop("message")) + + execute_response = cls( + message=message, + ) + + execute_response.additional_properties = d + return execute_response + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/src/unstract/api_deployments/sdk_docstudio/models/execution_message.py b/src/unstract/api_deployments/sdk_docstudio/models/execution_message.py new file mode 100644 index 0000000..2de8be7 --- /dev/null +++ b/src/unstract/api_deployments/sdk_docstudio/models/execution_message.py @@ -0,0 +1,135 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, TypeVar, cast + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..types import UNSET, Unset + +if TYPE_CHECKING: + from ..models.file_result import FileResult + + +T = TypeVar("T", bound="ExecutionMessage") + + +@_attrs_define +class ExecutionMessage: + """The execution's identity and, once it has finished, its per-file + results. + + Attributes: + error (str): + execution_id (str): + execution_status (str): + status_api (str): + result (list[FileResult] | None | Unset): + """ + + error: str + execution_id: str + execution_status: str + status_api: str + result: list[FileResult] | None | Unset = UNSET + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + error = self.error + + execution_id = self.execution_id + + execution_status = self.execution_status + + status_api = self.status_api + + result: list[dict[str, Any]] | None | Unset + if isinstance(self.result, Unset): + result = UNSET + elif isinstance(self.result, list): + result = [] + for result_type_0_item_data in self.result: + result_type_0_item = result_type_0_item_data.to_dict() + result.append(result_type_0_item) + + else: + result = self.result + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "error": error, + "execution_id": execution_id, + "execution_status": execution_status, + "status_api": status_api, + } + ) + if result is not UNSET: + field_dict["result"] = result + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + from ..models.file_result import FileResult + + d = dict(src_dict) + error = d.pop("error") + + execution_id = d.pop("execution_id") + + execution_status = d.pop("execution_status") + + status_api = d.pop("status_api") + + def _parse_result(data: object) -> list[FileResult] | None | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + try: + if not isinstance(data, list): + raise TypeError() + result_type_0 = [] + _result_type_0 = data + for result_type_0_item_data in _result_type_0: + result_type_0_item = FileResult.from_dict(result_type_0_item_data) + + result_type_0.append(result_type_0_item) + + return result_type_0 + except (TypeError, ValueError, AttributeError, KeyError): + pass + return cast(list[FileResult] | None | Unset, data) + + result = _parse_result(d.pop("result", UNSET)) + + execution_message = cls( + error=error, + execution_id=execution_id, + execution_status=execution_status, + status_api=status_api, + result=result, + ) + + execution_message.additional_properties = d + return execution_message + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/src/unstract/api_deployments/sdk_docstudio/models/file_result.py b/src/unstract/api_deployments/sdk_docstudio/models/file_result.py new file mode 100644 index 0000000..7559a51 --- /dev/null +++ b/src/unstract/api_deployments/sdk_docstudio/models/file_result.py @@ -0,0 +1,129 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar, cast + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..types import UNSET, Unset + +T = TypeVar("T", bound="FileResult") + + +@_attrs_define +class FileResult: + """ + Attributes: + file (str): + error (None | str | Unset): + file_execution_id (str | Unset): + metadata (Any | Unset): + metrics (Any | Unset): + result (Any | Unset): + status (str | Unset): + """ + + file: str + error: None | str | Unset = UNSET + file_execution_id: str | Unset = UNSET + metadata: Any | Unset = UNSET + metrics: Any | Unset = UNSET + result: Any | Unset = UNSET + status: str | Unset = UNSET + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + file = self.file + + error: None | str | Unset + if isinstance(self.error, Unset): + error = UNSET + else: + error = self.error + + file_execution_id = self.file_execution_id + + metadata = self.metadata + + metrics = self.metrics + + result = self.result + + status = self.status + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "file": file, + } + ) + if error is not UNSET: + field_dict["error"] = error + if file_execution_id is not UNSET: + field_dict["file_execution_id"] = file_execution_id + if metadata is not UNSET: + field_dict["metadata"] = metadata + if metrics is not UNSET: + field_dict["metrics"] = metrics + if result is not UNSET: + field_dict["result"] = result + if status is not UNSET: + field_dict["status"] = status + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + d = dict(src_dict) + file = d.pop("file") + + def _parse_error(data: object) -> None | str | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(None | str | Unset, data) + + error = _parse_error(d.pop("error", UNSET)) + + file_execution_id = d.pop("file_execution_id", UNSET) + + metadata = d.pop("metadata", UNSET) + + metrics = d.pop("metrics", UNSET) + + result = d.pop("result", UNSET) + + status = d.pop("status", UNSET) + + file_result = cls( + file=file, + error=error, + file_execution_id=file_execution_id, + metadata=metadata, + metrics=metrics, + result=result, + status=status, + ) + + file_result.additional_properties = d + return file_result + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/src/unstract/api_deployments/sdk_docstudio/models/status_response.py b/src/unstract/api_deployments/sdk_docstudio/models/status_response.py new file mode 100644 index 0000000..8073788 --- /dev/null +++ b/src/unstract/api_deployments/sdk_docstudio/models/status_response.py @@ -0,0 +1,109 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, TypeVar, cast + +from attrs import define as _attrs_define +from attrs import field as _attrs_field + +from ..types import UNSET, Unset + +if TYPE_CHECKING: + from ..models.file_result import FileResult + + +T = TypeVar("T", bound="StatusResponse") + + +@_attrs_define +class StatusResponse: + """ + Attributes: + status (str): + message (list[FileResult] | None | Unset): + """ + + status: str + message: list[FileResult] | None | Unset = UNSET + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + status = self.status + + message: list[dict[str, Any]] | None | Unset + if isinstance(self.message, Unset): + message = UNSET + elif isinstance(self.message, list): + message = [] + for message_type_0_item_data in self.message: + message_type_0_item = message_type_0_item_data.to_dict() + message.append(message_type_0_item) + + else: + message = self.message + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "status": status, + } + ) + if message is not UNSET: + field_dict["message"] = message + + return field_dict + + @classmethod + def from_dict(cls: type[T], src_dict: Mapping[str, Any]) -> T: + from ..models.file_result import FileResult + + d = dict(src_dict) + status = d.pop("status") + + def _parse_message(data: object) -> list[FileResult] | None | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + try: + if not isinstance(data, list): + raise TypeError() + message_type_0 = [] + _message_type_0 = data + for message_type_0_item_data in _message_type_0: + message_type_0_item = FileResult.from_dict(message_type_0_item_data) + + message_type_0.append(message_type_0_item) + + return message_type_0 + except (TypeError, ValueError, AttributeError, KeyError): + pass + return cast(list[FileResult] | None | Unset, data) + + message = _parse_message(d.pop("message", UNSET)) + + status_response = cls( + status=status, + message=message, + ) + + status_response.additional_properties = d + return status_response + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/src/unstract/api_deployments/sdk_docstudio/types.py b/src/unstract/api_deployments/sdk_docstudio/types.py new file mode 100644 index 0000000..3ca05bc --- /dev/null +++ b/src/unstract/api_deployments/sdk_docstudio/types.py @@ -0,0 +1,55 @@ +# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT. +"""Contains some shared types for properties""" + +from collections.abc import Mapping, MutableMapping +from http import HTTPStatus +from typing import IO, BinaryIO, Generic, Literal, TypeVar + +from attrs import define + + +class Unset: + def __bool__(self) -> Literal[False]: + return False + + +UNSET: Unset = Unset() + +# The types that `httpx.Client(files=)` can accept, copied from that library. +FileContent = IO[bytes] | bytes | str +FileTypes = ( + # (filename, file (or bytes), content_type) + tuple[str | None, FileContent, str | None] + # (filename, file (or bytes), content_type, headers) + | tuple[str | None, FileContent, str | None, Mapping[str, str]] +) +RequestFiles = list[tuple[str, FileTypes]] + + +@define +class File: + """Contains information for file uploads""" + + payload: BinaryIO + file_name: str | None = None + mime_type: str | None = None + + def to_tuple(self) -> FileTypes: + """Return a tuple representation that httpx will accept for multipart/form-data""" + return self.file_name, self.payload, self.mime_type + + +T = TypeVar("T") + + +@define +class Response(Generic[T]): + """A response from an endpoint""" + + status_code: HTTPStatus + content: bytes + headers: MutableMapping[str, str] + parsed: T | None + + +__all__ = ["UNSET", "File", "FileTypes", "RequestFiles", "Response", "Unset"] diff --git a/src/unstract/cli.py b/src/unstract/cli.py deleted file mode 100644 index 9634111..0000000 --- a/src/unstract/cli.py +++ /dev/null @@ -1,30 +0,0 @@ -"""Top-level ``unstract`` command group. - -Subcommands live in their own subpackages and are registered here so a -single console script (``unstract``) fronts all of them. ``unstract.clone`` -keeps its own group + ``main`` so ``python -m unstract.clone`` still works. -""" - -from __future__ import annotations - -from typing import Any - -import click - -from unstract.clone.cli import clone_cmd - - -@click.group(name="unstract") -def cli() -> None: - """Unstract command-line tools.""" - - -cli.add_command(clone_cmd, name="clone") - - -def main(argv: list[str] | None = None) -> Any: - return cli(args=argv, standalone_mode=True) - - -if __name__ == "__main__": - main() diff --git a/src/unstract/clone/cli.py b/src/unstract/clone/cli.py index 43f3a09..0406640 100644 --- a/src/unstract/clone/cli.py +++ b/src/unstract/clone/cli.py @@ -1,8 +1,7 @@ """Click-based CLI for ``unstract.clone``. -Single ``clone`` command, registered on the top-level ``unstract`` group -(``unstract.cli``) — the canonical invocation is ``unstract clone``. The -local group here only backs ``python -m unstract.clone``. +Single ``clone`` command, invoked as ``python -m unstract.clone``. The +``unstract`` CLI wraps the orchestrator directly rather than this module. Platform keys can be passed via flags (``--source-key`` / ``--target-key``) or env vars (``UNSTRACT_SRC_PLATFORM_KEY`` / ``UNSTRACT_TGT_PLATFORM_KEY``) diff --git a/tests/baseline/client_1_5_3.py b/tests/baseline/client_1_5_3.py new file mode 100644 index 0000000..dd76c91 --- /dev/null +++ b/tests/baseline/client_1_5_3.py @@ -0,0 +1,475 @@ +# Vendored from the released unstract-client 1.5.3 wheel on PyPI. DO NOT EDIT. +# Refresh with tools/refresh_baseline.sh when the parity baseline is intentionally moved. +"""This module provides an API client to invoke APIs deployed on the Unstract +platform. + +Classes: + APIDeploymentsClient: A class to invoke APIs deployed on the Unstract platform. + APIDeploymentsClientException: A class to handle exceptions raised by the + APIDeploymentsClient class. +""" + +import logging +import ntpath +import os +import time +from urllib.parse import urlparse + +import requests +from requests.exceptions import ConnectionError, JSONDecodeError, Timeout +from tenacity import ( + RetryCallState, + Retrying, + retry_if_exception_type, + retry_if_result, + stop_after_attempt, + wait_exponential_jitter, +) +from tenacity.wait import wait_base + +from unstract.api_deployments.utils import UnstractUtils + + +class APIDeploymentsClientException(Exception): + """A class to handle exceptions raised by the APIClient class.""" + + def __init__(self, message): + def __init__(self, value): + self.value = value + + def __str__(self): + return repr(self.value) + + def error_message(self): + return self.value + + +class _WaitRetryAfterOrExponentialJitter(wait_base): + """Wait strategy that respects Retry-After on 429, else exponential jitter. + + For 429 responses with a valid ``Retry-After`` header the server-requested + delay is used. In every other case the strategy delegates to + ``wait_exponential_jitter`` (additive jitter). + """ + + def __init__( + self, + initial: float, + max: float, + exp_base: float, + jitter: float, + ) -> None: + super().__init__() + self._exp_jitter = wait_exponential_jitter( + initial=initial, max=max, exp_base=exp_base, jitter=jitter + ) + + def __call__(self, retry_state: RetryCallState) -> float: + outcome = retry_state.outcome + if outcome and not outcome.failed: + response = outcome.result() + if response is not None and getattr(response, "status_code", None) == 429: + retry_after = response.headers.get("Retry-After") + if retry_after is not None: + try: + return float(retry_after) + except (ValueError, TypeError): + pass + return self._exp_jitter(retry_state) + + +class APIDeploymentsClient: + """A class to invoke APIs deployed on the Unstract platform.""" + + formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + ) + logger = logging.getLogger(__name__) + log_stream_handler = logging.StreamHandler() + log_stream_handler.setFormatter(formatter) + logger.addHandler(log_stream_handler) + + api_key = "" + api_timeout = 300 + in_progress_statuses = ["PENDING", "EXECUTING", "READY", "QUEUED", "INITIATED"] + + def __init__( + self, + api_url: str, + api_key: str, + api_timeout: int = 300, + logging_level: str = "INFO", + include_metadata: bool = False, + verify: bool = True, + max_retries: int = 4, + initial_delay: float = 2.0, + max_delay: float = 60.0, + backoff_factor: float = 2.0, + jitter: float = 1.0, + ): + """Initializes the APIClient class. + + Args: + api_key (str): The API key to authenticate the API request. + api_timeout (int): The timeout to wait for the API response. + logging_level (str): The logging level to log messages. + max_retries (int): Maximum number of retry attempts for failed requests. + initial_delay (float): Initial delay in seconds before the first retry. + max_delay (float): Maximum delay in seconds between retries. + backoff_factor (float): Multiplier applied to delay for each retry. + jitter (float): Maximum additive jitter in seconds added to each delay. + """ + if logging_level == "": + logging_level = os.getenv("UNSTRACT_API_CLIENT_LOGGING_LEVEL", "INFO") + if logging_level == "DEBUG": + self.logger.setLevel(logging.DEBUG) + elif logging_level == "INFO": + self.logger.setLevel(logging.INFO) + elif logging_level == "WARNING": + self.logger.setLevel(logging.WARNING) + elif logging_level == "ERROR": + self.logger.setLevel(logging.ERROR) + + # self.logger.setLevel(logging_level) + self.logger.debug("Logging level set to: " + logging_level) + + if api_key == "": + self.api_key = os.getenv("UNSTRACT_API_DEPLOYMENT_KEY", "") + else: + self.api_key = api_key + self.logger.debug("API key set to: " + UnstractUtils.redact_key(self.api_key)) + + self.api_timeout = api_timeout + self.api_url = api_url + self.__save_base_url(api_url) + self.include_metadata = include_metadata + self.verify = verify + self.max_retries = max_retries + self.initial_delay = initial_delay + self.max_delay = max_delay + self.backoff_factor = backoff_factor + self.jitter = jitter + + def _is_retryable_status(self, status_code: int) -> bool: + """Checks whether a status code should trigger a retry. + + Args: + status_code (int): The HTTP status code to check. + + Returns: + bool: True if the request should be retried. + """ + return status_code >= 500 or status_code == 429 + + def __save_base_url(self, full_url: str): + """Extracts the base URL from the full URL and saves it. + + Args: + full_url (str): The full URL of the API. + """ + parsed_url = urlparse(full_url) + self.base_url = parsed_url.scheme + "://" + parsed_url.netloc + self.logger.debug("Base URL: " + self.base_url) + + @staticmethod + def _rewind_files(files): + """Rewinds file objects so they can be re-sent on retry.""" + for file_tuple in files: + file_obj = file_tuple[1] + if hasattr(file_obj, "seek"): + file_obj.seek(0) + elif isinstance(file_obj, tuple) and len(file_obj) >= 2: + if hasattr(file_obj[1], "seek"): + file_obj[1].seek(0) + + def _request_with_retry(self, method: str, url: str, **kwargs) -> requests.Response: + """Makes an HTTP request with exponential backoff retry logic. + + Uses ``tenacity`` with additive jitter and Retry-After support. + + Args: + method (str): The HTTP method (e.g., "GET", "POST"). + url (str): The request URL. + **kwargs: Additional keyword arguments passed to requests.request(). + + Returns: + requests.Response: The response from the request. + + Raises: + ConnectionError: If a connection error persists after all retries. + Timeout: If a timeout persists after all retries. + """ + files = kwargs.get("files") + + def _before_sleep(retry_state: RetryCallState): + attempt = retry_state.attempt_number + delay = retry_state.next_action.sleep + outcome = retry_state.outcome + if outcome.failed: + exc = outcome.exception() + self.logger.warning( + "%s during request to %s. Retrying in %.1fs (attempt %d/%d).", + type(exc).__name__, + url, + delay, + attempt, + self.max_retries, + ) + else: + response = outcome.result() + self.logger.warning( + "Request to %s returned %d. Retrying in %.1fs (attempt %d/%d).", + url, + response.status_code, + delay, + attempt, + self.max_retries, + ) + # Rewind file objects before next attempt + if files: + self._rewind_files(files) + + def _retry_error_callback(retry_state: RetryCallState): + outcome = retry_state.outcome + if outcome.failed: + exc = outcome.exception() + self.logger.warning( + "%s during request to %s. Retries exhausted (%d/%d).", + type(exc).__name__, + url, + self.max_retries, + self.max_retries, + ) + raise exc + response = outcome.result() + self.logger.warning( + "Request to %s returned %d. Retries exhausted (%d/%d).", + url, + response.status_code, + self.max_retries, + self.max_retries, + ) + return response + + retrier = Retrying( + stop=stop_after_attempt(self.max_retries + 1), + wait=_WaitRetryAfterOrExponentialJitter( + initial=self.initial_delay, + max=self.max_delay, + exp_base=self.backoff_factor, + jitter=self.jitter, + ), + retry=( + retry_if_result(lambda r: self._is_retryable_status(r.status_code)) + | retry_if_exception_type((ConnectionError, Timeout)) + ), + before_sleep=_before_sleep, + retry_error_callback=_retry_error_callback, + sleep=time.sleep, + reraise=False, + ) + + return retrier(requests.request, method, url, **kwargs) + + def structure_file(self, file_paths: list[str]) -> dict: + """Invokes the API deployed on the Unstract platform. + + Args: + file_paths (list[str]): The file path to the file to be uploaded. + + Returns: + dict: The response from the API. + """ + self.logger.debug("Invoking API: " + self.api_url) + self.logger.debug("File paths: " + str(file_paths)) + + headers = { + "Authorization": "Bearer " + self.api_key, + } + + form_data = { + "timeout": self.api_timeout, + "include_metadata": self.include_metadata, + } + + files = [] + + try: + for file_path in file_paths: + record = ( + "files", + ( + ntpath.basename(file_path), + open(file_path, "rb"), + "application/octet-stream", + ), + ) + files.append(record) + except FileNotFoundError as e: + raise APIDeploymentsClientException("File not found: " + str(e)) + + if self.api_timeout == 0: + # Async mode: server returns immediately after queuing. + # A 5xx means queuing failed — safe to retry. + response = self._request_with_retry( + "POST", + self.api_url, + headers=headers, + data=form_data, + files=files, + verify=self.verify, + ) + else: + # Sync mode: server blocks during processing. + # A 5xx may mean it processed but response was lost — don't retry + # to avoid duplicate executions. + response = requests.post( + self.api_url, + headers=headers, + data=form_data, + files=files, + verify=self.verify, + ) + self.logger.debug(response.status_code) + self.logger.debug(response.text) + # The returned object is wrapped in a "message" key. + # Let's simplify the response. + obj_to_return = {} + + try: + response_data = response.json() + response_message = response_data.get("message", {}) + except JSONDecodeError: + self.logger.error( + "Failed to decode JSON response. Raw response: %s", + response.text, + exc_info=True, + ) + obj_to_return = { + "status_code": response.status_code, + "pending": False, + "execution_status": "", + "error": "Invalid JSON response from API", + "extraction_result": "", + } + return obj_to_return + if response.status_code == 401: + obj_to_return = { + "status_code": response.status_code, + "pending": False, + "execution_status": "", + "error": response_data.get("errors", [{}])[0].get( + "detail", "Unauthorized" + ), + "extraction_result": "", + } + return obj_to_return + + # If the execution status is pending, extract the execution ID from + # the response and return it in the response. + # Later, users can use the execution ID to check the status of the execution. + # The returned object is wrapped in a "message" key. + # Let's simplify the response. + # Construct response object + execution_status = response_message.get("execution_status", "") + error_message = response_message.get("error", "") + extraction_result = response_message.get("result", "") + status_api_endpoint = response_message.get("status_api") + + obj_to_return = { + "status_code": response.status_code, + "pending": False, + "execution_status": execution_status, + "error": error_message, + "extraction_result": extraction_result, + } + + # Check if the status is pending or if it's successful but lacks a result. + # The POST endpoint returns 200 for successful queuing (including + # PENDING/EXECUTING) and 422 only on setup errors — guard against + # incorrectly polling after an error response. + if 200 <= response.status_code < 300: + if execution_status in self.in_progress_statuses or ( + execution_status == "SUCCESS" and not extraction_result + ): + obj_to_return.update( + { + "status_check_api_endpoint": status_api_endpoint, + "pending": True, + } + ) + + return obj_to_return + + def check_execution_status(self, status_check_api_endpoint: str) -> dict: + """Checks the status of the execution. + + Args: + status_check_api_endpoint (str): + The API endpoint to check the status of the execution. + + Returns: + dict: The response from the API. + """ + + headers = { + "Authorization": "Bearer " + self.api_key, + } + status_call_url = self.base_url + status_check_api_endpoint + self.logger.debug("Checking execution status via endpoint: " + status_call_url) + response = self._request_with_retry( + "GET", + status_call_url, + headers=headers, + params={"include_metadata": self.include_metadata}, + verify=self.verify, + ) + self.logger.debug(response.status_code) + self.logger.debug(response.text) + + obj_to_return = {} + + try: + response_data = response.json() + except JSONDecodeError: + self.logger.error( + "Failed to decode JSON response. Raw response: %s", + response.text, + exc_info=True, + ) + obj_to_return = { + "status_code": response.status_code, + "pending": False, + "execution_status": "", + "error": "Invalid JSON response from API", + "extraction_result": "", + } + return obj_to_return + + # Construct response object + execution_status = response_data.get("status", "") + error_message = response_data.get("error", "") + extraction_result = response_data.get("message", "") + + obj_to_return = { + "status_code": response.status_code, + "pending": False, + "execution_status": execution_status, + "error": error_message, + "extraction_result": extraction_result, + } + + # If the execution status is pending, extract the execution ID from the response + # and return it in the response. + # Later, users can use the execution ID to check the status of the execution. + if obj_to_return["execution_status"] in self.in_progress_statuses: + obj_to_return["pending"] = True + elif self._is_retryable_status(response.status_code): + obj_to_return["pending"] = True + self.logger.warning( + "Status check returned %d after retries; " + "marking as pending to continue polling.", + response.status_code, + ) + + return obj_to_return diff --git a/tests/test_cli_top_level.py b/tests/test_cli_top_level.py deleted file mode 100644 index e55d26d..0000000 --- a/tests/test_cli_top_level.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Tests for the top-level ``unstract`` command group (``unstract.cli``).""" - -from __future__ import annotations - -from click.testing import CliRunner - -from unstract.cli import cli -from unstract.clone.report import CloneReport, Endpoint - - -def test_clone_invocation_via_top_level_group(monkeypatch): - captured: dict = {} - - def fake_clone(source, target, options=None): - captured["source"] = source - captured["target"] = target - return CloneReport( - source=Endpoint( - base_url=source.base_url, organization_id=source.organization_id - ), - target=Endpoint( - base_url=target.base_url, organization_id=target.organization_id - ), - ) - - # The clone command's callback resolves run_clone from unstract.clone.cli. - monkeypatch.setattr("unstract.clone.cli.run_clone", fake_clone) - - result = CliRunner().invoke( - cli, - [ - "clone", - "--source-url", - "http://src", - "--source-org", - "src", - "--source-key", - "sk", - "--target-url", - "http://tgt", - "--target-org", - "tgt", - "--target-key", - "tk", - ], - ) - - assert result.exit_code == 0, result.output - assert captured["source"].organization_id == "src" - assert captured["target"].organization_id == "tgt" diff --git a/tests/test_compat.py b/tests/test_compat.py new file mode 100644 index 0000000..34f8782 --- /dev/null +++ b/tests/test_compat.py @@ -0,0 +1,1190 @@ +"""Parity tests against the last released client. + +The transport underneath ``APIDeploymentsClient`` changed; its published +behaviour must not. These tests pin the seams where that could silently break: +the constructor and method signatures, what goes out on the wire, which +exceptions come back out, and the exact dict each method returns — the last one +by running the released client side by side over the same responses. +""" + +import ast +import hashlib +import importlib.util +import inspect +import io +import json +import re +import socket +import threading +from pathlib import Path +from unittest.mock import MagicMock, patch +from urllib.parse import parse_qs, urlparse + +import httpx +import pytest +import requests +from requests.exceptions import ( + ConnectionError, + ConnectTimeout, + ContentDecodingError, + MissingSchema, + ProxyError, + ReadTimeout, + RequestException, + Timeout, + TooManyRedirects, +) + +from unstract.api_deployments.client import ( + _EXECUTE_SEND_ONLY, + _STATUS_SEND_ONLY, + APIDeploymentsClient, + APIDeploymentsClientException, +) + +BASELINE_VERSION = "1.5.3" +BASELINE_PATH = Path(__file__).parent / "baseline" / "client_1_5_3.py" +BASELINE_SHA256 = "45201bb0de000e8f3a0e65f40cb0b08fec389514f7a17c8bb3410a3dc59229df" +SPEC_PATH = Path(__file__).parents[1] / "specs" / "docstudio-oss.json" + +API_URL = "https://api.example.com/deployment/api/testorg/testapi/" +STATUS_ENDPOINT = "/deployment/api/testorg/testapi/?execution_id=exec-123" + +#: Operations the facade wraps. The spec declares exactly these, and a new one +#: has to be added here deliberately rather than arriving unnoticed. +WRAPPED_OPERATIONS = frozenset({"execute", "status"}) + + +def _load_baseline(): + """Import the vendored released client under its own module name.""" + spec = importlib.util.spec_from_file_location("baseline_client", BASELINE_PATH) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +baseline = _load_baseline() + + +@pytest.fixture +def sample_file(tmp_path): + path = tmp_path / "sample.txt" + path.write_bytes(b"hello") + return str(path) + + +def _client(**kwargs): + kwargs.setdefault("api_url", API_URL) + kwargs.setdefault("api_key", "test-key") + kwargs.setdefault("logging_level", "ERROR") + kwargs.setdefault("max_retries", 0) + return APIDeploymentsClient(**kwargs) + + +def _baseline_client(**kwargs): + kwargs.setdefault("api_url", API_URL) + kwargs.setdefault("api_key", "test-key") + kwargs.setdefault("logging_level", "ERROR") + kwargs.setdefault("max_retries", 0) + return baseline.APIDeploymentsClient(**kwargs) + + +def _httpx_response(status_code=200, json_data=None, text=None): + if text is not None: + return httpx.Response(status_code, text=text) + return httpx.Response(status_code, json=json_data) + + +def _requests_response(status_code=200, json_data=None, text=None): + response = MagicMock() + response.status_code = status_code + if text is not None: + response.text = text + response.json.side_effect = requests.exceptions.JSONDecodeError( + "no json", text, 0 + ) + else: + response.text = json.dumps(json_data) + response.json.return_value = json_data + response.headers = {} + return response + + +# -------------------------------------------------------------------------- +# Transport error translation +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("raised", "expected"), + [ + (httpx.ConnectTimeout("connect timed out"), ConnectTimeout), + (httpx.ReadTimeout("read timed out"), ReadTimeout), + # Neither had a Timeout equivalent: a send that failed and a pool that + # could not hand out a connection both surfaced as ConnectionError. + (httpx.WriteTimeout("write timed out"), ConnectionError), + (httpx.PoolTimeout("pool timed out"), ConnectionError), + (httpx.ConnectError("refused"), ConnectionError), + (httpx.ReadError("reset"), ConnectionError), + (httpx.WriteError("broken pipe"), ConnectionError), + (httpx.ProtocolError("bad framing"), ConnectionError), + (httpx.ProxyError("proxy exploded"), ProxyError), + (httpx.UnsupportedProtocol("no scheme"), MissingSchema), + (httpx.TooManyRedirects("looping"), TooManyRedirects), + (httpx.DecodingError("bad gzip"), ContentDecodingError), + ], +) +def test_transport_errors_are_translated(raised, expected): + """Callers catch the ``requests`` classes; httpx's are not subclasses. + + ``ConnectTimeout`` is the case that makes ordering load-bearing, and it is + also both a ``ConnectionError`` and a ``Timeout`` — the plain ``Timeout`` + that httpx's hierarchy implies would stop matching half the callers. The + exact class matters too: a caller catching ``ReadTimeout`` sees nothing if + a broader ``Timeout`` is raised in its place. + """ + client = _client() + with patch.object( + client._transport.get_httpx_client(), "request", side_effect=raised + ): + with pytest.raises(expected) as caught: + client._send("get", "/anything") + assert type(caught.value) is expected + + +@pytest.mark.parametrize( + ("api_url", "expected"), + [ + ("::::", APIDeploymentsClientException), + ("/deployment/api/testorg/testapi/", MissingSchema), + # Parity: the released client raised this one too. + ("http://[bad", ValueError), + ], +) +def test_a_malformed_deployment_url_raises_the_class_this_client_chose( + api_url, expected +): + """A deliberate divergence, pinned here so it stays deliberate. + + The released client answered the first two with ``InvalidSchema``, which said + nothing about which URL was wrong. This is a configuration typo that never + reaches the wire, so the clearer class is worth the difference -- but a + caller wrapping construction in ``except RequestException`` no longer catches + the first, which is the part that has to be visible. + """ + with pytest.raises(expected) as caught: + client = _client(api_url=api_url) + client.check_execution_status(STATUS_ENDPOINT) + # MissingSchema is itself a ValueError, so the parity row can only tell the + # two apart by the exact class. + assert type(caught.value) is expected + + +def _httpx_request_errors(): + """Every httpx request failure, discovered rather than listed. + + A hand-written list is exactly as complete as it was the day it was + written; this one grows when httpx does. + """ + found, stack = [], [httpx.RequestError] + while stack: + cls = stack.pop() + found.append(cls) + stack.extend(cls.__subclasses__()) + return sorted(found, key=lambda cls: cls.__name__) + + +@pytest.mark.parametrize("cls", _httpx_request_errors(), ids=lambda cls: cls.__name__) +def test_no_httpx_failure_escapes_untranslated(cls): + """An httpx class reaching a caller is a class no caller catches.""" + client = _client() + with patch.object( + client._transport.get_httpx_client(), "request", side_effect=cls("boom") + ): + with pytest.raises(RequestException): + client._send("get", "/anything") + + +@pytest.mark.parametrize( + ("raised", "retried"), + [ + (httpx.PoolTimeout("pool timed out"), True), + (httpx.ProxyError("proxy exploded"), True), + # Retrying these cannot start working: the URL stays malformed, the + # redirect chain stays a loop, the body stays undecodable. + (httpx.UnsupportedProtocol("no scheme"), False), + (httpx.TooManyRedirects("looping"), False), + (httpx.DecodingError("bad gzip"), False), + ], +) +def test_translation_decides_what_gets_retried(raised, retried): + client = _client(max_retries=2, initial_delay=0, max_delay=0, jitter=0) + with patch.object( + client._transport.get_httpx_client(), "request", side_effect=raised + ) as request: + with pytest.raises(RequestException): + client._request_with_retry("get", "/anything") + assert (request.call_count > 1) is retried + + +def test_a_connect_timeout_is_still_a_connection_error(): + client = _client() + with patch.object( + client._transport.get_httpx_client(), + "request", + side_effect=httpx.ConnectTimeout("connect timed out"), + ): + with pytest.raises(ConnectionError): + client._send("get", "/anything") + with patch.object( + client._transport.get_httpx_client(), + "request", + side_effect=httpx.ConnectTimeout("connect timed out"), + ): + with pytest.raises(Timeout): + client._send("get", "/anything") + + +def test_translated_errors_keep_the_original_cause(): + client = _client() + original = httpx.ConnectError("refused") + with patch.object( + client._transport.get_httpx_client(), "request", side_effect=original + ): + with pytest.raises(ConnectionError) as excinfo: + client._send("get", "/anything") + assert excinfo.value.__cause__ is original + + +def test_structure_file_raises_translated_error(sample_file): + client = _client() + with patch.object( + client._transport.get_httpx_client(), + "request", + side_effect=httpx.ConnectError("refused"), + ): + with pytest.raises(ConnectionError): + client.structure_file([sample_file]) + + +def test_check_execution_status_raises_translated_error(): + client = _client() + with patch.object( + client._transport.get_httpx_client(), + "request", + side_effect=httpx.ReadTimeout("read timed out"), + ): + with pytest.raises(Timeout): + client.check_execution_status(STATUS_ENDPOINT) + + +def test_translation_happens_inside_the_retried_call(sample_file): + """Retry counts the transport failures, which requires translation first. + + ``_request_with_retry`` retries on the ``requests`` exception types. If the + httpx exception escaped the retried callable untranslated it would never + match, and transport-error retry would quietly stop working. + """ + client = _client(api_timeout=0, max_retries=2, initial_delay=0.001, max_delay=0.002) + with patch.object( + client._transport.get_httpx_client(), + "request", + side_effect=httpx.ConnectError("refused"), + ) as mock_request: + with pytest.raises(ConnectionError): + client.structure_file([sample_file]) + assert mock_request.call_count == 3 + + +# -------------------------------------------------------------------------- +# api_timeout is an execution mode, never a transport timeout +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("api_timeout", [-1, 0, 1, 300]) +def test_api_timeout_never_configures_the_transport(api_timeout): + """``api_timeout`` selects a backend execution mode. + + ``-1``/``0`` mean async; + handing either to the transport fails inside the connection layer. + """ + client = _client(api_timeout=api_timeout) + assert client._transport.get_httpx_client().timeout == httpx.Timeout(None) + + +@pytest.mark.parametrize("api_timeout", [-1, 0, 300]) +def test_api_timeout_never_reaches_the_transport_call(sample_file, api_timeout): + client = _client(api_timeout=api_timeout) + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(200, {"message": {}}) + client.structure_file([sample_file]) + _, kwargs = mock_send.call_args + assert "timeout" not in kwargs + + +# -------------------------------------------------------------------------- +# What goes out on the wire +# -------------------------------------------------------------------------- + + +def _captured_execute_kwargs(client, file_path, **request_params): + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(200, {"message": {}}) + client.structure_file([file_path], **request_params) + return mock_send.call_args + + +def _execute_parts(client, file_path, **request_params): + _, kwargs = _captured_execute_kwargs(client, file_path, **request_params) + return {name: value for name, value in kwargs["files"]} + + +def test_execute_sends_only_the_fields_the_client_sets(sample_file): + """A spec default written into the request pins a value the server would + otherwise choose, and the two diverge the moment the server's default + moves.""" + _, kwargs = _captured_execute_kwargs(_client(api_timeout=300), sample_file) + assert {name for name, _ in kwargs["files"]} == { + "files", + "include_metadata", + "timeout", + } + assert _EXECUTE_SEND_ONLY == { + "files", + "include_metadata", + "timeout", + "additional_properties", + } + + +def test_execute_multipart_values_match_the_released_client(sample_file): + _, kwargs = _captured_execute_kwargs( + _client(api_timeout=300, include_metadata=True), sample_file + ) + parts = {name: value for name, value in kwargs["files"]} + assert parts["timeout"][1] == b"300" + assert parts["include_metadata"][1] == b"True" + assert parts["files"][0] == "sample.txt" + assert parts["files"][2] == "application/octet-stream" + + +# -------------------------------------------------------------------------- +# Request parameters, added as keyword-only arguments +# -------------------------------------------------------------------------- + + +def _request_param_names(): + return [ + name + for name, p in inspect.signature( + APIDeploymentsClient.structure_file + ).parameters.items() + if p.kind is p.KEYWORD_ONLY + ] + + +def test_request_parameters_are_named_as_the_spec_names_them(): + """A rename here would need a translation table in every caller.""" + spec = json.loads(SPEC_PATH.read_text()) + declared = set(spec["components"]["schemas"]["ExecuteRequest"]["properties"]) + # ``files`` is built from ``file_paths``, not passed through. + assert set(_request_param_names()) == declared - {"files"} + + +def test_request_parameters_are_keyword_only(sample_file): + with pytest.raises(TypeError): + _client().structure_file([sample_file], 300) + + +def test_an_unset_parameter_is_not_sent(sample_file): + """Sending a default pins a value the server would otherwise choose.""" + parts = _execute_parts(_client(api_timeout=300), sample_file) + assert set(parts) == {"files", "include_metadata", "timeout"} + + +def test_a_requested_parameter_is_sent(sample_file): + parts = _execute_parts( + _client(api_timeout=300), + sample_file, + tags="a,b", + llm_profile_id="profile-1", + use_file_history=True, + ) + assert parts["tags"][1] == b"a,b" + assert parts["llm_profile_id"][1] == b"profile-1" + assert parts["use_file_history"][1] == b"True" + + +@pytest.mark.parametrize( + ("param", "value", "expected"), + [ + ("timeout", 0, b"0"), + ("include_metrics", False, b"False"), + ("include_extracted_text", False, b"False"), + ("tags", "", b""), + ], +) +def test_a_falsy_parameter_is_still_sent(sample_file, param, value, expected): + """``False``/``0``/``""`` are choices, not absences; a truthiness filter + eats them and silently hands the decision back to the server.""" + parts = _execute_parts(_client(api_timeout=300), sample_file, **{param: value}) + assert parts[param][1] == expected + + +def test_a_requested_parameter_overrides_the_constructor(sample_file): + parts = _execute_parts( + _client(api_timeout=300, include_metadata=False), + sample_file, + timeout=-1, + include_metadata=True, + ) + assert parts["timeout"][1] == b"-1" + assert parts["include_metadata"][1] == b"True" + + +def test_a_requested_timeout_selects_the_execution_mode(sample_file): + """``timeout`` is an execution mode: ``0`` queues, so a 5xx is safe to + retry. + + Passing it per request has to move that decision with it. + """ + with patch.object(APIDeploymentsClient, "_request_with_retry") as retried: + with patch.object(APIDeploymentsClient, "_send") as sent: + retried.return_value = sent.return_value = _httpx_response( + 200, {"message": {}} + ) + _client(api_timeout=300).structure_file([sample_file], timeout=0) + assert retried.called and not sent.called + + retried.reset_mock() + _client(api_timeout=0).structure_file([sample_file], timeout=300) + assert sent.called and not retried.called + + +def test_multipart_boundary_is_random_and_matches_the_body(sample_file): + """The generated builder pins ``boundary=+++`` in the header. A PDF + containing those bytes would corrupt the encoding, so the header is dropped + and the transport picks the boundary — as the released client did. + + Encoding happens inside the send, while the file handles are still + open. + """ + encoded = [] + + def encode(method, url, **kwargs): + assert "Content-Type" not in kwargs.get("headers", {}) + transport = httpx.Client(base_url="https://api.example.com") + request = transport.build_request(method, url, **kwargs) + encoded.append((request.headers["content-type"], request.read())) + return _httpx_response(200, {"message": {}}) + + with patch.object(APIDeploymentsClient, "_send", side_effect=encode): + _client().structure_file([sample_file]) + _client().structure_file([sample_file]) + + boundaries = [] + for content_type, body in encoded: + header_boundary = content_type.split("boundary=")[1] + assert body.split(b"\r\n")[0] == b"--" + header_boundary.encode() + assert header_boundary != "+++" + boundaries.append(header_boundary) + assert boundaries[0] != boundaries[1] + + +def _captured_status_params(client=None, endpoint=STATUS_ENDPOINT, **request_params): + client = client or _client() + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(200, {"status": "COMPLETED"}) + client.check_execution_status(endpoint, **request_params) + return mock_send.call_args[1]["params"] + + +def test_status_sends_only_the_fields_the_client_sets(): + assert set(_captured_status_params()) == {"execution_id", "include_metadata"} + + +@pytest.mark.parametrize( + ("api_url", "expected"), + [ + (API_URL, "https://api.example.com/deployment/api/testorg/testapi/"), + ( + "https://api.example.com/unstract/deployment/api/testorg/testapi/", + "https://api.example.com/unstract/deployment/api/testorg/testapi/", + ), + ], +) +def test_status_is_polled_under_the_deployment_urls_own_prefix(api_url, expected): + """A poll that misses is a paid execution whose result is never collected.""" + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(200, {"status": "COMPLETED"}) + _client(api_url=api_url).check_execution_status(STATUS_ENDPOINT) + assert mock_send.call_args[0][1] == expected + + +def test_the_status_endpoints_own_query_parameters_are_forwarded(): + """The endpoint is the service's instruction for reaching this execution. + + A region hint or a signature dropped from it polls somewhere the execution + is not, and the execution has already been paid for. + """ + params = _captured_status_params( + client=_client(), + endpoint=STATUS_ENDPOINT + "®ion=eu&sig=abc123", + ) + assert params["region"] == "eu" + assert params["sig"] == "abc123" + assert params["execution_id"] == "exec-123" + + +def test_a_deployment_url_without_the_spec_route_polls_the_endpoint_as_returned(): + """No prefix can be derived from a rewritten URL, and a guessed path polls + nothing.""" + client = _client(api_url="https://gw.example.com/v1/testorg/testapi/") + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(200, {"status": "COMPLETED"}) + client.check_execution_status("/v1/testorg/testapi/?execution_id=exec-123") + assert mock_send.call_args[0][1] == "https://gw.example.com/v1/testorg/testapi/" + + +def test_status_request_parameters_are_named_as_the_spec_names_them(): + """A rename here would need a translation table in every caller.""" + spec = json.loads(SPEC_PATH.read_text()) + execute = "/deployment/api/{org_name}/{api_name}/" + declared = { + p["name"] + for p in spec["paths"][execute]["get"]["parameters"] + if p["in"] == "query" + } + accepted = { + name + for name, p in inspect.signature( + APIDeploymentsClient.check_execution_status + ).parameters.items() + if p.kind is p.KEYWORD_ONLY + } + # `execution_id` is read out of the endpoint URL the server handed back. + assert accepted == declared - {"execution_id"} + + +def test_status_request_parameters_are_keyword_only(): + with pytest.raises(TypeError): + _client().check_execution_status(STATUS_ENDPOINT, True) + + +def test_a_requested_status_parameter_is_sent(): + params = _captured_status_params(include_metrics=True, include_extracted_text=False) + assert params["include_metrics"] == "True" + # False is a choice; a truthiness filter would drop it and hand the decision + # back to the server. + assert params["include_extracted_text"] == "False" + + +def test_a_requested_status_parameter_overrides_the_constructor(): + params = _captured_status_params( + _client(include_metadata=False), include_metadata=True + ) + assert params["include_metadata"] == "True" + assert _STATUS_SEND_ONLY == {"execution_id", "include_metadata"} + + +def test_status_url_matches_the_released_client(): + """The status URL is rebuilt from the spec route plus the execution id + instead of concatenating the server-supplied path. + + Same request either way, which is what this pins. + """ + client = _client(include_metadata=True) + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(200, {"status": "COMPLETED"}) + client.check_execution_status(STATUS_ENDPOINT) + args, kwargs = mock_send.call_args + # Encoded by the transport that will send it, rather than stringified here: + # httpx renders a bool as `true` where urlencoding one gives `True`, and + # normalising both sides is how a comparison stops seeing the difference. + sent = httpx.URL(client.base_url).join(args[1]).copy_merge_params(kwargs["params"]) + ours = urlparse(str(sent)) + ours_query = parse_qs(ours.query) + + published = urlparse(client.base_url + STATUS_ENDPOINT) + published_query = { + **parse_qs(published.query), + "include_metadata": [str(client.include_metadata)], + } + + assert args[0].lower() == "get" + assert (ours.scheme, ours.netloc, ours.path) == ( + published.scheme, + published.netloc, + published.path, + ) + assert ours_query == published_query + + +@pytest.mark.parametrize( + "endpoint", + [ + # What the server actually returns, and the only spelling the released + # client handled: it concatenated base URL and endpoint, so an absolute + # one produced `https://hosthttps://host/...` and a relative one with + # no leading slash produced `https://hostdeployment/...`. + "/deployment/api/testorg/testapi/?execution_id=exec-123", + "https://api.example.com/deployment/api/testorg/testapi/?execution_id=exec-123", + "deployment/api/testorg/testapi/?execution_id=exec-123", + ], +) +def test_the_status_endpoint_is_read_not_concatenated(endpoint): + """Only the execution id is taken from the server's endpoint; the route + comes from the spec. Every spelling therefore resolves to one request.""" + client = _client() + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(200, {"status": "COMPLETED"}) + client.check_execution_status(endpoint) + args, kwargs = mock_send.call_args + assert args[0].lower() == "get" + assert str(httpx.URL(client.base_url).join(args[1])) == API_URL + assert kwargs["params"]["execution_id"] == "exec-123" + + +@pytest.mark.parametrize( + "endpoint", + [ + "https://attacker.example/deployment/api/testorg/testapi/", + # A host can be spelled without a scheme, and a path beginning `//` is + # read as one by anything that resolves a reference. + "//attacker.example/deployment/api/testorg/testapi/", + "///attacker.example/deployment/api/testorg/testapi/", + "////attacker.example/deployment/api/testorg/testapi/", + "https:////attacker.example/deployment/api/testorg/testapi/", + "https://api.example.com//attacker.example/deployment/api/testorg/testapi/", + ], +) +def test_a_status_endpoint_naming_another_host_is_not_polled(endpoint): + """The reply names the path to poll. It does not get to name the host the + deployment key is sent to. + + The deployment URL here carries a prefix the spec route cannot account for, + which is the branch that reads the endpoint rather than rebuilding it. + """ + client = _client(api_url="https://api.example.com/other/testorg/testapi/") + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(200, {"status": "COMPLETED"}) + client.check_execution_status(f"{endpoint}?execution_id=exec-123") + args, _ = mock_send.call_args + sent = httpx.URL(client.base_url).join(args[1]) + + assert sent.host == "api.example.com" + + +def test_the_key_is_read_at_call_time(): + """A rotated key reaches the next request. + + The released client built its header on every call, so assigning ``api_key`` + took effect immediately; a transport that captured it once would answer with + the old one until the client was rebuilt. + """ + client = _client() + with patch.object(client._transport.get_httpx_client(), "request") as request: + request.return_value = _httpx_response(200, {}) + client._send("GET", API_URL) + before = request.call_args.kwargs["headers"]["Authorization"] + client.api_key = "rotated-key" + client._send("GET", API_URL) + after = request.call_args.kwargs["headers"]["Authorization"] + + assert before == "Bearer test-key" + assert after == "Bearer rotated-key" + + +@pytest.mark.parametrize( + "api_url", + [ + API_URL, + # Nothing normalises the deployment URL on the way in, so whatever the + # caller registered is what the released client posted to. + API_URL.rstrip("/"), + "https://api.example.com/unstract/deployment/api/testorg/testapi/", + "https://api.example.com/deployment/api/TestOrg/testapi/", + ], +) +def test_execute_url_matches_the_deployment_url(api_url): + """The deployment URL goes back out as given. + + A path prefix — an ingress route, an on-prem reverse proxy — is part of it + and no route template can carry it, so the URL cannot be rebuilt from one. + """ + client = _client(api_url=api_url) + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(200, {"message": {}}) + with patch("builtins.open", return_value=io.BytesIO(b"x")): + client.structure_file(["sample.txt"]) + args, _ = mock_send.call_args + + with patch.object(baseline, "requests") as mock_requests: + mock_requests.post.return_value = _requests_response(200, {"message": {}}) + with patch("builtins.open", return_value=io.BytesIO(b"x")): + _baseline_client(api_url=api_url).structure_file(["sample.txt"]) + + assert args[0].lower() == "post" + assert args[1] == api_url == mock_requests.post.call_args[0][0] + + +def _wire_requests(*calls, reply=b'{"status":"COMPLETED","message":{}}'): + """Run each call against a loopback server and return the raw requests. + + Below the client, the transport adds headers of its own -- and drops none + of them into any object the client can be asked for. A socket is the only + place both clients can be compared on what they actually send. One server + serves every call, so the ``Host`` header is the same for all of them. + """ + raw = [] + server = socket.socket() + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(("127.0.0.1", 0)) + server.listen(len(calls)) + + def serve(): + for _ in calls: + conn, _address = server.accept() + data = b"" + while b"\r\n\r\n" not in data: + chunk = conn.recv(65536) + if not chunk: + break + data += chunk + # The body has to be drained too: a client whose upload is never + # read can block on the socket instead of returning. + head, _, rest = data.partition(b"\r\n\r\n") + declared = _header_value(head, "content-length") + while declared and len(rest) < int(declared): + chunk = conn.recv(65536) + if not chunk: + break + rest += chunk + raw.append(head + b"\r\n\r\n" + rest) + body = reply + conn.sendall( + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n" + b"Content-Length: %d\r\n\r\n%s" % (len(body), body) + ) + conn.close() + + thread = threading.Thread(target=serve, daemon=True) + thread.start() + try: + url = f"http://127.0.0.1:{server.getsockname()[1]}/deployment/api/org/name/" + for call in calls: + call(url) + finally: + thread.join(timeout=10) + server.close() + + return raw + + +def _headers(head: bytes) -> dict[str, str]: + return { + name.lower(): value.strip() + for name, _, value in ( + line.partition(":") for line in head.decode().split("\r\n")[1:] + ) + } + + +def _header_value(head: bytes, name: str) -> str: + return _headers(head).get(name, "") + + +def _wire_heads(*calls): + """The request headers each call put on the wire.""" + return [_headers(raw.split(b"\r\n\r\n")[0]) for raw in _wire_requests(*calls)] + + +def _multipart_parts(raw: bytes) -> list[tuple[str, str, bytes]]: + """``(field, filename, content)`` for every part of a multipart request. + + The boundary itself is deliberately not compared: it is random per request + in both clients, so only what it delimits can be. + """ + head, _, body = raw.partition(b"\r\n\r\n") + boundary = _header_value(head, "content-type").partition("boundary=")[2] + parts = [] + for chunk in body.split(b"--" + boundary.encode()): + headers, _, content = chunk.partition(b"\r\n\r\n") + disposition = headers.decode("utf-8", errors="replace") + if "content-disposition" not in disposition.lower(): + continue + field = re.search(r'name="([^"]*)"', disposition) + filename = re.search(r'filename="([^"]*)"', disposition) + parts.append( + ( + field.group(1) if field else "", + filename.group(1) if filename else "", + content.removesuffix(b"\r\n"), + ) + ) + return parts + + +def test_wire_headers_match_the_released_client(): + """The headers no caller sets are still on the wire. + + ``Accept-Encoding`` is the load-bearing one: it decides whether responses + come back compressed at all. + """ + ours, theirs = _wire_heads( + lambda url: _client(api_url=url).check_execution_status(STATUS_ENDPOINT), + lambda url: _baseline_client(api_url=url).check_execution_status( + STATUS_ENDPOINT + ), + ) + + assert {name: ours[name] for name in theirs if name != "user-agent"} == { + name: value for name, value in theirs.items() if name != "user-agent" + } + assert ours.keys() == theirs.keys() + # The one accepted difference: the transport names itself, and nothing on + # the wire branches on it. + assert ours["user-agent"].startswith("python-httpx/") + + +def test_a_multi_file_upload_matches_the_released_client(tmp_path): + """The method takes a list, and the second file is where a transport swap + diverges: one part written, one dropped, or two parts sharing a name the + server then reads as one.""" + paths = [] + for name, content in (("first.txt", b"one"), ("second.txt", b"two")): + path = tmp_path / name + path.write_bytes(content) + paths.append(str(path)) + + ours, theirs = _wire_requests( + lambda url: _client(api_url=url, api_timeout=300).structure_file(paths), + lambda url: _baseline_client(api_url=url, api_timeout=300).structure_file( + paths + ), + ) + + # Sorted: the two clients order the fields differently, which no multipart + # parser reads as meaning. The order of the files among themselves is the + # part that carries meaning, and it is pinned below. + assert sorted(_multipart_parts(ours)) == sorted(_multipart_parts(theirs)) + uploaded = [part for part in _multipart_parts(ours) if part[0] == "files"] + assert [(filename, content) for _, filename, content in uploaded] == [ + ("first.txt", b"one"), + ("second.txt", b"two"), + ] + + +def test_redirects_are_followed(): + """The released client followed them on both verbs. + + Not following one turns a load balancer's 307 into a body the poll loop + reads as a finished execution with no status. + """ + assert _client()._transport.get_httpx_client().follow_redirects is True + + +def test_a_status_endpoint_without_an_execution_id_is_refused(): + """Polling with a blank id asks the service about an execution nobody has; + what it answers is not this execution's state.""" + with pytest.raises(APIDeploymentsClientException): + _client().check_execution_status("/deployment/api/testorg/testapi/") + + +def test_the_transport_is_untimed_by_default(): + """A connection that stalls forever is what the released client did. + + Bounding it by default would turn a hang into an exception callers have + never had to handle, and ``api_timeout`` cannot serve: it is an execution + mode the backend reads, not a socket timeout. + """ + assert _client()._transport.get_httpx_client().timeout == httpx.Timeout(None) + + +def test_transport_timeout_is_what_the_transport_uses(): + assert _client(transport_timeout=5)._transport.get_httpx_client().timeout == ( + httpx.Timeout(5) + ) + + +def test_transport_timeout_bounds_a_stalled_connection(): + """The call is made off the test thread, so the failure this pins -- a + request that never returns -- fails the test instead of hanging the run.""" + server = socket.socket() + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(("127.0.0.1", 0)) + server.listen(1) + accepted, outcome = [], [] + + def stall(): + conn, _address = server.accept() + accepted.append(conn) # held open, and answered by nobody + + def call(): + url = f"http://127.0.0.1:{server.getsockname()[1]}/deployment/api/org/name/" + client = _client(api_url=url, transport_timeout=0.2) + try: + client.check_execution_status(STATUS_ENDPOINT) + outcome.append(None) + except BaseException as e: # noqa: BLE001 - reported, not handled + outcome.append(e) + + stalling = threading.Thread(target=stall, daemon=True) + calling = threading.Thread(target=call, daemon=True) + stalling.start() + calling.start() + try: + calling.join(timeout=10) + assert outcome, "the request never returned" + assert isinstance(outcome[0], ReadTimeout) + finally: + stalling.join(timeout=5) + for conn in accepted: + conn.close() + server.close() + + +def test_deployment_route_rejects_an_unusable_url(): + from unstract.api_deployments.client import APIDeploymentsClientException + + client = _client(api_url="https://api.example.com/onlyone") + with pytest.raises(APIDeploymentsClientException): + _ = client._deployment_route + + +# -------------------------------------------------------------------------- +# Return shape, compared against the released client over the same responses +# -------------------------------------------------------------------------- + + +def _message(**fields): + return {"message": fields} + + +EXECUTE_CASES = [ + ("pending", 200, _message(execution_status="PENDING", status_api=STATUS_ENDPOINT)), + ( + "executing", + 200, + _message(execution_status="EXECUTING", status_api=STATUS_ENDPOINT), + ), + ("success", 200, _message(execution_status="SUCCESS", result=[{"file": "a"}])), + ( + "success_without_result", + 200, + _message(execution_status="SUCCESS", status_api=STATUS_ENDPOINT), + ), + ("error", 200, _message(execution_status="ERROR", error="boom")), + ("unauthorized", 401, {"errors": [{"detail": "Invalid token"}]}), + ("unprocessable", 422, _message(execution_status="ERROR", error="bad input")), + ("server_error", 500, _message(execution_status="ERROR", error="oops")), +] + + +@pytest.mark.parametrize( + ("name", "status_code", "body"), EXECUTE_CASES, ids=[c[0] for c in EXECUTE_CASES] +) +def test_structure_file_returns_what_the_released_client_returned( + sample_file, name, status_code, body +): + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(status_code, body) + ours = _client(api_timeout=300).structure_file([sample_file]) + + with patch.object(baseline, "requests") as mock_requests: + mock_requests.post.return_value = _requests_response(status_code, body) + theirs = _baseline_client(api_timeout=300).structure_file([sample_file]) + + assert ours == theirs + + +def test_structure_file_matches_on_a_non_json_body(sample_file): + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(502, text="gateway") + ours = _client(api_timeout=300).structure_file([sample_file]) + + with patch.object(baseline, "requests") as mock_requests: + mock_requests.post.return_value = _requests_response( + 502, text="gateway" + ) + theirs = _baseline_client(api_timeout=300).structure_file([sample_file]) + + assert ours == theirs + + +def test_structure_file_missing_file_still_raises(sample_file): + from unstract.api_deployments.client import APIDeploymentsClientException + + with pytest.raises(APIDeploymentsClientException): + _client().structure_file(["/nonexistent/file.txt"]) + with pytest.raises(baseline.APIDeploymentsClientException): + _baseline_client().structure_file(["/nonexistent/file.txt"]) + + +def test_structure_file_closes_its_handles(sample_file): + """The released client leaked these; closing them is invisible to + callers.""" + opened = [] + real_open = open + + def tracking_open(*args, **kwargs): + handle = real_open(*args, **kwargs) + opened.append(handle) + return handle + + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(200, {"message": {}}) + with patch("builtins.open", side_effect=tracking_open): + _client().structure_file([sample_file]) + + assert opened and all(handle.closed for handle in opened) + + +STATUS_CASES = [ + ("completed", 200, {"status": "COMPLETED", "message": [{"file": "a"}]}), + ("executing", 200, {"status": "EXECUTING", "message": ""}), + ("queued", 200, {"status": "QUEUED", "message": ""}), + ("error", 200, {"status": "ERROR", "error": "boom", "message": ""}), + ("already_acknowledged", 406, {"status": "", "error": "already acknowledged"}), + ("server_error", 500, {"status": "", "error": "oops"}), +] + + +@pytest.mark.parametrize( + ("name", "status_code", "body"), STATUS_CASES, ids=[c[0] for c in STATUS_CASES] +) +def test_check_execution_status_returns_what_the_released_client_returned( + name, status_code, body +): + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(status_code, body) + ours = _client().check_execution_status(STATUS_ENDPOINT) + + with patch.object(baseline, "requests") as mock_requests: + mock_requests.request.return_value = _requests_response(status_code, body) + theirs = _baseline_client().check_execution_status(STATUS_ENDPOINT) + + assert ours == theirs + + +def test_check_execution_status_matches_on_a_non_json_body(): + with patch.object(APIDeploymentsClient, "_send") as mock_send: + mock_send.return_value = _httpx_response(502, text="gateway") + ours = _client().check_execution_status(STATUS_ENDPOINT) + + with patch.object(baseline, "requests") as mock_requests: + mock_requests.request.return_value = _requests_response( + 502, text="gateway" + ) + theirs = _baseline_client().check_execution_status(STATUS_ENDPOINT) + + assert ours == theirs + + +# -------------------------------------------------------------------------- +# Construction and surface +# -------------------------------------------------------------------------- + + +def _baseline_class_node(): + tree = ast.parse(BASELINE_PATH.read_text()) + for node in tree.body: + if isinstance(node, ast.ClassDef) and node.name == "APIDeploymentsClient": + return node + raise AssertionError("APIDeploymentsClient not found in the baseline") + + +def _baseline_init_params(): + """Constructor parameters and defaults, read out of the baseline source. + + Parsed rather than imported so the comparison is against the + released text, not against whatever a shared import happened to + bind. + """ + for node in _baseline_class_node().body: + if isinstance(node, ast.FunctionDef) and node.name == "__init__": + args = node.args.args[1:] + defaults = [None] * (len(args) - len(node.args.defaults)) + [ + ast.literal_eval(d) for d in node.args.defaults + ] + return list(zip([a.arg for a in args], defaults)) + raise AssertionError("__init__ not found in the baseline") + + +def test_constructor_parameters_are_unchanged(): + """Names, order and defaults all matter: callers pass some positionally.""" + live = inspect.signature(APIDeploymentsClient.__init__).parameters + live_params = [ + (name, None if p.default is inspect.Parameter.empty else p.default) + for name, p in live.items() + # Keyword-only parameters are excluded: they cannot be reached by any + # existing call, so adding one leaves every released call shape intact. + if name != "self" and p.kind is not p.KEYWORD_ONLY + ] + assert live_params == _baseline_init_params() + + +def test_public_methods_are_unchanged(): + baseline_methods = { + node.name: node + for node in _baseline_class_node().body + if isinstance(node, ast.FunctionDef) and not node.name.startswith("_") + } + assert baseline_methods + + for name, node in baseline_methods.items(): + live = getattr(APIDeploymentsClient, name, None) + assert live is not None, f"{name} disappeared from the client" + # Keyword-only parameters are excluded: they cannot be reached by any + # existing call, so adding one leaves every released call shape intact. + live_args = [ + arg + for arg, p in inspect.signature(live).parameters.items() + if arg not in ("self", "cls") and p.kind is not p.KEYWORD_ONLY + ] + assert live_args == [a.arg for a in node.args.args[1:]], name + + +def test_class_attributes_are_unchanged(): + for node in _baseline_class_node().body: + if not isinstance(node, ast.Assign): + continue + try: + value = ast.literal_eval(node.value) + except ValueError: + continue # logger and friends: identity, not value + for target in node.targets: + assert getattr(APIDeploymentsClient, target.id) == value, target.id + + +def test_module_level_names_are_unchanged(): + import unstract.api_deployments.client as live + + tree = ast.parse(BASELINE_PATH.read_text()) + for node in tree.body: + if isinstance(node, ast.ClassDef) and not node.name.startswith("_"): + assert hasattr(live, node.name), node.name + + +def test_every_declared_operation_is_wrapped(): + """A new spec operation shows up here as a failure, not as silence. + + Compared whole rather than after subtracting an exception list: an entry + excusing an operation the spec no longer declares keeps passing forever, and + nothing about a green run says the list is still describing anything. + """ + spec = json.loads(SPEC_PATH.read_text()) + declared = { + operation["operationId"] + for path in spec["paths"].values() + for method, operation in path.items() + if method in {"get", "post", "put", "patch", "delete"} + } + assert declared == WRAPPED_OPERATIONS + + +def test_the_baseline_is_the_released_client_unmodified(): + # A digest, not a version string in a comment: an edited baseline can claim + # any provenance it likes, and every parity test here would still pass. + assert BASELINE_PATH.name == f"client_{BASELINE_VERSION.replace('.', '_')}.py" + assert hashlib.sha256(BASELINE_PATH.read_bytes()).hexdigest() == BASELINE_SHA256 diff --git a/tests/test_retry.py b/tests/test_retry.py index d0acb81..bdac497 100644 --- a/tests/test_retry.py +++ b/tests/test_retry.py @@ -212,7 +212,7 @@ def test_exception_outcome_uses_exponential(self): class TestRequestWithRetrySuccess: - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_success_on_first_try(self, mock_request, client): mock_request.return_value = _mock_response(200) resp = client._request_with_retry("GET", "https://api.example.com/test") @@ -220,7 +220,7 @@ def test_success_on_first_try(self, mock_request, client): assert mock_request.call_count == 1 @patch("unstract.api_deployments.client.time.sleep") - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_retry_on_503_then_success(self, mock_request, mock_sleep, client): mock_request.side_effect = [ _mock_response(503), @@ -232,7 +232,7 @@ def test_retry_on_503_then_success(self, mock_request, mock_sleep, client): assert mock_sleep.call_count == 1 @patch("unstract.api_deployments.client.time.sleep") - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_retry_on_500_then_success(self, mock_request, mock_sleep, client): mock_request.side_effect = [ _mock_response(500), @@ -245,7 +245,7 @@ def test_retry_on_500_then_success(self, mock_request, mock_sleep, client): assert mock_sleep.call_count == 2 @patch("unstract.api_deployments.client.time.sleep") - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_retry_on_429_then_success(self, mock_request, mock_sleep, client): mock_request.side_effect = [ _mock_response(429), @@ -256,7 +256,7 @@ def test_retry_on_429_then_success(self, mock_request, mock_sleep, client): assert mock_request.call_count == 2 @patch("unstract.api_deployments.client.time.sleep") - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_retry_on_502_then_success(self, mock_request, mock_sleep, client): mock_request.side_effect = [ _mock_response(502), @@ -267,7 +267,7 @@ def test_retry_on_502_then_success(self, mock_request, mock_sleep, client): assert mock_request.call_count == 2 @patch("unstract.api_deployments.client.time.sleep") - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_retry_on_504_then_success(self, mock_request, mock_sleep, client): mock_request.side_effect = [ _mock_response(504), @@ -283,7 +283,7 @@ def test_retry_on_504_then_success(self, mock_request, mock_sleep, client): class TestRequestWithRetryConnectionErrors: @patch("unstract.api_deployments.client.time.sleep") - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_retry_on_connection_error_then_success( self, mock_request, mock_sleep, client ): @@ -296,7 +296,7 @@ def test_retry_on_connection_error_then_success( assert mock_request.call_count == 2 @patch("unstract.api_deployments.client.time.sleep") - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_retry_on_timeout_then_success(self, mock_request, mock_sleep, client): mock_request.side_effect = [ Timeout("Request timed out"), @@ -307,7 +307,7 @@ def test_retry_on_timeout_then_success(self, mock_request, mock_sleep, client): assert mock_request.call_count == 2 @patch("unstract.api_deployments.client.time.sleep") - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_connection_error_exhausted_raises(self, mock_request, mock_sleep, client): mock_request.side_effect = ConnectionError("Connection refused") with pytest.raises(ConnectionError): @@ -315,7 +315,7 @@ def test_connection_error_exhausted_raises(self, mock_request, mock_sleep, clien assert mock_request.call_count == client.max_retries + 1 @patch("unstract.api_deployments.client.time.sleep") - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_timeout_exhausted_raises(self, mock_request, mock_sleep, client): mock_request.side_effect = Timeout("Request timed out") with pytest.raises(Timeout): @@ -328,7 +328,7 @@ def test_timeout_exhausted_raises(self, mock_request, mock_sleep, client): class TestRequestWithRetryExhaustion: @patch("unstract.api_deployments.client.time.sleep") - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_all_retries_exhausted_returns_last_response( self, mock_request, mock_sleep, client ): @@ -342,35 +342,35 @@ def test_all_retries_exhausted_returns_last_response( class TestNoRetryOnNonRetryable: - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_no_retry_on_200(self, mock_request, client): mock_request.return_value = _mock_response(200) resp = client._request_with_retry("GET", "https://api.example.com/test") assert resp.status_code == 200 assert mock_request.call_count == 1 - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_no_retry_on_400(self, mock_request, client): mock_request.return_value = _mock_response(400) resp = client._request_with_retry("GET", "https://api.example.com/test") assert resp.status_code == 400 assert mock_request.call_count == 1 - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_no_retry_on_401(self, mock_request, client): mock_request.return_value = _mock_response(401) resp = client._request_with_retry("GET", "https://api.example.com/test") assert resp.status_code == 401 assert mock_request.call_count == 1 - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_no_retry_on_404(self, mock_request, client): mock_request.return_value = _mock_response(404) resp = client._request_with_retry("GET", "https://api.example.com/test") assert resp.status_code == 404 assert mock_request.call_count == 1 - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_no_retry_on_422(self, mock_request, client): mock_request.return_value = _mock_response(422) resp = client._request_with_retry("GET", "https://api.example.com/test") @@ -382,7 +382,7 @@ def test_no_retry_on_422(self, mock_request, client): class TestTimeoutPassed: - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_no_default_timeout_set(self, mock_request, client): """api_timeout is a server-side parameter, not an HTTP socket timeout. @@ -393,7 +393,7 @@ def test_no_default_timeout_set(self, mock_request, client): _, kwargs = mock_request.call_args assert "timeout" not in kwargs - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_explicit_timeout_not_overridden(self, mock_request, client): """Callers can still pass an explicit HTTP socket timeout.""" mock_request.return_value = _mock_response(200) @@ -407,7 +407,7 @@ def test_explicit_timeout_not_overridden(self, mock_request, client): class TestRetryAfterHeader: @patch("unstract.api_deployments.client.time.sleep") - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_429_respects_retry_after_header(self, mock_request, mock_sleep, client): mock_request.side_effect = [ _mock_response(429, headers={"Retry-After": "5"}), @@ -418,7 +418,7 @@ def test_429_respects_retry_after_header(self, mock_request, mock_sleep, client) mock_sleep.assert_called_once_with(5.0) @patch("unstract.api_deployments.client.time.sleep") - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_429_invalid_retry_after_falls_back(self, mock_request, mock_sleep, client): mock_request.side_effect = [ _mock_response(429, headers={"Retry-After": "not-a-number"}), @@ -437,7 +437,7 @@ def test_429_invalid_retry_after_falls_back(self, mock_request, mock_sleep, clie class TestFileSeekOnRetry: @patch("unstract.api_deployments.client.time.sleep") - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_file_objects_rewound_on_retry(self, mock_request, mock_sleep, client): file_obj = io.BytesIO(b"test data") files = [("files", ("test.pdf", file_obj, "application/octet-stream"))] @@ -457,7 +457,7 @@ def test_file_objects_rewound_on_retry(self, mock_request, mock_sleep, client): class TestDisabledRetry: - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_no_retry_when_max_retries_zero(self, mock_request, client_no_retry): mock_request.return_value = _mock_response(503) resp = client_no_retry._request_with_retry( @@ -466,7 +466,7 @@ def test_no_retry_when_max_retries_zero(self, mock_request, client_no_retry): assert resp.status_code == 503 assert mock_request.call_count == 1 - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_connection_error_raises_immediately_when_disabled( self, mock_request, client_no_retry ): @@ -481,27 +481,27 @@ def test_connection_error_raises_immediately_when_disabled( class TestCheckExecutionStatusPendingFix: @patch("unstract.api_deployments.client.time.sleep") - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_503_after_exhaustion_sets_pending_true( self, mock_request, mock_sleep, client ): mock_request.return_value = _mock_response( 503, json_data={"status": "", "error": "Service Unavailable", "message": ""} ) - result = client.check_execution_status("/api/v1/status/123") + result = client.check_execution_status("/api/v1/status/?execution_id=123") assert result["pending"] is True assert result["status_code"] == 503 - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_200_with_pending_status_sets_pending_true(self, mock_request, client): mock_request.return_value = _mock_response( 200, json_data={"status": "EXECUTING", "error": "", "message": ""} ) - result = client.check_execution_status("/api/v1/status/123") + result = client.check_execution_status("/api/v1/status/?execution_id=123") assert result["pending"] is True assert result["status_code"] == 200 - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_200_with_completed_status_sets_pending_false(self, mock_request, client): mock_request.return_value = _mock_response( 200, @@ -511,10 +511,10 @@ def test_200_with_completed_status_sets_pending_false(self, mock_request, client "message": '{"result": "data"}', }, ) - result = client.check_execution_status("/api/v1/status/123") + result = client.check_execution_status("/api/v1/status/?execution_id=123") assert result["pending"] is False - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_422_with_executing_status_sets_pending_true(self, mock_request, client): """HTTP 422 is currently returned by Unstract for in-progress statuses. @@ -526,27 +526,27 @@ def test_422_with_executing_status_sets_pending_true(self, mock_request, client) mock_request.return_value = _mock_response( 422, json_data={"status": "EXECUTING", "error": "", "message": ""} ) - result = client.check_execution_status("/api/v1/status/123") + result = client.check_execution_status("/api/v1/status/?execution_id=123") assert result["pending"] is True assert result["status_code"] == 422 - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_422_with_pending_status_sets_pending_true(self, mock_request, client): """HTTP 422 with PENDING body status — still detected via body check.""" mock_request.return_value = _mock_response( 422, json_data={"status": "PENDING", "error": "", "message": ""} ) - result = client.check_execution_status("/api/v1/status/123") + result = client.check_execution_status("/api/v1/status/?execution_id=123") assert result["pending"] is True assert result["status_code"] == 422 - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_400_does_not_set_pending(self, mock_request, client): mock_request.return_value = _mock_response( 400, json_data={"status": "", "error": "Bad request", "message": ""} ) - result = client.check_execution_status("/api/v1/status/123") + result = client.check_execution_status("/api/v1/status/?execution_id=123") assert result["pending"] is False @@ -555,7 +555,7 @@ def test_400_does_not_set_pending(self, mock_request, client): class TestStructureFileUsesRetry: @patch("unstract.api_deployments.client.time.sleep") - @patch("unstract.api_deployments.client.requests.request") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_structure_file_retries_on_503_async_mode( self, mock_request, mock_sleep, tmp_path ): @@ -591,7 +591,7 @@ def test_structure_file_retries_on_503_async_mode( class TestStructureFileNoRetryInSyncMode: - @patch("unstract.api_deployments.client.requests.post") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_structure_file_no_retry_on_503_sync_mode(self, mock_post, tmp_path): """In sync mode (api_timeout>0), POST is NOT retried on 5xx.""" test_file = tmp_path / "test.pdf" @@ -620,7 +620,7 @@ def test_structure_file_no_retry_on_503_sync_mode(self, mock_post, tmp_path): # Only one call — no retries in sync mode assert mock_post.call_count == 1 - @patch("unstract.api_deployments.client.requests.post") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_structure_file_sync_mode_default_timeout(self, mock_post, tmp_path): """Default api_timeout=300 means sync mode — no POST retry.""" test_file = tmp_path / "test.pdf" @@ -654,7 +654,7 @@ def test_structure_file_sync_mode_default_timeout(self, mock_post, tmp_path): class TestStructureFile422DoesNotSetPending: - @patch("unstract.api_deployments.client.requests.post") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_422_pending_does_not_set_pending(self, mock_post, tmp_path): """POST 422 with PENDING status should NOT set pending=True. @@ -686,7 +686,7 @@ def test_422_pending_does_not_set_pending(self, mock_post, tmp_path): assert result["pending"] is False assert result["status_code"] == 422 - @patch("unstract.api_deployments.client.requests.post") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_422_executing_does_not_set_pending(self, mock_post, tmp_path): """POST 422 with EXECUTING status should NOT set pending=True. @@ -717,7 +717,7 @@ def test_422_executing_does_not_set_pending(self, mock_post, tmp_path): assert result["pending"] is False assert result["status_code"] == 422 - @patch("unstract.api_deployments.client.requests.post") + @patch("unstract.api_deployments.client.APIDeploymentsClient._send") def test_200_pending_sets_pending_true(self, mock_post, tmp_path): """POST 200 + PENDING correctly sets pending=True for polling.""" test_file = tmp_path / "test.pdf" diff --git a/tools/gen_sdk.sh b/tools/gen_sdk.sh new file mode 100755 index 0000000..714fb40 --- /dev/null +++ b/tools/gen_sdk.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Regenerate the transport layer from the committed OpenAPI spec. +# +# The generated tree is committed but NEVER hand-edited: regeneration overwrites +# it wholesale, so a fix applied there is lost on the next run. Fixes belong in +# the facade (client.py) or upstream in the spec. +# +# The spec itself is produced by the backend that serves these endpoints +# (`manage.py generate_docstudio_spec`); refresh it from there rather than +# editing it here. +# +# ./tools/gen_sdk.sh && git diff --stat src/unstract/api_deployments/sdk_docstudio +set -euo pipefail + +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +VENV="$REPO/.gen-venv" +# The generator shells out to ruff for its own post-processing. Without this it +# finds whatever ruff the caller happens to have, or none, and reports the miss +# as a warning -- which the gate below reads as an unparsable spec. +export PATH="$VENV/bin:$PATH" +OUT="src/unstract/api_deployments/sdk_docstudio" +# Pinned: unpinned, a generator upgrade and a spec change produce the same diff, +# and the drift gate can no longer tell them apart. +GENERATOR="openapi-python-client==0.29.0" + +if [ ! -x "$VENV/bin/openapi-python-client" ]; then + uv venv "$VENV" + uv pip install --python "$VENV/bin/python" "$GENERATOR" +fi + +want="${GENERATOR#*==}" +have="$("$VENV/bin/openapi-python-client" --version | awk '{print $NF}')" +if [ "$have" != "$want" ]; then + echo "generator is $have, expected $want — reinstalling" >&2 + uv pip install --python "$VENV/bin/python" "$GENERATOR" +fi + +rm -rf "${REPO:?}/$OUT" +log="$(mktemp)" +trap 'rm -f "$log"' EXIT +(cd "$REPO" && "$VENV/bin/openapi-python-client" generate \ + --path "$REPO/specs/docstudio-oss.json" --output-path "$REPO/$OUT" \ + --config "$REPO/tools/openapi-client.yaml" --overwrite --meta none) 2>&1 | tee "$log" + +# The generator downgrades a schema it cannot parse to a warning, drops +# the endpoint or model it belongs to, writes the rest and exits 0. The +# result is a client missing an operation and a spec that still looks fine. +if grep -qi warning "$log"; then + echo "the generator reported a problem above and still exited 0; whatever it could not parse is missing from the output" >&2 + exit 1 +fi + +# Stamp every file, so the rule survives contact with a reader who arrived via +# grep rather than via this script. +header='# Generated by tools/gen_sdk.sh from specs/docstudio-oss.json. DO NOT EDIT.' +find "$REPO/$OUT" -name '*.py' -print0 | while IFS= read -r -d '' f; do + printf '%s\n%s\n' "$header" "$(cat "$f")" > "$f.tmp" && mv "$f.tmp" "$f" +done + +echo "generated $OUT ($(find "$REPO/$OUT" -name '*.py' | wc -l) files)" diff --git a/tools/openapi-client.yaml b/tools/openapi-client.yaml new file mode 100644 index 0000000..d2196b9 --- /dev/null +++ b/tools/openapi-client.yaml @@ -0,0 +1,3 @@ +# openapi-python-client config. Kept minimal on purpose: every knob here is +# maintenance surface, and post-processing the generated code is a kill criterion. +literal_enums: true diff --git a/tools/refresh_baseline.sh b/tools/refresh_baseline.sh new file mode 100755 index 0000000..065cfb8 --- /dev/null +++ b/tools/refresh_baseline.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Refresh the vendored parity baseline in tests/baseline/. +# +# The compat suite compares this client against the last RELEASED one, not +# against the working tree — a baseline that moves with local edits measures +# nothing. It is vendored rather than downloaded at test time so the suite stays +# offline, and refreshing it is a deliberate act with a reviewable diff. +# +# ./tools/refresh_baseline.sh 1.5.3 +set -euo pipefail + +VERSION="${1:?usage: refresh_baseline.sh }" +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SLUG="${VERSION//./_}" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +(cd "$WORK" && pip download "unstract-client==$VERSION" --no-deps -q && unzip -o -q ./*.whl -d x) + +OUT="$REPO/tests/baseline/client_$SLUG.py" +{ + echo "# Vendored from the released unstract-client $VERSION wheel on PyPI. DO NOT EDIT." + echo "# Refresh with tools/refresh_baseline.sh when the parity baseline is intentionally moved." + cat "$WORK/x/unstract/api_deployments/client.py" +} > "$OUT" + +echo "wrote $OUT" +echo "in tests/test_compat.py set:" +echo " BASELINE_VERSION = \"$VERSION\"" +echo " BASELINE_SHA256 = \"$(sha256sum "$OUT" | cut -d' ' -f1)\"" diff --git a/uv.lock b/uv.lock index 99dffef..fe306ba 100644 --- a/uv.lock +++ b/uv.lock @@ -2,6 +2,28 @@ version = 1 revision = 3 requires-python = ">=3.11" +[[package]] +name = "anyio" +version = "4.14.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + [[package]] name = "certifi" version = "2026.1.4" @@ -267,6 +289,43 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e7/04/a94ebfb4eaaa08db56725a40de2887e95de4e8641b9e902c311bfa00aa39/filelock-3.24.2-py3-none-any.whl", hash = "sha256:667d7dc0b7d1e1064dd5f8f8e80bdac157a6482e8d2e02cd16fd3b6b33bd6556", size = 24152, upload-time = "2026-02-16T02:50:44Z" }, ] +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + [[package]] name = "identify" version = "2.6.16" @@ -799,7 +858,9 @@ wheels = [ name = "unstract-client" source = { editable = "." } dependencies = [ + { name = "attrs" }, { name = "click" }, + { name = "httpx" }, { name = "requests" }, { name = "rich" }, { name = "tenacity" }, @@ -837,7 +898,9 @@ test = [ [package.metadata] requires-dist = [ + { name = "attrs", specifier = ">=23.2" }, { name = "click", specifier = ">=8.1" }, + { name = "httpx", specifier = ">=0.27,<0.29" }, { name = "requests", specifier = ">=2.32.3" }, { name = "rich", specifier = ">=13.7" }, { name = "tenacity", specifier = ">=8.2.0" },