From c49234b7e1daec68d005774029c3121df960c61a Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Wed, 12 Aug 2026 13:10:35 +0530 Subject: [PATCH 1/7] feat(api): generate and commit the API deployment OpenAPI spec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The published Python clients and their generated SDKs are built from a spec of the deployment execute/status endpoints, which until now was produced by a script living outside this repo — so a route or serializer change here could silently invalidate it. The schema annotation for DeploymentExecution now lives next to the view, and `manage.py generate_docstudio_spec` writes specs/docstudio-oss.json. A unit test regenerates and compares, so drift fails in this repo's existing CI tier rather than in a client repo, with no database or extra CI job needed. The generated spec is unchanged from what the external script produced, apart from a root `tags` array — clients had nowhere to read group descriptions from. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014f9oEEYspPH4fmPULTnLkJ --- backend/api_v2/api_deployment_views.py | 98 ++++ backend/api_v2/deployment_spec_urls.py | 13 + .../commands/generate_docstudio_spec.py | 60 +++ backend/api_v2/tests/test_docstudio_spec.py | 38 ++ backend/backend/settings/base.py | 21 + backend/pyproject.toml | 1 + backend/uv.lock | 19 + specs/docstudio-oss.json | 442 ++++++++++++++++++ 8 files changed, 692 insertions(+) create mode 100644 backend/api_v2/deployment_spec_urls.py create mode 100644 backend/api_v2/management/commands/generate_docstudio_spec.py create mode 100644 backend/api_v2/tests/test_docstudio_spec.py create mode 100644 specs/docstudio-oss.json diff --git a/backend/api_v2/api_deployment_views.py b/backend/api_v2/api_deployment_views.py index d5cfa800a1..6fb1e08234 100644 --- a/backend/api_v2/api_deployment_views.py +++ b/backend/api_v2/api_deployment_views.py @@ -5,6 +5,13 @@ from django.db.models import F, OuterRef, QuerySet, Subquery from django.http import HttpResponse +from drf_spectacular.types import OpenApiTypes +from drf_spectacular.utils import ( + OpenApiParameter, + extend_schema, + extend_schema_field, + extend_schema_view, +) from permissions.membership_views import OwnerManagementMixin from permissions.permission import IsOwner, IsOwnerOrSharedUserOrSharedToOrg from permissions.resource_share_views import ResourceShareManagementMixin @@ -50,6 +57,97 @@ logger = logging.getLogger(__name__) +@extend_schema_field(OpenApiTypes.BINARY) +class UploadField(serializers.FileField): + """A bare ``FileField`` maps to ``format: uri`` — correct on output, wrong + for a multipart upload, and generators emit ``str`` for it. + """ + + +class ExecuteRequest(ExecutionRequestSerializer): + """Subclasses the real serializer so every backend param arrives free.""" + + # ``files`` arrives via ``request.FILES``, so no serializer declares it. + files = serializers.ListField(child=UploadField(), required=False) + + +class FileResult(serializers.Serializer): + file = serializers.CharField() + file_execution_id = serializers.CharField(required=False) + status = serializers.CharField(required=False) + result = serializers.JSONField(required=False) + metadata = serializers.JSONField(required=False) + metrics = serializers.JSONField(required=False) + error = serializers.CharField(required=False, allow_null=True) + + +class ExecutionMessage(serializers.Serializer): + execution_status = serializers.CharField() + execution_id = serializers.CharField(required=False) + workflow_id = serializers.CharField(required=False) + status_api = serializers.CharField(required=False, allow_null=True) + error = serializers.CharField(required=False, allow_null=True) + # The backend sends `result: null` while pending; without allow_null the + # generated deserialiser iterates None and crashes. + result = FileResult(many=True, required=False, allow_null=True) + + +class ExecuteResponse(serializers.Serializer): + message = ExecutionMessage() + + +class StatusResponse(serializers.Serializer): + status = serializers.CharField() + message = FileResult(many=True, required=False, allow_null=True) + + +class ErrorResponse(serializers.Serializer): + status = serializers.CharField(required=False) + message = serializers.JSONField(required=False, allow_null=True) + + +DEPLOYMENT_PATH_PARAMETERS = [ + OpenApiParameter( + "org_name", + str, + OpenApiParameter.PATH, + description="Organization identifier.", + ), + OpenApiParameter( + "api_name", str, OpenApiParameter.PATH, description="API deployment name." + ), +] + + +# The generated clients take their command names, module paths and request +# shapes from here, so this block is part of the public API surface. +@extend_schema_view( + post=extend_schema( + operation_id="execute", + tags=["deployment"], + parameters=DEPLOYMENT_PATH_PARAMETERS, + request={"multipart/form-data": ExecuteRequest}, + responses={ + 200: ExecuteResponse, + 422: ExecuteResponse, + 500: ErrorResponse, + }, + description="Execute an API deployment against one or more files.", + ), + get=extend_schema( + operation_id="status", + tags=["deployment"], + parameters=DEPLOYMENT_PATH_PARAMETERS + [ExecutionQuerySerializer], + # 406 means the result was already consumed — this GET is one-shot. + responses={ + 200: StatusResponse, + 406: ErrorResponse, + 422: StatusResponse, + 500: ErrorResponse, + }, + description="Poll the status of a previously started execution.", + ), +) class DeploymentExecution(views.APIView): def initialize_request(self, request: Request, *args: Any, **kwargs: Any) -> Request: """To remove csrf request for public API. diff --git a/backend/api_v2/deployment_spec_urls.py b/backend/api_v2/deployment_spec_urls.py new file mode 100644 index 0000000000..f61d714d2c --- /dev/null +++ b/backend/api_v2/deployment_spec_urls.py @@ -0,0 +1,13 @@ +"""URLconf used only to generate the API deployment OpenAPI spec. + +``api_v2.execution_urls`` is an included sub-urlconf, so generating against it +directly yields paths without the prefix it is mounted at — a spec describing +URLs the server does not serve. This mirrors the mount in ``base_urls``. +""" + +from django.conf import settings +from django.urls import include, path + +urlpatterns = [ + path(f"{settings.API_DEPLOYMENT_PATH_PREFIX}/", include("api_v2.execution_urls")) +] diff --git a/backend/api_v2/management/commands/generate_docstudio_spec.py b/backend/api_v2/management/commands/generate_docstudio_spec.py new file mode 100644 index 0000000000..601d59506c --- /dev/null +++ b/backend/api_v2/management/commands/generate_docstudio_spec.py @@ -0,0 +1,60 @@ +"""Regenerate the committed API deployment OpenAPI spec. + +The spec is the contract the published clients and their generated SDKs are +built from, so it is committed and CI fails on drift: change a route, a +serializer or the schema annotation, and regenerate in the same PR. + + python manage.py generate_docstudio_spec + python manage.py generate_docstudio_spec --check # CI: no write, drift is an error +""" + +import json +from pathlib import Path +from typing import Any + +from django.core.management.base import BaseCommand, CommandError +from drf_spectacular.generators import SchemaGenerator + +DEFAULT_OUT = Path(__file__).resolve().parents[4] / "specs" / "docstudio-oss.json" +URLCONF = "api_v2.deployment_spec_urls" + + +class Command(BaseCommand): + help = "Generate the API deployment OpenAPI spec." + + def add_arguments(self, parser: Any) -> None: + parser.add_argument("--out", type=Path, default=DEFAULT_OUT) + parser.add_argument( + "--check", + action="store_true", + help="Fail if the file on disk differs, instead of writing it.", + ) + + def handle(self, *args: Any, **options: Any) -> None: + schema = SchemaGenerator(urlconf=URLCONF).get_schema(request=None, public=True) + # Sorted keys are what make the committed artifact a usable drift signal. + rendered = json.dumps(schema, indent=2, sort_keys=True) + "\n" + + out: Path = options["out"] + if options["check"]: + current = out.read_text() if out.exists() else "" + if current != rendered: + raise CommandError( + f"{out} is out of date. Run `python manage.py " + f"generate_docstudio_spec` and commit the result." + ) + self.stdout.write(f"{out} is up to date") + return + + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(rendered) + operations = sum( + 1 + for methods in schema["paths"].values() + for method in methods + if method in {"get", "post", "put", "patch", "delete"} + ) + self.stdout.write( + f"{out}: {len(schema['paths'])} paths, {operations} operations, " + f"{len(schema.get('components', {}).get('schemas', {}))} schemas" + ) diff --git a/backend/api_v2/tests/test_docstudio_spec.py b/backend/api_v2/tests/test_docstudio_spec.py new file mode 100644 index 0000000000..602d8b5c9f --- /dev/null +++ b/backend/api_v2/tests/test_docstudio_spec.py @@ -0,0 +1,38 @@ +"""The committed spec is the contract the published clients are generated from. + +A route, serializer or schema-annotation change that is not regenerated ships a +spec describing an API the server no longer serves, so drift fails here rather +than in a client repo. +""" + +import json +from pathlib import Path + +from drf_spectacular.generators import SchemaGenerator + +from api_v2.management.commands.generate_docstudio_spec import DEFAULT_OUT, URLCONF + + +def _render() -> str: + schema = SchemaGenerator(urlconf=URLCONF).get_schema(request=None, public=True) + return json.dumps(schema, indent=2, sort_keys=True) + "\n" + + +def test_committed_spec_matches_the_code() -> None: + assert DEFAULT_OUT.exists(), f"{DEFAULT_OUT} is missing" + assert DEFAULT_OUT.read_text() == _render(), ( + f"{DEFAULT_OUT} is out of date. Run " + "`python manage.py generate_docstudio_spec` and commit the result." + ) + + +def test_spec_covers_the_deployment_routes() -> None: + """Guards the mount: generating against the included sub-urlconf silently + drops the prefix, leaving paths the server does not serve.""" + spec = json.loads(Path(DEFAULT_OUT).read_text()) + execute = "/deployment/api/{org_name}/{api_name}/" + + assert set(spec["paths"]) == {execute, f"{execute}mcp/"} + assert spec["paths"][execute]["post"]["operationId"] == "execute" + assert spec["paths"][execute]["get"]["operationId"] == "status" + assert [tag["name"] for tag in spec["tags"]] == ["deployment"] diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py index d14f87b304..bbc6b440d4 100644 --- a/backend/backend/settings/base.py +++ b/backend/backend/settings/base.py @@ -653,6 +653,27 @@ def filter(self, record): "DEFAULT_VERSION": "v1", "ALLOWED_VERSIONS": ["v1"], "VERSION_PARAM": "version", + "DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema", +} + +# Read only while generating the API deployment OpenAPI spec +# (``manage.py generate_docstudio_spec``); no effect at request time. +SPECTACULAR_SETTINGS = { + "TITLE": "Unstract Document Studio", + "VERSION": "v1", + "PREPROCESSING_HOOKS": ["drf_spectacular.hooks.preprocess_exclude_path_format"], + "SERVE_INCLUDE_SCHEMA": False, + # Group descriptions clients show in their help; without this the spec has + # no root `tags` array and the text has nowhere to live. + "TAGS": [ + { + "name": "deployment", + "description": ( + "Run an API deployment against one or more documents and poll " + "the result." + ), + } + ], } # These paths will work without authentication diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 7eee250b9b..005906884b 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -27,6 +27,7 @@ dependencies = [ "django-redis==5.4.0", "django-tenants==3.5.0", "drf-standardized-errors>=0.12.6", + "drf-spectacular>=0.28.0", # For the generated API deployment OpenAPI spec "drf-yasg>=1.21.8", # For API docs "psycopg2-binary==2.9.9", "python-dotenv==1.2.2", diff --git a/backend/uv.lock b/backend/uv.lock index c80f0ecea6..0c94322123 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -900,6 +900,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/db/be/3032490fa33b36ddc8c4b1da3252c6f974e7133f1a50de00c6b85cca203a/docker-6.1.3-py3-none-any.whl", hash = "sha256:aecd2277b8bf8e506e484f6ab7aec39abe0038e29fa4a6d3ba86c3fe01844ed9", size = 148096, upload-time = "2023-06-01T14:24:47.769Z" }, ] +[[package]] +name = "drf-spectacular" +version = "0.30.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "django" }, + { name = "djangorestframework" }, + { name = "inflection" }, + { name = "jsonschema" }, + { name = "pyyaml" }, + { name = "uritemplate" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/50/43/41d25039a6a53545420ebc98eb9f877ec9fe30c7bd03fefabcaf9b953af7/drf_spectacular-0.30.0.tar.gz", hash = "sha256:53e79e7ba00e240441b63c32273754a5368e4c2ab44a19f2595277cc1cd559c9", size = 252311, upload-time = "2026-07-06T11:29:46.264Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/56/74dd7b45bbde6d24494220b98d6961cb1200b63a1800332b430daa2c4551/drf_spectacular-0.30.0-py3-none-any.whl", hash = "sha256:006cf5921ebe20a9bd24f7c846261ebbf78780be5961b0d6e87afaa82afd62ff", size = 111150, upload-time = "2026-07-06T11:29:45.12Z" }, +] + [[package]] name = "drf-standardized-errors" version = "0.15.0" @@ -3688,6 +3705,7 @@ dependencies = [ { name = "django-redis" }, { name = "django-tenants" }, { name = "djangorestframework" }, + { name = "drf-spectacular" }, { name = "drf-standardized-errors" }, { name = "drf-yasg" }, { name = "google-cloud-recaptcha-enterprise" }, @@ -3755,6 +3773,7 @@ requires-dist = [ { name = "django-redis", specifier = "==5.4.0" }, { name = "django-tenants", specifier = "==3.5.0" }, { name = "djangorestframework", specifier = "==3.17.1" }, + { name = "drf-spectacular", specifier = ">=0.28.0" }, { name = "drf-standardized-errors", specifier = ">=0.12.6" }, { name = "drf-yasg", specifier = ">=1.21.8" }, { name = "google-cloud-recaptcha-enterprise", specifier = ">=1.28.2" }, diff --git a/specs/docstudio-oss.json b/specs/docstudio-oss.json new file mode 100644 index 0000000000..424b30ff80 --- /dev/null +++ b/specs/docstudio-oss.json @@ -0,0 +1,442 @@ +{ + "components": { + "schemas": { + "ErrorResponse": { + "properties": { + "message": { + "nullable": true + }, + "status": { + "type": "string" + } + }, + "type": "object" + }, + "ExecuteRequest": { + "description": "Subclasses the real serializer so every backend param arrives free.", + "properties": { + "custom_data": { + "nullable": true + }, + "files": { + "items": { + "format": "binary", + "type": "string" + }, + "type": "array" + }, + "hitl_packet_id": { + "nullable": true, + "type": "string" + }, + "hitl_queue_name": { + "nullable": true, + "type": "string" + }, + "include_extracted_text": { + "default": false, + "type": "boolean" + }, + "include_metadata": { + "default": false, + "type": "boolean" + }, + "include_metrics": { + "default": false, + "type": "boolean" + }, + "llm_profile_id": { + "nullable": true, + "type": "string" + }, + "presigned_urls": { + "items": { + "format": "uri", + "type": "string" + }, + "type": "array" + }, + "tags": { + "default": "", + "description": "Comma-separated list of tag names (EX:'tag1,tag2-name,tag3_name')", + "type": "string" + }, + "timeout": { + "default": -1, + "maximum": 300, + "minimum": -1, + "type": "integer" + }, + "use_file_history": { + "default": false, + "type": "boolean" + } + }, + "type": "object" + }, + "ExecuteResponse": { + "properties": { + "message": { + "$ref": "#/components/schemas/ExecutionMessage" + } + }, + "required": [ + "message" + ], + "type": "object" + }, + "ExecutionMessage": { + "properties": { + "error": { + "nullable": true, + "type": "string" + }, + "execution_id": { + "type": "string" + }, + "execution_status": { + "type": "string" + }, + "result": { + "items": { + "$ref": "#/components/schemas/FileResult" + }, + "nullable": true, + "type": "array" + }, + "status_api": { + "nullable": true, + "type": "string" + }, + "workflow_id": { + "type": "string" + } + }, + "required": [ + "execution_status" + ], + "type": "object" + }, + "FileResult": { + "properties": { + "error": { + "nullable": true, + "type": "string" + }, + "file": { + "type": "string" + }, + "file_execution_id": { + "type": "string" + }, + "metadata": {}, + "metrics": {}, + "result": {}, + "status": { + "type": "string" + } + }, + "required": [ + "file" + ], + "type": "object" + }, + "StatusResponse": { + "properties": { + "message": { + "items": { + "$ref": "#/components/schemas/FileResult" + }, + "nullable": true, + "type": "array" + }, + "status": { + "type": "string" + } + }, + "required": [ + "status" + ], + "type": "object" + } + }, + "securitySchemes": { + "basicAuth": { + "scheme": "basic", + "type": "http" + }, + "cookieAuth": { + "in": "cookie", + "name": "sessionid", + "type": "apiKey" + } + } + }, + "info": { + "title": "Unstract Document Studio", + "version": "v1" + }, + "openapi": "3.0.3", + "paths": { + "/deployment/api/{org_name}/{api_name}/": { + "get": { + "description": "Poll the status of a previously started execution.", + "operationId": "status", + "parameters": [ + { + "description": "API deployment name.", + "in": "path", + "name": "api_name", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "execution_id", + "required": true, + "schema": { + "minLength": 1, + "type": "string" + } + }, + { + "in": "query", + "name": "include_extracted_text", + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "in": "query", + "name": "include_metadata", + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "in": "query", + "name": "include_metrics", + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "description": "Organization identifier.", + "in": "path", + "name": "org_name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/StatusResponse" + } + } + }, + "description": "" + }, + "406": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/StatusResponse" + } + } + }, + "description": "" + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "" + } + }, + "security": [ + { + "cookieAuth": [] + }, + { + "basicAuth": [] + } + ], + "tags": [ + "deployment" + ] + }, + "post": { + "description": "Execute an API deployment against one or more files.", + "operationId": "execute", + "parameters": [ + { + "description": "API deployment name.", + "in": "path", + "name": "api_name", + "required": true, + "schema": { + "type": "string" + } + }, + { + "description": "Organization identifier.", + "in": "path", + "name": "org_name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "$ref": "#/components/schemas/ExecuteRequest" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ExecuteResponse" + } + } + }, + "description": "" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ExecuteResponse" + } + } + }, + "description": "" + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "" + } + }, + "security": [ + { + "cookieAuth": [] + }, + { + "basicAuth": [] + } + ], + "tags": [ + "deployment" + ] + } + }, + "/deployment/api/{org_name}/{api_name}/mcp/": { + "get": { + "description": "Refuse the SSE stream, but say who is here.\n\nUnder Streamable HTTP a client issues GET to open a server-to-client\nSSE stream, and a server that offers none must answer 405 (spec rev\n2025-06-18). Nothing here pushes messages \u2014 every tool call is\nrequest/response \u2014 so 405 is the honest answer, and returning\n``200 application/json`` instead would leave a conformant client\nparsing an identity document as an event stream.\n\nThe body is kept anyway: uptime checks and humans with curl probe this\npath, and a 405 may carry one. It stays deliberately free of tenant\ndetail \u2014 it reveals only that an MCP server is mounted here.\n\n``JsonResponse``, not DRF's ``Response``, for the same reason ``post``\nuses it: a DRF response runs content negotiation, so a client sending\n``Accept: text/html`` would be handed the browsable-API renderer.\n\nNo ``Allow`` header is set here. RFC 9110 asks for one on a 405, but a\nhandler cannot control it and pretending otherwise misleads a reader:\nDRF's ``finalize_response`` overwrites any handler-set value with\n``self.allowed_methods`` (``GET, POST, HEAD, OPTIONS``, since this view\ndefines both verbs), and ``RemoveAllowHeaderMiddleware`` \u2014 global in\n``MIDDLEWARE`` \u2014 then pops the header from every response before it\nleaves the process. So a client sees no ``Allow`` at all; a test driving\nthe view through ``APIRequestFactory`` bypasses that middleware and sees\nDRF's value.", + "operationId": "mcp_retrieve", + "parameters": [ + { + "in": "path", + "name": "api_name", + "required": true, + "schema": { + "pattern": "^[\\w-]+$", + "type": "string" + } + }, + { + "in": "path", + "name": "org_name", + "required": true, + "schema": { + "pattern": "^[\\w-]+$", + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "No response body" + } + }, + "tags": [ + "mcp" + ] + }, + "post": { + "description": "Handle a single JSON-RPC request.", + "operationId": "mcp_create", + "parameters": [ + { + "in": "path", + "name": "api_name", + "required": true, + "schema": { + "pattern": "^[\\w-]+$", + "type": "string" + } + }, + { + "in": "path", + "name": "org_name", + "required": true, + "schema": { + "pattern": "^[\\w-]+$", + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "No response body" + } + }, + "tags": [ + "mcp" + ] + } + } + }, + "tags": [ + { + "description": "Run an API deployment against one or more documents and poll the result.", + "name": "deployment" + } + ] +} From 482ca1b0d15b26f36b00ca50d0b4fc5debffabeb Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Wed, 12 Aug 2026 17:38:21 +0530 Subject: [PATCH 2/7] fix(api): publish the deployment contract the server actually implements The committed spec is what published clients are generated from, so the places where it disagreed with the server are places every SDK inherits. - Declare the bearer scheme the endpoints enforce. DRF's unset authentication default was being introspected as a decision and published session and basic auth, which these endpoints do not accept. - Declare the failures a caller has to handle (400/401/403/404/409/429) and describe the 406, so a generated client can branch on them. - Derive the response model from the serializer that builds the response, which drops `workflow_id` -- a field no code path produces. - Stop shadowing `files`: the real field carries the binary annotation, so a change to it now moves the spec. - Drop the MCP operations. MCP speaks JSON-RPC over one POST, so it had no REST shape to describe and was published with guessed responses, no security, and an internal docstring as its description. - Say in the shipped text that a status read is one-shot, and that documents may be supplied as files or presigned URLs. The gate had the same blind spots. It now resolves the real mount instead of comparing against a hand-written copy of it, fails when the generator reports a diagnostic instead of certifying its guess, and asserts the auth scheme and error statuses. Verified by mutation: moving the mount, adding a response field, changing the `files` constraint and dropping the auth annotation each redden the suite, and none of them did before. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014f9oEEYspPH4fmPULTnLkJ --- backend/api_v2/api_deployment_views.py | 109 ++++++--- backend/api_v2/deployment_spec_urls.py | 22 +- .../commands/generate_docstudio_spec.py | 51 +++- backend/api_v2/serializers.py | 11 +- backend/api_v2/tests/test_docstudio_spec.py | 83 +++++-- backend/backend/settings/base.py | 18 +- backend/mcp_server/views.py | 5 + backend/pyproject.toml | 4 +- backend/uv.lock | 2 +- specs/docstudio-oss.json | 219 ++++++++++-------- 10 files changed, 372 insertions(+), 152 deletions(-) diff --git a/backend/api_v2/api_deployment_views.py b/backend/api_v2/api_deployment_views.py index 6fb1e08234..4dce5967b2 100644 --- a/backend/api_v2/api_deployment_views.py +++ b/backend/api_v2/api_deployment_views.py @@ -5,11 +5,11 @@ from django.db.models import F, OuterRef, QuerySet, Subquery from django.http import HttpResponse -from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import ( OpenApiParameter, + OpenApiResponse, extend_schema, - extend_schema_field, + extend_schema_serializer, extend_schema_view, ) from permissions.membership_views import OwnerManagementMixin @@ -44,6 +44,7 @@ from api_v2.serializers import ( APIDeploymentListSerializer, APIDeploymentSerializer, + APIExecutionResponseSerializer, DeploymentResponseSerializer, ExecutionQuerySerializer, ExecutionRequestSerializer, @@ -57,18 +58,16 @@ logger = logging.getLogger(__name__) -@extend_schema_field(OpenApiTypes.BINARY) -class UploadField(serializers.FileField): - """A bare ``FileField`` maps to ``format: uri`` — correct on output, wrong - for a multipart upload, and generators emit ``str`` for it. - """ - - +# Declares no field of its own, so every backend parameter arrives free and a +# change to the real serializer moves the spec. It exists only to carry a +# caller-facing description in place of the implementation docstring, and to +# keep the published model name stable. +@extend_schema_serializer(component_name="ExecuteRequest") class ExecuteRequest(ExecutionRequestSerializer): - """Subclasses the real serializer so every backend param arrives free.""" + """The documents to run, and the options that shape the result. - # ``files`` arrives via ``request.FILES``, so no serializer declares it. - files = serializers.ListField(child=UploadField(), required=False) + Supply `files`, `presigned_urls`, or both. + """ class FileResult(serializers.Serializer): @@ -81,14 +80,19 @@ class FileResult(serializers.Serializer): error = serializers.CharField(required=False, allow_null=True) -class ExecutionMessage(serializers.Serializer): - execution_status = serializers.CharField() - execution_id = serializers.CharField(required=False) - workflow_id = serializers.CharField(required=False) - status_api = serializers.CharField(required=False, allow_null=True) - error = serializers.CharField(required=False, allow_null=True) - # The backend sends `result: null` while pending; without allow_null the - # generated deserialiser iterates None and crashes. +# Subclasses the serializer that builds the response, so a field added or +# removed there moves the spec. Docstrings on these annotation serializers are +# published as the client-facing model description, so they are written for +# the caller rather than the maintainer. +class ExecutionMessage(APIExecutionResponseSerializer): + """The execution's identity and, once it has finished, its per-file + results. + """ + + # The one field that has to be restated: the real declaration is an + # untyped JSONField, which gives generated clients nothing to work with. + # The backend also sends `result: null` while pending, and without + # allow_null the generated deserialiser iterates None and crashes. result = FileResult(many=True, required=False, allow_null=True) @@ -106,46 +110,95 @@ class ErrorResponse(serializers.Serializer): message = serializers.JSONField(required=False, allow_null=True) +# The pattern the route itself enforces, restated so a generated client can +# reject a mistyped identifier without a round trip. +PATH_SEGMENT = {"type": "string", "pattern": r"^[\w-]+$"} + DEPLOYMENT_PATH_PARAMETERS = [ OpenApiParameter( "org_name", - str, + PATH_SEGMENT, OpenApiParameter.PATH, description="Organization identifier.", ), OpenApiParameter( - "api_name", str, OpenApiParameter.PATH, description="API deployment name." + "api_name", + PATH_SEGMENT, + OpenApiParameter.PATH, + description="API deployment name.", ), ] +DEPLOYMENT_AUTH = [{"deploymentKey": []}] + +# Every failure a caller has to handle. Declared explicitly because a client +# generated without them treats an authentication or rate-limit response as an +# unknown status and has nothing to branch on. +DEPLOYMENT_ERRORS = { + 400: OpenApiResponse(ErrorResponse, description="The request failed validation."), + 401: OpenApiResponse(ErrorResponse, description="The API key is not valid."), + 403: OpenApiResponse(ErrorResponse, description="No API key was supplied."), + 404: OpenApiResponse(ErrorResponse, description="No such active deployment."), + 429: OpenApiResponse( + ErrorResponse, description="Too many concurrent executions; retry later." + ), + 500: ErrorResponse, +} + +EXECUTE_DESCRIPTION = ( + "Execute an API deployment against one or more documents.\n\n" + "Supply the documents either as `files` (multipart upload) or as " + "`presigned_urls` (HTTPS S3 URLs), or both — a request carrying neither is " + f"rejected, and the two together may not exceed " + f"{ExecutionRequestSerializer.MAX_FILES_ALLOWED} documents.\n\n" + "With the default `timeout` of -1 the call returns as soon as the " + "execution is queued; read the outcome from the status endpoint." +) + +STATUS_DESCRIPTION = ( + "Read the result of a previously started execution.\n\n" + "This read is one-shot: the first call that observes a completed execution " + "acknowledges it and the stored result is discarded, so every later call " + "for that execution answers 406. Poll while the execution is pending, and " + "keep the payload of the call that returns it — it cannot be fetched again." +) + + # The generated clients take their command names, module paths and request # shapes from here, so this block is part of the public API surface. @extend_schema_view( post=extend_schema( operation_id="execute", tags=["deployment"], + auth=DEPLOYMENT_AUTH, parameters=DEPLOYMENT_PATH_PARAMETERS, request={"multipart/form-data": ExecuteRequest}, responses={ 200: ExecuteResponse, + 409: OpenApiResponse( + ErrorResponse, description="The deployment has no active API key." + ), 422: ExecuteResponse, - 500: ErrorResponse, + **DEPLOYMENT_ERRORS, }, - description="Execute an API deployment against one or more files.", + description=EXECUTE_DESCRIPTION, ), get=extend_schema( operation_id="status", tags=["deployment"], + auth=DEPLOYMENT_AUTH, parameters=DEPLOYMENT_PATH_PARAMETERS + [ExecutionQuerySerializer], - # 406 means the result was already consumed — this GET is one-shot. responses={ 200: StatusResponse, - 406: ErrorResponse, + 406: OpenApiResponse( + ErrorResponse, + description="The result was already consumed by an earlier call.", + ), 422: StatusResponse, - 500: ErrorResponse, + **DEPLOYMENT_ERRORS, }, - description="Poll the status of a previously started execution.", + description=STATUS_DESCRIPTION, ), ) class DeploymentExecution(views.APIView): diff --git a/backend/api_v2/deployment_spec_urls.py b/backend/api_v2/deployment_spec_urls.py index f61d714d2c..0aa2d998bd 100644 --- a/backend/api_v2/deployment_spec_urls.py +++ b/backend/api_v2/deployment_spec_urls.py @@ -2,12 +2,26 @@ ``api_v2.execution_urls`` is an included sub-urlconf, so generating against it directly yields paths without the prefix it is mounted at — a spec describing -URLs the server does not serve. This mirrors the mount in ``base_urls``. +URLs the server does not serve. The mount is selected out of the served +urlconf rather than restated here, so a change to where the deployment API is +mounted moves the generated paths with it. """ -from django.conf import settings -from django.urls import include, path +from django.core.exceptions import ImproperlyConfigured + +from backend import base_urls + +DEPLOYMENT_URLCONF = "api_v2.execution_urls" urlpatterns = [ - path(f"{settings.API_DEPLOYMENT_PATH_PREFIX}/", include("api_v2.execution_urls")) + entry + for entry in base_urls.urlpatterns + if getattr(getattr(entry, "urlconf_name", None), "__name__", None) + == DEPLOYMENT_URLCONF ] + +if not urlpatterns: + raise ImproperlyConfigured( + f"{DEPLOYMENT_URLCONF} is not mounted in backend.base_urls; the API " + "deployment spec would be generated for no routes at all." + ) diff --git a/backend/api_v2/management/commands/generate_docstudio_spec.py b/backend/api_v2/management/commands/generate_docstudio_spec.py index 601d59506c..a091073a39 100644 --- a/backend/api_v2/management/commands/generate_docstudio_spec.py +++ b/backend/api_v2/management/commands/generate_docstudio_spec.py @@ -4,8 +4,12 @@ built from, so it is committed and CI fails on drift: change a route, a serializer or the schema annotation, and regenerate in the same PR. - python manage.py generate_docstudio_spec - python manage.py generate_docstudio_spec --check # CI: no write, drift is an error + uv run python manage.py generate_docstudio_spec # from backend/ + uv run python manage.py generate_docstudio_spec --check # no write, drift is an error + +The generated paths carry ``API_DEPLOYMENT_PATH_PREFIX``, so regenerate in an +environment that does not override it — the committed artifact describes the +deployment as it is served publicly, not as one installation mounts it. """ import json @@ -13,10 +17,44 @@ from typing import Any from django.core.management.base import BaseCommand, CommandError +from drf_spectacular.drainage import GENERATOR_STATS from drf_spectacular.generators import SchemaGenerator DEFAULT_OUT = Path(__file__).resolve().parents[4] / "specs" / "docstudio-oss.json" URLCONF = "api_v2.deployment_spec_urls" +REGENERATE = "uv run python manage.py generate_docstudio_spec" + + +class SpecGenerationFailed(CommandError): + """Raised when the generator had to guess.""" + + +def render_spec() -> str: + """The committed artifact, byte for byte. + + Shared with the drift test: two copies of this could disagree, and then + the gate rejects exactly the file the command it names produces. + """ + GENERATOR_STATS.reset() + schema = SchemaGenerator(urlconf=URLCONF).get_schema(request=None, public=True) + if GENERATOR_STATS: + # spectacular downgrades "unable to guess serializer" to a warning and + # writes a plausible, wrong operation. Nothing downstream can tell that + # apart from an annotation that is simply thin. + diagnostics = "\n".join( + f" {severity}: {message}" + for severity, cache in ( + ("error", GENERATOR_STATS._error_cache), + ("warning", GENERATOR_STATS._warn_cache), + ) + for message in cache + ) + raise SpecGenerationFailed( + f"The generator reported problems, so the spec would describe an " + f"API nobody implements:\n{diagnostics}" + ) + # Sorted keys are what make the committed artifact a usable drift signal. + return json.dumps(schema, indent=2, sort_keys=True) + "\n" class Command(BaseCommand): @@ -31,23 +69,22 @@ def add_arguments(self, parser: Any) -> None: ) def handle(self, *args: Any, **options: Any) -> None: - schema = SchemaGenerator(urlconf=URLCONF).get_schema(request=None, public=True) - # Sorted keys are what make the committed artifact a usable drift signal. - rendered = json.dumps(schema, indent=2, sort_keys=True) + "\n" + rendered = render_spec() out: Path = options["out"] if options["check"]: current = out.read_text() if out.exists() else "" if current != rendered: raise CommandError( - f"{out} is out of date. Run `python manage.py " - f"generate_docstudio_spec` and commit the result." + f"{out} is out of date. Run `{REGENERATE}` from `backend/` " + "and commit the result." ) self.stdout.write(f"{out} is up to date") return out.parent.mkdir(parents=True, exist_ok=True) out.write_text(rendered) + schema = json.loads(rendered) operations = sum( 1 for methods in schema["paths"].values() diff --git a/backend/api_v2/serializers.py b/backend/api_v2/serializers.py index 3db7f53db6..e376cd401b 100644 --- a/backend/api_v2/serializers.py +++ b/backend/api_v2/serializers.py @@ -6,6 +6,8 @@ from django.apps import apps from django.core.validators import RegexValidator +from drf_spectacular.types import OpenApiTypes +from drf_spectacular.utils import extend_schema_field from pipeline_v2.models import Pipeline from prompt_studio.prompt_profile_manager_v2.models import ProfileManager from rest_framework import serializers @@ -218,6 +220,13 @@ def to_representation(self, instance: APIKey) -> OrderedDict[str, Any]: return representation +@extend_schema_field(OpenApiTypes.BINARY) +class UploadField(FileField): + """A bare ``FileField`` maps to ``format: uri`` -- correct on output, wrong + for a multipart upload, and generators emit ``str`` for it. + """ + + class ExecutionRequestSerializer(TagParamsSerializer): """Execution request serializer. @@ -320,7 +329,7 @@ def validate_custom_data(self, value): return value files = ListField( - child=FileField(), + child=UploadField(), required=False, allow_empty=True, ) diff --git a/backend/api_v2/tests/test_docstudio_spec.py b/backend/api_v2/tests/test_docstudio_spec.py index 602d8b5c9f..b94f647450 100644 --- a/backend/api_v2/tests/test_docstudio_spec.py +++ b/backend/api_v2/tests/test_docstudio_spec.py @@ -6,33 +6,82 @@ """ import json -from pathlib import Path -from drf_spectacular.generators import SchemaGenerator +from django.urls import reverse +from drf_spectacular.drainage import GENERATOR_STATS -from api_v2.management.commands.generate_docstudio_spec import DEFAULT_OUT, URLCONF +from api_v2.management.commands.generate_docstudio_spec import ( + DEFAULT_OUT, + REGENERATE, + render_spec, +) -def _render() -> str: - schema = SchemaGenerator(urlconf=URLCONF).get_schema(request=None, public=True) - return json.dumps(schema, indent=2, sort_keys=True) + "\n" +def _committed() -> dict: + return json.loads(DEFAULT_OUT.read_text()) def test_committed_spec_matches_the_code() -> None: assert DEFAULT_OUT.exists(), f"{DEFAULT_OUT} is missing" - assert DEFAULT_OUT.read_text() == _render(), ( - f"{DEFAULT_OUT} is out of date. Run " - "`python manage.py generate_docstudio_spec` and commit the result." + assert DEFAULT_OUT.read_text() == render_spec(), ( + f"{DEFAULT_OUT} is out of date. Run `{REGENERATE}` from `backend/` and " + "commit the result." ) -def test_spec_covers_the_deployment_routes() -> None: - """Guards the mount: generating against the included sub-urlconf silently - drops the prefix, leaving paths the server does not serve.""" - spec = json.loads(Path(DEFAULT_OUT).read_text()) - execute = "/deployment/api/{org_name}/{api_name}/" +def test_generation_reports_no_diagnostics() -> None: + """A warned-about operation is published with guessed request and response + shapes, and the drift comparison certifies the guess.""" + render_spec() + assert not GENERATOR_STATS._error_cache + assert not GENERATOR_STATS._warn_cache - assert set(spec["paths"]) == {execute, f"{execute}mcp/"} - assert spec["paths"][execute]["post"]["operationId"] == "execute" - assert spec["paths"][execute]["get"]["operationId"] == "status" + +def test_spec_paths_are_the_urls_the_server_serves() -> None: + """Resolves the real mount rather than restating it: a spec generated for + URLs the server does not serve is the failure this file exists to catch.""" + served = reverse( + "api_deployment_execution", kwargs={"org_name": "ORG", "api_name": "API"} + ) + documented = [ + path.replace("{org_name}", "ORG").replace("{api_name}", "API").rstrip("/") + for path in _committed()["paths"] + ] + + assert documented == [served.rstrip("/")] + + +def test_spec_documents_the_deployment_operations() -> None: + spec = _committed() + (operations,) = spec["paths"].values() + + assert operations["post"]["operationId"] == "execute" + assert operations["get"]["operationId"] == "status" assert [tag["name"] for tag in spec["tags"]] == ["deployment"] + + +def test_operations_require_the_deployment_key() -> None: + """Without this the unset DRF authentication default is published as + though it were a decision, and no generated client can authenticate.""" + spec = _committed() + scheme = spec["components"]["securitySchemes"]["deploymentKey"] + + assert (scheme["type"], scheme["scheme"]) == ("http", "bearer") + for operation in spec["paths"].values(): + for method in ("get", "post"): + assert operation[method]["security"] == [{"deploymentKey": []}] + + +def test_clients_can_branch_on_every_failure_they_will_see() -> None: + for method in ("get", "post"): + (operation,) = (ops[method] for ops in _committed()["paths"].values()) + assert {"400", "401", "403", "404", "429", "500"} <= set(operation["responses"]) + + +def test_the_one_shot_read_is_documented_where_a_client_will_see_it() -> None: + """The semantics that a status read destroys the result must reach the + generated client, not live in a source comment.""" + (status_op,) = (ops["get"] for ops in _committed()["paths"].values()) + + assert "one-shot" in status_op["description"] + assert status_op["responses"]["406"]["description"].strip() diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py index bbc6b440d4..a00c2fcc5c 100644 --- a/backend/backend/settings/base.py +++ b/backend/backend/settings/base.py @@ -656,13 +656,27 @@ def filter(self, record): "DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema", } -# Read only while generating the API deployment OpenAPI spec -# (``manage.py generate_docstudio_spec``); no effect at request time. +# Read while generating the API deployment OpenAPI spec +# (``manage.py generate_docstudio_spec``). ``DEFAULT_SCHEMA_CLASS`` above is a +# project-wide DRF default, but DRF dereferences it only when a schema is +# generated, so neither has an effect at request time. SPECTACULAR_SETTINGS = { "TITLE": "Unstract Document Studio", "VERSION": "v1", "PREPROCESSING_HOOKS": ["drf_spectacular.hooks.preprocess_exclude_path_format"], "SERVE_INCLUDE_SCHEMA": False, + # DRF's unset ``DEFAULT_AUTHENTICATION_CLASSES`` would otherwise be + # introspected as a decision and publish session and basic auth, which + # these endpoints do not accept. + "APPEND_COMPONENTS": { + "securitySchemes": { + "deploymentKey": { + "type": "http", + "scheme": "bearer", + "description": "The API deployment's own key.", + } + } + }, # Group descriptions clients show in their help; without this the spec has # no root `tags` array and the text has nowhere to live. "TAGS": [ diff --git a/backend/mcp_server/views.py b/backend/mcp_server/views.py index 53ef1f4781..7c85f5d237 100644 --- a/backend/mcp_server/views.py +++ b/backend/mcp_server/views.py @@ -14,6 +14,7 @@ from typing import Any from api_v2.deployment_helper import DeploymentHelper +from drf_spectacular.utils import extend_schema from rest_framework.request import Request from mcp_server.context import MCPContext @@ -23,6 +24,10 @@ logger = logging.getLogger(__name__) +# MCP speaks JSON-RPC over one POST, so it has no REST surface worth +# describing; leaving it in would publish guessed request and response shapes +# to every client generated from the spec. +@extend_schema(exclude=True) class MCPServerView(BaseMCPView): """MCP JSON-RPC endpoint for a single API deployment. diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 005906884b..1cd74ecdcb 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -27,7 +27,9 @@ dependencies = [ "django-redis==5.4.0", "django-tenants==3.5.0", "drf-standardized-errors>=0.12.6", - "drf-spectacular>=0.28.0", # For the generated API deployment OpenAPI spec + # Pinned: its rendering is the committed spec, so an upgrade rewrites the + # contract published clients are generated from. + "drf-spectacular==0.30.0", "drf-yasg>=1.21.8", # For API docs "psycopg2-binary==2.9.9", "python-dotenv==1.2.2", diff --git a/backend/uv.lock b/backend/uv.lock index 0c94322123..52bd298262 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -3773,7 +3773,7 @@ requires-dist = [ { name = "django-redis", specifier = "==5.4.0" }, { name = "django-tenants", specifier = "==3.5.0" }, { name = "djangorestframework", specifier = "==3.17.1" }, - { name = "drf-spectacular", specifier = ">=0.28.0" }, + { name = "drf-spectacular", specifier = "==0.30.0" }, { name = "drf-standardized-errors", specifier = ">=0.12.6" }, { name = "drf-yasg", specifier = ">=1.21.8" }, { name = "google-cloud-recaptcha-enterprise", specifier = ">=1.28.2" }, diff --git a/specs/docstudio-oss.json b/specs/docstudio-oss.json index 424b30ff80..265b7fac10 100644 --- a/specs/docstudio-oss.json +++ b/specs/docstudio-oss.json @@ -13,7 +13,7 @@ "type": "object" }, "ExecuteRequest": { - "description": "Subclasses the real serializer so every backend param arrives free.", + "description": "The documents to run, and the options that shape the result.\n\nSupply `files`, `presigned_urls`, or both.", "properties": { "custom_data": { "nullable": true @@ -86,9 +86,9 @@ "type": "object" }, "ExecutionMessage": { + "description": "The execution's identity and, once it has finished, its per-file\nresults.", "properties": { "error": { - "nullable": true, "type": "string" }, "execution_id": { @@ -105,15 +105,14 @@ "type": "array" }, "status_api": { - "nullable": true, - "type": "string" - }, - "workflow_id": { "type": "string" } }, "required": [ - "execution_status" + "error", + "execution_id", + "execution_status", + "status_api" ], "type": "object" }, @@ -161,14 +160,10 @@ } }, "securitySchemes": { - "basicAuth": { - "scheme": "basic", + "deploymentKey": { + "description": "The API deployment's own key.", + "scheme": "bearer", "type": "http" - }, - "cookieAuth": { - "in": "cookie", - "name": "sessionid", - "type": "apiKey" } } }, @@ -180,7 +175,7 @@ "paths": { "/deployment/api/{org_name}/{api_name}/": { "get": { - "description": "Poll the status of a previously started execution.", + "description": "Read the result of a previously started execution.\n\nThis read is one-shot: the first call that observes a completed execution acknowledges it and the stored result is discarded, so every later call for that execution answers 406. Poll while the execution is pending, and keep the payload of the call that returns it \u2014 it cannot be fetched again.", "operationId": "status", "parameters": [ { @@ -189,6 +184,7 @@ "name": "api_name", "required": true, "schema": { + "pattern": "^[\\w-]+$", "type": "string" } }, @@ -231,6 +227,7 @@ "name": "org_name", "required": true, "schema": { + "pattern": "^[\\w-]+$", "type": "string" } } @@ -246,6 +243,46 @@ }, "description": "" }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "The request failed validation." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "The API key is not valid." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "No API key was supplied." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "No such active deployment." + }, "406": { "content": { "application/json": { @@ -254,7 +291,7 @@ } } }, - "description": "" + "description": "The result was already consumed by an earlier call." }, "422": { "content": { @@ -266,6 +303,16 @@ }, "description": "" }, + "429": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "Too many concurrent executions; retry later." + }, "500": { "content": { "application/json": { @@ -279,10 +326,7 @@ }, "security": [ { - "cookieAuth": [] - }, - { - "basicAuth": [] + "deploymentKey": [] } ], "tags": [ @@ -290,7 +334,7 @@ ] }, "post": { - "description": "Execute an API deployment against one or more files.", + "description": "Execute an API deployment against one or more documents.\n\nSupply the documents either as `files` (multipart upload) or as `presigned_urls` (HTTPS S3 URLs), or both \u2014 a request carrying neither is rejected, and the two together may not exceed 32 documents.\n\nWith the default `timeout` of -1 the call returns as soon as the execution is queued; read the outcome from the status endpoint.", "operationId": "execute", "parameters": [ { @@ -299,6 +343,7 @@ "name": "api_name", "required": true, "schema": { + "pattern": "^[\\w-]+$", "type": "string" } }, @@ -308,6 +353,7 @@ "name": "org_name", "required": true, "schema": { + "pattern": "^[\\w-]+$", "type": "string" } } @@ -332,6 +378,56 @@ }, "description": "" }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "The request failed validation." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "The API key is not valid." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "No API key was supplied." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "No such active deployment." + }, + "409": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "The deployment has no active API key." + }, "422": { "content": { "application/json": { @@ -342,6 +438,16 @@ }, "description": "" }, + "429": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "Too many concurrent executions; retry later." + }, "500": { "content": { "application/json": { @@ -355,82 +461,13 @@ }, "security": [ { - "cookieAuth": [] - }, - { - "basicAuth": [] + "deploymentKey": [] } ], "tags": [ "deployment" ] } - }, - "/deployment/api/{org_name}/{api_name}/mcp/": { - "get": { - "description": "Refuse the SSE stream, but say who is here.\n\nUnder Streamable HTTP a client issues GET to open a server-to-client\nSSE stream, and a server that offers none must answer 405 (spec rev\n2025-06-18). Nothing here pushes messages \u2014 every tool call is\nrequest/response \u2014 so 405 is the honest answer, and returning\n``200 application/json`` instead would leave a conformant client\nparsing an identity document as an event stream.\n\nThe body is kept anyway: uptime checks and humans with curl probe this\npath, and a 405 may carry one. It stays deliberately free of tenant\ndetail \u2014 it reveals only that an MCP server is mounted here.\n\n``JsonResponse``, not DRF's ``Response``, for the same reason ``post``\nuses it: a DRF response runs content negotiation, so a client sending\n``Accept: text/html`` would be handed the browsable-API renderer.\n\nNo ``Allow`` header is set here. RFC 9110 asks for one on a 405, but a\nhandler cannot control it and pretending otherwise misleads a reader:\nDRF's ``finalize_response`` overwrites any handler-set value with\n``self.allowed_methods`` (``GET, POST, HEAD, OPTIONS``, since this view\ndefines both verbs), and ``RemoveAllowHeaderMiddleware`` \u2014 global in\n``MIDDLEWARE`` \u2014 then pops the header from every response before it\nleaves the process. So a client sees no ``Allow`` at all; a test driving\nthe view through ``APIRequestFactory`` bypasses that middleware and sees\nDRF's value.", - "operationId": "mcp_retrieve", - "parameters": [ - { - "in": "path", - "name": "api_name", - "required": true, - "schema": { - "pattern": "^[\\w-]+$", - "type": "string" - } - }, - { - "in": "path", - "name": "org_name", - "required": true, - "schema": { - "pattern": "^[\\w-]+$", - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "No response body" - } - }, - "tags": [ - "mcp" - ] - }, - "post": { - "description": "Handle a single JSON-RPC request.", - "operationId": "mcp_create", - "parameters": [ - { - "in": "path", - "name": "api_name", - "required": true, - "schema": { - "pattern": "^[\\w-]+$", - "type": "string" - } - }, - { - "in": "path", - "name": "org_name", - "required": true, - "schema": { - "pattern": "^[\\w-]+$", - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "No response body" - } - }, - "tags": [ - "mcp" - ] - } } }, "tags": [ From 86761cc3b3d228151efe4476f61e77e6901dfd54 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Wed, 12 Aug 2026 18:28:25 +0530 Subject: [PATCH 3/7] Name the spec after the API, not one product The deployment endpoints are the public API surface, and the generated clients carry this title into their own documentation. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014f9oEEYspPH4fmPULTnLkJ --- backend/backend/settings/base.py | 2 +- specs/docstudio-oss.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py index a00c2fcc5c..cf21bd3e1e 100644 --- a/backend/backend/settings/base.py +++ b/backend/backend/settings/base.py @@ -661,7 +661,7 @@ def filter(self, record): # project-wide DRF default, but DRF dereferences it only when a schema is # generated, so neither has an effect at request time. SPECTACULAR_SETTINGS = { - "TITLE": "Unstract Document Studio", + "TITLE": "Unstract API", "VERSION": "v1", "PREPROCESSING_HOOKS": ["drf_spectacular.hooks.preprocess_exclude_path_format"], "SERVE_INCLUDE_SCHEMA": False, diff --git a/specs/docstudio-oss.json b/specs/docstudio-oss.json index 265b7fac10..edf3196660 100644 --- a/specs/docstudio-oss.json +++ b/specs/docstudio-oss.json @@ -168,7 +168,7 @@ } }, "info": { - "title": "Unstract Document Studio", + "title": "Unstract API", "version": "v1" }, "openapi": "3.0.3", From 05413c16f3536e2411fbdc50a8f910b3308179b8 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Thu, 13 Aug 2026 17:46:46 +0530 Subject: [PATCH 4/7] test: assert over every documented operation, not exactly one The spec describes one endpoint today, and five of these tests read it by unpacking a single item or by indexing get and post directly. The first endpoint added turns all five red for no reason, and a GET-only one raises KeyError. They now walk whatever the spec documents. The drift and diagnostics gates are unchanged. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014f9oEEYspPH4fmPULTnLkJ --- backend/api_v2/tests/test_docstudio_spec.py | 59 +++++++++++++++------ 1 file changed, 43 insertions(+), 16 deletions(-) diff --git a/backend/api_v2/tests/test_docstudio_spec.py b/backend/api_v2/tests/test_docstudio_spec.py index b94f647450..50b0e99cad 100644 --- a/backend/api_v2/tests/test_docstudio_spec.py +++ b/backend/api_v2/tests/test_docstudio_spec.py @@ -7,7 +7,7 @@ import json -from django.urls import reverse +from django.urls import resolve, reverse from drf_spectacular.drainage import GENERATOR_STATS from api_v2.management.commands.generate_docstudio_spec import ( @@ -17,10 +17,29 @@ ) +#: Keys under a path item that are operations. The rest -- `parameters`, +#: `summary`, vendor extensions -- describe the path, not a call. +_METHODS = ("get", "put", "post", "delete", "options", "head", "patch", "trace") + + def _committed() -> dict: return json.loads(DEFAULT_OUT.read_text()) +def _operations(spec: dict) -> list[tuple[str, str, dict]]: + """Every (path, method, operation) the spec documents. + + The spec grows an endpoint at a time, and a check written against exactly + one of them fails on the next addition without anything being wrong. + """ + return [ + (path, method, operation) + for path, path_item in spec["paths"].items() + for method, operation in path_item.items() + if method in _METHODS + ] + + def test_committed_spec_matches_the_code() -> None: assert DEFAULT_OUT.exists(), f"{DEFAULT_OUT} is missing" assert DEFAULT_OUT.read_text() == render_spec(), ( @@ -44,20 +63,22 @@ def test_spec_paths_are_the_urls_the_server_serves() -> None: "api_deployment_execution", kwargs={"org_name": "ORG", "api_name": "API"} ) documented = [ - path.replace("{org_name}", "ORG").replace("{api_name}", "API").rstrip("/") + path.replace("{org_name}", "ORG").replace("{api_name}", "API") for path in _committed()["paths"] ] - assert documented == [served.rstrip("/")] + assert served.rstrip("/") in [path.rstrip("/") for path in documented] + for path in documented: + # Raises Resolver404 if the spec documents a URL nothing answers. + resolve(path if path.endswith("/") else f"{path}/") def test_spec_documents_the_deployment_operations() -> None: spec = _committed() - (operations,) = spec["paths"].values() + documented = {operation["operationId"] for _, _, operation in _operations(spec)} - assert operations["post"]["operationId"] == "execute" - assert operations["get"]["operationId"] == "status" - assert [tag["name"] for tag in spec["tags"]] == ["deployment"] + assert {"execute", "status"} <= documented + assert "deployment" in [tag["name"] for tag in spec["tags"]] def test_operations_require_the_deployment_key() -> None: @@ -67,21 +88,27 @@ def test_operations_require_the_deployment_key() -> None: scheme = spec["components"]["securitySchemes"]["deploymentKey"] assert (scheme["type"], scheme["scheme"]) == ("http", "bearer") - for operation in spec["paths"].values(): - for method in ("get", "post"): - assert operation[method]["security"] == [{"deploymentKey": []}] + for path, method, operation in _operations(spec): + assert operation["security"] == [{"deploymentKey": []}], f"{method} {path}" def test_clients_can_branch_on_every_failure_they_will_see() -> None: - for method in ("get", "post"): - (operation,) = (ops[method] for ops in _committed()["paths"].values()) - assert {"400", "401", "403", "404", "429", "500"} <= set(operation["responses"]) + for path, method, operation in _operations(_committed()): + assert {"400", "401", "403", "404", "429", "500"} <= set( + operation["responses"] + ), f"{method} {path}" def test_the_one_shot_read_is_documented_where_a_client_will_see_it() -> None: """The semantics that a status read destroys the result must reach the generated client, not live in a source comment.""" - (status_op,) = (ops["get"] for ops in _committed()["paths"].values()) + reads = [ + operation + for _, _, operation in _operations(_committed()) + if operation["operationId"] == "status" + ] - assert "one-shot" in status_op["description"] - assert status_op["responses"]["406"]["description"].strip() + assert reads + for status_op in reads: + assert "one-shot" in status_op["description"] + assert status_op["responses"]["406"]["description"].strip() From 3ebfafbd03922de79e3382862d9c7213ed9ce397 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Tue, 18 Aug 2026 15:46:45 +0530 Subject: [PATCH 5/7] chore: drop drf-yasg now that spectacular generates the spec The `docs` app served a redoc UI over a schema drf-yasg introspected with `public=False`, so an anonymous caller saw nothing and no endpoint carried a `@swagger_auto_schema` annotation. Nothing generates or consumes it. Removes the dependency, the `docs` app and its two mounts. The `/doc/` route goes with it; drf-spectacular serves no UI, only the committed spec. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01BFSunNN6RKRA1xo6kWkztx --- backend/backend/public_urls.py | 2 -- backend/backend/public_urls_v2.py | 2 -- backend/backend/settings/base.py | 7 ++----- backend/docs/__init__.py | 0 backend/docs/urls.py | 20 -------------------- backend/pyproject.toml | 1 - backend/uv.lock | 20 -------------------- 7 files changed, 2 insertions(+), 50 deletions(-) delete mode 100644 backend/docs/__init__.py delete mode 100644 backend/docs/urls.py diff --git a/backend/backend/public_urls.py b/backend/backend/public_urls.py index 2ca815442e..3d9a8130a2 100644 --- a/backend/backend/public_urls.py +++ b/backend/backend/public_urls.py @@ -28,8 +28,6 @@ path(f"{path_prefix}/", include("account.urls")), # Connector OAuth path(f"{path_prefix}/", include("connector_auth.urls")), - # Docs - path(f"{path_prefix}/", include("docs.urls")), # API deployment path(f"{api_path_prefix}/", include("api.urls")), path(f"{api_path_prefix}/pipeline/", include("pipeline.public_api_urls")), diff --git a/backend/backend/public_urls_v2.py b/backend/backend/public_urls_v2.py index 7034e86141..336ea9cdbc 100644 --- a/backend/backend/public_urls_v2.py +++ b/backend/backend/public_urls_v2.py @@ -25,8 +25,6 @@ path("", include("account_v2.urls")), # Connector OAuth path("", include("connector_auth_v2.urls")), - # Docs - path("", include("docs.urls")), # Feature flags path("flags/", include("feature_flag.urls")), # Pipeline diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py index cf21bd3e1e..f91385bbca 100644 --- a/backend/backend/settings/base.py +++ b/backend/backend/settings/base.py @@ -371,9 +371,6 @@ def filter(self, record): # Connector OAuth # "connector_auth", "social_django", - # Doc generator - "drf_yasg", - "docs", # Plugins "plugins.apps.PluginsConfig", "feature_flag", @@ -677,8 +674,8 @@ def filter(self, record): } } }, - # Group descriptions clients show in their help; without this the spec has - # no root `tags` array and the text has nowhere to live. + # Group descriptions generated clients show in their help; without this + # the spec has no root `tags` array for the text to live in. "TAGS": [ { "name": "deployment", diff --git a/backend/docs/__init__.py b/backend/docs/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/backend/docs/urls.py b/backend/docs/urls.py deleted file mode 100644 index 83260b01db..0000000000 --- a/backend/docs/urls.py +++ /dev/null @@ -1,20 +0,0 @@ -from django.urls import path -from drf_yasg import openapi -from drf_yasg.views import get_schema_view - -schema_view = get_schema_view( - openapi.Info( - title="Unstract APIs", - default_version="v1", - description="", - ), - public=False, -) - -urlpatterns = [ - path( - "doc/", - schema_view.with_ui("redoc", cache_timeout=0), - name="schema-redoc", - ), -] diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 1cd74ecdcb..c1ca5b9ed4 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -30,7 +30,6 @@ dependencies = [ # Pinned: its rendering is the committed spec, so an upgrade rewrites the # contract published clients are generated from. "drf-spectacular==0.30.0", - "drf-yasg>=1.21.8", # For API docs "psycopg2-binary==2.9.9", "python-dotenv==1.2.2", "python-magic==0.4.27", # For file upload/download diff --git a/backend/uv.lock b/backend/uv.lock index 52bd298262..eb1952f9d2 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -930,24 +930,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/94/4e7721ff51cb10aa826cb27f3bf015e8d94d7f898c19c2b650d822019e5b/drf_standardized_errors-0.15.0-py3-none-any.whl", hash = "sha256:75dcfec11433a16c81f8c5948a5cd2932cd5b02f426f64ca82020a78c155b263", size = 25673, upload-time = "2025-06-09T07:47:55.042Z" }, ] -[[package]] -name = "drf-yasg" -version = "1.21.15" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "django" }, - { name = "djangorestframework" }, - { name = "inflection" }, - { name = "packaging" }, - { name = "pytz" }, - { name = "pyyaml" }, - { name = "uritemplate" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/4a/88/345135459b9cbaff0e8ee3270819e89ca92064a35a0a94a1cfce41c084db/drf_yasg-1.21.15.tar.gz", hash = "sha256:ef86838c4ef10dcd3ac1ebf2be601cbe02978b999671caa43667f7c9db961468", size = 5153419, upload-time = "2026-02-24T18:09:21.072Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f9/a4/400b0565cf25395f1d5e1a24e5d18dab8f4199e4212174341fea6f05747c/drf_yasg-1.21.15-py3-none-any.whl", hash = "sha256:7c7a7ab9feb0e13cdd6e25147d99adb0500a68bd96509ffd8f8cf7efd4bdc77e", size = 4856209, upload-time = "2026-02-24T18:09:18.982Z" }, -] - [[package]] name = "dropbox" version = "12.0.2" @@ -3707,7 +3689,6 @@ dependencies = [ { name = "djangorestframework" }, { name = "drf-spectacular" }, { name = "drf-standardized-errors" }, - { name = "drf-yasg" }, { name = "google-cloud-recaptcha-enterprise" }, { name = "gunicorn" }, { name = "httpx" }, @@ -3775,7 +3756,6 @@ requires-dist = [ { name = "djangorestframework", specifier = "==3.17.1" }, { name = "drf-spectacular", specifier = "==0.30.0" }, { name = "drf-standardized-errors", specifier = ">=0.12.6" }, - { name = "drf-yasg", specifier = ">=1.21.8" }, { name = "google-cloud-recaptcha-enterprise", specifier = ">=1.28.2" }, { name = "gunicorn", specifier = ">=23.0.0" }, { name = "httpx", specifier = ">=0.27.0" }, From 3198b529f3f614a1a19ef7c9878b6834685d0f0d Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Tue, 18 Aug 2026 15:46:56 +0530 Subject: [PATCH 6/7] refactor: move the deployment schema annotations out of the view The annotation serializers exist only to shape the published spec, so they sit in `api_v2/openapi_schema.py` rather than in `serializers.py`, where a request-time import of one would look ordinary. `api_deployment_views.py` keeps a single decorator. `deployment_spec_urls.py` now selects mounts from a tuple, so widening the spec to another endpoint is one entry plus its `@extend_schema`. The generated spec is unchanged: component names are all that reach it. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01BFSunNN6RKRA1xo6kWkztx --- backend/api_v2/api_deployment_views.py | 153 +---------------------- backend/api_v2/deployment_spec_urls.py | 26 ++-- backend/api_v2/openapi_schema.py | 161 +++++++++++++++++++++++++ backend/backend/settings/base.py | 18 +-- 4 files changed, 181 insertions(+), 177 deletions(-) create mode 100644 backend/api_v2/openapi_schema.py diff --git a/backend/api_v2/api_deployment_views.py b/backend/api_v2/api_deployment_views.py index 4dce5967b2..8f5d0763a9 100644 --- a/backend/api_v2/api_deployment_views.py +++ b/backend/api_v2/api_deployment_views.py @@ -5,13 +5,6 @@ from django.db.models import F, OuterRef, QuerySet, Subquery from django.http import HttpResponse -from drf_spectacular.utils import ( - OpenApiParameter, - OpenApiResponse, - extend_schema, - extend_schema_serializer, - extend_schema_view, -) from permissions.membership_views import OwnerManagementMixin from permissions.permission import IsOwner, IsOwnerOrSharedUserOrSharedToOrg from permissions.resource_share_views import ResourceShareManagementMixin @@ -40,11 +33,11 @@ contains_tool_not_found_error, ) from api_v2.models import APIDeployment +from api_v2.openapi_schema import DEPLOYMENT_EXECUTION_SCHEMA from api_v2.rate_limiter import APIDeploymentRateLimiter from api_v2.serializers import ( APIDeploymentListSerializer, APIDeploymentSerializer, - APIExecutionResponseSerializer, DeploymentResponseSerializer, ExecutionQuerySerializer, ExecutionRequestSerializer, @@ -58,149 +51,7 @@ logger = logging.getLogger(__name__) -# Declares no field of its own, so every backend parameter arrives free and a -# change to the real serializer moves the spec. It exists only to carry a -# caller-facing description in place of the implementation docstring, and to -# keep the published model name stable. -@extend_schema_serializer(component_name="ExecuteRequest") -class ExecuteRequest(ExecutionRequestSerializer): - """The documents to run, and the options that shape the result. - - Supply `files`, `presigned_urls`, or both. - """ - - -class FileResult(serializers.Serializer): - file = serializers.CharField() - file_execution_id = serializers.CharField(required=False) - status = serializers.CharField(required=False) - result = serializers.JSONField(required=False) - metadata = serializers.JSONField(required=False) - metrics = serializers.JSONField(required=False) - error = serializers.CharField(required=False, allow_null=True) - - -# Subclasses the serializer that builds the response, so a field added or -# removed there moves the spec. Docstrings on these annotation serializers are -# published as the client-facing model description, so they are written for -# the caller rather than the maintainer. -class ExecutionMessage(APIExecutionResponseSerializer): - """The execution's identity and, once it has finished, its per-file - results. - """ - - # The one field that has to be restated: the real declaration is an - # untyped JSONField, which gives generated clients nothing to work with. - # The backend also sends `result: null` while pending, and without - # allow_null the generated deserialiser iterates None and crashes. - result = FileResult(many=True, required=False, allow_null=True) - - -class ExecuteResponse(serializers.Serializer): - message = ExecutionMessage() - - -class StatusResponse(serializers.Serializer): - status = serializers.CharField() - message = FileResult(many=True, required=False, allow_null=True) - - -class ErrorResponse(serializers.Serializer): - status = serializers.CharField(required=False) - message = serializers.JSONField(required=False, allow_null=True) - - -# The pattern the route itself enforces, restated so a generated client can -# reject a mistyped identifier without a round trip. -PATH_SEGMENT = {"type": "string", "pattern": r"^[\w-]+$"} - -DEPLOYMENT_PATH_PARAMETERS = [ - OpenApiParameter( - "org_name", - PATH_SEGMENT, - OpenApiParameter.PATH, - description="Organization identifier.", - ), - OpenApiParameter( - "api_name", - PATH_SEGMENT, - OpenApiParameter.PATH, - description="API deployment name.", - ), -] - - -DEPLOYMENT_AUTH = [{"deploymentKey": []}] - -# Every failure a caller has to handle. Declared explicitly because a client -# generated without them treats an authentication or rate-limit response as an -# unknown status and has nothing to branch on. -DEPLOYMENT_ERRORS = { - 400: OpenApiResponse(ErrorResponse, description="The request failed validation."), - 401: OpenApiResponse(ErrorResponse, description="The API key is not valid."), - 403: OpenApiResponse(ErrorResponse, description="No API key was supplied."), - 404: OpenApiResponse(ErrorResponse, description="No such active deployment."), - 429: OpenApiResponse( - ErrorResponse, description="Too many concurrent executions; retry later." - ), - 500: ErrorResponse, -} - -EXECUTE_DESCRIPTION = ( - "Execute an API deployment against one or more documents.\n\n" - "Supply the documents either as `files` (multipart upload) or as " - "`presigned_urls` (HTTPS S3 URLs), or both — a request carrying neither is " - f"rejected, and the two together may not exceed " - f"{ExecutionRequestSerializer.MAX_FILES_ALLOWED} documents.\n\n" - "With the default `timeout` of -1 the call returns as soon as the " - "execution is queued; read the outcome from the status endpoint." -) - -STATUS_DESCRIPTION = ( - "Read the result of a previously started execution.\n\n" - "This read is one-shot: the first call that observes a completed execution " - "acknowledges it and the stored result is discarded, so every later call " - "for that execution answers 406. Poll while the execution is pending, and " - "keep the payload of the call that returns it — it cannot be fetched again." -) - - -# The generated clients take their command names, module paths and request -# shapes from here, so this block is part of the public API surface. -@extend_schema_view( - post=extend_schema( - operation_id="execute", - tags=["deployment"], - auth=DEPLOYMENT_AUTH, - parameters=DEPLOYMENT_PATH_PARAMETERS, - request={"multipart/form-data": ExecuteRequest}, - responses={ - 200: ExecuteResponse, - 409: OpenApiResponse( - ErrorResponse, description="The deployment has no active API key." - ), - 422: ExecuteResponse, - **DEPLOYMENT_ERRORS, - }, - description=EXECUTE_DESCRIPTION, - ), - get=extend_schema( - operation_id="status", - tags=["deployment"], - auth=DEPLOYMENT_AUTH, - parameters=DEPLOYMENT_PATH_PARAMETERS + [ExecutionQuerySerializer], - responses={ - 200: StatusResponse, - 406: OpenApiResponse( - ErrorResponse, - description="The result was already consumed by an earlier call.", - ), - 422: StatusResponse, - **DEPLOYMENT_ERRORS, - }, - description=STATUS_DESCRIPTION, - ), -) +@DEPLOYMENT_EXECUTION_SCHEMA class DeploymentExecution(views.APIView): def initialize_request(self, request: Request, *args: Any, **kwargs: Any) -> Request: """To remove csrf request for public API. diff --git a/backend/api_v2/deployment_spec_urls.py b/backend/api_v2/deployment_spec_urls.py index 0aa2d998bd..9dff155d9d 100644 --- a/backend/api_v2/deployment_spec_urls.py +++ b/backend/api_v2/deployment_spec_urls.py @@ -1,27 +1,29 @@ -"""URLconf used only to generate the API deployment OpenAPI spec. +"""URLconf the published OpenAPI spec is generated against. -``api_v2.execution_urls`` is an included sub-urlconf, so generating against it -directly yields paths without the prefix it is mounted at — a spec describing -URLs the server does not serve. The mount is selected out of the served -urlconf rather than restated here, so a change to where the deployment API is -mounted moves the generated paths with it. +Each entry is an included sub-urlconf: generating against one directly yields +paths without the prefix it is mounted at, i.e. a spec describing URLs the +server does not serve. The mounts are selected out of the served urlconf +rather than restated, so moving one moves the generated paths with it. + +Widening the spec to another endpoint means annotating its view with +``@extend_schema`` and adding its urlconf here. """ from django.core.exceptions import ImproperlyConfigured from backend import base_urls -DEPLOYMENT_URLCONF = "api_v2.execution_urls" +SPEC_URLCONFS = ("api_v2.execution_urls",) urlpatterns = [ entry for entry in base_urls.urlpatterns - if getattr(getattr(entry, "urlconf_name", None), "__name__", None) - == DEPLOYMENT_URLCONF + if getattr(getattr(entry, "urlconf_name", None), "__name__", None) in SPEC_URLCONFS ] -if not urlpatterns: +missing = set(SPEC_URLCONFS) - {entry.urlconf_name.__name__ for entry in urlpatterns} +if missing: raise ImproperlyConfigured( - f"{DEPLOYMENT_URLCONF} is not mounted in backend.base_urls; the API " - "deployment spec would be generated for no routes at all." + f"{', '.join(sorted(missing))} is not mounted in backend.base_urls; the " + "spec would be generated for routes the server does not serve." ) diff --git a/backend/api_v2/openapi_schema.py b/backend/api_v2/openapi_schema.py new file mode 100644 index 0000000000..3af658481b --- /dev/null +++ b/backend/api_v2/openapi_schema.py @@ -0,0 +1,161 @@ +"""OpenAPI annotations for the API deployment endpoints. + +The serializers here shape the published spec only; none of them is used to +parse a request or build a response. They live outside ``serializers.py`` so +that nothing at request time imports one by accident. + +Their docstrings are published as the client-facing model descriptions, so +they are written for the caller rather than the maintainer. +""" + +from drf_spectacular.utils import ( + OpenApiParameter, + OpenApiResponse, + extend_schema, + extend_schema_serializer, + extend_schema_view, +) +from rest_framework import serializers + +from api_v2.serializers import ( + APIExecutionResponseSerializer, + ExecutionQuerySerializer, + ExecutionRequestSerializer, +) + + +# Declares no field of its own, so a change to the real serializer moves the +# spec. It exists to carry a caller-facing description and a stable name. +@extend_schema_serializer(component_name="ExecuteRequest") +class ExecuteRequest(ExecutionRequestSerializer): + """The documents to run, and the options that shape the result. + + Supply `files`, `presigned_urls`, or both. + """ + + +class FileResult(serializers.Serializer): + file = serializers.CharField() + file_execution_id = serializers.CharField(required=False) + status = serializers.CharField(required=False) + result = serializers.JSONField(required=False) + metadata = serializers.JSONField(required=False) + metrics = serializers.JSONField(required=False) + error = serializers.CharField(required=False, allow_null=True) + + +class ExecutionMessage(APIExecutionResponseSerializer): + """The execution's identity and, once it has finished, its per-file + results. + """ + + # Restated because the real declaration is an untyped JSONField, and + # because a pending execution sends `result: null`, which a generated + # deserialiser iterates and crashes on without allow_null. + result = FileResult(many=True, required=False, allow_null=True) + + +class ExecuteResponse(serializers.Serializer): + message = ExecutionMessage() + + +class StatusResponse(serializers.Serializer): + status = serializers.CharField() + message = FileResult(many=True, required=False, allow_null=True) + + +class ErrorResponse(serializers.Serializer): + status = serializers.CharField(required=False) + message = serializers.JSONField(required=False, allow_null=True) + + +# Restates the route's own pattern so a client rejects a mistyped identifier +# without a round trip. +PATH_SEGMENT = {"type": "string", "pattern": r"^[\w-]+$"} + +DEPLOYMENT_PATH_PARAMETERS = [ + OpenApiParameter( + "org_name", + PATH_SEGMENT, + OpenApiParameter.PATH, + description="Organization identifier.", + ), + OpenApiParameter( + "api_name", + PATH_SEGMENT, + OpenApiParameter.PATH, + description="API deployment name.", + ), +] + + +DEPLOYMENT_AUTH = [{"deploymentKey": []}] + +# A client generated without these treats an authentication or rate-limit +# response as an unknown status and has nothing to branch on. +DEPLOYMENT_ERRORS = { + 400: OpenApiResponse(ErrorResponse, description="The request failed validation."), + 401: OpenApiResponse(ErrorResponse, description="The API key is not valid."), + 403: OpenApiResponse(ErrorResponse, description="No API key was supplied."), + 404: OpenApiResponse(ErrorResponse, description="No such active deployment."), + 429: OpenApiResponse( + ErrorResponse, description="Too many concurrent executions; retry later." + ), + 500: ErrorResponse, +} + +EXECUTE_DESCRIPTION = ( + "Execute an API deployment against one or more documents.\n\n" + "Supply the documents either as `files` (multipart upload) or as " + "`presigned_urls` (HTTPS S3 URLs), or both — a request carrying neither is " + f"rejected, and the two together may not exceed " + f"{ExecutionRequestSerializer.MAX_FILES_ALLOWED} documents.\n\n" + "With the default `timeout` of -1 the call returns as soon as the " + "execution is queued; read the outcome from the status endpoint." +) + +STATUS_DESCRIPTION = ( + "Read the result of a previously started execution.\n\n" + "This read is one-shot: the first call that observes a completed execution " + "acknowledges it and the stored result is discarded, so every later call " + "for that execution answers 406. Poll while the execution is pending, and " + "keep the payload of the call that returns it — it cannot be fetched again." +) + + +# Generated clients take their command names, module paths and request shapes +# from here, so this is part of the public API surface. +DEPLOYMENT_EXECUTION_SCHEMA = extend_schema_view( + post=extend_schema( + operation_id="execute", + tags=["deployment"], + auth=DEPLOYMENT_AUTH, + parameters=DEPLOYMENT_PATH_PARAMETERS, + request={"multipart/form-data": ExecuteRequest}, + responses={ + 200: ExecuteResponse, + 409: OpenApiResponse( + ErrorResponse, description="The deployment has no active API key." + ), + 422: ExecuteResponse, + **DEPLOYMENT_ERRORS, + }, + description=EXECUTE_DESCRIPTION, + ), + get=extend_schema( + operation_id="status", + tags=["deployment"], + auth=DEPLOYMENT_AUTH, + parameters=DEPLOYMENT_PATH_PARAMETERS + [ExecutionQuerySerializer], + responses={ + 200: StatusResponse, + 406: OpenApiResponse( + ErrorResponse, + description="The result was already consumed by an earlier call.", + ), + 422: StatusResponse, + **DEPLOYMENT_ERRORS, + }, + description=STATUS_DESCRIPTION, + ), +) diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py index f91385bbca..76cecc14b7 100644 --- a/backend/backend/settings/base.py +++ b/backend/backend/settings/base.py @@ -653,18 +653,15 @@ def filter(self, record): "DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema", } -# Read while generating the API deployment OpenAPI spec -# (``manage.py generate_docstudio_spec``). ``DEFAULT_SCHEMA_CLASS`` above is a -# project-wide DRF default, but DRF dereferences it only when a schema is -# generated, so neither has an effect at request time. +# Read only while generating the OpenAPI spec +# (``manage.py generate_docstudio_spec``); no effect at request time. SPECTACULAR_SETTINGS = { "TITLE": "Unstract API", "VERSION": "v1", "PREPROCESSING_HOOKS": ["drf_spectacular.hooks.preprocess_exclude_path_format"], "SERVE_INCLUDE_SCHEMA": False, - # DRF's unset ``DEFAULT_AUTHENTICATION_CLASSES`` would otherwise be - # introspected as a decision and publish session and basic auth, which - # these endpoints do not accept. + # Declared, because DRF's unset authentication default is otherwise + # introspected as a decision and publishes auth these endpoints reject. "APPEND_COMPONENTS": { "securitySchemes": { "deploymentKey": { @@ -709,13 +706,6 @@ def filter(self, record): # These path will work without organization in request ORGANIZATION_MIDDLEWARE_WHITELISTED_PATHS = [] -# API Doc Generator Settings -# https://drf-yasg.readthedocs.io/en/stable/settings.html -REDOC_SETTINGS = { - "PATH_IN_MIDDLE": True, - "REQUIRED_PROPS_FIRST": True, -} - # Social Auth Settings SOCIAL_AUTH_LOGIN_REDIRECT_URL = f"{WEB_APP_ORIGIN_URL}/oauth-status/?status=success" SOCIAL_AUTH_LOGIN_ERROR_URL = f"{WEB_APP_ORIGIN_URL}/oauth-status/?status=error" From 51c7cd45e61fc1589eabc1863f1724ef8738f99a Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Tue, 18 Aug 2026 16:12:32 +0530 Subject: [PATCH 7/7] test: name the downstream repos in the drift failure, and pin the response fields The drift test is the only gate, so its message has to reach the person or agent who then has to regenerate the client and the CLI; it now names both repos, as does the management command's `--check`. Adds one binding the annotation could not express by inheritance: the view returns the execution DTO as a dict rather than through `APIExecutionResponseSerializer`, so a renamed DTO field would reach clients as a field the server never sends. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01BFSunNN6RKRA1xo6kWkztx --- .../commands/generate_docstudio_spec.py | 9 +++++- backend/api_v2/tests/test_docstudio_spec.py | 30 +++++++++++++++---- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/backend/api_v2/management/commands/generate_docstudio_spec.py b/backend/api_v2/management/commands/generate_docstudio_spec.py index a091073a39..8a6e6f1205 100644 --- a/backend/api_v2/management/commands/generate_docstudio_spec.py +++ b/backend/api_v2/management/commands/generate_docstudio_spec.py @@ -23,6 +23,13 @@ DEFAULT_OUT = Path(__file__).resolve().parents[4] / "specs" / "docstudio-oss.json" URLCONF = "api_v2.deployment_spec_urls" REGENERATE = "uv run python manage.py generate_docstudio_spec" +# Named in every failure message: the repos that regenerate from this file are +# the ones a spec change actually breaks, and nothing there watches this repo. +DOWNSTREAM = ( + "The published client (Zipstack/unstract-python-client) and the CLI " + "(Zipstack/unstract-cli) are generated from this file — raise the matching " + "PRs there for anything that changes an operation id, a tag or a schema." +) class SpecGenerationFailed(CommandError): @@ -77,7 +84,7 @@ def handle(self, *args: Any, **options: Any) -> None: if current != rendered: raise CommandError( f"{out} is out of date. Run `{REGENERATE}` from `backend/` " - "and commit the result." + f"and commit the result.\n\n{DOWNSTREAM}" ) self.stdout.write(f"{out} is up to date") return diff --git a/backend/api_v2/tests/test_docstudio_spec.py b/backend/api_v2/tests/test_docstudio_spec.py index 50b0e99cad..23c2e4c5c3 100644 --- a/backend/api_v2/tests/test_docstudio_spec.py +++ b/backend/api_v2/tests/test_docstudio_spec.py @@ -5,17 +5,20 @@ than in a client repo. """ +import dataclasses import json from django.urls import resolve, reverse from drf_spectacular.drainage import GENERATOR_STATS +from workflow_manager.workflow_v2.dto import ExecutionResponse from api_v2.management.commands.generate_docstudio_spec import ( DEFAULT_OUT, + DOWNSTREAM, REGENERATE, render_spec, ) - +from api_v2.serializers import APIExecutionResponseSerializer #: Keys under a path item that are operations. The rest -- `parameters`, #: `summary`, vendor extensions -- describe the path, not a call. @@ -44,13 +47,14 @@ def test_committed_spec_matches_the_code() -> None: assert DEFAULT_OUT.exists(), f"{DEFAULT_OUT} is missing" assert DEFAULT_OUT.read_text() == render_spec(), ( f"{DEFAULT_OUT} is out of date. Run `{REGENERATE}` from `backend/` and " - "commit the result." + f"commit the result.\n\n{DOWNSTREAM}" ) def test_generation_reports_no_diagnostics() -> None: """A warned-about operation is published with guessed request and response - shapes, and the drift comparison certifies the guess.""" + shapes, and the drift comparison certifies the guess. + """ render_spec() assert not GENERATOR_STATS._error_cache assert not GENERATOR_STATS._warn_cache @@ -58,7 +62,8 @@ def test_generation_reports_no_diagnostics() -> None: def test_spec_paths_are_the_urls_the_server_serves() -> None: """Resolves the real mount rather than restating it: a spec generated for - URLs the server does not serve is the failure this file exists to catch.""" + URLs the server does not serve is the failure this file exists to catch. + """ served = reverse( "api_deployment_execution", kwargs={"org_name": "ORG", "api_name": "API"} ) @@ -83,7 +88,8 @@ def test_spec_documents_the_deployment_operations() -> None: def test_operations_require_the_deployment_key() -> None: """Without this the unset DRF authentication default is published as - though it were a decision, and no generated client can authenticate.""" + though it were a decision, and no generated client can authenticate. + """ spec = _committed() scheme = spec["components"]["securitySchemes"]["deploymentKey"] @@ -101,7 +107,8 @@ def test_clients_can_branch_on_every_failure_they_will_see() -> None: def test_the_one_shot_read_is_documented_where_a_client_will_see_it() -> None: """The semantics that a status read destroys the result must reach the - generated client, not live in a source comment.""" + generated client, not live in a source comment. + """ reads = [ operation for _, _, operation in _operations(_committed()) @@ -112,3 +119,14 @@ def test_the_one_shot_read_is_documented_where_a_client_will_see_it() -> None: for status_op in reads: assert "one-shot" in status_op["description"] assert status_op["responses"]["406"]["description"].strip() + + +def test_the_documented_response_fields_are_ones_the_code_produces() -> None: + """The view returns the execution DTO as a dict rather than through this + serializer, so a renamed DTO field would otherwise reach clients as a field + the server never sends. + """ + documented = set(APIExecutionResponseSerializer().get_fields()) + produced = {field.name for field in dataclasses.fields(ExecutionResponse)} + + assert documented <= produced, documented - produced