UN-4009 [MISC] Generate and commit the API deployment OpenAPI spec in-repo

backend/api_v2/api_deployment_views.py

@@ -33,6 +33,7 @@
     contains_tool_not_found_error,
 )
 from api_v2.models import APIDeployment
+from api_v2.openapi_schema import DEPLOYMENT_EXECUTION_SCHEMA
 from api_v2.rate_limiter import APIDeploymentRateLimiter
 from api_v2.serializers import (
     APIDeploymentListSerializer,
@@ -50,6 +51,7 @@
 logger = logging.getLogger(__name__)
 
 
+@DEPLOYMENT_EXECUTION_SCHEMA
 class DeploymentExecution(views.APIView):
     def initialize_request(self, request: Request, *args: Any, **kwargs: Any) -> Request:
         """To remove csrf request for public API.

backend/api_v2/deployment_spec_urls.py

@@ -0,0 +1,29 @@
+"""URLconf the published OpenAPI spec is generated against.
+
+Each entry is an included sub-urlconf: generating against one directly yields
+paths without the prefix it is mounted at, i.e. a spec describing URLs the
+server does not serve. The mounts are selected out of the served urlconf
+rather than restated, so moving one moves the generated paths with it.
+
+Widening the spec to another endpoint means annotating its view with
+``@extend_schema`` and adding its urlconf here.
+"""
+
+from django.core.exceptions import ImproperlyConfigured
+
+from backend import base_urls
+
+SPEC_URLCONFS = ("api_v2.execution_urls",)
+
+urlpatterns = [
+    entry
+    for entry in base_urls.urlpatterns
+    if getattr(getattr(entry, "urlconf_name", None), "__name__", None) in SPEC_URLCONFS
+]
+
+missing = set(SPEC_URLCONFS) - {entry.urlconf_name.__name__ for entry in urlpatterns}
+if missing:
+    raise ImproperlyConfigured(
+        f"{', '.join(sorted(missing))} is not mounted in backend.base_urls; the "
+        "spec would be generated for routes the server does not serve."
+    )

backend/api_v2/management/commands/generate_docstudio_spec.py

@@ -0,0 +1,104 @@
+"""Regenerate the committed API deployment OpenAPI spec.
+
+The spec is the contract the published clients and their generated SDKs are
+built from, so it is committed and CI fails on drift: change a route, a
+serializer or the schema annotation, and regenerate in the same PR.
+
+    uv run python manage.py generate_docstudio_spec           # from backend/
+    uv run python manage.py generate_docstudio_spec --check   # no write, drift is an error
+
+The generated paths carry ``API_DEPLOYMENT_PATH_PREFIX``, so regenerate in an
+environment that does not override it — the committed artifact describes the
+deployment as it is served publicly, not as one installation mounts it.
+"""
+
+import json
+from pathlib import Path
+from typing import Any
+
+from django.core.management.base import BaseCommand, CommandError
+from drf_spectacular.drainage import GENERATOR_STATS
+from drf_spectacular.generators import SchemaGenerator
+
+DEFAULT_OUT = Path(__file__).resolve().parents[4] / "specs" / "docstudio-oss.json"
+URLCONF = "api_v2.deployment_spec_urls"
+REGENERATE = "uv run python manage.py generate_docstudio_spec"
+# Named in every failure message: the repos that regenerate from this file are
+# the ones a spec change actually breaks, and nothing there watches this repo.
+DOWNSTREAM = (
+    "The published client (Zipstack/unstract-python-client) and the CLI "
+    "(Zipstack/unstract-cli) are generated from this file — raise the matching "
+    "PRs there for anything that changes an operation id, a tag or a schema."
+)
+
+
+class SpecGenerationFailed(CommandError):
+    """Raised when the generator had to guess."""
+
+
+def render_spec() -> str:
+    """The committed artifact, byte for byte.
+
+    Shared with the drift test: two copies of this could disagree, and then
+    the gate rejects exactly the file the command it names produces.
+    """
+    GENERATOR_STATS.reset()
+    schema = SchemaGenerator(urlconf=URLCONF).get_schema(request=None, public=True)
+    if GENERATOR_STATS:
+        # spectacular downgrades "unable to guess serializer" to a warning and
+        # writes a plausible, wrong operation. Nothing downstream can tell that
+        # apart from an annotation that is simply thin.
+        diagnostics = "\n".join(
+            f"  {severity}: {message}"
+            for severity, cache in (
+                ("error", GENERATOR_STATS._error_cache),
+                ("warning", GENERATOR_STATS._warn_cache),
+            )
+            for message in cache
+        )
+        raise SpecGenerationFailed(
+            f"The generator reported problems, so the spec would describe an "
+            f"API nobody implements:\n{diagnostics}"
+        )
+    # Sorted keys are what make the committed artifact a usable drift signal.
+    return json.dumps(schema, indent=2, sort_keys=True) + "\n"
+
+
+class Command(BaseCommand):
+    help = "Generate the API deployment OpenAPI spec."
+
+    def add_arguments(self, parser: Any) -> None:
+        parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
+        parser.add_argument(
+            "--check",
+            action="store_true",
+            help="Fail if the file on disk differs, instead of writing it.",
+        )
+
+    def handle(self, *args: Any, **options: Any) -> None:
+        rendered = render_spec()
+
+        out: Path = options["out"]
+        if options["check"]:
+            current = out.read_text() if out.exists() else ""
+            if current != rendered:
+                raise CommandError(
+                    f"{out} is out of date. Run `{REGENERATE}` from `backend/` "
+                    f"and commit the result.\n\n{DOWNSTREAM}"
+                )
+            self.stdout.write(f"{out} is up to date")
+            return
+
+        out.parent.mkdir(parents=True, exist_ok=True)
+        out.write_text(rendered)
+        schema = json.loads(rendered)
+        operations = sum(
+            1
+            for methods in schema["paths"].values()
+            for method in methods
+            if method in {"get", "post", "put", "patch", "delete"}
+        )
+        self.stdout.write(
+            f"{out}: {len(schema['paths'])} paths, {operations} operations, "
+            f"{len(schema.get('components', {}).get('schemas', {}))} schemas"
+        )

backend/api_v2/openapi_schema.py

@@ -0,0 +1,161 @@
+"""OpenAPI annotations for the API deployment endpoints.
+
+The serializers here shape the published spec only; none of them is used to
+parse a request or build a response. They live outside ``serializers.py`` so
+that nothing at request time imports one by accident.
+
+Their docstrings are published as the client-facing model descriptions, so
+they are written for the caller rather than the maintainer.
+"""
+
+from drf_spectacular.utils import (
+    OpenApiParameter,
+    OpenApiResponse,
+    extend_schema,
+    extend_schema_serializer,
+    extend_schema_view,
+)
+from rest_framework import serializers
+
+from api_v2.serializers import (
+    APIExecutionResponseSerializer,
+    ExecutionQuerySerializer,
+    ExecutionRequestSerializer,
+)
+
+
+# Declares no field of its own, so a change to the real serializer moves the
+# spec. It exists to carry a caller-facing description and a stable name.
+@extend_schema_serializer(component_name="ExecuteRequest")
+class ExecuteRequest(ExecutionRequestSerializer):
+    """The documents to run, and the options that shape the result.
+
+    Supply `files`, `presigned_urls`, or both.
+    """
+
+
+class FileResult(serializers.Serializer):
+    file = serializers.CharField()
+    file_execution_id = serializers.CharField(required=False)
+    status = serializers.CharField(required=False)
+    result = serializers.JSONField(required=False)
+    metadata = serializers.JSONField(required=False)
+    metrics = serializers.JSONField(required=False)
+    error = serializers.CharField(required=False, allow_null=True)
+
+
+class ExecutionMessage(APIExecutionResponseSerializer):
+    """The execution's identity and, once it has finished, its per-file
+    results.
+    """
+
+    # Restated because the real declaration is an untyped JSONField, and
+    # because a pending execution sends `result: null`, which a generated
+    # deserialiser iterates and crashes on without allow_null.
+    result = FileResult(many=True, required=False, allow_null=True)
+
+
+class ExecuteResponse(serializers.Serializer):
+    message = ExecutionMessage()
+
+
+class StatusResponse(serializers.Serializer):
+    status = serializers.CharField()
+    message = FileResult(many=True, required=False, allow_null=True)
+
+
+class ErrorResponse(serializers.Serializer):
+    status = serializers.CharField(required=False)
+    message = serializers.JSONField(required=False, allow_null=True)
+
+
+# Restates the route's own pattern so a client rejects a mistyped identifier
+# without a round trip.
+PATH_SEGMENT = {"type": "string", "pattern": r"^[\w-]+$"}
+
+DEPLOYMENT_PATH_PARAMETERS = [
+    OpenApiParameter(
+        "org_name",
+        PATH_SEGMENT,
+        OpenApiParameter.PATH,
+        description="Organization identifier.",
+    ),
+    OpenApiParameter(
+        "api_name",
+        PATH_SEGMENT,
+        OpenApiParameter.PATH,
+        description="API deployment name.",
+    ),
+]
+
+
+DEPLOYMENT_AUTH = [{"deploymentKey": []}]
+
+# A client generated without these treats an authentication or rate-limit
+# response as an unknown status and has nothing to branch on.
+DEPLOYMENT_ERRORS = {
+    400: OpenApiResponse(ErrorResponse, description="The request failed validation."),
+    401: OpenApiResponse(ErrorResponse, description="The API key is not valid."),
+    403: OpenApiResponse(ErrorResponse, description="No API key was supplied."),
+    404: OpenApiResponse(ErrorResponse, description="No such active deployment."),
+    429: OpenApiResponse(
+        ErrorResponse, description="Too many concurrent executions; retry later."
+    ),
+    500: ErrorResponse,
+}
+
+EXECUTE_DESCRIPTION = (
+    "Execute an API deployment against one or more documents.\n\n"
+    "Supply the documents either as `files` (multipart upload) or as "
+    "`presigned_urls` (HTTPS S3 URLs), or both — a request carrying neither is "
+    f"rejected, and the two together may not exceed "
+    f"{ExecutionRequestSerializer.MAX_FILES_ALLOWED} documents.\n\n"
+    "With the default `timeout` of -1 the call returns as soon as the "
+    "execution is queued; read the outcome from the status endpoint."
+)
+
+STATUS_DESCRIPTION = (
+    "Read the result of a previously started execution.\n\n"
+    "This read is one-shot: the first call that observes a completed execution "
+    "acknowledges it and the stored result is discarded, so every later call "
+    "for that execution answers 406. Poll while the execution is pending, and "
+    "keep the payload of the call that returns it — it cannot be fetched again."
+)
+
+
+# Generated clients take their command names, module paths and request shapes
+# from here, so this is part of the public API surface.
+DEPLOYMENT_EXECUTION_SCHEMA = extend_schema_view(
+    post=extend_schema(
+        operation_id="execute",
+        tags=["deployment"],
+        auth=DEPLOYMENT_AUTH,
+        parameters=DEPLOYMENT_PATH_PARAMETERS,
+        request={"multipart/form-data": ExecuteRequest},
+        responses={
+            200: ExecuteResponse,
+            409: OpenApiResponse(
+                ErrorResponse, description="The deployment has no active API key."
+            ),
+            422: ExecuteResponse,
+            **DEPLOYMENT_ERRORS,
+        },
+        description=EXECUTE_DESCRIPTION,
+    ),
+    get=extend_schema(
+        operation_id="status",
+        tags=["deployment"],
+        auth=DEPLOYMENT_AUTH,
+        parameters=DEPLOYMENT_PATH_PARAMETERS + [ExecutionQuerySerializer],
+        responses={
+            200: StatusResponse,
+            406: OpenApiResponse(
+                ErrorResponse,
+                description="The result was already consumed by an earlier call.",
+            ),
+            422: StatusResponse,
+            **DEPLOYMENT_ERRORS,
+        },
+        description=STATUS_DESCRIPTION,
+    ),
+)

backend/api_v2/serializers.py

@@ -6,6 +6,8 @@
 
 from django.apps import apps
 from django.core.validators import RegexValidator
+from drf_spectacular.types import OpenApiTypes
+from drf_spectacular.utils import extend_schema_field
 from pipeline_v2.models import Pipeline
 from prompt_studio.prompt_profile_manager_v2.models import ProfileManager
 from rest_framework import serializers
@@ -218,6 +220,13 @@ def to_representation(self, instance: APIKey) -> OrderedDict[str, Any]:
         return representation
 
 
+@extend_schema_field(OpenApiTypes.BINARY)
+class UploadField(FileField):
+    """A bare ``FileField`` maps to ``format: uri`` -- correct on output, wrong
+    for a multipart upload, and generators emit ``str`` for it.
+    """
+
+
 class ExecutionRequestSerializer(TagParamsSerializer):
     """Execution request serializer.
 
@@ -320,7 +329,7 @@ def validate_custom_data(self, value):
         return value
 
     files = ListField(
-        child=FileField(),
+        child=UploadField(),
         required=False,
         allow_empty=True,
     )