diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c8872e37..9769d929 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,6 +57,46 @@ jobs: [[ "$HEAD_SHA" =~ ^[0-9a-f]{40}$ ]] git diff --name-only -z "$BASE_SHA...$HEAD_SHA" | node scripts/ci-paths.mjs >> "$GITHUB_OUTPUT" + t4-api-generation: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Check out exact source + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Install pinned pnpm + uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 + with: + version: 11.10.0 + + - name: Install pinned Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 24.13.1 + + - name: Install dependencies from the committed lockfile + run: pnpm install --frozen-lockfile + + - name: Validate OpenAPI and deterministic generation + run: | + pnpm --filter @t4-code/t4-api-contract validate + pnpm --filter @t4-code/t4-api-contract generate:ci + pnpm --filter @t4-code/t4-api-contract check:generated + pnpm --filter @t4-code/t4-api-client typecheck + pnpm --filter @t4-code/client typecheck + pnpm --filter @t4-code/client exec vp test run test/t4-api-v1-conformance.test.ts + - name: Upload deterministic T4 API client + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: t4-api-generated-${{ github.run_id }} + path: artifacts/t4-api/generated/schema.ts + if-no-files-found: error + + + core: runs-on: ubuntu-24.04 timeout-minutes: 25 @@ -237,6 +277,23 @@ jobs: if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.cluster == 'true' }} runs-on: ubuntu-24.04 timeout-minutes: 25 + services: + postgres: + image: postgres:17.6-bookworm@sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3 + env: + POSTGRES_DB: t4_test + POSTGRES_USER: t4_test + POSTGRES_PASSWORD: t4_test_password + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U t4_test -d t4_test" + --health-interval 2s + --health-timeout 5s + --health-retries 15 + env: + T4_TEST_POSTGRES_URL: postgres://t4_test:t4_test_password@127.0.0.1:5432/t4_test + steps: - name: Check out source uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 @@ -294,6 +351,8 @@ jobs: - name: Run operator tests working-directory: packages/cluster-operator + env: + GOFLAGS: -mod=readonly run: go test ./... - name: Lint cluster chart @@ -521,7 +580,7 @@ jobs: verify: name: verify if: ${{ always() }} - needs: [changes, core, legacy-bridge-continuity, official-omp-gate0, cluster, tooling, android-debug, flutter, flutter-android, flutter-apple] + needs: [changes, t4-api-generation, core, legacy-bridge-continuity, official-omp-gate0, cluster, tooling, android-debug, flutter, flutter-android, flutter-apple] runs-on: ubuntu-24.04 timeout-minutes: 5 steps: @@ -529,6 +588,7 @@ jobs: shell: bash env: CHANGES_RESULT: ${{ needs.changes.result }} + T4_API_GENERATION_RESULT: ${{ needs.t4-api-generation.result }} CORE_RESULT: ${{ needs.core.result }} CONTINUITY_RESULT: ${{ needs.legacy-bridge-continuity.result }} OFFICIAL_OMP_GATE0_RESULT: ${{ needs.official-omp-gate0.result }} @@ -541,6 +601,7 @@ jobs: run: | set -euo pipefail test "$CHANGES_RESULT" = success + test "$T4_API_GENERATION_RESULT" = success test "$CORE_RESULT" = success for result in \ "$CONTINUITY_RESULT" \ diff --git a/.woodpecker.yml b/.woodpecker.yml index 4f8c556e..b6bd2403 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -11,6 +11,14 @@ clone: depth: 50 partial: false +services: + - name: postgres + image: mirror.gcr.io/library/postgres:17.6-bookworm@sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3 + environment: + POSTGRES_DB: t4_test + POSTGRES_USER: t4_test + POSTGRES_PASSWORD: t4_test_password + steps: dependencies: image: mirror.gcr.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 @@ -31,6 +39,32 @@ steps: memory: 2Gi ephemeral-storage: 4Gi + t4-api-contract: + depends_on: [dependencies] + image: mirror.gcr.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 + commands: + - corepack enable + - pnpm --filter @t4-code/t4-api-contract validate + - pnpm --filter @t4-code/t4-api-contract generate:ci + - test -f artifacts/t4-api/generated/schema.ts + - echo "T4_API_GENERATED_ARTIFACT=artifacts/t4-api/generated/schema.ts" + - pnpm --filter @t4-code/t4-api-contract check:generated + - pnpm --filter @t4-code/t4-api-client typecheck + - pnpm --filter @t4-code/client typecheck + - pnpm --filter @t4-code/client exec vp test run test/t4-api-v1-conformance.test.ts + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 256Mi + ephemeral-storage: 256Mi + limits: + cpu: "1" + memory: 2Gi + ephemeral-storage: 2Gi + bun-runtime: depends_on: [dependencies] image: mirror.gcr.io/oven/bun:1.3.14@sha256:e10577f0db68676a7024391c6e5cb4b879ebd17188ab750cf10024a6d700e5c4 @@ -50,7 +84,7 @@ steps: ephemeral-storage: 256Mi upstream-core: - depends_on: [cluster-ci-contracts, bun-runtime] + depends_on: [cluster-ci-contracts, t4-api-contract, bun-runtime] image: mirror.gcr.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 commands: - export PATH="$PWD/.ci:$PATH" @@ -204,6 +238,7 @@ steps: image: mirror.gcr.io/oven/bun:1.3.14@sha256:e10577f0db68676a7024391c6e5cb4b879ebd17188ab750cf10024a6d700e5c4 environment: T4_PROOF_OUTPUT_DIR: artifacts/cluster-proof/scenarios + T4_TEST_POSTGRES_URL: postgres://t4_test:t4_test_password@postgres:5432/t4_test commands: - export SSL_CERT_FILE="$PWD/.ci/ca-certificates.crt" - export NODE_EXTRA_CA_CERTS="$SSL_CERT_FILE" @@ -250,9 +285,9 @@ steps: image: mirror.gcr.io/library/golang:1.25.1-bookworm@sha256:c423747fbd96fd8f0b1102d947f51f9b266060217478e5f9bf86f145969562ee directory: packages/cluster-operator commands: - - GOMAXPROCS=1 GOFLAGS=-p=1 go test ./api/... ./controllers/... ./cmd/... + - GOMAXPROCS=1 GOFLAGS='-mod=readonly -p=1' go test ./api/... ./controllers/... ./cmd/... - mkdir -p ../../artifacts/cluster-proof - - CGO_ENABLED=0 GOMAXPROCS=1 GOFLAGS=-p=1 go test -c ./charttests -o ../../artifacts/cluster-proof/chart-contract.test + - CGO_ENABLED=0 GOMAXPROCS=1 GOFLAGS='-mod=readonly -p=1' go test -c ./charttests -o ../../artifacts/cluster-proof/chart-contract.test backend_options: kubernetes: serviceAccountName: woodpecker-ci-untrusted diff --git a/deploy/charts/t4-cluster/templates/_helpers.tpl b/deploy/charts/t4-cluster/templates/_helpers.tpl index 60450013..b5165665 100644 --- a/deploy/charts/t4-cluster/templates/_helpers.tpl +++ b/deploy/charts/t4-cluster/templates/_helpers.tpl @@ -33,3 +33,28 @@ app.kubernetes.io/instance: {{ .Release.Name | quote }} {{- define "t4-cluster.image" -}} {{- printf "%s@%s" .repository .digest -}} {{- end -}} + +{{- define "t4-cluster.validatePublicApi" -}} +{{- if .Values.publicApi.enabled -}} +{{- $existingSecret := required "publicApi.existingSecret is required when publicApi.enabled is true" .Values.publicApi.existingSecret -}} +{{- $postgresURLKey := required "publicApi.postgresURLKey is required when publicApi.enabled is true" .Values.publicApi.postgresURLKey -}} +{{- $credentialsKey := required "publicApi.credentialsKey is required when publicApi.enabled is true" .Values.publicApi.credentialsKey -}} +{{- if eq $postgresURLKey $credentialsKey -}} +{{- fail "publicApi.postgresURLKey and publicApi.credentialsKey must be different" -}} +{{- end -}} +{{- if not .Values.networkPolicy.enabled -}} +{{- fail "networkPolicy.enabled must be true when publicApi.enabled is true" -}} +{{- end -}} +{{- if empty .Values.networkPolicy.postgresPorts -}} +{{- fail "networkPolicy.postgresPorts must contain at least one port when publicApi.enabled is true" -}} +{{- end -}} +{{- $hasPostgresNamespaceSelector := not (empty .Values.networkPolicy.postgres.namespaceSelector) -}} +{{- $hasPostgresPodSelector := not (empty .Values.networkPolicy.postgres.podSelector) -}} +{{- if ne $hasPostgresNamespaceSelector $hasPostgresPodSelector -}} +{{- fail "networkPolicy.postgres namespaceSelector and podSelector must both be configured or both be empty" -}} +{{- end -}} +{{- if and (empty .Values.networkPolicy.postgresCIDRs) (not $hasPostgresNamespaceSelector) -}} +{{- fail "publicApi.enabled requires a PostgreSQL CIDR or complete selector destination" -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/deploy/charts/t4-cluster/templates/deployments.yaml b/deploy/charts/t4-cluster/templates/deployments.yaml index 9a131459..2f1155f8 100644 --- a/deploy/charts/t4-cluster/templates/deployments.yaml +++ b/deploy/charts/t4-cluster/templates/deployments.yaml @@ -1,4 +1,5 @@ {{- if .Values.enabled }} +{{- include "t4-cluster.validatePublicApi" . -}} {{- if eq .Values.session.omp.modelsKey .Values.session.omp.settingsKey }} {{- fail "session.omp.modelsKey and session.omp.settingsKey must be different" }} {{- end }} @@ -258,6 +259,14 @@ spec: valueFrom: {fieldRef: {fieldPath: metadata.name}} - name: POD_UID valueFrom: {fieldRef: {fieldPath: metadata.uid}} + {{- if .Values.publicApi.enabled }} + - name: T4_POSTGRES_URL_FILE + value: /var/run/secrets/t4-public-api/postgres-url + - name: T4_PUBLIC_API_CREDENTIALS_FILE + value: /var/run/secrets/t4-public-api/credentials.json + - name: T4_PUBLIC_API_RUNTIME_PROFILE + value: {{ first .Values.clusterHost.runtimeProfiles | quote }} + {{- end }} {{- if .Values.woodpecker.existingSecret }} - name: T4_WOODPECKER_TOKEN valueFrom: @@ -324,6 +333,11 @@ spec: - name: kubernetes-api-access mountPath: /var/run/secrets/kubernetes.io/serviceaccount readOnly: true + {{- if .Values.publicApi.enabled }} + - name: public-api + mountPath: /var/run/secrets/t4-public-api + readOnly: true + {{- end }} {{- if .Values.woodpecker.serviceAccountAudience }} - name: ci-identity mountPath: /var/run/secrets/t4-ci @@ -351,6 +365,17 @@ spec: items: - key: ca.crt path: ca.crt + {{- if .Values.publicApi.enabled }} + - name: public-api + secret: + secretName: {{ .Values.publicApi.existingSecret | quote }} + defaultMode: 0440 + items: + - key: {{ .Values.publicApi.postgresURLKey | quote }} + path: postgres-url + - key: {{ .Values.publicApi.credentialsKey | quote }} + path: credentials.json + {{- end }} {{- if .Values.woodpecker.serviceAccountAudience }} - name: ci-identity projected: diff --git a/deploy/charts/t4-cluster/templates/networkpolicies.yaml b/deploy/charts/t4-cluster/templates/networkpolicies.yaml index 99837f80..66e9a0b3 100644 --- a/deploy/charts/t4-cluster/templates/networkpolicies.yaml +++ b/deploy/charts/t4-cluster/templates/networkpolicies.yaml @@ -1,4 +1,6 @@ -{{- if and .Values.enabled .Values.networkPolicy.enabled }} +{{- if .Values.enabled }} +{{- include "t4-cluster.validatePublicApi" . -}} +{{- if .Values.networkPolicy.enabled }} apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: @@ -101,6 +103,27 @@ spec: {{- end }} {{- end }} {{- end }} + {{- if and .Values.publicApi.enabled .Values.networkPolicy.postgresPorts }} + {{- range .Values.networkPolicy.postgresCIDRs }} + - to: + - ipBlock: {cidr: {{ . | quote }}} + ports: + {{- range $.Values.networkPolicy.postgresPorts }} + - {protocol: TCP, port: {{ . }}} + {{- end }} + {{- end }} + {{- if and (not (empty .Values.networkPolicy.postgres.namespaceSelector)) (not (empty .Values.networkPolicy.postgres.podSelector)) }} + - to: + - namespaceSelector: + {{- toYaml .Values.networkPolicy.postgres.namespaceSelector | nindent 12 }} + podSelector: + {{- toYaml .Values.networkPolicy.postgres.podSelector | nindent 12 }} + ports: + {{- range .Values.networkPolicy.postgresPorts }} + - {protocol: TCP, port: {{ . }}} + {{- end }} + {{- end }} + {{- end }} --- apiVersion: networking.k8s.io/v1 kind: NetworkPolicy @@ -205,3 +228,4 @@ spec: ingress: [] {{- end }} {{- end }} +{{- end }} diff --git a/deploy/charts/t4-cluster/templates/rbac.yaml b/deploy/charts/t4-cluster/templates/rbac.yaml index 9c8af47f..2092d5fc 100644 --- a/deploy/charts/t4-cluster/templates/rbac.yaml +++ b/deploy/charts/t4-cluster/templates/rbac.yaml @@ -41,7 +41,7 @@ rules: verbs: [get, list, watch] - apiGroups: [cluster.t4.dev] resources: [t4workspaces, t4sessions] - verbs: [get, list, watch, create, delete] + verbs: [get, list, watch, create, patch, delete] - apiGroups: [''] resources: [pods, services, endpoints] verbs: [get, list, watch] diff --git a/deploy/charts/t4-cluster/values.schema.json b/deploy/charts/t4-cluster/values.schema.json index 9dd8e8ed..f625fb5b 100644 --- a/deploy/charts/t4-cluster/values.schema.json +++ b/deploy/charts/t4-cluster/values.schema.json @@ -10,6 +10,7 @@ "server", "session", "storage", + "publicApi", "woodpecker", "ingress", "networkPolicy", @@ -129,6 +130,17 @@ } } }, + "publicApi": { + "type": "object", + "required": ["enabled", "existingSecret", "postgresURLKey", "credentialsKey"], + "properties": { + "enabled": { "type": "boolean" }, + "existingSecret": { "$ref": "#/definitions/optionalDNSSubdomain" }, + "postgresURLKey": { "$ref": "#/definitions/configKey" }, + "credentialsKey": { "$ref": "#/definitions/configKey" } + }, + "additionalProperties": false + }, "woodpecker": { "type": "object", "required": ["existingSecret", "tokenKey", "configMap", "baseURLKey", "webBaseURLKey", "repositoriesKey", "serviceAccountAudience", "tokenExpirationSeconds"], @@ -172,7 +184,7 @@ }, "networkPolicy": { "type": "object", - "required": ["enabled", "kubernetesApiCIDRs", "modelRouteCIDRs", "modelRoutePorts", "modelRoute", "ciProviderCIDRs", "ciProviderPorts", "ciProvider", "dns", "gatewayIngress", "observability"], + "required": ["enabled", "kubernetesApiCIDRs", "modelRouteCIDRs", "modelRoutePorts", "modelRoute", "ciProviderCIDRs", "ciProviderPorts", "ciProvider", "postgresCIDRs", "postgresPorts", "postgres", "dns", "gatewayIngress", "observability"], "properties": { "enabled": { "type": "boolean" }, "kubernetesApiCIDRs": { "$ref": "#/definitions/cidrList" }, @@ -182,6 +194,9 @@ "ciProviderCIDRs": { "$ref": "#/definitions/cidrList" }, "ciProviderPorts": { "$ref": "#/definitions/tcpPortList" }, "ciProvider": { "$ref": "#/definitions/pairedSelectors" }, + "postgresCIDRs": { "$ref": "#/definitions/cidrList" }, + "postgresPorts": { "$ref": "#/definitions/tcpPortList" }, + "postgres": { "$ref": "#/definitions/pairedSelectors" }, "dns": { "type": "object", "required": ["namespaceSelector", "podSelector"], @@ -397,6 +412,41 @@ } } }, + { + "if": { + "properties": { + "publicApi": { "properties": { "enabled": { "const": true } } } + } + }, + "then": { + "properties": { + "publicApi": { + "properties": { + "existingSecret": { "minLength": 1 } + } + }, + "networkPolicy": { + "properties": { + "enabled": { "const": true }, + "postgresPorts": { "minItems": 1 } + }, + "anyOf": [ + { "properties": { "postgresCIDRs": { "minItems": 1 } } }, + { + "properties": { + "postgres": { + "properties": { + "namespaceSelector": { "minProperties": 1 }, + "podSelector": { "minProperties": 1 } + } + } + } + } + ] + } + } + } + }, { "if": { "properties": { "ingress": { "properties": { "enabled": { "const": true } } } } }, "then": { diff --git a/deploy/charts/t4-cluster/values.yaml b/deploy/charts/t4-cluster/values.yaml index b4c2bd1e..83b22b4f 100644 --- a/deploy/charts/t4-cluster/values.yaml +++ b/deploy/charts/t4-cluster/values.yaml @@ -82,6 +82,13 @@ storage: # Required when enabled. The administrator creates this backend-neutral RWX StorageClass. adminRWXStorageClass: "" +publicApi: + enabled: false + # Administrator-owned Secret; the chart only projects these two narrowly named files. + existingSecret: "" + postgresURLKey: postgres-url + credentialsKey: credentials.json + woodpecker: existingSecret: "" tokenKey: token @@ -121,6 +128,12 @@ networkPolicy: # Optional in-cluster CI endpoint identity. Set both selectors or neither. namespaceSelector: {} podSelector: {} + postgresCIDRs: [] + postgresPorts: [] + postgres: + # PostgreSQL may be selected by CIDR and/or in-cluster identity. Set both selectors or neither. + namespaceSelector: {} + podSelector: {} dns: namespaceSelector: matchLabels: diff --git a/docs/CLUSTER_OPERATOR.md b/docs/CLUSTER_OPERATOR.md index 67598da5..aae439e4 100644 --- a/docs/CLUSTER_OPERATOR.md +++ b/docs/CLUSTER_OPERATOR.md @@ -23,13 +23,15 @@ That declaration does not make a non-RWX backend safe. Missing classes, classes ## Install -Installing with defaults creates no controller, gateway, session workload, RBAC, Secret, or network policy: +Installing with defaults creates no controller, gateway, session workload, RBAC, Secret, or network policy. Use the lifecycle runner even for a fresh install so existing live objects and compatibility fixtures are checked against the proposed schemas before the CRDs are server-validated, established, and storage-version-checked and before Helm runs: ```sh -helm install t4-cluster deploy/charts/t4-cluster --namespace t4-system --create-namespace +scripts/cluster-ci/crd-lifecycle.sh install -- \ + helm install t4-cluster deploy/charts/t4-cluster \ + --namespace t4-system --create-namespace --skip-crds ``` -Helm processes files in `crds/` separately. Use `--skip-crds` if CRD lifecycle is administered independently. To enable the control plane, provide a private values file or a deployment controller values source: +Helm processes files in `crds/` separately on a direct install, but does not upgrade or delete them later. The release procedure therefore administers CRDs independently and always passes `--skip-crds` to Helm. To enable the control plane, provide a private values file or a deployment controller values source: ```yaml enabled: true @@ -112,11 +114,7 @@ The controller uses uncached, exact-name `get` permission for only that administ Reusable credential projection is unsupported because OMP and arbitrary session tools share one workload security boundary. `allowUnauthenticated: true` with empty `credentialSecret` and `credentialKey` is therefore mandatory. The two credential fields remain only as cutover sentinels: Helm, the controller, and the image entrypoint all reject an old credential-mode configuration instead of exposing it. Before OMP starts, the image requires the pinned `auth_credentials` table to be empty, rejects the pinned secret settings (`auth.broker.token`, `hindsight.apiToken`, `searxng.token`, and `dev.autoqaPush.token`), rejects broker token and encrypted snapshot files, and fails closed on unknown schemas or linked profile paths. It never deletes or rewrites durable user state. Because Helm rejects legacy credential values before rendering the new Deployment, a failed upgrade leaves the prior release unchanged. Stop development sessions and clear any legacy values and credential state explicitly before adopting this pre-production boundary. Provider authentication must live behind a private model gateway that authorizes the session through infrastructure identity or network policy while presenting an `auth: none` endpoint inside the session's allowed route. Do not expose that endpoint to the Internet or a shared untrusted network. The chart remains provider- and model-neutral. -Install or enable with immutable digests: - -```sh -helm upgrade --install t4-cluster deploy/charts/t4-cluster --namespace t4-system --values operator-values.yaml -``` +Install a new release with immutable digests by adding `--values operator-values.yaml` to the lifecycle-runner `install` command above. For an existing release, use the lifecycle-runner `upgrade` procedure below. Do not use `helm upgrade --install`: install and upgrade have different compatibility preflights. The controller always has two replicas and uses a Kubernetes Lease named from `t4-cluster-operator.cluster.t4.dev`; one replica reconciles at a time. The server defaults to three stateless replicas and supports a minimum of two. Its Deployment uses `maxUnavailable: 0`, a `minAvailable: 2` PDB, topology spread, anti-affinity, readiness draining, and an explicit `k3s-worker-02` exclusion. Session pods also exclude that node by default. Additional cluster-specific exclusions belong in deployment values, not this portable chart. @@ -150,28 +148,96 @@ The session runtime verifies and builds the exact OMP tag `t4code-17.0.5-appserv Session pods do not receive an automatically mounted ServiceAccount token. The explicit projected reviewer token can only create TokenReviews and is not resource-authorized. The namespace-local ConfigMap projection contains credential-free `auth: none` OMP models and credential-free settings; session Pods receive no provider Secret reference or reusable provider credential. All containers drop capabilities, disallow privilege escalation, use RuntimeDefault seccomp, and use read-only root filesystems. No per-session NodePort, LoadBalancer, host network, host PID, host display, or hostPath is created. -## Upgrade and rollback +## Upgrade, storage migration, and rollback -CRD changes must remain structural and additive. Helm installs files under `crds/` on first install but does not upgrade or delete them. Before every `helm upgrade`, an administrator must review and apply each newer CRD independently, wait for API discovery to serve the updated schemas, and only then upgrade workloads with the new immutable image digests: +The three namespaced CRDs have exactly one version: `cluster.t4.dev/v1alpha1`, with `served: true`, `storage: true`, a structural schema, and a status subresource. Changes in this version must remain additive: existing fields keep their meaning and validation, while new fields are optional or have safe defaults. The compatibility fixtures in `packages/cluster-operator/api/v1alpha1/testdata/compat/` are persisted old-object shapes, including status and finalizers, and must continue to validate and round-trip without losing any declared field. + +Helm does not upgrade CRDs. For every workload upgrade, run the fail-closed lifecycle command: ```sh -kubectl apply --server-side -f deploy/charts/t4-cluster/crds/ -kubectl wait --for=condition=Established crd/t4clusterhosts.cluster.t4.dev crd/t4workspaces.cluster.t4.dev crd/t4sessions.cluster.t4.dev +scripts/cluster-ci/crd-lifecycle.sh upgrade -- \ + helm upgrade t4-cluster deploy/charts/t4-cluster \ + --namespace t4-system --values operator-values.yaml --skip-crds ``` -Do not rely on `helm upgrade` to change CRDs. If a CRD pre-upgrade apply fails validation, stop the upgrade and keep the currently deployed controller/server images; do not force-replace the CRD or remove stored versions. +The command performs this exact order: + +1. Before any cluster access, validate every complete compatibility fixture (including `status`) against the proposed structural OpenAPI schema and CEL programs with the Kubernetes apiextensions validators. The check also rejects declared fields that the structural pruner would remove. +2. With `get` access to the three exact CRD definitions and `list` access to the corresponding namespaced T4 resources, detect which definitions already exist, enumerate every live CR for each existing definition across namespaces, and validate each complete object, including `status`, against the same proposed OpenAPI, CEL create, CEL unchanged-update, and pruning checks. A denied CRD read, denied or malformed object list, or incompatible object fails closed; an absent definition is safely empty on a fresh install. +3. Server-side dry-run the reviewed CRDs with strict validation. Upgrades stop before any mutation if a local proposed-schema validation, live-object enumeration, or CRD dry-run fails. +4. Apply the CRDs server-side without force conflicts. +5. Wait for all three CRDs to report `Established`. +6. Fetch the served `cluster.t4.dev/v1alpha1` OpenAPI v3 document three times and require every published resource schema to match the semantics generated from the proposed CRDs. Retained `Established=True` is not readiness when discovery still serves an old schema. +7. Server-side dry-run the compatibility fixtures against the converged admission path. +8. Require each CRD's `status.storedVersions` to be exactly `v1alpha1`. +9. Execute the supplied Helm command, which must use `--skip-crds`. + +The corresponding administrative checks are executable independently: ```sh -helm upgrade t4-cluster deploy/charts/t4-cluster --namespace t4-system --values operator-values.yaml +(cd packages/cluster-operator && \ + go run ./cmd/crd-preflight fixtures ../../deploy/charts/t4-cluster/crds api/v1alpha1/testdata/compat) +live_objects=$(mktemp) +trap 'rm -f "$live_objects"' EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM +for resource in t4clusterhosts.cluster.t4.dev t4workspaces.cluster.t4.dev t4sessions.cluster.t4.dev; do + installed_crd=$(kubectl get "crd/$resource" --ignore-not-found -o name) + if [ -z "$installed_crd" ]; then + continue + fi + kubectl get "$resource" --all-namespaces -o json >"$live_objects" + (cd packages/cluster-operator && \ + go run ./cmd/crd-preflight objects ../../deploy/charts/t4-cluster/crds <"$live_objects") +done +rm -f "$live_objects" +trap - EXIT HUP INT TERM +kubectl apply --server-side --dry-run=server --validate=strict \ + --field-manager=t4-crd-lifecycle -f deploy/charts/t4-cluster/crds/ +kubectl apply --server-side --validate=strict \ + --field-manager=t4-crd-lifecycle -f deploy/charts/t4-cluster/crds/ +kubectl wait --for=condition=Established --timeout=120s \ + crd/t4clusterhosts.cluster.t4.dev \ + crd/t4workspaces.cluster.t4.dev \ + crd/t4sessions.cluster.t4.dev +for observation in 1 2 3; do + kubectl get --raw /openapi/v3/apis/cluster.t4.dev/v1alpha1 | \ + (cd packages/cluster-operator && go run ./cmd/crd-preflight served ../../deploy/charts/t4-cluster/crds) +done +kubectl apply --dry-run=server --validate=strict --namespace default \ + -f packages/cluster-operator/api/v1alpha1/testdata/compat/ +for crd in t4clusterhosts.cluster.t4.dev t4workspaces.cluster.t4.dev t4sessions.cluster.t4.dev; do + test "$(kubectl get "crd/$crd" -o 'jsonpath={.status.storedVersions[*]}')" = v1alpha1 +done ``` -For an application rollback, retain the additive CRDs and use the previous known-compatible values and image digest set: +Do not rely on `helm upgrade` to change CRDs. Never use `kubectl replace --force`, `kubectl apply --force-conflicts`, delete/recreate a CRD, or alter `status.storedVersions` outside the verified migration sequence below. A preflight failure leaves the live CRDs, custom resources, controller/server workloads, and session workloads untouched. A failure after additive CRD apply but before Helm leaves the prior workloads running against the still-backward-compatible schema; investigate and rerun the gates rather than attempting CRD rollback. + +### Future `v1beta1` conversion and storage procedure + +There is no `v1beta1` API today. A future proposal is a separate incompatible lifecycle change and may proceed only through these gates: + +1. Back up all three CRDs and every custom resource, record per-kind object counts, and prove a restore in an isolated cluster. +2. Review separate `v1beta1` schemas, defaults, conversion semantics, downgrade semantics, and old/new/old fixture round-trips. Every field represented in either version must survive conversion; conversion may not manufacture product or runtime state in status. +3. Deploy a highly available conversion webhook with a PodDisruptionBudget, strict TLS/service identity, readiness checks, failure policy, metrics, and alerts. Prove conversion availability before adding a served version to any CRD. +4. Add `v1beta1` as served but not storage, retain `v1alpha1` as served and storage, wait for `Established`, and prove reads and writes through both versions plus all compatibility fixtures. Do not advance while any controller, gateway, backup, or recovery tool cannot read both versions. +5. In a later reviewed CRD apply, set `v1beta1` storage to true and `v1alpha1` storage to false while keeping both served. Reconfirm conversion health, then rewrite every object through the Kubernetes API using an approved storage-version migrator; merely changing `spec.versions[*].storage` does not migrate stored objects. +6. Compare pre/post object counts and read every rewritten object through both served versions. Verify identity, declared spec and status fields, finalizers, and conversion round-trips. Stop on any missing object or lossy read. +7. Only after that verification, explicitly retire the old storage record through the CRD status subresource for each CRD, for example `kubectl patch customresourcedefinition t4sessions.cluster.t4.dev --subresource=status --type=json -p='[{"op":"replace","path":"/status/storedVersions","value":["v1beta1"]}]'`. Repeat for hosts and workspaces. This status update records completed storage migration; it does not perform migration. +8. Read each CRD back and require `status.storedVersions` to be exactly `[v1beta1]`. Keep `v1alpha1` served, the webhook available, the verified backup, and dual-read binaries for the entire rollback window. Removing the old served version is a later explicit change after the window expires. + +If conversion or migration fails, stop forward rollout. Do not force-replace or downgrade CRDs. While both versions remain served and conversion is healthy, roll workloads back to the prior dual-read image set. If in-place compatibility cannot be proven, stop mutations and restore the verified backup under the reviewed recovery procedure before resuming service. + +### Workload rollback + +CRDs remain additive and installed. Roll back the known-compatible controller, server, T4 session runtime, and OMP digest set together: ```sh helm rollback t4-cluster REVISION --namespace t4-system ``` -Do not roll OMP independently of the T4 session runtime. Roll back the known-compatible T4/OMP image set together. The pinned authority boundary is not negotiated down. +Do not roll OMP independently of the T4 session runtime. The pinned authority boundary is not negotiated down. ## Uninstall diff --git a/packages/client/package.json b/packages/client/package.json index c960a2b3..8e29f7c2 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -17,6 +17,7 @@ "devDependencies": { "@t4-code/fixture-server": "workspace:*", "@t4-code/host-wire": "workspace:*", + "@t4-code/t4-api-client": "workspace:*", "@types/node": "catalog:", "@types/ws": "8.5.13", "vite-plus": "catalog:", diff --git a/packages/client/test/t4-api-v1-conformance-service.ts b/packages/client/test/t4-api-v1-conformance-service.ts new file mode 100644 index 00000000..312a2b84 --- /dev/null +++ b/packages/client/test/t4-api-v1-conformance-service.ts @@ -0,0 +1,513 @@ +import { createHmac, timingSafeEqual } from "node:crypto"; +import { expect } from "vite-plus/test"; + +const encoder = new TextEncoder(); +const PAGE_CURSOR_SECRET = "t4-api-v1-conformance-page-cursor-secret-2026"; +const COMMAND_BYTES_MAX = 32; +const COMMAND_REQUEST_BYTES_MAX = 256; +const METADATA_VALUE_BYTES_MAX = 32; + +function hasAtMostCodePoints(value: string, maximum: number): boolean { + let count = 0; + for (const _codePoint of value) { + count += 1; + if (count > maximum) return false; + } + return true; +} + + +function validLabels(value: unknown): boolean { + if (value === null || typeof value !== "object" || Array.isArray(value)) return false; + const entries = Object.entries(value as Record); + return entries.length <= 32 && entries.every(([key, item]) => + /^[a-z][a-z0-9.-]{0,62}$/u.test(key) && typeof item === "string" && hasAtMostCodePoints(item, 128)); +} + +function createViolation(body: Record, textField: "name" | "title"): { field: string; rule: string } | undefined { + if (Object.keys(body).some((key) => key !== textField && key !== "labels")) return { field: "body", rule: "schema" }; + const text = body[textField]; + if (typeof text !== "string" || text === "" || !hasAtMostCodePoints(text, 128)) return { field: textField, rule: "length" }; + if (body.labels !== undefined && !validLabels(body.labels)) return { field: "labels", rule: "bounds" }; + return undefined; +} + +function validMutation(body: Record, textField: "name" | "title"): boolean { + const keys = Object.keys(body); + if (keys.length < 1 || keys.some((key) => key !== textField && key !== "labels")) return false; + const text = body[textField]; + return (text === undefined || (typeof text === "string" && text !== "" && hasAtMostCodePoints(text, 128))) && + (body.labels === undefined || validLabels(body.labels)); +} + +function json(status: number, body: unknown, headers: Record = {}): Response { + return Response.json(body, { status, headers: { "T4-API-Version": "1.0", ...headers } }); +} + +function problem(status: number, code: string, message: string, extra: Record = {}): Response { + return json(status, { error: { code, message, requestId: `req-${code}`, retryable: status >= 500, ...extra } }); +} +const SNAPSHOT_BYTES_MAX = 16 * 1024 * 1024; + +function snapshotResponseAtBytes(sessionId: unknown, state: unknown, targetBytes: number): Response { + const entries = Array.from({ length: 4 }, (_, sequence) => ({ sequence, kind: "output", text: "" })); + const snapshot = { sessionId, cursor: "cursor-2", state, entries }; + let remaining = targetBytes - encoder.encode(JSON.stringify(snapshot)).byteLength; + for (const entry of entries) { + const astralCodePoints = Math.min(1_048_576, Math.floor(remaining / 4)); + entry.text = "💚".repeat(astralCodePoints); + remaining -= astralCodePoints * 4; + if (remaining > 0 && remaining < 4) { + entry.text += "x".repeat(remaining); + remaining = 0; + } + } + expect(remaining).toBe(0); + const body = JSON.stringify(snapshot); + expect(encoder.encode(body).byteLength).toBe(targetBytes); + return new Response(body, { status: 200, headers: { "Content-Type": "application/json", "T4-API-Version": "1.0" } }); +} + +function compareUtf16(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +export function canonicalJson(value: unknown): string { + if (value === null || typeof value === "boolean" || typeof value === "string") return JSON.stringify(value); + if (typeof value === "number") { + if (!Number.isFinite(value)) throw new TypeError("JCS only permits finite JSON numbers"); + return JSON.stringify(value); + } + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (typeof value === "object") { + return `{${Object.entries(value as Record) + .filter(([, item]) => item !== undefined) + .sort(([left], [right]) => compareUtf16(left, right)) + .map(([key, item]) => `${JSON.stringify(key)}:${canonicalJson(item)}`) + .join(",")}}`; + } + throw new TypeError("JCS identity requires a JSON value"); +} + +type PageCursorResult = { readonly offset: number } | { readonly error: "format" | "issued" }; + +function issuePageCursor(principal: string, collection: string, offset: number): string { + const payload = Buffer.from(canonicalJson({ principal, collection, offset })).toString("base64url"); + const signature = createHmac("sha256", PAGE_CURSOR_SECRET).update(payload).digest("base64url"); + return `page.${payload}.${signature}`; +} + +function decodePageCursor(value: string, principal: string, collection: string): PageCursorResult { + if (value.length > 512) return { error: "format" }; + const legacy = /^page-(?:0|[1-9][0-9]*)$/u.exec(value); + if (legacy !== null) return Number.isSafeInteger(Number(value.slice(5))) ? { error: "issued" } : { error: "format" }; + const match = /^page\.([A-Za-z0-9_-]+)\.([A-Za-z0-9_-]{43})$/u.exec(value); + if (match === null) return { error: "format" }; + const expected = createHmac("sha256", PAGE_CURSOR_SECRET).update(match[1]!).digest(); + const signature = Buffer.from(match[2]!, "base64url"); + if (signature.toString("base64url") !== match[2]) return { error: "format" }; + if (signature.byteLength !== expected.byteLength || !timingSafeEqual(signature, expected)) return { error: "issued" }; + let decoded: unknown; + try { decoded = JSON.parse(Buffer.from(match[1]!, "base64url").toString("utf8")); } catch { return { error: "format" }; } + if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded)) return { error: "format" }; + const payload = decoded as Record; + if (Object.keys(payload).length !== 3 || !Object.hasOwn(payload, "principal") || !Object.hasOwn(payload, "collection") || !Object.hasOwn(payload, "offset") || + Buffer.from(canonicalJson(payload)).toString("base64url") !== match[1] || + typeof payload.principal !== "string" || typeof payload.collection !== "string" || + !Number.isSafeInteger(payload.offset) || Number(payload.offset) < 1) return { error: "format" }; + if (payload.principal !== principal || payload.collection !== collection) return { error: "issued" }; + return { offset: Number(payload.offset) }; +} + +interface ReplayRecord { + readonly identity: string; + readonly body: unknown; + readonly replayStatus: number; + readonly cursor: string; +} + + +export interface T4ApiV1ConformanceOptions { + readonly invalidPayload?: "discovery" | "workspace" | "session" | "command"; + readonly snapshotBoundary?: "exact" | "over"; + readonly watchTransport?: "normal" | "bytewise" | "oversized" | "many-small"; +} + +export class T4ApiV1ConformanceService { + readonly origin = "https://t4-api.conformance.test"; + readonly calls: Array<{ method: string; path: string; authorization: string | null }> = []; + readonly abortedWatches: string[] = []; + readonly watchCursors: Array<{ query: string | null; header: string | null }> = []; + readonly watchQueries: Array<{ maxEvents: string | null; heartbeatSeconds: string | null }> = []; + + #workspaceSequence = 0; + #sessionSequence = 0; + #commandSequence = 0; + #eventSequence = 0; + readonly #workspaces = new Map>(); + readonly #sessions = new Map>(); + readonly #replays = new Map(); + + constructor(readonly options: T4ApiV1ConformanceOptions = {}) {} + + readonly fetch: typeof globalThis.fetch = async (input, init) => { + const request = new Request(input, init); + const url = new URL(request.url); + const authorization = request.headers.get("authorization"); + this.calls.push({ method: request.method, path: url.pathname, authorization }); + if (url.origin !== this.origin) return problem(400, "invalid_origin", "HTTPS API origin is fixed for this client"); + if (url.protocol !== "https:") return problem(400, "https_required", "HTTPS is required"); + if (authorization !== "Bearer token-a" && authorization !== "Bearer token-b" && authorization !== "Bearer token-denied") { + return problem(401, "unauthenticated", "A valid bearer credential is required"); + } + if (request.headers.get("T4-API-Version") !== "1") { + return problem(406, "incompatible_version", "No compatible T4 API major version", { supportedMajors: [1] }); + } + if (authorization === "Bearer token-denied") return problem(403, "forbidden", "Credential lacks the required operation scope"); + const tenant = authorization === "Bearer token-a" ? "tenant-a" : "tenant-b"; + + if (request.method === "GET" && url.pathname === "/v1") { + return json(200, this.#payload("discovery", { + apiVersion: "1.0", + serverBuild: { version: "0.1.30", revision: "fixture-revision-1" }, + supportedMajors: [1], + capabilities: { + "workspace.lifecycle": { supported: true, enabled: true, authorized: true, available: true }, + "session.lifecycle": { supported: true, enabled: true, authorized: true, available: true }, + "session.commands": { supported: true, enabled: true, authorized: true, available: true }, + "session.watch.sse": { supported: true, enabled: true, authorized: true, available: true }, + "optional.preview": { + supported: true, enabled: false, authorized: false, available: false, + deprecation: { message: "Use workspace.lifecycle", sinceVersion: "1.0", sunsetAt: "2027-01-01T00:00:00Z", replacement: "workspace.lifecycle" }, + }, + }, + limits: { + pageSizeDefault: 2, + pageSizeMax: 3, + commandBytesMax: COMMAND_BYTES_MAX, + commandRequestBytesMax: COMMAND_REQUEST_BYTES_MAX, + commandMetadataValueBytesMax: METADATA_VALUE_BYTES_MAX, + watchEventsMax: this.options.watchTransport === "many-small" ? 1000 : 4, + heartbeatSeconds: 15, + }, + })); + } + + if (request.method === "POST" && url.pathname === "/v1/workspaces") { + const parsed = await this.#jsonBody(request); + if (parsed instanceof Response) return parsed; + const body = parsed; + const violation = createViolation(body, "name"); + if (violation !== undefined) return this.#invalid(violation.field, violation.rule, "workspace create must match WorkspaceCreate"); + return this.#idempotent(request, tenant, "createWorkspace", [], body, 202, 200, () => { + const id = `ws-${++this.#workspaceSequence}`; + const workspace = { id, name: body.name, ...(body.labels === undefined ? {} : { labels: body.labels }), state: "accepted", revision: 1, tenant }; + this.#workspaces.set(id, workspace); + return workspace; + }); + } + + if (request.method === "GET" && url.pathname === "/v1/workspaces") { + const pageSize = Number(url.searchParams.get("pageSize") ?? "2"); + if (!Number.isInteger(pageSize) || pageSize < 1 || pageSize > 3) return this.#invalid("pageSize", "range", "pageSize must be between 1 and 3"); + const collection = "workspaces"; + const cursor = url.searchParams.get("cursor"); + let start = 0; + if (cursor !== null) { + const decoded = decodePageCursor(cursor, tenant, collection); + if ("error" in decoded) return this.#invalid("cursor", decoded.error, decoded.error === "format" ? "cursor must be a canonical bounded opaque token" : "cursor was not issued for this principal and collection"); + start = decoded.offset; + } + const visible = [...this.#workspaces.values()].filter((item) => item.tenant === tenant); + if (cursor !== null && start >= visible.length) return this.#invalid("cursor", "issued", "cursor is outside the current collection"); + const items = visible.slice(start, start + pageSize).map(({ tenant: _tenant, ...item }) => item); + const next = start + items.length < visible.length ? issuePageCursor(tenant, collection, start + items.length) : undefined; + return json(200, { items, ...(next === undefined ? {} : { nextCursor: next }) }); + } + + const workspaceMatch = url.pathname.match(/^\/v1\/workspaces\/([^/]+)$/u); + if (workspaceMatch) { + const id = decodeURIComponent(workspaceMatch[1]!); + if (request.method === "PATCH") { + const ifMatch = request.headers.get("If-Match"); + if (ifMatch === null || !/^[1-9][0-9]{0,18}$/u.test(ifMatch)) return problem(400, "invalid_request", "If-Match is invalid"); + } + const workspace = this.#workspaces.get(id); + if (request.method === "DELETE") { + return this.#idempotent(request, tenant, "deleteWorkspace", [id], null, 204, 204, () => { + this.#workspaces.delete(id); + return null; + }, () => workspace?.tenant === tenant ? undefined : problem(404, "not_found", "Workspace not found")); + } + if (workspace?.tenant !== tenant) return problem(404, "not_found", "Workspace not found"); + if (request.method === "GET") return json(200, this.#payload("workspace", this.#visible(workspace))); + if (request.method === "PATCH") { + const parsed = await this.#jsonBody(request); + if (parsed instanceof Response) return parsed; + const body = parsed; + if (!validMutation(body, "name")) return this.#invalid("body", "schema", "workspace mutation must match WorkspaceMutation"); + return this.#idempotent(request, tenant, "mutateWorkspace", [id], body, 200, 200, () => { + const updated = { ...workspace, ...(body.name === undefined ? {} : { name: body.name }), ...(body.labels === undefined ? {} : { labels: body.labels }), revision: Number(workspace.revision) + 1 }; + this.#workspaces.set(id, updated); + return updated; + }, () => request.headers.get("If-Match") === String(workspace.revision) ? undefined : problem(409, "revision_conflict", "Workspace revision changed")); + } + } + + const sessionsPath = url.pathname.match(/^\/v1\/workspaces\/([^/]+)\/sessions$/u); + if (sessionsPath) { + const workspaceId = decodeURIComponent(sessionsPath[1]!); + const workspace = this.#workspaces.get(workspaceId); + if (workspace?.tenant !== tenant) return problem(404, "not_found", "Workspace not found"); + if (request.method === "POST") { + const parsed = await this.#jsonBody(request); + if (parsed instanceof Response) return parsed; + const body = parsed; + const violation = createViolation(body, "title"); + if (violation !== undefined) return this.#invalid(violation.field, violation.rule, "session create must match SessionCreate"); + return this.#idempotent(request, tenant, "spawnSession", [workspaceId], body, 202, 200, () => { + const id = `ses-${++this.#sessionSequence}`; + const session = { id, workspaceId, title: body.title, ...(body.labels === undefined ? {} : { labels: body.labels }), state: "accepted", revision: 1, tenant }; + this.#sessions.set(id, session); + return session; + }); + } + if (request.method === "GET") { + const pageSize = Number(url.searchParams.get("pageSize") ?? "2"); + if (!Number.isInteger(pageSize) || pageSize < 1 || pageSize > 3) return this.#invalid("pageSize", "range", "pageSize must be between 1 and 3"); + const collection = `workspaces/${workspaceId}/sessions`; + const cursor = url.searchParams.get("cursor"); + let start = 0; + if (cursor !== null) { + const decoded = decodePageCursor(cursor, tenant, collection); + if ("error" in decoded) return this.#invalid("cursor", decoded.error, decoded.error === "format" ? "cursor must be a canonical bounded opaque token" : "cursor was not issued for this principal and collection"); + start = decoded.offset; + } + const visible = [...this.#sessions.values()].filter((item) => item.workspaceId === workspaceId && item.tenant === tenant); + if (cursor !== null && start >= visible.length) return this.#invalid("cursor", "issued", "cursor is outside the current collection"); + const items = visible.slice(start, start + pageSize).map((item) => this.#visible(item)); + const next = start + items.length < visible.length ? issuePageCursor(tenant, collection, start + items.length) : undefined; + return json(200, { items, ...(next === undefined ? {} : { nextCursor: next }) }); + } + } + + const sessionMatch = url.pathname.match(/^\/v1\/sessions\/([^/]+)$/u); + if (sessionMatch) { + const id = decodeURIComponent(sessionMatch[1]!); + if (request.method === "PATCH") { + const ifMatch = request.headers.get("If-Match"); + if (ifMatch === null || !/^[1-9][0-9]{0,18}$/u.test(ifMatch)) return problem(400, "invalid_request", "If-Match is invalid"); + } + const session = this.#sessions.get(id); + if (request.method === "DELETE") { + return this.#idempotent(request, tenant, "deleteSession", [id], null, 204, 204, () => { + this.#sessions.delete(id); + return null; + }, () => session?.tenant === tenant ? undefined : problem(404, "not_found", "Session not found")); + } + if (session?.tenant !== tenant) return problem(404, "not_found", "Session not found"); + if (request.method === "GET") return json(200, this.#payload("session", this.#visible(session))); + if (request.method === "PATCH") { + const parsed = await this.#jsonBody(request); + if (parsed instanceof Response) return parsed; + const body = parsed; + if (!validMutation(body, "title")) return this.#invalid("body", "schema", "session mutation must match SessionMutation"); + return this.#idempotent(request, tenant, "mutateSession", [id], body, 200, 200, () => { + const updated = { ...session, ...(body.title === undefined ? {} : { title: body.title }), ...(body.labels === undefined ? {} : { labels: body.labels }), revision: Number(session.revision) + 1 }; + this.#sessions.set(id, updated); + return updated; + }, () => request.headers.get("If-Match") === String(session.revision) ? undefined : problem(409, "revision_conflict", "Session revision changed")); + } + } + + const cancelMatch = url.pathname.match(/^\/v1\/sessions\/([^/]+)\/cancel$/u); + if (cancelMatch && request.method === "POST") { + const id = decodeURIComponent(cancelMatch[1]!); + const session = this.#sessions.get(id); + if (session?.tenant !== tenant) return problem(404, "not_found", "Session not found"); + return this.#idempotent(request, tenant, "cancelSession", [id], null, 202, 200, () => { + const cancelled = { ...session, state: "cancelled", revision: Number(session.revision) + 1 }; + this.#sessions.set(id, cancelled); + return cancelled; + }); + } + + const commandMatch = url.pathname.match(/^\/v1\/sessions\/([^/]+)\/commands$/u); + if (commandMatch && request.method === "POST") { + const id = decodeURIComponent(commandMatch[1]!); + const session = this.#sessions.get(id); + if (session?.tenant !== tenant) return problem(404, "not_found", "Session not found"); + const mediaError = this.#jsonMediaError(request); + if (mediaError !== undefined) return mediaError; + const text = await request.text(); + if (encoder.encode(text).byteLength > COMMAND_REQUEST_BYTES_MAX) return this.#invalid("body", "maxBytes", `request must not exceed ${COMMAND_REQUEST_BYTES_MAX} UTF-8 bytes`); + let decoded: unknown; + try { decoded = JSON.parse(text); } catch { return problem(400, "invalid_request", "Malformed JSON request"); } + if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded)) return this.#invalid("body", "schema", "command create must match CommandCreate"); + const body = decoded as Record; + if (Object.keys(body).some((key) => key !== "command" && key !== "metadata")) return this.#invalid("body", "schema", "command create must match CommandCreate"); + if (typeof body.command !== "string" || body.command.length < 1 || encoder.encode(body.command).byteLength > COMMAND_BYTES_MAX) return this.#invalid("command", "maxBytes", `command must contain 1 to ${COMMAND_BYTES_MAX} UTF-8 bytes`); + if (!this.#validMetadata(body.metadata)) return this.#invalid("metadata", "bounds", "metadata keys and values exceed the discovered bounds"); + const states: Record = { accepted: true, projected: true, dispatching: true, running: true, succeeded: true, failed: true, cancelling: true, cancelled: true, rejected: true, unavailable: true, indeterminate: true }; + const state = states[body.command] === true ? body.command : "accepted"; + return this.#idempotent(request, tenant, "submitCommand", [id], { metadata: {}, ...body }, 202, 200, () => ({ commandId: `cmd-${++this.#commandSequence}`, state })); + } + + const snapshotMatch = url.pathname.match(/^\/v1\/sessions\/([^/]+)\/snapshot$/u); + if (snapshotMatch && request.method === "GET") { + const session = this.#sessions.get(decodeURIComponent(snapshotMatch[1]!)); + if (session?.tenant !== tenant) return problem(404, "not_found", "Session not found"); + if (this.options.snapshotBoundary !== undefined) { + return snapshotResponseAtBytes(session.id, session.state, SNAPSHOT_BYTES_MAX + (this.options.snapshotBoundary === "over" ? 1 : 0)); + } + return json(200, { sessionId: session.id, cursor: "cursor-2", state: session.state, entries: [{ sequence: 2, kind: "output", text: "ready" }] }); + } + + const watchMatch = url.pathname.match(/^\/v1\/sessions\/([^/]+)\/events$/u); + if (watchMatch && request.method === "GET") { + if (request.headers.get("Accept") !== "text/event-stream") return problem(406, "incompatible_version", "Watch requests must accept text/event-stream", { supportedMajors: [1] }); + const id = decodeURIComponent(watchMatch[1]!); + const session = this.#sessions.get(id); + if (session?.tenant !== tenant) return problem(404, "not_found", "Session not found"); + const maxEvents = Number(url.searchParams.get("maxEvents") ?? "100"); + const heartbeat = Number(url.searchParams.get("heartbeatSeconds") ?? "15"); + this.watchQueries.push({ maxEvents: url.searchParams.get("maxEvents"), heartbeatSeconds: url.searchParams.get("heartbeatSeconds") }); + if (!Number.isInteger(maxEvents) || maxEvents < 1 || maxEvents > (this.options.watchTransport === "many-small" ? 1000 : 4)) return this.#invalid("maxEvents", "range", "maxEvents exceeds the discovered bound"); + if (!Number.isInteger(heartbeat) || heartbeat < 5 || heartbeat > 60) return this.#invalid("heartbeatSeconds", "range", "heartbeatSeconds must be between 5 and 60"); + const queryCursor = url.searchParams.get("cursor"); + const headerCursor = request.headers.get("Last-Event-ID"); + this.watchCursors.push({ query: queryCursor, header: headerCursor }); + if (queryCursor !== null && headerCursor !== null && queryCursor !== headerCursor) return problem(400, "invalid_request", "Reconnect cursors disagree"); + const cursor = queryCursor ?? headerCursor; + if (cursor === "expired") return problem(410, "cursor_expired", "Watch cursor is no longer retained", { resync: { snapshotUrl: `/v1/sessions/${id}/snapshot`, cursor: "cursor-2" } }); + const allFrames = cursor === "cursor-4" + ? [`id: cursor-4\r\nevent: heartbeat\r\ndata: {"type":"heartbeat","cursor":"cursor-4","observedAt":"2026-07-21T00:00:15Z"}\r\n\r\n`] + : cursor === "cursor-5" + ? [`id: cursor-6\nevent: command\ndata: {"type":"command","cursor":"cursor-6","commandId":"cmd-1","state":"accepted"}\n\n`] + : [ + `id: ${cursor ?? "cursor-3"}\r\nevent: heartbeat\r\ndata: {"type":"heartbeat","cursor":"${cursor ?? "cursor-3"}","observedAt":"2026-07-21T00:00:00Z"}\r\n\r\n`, + `id: cursor-4\r\nevent: session\r\ndata: {"type":"session","cursor":"cursor-4","state":"accepted","revision":2}\r\n\r\n`, + ]; + let payload: Uint8Array; + if (this.options.watchTransport === "oversized") { + payload = encoder.encode(`: ${"x".repeat(1024 * 1024)}\ndata: {"type":"heartbeat","cursor":"large-1","observedAt":"2026-07-21T00:00:00Z"}\n\n`); + } else if (this.options.watchTransport === "many-small") { + payload = encoder.encode(Array.from({ length: maxEvents }, () => `: ${"x".repeat(1010)}\ndata: {"type":"heartbeat","cursor":"bulk-0","observedAt":"2026-07-21T00:00:00Z"}\n\n`).join("")); + } else { + const prefix = this.options.watchTransport === "bytewise" ? ": 💚\n" : ""; + payload = encoder.encode(prefix + allFrames.slice(0, maxEvents).join("")); + } + const stream = new ReadableStream({ + start: (controller) => { + if (this.options.watchTransport === "bytewise") { + for (const byte of payload) controller.enqueue(Uint8Array.of(byte)); + } else { + const split = Math.max(1, payload.byteLength - 3); + controller.enqueue(payload.slice(0, split)); + controller.enqueue(payload.slice(split)); + } + controller.close(); + request.signal.addEventListener("abort", () => this.abortedWatches.push(id), { once: true }); + }, + }); + return new Response(stream, { status: 200, headers: { "Content-Type": "text/event-stream", "Cache-Control": "no-store", "T4-API-Version": "1.0" } }); + } + + return problem(404, "not_found", "Resource not found"); + }; + + expectNoCredentialLeak(): void { + expect(this.calls.every((call) => call.authorization === "Bearer token-a" || call.authorization === "Bearer token-b" || call.authorization === "Bearer token-denied")).toBe(true); + } + + #jsonMediaError(request: Request): Response | undefined { + const mediaType = request.headers.get("Content-Type")?.split(";", 1)[0]?.trim().toLowerCase(); + return mediaType === "application/json" ? undefined : problem(400, "invalid_request", "Content-Type must be application/json"); + } + + async #jsonBody(request: Request): Promise | Response> { + const mediaError = this.#jsonMediaError(request); + if (mediaError !== undefined) return mediaError; + try { + const value: unknown = await request.json(); + if (value === null || typeof value !== "object" || Array.isArray(value)) return problem(400, "invalid_request", "JSON request body must be an object"); + return value as Record; + } catch { + return problem(400, "invalid_request", "Malformed JSON request"); + } + } + + #payload(kind: T4ApiV1ConformanceOptions["invalidPayload"], value: Record): Record { + return this.options.invalidPayload === kind ? { ...value, unknown: true } : value; + } + + #invalid(field: string, rule: string, message: string): Response { + return problem(422, "invalid_request", "Request validation failed", { violations: [{ field, rule, message }] }); + } + + #visible(value: Record): Record { + const { tenant: _tenant, ...visible } = value; + return visible; + } + + #validMetadata(value: unknown): boolean { + if (value === undefined) return true; + if (value === null || typeof value !== "object" || Array.isArray(value)) return false; + const entries = Object.entries(value as Record); + return entries.length <= 32 && entries.every(([key, item]) => + /^[a-z][a-z0-9.-]{0,62}$/u.test(key) && + (item === null || typeof item === "boolean" || + (typeof item === "number" && Number.isSafeInteger(item)) || + (typeof item === "string" && encoder.encode(item).byteLength <= METADATA_VALUE_BYTES_MAX))); + } + + + #idempotent( + request: Request, + principal: string, + operationId: string, + targets: readonly string[], + body: unknown, + firstStatus: number, + replayStatus: number, + create: () => unknown, + validate?: () => Response | undefined, + ): Response { + const key = request.headers.get("Idempotency-Key"); + if (key === null) return problem(400, "idempotency_key_required", "Idempotency-Key is required"); + if (!/^[A-Za-z0-9._~-]{16,128}$/u.test(key)) return problem(400, "invalid_request", "Idempotency-Key is invalid"); + const preconditions = request.headers.has("If-Match") ? { "if-match": request.headers.get("If-Match") } : {}; + const identity = canonicalJson({ operationId, targets, preconditions, body }); + const scope = canonicalJson({ principal, operationId, targets, key }); + const prior = this.#replays.get(scope); + if (prior !== undefined) { + if (prior.identity !== identity) return problem(409, "idempotency_conflict", "Idempotency key was reused with a different canonical request"); + const headers = { "T4-API-Version": "1.0", "Idempotency-Replayed": "true", "T4-Event-Cursor": prior.cursor }; + return prior.body === null + ? new Response(null, { status: prior.replayStatus, headers }) + : json(prior.replayStatus, prior.body, headers); + } + const validation = validate?.(); + if (validation !== undefined) return validation; + const created = create(); + const cursor = `event-${++this.#eventSequence}`; + const visibleValue = created !== null && typeof created === "object" && !Array.isArray(created) + ? this.#visible(created as Record) + : created; + const responseKind = operationId === "submitCommand" + ? "command" + : operationId === "spawnSession" || operationId === "mutateSession" || operationId === "cancelSession" + ? "session" + : operationId === "createWorkspace" || operationId === "mutateWorkspace" + ? "workspace" + : undefined; + const visible = visibleValue !== null && typeof visibleValue === "object" && !Array.isArray(visibleValue) && responseKind !== undefined + ? this.#payload(responseKind, visibleValue as Record) + : visibleValue; + this.#replays.set(scope, { identity, body: visible, replayStatus, cursor }); + const headers = { "Idempotency-Replayed": "false", "T4-Event-Cursor": cursor }; + return visible === null + ? new Response(null, { status: firstStatus, headers: { "T4-API-Version": "1.0", ...headers } }) + : json(firstStatus, visible, headers); + } +} diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts new file mode 100644 index 00000000..ea8afef9 --- /dev/null +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -0,0 +1,1662 @@ +import { readFileSync } from "node:fs"; + +import { describe, expect, it, vi } from "vite-plus/test"; + +import { + T4ApiError, + createT4ApiClient, + type components, +} from "@t4-code/t4-api-client"; +import { T4ApiV1ConformanceService, canonicalJson } from "./t4-api-v1-conformance-service.ts"; + +type WorkspaceCreate = components["schemas"]["WorkspaceCreate"]; +type SessionCreate = components["schemas"]["SessionCreate"]; +type CommandCreate = components["schemas"]["CommandCreate"]; + +type WatchEvent = components["schemas"]["WatchEvent"]; + +function exhaustWatchEvent(event: WatchEvent): string { + switch (event.type) { + case "heartbeat": return event.observedAt; + case "session": return `${event.state}:${event.revision}`; + case "command": return `${event.commandId}:${event.state}`; + default: { + const unreachable: never = event; + return unreachable; + } + } +} + +const VERSION_HEADERS = { "T4-API-Version": "1" } as const; + +const COMMAND_STATES = [ + "accepted", "projected", "dispatching", "running", "succeeded", "failed", + "cancelling", "cancelled", "rejected", "unavailable", "indeterminate", +] as const satisfies readonly components["schemas"]["CommandState"][]; +const DISCOVERY = { + apiVersion: "1.0", + serverBuild: { version: "0.1.30", revision: "fixture-revision-1" }, + supportedMajors: [1], + capabilities: { + "workspace.lifecycle": { supported: true, enabled: true, authorized: true, available: true }, + "optional.preview": { + supported: true, enabled: false, authorized: false, available: false, + deprecation: { message: "Use workspace.lifecycle", sinceVersion: "1.0", sunsetAt: "2027-01-01T00:00:00Z", replacement: "workspace.lifecycle" }, + }, + }, + limits: { pageSizeDefault: 1, pageSizeMax: 1, commandBytesMax: 1, commandRequestBytesMax: 1, commandMetadataValueBytesMax: 1, watchEventsMax: 1, heartbeatSeconds: 5 }, +} as const; + +function withSelectedVersion(init: ResponseInit = {}): ResponseInit { + const headers = new Headers(init.headers); + if (!headers.has("T4-API-Version")) headers.set("T4-API-Version", "1.0"); + if (headers.get("Content-Type")?.split(";", 1)[0]?.trim().toLowerCase() === "text/event-stream" && !headers.has("Cache-Control")) headers.set("Cache-Control", "no-store"); + return { ...init, headers }; +} + +function jsonResponse(body: unknown, init: ResponseInit = {}): Response { + return Response.json(body, withSelectedVersion(init)); +} + +function apiResponse(body?: BodyInit | null, init: ResponseInit = {}): Response { + return new Response(body, withSelectedVersion(init)); +} + +function idempotencyHeaders(key: string): Readonly<{ "T4-API-Version": "1"; "Idempotency-Key": string }> { + return { ...VERSION_HEADERS, "Idempotency-Key": key }; +} + +function mutationHeaders(revision: number, key: string): Readonly<{ "T4-API-Version": "1"; "If-Match": string; "Idempotency-Key": string }> { + return { ...VERSION_HEADERS, "If-Match": String(revision), "Idempotency-Key": key }; +} + +function requireEventCursor(response: Response): string { + const cursor = response.headers.get("T4-Event-Cursor"); + expect(cursor).toMatch(/^[A-Za-z0-9._~-]+$/u); + return cursor ?? ""; +} + +function requireData(result: { data?: T; error?: unknown }): T { + expect(result.error).toBeUndefined(); + expect(result.data).toBeDefined(); + return result.data!; +} + +async function seededClient(service: T4ApiV1ConformanceService, key: string) { + const client = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); + requireData(await client.http.POST("/v1/workspaces", { + body: { name: "workspace" }, params: { header: idempotencyHeaders(`workspace-${key}-0001`) }, + })); + requireData(await client.http.POST("/v1/workspaces/{workspaceId}/sessions", { + body: { title: "agent" }, params: { header: idempotencyHeaders(`session-${key}-00001`), path: { workspaceId: "ws-1" } }, + })); + return client; +} + +describe("generated T4 API v1 client conformance", () => { + it("negotiates truthful discovery metadata and rejects an incompatible major", async () => { + const service = new T4ApiV1ConformanceService(); + const client = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); + const discovery = requireData(await client.http.GET("/v1", { params: { header: VERSION_HEADERS } })); + expect(discovery).toMatchObject({ + apiVersion: "1.0", + serverBuild: { version: "0.1.30", revision: "fixture-revision-1" }, + supportedMajors: [1], + capabilities: { + "workspace.lifecycle": { supported: true, enabled: true, authorized: true, available: true }, + "optional.preview": { supported: true, enabled: false, authorized: false, available: false }, + }, + limits: { pageSizeMax: 3, commandBytesMax: 32, watchEventsMax: 4, heartbeatSeconds: 15 }, + }); + + const incompatible = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 2, fetch: service.fetch }); + const rejected = await incompatible.http.GET("/v1", { params: { header: VERSION_HEADERS } }); + expect(rejected.response.status).toBe(406); + expect(rejected.error).toMatchObject({ error: { code: "incompatible_version", retryable: false, supportedMajors: [1] } }); + + const silentDowngrade = createT4ApiClient({ + baseUrl: "https://silent-downgrade.test", credential: "token-a", majorVersion: 2, + fetch: async () => jsonResponse(DISCOVERY), + }); + await expect(silentDowngrade.http.GET("/v1", { params: { header: VERSION_HEADERS } })).rejects.toMatchObject({ + code: "indeterminate", status: 502, retryable: false, + }); + + const unauthenticated = await service.fetch(`${service.origin}/v1`, { headers: { "T4-API-Version": "1" } }); + expect(unauthenticated.status).toBe(401); + expect(await unauthenticated.json()).toMatchObject({ error: { code: "unauthenticated", retryable: false } }); + + const denied = createT4ApiClient({ baseUrl: service.origin, credential: "token-denied", majorVersion: 1, fetch: service.fetch }); + const forbidden = await denied.http.GET("/v1", { params: { header: VERSION_HEADERS } }); + expect(forbidden.response.status).toBe(403); + expect(forbidden.error).toMatchObject({ error: { code: "forbidden" } }); + expect(discovery.limits).toMatchObject({ commandBytesMax: 32, commandRequestBytesMax: 256, commandMetadataValueBytesMax: 32 }); + }); + + it("creates, canonically replays, conflicts, mutates, paginates, isolates, and deletes workspaces", async () => { + const service = new T4ApiV1ConformanceService(); + const owner = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); + const other = createT4ApiClient({ baseUrl: service.origin, credential: "token-b", majorVersion: 1, fetch: service.fetch }); + const body = { name: "primary", labels: { beta: "2", alpha: "1" } } satisfies WorkspaceCreate; + const first = await owner.http.POST("/v1/workspaces", { + body, + params: { header: idempotencyHeaders("workspace-create-0001") }, + }); + expect(first.response.status).toBe(202); + const workspace = requireData(first); + expect(workspace).toMatchObject({ id: "ws-1", state: "accepted", revision: 1 }); + expect(workspace).not.toHaveProperty("tenant"); + + const replay = await owner.http.POST("/v1/workspaces", { + body: { labels: { alpha: "1", beta: "2" }, name: "primary" }, + params: { header: idempotencyHeaders("workspace-create-0001") }, + }); + expect(replay.response.status).toBe(200); + expect(replay.response.headers.get("Idempotency-Replayed")).toBe("true"); + expect(replay.data).toEqual(workspace); + const conflict = await owner.http.POST("/v1/workspaces", { + body: { name: "different" }, + params: { header: idempotencyHeaders("workspace-create-0001") }, + }); + expect(conflict.response.status).toBe(409); + expect(conflict.error).toMatchObject({ error: { code: "idempotency_conflict", retryable: false } }); + + const omitted = await owner.http.POST("/v1/workspaces", { + body: { name: "primary" }, params: { header: idempotencyHeaders("workspace-omitted-0001") }, + }); + expect(omitted.response.status).toBe(202); + const explicitEmpty = await owner.http.POST("/v1/workspaces", { + body: { name: "primary", labels: {} }, params: { header: idempotencyHeaders("workspace-omitted-0001") }, + }); + expect(explicitEmpty.response.status).toBe(409); + expect(canonicalJson({ z: [1, 2], a: { y: 2, x: 1 } })).toBe('{"a":{"x":1,"y":2},"z":[1,2]}'); + expect(canonicalJson({ z: [1, 2] })).not.toBe(canonicalJson({ z: [2, 1] })); + + for (const name of ["second", "third", "fourth"]) { + requireData(await owner.http.POST("/v1/workspaces", { + body: { name }, + params: { header: idempotencyHeaders(`workspace-${name}-0001`) }, + })); + } + const pageOne = requireData(await owner.http.GET("/v1/workspaces", { + params: { header: VERSION_HEADERS, query: { pageSize: 3 } }, + })); + expect(pageOne.items).toHaveLength(3); + expect(pageOne.nextCursor).toMatch(/^[A-Za-z0-9._~-]+$/u); + const pageTwo = requireData(await owner.http.GET("/v1/workspaces", { + params: { header: VERSION_HEADERS, query: { pageSize: 3, cursor: pageOne.nextCursor! } }, + })); + expect(pageTwo.items).toHaveLength(2); + expect(pageTwo.nextCursor).toBeUndefined(); + + const isolated = await other.http.GET("/v1/workspaces/{workspaceId}", { + params: { header: VERSION_HEADERS, path: { workspaceId: "ws-1" } }, + }); + expect(isolated.response.status).toBe(404); + expect(isolated.error).toMatchObject({ error: { code: "not_found" } }); + const invalid = await owner.http.POST("/v1/workspaces", { + body: { name: "x".repeat(129) }, + params: { header: idempotencyHeaders("workspace-invalid-0001") }, + }); + expect(invalid.response.status).toBe(422); + expect(invalid.error).toMatchObject({ error: { code: "invalid_request", violations: [{ field: "name", rule: "length" }] } }); + + const updated = requireData(await owner.http.PATCH("/v1/workspaces/{workspaceId}", { + params: { header: mutationHeaders(1, "workspace-patch-0001"), path: { workspaceId: "ws-1" } }, + body: { name: "renamed" }, + })); + expect(updated).toMatchObject({ name: "renamed", revision: 2 }); + const updatedReplay = await owner.http.PATCH("/v1/workspaces/{workspaceId}", { + params: { header: mutationHeaders(1, "workspace-patch-0001"), path: { workspaceId: "ws-1" } }, body: { name: "renamed" }, + }); + expect(updatedReplay.response.status).toBe(200); + expect(updatedReplay.response.headers.get("Idempotency-Replayed")).toBe("true"); + expect(updatedReplay.data).toEqual(updated); + const stale = await owner.http.PATCH("/v1/workspaces/{workspaceId}", { + params: { header: mutationHeaders(1, "workspace-patch-0002"), path: { workspaceId: "ws-1" } }, + body: { name: "stale" }, + }); + expect(stale.response.status).toBe(409); + expect(stale.error).toMatchObject({ error: { code: "revision_conflict" } }); + const targetOne = await owner.http.PATCH("/v1/workspaces/{workspaceId}", { + params: { header: mutationHeaders(2, "workspace-shared-0001"), path: { workspaceId: "ws-1" } }, body: { name: "target" }, + }); + const targetTwo = await owner.http.PATCH("/v1/workspaces/{workspaceId}", { + params: { header: mutationHeaders(1, "workspace-shared-0001"), path: { workspaceId: "ws-2" } }, body: { name: "target" }, + }); + expect(targetOne.response.status).toBe(200); + expect(targetTwo.response.status).toBe(200); + const changedPrecondition = await owner.http.PATCH("/v1/workspaces/{workspaceId}", { + params: { header: mutationHeaders(3, "workspace-shared-0001"), path: { workspaceId: "ws-1" } }, body: { name: "target" }, + }); + expect(changedPrecondition.response.status).toBe(409); + expect(changedPrecondition.error).toMatchObject({ error: { code: "idempotency_conflict" } }); + expect((await owner.http.DELETE("/v1/workspaces/{workspaceId}", { + params: { header: idempotencyHeaders("workspace-delete-0001"), path: { workspaceId: "ws-1" } }, + })).response.status).toBe(204); + service.expectNoCredentialLeak(); + }); + + it("replays successful workspace deletion only within the original principal and request scope", async () => { + const service = new T4ApiV1ConformanceService(); + const owner = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); + const other = createT4ApiClient({ baseUrl: service.origin, credential: "token-b", majorVersion: 1, fetch: service.fetch }); + requireData(await owner.http.POST("/v1/workspaces", { + body: { name: "workspace" }, params: { header: idempotencyHeaders("workspace-delete-setup") }, + })); + const first = await owner.http.DELETE("/v1/workspaces/{workspaceId}", { + params: { header: idempotencyHeaders("workspace-delete-replay"), path: { workspaceId: "ws-1" } }, + }); + const replay = await owner.http.DELETE("/v1/workspaces/{workspaceId}", { + params: { header: idempotencyHeaders("workspace-delete-replay"), path: { workspaceId: "ws-1" } }, + }); + expect(first.response.status).toBe(204); + expect(first.response.headers.get("Idempotency-Replayed")).toBe("false"); + expect(replay.response.status).toBe(204); + expect(replay.response.headers.get("Idempotency-Replayed")).toBe("true"); + expect((await owner.http.DELETE("/v1/workspaces/{workspaceId}", { + params: { header: idempotencyHeaders("workspace-delete-fresh"), path: { workspaceId: "ws-1" } }, + })).response.status).toBe(404); + expect((await other.http.DELETE("/v1/workspaces/{workspaceId}", { + params: { header: idempotencyHeaders("workspace-delete-replay"), path: { workspaceId: "ws-1" } }, + })).response.status).toBe(404); + }); + + it("spawns, paginates, and mutates sessions, then submits idempotent stable command states", async () => { + const service = new T4ApiV1ConformanceService(); + const client = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); + requireData(await client.http.POST("/v1/workspaces", { + body: { name: "workspace" }, params: { header: idempotencyHeaders("workspace-setup-0001") }, + })); + const body = { title: "agent" } satisfies SessionCreate; + const first = await client.http.POST("/v1/workspaces/{workspaceId}/sessions", { + params: { header: idempotencyHeaders("session-spawn-0001"), path: { workspaceId: "ws-1" } }, + body, + }); + expect(first.response.status).toBe(202); + const session = requireData(first); + const replay = await client.http.POST("/v1/workspaces/{workspaceId}/sessions", { + params: { header: idempotencyHeaders("session-spawn-0001"), path: { workspaceId: "ws-1" } }, + body, + }); + expect(replay.data).toEqual(session); + for (const [key, title] of [["session-spawn-0002", "agent-two"], ["session-spawn-0003", "agent-three"]] as const) { + requireData(await client.http.POST("/v1/workspaces/{workspaceId}/sessions", { + params: { header: idempotencyHeaders(key), path: { workspaceId: "ws-1" } }, body: { title }, + })); + } + const sessionPageOne = requireData(await client.http.GET("/v1/workspaces/{workspaceId}/sessions", { + params: { header: VERSION_HEADERS, path: { workspaceId: "ws-1" }, query: { pageSize: 2 } }, + })); + expect(sessionPageOne.items).toHaveLength(2); + expect(sessionPageOne.nextCursor).toMatch(/^[A-Za-z0-9._~-]+$/u); + const sessionPageTwo = requireData(await client.http.GET("/v1/workspaces/{workspaceId}/sessions", { + params: { header: VERSION_HEADERS, path: { workspaceId: "ws-1" }, query: { pageSize: 2, cursor: sessionPageOne.nextCursor! } }, + })); + expect(sessionPageTwo.items).toHaveLength(1); + + const mutated = requireData(await client.http.PATCH("/v1/sessions/{sessionId}", { + params: { header: mutationHeaders(1, "session-patch-0001"), path: { sessionId: "ses-1" } }, body: { title: "renamed-agent" }, + })); + expect(mutated).toMatchObject({ title: "renamed-agent", revision: 2 }); + const mutatedReplay = await client.http.PATCH("/v1/sessions/{sessionId}", { + params: { header: mutationHeaders(1, "session-patch-0001"), path: { sessionId: "ses-1" } }, body: { title: "renamed-agent" }, + }); + expect(mutatedReplay.response.status).toBe(200); + expect(mutatedReplay.response.headers.get("Idempotency-Replayed")).toBe("true"); + expect(mutatedReplay.data).toEqual(mutated); + const stale = await client.http.PATCH("/v1/sessions/{sessionId}", { + params: { header: mutationHeaders(1, "session-patch-0002"), path: { sessionId: "ses-1" } }, body: { title: "stale-agent" }, + }); + expect(stale.response.status).toBe(409); + expect(stale.error).toMatchObject({ error: { code: "revision_conflict" } }); + + for (const state of COMMAND_STATES) { + const command = { command: state, metadata: {} } satisfies CommandCreate; + const result = await client.http.POST("/v1/sessions/{sessionId}/commands", { + params: { header: idempotencyHeaders(`command-${state}-0001`), path: { sessionId: "ses-1" } }, body: command, + }); + expect(requireData(result).state).toBe(state); + const commandReplay = await client.http.POST("/v1/sessions/{sessionId}/commands", { + params: { header: idempotencyHeaders(`command-${state}-0001`), path: { sessionId: "ses-1" } }, body: command, + }); + expect(commandReplay.data).toEqual(result.data); + expect(commandReplay.response.headers.get("Idempotency-Replayed")).toBe("true"); + } + const oversized = await client.http.POST("/v1/sessions/{sessionId}/commands", { + params: { header: idempotencyHeaders("command-oversized-0001"), path: { sessionId: "ses-1" } }, + body: { command: "a".repeat(33), metadata: {} }, + }); + expect(oversized.response.status).toBe(422); + expect(oversized.error).toMatchObject({ error: { violations: [{ field: "command", rule: "maxBytes" }] } }); + const multibyteOversized = await client.http.POST("/v1/sessions/{sessionId}/commands", { + params: { header: idempotencyHeaders("command-multibyte-0001"), path: { sessionId: "ses-1" } }, + body: { command: "é".repeat(17), metadata: {} }, + }); + expect(multibyteOversized.response.status).toBe(422); + const metadataOversized = await client.http.POST("/v1/sessions/{sessionId}/commands", { + params: { header: idempotencyHeaders("command-metadata-0001"), path: { sessionId: "ses-1" } }, + body: { command: "ok", metadata: { source: "é".repeat(17) } }, + }); + expect(metadataOversized.response.status).toBe(422); + const requestOversized = await client.http.POST("/v1/sessions/{sessionId}/commands", { + params: { header: idempotencyHeaders("command-request-0001"), path: { sessionId: "ses-1" } }, + body: { command: "ok", metadata: Object.fromEntries(Array.from({ length: 8 }, (_, index) => [`field-${index}`, "x".repeat(32)])) }, + }); + expect(requestOversized.response.status).toBe(422); + const defaultedMetadata = await client.http.POST("/v1/sessions/{sessionId}/commands", { + params: { header: idempotencyHeaders("command-default-0001"), path: { sessionId: "ses-1" } }, body: { command: "ok" }, + }); + const explicitMetadata = await client.http.POST("/v1/sessions/{sessionId}/commands", { + params: { header: idempotencyHeaders("command-default-0001"), path: { sessionId: "ses-1" } }, body: { command: "ok", metadata: {} }, + }); + expect(explicitMetadata.data).toEqual(defaultedMetadata.data); + const cancelled = await client.http.POST("/v1/sessions/{sessionId}/cancel", { + params: { header: idempotencyHeaders("session-cancel-0001"), path: { sessionId: "ses-1" } }, + }); + expect(cancelled.response.status).toBe(202); + expect(cancelled.data).toMatchObject({ state: "cancelled" }); + }); + + it("returns one durable event cursor for each committed mutation and its replay", async () => { + const service = new T4ApiV1ConformanceService(); + const client = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); + + const workspaceFirst = await client.http.POST("/v1/workspaces", { + body: { name: "workspace" }, params: { header: idempotencyHeaders("cursor-workspace-create") }, + }); + const workspaceReplay = await client.http.POST("/v1/workspaces", { + body: { name: "workspace" }, params: { header: idempotencyHeaders("cursor-workspace-create") }, + }); + expect(requireEventCursor(workspaceReplay.response)).toBe(requireEventCursor(workspaceFirst.response)); + + const workspacePatchFirst = await client.http.PATCH("/v1/workspaces/{workspaceId}", { + body: { name: "renamed" }, params: { header: mutationHeaders(1, "cursor-workspace-patch"), path: { workspaceId: "ws-1" } }, + }); + const workspacePatchReplay = await client.http.PATCH("/v1/workspaces/{workspaceId}", { + body: { name: "renamed" }, params: { header: mutationHeaders(1, "cursor-workspace-patch"), path: { workspaceId: "ws-1" } }, + }); + expect(requireEventCursor(workspacePatchReplay.response)).toBe(requireEventCursor(workspacePatchFirst.response)); + + const sessionFirst = await client.http.POST("/v1/workspaces/{workspaceId}/sessions", { + body: { title: "session" }, params: { header: idempotencyHeaders("cursor-session-create"), path: { workspaceId: "ws-1" } }, + }); + const sessionReplay = await client.http.POST("/v1/workspaces/{workspaceId}/sessions", { + body: { title: "session" }, params: { header: idempotencyHeaders("cursor-session-create"), path: { workspaceId: "ws-1" } }, + }); + expect(requireEventCursor(sessionReplay.response)).toBe(requireEventCursor(sessionFirst.response)); + + const sessionPatchFirst = await client.http.PATCH("/v1/sessions/{sessionId}", { + body: { title: "retitled" }, params: { header: mutationHeaders(1, "cursor-session-patch"), path: { sessionId: "ses-1" } }, + }); + const sessionPatchReplay = await client.http.PATCH("/v1/sessions/{sessionId}", { + body: { title: "retitled" }, params: { header: mutationHeaders(1, "cursor-session-patch"), path: { sessionId: "ses-1" } }, + }); + expect(requireEventCursor(sessionPatchReplay.response)).toBe(requireEventCursor(sessionPatchFirst.response)); + + for (const [operation, invoke] of [ + ["cancel", () => client.http.POST("/v1/sessions/{sessionId}/cancel", { params: { header: idempotencyHeaders("cursor-session-cancel"), path: { sessionId: "ses-1" } } })], + ["command", () => client.http.POST("/v1/sessions/{sessionId}/commands", { body: { command: "run" }, params: { header: idempotencyHeaders("cursor-session-command"), path: { sessionId: "ses-1" } } })], + ["delete", () => client.http.DELETE("/v1/sessions/{sessionId}", { params: { header: idempotencyHeaders("cursor-session-delete"), path: { sessionId: "ses-1" } } })], + ] as const) { + const first = await invoke(); + const replay = await invoke(); + expect(requireEventCursor(replay.response), operation).toBe(requireEventCursor(first.response)); + } + }); + + it("replays successful session deletion only within the original principal and request scope", async () => { + const service = new T4ApiV1ConformanceService(); + const owner = await seededClient(service, "delete-replay"); + const other = createT4ApiClient({ baseUrl: service.origin, credential: "token-b", majorVersion: 1, fetch: service.fetch }); + const first = await owner.http.DELETE("/v1/sessions/{sessionId}", { + params: { header: idempotencyHeaders("session-delete-replay"), path: { sessionId: "ses-1" } }, + }); + const replay = await owner.http.DELETE("/v1/sessions/{sessionId}", { + params: { header: idempotencyHeaders("session-delete-replay"), path: { sessionId: "ses-1" } }, + }); + expect(first.response.status).toBe(204); + expect(first.response.headers.get("Idempotency-Replayed")).toBe("false"); + expect(replay.response.status).toBe(204); + expect(replay.response.headers.get("Idempotency-Replayed")).toBe("true"); + expect((await owner.http.DELETE("/v1/sessions/{sessionId}", { + params: { header: idempotencyHeaders("session-delete-fresh"), path: { sessionId: "ses-1" } }, + })).response.status).toBe(404); + expect((await other.http.DELETE("/v1/sessions/{sessionId}", { + params: { header: idempotencyHeaders("session-delete-replay"), path: { sessionId: "ses-1" } }, + })).response.status).toBe(404); + }); + + it("validates mutations before state changes and idempotency recording", async () => { + const workspaceService = new T4ApiV1ConformanceService(); + const workspaceClient = createT4ApiClient({ baseUrl: workspaceService.origin, credential: "token-a", majorVersion: 1, fetch: workspaceService.fetch }); + requireData(await workspaceClient.http.POST("/v1/workspaces", { + body: { name: "original" }, params: { header: idempotencyHeaders("mutation-workspace-setup") }, + })); + const invalidWorkspaceBodies = [ + {}, { unknown: true }, { name: 42 }, { name: "" }, { name: "😀".repeat(129) }, + { labels: { Invalid: "value" } }, { labels: { valid: "😀".repeat(129) } }, + ]; + for (const [index, body] of invalidWorkspaceBodies.entries()) { + const result = await workspaceClient.http.PATCH("/v1/workspaces/{workspaceId}", { + params: { header: mutationHeaders(1, `invalid-workspace-${index}`), path: { workspaceId: "ws-1" } }, body: body as never, + }); + expect(result.response.status).toBe(422); + expect(result.error).toMatchObject({ error: { code: "invalid_request" } }); + } + expect(requireData(await workspaceClient.http.GET("/v1/workspaces/{workspaceId}", { + params: { header: VERSION_HEADERS, path: { workspaceId: "ws-1" } }, + }))).toMatchObject({ name: "original", revision: 1 }); + expect((await workspaceClient.http.PATCH("/v1/workspaces/{workspaceId}", { + params: { header: mutationHeaders(1, "invalid-workspace-0"), path: { workspaceId: "ws-1" } }, body: { name: "valid" }, + })).response.status).toBe(200); + + const sessionService = new T4ApiV1ConformanceService(); + const sessionClient = await seededClient(sessionService, "mutation-validation"); + const invalidSessionBodies = [ + {}, { unknown: true }, { title: 42 }, { title: "" }, { title: "😀".repeat(129) }, + { labels: { Invalid: "value" } }, { labels: { valid: "😀".repeat(129) } }, + ]; + for (const [index, body] of invalidSessionBodies.entries()) { + const result = await sessionClient.http.PATCH("/v1/sessions/{sessionId}", { + params: { header: mutationHeaders(1, `invalid-session-${index}`), path: { sessionId: "ses-1" } }, body: body as never, + }); + expect(result.response.status).toBe(422); + expect(result.error).toMatchObject({ error: { code: "invalid_request" } }); + } + expect(requireData(await sessionClient.http.GET("/v1/sessions/{sessionId}", { + params: { header: VERSION_HEADERS, path: { sessionId: "ses-1" } }, + }))).toMatchObject({ title: "agent", revision: 1 }); + expect((await sessionClient.http.PATCH("/v1/sessions/{sessionId}", { + params: { header: mutationHeaders(1, "invalid-session-0"), path: { sessionId: "ses-1" } }, body: { title: "valid" }, + })).response.status).toBe(200); + }); + + it("replays exact PATCH results after an intervening revision", async () => { + const workspaceService = new T4ApiV1ConformanceService(); + const workspaceClient = createT4ApiClient({ baseUrl: workspaceService.origin, credential: "token-a", majorVersion: 1, fetch: workspaceService.fetch }); + requireData(await workspaceClient.http.POST("/v1/workspaces", { + body: { name: "original" }, params: { header: idempotencyHeaders("workspace-replay-setup") }, + })); + const originalWorkspace = requireData(await workspaceClient.http.PATCH("/v1/workspaces/{workspaceId}", { + body: { name: "first" }, params: { header: mutationHeaders(1, "workspace-replay-original"), path: { workspaceId: "ws-1" } }, + })); + requireData(await workspaceClient.http.PATCH("/v1/workspaces/{workspaceId}", { + body: { name: "second" }, params: { header: mutationHeaders(2, "workspace-replay-intervening"), path: { workspaceId: "ws-1" } }, + })); + const workspaceReplay = await workspaceClient.http.PATCH("/v1/workspaces/{workspaceId}", { + body: { name: "first" }, params: { header: mutationHeaders(1, "workspace-replay-original"), path: { workspaceId: "ws-1" } }, + }); + expect(workspaceReplay.response.status).toBe(200); + expect(workspaceReplay.response.headers.get("Idempotency-Replayed")).toBe("true"); + expect(workspaceReplay.data).toEqual(originalWorkspace); + + const sessionService = new T4ApiV1ConformanceService(); + const sessionClient = await seededClient(sessionService, "patch-replay-intervening"); + const originalSession = requireData(await sessionClient.http.PATCH("/v1/sessions/{sessionId}", { + body: { title: "first" }, params: { header: mutationHeaders(1, "session-replay-original"), path: { sessionId: "ses-1" } }, + })); + requireData(await sessionClient.http.PATCH("/v1/sessions/{sessionId}", { + body: { title: "second" }, params: { header: mutationHeaders(2, "session-replay-intervening"), path: { sessionId: "ses-1" } }, + })); + const sessionReplay = await sessionClient.http.PATCH("/v1/sessions/{sessionId}", { + body: { title: "first" }, params: { header: mutationHeaders(1, "session-replay-original"), path: { sessionId: "ses-1" } }, + }); + expect(sessionReplay.response.status).toBe(200); + expect(sessionReplay.response.headers.get("Idempotency-Replayed")).toBe("true"); + expect(sessionReplay.data).toEqual(originalSession); + }); + + it("validates creates before idempotency and round-trips valid labels", async () => { + const invalidLabels = [ + null, [], Object.fromEntries(Array.from({ length: 33 }, (_, index) => [`key-${index}`, "value"])), + { Invalid: "value" }, { valid: 1 }, { valid: "😀".repeat(129) }, + ]; + const workspaceService = new T4ApiV1ConformanceService(); + const workspaceClient = createT4ApiClient({ baseUrl: workspaceService.origin, credential: "token-a", majorVersion: 1, fetch: workspaceService.fetch }); + const workspaceBodies: unknown[] = [{ name: "workspace", unknown: true }, ...invalidLabels.map((labels) => ({ name: "workspace", labels }))]; + for (const [index, body] of workspaceBodies.entries()) { + const key = `invalid-workspace-create-${index}`; + const invalid = await workspaceClient.http.POST("/v1/workspaces", { body: body as never, params: { header: idempotencyHeaders(key) } }); + expect(invalid.response.status).toBe(422); + expect(invalid.error).toMatchObject({ error: { code: "invalid_request" } }); + expect((await workspaceClient.http.POST("/v1/workspaces", { + body: { name: `valid-${index}` }, params: { header: idempotencyHeaders(key) }, + })).response.status).toBe(202); + } + const workspaceLabels = { team: "😀".repeat(128) }; + expect(requireData(await workspaceClient.http.POST("/v1/workspaces", { + body: { name: "labelled", labels: workspaceLabels }, params: { header: idempotencyHeaders("valid-workspace-labels") }, + }))).toMatchObject({ labels: workspaceLabels }); + + const sessionService = new T4ApiV1ConformanceService(); + const sessionClient = createT4ApiClient({ baseUrl: sessionService.origin, credential: "token-a", majorVersion: 1, fetch: sessionService.fetch }); + requireData(await sessionClient.http.POST("/v1/workspaces", { + body: { name: "workspace" }, params: { header: idempotencyHeaders("session-create-workspace") }, + })); + const sessionBodies: unknown[] = [{ title: "session", unknown: true }, ...invalidLabels.map((labels) => ({ title: "session", labels }))]; + for (const [index, body] of sessionBodies.entries()) { + const key = `invalid-session-create-${index}`; + const invalid = await sessionClient.http.POST("/v1/workspaces/{workspaceId}/sessions", { + body: body as never, params: { header: idempotencyHeaders(key), path: { workspaceId: "ws-1" } }, + }); + expect(invalid.response.status).toBe(422); + expect(invalid.error).toMatchObject({ error: { code: "invalid_request" } }); + expect((await sessionClient.http.POST("/v1/workspaces/{workspaceId}/sessions", { + body: { title: `valid-${index}` }, params: { header: idempotencyHeaders(key), path: { workspaceId: "ws-1" } }, + })).response.status).toBe(202); + } + const sessionLabels = { team: "😀".repeat(128) }; + expect(requireData(await sessionClient.http.POST("/v1/workspaces/{workspaceId}/sessions", { + body: { title: "labelled", labels: sessionLabels }, params: { header: idempotencyHeaders("valid-session-labels"), path: { workspaceId: "ws-1" } }, + }))).toMatchObject({ labels: sessionLabels }); + + const commandClient = await seededClient(new T4ApiV1ConformanceService(), "command-create-validation"); + const invalidCommand = await commandClient.http.POST("/v1/sessions/{sessionId}/commands", { + body: { command: "ok", unknown: true } as never, + params: { header: idempotencyHeaders("invalid-command-create"), path: { sessionId: "ses-1" } }, + }); + expect(invalidCommand.response.status).toBe(422); + expect((await commandClient.http.POST("/v1/sessions/{sessionId}/commands", { + body: { command: "ok" }, params: { header: idempotencyHeaders("invalid-command-create"), path: { sessionId: "ses-1" } }, + })).response.status).toBe(202); + }); + + it("rejects undeclared JSON request media before mutation and idempotency", async () => { + for (const contentType of [undefined, "text/plain"]) { + const mediaName = contentType === undefined ? "missing" : "plain"; + const service = new T4ApiV1ConformanceService(); + await seededClient(service, `request-media-${mediaName}`); + const cases = [ + ["POST", "/v1/workspaces", '{"name":"media-workspace"}', {}, 202, { id: "ws-2" }], + ["POST", "/v1/workspaces/ws-1/sessions", '{"title":"media-session"}', {}, 202, { id: "ses-2" }], + ["PATCH", "/v1/workspaces/ws-1", '{"name":"renamed"}', { "If-Match": "1" }, 200, { revision: 2 }], + ["PATCH", "/v1/sessions/ses-1", '{"title":"retitled"}', { "If-Match": "1" }, 200, { revision: 2 }], + ["POST", "/v1/sessions/ses-1/commands", '{"command":"ok"}', {}, 202, { commandId: "cmd-1" }], + ] as const; + for (const [index, [method, path, body, routeHeaders, successStatus, successBody]] of cases.entries()) { + const key = `request-media-${mediaName}-${index}`; + const headers = { Authorization: "Bearer token-a", "T4-API-Version": "1", "Idempotency-Key": key, ...routeHeaders }; + const invalid = await service.fetch(`${service.origin}${path}`, { + method, headers: { ...headers, ...(contentType === undefined ? {} : { "Content-Type": contentType }) }, body, + }); + expect(invalid.status).toBe(400); + expect(await invalid.json()).toMatchObject({ error: { code: "invalid_request" } }); + const valid = await service.fetch(`${service.origin}${path}`, { + method, headers: { ...headers, "Content-Type": "Application/JSON; Charset=UTF-8" }, body, + }); + expect(valid.status).toBe(successStatus); + expect(valid.headers.get("Idempotency-Replayed")).toBe("false"); + expect(await valid.json()).toMatchObject(successBody); + } + } + }); + + it("validates If-Match syntax before PATCH idempotency lookup", async () => { + const service = new T4ApiV1ConformanceService(); + const client = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); + for (const [index, path] of ["/v1/workspaces/missing", "/v1/sessions/missing"].entries()) { + for (const ifMatch of [undefined, "invalid"]) { + const response = await service.fetch(`${service.origin}${path}`, { + method: "PATCH", + headers: { + Authorization: "Bearer token-a", "T4-API-Version": "1", "Idempotency-Key": `missing-if-match-${index}-${ifMatch ?? "absent"}`, "Content-Type": "application/json", + ...(ifMatch === undefined ? {} : { "If-Match": ifMatch }), + }, + body: path.includes("workspaces") ? '{"name":"updated"}' : '{"title":"updated"}', + }); + expect(response.status).toBe(400); + expect((await response.json()) as unknown).toMatchObject({ error: { code: "invalid_request" } }); + } + } + requireData(await client.http.POST("/v1/workspaces", { + body: { name: "workspace" }, params: { header: idempotencyHeaders("if-match-workspace-setup") }, + })); + for (const [index, ifMatch] of [undefined, "", "0", "01", "abc", "1.0", "11111111111111111111"].entries()) { + const key = `invalid-if-match-${index}`; + const response = await service.fetch(`${service.origin}/v1/workspaces/ws-1`, { + method: "PATCH", + headers: { + Authorization: "Bearer token-a", "T4-API-Version": "1", "Idempotency-Key": key, "Content-Type": "application/json", + ...(ifMatch === undefined ? {} : { "If-Match": ifMatch }), + }, + body: '{"name":"updated"}', + }); + expect(response.status).toBe(400); + expect((await response.json()) as unknown).toMatchObject({ error: { code: "invalid_request" } }); + } + const first = await client.http.PATCH("/v1/workspaces/{workspaceId}", { + body: { name: "updated" }, params: { header: mutationHeaders(1, "if-match-replay-order"), path: { workspaceId: "ws-1" } }, + }); + expect(first.response.status).toBe(200); + const missingOnReplay = await service.fetch(`${service.origin}/v1/workspaces/ws-1`, { + method: "PATCH", + headers: { Authorization: "Bearer token-a", "T4-API-Version": "1", "Idempotency-Key": "if-match-replay-order", "Content-Type": "application/json" }, + body: '{"name":"updated"}', + }); + expect(missingOnReplay.status).toBe(400); + }); + + it("rejects an invalid declared watch cache header", async () => { + let cacheBodyCancelled = false; + const cacheClient = createT4ApiClient({ + baseUrl: "https://watch-cache.test", credential: "token-a", majorVersion: 1, + fetch: async () => apiResponse(new ReadableStream({ + start(controller) { controller.enqueue(new TextEncoder().encode('data: {"type":"heartbeat","cursor":"cache","observedAt":"2026-07-21T00:00:00Z"}\n\n')); }, + cancel() { cacheBodyCancelled = true; }, + }), { headers: { "Content-Type": "text/event-stream", "Cache-Control": "max-age=3600" } }), + }); + await expect(cacheClient.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ status: 502, retryable: false }); + expect(cacheBodyCancelled).toBe(true); + }); + + it("requires no-store through the exported HTTP watch operation", async () => { + for (const cacheControl of [undefined, "max-age=3600"]) { + const client = createT4ApiClient({ + baseUrl: "https://raw-watch-cache.test", credential: "token-a", majorVersion: 1, + fetch: async () => new Response('data: {"type":"heartbeat","cursor":"raw","observedAt":"2026-07-21T00:00:00Z"}\n\n', { headers: { + "Content-Type": "text/event-stream", "T4-API-Version": "1.0", ...(cacheControl === undefined ? {} : { "Cache-Control": cacheControl }), + } }), + }); + await expect(client.http.GET("/v1/sessions/{sessionId}/events", { + params: { header: VERSION_HEADERS, path: { sessionId: "ses-1" }, query: { maxEvents: 1, heartbeatSeconds: 5 } }, + })).rejects.toMatchObject({ status: 502, retryable: false }); + } + }); + + it("rejects invalid RFC 3339 calendar timestamps", async () => { + + for (const observedAt of [ + "2026-02-31T00:00:00Z", "2026-13-01T00:00:00Z", "2025-02-29T00:00:00Z", + "2026-04-31T00:00:00Z", "2026-01-01T24:00:00Z", "2026-01-01T00:60:00Z", + "2026-01-01T00:00:00+24:00", "2026-01-01T00:00:00+01:60", + "2026-06-30T23:59:60Z", "2026-12-31T23:59:60Z", "2026-06-30T15:59:60-08:00", + ]) { + const calendarClient = createT4ApiClient({ + baseUrl: "https://watch-calendar.test", credential: "token-a", majorVersion: 1, + fetch: async () => apiResponse(`data: {"type":"heartbeat","cursor":"calendar","observedAt":"${observedAt}"}\n\n`, { headers: { "Content-Type": "text/event-stream" } }), + }); + await expect(calendarClient.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ status: 502, retryable: false }); + } + }); + + it("accepts an RFC 3339 leap second after applying its offset", async () => { + const client = createT4ApiClient({ + baseUrl: "https://watch-leap-second.test", credential: "token-a", majorVersion: 1, + fetch: async () => apiResponse('data: {"type":"heartbeat","cursor":"leap","observedAt":"1990-12-31T15:59:60-08:00"}\n\n', { headers: { "Content-Type": "text/event-stream" } }), + }); + await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).resolves.toMatchObject({ value: { cursor: "leap" } }); + }); + + it("accepts lowercase RFC 3339 delimiters", async () => { + const client = createT4ApiClient({ + baseUrl: "https://watch-lowercase-time.test", credential: "token-a", majorVersion: 1, + fetch: async () => apiResponse('data: {"type":"heartbeat","cursor":"lowercase","observedAt":"2026-07-21t00:00:00z"}\n\n', { headers: { "Content-Type": "text/event-stream" } }), + }); + await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).resolves.toMatchObject({ value: { cursor: "lowercase" } }); + }); + + it("uses typed low-level stream parsing and the SSE Accept header", async () => { + const service = new T4ApiV1ConformanceService(); + const client = await seededClient(service, "raw-stream"); + const wrongAccept = await service.fetch(`${service.origin}/v1/sessions/ses-1/events?maxEvents=1&heartbeatSeconds=5`, { + headers: { Authorization: "Bearer token-a", "T4-API-Version": "1", Accept: "application/json" }, + }); + expect(wrongAccept.status).toBe(406); + const streamed = await client.http.GET("/v1/sessions/{sessionId}/events", { + params: { header: VERSION_HEADERS, path: { sessionId: "ses-1" }, query: { maxEvents: 1, heartbeatSeconds: 5 } }, + parseAs: "stream", + }); + expect(streamed.response.headers.get("Content-Type")).toBe("text/event-stream"); + expect(streamed.response.headers.get("Cache-Control")).toBe("no-store"); + expect(streamed.data).toBeInstanceOf(ReadableStream); + await streamed.data?.cancel(); + }); + + it("rejects malformed present Retry-After on ordinary and watch 503 responses", async () => { + for (const retryAfter of ["", "not-a-date", "-1", "x".repeat(129)]) { + const body = { error: { code: "unavailable", message: "later", requestId: "r", retryable: true } }; + const ordinary = createT4ApiClient({ + baseUrl: "https://ordinary-retry-after.test", credential: "token-a", majorVersion: 1, + fetch: async () => jsonResponse(body, { status: 503, headers: { "Retry-After": retryAfter } }), + }); + await expect(ordinary.http.GET("/v1", { params: { header: VERSION_HEADERS } })).rejects.toMatchObject({ status: 502, retryable: false }); + + const watch = createT4ApiClient({ + baseUrl: "https://watch-retry-after.test", credential: "token-a", majorVersion: 1, + fetch: async () => jsonResponse(body, { status: 503, headers: { "Retry-After": retryAfter } }), + }); + await expect(watch.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0, retryBackoffMs: 0 }).next()).rejects.toMatchObject({ status: 502, retryable: false }); + } + }); + + it("accepts every RFC 9110 HTTP-date form in Retry-After without dropping its delay", async () => { + for (const retryAfter of [ + "Sun, 06 Nov 1994 08:49:37 GMT", + "Sunday, 06-Nov-94 08:49:37 GMT", + "Sun Nov 6 08:49:37 1994", + ]) { + const client = createT4ApiClient({ + baseUrl: "https://http-date-retry-after.test", credential: "token-a", majorVersion: 1, + fetch: async () => jsonResponse({ error: { code: "unavailable", message: "later", requestId: "r", retryable: true } }, { status: 503, headers: { "Retry-After": retryAfter } }), + }); + await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ + status: 502, cause: { status: 503, retryAfterMs: 0 }, + }); + } + }); + + it("resolves RFC 850 two-digit years relative to the current year", async () => { + vi.useFakeTimers(); + try { + vi.setSystemTime(new Date("2026-07-21T00:00:00Z")); + const client = createT4ApiClient({ + baseUrl: "https://rfc850-year.test", credential: "token-a", majorVersion: 1, + fetch: async () => jsonResponse({ error: { code: "unavailable", message: "later", requestId: "r", retryable: true } }, { status: 503, headers: { "Retry-After": "Saturday, 01-Jan-50 00:00:00 GMT" } }), + }); + await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ + status: 502, cause: { status: 503, retryAfterMs: 30_000 }, + }); + } finally { + vi.useRealTimers(); + } + }); + + it("applies the RFC 850 pivot to the complete timestamp", async () => { + vi.useFakeTimers(); + try { + vi.setSystemTime(new Date("2026-07-21T00:00:00Z")); + const client = createT4ApiClient({ + baseUrl: "https://rfc850-boundary.test", credential: "token-a", majorVersion: 1, + fetch: async () => jsonResponse({ error: { code: "unavailable", message: "later", requestId: "r", retryable: true } }, { status: 503, headers: { "Retry-After": "Friday, 31-Dec-76 00:00:00 GMT" } }), + }); + await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ + status: 502, cause: { status: 503, retryAfterMs: 0 }, + }); + } finally { + vi.useRealTimers(); + } + }); + + it("takes a snapshot and watches bounded SSE with heartbeat, reconnect, cancellation, and typed resync", async () => { + const service = new T4ApiV1ConformanceService(); + const client = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); + requireData(await client.http.POST("/v1/workspaces", { + body: { name: "workspace" }, params: { header: idempotencyHeaders("workspace-watch-0001") }, + })); + requireData(await client.http.POST("/v1/workspaces/{workspaceId}/sessions", { + params: { header: idempotencyHeaders("session-watch-0001"), path: { workspaceId: "ws-1" } }, body: { title: "agent" }, + })); + const snapshot = requireData(await client.http.GET("/v1/sessions/{sessionId}/snapshot", { + params: { header: VERSION_HEADERS, path: { sessionId: "ses-1" } }, + })); + expect(snapshot).toMatchObject({ sessionId: "ses-1", cursor: "cursor-2", entries: [{ sequence: 2 }] }); + + const received: Array = []; + for await (const event of client.watchSession("ses-1", { cursor: snapshot.cursor, maxEvents: 3, heartbeatSeconds: 20, maxReconnectAttempts: 2, retryBackoffMs: 0 })) { + received.push(event); + } + expect(received).toEqual([ + expect.objectContaining({ type: "heartbeat", cursor: "cursor-2" }), + expect.objectContaining({ type: "session", cursor: "cursor-4", state: "accepted" }), + expect.objectContaining({ type: "heartbeat", cursor: "cursor-4" }), + ]); + expect(received.map(exhaustWatchEvent)).toEqual(["2026-07-21T00:00:00Z", "accepted:2", "2026-07-21T00:00:15Z"]); + expect(service.watchCursors[0]).toEqual({ query: "cursor-2", header: "cursor-2" }); + expect(service.watchCursors[1]).toEqual({ query: "cursor-4", header: "cursor-4" }); + expect(service.watchQueries.slice(0, 2)).toEqual([ + { maxEvents: "3", heartbeatSeconds: "20" }, + { maxEvents: "1", heartbeatSeconds: "20" }, + ]); + + await expect(async () => { + for await (const _event of client.watchSession("ses-1", { cursor: "expired", maxEvents: 1 })) { + throw new Error("expired cursor unexpectedly produced an event"); + } + }).rejects.toMatchObject({ + name: "T4ApiError", + code: "cursor_expired", + status: 410, + resync: { snapshotUrl: "/v1/sessions/ses-1/snapshot", cursor: "cursor-2" }, + } satisfies Partial); + }); + + it("enforces the ResourceId bound in cursor-expired resync snapshot URLs", async () => { + const resourceId128 = `s${"a".repeat(127)}`; + const resourceId129 = `s${"a".repeat(128)}`; + const cursorExpired = (sessionId: string) => ({ error: { + code: "cursor_expired", message: "expired", requestId: "r", retryable: false, + resync: { snapshotUrl: `/v1/sessions/${sessionId}/snapshot`, cursor: "cursor-1" }, + } }); + + for (const [sessionId, expected] of [[resourceId128, "cursor_expired"], [resourceId129, "indeterminate"]] as const) { + const client = createT4ApiClient({ + baseUrl: "https://resync-bound.test", credential: "token-a", majorVersion: 1, + fetch: async () => jsonResponse(cursorExpired(sessionId), { status: 410 }), + }); + await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ + code: expected, status: expected === "cursor_expired" ? 410 : 502, + }); + } + + const contract = JSON.parse(readFileSync(new URL("../../t4-api-contract/openapi.json", import.meta.url), "utf8")) as { + components: { schemas: { Resync: { properties: { snapshotUrl: { pattern: string; maxLength: number } } } } }; + }; + const snapshotUrlSchema = contract.components.schemas.Resync.properties.snapshotUrl; + const pattern = new RegExp(snapshotUrlSchema.pattern, "u"); + expect(pattern.test(`/v1/sessions/${resourceId128}/snapshot`)).toBe(true); + expect(pattern.test(`/v1/sessions/${resourceId129}/snapshot`)).toBe(false); + expect(`/v1/sessions/${resourceId128}/snapshot`.length).toBeLessThanOrEqual(snapshotUrlSchema.maxLength); + }); + + it("accepts every command lifecycle watch state and rejects removed states", async () => { + for (const [index, state] of COMMAND_STATES.entries()) { + const cursor = `command-state-${index}`; + const client = createT4ApiClient({ + baseUrl: "https://command-states.test", credential: "token-a", majorVersion: 1, + fetch: async () => apiResponse(`data: ${JSON.stringify({ type: "command", cursor, commandId: "cmd-1", state })}\n\n`, { headers: { "Content-Type": "text/event-stream" } }), + }); + await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).resolves.toMatchObject({ value: { state } }); + } + + const removed = createT4ApiClient({ + baseUrl: "https://removed-command-state.test", credential: "token-a", majorVersion: 1, + fetch: async () => apiResponse('data: {"type":"command","cursor":"removed-1","commandId":"cmd-1","state":"conflict"}\n\n', { headers: { "Content-Type": "text/event-stream" } }), + }); + await expect(removed.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ code: "indeterminate", status: 502 }); + }); + + it("rejects malformed mutation idempotency and watch query contracts", async () => { + const service = new T4ApiV1ConformanceService(); + const baseHeaders = { Authorization: "Bearer token-a", "T4-API-Version": "1", "Content-Type": "application/json" }; + const created = await service.fetch(`${service.origin}/v1/workspaces`, { method: "POST", headers: { ...baseHeaders, "Idempotency-Key": "workspace-raw-0001" }, body: '{"name":"workspace"}' }); + expect(created.status).toBe(202); + const missingSpawn = await service.fetch(`${service.origin}/v1/workspaces/ws-1/sessions`, { method: "POST", headers: baseHeaders, body: '{"title":"agent"}' }); + expect(missingSpawn.status).toBe(400); + expect(await missingSpawn.json()).toMatchObject({ error: { code: "idempotency_key_required" } }); + const invalidSpawn = await service.fetch(`${service.origin}/v1/workspaces/ws-1/sessions`, { method: "POST", headers: { ...baseHeaders, "Idempotency-Key": "short" }, body: '{"title":"agent"}' }); + expect(invalidSpawn.status).toBe(400); + const spawned = await service.fetch(`${service.origin}/v1/workspaces/ws-1/sessions`, { method: "POST", headers: { ...baseHeaders, "Idempotency-Key": "session-raw-00001" }, body: '{"title":"agent"}' }); + expect(spawned.status).toBe(202); + for (const [method, path, body] of [ + ["PATCH", "/v1/workspaces/ws-1", '{"name":"x"}'], + ["DELETE", "/v1/workspaces/ws-1", undefined], + ["PATCH", "/v1/sessions/ses-1", '{"title":"x"}'], + ["DELETE", "/v1/sessions/ses-1", undefined], + ["POST", "/v1/sessions/ses-1/cancel", undefined], + ["POST", "/v1/sessions/ses-1/commands", '{"command":"ok"}'], + ] as const) { + const response = await service.fetch(`${service.origin}${path}`, { method, headers: { ...baseHeaders, "If-Match": "1" }, ...(body === undefined ? {} : { body }) }); + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ error: { code: "idempotency_key_required" } }); + } + for (const [method, path, headers] of [ + ["POST", "/v1/workspaces", { ...baseHeaders, "Idempotency-Key": "malformed-workspace-01" }], + ["PATCH", "/v1/workspaces/ws-1", { ...baseHeaders, "If-Match": "1", "Idempotency-Key": "malformed-workspace-02" }], + ["POST", "/v1/workspaces/ws-1/sessions", { ...baseHeaders, "Idempotency-Key": "malformed-session-0001" }], + ["PATCH", "/v1/sessions/ses-1", { ...baseHeaders, "If-Match": "1", "Idempotency-Key": "malformed-session-0002" }], + ] as const) { + const response = await service.fetch(`${service.origin}${path}`, { method, headers, body: "{" }); + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ error: { code: "invalid_request" } }); + } + for (const query of ["maxEvents=0", "maxEvents=5", "heartbeatSeconds=4", "heartbeatSeconds=61"]) { + const response = await service.fetch(`${service.origin}/v1/sessions/ses-1/events?${query}`, { headers: { ...baseHeaders, Accept: "text/event-stream" } }); + expect(response.status).toBe(422); + expect(await response.json()).toMatchObject({ error: { code: "invalid_request", violations: expect.any(Array) } }); + } + }); + + it("rejects malformed workspace and session pagination cursors", async () => { + const service = new T4ApiV1ConformanceService(); + const headers = { Authorization: "Bearer token-a", "T4-API-Version": "1", "Content-Type": "application/json" }; + expect((await service.fetch(`${service.origin}/v1/workspaces`, { + method: "POST", headers: { ...headers, "Idempotency-Key": "pagination-workspace" }, body: '{"name":"workspace"}', + })).status).toBe(202); + expect((await service.fetch(`${service.origin}/v1/workspaces/ws-1/sessions`, { + method: "POST", headers: { ...headers, "Idempotency-Key": "pagination-session-1" }, body: '{"title":"session"}', + })).status).toBe(202); + + for (const path of ["/v1/workspaces", "/v1/workspaces/ws-1/sessions"]) { + for (const cursor of ["foo", "page--1", "page-01", "page-9007199254740992"]) { + const response = await service.fetch(`${service.origin}${path}?cursor=${encodeURIComponent(cursor)}`, { headers }); + expect(response.status, `${path} accepted ${cursor}`).toBe(422); + expect(await response.json()).toMatchObject({ error: { code: "invalid_request", violations: [{ field: "cursor", rule: "format" }] } }); + } + } + }); + + it("accepts only pagination cursors issued for the principal and collection", async () => { + const service = new T4ApiV1ConformanceService(); + const headers = { Authorization: "Bearer token-a", "T4-API-Version": "1", "Content-Type": "application/json" }; + for (const [key, name] of [["issued-workspace-1", "one"], ["issued-workspace-2", "two"], ["issued-workspace-3", "three"]]) { + expect((await service.fetch(`${service.origin}/v1/workspaces`, { + method: "POST", headers: { ...headers, "Idempotency-Key": key }, body: JSON.stringify({ name }), + })).status).toBe(202); + } + for (const [key, title] of [["issued-session-01", "one"], ["issued-session-02", "two"]]) { + expect((await service.fetch(`${service.origin}/v1/workspaces/ws-1/sessions`, { + method: "POST", headers: { ...headers, "Idempotency-Key": key }, body: JSON.stringify({ title }), + })).status).toBe(202); + } + + const workspacePage = await service.fetch(`${service.origin}/v1/workspaces?pageSize=1`, { headers }); + const workspaceCursor = (await workspacePage.json() as { nextCursor: string }).nextCursor; + expect((await service.fetch(`${service.origin}/v1/workspaces?pageSize=1&cursor=${workspaceCursor}`, { headers })).status).toBe(200); + const base64UrlAlphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; + const signatureSegment = workspaceCursor.split(".").at(-1)!; + const encodedRemainder = signatureSegment.length % 4; + const unusedBits = encodedRemainder === 2 ? 4 : encodedRemainder === 3 ? 2 : 0; + const terminalIndex = base64UrlAlphabet.indexOf(signatureSegment.at(-1)!); + expect(terminalIndex).toBeGreaterThanOrEqual(0); + const aliasMask = (1 << unusedBits) - 1; + expect(terminalIndex & aliasMask).toBe(0); + const signatureAliases = base64UrlAlphabet.slice( + terminalIndex & ~aliasMask, + (terminalIndex & ~aliasMask) + aliasMask + 1, + ).replace(signatureSegment.at(-1)!, ""); + expect(signatureAliases.length).toBeGreaterThan(0); + expect(signatureAliases).toHaveLength(3); + for (const signatureAlias of signatureAliases) { + const noncanonicalCursor = `${workspaceCursor.slice(0, -1)}${signatureAlias}`; + const noncanonical = await service.fetch(`${service.origin}/v1/workspaces?pageSize=1&cursor=${noncanonicalCursor}`, { headers }); + expect(noncanonical.status).toBe(422); + expect(await noncanonical.json()).toMatchObject({ error: { code: "invalid_request" } }); + } + const sessionPage = await service.fetch(`${service.origin}/v1/workspaces/ws-1/sessions?pageSize=1`, { headers }); + const sessionCursor = (await sessionPage.json() as { nextCursor: string }).nextCursor; + + for (const [path, cursor, authorization] of [ + ["/v1/workspaces", "page-999", "Bearer token-a"], + ["/v1/workspaces/ws-1/sessions", "page-999", "Bearer token-a"], + ["/v1/workspaces/ws-1/sessions", workspaceCursor, "Bearer token-a"], + ["/v1/workspaces", sessionCursor, "Bearer token-a"], + ["/v1/workspaces", workspaceCursor, "Bearer token-b"], + ]) { + const response = await service.fetch(`${service.origin}${path}?pageSize=1&cursor=${cursor}`, { headers: { ...headers, Authorization: authorization } }); + expect(response.status, `${path} accepted unissued cursor ${cursor}`).toBe(422); + expect(await response.json()).toMatchObject({ error: { code: "invalid_request", violations: [{ field: "cursor", rule: "issued" }] } }); + } + + const laterWorkspacePage = await service.fetch(`${service.origin}/v1/workspaces?pageSize=2`, { headers }); + const staleWorkspaceCursor = (await laterWorkspacePage.json() as { nextCursor: string }).nextCursor; + expect((await service.fetch(`${service.origin}/v1/workspaces/ws-3`, { + method: "DELETE", headers: { ...headers, "Idempotency-Key": "shrink-workspace-3" }, + })).status).toBe(204); + expect((await service.fetch(`${service.origin}/v1/sessions/ses-2`, { + method: "DELETE", headers: { ...headers, "Idempotency-Key": "shrink-session-02" }, + })).status).toBe(204); + for (const [path, cursor] of [ + ["/v1/workspaces", staleWorkspaceCursor], + ["/v1/workspaces/ws-1/sessions", sessionCursor], + ]) { + const response = await service.fetch(`${service.origin}${path}?pageSize=1&cursor=${cursor}`, { headers }); + expect(response.status, `${path} accepted a cursor past the shrunken collection`).toBe(422); + expect(await response.json()).toMatchObject({ error: { code: "invalid_request", violations: [{ field: "cursor", rule: "issued" }] } }); + } + }); + + it("parses service frames incrementally across split UTF-8 and per-frame bounds", async () => { + const bytewiseService = new T4ApiV1ConformanceService({ watchTransport: "bytewise" }); + const bytewiseClient = await seededClient(bytewiseService, "bytewise"); + const event = await bytewiseClient.watchSession("ses-1", { maxEvents: 1, heartbeatSeconds: 20, maxReconnectAttempts: 0 }).next(); + expect(event.value).toMatchObject({ type: "heartbeat", cursor: "cursor-3" }); + expect(bytewiseService.watchQueries[0]).toEqual({ maxEvents: "1", heartbeatSeconds: "20" }); + + const manyService = new T4ApiV1ConformanceService({ watchTransport: "many-small" }); + const manyClient = await seededClient(manyService, "many-small"); + let count = 0; + for await (const _event of manyClient.watchSession("ses-1", { maxEvents: 1000, maxReconnectAttempts: 0 })) count += 1; + expect(count).toBe(1000); + + const oversizedService = new T4ApiV1ConformanceService({ watchTransport: "oversized" }); + const oversizedClient = await seededClient(oversizedService, "oversized"); + await expect(oversizedClient.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ code: "indeterminate", status: 502 }); + + let ongoingOversizedCancelled = false; + const ongoingOversizedClient = createT4ApiClient({ + baseUrl: "https://ongoing-oversized-event.test", credential: "token-a", majorVersion: 1, + fetch: async () => apiResponse(new ReadableStream({ + pull(controller) { controller.enqueue(new Uint8Array(600 * 1024)); }, + cancel() { ongoingOversizedCancelled = true; }, + }), { status: 200, headers: { "Content-Type": "text/event-stream" } }), + }); + await expect(ongoingOversizedClient.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ + code: "indeterminate", status: 502, retryable: false, + }); + expect(ongoingOversizedCancelled).toBe(true); + }); + + it("fails closed on unknown watch fields and incomplete typed errors", async () => { + const eventFetch: typeof globalThis.fetch = async () => apiResponse('data: {"type":"heartbeat","cursor":"c1","observedAt":"2026-07-21T00:00:00Z","unknown":true}\n\n', { headers: { "Content-Type": "text/event-stream" } }); + const eventClient = createT4ApiClient({ baseUrl: "https://unknown.test", credential: "token-a", majorVersion: 1, fetch: eventFetch }); + await expect(eventClient.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ code: "indeterminate", status: 502 }); + for (const [status, code] of [[410, "cursor_expired"], [406, "incompatible_version"], [422, "invalid_request"]] as const) { + const fetch: typeof globalThis.fetch = async () => jsonResponse({ error: { code, message: "incomplete", requestId: "r", retryable: false } }, { status }); + const client = createT4ApiClient({ baseUrl: "https://errors.test", credential: "token-a", majorVersion: 1, fetch }); + await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ code: "indeterminate", status: 502, retryable: false }); + } + for (const [status, code] of [[401, "not_found"], [403, "unauthenticated"], [404, "forbidden"], [409, "unavailable"], [503, "revision_conflict"]] as const) { + const fetch: typeof globalThis.fetch = async () => jsonResponse({ error: { code, message: "wrong status class", requestId: "r", retryable: false } }, { status }); + const client = createT4ApiClient({ baseUrl: "https://status-errors.test", credential: "token-a", majorVersion: 1, fetch }); + await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ code: "indeterminate", status: 502, retryable: false }); + } + }); + + it("fails closed on malformed, oversized, status-inconsistent, and unknown-field HTTP errors", async () => { + const validErrors = [ + [400, { error: { code: "invalid_request", message: "bad", requestId: "r", retryable: false } }], + [401, { error: { code: "unauthenticated", message: "bad", requestId: "r", retryable: false } }], + [403, { error: { code: "forbidden", message: "bad", requestId: "r", retryable: false } }], + [404, { error: { code: "not_found", message: "bad", requestId: "r", retryable: false } }], + [406, { error: { code: "incompatible_version", message: "bad", requestId: "r", retryable: false, supportedMajors: [1] } }], + [409, { error: { code: "revision_conflict", message: "bad", requestId: "r", retryable: false } }], + [422, { error: { code: "invalid_request", message: "bad", requestId: "r", retryable: false, violations: [{ field: "name", rule: "length", message: "bad" }] } }], + [503, { error: { code: "unavailable", message: "bad", requestId: "r", retryable: true } }], + ] as const; + for (const [status, body] of validErrors) { + const fetch: typeof globalThis.fetch = async () => jsonResponse({ error: { ...body.error, unknown: true } }, { status }); + const client = createT4ApiClient({ baseUrl: "https://ordinary-errors.test", credential: "token-a", majorVersion: 1, fetch }); + await expect(client.http.POST("/v1/workspaces", { + body: { name: "workspace" }, params: { header: idempotencyHeaders("ordinary-errors-0001") }, + })).rejects.toMatchObject({ code: "indeterminate", status: 502, retryable: false }); + } + + const malformedClient = createT4ApiClient({ + baseUrl: "https://malformed-error.test", credential: "token-a", majorVersion: 1, + fetch: async () => apiResponse("{", { status: 404, headers: { "Content-Type": "application/json" } }), + }); + await expect(malformedClient.http.GET("/v1/workspaces/{workspaceId}", { + params: { header: VERSION_HEADERS, path: { workspaceId: "missing" } }, + })).rejects.toMatchObject({ code: "indeterminate", status: 502, retryable: false }); + + const oversizedClient = createT4ApiClient({ + baseUrl: "https://oversized-error.test", credential: "token-a", majorVersion: 1, + fetch: async () => apiResponse("x".repeat(1024 * 1024 + 1), { status: 404, headers: { "Content-Type": "application/json" } }), + }); + await expect(oversizedClient.http.GET("/v1/workspaces/{workspaceId}", { + params: { header: VERSION_HEADERS, path: { workspaceId: "missing" } }, + })).rejects.toMatchObject({ code: "indeterminate", status: 502, retryable: false }); + + const streamedChunk = new Uint8Array(600 * 1024); + for (const contentType of ["application/json", "text/plain"]) { + let cancelled = false; + const streamedClient = createT4ApiClient({ + baseUrl: "https://streamed-error.test", credential: "token-a", majorVersion: 1, + fetch: async () => apiResponse(new ReadableStream({ + pull(controller) { controller.enqueue(streamedChunk); }, + cancel() { cancelled = true; }, + }), { status: 404, headers: { "Content-Type": contentType } }), + }); + await expect(streamedClient.http.GET("/v1/workspaces/{workspaceId}", { + params: { header: VERSION_HEADERS, path: { workspaceId: "missing" } }, + })).rejects.toMatchObject({ code: "indeterminate", status: 502, retryable: false }); + expect(cancelled).toBe(true); + } + + const statusClient = createT4ApiClient({ + baseUrl: "https://undeclared-error.test", credential: "token-a", majorVersion: 1, + fetch: async () => jsonResponse(validErrors[3][1], { status: 404 }), + }); + await expect(statusClient.http.GET("/v1", { params: { header: VERSION_HEADERS } })).rejects.toMatchObject({ code: "indeterminate", status: 502, retryable: false }); + }); + + it("validates successful JSON routes relative to the base path and rejects undeclared media and statuses", async () => { + const discovery = DISCOVERY; + const prefixed = createT4ApiClient({ + baseUrl: "https://prefixed.test/api", credential: "token-a", majorVersion: 1, + fetch: async (input) => { + expect(new Request(input).url).toBe("https://prefixed.test/api/v1"); + return jsonResponse({ ...discovery, unknown: true }); + }, + }); + await expect(prefixed.http.GET("/v1", { params: { header: VERSION_HEADERS } })).rejects.toMatchObject({ code: "indeterminate", status: 502 }); + + const invalidSnapshot = createT4ApiClient({ + baseUrl: "https://snapshot.test/api", credential: "token-a", majorVersion: 1, + fetch: async (input) => { + expect(new URL(new Request(input).url).pathname).toBe("/api/v1/sessions/ses-1/snapshot"); + return jsonResponse({ sessionId: "ses-1", cursor: "cursor-1", state: "ready", entries: [], unknown: true }); + }, + }); + await expect(invalidSnapshot.http.GET("/v1/sessions/{sessionId}/snapshot", { + params: { header: VERSION_HEADERS, path: { sessionId: "ses-1" } }, + })).rejects.toMatchObject({ code: "indeterminate", status: 502 }); + + const astralText = "😀".repeat(524_289); + const unicodeSnapshot = createT4ApiClient({ + baseUrl: "https://unicode-snapshot.test", credential: "token-a", majorVersion: 1, + fetch: async () => jsonResponse({ sessionId: "ses-1", cursor: "cursor-1", state: "ready", entries: [{ sequence: 0, kind: "output", text: astralText }] }), + }); + const unicode = requireData(await unicodeSnapshot.http.GET("/v1/sessions/{sessionId}/snapshot", { + params: { header: VERSION_HEADERS, path: { sessionId: "ses-1" } }, + })); + expect(unicode.entries[0]?.text).toBe(astralText); + + const astral128 = "😀".repeat(128); + const unicodeWorkspace = createT4ApiClient({ + baseUrl: "https://unicode-workspace.test", credential: "token-a", majorVersion: 1, + fetch: async () => jsonResponse({ id: "wsp-1", name: astral128, state: "ready", revision: 1, labels: { team: astral128 } }), + }); + const workspace = requireData(await unicodeWorkspace.http.GET("/v1/workspaces/{workspaceId}", { + params: { header: VERSION_HEADERS, path: { workspaceId: "wsp-1" } }, + })); + expect(workspace).toMatchObject({ name: astral128, labels: { team: astral128 } }); + + const unicodeSession = createT4ApiClient({ + baseUrl: "https://unicode-session.test", credential: "token-a", majorVersion: 1, + fetch: async () => jsonResponse({ id: "ses-1", workspaceId: "wsp-1", title: astral128, state: "ready", revision: 1, labels: { team: astral128 } }), + }); + const session = requireData(await unicodeSession.http.GET("/v1/sessions/{sessionId}", { + params: { header: VERSION_HEADERS, path: { sessionId: "ses-1" } }, + })); + expect(session).toMatchObject({ title: astral128, labels: { team: astral128 } }); + + const errorMessage = "😀".repeat(1024); + const errorRequestId = "😀".repeat(128); + const violationField = "😀".repeat(256); + const violationMessage = "😀".repeat(512); + const unicodeError = createT4ApiClient({ + baseUrl: "https://unicode-error.test", credential: "token-a", majorVersion: 1, + fetch: async () => jsonResponse({ error: { + code: "invalid_request", message: errorMessage, requestId: errorRequestId, retryable: false, + violations: [{ field: violationField, rule: "range", message: violationMessage }], + } }, { status: 422 }), + }); + await expect(unicodeError.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ + code: "invalid_request", status: 422, message: errorMessage, requestId: errorRequestId, + violations: [{ field: violationField, rule: "range", message: violationMessage }], + }); + + for (const response of [ + apiResponse(JSON.stringify(discovery), { status: 200, headers: { "Content-Type": "text/plain" } }), + jsonResponse(discovery, { status: 201 }), + ]) { + const client = createT4ApiClient({ baseUrl: "https://dispatch.test", credential: "token-a", majorVersion: 1, fetch: async () => response.clone() }); + await expect(client.http.GET("/v1", { params: { header: VERSION_HEADERS } })).rejects.toMatchObject({ code: "indeterminate", status: 502 }); + } + + const unknownRoute = createT4ApiClient({ + baseUrl: "https://dispatch.test", credential: "token-a", majorVersion: 1, + fetch: async () => jsonResponse(discovery), + }); + await expect(unknownRoute.http.GET("/v1/undeclared" as "/v1", { params: { header: VERSION_HEADERS } })).rejects.toMatchObject({ code: "indeterminate", status: 502 }); + }); + + it("accepts only the exact watch status matrix under a prefixed base path", async () => { + const prefixed = createT4ApiClient({ + baseUrl: "https://watch-status.test/api", credential: "token-a", majorVersion: 1, + fetch: async (input) => { + expect(new URL(new Request(input).url).pathname).toBe("/api/v1/sessions/ses-1/events"); + return apiResponse('data: {"type":"heartbeat","cursor":"prefixed-1","observedAt":"2026-07-21T00:00:00Z"}\n\n', { status: 200, headers: { "Content-Type": "text/event-stream" } }); + }, + }); + await expect(prefixed.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).resolves.toMatchObject({ value: { cursor: "prefixed-1" } }); + + const declared = [ + [400, { error: { code: "invalid_request", message: "bad", requestId: "r", retryable: false } }], + [401, { error: { code: "unauthenticated", message: "bad", requestId: "r", retryable: false } }], + [403, { error: { code: "forbidden", message: "bad", requestId: "r", retryable: false } }], + [404, { error: { code: "not_found", message: "bad", requestId: "r", retryable: false } }], + [406, { error: { code: "incompatible_version", message: "bad", requestId: "r", retryable: false, supportedMajors: [1] } }], + [410, { error: { code: "cursor_expired", message: "bad", requestId: "r", retryable: false, resync: { snapshotUrl: "/v1/sessions/ses-1/snapshot", cursor: "cursor-1" } } }], + [422, { error: { code: "invalid_request", message: "bad", requestId: "r", retryable: false, violations: [{ field: "maxEvents", rule: "range", message: "bad" }] } }], + [503, { error: { code: "unavailable", message: "bad", requestId: "r", retryable: false } }], + ] as const; + for (const [status, body] of declared) { + const client = createT4ApiClient({ baseUrl: "https://watch-status.test", credential: "token-a", majorVersion: 1, fetch: async () => jsonResponse(body, { status }) }); + await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ status, code: body.error.code }); + } + + for (const response of [ + apiResponse('data: {"type":"heartbeat","cursor":"wrong-media-1","observedAt":"2026-07-21T00:00:00Z"}\n\n', { status: 200, headers: { "Content-Type": "application/json" } }), + apiResponse('data: {"type":"heartbeat","cursor":"wrong-1","observedAt":"2026-07-21T00:00:00Z"}\n\n', { status: 201, headers: { "Content-Type": "text/event-stream" } }), + apiResponse(null, { status: 204, headers: { "Content-Type": "text/event-stream" } }), + jsonResponse({ error: { code: "revision_conflict", message: "bad", requestId: "r", retryable: false } }, { status: 409 }), + ]) { + let attempts = 0; + const client = createT4ApiClient({ + baseUrl: "https://watch-status.test", credential: "token-a", majorVersion: 1, + fetch: async () => { attempts += 1; return response.clone(); }, + }); + await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 3, retryBackoffMs: 0 }).next()).rejects.toMatchObject({ + status: 502, code: "indeterminate", retryable: false, + }); + expect(attempts).toBe(1); + } + }); + + it("declares truthful discovery and required public response headers while preserving the 401 exception", () => { + const contract = JSON.parse(readFileSync(new URL("../../t4-api-contract/openapi.json", import.meta.url), "utf8")) as { + paths: Record }; get?: { responses: Record }> } }>; + components: { + headers: Record; + responses: Record }>; + schemas: Record }>; + }; + }; + expect(contract.paths["/v1/workspaces/{workspaceId}"]?.patch?.responses["200"]?.$ref).toBe("#/components/responses/WorkspaceReplay"); + expect(contract.paths["/v1/sessions/{sessionId}"]?.patch?.responses["200"]?.$ref).toBe("#/components/responses/SessionReplay"); + expect(contract.components.headers.SelectedVersion?.required).toBe(true); + expect(contract.components.headers.IdempotencyReplayed?.required).toBe(true); + expect(contract.components.headers.EventCursor?.required).toBe(true); + for (const response of ["WorkspaceAccepted", "WorkspaceReplay", "SessionAccepted", "SessionReplay", "CommandAccepted", "CommandReplay", "Deleted"]) { + expect(contract.components.responses[response]?.headers?.["T4-Event-Cursor"]?.$ref).toBe("#/components/headers/EventCursor"); + } + expect(contract.paths["/v1/sessions/{sessionId}/events"]?.get?.responses["200"]?.headers?.["Cache-Control"]?.required).toBe(true); + expect(contract.components.responses.Error401?.headers?.["T4-API-Version"]).toBeUndefined(); + + const discovery = contract.components.schemas.Discovery!; + expect(discovery.required).toContain("serverBuild"); + expect(discovery.properties?.capabilities).toEqual({ $ref: "#/components/schemas/Capabilities" }); + expect(contract.components.schemas.Capabilities).toMatchObject({ + maxProperties: 128, + propertyNames: { pattern: "^[a-z][a-z0-9.-]*$", maxLength: 128 }, + additionalProperties: { $ref: "#/components/schemas/CapabilityStatus" }, + }); + expect(contract.components.schemas.CapabilityStatus).toMatchObject({ + additionalProperties: false, + required: ["supported", "enabled", "authorized", "available"], + }); + }); + + it("treats malformed and oversized watch 503 envelopes as terminal protocol errors", async () => { + for (const body of ["{", "x".repeat(1024 * 1024 + 1)]) { + let attempts = 0; + const client = createT4ApiClient({ + baseUrl: "https://invalid-503.test", credential: "token-a", majorVersion: 1, + fetch: async () => { + attempts += 1; + return apiResponse(body, { status: 503, headers: { "Content-Type": "application/json", "Retry-After": "0" } }); + }, + }); + await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 3, retryBackoffMs: 0 }).next()).rejects.toMatchObject({ + code: "indeterminate", status: 502, retryable: false, + }); + expect(attempts).toBe(1); + } + + let oversizedCancelled = false; + const oversizedChunk = new Uint8Array(600 * 1024); + const streamedOversizedClient = createT4ApiClient({ + baseUrl: "https://streamed-invalid-503.test", credential: "token-a", majorVersion: 1, + fetch: async () => apiResponse(new ReadableStream({ + pull(controller) { controller.enqueue(oversizedChunk); }, + cancel() { oversizedCancelled = true; }, + }), { status: 503, headers: { "Content-Type": "application/json" } }), + }); + await expect(streamedOversizedClient.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 3, retryBackoffMs: 0 }).next()).rejects.toMatchObject({ + code: "indeterminate", status: 502, retryable: false, + }); + expect(oversizedCancelled).toBe(true); + + let nonJsonCancelled = false; + const nonJsonClient = createT4ApiClient({ + baseUrl: "https://invalid-503-media.test", credential: "token-a", majorVersion: 1, + fetch: async () => apiResponse(new ReadableStream({ + pull(controller) { controller.enqueue(new Uint8Array(1024)); }, + cancel() { nonJsonCancelled = true; }, + }), { status: 503, headers: { "Content-Type": "text/plain" } }), + }); + await expect(nonJsonClient.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 3, retryBackoffMs: 0 }).next()).rejects.toMatchObject({ + code: "indeterminate", status: 502, retryable: false, + }); + expect(nonJsonCancelled).toBe(true); + }); + + it("rejects unknown fields in every public JSON response family", async () => { + const discoveryService = new T4ApiV1ConformanceService({ invalidPayload: "discovery" }); + const discoveryClient = createT4ApiClient({ baseUrl: discoveryService.origin, credential: "token-a", majorVersion: 1, fetch: discoveryService.fetch }); + await expect(discoveryClient.http.GET("/v1", { params: { header: VERSION_HEADERS } })).rejects.toMatchObject({ code: "indeterminate", status: 502 }); + + const workspaceService = new T4ApiV1ConformanceService({ invalidPayload: "workspace" }); + const workspaceClient = createT4ApiClient({ baseUrl: workspaceService.origin, credential: "token-a", majorVersion: 1, fetch: workspaceService.fetch }); + await expect(workspaceClient.http.POST("/v1/workspaces", { + body: { name: "workspace" }, params: { header: idempotencyHeaders("invalid-workspace-0001") }, + })).rejects.toMatchObject({ code: "indeterminate", status: 502 }); + + const sessionService = new T4ApiV1ConformanceService({ invalidPayload: "session" }); + const sessionClient = createT4ApiClient({ baseUrl: sessionService.origin, credential: "token-a", majorVersion: 1, fetch: sessionService.fetch }); + requireData(await sessionClient.http.POST("/v1/workspaces", { + body: { name: "workspace" }, params: { header: idempotencyHeaders("invalid-session-work-01") }, + })); + await expect(sessionClient.http.POST("/v1/workspaces/{workspaceId}/sessions", { + body: { title: "agent" }, params: { header: idempotencyHeaders("invalid-session-00001"), path: { workspaceId: "ws-1" } }, + })).rejects.toMatchObject({ code: "indeterminate", status: 502 }); + + const commandService = new T4ApiV1ConformanceService({ invalidPayload: "command" }); + const commandClient = await seededClient(commandService, "invalid-command"); + await expect(commandClient.http.POST("/v1/sessions/{sessionId}/commands", { + body: { command: "ok", metadata: {} }, params: { header: idempotencyHeaders("invalid-command-0001"), path: { sessionId: "ses-1" } }, + })).rejects.toMatchObject({ code: "indeterminate", status: 502 }); + }); + + it("requires declared selected-version and replay response headers", async () => { + const discovery = DISCOVERY; + for (const selectedVersion of [undefined, "1", "2.0", `1.${"0".repeat(15)}`]) { + const client = createT4ApiClient({ + baseUrl: "https://response-version.test", credential: "token-a", majorVersion: 1, + fetch: async () => Response.json(discovery, selectedVersion === undefined ? {} : { headers: { "T4-API-Version": selectedVersion } }), + }); + await expect(client.http.GET("/v1", { params: { header: VERSION_HEADERS } })).rejects.toMatchObject({ status: 502, code: "indeterminate", retryable: false }); + } + + const unauthenticated = createT4ApiClient({ + baseUrl: "https://response-version.test", credential: "token-a", majorVersion: 1, + fetch: async () => Response.json({ error: { code: "unauthenticated", message: "bad", requestId: "r", retryable: false } }, { status: 401 }), + }); + expect((await unauthenticated.http.GET("/v1", { params: { header: VERSION_HEADERS } })).response.status).toBe(401); + + const workspace = { id: "wsp-1", name: "workspace", state: "ready", revision: 1 }; + const missingReplay = createT4ApiClient({ + baseUrl: "https://response-replay.test", credential: "token-a", majorVersion: 1, + fetch: async () => jsonResponse(workspace, { headers: { "T4-API-Version": "1.0" } }), + }); + await expect(missingReplay.http.PATCH("/v1/workspaces/{workspaceId}", { + params: { header: mutationHeaders(1, "response-replay-patch"), path: { workspaceId: "wsp-1" } }, body: { name: "workspace" }, + })).rejects.toMatchObject({ status: 502, retryable: false }); + + const missingEventCursor = createT4ApiClient({ + baseUrl: "https://response-event-cursor.test", credential: "token-a", majorVersion: 1, + fetch: async () => jsonResponse(workspace, { headers: { "T4-API-Version": "1.0", "Idempotency-Replayed": "false" } }), + }); + await expect(missingEventCursor.http.PATCH("/v1/workspaces/{workspaceId}", { + params: { header: mutationHeaders(1, "response-event-cursor"), path: { workspaceId: "wsp-1" } }, body: { name: "workspace" }, + })).rejects.toMatchObject({ status: 502, retryable: false }); + + for (const replayed of [undefined, "TRUE", "0"]) { + const client = createT4ApiClient({ + baseUrl: "https://delete-replay-header.test", credential: "token-a", majorVersion: 1, + fetch: async () => apiResponse(null, { status: 204, headers: { + "T4-API-Version": "1.0", ...(replayed === undefined ? {} : { "Idempotency-Replayed": replayed }), + } }), + }); + await expect(client.http.DELETE("/v1/workspaces/{workspaceId}", { + params: { header: idempotencyHeaders("response-delete-key"), path: { workspaceId: "wsp-1" } }, + })).rejects.toMatchObject({ status: 502, retryable: false }); + } + + const missingWatchVersion = createT4ApiClient({ + baseUrl: "https://watch-response-version.test", credential: "token-a", majorVersion: 1, + fetch: async () => new Response('data: {"type":"heartbeat","cursor":"c1","observedAt":"2026-07-21T00:00:00Z"}\n\n', { headers: { "Content-Type": "text/event-stream" } }), + }); + await expect(missingWatchVersion.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ status: 502, retryable: false }); + + const missingWatchErrorVersion = createT4ApiClient({ + baseUrl: "https://watch-error-version.test", credential: "token-a", majorVersion: 1, + fetch: async () => Response.json({ error: { code: "unavailable", message: "later", requestId: "r", retryable: true } }, { status: 503 }), + }); + await expect(missingWatchErrorVersion.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0, retryBackoffMs: 0 }).next()).rejects.toMatchObject({ status: 502, retryable: false }); + }); + + it("aligns the snapshot aggregate schema bound with high-level response validation", async () => { + const contract = JSON.parse(readFileSync(new URL("../../t4-api-contract/openapi.json", import.meta.url), "utf8")) as { + components: { schemas: { SessionSnapshot: { "x-t4-maxUtf8Bytes"?: number } } }; + }; + expect(contract.components.schemas.SessionSnapshot["x-t4-maxUtf8Bytes"]).toBe(16 * 1024 * 1024); + + for (const boundary of ["exact", "over"] as const) { + const service = new T4ApiV1ConformanceService({ snapshotBoundary: boundary }); + const client = await seededClient(service, `snapshot-${boundary}`); + const response = client.http.GET("/v1/sessions/{sessionId}/snapshot", { + params: { header: VERSION_HEADERS, path: { sessionId: "ses-1" } }, + }); + if (boundary === "exact") { + const snapshot = requireData(await response); + expect(snapshot.entries).toHaveLength(4); + expect(snapshot.entries[0]?.text).toContain("💚"); + } else { + await expect(response).rejects.toMatchObject({ code: "indeterminate", status: 502, retryable: false }); + } + } + }); + + it("rejects duplicate durable cursors within one SSE stream and after reconnect", async () => { + const event = 'data: {"type":"session","cursor":"duplicate-1","state":"ready","revision":1}\n\n'; + for (const delivery of ["same-stream", "reconnect"] as const) { + let attempts = 0; + const client = createT4ApiClient({ + baseUrl: `https://duplicate-${delivery}.test`, credential: "token-a", majorVersion: 1, + fetch: async () => { + attempts += 1; + return apiResponse(delivery === "same-stream" ? `${event}${event}` : event, { headers: { "Content-Type": "text/event-stream" } }); + }, + }); + const stream = client.watchSession("ses-1", { maxEvents: 2, maxReconnectAttempts: 1, retryBackoffMs: 0 }); + await expect(stream.next()).resolves.toMatchObject({ value: { cursor: "duplicate-1" }, done: false }); + await expect(stream.next()).rejects.toMatchObject({ code: "indeterminate", status: 502, retryable: false }); + expect(attempts).toBe(delivery === "same-stream" ? 1 : 2); + } + }); + + it("accepts current-cursor heartbeats without allowing them to advance the durable cursor", async () => { + const heartbeat = (cursor: string) => `data: ${JSON.stringify({ type: "heartbeat", cursor, observedAt: "2026-07-21T00:00:00Z" })}\n\n`; + const startingRequests: Array<{ query: string | null; header: string | null }> = []; + const starting = createT4ApiClient({ + baseUrl: "https://starting-heartbeat.test", credential: "token-a", majorVersion: 1, + fetch: async (input, init) => { + const request = new Request(input, init); + startingRequests.push({ query: new URL(request.url).searchParams.get("cursor"), header: request.headers.get("Last-Event-ID") }); + return apiResponse(heartbeat("duplicate-1"), { headers: { "Content-Type": "text/event-stream" } }); + }, + }); + await expect(starting.watchSession("ses-1", { cursor: "duplicate-1", maxEvents: 1, maxReconnectAttempts: 0 }).next()).resolves.toMatchObject({ + value: { type: "heartbeat", cursor: "duplicate-1" }, done: false, + }); + expect(startingRequests).toEqual([{ query: "duplicate-1", header: "duplicate-1" }]); + + const reconnectRequests: Array<{ query: string | null; header: string | null }> = []; + const reconnecting = createT4ApiClient({ + baseUrl: "https://reconnect-heartbeat.test", credential: "token-a", majorVersion: 1, + fetch: async (input, init) => { + const request = new Request(input, init); + reconnectRequests.push({ query: new URL(request.url).searchParams.get("cursor"), header: request.headers.get("Last-Event-ID") }); + return apiResponse(heartbeat("duplicate-1"), { headers: { "Content-Type": "text/event-stream" } }); + }, + }); + const liveness: WatchEvent[] = []; + for await (const event of reconnecting.watchSession("ses-1", { maxEvents: 2, maxReconnectAttempts: 1, retryBackoffMs: 0 })) liveness.push(event); + expect(liveness.map((event) => event.cursor)).toEqual(["duplicate-1", "duplicate-1"]); + expect(reconnectRequests).toEqual([{ query: null, header: null }, { query: "duplicate-1", header: "duplicate-1" }]); + + const advancing = createT4ApiClient({ + baseUrl: "https://advancing-heartbeat.test", credential: "token-a", majorVersion: 1, + fetch: async () => apiResponse(heartbeat("different-2"), { headers: { "Content-Type": "text/event-stream" } }), + }); + await expect(advancing.watchSession("ses-1", { cursor: "duplicate-1", maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ + code: "indeterminate", status: 502, retryable: false, + }); + }); + + it("rejects duplicate version entries and malformed capability maps", async () => { + const status = { supported: true, enabled: true, authorized: true, available: true }; + for (const discovery of [ + { ...DISCOVERY, supportedMajors: [1, 1] }, + { ...DISCOVERY, capabilities: { Invalid: status } }, + { ...DISCOVERY, capabilities: { "session.watch.sse": { ...status, unknown: true } } }, + ]) { + const client = createT4ApiClient({ + baseUrl: "https://invalid-discovery.test", credential: "token-a", majorVersion: 1, + fetch: async () => jsonResponse(discovery, { headers: { "T4-API-Version": "1.0" } }), + }); + await expect(client.http.GET("/v1", { params: { header: VERSION_HEADERS } })).rejects.toMatchObject({ status: 502, retryable: false }); + } + const duplicateError = createT4ApiClient({ + baseUrl: "https://duplicate-error.test", credential: "token-a", majorVersion: 1, + fetch: async () => jsonResponse({ error: { + code: "incompatible_version", message: "bad", requestId: "r", retryable: false, supportedMajors: [1, 1], + } }, { status: 406, headers: { "T4-API-Version": "1.0" } }), + }); + await expect(duplicateError.http.GET("/v1", { params: { header: VERSION_HEADERS } })).rejects.toMatchObject({ status: 502, code: "indeterminate", retryable: false }); + }); + + it("discards unterminated SSE data at EOF before reconnecting", async () => { + for (const unterminated of [ + 'data: {"type":"heartbeat","cursor":"unterminated","observedAt":"2026-07-21T00:00:00Z"}', + "data: {not-json", + ]) { + let attempts = 0; + const client = createT4ApiClient({ + baseUrl: "https://unterminated-sse.test", credential: "token-a", majorVersion: 1, + fetch: async () => { + attempts += 1; + const body = attempts === 1 + ? unterminated + : 'data: {"type":"heartbeat","cursor":"complete","observedAt":"2026-07-21T00:00:00Z"}\n\n'; + return apiResponse(body, { headers: { "Content-Type": "text/event-stream", "T4-API-Version": "1.0" } }); + }, + }); + await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 1, retryBackoffMs: 0 }).next()).resolves.toMatchObject({ value: { cursor: "complete" } }); + expect(attempts).toBe(2); + } + }); + + it("accepts exact code-point request lengths and rejects one more", async () => { + const service = new T4ApiV1ConformanceService(); + const client = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); + const exact = "😀".repeat(128); + const workspace = await client.http.POST("/v1/workspaces", { + body: { name: exact }, params: { header: idempotencyHeaders("unicode-workspace-exact") }, + }); + expect(workspace.response.status).toBe(202); + expect((await client.http.POST("/v1/workspaces", { + body: { name: `${exact}😀` }, params: { header: idempotencyHeaders("unicode-workspace-over") }, + })).response.status).toBe(422); + const session = await client.http.POST("/v1/workspaces/{workspaceId}/sessions", { + body: { title: exact }, params: { header: idempotencyHeaders("unicode-session-exact"), path: { workspaceId: requireData(workspace).id } }, + }); + expect(session.response.status).toBe(202); + expect((await client.http.POST("/v1/workspaces/{workspaceId}/sessions", { + body: { title: `${exact}😀` }, params: { header: idempotencyHeaders("unicode-session-over"), path: { workspaceId: requireData(workspace).id } }, + })).response.status).toBe(422); + }); + + it("fails terminally and cancels an invalid UTF-8 SSE body", async () => { + let cancelled = false; + const client = createT4ApiClient({ + baseUrl: "https://invalid-utf8-sse.test", credential: "token-a", majorVersion: 1, + fetch: async () => apiResponse(new ReadableStream({ + start(controller) { controller.enqueue(new Uint8Array([100, 97, 116, 97, 58, 32, 195, 40, 10, 10])); }, + cancel() { cancelled = true; }, + }), { headers: { "Content-Type": "text/event-stream", "T4-API-Version": "1.0" } }), + }); + await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 3 }).next()).rejects.toMatchObject({ status: 502, code: "indeterminate", retryable: false }); + expect(cancelled).toBe(true); + }); + + it("scopes one DELETE key independently by live target ID", async () => { + const service = new T4ApiV1ConformanceService(); + const client = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); + for (const [key, name] of [["delete-target-workspace-a", "first"], ["delete-target-workspace-b", "second"]] as const) { + requireData(await client.http.POST("/v1/workspaces", { body: { name }, params: { header: idempotencyHeaders(key) } })); + } + for (const workspaceId of ["ws-1", "ws-2"] as const) { + expect((await client.http.DELETE("/v1/workspaces/{workspaceId}", { + params: { header: idempotencyHeaders("delete-shared-workspace"), path: { workspaceId } }, + })).response.status).toBe(204); + } + + const sessionService = new T4ApiV1ConformanceService(); + const sessionClient = createT4ApiClient({ baseUrl: sessionService.origin, credential: "token-a", majorVersion: 1, fetch: sessionService.fetch }); + requireData(await sessionClient.http.POST("/v1/workspaces", { body: { name: "workspace" }, params: { header: idempotencyHeaders("delete-session-workspace") } })); + for (const [key, title] of [["delete-target-session-a", "first"], ["delete-target-session-b", "second"]] as const) { + requireData(await sessionClient.http.POST("/v1/workspaces/{workspaceId}/sessions", { + body: { title }, params: { header: idempotencyHeaders(key), path: { workspaceId: "ws-1" } }, + })); + } + for (const sessionId of ["ses-1", "ses-2"] as const) { + expect((await sessionClient.http.DELETE("/v1/sessions/{sessionId}", { + params: { header: idempotencyHeaders("delete-shared-session"), path: { sessionId } }, + })).response.status).toBe(204); + } + }); + + it("bounds automatic EOF retry and supports explicit abort", async () => { + let attempts = 0; + const eofFetch: typeof globalThis.fetch = async () => { + attempts += 1; + return apiResponse(new ReadableStream({ start(controller) { controller.close(); } }), { headers: { "Content-Type": "text/event-stream" } }); + }; + const eofClient = createT4ApiClient({ baseUrl: "https://eof.test", credential: "token-a", majorVersion: 1, fetch: eofFetch }); + await expect(eofClient.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 2, retryBackoffMs: 0 }).next()).rejects.toMatchObject({ code: "indeterminate", status: 502 }); + expect(attempts).toBe(3); + + const reconnectHeaders: Array = []; + let networkAttempt = 0; + const networkFetch: typeof globalThis.fetch = async (_input, init) => { + reconnectHeaders.push(new Headers(init?.headers).get("Last-Event-ID")); + networkAttempt += 1; + if (networkAttempt === 1) { + let sent = false; + return apiResponse(new ReadableStream({ + pull(stream) { + if (sent) stream.error(new TypeError("transient network loss")); + else { + sent = true; + stream.enqueue(new TextEncoder().encode('data: {"type":"heartbeat","cursor":"network-1","observedAt":"2026-07-21T00:00:00Z"}\n\n')); + } + }, + }), { headers: { "Content-Type": "text/event-stream" } }); + } + return apiResponse('data: {"type":"session","cursor":"network-2","state":"ready","revision":2}\n\n', { headers: { "Content-Type": "text/event-stream" } }); + }; + const networkClient = createT4ApiClient({ baseUrl: "https://network.test", credential: "token-a", majorVersion: 1, fetch: networkFetch }); + const networkEvents: WatchEvent[] = []; + for await (const event of networkClient.watchSession("ses-1", { maxEvents: 2, maxReconnectAttempts: 1, retryBackoffMs: 0 })) networkEvents.push(event); + expect(networkEvents.map((event) => event.cursor)).toEqual(["network-1", "network-2"]); + expect(reconnectHeaders).toEqual([null, "network-1"]); + + let retryableAttempts = 0; + const retryableFetch: typeof globalThis.fetch = async () => { + retryableAttempts += 1; + if (retryableAttempts === 1) return jsonResponse({ error: { code: "unavailable", message: "retry", requestId: "r", retryable: true } }, { status: 503, headers: { "Retry-After": "0" } }); + return apiResponse('data: {"type":"heartbeat","cursor":"retry-1","observedAt":"2026-07-21T00:00:00Z"}\n\n', { headers: { "Content-Type": "text/event-stream" } }); + }; + const retryableClient = createT4ApiClient({ baseUrl: "https://retryable.test", credential: "token-a", majorVersion: 1, fetch: retryableFetch }); + expect((await retryableClient.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 1, retryBackoffMs: 0 }).next()).value).toMatchObject({ cursor: "retry-1" }); + expect(retryableAttempts).toBe(2); + + let progressAttempts = 0; + const progressFetch: typeof globalThis.fetch = async () => { + progressAttempts += 1; + if (progressAttempts === 2 || progressAttempts === 4) { + const cursor = progressAttempts === 2 ? "progress-1" : "progress-2"; + return apiResponse(`data: {"type":"session","cursor":"${cursor}","state":"ready","revision":${progressAttempts / 2}}\n\n`, { headers: { "Content-Type": "text/event-stream" } }); + } + return apiResponse(new ReadableStream({ start(controller) { controller.close(); } }), { headers: { "Content-Type": "text/event-stream" } }); + }; + const progressClient = createT4ApiClient({ baseUrl: "https://progress.test", credential: "token-a", majorVersion: 1, fetch: progressFetch }); + const progressEvents: WatchEvent[] = []; + for await (const event of progressClient.watchSession("ses-1", { maxEvents: 2, maxReconnectAttempts: 1, retryBackoffMs: 0 })) progressEvents.push(event); + expect(progressEvents.map((event) => event.cursor)).toEqual(["progress-1", "progress-2"]); + expect(progressAttempts).toBe(4); + + vi.useFakeTimers(); + try { + const retryAfterController = new AbortController(); + let retryAfterAttempts = 0; + const retryAfterFetch: typeof globalThis.fetch = async () => { + retryAfterAttempts += 1; + return jsonResponse({ error: { code: "unavailable", message: "later", requestId: "r", retryable: true } }, { status: 503, headers: { "Retry-After": "30" } }); + }; + const retryAfterClient = createT4ApiClient({ baseUrl: "https://retry-after.test", credential: "token-a", majorVersion: 1, fetch: retryAfterFetch }); + const retryAfterPending = retryAfterClient.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 2, retryBackoffMs: 0, signal: retryAfterController.signal }).next(); + const retryAfterAssertion = expect(retryAfterPending).resolves.toMatchObject({ done: true }); + await vi.advanceTimersByTimeAsync(0); + expect(retryAfterAttempts).toBe(1); + retryAfterController.abort(); + await retryAfterAssertion; + } finally { + vi.useRealTimers(); + } + + const controller = new AbortController(); + const abortFetch: typeof globalThis.fetch = async (_input, init) => apiResponse(new ReadableStream({ + start(stream) { init?.signal?.addEventListener("abort", () => stream.close(), { once: true }); }, + }), { headers: { "Content-Type": "text/event-stream" } }); + const abortClient = createT4ApiClient({ baseUrl: "https://abort.test", credential: "token-a", majorVersion: 1, fetch: abortFetch }); + const pending = abortClient.watchSession("ses-1", { signal: controller.signal }).next(); + controller.abort("done"); + await expect(pending).resolves.toMatchObject({ done: true }); + }); +}); diff --git a/packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4clusterhost.yaml b/packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4clusterhost.yaml new file mode 100644 index 00000000..d618128d --- /dev/null +++ b/packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4clusterhost.yaml @@ -0,0 +1,25 @@ +apiVersion: cluster.t4.dev/v1alpha1 +kind: T4ClusterHost +metadata: + name: legacy-host +spec: + storageClassName: legacy-rwx + runtimeProfiles: + - default + - gui + ciProvider: + secretRef: + name: legacy-ci-token + configMapRef: + name: legacy-ci-config + allowedOrigins: + - https://t4.example.test +status: + observedGeneration: 3 + conditions: + - type: Available + status: "True" + observedGeneration: 3 + lastTransitionTime: "2026-01-15T12:00:00Z" + reason: Reconciled + message: Legacy host remains available diff --git a/packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4session.yaml b/packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4session.yaml new file mode 100644 index 00000000..dde06302 --- /dev/null +++ b/packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4session.yaml @@ -0,0 +1,29 @@ +apiVersion: cluster.t4.dev/v1alpha1 +kind: T4Session +metadata: + name: legacy-session + finalizers: + - cluster.t4.dev/session-cleanup +spec: + hostRef: legacy-host + workspaceRef: legacy-workspace + title: Legacy session + runtimeProfile: gui + initialPromptSecretRef: + name: legacy-initial-prompt + ci: + repositoryId: legacy/project + ref: refs/heads/main + commit: 0123456789abcdef0123456789abcdef01234567 +status: + observedGeneration: 7 + podName: legacy-session-pod + serviceName: legacy-session + phase: Running + conditions: + - type: Available + status: "True" + observedGeneration: 7 + lastTransitionTime: "2026-01-15T12:00:00Z" + reason: RuntimeReady + message: Legacy session infrastructure is running diff --git a/packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4workspace.yaml b/packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4workspace.yaml new file mode 100644 index 00000000..dcd9ac1c --- /dev/null +++ b/packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4workspace.yaml @@ -0,0 +1,29 @@ +apiVersion: cluster.t4.dev/v1alpha1 +kind: T4Workspace +metadata: + name: legacy-workspace + finalizers: + - cluster.t4.dev/workspace-protection +spec: + hostRef: legacy-host + displayName: Legacy workspace + owner: user:legacy-owner + repository: + repositoryId: legacy/project + ref: refs/heads/main + commit: 0123456789abcdef0123456789abcdef01234567 + size: 20Gi + retentionPolicy: Retain +status: + observedGeneration: 4 + pvcName: legacy-workspace-data + pvcPhase: Bound + capacity: 20Gi + phase: Ready + conditions: + - type: Ready + status: "True" + observedGeneration: 4 + lastTransitionTime: "2026-01-15T12:00:00Z" + reason: StorageBound + message: Legacy workspace storage is ready diff --git a/packages/cluster-operator/api/v1alpha1/types_test.go b/packages/cluster-operator/api/v1alpha1/types_test.go index fde778fb..4bb6390a 100644 --- a/packages/cluster-operator/api/v1alpha1/types_test.go +++ b/packages/cluster-operator/api/v1alpha1/types_test.go @@ -1,8 +1,10 @@ package v1alpha1_test import ( + "encoding/json" "os" "path/filepath" + "reflect" "strings" "testing" @@ -11,6 +13,7 @@ import ( structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" "sigs.k8s.io/yaml" clusterv1alpha1 "github.com/LycaonLLC/t4-code/packages/cluster-operator/api/v1alpha1" @@ -141,6 +144,107 @@ func TestCRDSchemasAreStructuralBoundedAndValidated(t *testing.T) { } } +func TestOldObjectsDefaultAndRoundTripDeclaredFields(t *testing.T) { + tests := []struct { + fixture string + crd string + }{ + {"v1alpha1-t4clusterhost.yaml", "t4clusterhosts.cluster.t4.dev.yaml"}, + {"v1alpha1-t4workspace.yaml", "t4workspaces.cluster.t4.dev.yaml"}, + {"v1alpha1-t4session.yaml", "t4sessions.cluster.t4.dev.yaml"}, + } + + scheme := runtime.NewScheme() + if err := clusterv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + decoder := serializer.NewCodecFactory(scheme).UniversalDeserializer() + + for _, tc := range tests { + t.Run(tc.fixture, func(t *testing.T) { + fixtureRaw, err := os.ReadFile(filepath.Join("testdata", "compat", tc.fixture)) + if err != nil { + t.Fatal(err) + } + var declared map[string]interface{} + if err := yaml.Unmarshal(fixtureRaw, &declared); err != nil { + t.Fatalf("decode fixture: %v", err) + } + + crdRaw, err := os.ReadFile(filepath.Join("..", "..", "..", "..", "deploy", "charts", "t4-cluster", "crds", tc.crd)) + if err != nil { + t.Fatal(err) + } + var crd apiextensionsv1.CustomResourceDefinition + if err := yaml.Unmarshal(crdRaw, &crd); err != nil { + t.Fatalf("decode CRD: %v", err) + } + if len(crd.Spec.Versions) != 1 || crd.Spec.Versions[0].Name != "v1alpha1" || !crd.Spec.Versions[0].Served || !crd.Spec.Versions[0].Storage { + t.Fatalf("storage contract changed: %#v", crd.Spec.Versions) + } + + if tc.fixture == "v1alpha1-t4session.yaml" { + specSchema := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["spec"] + guiSchema := specSchema.Properties["guiEnabled"] + if guiSchema.Default == nil || string(guiSchema.Default.Raw) != "false" { + t.Fatalf("guiEnabled schema default = %#v, want false", guiSchema.Default) + } + } + + admittedJSON, err := json.Marshal(declared) + if err != nil { + t.Fatal(err) + } + object, gvk, err := decoder.Decode(admittedJSON, nil, nil) + if err != nil { + t.Fatalf("decode through registered v1alpha1 API: %v", err) + } + if gvk.GroupVersion() != clusterv1alpha1.GroupVersion { + t.Fatalf("decoded version = %s", gvk.GroupVersion()) + } + roundTripJSON, err := json.Marshal(object) + if err != nil { + t.Fatal(err) + } + var roundTripped map[string]interface{} + if err := json.Unmarshal(roundTripJSON, &roundTripped); err != nil { + t.Fatal(err) + } + assertDeclaredFieldsPreserved(t, "$", declared, roundTripped) + }) + } +} + +func assertDeclaredFieldsPreserved(t *testing.T, path string, declared, roundTripped interface{}) { + t.Helper() + switch expected := declared.(type) { + case map[string]interface{}: + actual, ok := roundTripped.(map[string]interface{}) + if !ok { + t.Fatalf("%s changed type: got %T", path, roundTripped) + } + for key, expectedValue := range expected { + actualValue, found := actual[key] + if !found { + t.Fatalf("%s.%s was lost", path, key) + } + assertDeclaredFieldsPreserved(t, path+"."+key, expectedValue, actualValue) + } + case []interface{}: + actual, ok := roundTripped.([]interface{}) + if !ok || len(actual) != len(expected) { + t.Fatalf("%s changed array shape: got %#v", path, roundTripped) + } + for index := range expected { + assertDeclaredFieldsPreserved(t, path, expected[index], actual[index]) + } + default: + if !reflect.DeepEqual(declared, roundTripped) { + t.Fatalf("%s changed from %#v to %#v", path, declared, roundTripped) + } + } +} + func TestCRDsHaveCrossFieldCELAndForbidClientRuntimeAuthority(t *testing.T) { for _, name := range []string{"t4clusterhosts.cluster.t4.dev.yaml", "t4workspaces.cluster.t4.dev.yaml", "t4sessions.cluster.t4.dev.yaml"} { raw, err := os.ReadFile(filepath.Join("..", "..", "..", "..", "deploy", "charts", "t4-cluster", "crds", name)) diff --git a/packages/cluster-operator/charttests/chart_contract_test.go b/packages/cluster-operator/charttests/chart_contract_test.go index 611bd493..ba034664 100644 --- a/packages/cluster-operator/charttests/chart_contract_test.go +++ b/packages/cluster-operator/charttests/chart_contract_test.go @@ -143,33 +143,43 @@ func TestEachDeploymentUsesZeroUnavailableAndConfiguredAPIAudience(t *testing.T) func TestValuesSchemaRejectsUnsafeNamesProfilesCIDRsAndHalfSelectors(t *testing.T) { for name, values := range map[string][]string{ - "cluster host name": {"--set-string", "clusterHost.name=Bad_Name"}, - "storage class name": {"--set-string", "storage.adminRWXStorageClass=Bad_Name"}, - "runtime profile": {"--set-string", "clusterHost.runtimeProfiles[0]=-bad"}, - "Woodpecker Secret name": {"--set-string", "woodpecker.existingSecret=Bad_Name", "--set-string", "woodpecker.configMap=woodpecker-config"}, - "Woodpecker ConfigMap name": {"--set-string", "woodpecker.existingSecret=woodpecker-token", "--set-string", "woodpecker.configMap=Bad_Name"}, - "Woodpecker key": {"--set-string", "woodpecker.existingSecret=woodpecker-token", "--set-string", "woodpecker.configMap=woodpecker-config", "--set-string", "woodpecker.tokenKey=bad/key"}, - "Woodpecker audience": {"--set-string", "woodpecker.serviceAccountAudience=/bad", "--set-string", "woodpecker.configMap=woodpecker-config"}, - "IPv4 default route": {"--set-string", "server.trustedProxyCIDRs[0]=0.0.0.0/0"}, - "IPv6 default route": {"--set-string", "server.trustedProxyCIDRs[0]=::/0"}, - "gateway half selector": {"--set-string", "networkPolicy.gatewayIngress.namespaceSelector.matchLabels.scope=gateway"}, - "observability half selector": {"--set-string", "networkPolicy.observability.podSelector.matchLabels.scope=metrics"}, - "OMP ConfigMap name": {"--set-string", "session.omp.configMap=Bad_Name"}, - "OMP models key": {"--set-string", "session.omp.modelsKey=bad/key"}, - "legacy OMP credential Secret": {"--set-string", "session.omp.credentialSecret=omp-runtime-credential"}, - "legacy OMP credential key": {"--set-string", "session.omp.credentialKey=MODEL_API_KEY"}, - "identical OMP projection keys": {"--set-string", "session.omp.settingsKey=provider-models"}, - "model route port zero": {"--set", "networkPolicy.modelRoutePorts[0]=0"}, - "model route port above TCP range": {"--set", "networkPolicy.modelRoutePorts[0]=65536"}, - "duplicate model route port": {"--set", "networkPolicy.modelRoutePorts[0]=19481", "--set", "networkPolicy.modelRoutePorts[1]=19481"}, - "noninteger model route port": {"--set-string", "networkPolicy.modelRoutePorts[0]=https"}, - "model route half selector": {"--set-string", "networkPolicy.modelRoute.namespaceSelector.matchLabels.scope=linkedin-bot"}, - "CI provider port zero": {"--set", "networkPolicy.ciProviderPorts[0]=0"}, - "CI provider port above TCP range": {"--set", "networkPolicy.ciProviderPorts[0]=65536"}, - "duplicate CI provider port": {"--set", "networkPolicy.ciProviderPorts[0]=8080", "--set", "networkPolicy.ciProviderPorts[1]=8080"}, - "noninteger CI provider port": {"--set-string", "networkPolicy.ciProviderPorts[0]=http"}, - "CI provider half selector": {"--set-string", "networkPolicy.ciProvider.namespaceSelector.matchLabels.scope=linkedin-bot"}, - "legacy OMP credential mode": {"--set", "session.omp.allowUnauthenticated=false"}, + "cluster host name": {"--set-string", "clusterHost.name=Bad_Name"}, + "storage class name": {"--set-string", "storage.adminRWXStorageClass=Bad_Name"}, + "runtime profile": {"--set-string", "clusterHost.runtimeProfiles[0]=-bad"}, + "public API Secret name": {"--set-string", "publicApi.existingSecret=Bad_Name"}, + "public API PostgreSQL key": {"--set-string", "publicApi.postgresURLKey=bad/key"}, + "public API credentials key": {"--set-string", "publicApi.credentialsKey=bad/key"}, + "Woodpecker Secret name": {"--set-string", "woodpecker.existingSecret=Bad_Name", "--set-string", "woodpecker.configMap=woodpecker-config"}, + "Woodpecker ConfigMap name": {"--set-string", "woodpecker.existingSecret=woodpecker-token", "--set-string", "woodpecker.configMap=Bad_Name"}, + "Woodpecker key": {"--set-string", "woodpecker.existingSecret=woodpecker-token", "--set-string", "woodpecker.configMap=woodpecker-config", "--set-string", "woodpecker.tokenKey=bad/key"}, + "Woodpecker audience": {"--set-string", "woodpecker.serviceAccountAudience=/bad", "--set-string", "woodpecker.configMap=woodpecker-config"}, + "IPv4 default route": {"--set-string", "server.trustedProxyCIDRs[0]=0.0.0.0/0"}, + "IPv6 default route": {"--set-string", "server.trustedProxyCIDRs[0]=::/0"}, + "PostgreSQL IPv4 default route": {"--set-string", "networkPolicy.postgresCIDRs[0]=0.0.0.0/0"}, + "PostgreSQL IPv6 default route": {"--set-string", "networkPolicy.postgresCIDRs[0]=::/0"}, + "gateway half selector": {"--set-string", "networkPolicy.gatewayIngress.namespaceSelector.matchLabels.scope=gateway"}, + "observability half selector": {"--set-string", "networkPolicy.observability.podSelector.matchLabels.scope=metrics"}, + "OMP ConfigMap name": {"--set-string", "session.omp.configMap=Bad_Name"}, + "OMP models key": {"--set-string", "session.omp.modelsKey=bad/key"}, + "legacy OMP credential Secret": {"--set-string", "session.omp.credentialSecret=omp-runtime-credential"}, + "legacy OMP credential key": {"--set-string", "session.omp.credentialKey=MODEL_API_KEY"}, + "identical OMP projection keys": {"--set-string", "session.omp.settingsKey=provider-models"}, + "model route port zero": {"--set", "networkPolicy.modelRoutePorts[0]=0"}, + "model route port above TCP range": {"--set", "networkPolicy.modelRoutePorts[0]=65536"}, + "duplicate model route port": {"--set", "networkPolicy.modelRoutePorts[0]=19481", "--set", "networkPolicy.modelRoutePorts[1]=19481"}, + "noninteger model route port": {"--set-string", "networkPolicy.modelRoutePorts[0]=https"}, + "model route half selector": {"--set-string", "networkPolicy.modelRoute.namespaceSelector.matchLabels.scope=linkedin-bot"}, + "CI provider port zero": {"--set", "networkPolicy.ciProviderPorts[0]=0"}, + "CI provider port above TCP range": {"--set", "networkPolicy.ciProviderPorts[0]=65536"}, + "duplicate CI provider port": {"--set", "networkPolicy.ciProviderPorts[0]=8080", "--set", "networkPolicy.ciProviderPorts[1]=8080"}, + "noninteger CI provider port": {"--set-string", "networkPolicy.ciProviderPorts[0]=http"}, + "CI provider half selector": {"--set-string", "networkPolicy.ciProvider.namespaceSelector.matchLabels.scope=linkedin-bot"}, + "PostgreSQL port zero": {"--set", "networkPolicy.postgresPorts[0]=0"}, + "PostgreSQL port above TCP range": {"--set", "networkPolicy.postgresPorts[0]=65536"}, + "duplicate PostgreSQL port": {"--set", "networkPolicy.postgresPorts[0]=5432", "--set", "networkPolicy.postgresPorts[1]=5432"}, + "noninteger PostgreSQL port": {"--set-string", "networkPolicy.postgresPorts[0]=postgres"}, + "PostgreSQL half selector": {"--set-string", "networkPolicy.postgres.namespaceSelector.matchLabels.scope=database"}, + "legacy OMP credential mode": {"--set", "session.omp.allowUnauthenticated=false"}, } { t.Run(name, func(t *testing.T) { helmTemplateMustFail(t, append(enabledValues(), values...)...) @@ -178,7 +188,7 @@ func TestValuesSchemaRejectsUnsafeNamesProfilesCIDRsAndHalfSelectors(t *testing. } func TestValuesSchemaBoundsRoutePortLists(t *testing.T) { - for _, field := range []string{"modelRoutePorts", "ciProviderPorts"} { + for _, field := range []string{"modelRoutePorts", "ciProviderPorts", "postgresPorts"} { t.Run(field, func(t *testing.T) { values := enabledValues() for index := 0; index < 17; index++ { @@ -333,11 +343,177 @@ func TestRBACSeparatesControllerMutationFromServerProjection(t *testing.T) { if strings.Contains(controllerRole, "resources: [secrets]") { t.Fatal("controller role can read provider credential Secrets") } - assertContains(t, serverRole, "t4clusterhosts", "t4workspaces", "t4sessions", "create", "list", "watch") - if strings.Contains(serverRole, "secrets") || strings.Contains(serverRole, "persistentvolumeclaims") || strings.Contains(serverRole, "t4sessions/status") { - t.Fatal("server role can read secrets or mutate controller-owned infrastructure/status") + assertContains(t, serverRole, + "resources: [t4clusterhosts]", + "verbs: [get, list, watch]", + "resources: [t4workspaces, t4sessions]", + "verbs: [get, list, watch, create, patch, delete]", + ) + assertCount(t, serverRole, "patch", 1) + if strings.Contains(serverRole, "secrets") || + strings.Contains(serverRole, "persistentvolumeclaims") || + strings.Contains(serverRole, "t4sessions/status") || + strings.Contains(serverRole, "update") { + t.Fatal("server role can read secrets or mutate controller-owned infrastructure/status beyond exact CR patch operations") + } +} + +func TestPublicAPIRequiresDurableSecretAndExplicitPostgresEgress(t *testing.T) { + disabled := helmTemplate(t, append(enabledValues(), + "--set-string", "publicApi.existingSecret=t4-public-api", + "--set", "networkPolicy.postgresCIDRs[0]=198.51.100.20/32", + "--set", "networkPolicy.postgresPorts[0]=5432", + )...) + disabledServer := documentContainingKind(t, disabled, "Deployment", "name: \"release-name-t4-cluster-server\"") + disabledEgress := documentContainingKind(t, disabled, "NetworkPolicy", "name: \"release-name-t4-cluster-server-egress\"") + if strings.Contains(disabledServer, "T4_POSTGRES_URL_FILE") || + strings.Contains(disabledServer, "T4_PUBLIC_API_CREDENTIALS_FILE") || + strings.Contains(disabledServer, "T4_PUBLIC_API_RUNTIME_PROFILE") || + strings.Contains(disabledEgress, "198.51.100.20/32") || + strings.Contains(disabledEgress, "port: 5432") { + t.Fatal("disabled public API rendered credentials or PostgreSQL egress") + } + + withoutSecret := append(enabledValues(), + "--set", "publicApi.enabled=true", + "--set", "networkPolicy.postgresCIDRs[0]=198.51.100.20/32", + "--set", "networkPolicy.postgresPorts[0]=5432", + ) + helmTemplateMustFail(t, withoutSecret...) + + withoutDatabaseRoute := append(enabledValues(), + "--set", "publicApi.enabled=true", + "--set-string", "publicApi.existingSecret=t4-public-api", + ) + helmTemplateMustFail(t, withoutDatabaseRoute...) + + values := append(enabledValues(), + "--set", "publicApi.enabled=true", + "--set-string", "publicApi.existingSecret=t4-public-api", + "--set-string", "publicApi.postgresURLKey=postgres-url", + "--set-string", "publicApi.credentialsKey=credentials.json", + "--set-string", "clusterHost.runtimeProfiles[0]=review-admitted", + "--set-string", "clusterHost.runtimeProfiles[1]=fallback-admitted", + "--set", "networkPolicy.postgresCIDRs[0]=198.51.100.20/32", + "--set", "networkPolicy.postgresPorts[0]=5432", + ) + output := helmTemplate(t, values...) + server := documentContainingKind(t, output, "Deployment", "name: \"release-name-t4-cluster-server\"") + assertContains(t, server, + "name: T4_POSTGRES_URL_FILE", + "name: T4_PUBLIC_API_CREDENTIALS_FILE", + "name: T4_PUBLIC_API_RUNTIME_PROFILE", + "value: \"review-admitted\"", + "secretName: \"t4-public-api\"", + "key: \"postgres-url\"", + "key: \"credentials.json\"", + "/var/run/secrets/t4-public-api/postgres-url", + "/var/run/secrets/t4-public-api/credentials.json", + ) + if strings.Contains(server, "value: \"fallback-admitted\"") { + t.Fatalf("public API runtime profile was not derived from the first host-admitted profile:\n%s", server) + } + host := documentContainingKind(t, output, "T4ClusterHost", "name: \"t4-cluster\"") + assertContains(t, host, "- review-admitted") + serverEgress := documentContainingKind(t, output, "NetworkPolicy", "name: \"release-name-t4-cluster-server-egress\"") + assertContains(t, serverEgress, "198.51.100.20/32", "port: 5432") + if strings.Contains(serverEgress, "0.0.0.0/0") || strings.Contains(serverEgress, "::/0") { + t.Fatalf("public API PostgreSQL egress is broader than its configured destination:\n%s", serverEgress) + } + + selectorValues := append(enabledValues(), + "--set", "publicApi.enabled=true", + "--set-string", "publicApi.existingSecret=t4-public-api", + "--set", "networkPolicy.postgresPorts[0]=5433", + "--set-string", "networkPolicy.postgres.namespaceSelector.matchLabels.scope=database-namespace", + "--set-string", "networkPolicy.postgres.podSelector.matchLabels.app=postgres", + ) + selectorOutput := helmTemplate(t, selectorValues...) + selectorEgress := documentContainingKind(t, selectorOutput, "NetworkPolicy", "name: \"release-name-t4-cluster-server-egress\"") + assertContains(t, selectorEgress, "scope: database-namespace", "app: postgres", "port: 5433") +} + +func TestPublicAPIRejectsIncompleteSecretKeysAndPostgresRoutes(t *testing.T) { + for name, values := range map[string][]string{ + "empty PostgreSQL key": {"--set", "publicApi.enabled=true", "--set-string", "publicApi.existingSecret=t4-public-api", "--set-string", "publicApi.postgresURLKey=", "--set", "networkPolicy.postgresCIDRs[0]=198.51.100.20/32", "--set", "networkPolicy.postgresPorts[0]=5432"}, + "empty credentials key": {"--set", "publicApi.enabled=true", "--set-string", "publicApi.existingSecret=t4-public-api", "--set-string", "publicApi.credentialsKey=", "--set", "networkPolicy.postgresCIDRs[0]=198.51.100.20/32", "--set", "networkPolicy.postgresPorts[0]=5432"}, + "destination without port": {"--set", "publicApi.enabled=true", "--set-string", "publicApi.existingSecret=t4-public-api", "--set", "networkPolicy.postgresCIDRs[0]=198.51.100.20/32", "--set-string", "networkPolicy.postgresPorts="}, + "port without destination": {"--set", "publicApi.enabled=true", "--set-string", "publicApi.existingSecret=t4-public-api", "--set", "networkPolicy.postgresPorts[0]=5432"}, + "disabled network policy": {"--set", "publicApi.enabled=true", "--set-string", "publicApi.existingSecret=t4-public-api", "--set", "networkPolicy.enabled=false", "--set", "networkPolicy.postgresCIDRs[0]=198.51.100.20/32", "--set", "networkPolicy.postgresPorts[0]=5432"}, + "half selector": {"--set", "publicApi.enabled=true", "--set-string", "publicApi.existingSecret=t4-public-api", "--set", "networkPolicy.postgresPorts[0]=5432", "--set-string", "networkPolicy.postgres.namespaceSelector.matchLabels.scope=database"}, + "selector without port": {"--set", "publicApi.enabled=true", "--set-string", "publicApi.existingSecret=t4-public-api", "--set-string", "networkPolicy.postgres.namespaceSelector.matchLabels.scope=database", "--set-string", "networkPolicy.postgres.podSelector.matchLabels.app=postgres"}, + } { + t.Run(name, func(t *testing.T) { helmTemplateMustFail(t, append(enabledValues(), values...)...) }) } } + +func TestPublicAPIRenderGuardsSurviveSchemaBypass(t *testing.T) { + complete := []string{ + "--skip-schema-validation", + "--set", "publicApi.enabled=true", + "--set-string", "publicApi.existingSecret=t4-public-api", + "--set-string", "publicApi.postgresURLKey=postgres-url", + "--set-string", "publicApi.credentialsKey=credentials.json", + "--set", "networkPolicy.postgresCIDRs[0]=198.51.100.20/32", + "--set", "networkPolicy.postgresPorts[0]=5432", + } + for name, test := range map[string]struct { + flag string + override string + message string + }{ + "missing Secret": { + flag: "--set-string", + override: "publicApi.existingSecret=", + message: "publicApi.existingSecret is required when publicApi.enabled is true", + }, + "missing PostgreSQL key": { + flag: "--set-string", + override: "publicApi.postgresURLKey=", + message: "publicApi.postgresURLKey is required when publicApi.enabled is true", + }, + "missing credentials key": { + flag: "--set-string", + override: "publicApi.credentialsKey=", + message: "publicApi.credentialsKey is required when publicApi.enabled is true", + }, + "disabled network policy": { + flag: "--set", + override: "networkPolicy.enabled=false", + message: "networkPolicy.enabled must be true when publicApi.enabled is true", + }, + "missing PostgreSQL ports": { + flag: "--set-string", + override: "networkPolicy.postgresPorts=", + message: "networkPolicy.postgresPorts must contain at least one port when publicApi.enabled is true", + }, + "missing PostgreSQL destination": { + flag: "--set-string", + override: "networkPolicy.postgresCIDRs=", + message: "publicApi.enabled requires a PostgreSQL CIDR or complete selector destination", + }, + } { + t.Run(name, func(t *testing.T) { + values := append(enabledValues(), complete...) + values = append(values, test.flag, test.override) + helmTemplateMustFailBeforeWorkloads(t, test.message, values...) + }) + } +} + +func TestPublicAPIRejectsCollidingSecretProjectionKeysUnderSchemaBypass(t *testing.T) { + values := append(enabledValues(), + "--skip-schema-validation", + "--set", "publicApi.enabled=true", + "--set-string", "publicApi.existingSecret=t4-public-api", + "--set-string", "publicApi.postgresURLKey=credentials.json", + "--set-string", "publicApi.credentialsKey=credentials.json", + "--set", "networkPolicy.postgresCIDRs[0]=198.51.100.20/32", + "--set", "networkPolicy.postgresPorts[0]=5432", + ) + helmTemplateMustFailBeforeWorkloads(t, "publicApi.postgresURLKey and publicApi.credentialsKey must be different", values...) +} + func TestUnauthenticatedOMPControllerCannotReadSecrets(t *testing.T) { output := helmTemplate(t, enabledValues()...) controllerRole := documentContaining(t, output, "name: \"release-name-t4-cluster-controller\"") @@ -487,7 +663,21 @@ func TestCRDsRemainExplicitAcrossUpgradeAndUninstall(t *testing.T) { if err != nil { t.Fatal(err) } - for _, required := range []string{"helm upgrade", "helm rollback", "helm uninstall", "kubectl apply --server-side -f deploy/charts/t4-cluster/crds/", "condition=Established", "Do not rely on `helm upgrade` to change CRDs", "Retain", "Delete", "CRDs are not removed"} { + for _, required := range []string{ + "scripts/cluster-ci/crd-lifecycle.sh upgrade", + "helm upgrade", + "--skip-crds", + "helm rollback", + "helm uninstall", + "kubectl apply --server-side --dry-run=server", + "condition=Established", + "status.storedVersions", + "Do not rely on `helm upgrade` to change CRDs", + "Future `v1beta1` conversion and storage procedure", + "Retain", + "Delete", + "CRDs are not removed", + } { if !strings.Contains(string(docs), required) { t.Fatalf("operator guide lacks upgrade/uninstall contract %q", required) } @@ -708,6 +898,23 @@ func helmTemplateMustFail(t *testing.T, extra ...string) { } } +func helmTemplateMustFailBeforeWorkloads(t *testing.T, message string, extra ...string) { + t.Helper() + args := []string{"template", "release-name", filepath.Join(repoRoot(t), "deploy", "charts", "t4-cluster"), "--namespace", "t4-system"} + args = append(args, extra...) + command := exec.Command("helm", args...) + output, err := command.CombinedOutput() + if err == nil { + t.Fatalf("helm unexpectedly accepted invalid values: %s", output) + } + if !strings.Contains(string(output), message) { + t.Fatalf("helm failure lacks deterministic guard %q: %s", message, output) + } + if strings.Contains(string(output), "kind: Deployment") || strings.Contains(string(output), "kind: NetworkPolicy") { + t.Fatalf("helm rendered workloads or network policies before public API validation failed: %s", output) + } +} + func enabledValues() []string { return []string{ "--set", "enabled=true", diff --git a/packages/cluster-operator/cmd/crd-preflight/main.go b/packages/cluster-operator/cmd/crd-preflight/main.go new file mode 100644 index 00000000..0a55241b --- /dev/null +++ b/packages/cluster-operator/cmd/crd-preflight/main.go @@ -0,0 +1,382 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "reflect" + "strings" + + apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" + "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel" + "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/pruning" + apiservervalidation "k8s.io/apiextensions-apiserver/pkg/apiserver/validation" + "k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder" + "k8s.io/apimachinery/pkg/runtime" + kubejson "k8s.io/apimachinery/pkg/util/json" + "k8s.io/apimachinery/pkg/util/validation/field" + "sigs.k8s.io/yaml" +) + +const ( + perCallCELCostLimit = 1_000_000 + runtimeCELCostBudget = 10_000_000 +) + +type groupVersionKind struct { + Group string + Version string + Kind string +} + +type candidateSchema struct { + crd *apiextensionsv1.CustomResourceDefinition + version *apiextensionsv1.CustomResourceDefinitionVersion + internal *apiextensions.JSONSchemaProps + structural *structuralschema.Structural +} + +func main() { + if len(os.Args) < 2 { + usage() + } + var err error + switch os.Args[1] { + case "fixtures": + if len(os.Args) != 4 { + usage() + } + err = validateFixtures(os.Args[2], os.Args[3]) + case "objects": + if len(os.Args) != 3 { + usage() + } + err = validateObjects(os.Args[2], os.Stdin) + case "served": + if len(os.Args) != 3 { + usage() + } + err = verifyServedSchemas(os.Args[2], os.Stdin) + default: + usage() + } + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func usage() { + fmt.Fprintln(os.Stderr, "usage: crd-preflight fixtures CRD_DIRECTORY FIXTURE_DIRECTORY | objects CRD_DIRECTORY | served CRD_DIRECTORY") + os.Exit(64) +} + +func validateFixtures(crdDirectory, fixtureDirectory string) error { + candidates, err := loadCandidates(crdDirectory) + if err != nil { + return err + } + paths, err := yamlPaths(fixtureDirectory) + if err != nil { + return err + } + var validationErrors []error + for _, path := range paths { + object, err := decodeYAMLObject(path) + if err != nil { + validationErrors = append(validationErrors, err) + continue + } + validationErrors = append(validationErrors, validateCandidateObject(path, "fixture", object, candidates)...) + } + return errors.Join(validationErrors...) +} + +func validateObjects(crdDirectory string, input io.Reader) error { + candidates, err := loadCandidates(crdDirectory) + if err != nil { + return err + } + var list map[string]interface{} + decoder := json.NewDecoder(input) + decoder.UseNumber() + if err := decoder.Decode(&list); err != nil { + return fmt.Errorf("decode live object list: %w", err) + } + if err := kubejson.ConvertMapNumbers(list, 0); err != nil { + return fmt.Errorf("decode live object list numbers: %w", err) + } + var trailing interface{} + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + if err == nil { + return errors.New("decode live object list: multiple JSON documents") + } + return fmt.Errorf("decode live object list trailing data: %w", err) + } + rawItems, found := list["items"] + if !found { + return errors.New("decode live object list: missing items") + } + items, ok := rawItems.([]interface{}) + if !ok { + return errors.New("decode live object list: items is not an array") + } + var validationErrors []error + for index, rawItem := range items { + object, ok := rawItem.(map[string]interface{}) + if !ok { + validationErrors = append(validationErrors, fmt.Errorf("live object item %d is not an object", index)) + continue + } + path, err := liveObjectPath(index, object) + if err != nil { + validationErrors = append(validationErrors, err) + continue + } + validationErrors = append(validationErrors, validateCandidateObject(path, "object", object, candidates)...) + } + return errors.Join(validationErrors...) +} + +func liveObjectPath(index int, object map[string]interface{}) (string, error) { + metadata, ok := object["metadata"].(map[string]interface{}) + if !ok { + return "", fmt.Errorf("live object item %d has no metadata", index) + } + name, _ := metadata["name"].(string) + namespace, _ := metadata["namespace"].(string) + if name == "" || namespace == "" { + return "", fmt.Errorf("live object item %d must have namespace and name", index) + } + return fmt.Sprintf("live object %s/%s", namespace, name), nil +} + +func validateCandidateObject(path, fieldRoot string, object map[string]interface{}, candidates map[groupVersionKind]*candidateSchema) []error { + apiVersion, _ := object["apiVersion"].(string) + kind, _ := object["kind"].(string) + group, version, ok := strings.Cut(apiVersion, "/") + if !ok || group == "" || version == "" || kind == "" { + return []error{fmt.Errorf("%s: apiVersion and kind must identify a grouped resource", path)} + } + candidate, found := candidates[groupVersionKind{Group: group, Version: version, Kind: kind}] + if !found { + return []error{fmt.Errorf("%s: no proposed schema for %s %s", path, apiVersion, kind)} + } + return validateObject(path, fieldRoot, object, candidate) +} + +func validateObject(path, fieldRoot string, object map[string]interface{}, candidate *candidateSchema) []error { + var result []error + validator, _, err := apiservervalidation.NewSchemaValidator(candidate.internal) + if err != nil { + return []error{fmt.Errorf("%s: build OpenAPI validator: %w", path, err)} + } + if errs := apiservervalidation.ValidateCustomResource(field.NewPath(fieldRoot), object, validator); len(errs) > 0 { + result = append(result, fmt.Errorf("%s: proposed OpenAPI validation failed: %w", path, errs.ToAggregate())) + } + celValidator := cel.NewValidator(candidate.structural, true, perCallCELCostLimit) + if celValidator != nil { + objectPath := field.NewPath(fieldRoot) + createErrors, _ := celValidator.Validate(context.Background(), objectPath, candidate.structural, object, nil, runtimeCELCostBudget) + if len(createErrors) > 0 { + result = append(result, fmt.Errorf("%s: proposed CEL create validation failed: %w", path, createErrors.ToAggregate())) + } + updateErrors, _ := celValidator.Validate(context.Background(), objectPath, candidate.structural, object, object, runtimeCELCostBudget) + if len(updateErrors) > 0 { + result = append(result, fmt.Errorf("%s: proposed CEL unchanged-update validation failed: %w", path, updateErrors.ToAggregate())) + } + } + unknownFields := pruning.PruneWithOptions( + runtime.DeepCopyJSONValue(object), + candidate.structural, + true, + structuralschema.UnknownFieldPathOptions{TrackUnknownFieldPaths: true}, + ) + if len(unknownFields) > 0 { + result = append(result, fmt.Errorf("%s: proposed structural schema would prune declared fields: %s", path, strings.Join(unknownFields, ", "))) + } + return result +} + +func verifyServedSchemas(crdDirectory string, discovery io.Reader) error { + candidates, err := loadCandidates(crdDirectory) + if err != nil { + return err + } + var document map[string]interface{} + decoder := json.NewDecoder(discovery) + decoder.UseNumber() + if err := decoder.Decode(&document); err != nil { + return fmt.Errorf("decode served OpenAPI v3 document: %w", err) + } + for gvk, candidate := range candidates { + actual, err := schemaForGVK(document, gvk) + if err != nil { + return fmt.Errorf("served schema for %s/%s %s: %w", gvk.Group, gvk.Version, gvk.Kind, err) + } + expected, err := generatedSchema(candidate, gvk) + if err != nil { + return fmt.Errorf("generate proposed schema for %s/%s %s: %w", gvk.Group, gvk.Version, gvk.Kind, err) + } + normalizePublishedSchema(expected) + normalizePublishedSchema(actual) + if !reflect.DeepEqual(expected, actual) { + return fmt.Errorf("served OpenAPI semantics for %s/%s %s do not match the proposed CRD", gvk.Group, gvk.Version, gvk.Kind) + } + } + return nil +} + +func generatedSchema(candidate *candidateSchema, gvk groupVersionKind) (map[string]interface{}, error) { + openapi, err := builder.BuildOpenAPIV3(candidate.crd, candidate.version.Name, builder.Options{}) + if err != nil { + return nil, err + } + raw, err := json.Marshal(openapi) + if err != nil { + return nil, err + } + var document map[string]interface{} + decoder := json.NewDecoder(strings.NewReader(string(raw))) + decoder.UseNumber() + if err := decoder.Decode(&document); err != nil { + return nil, err + } + return schemaForGVK(document, gvk) +} + +func schemaForGVK(document map[string]interface{}, wanted groupVersionKind) (map[string]interface{}, error) { + components, ok := document["components"].(map[string]interface{}) + if !ok { + return nil, errors.New("document has no components") + } + schemas, ok := components["schemas"].(map[string]interface{}) + if !ok { + return nil, errors.New("document has no component schemas") + } + for _, value := range schemas { + schema, ok := value.(map[string]interface{}) + if !ok { + continue + } + gvks, ok := schema["x-kubernetes-group-version-kind"].([]interface{}) + if !ok { + continue + } + for _, value := range gvks { + gvk, ok := value.(map[string]interface{}) + if ok && gvk["group"] == wanted.Group && gvk["version"] == wanted.Version && gvk["kind"] == wanted.Kind { + return runtime.DeepCopyJSONValue(schema).(map[string]interface{}), nil + } + } + } + return nil, errors.New("matching x-kubernetes-group-version-kind was not published") +} + +func normalizePublishedSchema(schema map[string]interface{}) { + delete(schema, "x-kubernetes-group-version-kind") + if properties, ok := schema["properties"].(map[string]interface{}); ok { + delete(properties, "apiVersion") + delete(properties, "kind") + delete(properties, "metadata") + } + stripDescriptions(schema) +} + +func stripDescriptions(value interface{}) { + switch value := value.(type) { + case map[string]interface{}: + delete(value, "description") + for _, child := range value { + stripDescriptions(child) + } + case []interface{}: + for _, child := range value { + stripDescriptions(child) + } + } +} + +func loadCandidates(directory string) (map[groupVersionKind]*candidateSchema, error) { + paths, err := yamlPaths(directory) + if err != nil { + return nil, err + } + result := make(map[groupVersionKind]*candidateSchema) + for _, path := range paths { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var crd apiextensionsv1.CustomResourceDefinition + if err := yaml.Unmarshal(raw, &crd); err != nil { + return nil, fmt.Errorf("%s: decode CRD: %w", path, err) + } + for index := range crd.Spec.Versions { + version := &crd.Spec.Versions[index] + if !version.Served { + continue + } + if version.Schema == nil || version.Schema.OpenAPIV3Schema == nil { + return nil, fmt.Errorf("%s: served version %s has no OpenAPI schema", path, version.Name) + } + internal := &apiextensions.JSONSchemaProps{} + if err := apiextensionsv1.Convert_v1_JSONSchemaProps_To_apiextensions_JSONSchemaProps(version.Schema.OpenAPIV3Schema, internal, nil); err != nil { + return nil, fmt.Errorf("%s: convert schema for %s: %w", path, version.Name, err) + } + structural, err := structuralschema.NewStructural(internal) + if err != nil { + return nil, fmt.Errorf("%s: schema for %s is not structural: %w", path, version.Name, err) + } + gvk := groupVersionKind{Group: crd.Spec.Group, Version: version.Name, Kind: crd.Spec.Names.Kind} + if _, duplicate := result[gvk]; duplicate { + return nil, fmt.Errorf("%s: duplicate proposed schema for %s/%s %s", path, gvk.Group, gvk.Version, gvk.Kind) + } + result[gvk] = &candidateSchema{crd: &crd, version: version, internal: internal, structural: structural} + } + } + if len(result) == 0 { + return nil, fmt.Errorf("%s: no served CRD schemas found", directory) + } + return result, nil +} + +func decodeYAMLObject(path string) (map[string]interface{}, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + jsonRaw, err := yaml.YAMLToJSON(raw) + if err != nil { + return nil, fmt.Errorf("%s: decode YAML: %w", path, err) + } + var object map[string]interface{} + if err := json.Unmarshal(jsonRaw, &object); err != nil { + return nil, fmt.Errorf("%s: decode object: %w", path, err) + } + return object, nil +} + +func yamlPaths(directory string) ([]string, error) { + entries, err := os.ReadDir(directory) + if err != nil { + return nil, err + } + paths := make([]string, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() { + continue + } + extension := strings.ToLower(filepath.Ext(entry.Name())) + if extension == ".yaml" || extension == ".yml" { + paths = append(paths, filepath.Join(directory, entry.Name())) + } + } + return paths, nil +} diff --git a/packages/cluster-operator/cmd/crd-preflight/main_test.go b/packages/cluster-operator/cmd/crd-preflight/main_test.go new file mode 100644 index 00000000..1c08fa28 --- /dev/null +++ b/packages/cluster-operator/cmd/crd-preflight/main_test.go @@ -0,0 +1,362 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +const candidateCRD = `apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: widgets.cluster.t4.dev +spec: + group: cluster.t4.dev + scope: Namespaced + names: + plural: widgets + singular: widget + kind: Widget + listKind: WidgetList + versions: + - name: v1alpha1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + required: [spec] + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + type: object + required: [code] + x-kubernetes-validations: + - rule: self.code.startsWith('ok') + message: code must start with ok + properties: + code: + type: string + maxLength: 3 + status: + type: object + properties: + phase: + type: string + enum: [Ready] +` + +func TestValidateFixturesRejectsProposedSpecTighteningAndCEL(t *testing.T) { + for _, test := range []struct { + name string + code string + expectedError string + }{ + {name: "openapi maxLength", code: "okay", expectedError: "proposed OpenAPI validation failed"}, + {name: "CEL rule", code: "bad", expectedError: "proposed CEL create validation failed"}, + } { + t.Run(test.name, func(t *testing.T) { + crds, fixtures := writeCandidate(t, `apiVersion: cluster.t4.dev/v1alpha1 +kind: Widget +metadata: + name: legacy +spec: + code: `+test.code+"\n"+`status: + phase: Ready +`) + err := validateFixtures(crds, fixtures) + if err == nil { + t.Fatal("fixture incompatible with the proposed spec schema was accepted") + } + if !strings.Contains(err.Error(), test.expectedError) { + t.Fatalf("validation error %q does not identify %q", err, test.expectedError) + } + }) + } +} + +func TestValidateFixturesRejectsPersistedStatusAgainstProposedSchema(t *testing.T) { + crds, fixtures := writeCandidate(t, `apiVersion: cluster.t4.dev/v1alpha1 +kind: Widget +metadata: + name: legacy +spec: + code: ok +status: + phase: Legacy +`) + if err := validateFixtures(crds, fixtures); err == nil { + t.Fatal("persisted status incompatible with the proposed status schema was accepted") + } +} + +func TestValidateFixturesRejectsUnchangedLegacyValuesUnderTransitionCEL(t *testing.T) { + tests := []struct { + name string + fixture string + expectedPath string + candidate func(string) string + }{ + { + name: "spec transition rule", + expectedPath: "fixture.spec.code", + fixture: `apiVersion: cluster.t4.dev/v1alpha1 +kind: Widget +metadata: + name: legacy +spec: + code: bad +status: + phase: Ready +`, + candidate: func(crd string) string { + crd = strings.Replace(crd, "rule: self.code.startsWith('ok')", `rule: "true"`, 1) + return strings.Replace(crd, " maxLength: 3", " maxLength: 3\n x-kubernetes-validations:\n - rule: oldSelf.startsWith('ok')", 1) + }, + }, + { + name: "status transition rule", + expectedPath: "fixture.status.phase", + fixture: `apiVersion: cluster.t4.dev/v1alpha1 +kind: Widget +metadata: + name: legacy +spec: + code: ok +status: + phase: Pending +`, + candidate: func(crd string) string { + return strings.Replace(crd, " enum: [Ready]", " enum: [Ready, Pending]\n x-kubernetes-validations:\n - rule: oldSelf == 'Ready'", 1) + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + crds, fixtures := writeCandidate(t, test.fixture) + if err := os.WriteFile(filepath.Join(crds, "widget.yaml"), []byte(test.candidate(candidateCRD)), 0o644); err != nil { + t.Fatal(err) + } + err := validateFixtures(crds, fixtures) + if err == nil { + t.Fatal("unchanged persisted value blocked by transition CEL was accepted") + } + if !strings.Contains(err.Error(), "proposed CEL unchanged-update validation failed") { + t.Fatalf("validation error does not identify unchanged-update semantics: %v", err) + } + if !strings.Contains(err.Error(), test.expectedPath) { + t.Fatalf("validation error %q does not identify field %q", err, test.expectedPath) + } + }) + } +} + +func TestValidateObjectsRejectsLiveDataOmittedFromFixtures(t *testing.T) { + tests := []struct { + name string + candidate func(string) string + fixtureCode string + fixturePhase string + liveCode string + livePhase string + liveExtra string + expectedError string + }{ + { + name: "spec OpenAPI narrowing", + candidate: func(crd string) string { + return strings.Replace(crd, " maxLength: 3", " maxLength: 3\n enum: [ok1]", 1) + }, + fixtureCode: "ok1", fixturePhase: "Ready", liveCode: "ok2", livePhase: "Ready", + expectedError: "proposed OpenAPI validation failed", + }, + { + name: "status OpenAPI narrowing", + candidate: func(crd string) string { return crd }, + fixtureCode: "ok", fixturePhase: "Ready", liveCode: "ok", livePhase: "Legacy", + expectedError: "proposed OpenAPI validation failed", + }, + { + name: "spec CEL create semantics", + candidate: func(crd string) string { return crd }, + fixtureCode: "ok", fixturePhase: "Ready", liveCode: "bad", livePhase: "Ready", + expectedError: "proposed CEL create validation failed", + }, + { + name: "status CEL create semantics", + candidate: func(crd string) string { + return strings.Replace(crd, " phase:\n type: string\n enum: [Ready]", " phase:\n type: string\n enum: [Ready, Legacy]\n x-kubernetes-validations:\n - rule: self == 'Ready'", 1) + }, + fixtureCode: "ok", fixturePhase: "Ready", liveCode: "ok", livePhase: "Legacy", + expectedError: "proposed CEL create validation failed", + }, + { + name: "spec CEL unchanged-update semantics", + candidate: func(crd string) string { + crd = strings.Replace(crd, "rule: self.code.startsWith('ok')", `rule: "true"`, 1) + return strings.Replace(crd, " maxLength: 3", " maxLength: 3\n x-kubernetes-validations:\n - rule: oldSelf.startsWith('ok')", 1) + }, + fixtureCode: "ok", fixturePhase: "Ready", liveCode: "bad", livePhase: "Ready", + expectedError: "proposed CEL unchanged-update validation failed", + }, + { + name: "status CEL unchanged-update semantics", + candidate: func(crd string) string { + return strings.Replace(crd, " enum: [Ready]", " enum: [Ready, Legacy]\n x-kubernetes-validations:\n - rule: oldSelf == 'Ready'", 1) + }, + fixtureCode: "ok", fixturePhase: "Ready", liveCode: "ok", livePhase: "Legacy", + expectedError: "proposed CEL unchanged-update validation failed", + }, + { + name: "spec pruning", + candidate: func(crd string) string { return crd }, + fixtureCode: "ok", fixturePhase: "Ready", liveCode: "ok", livePhase: "Ready", liveExtra: `,"removedSpec":"legacy"`, + expectedError: "proposed structural schema would prune declared fields", + }, + { + name: "status pruning", + candidate: func(crd string) string { return crd }, + fixtureCode: "ok", fixturePhase: "Ready", liveCode: "ok", livePhase: "Ready", liveExtra: `,"removedStatus":"legacy"`, + expectedError: "proposed structural schema would prune declared fields", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fixture := fmt.Sprintf("apiVersion: cluster.t4.dev/v1alpha1\nkind: Widget\nmetadata:\n name: curated\n namespace: tenant-a\nspec:\n code: %s\nstatus:\n phase: %s\n", test.fixtureCode, test.fixturePhase) + crds, fixtures := writeCandidate(t, fixture) + if err := os.WriteFile(filepath.Join(crds, "widget.yaml"), []byte(test.candidate(candidateCRD)), 0o644); err != nil { + t.Fatal(err) + } + if err := validateFixtures(crds, fixtures); err != nil { + t.Fatalf("curated fixture should not expose the live-only incompatibility: %v", err) + } + live := fmt.Sprintf(`{"apiVersion":"v1","kind":"WidgetList","items":[{"apiVersion":"cluster.t4.dev/v1alpha1","kind":"Widget","metadata":{"name":"live","namespace":"tenant-b"},"spec":{"code":%q%s},"status":{"phase":%q%s}}]}`, + test.liveCode, map[bool]string{true: test.liveExtra, false: ""}[test.name == "spec pruning"], test.livePhase, map[bool]string{true: test.liveExtra, false: ""}[test.name == "status pruning"]) + err := validateObjects(crds, strings.NewReader(live)) + if err == nil { + t.Fatal("live object incompatible with the proposed schema was accepted") + } + if !strings.Contains(err.Error(), "tenant-b/live") || !strings.Contains(err.Error(), test.expectedError) { + t.Fatalf("validation error %q does not identify live object and %q", err, test.expectedError) + } + }) + } +} + +func TestValidateObjectsAcceptsKubernetesJSONNumbers(t *testing.T) { + crds, _ := writeCandidate(t, `apiVersion: cluster.t4.dev/v1alpha1 +kind: Widget +metadata: + name: curated +spec: + code: ok +`) + candidate := strings.Replace(candidateCRD, " required: [code]", " required: [code, count]", 1) + candidate = strings.Replace(candidate, " maxLength: 3", " maxLength: 3\n count:\n type: integer\n x-kubernetes-validations:\n - rule: self == 9223372036854775807", 1) + if err := os.WriteFile(filepath.Join(crds, "widget.yaml"), []byte(candidate), 0o644); err != nil { + t.Fatal(err) + } + live := strings.NewReader(`{"apiVersion":"v1","kind":"WidgetList","items":[{"apiVersion":"cluster.t4.dev/v1alpha1","kind":"Widget","metadata":{"name":"live","namespace":"tenant-b"},"spec":{"code":"ok","count":9223372036854775807},"status":{"phase":"Ready"}}]}`) + if err := validateObjects(crds, live); err != nil { + t.Fatalf("ordinary Kubernetes JSON integer was rejected: %v", err) + } +} + +func TestVerifyServedSchemasRejectsRetainedEstablishedWithStaleSchema(t *testing.T) { + crds, _ := writeCandidate(t, `apiVersion: cluster.t4.dev/v1alpha1 +kind: Widget +metadata: + name: legacy +spec: + code: ok +`) + staleDiscovery := strings.NewReader(`{ + "openapi": "3.0.0", + "components": {"schemas": { + "cluster.t4.dev.v1alpha1.Widget": { + "type": "object", + "required": ["spec"], + "properties": { + "apiVersion": {"type": "string"}, + "kind": {"type": "string"}, + "metadata": {"type": "object"}, + "spec": { + "type": "object", + "required": ["code"], + "x-kubernetes-validations": [{"rule": "self.code.startsWith('ok')", "message": "code must start with ok"}], + "properties": {"code": {"type": "string", "maxLength": 8}} + }, + "status": {"type": "object", "properties": {"phase": {"type": "string", "enum": ["Ready"]}}} + }, + "x-kubernetes-group-version-kind": [{"group":"cluster.t4.dev","version":"v1alpha1","kind":"Widget"}] + } + }} +}`) + if err := verifyServedSchemas(crds, staleDiscovery); err == nil { + t.Fatal("stale served OpenAPI schema was accepted after Established") + } +} + +func TestVerifyServedSchemasAcceptsExactProposedSemantics(t *testing.T) { + crds, _ := writeCandidate(t, `apiVersion: cluster.t4.dev/v1alpha1 +kind: Widget +metadata: + name: legacy +spec: + code: ok +`) + discovery := strings.NewReader(`{ + "openapi": "3.0.0", + "components": {"schemas": { + "cluster.t4.dev.v1alpha1.Widget": { + "type": "object", + "required": ["spec"], + "properties": { + "apiVersion": {"type": "string"}, + "kind": {"type": "string"}, + "metadata": {"type": "object"}, + "spec": { + "type": "object", + "required": ["code"], + "x-kubernetes-validations": [{"rule": "self.code.startsWith('ok')", "message": "code must start with ok"}], + "properties": {"code": {"type": "string", "maxLength": 3}} + }, + "status": {"type": "object", "properties": {"phase": {"type": "string", "enum": ["Ready"]}}} + }, + "x-kubernetes-group-version-kind": [{"group":"cluster.t4.dev","version":"v1alpha1","kind":"Widget"}] + } + }} +}`) + if err := verifyServedSchemas(crds, discovery); err != nil { + t.Fatalf("exact proposed schema rejected: %v", err) + } +} + +func writeCandidate(t *testing.T, fixture string) (string, string) { + t.Helper() + root := t.TempDir() + crds := filepath.Join(root, "crds") + fixtures := filepath.Join(root, "fixtures") + for _, directory := range []string{crds, fixtures} { + if err := os.Mkdir(directory, 0o755); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(crds, "widget.yaml"), []byte(candidateCRD), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(fixtures, "widget.yaml"), []byte(fixture), 0o644); err != nil { + t.Fatal(err) + } + return crds, fixtures +} + diff --git a/packages/cluster-operator/cmd/manager/main.go b/packages/cluster-operator/cmd/manager/main.go index d216db5a..f33af52c 100644 --- a/packages/cluster-operator/cmd/manager/main.go +++ b/packages/cluster-operator/cmd/manager/main.go @@ -70,7 +70,7 @@ func main() { ctrl.Log.Error(err, "unable to register T4ClusterHost controller") os.Exit(1) } - if err := (&controllers.WorkspaceReconciler{Client: manager.GetClient(), Scheme: manager.GetScheme()}).SetupWithManager(manager); err != nil { + if err := (&controllers.WorkspaceReconciler{Client: manager.GetClient(), APIReader: manager.GetAPIReader(), Scheme: manager.GetScheme()}).SetupWithManager(manager); err != nil { ctrl.Log.Error(err, "unable to register T4Workspace controller") os.Exit(1) } diff --git a/packages/cluster-operator/controllers/helpers.go b/packages/cluster-operator/controllers/helpers.go index ec5681af..8fef978c 100644 --- a/packages/cluster-operator/controllers/helpers.go +++ b/packages/cluster-operator/controllers/helpers.go @@ -18,6 +18,7 @@ import ( const ( ReasonStorageClassNotFound = "StorageClassNotFound" ReasonStorageClassNotRWX = "StorageClassNotRWX" + ReasonStorageClassMismatch = "StorageClassMismatch" ReasonStorageReady = "StorageClassSupportsRWX" ) @@ -82,6 +83,13 @@ func pvcHasRWX(pvc *corev1.PersistentVolumeClaim) bool { return false } +func pvcStorageClassName(pvc *corev1.PersistentVolumeClaim) string { + if pvc.Spec.StorageClassName == nil { + return "" + } + return *pvc.Spec.StorageClassName +} + func hasString(values []string, wanted string) bool { for _, value := range values { if value == wanted { diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index 1a54e2d0..2c9db16f 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -2,8 +2,10 @@ package controllers_test import ( "context" + "reflect" "strings" "testing" + "time" corev1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" @@ -11,6 +13,7 @@ import ( apiresource "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" utilvalidation "k8s.io/apimachinery/pkg/util/validation" ctrl "sigs.k8s.io/controller-runtime" @@ -69,6 +72,172 @@ func TestWorkspaceReconcileIsIdempotentAcrossDuplicateEvents(t *testing.T) { } } +func TestWorkspaceCreateAlreadyExistsRefetchesForeignPVC(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + base := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Workspace{}, &corev1.PersistentVolumeClaim{}). + WithObjects(testHost(), rwxStorageClass(), workspace).Build() + c := &createAlreadyExistsClient{Client: base, raceKind: "PVC", hideWinnerFromCache: true} + r := &controllers.WorkspaceReconciler{Client: c, APIReader: base, Scheme: scheme} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + if c.winner == nil { + t.Fatal("PVC create race was not exercised") + } + var pvc corev1.PersistentVolumeClaim + if err := base.Get(ctx, client.ObjectKeyFromObject(c.winner), &pvc); err != nil { + t.Fatalf("foreign PVC winner was deleted: %v", err) + } + want := c.winner.(*corev1.PersistentVolumeClaim) + if !reflect.DeepEqual(pvc.ObjectMeta, want.ObjectMeta) || !reflect.DeepEqual(pvc.Spec, want.Spec) { + t.Fatalf("foreign PVC winner was mutated: %#v", pvc) + } + var failed clusterv1alpha1.T4Workspace + if err := base.Get(ctx, client.ObjectKeyFromObject(workspace), &failed); err != nil { + t.Fatal(err) + } + storageReady := findCondition(failed.Status.Conditions, "StorageReady") + if failed.Status.PVCName != "" || storageReady == nil || storageReady.Status != metav1.ConditionFalse || storageReady.Reason != "PVCOwnershipConflict" || storageReady.ObservedGeneration != failed.Generation { + t.Fatalf("foreign PVC winner was published as authoritative: status=%#v StorageReady=%#v", failed.Status, storageReady) + } +} + +func TestWorkspaceReadinessRejectsAuthoritativeForeignPVCReplacement(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + cachedPVC := ownedWorkspacePVC(workspace) + cachedPVC.UID = "cached-pvc-uid" + authoritativePVC := cachedPVC.DeepCopy() + authoritativePVC.UID = "replacement-pvc-uid" + authoritativePVC.Annotations = nil + authoritativePVC.OwnerReferences = nil + cacheClient := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Workspace{}, &corev1.PersistentVolumeClaim{}). + WithObjects(testHost(), rwxStorageClass(), workspace, cachedPVC).Build() + r := &controllers.WorkspaceReconciler{ + Client: cacheClient, + APIReader: &pvcOverrideReader{Reader: cacheClient, pvc: authoritativePVC}, + Scheme: scheme, + } + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + var got clusterv1alpha1.T4Workspace + if err := cacheClient.Get(ctx, client.ObjectKeyFromObject(workspace), &got); err != nil { + t.Fatal(err) + } + storageReady := findCondition(got.Status.Conditions, "StorageReady") + ready := findCondition(got.Status.Conditions, "Ready") + if got.Status.PVCName != "" || got.Status.PVCPhase != "" || !got.Status.Capacity.IsZero() || + storageReady == nil || storageReady.Status != metav1.ConditionFalse || storageReady.Reason != "PVCOwnershipConflict" || storageReady.ObservedGeneration != got.Generation || + ready == nil || ready.Status != metav1.ConditionFalse || ready.ObservedGeneration != got.Generation { + t.Fatalf("stale cached PVC published workspace authority: status=%#v StorageReady=%#v Ready=%#v", got.Status, storageReady, ready) + } +} + + +func TestWorkspacePendingPVCPolicyFailsBeforeAuthority(t *testing.T) { + for _, test := range []struct { + name string + mutate func(*corev1.PersistentVolumeClaim) + reason string + }{ + {name: "wrong class", reason: controllers.ReasonStorageClassMismatch, mutate: func(pvc *corev1.PersistentVolumeClaim) { pvc.Spec.StorageClassName = ptr("other-rwx") }}, + {name: "wrong access", reason: "PVCNotRWX", mutate: func(pvc *corev1.PersistentVolumeClaim) { pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} }}, + } { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + pvc := ownedWorkspacePVC(workspace) + pvc.Status.Phase = corev1.ClaimPending + test.mutate(pvc) + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Workspace{}, &corev1.PersistentVolumeClaim{}).WithObjects(testHost(), rwxStorageClass(), workspace, pvc).Build() + r := &controllers.WorkspaceReconciler{Client: c, APIReader: c, Scheme: scheme} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + var failed clusterv1alpha1.T4Workspace + if err := c.Get(ctx, client.ObjectKeyFromObject(workspace), &failed); err != nil { + t.Fatal(err) + } + condition := findCondition(failed.Status.Conditions, "StorageReady") + if failed.Status.PVCName != "" || condition == nil || condition.Status != metav1.ConditionFalse || condition.Reason != test.reason { + t.Fatalf("incompatible Pending PVC published authority: status=%#v condition=%#v", failed.Status, condition) + } + }) + } +} + +func TestWorkspaceDeletionUsesAuthoritativePVCReader(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + workspace.Finalizers = []string{clusterv1alpha1.WorkspaceFinalizer} + pvc := ownedWorkspacePVC(workspace) + pvc.OwnerReferences = nil + base := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Workspace{}).WithObjects(workspace, pvc).Build() + if err := base.Delete(ctx, workspace); err != nil { + t.Fatal(err) + } + c := &createAlreadyExistsClient{Client: base, raceKind: "PVC", winner: pvc, hideWinnerFromCache: true} + r := &controllers.WorkspaceReconciler{Client: c, APIReader: base, Scheme: scheme} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + var waiting clusterv1alpha1.T4Workspace + if err := base.Get(ctx, client.ObjectKeyFromObject(workspace), &waiting); err != nil { + t.Fatalf("workspace finalizer ignored authoritative PVC conflict: %v", err) + } + condition := findCondition(waiting.Status.Conditions, "Ready") + if condition == nil || condition.Reason != "CleanupOwnershipConflict" || !contains(waiting.Finalizers, clusterv1alpha1.WorkspaceFinalizer) { + t.Fatalf("authoritative PVC conflict not retained: %#v", waiting) + } +} + +func TestWorkspaceDeletionUsesAuthoritativeSessionReader(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + workspace.Finalizers = []string{clusterv1alpha1.WorkspaceFinalizer} + pvc := ownedWorkspacePVC(workspace) + session := testSession() + session.Spec.WorkspaceRef = workspace.Name + cacheClient := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Workspace{}).WithObjects(workspace, pvc).Build() + apiReader := fake.NewClientBuilder().WithScheme(scheme).WithObjects(pvc, session).Build() + if err := cacheClient.Delete(ctx, workspace); err != nil { + t.Fatal(err) + } + r := &controllers.WorkspaceReconciler{Client: cacheClient, APIReader: apiReader, Scheme: scheme} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + var waiting clusterv1alpha1.T4Workspace + if err := cacheClient.Get(ctx, client.ObjectKeyFromObject(workspace), &waiting); err != nil { + t.Fatalf("workspace finalizer ignored authoritative session: %v", err) + } + ready := findCondition(waiting.Status.Conditions, "Ready") + if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != "SessionsRemain" || ready.ObservedGeneration != waiting.Generation { + t.Fatalf("Ready = %#v, want current-generation False/SessionsRemain", ready) + } + if !contains(waiting.Finalizers, clusterv1alpha1.WorkspaceFinalizer) { + t.Fatal("workspace finalizer was removed while authoritative session remains") + } + var remainingPVC corev1.PersistentVolumeClaim + if err := cacheClient.Get(ctx, client.ObjectKeyFromObject(pvc), &remainingPVC); err != nil { + t.Fatalf("workspace PVC was deleted while authoritative session remains: %v", err) + } +} + func TestRetainWorkspaceCreatesPVCWithoutGarbageCollectableOwner(t *testing.T) { scheme := testScheme(t) workspace := testWorkspace(clusterv1alpha1.RetentionPolicyRetain) @@ -151,7 +320,7 @@ func TestRetainDeletionOrphansPVCBeforeRemovingFinalizer(t *testing.T) { APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Workspace", Name: workspace.Name, UID: workspace.UID, Controller: ptr(true), }}, }, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, } c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Workspace{}).WithObjects(testHost(), rwxStorageClass(), workspace, pvc).Build() if err := c.Delete(context.Background(), workspace); err != nil { @@ -185,7 +354,7 @@ func TestWorkspaceDeletionWaitsForSessionResources(t *testing.T) { Name: controllers.WorkspacePVCName(workspace), Namespace: workspace.Namespace, OwnerReferences: []metav1.OwnerReference{{APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Workspace", Name: workspace.Name, UID: workspace.UID, Controller: ptr(true)}}, }, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, } session := testSession() session.Spec.WorkspaceRef = workspace.Name @@ -214,6 +383,107 @@ func TestWorkspaceDeletionWaitsForSessionResources(t *testing.T) { } } +func TestSessionPodCreateRevalidatesAuthoritativePVC(t *testing.T) { + tests := []struct { + name string + mutate func(*corev1.PersistentVolumeClaim) + }{ + {name: "replacement UID", mutate: func(pvc *corev1.PersistentVolumeClaim) { pvc.UID = "replacement-pvc-uid" }}, + {name: "foreign owner", mutate: func(pvc *corev1.PersistentVolumeClaim) { + pvc.OwnerReferences = []metav1.OwnerReference{{APIVersion: "v1", Kind: "Secret", Name: "foreign", UID: "foreign-uid", Controller: ptr(true)}} + }}, + {name: "storage class drift", mutate: func(pvc *corev1.PersistentVolumeClaim) { pvc.Spec.StorageClassName = ptr("other-rwx") }}, + {name: "access mode drift", mutate: func(pvc *corev1.PersistentVolumeClaim) { pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} }}, + {name: "unbound replacement", mutate: func(pvc *corev1.PersistentVolumeClaim) { pvc.Status.Phase = corev1.ClaimPending }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + workspace.Status.PVCName = controllers.WorkspacePVCName(workspace) + cachedPVC := ownedWorkspacePVC(workspace) + cachedPVC.UID = "cached-pvc-uid" + authoritativePVC := cachedPVC.DeepCopy() + test.mutate(authoritativePVC) + session := testSession() + session.UID = "session-uid" + cacheClient := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(testHost(), workspace, cachedPVC, session).Build() + r := configuredSessionReconciler(cacheClient, scheme) + r.APIReader = &pvcOverrideReader{Reader: cacheClient, pvc: authoritativePVC} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + var pods corev1.PodList + if err := cacheClient.List(ctx, &pods, client.InNamespace(session.Namespace)); err != nil { + t.Fatal(err) + } + if len(pods.Items) != 0 { + t.Fatalf("created %d Pods after authoritative PVC %s", len(pods.Items), test.name) + } + var got clusterv1alpha1.T4Session + if err := cacheClient.Get(ctx, client.ObjectKeyFromObject(session), &got); err != nil { + t.Fatal(err) + } + workspaceReady := findCondition(got.Status.Conditions, "WorkspaceReady") + if got.Status.PodName != "" || workspaceReady == nil || workspaceReady.Status != metav1.ConditionFalse || workspaceReady.Reason != "PVCAuthorityChanged" || workspaceReady.ObservedGeneration != got.Generation { + t.Fatalf("authoritative PVC %s published session authority: status=%#v WorkspaceReady=%#v", test.name, got.Status, workspaceReady) + } + }) + } +} + +func TestSessionExistingPodRepairRejectsAuthoritativeForeignPVCReplacement(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + workspace.Status.PVCName = controllers.WorkspacePVCName(workspace) + cachedPVC := ownedWorkspacePVC(workspace) + cachedPVC.UID = "cached-pvc-uid" + session := testSession() + session.UID = "session-uid" + cacheClient := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(testHost(), workspace, cachedPVC, session).Build() + r := configuredSessionReconciler(cacheClient, scheme) + reconcileMany(t, 2, func() error { + _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + var pod corev1.Pod + if err := cacheClient.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: controllers.SessionPodName(session)}, &pod); err != nil { + t.Fatal(err) + } + pod.Labels = nil + if err := cacheClient.Update(ctx, &pod); err != nil { + t.Fatal(err) + } + authoritativePVC := cachedPVC.DeepCopy() + authoritativePVC.UID = "replacement-pvc-uid" + authoritativePVC.Annotations = nil + authoritativePVC.OwnerReferences = nil + r.APIReader = &pvcOverrideReader{Reader: cacheClient, pvc: authoritativePVC} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + if err := cacheClient.Get(ctx, client.ObjectKeyFromObject(&pod), &pod); !apierrors.IsNotFound(err) { + t.Fatalf("existing Pod survived authoritative PVC replacement: %v", err) + } + var got clusterv1alpha1.T4Session + if err := cacheClient.Get(ctx, client.ObjectKeyFromObject(session), &got); err != nil { + t.Fatal(err) + } + workspaceReady := findCondition(got.Status.Conditions, "WorkspaceReady") + if got.Status.PodName != "" || workspaceReady == nil || workspaceReady.Status != metav1.ConditionFalse || workspaceReady.Reason != "PVCAuthorityChanged" || workspaceReady.ObservedGeneration != got.Generation { + t.Fatalf("existing Pod repair retained stale PVC authority: status=%#v WorkspaceReady=%#v", got.Status, workspaceReady) + } +} + + func TestSessionFailsClosedWhenAnyOMPReferenceIsMissing(t *testing.T) { for _, test := range []struct { name string @@ -231,7 +501,7 @@ func TestSessionFailsClosedWhenAnyOMPReferenceIsMissing(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -293,7 +563,7 @@ func TestSessionCredentialModeUpgradeStopsExistingAuthorityAndClearsRoute(t *tes workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -355,7 +625,7 @@ func TestSessionRejectsCredentialBearingModelsConfiguration(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -406,7 +676,7 @@ func TestSessionRejectsCredentialBearingSettingsConfiguration(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -455,7 +725,7 @@ func TestSessionRuntimeImageMustBeImmutableDigest(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -526,7 +796,7 @@ func TestSessionAuthorityRevocationDeletesOwnedPodAndService(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -573,7 +843,7 @@ func TestSessionNamesProduceSafeRuntimeIdentities(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -618,7 +888,7 @@ func TestSessionWaitsForBoundRWXThenCreatesExactlyOnePodAndService(t *testing.T) workspace.Status.Phase = clusterv1alpha1.InfrastructurePending pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimPending}, } session := testSession() @@ -753,7 +1023,7 @@ func TestSessionOMPModeOmitsCredentialSecretReference(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -789,7 +1059,7 @@ func TestSessionRejectsUnownedDeterministicResources(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -824,7 +1094,7 @@ func TestSessionRecreatesPodWhenImmutableDesiredStateChanges(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -879,7 +1149,7 @@ func TestSessionPodHashIncludesEveryOMPReference(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -925,7 +1195,7 @@ func TestSessionRecreatesPodWhenOMPResourceVersionChanges(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -968,7 +1238,7 @@ func TestSessionRuntimeReferenceRevocationStopsAuthority(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -1022,7 +1292,7 @@ func TestSessionFailsClosedWhenOMPConfigMapIsMissing(t *testing.T) { scheme := testScheme(t) workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) workspace.Status.PVCName = "workspace-a-data" - pvc := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}} + pvc := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}} session := testSession() objects := []client.Object{testHost(), rwxStorageClass(), workspace, pvc, session} if test.configMap != nil { @@ -1052,7 +1322,7 @@ func TestSessionRecreatesExternallyExposedOwnedService(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -1100,7 +1370,7 @@ func TestSessionRestoresRequiredPodSelectorLabelsBeforeAvailability(t *testing.T workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -1142,7 +1412,7 @@ func TestSessionRestoresRequiredPodSelectorLabelsBeforeAvailability(t *testing.T func TestWorkspaceDeletionRefusesForeignDeterministicPVC(t *testing.T) { for _, policy := range []clusterv1alpha1.RetentionPolicy{clusterv1alpha1.RetentionPolicyRetain, clusterv1alpha1.RetentionPolicyDelete} { - for _, mismatch := range []string{"uid-annotation", "controller-owner"} { + for _, mismatch := range []string{"uid-annotation", "controller-owner", "foreign-non-controller-owner"} { t.Run(string(policy)+"/"+mismatch, func(t *testing.T) { scheme := testScheme(t) workspace := testWorkspace(policy) @@ -1154,10 +1424,14 @@ func TestWorkspaceDeletionRefusesForeignDeterministicPVC(t *testing.T) { }} if mismatch == "uid-annotation" { pvc.Annotations[clusterv1alpha1.WorkspaceUIDAnnotation] = "foreign-workspace-uid" - } else { + } else if mismatch == "controller-owner" { pvc.OwnerReferences = []metav1.OwnerReference{{ APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Workspace", Name: "foreign", UID: "foreign-workspace-uid", Controller: ptr(true), }} + } else { + pvc.OwnerReferences = []metav1.OwnerReference{{ + APIVersion: "example.test/v1", Kind: "Foreign", Name: "foreign", UID: "foreign-uid", + }} } expectedOwnerCount := len(pvc.OwnerReferences) c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Workspace{}).WithObjects(workspace, pvc).Build() @@ -1207,6 +1481,14 @@ func TestSessionDeletionRefusesForeignDeterministicResources(t *testing.T) { service.OwnerReferences = nil } c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Session{}).WithObjects(session, pod, service).Build() + beforePod := &corev1.Pod{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(pod), beforePod); err != nil { + t.Fatal(err) + } + beforeService := &corev1.Service{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(service), beforeService); err != nil { + t.Fatal(err) + } if err := c.Delete(context.Background(), session); err != nil { t.Fatal(err) } @@ -1214,7 +1496,28 @@ func TestSessionDeletionRefusesForeignDeterministicResources(t *testing.T) { if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { t.Fatal(err) } - assertObjectCounts(t, c, 1, 1) + if foreignKind == "Pod" { + assertObjectCounts(t, c, 1, 0) + } else { + assertObjectCounts(t, c, 0, 1) + } + if foreignKind == "Pod" { + var got corev1.Pod + if err := c.Get(context.Background(), client.ObjectKeyFromObject(pod), &got); err != nil { + t.Fatalf("foreign Pod was deleted: %v", err) + } + if !reflect.DeepEqual(got.ObjectMeta, beforePod.ObjectMeta) || !reflect.DeepEqual(got.Spec, beforePod.Spec) { + t.Fatalf("foreign Pod was mutated during finalizer cleanup: %#v", got) + } + } else { + var got corev1.Service + if err := c.Get(context.Background(), client.ObjectKeyFromObject(service), &got); err != nil { + t.Fatalf("foreign Service was deleted: %v", err) + } + if !reflect.DeepEqual(got.ObjectMeta, beforeService.ObjectMeta) || !reflect.DeepEqual(got.Spec, beforeService.Spec) { + t.Fatalf("foreign Service was mutated during finalizer cleanup: %#v", got) + } + } var waiting clusterv1alpha1.T4Session if err := c.Get(context.Background(), client.ObjectKeyFromObject(session), &waiting); err != nil { t.Fatalf("session finalizer was removed on cleanup conflict: %v", err) @@ -1227,6 +1530,51 @@ func TestSessionDeletionRefusesForeignDeterministicResources(t *testing.T) { } } +func TestSessionDeletionUsesAuthoritativeChildReader(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + session := testSession() + session.UID = "session-uid" + session.Finalizers = []string{clusterv1alpha1.SessionFinalizer} + pod, service := ownedSessionResources(session) + pod.OwnerReferences = nil + base := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Session{}).WithObjects(session, pod, service).Build() + if err := base.Delete(ctx, session); err != nil { + t.Fatal(err) + } + c := &createAlreadyExistsClient{Client: base, raceKind: "Pod", winner: pod, hideWinnerFromCache: true} + r := configuredSessionReconciler(c, scheme) + r.APIReader = base + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + assertObjectCounts(t, base, 1, 0) + var waiting clusterv1alpha1.T4Session + if err := base.Get(ctx, client.ObjectKeyFromObject(session), &waiting); err != nil { + t.Fatalf("session finalizer ignored authoritative Pod conflict: %v", err) + } + condition := findCondition(waiting.Status.Conditions, "Available") + if condition == nil || condition.Reason != "CleanupOwnershipConflict" || !contains(waiting.Finalizers, clusterv1alpha1.SessionFinalizer) { + t.Fatalf("authoritative child conflict not retained: %#v", waiting) + } +} + +func TestSessionDependencyCleanupUsesAuthoritativeChildReader(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + session := testSession() + session.UID = "session-uid" + pod, service := ownedSessionResources(session) + base := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.Pod{}).WithObjects(session, pod, service).Build() + c := &createAlreadyExistsClient{Client: base, raceKind: "Pod", winner: pod, hideWinnerFromCache: true} + r := configuredSessionReconciler(c, scheme) + r.APIReader = base + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + assertObjectCounts(t, base, 0, 0) +} + func TestSessionDeletionCleansResourcesBeforeFinalizer(t *testing.T) { scheme := testScheme(t) session := testSession() @@ -1254,6 +1602,708 @@ func TestSessionDeletionCleansResourcesBeforeFinalizer(t *testing.T) { } } +func TestSessionDependencyRevocationCleansOwnedResourcesAndConvergesAfterRestart(t *testing.T) { + for _, test := range []struct { + name string + conditionType string + wantReason string + revoke func(context.Context, client.Client) error + }{ + {name: "missing Host", conditionType: "HostReady", wantReason: "HostNotFound", revoke: func(ctx context.Context, c client.Client) error { + return c.Delete(ctx, &clusterv1alpha1.T4ClusterHost{ObjectMeta: metav1.ObjectMeta{Name: "host-a", Namespace: "team"}}) + }}, + {name: "invalid Host runtime profile", conditionType: "RuntimeConfigured", wantReason: "RuntimeProfileNotAllowed", revoke: func(ctx context.Context, c client.Client) error { + var host clusterv1alpha1.T4ClusterHost + if err := c.Get(ctx, types.NamespacedName{Namespace: "team", Name: "host-a"}, &host); err != nil { + return err + } + host.Spec.RuntimeProfiles = nil + return c.Update(ctx, &host) + }}, + {name: "missing Workspace", conditionType: "WorkspaceReady", wantReason: "WorkspaceNotFound", revoke: func(ctx context.Context, c client.Client) error { + return c.Delete(ctx, &clusterv1alpha1.T4Workspace{ObjectMeta: metav1.ObjectMeta{Name: "workspace-a", Namespace: "team"}}) + }}, + {name: "mismatched Workspace Host", conditionType: "WorkspaceReady", wantReason: "HostMismatch", revoke: func(ctx context.Context, c client.Client) error { + var workspace clusterv1alpha1.T4Workspace + if err := c.Get(ctx, types.NamespacedName{Namespace: "team", Name: "workspace-a"}, &workspace); err != nil { + return err + } + workspace.Spec.HostRef = "host-b" + return c.Update(ctx, &workspace) + }}, + {name: "missing OMP ConfigMap", conditionType: "RuntimeConfigured", wantReason: "OMPConfigMapNotFound", revoke: func(ctx context.Context, c client.Client) error { + return c.Delete(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "omp-runtime-config", Namespace: "team"}}) + }}, + {name: "mismatched Host storage class", conditionType: "WorkspaceReady", wantReason: "StorageClassMismatch", revoke: func(ctx context.Context, c client.Client) error { + otherClass := &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: "other-rwx", Annotations: map[string]string{clusterv1alpha1.RWXStorageClassAnnotation: string(corev1.ReadWriteMany)}}, Provisioner: "example.invalid/csi"} + if err := c.Create(ctx, otherClass); err != nil { + return err + } + var host clusterv1alpha1.T4ClusterHost + if err := c.Get(ctx, types.NamespacedName{Namespace: "team", Name: "host-a"}, &host); err != nil { + return err + } + host.Spec.StorageClassName = otherClass.Name + return c.Update(ctx, &host) + }}, + {name: "classless Workspace PVC", conditionType: "WorkspaceReady", wantReason: "StorageClassMismatch", revoke: func(ctx context.Context, c client.Client) error { + var pvc corev1.PersistentVolumeClaim + if err := c.Get(ctx, types.NamespacedName{Namespace: "team", Name: "workspace-a-data"}, &pvc); err != nil { + return err + } + pvc.Spec.StorageClassName = nil + return c.Update(ctx, &pvc) + }}, + } { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.Status.PVCName = "workspace-a-data" + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } + session := testSession() + session.UID = "session-uid" + foreignPod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foreign-pod", Namespace: "team"}} + foreignService := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "foreign-service", Namespace: "team"}} + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(testHost(), workspace, pvc, session, foreignPod, foreignService).Build() + r := configuredSessionReconciler(c, scheme) + reconcileMany(t, 2, func() error { + _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + if err := test.revoke(ctx, c); err != nil { + t.Fatal(err) + } + + result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + if err != nil { + t.Fatal(err) + } + if result.RequeueAfter <= 0 || result.RequeueAfter > 30*time.Second { + t.Fatalf("revoked dependency requeue = %s, want bounded positive retry", result.RequeueAfter) + } + for _, object := range []client.Object{ + &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionPodName(session), Namespace: session.Namespace}}, + &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionServiceName(session), Namespace: session.Namespace}}, + } { + if err := c.Get(ctx, client.ObjectKeyFromObject(object), object); !apierrors.IsNotFound(err) { + t.Fatalf("owned stale %T remained after dependency revocation: %v", object, err) + } + } + for _, object := range []client.Object{foreignPod.DeepCopy(), foreignService.DeepCopy()} { + if err := c.Get(ctx, client.ObjectKeyFromObject(object), object); err != nil { + t.Fatalf("unowned %T was removed: %v", object, err) + } + } + + var failed clusterv1alpha1.T4Session + if err := c.Get(ctx, client.ObjectKeyFromObject(session), &failed); err != nil { + t.Fatal(err) + } + condition := findCondition(failed.Status.Conditions, test.conditionType) + available := findCondition(failed.Status.Conditions, "Available") + if failed.Status.ObservedGeneration != failed.Generation || failed.Status.PodName != "" || failed.Status.ServiceName != "" || failed.Status.Phase != clusterv1alpha1.InfrastructureFailed || + condition == nil || condition.Status != metav1.ConditionFalse || condition.Reason != test.wantReason || condition.ObservedGeneration != failed.Generation || + available == nil || available.Status != metav1.ConditionFalse || available.Reason != test.wantReason || available.ObservedGeneration != failed.Generation { + t.Fatalf("revoked session did not converge: status=%#v condition=%#v available=%#v", failed.Status, condition, available) + } + stableStatus := failed.Status + restarted := *r + reconcileMany(t, 2, func() error { + _, err := restarted.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + if err := c.Get(ctx, client.ObjectKeyFromObject(session), &failed); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(failed.Status, stableStatus) { + t.Fatalf("duplicate/restart reconciliation changed converged status: got %#v, want %#v", failed.Status, stableStatus) + } + }) + } +} + +func TestWorkspaceHostStorageClassDriftFailsClosedWithoutRecreatingPVC(t *testing.T) { + oldClass := "portable-rwx" + for _, test := range []struct { + name string + claimClass *string + }{ + {name: "different class", claimClass: &oldClass}, + {name: "class omitted"}, + } { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + workspace.Status.ObservedGeneration = workspace.Generation + workspace.Status.PVCName = controllers.WorkspacePVCName(workspace) + workspace.Status.Phase = clusterv1alpha1.InfrastructureReady + workspace.Status.Conditions = []metav1.Condition{ + {Type: "StorageReady", Status: metav1.ConditionTrue, Reason: controllers.ReasonStorageReady, ObservedGeneration: workspace.Generation}, + {Type: "Ready", Status: metav1.ConditionTrue, Reason: "PVCBound", ObservedGeneration: workspace.Generation}, + } + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: workspace.Status.PVCName, Namespace: workspace.Namespace, + Annotations: map[string]string{clusterv1alpha1.WorkspaceUIDAnnotation: string(workspace.UID)}, + OwnerReferences: []metav1.OwnerReference{{APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Workspace", Name: workspace.Name, UID: workspace.UID, Controller: ptr(true)}}, + }, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: test.claimClass, AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } + host := testHost() + host.Spec.StorageClassName = "other-rwx" + otherClass := &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: "other-rwx", Annotations: map[string]string{clusterv1alpha1.RWXStorageClassAnnotation: string(corev1.ReadWriteMany)}}, Provisioner: "example.invalid/csi"} + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Workspace{}, &corev1.PersistentVolumeClaim{}). + WithObjects(host, otherClass, workspace, pvc).Build() + r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} + + result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}) + if err != nil { + t.Fatal(err) + } + if result.RequeueAfter <= 0 || result.RequeueAfter > 30*time.Second { + t.Fatalf("storage drift requeue = %s, want bounded positive retry", result.RequeueAfter) + } + var got clusterv1alpha1.T4Workspace + if err := c.Get(ctx, client.ObjectKeyFromObject(workspace), &got); err != nil { + t.Fatal(err) + } + storageReady := findCondition(got.Status.Conditions, "StorageReady") + ready := findCondition(got.Status.Conditions, "Ready") + if got.Status.Phase != clusterv1alpha1.InfrastructureFailed || storageReady == nil || storageReady.Status != metav1.ConditionFalse || storageReady.Reason != "StorageClassMismatch" || ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != "StorageClassMismatch" { + t.Fatalf("storage drift remained Ready: status=%#v StorageReady=%#v Ready=%#v", got.Status, storageReady, ready) + } + var retained corev1.PersistentVolumeClaim + if err := c.Get(ctx, client.ObjectKeyFromObject(pvc), &retained); err != nil { + t.Fatalf("storage drift removed the data PVC: %v", err) + } + if !reflect.DeepEqual(retained.Spec.StorageClassName, test.claimClass) { + t.Fatalf("storage drift recreated or mutated PVC class: got %#v, want %#v", retained.Spec.StorageClassName, test.claimClass) + } + }) + } +} + +func TestHostDependencyRecoveryReplacesFalseConditions(t *testing.T) { + t.Run("Workspace", func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Workspace{}, &corev1.PersistentVolumeClaim{}). + WithObjects(workspace).Build() + r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + if err := c.Create(ctx, testHost()); err != nil { + t.Fatal(err) + } + if err := c.Create(ctx, rwxStorageClass()); err != nil { + t.Fatal(err) + } + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + var pvc corev1.PersistentVolumeClaim + if err := c.Get(ctx, types.NamespacedName{Namespace: workspace.Namespace, Name: controllers.WorkspacePVCName(workspace)}, &pvc); err != nil { + t.Fatal(err) + } + pvc.Status.Phase = corev1.ClaimBound + if err := c.Status().Update(ctx, &pvc); err != nil { + t.Fatal(err) + } + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + var recovered clusterv1alpha1.T4Workspace + if err := c.Get(ctx, client.ObjectKeyFromObject(workspace), &recovered); err != nil { + t.Fatal(err) + } + hostReady := findCondition(recovered.Status.Conditions, "HostReady") + if hostReady == nil || hostReady.Status != metav1.ConditionTrue || hostReady.ObservedGeneration != recovered.Generation { + t.Fatalf("recovered Workspace retained stale HostReady: %#v", hostReady) + } + }) + + t.Run("Session", func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.Status.PVCName = "workspace-a-data" + pvc := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: workspace.Namespace}, Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}} + session := testSession() + session.UID = "session-uid" + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(workspace, pvc, session).Build() + r := configuredSessionReconciler(c, scheme) + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + if err := c.Create(ctx, testHost()); err != nil { + t.Fatal(err) + } + reconcileMany(t, 2, func() error { + _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + var recovered clusterv1alpha1.T4Session + if err := c.Get(ctx, client.ObjectKeyFromObject(session), &recovered); err != nil { + t.Fatal(err) + } + hostReady := findCondition(recovered.Status.Conditions, "HostReady") + if hostReady == nil || hostReady.Status != metav1.ConditionTrue || hostReady.ObservedGeneration != recovered.Generation { + t.Fatalf("recovered Session retained stale HostReady: %#v", hostReady) + } + }) + + t.Run("static runtime failure", func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + session := testSession() + session.Status.ObservedGeneration = session.Generation + session.Status.Phase = clusterv1alpha1.InfrastructureFailed + session.Status.Conditions = []metav1.Condition{{Type: "HostReady", Status: metav1.ConditionFalse, Reason: "HostNotFound", ObservedGeneration: session.Generation}} + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}). + WithObjects(testHost(), session).Build() + r := configuredSessionReconciler(c, scheme) + r.RuntimeImage = "registry.example/session:latest" + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + var failed clusterv1alpha1.T4Session + if err := c.Get(ctx, client.ObjectKeyFromObject(session), &failed); err != nil { + t.Fatal(err) + } + hostReady := findCondition(failed.Status.Conditions, "HostReady") + if hostReady == nil || hostReady.Status != metav1.ConditionUnknown || hostReady.Reason != "NotEvaluated" || hostReady.ObservedGeneration != failed.Generation { + t.Fatalf("static runtime failure retained stale HostReady: %#v", hostReady) + } + }) +} + +func TestWorkspaceFailureRevokesPublishedPVCAuthorityAndRefreshesConditions(t *testing.T) { + for _, test := range []struct { + name string + objects func(*clusterv1alpha1.T4Workspace) []client.Object + wantHostStatus metav1.ConditionStatus + wantStorageReason string + }{ + { + name: "missing Host", + objects: func(workspace *clusterv1alpha1.T4Workspace) []client.Object { + return []client.Object{workspace} + }, + wantHostStatus: metav1.ConditionFalse, + wantStorageReason: "NotEvaluated", + }, + { + name: "missing StorageClass", + objects: func(workspace *clusterv1alpha1.T4Workspace) []client.Object { + return []client.Object{testHost(), workspace} + }, + wantHostStatus: metav1.ConditionTrue, + wantStorageReason: controllers.ReasonStorageClassNotFound, + }, + { + name: "non-RWX StorageClass", + objects: func(workspace *clusterv1alpha1.T4Workspace) []client.Object { + class := rwxStorageClass() + class.Annotations = nil + return []client.Object{testHost(), class, workspace} + }, + wantHostStatus: metav1.ConditionTrue, + wantStorageReason: controllers.ReasonStorageClassNotRWX, + }, + { + name: "PVC class drift", + objects: func(workspace *clusterv1alpha1.T4Workspace) []client.Object { + pvc := ownedWorkspacePVC(workspace) + pvc.Spec.StorageClassName = ptr("old-rwx") + return []client.Object{testHost(), rwxStorageClass(), workspace, pvc} + }, + wantHostStatus: metav1.ConditionTrue, + wantStorageReason: controllers.ReasonStorageClassMismatch, + }, + { + name: "PVC ownership conflict", + objects: func(workspace *clusterv1alpha1.T4Workspace) []client.Object { + pvc := ownedWorkspacePVC(workspace) + pvc.OwnerReferences = []metav1.OwnerReference{{APIVersion: "example.test/v1", Kind: "Foreign", Name: "foreign", UID: "foreign-uid", Controller: ptr(true)}} + return []client.Object{testHost(), rwxStorageClass(), workspace, pvc} + }, + wantHostStatus: metav1.ConditionTrue, + wantStorageReason: "PVCOwnershipConflict", + }, + } { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + workspace.Generation = 7 + workspace.Status.ObservedGeneration = 6 + workspace.Status.PVCName = controllers.WorkspacePVCName(workspace) + workspace.Status.PVCPhase = corev1.ClaimBound + workspace.Status.Capacity = apiresource.MustParse("10Gi") + workspace.Status.Phase = clusterv1alpha1.InfrastructureReady + workspace.Status.Conditions = []metav1.Condition{ + {Type: "HostReady", Status: metav1.ConditionTrue, Reason: "Stale", ObservedGeneration: 6}, + {Type: "StorageReady", Status: metav1.ConditionTrue, Reason: "Stale", ObservedGeneration: 6}, + {Type: "Ready", Status: metav1.ConditionTrue, Reason: "Stale", ObservedGeneration: 6}, + } + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Workspace{}, &corev1.PersistentVolumeClaim{}).WithObjects(test.objects(workspace)...).Build() + r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + var failed clusterv1alpha1.T4Workspace + if err := c.Get(ctx, client.ObjectKeyFromObject(workspace), &failed); err != nil { + t.Fatal(err) + } + if failed.Status.PVCName != "" || failed.Status.PVCPhase != "" || !failed.Status.Capacity.IsZero() { + t.Fatalf("failed Workspace retained PVC authority: %#v", failed.Status) + } + hostReady := findCondition(failed.Status.Conditions, "HostReady") + storageReady := findCondition(failed.Status.Conditions, "StorageReady") + ready := findCondition(failed.Status.Conditions, "Ready") + if hostReady == nil || hostReady.Status != test.wantHostStatus || hostReady.ObservedGeneration != failed.Generation || + storageReady == nil || storageReady.Status == metav1.ConditionTrue || storageReady.Reason != test.wantStorageReason || storageReady.ObservedGeneration != failed.Generation || + ready == nil || ready.Status != metav1.ConditionFalse || ready.ObservedGeneration != failed.Generation { + t.Fatalf("failure conditions are stale: HostReady=%#v StorageReady=%#v Ready=%#v", hostReady, storageReady, ready) + } + }) + } +} + +func TestSessionRejectsWorkspacePVCWithoutExactIdentityAndOwnership(t *testing.T) { + for _, test := range []struct { + name string + mutatePVC func(*clusterv1alpha1.T4Workspace, *corev1.PersistentVolumeClaim) + }{ + {name: "foreign deterministic PVC", mutatePVC: func(_ *clusterv1alpha1.T4Workspace, pvc *corev1.PersistentVolumeClaim) { + pvc.OwnerReferences = []metav1.OwnerReference{{APIVersion: "example.test/v1", Kind: "Foreign", Name: "foreign", UID: "foreign-uid", Controller: ptr(true)}} + }}, + {name: "tampered Workspace status name", mutatePVC: func(workspace *clusterv1alpha1.T4Workspace, pvc *corev1.PersistentVolumeClaim) { + workspace.Status.PVCName = "foreign-data" + pvc.Name = workspace.Status.PVCName + }}, + } { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + workspace.Status.PVCName = controllers.WorkspacePVCName(workspace) + pvc := ownedWorkspacePVC(workspace) + test.mutatePVC(workspace, pvc) + session := testSession() + session.UID = "session-uid" + pod, service := ownedSessionResources(session) + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(testHost(), workspace, pvc, session, pod, service).Build() + r := configuredSessionReconciler(c, scheme) + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + assertObjectCounts(t, c, 0, 0) + var failed clusterv1alpha1.T4Session + if err := c.Get(ctx, client.ObjectKeyFromObject(session), &failed); err != nil { + t.Fatal(err) + } + condition := findCondition(failed.Status.Conditions, "WorkspaceReady") + if failed.Status.PodName != "" || failed.Status.ServiceName != "" || condition == nil || condition.Status != metav1.ConditionFalse || condition.ObservedGeneration != failed.Generation { + t.Fatalf("untrusted Workspace PVC retained session authority: status=%#v WorkspaceReady=%#v", failed.Status, condition) + } + }) + } +} + +func TestSessionFailureAndPendingRefreshEveryCondition(t *testing.T) { + t.Run("failure", func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + session := testSession() + session.Generation = 8 + session.Status.ObservedGeneration = 7 + session.Status.Conditions = staleSessionConditions(7) + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Session{}).WithObjects(session).Build() + r := configuredSessionReconciler(c, scheme) + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + var failed clusterv1alpha1.T4Session + if err := c.Get(ctx, client.ObjectKeyFromObject(session), &failed); err != nil { + t.Fatal(err) + } + assertCurrentSessionConditions(t, &failed, map[string]metav1.ConditionStatus{ + "HostReady": metav1.ConditionFalse, "WorkspaceReady": metav1.ConditionUnknown, "RuntimeConfigured": metav1.ConditionUnknown, "Available": metav1.ConditionFalse, + }) + }) + + t.Run("pending", func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + workspace.Status.PVCName = controllers.WorkspacePVCName(workspace) + pvc := ownedWorkspacePVC(workspace) + session := testSession() + session.UID = "session-uid" + session.Generation = 8 + session.Status.ObservedGeneration = 7 + session.Status.Conditions = staleSessionConditions(7) + _, service := ownedSessionResources(session) + service.Spec.Type = corev1.ServiceTypeNodePort + service.Spec.Ports = []corev1.ServicePort{{Name: "host", Port: 8787, NodePort: 32080}} + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(testHost(), workspace, pvc, session, service).Build() + r := configuredSessionReconciler(c, scheme) + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + var pending clusterv1alpha1.T4Session + if err := c.Get(ctx, client.ObjectKeyFromObject(session), &pending); err != nil { + t.Fatal(err) + } + assertCurrentSessionConditions(t, &pending, map[string]metav1.ConditionStatus{ + "HostReady": metav1.ConditionTrue, "WorkspaceReady": metav1.ConditionTrue, "RuntimeConfigured": metav1.ConditionTrue, "Available": metav1.ConditionFalse, + }) + }) +} + +func TestSessionResourcesWithAnyForeignOwnerFailClosed(t *testing.T) { + for _, path := range []string{"normal", "dependency-cleanup"} { + for _, kind := range []string{"Pod", "Service"} { + t.Run(path+"/"+kind, func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + session := testSession() + session.UID = "session-uid" + pod, service := ownedSessionResources(session) + foreignOwner := metav1.OwnerReference{APIVersion: "example.test/v1", Kind: "Foreign", Name: "foreign", UID: "foreign-uid"} + if kind == "Pod" { + pod.OwnerReferences = append(pod.OwnerReferences, foreignOwner) + } else { + service.OwnerReferences = append(service.OwnerReferences, foreignOwner) + } + objects := []client.Object{session, pod, service} + if path == "normal" { + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + workspace.Status.PVCName = controllers.WorkspacePVCName(workspace) + objects = append(objects, testHost(), workspace, ownedWorkspacePVC(workspace)) + } + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(objects...).Build() + r := configuredSessionReconciler(c, scheme) + beforePod := pod.DeepCopy() + beforeService := service.DeepCopy() + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + if kind == "Pod" { + var got corev1.Pod + if err := c.Get(ctx, client.ObjectKeyFromObject(pod), &got); err != nil { + t.Fatalf("foreign-owned Pod was deleted: %v", err) + } + if !reflect.DeepEqual(got.ObjectMeta, beforePod.ObjectMeta) || !reflect.DeepEqual(got.Spec, beforePod.Spec) { + t.Fatalf("Pod with foreign OwnerReference was mutated: %#v", got) + } + } else { + var got corev1.Service + if err := c.Get(ctx, client.ObjectKeyFromObject(service), &got); err != nil { + t.Fatalf("foreign-owned Service was deleted: %v", err) + } + if !reflect.DeepEqual(got.ObjectMeta, beforeService.ObjectMeta) || !reflect.DeepEqual(got.Spec, beforeService.Spec) { + t.Fatalf("Service with foreign OwnerReference was mutated: %#v", got) + } + } + var sibling client.Object + if kind == "Pod" { + sibling = &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionServiceName(session), Namespace: session.Namespace}} + } else { + sibling = &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionPodName(session), Namespace: session.Namespace}} + } + if err := c.Get(ctx, client.ObjectKeyFromObject(sibling), sibling); !apierrors.IsNotFound(err) { + t.Fatalf("exclusively owned sibling remained after ownership conflict: %v", err) + } + var failed clusterv1alpha1.T4Session + if err := c.Get(ctx, client.ObjectKeyFromObject(session), &failed); err != nil { + t.Fatal(err) + } + available := findCondition(failed.Status.Conditions, "Available") + if available == nil || available.Status != metav1.ConditionFalse || !strings.Contains(available.Reason, "OwnershipConflict") { + t.Fatalf("foreign owner did not produce stable ownership conflict: %#v", available) + } + if path == "normal" { + assertCurrentSessionConditions(t, &failed, map[string]metav1.ConditionStatus{ + "HostReady": metav1.ConditionTrue, "WorkspaceReady": metav1.ConditionTrue, "RuntimeConfigured": metav1.ConditionTrue, "Available": metav1.ConditionFalse, + }) + } + }) + } + } +} + +func TestSessionCreateAlreadyExistsRefetchesForeignWinner(t *testing.T) { + for _, kind := range []string{"Pod", "Service"} { + t.Run(kind, func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + workspace.Status.PVCName = controllers.WorkspacePVCName(workspace) + session := testSession() + session.UID = "session-uid" + pod, service := ownedSessionResources(session) + objects := []client.Object{testHost(), workspace, ownedWorkspacePVC(workspace), session} + if kind == "Pod" { + objects = append(objects, service) + } else { + objects = append(objects, pod) + } + base := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(objects...).Build() + c := &createAlreadyExistsClient{Client: base, raceKind: kind, hideWinnerFromCache: true} + r := configuredSessionReconciler(c, scheme) + r.APIReader = base + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + if c.winner == nil { + t.Fatalf("%s create race was not exercised", kind) + } + if kind == "Pod" { + var got corev1.Pod + if err := base.Get(ctx, client.ObjectKeyFromObject(c.winner), &got); err != nil { + t.Fatalf("foreign Pod winner was deleted: %v", err) + } + want := c.winner.(*corev1.Pod) + if !reflect.DeepEqual(got.ObjectMeta, want.ObjectMeta) || !reflect.DeepEqual(got.Spec, want.Spec) { + t.Fatalf("foreign Pod winner was mutated: %#v", got) + } + assertObjectCounts(t, base, 1, 0) + } else { + var got corev1.Service + if err := base.Get(ctx, client.ObjectKeyFromObject(c.winner), &got); err != nil { + t.Fatalf("foreign Service winner was deleted: %v", err) + } + want := c.winner.(*corev1.Service) + if !reflect.DeepEqual(got.ObjectMeta, want.ObjectMeta) || !reflect.DeepEqual(got.Spec, want.Spec) { + t.Fatalf("foreign Service winner was mutated: %#v", got) + } + assertObjectCounts(t, base, 0, 1) + } + var failed clusterv1alpha1.T4Session + if err := base.Get(ctx, client.ObjectKeyFromObject(session), &failed); err != nil { + t.Fatal(err) + } + assertCurrentSessionConditions(t, &failed, map[string]metav1.ConditionStatus{ + "HostReady": metav1.ConditionTrue, "WorkspaceReady": metav1.ConditionTrue, "RuntimeConfigured": metav1.ConditionTrue, "Available": metav1.ConditionFalse, + }) + available := findCondition(failed.Status.Conditions, "Available") + if available.Reason != kind+"OwnershipConflict" { + t.Fatalf("Available reason = %q, want %sOwnershipConflict", available.Reason, kind) + } + }) + } +} + +func TestWorkspaceTerminalPVCFailuresRevokePublishedAuthority(t *testing.T) { + for _, test := range []struct { + name string + mutate func(*corev1.PersistentVolumeClaim) + reason string + }{ + {name: "Bound without RWX", reason: "PVCNotRWX", mutate: func(pvc *corev1.PersistentVolumeClaim) { pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} }}, + {name: "Lost", reason: "PVCLost", mutate: func(pvc *corev1.PersistentVolumeClaim) { pvc.Status.Phase = corev1.ClaimLost }}, + } { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + pvc := ownedWorkspacePVC(workspace) + test.mutate(pvc) + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Workspace{}, &corev1.PersistentVolumeClaim{}). + WithObjects(testHost(), rwxStorageClass(), workspace, pvc).Build() + r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + var failed clusterv1alpha1.T4Workspace + if err := c.Get(ctx, client.ObjectKeyFromObject(workspace), &failed); err != nil { + t.Fatal(err) + } + storageReady := findCondition(failed.Status.Conditions, "StorageReady") + ready := findCondition(failed.Status.Conditions, "Ready") + if failed.Status.PVCName != "" || failed.Status.PVCPhase != "" || !failed.Status.Capacity.IsZero() || + storageReady == nil || storageReady.Status != metav1.ConditionFalse || storageReady.Reason != test.reason || storageReady.ObservedGeneration != failed.Generation || + ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != test.reason || ready.ObservedGeneration != failed.Generation { + t.Fatalf("terminal PVC failure retained authority: status=%#v StorageReady=%#v Ready=%#v", failed.Status, storageReady, ready) + } + }) + } +} + + +func ownedWorkspacePVC(workspace *clusterv1alpha1.T4Workspace) *corev1.PersistentVolumeClaim { + return &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: controllers.WorkspacePVCName(workspace), Namespace: workspace.Namespace, + Annotations: map[string]string{clusterv1alpha1.WorkspaceUIDAnnotation: string(workspace.UID)}, + OwnerReferences: []metav1.OwnerReference{{APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Workspace", Name: workspace.Name, UID: workspace.UID, Controller: ptr(true)}}, + }, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } +} + +func ownedSessionResources(session *clusterv1alpha1.T4Session) (*corev1.Pod, *corev1.Service) { + owner := metav1.OwnerReference{APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Session", Name: session.Name, UID: session.UID, Controller: ptr(true)} + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionPodName(session), Namespace: session.Namespace, OwnerReferences: []metav1.OwnerReference{owner}}} + service := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionServiceName(session), Namespace: session.Namespace, OwnerReferences: []metav1.OwnerReference{owner}}, Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeClusterIP}} + return pod, service +} + +func staleSessionConditions(generation int64) []metav1.Condition { + return []metav1.Condition{ + {Type: "HostReady", Status: metav1.ConditionTrue, Reason: "Stale", ObservedGeneration: generation}, + {Type: "WorkspaceReady", Status: metav1.ConditionTrue, Reason: "Stale", ObservedGeneration: generation}, + {Type: "RuntimeConfigured", Status: metav1.ConditionTrue, Reason: "Stale", ObservedGeneration: generation}, + {Type: "Available", Status: metav1.ConditionTrue, Reason: "Stale", ObservedGeneration: generation}, + } +} + +func assertCurrentSessionConditions(t *testing.T, session *clusterv1alpha1.T4Session, want map[string]metav1.ConditionStatus) { + t.Helper() + for conditionType, status := range want { + condition := findCondition(session.Status.Conditions, conditionType) + if condition == nil || condition.Status != status || condition.ObservedGeneration != session.Generation { + t.Fatalf("%s = %#v, want %s at generation %d", conditionType, condition, status, session.Generation) + } + } +} + func configuredSessionReconciler(c client.Client, scheme *runtime.Scheme) *controllers.SessionReconciler { for _, object := range []client.Object{ &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "omp-runtime-config", Namespace: "team"}, Data: map[string]string{ @@ -1383,4 +2433,71 @@ func hasReadOnlyMount(mounts []corev1.VolumeMount, name, path string) bool { return false } +type pvcOverrideReader struct { + client.Reader + pvc *corev1.PersistentVolumeClaim +} + +func (r *pvcOverrideReader) Get(ctx context.Context, key client.ObjectKey, object client.Object, options ...client.GetOption) error { + if pvc, ok := object.(*corev1.PersistentVolumeClaim); ok && key == client.ObjectKeyFromObject(r.pvc) { + r.pvc.DeepCopyInto(pvc) + return nil + } + return r.Reader.Get(ctx, key, object, options...) +} + +type createAlreadyExistsClient struct { + client.Client + raceKind string + winner client.Object + hideWinnerFromCache bool +} + +func (c *createAlreadyExistsClient) Get(ctx context.Context, key client.ObjectKey, object client.Object, options ...client.GetOption) error { + if c.hideWinnerFromCache && c.winner != nil && key == client.ObjectKeyFromObject(c.winner) { + switch object.(type) { + case *corev1.Pod: + if c.raceKind == "Pod" { + return apierrors.NewNotFound(schema.GroupResource{Resource: "pods"}, key.Name) + } + case *corev1.Service: + if c.raceKind == "Service" { + return apierrors.NewNotFound(schema.GroupResource{Resource: "services"}, key.Name) + } + case *corev1.PersistentVolumeClaim: + if c.raceKind == "PVC" { + return apierrors.NewNotFound(schema.GroupResource{Resource: "persistentvolumeclaims"}, key.Name) + } + } + } + return c.Client.Get(ctx, key, object, options...) +} + +func (c *createAlreadyExistsClient) Create(ctx context.Context, object client.Object, options ...client.CreateOption) error { + var winner client.Object + switch object := object.(type) { + case *corev1.Pod: + if c.raceKind == "Pod" { + winner = object.DeepCopy() + } + case *corev1.Service: + if c.raceKind == "Service" { + winner = object.DeepCopy() + } + case *corev1.PersistentVolumeClaim: + if c.raceKind == "PVC" { + winner = object.DeepCopy() + } + } + if winner == nil || c.winner != nil { + return c.Client.Create(ctx, object, options...) + } + winner.SetOwnerReferences(nil) + if err := c.Client.Create(ctx, winner, options...); err != nil { + return err + } + c.winner = winner.DeepCopyObject().(client.Object) + return apierrors.NewAlreadyExists(schema.GroupResource{Resource: strings.ToLower(c.raceKind) + "s"}, object.GetName()) +} + func ptr[T any](value T) *T { return &value } diff --git a/packages/cluster-operator/controllers/session_controller.go b/packages/cluster-operator/controllers/session_controller.go index 9336a3f4..345dbeb0 100644 --- a/packages/cluster-operator/controllers/session_controller.go +++ b/packages/cluster-operator/controllers/session_controller.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "encoding/json" + "errors" "fmt" "net/url" "reflect" @@ -25,6 +26,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" clusterv1alpha1 "github.com/LycaonLLC/t4-code/packages/cluster-operator/api/v1alpha1" ) @@ -36,6 +38,13 @@ const ( SessionReviewerTokenExpirationSeconds int64 = 3600 ) +const ( + sessionHostRefIndexField = "t4.session.spec.hostRef" + sessionWorkspaceRefIndexField = "t4.session.spec.workspaceRef" +) + +var errSessionResourceOwnershipConflict = errors.New("session resource ownership conflict") + var ( configMapKeyPattern = regexp.MustCompile(`^[-._A-Za-z0-9]+$`) runtimeImagePattern = regexp.MustCompile(`^(?:(?:[A-Za-z0-9](?:[A-Za-z0-9.-]*[A-Za-z0-9])?|\[[A-Fa-f0-9:]+\])(?::[0-9]+)?/)?[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*(?:/[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*)*@sha256:[a-f0-9]{64}$`) @@ -283,6 +292,10 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) var session clusterv1alpha1.T4Session found := false defer func() { + if errors.Is(err, errSessionResourceOwnershipConflict) { + result = ctrl.Result{RequeueAfter: 30 * time.Second} + err = nil + } observeReconcile(metricKindSession, request.NamespacedName, session.Status.Conditions, conditionObjectPresent(&session, found, err), err) }() if err := r.Get(ctx, request.NamespacedName, &session); err != nil { @@ -301,19 +314,22 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "RuntimeConfigured", reason, message) + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, false, false, "RuntimeConfigured", reason, message) } if reason, message := r.OMPConfig.validationFailure(); reason != "" { if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "RuntimeConfigured", reason, message) + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, false, false, "RuntimeConfigured", reason, message) } var host clusterv1alpha1.T4ClusterHost if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: session.Spec.HostRef}, &host); err != nil { if apierrors.IsNotFound(err) { - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "HostReady", "HostNotFound", "referenced T4ClusterHost does not exist") + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, false, false, "HostReady", "HostNotFound", "referenced T4ClusterHost does not exist") } return ctrl.Result{}, err } @@ -321,7 +337,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "RuntimeConfigured", "RuntimeProfileNotAllowed", "runtime profile is not allowed by the referenced T4ClusterHost") + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "RuntimeConfigured", "RuntimeProfileNotAllowed", "runtime profile is not allowed by the referenced T4ClusterHost") } var storageClass storagev1.StorageClass if err := r.Get(ctx, types.NamespacedName{Name: host.Spec.StorageClassName}, &storageClass); err != nil { @@ -329,7 +345,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", ReasonStorageClassNotFound, fmt.Sprintf("StorageClass %q selected by the referenced T4ClusterHost does not exist", host.Spec.StorageClassName)) + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", ReasonStorageClassNotFound, fmt.Sprintf("StorageClass %q selected by the referenced T4ClusterHost does not exist", host.Spec.StorageClassName)) } return ctrl.Result{}, err } @@ -337,30 +353,63 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", ReasonStorageClassNotRWX, fmt.Sprintf("StorageClass %q selected by the referenced T4ClusterHost is not administrator-declared ReadWriteMany", host.Spec.StorageClassName)) + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", ReasonStorageClassNotRWX, fmt.Sprintf("StorageClass %q selected by the referenced T4ClusterHost is not administrator-declared ReadWriteMany", host.Spec.StorageClassName)) } var workspace clusterv1alpha1.T4Workspace if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: session.Spec.WorkspaceRef}, &workspace); err != nil { if apierrors.IsNotFound(err) { - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "WorkspaceNotFound", "referenced T4Workspace does not exist") + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", "WorkspaceNotFound", "referenced T4Workspace does not exist") } return ctrl.Result{}, err } if workspace.Spec.HostRef != session.Spec.HostRef { - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "HostMismatch", "session and workspace must reference the same T4ClusterHost") + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", "HostMismatch", "session and workspace must reference the same T4ClusterHost") } if workspace.Status.PVCName == "" { - return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "PVCNotDeclared", "workspace controller has not declared a PVC") + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", "PVCNotDeclared", "workspace controller has not declared a PVC") + } + if workspace.UID != "" && workspace.Status.PVCName != WorkspacePVCName(&workspace) { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", "PVCIdentityMismatch", "workspace status does not reference its deterministic PVC") } var pvc corev1.PersistentVolumeClaim if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: workspace.Status.PVCName}, &pvc); err != nil { if apierrors.IsNotFound(err) { - return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "PVCNotFound", "workspace PVC does not exist") + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", "PVCNotFound", "workspace PVC does not exist") } return ctrl.Result{}, err } + if workspace.UID != "" && !workspaceOwnsPVC(&workspace, &pvc) { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", "PVCOwnershipConflict", "workspace PVC identity or ownership is not authoritative") + } + if pvcStorageClassName(&pvc) != host.Spec.StorageClassName { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", ReasonStorageClassMismatch, fmt.Sprintf("workspace PVC uses StorageClass %q instead of host-selected %q", pvcStorageClassName(&pvc), host.Spec.StorageClassName)) + } if pvc.Status.Phase != corev1.ClaimBound || !pvcHasRWX(&pvc) { - return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "PVCNotBoundRWX", "workspace PVC must be Bound and ReadWriteMany before a session starts") + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", "PVCNotBoundRWX", "workspace PVC must be Bound and ReadWriteMany before a session starts") } runtimeVersions, reason, message, err := r.loadOMPResourceVersions(ctx, session.Namespace) if err != nil { @@ -370,9 +419,20 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "RuntimeConfigured", reason, message) + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, true, "RuntimeConfigured", reason, message) + } + reason, message, err = r.authoritativePVCValidation(ctx, &workspace, &pvc, host.Spec.StorageClassName) + if err != nil { + return ctrl.Result{}, err + } + if reason != "" { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", reason, message) } + serviceName := SessionServiceName(&session) podName := SessionPodName(&session) labels := map[string]string{ @@ -392,17 +452,31 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) return ctrl.Result{}, err } var service corev1.Service - if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: serviceName}, &service); apierrors.IsNotFound(err) { + serviceKey := types.NamespacedName{Namespace: session.Namespace, Name: serviceName} + if err := r.Get(ctx, serviceKey, &service); apierrors.IsNotFound(err) { service = desiredService - if err := r.Create(ctx, &service); err != nil && !apierrors.IsAlreadyExists(err) { - return ctrl.Result{}, err + if err := r.Create(ctx, &service); err != nil { + if !apierrors.IsAlreadyExists(err) { + return ctrl.Result{}, err + } + reader := r.APIReader + if reader == nil { + reader = r.Client + } + if err := reader.Get(ctx, serviceKey, &service); err != nil { + return ctrl.Result{}, err + } } } else if err != nil { return ctrl.Result{}, err - } else if !metav1.IsControlledBy(&service, &session) { - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "Available", "ServiceOwnershipConflict", "deterministic session Service is not controlled by this session") + } + if !sessionExclusivelyOwnsResource(&service, &session) { + if err := r.deleteOwnedSessionResourcesAfterVerifiedDependencies(ctx, &session, "ServiceOwnershipConflict", "deterministic session Service has an unexpected owner"); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } else if !serviceExposureIsInternal(&service) { - if err := r.Delete(ctx, &service); err != nil && !apierrors.IsNotFound(err) { + if err := deleteWithPreconditions(ctx, r.Client, &service); err != nil && !apierrors.IsNotFound(err) { return ctrl.Result{}, err } if err := r.updateSessionPending(ctx, &session, podName, serviceName, "ServiceExposureChanged", "session Service is being recreated with ClusterIP-only exposure"); err != nil { @@ -426,15 +500,39 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) return ctrl.Result{}, err } var pod corev1.Pod - if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: podName}, &pod); apierrors.IsNotFound(err) { - pod = desiredPod - if err := r.Create(ctx, &pod); err != nil && !apierrors.IsAlreadyExists(err) { + podKey := types.NamespacedName{Namespace: session.Namespace, Name: podName} + if err := r.Get(ctx, podKey, &pod); apierrors.IsNotFound(err) { + reason, message, err := r.authoritativePVCValidation(ctx, &workspace, &pvc, host.Spec.StorageClassName) + if err != nil { return ctrl.Result{}, err } + if reason != "" { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", reason, message) + } + pod = desiredPod + if err := r.Create(ctx, &pod); err != nil { + if !apierrors.IsAlreadyExists(err) { + return ctrl.Result{}, err + } + reader := r.APIReader + if reader == nil { + reader = r.Client + } + if err := reader.Get(ctx, podKey, &pod); err != nil { + return ctrl.Result{}, err + } + } } else if err != nil { return ctrl.Result{}, err - } else if !metav1.IsControlledBy(&pod, &session) { - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "Available", "PodOwnershipConflict", "deterministic session Pod is not controlled by this session") + } + if !sessionExclusivelyOwnsResource(&pod, &session) { + if err := r.deleteOwnedSessionResourcesAfterVerifiedDependencies(ctx, &session, "PodOwnershipConflict", "deterministic session Pod has an unexpected owner"); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } else if !labelsContain(pod.Labels, desiredPod.Labels) { if pod.Labels == nil { pod.Labels = map[string]string{} @@ -450,7 +548,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) } return ctrl.Result{RequeueAfter: time.Second}, nil } else if pod.Annotations[clusterv1alpha1.SessionPodSpecHashAnnotation] != desiredPod.Annotations[clusterv1alpha1.SessionPodSpecHashAnnotation] { - if err := r.Delete(ctx, &pod); err != nil && !apierrors.IsNotFound(err) { + if err := deleteWithPreconditions(ctx, r.Client, &pod); err != nil && !apierrors.IsNotFound(err) { return ctrl.Result{}, err } if err := r.updateSessionPending(ctx, &session, podName, serviceName, "PodSpecChanged", "session Pod is being recreated to apply immutable desired state"); err != nil { @@ -459,6 +557,17 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) return ctrl.Result{RequeueAfter: time.Second}, nil } + reason, message, err = r.authoritativePVCValidation(ctx, &workspace, &pvc, host.Spec.StorageClassName) + if err != nil { + return ctrl.Result{}, err + } + if reason != "" { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", reason, message) + } + original := session.Status if session.Status.Conditions != nil { original.Conditions = append([]metav1.Condition(nil), session.Status.Conditions...) @@ -466,6 +575,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) session.Status.ObservedGeneration = session.Generation session.Status.PodName = podName session.Status.ServiceName = serviceName + meta.SetStatusCondition(&session.Status.Conditions, condition("HostReady", metav1.ConditionTrue, "HostResolved", "referenced T4ClusterHost is available", session.Generation)) meta.SetStatusCondition(&session.Status.Conditions, condition("WorkspaceReady", metav1.ConditionTrue, "PVCBoundRWX", "workspace PVC is Bound and ReadWriteMany", session.Generation)) meta.SetStatusCondition(&session.Status.Conditions, condition("RuntimeConfigured", metav1.ConditionTrue, "OMPReferencesReady", "administrator-owned OMP runtime references are configured", session.Generation)) if podReady(&pod) { @@ -489,6 +599,36 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } +func (r *SessionReconciler) authoritativePVCValidation(ctx context.Context, workspace *clusterv1alpha1.T4Workspace, cachedPVC *corev1.PersistentVolumeClaim, storageClassName string) (string, string, error) { + reader := r.APIReader + if reader == nil { + reader = r.Client + } + var authoritativePVC corev1.PersistentVolumeClaim + if err := reader.Get(ctx, client.ObjectKeyFromObject(cachedPVC), &authoritativePVC); err != nil { + if apierrors.IsNotFound(err) { + return "PVCAuthorityChanged", "workspace PVC does not exist in authoritative API state", nil + } + return "", "", err + } + if authoritativePVC.UID != cachedPVC.UID { + return "PVCAuthorityChanged", "authoritative workspace PVC UID differs from the validated cached PVC", nil + } + if workspace.UID != "" && !workspaceOwnsPVC(workspace, &authoritativePVC) { + return "PVCAuthorityChanged", "authoritative workspace PVC owner reference does not belong to the workspace", nil + } + if pvcStorageClassName(&authoritativePVC) != storageClassName { + return "PVCAuthorityChanged", fmt.Sprintf("authoritative workspace PVC uses StorageClass %q instead of host-selected %q", pvcStorageClassName(&authoritativePVC), storageClassName), nil + } + if !pvcHasRWX(&authoritativePVC) { + return "PVCAuthorityChanged", "authoritative workspace PVC does not request ReadWriteMany", nil + } + if authoritativePVC.Status.Phase != corev1.ClaimBound { + return "PVCAuthorityChanged", "authoritative workspace PVC is not Bound", nil + } + return "", "", nil +} + func (r *SessionReconciler) desiredPod(session *clusterv1alpha1.T4Session, pvcName, podName string, labels map[string]string, runtimeVersions ompResourceVersions) (corev1.Pod, error) { falseValue := false trueValue := true @@ -622,6 +762,9 @@ func (r *SessionReconciler) updateSessionPending(ctx context.Context, session *c original.Conditions = append([]metav1.Condition(nil), session.Status.Conditions...) } session.Status.ObservedGeneration = session.Generation + meta.SetStatusCondition(&session.Status.Conditions, condition("HostReady", metav1.ConditionTrue, "HostResolved", "referenced T4ClusterHost is available", session.Generation)) + meta.SetStatusCondition(&session.Status.Conditions, condition("WorkspaceReady", metav1.ConditionTrue, "PVCBoundRWX", "workspace PVC is Bound and ReadWriteMany", session.Generation)) + meta.SetStatusCondition(&session.Status.Conditions, condition("RuntimeConfigured", metav1.ConditionTrue, "OMPReferencesReady", "administrator-owned OMP runtime references are configured", session.Generation)) session.Status.PodName = podName session.Status.ServiceName = serviceName session.Status.Phase = clusterv1alpha1.InfrastructurePending @@ -653,35 +796,46 @@ func (r *SessionReconciler) reconcileDelete(ctx context.Context, session *cluste &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: SessionServiceName(session), Namespace: session.Namespace}}, } existing := make([]client.Object, 0, len(objects)) + var ownershipConflict client.Object + reader := r.APIReader + if reader == nil { + reader = r.Client + } for _, object := range objects { - err := r.Get(ctx, client.ObjectKeyFromObject(object), object) + err := reader.Get(ctx, client.ObjectKeyFromObject(object), object) if apierrors.IsNotFound(err) { continue } if err != nil { return ctrl.Result{}, err } - if !metav1.IsControlledBy(object, session) { - before := session.Status - if session.Status.Conditions != nil { - before.Conditions = append([]metav1.Condition(nil), session.Status.Conditions...) - } - meta.SetStatusCondition(&session.Status.Conditions, condition("Available", metav1.ConditionFalse, "CleanupOwnershipConflict", fmt.Sprintf("deterministic %T is not controlled by this session", object), session.Generation)) - if !reflect.DeepEqual(before, session.Status) { - if err := r.Status().Update(ctx, session); err != nil { - return ctrl.Result{}, err - } + if !sessionExclusivelyOwnsResource(object, session) { + if ownershipConflict == nil { + ownershipConflict = object } - return ctrl.Result{RequeueAfter: 30 * time.Second}, nil + continue } existing = append(existing, object) } for _, object := range existing { if object.GetDeletionTimestamp().IsZero() { - if err := r.Delete(ctx, object); err != nil && !apierrors.IsNotFound(err) { + if err := deleteWithPreconditions(ctx, r.Client, object); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + } + } + if ownershipConflict != nil { + before := session.Status + if session.Status.Conditions != nil { + before.Conditions = append([]metav1.Condition(nil), session.Status.Conditions...) + } + meta.SetStatusCondition(&session.Status.Conditions, condition("Available", metav1.ConditionFalse, "CleanupOwnershipConflict", fmt.Sprintf("deterministic %T is not controlled by this session", ownershipConflict), session.Generation)) + if !reflect.DeepEqual(before, session.Status) { + if err := r.Status().Update(ctx, session); err != nil { return ctrl.Result{}, err } } + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } if len(existing) > 0 { return ctrl.Result{RequeueAfter: time.Second}, nil @@ -691,46 +845,122 @@ func (r *SessionReconciler) reconcileDelete(ctx context.Context, session *cluste } func (r *SessionReconciler) deleteOwnedSessionResources(ctx context.Context, session *clusterv1alpha1.T4Session) error { + reader := r.APIReader + if reader == nil { + reader = r.Client + } + return r.deleteOwnedSessionResourcesWithFailure(ctx, reader, session, true, false, false, "ResourceOwnershipConflict", "one or more deterministic session resources have an unexpected owner") +} + +func (r *SessionReconciler) deleteOwnedSessionResourcesAfterVerifiedDependencies(ctx context.Context, session *clusterv1alpha1.T4Session, reason, message string) error { + reader := r.APIReader + if reader == nil { + reader = r.Client + } + return r.deleteOwnedSessionResourcesWithFailure(ctx, reader, session, false, true, true, reason, message) +} + +func (r *SessionReconciler) deleteOwnedSessionResourcesWithFailure(ctx context.Context, reader client.Reader, session *clusterv1alpha1.T4Session, deleteWithoutConflict, hostReady, workspaceReady bool, reason, message string) error { objects := []client.Object{ &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: SessionPodName(session), Namespace: session.Namespace}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: SessionServiceName(session), Namespace: session.Namespace}}, } + owned := make([]client.Object, 0, len(objects)) + ownershipConflict := false for _, object := range objects { - if err := r.Get(ctx, client.ObjectKeyFromObject(object), object); err != nil { + if err := reader.Get(ctx, client.ObjectKeyFromObject(object), object); err != nil { if err := client.IgnoreNotFound(err); err != nil { return err } continue } - if !metav1.IsControlledBy(object, session) { + if !sessionExclusivelyOwnsResource(object, session) { + ownershipConflict = true continue } - if err := r.Delete(ctx, object); err != nil && !apierrors.IsNotFound(err) { + owned = append(owned, object) + } + if ownershipConflict || deleteWithoutConflict { + for _, object := range owned { + if err := deleteWithPreconditions(ctx, r.Client, object); err != nil && !apierrors.IsNotFound(err) { + return err + } + } + } + if ownershipConflict { + if err := r.updateSessionFailure(ctx, session, hostReady, workspaceReady, "Available", reason, message); err != nil { return err } + return errSessionResourceOwnershipConflict } return nil } -func (r *SessionReconciler) updateSessionFailure(ctx context.Context, session *clusterv1alpha1.T4Session, conditionType, reason, message string) error { +func deleteWithPreconditions(ctx context.Context, writer client.Client, object client.Object) error { + preconditions := metav1.Preconditions{} + if uid := object.GetUID(); uid != "" { + preconditions.UID = &uid + } + if resourceVersion := object.GetResourceVersion(); resourceVersion != "" { + preconditions.ResourceVersion = &resourceVersion + } + options := &client.DeleteOptions{} + if preconditions.UID != nil || preconditions.ResourceVersion != nil { + options.Preconditions = &preconditions + } + return writer.Delete(ctx, object, options) +} + +func (r *SessionReconciler) updateSessionFailure(ctx context.Context, session *clusterv1alpha1.T4Session, hostReady, workspaceReady bool, conditionType, reason, message string) error { original := session.Status if session.Status.Conditions != nil { original.Conditions = append([]metav1.Condition(nil), session.Status.Conditions...) } + if conditionType == "HostReady" { + meta.SetStatusCondition(&session.Status.Conditions, condition("HostReady", metav1.ConditionFalse, reason, message, session.Generation)) + } else if hostReady { + meta.SetStatusCondition(&session.Status.Conditions, condition("HostReady", metav1.ConditionTrue, "HostResolved", "referenced T4ClusterHost is available", session.Generation)) + } else { + meta.SetStatusCondition(&session.Status.Conditions, condition("HostReady", metav1.ConditionUnknown, "NotEvaluated", "host dependency was not evaluated", session.Generation)) + } + if conditionType == "WorkspaceReady" { + meta.SetStatusCondition(&session.Status.Conditions, condition("WorkspaceReady", metav1.ConditionFalse, reason, message, session.Generation)) + } else if workspaceReady { + meta.SetStatusCondition(&session.Status.Conditions, condition("WorkspaceReady", metav1.ConditionTrue, "PVCBoundRWX", "workspace PVC is Bound and ReadWriteMany", session.Generation)) + } else { + meta.SetStatusCondition(&session.Status.Conditions, condition("WorkspaceReady", metav1.ConditionUnknown, "NotEvaluated", "workspace dependency was not evaluated", session.Generation)) + } + if conditionType == "RuntimeConfigured" { + meta.SetStatusCondition(&session.Status.Conditions, condition("RuntimeConfigured", metav1.ConditionFalse, reason, message, session.Generation)) + } else if workspaceReady { + meta.SetStatusCondition(&session.Status.Conditions, condition("RuntimeConfigured", metav1.ConditionTrue, "OMPReferencesReady", "administrator-owned OMP runtime references are configured", session.Generation)) + } else { + meta.SetStatusCondition(&session.Status.Conditions, condition("RuntimeConfigured", metav1.ConditionUnknown, "NotEvaluated", "runtime configuration was not evaluated", session.Generation)) + } session.Status.ObservedGeneration = session.Generation session.Status.PodName = "" session.Status.ServiceName = "" session.Status.Phase = clusterv1alpha1.InfrastructureFailed - meta.SetStatusCondition(&session.Status.Conditions, condition(conditionType, metav1.ConditionFalse, reason, message, session.Generation)) - if conditionType != "Available" { - meta.SetStatusCondition(&session.Status.Conditions, condition("Available", metav1.ConditionFalse, reason, message, session.Generation)) - } + meta.SetStatusCondition(&session.Status.Conditions, condition("Available", metav1.ConditionFalse, reason, message, session.Generation)) if reflect.DeepEqual(original, session.Status) { return nil } return r.Status().Update(ctx, session) } +func sessionExclusivelyOwnsResource(object metav1.Object, session *clusterv1alpha1.T4Session) bool { + controller := metav1.GetControllerOf(object) + if controller == nil || controller.APIVersion != clusterv1alpha1.GroupVersion.String() || controller.Kind != "T4Session" || controller.Name != session.Name || controller.UID != session.UID { + return false + } + for _, reference := range object.GetOwnerReferences() { + if reference.APIVersion != clusterv1alpha1.GroupVersion.String() || reference.Kind != "T4Session" || reference.Name != session.Name || reference.UID != session.UID { + return false + } + } + return true +} + func serviceExposureIsInternal(service *corev1.Service) bool { if service.Spec.Type != corev1.ServiceTypeClusterIP || service.Spec.ClusterIP == corev1.ClusterIPNone || service.Spec.ExternalName != "" || len(service.Spec.ExternalIPs) != 0 || service.Spec.LoadBalancerIP != "" || len(service.Spec.LoadBalancerSourceRanges) != 0 || @@ -754,9 +984,62 @@ func labelsContain(actual, required map[string]string) bool { return true } +func indexSessionByHostRef(object client.Object) []string { + session, ok := object.(*clusterv1alpha1.T4Session) + if !ok || session.Spec.HostRef == "" { + return nil + } + return []string{session.Spec.HostRef} +} + +func indexSessionByWorkspaceRef(object client.Object) []string { + session, ok := object.(*clusterv1alpha1.T4Session) + if !ok || session.Spec.WorkspaceRef == "" { + return nil + } + return []string{session.Spec.WorkspaceRef} +} + +func (r *SessionReconciler) sessionRequestsForHost(ctx context.Context, object client.Object) []ctrl.Request { + host, ok := object.(*clusterv1alpha1.T4ClusterHost) + if !ok || host.Name == "" || host.Namespace == "" { + return nil + } + return r.sessionRequestsForReference(ctx, host.Namespace, sessionHostRefIndexField, host.Name, "clusterHost", client.ObjectKeyFromObject(host)) +} + +func (r *SessionReconciler) sessionRequestsForWorkspace(ctx context.Context, object client.Object) []ctrl.Request { + workspace, ok := object.(*clusterv1alpha1.T4Workspace) + if !ok || workspace.Name == "" || workspace.Namespace == "" { + return nil + } + return r.sessionRequestsForReference(ctx, workspace.Namespace, sessionWorkspaceRefIndexField, workspace.Name, "workspace", client.ObjectKeyFromObject(workspace)) +} + +func (r *SessionReconciler) sessionRequestsForReference(ctx context.Context, namespace, field, value, dependencyKind string, dependencyKey types.NamespacedName) []ctrl.Request { + var sessions clusterv1alpha1.T4SessionList + if err := r.List(ctx, &sessions, client.InNamespace(namespace), client.MatchingFields{field: value}); err != nil { + ctrl.LoggerFrom(ctx).Error(err, "unable to map dependency to sessions", dependencyKind, dependencyKey) + return nil + } + requests := make([]ctrl.Request, 0, len(sessions.Items)) + for i := range sessions.Items { + requests = append(requests, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(&sessions.Items[i])}) + } + return requests +} + func (r *SessionReconciler) SetupWithManager(manager ctrl.Manager) error { + if err := manager.GetFieldIndexer().IndexField(context.Background(), &clusterv1alpha1.T4Session{}, sessionHostRefIndexField, indexSessionByHostRef); err != nil { + return fmt.Errorf("index T4Session by host reference: %w", err) + } + if err := manager.GetFieldIndexer().IndexField(context.Background(), &clusterv1alpha1.T4Session{}, sessionWorkspaceRefIndexField, indexSessionByWorkspaceRef); err != nil { + return fmt.Errorf("index T4Session by workspace reference: %w", err) + } return ctrl.NewControllerManagedBy(manager). For(&clusterv1alpha1.T4Session{}). + Watches(&clusterv1alpha1.T4ClusterHost{}, handler.EnqueueRequestsFromMapFunc(r.sessionRequestsForHost)). + Watches(&clusterv1alpha1.T4Workspace{}, handler.EnqueueRequestsFromMapFunc(r.sessionRequestsForWorkspace)). Owns(&corev1.Pod{}). Owns(&corev1.Service{}). Complete(r) diff --git a/packages/cluster-operator/controllers/session_controller_test.go b/packages/cluster-operator/controllers/session_controller_test.go new file mode 100644 index 00000000..cba15fef --- /dev/null +++ b/packages/cluster-operator/controllers/session_controller_test.go @@ -0,0 +1,62 @@ +package controllers + +import ( + "context" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + clusterv1alpha1 "github.com/LycaonLLC/t4-code/packages/cluster-operator/api/v1alpha1" +) + +func TestSessionRequestsForHostOnlyEnqueuesAffectedSessions(t *testing.T) { + scheme := runtime.NewScheme() + if err := clusterv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + objects := []client.Object{ + &clusterv1alpha1.T4Session{ObjectMeta: metav1.ObjectMeta{Name: "session-a", Namespace: "team"}, Spec: clusterv1alpha1.T4SessionSpec{HostRef: "host-a", WorkspaceRef: "workspace-a"}}, + &clusterv1alpha1.T4Session{ObjectMeta: metav1.ObjectMeta{Name: "session-b", Namespace: "team"}, Spec: clusterv1alpha1.T4SessionSpec{HostRef: "host-a", WorkspaceRef: "workspace-b"}}, + &clusterv1alpha1.T4Session{ObjectMeta: metav1.ObjectMeta{Name: "other-host", Namespace: "team"}, Spec: clusterv1alpha1.T4SessionSpec{HostRef: "host-b", WorkspaceRef: "workspace-a"}}, + &clusterv1alpha1.T4Session{ObjectMeta: metav1.ObjectMeta{Name: "other-namespace", Namespace: "other"}, Spec: clusterv1alpha1.T4SessionSpec{HostRef: "host-a", WorkspaceRef: "workspace-a"}}, + } + c := fake.NewClientBuilder().WithScheme(scheme). + WithIndex(&clusterv1alpha1.T4Session{}, sessionHostRefIndexField, indexSessionByHostRef). + WithIndex(&clusterv1alpha1.T4Session{}, sessionWorkspaceRefIndexField, indexSessionByWorkspaceRef). + WithObjects(objects...).Build() + r := &SessionReconciler{Client: c, Scheme: scheme} + + requests := r.sessionRequestsForHost(context.Background(), &clusterv1alpha1.T4ClusterHost{ObjectMeta: metav1.ObjectMeta{Name: "host-a", Namespace: "team"}}) + assertRequestSet(t, requests, []types.NamespacedName{ + {Namespace: "team", Name: "session-a"}, + {Namespace: "team", Name: "session-b"}, + }) +} + +func TestSessionRequestsForWorkspaceOnlyEnqueuesAffectedSessions(t *testing.T) { + scheme := runtime.NewScheme() + if err := clusterv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + objects := []client.Object{ + &clusterv1alpha1.T4Session{ObjectMeta: metav1.ObjectMeta{Name: "session-a", Namespace: "team"}, Spec: clusterv1alpha1.T4SessionSpec{HostRef: "host-a", WorkspaceRef: "workspace-a"}}, + &clusterv1alpha1.T4Session{ObjectMeta: metav1.ObjectMeta{Name: "session-b", Namespace: "team"}, Spec: clusterv1alpha1.T4SessionSpec{HostRef: "host-b", WorkspaceRef: "workspace-a"}}, + &clusterv1alpha1.T4Session{ObjectMeta: metav1.ObjectMeta{Name: "other-workspace", Namespace: "team"}, Spec: clusterv1alpha1.T4SessionSpec{HostRef: "host-a", WorkspaceRef: "workspace-b"}}, + &clusterv1alpha1.T4Session{ObjectMeta: metav1.ObjectMeta{Name: "other-namespace", Namespace: "other"}, Spec: clusterv1alpha1.T4SessionSpec{HostRef: "host-a", WorkspaceRef: "workspace-a"}}, + } + c := fake.NewClientBuilder().WithScheme(scheme). + WithIndex(&clusterv1alpha1.T4Session{}, sessionHostRefIndexField, indexSessionByHostRef). + WithIndex(&clusterv1alpha1.T4Session{}, sessionWorkspaceRefIndexField, indexSessionByWorkspaceRef). + WithObjects(objects...).Build() + r := &SessionReconciler{Client: c, Scheme: scheme} + + requests := r.sessionRequestsForWorkspace(context.Background(), &clusterv1alpha1.T4Workspace{ObjectMeta: metav1.ObjectMeta{Name: "workspace-a", Namespace: "team"}}) + assertRequestSet(t, requests, []types.NamespacedName{ + {Namespace: "team", Name: "session-a"}, + {Namespace: "team", Name: "session-b"}, + }) +} diff --git a/packages/cluster-operator/controllers/workspace_controller.go b/packages/cluster-operator/controllers/workspace_controller.go index fbc60f45..e1b616dc 100644 --- a/packages/cluster-operator/controllers/workspace_controller.go +++ b/packages/cluster-operator/controllers/workspace_controller.go @@ -10,6 +10,7 @@ import ( storagev1 "k8s.io/api/storage/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" meta "k8s.io/apimachinery/pkg/api/meta" + apiresource "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -23,7 +24,8 @@ import ( type WorkspaceReconciler struct { client.Client - Scheme *runtime.Scheme + APIReader client.Reader + Scheme *runtime.Scheme } const ( @@ -73,8 +75,9 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, request ctrl.Reques } pvcName := WorkspacePVCName(&workspace) + pvcKey := types.NamespacedName{Namespace: workspace.Namespace, Name: pvcName} var pvc corev1.PersistentVolumeClaim - err = r.Get(ctx, types.NamespacedName{Namespace: workspace.Namespace, Name: pvcName}, &pvc) + err = r.Get(ctx, pvcKey, &pvc) if apierrors.IsNotFound(err) { volumeMode := corev1.PersistentVolumeFilesystem pvc = corev1.PersistentVolumeClaim{ @@ -98,13 +101,27 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, request ctrl.Reques return ctrl.Result{}, err } } - if err := r.Create(ctx, &pvc); err != nil && !apierrors.IsAlreadyExists(err) { - return ctrl.Result{}, err + if err := r.Create(ctx, &pvc); err != nil { + if !apierrors.IsAlreadyExists(err) { + return ctrl.Result{}, err + } + reader := r.APIReader + if reader == nil { + reader = r.Client + } + if err := reader.Get(ctx, pvcKey, &pvc); err != nil { + return ctrl.Result{}, err + } } } else if err != nil { return ctrl.Result{}, err - } else if !workspaceOwnsPVC(&workspace, &pvc) { + } + if !workspaceOwnsPVC(&workspace, &pvc) { return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCOwnershipConflict", "deterministic workspace PVC does not belong to this workspace") + } else if !pvcHasRWX(&pvc) { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCNotRWX", "workspace PVC does not request ReadWriteMany") + } else if pvcStorageClassName(&pvc) != storageClassName { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", ReasonStorageClassMismatch, fmt.Sprintf("workspace PVC uses StorageClass %q instead of host-selected %q; data-bearing PVCs are never recreated automatically", pvcStorageClassName(&pvc), storageClassName)) } else if workspace.Spec.RetentionPolicy == clusterv1alpha1.RetentionPolicyRetain && metav1.IsControlledBy(&pvc, &workspace) { before := pvc.DeepCopy() pvc.OwnerReferences = removeWorkspaceOwnerReference(pvc.OwnerReferences, workspace.UID) @@ -115,6 +132,31 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, request ctrl.Reques return ctrl.Result{Requeue: true}, nil } } + reader := r.APIReader + if reader == nil { + reader = r.Client + } + var authoritativePVC corev1.PersistentVolumeClaim + if err := reader.Get(ctx, pvcKey, &authoritativePVC); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCNotFound", "workspace PVC does not exist in authoritative API state") + } + return ctrl.Result{}, err + } + if authoritativePVC.UID != pvc.UID || !workspaceOwnsPVC(&workspace, &authoritativePVC) { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCOwnershipConflict", "authoritative workspace PVC identity or ownership does not belong to this workspace") + } + if !pvcHasRWX(&authoritativePVC) { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCNotRWX", "authoritative workspace PVC does not request ReadWriteMany") + } + if pvcStorageClassName(&authoritativePVC) != storageClassName { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", ReasonStorageClassMismatch, fmt.Sprintf("authoritative workspace PVC uses StorageClass %q instead of host-selected %q; data-bearing PVCs are never recreated automatically", pvcStorageClassName(&authoritativePVC), storageClassName)) + } + pvc = authoritativePVC + + if pvc.Status.Phase == corev1.ClaimLost { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCLost", "workspace PVC lost its volume") + } original := workspace.Status original.Capacity = workspace.Status.Capacity.DeepCopy() @@ -126,19 +168,12 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, request ctrl.Reques workspace.Status.PVCPhase = pvc.Status.Phase capacity := pvc.Status.Capacity[corev1.ResourceStorage] workspace.Status.Capacity = capacity.DeepCopy() + meta.SetStatusCondition(&workspace.Status.Conditions, condition("HostReady", metav1.ConditionTrue, "HostResolved", "referenced T4ClusterHost is available", workspace.Generation)) meta.SetStatusCondition(&workspace.Status.Conditions, condition("StorageReady", metav1.ConditionTrue, ReasonStorageReady, "RWX StorageClass and workspace PVC are accepted", workspace.Generation)) switch pvc.Status.Phase { case corev1.ClaimBound: - if !pvcHasRWX(&pvc) { - workspace.Status.Phase = clusterv1alpha1.InfrastructureFailed - meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionFalse, "PVCNotRWX", "bound workspace PVC does not request ReadWriteMany", workspace.Generation)) - } else { - workspace.Status.Phase = clusterv1alpha1.InfrastructureReady - meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionTrue, "PVCBound", "workspace PVC is bound with ReadWriteMany access", workspace.Generation)) - } - case corev1.ClaimLost: - workspace.Status.Phase = clusterv1alpha1.InfrastructureFailed - meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionFalse, "PVCLost", "workspace PVC lost its volume", workspace.Generation)) + workspace.Status.Phase = clusterv1alpha1.InfrastructureReady + meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionTrue, "PVCBound", "workspace PVC is bound with ReadWriteMany access", workspace.Generation)) default: workspace.Status.Phase = clusterv1alpha1.InfrastructurePending meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionFalse, "PVCBinding", "workspace PVC is waiting to bind", workspace.Generation)) @@ -158,6 +193,11 @@ func workspaceOwnsPVC(workspace *clusterv1alpha1.T4Workspace, pvc *corev1.Persis if pvc.Annotations[clusterv1alpha1.WorkspaceUIDAnnotation] != string(workspace.UID) { return false } + for _, reference := range pvc.OwnerReferences { + if reference.APIVersion != clusterv1alpha1.GroupVersion.String() || reference.Kind != "T4Workspace" || reference.Name != workspace.Name || reference.UID != workspace.UID { + return false + } + } controller := metav1.GetControllerOf(pvc) if workspace.Spec.RetentionPolicy == clusterv1alpha1.RetentionPolicyDelete { return controller != nil && controller.UID == workspace.UID @@ -192,7 +232,11 @@ func (r *WorkspaceReconciler) reconcileDelete(ctx context.Context, workspace *cl } } var sessions clusterv1alpha1.T4SessionList - if err := r.List(ctx, &sessions, client.InNamespace(workspace.Namespace)); err != nil { + sessionReader := r.APIReader + if sessionReader == nil { + sessionReader = r.Client + } + if err := sessionReader.List(ctx, &sessions, client.InNamespace(workspace.Namespace)); err != nil { return ctrl.Result{}, err } remainingSessions := 0 @@ -216,7 +260,11 @@ func (r *WorkspaceReconciler) reconcileDelete(ctx context.Context, workspace *cl } pvcKey := types.NamespacedName{Namespace: workspace.Namespace, Name: WorkspacePVCName(workspace)} var pvc corev1.PersistentVolumeClaim - err := r.Get(ctx, pvcKey, &pvc) + reader := r.APIReader + if reader == nil { + reader = r.Client + } + err := reader.Get(ctx, pvcKey, &pvc) if err == nil && !workspaceOwnsPVC(workspace, &pvc) { before := workspace.Status if workspace.Status.Conditions != nil { @@ -248,7 +296,7 @@ func (r *WorkspaceReconciler) reconcileDelete(ctx context.Context, workspace *cl } } else { if err == nil { - if err := r.Delete(ctx, &pvc); err != nil && !apierrors.IsNotFound(err) { + if err := deleteWithPreconditions(ctx, r.Client, &pvc); err != nil && !apierrors.IsNotFound(err) { return ctrl.Result{}, err } return ctrl.Result{RequeueAfter: time.Second}, nil @@ -268,11 +316,18 @@ func (r *WorkspaceReconciler) updateWorkspaceFailure(ctx context.Context, worksp original.Conditions = append([]metav1.Condition(nil), workspace.Status.Conditions...) } workspace.Status.ObservedGeneration = workspace.Generation + workspace.Status.PVCName = "" + workspace.Status.PVCPhase = "" + workspace.Status.Capacity = apiresource.Quantity{} workspace.Status.Phase = clusterv1alpha1.InfrastructureFailed - meta.SetStatusCondition(&workspace.Status.Conditions, condition(conditionType, metav1.ConditionFalse, reason, message, workspace.Generation)) - if conditionType != "Ready" { - meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionFalse, reason, message, workspace.Generation)) + if conditionType == "HostReady" { + meta.SetStatusCondition(&workspace.Status.Conditions, condition("HostReady", metav1.ConditionFalse, reason, message, workspace.Generation)) + meta.SetStatusCondition(&workspace.Status.Conditions, condition("StorageReady", metav1.ConditionUnknown, "NotEvaluated", "storage dependency was not evaluated because the referenced host is unavailable", workspace.Generation)) + } else { + meta.SetStatusCondition(&workspace.Status.Conditions, condition("HostReady", metav1.ConditionTrue, "HostResolved", "referenced T4ClusterHost is available", workspace.Generation)) + meta.SetStatusCondition(&workspace.Status.Conditions, condition("StorageReady", metav1.ConditionFalse, reason, message, workspace.Generation)) } + meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionFalse, reason, message, workspace.Generation)) if reflect.DeepEqual(original, workspace.Status) { return nil } @@ -337,6 +392,23 @@ func (r *WorkspaceReconciler) workspaceRequestsForStorageClass(ctx context.Conte return requests } +func (r *WorkspaceReconciler) workspaceRequestsForHost(ctx context.Context, object client.Object) []ctrl.Request { + host, ok := object.(*clusterv1alpha1.T4ClusterHost) + if !ok || host.Name == "" || host.Namespace == "" { + return nil + } + var workspaces clusterv1alpha1.T4WorkspaceList + if err := r.List(ctx, &workspaces, client.InNamespace(host.Namespace), client.MatchingFields{workspaceHostRefIndexField: host.Name}); err != nil { + ctrl.LoggerFrom(ctx).Error(err, "unable to map cluster host to workspaces", "clusterHost", client.ObjectKeyFromObject(host)) + return nil + } + requests := make([]ctrl.Request, 0, len(workspaces.Items)) + for i := range workspaces.Items { + requests = append(requests, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(&workspaces.Items[i])}) + } + return requests +} + func (r *WorkspaceReconciler) SetupWithManager(manager ctrl.Manager) error { if err := manager.GetFieldIndexer().IndexField(context.Background(), &clusterv1alpha1.T4ClusterHost{}, hostStorageClassIndexField, indexHostByStorageClass); err != nil { return fmt.Errorf("index T4ClusterHost by StorageClass: %w", err) @@ -346,6 +418,7 @@ func (r *WorkspaceReconciler) SetupWithManager(manager ctrl.Manager) error { } return ctrl.NewControllerManagedBy(manager). For(&clusterv1alpha1.T4Workspace{}). + Watches(&clusterv1alpha1.T4ClusterHost{}, handler.EnqueueRequestsFromMapFunc(r.workspaceRequestsForHost)). Watches(&corev1.PersistentVolumeClaim{}, handler.EnqueueRequestsFromMapFunc(workspaceRequestsForPVC)). Watches(&storagev1.StorageClass{}, handler.EnqueueRequestsFromMapFunc(r.workspaceRequestsForStorageClass)). Complete(r) diff --git a/packages/cluster-operator/controllers/workspace_controller_test.go b/packages/cluster-operator/controllers/workspace_controller_test.go index 412705d0..27d07759 100644 --- a/packages/cluster-operator/controllers/workspace_controller_test.go +++ b/packages/cluster-operator/controllers/workspace_controller_test.go @@ -9,6 +9,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -107,3 +108,42 @@ func TestWorkspaceRequestsForStorageClassOnlyEnqueuesAffectedWorkspaces(t *testi } } } + +func TestWorkspaceRequestsForHostOnlyEnqueuesAffectedWorkspaces(t *testing.T) { + scheme := runtime.NewScheme() + if err := clusterv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + objects := []client.Object{ + &clusterv1alpha1.T4Workspace{ObjectMeta: metav1.ObjectMeta{Name: "workspace-a", Namespace: "team"}, Spec: clusterv1alpha1.T4WorkspaceSpec{HostRef: "host-a"}}, + &clusterv1alpha1.T4Workspace{ObjectMeta: metav1.ObjectMeta{Name: "workspace-b", Namespace: "team"}, Spec: clusterv1alpha1.T4WorkspaceSpec{HostRef: "host-a"}}, + &clusterv1alpha1.T4Workspace{ObjectMeta: metav1.ObjectMeta{Name: "other-host", Namespace: "team"}, Spec: clusterv1alpha1.T4WorkspaceSpec{HostRef: "host-b"}}, + &clusterv1alpha1.T4Workspace{ObjectMeta: metav1.ObjectMeta{Name: "other-namespace", Namespace: "other"}, Spec: clusterv1alpha1.T4WorkspaceSpec{HostRef: "host-a"}}, + } + c := fake.NewClientBuilder().WithScheme(scheme). + WithIndex(&clusterv1alpha1.T4Workspace{}, workspaceHostRefIndexField, indexWorkspaceByHostRef). + WithObjects(objects...).Build() + r := &WorkspaceReconciler{Client: c, Scheme: scheme} + + requests := r.workspaceRequestsForHost(context.Background(), &clusterv1alpha1.T4ClusterHost{ObjectMeta: metav1.ObjectMeta{Name: "host-a", Namespace: "team"}}) + assertRequestSet(t, requests, []types.NamespacedName{ + {Namespace: "team", Name: "workspace-a"}, + {Namespace: "team", Name: "workspace-b"}, + }) +} + +func assertRequestSet(t *testing.T, requests []ctrl.Request, want []types.NamespacedName) { + t.Helper() + got := make(map[types.NamespacedName]int, len(requests)) + for _, request := range requests { + got[request.NamespacedName]++ + } + if len(got) != len(want) { + t.Fatalf("requests = %#v, want exactly %v", requests, want) + } + for _, key := range want { + if got[key] != 1 { + t.Fatalf("requests = %#v, want %v exactly once", requests, key) + } + } +} diff --git a/packages/cluster-operator/go.mod b/packages/cluster-operator/go.mod index c85639c9..f4504858 100644 --- a/packages/cluster-operator/go.mod +++ b/packages/cluster-operator/go.mod @@ -14,23 +14,32 @@ require ( ) require ( + cel.dev/expr v0.19.1 // indirect + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.23.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/google/btree v1.1.3 // indirect + github.com/google/cel-go v0.23.2 // indirect github.com/google/gnostic-models v0.6.9 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -41,10 +50,22 @@ require ( github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect + github.com/spf13/cobra v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stoewer/go-strcase v1.3.0 // indirect github.com/x448/float16 v0.8.4 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect + go.opentelemetry.io/otel v1.33.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 // indirect + go.opentelemetry.io/otel/metric v1.33.0 // indirect + go.opentelemetry.io/otel/sdk v1.33.0 // indirect + go.opentelemetry.io/otel/trace v1.33.0 // indirect + go.opentelemetry.io/proto/otlp v1.4.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect golang.org/x/net v0.38.0 // indirect golang.org/x/oauth2 v0.27.0 // indirect golang.org/x/sync v0.12.0 // indirect @@ -53,12 +74,19 @@ require ( golang.org/x/text v0.23.0 // indirect golang.org/x/time v0.9.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect + google.golang.org/grpc v1.68.1 // indirect google.golang.org/protobuf v1.36.5 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiserver v0.33.2 // indirect + k8s.io/component-base v0.33.2 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect diff --git a/packages/cluster-operator/go.sum b/packages/cluster-operator/go.sum index e291ac10..6883ecd7 100644 --- a/packages/cluster-operator/go.sum +++ b/packages/cluster-operator/go.sum @@ -1,9 +1,16 @@ +cel.dev/expr v0.19.1 h1:NciYrtDRIR0lNCnH1LFJegdjspNx9fI59O7TWcua/W4= +cel.dev/expr v0.19.1/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= +github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= +github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -14,12 +21,17 @@ github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8 github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= @@ -36,6 +48,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/cel-go v0.23.2 h1:UdEe3CvQh3Nv+E/j9r1Y//WO0K0cSyD7/y0bzyLIMI4= +github.com/google/cel-go v0.23.2/go.mod h1:52Pb6QsDbC5kvgxvZhiL9QX1oZEkcUF/ZqaPx1J5Wwo= github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -48,6 +62,11 @@ github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgY github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -92,8 +111,13 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -109,6 +133,24 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= +go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw= +go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0/go.mod h1:cpgtDBaqD/6ok/UG0jT15/uKjAY8mRA53diogHBg3UI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 h1:5pojmb1U1AogINhN3SurB+zm/nIcusopeBNp42f45QM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0/go.mod h1:57gTHJSE5S1tqg+EKsLPlTWhpHMsWlVmer+LA926XiA= +go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ= +go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M= +go.opentelemetry.io/otel/sdk v1.33.0 h1:iax7M131HuAm9QkZotNHEfstof92xM+N8sr3uHXc2IM= +go.opentelemetry.io/otel/sdk v1.33.0/go.mod h1:A1Q5oi7/9XaMlIWzPSxLRWOI8nG3FnzHJNbiENQuihM= +go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s= +go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck= +go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg= +go.opentelemetry.io/proto/otlp v1.4.0/go.mod h1:PPBWZIP98o2ElSqI35IHfu7hIhSwvc5N38Jw8pXuGFY= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -118,6 +160,8 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -158,6 +202,13 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ= +google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 h1:CkkIfIt50+lT6NHAVoRYEyAvQGFM7xEwXUUywFvEb3Q= +google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 h1:8ZmaLZE4XWrtU3MyClkYqqtl6Oegr3235h7jxsDyqCY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= +google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0= +google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw= google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -176,14 +227,20 @@ k8s.io/apiextensions-apiserver v0.33.2 h1:6gnkIbngnaUflR3XwE1mCefN3YS8yTD631JXQh k8s.io/apiextensions-apiserver v0.33.2/go.mod h1:IvVanieYsEHJImTKXGP6XCOjTwv2LUMos0YWc9O+QP8= k8s.io/apimachinery v0.33.2 h1:IHFVhqg59mb8PJWTLi8m1mAoepkUNYmptHsV+Z1m5jY= k8s.io/apimachinery v0.33.2/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= +k8s.io/apiserver v0.33.2 h1:KGTRbxn2wJagJowo29kKBp4TchpO1DRO3g+dB/KOJN4= +k8s.io/apiserver v0.33.2/go.mod h1:9qday04wEAMLPWWo9AwqCZSiIn3OYSZacDyu/AcoM/M= k8s.io/client-go v0.33.2 h1:z8CIcc0P581x/J1ZYf4CNzRKxRvQAwoAolYPbtQes+E= k8s.io/client-go v0.33.2/go.mod h1:9mCgT4wROvL948w6f6ArJNb7yQd7QsvqavDeZHvNmHo= +k8s.io/component-base v0.33.2 h1:sCCsn9s/dG3ZrQTX/Us0/Sx2R0G5kwa0wbZFYoVp/+0= +k8s.io/component-base v0.33.2/go.mod h1:/41uw9wKzuelhN+u+/C59ixxf4tYQKW7p32ddkYNe2k= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8= sigs.k8s.io/controller-runtime v0.21.0/go.mod h1:OSg14+F65eWqIu4DceX7k/+QRAbTTvxeQSNSOQpukWM= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= diff --git a/packages/cluster-server/migrations/001_durable_gateway.down.sql b/packages/cluster-server/migrations/001_durable_gateway.down.sql new file mode 100644 index 00000000..c3967522 --- /dev/null +++ b/packages/cluster-server/migrations/001_durable_gateway.down.sql @@ -0,0 +1,11 @@ +DROP TABLE IF EXISTS t4_stale_create_cleanups; +DROP TABLE IF EXISTS t4_kubernetes_status_cursors; +DROP TABLE IF EXISTS t4_outbox; +DROP TABLE IF EXISTS t4_owner_leases; +DROP TABLE IF EXISTS t4_snapshot_entries; +DROP TABLE IF EXISTS t4_event_retention; +DROP TABLE IF EXISTS t4_events; +DROP TABLE IF EXISTS t4_session_intents; +DROP TABLE IF EXISTS t4_workspace_intents; +DROP TABLE IF EXISTS t4_commands; +DROP TABLE IF EXISTS t4_schema_migrations; diff --git a/packages/cluster-server/migrations/001_durable_gateway.sql b/packages/cluster-server/migrations/001_durable_gateway.sql new file mode 100644 index 00000000..b9cfcc17 --- /dev/null +++ b/packages/cluster-server/migrations/001_durable_gateway.sql @@ -0,0 +1,110 @@ +CREATE TABLE IF NOT EXISTS t4_schema_migrations (version integer PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT clock_timestamp()); +CREATE TABLE IF NOT EXISTS t4_commands ( + command_id text PRIMARY KEY, principal_id text NOT NULL, operation text NOT NULL, target_scope text NOT NULL, + idempotency_key text NOT NULL, fingerprint text NOT NULL, lifecycle_state text NOT NULL CHECK (lifecycle_state IN ('accepted','projected','dispatching','running','succeeded','failed','cancelling','cancelled','rejected','unavailable','indeterminate')), + response_status integer NOT NULL, response_body jsonb, error_body jsonb, created_at timestamptz NOT NULL DEFAULT clock_timestamp(), updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE (principal_id, operation, target_scope, idempotency_key) +); +ALTER TABLE t4_commands DROP CONSTRAINT IF EXISTS t4_commands_lifecycle_state_check; +UPDATE t4_commands SET lifecycle_state = 'projected', updated_at = clock_timestamp() WHERE lifecycle_state = 'completed'; +ALTER TABLE t4_commands ADD CONSTRAINT t4_commands_lifecycle_state_check CHECK (lifecycle_state IN ('accepted','projected','dispatching','running','succeeded','failed','cancelling','cancelled','rejected','unavailable','indeterminate')) NOT VALID; +ALTER TABLE t4_commands VALIDATE CONSTRAINT t4_commands_lifecycle_state_check; +CREATE TABLE IF NOT EXISTS t4_workspace_intents ( + workspace_id text PRIMARY KEY, principal_id text NOT NULL, name text NOT NULL, labels jsonb NOT NULL DEFAULT '{}'::jsonb, + state text NOT NULL CHECK (state IN ('accepted','provisioning','ready','deleting','deleted','failed','unavailable','indeterminate')), + revision bigint NOT NULL CHECK (revision > 0), generation bigint NOT NULL CHECK (generation > 0), deletion_requested boolean NOT NULL DEFAULT false, + kube_uid text, kube_resource_version text, kube_generation bigint, kube_observed_generation bigint, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), updated_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +ALTER TABLE t4_workspace_intents ADD COLUMN IF NOT EXISTS kube_uid text; +ALTER TABLE t4_workspace_intents ADD COLUMN IF NOT EXISTS kube_resource_version text; +ALTER TABLE t4_workspace_intents ADD COLUMN IF NOT EXISTS kube_generation bigint; +ALTER TABLE t4_workspace_intents ADD COLUMN IF NOT EXISTS kube_observed_generation bigint; +UPDATE t4_workspace_intents SET kube_uid = NULL, kube_resource_version = NULL, kube_generation = NULL, kube_observed_generation = NULL + WHERE NOT ((kube_uid IS NULL AND kube_resource_version IS NULL AND kube_generation IS NULL AND kube_observed_generation IS NULL) + OR (kube_uid IS NOT NULL AND octet_length(kube_uid) BETWEEN 1 AND 256 AND kube_resource_version IS NOT NULL AND octet_length(kube_resource_version) BETWEEN 1 AND 256 AND kube_generation IS NOT NULL AND kube_generation > 0 AND kube_observed_generation IS NOT NULL AND kube_observed_generation >= 0)); +ALTER TABLE t4_workspace_intents DROP CONSTRAINT IF EXISTS t4_workspace_intents_kube_identity_check; +ALTER TABLE t4_workspace_intents ADD CONSTRAINT t4_workspace_intents_kube_identity_check CHECK ((kube_uid IS NULL AND kube_resource_version IS NULL AND kube_generation IS NULL AND kube_observed_generation IS NULL) OR (kube_uid IS NOT NULL AND octet_length(kube_uid) BETWEEN 1 AND 256 AND kube_resource_version IS NOT NULL AND octet_length(kube_resource_version) BETWEEN 1 AND 256 AND kube_generation IS NOT NULL AND kube_generation > 0 AND kube_observed_generation IS NOT NULL AND kube_observed_generation >= 0)) NOT VALID; +ALTER TABLE t4_workspace_intents VALIDATE CONSTRAINT t4_workspace_intents_kube_identity_check; +CREATE INDEX IF NOT EXISTS t4_workspace_intents_principal_page ON t4_workspace_intents (principal_id, workspace_id); +CREATE TABLE IF NOT EXISTS t4_session_intents ( + session_id text PRIMARY KEY, workspace_id text NOT NULL REFERENCES t4_workspace_intents(workspace_id), principal_id text NOT NULL, + title text NOT NULL, labels jsonb NOT NULL DEFAULT '{}'::jsonb, + state text NOT NULL CHECK (state IN ('accepted','provisioning','ready','cancelling','cancelled','failed','unavailable','indeterminate')), + revision bigint NOT NULL CHECK (revision > 0), generation bigint NOT NULL CHECK (generation > 0), cancellation_requested boolean NOT NULL DEFAULT false, + deletion_requested boolean NOT NULL DEFAULT false, kube_uid text, kube_resource_version text, kube_generation bigint, kube_observed_generation bigint, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), updated_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +ALTER TABLE t4_session_intents ADD COLUMN IF NOT EXISTS kube_uid text; +ALTER TABLE t4_session_intents ADD COLUMN IF NOT EXISTS kube_resource_version text; +ALTER TABLE t4_session_intents ADD COLUMN IF NOT EXISTS kube_generation bigint; +ALTER TABLE t4_session_intents ADD COLUMN IF NOT EXISTS kube_observed_generation bigint; +UPDATE t4_session_intents SET kube_uid = NULL, kube_resource_version = NULL, kube_generation = NULL, kube_observed_generation = NULL + WHERE NOT ((kube_uid IS NULL AND kube_resource_version IS NULL AND kube_generation IS NULL AND kube_observed_generation IS NULL) + OR (kube_uid IS NOT NULL AND octet_length(kube_uid) BETWEEN 1 AND 256 AND kube_resource_version IS NOT NULL AND octet_length(kube_resource_version) BETWEEN 1 AND 256 AND kube_generation IS NOT NULL AND kube_generation > 0 AND kube_observed_generation IS NOT NULL AND kube_observed_generation >= 0)); +ALTER TABLE t4_session_intents DROP CONSTRAINT IF EXISTS t4_session_intents_kube_identity_check; +ALTER TABLE t4_session_intents ADD CONSTRAINT t4_session_intents_kube_identity_check CHECK ((kube_uid IS NULL AND kube_resource_version IS NULL AND kube_generation IS NULL AND kube_observed_generation IS NULL) OR (kube_uid IS NOT NULL AND octet_length(kube_uid) BETWEEN 1 AND 256 AND kube_resource_version IS NOT NULL AND octet_length(kube_resource_version) BETWEEN 1 AND 256 AND kube_generation IS NOT NULL AND kube_generation > 0 AND kube_observed_generation IS NOT NULL AND kube_observed_generation >= 0)) NOT VALID; +ALTER TABLE t4_session_intents VALIDATE CONSTRAINT t4_session_intents_kube_identity_check; +CREATE INDEX IF NOT EXISTS t4_session_intents_principal_workspace_page ON t4_session_intents (principal_id, workspace_id, session_id); +CREATE TABLE IF NOT EXISTS t4_events ( + sequence bigserial PRIMARY KEY, principal_id text NOT NULL, session_id text, event_type text NOT NULL CHECK (event_type IN ('session','command')), + payload jsonb NOT NULL, owner_epoch bigint NOT NULL DEFAULT 0 CHECK (owner_epoch >= 0), created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX IF NOT EXISTS t4_events_session_sequence ON t4_events (principal_id, session_id, sequence); +CREATE TABLE IF NOT EXISTS t4_event_retention ( + principal_id text NOT NULL, session_id text NOT NULL, first_retained_sequence bigint NOT NULL CHECK (first_retained_sequence >= 0), + latest_sequence bigint NOT NULL CHECK (latest_sequence >= 0), updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), PRIMARY KEY (principal_id, session_id) +); +CREATE TABLE IF NOT EXISTS t4_snapshot_entries ( + principal_id text NOT NULL, session_id text NOT NULL, entry_sequence bigint NOT NULL CHECK (entry_sequence >= 0), + kind text NOT NULL CHECK (kind IN ('input','output','status')), text_value text NOT NULL, created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (principal_id, session_id, entry_sequence) +); +CREATE TABLE IF NOT EXISTS t4_owner_leases ( + lease_name text PRIMARY KEY, owner_id text NOT NULL, epoch bigint NOT NULL CHECK (epoch > 0), expires_at timestamptz NOT NULL, updated_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE IF NOT EXISTS t4_outbox ( + outbox_id bigserial PRIMARY KEY, command_id text NOT NULL UNIQUE REFERENCES t4_commands(command_id), principal_id text NOT NULL, + idempotency_key text NOT NULL, mutation_kind text NOT NULL CHECK (mutation_kind IN ('workspace.create','workspace.patch','workspace.delete','session.create','session.patch','session.cancel','session.delete','command.submit')), + target_id text NOT NULL, target_revision bigint NOT NULL CHECK (target_revision > 0), mutation jsonb NOT NULL, + state text NOT NULL DEFAULT 'pending' CHECK (state IN ('pending','claimed','applied','skipped','failed')), owner_id text, owner_epoch bigint, + attempts integer NOT NULL DEFAULT 0 CHECK (attempts >= 0), next_attempt_at timestamptz NOT NULL DEFAULT clock_timestamp(), claimed_at timestamptz, + terminal_result jsonb, last_error text, created_at timestamptz NOT NULL DEFAULT clock_timestamp(), updated_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +UPDATE t4_outbox SET state = 'pending', owner_id = NULL, owner_epoch = NULL, claimed_at = NULL, next_attempt_at = clock_timestamp(), updated_at = clock_timestamp() WHERE state = 'claimed' AND (owner_id IS NULL OR btrim(owner_id) = '' OR owner_epoch IS NULL OR owner_epoch <= 0 OR claimed_at IS NULL); +UPDATE t4_outbox SET owner_id = NULL, owner_epoch = NULL, claimed_at = NULL, updated_at = clock_timestamp() WHERE state <> 'claimed' AND (owner_id IS NOT NULL OR owner_epoch IS NOT NULL OR claimed_at IS NOT NULL); +ALTER TABLE t4_outbox DROP CONSTRAINT IF EXISTS t4_outbox_owner_tuple_check; +ALTER TABLE t4_outbox DROP CONSTRAINT IF EXISTS t4_outbox_claimed_identity_check; +ALTER TABLE t4_outbox DROP CONSTRAINT IF EXISTS t4_outbox_nonclaimed_identity_check; +ALTER TABLE t4_outbox ADD CONSTRAINT t4_outbox_owner_tuple_check CHECK ((owner_id IS NULL AND owner_epoch IS NULL) OR (owner_id IS NOT NULL AND btrim(owner_id) <> '' AND owner_epoch IS NOT NULL AND owner_epoch > 0)) NOT VALID; +ALTER TABLE t4_outbox ADD CONSTRAINT t4_outbox_claimed_identity_check CHECK (state <> 'claimed' OR (owner_id IS NOT NULL AND btrim(owner_id) <> '' AND owner_epoch IS NOT NULL AND owner_epoch > 0 AND claimed_at IS NOT NULL)) NOT VALID; +ALTER TABLE t4_outbox ADD CONSTRAINT t4_outbox_nonclaimed_identity_check CHECK (state = 'claimed' OR (owner_id IS NULL AND owner_epoch IS NULL AND claimed_at IS NULL)) NOT VALID; +ALTER TABLE t4_outbox VALIDATE CONSTRAINT t4_outbox_owner_tuple_check; +ALTER TABLE t4_outbox VALIDATE CONSTRAINT t4_outbox_claimed_identity_check; +ALTER TABLE t4_outbox VALIDATE CONSTRAINT t4_outbox_nonclaimed_identity_check; +CREATE INDEX IF NOT EXISTS t4_outbox_ready ON t4_outbox (state, next_attempt_at, outbox_id); +CREATE TABLE IF NOT EXISTS t4_kubernetes_status_cursors ( + collection text PRIMARY KEY CHECK (collection IN ('t4workspaces','t4sessions')), resource_version text NOT NULL CHECK (octet_length(resource_version) BETWEEN 1 AND 256), + owner_epoch bigint NOT NULL CHECK (owner_epoch > 0), updated_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE IF NOT EXISTS t4_stale_create_cleanups ( + cleanup_id bigserial PRIMARY KEY, outbox_id bigint NOT NULL REFERENCES t4_outbox(outbox_id), + resource_type text NOT NULL CHECK (resource_type IN ('t4workspaces','t4sessions')), target_id text NOT NULL, + uid text NOT NULL CHECK (octet_length(uid) BETWEEN 1 AND 256), resource_version text NOT NULL CHECK (octet_length(resource_version) BETWEEN 1 AND 256), + state text NOT NULL DEFAULT 'pending' CHECK (state IN ('pending','claimed','applied')), owner_id text, owner_epoch bigint, + attempts integer NOT NULL DEFAULT 0 CHECK (attempts >= 0), next_attempt_at timestamptz NOT NULL DEFAULT clock_timestamp(), claimed_at timestamptz, + last_error text, created_at timestamptz NOT NULL DEFAULT clock_timestamp(), updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE (outbox_id, uid) +); +UPDATE t4_stale_create_cleanups cleanup SET state = 'applied', owner_id = NULL, owner_epoch = NULL, claimed_at = NULL, last_error = NULL, updated_at = clock_timestamp() + WHERE cleanup.state <> 'applied' AND EXISTS (SELECT 1 FROM t4_outbox item WHERE item.outbox_id = cleanup.outbox_id AND item.state = 'applied'); +UPDATE t4_stale_create_cleanups SET state = 'pending', owner_id = NULL, owner_epoch = NULL, claimed_at = NULL, next_attempt_at = clock_timestamp(), updated_at = clock_timestamp() WHERE state = 'claimed' AND (owner_id IS NULL OR btrim(owner_id) = '' OR owner_epoch IS NULL OR owner_epoch <= 0 OR claimed_at IS NULL); +UPDATE t4_stale_create_cleanups SET owner_id = NULL, owner_epoch = NULL, claimed_at = NULL, updated_at = clock_timestamp() WHERE state <> 'claimed' AND (owner_id IS NOT NULL OR owner_epoch IS NOT NULL OR claimed_at IS NOT NULL); +ALTER TABLE t4_stale_create_cleanups DROP CONSTRAINT IF EXISTS t4_stale_create_cleanups_claim_check; +ALTER TABLE t4_stale_create_cleanups DROP CONSTRAINT IF EXISTS t4_stale_create_cleanups_nonclaimed_check; +ALTER TABLE t4_stale_create_cleanups ADD CONSTRAINT t4_stale_create_cleanups_claim_check CHECK (state <> 'claimed' OR (owner_id IS NOT NULL AND btrim(owner_id) <> '' AND owner_epoch IS NOT NULL AND owner_epoch > 0 AND claimed_at IS NOT NULL)) NOT VALID; +ALTER TABLE t4_stale_create_cleanups ADD CONSTRAINT t4_stale_create_cleanups_nonclaimed_check CHECK (state = 'claimed' OR (owner_id IS NULL AND owner_epoch IS NULL AND claimed_at IS NULL)) NOT VALID; +ALTER TABLE t4_stale_create_cleanups VALIDATE CONSTRAINT t4_stale_create_cleanups_claim_check; +ALTER TABLE t4_stale_create_cleanups VALIDATE CONSTRAINT t4_stale_create_cleanups_nonclaimed_check; +CREATE INDEX IF NOT EXISTS t4_stale_create_cleanups_ready ON t4_stale_create_cleanups (state, next_attempt_at, cleanup_id); +INSERT INTO t4_schema_migrations(version) VALUES (1) ON CONFLICT (version) DO NOTHING; diff --git a/packages/cluster-server/src/config.ts b/packages/cluster-server/src/config.ts index 06a8d863..fea3d530 100644 --- a/packages/cluster-server/src/config.ts +++ b/packages/cluster-server/src/config.ts @@ -5,6 +5,7 @@ import { isIP } from "node:net"; const DNS_LABEL = /^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$/u; const DNS_SUBDOMAIN = /^(?:[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?)(?:\.[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?)*$/u; const AUDIENCE = /^[A-Za-z0-9][A-Za-z0-9._:/-]*$/u; +const RUNTIME_PROFILE = /^[a-z0-9](?:[-._a-z0-9]{0,62}[a-z0-9])?$/u; export interface ClusterServerConfig { readonly namespace: string; readonly podName: string; @@ -21,6 +22,9 @@ export interface ClusterServerConfig { readonly identityProvider: "tailscale"; readonly trustedProxyAddresses: readonly string[]; readonly trustedProxyCidrs: readonly string[]; + readonly postgres?: { readonly url?: string; readonly urlFile?: string }; + readonly publicApiCredentialsFile?: string; + readonly publicApiRuntimeProfile?: string; readonly woodpecker?: { readonly baseUrl: string; readonly webBaseUrl?: string; @@ -132,6 +136,16 @@ export function clusterServerConfigFromEnv(env: Readonly { if (!token || /\s/u.test(token)) throw new Error("Kubernetes token file is invalid"); return token; } +function postgresUrl(value: string): string { + const trimmed = value.trim(); + if (trimmed !== value || new TextEncoder().encode(value).byteLength > 8192) throw new Error("PostgreSQL URL is invalid"); + let parsed: URL; + try { parsed = new URL(value); } catch { throw new Error("PostgreSQL URL is invalid"); } + if (!["postgres:", "postgresql:"].includes(parsed.protocol) || !parsed.hostname || !parsed.username || !parsed.password || parsed.pathname.length < 2 || parsed.hash) + throw new Error("PostgreSQL URL is invalid"); + return value; +} +export async function loadPostgresUrl(config: ClusterServerConfig): Promise { + if (!config.postgres) throw new Error("PostgreSQL configuration is required"); + if (Boolean(config.postgres.url) === Boolean(config.postgres.urlFile)) throw new Error("PostgreSQL configuration requires exactly one credential source"); + return postgresUrl(config.postgres.url ?? (await readBoundedRegularFile(config.postgres.urlFile!, 8192, "PostgreSQL URL")).trim()); +} export async function loadKubernetesCa(config: ClusterServerConfig): Promise { const ca = await readBoundedRegularFile(config.kubernetesCaPath, 1024 * 1024, "Kubernetes CA"); if (!ca.includes("BEGIN CERTIFICATE")) throw new Error("Kubernetes CA file is invalid"); diff --git a/packages/cluster-server/src/index.ts b/packages/cluster-server/src/index.ts index 4f81c495..55c590f7 100644 --- a/packages/cluster-server/src/index.ts +++ b/packages/cluster-server/src/index.ts @@ -1,6 +1,11 @@ export * from "./ci-projection-runner.ts"; export * from "./config.ts"; export * from "./gateway.ts"; +export * from "./ledger.ts"; +export * from "./outbox-worker.ts"; +export * from "./public-api-auth.ts"; +export * from "./public-api-v1.ts"; +export * from "./public-kubernetes-applier.ts"; export * from "./kubernetes-client.ts"; export * from "./kubernetes-projection.ts"; export * from "./kubernetes-runner.ts"; diff --git a/packages/cluster-server/src/kubernetes-client.ts b/packages/cluster-server/src/kubernetes-client.ts index bd43edc8..7b575221 100644 --- a/packages/cluster-server/src/kubernetes-client.ts +++ b/packages/cluster-server/src/kubernetes-client.ts @@ -122,6 +122,15 @@ export class KubernetesApiClient { }) as KubernetesResource; } + async patch(resource: string, name: string, body: unknown, signal?: AbortSignal): Promise { + return await this.request(`${this.#collection(resource)}/${encodeURIComponent(name)}`, { + method: "PATCH", + headers: { "content-type": "application/merge-patch+json" }, + body: JSON.stringify(body), + signal, + }) as KubernetesResource; + } + async get(resource: string, name: string, signal?: AbortSignal): Promise { return await this.request(`${this.#collection(resource)}/${encodeURIComponent(name)}`, { signal }) as KubernetesResource; } diff --git a/packages/cluster-server/src/kubernetes-runner.ts b/packages/cluster-server/src/kubernetes-runner.ts index 3f23a5ef..7268594a 100644 --- a/packages/cluster-server/src/kubernetes-runner.ts +++ b/packages/cluster-server/src/kubernetes-runner.ts @@ -1,5 +1,16 @@ import { KubernetesApiClient, KubernetesApiError } from "./kubernetes-client.ts"; -import { ClusterInfrastructureProjection, KubernetesAuthorityInvalidatedError } from "./kubernetes-projection.ts"; +import { + ClusterInfrastructureProjection, + KubernetesAuthorityInvalidatedError, + type InfrastructureList, + type KubernetesResource, + type KubernetesWatchEvent, +} from "./kubernetes-projection.ts"; +import type { + AuthoritativeKubernetesStatusIngress, + AuthoritativeKubernetesStatusObservation, + KubernetesStatusCollection, +} from "./ledger.ts"; export interface KubernetesProjectionRunnerOptions { readonly client: KubernetesApiClient; @@ -7,6 +18,10 @@ export interface KubernetesProjectionRunnerOptions { readonly hostName: string; readonly onSynchronized?: () => void; readonly onError?: (error: unknown) => void; + readonly statusIngress?: ( + collection: KubernetesStatusCollection, + observation: AuthoritativeKubernetesStatusIngress, + ) => Promise; readonly retryMs?: number; } @@ -24,6 +39,7 @@ function delay(milliseconds: number, signal: AbortSignal): Promise { export class KubernetesProjectionRunner { readonly #options: KubernetesProjectionRunnerOptions; readonly #synchronized = new Set(); + readonly #workspaceOwners = new Map(); #abort?: AbortController; #running?: Promise; #relist?: Promise; @@ -34,6 +50,7 @@ export class KubernetesProjectionRunner { this.#abort = new AbortController(); const signal = this.#abort.signal; const initial = await this.#options.client.listInfrastructure(this.#options.hostName, signal); + await this.#prepareSnapshot(initial, signal); this.#options.projection.replace(initial); this.#synchronized.clear(); for (const resource of ["t4clusterhosts", "t4workspaces", "t4sessions"]) this.#synchronized.add(resource); @@ -48,6 +65,143 @@ export class KubernetesProjectionRunner { this.#abort = undefined; this.#relist = undefined; this.#synchronized.clear(); + this.#workspaceOwners.clear(); + } + + async #prepareSnapshot(snapshot: InfrastructureList, signal: AbortSignal): Promise { + await this.#reconcileLegacySessionAnnotations(snapshot.sessions, signal); + const workspaceOwners = new Map(); + for (const workspace of snapshot.workspaces) { + const workspaceId = this.#requiredString(workspace.metadata.name, "workspace name"); + const owner = this.#requiredString(workspace.spec?.owner, "workspace owner"); + workspaceOwners.set(workspaceId, owner); + } + const ingress = this.#options.statusIngress; + let ingressAvailable = ingress !== undefined; + if (ingress) { + for (const workspace of snapshot.workspaces) { + if (signal.aborted) throw signal.reason; + if (ingressAvailable) ingressAvailable = await this.#emitStatus("t4workspaces", this.#statusObservation("t4workspaces", workspace, workspaceOwners)); + } + for (const session of snapshot.sessions) { + if (signal.aborted) throw signal.reason; + if (ingressAvailable) ingressAvailable = await this.#emitStatus("t4sessions", this.#statusObservation("t4sessions", session, workspaceOwners)); + } + if (ingressAvailable) ingressAvailable = await this.#emitStatus("t4workspaces", { + relisted: true, + resourceVersion: this.#requiredString(snapshot.resourceVersions?.t4workspaces ?? snapshot.resourceVersion, "workspace list resourceVersion"), + resourceIds: [...workspaceOwners.keys()], + }); + if (ingressAvailable) await this.#emitStatus("t4sessions", { + relisted: true, + resourceVersion: this.#requiredString(snapshot.resourceVersions?.t4sessions ?? snapshot.resourceVersion, "session list resourceVersion"), + resourceIds: snapshot.sessions.map(session => this.#requiredString(session.metadata.name, "session name")), + }); + } + this.#workspaceOwners.clear(); + for (const [workspaceId, owner] of workspaceOwners) this.#workspaceOwners.set(workspaceId, owner); + } + + async #reconcileLegacySessionAnnotations(sessions: readonly KubernetesResource[], signal: AbortSignal): Promise { + for (const session of sessions) { + const annotations = session.metadata.annotations; + if (!annotations || !Object.prototype.hasOwnProperty.call(annotations, "cluster.t4.dev/pending-command")) continue; + const resourceVersion = this.#requiredString(session.metadata.resourceVersion, "session resourceVersion"); + await this.#options.client.patch("t4sessions", session.metadata.name, { + metadata: { + resourceVersion, + annotations: { "cluster.t4.dev/pending-command": null }, + }, + }, signal); + } + } + + #statusObservation( + collection: KubernetesStatusCollection, + resource: KubernetesResource, + workspaceOwners: ReadonlyMap, + deleted = false, + ): AuthoritativeKubernetesStatusObservation { + const expectedKind = collection === "t4workspaces" ? "T4Workspace" : "T4Session"; + if (resource.kind !== expectedKind) throw new Error(`Kubernetes ${collection} status kind is invalid`); + const generation = resource.metadata.generation; + if (generation === undefined || !Number.isSafeInteger(generation) || generation < 1) + throw new Error(`Kubernetes ${collection} generation is invalid`); + const observedGeneration = resource.status?.observedGeneration ?? 0; + if (typeof observedGeneration !== "number" || !Number.isSafeInteger(observedGeneration) + || observedGeneration < 0 || observedGeneration > generation) + throw new Error(`Kubernetes ${collection} observedGeneration is invalid`); + const rawPhase = deleted || resource.metadata.deletionTimestamp + ? "Terminating" + : resource.status?.phase; + const phase = collection === "t4workspaces" + ? rawPhase === "Pending" || rawPhase === "Ready" || rawPhase === "Failed" || rawPhase === "Terminating" ? rawPhase : "Unknown" + : rawPhase === "Pending" || rawPhase === "Running" || rawPhase === "Failed" || rawPhase === "Terminating" ? rawPhase : "Unknown"; + const resourceId = this.#requiredString(resource.metadata.name, `${collection} name`); + const uid = this.#requiredString(resource.metadata.uid, `${collection} UID`); + const resourceVersion = this.#requiredString(resource.metadata.resourceVersion, `${collection} resourceVersion`); + if (collection === "t4workspaces") { + return { + resourceId, + principalId: this.#requiredString(resource.spec?.owner, "workspace owner"), + uid, + resourceVersion, + generation: BigInt(generation), + observedGeneration: BigInt(observedGeneration), + phase, + ...(deleted ? { deleted: true } : {}), + }; + } + const workspaceId = this.#requiredString(resource.spec?.workspaceRef, "session workspace reference"); + return { + resourceId, + principalId: this.#requiredString(workspaceOwners.get(workspaceId), "session workspace owner"), + workspaceId, + uid, + resourceVersion, + generation: BigInt(generation), + observedGeneration: BigInt(observedGeneration), + phase, + ...(deleted ? { deleted: true } : {}), + }; + } + + #requiredString(value: unknown, name: string): string { + if (typeof value !== "string" || value.length === 0) throw new Error(`Kubernetes ${name} is invalid`); + return value; + } + + #statusIngressLeaseUnavailable(error: unknown): boolean { + return error instanceof Error && ( + error.message === "outbox lease is held by another owner" + || error.message === "Kubernetes status owner lease is no longer current" + ); + } + + async #emitStatus(collection: KubernetesStatusCollection, observation: AuthoritativeKubernetesStatusIngress): Promise { + const ingress = this.#options.statusIngress; + if (!ingress) return false; + try { + await ingress(collection, observation); + return true; + } catch (error) { + if (this.#statusIngressLeaseUnavailable(error)) return false; + throw error; + } + } + + async #ingestWatchStatus(collection: KubernetesStatusCollection, event: KubernetesWatchEvent): Promise { + if (this.#options.statusIngress) { + await this.#emitStatus(collection, this.#statusObservation(collection, event.object, this.#workspaceOwners, event.type === "DELETED")); + } + if (collection === "t4workspaces") { + const workspaceId = this.#requiredString(event.object.metadata.name, "workspace name"); + if (event.type === "DELETED") this.#workspaceOwners.delete(workspaceId); + else { + const owner = this.#requiredString(event.object.spec?.owner, "workspace owner"); + this.#workspaceOwners.set(workspaceId, owner); + } + } } async #run(signal: AbortSignal): Promise { @@ -68,29 +222,52 @@ export class KubernetesProjectionRunner { generation: AbortController, ): Promise { let version = this.#options.projection.resourceVersionFor(resource); + const statusCollection = resource === "t4workspaces" || resource === "t4sessions" ? resource : undefined; while (!generation.signal.aborted) { + let pendingStatus = Promise.resolve(); + let statusFailed = false; + let statusError: unknown; + const applyEvent = (event: KubernetesWatchEvent): void => { + this.#options.projection.applyWatch(event); + version = event.object.metadata.resourceVersion ?? version; + }; + const onEvent = this.#options.statusIngress && statusCollection + ? (event: KubernetesWatchEvent): void => { + pendingStatus = pendingStatus.then(async () => { + if (statusFailed) return; + await this.#ingestWatchStatus(statusCollection, event); + applyEvent(event); + }).catch(error => { + statusFailed = true; + statusError = error; + generation.abort(error); + }); + } + : applyEvent; try { await this.#options.client.watch( resource, version, - event => { - this.#options.projection.applyWatch(event); - version = event.object.metadata.resourceVersion ?? version; - }, + onEvent, generation.signal, () => { this.#synchronized.add(resource); if (this.#synchronized.size === resourceCount) this.#options.onSynchronized?.(); }, ); + await pendingStatus; + if (statusFailed) throw statusError; if (generation.signal.aborted) return; this.#synchronized.delete(resource); this.#options.onError?.(new Error(`Kubernetes ${resource} watch ended`)); - } catch (error) { - if (generation.signal.aborted || rootSignal.aborted) return; + } catch (watchError) { + await pendingStatus; + const error = statusFailed ? statusError : watchError; + if (rootSignal.aborted || generation.signal.aborted && !statusFailed) return; this.#synchronized.delete(resource); this.#options.onError?.(error); - const requiresRelist = error instanceof KubernetesAuthorityInvalidatedError + const requiresRelist = statusFailed + || error instanceof KubernetesAuthorityInvalidatedError || error instanceof KubernetesApiError && error.status === 410; if (requiresRelist) { try { await this.#restartGeneration(generation, rootSignal); } @@ -113,6 +290,7 @@ export class KubernetesProjectionRunner { while (!signal.aborted) { try { const snapshot = await this.#options.client.listInfrastructure(this.#options.hostName, signal); + await this.#prepareSnapshot(snapshot, signal); this.#options.projection.replace(snapshot); return; } catch (error) { diff --git a/packages/cluster-server/src/ledger.ts b/packages/cluster-server/src/ledger.ts new file mode 100644 index 00000000..53af9f4f --- /dev/null +++ b/packages/cluster-server/src/ledger.ts @@ -0,0 +1,729 @@ +import { createHash, createHmac, randomUUID, timingSafeEqual } from "node:crypto"; +import { SQL } from "bun"; + +export interface DurableLedgerOptions { + readonly url: string; + readonly cursorSecret: string; + readonly eventRetention?: number; +} +export interface WorkspaceRecord { + readonly id: string; + readonly name: string; + readonly state: "accepted" | "provisioning" | "ready" | "deleting" | "deleted" | "failed" | "unavailable" | "indeterminate"; + readonly revision: number; + readonly labels?: Readonly>; +} +export interface SessionRecord { + readonly id: string; + readonly workspaceId: string; + readonly title: string; + readonly state: "accepted" | "provisioning" | "ready" | "cancelling" | "cancelled" | "failed" | "unavailable" | "indeterminate"; + readonly revision: number; + readonly labels?: Readonly>; +} +export interface LedgerMutationResult { + readonly kind: "accepted" | "replay" | "idempotency_conflict" | "revision_conflict" | "not_found"; + readonly commandId?: string; + readonly value?: T; +} +export interface DurableEvent { + readonly sequence: bigint; + readonly type: "session" | "command"; + readonly payload: Readonly>; +} +export interface EventWindow { + readonly events: readonly DurableEvent[]; + readonly cursor: string; + readonly expired: boolean; + readonly resyncCursor: string; +} +export interface OutboxClaim { + readonly outboxId: bigint; + readonly commandId: string; + readonly principalId: string; + readonly idempotencyKey: string; + readonly kind: "workspace.create" | "workspace.patch" | "workspace.delete" | "session.create" | "session.patch" | "session.cancel" | "session.delete" | "command.submit"; + readonly targetId: string; + readonly targetRevision: bigint; + readonly mutation: Readonly>; + readonly ownerId: string; + readonly ownerEpoch: bigint; + readonly expiresAt: number; + readonly claimedAt: string; +} +export interface OwnerLease { + readonly ownerId: string; + readonly epoch: bigint; +} +export type KubernetesStatusCollection = "t4workspaces" | "t4sessions"; +export interface AuthoritativeKubernetesStatusObservation { + readonly resourceId: string; + readonly principalId: string; + readonly workspaceId?: string; + readonly uid: string; + readonly resourceVersion: string; + readonly generation: bigint; + readonly observedGeneration: bigint; + readonly phase: "Pending" | "Ready" | "Running" | "Failed" | "Terminating" | "Unknown"; + readonly deleted?: boolean; +} +export interface AuthoritativeKubernetesStatusSnapshot { + readonly relisted: true; + readonly resourceVersion: string; + readonly resourceIds: readonly string[]; +} +export type AuthoritativeKubernetesStatusIngress = AuthoritativeKubernetesStatusObservation | AuthoritativeKubernetesStatusSnapshot; +export interface StaleCreateCleanup { + readonly resourceType: KubernetesStatusCollection; + readonly targetId: string; + readonly uid: string; + readonly resourceVersion: string; +} +export interface StaleCreateCleanupClaim extends StaleCreateCleanup { + readonly cleanupId: bigint; + readonly ownerId: string; + readonly ownerEpoch: bigint; + readonly expiresAt: number; + readonly claimedAt: string; +} + +interface CommandRow { + readonly fingerprint: string; + readonly response_body: unknown; +} +interface WorkspaceRow { + readonly workspace_id: string; + readonly name: string; + readonly state: WorkspaceRecord["state"]; + readonly revision: bigint; + readonly labels: Record; +} +interface SessionRow { + readonly session_id: string; + readonly workspace_id: string; + readonly title: string; + readonly state: SessionRecord["state"]; + readonly revision: bigint; + readonly labels: Record; + readonly cancellation_requested: boolean; + readonly deletion_requested: boolean; +} +interface EventRow { + readonly sequence: bigint; + readonly event_type: "session" | "command"; + readonly payload: Record; +} +interface RetentionRow { readonly first_retained_sequence: bigint; readonly latest_sequence: bigint; } +interface LeaseRow { readonly owner_id: string; readonly epoch: bigint; readonly expired: boolean; } +interface OutboxRow { + readonly outbox_id: bigint; + readonly command_id: string; + readonly principal_id: string; + readonly idempotency_key: string; + readonly mutation_kind: OutboxClaim["kind"]; + readonly target_id: string; + readonly target_revision: bigint; + readonly mutation: Record; +} +interface LockedOutboxRow extends OutboxRow { + readonly state: "pending" | "claimed" | "applied" | "skipped" | "failed"; + readonly owner_id: string | null; + readonly owner_epoch: bigint | null; + readonly claimed_at: string | null; +} +interface CleanupRow { + readonly cleanup_id: bigint; + readonly resource_type: KubernetesStatusCollection; + readonly target_id: string; + readonly uid: string; + readonly resource_version: string; +} +interface MissingWorkspaceStatusRow { readonly workspace_id: string; } +interface MissingSessionStatusRow { readonly session_id: string; readonly principal_id: string; readonly revision: bigint; } +interface KubernetesStatusRow { + readonly state: WorkspaceRecord["state"] | SessionRecord["state"]; + readonly revision: bigint; + readonly generation: bigint; + readonly kube_uid: string | null; + readonly kube_resource_version: string | null; + readonly kube_generation: bigint | null; + readonly kube_observed_generation: bigint | null; + readonly cancellation_requested?: boolean; + readonly deletion_requested: boolean; +} + +const MIGRATION_URL = new URL("../migrations/001_durable_gateway.sql", import.meta.url); +const ROLLBACK_URL = new URL("../migrations/001_durable_gateway.down.sql", import.meta.url); +const OUTBOX_LEASE = "gateway-outbox"; +const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER); +const SERIALIZABLE_ATTEMPTS = 3; +const STALE_APPLY_DEADLINE_SECONDS = 5; + +function persistedBigint(value: unknown, allowZero = false): bigint { + if (typeof value !== "bigint") throw new Error("persisted ledger bigint is invalid"); + if (allowZero ? value < 0n : value <= 0n) throw new Error("persisted ledger bigint is invalid"); + return value; +} +function safeNumber(value: unknown, allowZero = false): number { + const decoded = persistedBigint(value, allowZero); + if (decoded > MAX_SAFE_BIGINT) throw new Error("persisted ledger integer is outside the public API range"); + return Number(decoded); +} +function postgresSqlState(error: unknown): string | undefined { + if (!error || typeof error !== "object") return undefined; + const value = error as { readonly code?: unknown; readonly errno?: unknown }; + if (typeof value.code === "string" && /^[0-9A-Z]{5}$/u.test(value.code)) return value.code; + return typeof value.errno === "string" && /^[0-9A-Z]{5}$/u.test(value.errno) ? value.errno : undefined; +} +function isIdempotencyUniqueRace(error: unknown): boolean { + if (postgresSqlState(error) !== "23505" || !error || typeof error !== "object") return false; + const value = error as { readonly constraint?: unknown; readonly constraint_name?: unknown }; + const constraint = value.constraint ?? value.constraint_name; + return constraint === "t4_commands_principal_id_operation_target_scope_idempotency_key"; +} +function labels(value: Record): Record | undefined { + return Object.keys(value).length > 0 ? value : undefined; +} +function workspace(row: WorkspaceRow): WorkspaceRecord { + return { id: row.workspace_id, name: row.name, state: row.state, revision: safeNumber(row.revision), ...(labels(row.labels) ? { labels: labels(row.labels) } : {}) }; +} +function session(row: SessionRow): SessionRecord { + return { id: row.session_id, workspaceId: row.workspace_id, title: row.title, state: row.state, revision: safeNumber(row.revision), ...(labels(row.labels) ? { labels: labels(row.labels) } : {}) }; +} +function stableJson(value: unknown): string { + if (value === null || typeof value === "boolean" || typeof value === "string") return JSON.stringify(value); + if (typeof value === "number") { + if (!Number.isFinite(value)) throw new Error("canonical JSON cannot contain a non-finite number"); + return JSON.stringify(Object.is(value, -0) ? 0 : value); + } + if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`; + if (value && typeof value === "object") { + return `{${Object.keys(value as Record).sort().map(key => `${JSON.stringify(key)}:${stableJson((value as Record)[key])}`).join(",")}}`; + } + throw new Error("canonical JSON value is invalid"); +} +function semanticJsonEqual(left: unknown, right: unknown): boolean { + try { return stableJson(left) === stableJson(right); } + catch { return false; } +} +export function canonicalRequestFingerprint(value: unknown): string { + return `sha256:${createHash("sha256").update(stableJson(value), "utf8").digest("hex")}`; +} + +export class PostgresLedger { + readonly #sql: SQL; + readonly #cursorSecret: Uint8Array; + readonly #eventRetention: number; + + constructor(options: DurableLedgerOptions) { + const secret = new TextEncoder().encode(options.cursorSecret); + if (secret.byteLength < 32 || secret.byteLength > 4096) throw new Error("cursor secret must contain 32 to 4096 UTF-8 bytes"); + this.#cursorSecret = secret; + this.#eventRetention = options.eventRetention ?? 10_000; + if (!Number.isSafeInteger(this.#eventRetention) || this.#eventRetention < 1 || this.#eventRetention > 100_000) throw new Error("event retention is invalid"); + this.#sql = new SQL(options.url, { bigint: true, max: 8 }); + } + + async migrate(): Promise { + const migration = await Bun.file(MIGRATION_URL).text(); + await this.#serializable(async transaction => { + await transaction`SELECT pg_advisory_xact_lock(741405)`; + await transaction.unsafe(migration); + }); + const rows = await this.#sql<{ version: number }[]>`SELECT version FROM t4_schema_migrations ORDER BY version DESC LIMIT 1`; + if (rows[0]?.version !== 1) throw new Error("durable gateway schema migration did not reach version 1"); + } + async rollback(): Promise { + const rollback = await Bun.file(ROLLBACK_URL).text(); + await this.#serializable(async transaction => { + await transaction`SELECT pg_advisory_xact_lock(741405)`; + await transaction.unsafe(rollback); + }); + } + async close(): Promise { await this.#sql.close(); } + + async createWorkspace(principalId: string, idempotencyKey: string, fingerprint: string, input: { name: string; labels?: Record }): Promise> { + return await this.#serializable(async transaction => { + await this.#lockIdempotency(transaction, principalId, "createWorkspace", "workspaces", idempotencyKey); + const prior = await transaction`SELECT fingerprint, response_body FROM t4_commands WHERE principal_id = ${principalId} AND operation = 'createWorkspace' AND target_scope = 'workspaces' AND idempotency_key = ${idempotencyKey} FOR UPDATE`; + if (prior[0]) return prior[0].fingerprint === fingerprint + ? { kind: "replay", value: prior[0].response_body as WorkspaceRecord } + : { kind: "idempotency_conflict" }; + const commandId = `cmd_${randomUUID()}`; + const workspaceId = `ws-${randomUUID()}`; + const value: WorkspaceRecord = { id: workspaceId, name: input.name, state: "accepted", revision: 1, ...(input.labels ? { labels: input.labels } : {}) }; + await transaction`INSERT INTO t4_commands (command_id, principal_id, operation, target_scope, idempotency_key, fingerprint, lifecycle_state, response_status, response_body) VALUES (${commandId}, ${principalId}, 'createWorkspace', 'workspaces', ${idempotencyKey}, ${fingerprint}, 'accepted', 202, ${value})`; + await transaction`INSERT INTO t4_workspace_intents (workspace_id, principal_id, name, labels, state, revision, generation) VALUES (${workspaceId}, ${principalId}, ${input.name}, ${input.labels ?? {}}, 'accepted', 1, 1)`; + await transaction`INSERT INTO t4_outbox (command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation) VALUES (${commandId}, ${principalId}, ${idempotencyKey}, 'workspace.create', ${workspaceId}, 1, ${value})`; + return { kind: "accepted", commandId, value }; + }); + } + + async getWorkspace(principalId: string, workspaceId: string): Promise { + const rows = await this.#sql`SELECT workspace_id, name, state, revision, labels FROM t4_workspace_intents WHERE principal_id = ${principalId} AND workspace_id = ${workspaceId} AND state <> 'deleted'`; + return rows[0] ? workspace(rows[0]) : undefined; + } + async listWorkspaces(principalId: string, afterId: string | undefined, limit: number): Promise { + const rows = afterId + ? await this.#sql`SELECT workspace_id, name, state, revision, labels FROM t4_workspace_intents WHERE principal_id = ${principalId} AND workspace_id > ${afterId} AND state <> 'deleted' ORDER BY workspace_id LIMIT ${limit}` + : await this.#sql`SELECT workspace_id, name, state, revision, labels FROM t4_workspace_intents WHERE principal_id = ${principalId} AND state <> 'deleted' ORDER BY workspace_id LIMIT ${limit}`; + return rows.map(workspace); + } + + async patchWorkspace(principalId: string, workspaceId: string, expectedRevision: number, idempotencyKey: string, fingerprint: string, input: { name?: string; labels?: Record }): Promise> { + return await this.#serializable(async transaction => { + const scope = `workspace:${workspaceId}`; + await this.#lockIdempotency(transaction, principalId, "mutateWorkspace", scope, idempotencyKey); + const prior = await transaction`SELECT fingerprint, response_body FROM t4_commands WHERE principal_id = ${principalId} AND operation = 'mutateWorkspace' AND target_scope = ${scope} AND idempotency_key = ${idempotencyKey} FOR UPDATE`; + if (prior[0]) return prior[0].fingerprint === fingerprint ? { kind: "replay", value: prior[0].response_body as WorkspaceRecord } : { kind: "idempotency_conflict" }; + const rows = await transaction`SELECT workspace_id, name, state, revision, labels FROM t4_workspace_intents WHERE principal_id = ${principalId} AND workspace_id = ${workspaceId} AND state <> 'deleted' FOR UPDATE`; + const current = rows[0]; + if (!current) return { kind: "not_found" }; + if (current.revision !== BigInt(expectedRevision)) return { kind: "revision_conflict" }; + const revision = current.revision + 1n; + const value: WorkspaceRecord = { id: workspaceId, name: input.name ?? current.name, state: current.state, revision: safeNumber(revision), ...(input.labels ?? labels(current.labels) ? { labels: input.labels ?? current.labels } : {}) }; + const commandId = `cmd_${randomUUID()}`; + await transaction`INSERT INTO t4_commands (command_id, principal_id, operation, target_scope, idempotency_key, fingerprint, lifecycle_state, response_status, response_body) VALUES (${commandId}, ${principalId}, 'mutateWorkspace', ${scope}, ${idempotencyKey}, ${fingerprint}, 'accepted', 200, ${value})`; + await transaction`UPDATE t4_workspace_intents SET name = ${value.name}, labels = ${value.labels ?? {}}, revision = ${revision}, generation = generation + 1, updated_at = clock_timestamp() WHERE workspace_id = ${workspaceId}`; + await transaction`INSERT INTO t4_outbox (command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation) VALUES (${commandId}, ${principalId}, ${idempotencyKey}, 'workspace.patch', ${workspaceId}, ${revision}, ${value})`; + return { kind: "accepted", commandId, value }; + }); + } + + async deleteWorkspace(principalId: string, workspaceId: string, idempotencyKey: string, fingerprint: string): Promise> { + return await this.#serializable(async transaction => { + const scope = `workspace:${workspaceId}`; + await this.#lockIdempotency(transaction, principalId, "deleteWorkspace", scope, idempotencyKey); + const prior = await transaction`SELECT fingerprint, response_body FROM t4_commands WHERE principal_id = ${principalId} AND operation = 'deleteWorkspace' AND target_scope = ${scope} AND idempotency_key = ${idempotencyKey} FOR UPDATE`; + if (prior[0]) return prior[0].fingerprint === fingerprint ? { kind: "replay" } : { kind: "idempotency_conflict" }; + const rows = await transaction`SELECT workspace_id, name, state, revision, labels FROM t4_workspace_intents WHERE principal_id = ${principalId} AND workspace_id = ${workspaceId} AND state <> 'deleted' FOR UPDATE`; + if (!rows[0]) return { kind: "not_found" }; + const revision = rows[0].revision + 1n; + const commandId = `cmd_${randomUUID()}`; + await transaction`INSERT INTO t4_commands (command_id, principal_id, operation, target_scope, idempotency_key, fingerprint, lifecycle_state, response_status) VALUES (${commandId}, ${principalId}, 'deleteWorkspace', ${scope}, ${idempotencyKey}, ${fingerprint}, 'accepted', 204)`; + await transaction`UPDATE t4_workspace_intents SET state = 'deleting', deletion_requested = true, revision = ${revision}, generation = generation + 1, updated_at = clock_timestamp() WHERE workspace_id = ${workspaceId}`; + await transaction`INSERT INTO t4_outbox (command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation) VALUES (${commandId}, ${principalId}, ${idempotencyKey}, 'workspace.delete', ${workspaceId}, ${revision}, ${{ workspaceId }})`; + return { kind: "accepted", commandId }; + }); + } + + async createSession(principalId: string, workspaceId: string, idempotencyKey: string, fingerprint: string, input: { title: string; labels?: Record }): Promise> { + return await this.#serializable(async transaction => { + const scope = `workspace:${workspaceId}:sessions`; + await this.#lockIdempotency(transaction, principalId, "spawnSession", scope, idempotencyKey); + const prior = await transaction`SELECT fingerprint, response_body FROM t4_commands WHERE principal_id = ${principalId} AND operation = 'spawnSession' AND target_scope = ${scope} AND idempotency_key = ${idempotencyKey} FOR UPDATE`; + if (prior[0]) return prior[0].fingerprint === fingerprint ? { kind: "replay", value: prior[0].response_body as SessionRecord } : { kind: "idempotency_conflict" }; + const workspaceRows = await transaction<{ workspace_id: string }[]>`SELECT workspace_id FROM t4_workspace_intents WHERE principal_id = ${principalId} AND workspace_id = ${workspaceId} AND state NOT IN ('deleting','deleted') FOR UPDATE`; + if (!workspaceRows[0]) return { kind: "not_found" }; + const commandId = `cmd_${randomUUID()}`; + const sessionId = `ss-${randomUUID()}`; + const value: SessionRecord = { id: sessionId, workspaceId, title: input.title, state: "accepted", revision: 1, ...(input.labels ? { labels: input.labels } : {}) }; + await transaction`INSERT INTO t4_commands (command_id, principal_id, operation, target_scope, idempotency_key, fingerprint, lifecycle_state, response_status, response_body) VALUES (${commandId}, ${principalId}, 'spawnSession', ${scope}, ${idempotencyKey}, ${fingerprint}, 'accepted', 202, ${value})`; + await transaction`INSERT INTO t4_session_intents (session_id, workspace_id, principal_id, title, labels, state, revision, generation) VALUES (${sessionId}, ${workspaceId}, ${principalId}, ${input.title}, ${input.labels ?? {}}, 'accepted', 1, 1)`; + await transaction`INSERT INTO t4_outbox (command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation) VALUES (${commandId}, ${principalId}, ${idempotencyKey}, 'session.create', ${sessionId}, 1, ${value})`; + await this.#appendEvent(transaction, principalId, sessionId, "session", { state: "accepted", revision: 1 }, 0n); + return { kind: "accepted", commandId, value }; + }); + } + + async getSession(principalId: string, sessionId: string): Promise { + const rows = await this.#sql`SELECT session_id, workspace_id, title, state, revision, labels, cancellation_requested FROM t4_session_intents WHERE principal_id = ${principalId} AND session_id = ${sessionId}`; + return rows[0] ? session(rows[0]) : undefined; + } + async listSessions(principalId: string, workspaceId: string, afterId: string | undefined, limit: number): Promise { + if (!await this.getWorkspace(principalId, workspaceId)) return undefined; + const rows = afterId + ? await this.#sql`SELECT session_id, workspace_id, title, state, revision, labels, cancellation_requested FROM t4_session_intents WHERE principal_id = ${principalId} AND workspace_id = ${workspaceId} AND session_id > ${afterId} ORDER BY session_id LIMIT ${limit}` + : await this.#sql`SELECT session_id, workspace_id, title, state, revision, labels, cancellation_requested FROM t4_session_intents WHERE principal_id = ${principalId} AND workspace_id = ${workspaceId} ORDER BY session_id LIMIT ${limit}`; + return rows.map(session); + } + + async patchSession(principalId: string, sessionId: string, expectedRevision: number, idempotencyKey: string, fingerprint: string, input: { title?: string; labels?: Record }): Promise> { + return await this.#serializable(async transaction => { + const scope = `session:${sessionId}`; + await this.#lockIdempotency(transaction, principalId, "mutateSession", scope, idempotencyKey); + const prior = await transaction`SELECT fingerprint, response_body FROM t4_commands WHERE principal_id = ${principalId} AND operation = 'mutateSession' AND target_scope = ${scope} AND idempotency_key = ${idempotencyKey} FOR UPDATE`; + if (prior[0]) return prior[0].fingerprint === fingerprint ? { kind: "replay", value: prior[0].response_body as SessionRecord } : { kind: "idempotency_conflict" }; + const rows = await transaction`SELECT session_id, workspace_id, title, state, revision, labels, cancellation_requested FROM t4_session_intents WHERE principal_id = ${principalId} AND session_id = ${sessionId} AND state NOT IN ('cancelling','cancelled','failed','unavailable','indeterminate') FOR UPDATE`; + const current = rows[0]; + if (!current) return { kind: "not_found" }; + if (current.revision !== BigInt(expectedRevision)) return { kind: "revision_conflict" }; + const revision = current.revision + 1n; + const value: SessionRecord = { id: sessionId, workspaceId: current.workspace_id, title: input.title ?? current.title, state: current.state, revision: safeNumber(revision), ...(input.labels ?? labels(current.labels) ? { labels: input.labels ?? current.labels } : {}) }; + const commandId = `cmd_${randomUUID()}`; + await transaction`INSERT INTO t4_commands (command_id, principal_id, operation, target_scope, idempotency_key, fingerprint, lifecycle_state, response_status, response_body) VALUES (${commandId}, ${principalId}, 'mutateSession', ${scope}, ${idempotencyKey}, ${fingerprint}, 'accepted', 200, ${value})`; + await transaction`UPDATE t4_session_intents SET title = ${value.title}, labels = ${value.labels ?? {}}, revision = ${revision}, generation = generation + 1, updated_at = clock_timestamp() WHERE session_id = ${sessionId}`; + await transaction`INSERT INTO t4_outbox (command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation) VALUES (${commandId}, ${principalId}, ${idempotencyKey}, 'session.patch', ${sessionId}, ${revision}, ${value})`; + await this.#appendEvent(transaction, principalId, sessionId, "session", { state: value.state, revision: value.revision }, 0n); + return { kind: "accepted", commandId, value }; + }); + } + + async cancelSession(principalId: string, sessionId: string, idempotencyKey: string, fingerprint: string, deletion = false): Promise> { + return await this.#serializable(async transaction => { + const operation = deletion ? "deleteSession" : "cancelSession"; + const scope = `session:${sessionId}`; + await this.#lockIdempotency(transaction, principalId, operation, scope, idempotencyKey); + const prior = await transaction`SELECT fingerprint, response_body FROM t4_commands WHERE principal_id = ${principalId} AND operation = ${operation} AND target_scope = ${scope} AND idempotency_key = ${idempotencyKey} FOR UPDATE`; + if (prior[0]) return prior[0].fingerprint === fingerprint ? { kind: "replay", value: prior[0].response_body as SessionRecord } : { kind: "idempotency_conflict" }; + const rows = await transaction`SELECT session_id, workspace_id, title, state, revision, labels, cancellation_requested, deletion_requested FROM t4_session_intents WHERE principal_id = ${principalId} AND session_id = ${sessionId} FOR UPDATE`; + const current = rows[0]; + if (!current) return { kind: "not_found" }; + if (current.cancellation_requested || current.state === "cancelling" || current.state === "cancelled" || current.state === "failed" || current.state === "unavailable" || current.state === "indeterminate") { + const value = session(current); + const commandId = `cmd_${randomUUID()}`; + const dispatchDeletion = deletion && !current.deletion_requested; + await transaction`INSERT INTO t4_commands (command_id, principal_id, operation, target_scope, idempotency_key, fingerprint, lifecycle_state, response_status, response_body) VALUES (${commandId}, ${principalId}, ${operation}, ${scope}, ${idempotencyKey}, ${fingerprint}, ${dispatchDeletion ? "accepted" : "projected"}, ${deletion ? 204 : 202}, ${value})`; + if (dispatchDeletion) { + await transaction`UPDATE t4_session_intents SET deletion_requested = true, generation = generation + 1, updated_at = clock_timestamp() WHERE session_id = ${sessionId}`; + await transaction`INSERT INTO t4_outbox (command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation) VALUES (${commandId}, ${principalId}, ${idempotencyKey}, 'session.delete', ${sessionId}, ${current.revision}, ${{ sessionId, revision: safeNumber(current.revision) }})`; + } + await this.#terminalizePendingSessionCommands(transaction, principalId, sessionId, "session cancelled or deleted", 0n); + await this.#appendEvent(transaction, principalId, sessionId, "session", { state: current.state, revision: safeNumber(current.revision) }, 0n); + return { kind: "accepted", commandId, value }; + } + const revision = current.revision + 1n; + const value: SessionRecord = { id: sessionId, workspaceId: current.workspace_id, title: current.title, state: "cancelling", revision: safeNumber(revision), ...(labels(current.labels) ? { labels: current.labels } : {}) }; + const commandId = `cmd_${randomUUID()}`; + await transaction`INSERT INTO t4_commands (command_id, principal_id, operation, target_scope, idempotency_key, fingerprint, lifecycle_state, response_status, response_body) VALUES (${commandId}, ${principalId}, ${operation}, ${scope}, ${idempotencyKey}, ${fingerprint}, 'accepted', ${deletion ? 204 : 202}, ${value})`; + await transaction`UPDATE t4_session_intents SET state = 'cancelling', cancellation_requested = true, deletion_requested = deletion_requested OR ${deletion}, revision = ${revision}, generation = generation + 1, updated_at = clock_timestamp() WHERE session_id = ${sessionId}`; + await transaction`UPDATE t4_outbox SET state = 'skipped', owner_id = NULL, owner_epoch = NULL, claimed_at = NULL, terminal_result = ${{ reason: "superseded by cancellation" }}, updated_at = clock_timestamp() WHERE target_id = ${sessionId} AND mutation_kind = 'session.create' AND state IN ('pending','claimed')`; + await this.#terminalizePendingSessionCommands(transaction, principalId, sessionId, "session cancelled or deleted", 0n); + await transaction`INSERT INTO t4_outbox (command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation) VALUES (${commandId}, ${principalId}, ${idempotencyKey}, ${deletion ? "session.delete" : "session.cancel"}, ${sessionId}, ${revision}, ${{ sessionId, revision: safeNumber(revision) }})`; + await this.#appendEvent(transaction, principalId, sessionId, "session", { state: "cancelling", revision: safeNumber(revision) }, 0n); + return { kind: "accepted", commandId, value }; + }); + } + + async submitCommand(principalId: string, sessionId: string, idempotencyKey: string, fingerprint: string, input: { command: string; metadata: Record }): Promise> { + return await this.#serializable(async transaction => { + const scope = `session:${sessionId}:commands`; + await this.#lockIdempotency(transaction, principalId, "submitCommand", scope, idempotencyKey); + const prior = await transaction`SELECT fingerprint, response_body FROM t4_commands WHERE principal_id = ${principalId} AND operation = 'submitCommand' AND target_scope = ${scope} AND idempotency_key = ${idempotencyKey} FOR UPDATE`; + if (prior[0]) return prior[0].fingerprint === fingerprint ? { kind: "replay", value: prior[0].response_body as { commandId: string; state: "accepted" } } : { kind: "idempotency_conflict" }; + const sessionRows = await transaction<{ session_id: string }[]>`SELECT session_id FROM t4_session_intents WHERE principal_id = ${principalId} AND session_id = ${sessionId} AND state NOT IN ('cancelling','cancelled','failed','unavailable','indeterminate') FOR UPDATE`; + if (!sessionRows[0]) return { kind: "not_found" }; + const commandId = `cmd_${randomUUID()}`; + const value = { commandId, state: "accepted" as const }; + await transaction`INSERT INTO t4_commands (command_id, principal_id, operation, target_scope, idempotency_key, fingerprint, lifecycle_state, response_status, response_body) VALUES (${commandId}, ${principalId}, 'submitCommand', ${scope}, ${idempotencyKey}, ${fingerprint}, 'accepted', 202, ${value})`; + await transaction`INSERT INTO t4_outbox (command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation) SELECT ${commandId}, ${principalId}, ${idempotencyKey}, 'command.submit', ${sessionId}, revision, ${{ sessionId, ...input }} FROM t4_session_intents WHERE session_id = ${sessionId}`; + const entryRows = await transaction<{ next: bigint }[]>`SELECT COALESCE(MAX(entry_sequence), -1) + 1 AS next FROM t4_snapshot_entries WHERE principal_id = ${principalId} AND session_id = ${sessionId}`; + await transaction`INSERT INTO t4_snapshot_entries (principal_id, session_id, entry_sequence, kind, text_value) VALUES (${principalId}, ${sessionId}, ${entryRows[0]?.next ?? 0n}, 'input', ${input.command})`; + await this.#appendEvent(transaction, principalId, sessionId, "command", { commandId, state: "accepted" }, 0n); + return { kind: "accepted", commandId, value }; + }); + } + + async snapshot(principalId: string, sessionId: string): Promise<{ session: SessionRecord; cursor: string; entries: readonly { sequence: number; kind: "input" | "output" | "status"; text: string }[] } | undefined> { + return await this.#sql.begin("ISOLATION LEVEL REPEATABLE READ", async rawTransaction => { + const transaction = rawTransaction as unknown as SQL; + const sessionRows = await transaction`SELECT session_id, workspace_id, title, state, revision, labels, cancellation_requested FROM t4_session_intents WHERE principal_id = ${principalId} AND session_id = ${sessionId}`; + if (!sessionRows[0]) return undefined; + const rows = await transaction<{ entry_sequence: bigint; kind: "input" | "output" | "status"; text_value: string }[]>`SELECT entry_sequence, kind, text_value FROM t4_snapshot_entries WHERE principal_id = ${principalId} AND session_id = ${sessionId} ORDER BY entry_sequence DESC LIMIT 1000`; + const retention = await transaction`SELECT first_retained_sequence, latest_sequence FROM t4_event_retention WHERE principal_id = ${principalId} AND session_id = ${sessionId}`; + const latest = retention[0] ? persistedBigint(retention[0].latest_sequence, true) : 0n; + return { session: session(sessionRows[0]), cursor: this.watchCursor(principalId, sessionId, latest), entries: rows.reverse().map(row => ({ sequence: safeNumber(row.entry_sequence, true), kind: row.kind, text: row.text_value })) }; + }); + } + + async eventWindow(principalId: string, sessionId: string, cursor: string | undefined, limit: number): Promise { + let after = 0n; + if (cursor !== undefined) { + const decoded = this.#decodeCursor(cursor, "watch", principalId, sessionId); + if (typeof decoded !== "bigint") throw new Error("invalid_cursor"); + after = decoded; + } + return await this.#sql.begin("ISOLATION LEVEL REPEATABLE READ", async rawTransaction => { + const transaction = rawTransaction as unknown as SQL; + const sessionRows = await transaction<{ session_id: string }[]>`SELECT session_id FROM t4_session_intents WHERE principal_id = ${principalId} AND session_id = ${sessionId}`; + if (!sessionRows[0]) return undefined; + const retentionRows = await transaction`SELECT first_retained_sequence, latest_sequence FROM t4_event_retention WHERE principal_id = ${principalId} AND session_id = ${sessionId}`; + const first = retentionRows[0] ? persistedBigint(retentionRows[0].first_retained_sequence, true) : 0n; + const latest = retentionRows[0] ? persistedBigint(retentionRows[0].latest_sequence, true) : 0n; + const resyncCursor = this.watchCursor(principalId, sessionId, latest); + if (cursor !== undefined && first > 0n && after < first - 1n) return { events: [], cursor: resyncCursor, expired: true, resyncCursor }; + const rows = await transaction`SELECT sequence, event_type, payload FROM t4_events WHERE principal_id = ${principalId} AND session_id = ${sessionId} AND sequence > ${after} ORDER BY sequence LIMIT ${limit}`; + const delivered = rows.at(-1) ? persistedBigint(rows.at(-1)!.sequence) : after; + return { events: rows.map(row => ({ sequence: persistedBigint(row.sequence), type: row.event_type, payload: row.payload })), cursor: this.watchCursor(principalId, sessionId, delivered), expired: false, resyncCursor }; + }); + } + + watchCursor(principalId: string, sessionId: string, sequence: bigint): string { return this.#cursor("watch", principalId, sessionId, sequence); } + pageCursor(principalId: string, scope: string, id: string): string { return this.#cursor("page", principalId, scope, id); } + readPageCursor(cursor: string, principalId: string, scope: string): string | undefined { + const result = this.#decodeCursor(cursor, "page", principalId, scope); + return typeof result === "string" ? result : undefined; + } + + async tryAcquireLease(ownerId: string, leaseSeconds = 30): Promise { + if (!ownerId.trim() || Buffer.byteLength(ownerId, "utf8") > 256) throw new Error("outbox owner id is invalid"); + if (!Number.isSafeInteger(leaseSeconds) || leaseSeconds <= 0) throw new Error("outbox lease duration is invalid"); + const rows = await this.#serializable(async transaction => { + const current = await transaction`SELECT owner_id, epoch, expires_at <= clock_timestamp() AS expired FROM t4_owner_leases WHERE lease_name = ${OUTBOX_LEASE} FOR UPDATE`; + if (!current[0]) return await transaction`INSERT INTO t4_owner_leases (lease_name, owner_id, epoch, expires_at) VALUES (${OUTBOX_LEASE}, ${ownerId}, 1, clock_timestamp() + (${leaseSeconds} * interval '1 second')) RETURNING owner_id, epoch, false AS expired`; + if (current[0].owner_id === ownerId && !current[0].expired) return await transaction`UPDATE t4_owner_leases SET expires_at = clock_timestamp() + (${leaseSeconds} * interval '1 second'), updated_at = clock_timestamp() WHERE lease_name = ${OUTBOX_LEASE} RETURNING owner_id, epoch, false AS expired`; + if (!current[0].expired) return undefined; + return await transaction`UPDATE t4_owner_leases SET owner_id = ${ownerId}, epoch = epoch + 1, expires_at = clock_timestamp() + (${leaseSeconds} * interval '1 second'), updated_at = clock_timestamp() WHERE lease_name = ${OUTBOX_LEASE} RETURNING owner_id, epoch, false AS expired`; + }); + return rows?.[0] ? { ownerId: rows[0].owner_id, epoch: persistedBigint(rows[0].epoch) } : undefined; + } + + async acquireLease(ownerId: string, leaseSeconds = 30): Promise { + const lease = await this.tryAcquireLease(ownerId, leaseSeconds); + if (!lease) throw new Error("outbox lease is held by another owner"); + return lease; + } + + async claimNext(lease: OwnerLease): Promise { + return await this.#serializable(async transaction => { + const current = await transaction`SELECT owner_id, epoch, expires_at <= clock_timestamp() AS expired FROM t4_owner_leases WHERE lease_name = ${OUTBOX_LEASE} FOR UPDATE`; + if (current[0]?.owner_id !== lease.ownerId || current[0].epoch !== lease.epoch || current[0].expired) return undefined; + const rows = await transaction`SELECT outbox_id, command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation FROM t4_outbox WHERE (state = 'pending' OR (state = 'claimed' AND owner_epoch < ${lease.epoch}) OR (state = 'claimed' AND owner_id = ${lease.ownerId} AND owner_epoch = ${lease.epoch} AND claimed_at <= clock_timestamp() - (${STALE_APPLY_DEADLINE_SECONDS} * interval '1 second'))) AND next_attempt_at <= clock_timestamp() ORDER BY outbox_id FOR UPDATE SKIP LOCKED LIMIT 1`; + const row = rows[0]; + if (!row) return undefined; + let superseded = false; + if (row.mutation_kind.startsWith("session.")) { + const intent = await transaction<{ revision: bigint; cancellation_requested: boolean }[]>`SELECT revision, cancellation_requested FROM t4_session_intents WHERE session_id = ${row.target_id}`; + superseded = !intent[0] || intent[0].revision !== row.target_revision || row.mutation_kind === "session.create" && intent[0].cancellation_requested; + } else if (row.mutation_kind === "command.submit") { + const intent = await transaction<{ state: SessionRecord["state"]; cancellation_requested: boolean; deletion_requested: boolean }[]>`SELECT state, cancellation_requested, deletion_requested FROM t4_session_intents WHERE session_id = ${row.target_id}`; + if (intent[0] && (intent[0].cancellation_requested || intent[0].deletion_requested || intent[0].state === "cancelling" || intent[0].state === "cancelled")) { + await this.#terminalizePendingSessionCommands(transaction, row.principal_id, row.target_id, "session cancelled or deleted", lease.epoch); + return undefined; + } + } else if (row.mutation_kind.startsWith("workspace.")) { + const intent = await transaction<{ revision: bigint; deletion_requested: boolean }[]>`SELECT revision, deletion_requested FROM t4_workspace_intents WHERE workspace_id = ${row.target_id}`; + superseded = !intent[0] || intent[0].revision !== row.target_revision || row.mutation_kind === "workspace.create" && intent[0].deletion_requested; + } + if (superseded) { + await transaction`UPDATE t4_outbox SET state = 'skipped', owner_id = NULL, owner_epoch = NULL, claimed_at = NULL, terminal_result = ${{ reason: "superseded intent", ownerId: lease.ownerId, ownerEpoch: lease.epoch.toString() }}, updated_at = clock_timestamp() WHERE outbox_id = ${row.outbox_id}`; + return undefined; + } + const claimed = await transaction<{ claimed_at: string; apply_expires_at_ms: bigint }[]>`UPDATE t4_outbox SET state = 'claimed', owner_id = ${lease.ownerId}, owner_epoch = ${lease.epoch}, attempts = attempts + 1, claimed_at = clock_timestamp(), updated_at = clock_timestamp() WHERE outbox_id = ${row.outbox_id} RETURNING floor(extract(epoch FROM claimed_at) * 1000000)::numeric::text AS claimed_at, floor(extract(epoch FROM LEAST(claimed_at + (${STALE_APPLY_DEADLINE_SECONDS} * interval '1 second'), (SELECT expires_at FROM t4_owner_leases WHERE lease_name = ${OUTBOX_LEASE}))) * 1000)::bigint AS apply_expires_at_ms`; + return { outboxId: row.outbox_id, commandId: row.command_id, principalId: row.principal_id, idempotencyKey: row.idempotency_key, kind: row.mutation_kind, targetId: row.target_id, targetRevision: row.target_revision, mutation: row.mutation, ownerId: lease.ownerId, ownerEpoch: lease.epoch, expiresAt: safeNumber(claimed[0]!.apply_expires_at_ms), claimedAt: claimed[0]!.claimed_at }; + }); + } + + async claimIsCurrent(claim: OutboxClaim): Promise { + const rows = await this.#sql<{ mutation: Record }[]>`SELECT item.mutation FROM t4_owner_leases lease JOIN t4_outbox item ON item.outbox_id = ${claim.outboxId} WHERE lease.lease_name = ${OUTBOX_LEASE} AND lease.owner_id = ${claim.ownerId} AND lease.epoch = ${claim.ownerEpoch} AND lease.expires_at > clock_timestamp() AND item.state = 'claimed' AND item.owner_id = lease.owner_id AND item.owner_epoch = lease.epoch AND floor(extract(epoch FROM item.claimed_at) * 1000000)::numeric::text = ${claim.claimedAt} AND item.command_id = ${claim.commandId} AND item.principal_id = ${claim.principalId} AND item.idempotency_key = ${claim.idempotencyKey} AND item.mutation_kind = ${claim.kind} AND item.target_id = ${claim.targetId} AND item.target_revision = ${claim.targetRevision} LIMIT 1`; + return rows[0] !== undefined && semanticJsonEqual(rows[0].mutation, claim.mutation); + } + + async acknowledge(claim: OutboxClaim): Promise { + return await this.#serializable(async transaction => { + const lease = await transaction`SELECT owner_id, epoch, expires_at <= clock_timestamp() AS expired FROM t4_owner_leases WHERE lease_name = ${OUTBOX_LEASE} FOR UPDATE`; + if (lease[0]?.owner_id !== claim.ownerId || lease[0].epoch !== claim.ownerEpoch || lease[0].expired) return false; + const rows = await transaction`SELECT outbox_id, command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation, state, owner_id, owner_epoch, floor(extract(epoch FROM claimed_at) * 1000000)::numeric::text AS claimed_at FROM t4_outbox WHERE outbox_id = ${claim.outboxId} FOR UPDATE`; + const row = rows[0]; + if (!row || row.state !== "claimed" || row.owner_id !== claim.ownerId || row.owner_epoch !== claim.ownerEpoch || row.claimed_at !== claim.claimedAt + || row.command_id !== claim.commandId || row.principal_id !== claim.principalId || row.idempotency_key !== claim.idempotencyKey + || row.mutation_kind !== claim.kind || row.target_id !== claim.targetId || row.target_revision !== claim.targetRevision + || !semanticJsonEqual(row.mutation, claim.mutation)) return false; + await transaction`UPDATE t4_outbox SET state = 'applied', owner_id = NULL, owner_epoch = NULL, claimed_at = NULL, terminal_result = ${{ applied: true, ownerId: claim.ownerId, ownerEpoch: claim.ownerEpoch.toString() }}, updated_at = clock_timestamp() WHERE outbox_id = ${row.outbox_id}`; + await transaction`UPDATE t4_stale_create_cleanups SET state = 'applied', owner_id = NULL, owner_epoch = NULL, claimed_at = NULL, last_error = NULL, updated_at = clock_timestamp() WHERE outbox_id = ${row.outbox_id} AND state <> 'applied'`; + const projected = await transaction<{ command_id: string }[]>`UPDATE t4_commands SET lifecycle_state = 'projected', updated_at = clock_timestamp() WHERE command_id = ${row.command_id} AND lifecycle_state = 'accepted' RETURNING command_id`; + if (projected[0] && row.mutation_kind === "command.submit") await this.#appendEvent(transaction, row.principal_id, row.target_id, "command", { commandId: row.command_id, state: "projected" }, claim.ownerEpoch); + if (row.mutation_kind === "workspace.create") await transaction`UPDATE t4_workspace_intents SET state = 'provisioning', updated_at = clock_timestamp() WHERE workspace_id = ${row.target_id} AND revision = ${row.target_revision} AND state = 'accepted' AND deletion_requested = false`; + if (row.mutation_kind === "session.create") { + const transitioned = await transaction<{ session_id: string }[]>`UPDATE t4_session_intents SET state = 'provisioning', updated_at = clock_timestamp() WHERE session_id = ${row.target_id} AND revision = ${row.target_revision} AND state = 'accepted' AND cancellation_requested = false RETURNING session_id`; + if (transitioned[0]) await this.#appendEvent(transaction, row.principal_id, row.target_id, "session", { state: "provisioning", revision: safeNumber(row.target_revision) }, claim.ownerEpoch); + } + return true; + }); + } + + async recordFailure(claim: OutboxClaim, message: string): Promise { + return await this.#serializable(async transaction => { + const lease = await transaction`SELECT owner_id, epoch, expires_at <= clock_timestamp() AS expired FROM t4_owner_leases WHERE lease_name = ${OUTBOX_LEASE} FOR UPDATE`; + if (lease[0]?.owner_id !== claim.ownerId || lease[0].epoch !== claim.ownerEpoch || lease[0].expired) return false; + const rows = await transaction`SELECT outbox_id, command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation, state, owner_id, owner_epoch, floor(extract(epoch FROM claimed_at) * 1000000)::numeric::text AS claimed_at FROM t4_outbox WHERE outbox_id = ${claim.outboxId} FOR UPDATE`; + const row = rows[0]; + if (!row || row.state !== "claimed" || row.owner_id !== claim.ownerId || row.owner_epoch !== claim.ownerEpoch || row.claimed_at !== claim.claimedAt + || row.command_id !== claim.commandId || row.principal_id !== claim.principalId || row.idempotency_key !== claim.idempotencyKey + || row.mutation_kind !== claim.kind || row.target_id !== claim.targetId || row.target_revision !== claim.targetRevision + || !semanticJsonEqual(row.mutation, claim.mutation)) return false; + await transaction`UPDATE t4_outbox SET state = 'pending', owner_id = NULL, owner_epoch = NULL, claimed_at = NULL, last_error = ${message.slice(0, 1024)}, next_attempt_at = clock_timestamp() + interval '1 second', updated_at = clock_timestamp() WHERE outbox_id = ${row.outbox_id}`; + return true; + }); + } + + async ingestKubernetesStatus(lease: OwnerLease, collection: KubernetesStatusCollection, ingress: AuthoritativeKubernetesStatusIngress): Promise { + if (collection !== "t4workspaces" && collection !== "t4sessions") throw new Error("Kubernetes status collection is invalid"); + if ("relisted" in ingress) { + if (!ingress.resourceVersion || Buffer.byteLength(ingress.resourceVersion, "utf8") > 256 + || !Array.isArray(ingress.resourceIds) || ingress.resourceIds.some(resourceId => typeof resourceId !== "string" || !resourceId || Buffer.byteLength(resourceId, "utf8") > 256)) throw new Error("Kubernetes status relist identity is invalid"); + const present = new Set(ingress.resourceIds); + await this.#serializable(async transaction => { + const currentLease = await transaction`SELECT owner_id, epoch, expires_at <= clock_timestamp() AS expired FROM t4_owner_leases WHERE lease_name = ${OUTBOX_LEASE} FOR UPDATE`; + if (currentLease[0]?.owner_id !== lease.ownerId || currentLease[0].epoch !== lease.epoch || currentLease[0].expired) throw new Error("Kubernetes status owner lease is no longer current"); + if (collection === "t4workspaces") { + const missing = await transaction`SELECT workspace_id FROM t4_workspace_intents WHERE state = 'deleting' AND deletion_requested = true FOR UPDATE`; + for (const row of missing) { + if (!present.has(row.workspace_id)) await transaction`UPDATE t4_workspace_intents SET state = 'deleted', updated_at = clock_timestamp() WHERE workspace_id = ${row.workspace_id} AND state = 'deleting'`; + } + } else { + const missing = await transaction`SELECT session_id, principal_id, revision FROM t4_session_intents WHERE state <> 'cancelled' AND (state = 'cancelling' AND cancellation_requested = true OR deletion_requested = true) FOR UPDATE`; + for (const row of missing) { + if (present.has(row.session_id)) continue; + const transitioned = await transaction<{ session_id: string }[]>`UPDATE t4_session_intents SET state = 'cancelled', updated_at = clock_timestamp() WHERE session_id = ${row.session_id} AND state <> 'cancelled' RETURNING session_id`; + if (transitioned[0]) { + await this.#terminalizePendingSessionCommands(transaction, row.principal_id, row.session_id, "session absent from authoritative relist", lease.epoch); + await this.#appendEvent(transaction, row.principal_id, row.session_id, "session", { state: "cancelled", revision: safeNumber(row.revision) }, lease.epoch); + } + } + } + await transaction`INSERT INTO t4_kubernetes_status_cursors (collection, resource_version, owner_epoch) VALUES (${collection}, ${ingress.resourceVersion}, ${lease.epoch}) ON CONFLICT (collection) DO UPDATE SET resource_version = EXCLUDED.resource_version, owner_epoch = EXCLUDED.owner_epoch, updated_at = clock_timestamp()`; + }); + return; + } + const observation = ingress; + if (!observation.resourceId || !observation.principalId || !observation.uid || !observation.resourceVersion || Buffer.byteLength(observation.uid, "utf8") > 256 || Buffer.byteLength(observation.resourceVersion, "utf8") > 256) throw new Error("Kubernetes status identity is incomplete or unbounded"); + if (typeof observation.generation !== "bigint" || observation.generation <= 0n || typeof observation.observedGeneration !== "bigint" || observation.observedGeneration < 0n) throw new Error("Kubernetes status generation is invalid"); + if (collection === "t4sessions" && !observation.workspaceId) throw new Error("Kubernetes session status lacks workspace identity"); + await this.#serializable(async transaction => { + const currentLease = await transaction`SELECT owner_id, epoch, expires_at <= clock_timestamp() AS expired FROM t4_owner_leases WHERE lease_name = ${OUTBOX_LEASE} FOR UPDATE`; + if (currentLease[0]?.owner_id !== lease.ownerId || currentLease[0].epoch !== lease.epoch || currentLease[0].expired) throw new Error("Kubernetes status owner lease is no longer current"); + const rows = collection === "t4workspaces" + ? await transaction`SELECT state, revision, generation, kube_uid, kube_resource_version, kube_generation, kube_observed_generation, deletion_requested FROM t4_workspace_intents WHERE workspace_id = ${observation.resourceId} AND principal_id = ${observation.principalId} FOR UPDATE` + : await transaction`SELECT state, revision, generation, kube_uid, kube_resource_version, kube_generation, kube_observed_generation, cancellation_requested, deletion_requested FROM t4_session_intents WHERE session_id = ${observation.resourceId} AND principal_id = ${observation.principalId} AND workspace_id = ${observation.workspaceId!} FOR UPDATE`; + const row = rows[0]; + await transaction`INSERT INTO t4_kubernetes_status_cursors (collection, resource_version, owner_epoch) VALUES (${collection}, ${observation.resourceVersion}, ${lease.epoch}) ON CONFLICT (collection) DO UPDATE SET resource_version = EXCLUDED.resource_version, owner_epoch = EXCLUDED.owner_epoch, updated_at = clock_timestamp()`; + if (!row) return; + if (row.kube_uid !== null && row.kube_uid !== observation.uid) throw new Error("Kubernetes status UID conflicts with the durable resource identity"); + if (row.kube_resource_version === observation.resourceVersion && !observation.deleted) return; + if (!observation.deleted && (row.kube_generation !== null && observation.generation < row.kube_generation + || row.kube_generation === observation.generation && row.kube_observed_generation !== null && observation.observedGeneration < row.kube_observed_generation)) return; + const statusIsCurrent = observation.observedGeneration >= observation.generation && observation.generation >= row.generation; + if (collection === "t4workspaces") { + const absorbed = row.deletion_requested || row.state === "deleting" || row.state === "deleted"; + const mapped: WorkspaceRecord["state"] = observation.phase === "Pending" ? "provisioning" + : observation.phase === "Ready" || observation.phase === "Running" ? "ready" + : observation.phase === "Failed" ? "failed" + : observation.phase === "Terminating" ? "deleting" : "indeterminate"; + const nextState: WorkspaceRecord["state"] = observation.deleted ? "deleted" : statusIsCurrent && !absorbed ? mapped : row.state as WorkspaceRecord["state"]; + await transaction`UPDATE t4_workspace_intents SET kube_uid = ${observation.uid}, kube_resource_version = ${observation.resourceVersion}, kube_generation = ${observation.generation}, kube_observed_generation = ${observation.observedGeneration}, state = ${nextState}, updated_at = clock_timestamp() WHERE workspace_id = ${observation.resourceId}`; + } else { + const absorbed = row.cancellation_requested === true || row.deletion_requested || row.state === "cancelling" || row.state === "cancelled"; + const mapped: SessionRecord["state"] = observation.phase === "Pending" ? "provisioning" + : observation.phase === "Ready" || observation.phase === "Running" ? "ready" + : observation.phase === "Failed" ? "failed" + : observation.phase === "Terminating" ? "cancelling" : "indeterminate"; + const nextState: SessionRecord["state"] = observation.deleted ? "cancelled" : statusIsCurrent && !absorbed ? mapped : row.state as SessionRecord["state"]; + await transaction`UPDATE t4_session_intents SET kube_uid = ${observation.uid}, kube_resource_version = ${observation.resourceVersion}, kube_generation = ${observation.generation}, kube_observed_generation = ${observation.observedGeneration}, state = ${nextState}, updated_at = clock_timestamp() WHERE session_id = ${observation.resourceId}`; + if (nextState !== row.state) { + if (nextState === "cancelled") await this.#terminalizePendingSessionCommands(transaction, observation.principalId, observation.resourceId, "session deleted in Kubernetes", lease.epoch); + await this.#appendEvent(transaction, observation.principalId, observation.resourceId, "session", { state: nextState, revision: safeNumber(row.revision) }, lease.epoch); + } + } + }); + } + + async persistStaleCreateCleanup(claim: OutboxClaim, cleanup: StaleCreateCleanup): Promise { + if ((cleanup.resourceType !== "t4workspaces" && cleanup.resourceType !== "t4sessions") || !cleanup.targetId || !cleanup.uid || !cleanup.resourceVersion + || Buffer.byteLength(cleanup.uid, "utf8") > 256 || Buffer.byteLength(cleanup.resourceVersion, "utf8") > 256) return false; + return await this.#serializable(async transaction => { + const rows = await transaction`SELECT outbox_id, command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation, state, owner_id, owner_epoch, floor(extract(epoch FROM claimed_at) * 1000000)::numeric::text AS claimed_at FROM t4_outbox WHERE outbox_id = ${claim.outboxId} FOR UPDATE`; + const row = rows[0]; + const resourceType = row?.mutation_kind === "workspace.create" ? "t4workspaces" : row?.mutation_kind === "session.create" ? "t4sessions" : undefined; + if (!row || row.state === "applied" || resourceType !== cleanup.resourceType || row.target_id !== cleanup.targetId || row.command_id !== claim.commandId || row.principal_id !== claim.principalId + || row.idempotency_key !== claim.idempotencyKey || row.mutation_kind !== claim.kind || row.target_id !== claim.targetId || row.target_revision !== claim.targetRevision + || !semanticJsonEqual(row.mutation, claim.mutation)) return false; + await transaction`INSERT INTO t4_stale_create_cleanups (outbox_id, resource_type, target_id, uid, resource_version) VALUES (${row.outbox_id}, ${cleanup.resourceType}, ${cleanup.targetId}, ${cleanup.uid}, ${cleanup.resourceVersion}) ON CONFLICT (outbox_id, uid) DO UPDATE SET resource_version = EXCLUDED.resource_version, state = CASE WHEN t4_stale_create_cleanups.state = 'applied' THEN 'applied' ELSE 'pending' END, owner_id = NULL, owner_epoch = NULL, claimed_at = NULL, next_attempt_at = clock_timestamp(), updated_at = clock_timestamp()`; + return true; + }); + } + + async claimStaleCreateCleanup(lease: OwnerLease): Promise { + return await this.#serializable(async transaction => { + const current = await transaction`SELECT owner_id, epoch, expires_at <= clock_timestamp() AS expired FROM t4_owner_leases WHERE lease_name = ${OUTBOX_LEASE} FOR UPDATE`; + if (current[0]?.owner_id !== lease.ownerId || current[0].epoch !== lease.epoch || current[0].expired) return undefined; + const rows = await transaction`SELECT cleanup.cleanup_id, cleanup.resource_type, cleanup.target_id, cleanup.uid, cleanup.resource_version FROM t4_stale_create_cleanups cleanup JOIN t4_outbox item ON item.outbox_id = cleanup.outbox_id WHERE item.state <> 'applied' AND (cleanup.state = 'pending' OR (cleanup.state = 'claimed' AND cleanup.owner_epoch < ${lease.epoch}) OR (cleanup.state = 'claimed' AND cleanup.owner_id = ${lease.ownerId} AND cleanup.owner_epoch = ${lease.epoch} AND cleanup.claimed_at <= clock_timestamp() - (${STALE_APPLY_DEADLINE_SECONDS} * interval '1 second'))) AND cleanup.next_attempt_at <= clock_timestamp() ORDER BY cleanup.cleanup_id FOR UPDATE OF cleanup SKIP LOCKED LIMIT 1`; + const row = rows[0]; + if (!row) return undefined; + const claimed = await transaction<{ claimed_at: string; apply_expires_at_ms: bigint }[]>`UPDATE t4_stale_create_cleanups SET state = 'claimed', owner_id = ${lease.ownerId}, owner_epoch = ${lease.epoch}, attempts = attempts + 1, claimed_at = clock_timestamp(), updated_at = clock_timestamp() WHERE cleanup_id = ${row.cleanup_id} RETURNING floor(extract(epoch FROM claimed_at) * 1000000)::numeric::text AS claimed_at, floor(extract(epoch FROM LEAST(claimed_at + (${STALE_APPLY_DEADLINE_SECONDS} * interval '1 second'), (SELECT expires_at FROM t4_owner_leases WHERE lease_name = ${OUTBOX_LEASE}))) * 1000)::bigint AS apply_expires_at_ms`; + return { cleanupId: row.cleanup_id, resourceType: row.resource_type, targetId: row.target_id, uid: row.uid, resourceVersion: row.resource_version, ownerId: lease.ownerId, ownerEpoch: lease.epoch, expiresAt: safeNumber(claimed[0]!.apply_expires_at_ms), claimedAt: claimed[0]!.claimed_at }; + }); + } + + async staleCreateCleanupClaimIsCurrent(claim: StaleCreateCleanupClaim): Promise { + const rows = await this.#sql<{ valid: boolean }[]>`SELECT EXISTS (SELECT 1 FROM t4_owner_leases lease JOIN t4_stale_create_cleanups cleanup ON cleanup.cleanup_id = ${claim.cleanupId} JOIN t4_outbox item ON item.outbox_id = cleanup.outbox_id WHERE lease.lease_name = ${OUTBOX_LEASE} AND lease.owner_id = ${claim.ownerId} AND lease.epoch = ${claim.ownerEpoch} AND lease.expires_at > clock_timestamp() AND item.state <> 'applied' AND cleanup.state = 'claimed' AND cleanup.owner_id = lease.owner_id AND cleanup.owner_epoch = lease.epoch AND floor(extract(epoch FROM cleanup.claimed_at) * 1000000)::numeric::text = ${claim.claimedAt} AND cleanup.resource_type = ${claim.resourceType} AND cleanup.target_id = ${claim.targetId} AND cleanup.uid = ${claim.uid} AND cleanup.resource_version = ${claim.resourceVersion}) AS valid`; + return rows[0]?.valid === true; + } + + async acknowledgeStaleCreateCleanup(claim: StaleCreateCleanupClaim): Promise { + return await this.#finishStaleCreateCleanup(claim, undefined); + } + + async recordStaleCreateCleanupFailure(claim: StaleCreateCleanupClaim, message: string): Promise { + return await this.#finishStaleCreateCleanup(claim, message.slice(0, 1024)); + } + + async #finishStaleCreateCleanup(claim: StaleCreateCleanupClaim, failure: string | undefined): Promise { + return await this.#serializable(async transaction => { + const lease = await transaction`SELECT owner_id, epoch, expires_at <= clock_timestamp() AS expired FROM t4_owner_leases WHERE lease_name = ${OUTBOX_LEASE} FOR UPDATE`; + if (lease[0]?.owner_id !== claim.ownerId || lease[0].epoch !== claim.ownerEpoch || lease[0].expired) return false; + const rows = await transaction<{ cleanup_id: bigint }[]>`SELECT cleanup_id FROM t4_stale_create_cleanups WHERE cleanup_id = ${claim.cleanupId} AND state = 'claimed' AND owner_id = ${claim.ownerId} AND owner_epoch = ${claim.ownerEpoch} AND floor(extract(epoch FROM claimed_at) * 1000000)::numeric::text = ${claim.claimedAt} AND resource_type = ${claim.resourceType} AND target_id = ${claim.targetId} AND uid = ${claim.uid} AND resource_version = ${claim.resourceVersion} FOR UPDATE`; + if (!rows[0]) return false; + if (failure === undefined) await transaction`UPDATE t4_stale_create_cleanups SET state = 'applied', owner_id = NULL, owner_epoch = NULL, claimed_at = NULL, last_error = NULL, updated_at = clock_timestamp() WHERE cleanup_id = ${claim.cleanupId}`; + else await transaction`UPDATE t4_stale_create_cleanups SET state = 'pending', owner_id = NULL, owner_epoch = NULL, claimed_at = NULL, last_error = ${failure}, next_attempt_at = clock_timestamp() + interval '1 second', updated_at = clock_timestamp() WHERE cleanup_id = ${claim.cleanupId}`; + return true; + }); + } + + async #terminalizePendingSessionCommands(transaction: SQL, principalId: string, sessionId: string, reason: string, ownerEpoch: bigint): Promise { + const skipped = await transaction<{ command_id: string }[]>`UPDATE t4_outbox SET state = 'skipped', owner_id = NULL, owner_epoch = NULL, claimed_at = NULL, terminal_result = ${{ reason }}, updated_at = clock_timestamp() WHERE principal_id = ${principalId} AND target_id = ${sessionId} AND mutation_kind = 'command.submit' AND state IN ('pending','claimed') RETURNING command_id`; + for (const row of skipped) { + const terminal = await transaction<{ command_id: string }[]>`UPDATE t4_commands SET lifecycle_state = 'cancelled', updated_at = clock_timestamp() WHERE command_id = ${row.command_id} AND lifecycle_state = 'accepted' RETURNING command_id`; + if (terminal[0]) await this.#appendEvent(transaction, principalId, sessionId, "command", { commandId: row.command_id, state: "cancelled" }, ownerEpoch); + } + } + + async #serializable(operation: (transaction: SQL) => Promise): Promise { + for (let attempt = 1; ; attempt++) { + try { + return await this.#sql.begin("ISOLATION LEVEL SERIALIZABLE", async transaction => await operation(transaction as unknown as SQL)); + } catch (error) { + const code = postgresSqlState(error); + const retryable = code === "40001" || code === "40P01" || isIdempotencyUniqueRace(error); + if (attempt >= SERIALIZABLE_ATTEMPTS || !retryable) throw error; + } + } + } + + async #lockIdempotency(transaction: SQL, principalId: string, operation: string, scope: string, idempotencyKey: string): Promise { + const identity = stableJson([principalId, operation, scope, idempotencyKey]); + await transaction`SELECT pg_advisory_xact_lock(hashtextextended(${identity}, 0))`; + } + + #cursor(kind: "watch" | "page", principalId: string, scope: string, position: bigint | string): string { + const payload = Buffer.from(JSON.stringify({ k: kind, p: createHash("sha256").update(principalId).digest("base64url"), s: scope, q: position.toString() }), "utf8").toString("base64url"); + const signature = createHmac("sha256", this.#cursorSecret).update(payload).digest("base64url"); + return `${payload}.${signature}`; + } + #decodeCursor(cursor: string, kind: "watch" | "page", principalId: string, scope: string): bigint | string | undefined { + if (cursor.length > 512 || !/^[A-Za-z0-9._~-]+$/u.test(cursor)) return undefined; + const [payload, signature, extra] = cursor.split("."); + if (!payload || !signature || extra) return undefined; + const expected = createHmac("sha256", this.#cursorSecret).update(payload).digest(); + let supplied: Buffer; + try { supplied = Buffer.from(signature, "base64url"); } catch { return undefined; } + if (supplied.byteLength !== expected.byteLength || !timingSafeEqual(supplied, expected)) return undefined; + let value: { k?: unknown; p?: unknown; s?: unknown; q?: unknown }; + try { value = JSON.parse(Buffer.from(payload, "base64url").toString("utf8")) as typeof value; } catch { return undefined; } + if (value.k !== kind || value.p !== createHash("sha256").update(principalId).digest("base64url") || value.s !== scope || typeof value.q !== "string") return undefined; + if (kind === "page") return value.q; + if (!/^(?:0|[1-9][0-9]*)$/u.test(value.q)) return undefined; + return BigInt(value.q); + } + + async #appendEvent(transaction: SQL, principalId: string, sessionId: string, type: "session" | "command", payload: Record, ownerEpoch: bigint): Promise { + const inserted = await transaction<{ sequence: bigint }[]>`INSERT INTO t4_events (principal_id, session_id, event_type, payload, owner_epoch) VALUES (${principalId}, ${sessionId}, ${type}, ${payload}, ${ownerEpoch}) RETURNING sequence`; + const sequence = persistedBigint(inserted[0]?.sequence); + await transaction`DELETE FROM t4_events WHERE principal_id = ${principalId} AND session_id = ${sessionId} AND sequence < COALESCE((SELECT sequence FROM t4_events WHERE principal_id = ${principalId} AND session_id = ${sessionId} ORDER BY sequence DESC OFFSET ${this.#eventRetention - 1} LIMIT 1), 0)`; + const retained = await transaction<{ first: bigint; latest: bigint }[]>`SELECT COALESCE(MIN(sequence), 0) AS first, COALESCE(MAX(sequence), 0) AS latest FROM t4_events WHERE principal_id = ${principalId} AND session_id = ${sessionId}`; + const first = persistedBigint(retained[0]?.first, true); + const latest = persistedBigint(retained[0]?.latest, true); + await transaction`INSERT INTO t4_event_retention (principal_id, session_id, first_retained_sequence, latest_sequence) VALUES (${principalId}, ${sessionId}, ${first}, ${latest}) ON CONFLICT (principal_id, session_id) DO UPDATE SET first_retained_sequence = EXCLUDED.first_retained_sequence, latest_sequence = EXCLUDED.latest_sequence, updated_at = clock_timestamp()`; + return sequence; + } +} diff --git a/packages/cluster-server/src/main.ts b/packages/cluster-server/src/main.ts index 0ac12d7f..8c8f0361 100755 --- a/packages/cluster-server/src/main.ts +++ b/packages/cluster-server/src/main.ts @@ -1,6 +1,6 @@ #!/usr/bin/env bun import { CiProjectionRunner } from "./ci-projection-runner.ts"; -import { clusterServerConfigFromEnv, loadKubernetesCa } from "./config.ts"; +import { clusterServerConfigFromEnv, loadKubernetesCa, loadPostgresUrl } from "./config.ts"; import { ClusterGateway } from "./gateway.ts"; import { KubernetesApiClient, KubernetesGatewayMutationBackend } from "./kubernetes-client.ts"; import { ClusterInfrastructureProjection } from "./kubernetes-projection.ts"; @@ -10,6 +10,11 @@ import { WebSocketPodHostConnector } from "./pod-host-router.ts"; import { startClusterHttpServers, type ClusterHttpServers } from "./server.ts"; import { SessionAuthorityRunner } from "./session-authority-runner.ts"; import { WoodpeckerProvider } from "./woodpecker.ts"; +import { PostgresLedger } from "./ledger.ts"; +import { DurableOutboxWorker } from "./outbox-worker.ts"; +import { loadPublicApiSecrets } from "./public-api-auth.ts"; +import { T4PublicApiV1 } from "./public-api-v1.ts"; +import { PublicKubernetesOutboxApplier } from "./public-kubernetes-applier.ts"; export async function runClusterServer(env: Readonly> = process.env): Promise { const config = clusterServerConfigFromEnv(env); @@ -23,11 +28,17 @@ export async function runClusterServer(env: Readonly { + if (!ledger) return; + const lease = await ledger.tryAcquireLease(config.epoch); + if (lease) await ledger.ingestKubernetesStatus(lease, collection, observation); + }, onSynchronized: () => health.markKubernetesSynced(), onError: error => { health.markKubernetesUnavailable(); @@ -40,8 +51,38 @@ export async function runClusterServer(env: Readonly | undefined; let signalsInstalled = false; try { + if (config.postgres) { + if (!config.publicApiCredentialsFile || !config.publicApiRuntimeProfile) throw new Error("public API durable storage configuration is incomplete"); + const [databaseUrl, publicApiSecrets] = await Promise.all([ + loadPostgresUrl(config), + loadPublicApiSecrets(config.publicApiCredentialsFile), + ]); + ledger = new PostgresLedger({ url: databaseUrl, cursorSecret: publicApiSecrets.cursorSecret }); + await ledger.migrate(); + publicApi = new T4PublicApiV1({ ledger, authenticator: publicApiSecrets.authenticator, allowedOrigins: () => projection.allowedOrigins() }); + const outbox = new DurableOutboxWorker({ + ledger, + ownerId: config.epoch, + applier: new PublicKubernetesOutboxApplier({ client: kubernetes, hostRef: config.hostName, runtimeProfile: config.publicApiRuntimeProfile }), + }); + outboxAbort = new AbortController(); + outboxTask = (async () => { + while (!outboxAbort!.signal.aborted) { + try { + await outbox.acquireLease(); + await outbox.drain(); + } catch (error) { + logger.warn("durable outbox recovery retrying", { condition: error instanceof Error ? error.name : "unknown", result: "failure" }); + } + if (!outboxAbort!.signal.aborted) await Bun.sleep(1_000); + } + })(); + } await runner.start(); const connector = new WebSocketPodHostConnector({ identityTokenFile: config.identityTokenPath }); authority = new SessionAuthorityRunner({ @@ -67,6 +108,7 @@ export async function runClusterServer(env: Readonly>; +} +export interface OutboxApplyContext { + readonly claimIsCurrent?: () => Promise; + readonly signal?: AbortSignal; + readonly persistStaleCreateCleanup?: (cleanup: StaleCreateCleanup) => Promise; +} +export interface StaleCreateCleanupReplayContext { + readonly claimIsCurrent: () => Promise; + readonly signal?: AbortSignal; +} +export interface KubernetesOutboxApplier { + apply(mutation: OutboxMutation, fence: OwnerLease, context?: OutboxApplyContext): Promise; + replayStaleCreateCleanup?(cleanup: StaleCreateCleanup, context: StaleCreateCleanupReplayContext): Promise; +} +export interface DurableOutboxWorkerOptions { + readonly ledger: PostgresLedger; + readonly ownerId: string; + readonly applier: KubernetesOutboxApplier; +} + +export class DurableOutboxWorker { + readonly #ledger: PostgresLedger; + readonly #ownerId: string; + readonly #applier: KubernetesOutboxApplier; + #lease: OwnerLease | undefined; + + constructor(options: DurableOutboxWorkerOptions) { + if (!options.ownerId.trim() || Buffer.byteLength(options.ownerId, "utf8") > 256) throw new Error("outbox owner id is invalid"); + this.#ledger = options.ledger; + this.#ownerId = options.ownerId; + this.#applier = options.applier; + } + + async tryAcquireLease(): Promise { + this.#lease = await this.#ledger.tryAcquireLease(this.#ownerId); + return this.#lease; + } + + async acquireLease(): Promise { + this.#lease = await this.#ledger.acquireLease(this.#ownerId); + return this.#lease; + } + + async claimNext(): Promise { + if (!this.#lease) throw new Error("outbox lease has not been acquired"); + return await this.#ledger.claimNext(this.#lease); + } + + async applyClaim(claim: OutboxClaim): Promise<"applied" | "fenced" | "retry"> { + if (!await this.#ledger.claimIsCurrent(claim)) return "fenced"; + const mutation: OutboxMutation = { + idempotencyToken: `outbox:${claim.outboxId}`, + commandId: claim.commandId, + principalId: claim.principalId, + kind: claim.kind, + targetId: claim.targetId, + targetRevision: claim.targetRevision, + payload: claim.mutation, + }; + const timeout = Math.max(0, claim.expiresAt - Date.now()); + const signal = timeout === 0 ? AbortSignal.abort(new Error("outbox lease has expired")) : AbortSignal.timeout(timeout); + try { + await this.#applier.apply(mutation, { ownerId: claim.ownerId, epoch: claim.ownerEpoch }, { + claimIsCurrent: () => this.#ledger.claimIsCurrent(claim), + signal, + persistStaleCreateCleanup: async cleanup => { + if (!await this.#ledger.persistStaleCreateCleanup(claim, cleanup)) throw new Error("stale create cleanup does not match its durable outbox claim"); + }, + }); + } catch (error) { + if (!await this.#ledger.claimIsCurrent(claim)) return "fenced"; + await this.#ledger.recordFailure(claim, error instanceof Error ? error.message : "outbox application failed"); + return "retry"; + } + return await this.#ledger.acknowledge(claim) ? "applied" : "fenced"; + } + + async #applyStaleCreateCleanup(claim: StaleCreateCleanupClaim): Promise<"applied" | "fenced" | "retry"> { + if (!this.#applier.replayStaleCreateCleanup || !await this.#ledger.staleCreateCleanupClaimIsCurrent(claim)) return "fenced"; + const timeout = Math.max(0, claim.expiresAt - Date.now()); + const signal = timeout === 0 ? AbortSignal.abort(new Error("outbox lease has expired")) : AbortSignal.timeout(timeout); + try { + await this.#applier.replayStaleCreateCleanup(claim, { + claimIsCurrent: () => this.#ledger.staleCreateCleanupClaimIsCurrent(claim), + signal, + }); + } catch (error) { + if (!await this.#ledger.staleCreateCleanupClaimIsCurrent(claim)) return "fenced"; + await this.#ledger.recordStaleCreateCleanupFailure(claim, error instanceof Error ? error.message : "stale create cleanup failed"); + return "retry"; + } + return await this.#ledger.acknowledgeStaleCreateCleanup(claim) ? "applied" : "fenced"; + } + + async drain(maximum = 100): Promise { + if (!this.#lease) await this.acquireLease(); + if (!Number.isSafeInteger(maximum) || maximum < 1 || maximum > 10_000) throw new Error("outbox drain bound is invalid"); + let applied = 0; + let emptyPasses = 0; + for (let index = 0; index < maximum && emptyPasses < 2; index++) { + if (this.#applier.replayStaleCreateCleanup) { + const cleanup = await this.#ledger.claimStaleCreateCleanup(this.#lease!); + if (cleanup) { + emptyPasses = 0; + const result = await this.#applyStaleCreateCleanup(cleanup); + if (result === "fenced") break; + continue; + } + } + const claim = await this.claimNext(); + if (!claim) { + emptyPasses++; + continue; + } + emptyPasses = 0; + const result = await this.applyClaim(claim); + if (result === "applied") applied++; + if (result === "fenced") break; + } + return applied; + } +} diff --git a/packages/cluster-server/src/public-api-auth.ts b/packages/cluster-server/src/public-api-auth.ts new file mode 100644 index 00000000..cd733965 --- /dev/null +++ b/packages/cluster-server/src/public-api-auth.ts @@ -0,0 +1,69 @@ +import { timingSafeEqual } from "node:crypto"; +import { isAbsolute } from "node:path"; +import { readBoundedRegularFile } from "./config.ts"; +import type { ApiPrincipal, PrincipalAuthenticator } from "./public-api-v1.ts"; + +const SCOPE = /^[a-z][a-z0-9.-]{0,127}$/u; +interface CredentialRecord { + readonly token: string; + readonly principal: ApiPrincipal; +} +interface PublicApiSecrets { + readonly cursorSecret: string; + readonly authenticator: PrincipalAuthenticator; +} + +function decodeSecrets(value: string): { cursorSecret: string; credentials: readonly CredentialRecord[] } { + let root: unknown; + try { root = JSON.parse(value); } catch { throw new Error("public API credentials file is invalid"); } + if (!root || typeof root !== "object" || Array.isArray(root)) throw new Error("public API credentials file is invalid"); + const object = root as Record; + if (Object.keys(object).some(key => key !== "cursorSecret" && key !== "principals") || typeof object.cursorSecret !== "string") throw new Error("public API credentials file is invalid"); + const secretBytes = new TextEncoder().encode(object.cursorSecret); + if (secretBytes.byteLength < 32 || secretBytes.byteLength > 4096) throw new Error("public API credentials file is invalid"); + if (!Array.isArray(object.principals) || object.principals.length < 1 || object.principals.length > 128) throw new Error("public API credentials file is invalid"); + const principals = new Set(); + const tokenHashes = new Set(); + const credentials: CredentialRecord[] = []; + for (const raw of object.principals) { + if (!raw || typeof raw !== "object" || Array.isArray(raw)) throw new Error("public API credentials file is invalid"); + const entry = raw as Record; + if (Object.keys(entry).some(key => !["token", "principal", "scopes"].includes(key)) || typeof entry.token !== "string" || typeof entry.principal !== "string" || !Array.isArray(entry.scopes)) throw new Error("public API credentials file is invalid"); + const tokenBytes = new TextEncoder().encode(entry.token); + if (tokenBytes.byteLength < 32 || tokenBytes.byteLength > 16_384 || /\s/u.test(entry.token)) throw new Error("public API credentials file is invalid"); + if (!entry.principal || entry.principal !== entry.principal.trim() || new TextEncoder().encode(entry.principal).byteLength > 256 || /\p{Cc}/u.test(entry.principal)) throw new Error("public API credentials file is invalid"); + if (principals.has(entry.principal)) throw new Error("public API credentials file is invalid"); + principals.add(entry.principal); + const tokenIdentity = Buffer.from(tokenBytes).toString("base64"); + if (tokenHashes.has(tokenIdentity)) throw new Error("public API credentials file is invalid"); + tokenHashes.add(tokenIdentity); + const scopes = entry.scopes.map(scope => { + if (typeof scope !== "string" || !SCOPE.test(scope)) throw new Error("public API credentials file is invalid"); + return scope; + }); + if (scopes.length < 1 || scopes.length > 128 || new Set(scopes).size !== scopes.length) throw new Error("public API credentials file is invalid"); + credentials.push({ token: entry.token, principal: { id: entry.principal, scopes: new Set(scopes) } }); + } + return { cursorSecret: object.cursorSecret, credentials }; +} + +class SecretFileAuthenticator implements PrincipalAuthenticator { + readonly #path: string; + constructor(path: string) { this.#path = path; } + async authenticate(token: string): Promise { + const presented = Buffer.from(token, "utf8"); + const { credentials } = decodeSecrets(await readBoundedRegularFile(this.#path, 65_536, "public API credentials")); + let principal: ApiPrincipal | undefined; + for (const credential of credentials) { + const expected = Buffer.from(credential.token, "utf8"); + if (expected.byteLength === presented.byteLength && timingSafeEqual(expected, presented)) principal = credential.principal; + } + return principal; + } +} + +export async function loadPublicApiSecrets(path: string): Promise { + if (!isAbsolute(path)) throw new Error("public API credentials file path must be absolute"); + const decoded = decodeSecrets(await readBoundedRegularFile(path, 65_536, "public API credentials")); + return { cursorSecret: decoded.cursorSecret, authenticator: new SecretFileAuthenticator(path) }; +} diff --git a/packages/cluster-server/src/public-api-v1.ts b/packages/cluster-server/src/public-api-v1.ts new file mode 100644 index 00000000..73080618 --- /dev/null +++ b/packages/cluster-server/src/public-api-v1.ts @@ -0,0 +1,441 @@ +import { randomUUID } from "node:crypto"; +import type { components } from "../../t4-api-client/src/generated/schema.ts"; +import { + canonicalRequestFingerprint, + type LedgerMutationResult, + type PostgresLedger, +} from "./ledger.ts"; + +type Discovery = components["schemas"]["Discovery"]; +type ErrorCode = components["schemas"]["ApiError"]["code"]; +type CommandCreate = components["schemas"]["CommandCreate"]; +type WorkspaceCreate = components["schemas"]["WorkspaceCreate"]; +type WorkspaceMutation = components["schemas"]["WorkspaceMutation"]; +type SessionCreate = components["schemas"]["SessionCreate"]; +type SessionMutation = components["schemas"]["SessionMutation"]; + +export interface ApiPrincipal { + readonly id: string; + readonly scopes: ReadonlySet; +} +export interface PrincipalAuthenticator { + authenticate(token: string): Promise; +} +export interface T4PublicApiV1Options { + readonly ledger: PostgresLedger; + readonly authenticator: PrincipalAuthenticator; + readonly allowedOrigins?: readonly string[] | (() => readonly string[]); +} + +const CAPABILITY_REQUIRED_SCOPES = [ + ["workspace.lifecycle", ["workspaces.read", "workspaces.write"]], + ["session.lifecycle", ["sessions.read", "sessions.write"]], + ["session.commands", ["commands.write"]], + ["session.watch.sse", ["events.read"]], +] as const; +const DISCOVERY: Omit = { + apiVersion: "1.0", + serverBuild: { version: "0.1.30", revision: "cluster" }, + supportedMajors: [1], + limits: { + pageSizeDefault: 25, + pageSizeMax: 100, + commandBytesMax: 262_144, + commandRequestBytesMax: 1_048_576, + commandMetadataValueBytesMax: 262_144, + watchEventsMax: 1_000, + heartbeatSeconds: 15, + }, +}; +function discoveryFor(principal: ApiPrincipal): Discovery { + const capabilities: Discovery["capabilities"] = {}; + for (const [capability, requiredScopes] of CAPABILITY_REQUIRED_SCOPES) { + capabilities[capability] = { + supported: true, + enabled: true, + authorized: requiredScopes.every(scope => principal.scopes.has(scope)), + available: true, + }; + } + return { ...DISCOVERY, capabilities }; +} +const RESOURCE_ID = /^[A-Za-z0-9][A-Za-z0-9._~-]{0,127}$/u; +const IDEMPOTENCY_KEY = /^[A-Za-z0-9._~-]{16,128}$/u; +const VERSION = /^[1-9][0-9]{0,3}(?:\.[0-9]+)?$/u; +const LABEL_KEY = /^[a-z][a-z0-9.-]{0,62}$/u; +const CRD_OWNER = /^[A-Za-z0-9][A-Za-z0-9._:@/-]*$/u; +const encoder = new TextEncoder(); + +interface ApiContext { readonly requestId: string; readonly principal: ApiPrincipal; } +interface ParsedObject { readonly value?: Record; readonly response?: Response; } + +function responseHeaders(extra: Record = {}): Headers { + return new Headers({ "cache-control": "no-store", "t4-api-version": "1.0", ...extra }); +} +function jsonResponse(status: number, body: unknown, extra: Record = {}): Response { + const headers = responseHeaders({ "content-type": "application/json", ...extra }); + return new Response(JSON.stringify(body), { status, headers }); +} +function apiError(requestId: string, status: number, code: ErrorCode, message: string, details: Record = {}): Response { + return jsonResponse(status, { error: { code, message, requestId, retryable: status === 503, ...details } }); +} +function notFound(requestId: string): Response { return apiError(requestId, 404, "not_found", "resource was not found"); } +function resultResponse(result: LedgerMutationResult, requestId: string, acceptedStatus: number): Response { + if (result.kind === "idempotency_conflict") return apiError(requestId, 409, "idempotency_conflict", "idempotency key was reused with another canonical request"); + if (result.kind === "revision_conflict") return apiError(requestId, 409, "revision_conflict", "resource revision does not match If-Match"); + if (result.kind === "not_found") return notFound(requestId); + const replayed = result.kind === "replay"; + const status = replayed ? 200 : acceptedStatus; + return jsonResponse(status, result.value, { "idempotency-replayed": replayed ? "true" : "false" }); +} +function deletedResponse(result: LedgerMutationResult, requestId: string): Response { + if (result.kind === "idempotency_conflict") return apiError(requestId, 409, "idempotency_conflict", "idempotency key was reused with another canonical request"); + if (result.kind === "not_found") return notFound(requestId); + return new Response(null, { status: 204, headers: responseHeaders({ "idempotency-replayed": result.kind === "replay" ? "true" : "false" }) }); +} +function withinCodePointLimit(value: string, maximum: number): boolean { + let length = 0; + let offset = 0; + while (offset < value.length) { + const codePoint = value.codePointAt(offset); + if (codePoint === undefined) break; + offset += codePoint > 0xffff ? 2 : 1; + if (++length > maximum) return false; + } + return true; +} +function labels(value: unknown): Record | undefined { + if (value === undefined) return undefined; + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("labels must be an object"); + const entries = Object.entries(value as Record); + if (entries.length > 32) throw new Error("labels exceeds 32 properties"); + const output: Record = {}; + for (const [key, item] of entries) { + if (!LABEL_KEY.test(key) || typeof item !== "string" || !withinCodePointLimit(item, 128)) throw new Error("labels is invalid"); + output[key] = item; + } + return output; +} +function boundedString(value: unknown, name: string, maximum: number): string { + if (typeof value !== "string" || value.length < 1 || !withinCodePointLimit(value, maximum)) throw new Error(`${name} is invalid`); + return value; +} +function exactKeys(value: Record, allowed: readonly string[], required: readonly string[], minimum = 0): void { + const keys = Object.keys(value); + if (keys.length < minimum || keys.some(key => !allowed.includes(key)) || required.some(key => !Object.hasOwn(value, key))) throw new Error("request object fields are invalid"); +} +function integerQuery(value: string | null, fallback: number, minimum: number, maximum: number): number | undefined { + if (value === null) return fallback; + if (!/^[1-9][0-9]*$/u.test(value)) return undefined; + const parsed = Number(value); + return Number.isSafeInteger(parsed) && parsed >= minimum && parsed <= maximum ? parsed : undefined; +} +function acceptsEventStream(value: string | null): boolean { + if (!value) return false; + let selectedSpecificity = -1; + let selectedQuality = 0; + for (const range of value.split(",")) { + const [rawMediaType = "", ...parameters] = range.split(";"); + const mediaType = rawMediaType.trim().toLowerCase(); + const specificity = mediaType === "text/event-stream" ? 2 : mediaType === "text/*" ? 1 : mediaType === "*/*" ? 0 : -1; + if (specificity < 0) continue; + let quality = 1; + let valid = true; + let qualitySeen = false; + for (const rawParameter of parameters) { + const parameter = rawParameter.trim(); + const separator = parameter.indexOf("="); + if (separator < 1) { valid = false; break; } + const name = parameter.slice(0, separator).trim().toLowerCase(); + const parameterValue = parameter.slice(separator + 1).trim(); + if (name === "q") { + if (qualitySeen || !/^(?:0(?:\.[0-9]{0,3})?|1(?:\.0{0,3})?)$/u.test(parameterValue)) { valid = false; break; } + qualitySeen = true; + quality = Number(parameterValue); + } else if (!qualitySeen) { + valid = false; + break; + } + } + if (!valid) continue; + if (specificity > selectedSpecificity) { + selectedSpecificity = specificity; + selectedQuality = quality; + } else if (specificity === selectedSpecificity) { + selectedQuality = Math.max(selectedQuality, quality); + } + } + return selectedQuality > 0; +} + +export class T4PublicApiV1 { + readonly #ledger: PostgresLedger; + readonly #authenticator: PrincipalAuthenticator; + readonly #allowedOrigins: () => readonly string[]; + + constructor(options: T4PublicApiV1Options) { + this.#ledger = options.ledger; + this.#authenticator = options.authenticator; + const allowedOrigins = options.allowedOrigins; + this.#allowedOrigins = typeof allowedOrigins === "function" ? allowedOrigins : () => allowedOrigins ?? []; + } + + async handle(request: Request): Promise { + const requestId = `req_${randomUUID()}`; + try { + if (new URL(request.url).protocol !== "https:") return apiError(requestId, 400, "https_required", "HTTPS is required"); + const origin = request.headers.get("origin"); + if (origin && !this.#allowedOrigins().includes(origin)) return apiError(requestId, 400, "invalid_origin", "request origin is not allowed"); + const authorization = request.headers.get("authorization"); + if (!authorization || !/^Bearer /iu.test(authorization) || authorization.length > 16_391) return apiError(requestId, 401, "unauthenticated", "valid bearer authentication is required"); + const token = authorization.slice(7); + if (!token || /\s/u.test(token)) return apiError(requestId, 401, "unauthenticated", "valid bearer authentication is required"); + const principal = await this.#authenticator.authenticate(token); + if (!principal || typeof principal.id !== "string" || !CRD_OWNER.test(principal.id) || !withinCodePointLimit(principal.id, 256)) return apiError(requestId, 401, "unauthenticated", "valid bearer authentication is required"); + const selectedVersion = request.headers.get("t4-api-version"); + if (!selectedVersion || selectedVersion.length > 16 || !VERSION.test(selectedVersion)) return apiError(requestId, 400, "invalid_request", "T4-API-Version is required and must be valid"); + if (selectedVersion.split(".", 1)[0] !== "1") return apiError(requestId, 406, "incompatible_version", "requested API major is unsupported", { supportedMajors: [1] }); + return await this.#route(request, new URL(request.url), { requestId, principal }); + } catch { + return apiError(requestId, 503, "unavailable", "durable gateway is temporarily unavailable"); + } + } + + async #route(request: Request, url: URL, context: ApiContext): Promise { + const segments = url.pathname.split("/").filter(Boolean).map(value => { + try { return decodeURIComponent(value); } catch { return ""; } + }); + if (segments[0] !== "v1") return notFound(context.requestId); + if (segments.length === 1 && request.method === "GET") { + if (!this.#scope(context, "discovery.read")) return apiError(context.requestId, 403, "forbidden", "credential lacks the required operation scope"); + return jsonResponse(200, discoveryFor(context.principal)); + } + if (segments.length === 2 && segments[1] === "workspaces") { + if (request.method === "GET") return await this.#listWorkspaces(url, context); + if (request.method === "POST") return await this.#createWorkspace(request, context); + } + if (segments.length === 3 && segments[1] === "workspaces" && RESOURCE_ID.test(segments[2]!)) { + if (request.method === "GET") return await this.#getWorkspace(segments[2]!, context); + if (request.method === "PATCH") return await this.#patchWorkspace(request, segments[2]!, context); + if (request.method === "DELETE") return await this.#deleteWorkspace(request, segments[2]!, context); + } + if (segments.length === 4 && segments[1] === "workspaces" && RESOURCE_ID.test(segments[2]!) && segments[3] === "sessions") { + if (request.method === "GET") return await this.#listSessions(url, segments[2]!, context); + if (request.method === "POST") return await this.#createSession(request, segments[2]!, context); + } + if (segments.length === 3 && segments[1] === "sessions" && RESOURCE_ID.test(segments[2]!)) { + if (request.method === "GET") return await this.#getSession(segments[2]!, context); + if (request.method === "PATCH") return await this.#patchSession(request, segments[2]!, context); + if (request.method === "DELETE") return await this.#cancelSession(request, segments[2]!, context, true); + } + if (segments.length === 4 && segments[1] === "sessions" && RESOURCE_ID.test(segments[2]!)) { + if (segments[3] === "cancel" && request.method === "POST") return await this.#cancelSession(request, segments[2]!, context, false); + if (segments[3] === "commands" && request.method === "POST") return await this.#submitCommand(request, segments[2]!, context); + if (segments[3] === "snapshot" && request.method === "GET") return await this.#snapshot(segments[2]!, context); + if (segments[3] === "events" && request.method === "GET") return await this.#events(request, url, segments[2]!, context); + } + return notFound(context.requestId); + } + + #scope(context: ApiContext, required: string): boolean { return context.principal.scopes.has(required); } + #requireScope(context: ApiContext, required: string): Response | undefined { + return this.#scope(context, required) ? undefined : apiError(context.requestId, 403, "forbidden", "credential lacks the required operation scope"); + } + #key(request: Request, context: ApiContext): string | Response { + const value = request.headers.get("idempotency-key"); + return value && IDEMPOTENCY_KEY.test(value) ? value : apiError(context.requestId, 400, "idempotency_key_required", "a valid Idempotency-Key is required"); + } + #ifMatch(request: Request, context: ApiContext): number | Response { + const value = request.headers.get("if-match"); + if (!value || !/^[1-9][0-9]{0,15}$/u.test(value)) return apiError(context.requestId, 400, "invalid_request", "a valid If-Match revision is required"); + const revision = Number(value); + return Number.isSafeInteger(revision) ? revision : apiError(context.requestId, 400, "invalid_request", "a valid If-Match revision is required"); + } + async #body(request: Request, context: ApiContext): Promise { + const contentType = request.headers.get("content-type")?.toLowerCase(); + if (!contentType || !/^application\/json(?:\s*;\s*charset=utf-8)?$/u.test(contentType)) return { response: apiError(context.requestId, 400, "invalid_request", "request content type must be application/json") }; + const announced = request.headers.get("content-length"); + if (announced && (!/^[0-9]+$/u.test(announced) || Number(announced) > DISCOVERY.limits.commandRequestBytesMax)) { + await request.body?.cancel().catch(() => undefined); + return { response: apiError(context.requestId, 400, "invalid_request", "request body exceeds the maximum size") }; + } + const reader = request.body?.getReader(); + const decoder = new TextDecoder(); + let bytesRead = 0; + let text = ""; + if (reader) { + try { + while (true) { + const chunk = await reader.read(); + if (chunk.done) break; + bytesRead += chunk.value.byteLength; + if (bytesRead > DISCOVERY.limits.commandRequestBytesMax) { + await reader.cancel().catch(() => undefined); + return { response: apiError(context.requestId, 400, "invalid_request", "request body exceeds the maximum size") }; + } + text += decoder.decode(chunk.value, { stream: true }); + } + text += decoder.decode(); + } finally { + reader.releaseLock(); + } + } + let value: unknown; + try { value = JSON.parse(text); } catch { return { response: apiError(context.requestId, 400, "invalid_request", "request body is malformed JSON") }; } + return value && typeof value === "object" && !Array.isArray(value) + ? { value: value as Record } + : { response: apiError(context.requestId, 422, "invalid_request", "request body must be an object", { violations: [{ field: "body", rule: "type", message: "must be an object" }] }) }; + } + #validation(context: ApiContext, message: string): Response { + return apiError(context.requestId, 422, "invalid_request", message, { violations: [{ field: "body", rule: "schema", message }] }); + } + + async #createWorkspace(request: Request, context: ApiContext): Promise { + const forbidden = this.#requireScope(context, "workspaces.write"); if (forbidden) return forbidden; + const key = this.#key(request, context); if (key instanceof Response) return key; + const parsed = await this.#body(request, context); if (parsed.response) return parsed.response; + let input: WorkspaceCreate; + try { exactKeys(parsed.value!, ["name", "labels"], ["name"]); input = { name: boundedString(parsed.value!.name, "name", 128), ...(parsed.value!.labels !== undefined ? { labels: labels(parsed.value!.labels) } : {}) }; } + catch (error) { return this.#validation(context, error instanceof Error ? error.message : "workspace request is invalid"); } + const fingerprint = canonicalRequestFingerprint({ operation: "createWorkspace", target: "workspaces", body: input }); + return resultResponse(await this.#ledger.createWorkspace(context.principal.id, key, fingerprint, input), context.requestId, 202); + } + async #getWorkspace(workspaceId: string, context: ApiContext): Promise { + const forbidden = this.#requireScope(context, "workspaces.read"); if (forbidden) return forbidden; + const value = await this.#ledger.getWorkspace(context.principal.id, workspaceId); + return value ? jsonResponse(200, value) : notFound(context.requestId); + } + async #listWorkspaces(url: URL, context: ApiContext): Promise { + const forbidden = this.#requireScope(context, "workspaces.read"); if (forbidden) return forbidden; + const size = integerQuery(url.searchParams.get("pageSize"), DISCOVERY.limits.pageSizeDefault, 1, DISCOVERY.limits.pageSizeMax); + if (!size) return this.#validation(context, "pageSize is outside the supported bounds"); + const rawCursor = url.searchParams.get("cursor"); + const after = rawCursor ? this.#ledger.readPageCursor(rawCursor, context.principal.id, "workspaces") : undefined; + if (rawCursor && after === undefined) return this.#validation(context, "cursor is invalid for this identity and operation"); + const items = await this.#ledger.listWorkspaces(context.principal.id, after, size + 1); + return jsonResponse(200, { items: items.slice(0, size), ...(items.length > size ? { nextCursor: this.#ledger.pageCursor(context.principal.id, "workspaces", items[size - 1]!.id) } : {}) }); + } + async #patchWorkspace(request: Request, workspaceId: string, context: ApiContext): Promise { + const forbidden = this.#requireScope(context, "workspaces.write"); if (forbidden) return forbidden; + const revision = this.#ifMatch(request, context); if (revision instanceof Response) return revision; + const key = this.#key(request, context); if (key instanceof Response) return key; + const parsed = await this.#body(request, context); if (parsed.response) return parsed.response; + let input: WorkspaceMutation; + try { exactKeys(parsed.value!, ["name", "labels"], [], 1); input = { ...(parsed.value!.name !== undefined ? { name: boundedString(parsed.value!.name, "name", 128) } : {}), ...(parsed.value!.labels !== undefined ? { labels: labels(parsed.value!.labels) } : {}) }; } + catch (error) { return this.#validation(context, error instanceof Error ? error.message : "workspace mutation is invalid"); } + const fingerprint = canonicalRequestFingerprint({ operation: "mutateWorkspace", target: workspaceId, ifMatch: revision, body: input }); + return resultResponse(await this.#ledger.patchWorkspace(context.principal.id, workspaceId, revision, key, fingerprint, input), context.requestId, 200); + } + async #deleteWorkspace(request: Request, workspaceId: string, context: ApiContext): Promise { + const forbidden = this.#requireScope(context, "workspaces.write"); if (forbidden) return forbidden; + const key = this.#key(request, context); if (key instanceof Response) return key; + const fingerprint = canonicalRequestFingerprint({ operation: "deleteWorkspace", target: workspaceId }); + return deletedResponse(await this.#ledger.deleteWorkspace(context.principal.id, workspaceId, key, fingerprint), context.requestId); + } + + async #createSession(request: Request, workspaceId: string, context: ApiContext): Promise { + const forbidden = this.#requireScope(context, "sessions.write"); if (forbidden) return forbidden; + const key = this.#key(request, context); if (key instanceof Response) return key; + const parsed = await this.#body(request, context); if (parsed.response) return parsed.response; + let input: SessionCreate; + try { exactKeys(parsed.value!, ["title", "labels"], ["title"]); input = { title: boundedString(parsed.value!.title, "title", 128), ...(parsed.value!.labels !== undefined ? { labels: labels(parsed.value!.labels) } : {}) }; } + catch (error) { return this.#validation(context, error instanceof Error ? error.message : "session request is invalid"); } + const fingerprint = canonicalRequestFingerprint({ operation: "spawnSession", target: workspaceId, body: input }); + return resultResponse(await this.#ledger.createSession(context.principal.id, workspaceId, key, fingerprint, input), context.requestId, 202); + } + async #getSession(sessionId: string, context: ApiContext): Promise { + const forbidden = this.#requireScope(context, "sessions.read"); if (forbidden) return forbidden; + const value = await this.#ledger.getSession(context.principal.id, sessionId); + return value ? jsonResponse(200, value) : notFound(context.requestId); + } + async #listSessions(url: URL, workspaceId: string, context: ApiContext): Promise { + const forbidden = this.#requireScope(context, "sessions.read"); if (forbidden) return forbidden; + const size = integerQuery(url.searchParams.get("pageSize"), DISCOVERY.limits.pageSizeDefault, 1, DISCOVERY.limits.pageSizeMax); + if (!size) return this.#validation(context, "pageSize is outside the supported bounds"); + const scope = `workspace:${workspaceId}:sessions`; + const rawCursor = url.searchParams.get("cursor"); + const after = rawCursor ? this.#ledger.readPageCursor(rawCursor, context.principal.id, scope) : undefined; + if (rawCursor && after === undefined) return this.#validation(context, "cursor is invalid for this identity and operation"); + const items = await this.#ledger.listSessions(context.principal.id, workspaceId, after, size + 1); + if (!items) return notFound(context.requestId); + return jsonResponse(200, { items: items.slice(0, size), ...(items.length > size ? { nextCursor: this.#ledger.pageCursor(context.principal.id, scope, items[size - 1]!.id) } : {}) }); + } + async #patchSession(request: Request, sessionId: string, context: ApiContext): Promise { + const forbidden = this.#requireScope(context, "sessions.write"); if (forbidden) return forbidden; + const revision = this.#ifMatch(request, context); if (revision instanceof Response) return revision; + const key = this.#key(request, context); if (key instanceof Response) return key; + const parsed = await this.#body(request, context); if (parsed.response) return parsed.response; + let input: SessionMutation; + try { exactKeys(parsed.value!, ["title", "labels"], [], 1); input = { ...(parsed.value!.title !== undefined ? { title: boundedString(parsed.value!.title, "title", 128) } : {}), ...(parsed.value!.labels !== undefined ? { labels: labels(parsed.value!.labels) } : {}) }; } + catch (error) { return this.#validation(context, error instanceof Error ? error.message : "session mutation is invalid"); } + const fingerprint = canonicalRequestFingerprint({ operation: "mutateSession", target: sessionId, ifMatch: revision, body: input }); + return resultResponse(await this.#ledger.patchSession(context.principal.id, sessionId, revision, key, fingerprint, input), context.requestId, 200); + } + async #cancelSession(request: Request, sessionId: string, context: ApiContext, deletion: boolean): Promise { + const forbidden = this.#requireScope(context, "sessions.write"); if (forbidden) return forbidden; + const key = this.#key(request, context); if (key instanceof Response) return key; + const fingerprint = canonicalRequestFingerprint({ operation: deletion ? "deleteSession" : "cancelSession", target: sessionId }); + const result = await this.#ledger.cancelSession(context.principal.id, sessionId, key, fingerprint, deletion); + return deletion ? deletedResponse(result, context.requestId) : resultResponse(result, context.requestId, 202); + } + async #submitCommand(request: Request, sessionId: string, context: ApiContext): Promise { + const forbidden = this.#requireScope(context, "commands.write"); if (forbidden) return forbidden; + const key = this.#key(request, context); if (key instanceof Response) return key; + const parsed = await this.#body(request, context); if (parsed.response) return parsed.response; + let input: CommandCreate; + try { + exactKeys(parsed.value!, ["command", "metadata"], ["command"]); + const command = boundedString(parsed.value!.command, "command", 262_144); + if (encoder.encode(command).byteLength > DISCOVERY.limits.commandBytesMax) throw new Error("command exceeds UTF-8 byte limit"); + const metadataValue = parsed.value!.metadata ?? {}; + if (!metadataValue || typeof metadataValue !== "object" || Array.isArray(metadataValue) || Object.keys(metadataValue).length > 32) throw new Error("metadata is invalid"); + const metadata: Record = {}; + for (const [metadataKey, value] of Object.entries(metadataValue as Record)) { + if (!LABEL_KEY.test(metadataKey) || value !== null && !["string", "number", "boolean"].includes(typeof value) || typeof value === "number" && (!Number.isSafeInteger(value) || !Number.isFinite(value))) throw new Error("metadata is invalid"); + if (typeof value === "string" && encoder.encode(value).byteLength > DISCOVERY.limits.commandMetadataValueBytesMax) throw new Error("metadata value exceeds UTF-8 byte limit"); + metadata[metadataKey] = value as string | number | boolean | null; + } + input = { command, metadata }; + } catch (error) { return this.#validation(context, error instanceof Error ? error.message : "command request is invalid"); } + const fingerprint = canonicalRequestFingerprint({ operation: "submitCommand", target: sessionId, body: input }); + return resultResponse(await this.#ledger.submitCommand(context.principal.id, sessionId, key, fingerprint, { command: input.command, metadata: input.metadata ?? {} }), context.requestId, 202); + } + async #snapshot(sessionId: string, context: ApiContext): Promise { + const forbidden = this.#requireScope(context, "events.read"); if (forbidden) return forbidden; + const snapshot = await this.#ledger.snapshot(context.principal.id, sessionId); + return snapshot ? jsonResponse(200, { sessionId, cursor: snapshot.cursor, state: snapshot.session.state, entries: snapshot.entries }) : notFound(context.requestId); + } + async #events(request: Request, url: URL, sessionId: string, context: ApiContext): Promise { + const forbidden = this.#requireScope(context, "events.read"); if (forbidden) return forbidden; + if (!acceptsEventStream(request.headers.get("accept"))) return apiError(context.requestId, 406, "incompatible_version", "Accept must allow text/event-stream", { supportedMajors: [1] }); + const maxEvents = integerQuery(url.searchParams.get("maxEvents"), 100, 1, DISCOVERY.limits.watchEventsMax); + const heartbeat = integerQuery(url.searchParams.get("heartbeatSeconds"), DISCOVERY.limits.heartbeatSeconds, 5, 60); + if (!maxEvents || !heartbeat) return this.#validation(context, "watch bounds are invalid"); + const hasQueryCursor = url.searchParams.has("cursor"); + const queryCursor = url.searchParams.get("cursor"); + const headerCursor = request.headers.get("last-event-id"); + if ((hasQueryCursor && !queryCursor) || (headerCursor !== null && !headerCursor)) return this.#validation(context, "watch cursor is invalid for this identity and session"); + if (hasQueryCursor && headerCursor !== null && queryCursor !== headerCursor) return apiError(context.requestId, 400, "invalid_request", "cursor and Last-Event-ID must agree"); + let window; + try { window = await this.#ledger.eventWindow(context.principal.id, sessionId, queryCursor ?? headerCursor ?? undefined, maxEvents); } + catch (error) { + if (error instanceof Error && error.message === "invalid_cursor") return this.#validation(context, "watch cursor is invalid for this identity and session"); + throw error; + } + if (!window) return notFound(context.requestId); + if (window.expired) return apiError(context.requestId, 410, "cursor_expired", "watch cursor is outside retained history", { resync: { snapshotUrl: `/v1/sessions/${sessionId}/snapshot`, cursor: window.resyncCursor } }); + const frames: string[] = []; + for (const event of window.events) { + const cursor = this.#ledger.watchCursor(context.principal.id, sessionId, event.sequence); + const data = event.type === "session" + ? { type: "session", cursor, state: event.payload.state, revision: event.payload.revision } + : { type: "command", cursor, commandId: event.payload.commandId, state: event.payload.state }; + frames.push(`id: ${cursor}\nevent: ${event.type}\ndata: ${JSON.stringify(data)}\n\n`); + } + if (frames.length === 0) { + const data = { type: "heartbeat", cursor: window.cursor, observedAt: new Date().toISOString() }; + frames.push(`id: ${window.cursor}\nevent: heartbeat\ndata: ${JSON.stringify(data)}\n\n`); + } + return new Response(frames.join(""), { status: 200, headers: responseHeaders({ "content-type": "text/event-stream" }) }); + } +} diff --git a/packages/cluster-server/src/public-kubernetes-applier.ts b/packages/cluster-server/src/public-kubernetes-applier.ts new file mode 100644 index 00000000..43add538 --- /dev/null +++ b/packages/cluster-server/src/public-kubernetes-applier.ts @@ -0,0 +1,291 @@ +import { semanticResourceHash, KubernetesApiClient, KubernetesApiError } from "./kubernetes-client.ts"; +import type { KubernetesResource } from "./kubernetes-projection.ts"; +import type { + KubernetesOutboxApplier, + OutboxApplyContext, + OutboxMutation, + StaleCreateCleanup, + StaleCreateCleanupReplayContext, +} from "./outbox-worker.ts"; +import type { OwnerLease } from "./ledger.ts"; + +export interface PublicKubernetesOutboxApplierOptions { + readonly client: KubernetesApiClient; + readonly hostRef: string; + readonly runtimeProfile?: string; +} + +type ResourceType = "t4workspaces" | "t4sessions"; +const API_VERSION = "cluster.t4.dev/v1alpha1"; +const LEDGER_ANNOTATIONS = [ + "cluster.t4.dev/ledger-command-id", + "cluster.t4.dev/ledger-outbox-token", + "cluster.t4.dev/ledger-owner", + "cluster.t4.dev/ledger-owner-epoch", + "cluster.t4.dev/ledger-semantic-hash", +] as const; + +function record(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +} +function boundedAnnotation(name: string, value: string): string { + if (!value || new TextEncoder().encode(value).byteLength > 256) throw new Error(`${name} is invalid`); + return value; +} +function annotations(mutation: OutboxMutation, fence: OwnerLease): Record { + if (fence.epoch < 0n) throw new Error("ledger owner epoch is invalid"); + return { + "cluster.t4.dev/ledger-command-id": boundedAnnotation("ledger command id", mutation.commandId), + "cluster.t4.dev/ledger-outbox-token": boundedAnnotation("ledger outbox token", mutation.idempotencyToken), + "cluster.t4.dev/ledger-owner": boundedAnnotation("ledger owner", fence.ownerId), + "cluster.t4.dev/ledger-owner-epoch": fence.epoch.toString(), + "cluster.t4.dev/ledger-semantic-hash": semanticResourceHash(mutation.payload), + }; +} +function assertResourceIdentity(resource: KubernetesResource, resourceType: ResourceType, name: string): void { + const kind = resourceType === "t4workspaces" ? "T4Workspace" : "T4Session"; + if (resource.apiVersion !== API_VERSION || resource.kind !== kind || resource.metadata?.name !== name) + throw new Error("Kubernetes resource identity is invalid"); +} +function assertLedgerIdentity(resource: KubernetesResource, fence: OwnerLease): void { + const values = resource.metadata.annotations; + if (!values || LEDGER_ANNOTATIONS.some(key => typeof values[key] !== "string" || values[key].length === 0)) + throw new Error("Kubernetes ledger identity is invalid"); + const epoch = values["cluster.t4.dev/ledger-owner-epoch"]!; + if (!/^(?:0|[1-9][0-9]*)$/u.test(epoch)) throw new Error("Kubernetes ledger identity is invalid"); + const ownerEpoch = BigInt(epoch); + if (ownerEpoch > fence.epoch || ownerEpoch === fence.epoch && values["cluster.t4.dev/ledger-owner"] !== fence.ownerId) + throw new Error("Kubernetes resource is fenced by another ledger owner"); +} + +export class PublicKubernetesOutboxApplier implements KubernetesOutboxApplier { + readonly #client: KubernetesApiClient; + readonly #hostRef: string; + readonly #runtimeProfile: string; + constructor(options: PublicKubernetesOutboxApplierOptions) { + this.#client = options.client; + this.#hostRef = boundedAnnotation("Kubernetes host reference", options.hostRef); + const runtimeProfile = options.runtimeProfile ?? "default"; + if (runtimeProfile.length > 64 || !/^[a-z0-9](?:[a-z0-9._-]*[a-z0-9])?$/u.test(runtimeProfile)) + throw new Error("Kubernetes runtime profile is invalid"); + this.#runtimeProfile = runtimeProfile; + } + + async apply(mutation: OutboxMutation, fence: OwnerLease, context?: OutboxApplyContext): Promise { + if (mutation.kind === "workspace.create") return await this.#createWorkspace(mutation, fence, context); + if (mutation.kind === "session.create") return await this.#createSession(mutation, fence, context); + if (mutation.kind === "workspace.patch") return await this.#patchWorkspace(mutation, fence, context); + if (mutation.kind === "session.patch") return await this.#patchSession(mutation, fence, context); + if (mutation.kind === "command.submit") return await this.#submitCommand(mutation, fence, context); + if (mutation.kind === "workspace.delete") return await this.#delete("t4workspaces", mutation, fence, context); + return await this.#delete("t4sessions", mutation, fence, context); + } + + async replayStaleCreateCleanup(cleanup: StaleCreateCleanup, context: StaleCreateCleanupReplayContext): Promise { + if (context.signal?.aborted) throw context.signal.reason; + if (!await context.claimIsCurrent()) throw new Error("outbox lease is no longer current"); + if (context.signal?.aborted) throw context.signal.reason; + let current: KubernetesResource; + try { + current = await this.#client.get(cleanup.resourceType, cleanup.targetId, context.signal); + } catch (error) { + if (error instanceof KubernetesApiError && error.status === 404) { + if (!await context.claimIsCurrent()) throw new Error("outbox lease is no longer current"); + if (context.signal?.aborted) throw context.signal.reason; + return; + } + throw error; + } + if (!await context.claimIsCurrent()) throw new Error("outbox lease is no longer current"); + if (context.signal?.aborted) throw context.signal.reason; + if (current.metadata.uid !== cleanup.uid) return; + assertResourceIdentity(current, cleanup.resourceType, cleanup.targetId); + const resourceVersion = current.metadata.resourceVersion; + if (!resourceVersion) throw new Error("Kubernetes stale-create cleanup resourceVersion is unavailable"); + try { + await this.#client.delete(cleanup.resourceType, cleanup.targetId, { uid: cleanup.uid, resourceVersion }, context.signal); + } catch (error) { + if (!(error instanceof KubernetesApiError) || error.status !== 404) throw error; + } + } + + async #assertCurrent(context?: OutboxApplyContext): Promise { + if (context?.signal?.aborted) throw context.signal.reason; + if (context?.claimIsCurrent && !await context.claimIsCurrent()) throw new Error("outbox lease is no longer current"); + if (context?.signal?.aborted) throw context.signal.reason; + } + + async #createWorkspace(mutation: OutboxMutation, fence: OwnerLease, context?: OutboxApplyContext): Promise { + const body = { + apiVersion: API_VERSION, + kind: "T4Workspace", + metadata: { name: mutation.targetId, annotations: annotations(mutation, fence) }, + spec: { + hostRef: this.#hostRef, + owner: mutation.principalId, + displayName: String(mutation.payload.name), + retentionPolicy: "Retain", + size: "20Gi", + }, + }; + await this.#createIdempotently("t4workspaces", mutation, fence, body, context); + } + async #createSession(mutation: OutboxMutation, fence: OwnerLease, context?: OutboxApplyContext): Promise { + const workspaceId = String(mutation.payload.workspaceId); + await this.#ownedWorkspace(workspaceId, mutation, fence, context?.signal); + const body = { + apiVersion: API_VERSION, + kind: "T4Session", + metadata: { name: mutation.targetId, annotations: annotations(mutation, fence) }, + spec: { + hostRef: this.#hostRef, + workspaceRef: workspaceId, + title: String(mutation.payload.title), + runtimeProfile: this.#runtimeProfile, + guiEnabled: true, + }, + }; + await this.#createIdempotently("t4sessions", mutation, fence, body, context); + } + async #patchWorkspace(mutation: OutboxMutation, fence: OwnerLease, context?: OutboxApplyContext): Promise { + const current = await this.#owned("t4workspaces", mutation, fence, context?.signal); + const spec = record(current.spec); + await this.#assertCurrent(context); + await this.#client.patch("t4workspaces", mutation.targetId, { + metadata: { resourceVersion: current.metadata.resourceVersion, annotations: annotations(mutation, fence) }, + spec: { ...spec, displayName: String(mutation.payload.name) }, + }, context?.signal); + } + async #patchSession(mutation: OutboxMutation, fence: OwnerLease, context?: OutboxApplyContext): Promise { + const current = await this.#owned("t4sessions", mutation, fence, context?.signal); + await this.#assertCurrent(context); + await this.#client.patch("t4sessions", mutation.targetId, { + metadata: { + resourceVersion: current.metadata.resourceVersion, + annotations: { ...annotations(mutation, fence), "cluster.t4.dev/pending-command": null }, + }, + spec: { title: String(mutation.payload.title) }, + }, context?.signal); + } + async #submitCommand(mutation: OutboxMutation, fence: OwnerLease, context?: OutboxApplyContext): Promise { + const current = await this.#owned("t4sessions", mutation, fence, context?.signal); + await this.#assertCurrent(context); + await this.#client.patch("t4sessions", mutation.targetId, { + metadata: { + resourceVersion: current.metadata.resourceVersion, + annotations: { + ...annotations(mutation, fence), + "cluster.t4.dev/pending-command": null, + "cluster.t4.dev/pending-command-id": boundedAnnotation("pending command id", mutation.commandId), + "cluster.t4.dev/pending-command-epoch": fence.epoch.toString(), + }, + }, + }, context?.signal); + } + async #delete(resourceType: ResourceType, mutation: OutboxMutation, fence: OwnerLease, context?: OutboxApplyContext): Promise { + let current: KubernetesResource; + try { + current = await this.#owned(resourceType, mutation, fence, context?.signal); + } catch (error) { + if (error instanceof KubernetesApiError && error.status === 404) return; + throw error; + } + if (!current.metadata.uid) throw new Error("Kubernetes delete precondition is unavailable"); + await this.#assertCurrent(context); + try { + await this.#client.delete(resourceType, mutation.targetId, { uid: current.metadata.uid, resourceVersion: current.metadata.resourceVersion! }, context?.signal); + } catch (error) { + if (!(error instanceof KubernetesApiError) || error.status !== 404) throw error; + } + } + async #owned(resourceType: ResourceType, mutation: OutboxMutation, fence: OwnerLease, signal?: AbortSignal): Promise { + const current = await this.#client.get(resourceType, mutation.targetId, signal); + assertResourceIdentity(current, resourceType, mutation.targetId); + assertLedgerIdentity(current, fence); + if (!current.metadata.resourceVersion) throw new Error("Kubernetes resource identity is missing resourceVersion"); + const spec = record(current.spec); + if (spec.hostRef !== this.#hostRef) throw new Error("Kubernetes resource belongs to another cluster host"); + if (resourceType === "t4workspaces") { + if (spec.owner !== mutation.principalId) throw new Error("Kubernetes workspace belongs to another principal"); + } else { + const workspaceId = spec.workspaceRef; + if (typeof workspaceId !== "string" || !workspaceId || mutation.payload.workspaceId !== undefined && mutation.payload.workspaceId !== workspaceId) + throw new Error("Kubernetes session workspace identity is invalid"); + await this.#ownedWorkspace(workspaceId, mutation, fence, signal); + } + return current; + } + async #ownedWorkspace(workspaceId: string, mutation: OutboxMutation, fence: OwnerLease, signal?: AbortSignal): Promise { + const workspace = await this.#client.get("t4workspaces", workspaceId, signal); + assertResourceIdentity(workspace, "t4workspaces", workspaceId); + assertLedgerIdentity(workspace, fence); + const spec = record(workspace.spec); + if (spec.hostRef !== this.#hostRef || spec.owner !== mutation.principalId) + throw new Error("Kubernetes workspace ownership identity is invalid"); + return workspace; + } + async #persistCreateCleanup(resourceType: ResourceType, targetId: string, created: KubernetesResource, context?: OutboxApplyContext): Promise { + const uid = created.metadata?.uid; + const resourceVersion = created.metadata?.resourceVersion; + if (!uid || !resourceVersion) return undefined; + assertResourceIdentity(created, resourceType, targetId); + const cleanup: StaleCreateCleanup = { resourceType, targetId, uid, resourceVersion }; + await context?.persistStaleCreateCleanup?.(cleanup); + return cleanup; + } + + async #cleanupCreate(cleanup: StaleCreateCleanup): Promise { + try { + await this.#client.delete( + cleanup.resourceType, + cleanup.targetId, + { uid: cleanup.uid, resourceVersion: cleanup.resourceVersion }, + AbortSignal.timeout(5_000), + ); + } catch (error) { + if (!(error instanceof KubernetesApiError) || error.status !== 404) throw error; + } + } + + async #createIdempotently(resourceType: ResourceType, mutation: OutboxMutation, fence: OwnerLease, body: KubernetesResource, context?: OutboxApplyContext): Promise { + await this.#assertCurrent(context); + let created: KubernetesResource; + try { + created = await this.#client.create(resourceType, body, context?.signal); + } catch (error) { + if (!(error instanceof KubernetesApiError) || error.status !== 409) throw error; + try { + const current = await this.#client.get(resourceType, mutation.targetId, context?.signal); + assertResourceIdentity(current, resourceType, mutation.targetId); + assertLedgerIdentity(current, fence); + if (!current.metadata.resourceVersion) throw new Error("resourceVersion conflict"); + const spec = record(current.spec); + if (semanticResourceHash(spec) !== semanticResourceHash(body.spec)) throw new Error("spec conflict"); + if (spec.hostRef !== this.#hostRef) throw new Error("host conflict"); + if (resourceType === "t4workspaces" && spec.owner !== mutation.principalId) throw new Error("principal conflict"); + if (resourceType === "t4sessions" && spec.workspaceRef !== mutation.payload.workspaceId) throw new Error("workspace conflict"); + if (resourceType === "t4sessions") await this.#ownedWorkspace(String(spec.workspaceRef), mutation, fence, context?.signal); + const currentAnnotations = current.metadata.annotations ?? {}; + const expectedAnnotations = annotations(mutation, fence); + for (const key of ["cluster.t4.dev/ledger-command-id", "cluster.t4.dev/ledger-outbox-token", "cluster.t4.dev/ledger-semantic-hash"] as const) { + if (currentAnnotations[key] !== expectedAnnotations[key]) throw new Error("ledger conflict"); + } + await this.#assertCurrent(context); + await this.#client.patch(resourceType, mutation.targetId, { + metadata: { resourceVersion: current.metadata.resourceVersion, annotations: expectedAnnotations }, + }, context?.signal); + await this.#assertCurrent(context); + return; + } catch { + throw new Error("Kubernetes resource conflicts with durable ledger intent"); + } + } + const cleanup = await this.#persistCreateCleanup(resourceType, mutation.targetId, created, context); + try { + await this.#assertCurrent(context); + } catch (error) { + if (cleanup) await this.#cleanupCreate(cleanup); + throw error; + } + } +} diff --git a/packages/cluster-server/src/server.ts b/packages/cluster-server/src/server.ts index b1555972..86942a30 100644 --- a/packages/cluster-server/src/server.ts +++ b/packages/cluster-server/src/server.ts @@ -3,10 +3,12 @@ import { normalizeIpAddress } from "@t4-code/host-service"; import type { GatewayConnection, ClusterGateway } from "./gateway.ts"; import { ClusterInfrastructureProjection } from "./kubernetes-projection.ts"; import { ClusterMetrics, ClusterServerHealth, JsonLogger, createAdminHandler } from "./observability.ts"; +import type { T4PublicApiV1 } from "./public-api-v1.ts"; interface SocketData { connection?: GatewayConnection; principal: string; } export interface ClusterHttpServersOptions { readonly gateway: ClusterGateway; + readonly publicApi?: T4PublicApiV1; readonly projection: ClusterInfrastructureProjection; readonly gatewayPort: number; readonly adminPort: number; @@ -66,8 +68,21 @@ export function startClusterHttpServers(options: ClusterHttpServersOptions): Clu const gatewayServer = Bun.serve({ hostname: "0.0.0.0", port: options.gatewayPort, - fetch(request, server) { + async fetch(request, server) { const url = new URL(request.url); + if (url.pathname !== "/v1/ws" && (url.pathname === "/v1" || url.pathname.startsWith("/v1/"))) { + if (!options.publicApi) return new Response("not found", { status: 404 }); + if (draining) return new Response("draining", { status: 503 }); + const remoteAddress = server.requestIP(request)?.address; + if (!remoteAddress || !trustedSource(remoteAddress) || request.headers.get("x-forwarded-proto") !== "https") { + const insecure = new URL(request.url); + insecure.protocol = "http:"; + return await options.publicApi.handle(new Request(insecure, request)); + } + const secure = new URL(request.url); + secure.protocol = "https:"; + return await options.publicApi.handle(new Request(secure, request)); + } if (url.pathname !== "/v1/ws") return new Response("not found", { status: 404 }); if (request.method !== "GET") return new Response("method not allowed", { status: 405 }); if (draining) return new Response("draining", { status: 503 }); diff --git a/packages/cluster-server/test/config.test.ts b/packages/cluster-server/test/config.test.ts index 611ff078..7ff98f0f 100644 --- a/packages/cluster-server/test/config.test.ts +++ b/packages/cluster-server/test/config.test.ts @@ -110,6 +110,17 @@ describe("cluster server configuration", () => { })).toThrow("exactly one"); }); + it("requires a chart-admitted runtime profile with durable public API configuration", () => { + const durable = { + ...BASE_ENV, + T4_POSTGRES_URL: "postgres://t4:secret@postgres.t4-system.svc:5432/t4", + T4_PUBLIC_API_CREDENTIALS_FILE: "/var/run/secrets/t4-public-api/credentials.json", + T4_PUBLIC_API_RUNTIME_PROFILE: "review-admitted", + }; + expect(clusterServerConfigFromEnv(durable)).toMatchObject({ publicApiRuntimeProfile: "review-admitted" }); + expect(() => clusterServerConfigFromEnv({ ...durable, T4_PUBLIC_API_RUNTIME_PROFILE: undefined })).toThrow("runtime profile"); + expect(() => clusterServerConfigFromEnv({ ...durable, T4_PUBLIC_API_RUNTIME_PROFILE: "-not-admitted" })).toThrow("runtime profile"); + }); }); describe("trusted cluster gateway proxy sources", () => { diff --git a/packages/cluster-server/test/durable-gateway-postgres.test.ts b/packages/cluster-server/test/durable-gateway-postgres.test.ts new file mode 100644 index 00000000..386ccb44 --- /dev/null +++ b/packages/cluster-server/test/durable-gateway-postgres.test.ts @@ -0,0 +1,1259 @@ +import { SQL } from "bun"; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vite-plus/test"; +interface DurableLedgerOptions { + readonly url: string; + readonly cursorSecret: string; + readonly eventRetention: number; +} +interface PostgresLedgerLike { + migrate(): Promise; + close(): Promise; + acquireLease(ownerId: string, leaseSeconds?: number): Promise<{ ownerId: string; epoch: bigint }>; + claimNext(lease: { ownerId: string; epoch: bigint }): Promise; + claimIsCurrent(claim: OutboxClaim): Promise; + acknowledge(claim: OutboxClaim): Promise; + recordFailure(claim: OutboxClaim, message: string): Promise; +} +interface ApiPrincipal { + readonly id: string; + readonly scopes: ReadonlySet; +} +interface PrincipalAuthenticator { + authenticate(token: string): Promise; +} +interface T4PublicApiV1Like { + handle(request: Request): Promise; +} +interface OutboxMutation { + readonly idempotencyToken: string; + readonly targetId: string; + readonly kind: string; + readonly payload: Readonly>; +} +interface KubernetesOutboxApplier { + apply(mutation: OutboxMutation, fence: { ownerId: string; epoch: bigint }): Promise; +} +interface OutboxClaim { + readonly outboxId: bigint; + readonly commandId: string; + readonly principalId: string; + readonly idempotencyKey: string; + readonly kind: string; + readonly targetId: string; + readonly targetRevision: bigint; + readonly mutation: Readonly>; + readonly ownerId: string; + readonly ownerEpoch: bigint; + readonly expiresAt: number; +} +interface AuthoritativeKubernetesStatusObservation { + readonly resourceId: string; + readonly principalId: string; + readonly workspaceId?: string; + readonly uid: string; + readonly resourceVersion: string; + readonly generation: bigint; + readonly observedGeneration: bigint; + readonly phase: "Pending" | "Ready" | "Running" | "Failed" | "Terminating" | "Unknown"; +} +interface AuthoritativeKubernetesStatusIngress { + ingestKubernetesStatus( + lease: { ownerId: string; epoch: bigint }, + collection: "t4workspaces" | "t4sessions", + observation: AuthoritativeKubernetesStatusObservation, + ): Promise; +} +interface DurableOutboxWorkerLike { + acquireLease(): Promise<{ ownerId: string; epoch: bigint }>; + drain(): Promise; + claimNext(): Promise; + applyClaim(claim: OutboxClaim): Promise; +} +type LedgerConstructor = new (options: DurableLedgerOptions) => PostgresLedgerLike; +type ApiConstructor = new (options: { ledger: PostgresLedgerLike; authenticator: PrincipalAuthenticator; allowedOrigins?: readonly string[] }) => T4PublicApiV1Like; +type WorkerConstructor = new (options: { ledger: PostgresLedgerLike; ownerId: string; applier: KubernetesOutboxApplier }) => DurableOutboxWorkerLike; + +const DATABASE_URL = process.env.T4_TEST_POSTGRES_URL; +const VERSION = "1"; +const OWNER_TOKEN = "opaque-owner-token-with-sufficient-entropy"; +const MIXED_CASE_OWNER_TOKEN = "Opaque-Owner-Token-With-Mixed-Case-Entropy"; +const OTHER_TOKEN = "opaque-other-token-with-sufficient-entropy"; +const INVALID_PRINCIPAL_TOKEN = "invalid-principal-token-with-sufficient-entropy"; +const STALE_APPLY_DEADLINE_SECONDS = 5; +const ALL_SCOPES = new Set([ + "discovery.read", + "workspaces.read", + "workspaces.write", + "sessions.read", + "sessions.write", + "commands.write", + "events.read", +]); +const CURSOR_SECRET = "test-cursor-secret-that-is-at-least-thirty-two-bytes"; +const LEDGER_MODULE_PATH = "../src/ledger.ts"; +const API_MODULE_PATH = "../src/public-api-v1.ts"; +const WORKER_MODULE_PATH = "../src/outbox-worker.ts"; + +let PostgresLedger: LedgerConstructor | undefined; +let T4PublicApiV1: ApiConstructor | undefined; +let DurableOutboxWorker: WorkerConstructor | undefined; + +class TestAuthenticator implements PrincipalAuthenticator { + readonly #principals: Record = { + [OWNER_TOKEN]: { id: "owner@example.com", scopes: ALL_SCOPES }, + [MIXED_CASE_OWNER_TOKEN]: { id: "owner@example.com", scopes: ALL_SCOPES }, + [OTHER_TOKEN]: { id: "other@example.com", scopes: ALL_SCOPES }, + [INVALID_PRINCIPAL_TOKEN]: { id: "Not A CRD Owner", scopes: ALL_SCOPES }, + }; + async authenticate(token: string): Promise { + return this.#principals[token]; + } +} + +class ObservableKubernetes implements KubernetesOutboxApplier { + readonly attempts: OutboxMutation[] = []; + readonly applied = new Map(); + currentEpoch = 0n; + async apply(mutation: OutboxMutation, fence: { ownerId: string; epoch: bigint }): Promise { + this.attempts.push(mutation); + if (fence.epoch !== this.currentEpoch) throw new Error("stale owner epoch"); + this.applied.set(mutation.idempotencyToken, mutation); + } +} +class PausedCreateKubernetes implements KubernetesOutboxApplier { + readonly resources = new Set(); + readonly createStarted = Promise.withResolvers(); + readonly releaseCreate = Promise.withResolvers(); + async apply(mutation: OutboxMutation): Promise { + if (mutation.kind === "session.create") { + this.createStarted.resolve(); + await this.releaseCreate.promise; + this.resources.add(mutation.targetId); + return; + } + if (mutation.kind === "workspace.create") this.resources.add(mutation.targetId); + if (mutation.kind === "session.cancel" || mutation.kind === "session.delete") this.resources.delete(mutation.targetId); + } +} + + +function ledgerOptions(overrides: Partial = {}): DurableLedgerOptions { + if (!DATABASE_URL) throw new Error("T4_TEST_POSTGRES_URL is required for durable gateway tests"); + return { + url: DATABASE_URL, + cursorSecret: CURSOR_SECRET, + eventRetention: 3, + ...overrides, + }; +} + +async function newLedger(overrides: Partial = {}): Promise { + if (!PostgresLedger) throw new Error("PostgreSQL ledger implementation is unavailable"); + const ledger = new PostgresLedger(ledgerOptions(overrides)); + await ledger.migrate(); + return ledger; +} + +function newApi(ledger: PostgresLedgerLike, allowedOrigins?: readonly string[]): T4PublicApiV1Like { + if (!T4PublicApiV1) throw new Error("public T4 API v1 implementation is unavailable"); + return new T4PublicApiV1({ ledger, authenticator: new TestAuthenticator(), ...(allowedOrigins ? { allowedOrigins } : {}) }); +} + +function newWorker(ledger: PostgresLedgerLike, ownerId: string, applier: KubernetesOutboxApplier): DurableOutboxWorkerLike { + if (!DurableOutboxWorker) throw new Error("durable outbox worker implementation is unavailable"); + return new DurableOutboxWorker({ ledger, ownerId, applier }); +} + +function headers(token = OWNER_TOKEN, additions: Record = {}): Headers { + return new Headers({ + authorization: `Bearer ${token}`, + "t4-api-version": VERSION, + ...additions, + }); +} + +async function call( + api: T4PublicApiV1Like, + method: string, + path: string, + options: { token?: string; key?: string; ifMatch?: string; body?: unknown; rawBody?: string; headers?: Record } = {}, +): Promise { + const requestHeaders = headers(options.token, options.headers); + if (options.key) requestHeaders.set("idempotency-key", options.key); + if (options.ifMatch) requestHeaders.set("if-match", options.ifMatch); + let body: string | undefined; + if (options.rawBody !== undefined) body = options.rawBody; + else if (options.body !== undefined) body = JSON.stringify(options.body); + if (body !== undefined) requestHeaders.set("content-type", "application/json"); + return await api.handle(new Request(`https://t4.example.test${path}`, { method, headers: requestHeaders, body })); +} + +async function waitForDatabase(predicate: () => Promise): Promise { + for (let attempt = 0; attempt < 200; attempt++) { + if (await predicate()) return; + await Bun.sleep(10); + } + throw new Error("timed out waiting for PostgreSQL concurrency fixture"); +} + +interface JsonDocument extends Record { + id: string; + cursor: string; + state: string; + revision: number; + error: { code: string; message: string; [key: string]: unknown }; +} + +async function json(response: Response): Promise { + return await response.json() as JsonDocument; +} + +function eventData(responseBody: string): JsonDocument[] { + return responseBody + .split("\n\n") + .flatMap(frame => frame.split("\n").filter(line => line.startsWith("data: ")).map(line => JSON.parse(line.slice(6)) as JsonDocument)); +} + +let admin: SQL; + +beforeAll(async () => { + if (!DATABASE_URL) return; + admin = new SQL(DATABASE_URL, { max: 1, bigint: true }); + await admin.unsafe("DROP SCHEMA public CASCADE; CREATE SCHEMA public"); + const modules = await Promise.all([ + import(/* @vite-ignore */ LEDGER_MODULE_PATH).catch(() => undefined), + import(/* @vite-ignore */ API_MODULE_PATH).catch(() => undefined), + import(/* @vite-ignore */ WORKER_MODULE_PATH).catch(() => undefined), + ]); + PostgresLedger = (modules[0] as { PostgresLedger?: LedgerConstructor } | undefined)?.PostgresLedger; + T4PublicApiV1 = (modules[1] as { T4PublicApiV1?: ApiConstructor } | undefined)?.T4PublicApiV1; + DurableOutboxWorker = (modules[2] as { DurableOutboxWorker?: WorkerConstructor } | undefined)?.DurableOutboxWorker; +}); + +beforeEach(async () => { + if (!PostgresLedger) return; + const ledger = new PostgresLedger(ledgerOptions()); + await ledger.migrate(); + await ledger.close(); + await admin.unsafe("TRUNCATE t4_outbox, t4_event_retention, t4_events, t4_snapshot_entries, t4_session_intents, t4_workspace_intents, t4_commands, t4_owner_leases RESTART IDENTITY CASCADE"); +}); + +afterAll(async () => { + await admin?.close(); +}); + +const postgresSuite = DATABASE_URL ? describe.sequential : describe.skip; +postgresSuite("PostgreSQL-backed public T4 API v1", () => { + it("replays an identical create after process restart without a second Kubernetes mutation", async () => { + expect(PostgresLedger, "PostgreSQL ledger implementation is required").toBeDefined(); + expect(T4PublicApiV1, "public T4 API v1 implementation is required").toBeDefined(); + expect(DurableOutboxWorker, "durable outbox worker implementation is required").toBeDefined(); + let ledger = await newLedger(); + let api = newApi(ledger); + const key = "restart-workspace-key-0001"; + const accepted = await call(api, "POST", "/v1/workspaces", { key, body: { name: "durable workspace", labels: { team: "runtime" } } }); + expect(accepted.status).toBe(202); + expect(accepted.headers.get("idempotency-replayed")).toBe("false"); + const acceptedBody = await json(accepted); + + const kubernetes = new ObservableKubernetes(); + const firstWorker = newWorker(ledger, "worker-a", kubernetes); + const firstLease = await firstWorker.acquireLease(); + kubernetes.currentEpoch = firstLease.epoch; + expect(await firstWorker.drain()).toBeGreaterThan(0); + expect(kubernetes.applied.size).toBe(1); + await ledger.close(); + + ledger = await newLedger(); + api = newApi(ledger); + const replay = await call(api, "POST", "/v1/workspaces", { key, body: { labels: { team: "runtime" }, name: "durable workspace" } }); + expect(replay.status).toBe(200); + expect(replay.headers.get("idempotency-replayed")).toBe("true"); + expect(await json(replay)).toEqual(acceptedBody); + const secondWorker = newWorker(ledger, "worker-a", kubernetes); + const secondLease = await secondWorker.acquireLease(); + kubernetes.currentEpoch = secondLease.epoch; + expect(await secondWorker.drain()).toBe(0); + expect(kubernetes.applied.size).toBe(1); + await ledger.close(); + }); + + it("uses validated JCS identity, reports conflicts, and scopes keys by principal", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const key = "principal-jcs-key-0000001"; + const first = await call(api, "POST", "/v1/workspaces", { key, body: { name: "one", labels: { b: "2", a: "1" } } }); + expect(first.status).toBe(202); + const same = await call(api, "POST", "/v1/workspaces", { key, body: { labels: { a: "1", b: "2" }, name: "one" } }); + expect(same.status).toBe(200); + expect(same.headers.get("idempotency-replayed")).toBe("true"); + const conflict = await call(api, "POST", "/v1/workspaces", { key, body: { name: "two", labels: { a: "1", b: "2" } } }); + expect(conflict.status).toBe(409); + expect((await json(conflict)).error.code).toBe("idempotency_conflict"); + const isolated = await call(api, "POST", "/v1/workspaces", { token: OTHER_TOKEN, key, body: { name: "other" } }); + expect(isolated.status).toBe(202); + await ledger.close(); + }); + + it("recovers a commit-before-CR crash through one idempotent fenced application", async () => { + let ledger = await newLedger(); + let api = newApi(ledger); + const workspace = await json(await call(api, "POST", "/v1/workspaces", { + key: "crash-parent-workspace-001", + body: { name: "crash parent" }, + })); + const sessionResponse = await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { + key: "commit-before-crash-0001", + body: { title: "recover me" }, + }); + expect(sessionResponse.status).toBe(202); + const session = await json(sessionResponse); + await ledger.close(); + + ledger = await newLedger(); + api = newApi(ledger); + const kubernetes = new ObservableKubernetes(); + const worker = newWorker(ledger, "recovery-worker", kubernetes); + const lease = await worker.acquireLease(); + kubernetes.currentEpoch = lease.epoch; + expect(await worker.drain()).toBeGreaterThan(0); + expect([...kubernetes.applied.values()].filter(value => value.targetId === session.id)).toHaveLength(1); + const replay = await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { + key: "commit-before-crash-0001", + body: { title: "recover me" }, + }); + expect(replay.status).toBe(200); + expect(await json(replay)).toEqual(session); + await ledger.close(); + }); + + it("fences a stale owner from applying, acknowledging, or publishing events", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "fence-workspace-key-001", body: { name: "fence" } })); + const session = await json(await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: "fence-session-key-0001", body: { title: "fenced" } })); + const kubernetes = new ObservableKubernetes(); + const stale = newWorker(ledger, "stale", kubernetes); + const staleLease = await stale.acquireLease(); + kubernetes.currentEpoch = staleLease.epoch; + const claim = await stale.claimNext(); + expect(claim).toBeDefined(); + + await admin`UPDATE t4_owner_leases SET expires_at = clock_timestamp() - interval '1 second' WHERE lease_name = 'gateway-outbox'`; + const current = newWorker(ledger, "current", kubernetes); + const currentLease = await current.acquireLease(); + kubernetes.currentEpoch = currentLease.epoch; + expect(await stale.applyClaim(claim as OutboxClaim)).toBe("fenced"); + expect(kubernetes.attempts).toHaveLength(0); + expect(await current.drain()).toBeGreaterThan(0); + const events = await admin`SELECT owner_epoch FROM t4_events WHERE session_id = ${session.id} ORDER BY sequence` as { owner_epoch: bigint }[]; + expect(events.every(row => row.owner_epoch === currentLease.epoch || row.owner_epoch === 0n)).toBe(true); + await ledger.close(); + }); + + it("resumes SSE strictly after a durable cursor and returns typed 410 after retention expiry", async () => { + const ledger = await newLedger({ eventRetention: 3 }); + const api = newApi(ledger); + const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "sse-workspace-key-00001", body: { name: "sse" } })); + const session = await json(await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: "sse-session-key-000001", body: { title: "events" } })); + const snapshot = await json(await call(api, "GET", `/v1/sessions/${session.id}/snapshot`)); + for (let index = 0; index < 2; index++) { + const response = await call(api, "POST", `/v1/sessions/${session.id}/commands`, { + key: `sse-command-key-00000${index}`, + body: { command: `echo ${index}` }, + }); + expect(response.status).toBe(202); + } + const first = await call(api, "GET", `/v1/sessions/${session.id}/events?cursor=${encodeURIComponent(snapshot.cursor)}&maxEvents=1&heartbeatSeconds=5`, { headers: { accept: "text/event-stream" } }); + expect(first.status).toBe(200); + const firstEvents = eventData(await first.text()); + expect(firstEvents).toHaveLength(1); + const deliveredCursor = firstEvents[0].cursor as string; + const second = await call(api, "GET", `/v1/sessions/${session.id}/events?maxEvents=100&heartbeatSeconds=5`, { + headers: { "last-event-id": deliveredCursor, accept: "text/event-stream" }, + }); + expect(second.status).toBe(200); + const secondEvents = eventData(await second.text()); + expect(secondEvents.map(value => value.cursor)).not.toContain(deliveredCursor); + expect(new Set([...firstEvents, ...secondEvents].map(value => value.cursor)).size).toBe(firstEvents.length + secondEvents.length); + + for (let index = 2; index < 7; index++) { + await call(api, "POST", `/v1/sessions/${session.id}/commands`, { + key: `sse-command-key-00000${index}`, + body: { command: `echo ${index}` }, + }); + } + const expired = await call(api, "GET", `/v1/sessions/${session.id}/events?cursor=${encodeURIComponent(deliveredCursor)}&maxEvents=1&heartbeatSeconds=5`, { headers: { accept: "text/event-stream" } }); + expect(expired.status).toBe(410); + const expiredBody = await json(expired); + expect(expiredBody.error).toMatchObject({ + code: "cursor_expired", + resync: { snapshotUrl: `/v1/sessions/${session.id}/snapshot` }, + }); + await ledger.close(); + }); + + it("persists cancellation before dispatch and keeps lifecycle monotonic across restart", async () => { + let ledger = await newLedger(); + let api = newApi(ledger); + const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "cancel-workspace-key-001", body: { name: "cancel" } })); + const session = await json(await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: "cancel-session-key-0001", body: { title: "cancel me" } })); + const cancellation = await call(api, "POST", `/v1/sessions/${session.id}/cancel`, { key: "cancel-race-key-0000001" }); + expect(cancellation.status).toBe(202); + expect(await json(cancellation)).toMatchObject({ id: session.id, state: "cancelling", revision: 2 }); + await ledger.close(); + + ledger = await newLedger(); + api = newApi(ledger); + const persisted = await call(api, "GET", `/v1/sessions/${session.id}`); + expect(await json(persisted)).toMatchObject({ id: session.id, state: "cancelling", revision: 2 }); + const kubernetes = new ObservableKubernetes(); + const worker = newWorker(ledger, "cancel-worker", kubernetes); + const lease = await worker.acquireLease(); + kubernetes.currentEpoch = lease.epoch; + await worker.drain(); + const appliedForSession = [...kubernetes.applied.values()].filter(value => value.targetId === session.id); + expect(appliedForSession.map(value => value.kind)).toEqual(["session.cancel"]); + await ledger.close(); + }); + + it("preserves snapshots, status, ordered events, and replay responses on restart", async () => { + let ledger = await newLedger(); + let api = newApi(ledger); + const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "durable-status-workspace", body: { name: "status" } })); + const session = await json(await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: "durable-status-session-01", body: { title: "status" } })); + const command = await json(await call(api, "POST", `/v1/sessions/${session.id}/commands`, { key: "durable-status-command-01", body: { command: "printf durable" } })); + const before = await json(await call(api, "GET", `/v1/sessions/${session.id}/snapshot`)); + await ledger.close(); + + ledger = await newLedger(); + api = newApi(ledger); + expect(await json(await call(api, "GET", `/v1/sessions/${session.id}`))).toEqual(session); + expect(await json(await call(api, "GET", `/v1/sessions/${session.id}/snapshot`))).toEqual(before); + const replay = await call(api, "POST", `/v1/sessions/${session.id}/commands`, { key: "durable-status-command-01", body: { command: "printf durable" } }); + expect(replay.status).toBe(200); + expect(replay.headers.get("idempotency-replayed")).toBe("true"); + expect(await json(replay)).toEqual(command); + await ledger.close(); + }); + + it("rolls back command, intent, event, and outbox together on transaction failure", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + await admin.unsafe(` + CREATE OR REPLACE FUNCTION t4_test_reject_outbox() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF NEW.idempotency_key = 'rollback-key-000000001' THEN RAISE EXCEPTION 'injected rollback'; END IF; + RETURN NEW; + END $$; + DROP TRIGGER IF EXISTS t4_test_reject_outbox_trigger ON t4_outbox; + CREATE TRIGGER t4_test_reject_outbox_trigger BEFORE INSERT ON t4_outbox + FOR EACH ROW EXECUTE FUNCTION t4_test_reject_outbox(); + `); + const response = await call(api, "POST", "/v1/workspaces", { key: "rollback-key-000000001", body: { name: "must not persist" } }); + expect(response.status).toBe(503); + const rows = await admin.unsafe(` + SELECT + (SELECT count(*) FROM t4_commands WHERE idempotency_key = 'rollback-key-000000001') AS commands, + (SELECT count(*) FROM t4_workspace_intents WHERE name = 'must not persist') AS intents, + (SELECT count(*) FROM t4_outbox WHERE idempotency_key = 'rollback-key-000000001') AS outbox + `); + expect(rows[0]).toMatchObject({ commands: 0n, intents: 0n, outbox: 0n }); + await admin.unsafe("DROP TRIGGER t4_test_reject_outbox_trigger ON t4_outbox; DROP FUNCTION t4_test_reject_outbox()"); + await ledger.close(); + }); + + it("retries the whole serializable mutation after a real PostgreSQL 40001 exactly once", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + await admin.unsafe(` + DROP SEQUENCE IF EXISTS t4_test_serialization_attempt; + CREATE SEQUENCE t4_test_serialization_attempt; + CREATE OR REPLACE FUNCTION t4_test_serialize_once() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF NEW.idempotency_key = 'serialization-retry-key-01' AND nextval('t4_test_serialization_attempt') = 1 THEN + RAISE SQLSTATE '40001' USING MESSAGE = 'forced serialization rollback'; + END IF; + RETURN NEW; + END $$; + DROP TRIGGER IF EXISTS t4_test_serialize_once_trigger ON t4_outbox; + CREATE TRIGGER t4_test_serialize_once_trigger BEFORE INSERT ON t4_outbox + FOR EACH ROW EXECUTE FUNCTION t4_test_serialize_once(); + `); + const response = await call(api, "POST", "/v1/workspaces", { + key: "serialization-retry-key-01", + body: { name: "retry exactly once" }, + }); + expect(response.status).toBe(202); + + const accepted = await json(response); + const rows = await admin.unsafe(` + SELECT + (SELECT last_value FROM t4_test_serialization_attempt) AS attempts, + (SELECT count(*) FROM t4_commands WHERE idempotency_key = 'serialization-retry-key-01') AS commands, + (SELECT count(*) FROM t4_workspace_intents WHERE workspace_id = '${accepted.id}') AS intents, + (SELECT count(*) FROM t4_outbox WHERE idempotency_key = 'serialization-retry-key-01') AS outbox + `); + expect(rows[0]).toMatchObject({ attempts: 2n, commands: 1n, intents: 1n, outbox: 1n }); + await admin.unsafe("DROP TRIGGER t4_test_serialize_once_trigger ON t4_outbox; DROP FUNCTION t4_test_serialize_once(); DROP SEQUENCE t4_test_serialization_attempt"); + await ledger.close(); + }); + + it("retries the whole serializable mutation after a real PostgreSQL 40P01 exactly once", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + await admin.unsafe(` + DROP SEQUENCE IF EXISTS t4_test_deadlock_attempt; + CREATE SEQUENCE t4_test_deadlock_attempt; + CREATE OR REPLACE FUNCTION t4_test_deadlock_once() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF NEW.idempotency_key = 'deadlock-retry-key-00001' AND nextval('t4_test_deadlock_attempt') = 1 THEN + RAISE SQLSTATE '40P01' USING MESSAGE = 'forced deadlock rollback'; + END IF; + RETURN NEW; + END $$; + CREATE TRIGGER t4_test_deadlock_once_trigger BEFORE INSERT ON t4_outbox + FOR EACH ROW EXECUTE FUNCTION t4_test_deadlock_once(); + `); + const response = await call(api, "POST", "/v1/workspaces", { key: "deadlock-retry-key-00001", body: { name: "retry deadlock once" } }); + expect(response.status).toBe(202); + const rows = await admin`SELECT + (SELECT last_value FROM t4_test_deadlock_attempt) AS attempts, + (SELECT count(*) FROM t4_commands WHERE idempotency_key = 'deadlock-retry-key-00001') AS commands, + (SELECT count(*) FROM t4_outbox WHERE idempotency_key = 'deadlock-retry-key-00001') AS outbox` as Array<{ attempts: bigint; commands: bigint; outbox: bigint }>; + expect(rows[0]).toEqual({ attempts: 2n, commands: 1n, outbox: 1n }); + await admin.unsafe("DROP TRIGGER t4_test_deadlock_once_trigger ON t4_outbox; DROP FUNCTION t4_test_deadlock_once(); DROP SEQUENCE t4_test_deadlock_attempt"); + await ledger.close(); + }); + + it("serializes two first-use requests for the same absent idempotency row", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + await admin.unsafe(` + DROP SEQUENCE IF EXISTS t4_test_absent_key_attempt; + CREATE SEQUENCE t4_test_absent_key_attempt; + CREATE OR REPLACE FUNCTION t4_test_pause_first_absent_key() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF NEW.idempotency_key = 'absent-idempotency-key-01' THEN + PERFORM nextval('t4_test_absent_key_attempt'); + IF currval('t4_test_absent_key_attempt') = 1 THEN PERFORM pg_advisory_xact_lock(741406); END IF; + END IF; + RETURN NEW; + END $$; + DROP TRIGGER IF EXISTS t4_test_pause_absent_key_trigger ON t4_outbox; + CREATE TRIGGER t4_test_pause_absent_key_trigger BEFORE INSERT ON t4_outbox + FOR EACH ROW EXECUTE FUNCTION t4_test_pause_first_absent_key(); + `); + const blocker = new SQL(DATABASE_URL!, { max: 1, bigint: true }); + await blocker`SELECT pg_advisory_lock(741406)`; + const first = call(api, "POST", "/v1/workspaces", { key: "absent-idempotency-key-01", body: { name: "one durable winner" } }); + let second: Promise | undefined; + let responses: Response[] | undefined; + try { + await waitForDatabase(async () => (await admin`SELECT is_called FROM t4_test_absent_key_attempt` as Array<{ is_called: boolean }>)[0]?.is_called === true); + const secondRequest = call(api, "POST", "/v1/workspaces", { key: "absent-idempotency-key-01", body: { name: "one durable winner" } }); + second = secondRequest; + await waitForDatabase(async () => { + const blocked = await admin`SELECT count(*) AS count FROM pg_stat_activity WHERE datname = current_database() AND wait_event = 'advisory' AND query LIKE 'SELECT pg_advisory_xact_lock(hashtextextended%'` as Array<{ count: bigint }>; + return blocked[0]?.count === 1n; + }); + await blocker`SELECT pg_advisory_unlock(741406)`; + responses = await Promise.all([first, secondRequest]); + } finally { + await blocker.close(); + await Promise.allSettled([first, ...(second ? [second] : [])]); + await admin.unsafe("DROP TRIGGER IF EXISTS t4_test_pause_absent_key_trigger ON t4_outbox; DROP FUNCTION IF EXISTS t4_test_pause_first_absent_key(); DROP SEQUENCE IF EXISTS t4_test_absent_key_attempt"); + } + expect(responses).toBeDefined(); + expect(responses!.map(value => value.status).sort()).toEqual([200, 202]); + expect(await json(responses![0]!)).toEqual(await json(responses![1]!)); + const rows = await admin`SELECT + (SELECT count(*) FROM t4_commands WHERE idempotency_key = 'absent-idempotency-key-01') AS commands, + (SELECT count(*) FROM t4_outbox WHERE idempotency_key = 'absent-idempotency-key-01') AS outbox` as Array<{ commands: bigint; outbox: bigint }>; + expect(rows[0]).toEqual({ commands: 1n, outbox: 1n }); + await ledger.close(); + }); + + it("replays stored patch, create, and command responses before mutable state checks", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "replay-state-workspace-01", body: { name: "original" } })); + const firstWorkspacePatch = await json(await call(api, "PATCH", `/v1/workspaces/${workspace.id}`, { + key: "replay-workspace-patch-01", ifMatch: "1", body: { name: "first patch" }, + })); + expect((await call(api, "PATCH", `/v1/workspaces/${workspace.id}`, { + key: "replay-workspace-patch-02", ifMatch: "2", body: { name: "second patch" }, + })).status).toBe(200); + const workspaceReplay = await call(api, "PATCH", `/v1/workspaces/${workspace.id}`, { + key: "replay-workspace-patch-01", ifMatch: "1", body: { name: "first patch" }, + }); + expect(workspaceReplay.status).toBe(200); + expect(workspaceReplay.headers.get("idempotency-replayed")).toBe("true"); + expect(await json(workspaceReplay)).toEqual(firstWorkspacePatch); + const workspaceMismatch = await call(api, "PATCH", `/v1/workspaces/${workspace.id}`, { + key: "replay-workspace-patch-01", ifMatch: "1", body: { name: "different" }, + }); + expect((await json(workspaceMismatch)).error.code).toBe("idempotency_conflict"); + + const sessionKey = "replay-session-create-001"; + const session = await json(await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: sessionKey, body: { title: "original" } })); + const firstSessionPatch = await json(await call(api, "PATCH", `/v1/sessions/${session.id}`, { + key: "replay-session-patch-001", ifMatch: "1", body: { title: "first patch" }, + })); + expect((await call(api, "PATCH", `/v1/sessions/${session.id}`, { + key: "replay-session-patch-002", ifMatch: "2", body: { title: "second patch" }, + })).status).toBe(200); + const commandKey = "replay-command-state-001"; + const command = await json(await call(api, "POST", `/v1/sessions/${session.id}/commands`, { key: commandKey, body: { command: "printf original" } })); + expect((await call(api, "POST", `/v1/sessions/${session.id}/cancel`, { key: "replay-cancel-state-001" })).status).toBe(202); + const sessionReplay = await call(api, "PATCH", `/v1/sessions/${session.id}`, { + key: "replay-session-patch-001", ifMatch: "1", body: { title: "first patch" }, + }); + expect(sessionReplay.status).toBe(200); + expect(await json(sessionReplay)).toEqual(firstSessionPatch); + const commandReplay = await call(api, "POST", `/v1/sessions/${session.id}/commands`, { key: commandKey, body: { command: "printf original" } }); + expect(commandReplay.status).toBe(200); + expect(await json(commandReplay)).toEqual(command); + const commandMismatch = await call(api, "POST", `/v1/sessions/${session.id}/commands`, { key: commandKey, body: { command: "printf different" } }); + expect((await json(commandMismatch)).error.code).toBe("idempotency_conflict"); + expect((await call(api, "DELETE", `/v1/workspaces/${workspace.id}`, { key: "replay-delete-parent-001" })).status).toBe(204); + const createReplay = await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: sessionKey, body: { title: "original" } }); + expect(createReplay.status).toBe(200); + expect(await json(createReplay)).toEqual(session); + const createMismatch = await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: sessionKey, body: { title: "different" } }); + expect((await json(createMismatch)).error.code).toBe("idempotency_conflict"); + await ledger.close(); + }); + + it("keeps cancelling and terminal sessions monotonic across distinct requests", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "monotonic-workspace-key-1", body: { name: "monotonic" } })); + const session = await json(await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: "monotonic-session-key-01", body: { title: "monotonic" } })); + const firstCancel = await json(await call(api, "POST", `/v1/sessions/${session.id}/cancel`, { key: "monotonic-cancel-key-001" })); + const repeated = await call(api, "POST", `/v1/sessions/${session.id}/cancel`, { key: "monotonic-cancel-key-002" }); + expect(repeated.status).toBe(202); + const repeatedReplay = await call(api, "POST", `/v1/sessions/${session.id}/cancel`, { key: "monotonic-cancel-key-002" }); + expect(repeatedReplay.status).toBe(200); + expect(await json(repeatedReplay)).toEqual(firstCancel); + expect(await json(repeated)).toEqual(firstCancel); + expect((await call(api, "PATCH", `/v1/sessions/${session.id}`, { key: "monotonic-patch-key-0001", ifMatch: String(firstCancel.revision), body: { title: "forbidden" } })).status).toBe(404); + expect((await call(api, "POST", `/v1/sessions/${session.id}/commands`, { key: "monotonic-command-key-01", body: { command: "forbidden" } })).status).toBe(404); + await admin`UPDATE t4_session_intents SET state = 'cancelled' WHERE session_id = ${session.id}`; + const terminalCancel = await call(api, "POST", `/v1/sessions/${session.id}/cancel`, { key: "monotonic-cancel-key-003" }); + expect(terminalCancel.status).toBe(202); + const firstDelete = await call(api, "DELETE", `/v1/sessions/${session.id}`, { key: "monotonic-delete-key-001" }); + expect(firstDelete.status).toBe(204); + expect(firstDelete.headers.get("idempotency-replayed")).toBe("false"); + const deleteReplay = await call(api, "DELETE", `/v1/sessions/${session.id}`, { key: "monotonic-delete-key-001" }); + expect(deleteReplay.status).toBe(204); + expect(deleteReplay.headers.get("idempotency-replayed")).toBe("true"); + const after = await admin`SELECT + (SELECT count(*) FROM t4_commands WHERE target_scope = ${`session:${session.id}`}) AS commands, + (SELECT count(*) FROM t4_outbox WHERE target_id = ${session.id}) AS outbox, + (SELECT count(*) FROM t4_events WHERE session_id = ${session.id}) AS events, + (SELECT revision FROM t4_session_intents WHERE session_id = ${session.id}) AS revision` as Array<{ commands: bigint; outbox: bigint; events: bigint; revision: bigint }>; + expect(after[0]).toEqual({ commands: 4n, outbox: 3n, events: 3n, revision: BigInt(firstCancel.revision) }); + await ledger.close(); + }); + + it("renews, excludes, expires, and fences PostgreSQL leases", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const first = await ledger.acquireLease("lease-owner-a", 30); + const renewed = await ledger.acquireLease("lease-owner-a", 30); + expect(renewed).toEqual(first); + await expect(ledger.acquireLease("lease-owner-b", 30)).rejects.toThrow("lease"); + await call(api, "POST", "/v1/workspaces", { key: "lease-fencing-workspace-1", body: { name: "lease" } }); + const claim = await ledger.claimNext(first); + expect(claim).toBeDefined(); + await admin`UPDATE t4_owner_leases SET expires_at = clock_timestamp() - interval '1 second' WHERE lease_name = 'gateway-outbox'`; + expect(await ledger.claimIsCurrent(claim!)).toBe(false); + expect(await ledger.acknowledge(claim!)).toBe(false); + expect(await ledger.recordFailure(claim!, "expired")).toBe(false); + const takeover = await ledger.acquireLease("lease-owner-b", 30); + expect(takeover.epoch).toBe(first.epoch + 1n); + await ledger.close(); + }); + + it("rejects forged acknowledge claims unless command, kind, and target match the locked outbox row", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const first = await json(await call(api, "POST", "/v1/workspaces", { key: "ack-binding-workspace-01", body: { name: "claimed" } })); + const unrelated = await json(await call(api, "POST", "/v1/workspaces", { key: "ack-binding-workspace-02", body: { name: "unrelated" } })); + const lease = await ledger.acquireLease("ack-binding-owner", 30); + const claim = await ledger.claimNext(lease); + expect(claim).toBeDefined(); + expect(claim!.targetId).toBe(first.id); + const commands = await admin`SELECT command_id, idempotency_key, lifecycle_state FROM t4_commands WHERE idempotency_key IN ('ack-binding-workspace-01', 'ack-binding-workspace-02')` as Array<{ command_id: string; idempotency_key: string; lifecycle_state: string }>; + const unrelatedCommandId = commands.find(row => row.idempotency_key === "ack-binding-workspace-02")!.command_id; + + expect(await ledger.acknowledge({ ...claim!, commandId: unrelatedCommandId })).toBe(false); + expect(await ledger.acknowledge({ ...claim!, kind: "workspace.patch" })).toBe(false); + expect(await ledger.acknowledge({ ...claim!, targetId: unrelated.id })).toBe(false); + const unchanged = await admin`SELECT + (SELECT state FROM t4_outbox WHERE outbox_id = ${claim!.outboxId}) AS outbox_state, + (SELECT lifecycle_state FROM t4_commands WHERE command_id = ${claim!.commandId}) AS claimed_command_state, + (SELECT lifecycle_state FROM t4_commands WHERE command_id = ${unrelatedCommandId}) AS unrelated_command_state` as Array<{ outbox_state: string; claimed_command_state: string; unrelated_command_state: string }>; + expect(unchanged[0]).toEqual({ outbox_state: "claimed", claimed_command_state: "accepted", unrelated_command_state: "accepted" }); + expect(await ledger.acknowledge(claim!)).toBe(true); + await ledger.close(); + }); + + it("rejects claimed outbox rows with null or empty owners and nonpositive owner epochs at the table boundary", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + await call(api, "POST", "/v1/workspaces", { key: "claimed-row-constraint-01", body: { name: "claim constraints" } }); + const rows = await admin`SELECT outbox_id FROM t4_outbox WHERE idempotency_key = 'claimed-row-constraint-01'` as Array<{ outbox_id: bigint }>; + const outboxId = rows[0]!.outbox_id; + await expect(admin`UPDATE t4_outbox SET state = 'claimed', owner_id = NULL, owner_epoch = 1, claimed_at = clock_timestamp() WHERE outbox_id = ${outboxId}`).rejects.toMatchObject({ code: "23514" }); + await expect(admin`UPDATE t4_outbox SET state = 'claimed', owner_id = 'constraint-owner', owner_epoch = NULL, claimed_at = clock_timestamp() WHERE outbox_id = ${outboxId}`).rejects.toMatchObject({ code: "23514" }); + await expect(admin`UPDATE t4_outbox SET state = 'claimed', owner_id = 'constraint-owner', owner_epoch = 0, claimed_at = clock_timestamp() WHERE outbox_id = ${outboxId}`).rejects.toMatchObject({ code: "23514" }); + await expect(admin`UPDATE t4_outbox SET state = 'claimed', owner_id = '', owner_epoch = 1, claimed_at = clock_timestamp() WHERE outbox_id = ${outboxId}`).rejects.toMatchObject({ code: "23514" }); + await expect(admin`UPDATE t4_outbox SET state = 'claimed', owner_id = 'constraint-owner', owner_epoch = -1, claimed_at = clock_timestamp() WHERE outbox_id = ${outboxId}`).rejects.toMatchObject({ code: "23514" }); + const unchanged = await admin`SELECT state, owner_id, owner_epoch FROM t4_outbox WHERE outbox_id = ${outboxId}` as Array<{ state: string; owner_id: string | null; owner_epoch: bigint | null }>; + expect(unchanged[0]).toEqual({ state: "pending", owner_id: null, owner_epoch: null }); + await ledger.close(); + }); + + it("retries a current recordFailure serialization failure while preventing a stale owner from resetting the reclaimed item", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + await call(api, "POST", "/v1/workspaces", { key: "failure-takeover-workspace", body: { name: "failure takeover" } }); + const staleLease = await ledger.acquireLease("failure-stale-owner", 30); + const staleClaim = await ledger.claimNext(staleLease); + expect(staleClaim).toBeDefined(); + await admin`UPDATE t4_owner_leases SET expires_at = clock_timestamp() - interval '1 second' WHERE lease_name = 'gateway-outbox'`; + const currentLease = await ledger.acquireLease("failure-current-owner", 30); + const currentClaim = await ledger.claimNext(currentLease); + expect(currentClaim?.outboxId).toBe(staleClaim!.outboxId); + expect(currentClaim?.ownerEpoch).toBeGreaterThan(staleClaim!.ownerEpoch); + + expect(await ledger.recordFailure(staleClaim!, "late stale failure")).toBe(false); + const retained = await admin`SELECT state, owner_id, owner_epoch, attempts, last_error FROM t4_outbox WHERE outbox_id = ${staleClaim!.outboxId}` as Array<{ state: string; owner_id: string; owner_epoch: bigint; attempts: number; last_error: string | null }>; + expect(retained[0]).toEqual({ state: "claimed", owner_id: currentLease.ownerId, owner_epoch: currentLease.epoch, attempts: 2, last_error: null }); + + let retryFixtureInstalled = false; + try { + await admin.unsafe(` + CREATE SEQUENCE t4_test_record_failure_attempt; + CREATE FUNCTION t4_test_fail_first_record_failure() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF OLD.state = 'claimed' AND NEW.state = 'pending' AND nextval('t4_test_record_failure_attempt') = 1 THEN + RAISE EXCEPTION 'injected recordFailure serialization failure' USING ERRCODE = '40001'; + END IF; + RETURN NEW; + END $$; + CREATE TRIGGER t4_test_fail_first_record_failure_trigger BEFORE UPDATE ON t4_outbox + FOR EACH ROW EXECUTE FUNCTION t4_test_fail_first_record_failure(); + `); + retryFixtureInstalled = true; + expect(await ledger.recordFailure(currentClaim!, "retryable current failure")).toBe(true); + const released = await admin`SELECT + (SELECT last_value FROM t4_test_record_failure_attempt) AS retry_attempts, + state, owner_id, owner_epoch, attempts, last_error + FROM t4_outbox WHERE outbox_id = ${currentClaim!.outboxId}` as Array<{ retry_attempts: bigint; state: string; owner_id: string | null; owner_epoch: bigint | null; attempts: number; last_error: string | null }>; + expect(released[0]).toEqual({ retry_attempts: 2n, state: "pending", owner_id: null, owner_epoch: null, attempts: 2, last_error: "retryable current failure" }); + } finally { + if (retryFixtureInstalled) await admin.unsafe("DROP TRIGGER IF EXISTS t4_test_fail_first_record_failure_trigger ON t4_outbox; DROP FUNCTION IF EXISTS t4_test_fail_first_record_failure(); DROP SEQUENCE IF EXISTS t4_test_record_failure_attempt"); + await ledger.close(); + } + }); + + it("persists and exactly replays two distinct DELETE requests while enqueueing one provider deletion and one revision transition", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "double-delete-workspace-01", body: { name: "double delete" } })); + const session = await json(await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: "double-delete-session-001", body: { title: "double delete" } })); + const firstKey = "double-delete-request-001"; + const secondKey = "double-delete-request-002"; + const first = await call(api, "DELETE", `/v1/sessions/${session.id}`, { key: firstKey }); + const second = await call(api, "DELETE", `/v1/sessions/${session.id}`, { key: secondKey }); + expect([first.status, second.status]).toEqual([204, 204]); + expect([first.headers.get("idempotency-replayed"), second.headers.get("idempotency-replayed")]).toEqual(["false", "false"]); + const firstBody = await first.text(); + const secondBody = await second.text(); + const firstReplay = await call(api, "DELETE", `/v1/sessions/${session.id}`, { key: firstKey }); + const secondReplay = await call(api, "DELETE", `/v1/sessions/${session.id}`, { key: secondKey }); + expect([firstReplay.status, secondReplay.status]).toEqual([204, 204]); + expect([firstReplay.headers.get("idempotency-replayed"), secondReplay.headers.get("idempotency-replayed")]).toEqual(["true", "true"]); + expect([await firstReplay.text(), await secondReplay.text()]).toEqual([firstBody, secondBody]); + + const persisted = await admin`SELECT + (SELECT count(*) FROM t4_commands WHERE operation = 'deleteSession' AND target_scope = ${`session:${session.id}`}) AS commands, + (SELECT count(*) FROM t4_outbox WHERE mutation_kind = 'session.delete' AND target_id = ${session.id}) AS deletions, + (SELECT revision FROM t4_session_intents WHERE session_id = ${session.id}) AS revision, + (SELECT generation FROM t4_session_intents WHERE session_id = ${session.id}) AS generation, + (SELECT deletion_requested FROM t4_session_intents WHERE session_id = ${session.id}) AS deletion_requested` as Array<{ commands: bigint; deletions: bigint; revision: bigint; generation: bigint; deletion_requested: boolean }>; + expect(persisted[0]).toEqual({ commands: 2n, deletions: 1n, revision: 2n, generation: 2n, deletion_requested: true }); + const kubernetes = new ObservableKubernetes(); + const worker = newWorker(ledger, "double-delete-owner", kubernetes); + const lease = await worker.acquireLease(); + kubernetes.currentEpoch = lease.epoch; + await worker.drain(); + expect(kubernetes.attempts.filter(mutation => mutation.kind === "session.delete" && mutation.targetId === session.id)).toHaveLength(1); + await ledger.close(); + }); + + it("reclaims a same-owner claim only after the named stale-apply deadline and immediately fences the abandoned claim", async () => { + let ledger = await newLedger(); + let api = newApi(ledger); + await call(api, "POST", "/v1/workspaces", { key: "same-owner-restart-claim", body: { name: "restart claim" } }); + const firstWorker = newWorker(ledger, "stable-restart-owner", new ObservableKubernetes()); + const firstLease = await firstWorker.acquireLease(); + const abandoned = await firstWorker.claimNext(); + expect(abandoned).toBeDefined(); + await ledger.close(); + + ledger = await newLedger(); + api = newApi(ledger); + const kubernetes = new ObservableKubernetes(); + const restarted = newWorker(ledger, "stable-restart-owner", kubernetes); + const restartedLease = await restarted.acquireLease(); + expect(restartedLease).toEqual(firstLease); + await expect(ledger.acquireLease("concurrent-thief", 30)).rejects.toThrow("lease"); + expect(await restarted.claimNext()).toBeUndefined(); + await admin`UPDATE t4_outbox SET claimed_at = clock_timestamp() - (${STALE_APPLY_DEADLINE_SECONDS - 1} * interval '1 second') WHERE outbox_id = ${abandoned!.outboxId}`; + expect(await restarted.claimNext()).toBeUndefined(); + await admin`UPDATE t4_outbox SET claimed_at = clock_timestamp() - (${STALE_APPLY_DEADLINE_SECONDS + 1} * interval '1 second') WHERE outbox_id = ${abandoned!.outboxId}`; + const reclaimed = await restarted.claimNext(); + expect(reclaimed?.outboxId).toBe(abandoned!.outboxId); + expect(reclaimed?.ownerEpoch).toBe(firstLease.epoch); + expect(await ledger.claimIsCurrent(abandoned!)).toBe(false); + kubernetes.currentEpoch = restartedLease.epoch; + expect(await restarted.applyClaim(reclaimed!)).toBe("applied"); + expect(kubernetes.attempts).toHaveLength(1); + await ledger.close(); + }); + + it("case-folds lowercase and mixed-case Bearer schemes without altering mixed-case credential bytes", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + for (const scheme of ["bearer", "bEaReR"]) { + const requestHeaders = headers(); + requestHeaders.set("authorization", `${scheme} ${MIXED_CASE_OWNER_TOKEN}`); + const response = await api.handle(new Request("https://t4.example.test/v1", { headers: requestHeaders })); + expect(response.status).toBe(200); + } + await ledger.close(); + }); + + it("returns the generated incompatible-version Error406 envelope for malformed or non-SSE Accept", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "accept-406-workspace-01", body: { name: "accept contract" } })); + const session = await json(await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: "accept-406-session-001", body: { title: "accept contract" } })); + for (const accept of ["application/json", "text/event-stream;q=bogus"]) { + const response = await call(api, "GET", `/v1/sessions/${session.id}/events`, { headers: { accept } }); + expect(response.status).toBe(406); + expect(response.headers.get("content-type")).toContain("application/json"); + expect(response.headers.get("t4-api-version")).toBe("1.0"); + expect(await json(response)).toEqual({ + error: { + code: "incompatible_version", + message: expect.any(String), + requestId: expect.stringMatching(/^req_[0-9a-f-]{36}$/u), + retryable: false, + supportedMajors: [1], + }, + }); + } + await ledger.close(); + }); + + it("marks successful Kubernetes outbox application projected, never completed, while public command replay stays accepted", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "projected-workspace-0001", body: { name: "projected" } })); + const session = await json(await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: "projected-session-00001", body: { title: "projected" } })); + const kubernetes = new ObservableKubernetes(); + const worker = newWorker(ledger, "projection-lifecycle-owner", kubernetes); + const lease = await worker.acquireLease(); + kubernetes.currentEpoch = lease.epoch; + expect(await worker.drain()).toBeGreaterThan(0); + + const key = "projected-command-00001"; + const input = { command: "printf projected", metadata: { contract: "lifecycle" } }; + const acceptedResponse = await call(api, "POST", `/v1/sessions/${session.id}/commands`, { key, body: input }); + expect(acceptedResponse.status).toBe(202); + const accepted = await json(acceptedResponse); + expect(accepted).toMatchObject({ state: "accepted" }); + expect(await worker.drain()).toBe(1); + const replay = await call(api, "POST", `/v1/sessions/${session.id}/commands`, { key, body: input }); + expect(replay.status).toBe(200); + expect(replay.headers.get("idempotency-replayed")).toBe("true"); + expect(await json(replay)).toEqual(accepted); + const lifecycle = await admin`SELECT command.lifecycle_state, item.state AS outbox_state + FROM t4_commands command JOIN t4_outbox item USING (command_id) + WHERE command.command_id = ${String(accepted.commandId)}` as Array<{ lifecycle_state: string; outbox_state: string }>; + expect(lifecycle[0]).toEqual({ lifecycle_state: "projected", outbox_state: "applied" }); + expect(kubernetes.attempts.filter(mutation => mutation.kind === "command.submit" && mutation.targetId === session.id)).toHaveLength(1); + await ledger.close(); + }); + + it("atomically converges one complete session status event and opaque cursor while cancellation absorbs later status", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "status-ingress-workspace", body: { name: "status ingress" } })); + const session = await json(await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: "status-ingress-session-01", body: { title: "status ingress" } })); + const lease = await ledger.acquireLease("status-ingress-owner", 30); + const ingress = ledger as PostgresLedgerLike & AuthoritativeKubernetesStatusIngress; + let rejectionTriggerInstalled = false; + try { + const eventsBeforeWorkspace = await admin`SELECT count(*) AS count FROM t4_events` as Array<{ count: bigint }>; + await ingress.ingestKubernetesStatus(lease, "t4workspaces", { + resourceId: workspace.id, + principalId: "owner@example.com", + uid: "workspace-authority-uid", + resourceVersion: "workspace-rv-101", + generation: 1n, + observedGeneration: 1n, + phase: "Ready", + }); + const workspaceConvergence = await admin`SELECT + (SELECT state FROM t4_workspace_intents WHERE workspace_id = ${workspace.id}) AS state, + (SELECT count(*) FROM t4_events) AS event_count, + (SELECT resource_version FROM t4_kubernetes_status_cursors WHERE collection = 't4workspaces') AS cursor` as Array<{ state: string; event_count: bigint; cursor: string }>; + expect(workspaceConvergence[0]).toEqual({ state: "ready", event_count: eventsBeforeWorkspace[0]!.count, cursor: "workspace-rv-101" }); + + const beforeSessionStatus = await admin`SELECT count(*) AS event_count, max(sequence) AS latest_event FROM t4_events WHERE session_id = ${session.id}` as Array<{ event_count: bigint; latest_event: bigint }>; + await ingress.ingestKubernetesStatus(lease, "t4sessions", { + resourceId: session.id, + principalId: "owner@example.com", + workspaceId: workspace.id, + uid: "session-authority-uid", + resourceVersion: "900", + generation: 1n, + observedGeneration: 1n, + phase: "Running", + }); + const converged = await admin`SELECT + (SELECT state FROM t4_session_intents WHERE session_id = ${session.id}) AS state, + (SELECT count(*) FROM t4_events WHERE session_id = ${session.id} AND sequence > ${beforeSessionStatus[0]!.latest_event}) AS new_event_count, + (SELECT sequence FROM t4_events WHERE session_id = ${session.id} ORDER BY sequence DESC LIMIT 1) AS latest_event, + (SELECT payload FROM t4_events WHERE session_id = ${session.id} ORDER BY sequence DESC LIMIT 1) AS event_payload, + (SELECT owner_epoch FROM t4_events WHERE session_id = ${session.id} ORDER BY sequence DESC LIMIT 1) AS event_epoch, + (SELECT latest_sequence FROM t4_event_retention WHERE principal_id = 'owner@example.com' AND session_id = ${session.id}) AS retained_event, + (SELECT resource_version FROM t4_kubernetes_status_cursors WHERE collection = 't4sessions') AS cursor` as Array<{ state: string; new_event_count: bigint; latest_event: bigint; event_payload: Record; event_epoch: bigint; retained_event: bigint; cursor: string }>; + expect(converged[0]!.state).toBe("ready"); + expect(converged[0]!.new_event_count).toBe(1n); + expect(converged[0]!.event_payload).toEqual({ state: "ready", revision: session.revision }); + expect(converged[0]!.event_epoch).toBe(lease.epoch); + expect(converged[0]!.cursor).toBe("900"); + expect(converged[0]!.latest_event).toBeGreaterThan(beforeSessionStatus[0]!.latest_event); + expect(converged[0]!.retained_event).toBe(converged[0]!.latest_event); + + await admin.unsafe(` + CREATE OR REPLACE FUNCTION t4_test_reject_authoritative_status_event() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF NEW.event_type = 'session' AND NEW.payload->>'state' = 'failed' THEN RAISE EXCEPTION 'injected authoritative status event failure'; END IF; + RETURN NEW; + END $$; + CREATE TRIGGER t4_test_reject_authoritative_status_event_trigger BEFORE INSERT ON t4_events + FOR EACH ROW EXECUTE FUNCTION t4_test_reject_authoritative_status_event(); + `); + rejectionTriggerInstalled = true; + const beforeFailure = await admin`SELECT + state, revision, kube_uid, kube_resource_version, kube_generation, kube_observed_generation, + (SELECT count(*) FROM t4_events WHERE session_id = ${session.id}) AS event_count, + (SELECT min(sequence) FROM t4_events WHERE session_id = ${session.id}) AS first_event, + (SELECT max(sequence) FROM t4_events WHERE session_id = ${session.id}) AS latest_event, + (SELECT first_retained_sequence FROM t4_event_retention WHERE principal_id = 'owner@example.com' AND session_id = ${session.id}) AS first_retained_event, + (SELECT latest_sequence FROM t4_event_retention WHERE principal_id = 'owner@example.com' AND session_id = ${session.id}) AS latest_retained_event, + (SELECT resource_version FROM t4_kubernetes_status_cursors WHERE collection = 't4sessions') AS cursor_resource_version, + (SELECT owner_epoch FROM t4_kubernetes_status_cursors WHERE collection = 't4sessions') AS cursor_owner_epoch + FROM t4_session_intents WHERE session_id = ${session.id}`; + await expect(ingress.ingestKubernetesStatus(lease, "t4sessions", { + resourceId: session.id, + principalId: "owner@example.com", + workspaceId: workspace.id, + uid: "session-authority-uid", + resourceVersion: "901", + generation: 2n, + observedGeneration: 2n, + phase: "Failed", + })).rejects.toThrow("injected authoritative status event failure"); + const afterFailure = await admin`SELECT + state, revision, kube_uid, kube_resource_version, kube_generation, kube_observed_generation, + (SELECT count(*) FROM t4_events WHERE session_id = ${session.id}) AS event_count, + (SELECT min(sequence) FROM t4_events WHERE session_id = ${session.id}) AS first_event, + (SELECT max(sequence) FROM t4_events WHERE session_id = ${session.id}) AS latest_event, + (SELECT first_retained_sequence FROM t4_event_retention WHERE principal_id = 'owner@example.com' AND session_id = ${session.id}) AS first_retained_event, + (SELECT latest_sequence FROM t4_event_retention WHERE principal_id = 'owner@example.com' AND session_id = ${session.id}) AS latest_retained_event, + (SELECT resource_version FROM t4_kubernetes_status_cursors WHERE collection = 't4sessions') AS cursor_resource_version, + (SELECT owner_epoch FROM t4_kubernetes_status_cursors WHERE collection = 't4sessions') AS cursor_owner_epoch + FROM t4_session_intents WHERE session_id = ${session.id}`; + expect(afterFailure).toEqual(beforeFailure); + + const cancellationResponse = await call(api, "POST", `/v1/sessions/${session.id}/cancel`, { key: "status-ingress-cancel-001" }); + expect(cancellationResponse.status).toBe(202); + const cancelling = await json(cancellationResponse); + expect(cancelling).toMatchObject({ id: session.id, state: "cancelling", revision: session.revision + 1 }); + const afterCancellation = await admin`SELECT count(*) AS event_count FROM t4_events WHERE session_id = ${session.id}` as Array<{ event_count: bigint }>; + await ingress.ingestKubernetesStatus(lease, "t4sessions", { + resourceId: session.id, + principalId: "owner@example.com", + workspaceId: workspace.id, + uid: "session-authority-uid", + resourceVersion: "10", + generation: 1n, + observedGeneration: 1n, + phase: "Running", + }); + const afterLowerOpaqueVersion = await admin`SELECT + (SELECT state FROM t4_session_intents WHERE session_id = ${session.id}) AS state, + (SELECT count(*) FROM t4_events WHERE session_id = ${session.id}) AS event_count, + (SELECT resource_version FROM t4_kubernetes_status_cursors WHERE collection = 't4sessions') AS cursor_resource_version, + (SELECT owner_epoch FROM t4_kubernetes_status_cursors WHERE collection = 't4sessions') AS cursor_owner_epoch` as Array<{ state: string; event_count: bigint; cursor_resource_version: string; cursor_owner_epoch: bigint }>; + expect(afterLowerOpaqueVersion[0]).toEqual({ state: "cancelling", event_count: afterCancellation[0]!.event_count, cursor_resource_version: "10", cursor_owner_epoch: lease.epoch }); + } finally { + if (rejectionTriggerInstalled) await admin.unsafe("DROP TRIGGER IF EXISTS t4_test_reject_authoritative_status_event_trigger ON t4_events; DROP FUNCTION IF EXISTS t4_test_reject_authoritative_status_event()"); + await ledger.close(); + } + }); + + it("allows exact Unicode code-point bounds and denies browser Origin by default", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const astral = "😀".repeat(128); + const exact = await call(api, "POST", "/v1/workspaces", { + key: "unicode-codepoint-bound-01", body: { name: astral, labels: { emoji: astral } }, + }); + expect(exact.status).toBe(202); + expect((await call(api, "POST", "/v1/workspaces", { + key: "unicode-codepoint-bound-02", body: { name: `${astral}😀` }, + })).status).toBe(422); + const browser = await call(api, "GET", "/v1", { headers: { origin: "https://browser.example.test" } }); + expect(browser.status).toBe(400); + expect((await json(browser)).error.code).toBe("invalid_origin"); + expect((await call(api, "GET", "/v1")).status).toBe(200); + const allowlisted = newApi(ledger, ["https://browser.example.test"]); + expect((await call(allowlisted, "GET", "/v1", { headers: { origin: "https://browser.example.test" } })).status).toBe(200); + expect((await call(allowlisted, "GET", "/v1", { headers: { origin: "https://other.example.test" } })).status).toBe(400); + await ledger.close(); + }); + + it("converges a paused stale session create followed by cancel and expired-lease takeover", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const kubernetes = new PausedCreateKubernetes(); + const stale = newWorker(ledger, "paused-owner", kubernetes); + await stale.acquireLease(); + const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "paused-race-workspace-01", body: { name: "paused" } })); + expect(await stale.drain()).toBe(1); + const session = await json(await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: "paused-race-session-001", body: { title: "paused" } })); + const createClaim = await stale.claimNext(); + expect(createClaim).toBeDefined(); + const staleApply = stale.applyClaim(createClaim!); + await kubernetes.createStarted.promise; + expect((await call(api, "POST", `/v1/sessions/${session.id}/cancel`, { key: "paused-race-cancel-0001" })).status).toBe(202); + await admin`UPDATE t4_owner_leases SET expires_at = clock_timestamp() - interval '1 second' WHERE lease_name = 'gateway-outbox'`; + const current = newWorker(ledger, "takeover-owner", kubernetes); + const currentLease = await current.acquireLease(); + expect(currentLease.epoch).toBeGreaterThan(createClaim!.ownerEpoch); + kubernetes.releaseCreate.resolve(); + expect(await staleApply).toBe("fenced"); + expect(kubernetes.resources.has(session.id)).toBe(true); + expect(await current.drain()).toBeGreaterThan(0); + expect(kubernetes.resources.has(session.id)).toBe(false); + await ledger.close(); + }); + + it("reads snapshot entries and cursor from one database snapshot", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "atomic-snapshot-workspace", body: { name: "atomic snapshot" } })); + const session = await json(await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: "atomic-snapshot-session-1", body: { title: "atomic snapshot" } })); + await call(api, "POST", `/v1/sessions/${session.id}/commands`, { key: "atomic-snapshot-command-1", body: { command: "before snapshot" } }); + const blocker = new SQL(DATABASE_URL!, { max: 1, bigint: true }); + await blocker`SELECT pg_advisory_lock(741407)`; + await admin.unsafe(` + CREATE SEQUENCE t4_test_snapshot_pause_reached; + ALTER TABLE t4_event_retention RENAME TO t4_event_retention_base; + CREATE FUNCTION t4_test_pause_snapshot(value bigint) RETURNS bigint LANGUAGE plpgsql VOLATILE AS $$ + BEGIN PERFORM nextval('t4_test_snapshot_pause_reached'); PERFORM pg_advisory_xact_lock(741407); RETURN value; END $$; + CREATE VIEW t4_event_retention AS + SELECT principal_id, session_id, first_retained_sequence, t4_test_pause_snapshot(latest_sequence) AS latest_sequence, updated_at + FROM t4_event_retention_base; + `); + const pendingSnapshot = call(api, "GET", `/v1/sessions/${session.id}/snapshot`); + const concurrent = new SQL(DATABASE_URL!, { max: 1, bigint: true }); + const concurrentCommandId = "cmd_concurrent_snapshot_boundary"; + let snapshotResponse: Response | undefined; + try { + await waitForDatabase(async () => (await admin`SELECT is_called FROM t4_test_snapshot_pause_reached` as Array<{ is_called: boolean }>)[0]?.is_called === true); + await concurrent.begin(async transaction => { + const entries = await transaction<{ next: bigint }[]>`SELECT COALESCE(MAX(entry_sequence), -1) + 1 AS next FROM t4_snapshot_entries WHERE principal_id = 'owner@example.com' AND session_id = ${session.id}`; + await transaction`INSERT INTO t4_snapshot_entries (principal_id, session_id, entry_sequence, kind, text_value) VALUES ('owner@example.com', ${session.id}, ${entries[0]?.next ?? 0n}, 'input', 'concurrent snapshot command')`; + const events = await transaction<{ sequence: bigint }[]>`INSERT INTO t4_events (principal_id, session_id, event_type, payload, owner_epoch) VALUES ('owner@example.com', ${session.id}, 'command', ${{ commandId: concurrentCommandId, state: "accepted" }}, 0) RETURNING sequence`; + await transaction`UPDATE t4_event_retention_base SET latest_sequence = ${events[0]!.sequence}, updated_at = clock_timestamp() WHERE principal_id = 'owner@example.com' AND session_id = ${session.id}`; + }); + await blocker`SELECT pg_advisory_unlock(741407)`; + snapshotResponse = await pendingSnapshot; + } finally { + await blocker.close(); + await Promise.allSettled([pendingSnapshot]); + await concurrent.close(); + await admin.unsafe("DROP VIEW IF EXISTS t4_event_retention; ALTER TABLE t4_event_retention_base RENAME TO t4_event_retention; DROP FUNCTION IF EXISTS t4_test_pause_snapshot(bigint); DROP SEQUENCE IF EXISTS t4_test_snapshot_pause_reached"); + } + expect(snapshotResponse).toBeDefined(); + expect(snapshotResponse!.status).toBe(200); + const snapshot = await json(snapshotResponse!); + const afterBoundary = await call(api, "GET", `/v1/sessions/${session.id}/events?cursor=${encodeURIComponent(snapshot.cursor)}`, { headers: { accept: "text/event-stream" } }); + expect(afterBoundary.status).toBe(200); + expect(eventData(await afterBoundary.text()).map(value => value.commandId)).toContain(concurrentCommandId); + await ledger.close(); + }); + + it("reads retention floor, events, and delivered cursor atomically across pruning", async () => { + const ledger = await newLedger({ eventRetention: 100 }); + const api = newApi(ledger); + const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "atomic-window-workspace-1", body: { name: "atomic window" } })); + const session = await json(await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: "atomic-window-session-001", body: { title: "atomic window" } })); + const snapshot = await json(await call(api, "GET", `/v1/sessions/${session.id}/snapshot`)); + const command = await json(await call(api, "POST", `/v1/sessions/${session.id}/commands`, { key: "atomic-window-command-001", body: { command: "must survive concurrent prune" } })); + const target = await admin`SELECT sequence FROM t4_events WHERE principal_id = 'owner@example.com' AND session_id = ${session.id} AND payload->>'commandId' = ${command.commandId}` as Array<{ sequence: bigint }>; + expect(target[0]).toBeDefined(); + const blocker = new SQL(DATABASE_URL!, { max: 1, bigint: true }); + const releaseBlocker = Promise.withResolvers(); + const blockerReady = Promise.withResolvers(); + const blockerTask = blocker.begin(async transaction => { + await transaction`SELECT pg_advisory_xact_lock(741408)`; + blockerReady.resolve(); + await releaseBlocker.promise; + }); + await blockerReady.promise; + await admin.unsafe(` + CREATE SEQUENCE t4_test_event_pause_reached; + ALTER TABLE t4_event_retention RENAME TO t4_event_retention_base; + CREATE FUNCTION t4_test_pause_event(value bigint) RETURNS bigint LANGUAGE plpgsql VOLATILE AS $$ + BEGIN PERFORM nextval('t4_test_event_pause_reached'); PERFORM pg_advisory_xact_lock(741408); RETURN value; END $$; + CREATE VIEW t4_event_retention AS + SELECT principal_id, session_id, first_retained_sequence, t4_test_pause_event(latest_sequence) AS latest_sequence, updated_at + FROM t4_event_retention_base; + `); + const pendingWindow = call(api, "GET", `/v1/sessions/${session.id}/events?cursor=${encodeURIComponent(snapshot.cursor)}&maxEvents=1`, { headers: { accept: "text/event-stream" } }); + const concurrent = new SQL(DATABASE_URL!, { max: 1, bigint: true }); + let response: Response | undefined; + try { + await waitForDatabase(async () => (await admin`SELECT is_called FROM t4_test_event_pause_reached` as Array<{ is_called: boolean }>)[0]?.is_called === true); + await concurrent.begin(async transaction => { + await transaction`DELETE FROM t4_events WHERE sequence = ${target[0]!.sequence}`; + await transaction`UPDATE t4_event_retention_base SET first_retained_sequence = ${target[0]!.sequence + 1n} WHERE principal_id = 'owner@example.com' AND session_id = ${session.id}`; + }); + releaseBlocker.resolve(); + await blockerTask; + response = await pendingWindow; + } finally { + releaseBlocker.resolve(); + await blockerTask; + await blocker.close(); + await Promise.allSettled([pendingWindow]); + await concurrent.close(); + await admin.unsafe("DROP VIEW IF EXISTS t4_event_retention; ALTER TABLE t4_event_retention_base RENAME TO t4_event_retention; DROP FUNCTION IF EXISTS t4_test_pause_event(bigint); DROP SEQUENCE IF EXISTS t4_test_event_pause_reached"); + } + expect(response).toBeDefined(); + expect(response!.status).toBe(200); + expect(eventData(await response!.text()).map(value => value.commandId)).toContain(command.commandId); + await ledger.close(); + }); + + it("stops a chunked request body as soon as the byte limit is crossed", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + let pulls = 0; + let cancelledAt: number | undefined; + const body = new ReadableStream({ + pull(controller) { + pulls++; + controller.enqueue(new Uint8Array(600_000)); + }, + cancel() { cancelledAt = pulls; }, + }); + const response = await api.handle(new Request("https://t4.example.test/v1/workspaces", { + method: "POST", + headers: headers(OWNER_TOKEN, { "content-type": "application/json", "idempotency-key": "streaming-body-limit-key" }), + body, + duplex: "half", + } as RequestInit & { duplex: "half" })); + expect(response.status).toBe(400); + expect((await json(response)).error.message).toContain("maximum size"); + expect(cancelledAt).toBeDefined(); + const pullsAfterCancellation = pulls; + await Promise.resolve(); + expect(pulls).toBe(pullsAfterCancellation); + await ledger.close(); + }); + + it("requires an acceptable SSE media range and maps watch database failures to 503", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "sse-accept-workspace-01", body: { name: "sse accept" } })); + const session = await json(await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: "sse-accept-session-001", body: { title: "sse accept" } })); + for (const accept of [undefined, "application/json", "text/plain;q=1, text/event-stream;q=0"]) { + const response = await call(api, "GET", `/v1/sessions/${session.id}/events`, accept ? { headers: { accept } } : {}); + expect(response.status).toBe(406); + } + expect((await call(api, "GET", `/v1/sessions/${session.id}/events`, { headers: { accept: "text/event-stream, application/json;q=0.5" } })).status).toBe(200); + expect((await call(api, "GET", `/v1/sessions/${session.id}/events?cursor=invalid`, { headers: { accept: "text/event-stream" } })).status).toBe(422); + await admin`ALTER TABLE t4_event_retention RENAME TO t4_event_retention_unavailable`; + const unavailable = await call(api, "GET", `/v1/sessions/${session.id}/events`, { headers: { accept: "text/event-stream" } }); + expect(unavailable.status).toBe(503); + expect((await json(unavailable)).error).toMatchObject({ code: "unavailable", retryable: true }); + await admin`ALTER TABLE t4_event_retention_unavailable RENAME TO t4_event_retention`; + await ledger.close(); + }); + + it("rejects authenticated principals that cannot be projected into CRD spec.owner", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const before = await admin`SELECT count(*) AS count FROM t4_commands` as Array<{ count: bigint }>; + const response = await call(api, "POST", "/v1/workspaces", { + token: INVALID_PRINCIPAL_TOKEN, + key: "invalid-principal-owner-01", + body: { name: "must not commit" }, + }); + expect(response.status).toBe(401); + const after = await admin`SELECT count(*) AS count FROM t4_commands` as Array<{ count: bigint }>; + expect(after).toEqual(before); + await ledger.close(); + }); + + it("rejects malformed, oversized, unsupported-version input before ledger or cluster mutation", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const before = await admin`SELECT count(*)::text AS count FROM t4_commands`; + const malformed = await call(api, "POST", "/v1/workspaces", { key: "malformed-input-key-001", rawBody: "{" }); + expect(malformed.status).toBe(400); + const oversized = await call(api, "POST", "/v1/workspaces", { key: "oversized-input-key-001", body: { name: "x".repeat(129) } }); + expect(oversized.status).toBe(422); + const unsupported = await call(api, "POST", "/v1/workspaces", { + key: "unsupported-version-key", + body: { name: "unsupported" }, + headers: { "t4-api-version": "2" }, + }); + expect(unsupported.status).toBe(406); + expect((await json(unsupported)).error).toMatchObject({ code: "incompatible_version", supportedMajors: [1] }); + const after = await admin`SELECT count(*)::text AS count FROM t4_commands`; + expect(after[0].count).toBe(before[0].count); + await ledger.close(); + }); + + it("returns indistinguishable scoped 404s without accepting a user-provided principal", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "auth-isolation-key-0001", body: { name: "private" } })); + const outside = await call(api, "GET", `/v1/workspaces/${workspace.id}`, { token: OTHER_TOKEN, headers: { "x-t4-principal": "owner@example.com" } }); + const absent = await call(api, "GET", "/v1/workspaces/absent-resource", { token: OTHER_TOKEN }); + expect(outside.status).toBe(404); + expect(absent.status).toBe(404); + const outsideBody = await json(outside); + const absentBody = await json(absent); + expect(outsideBody.error.code).toBe("not_found"); + expect(outsideBody.error.message).toBe(absentBody.error.message); + expect(Object.keys(outsideBody.error).sort()).toEqual(Object.keys(absentBody.error).sort()); + const unauthenticated = await api.handle(new Request("https://t4.example.test/v1", { headers: { "t4-api-version": "1" } })); + expect(unauthenticated.status).toBe(401); + await ledger.close(); + }); +}); diff --git a/packages/cluster-server/test/kubernetes-client.test.ts b/packages/cluster-server/test/kubernetes-client.test.ts index 7adc0c65..631d8064 100644 --- a/packages/cluster-server/test/kubernetes-client.test.ts +++ b/packages/cluster-server/test/kubernetes-client.test.ts @@ -9,6 +9,10 @@ import { KubernetesTokenReviewer, semanticResourceHash, } from "../src/kubernetes-client.ts"; +import { PublicKubernetesOutboxApplier } from "../src/public-kubernetes-applier.ts"; +import { ClusterInfrastructureProjection } from "../src/kubernetes-projection.ts"; +import { KubernetesProjectionRunner } from "../src/kubernetes-runner.ts"; +import type { OutboxMutation } from "../src/outbox-worker.ts"; const PRINCIPAL = "owner@example.com"; @@ -216,6 +220,503 @@ describe("namespaced Kubernetes client", () => { }); }); +describe("durable public Kubernetes applier", () => { + const fence = { ownerId: "worker-a", epoch: 7n }; + const workspaceMutation: OutboxMutation = { + idempotencyToken: "outbox:41", + commandId: "command-41", + principalId: PRINCIPAL, + kind: "workspace.patch", + targetId: "workspace-one", + targetRevision: 2n, + payload: { id: "workspace-one", name: "patched", revision: 2 }, + }; + const priorAnnotations = { + "cluster.t4.dev/ledger-command-id": "prior-command", + "cluster.t4.dev/ledger-outbox-token": "outbox:prior", + "cluster.t4.dev/ledger-owner": "prior-worker", + "cluster.t4.dev/ledger-owner-epoch": "6", + "cluster.t4.dev/ledger-semantic-hash": "sha256:prior", + }; + + it("does not compensate an objectless non-success POST with fallback GET or DELETE", async () => { + const requests: Array<{ url: string; init?: RequestInit }> = []; + const fetch = (async (input: string | URL | Request, init?: RequestInit) => { + requests.push({ url: String(input), init }); + return init?.method === "POST" + ? Response.json({ reason: "ServiceUnavailable" }, { status: 503 }) + : Response.json({ reason: "NotFound" }, { status: 404 }); + }) as typeof globalThis.fetch; + const applier = new PublicKubernetesOutboxApplier({ + client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch }), + hostRef: "primary", + }); + const createMutation: OutboxMutation = { + ...workspaceMutation, + kind: "workspace.create", + payload: { id: "workspace-one", name: "created", revision: 1 }, + }; + + await expect(applier.apply(createMutation, fence)).rejects.toMatchObject({ status: 503 }); + expect(requests.map(request => request.init?.method ?? "GET")).toEqual(["POST"]); + }); + + it("cleans a stale successful create by its returned UID while protecting a replacement UID", async () => { + const requests: Array<{ url: string; init?: RequestInit }> = []; + const replacementUid = "workspace-replacement-uid"; + let replacementProtected = false; + const fetch = (async (input: string | URL | Request, init?: RequestInit) => { + requests.push({ url: String(input), init }); + if (init?.method === "POST") { + const posted = JSON.parse(String(init.body)); + return Response.json({ + ...posted, + metadata: { ...posted.metadata, uid: "workspace-created-uid", resourceVersion: "81" }, + }, { status: 201 }); + } + if (init?.method === "DELETE") { + const deletion = JSON.parse(String(init.body)); + replacementProtected = deletion.preconditions.uid !== replacementUid; + return replacementProtected + ? Response.json({ reason: "Conflict" }, { status: 409 }) + : Response.json({}); + } + return Response.json({ + apiVersion: "cluster.t4.dev/v1alpha1", + kind: "T4Workspace", + metadata: { name: "workspace-one", uid: replacementUid, resourceVersion: "82" }, + }, { status: 200 }); + }) as typeof globalThis.fetch; + const applier = new PublicKubernetesOutboxApplier({ + client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch }), + hostRef: "primary", + }); + const createMutation: OutboxMutation = { + ...workspaceMutation, + kind: "workspace.create", + payload: { id: "workspace-one", name: "created", revision: 1 }, + }; + let currencyChecks = 0; + + await expect(applier.apply(createMutation, fence, { + claimIsCurrent: async () => ++currencyChecks === 1, + })).rejects.toThrow("no longer current"); + expect(requests.map(request => request.init?.method ?? "GET")).toEqual(["POST", "DELETE"]); + expect(JSON.parse(String(requests[1]?.init?.body))).toMatchObject({ + preconditions: { uid: "workspace-created-uid", resourceVersion: "81" }, + }); + expect(replacementProtected).toBe(true); + }); + + it.each([409, 503])("persists and safely replays exact-UID stale-create cleanup after DELETE %i", async cleanupStatus => { + interface StaleCreateCleanup { + readonly resourceType: "t4workspaces" | "t4sessions"; + readonly targetId: string; + readonly uid: string; + readonly resourceVersion: string; + } + interface StaleCreateCleanupReplayer { + replayStaleCreateCleanup(cleanup: StaleCreateCleanup, context: { claimIsCurrent(): Promise }): Promise; + } + const durableCleanups: StaleCreateCleanup[] = []; + const requests: Array<{ url: string; init?: RequestInit }> = []; + let phase: "initial" | "same-uid" | "replacement-uid" = "initial"; + const fetch = (async (input: string | URL | Request, init?: RequestInit) => { + requests.push({ url: String(input), init }); + if (phase === "initial" && init?.method === "POST") { + const posted = JSON.parse(String(init.body)); + return Response.json({ + ...posted, + metadata: { ...posted.metadata, uid: "workspace-created-uid", resourceVersion: "91" }, + }, { status: 201 }); + } + if (phase === "initial") { + return Response.json({ reason: cleanupStatus === 409 ? "Conflict" : "ServiceUnavailable" }, { status: cleanupStatus }); + } + if (!init?.method) { + return Response.json({ + apiVersion: "cluster.t4.dev/v1alpha1", + kind: "T4Workspace", + metadata: { + name: "workspace-one", + uid: phase === "same-uid" ? "workspace-created-uid" : "workspace-replacement-uid", + resourceVersion: phase === "same-uid" ? "92" : "93", + }, + }, { status: 200 }); + } + return Response.json({}, { status: 200 }); + }) as typeof globalThis.fetch; + const applier = new PublicKubernetesOutboxApplier({ + client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch }), + hostRef: "primary", + }); + const createMutation: OutboxMutation = { + ...workspaceMutation, + kind: "workspace.create", + payload: { id: "workspace-one", name: "created", revision: 1 }, + }; + let currencyChecks = 0; + const cleanupAwareContext = { + claimIsCurrent: async () => ++currencyChecks === 1, + persistStaleCreateCleanup: async (cleanup: StaleCreateCleanup) => { durableCleanups.push(cleanup); }, + }; + + await expect(applier.apply(createMutation, fence, cleanupAwareContext)).rejects.toThrow(); + expect(requests.map(request => request.init?.method ?? "GET")).toEqual(["POST", "DELETE"]); + expect(durableCleanups).toEqual([{ + resourceType: "t4workspaces", + targetId: "workspace-one", + uid: "workspace-created-uid", + resourceVersion: "91", + }]); + + const replayer = applier as unknown as StaleCreateCleanupReplayer; + let replayCurrencyChecks = 0; + const currentOwner = { claimIsCurrent: async () => { replayCurrencyChecks += 1; return true; } }; + phase = "same-uid"; + const sameUidRequestStart = requests.length; + await expect(replayer.replayStaleCreateCleanup(durableCleanups[0]!, currentOwner)).resolves.toBeUndefined(); + const sameUidRequests = requests.slice(sameUidRequestStart); + expect(sameUidRequests.map(request => request.init?.method ?? "GET")).toEqual(["GET", "DELETE"]); + expect(sameUidRequests[0]?.url).toBe("https://kubernetes.default.svc/apis/cluster.t4.dev/v1alpha1/namespaces/development/t4workspaces/workspace-one"); + expect(JSON.parse(String(sameUidRequests[1]?.init?.body))).toEqual({ + apiVersion: "v1", + kind: "DeleteOptions", + propagationPolicy: "Foreground", + preconditions: { uid: "workspace-created-uid", resourceVersion: "92" }, + }); + expect(replayCurrencyChecks).toBeGreaterThanOrEqual(1); + const currencyChecksBeforeReplacement = replayCurrencyChecks; + + phase = "replacement-uid"; + const replacementRequestStart = requests.length; + await expect(replayer.replayStaleCreateCleanup(durableCleanups[0]!, currentOwner)).resolves.toBeUndefined(); + const replacementRequests = requests.slice(replacementRequestStart); + expect(replacementRequests.map(request => request.init?.method ?? "GET")).toEqual(["GET"]); + expect(replacementRequests[0]?.url).toBe("https://kubernetes.default.svc/apis/cluster.t4.dev/v1alpha1/namespaces/development/t4workspaces/workspace-one"); + expect(replayCurrencyChecks).toBeGreaterThan(currencyChecksBeforeReplacement); + }); + + it("carries observed resourceVersion and owner epoch on every existing-resource PATCH", async () => { + const values = recordingFetch([{ + apiVersion: "cluster.t4.dev/v1alpha1", + kind: "T4Workspace", + metadata: { name: "workspace-one", resourceVersion: "41", annotations: priorAnnotations }, + spec: { hostRef: "primary", owner: PRINCIPAL, displayName: "old" }, + }, {}]); + const applier = new PublicKubernetesOutboxApplier({ + client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch: values.fetch }), + hostRef: "primary", + }); + await applier.apply(workspaceMutation, fence); + const patch = JSON.parse(String(values.requests[1]?.init?.body)); + expect(patch.metadata).toMatchObject({ + resourceVersion: "41", + annotations: { + "cluster.t4.dev/ledger-owner": "worker-a", + "cluster.t4.dev/ledger-owner-epoch": "7", + }, + }); + }); + + it("lets Kubernetes reject a paused stale resourceVersion after a current-owner mutation", async () => { + let actualVersion = "51"; + const patchStarted = Promise.withResolvers(); + const releasePatch = Promise.withResolvers(); + const fetch = (async (_input: string | URL | Request, init?: RequestInit) => { + if (init?.method !== "PATCH") return Response.json({ + apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Workspace", + metadata: { name: "workspace-one", resourceVersion: "51", annotations: priorAnnotations }, + spec: { hostRef: "primary", owner: PRINCIPAL }, + }); + patchStarted.resolve(); + await releasePatch.promise; + const body = JSON.parse(String(init.body)); + return body.metadata?.resourceVersion === actualVersion + ? Response.json({ metadata: { resourceVersion: "52" } }) + : Response.json({ reason: "Conflict" }, { status: 409 }); + }) as typeof globalThis.fetch; + const applier = new PublicKubernetesOutboxApplier({ + client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch }), + hostRef: "primary", + }); + const stale = applier.apply(workspaceMutation, fence); + await patchStarted.promise; + actualVersion = "52"; + releasePatch.resolve(); + await expect(stale).rejects.toMatchObject({ status: 409 }); + }); + + it("validates full kind, host, principal relation, and ledger identity on create-409 replay", async () => { + const createMutation: OutboxMutation = { + ...workspaceMutation, + kind: "workspace.create", + payload: { id: "workspace-one", name: "created", revision: 1 }, + }; + const requiredAnnotations = { + "cluster.t4.dev/ledger-command-id": createMutation.commandId, + "cluster.t4.dev/ledger-outbox-token": createMutation.idempotencyToken, + "cluster.t4.dev/ledger-owner": "expired-worker", + "cluster.t4.dev/ledger-owner-epoch": (fence.epoch - 1n).toString(), + "cluster.t4.dev/ledger-semantic-hash": semanticResourceHash(createMutation.payload), + }; + const exactExisting = { + apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Workspace", + metadata: { name: "workspace-one", resourceVersion: "9", annotations: requiredAnnotations }, + spec: { hostRef: "primary", owner: PRINCIPAL, displayName: "created", retentionPolicy: "Retain", size: "20Gi" }, + }; + const exact = conflictFetch(exactExisting); + await new PublicKubernetesOutboxApplier({ + client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch: exact.fetch }), + hostRef: "primary", + }).apply(createMutation, fence); + const adoption = JSON.parse(String(exact.requests[2]?.init?.body)); + expect(adoption.metadata).toMatchObject({ + resourceVersion: "9", + annotations: { + "cluster.t4.dev/ledger-owner": fence.ownerId, + "cluster.t4.dev/ledger-owner-epoch": fence.epoch.toString(), + }, + }); + + for (const conflicting of [ + { ...exactExisting, kind: "T4Session" }, + { ...exactExisting, spec: { ...exactExisting.spec, hostRef: "other" } }, + { ...exactExisting, spec: { ...exactExisting.spec, owner: "other@example.com" } }, + { ...exactExisting, metadata: { ...exactExisting.metadata, annotations: { ...requiredAnnotations, "cluster.t4.dev/ledger-command-id": "copied-wrong-command" } } }, + ]) { + const values = conflictFetch(conflicting); + const applier = new PublicKubernetesOutboxApplier({ + client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch: values.fetch }), + hostRef: "primary", + }); + await expect(applier.apply(createMutation, fence)).rejects.toThrow("conflict"); + } + }); + + it("rejects patching existing resources with missing ledger identity or the wrong API kind", async () => { + for (const existing of [ + { apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Workspace", metadata: { name: "workspace-one", resourceVersion: "8", annotations: { "cluster.t4.dev/ledger-owner-epoch": "6" } }, spec: { hostRef: "primary", owner: PRINCIPAL } }, + { apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Session", metadata: { name: "workspace-one", resourceVersion: "8", annotations: priorAnnotations }, spec: { hostRef: "primary", owner: PRINCIPAL } }, + ]) { + const values = recordingFetch([existing]); + const applier = new PublicKubernetesOutboxApplier({ + client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch: values.fetch }), + hostRef: "primary", + }); + await expect(applier.apply(workspaceMutation, fence)).rejects.toThrow("identity"); + expect(values.requests).toHaveLength(1); + } + }); + it("uses the configured host-admitted runtime profile for durable session creation", async () => { + const values = recordingFetch([{ + apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Workspace", + metadata: { name: "workspace-one", resourceVersion: "55", annotations: priorAnnotations }, + spec: { hostRef: "primary", owner: PRINCIPAL }, + }, {}]); + const options = { + client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch: values.fetch }), + hostRef: "primary", + runtimeProfile: "review-admitted", + }; + const applier = new PublicKubernetesOutboxApplier(options); + await applier.apply({ + ...workspaceMutation, + kind: "session.create", + targetId: "session-one", + payload: { id: "session-one", workspaceId: "workspace-one", title: "session", revision: 1 }, + }, fence); + const created = JSON.parse(String(values.requests[1]?.init?.body)); + expect(created.spec.runtimeProfile).toBe("review-admitted"); + }); + + it("projects only a bounded durable command pointer and keeps command payload in PostgreSQL", async () => { + const secretCommand = `printf ${"secret-payload".repeat(8_192)}`; + const values = recordingFetch([{ + apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Session", + metadata: { name: "session-one", resourceVersion: "61", annotations: priorAnnotations }, + spec: { hostRef: "primary", workspaceRef: "workspace-one" }, + }, { + apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Workspace", + metadata: { name: "workspace-one", resourceVersion: "60", annotations: priorAnnotations }, + spec: { hostRef: "primary", owner: PRINCIPAL }, + }, {}]); + const applier = new PublicKubernetesOutboxApplier({ + client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch: values.fetch }), + hostRef: "primary", + }); + const commandMutation: OutboxMutation = { + ...workspaceMutation, + kind: "command.submit", + targetId: "session-one", + payload: { sessionId: "session-one", command: secretCommand, metadata: { secret: "never-project" } }, + }; + await applier.apply(commandMutation, fence); + const patchRequest = values.requests.at(-1)!; + const patch = JSON.parse(String(patchRequest.init?.body)); + expect(patch.metadata.annotations["cluster.t4.dev/pending-command-id"]).toBe(commandMutation.commandId); + expect(patch.metadata.annotations["cluster.t4.dev/pending-command-epoch"]).toBe(fence.epoch.toString()); + expect(JSON.stringify(patch)).not.toContain("secret-payload"); + expect(JSON.stringify(patch)).not.toContain("never-project"); + expect(new TextEncoder().encode(JSON.stringify(patch.metadata)).byteLength).toBeLessThan(4_096); + }); + + it("removes the legacy full command payload on submit while replacing the bounded pointer", async () => { + const values = recordingFetch([{ + apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Session", + metadata: { name: "session-one", resourceVersion: "66", annotations: { + ...priorAnnotations, + "cluster.t4.dev/pending-command": "legacy full command payload", + "cluster.t4.dev/pending-command-id": "prior-command-id", + "cluster.t4.dev/pending-command-epoch": "6", + } }, + spec: { hostRef: "primary", workspaceRef: "workspace-one" }, + }, { + apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Workspace", + metadata: { name: "workspace-one", resourceVersion: "65", annotations: priorAnnotations }, + spec: { hostRef: "primary", owner: PRINCIPAL }, + }, {}]); + const applier = new PublicKubernetesOutboxApplier({ + client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch: values.fetch }), + hostRef: "primary", + }); + const commandMutation: OutboxMutation = { + ...workspaceMutation, + kind: "command.submit", + targetId: "session-one", + payload: { sessionId: "session-one", command: "printf replacement" }, + }; + + await applier.apply(commandMutation, fence); + expect(new Headers(values.requests.at(-1)?.init?.headers).get("content-type")).toBe("application/merge-patch+json"); + const annotations = JSON.parse(String(values.requests.at(-1)?.init?.body)).metadata.annotations; + expect(annotations["cluster.t4.dev/pending-command"]).toBeNull(); + expect(annotations["cluster.t4.dev/pending-command-id"]).toBe(commandMutation.commandId); + expect(annotations["cluster.t4.dev/pending-command-epoch"]).toBe(fence.epoch.toString()); + }); + + it("removes the legacy full command payload on session patch without clearing the bounded pointer", async () => { + const values = recordingFetch([{ + apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Session", + metadata: { name: "session-one", resourceVersion: "68", annotations: { + ...priorAnnotations, + "cluster.t4.dev/pending-command": "legacy full command payload", + "cluster.t4.dev/pending-command-id": "bounded-command-id", + "cluster.t4.dev/pending-command-epoch": "6", + } }, + spec: { hostRef: "primary", workspaceRef: "workspace-one" }, + }, { + apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Workspace", + metadata: { name: "workspace-one", resourceVersion: "67", annotations: priorAnnotations }, + spec: { hostRef: "primary", owner: PRINCIPAL }, + }, {}]); + const applier = new PublicKubernetesOutboxApplier({ + client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch: values.fetch }), + hostRef: "primary", + }); + + await applier.apply({ + ...workspaceMutation, + kind: "session.patch", + targetId: "session-one", + payload: { id: "session-one", workspaceId: "workspace-one", title: "patched", revision: 2 }, + }, fence); + expect(new Headers(values.requests.at(-1)?.init?.headers).get("content-type")).toBe("application/merge-patch+json"); + const annotations = JSON.parse(String(values.requests.at(-1)?.init?.body)).metadata.annotations; + expect(annotations["cluster.t4.dev/pending-command"]).toBeNull(); + expect(annotations).not.toHaveProperty("cluster.t4.dev/pending-command-id"); + expect(annotations).not.toHaveProperty("cluster.t4.dev/pending-command-epoch"); + }); + + it("verifies a session kind and its owning workspace relation before patching", async () => { + const values = recordingFetch([{ + apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Session", + metadata: { name: "session-one", resourceVersion: "71", annotations: { + "cluster.t4.dev/ledger-command-id": "prior-command", + "cluster.t4.dev/ledger-outbox-token": "outbox:prior", + "cluster.t4.dev/ledger-owner": "prior-worker", + "cluster.t4.dev/ledger-owner-epoch": "6", + "cluster.t4.dev/ledger-semantic-hash": "sha256:prior", + } }, + spec: { hostRef: "primary", workspaceRef: "workspace-one" }, + }, { + apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Workspace", + metadata: { name: "workspace-one", resourceVersion: "70", annotations: priorAnnotations }, + spec: { hostRef: "primary", owner: PRINCIPAL }, + }, {}]); + const applier = new PublicKubernetesOutboxApplier({ + client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch: values.fetch }), + hostRef: "primary", + }); + await applier.apply({ + ...workspaceMutation, + kind: "session.patch", + targetId: "session-one", + payload: { id: "session-one", workspaceId: "workspace-one", title: "patched", revision: 2 }, + }, fence); + expect(values.requests.map(value => value.init?.method ?? "GET")).toEqual(["GET", "GET", "PATCH"]); + expect(values.requests[1]?.url).toContain("/t4workspaces/workspace-one"); + }); +}); + +describe("Kubernetes legacy annotation reconciliation", () => { + + it("reconciles legacy full command payload annotations on untouched listed sessions", async () => { + const patches: Array<{ resource: string; name: string; body: unknown }> = []; + const client = { + listInfrastructure: async () => ({ + host: { + apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4ClusterHost", + metadata: { name: "primary", uid: "host-uid", resourceVersion: "100" }, spec: {}, + }, + workspaces: [{ + apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Workspace", + metadata: { name: "workspace-one", uid: "workspace-uid", resourceVersion: "101" }, + spec: { hostRef: "primary", owner: PRINCIPAL, displayName: "Workspace", retentionPolicy: "Retain", size: "20Gi" }, + }], + sessions: [{ + apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Session", + metadata: { name: "session-legacy", uid: "session-uid", resourceVersion: "102", annotations: { + "cluster.t4.dev/pending-command": "legacy full command payload", + "cluster.t4.dev/pending-command-id": "bounded-command-id", + "cluster.t4.dev/pending-command-epoch": "6", + } }, + spec: { hostRef: "primary", workspaceRef: "workspace-one", title: "Session", runtimeProfile: "default", guiEnabled: true }, + }], + resourceVersion: "102", + resourceVersions: { t4clusterhosts: "100", t4workspaces: "101", t4sessions: "102" }, + }), + patch: async (resource: string, name: string, body: unknown) => { + patches.push({ resource, name, body }); + return { metadata: { name, resourceVersion: "103" } }; + }, + watch: (_resource: string, _version: string, _onEvent: unknown, signal: AbortSignal) => new Promise(resolve => { + if (signal.aborted) resolve(); + else signal.addEventListener("abort", () => resolve(), { once: true }); + }), + } as unknown as KubernetesApiClient; + const projection = new ClusterInfrastructureProjection({ epoch: "replica-one", namespace: "development" }); + const runner = new KubernetesProjectionRunner({ client, projection, hostName: "primary", retryMs: 0 }); + + try { + await runner.start(); + expect(patches).toHaveLength(1); + expect(patches[0]).toEqual({ + resource: "t4sessions", + name: "session-legacy", + body: { + metadata: { + resourceVersion: "102", + annotations: { "cluster.t4.dev/pending-command": null }, + }, + }, + }); + } finally { + await runner.stop(); + } + }); +}); + describe("Kubernetes projected identity review", () => { it("submits the presented bearer with the fixed audience and requires the exact server ServiceAccount", async () => { const directory = await mkdtemp(join(tmpdir(), "t4-token-review-")); diff --git a/packages/t4-api-client/package.json b/packages/t4-api-client/package.json new file mode 100644 index 00000000..9c844512 --- /dev/null +++ b/packages/t4-api-client/package.json @@ -0,0 +1,23 @@ +{ + "name": "@t4-code/t4-api-client", + "version": "0.1.30", + "private": true, + "type": "module", + "exports": { + ".": "./src/index.ts", + "./generated": "./src/generated/schema.ts" + }, + "scripts": { + "build": "tsgo --noEmit", + "typecheck": "tsgo --noEmit", + "test": "vp test run --passWithNoTests" + }, + "dependencies": { + "openapi-fetch": "0.17.0" + }, + "devDependencies": { + "@types/node": "catalog:", + "typescript": "catalog:", + "vite-plus": "catalog:" + } +} diff --git a/packages/t4-api-client/src/generated/schema.ts b/packages/t4-api-client/src/generated/schema.ts new file mode 100755 index 00000000..ad07c82b --- /dev/null +++ b/packages/t4-api-client/src/generated/schema.ts @@ -0,0 +1,1069 @@ +export interface paths { + "/v1": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Negotiate T4 API v1 and discover capabilities and bounds */ + get: operations["discoverV1"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/sessions/{sessionId}": { + parameters: { + query?: never; + header?: never; + path: { + sessionId: components["parameters"]["SessionId"]; + }; + cookie?: never; + }; + get: operations["getSession"]; + put?: never; + post?: never; + delete: operations["deleteSession"]; + options?: never; + head?: never; + patch: operations["mutateSession"]; + trace?: never; + }; + "/v1/sessions/{sessionId}/cancel": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["cancelSession"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/sessions/{sessionId}/commands": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["submitCommand"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/sessions/{sessionId}/events": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Watch a bounded Server-Sent Events stream over HTTPS + * @description The response is standard Server-Sent Events over HTTPS. Clients reconnect with the last event cursor in the cursor query parameter or Last-Event-ID header. Heartbeat events bound idle detection. Clients explicitly cancel by aborting the HTTPS request. A retained cursor expiry returns a typed 410 response with a snapshot resync target. + */ + get: operations["watchSessionEvents"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/sessions/{sessionId}/snapshot": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["getSessionSnapshot"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/workspaces": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["listWorkspaces"]; + put?: never; + post: operations["createWorkspace"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/workspaces/{workspaceId}": { + parameters: { + query?: never; + header?: never; + path: { + workspaceId: components["parameters"]["WorkspaceId"]; + }; + cookie?: never; + }; + get: operations["getWorkspace"]; + put?: never; + post?: never; + delete: operations["deleteWorkspace"]; + options?: never; + head?: never; + patch: operations["mutateWorkspace"]; + trace?: never; + }; + "/v1/workspaces/{workspaceId}/sessions": { + parameters: { + query?: never; + header?: never; + path: { + workspaceId: components["parameters"]["WorkspaceId"]; + }; + cookie?: never; + }; + get: operations["listSessions"]; + put?: never; + post: operations["spawnSession"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; +} +export type webhooks = Record; +export interface components { + schemas: { + ApiError: { + /** @enum {string} */ + code: "invalid_request" | "unauthenticated" | "forbidden" | "not_found" | "idempotency_key_required" | "idempotency_conflict" | "revision_conflict" | "incompatible_version" | "cursor_expired" | "unavailable" | "indeterminate" | "invalid_origin" | "https_required"; + message: string; + requestId: string; + resync?: components["schemas"]["Resync"]; + retryable: boolean; + supportedMajors?: number[]; + violations?: components["schemas"]["FieldViolation"][]; + }; + BadRequestApiError: components["schemas"]["ApiError"] & { + /** @enum {unknown} */ + code?: "invalid_request" | "idempotency_key_required" | "invalid_origin" | "https_required"; + }; + BadRequestErrorEnvelope: { + error: components["schemas"]["BadRequestApiError"]; + }; + Capabilities: { + [key: string]: components["schemas"]["CapabilityStatus"]; + }; + CapabilityDeprecation: { + message: string; + replacement?: string; + sinceVersion?: string; + /** Format: date-time */ + sunsetAt?: string; + }; + CapabilityStatus: { + authorized: boolean; + available: boolean; + deprecation?: components["schemas"]["CapabilityDeprecation"]; + enabled: boolean; + supported: boolean; + }; + CommandCreate: { + command: string; + /** @default {} */ + metadata?: { + [key: string]: string | number | boolean | null; + }; + }; + CommandResult: { + commandId: components["schemas"]["ResourceId"]; + state: components["schemas"]["CommandState"]; + }; + /** @enum {string} */ + CommandState: "accepted" | "projected" | "dispatching" | "running" | "succeeded" | "failed" | "cancelling" | "cancelled" | "rejected" | "unavailable" | "indeterminate"; + CommandWatchEvent: { + commandId: components["schemas"]["ResourceId"]; + cursor: components["schemas"]["Cursor"]; + state: components["schemas"]["CommandState"]; + /** @constant */ + type: "command"; + }; + ConflictApiError: components["schemas"]["ApiError"] & { + /** @enum {unknown} */ + code?: "idempotency_conflict" | "revision_conflict"; + }; + ConflictErrorEnvelope: { + error: components["schemas"]["ConflictApiError"]; + }; + /** @description Opaque server-issued header-safe SSE cursor. */ + Cursor: string; + CursorExpiredApiError: components["schemas"]["ApiError"] & { + /** @constant */ + code?: "cursor_expired"; + resync: components["schemas"]["Resync"]; + }; + CursorExpiredErrorEnvelope: { + error: components["schemas"]["CursorExpiredApiError"]; + }; + Discovery: { + apiVersion: string; + capabilities: components["schemas"]["Capabilities"]; + limits: { + commandBytesMax: number; + commandMetadataValueBytesMax: number; + commandRequestBytesMax: number; + heartbeatSeconds: number; + pageSizeDefault: number; + pageSizeMax: number; + watchEventsMax: number; + }; + serverBuild: components["schemas"]["ServerBuild"]; + supportedMajors: number[]; + }; + ErrorEnvelope: { + error: components["schemas"]["ApiError"]; + }; + FieldViolation: { + field: string; + message: string; + rule: string; + }; + ForbiddenApiError: components["schemas"]["ApiError"] & { + /** @constant */ + code?: "forbidden"; + }; + ForbiddenErrorEnvelope: { + error: components["schemas"]["ForbiddenApiError"]; + }; + HeartbeatWatchEvent: { + cursor: components["schemas"]["Cursor"]; + /** Format: date-time */ + observedAt: string; + /** @constant */ + type: "heartbeat"; + }; + IncompatibleVersionApiError: components["schemas"]["ApiError"] & { + /** @constant */ + code?: "incompatible_version"; + supportedMajors: number[]; + }; + IncompatibleVersionErrorEnvelope: { + error: components["schemas"]["IncompatibleVersionApiError"]; + }; + InvalidRequestApiError: components["schemas"]["ApiError"] & { + /** @constant */ + code?: "invalid_request"; + violations: components["schemas"]["FieldViolation"][]; + }; + InvalidRequestErrorEnvelope: { + error: components["schemas"]["InvalidRequestApiError"]; + }; + Labels: { + [key: string]: string; + }; + NotFoundApiError: components["schemas"]["ApiError"] & { + /** @constant */ + code?: "not_found"; + }; + NotFoundErrorEnvelope: { + error: components["schemas"]["NotFoundApiError"]; + }; + ResourceId: string; + Resync: { + cursor: components["schemas"]["Cursor"]; + snapshotUrl: string; + }; + Revision: number; + ServerBuild: { + revision: string; + version: string; + }; + Session: { + id: components["schemas"]["ResourceId"]; + labels?: components["schemas"]["Labels"]; + revision: components["schemas"]["Revision"]; + state: components["schemas"]["SessionState"]; + title: string; + workspaceId: components["schemas"]["ResourceId"]; + }; + SessionCreate: { + labels?: components["schemas"]["Labels"]; + title: string; + }; + SessionMutation: { + labels?: components["schemas"]["Labels"]; + title?: string; + }; + SessionPage: { + items: components["schemas"]["Session"][]; + nextCursor?: components["schemas"]["Cursor"]; + }; + SessionSnapshot: { + cursor: components["schemas"]["Cursor"]; + entries: components["schemas"]["SnapshotEntry"][]; + sessionId: components["schemas"]["ResourceId"]; + state: components["schemas"]["SessionState"]; + }; + /** @enum {string} */ + SessionState: "accepted" | "provisioning" | "ready" | "cancelling" | "cancelled" | "failed" | "unavailable" | "indeterminate"; + SessionWatchEvent: { + cursor: components["schemas"]["Cursor"]; + revision: components["schemas"]["Revision"]; + state: components["schemas"]["SessionState"]; + /** @constant */ + type: "session"; + }; + SnapshotEntry: { + /** @enum {string} */ + kind: "input" | "output" | "status"; + sequence: number; + text: string; + }; + UnauthenticatedApiError: components["schemas"]["ApiError"] & { + /** @constant */ + code?: "unauthenticated"; + }; + UnauthenticatedErrorEnvelope: { + error: components["schemas"]["UnauthenticatedApiError"]; + }; + UnavailableApiError: components["schemas"]["ApiError"] & { + /** @enum {unknown} */ + code?: "unavailable" | "indeterminate"; + }; + UnavailableErrorEnvelope: { + error: components["schemas"]["UnavailableApiError"]; + }; + WatchEvent: components["schemas"]["HeartbeatWatchEvent"] | components["schemas"]["SessionWatchEvent"] | components["schemas"]["CommandWatchEvent"]; + Workspace: { + id: components["schemas"]["ResourceId"]; + labels?: components["schemas"]["Labels"]; + name: string; + revision: components["schemas"]["Revision"]; + state: components["schemas"]["WorkspaceState"]; + }; + WorkspaceCreate: { + labels?: components["schemas"]["Labels"]; + name: string; + }; + WorkspaceMutation: { + labels?: components["schemas"]["Labels"]; + name?: string; + }; + WorkspacePage: { + items: components["schemas"]["Workspace"][]; + nextCursor?: components["schemas"]["Cursor"]; + }; + /** @enum {string} */ + WorkspaceState: "accepted" | "provisioning" | "ready" | "deleting" | "deleted" | "failed" | "unavailable" | "indeterminate"; + }; + responses: { + /** @description Command intent accepted with a stable outcome state */ + CommandAccepted: { + headers: { + "Idempotency-Replayed": components["headers"]["IdempotencyReplayed"]; + "T4-API-Version": components["headers"]["SelectedVersion"]; + "T4-Event-Cursor": components["headers"]["EventCursor"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["CommandResult"]; + }; + }; + /** @description Identical command request replay */ + CommandReplay: { + headers: { + "Idempotency-Replayed": components["headers"]["IdempotencyReplayed"]; + "T4-API-Version": components["headers"]["SelectedVersion"]; + "T4-Event-Cursor": components["headers"]["EventCursor"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["CommandResult"]; + }; + }; + /** @description Idempotent deletion accepted or resource already absent */ + Deleted: { + headers: { + "Idempotency-Replayed": components["headers"]["IdempotencyReplayed"]; + "T4-API-Version": components["headers"]["SelectedVersion"]; + "T4-Event-Cursor": components["headers"]["EventCursor"]; + [name: string]: unknown; + }; + content?: never; + }; + /** @description Negotiated v1 discovery */ + Discovery: { + headers: { + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Discovery"]; + }; + }; + /** @description Malformed request or missing idempotency key */ + Error400: { + headers: { + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["BadRequestErrorEnvelope"]; + }; + }; + /** @description Missing or invalid opaque bearer credential */ + Error401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UnauthenticatedErrorEnvelope"]; + }; + }; + /** @description Credential lacks the deny-by-default operation or resource scope */ + Error403: { + headers: { + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ForbiddenErrorEnvelope"]; + }; + }; + /** @description Resource absent or outside caller scope (scope existence is not disclosed) */ + Error404: { + headers: { + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["NotFoundErrorEnvelope"]; + }; + }; + /** @description Requested major is incompatible; no silent downgrade */ + Error406: { + headers: { + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["IncompatibleVersionErrorEnvelope"]; + }; + }; + /** @description Idempotency or resource revision conflict */ + Error409: { + headers: { + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ConflictErrorEnvelope"]; + }; + }; + /** @description Watch cursor expired; resync from the typed snapshot target */ + Error410: { + headers: { + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["CursorExpiredErrorEnvelope"]; + }; + }; + /** @description Bounded semantic validation failure with stable field violations */ + Error422: { + headers: { + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["InvalidRequestErrorEnvelope"]; + }; + }; + /** @description Service temporarily unavailable or outcome indeterminate. Retryable watch failures may advertise Retry-After; clients honor it with a 30 second ceiling. */ + Error503: { + headers: { + /** @description RFC 9110 delay in seconds or HTTP date; clients bound the applied delay. */ + "Retry-After"?: string; + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["UnavailableErrorEnvelope"]; + }; + }; + /** @description Session */ + Session: { + headers: { + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Session"]; + }; + }; + /** @description Session intent accepted */ + SessionAccepted: { + headers: { + "Idempotency-Replayed": components["headers"]["IdempotencyReplayed"]; + "T4-API-Version": components["headers"]["SelectedVersion"]; + "T4-Event-Cursor": components["headers"]["EventCursor"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Session"]; + }; + }; + /** @description Bounded session page */ + SessionPage: { + headers: { + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SessionPage"]; + }; + }; + /** @description Identical session mutation replay */ + SessionReplay: { + headers: { + "Idempotency-Replayed": components["headers"]["IdempotencyReplayed"]; + "T4-API-Version": components["headers"]["SelectedVersion"]; + "T4-Event-Cursor": components["headers"]["EventCursor"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Session"]; + }; + }; + /** @description Bounded session snapshot and reconnect cursor */ + Snapshot: { + headers: { + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SessionSnapshot"]; + }; + }; + /** @description Workspace */ + Workspace: { + headers: { + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Workspace"]; + }; + }; + /** @description Workspace intent accepted */ + WorkspaceAccepted: { + headers: { + "Idempotency-Replayed": components["headers"]["IdempotencyReplayed"]; + "T4-API-Version": components["headers"]["SelectedVersion"]; + "T4-Event-Cursor": components["headers"]["EventCursor"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Workspace"]; + }; + }; + /** @description Bounded workspace page */ + WorkspacePage: { + headers: { + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["WorkspacePage"]; + }; + }; + /** @description Identical workspace request replay */ + WorkspaceReplay: { + headers: { + "Idempotency-Replayed": components["headers"]["IdempotencyReplayed"]; + "T4-API-Version": components["headers"]["SelectedVersion"]; + "T4-Event-Cursor": components["headers"]["EventCursor"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Workspace"]; + }; + }; + }; + parameters: { + /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ + ApiVersion: string; + HeartbeatSeconds: number; + /** @description Opaque caller-generated key scoped by the server to the authenticated principal, operation ID, and target resource IDs. Idempotency is evaluated only after path, query, header, and request-schema validation and application of schema defaults. Request identity is the operation ID, target IDs, relevant precondition headers, and the RFC 8785 JSON Canonicalization Scheme (JCS) bytes of the validated JSON body. JCS object member order is insignificant, array order is significant, omitted fields remain distinct unless the request schema defaulted them, and no Unicode normalization is performed beyond RFC 8785. Reusing a key with identical identity replays the original terminal response and advertises Idempotency-Replayed; reusing it with a different identity returns 409 idempotency_conflict. */ + IdempotencyKey: string; + /** @description Decimal resource revision for optimistic mutation. */ + IfMatch: string; + /** @description Standard SSE reconnect header. Must agree with cursor when both are present. */ + LastEventId: components["schemas"]["Cursor"]; + MaxEvents: number; + /** @description Opaque page cursor, valid only for the same identity and list operation. */ + PageCursor: components["schemas"]["Cursor"]; + PageSize: number; + SessionId: components["schemas"]["ResourceId"]; + /** @description Opaque last-consumed watch event cursor. */ + WatchCursor: components["schemas"]["Cursor"]; + WorkspaceId: components["schemas"]["ResourceId"]; + }; + requestBodies: never; + headers: { + /** @description Durable event cursor committed with the accepted mutation and preserved by replay. */ + EventCursor: components["schemas"]["Cursor"]; + /** @description True when this response replays a prior identical request. */ + IdempotencyReplayed: "true" | "false"; + /** @description Selected compatible T4 API minor version. */ + SelectedVersion: string; + }; + pathItems: never; +} +export type $defs = Record; +export interface operations { + discoverV1: { + parameters: { + query?: never; + header: { + /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ + "T4-API-Version": components["parameters"]["ApiVersion"]; + }; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: components["responses"]["Discovery"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 406: components["responses"]["Error406"]; + 503: components["responses"]["Error503"]; + }; + }; + getSession: { + parameters: { + query?: never; + header: { + /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ + "T4-API-Version": components["parameters"]["ApiVersion"]; + }; + path: { + sessionId: components["parameters"]["SessionId"]; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: components["responses"]["Session"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + 406: components["responses"]["Error406"]; + 503: components["responses"]["Error503"]; + }; + }; + deleteSession: { + parameters: { + query?: never; + header: { + /** @description Opaque caller-generated key scoped by the server to the authenticated principal, operation ID, and target resource IDs. Idempotency is evaluated only after path, query, header, and request-schema validation and application of schema defaults. Request identity is the operation ID, target IDs, relevant precondition headers, and the RFC 8785 JSON Canonicalization Scheme (JCS) bytes of the validated JSON body. JCS object member order is insignificant, array order is significant, omitted fields remain distinct unless the request schema defaulted them, and no Unicode normalization is performed beyond RFC 8785. Reusing a key with identical identity replays the original terminal response and advertises Idempotency-Replayed; reusing it with a different identity returns 409 idempotency_conflict. */ + "Idempotency-Key": components["parameters"]["IdempotencyKey"]; + /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ + "T4-API-Version": components["parameters"]["ApiVersion"]; + }; + path: { + sessionId: components["parameters"]["SessionId"]; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 204: components["responses"]["Deleted"]; + 400: components["responses"]["Error400"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + 406: components["responses"]["Error406"]; + 409: components["responses"]["Error409"]; + 503: components["responses"]["Error503"]; + }; + }; + mutateSession: { + parameters: { + query?: never; + header: { + /** @description Opaque caller-generated key scoped by the server to the authenticated principal, operation ID, and target resource IDs. Idempotency is evaluated only after path, query, header, and request-schema validation and application of schema defaults. Request identity is the operation ID, target IDs, relevant precondition headers, and the RFC 8785 JSON Canonicalization Scheme (JCS) bytes of the validated JSON body. JCS object member order is insignificant, array order is significant, omitted fields remain distinct unless the request schema defaulted them, and no Unicode normalization is performed beyond RFC 8785. Reusing a key with identical identity replays the original terminal response and advertises Idempotency-Replayed; reusing it with a different identity returns 409 idempotency_conflict. */ + "Idempotency-Key": components["parameters"]["IdempotencyKey"]; + /** @description Decimal resource revision for optimistic mutation. */ + "If-Match": components["parameters"]["IfMatch"]; + /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ + "T4-API-Version": components["parameters"]["ApiVersion"]; + }; + path: { + sessionId: components["parameters"]["SessionId"]; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["SessionMutation"]; + }; + }; + responses: { + 200: components["responses"]["SessionReplay"]; + 400: components["responses"]["Error400"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + 406: components["responses"]["Error406"]; + 409: components["responses"]["Error409"]; + 422: components["responses"]["Error422"]; + 503: components["responses"]["Error503"]; + }; + }; + cancelSession: { + parameters: { + query?: never; + header: { + /** @description Opaque caller-generated key scoped by the server to the authenticated principal, operation ID, and target resource IDs. Idempotency is evaluated only after path, query, header, and request-schema validation and application of schema defaults. Request identity is the operation ID, target IDs, relevant precondition headers, and the RFC 8785 JSON Canonicalization Scheme (JCS) bytes of the validated JSON body. JCS object member order is insignificant, array order is significant, omitted fields remain distinct unless the request schema defaulted them, and no Unicode normalization is performed beyond RFC 8785. Reusing a key with identical identity replays the original terminal response and advertises Idempotency-Replayed; reusing it with a different identity returns 409 idempotency_conflict. */ + "Idempotency-Key": components["parameters"]["IdempotencyKey"]; + /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ + "T4-API-Version": components["parameters"]["ApiVersion"]; + }; + path: { + sessionId: components["parameters"]["SessionId"]; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: components["responses"]["SessionReplay"]; + 202: components["responses"]["SessionAccepted"]; + 400: components["responses"]["Error400"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + 406: components["responses"]["Error406"]; + 409: components["responses"]["Error409"]; + 503: components["responses"]["Error503"]; + }; + }; + submitCommand: { + parameters: { + query?: never; + header: { + /** @description Opaque caller-generated key scoped by the server to the authenticated principal, operation ID, and target resource IDs. Idempotency is evaluated only after path, query, header, and request-schema validation and application of schema defaults. Request identity is the operation ID, target IDs, relevant precondition headers, and the RFC 8785 JSON Canonicalization Scheme (JCS) bytes of the validated JSON body. JCS object member order is insignificant, array order is significant, omitted fields remain distinct unless the request schema defaulted them, and no Unicode normalization is performed beyond RFC 8785. Reusing a key with identical identity replays the original terminal response and advertises Idempotency-Replayed; reusing it with a different identity returns 409 idempotency_conflict. */ + "Idempotency-Key": components["parameters"]["IdempotencyKey"]; + /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ + "T4-API-Version": components["parameters"]["ApiVersion"]; + }; + path: { + sessionId: components["parameters"]["SessionId"]; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["CommandCreate"]; + }; + }; + responses: { + 200: components["responses"]["CommandReplay"]; + 202: components["responses"]["CommandAccepted"]; + 400: components["responses"]["Error400"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + 406: components["responses"]["Error406"]; + 409: components["responses"]["Error409"]; + 422: components["responses"]["Error422"]; + 503: components["responses"]["Error503"]; + }; + }; + watchSessionEvents: { + parameters: { + query?: { + /** @description Opaque last-consumed watch event cursor. */ + cursor?: components["parameters"]["WatchCursor"]; + heartbeatSeconds?: components["parameters"]["HeartbeatSeconds"]; + maxEvents?: components["parameters"]["MaxEvents"]; + }; + header: { + /** @description Standard SSE reconnect header. Must agree with cursor when both are present. */ + "Last-Event-ID"?: components["parameters"]["LastEventId"]; + /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ + "T4-API-Version": components["parameters"]["ApiVersion"]; + }; + path: { + sessionId: components["parameters"]["SessionId"]; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Bounded SSE event stream. Every non-empty SSE data field is one JSON value conforming to WatchEvent; clients MUST reject unknown fields and schema-invalid event payloads before delivery. Transport chunk boundaries do not delimit frames. */ + 200: { + headers: { + "Cache-Control": "no-store"; + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + /** @example id: cur_01J...\nevent: heartbeat\ndata: {"type":"heartbeat","cursor":"cur_01J...","observedAt":"2026-07-21T00:00:00Z"}\n\n */ + "text/event-stream": string; + }; + }; + 400: components["responses"]["Error400"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + 406: components["responses"]["Error406"]; + 410: components["responses"]["Error410"]; + 422: components["responses"]["Error422"]; + 503: components["responses"]["Error503"]; + }; + }; + getSessionSnapshot: { + parameters: { + query?: never; + header: { + /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ + "T4-API-Version": components["parameters"]["ApiVersion"]; + }; + path: { + sessionId: components["parameters"]["SessionId"]; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: components["responses"]["Snapshot"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + 406: components["responses"]["Error406"]; + 503: components["responses"]["Error503"]; + }; + }; + listWorkspaces: { + parameters: { + query?: { + /** @description Opaque page cursor, valid only for the same identity and list operation. */ + cursor?: components["parameters"]["PageCursor"]; + pageSize?: components["parameters"]["PageSize"]; + }; + header: { + /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ + "T4-API-Version": components["parameters"]["ApiVersion"]; + }; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: components["responses"]["WorkspacePage"]; + 400: components["responses"]["Error400"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 406: components["responses"]["Error406"]; + 422: components["responses"]["Error422"]; + 503: components["responses"]["Error503"]; + }; + }; + createWorkspace: { + parameters: { + query?: never; + header: { + /** @description Opaque caller-generated key scoped by the server to the authenticated principal, operation ID, and target resource IDs. Idempotency is evaluated only after path, query, header, and request-schema validation and application of schema defaults. Request identity is the operation ID, target IDs, relevant precondition headers, and the RFC 8785 JSON Canonicalization Scheme (JCS) bytes of the validated JSON body. JCS object member order is insignificant, array order is significant, omitted fields remain distinct unless the request schema defaulted them, and no Unicode normalization is performed beyond RFC 8785. Reusing a key with identical identity replays the original terminal response and advertises Idempotency-Replayed; reusing it with a different identity returns 409 idempotency_conflict. */ + "Idempotency-Key": components["parameters"]["IdempotencyKey"]; + /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ + "T4-API-Version": components["parameters"]["ApiVersion"]; + }; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["WorkspaceCreate"]; + }; + }; + responses: { + 200: components["responses"]["WorkspaceReplay"]; + 202: components["responses"]["WorkspaceAccepted"]; + 400: components["responses"]["Error400"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 406: components["responses"]["Error406"]; + 409: components["responses"]["Error409"]; + 422: components["responses"]["Error422"]; + 503: components["responses"]["Error503"]; + }; + }; + getWorkspace: { + parameters: { + query?: never; + header: { + /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ + "T4-API-Version": components["parameters"]["ApiVersion"]; + }; + path: { + workspaceId: components["parameters"]["WorkspaceId"]; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: components["responses"]["Workspace"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + 406: components["responses"]["Error406"]; + 503: components["responses"]["Error503"]; + }; + }; + deleteWorkspace: { + parameters: { + query?: never; + header: { + /** @description Opaque caller-generated key scoped by the server to the authenticated principal, operation ID, and target resource IDs. Idempotency is evaluated only after path, query, header, and request-schema validation and application of schema defaults. Request identity is the operation ID, target IDs, relevant precondition headers, and the RFC 8785 JSON Canonicalization Scheme (JCS) bytes of the validated JSON body. JCS object member order is insignificant, array order is significant, omitted fields remain distinct unless the request schema defaulted them, and no Unicode normalization is performed beyond RFC 8785. Reusing a key with identical identity replays the original terminal response and advertises Idempotency-Replayed; reusing it with a different identity returns 409 idempotency_conflict. */ + "Idempotency-Key": components["parameters"]["IdempotencyKey"]; + /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ + "T4-API-Version": components["parameters"]["ApiVersion"]; + }; + path: { + workspaceId: components["parameters"]["WorkspaceId"]; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 204: components["responses"]["Deleted"]; + 400: components["responses"]["Error400"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + 406: components["responses"]["Error406"]; + 409: components["responses"]["Error409"]; + 503: components["responses"]["Error503"]; + }; + }; + mutateWorkspace: { + parameters: { + query?: never; + header: { + /** @description Opaque caller-generated key scoped by the server to the authenticated principal, operation ID, and target resource IDs. Idempotency is evaluated only after path, query, header, and request-schema validation and application of schema defaults. Request identity is the operation ID, target IDs, relevant precondition headers, and the RFC 8785 JSON Canonicalization Scheme (JCS) bytes of the validated JSON body. JCS object member order is insignificant, array order is significant, omitted fields remain distinct unless the request schema defaulted them, and no Unicode normalization is performed beyond RFC 8785. Reusing a key with identical identity replays the original terminal response and advertises Idempotency-Replayed; reusing it with a different identity returns 409 idempotency_conflict. */ + "Idempotency-Key": components["parameters"]["IdempotencyKey"]; + /** @description Decimal resource revision for optimistic mutation. */ + "If-Match": components["parameters"]["IfMatch"]; + /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ + "T4-API-Version": components["parameters"]["ApiVersion"]; + }; + path: { + workspaceId: components["parameters"]["WorkspaceId"]; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["WorkspaceMutation"]; + }; + }; + responses: { + 200: components["responses"]["WorkspaceReplay"]; + 400: components["responses"]["Error400"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + 406: components["responses"]["Error406"]; + 409: components["responses"]["Error409"]; + 422: components["responses"]["Error422"]; + 503: components["responses"]["Error503"]; + }; + }; + listSessions: { + parameters: { + query?: { + /** @description Opaque page cursor, valid only for the same identity and list operation. */ + cursor?: components["parameters"]["PageCursor"]; + pageSize?: components["parameters"]["PageSize"]; + }; + header: { + /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ + "T4-API-Version": components["parameters"]["ApiVersion"]; + }; + path: { + workspaceId: components["parameters"]["WorkspaceId"]; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: components["responses"]["SessionPage"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + 406: components["responses"]["Error406"]; + 422: components["responses"]["Error422"]; + 503: components["responses"]["Error503"]; + }; + }; + spawnSession: { + parameters: { + query?: never; + header: { + /** @description Opaque caller-generated key scoped by the server to the authenticated principal, operation ID, and target resource IDs. Idempotency is evaluated only after path, query, header, and request-schema validation and application of schema defaults. Request identity is the operation ID, target IDs, relevant precondition headers, and the RFC 8785 JSON Canonicalization Scheme (JCS) bytes of the validated JSON body. JCS object member order is insignificant, array order is significant, omitted fields remain distinct unless the request schema defaulted them, and no Unicode normalization is performed beyond RFC 8785. Reusing a key with identical identity replays the original terminal response and advertises Idempotency-Replayed; reusing it with a different identity returns 409 idempotency_conflict. */ + "Idempotency-Key": components["parameters"]["IdempotencyKey"]; + /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ + "T4-API-Version": components["parameters"]["ApiVersion"]; + }; + path: { + workspaceId: components["parameters"]["WorkspaceId"]; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["SessionCreate"]; + }; + }; + responses: { + 200: components["responses"]["SessionReplay"]; + 202: components["responses"]["SessionAccepted"]; + 400: components["responses"]["Error400"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + 406: components["responses"]["Error406"]; + 409: components["responses"]["Error409"]; + 422: components["responses"]["Error422"]; + 503: components["responses"]["Error503"]; + }; + }; +} + diff --git a/packages/t4-api-client/src/index.ts b/packages/t4-api-client/src/index.ts new file mode 100644 index 00000000..412d6756 --- /dev/null +++ b/packages/t4-api-client/src/index.ts @@ -0,0 +1,935 @@ +import createClient, { type Client } from "openapi-fetch"; + +import type { components, paths } from "./generated/schema.ts"; + +export type { components, operations, paths } from "./generated/schema.ts"; + +const MAX_CREDENTIAL_LENGTH = 4096; +const MAX_ERROR_BYTES = 1024 * 1024; +const MAX_EVENT_BYTES = 1024 * 1024; +const MAX_JSON_RESPONSE_BYTES = 16 * 1024 * 1024; +const CURSOR_PATTERN = /^[A-Za-z0-9._~-]+$/u; +const SELECTED_VERSION_MAX_LENGTH = 16; +const INVALID_RETRY_AFTER = Symbol("invalid Retry-After"); +const HTTP_MONTHS: Readonly> = { + Jan: 0, Feb: 1, Mar: 2, Apr: 3, May: 4, Jun: 5, + Jul: 6, Aug: 7, Sep: 8, Oct: 9, Nov: 10, Dec: 11, +}; +const HTTP_WEEKDAYS: Readonly> = { + Sun: 0, Sunday: 0, Mon: 1, Monday: 1, Tue: 2, Tuesday: 2, + Wed: 3, Wednesday: 3, Thu: 4, Thursday: 4, Fri: 5, Friday: 5, + Sat: 6, Saturday: 6, +}; +const SELECTED_VERSION_PATTERN = /^1\.[0-9]+$/u; +const ERROR_CODES = { + invalid_request: true, unauthenticated: true, forbidden: true, not_found: true, + idempotency_key_required: true, idempotency_conflict: true, revision_conflict: true, + incompatible_version: true, cursor_expired: true, unavailable: true, indeterminate: true, + invalid_origin: true, https_required: true, +} as const satisfies Record; +const ERROR_FIELDS: Record = { + code: true, message: true, requestId: true, retryable: true, + violations: true, supportedMajors: true, resync: true, +}; +const ERROR_CODES_BY_STATUS: Readonly>>> = { + 400: { invalid_request: true, idempotency_key_required: true, invalid_origin: true, https_required: true }, + 401: { unauthenticated: true }, + 403: { forbidden: true }, + 404: { not_found: true }, + 406: { incompatible_version: true }, + 409: { idempotency_conflict: true, revision_conflict: true }, + 410: { cursor_expired: true }, + 422: { invalid_request: true }, + 503: { unavailable: true, indeterminate: true }, +}; +const WORKSPACE_STATES = { + accepted: true, provisioning: true, ready: true, deleting: true, + deleted: true, failed: true, unavailable: true, indeterminate: true, +} as const satisfies Record; +const SESSION_STATES = { + accepted: true, provisioning: true, ready: true, cancelling: true, + cancelled: true, failed: true, unavailable: true, indeterminate: true, +} as const satisfies Record; +const COMMAND_STATES = { + accepted: true, projected: true, dispatching: true, running: true, succeeded: true, failed: true, + cancelling: true, cancelled: true, rejected: true, unavailable: true, indeterminate: true, +} as const satisfies Record; + +type ApiError = components["schemas"]["ApiError"]; +type Resync = components["schemas"]["Resync"]; +type WatchEvent = components["schemas"]["WatchEvent"]; + +export interface T4ApiClientOptions { + readonly baseUrl: string; + readonly credential: string; + readonly majorVersion: number; + readonly fetch?: typeof globalThis.fetch; +} + +export interface WatchSessionOptions { + readonly cursor?: components["schemas"]["Cursor"]; + readonly maxEvents?: number; + readonly heartbeatSeconds?: number; + readonly signal?: AbortSignal; + readonly maxReconnectAttempts?: number; + readonly retryBackoffMs?: number; +} + +export interface T4ApiClient { + /** + * Low-level generated client. Event-stream calls must pass `parseAs: "stream"`; + * use `watchSession` for bounded, validated event decoding and reconnects. + */ + readonly http: Client; + watchSession(sessionId: string, options?: WatchSessionOptions): AsyncGenerator; +} + +export class T4ApiError extends Error { + readonly code: ApiError["code"]; + readonly status: number; + readonly requestId: string; + readonly retryable: boolean; + readonly violations?: ApiError["violations"]; + readonly supportedMajors?: ApiError["supportedMajors"]; + readonly resync?: Resync; + readonly retryAfterMs?: number; + + constructor(status: number, error: ApiError, options?: ErrorOptions & { readonly retryAfterMs?: number }) { + super(error.message, options); + this.name = "T4ApiError"; + this.code = error.code; + this.status = status; + this.requestId = error.requestId; + this.retryable = error.retryable; + if (error.violations !== undefined) this.violations = error.violations; + if (error.supportedMajors !== undefined) this.supportedMajors = error.supportedMajors; + if (error.resync !== undefined) this.resync = error.resync; + if (options?.retryAfterMs !== undefined) this.retryAfterMs = options.retryAfterMs; + } +} + +function normalizedBaseUrl(value: string): string { + let url: URL; + try { + url = new URL(value); + } catch (error) { + throw new TypeError("T4 API baseUrl must be an absolute HTTPS URL", { cause: error }); + } + if ( + url.protocol !== "https:" || + url.username !== "" || + url.password !== "" || + url.search !== "" || + url.hash !== "" + ) { + throw new TypeError("T4 API baseUrl must be a credential-free HTTPS URL without query or fragment"); + } + url.pathname = url.pathname.replace(/\/+$/u, ""); + return url.toString().replace(/\/$/u, ""); +} + +function requiredCredential(value: string): string { + if ( + value.length === 0 || + value.length > MAX_CREDENTIAL_LENGTH || + !/^[A-Za-z0-9._~+/-]+=*$/u.test(value) + ) { + throw new TypeError("credential must be an opaque bearer token of at most 4096 characters"); + } + return value; +} + +function requiredMajor(value: number): string { + if (!Number.isSafeInteger(value) || value < 1 || value > 9999) { + throw new TypeError("majorVersion must be an integer between 1 and 9999"); + } + return String(value); +} + +function boundedInteger(value: number | undefined, fallback: number, minimum: number, maximum: number, label: string): number { + const selected = value ?? fallback; + if (!Number.isSafeInteger(selected) || selected < minimum || selected > maximum) { + throw new RangeError(`${label} must be an integer between ${minimum} and ${maximum}`); + } + return selected; +} + +function requiredSessionId(value: string): string { + if (value.length < 1 || value.length > 128 || !/^[A-Za-z0-9][A-Za-z0-9._~-]*$/u.test(value)) { + throw new TypeError("sessionId is invalid"); + } + return value; +} + +function record(value: unknown): Record | undefined { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? value as Record + : undefined; +} + +function validViolation(value: unknown): boolean { + const violation = record(value); + return violation !== undefined && + Object.keys(violation).every((key) => key === "field" || key === "rule" || key === "message") && + typeof violation.field === "string" && violation.field !== "" && hasAtMostCodePoints(violation.field, 256) && + typeof violation.rule === "string" && hasAtMostCodePoints(violation.rule, 64) && /^[A-Za-z][A-Za-z0-9._-]{0,63}$/u.test(violation.rule) && + typeof violation.message === "string" && violation.message !== "" && hasAtMostCodePoints(violation.message, 512); +} + +function validResync(value: unknown): value is Resync { + const resync = record(value); + if (resync === undefined || Object.keys(resync).some((key) => key !== "snapshotUrl" && key !== "cursor") || typeof resync.snapshotUrl !== "string") return false; + const snapshot = /^\/v1\/sessions\/([^/]+)\/snapshot$/u.exec(resync.snapshotUrl); + return hasAtMostCodePoints(resync.snapshotUrl, 150) && snapshot !== null && validResourceId(snapshot[1]) && + typeof resync.cursor === "string" && hasAtMostCodePoints(resync.cursor, 512) && CURSOR_PATTERN.test(resync.cursor); +} + +function apiError(value: unknown, status: number): ApiError | undefined { + const envelope = record(value); + const error = record(envelope?.error); + if ( + envelope === undefined || Object.keys(envelope).some((key) => key !== "error") || + error === undefined || Object.keys(error).some((key) => ERROR_FIELDS[key] !== true) || + typeof error.code !== "string" || ERROR_CODES[error.code as ApiError["code"]] !== true || + ERROR_CODES_BY_STATUS[status]?.[error.code] !== true || + typeof error.message !== "string" || error.message === "" || !hasAtMostCodePoints(error.message, 1024) || + typeof error.requestId !== "string" || error.requestId === "" || !hasAtMostCodePoints(error.requestId, 128) || + typeof error.retryable !== "boolean" || + (error.violations !== undefined && (!Array.isArray(error.violations) || error.violations.length > 64 || !error.violations.every(validViolation))) || + (error.supportedMajors !== undefined && (!Array.isArray(error.supportedMajors) || error.supportedMajors.length > 8 || !hasUniqueItems(error.supportedMajors) || !error.supportedMajors.every((major) => Number.isSafeInteger(major) && Number(major) >= 1))) || + (error.resync !== undefined && !validResync(error.resync)) || + (status === 406 && (error.code !== "incompatible_version" || !Array.isArray(error.supportedMajors) || error.supportedMajors.length < 1)) || + (status === 410 && (error.code !== "cursor_expired" || !validResync(error.resync))) || + (status === 422 && (error.code !== "invalid_request" || !Array.isArray(error.violations) || error.violations.length < 1)) + ) return undefined; + return error as ApiError; +} + +async function boundedResponseText(response: Response, maximumBytes = MAX_ERROR_BYTES): Promise { + if (response.body === null) return ""; + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let length = 0; + try { + while (true) { + const chunk = await reader.read(); + if (chunk.done) break; + length += chunk.value.byteLength; + if (length > maximumBytes) { + void reader.cancel().catch(() => {}); + return undefined; + } + chunks.push(chunk.value); + } + } finally { + reader.releaseLock(); + } + const bytes = new Uint8Array(length); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + try { return new TextDecoder("utf-8", { fatal: true }).decode(bytes); } catch { return undefined; } +} + +function matchingHttpDate( + timestamp: number, + parts: RegExpExecArray, + indexes: { readonly weekday: number; readonly day: number; readonly alternateDay?: number; readonly month: number; readonly year: number; readonly hour: number; readonly minute: number; readonly second: number }, + expectedYear = Number(parts[indexes.year]), +): boolean { + const date = new Date(timestamp); + return date.getUTCDay() === HTTP_WEEKDAYS[parts[indexes.weekday]!] && + date.getUTCDate() === Number(parts[indexes.day] ?? parts[indexes.alternateDay ?? indexes.day]) && + date.getUTCMonth() === HTTP_MONTHS[parts[indexes.month]!] && + date.getUTCFullYear() === expectedYear && + date.getUTCHours() === Number(parts[indexes.hour]) && + date.getUTCMinutes() === Number(parts[indexes.minute]) && + date.getUTCSeconds() === Number(parts[indexes.second]); +} + +function httpDateTimestamp(value: string): number | undefined { + const imf = /^(Sun|Mon|Tue|Wed|Thu|Fri|Sat), ([0-9]{2}) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) ([0-9]{4}) ([0-9]{2}):([0-9]{2}):([0-9]{2}) GMT$/u.exec(value); + if (imf !== null) { + const timestamp = Date.parse(value); + return Number.isFinite(timestamp) && matchingHttpDate(timestamp, imf, { weekday: 1, day: 2, month: 3, year: 4, hour: 5, minute: 6, second: 7 }) ? timestamp : undefined; + } + const rfc850 = /^(Sunday|Monday|Tuesday|Wednesday|Thursday|Friday|Saturday), ([0-9]{2})-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-([0-9]{2}) ([0-9]{2}):([0-9]{2}):([0-9]{2}) GMT$/u.exec(value); + if (rfc850 !== null) { + const now = new Date(Date.now()); + let year = Math.floor(now.getUTCFullYear() / 100) * 100 + Number(rfc850[4]); + const date = new Date(0); + date.setUTCFullYear(year, HTTP_MONTHS[rfc850[3]!]!, Number(rfc850[2])); + date.setUTCHours(Number(rfc850[5]), Number(rfc850[6]), Number(rfc850[7]), 0); + const fiftyYearsFromNow = new Date(now.getTime()); + fiftyYearsFromNow.setUTCFullYear(now.getUTCFullYear() + 50); + if (date.getTime() > fiftyYearsFromNow.getTime()) { + year -= 100; + date.setUTCFullYear(year); + } + const timestamp = date.getTime(); + return matchingHttpDate(timestamp, rfc850, { weekday: 1, day: 2, month: 3, year: 4, hour: 5, minute: 6, second: 7 }, year) ? timestamp : undefined; + } + const asctime = /^(Sun|Mon|Tue|Wed|Thu|Fri|Sat) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) (?:([0-9]{2})| ([0-9])) ([0-9]{2}):([0-9]{2}):([0-9]{2}) ([0-9]{4})$/u.exec(value); + if (asctime === null) return undefined; + const timestamp = Date.parse(value); + return Number.isFinite(timestamp) && matchingHttpDate(timestamp, asctime, { weekday: 1, day: 3, alternateDay: 4, month: 2, year: 8, hour: 5, minute: 6, second: 7 }) ? timestamp : undefined; +} + +function retryAfterMilliseconds(response: Response): number | undefined | typeof INVALID_RETRY_AFTER { + const header = response.headers.get("Retry-After"); + if (header === null) return undefined; + const value = header.trim(); + if (value === "" || !hasAtMostCodePoints(value, 128)) return INVALID_RETRY_AFTER; + if (/^[0-9]+$/u.test(value)) return Math.min(30_000, Number(value) * 1000); + const timestamp = httpDateTimestamp(value); + if (timestamp === undefined) return INVALID_RETRY_AFTER; + return Math.min(30_000, Math.max(0, timestamp - Date.now())); +} + + +function protocolError(status: number, message: string): T4ApiError { + return new T4ApiError(status === 503 ? 502 : status, { + code: "indeterminate", + message, + requestId: "unavailable", + retryable: false, + }); +} + +async function parsedError(response: Response): Promise { + if (response.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase() !== "application/json") { + void response.body?.cancel().catch(() => {}); + return undefined; + } + const text = await boundedResponseText(response); + if (text === undefined) return undefined; + try { + const decoded = apiError(JSON.parse(text), response.status); + if (decoded === undefined) return undefined; + const retryAfterMs = retryAfterMilliseconds(response); + if (retryAfterMs === INVALID_RETRY_AFTER) return undefined; + return retryAfterMs === undefined + ? new T4ApiError(response.status, decoded) + : new T4ApiError(response.status, decoded, { retryAfterMs }); + } catch { + return undefined; + } +} + + +function hasOnlyKeys(value: Record, keys: Readonly>): boolean { + return Object.keys(value).every((key) => keys[key] === true); +} + +function hasUniqueItems(value: readonly unknown[]): boolean { + for (let index = 0; index < value.length; index += 1) { + for (let prior = 0; prior < index; prior += 1) { + if (value[index] === value[prior]) return false; + } + } + return true; +} + +function validResourceId(value: unknown): value is string { + return typeof value === "string" && value !== "" && hasAtMostCodePoints(value, 128) && /^[A-Za-z0-9][A-Za-z0-9._~-]*$/u.test(value); +} + +function hasAtMostCodePoints(value: string, maximum: number): boolean { + let count = 0; + for (const _codePoint of value) { + count += 1; + if (count > maximum) return false; + } + return true; +} + +function validCursor(value: unknown): value is string { + return typeof value === "string" && value !== "" && hasAtMostCodePoints(value, 512) && CURSOR_PATTERN.test(value); +} + +function validLabels(value: unknown): boolean { + if (value === undefined) return true; + const labels = record(value); + return labels !== undefined && Object.keys(labels).length <= 32 && Object.entries(labels).every(([key, item]) => + /^[a-z][a-z0-9.-]{0,62}$/u.test(key) && typeof item === "string" && hasAtMostCodePoints(item, 128)); +} + +function validWorkspace(value: unknown): boolean { + const item = record(value); + return item !== undefined && hasOnlyKeys(item, { id: true, name: true, state: true, revision: true, labels: true }) && + validResourceId(item.id) && typeof item.name === "string" && item.name !== "" && hasAtMostCodePoints(item.name, 128) && + typeof item.state === "string" && WORKSPACE_STATES[item.state as components["schemas"]["WorkspaceState"]] === true && + Number.isSafeInteger(item.revision) && Number(item.revision) >= 1 && validLabels(item.labels); +} + +function validSession(value: unknown): boolean { + const item = record(value); + return item !== undefined && hasOnlyKeys(item, { id: true, workspaceId: true, title: true, state: true, revision: true, labels: true }) && + validResourceId(item.id) && validResourceId(item.workspaceId) && typeof item.title === "string" && item.title !== "" && hasAtMostCodePoints(item.title, 128) && + typeof item.state === "string" && SESSION_STATES[item.state as components["schemas"]["SessionState"]] === true && + Number.isSafeInteger(item.revision) && Number(item.revision) >= 1 && validLabels(item.labels); +} + +function validCommandResult(value: unknown): boolean { + const item = record(value); + return item !== undefined && hasOnlyKeys(item, { commandId: true, state: true }) && validResourceId(item.commandId) && + typeof item.state === "string" && COMMAND_STATES[item.state as components["schemas"]["CommandState"]] === true; +} + +function validCapabilityDeprecation(value: unknown): boolean { + if (value === undefined) return true; + const deprecation = record(value); + return deprecation !== undefined && hasOnlyKeys(deprecation, { message: true, sinceVersion: true, sunsetAt: true, replacement: true }) && + typeof deprecation.message === "string" && deprecation.message !== "" && hasAtMostCodePoints(deprecation.message, 1024) && + (deprecation.sinceVersion === undefined || (typeof deprecation.sinceVersion === "string" && deprecation.sinceVersion !== "" && hasAtMostCodePoints(deprecation.sinceVersion, 128))) && + (deprecation.sunsetAt === undefined || (typeof deprecation.sunsetAt === "string" && validRfc3339DateTime(deprecation.sunsetAt))) && + (deprecation.replacement === undefined || (typeof deprecation.replacement === "string" && /^[a-z][a-z0-9.-]{0,127}$/u.test(deprecation.replacement))); +} + +function validCapabilityStatus(value: unknown): boolean { + const status = record(value); + return status !== undefined && hasOnlyKeys(status, { supported: true, enabled: true, authorized: true, available: true, deprecation: true }) && + typeof status.supported === "boolean" && typeof status.enabled === "boolean" && + typeof status.authorized === "boolean" && typeof status.available === "boolean" && validCapabilityDeprecation(status.deprecation); +} + +function validDiscovery(value: unknown): boolean { + const discovery = record(value); + const serverBuild = record(discovery?.serverBuild); + const capabilities = record(discovery?.capabilities); + const limits = record(discovery?.limits); + return discovery !== undefined && hasOnlyKeys(discovery, { apiVersion: true, serverBuild: true, supportedMajors: true, capabilities: true, limits: true }) && + typeof discovery.apiVersion === "string" && /^1\.[0-9]+$/u.test(discovery.apiVersion) && + serverBuild !== undefined && hasOnlyKeys(serverBuild, { version: true, revision: true }) && + typeof serverBuild.version === "string" && serverBuild.version !== "" && hasAtMostCodePoints(serverBuild.version, 128) && + typeof serverBuild.revision === "string" && serverBuild.revision !== "" && hasAtMostCodePoints(serverBuild.revision, 128) && + Array.isArray(discovery.supportedMajors) && discovery.supportedMajors.length >= 1 && discovery.supportedMajors.length <= 8 && hasUniqueItems(discovery.supportedMajors) && discovery.supportedMajors.every((major) => Number.isSafeInteger(major) && Number(major) >= 1) && + capabilities !== undefined && Object.keys(capabilities).length <= 128 && Object.entries(capabilities).every(([id, status]) => /^[a-z][a-z0-9.-]{0,127}$/u.test(id) && validCapabilityStatus(status)) && + limits !== undefined && hasOnlyKeys(limits, { pageSizeDefault: true, pageSizeMax: true, commandBytesMax: true, commandRequestBytesMax: true, commandMetadataValueBytesMax: true, watchEventsMax: true, heartbeatSeconds: true }) && + [limits.pageSizeDefault, limits.pageSizeMax, limits.commandBytesMax, limits.commandRequestBytesMax, limits.commandMetadataValueBytesMax, limits.watchEventsMax, limits.heartbeatSeconds].every((limit) => Number.isSafeInteger(limit) && Number(limit) >= 1) && + Number(limits.pageSizeDefault) <= Number(limits.pageSizeMax) && Number(limits.commandBytesMax) <= Number(limits.commandRequestBytesMax) && Number(limits.commandMetadataValueBytesMax) <= Number(limits.commandRequestBytesMax) && + Number(limits.pageSizeMax) <= 100 && Number(limits.commandBytesMax) <= 262144 && Number(limits.commandRequestBytesMax) <= 1048576 && Number(limits.commandMetadataValueBytesMax) <= 262144 && Number(limits.watchEventsMax) <= 1000 && Number(limits.heartbeatSeconds) >= 5 && Number(limits.heartbeatSeconds) <= 60; +} + +function validSnapshot(value: unknown): boolean { + const snapshot = record(value); + return snapshot !== undefined && hasOnlyKeys(snapshot, { sessionId: true, cursor: true, state: true, entries: true }) && + validResourceId(snapshot.sessionId) && validCursor(snapshot.cursor) && + typeof snapshot.state === "string" && SESSION_STATES[snapshot.state as components["schemas"]["SessionState"]] === true && + Array.isArray(snapshot.entries) && snapshot.entries.length <= 1000 && snapshot.entries.every((value) => { + const entry = record(value); + return entry !== undefined && hasOnlyKeys(entry, { sequence: true, kind: true, text: true }) && + Number.isSafeInteger(entry.sequence) && Number(entry.sequence) >= 0 && + (entry.kind === "input" || entry.kind === "output" || entry.kind === "status") && + typeof entry.text === "string" && hasAtMostCodePoints(entry.text, 1_048_576); + }); +} + +type SuccessValidator = ((value: unknown) => boolean) | "empty" | "event-stream"; + +interface ResponseContract { + readonly success: Readonly>; + readonly errors: readonly number[]; + readonly replaySuccess?: readonly number[]; +} + +const DISCOVERY_RESPONSE: ResponseContract = { success: { 200: validDiscovery }, errors: [401, 403, 406, 503] }; +const WORKSPACE_LIST_RESPONSE: ResponseContract = { success: { 200: validWorkspacePage }, errors: [400, 401, 403, 406, 422, 503] }; +const WORKSPACE_CREATE_RESPONSE: ResponseContract = { success: { 200: validWorkspace, 202: validWorkspace }, errors: [400, 401, 403, 406, 409, 422, 503], replaySuccess: [200, 202] }; +const WORKSPACE_GET_RESPONSE: ResponseContract = { success: { 200: validWorkspace }, errors: [401, 403, 404, 406, 503] }; +const WORKSPACE_MUTATE_RESPONSE: ResponseContract = { success: { 200: validWorkspace }, errors: [400, 401, 403, 404, 406, 409, 422, 503], replaySuccess: [200] }; +const DELETE_RESPONSE: ResponseContract = { success: { 204: "empty" }, errors: [400, 401, 403, 404, 406, 409, 503], replaySuccess: [204] }; +const SESSION_LIST_RESPONSE: ResponseContract = { success: { 200: validSessionPage }, errors: [401, 403, 404, 406, 422, 503] }; +const SESSION_CREATE_RESPONSE: ResponseContract = { success: { 200: validSession, 202: validSession }, errors: [400, 401, 403, 404, 406, 409, 422, 503], replaySuccess: [200, 202] }; +const SESSION_GET_RESPONSE: ResponseContract = { success: { 200: validSession }, errors: [401, 403, 404, 406, 503] }; +const SESSION_MUTATE_RESPONSE: ResponseContract = { success: { 200: validSession }, errors: [400, 401, 403, 404, 406, 409, 422, 503], replaySuccess: [200] }; +const SESSION_CANCEL_RESPONSE: ResponseContract = { success: { 200: validSession, 202: validSession }, errors: [400, 401, 403, 404, 406, 409, 503], replaySuccess: [200, 202] }; +const COMMAND_RESPONSE: ResponseContract = { success: { 200: validCommandResult, 202: validCommandResult }, errors: [400, 401, 403, 404, 406, 409, 422, 503], replaySuccess: [200, 202] }; +const SNAPSHOT_RESPONSE: ResponseContract = { success: { 200: validSnapshot }, errors: [401, 403, 404, 406, 503] }; +const EVENTS_RESPONSE: ResponseContract = { success: { 200: "event-stream" }, errors: [400, 401, 403, 404, 406, 410, 422, 503] }; + +function validWorkspacePage(value: unknown): boolean { + const page = record(value); + return page !== undefined && hasOnlyKeys(page, { items: true, nextCursor: true }) && + Array.isArray(page.items) && page.items.length <= 100 && page.items.every(validWorkspace) && + (page.nextCursor === undefined || validCursor(page.nextCursor)); +} + +function validSessionPage(value: unknown): boolean { + const page = record(value); + return page !== undefined && hasOnlyKeys(page, { items: true, nextCursor: true }) && + Array.isArray(page.items) && page.items.length <= 100 && page.items.every(validSession) && + (page.nextCursor === undefined || validCursor(page.nextCursor)); +} + +function relativeApiPath(request: Request, baseUrl: string): string | undefined { + const requestUrl = new URL(request.url); + const base = new URL(baseUrl); + if (requestUrl.origin !== base.origin) return undefined; + const prefix = base.pathname === "/" ? "" : base.pathname; + if (prefix === "") return requestUrl.pathname; + if (!requestUrl.pathname.startsWith(`${prefix}/`)) return undefined; + return requestUrl.pathname.slice(prefix.length); +} + +function responseContract(method: string, path: string): ResponseContract | undefined { + if (path === "/v1" && method === "GET") return DISCOVERY_RESPONSE; + if (path === "/v1/workspaces") { + if (method === "GET") return WORKSPACE_LIST_RESPONSE; + if (method === "POST") return WORKSPACE_CREATE_RESPONSE; + return undefined; + } + if (/^\/v1\/workspaces\/[A-Za-z0-9._~-]+$/u.test(path)) { + if (method === "GET") return WORKSPACE_GET_RESPONSE; + if (method === "PATCH") return WORKSPACE_MUTATE_RESPONSE; + if (method === "DELETE") return DELETE_RESPONSE; + return undefined; + } + if (/^\/v1\/workspaces\/[A-Za-z0-9._~-]+\/sessions$/u.test(path)) { + if (method === "GET") return SESSION_LIST_RESPONSE; + if (method === "POST") return SESSION_CREATE_RESPONSE; + return undefined; + } + if (/^\/v1\/sessions\/[A-Za-z0-9._~-]+$/u.test(path)) { + if (method === "GET") return SESSION_GET_RESPONSE; + if (method === "PATCH") return SESSION_MUTATE_RESPONSE; + if (method === "DELETE") return DELETE_RESPONSE; + return undefined; + } + if (/^\/v1\/sessions\/[A-Za-z0-9._~-]+\/cancel$/u.test(path) && method === "POST") return SESSION_CANCEL_RESPONSE; + if (/^\/v1\/sessions\/[A-Za-z0-9._~-]+\/commands$/u.test(path) && method === "POST") return COMMAND_RESPONSE; + if (/^\/v1\/sessions\/[A-Za-z0-9._~-]+\/snapshot$/u.test(path) && method === "GET") return SNAPSHOT_RESPONSE; + if (/^\/v1\/sessions\/[A-Za-z0-9._~-]+\/events$/u.test(path) && method === "GET") return EVENTS_RESPONSE; + return undefined; +} + +function validSelectedVersion(response: Response, expectedMajor?: string): boolean { + const selected = response.headers.get("T4-API-Version"); + return selected !== null && selected.length <= SELECTED_VERSION_MAX_LENGTH && SELECTED_VERSION_PATTERN.test(selected) && + (expectedMajor === undefined || selected.startsWith(`${expectedMajor}.`)); +} + +function validReplayHeader(response: Response): boolean { + const replayed = response.headers.get("Idempotency-Replayed"); + return replayed === "true" || replayed === "false"; +} + + +async function validateResponse(request: Request, response: Response, baseUrl: string): Promise { + const path = relativeApiPath(request, baseUrl); + const contract = path === undefined ? undefined : responseContract(request.method, path); + if (contract === undefined) { + void response.body?.cancel().catch(() => {}); + throw protocolError(502, "T4 API returned a response for an undeclared route"); + } + if (!response.ok) { + if (!contract.errors.includes(response.status)) { + void response.body?.cancel().catch(() => {}); + throw protocolError(502, "T4 API returned an undeclared error status"); + } + if (response.status !== 401 && !validSelectedVersion(response)) { + void response.body?.cancel().catch(() => {}); + throw protocolError(502, "T4 API returned a missing or invalid selected version"); + } + if (await parsedError(response.clone()) === undefined) { + void response.body?.cancel().catch(() => {}); + throw protocolError(502, "T4 API returned an invalid or oversized error envelope"); + } + return; + } + const validator = contract.success[response.status]; + if (validator === undefined) { + void response.body?.cancel().catch(() => {}); + throw protocolError(502, "T4 API returned an undeclared success status"); + } + if (!validSelectedVersion(response, request.headers.get("T4-API-Version") ?? "")) { + void response.body?.cancel().catch(() => {}); + throw protocolError(502, "T4 API returned a missing or invalid selected version"); + } + if (contract.replaySuccess?.includes(response.status) === true && !validReplayHeader(response)) { + void response.body?.cancel().catch(() => {}); + throw protocolError(502, "T4 API returned a missing or invalid replay header"); + } + if (contract.replaySuccess?.includes(response.status) === true && !validCursor(response.headers.get("T4-Event-Cursor"))) { + void response.body?.cancel().catch(() => {}); + throw protocolError(502, "T4 API returned a missing or invalid durable event cursor"); + } + if (validator === "empty") { + if (response.body !== null || response.headers.has("content-type")) { + void response.body?.cancel().catch(() => {}); + throw protocolError(502, "T4 API returned content for a bodyless response"); + } + return; + } + if (validator === "event-stream") { + if (response.headers.get("Cache-Control") !== "no-store") { + void response.body?.cancel().catch(() => {}); + throw protocolError(502, "T4 API watch did not return Cache-Control: no-store"); + } + if (response.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase() !== "text/event-stream") { + void response.body?.cancel().catch(() => {}); + throw protocolError(502, "T4 API returned an undeclared success media type"); + } + return; + } + if (response.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase() !== "application/json") { + void response.body?.cancel().catch(() => {}); + throw protocolError(502, "T4 API returned an undeclared success media type"); + } + const text = await boundedResponseText(response.clone(), MAX_JSON_RESPONSE_BYTES); + let value: unknown; + try { value = text === undefined ? undefined : JSON.parse(text); } catch { value = undefined; } + if (text === undefined || !validator(value)) { + void response.body?.cancel().catch(() => {}); + throw protocolError(502, "T4 API returned an invalid or oversized JSON response"); + } +} + +function isKnownUtcLeapSecondDate(year: number, month: number, day: number): boolean { + // Update this IERS insertion-date table when a new UTC leap second is announced. + switch (year * 10_000 + month * 100 + day) { + case 19720630: case 19721231: case 19731231: case 19741231: case 19751231: + case 19761231: case 19771231: case 19781231: case 19791231: case 19810630: + case 19820630: case 19830630: case 19850630: case 19871231: case 19891231: + case 19901231: case 19920630: case 19930630: case 19940630: case 19951231: + case 19970630: case 19981231: case 20051231: case 20081231: case 20120630: + case 20150630: case 20161231: + return true; + default: + return false; + } +} + +function validRfc3339DateTime(value: string): boolean { + if (!hasAtMostCodePoints(value, 64)) return false; + const match = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d+)?(?:Z|([+-])(\d{2}):(\d{2}))$/iu.exec(value); + if (match === null) return false; + const year = Number(match[1]); + const month = Number(match[2]); + const day = Number(match[3]); + const hour = Number(match[4]); + const minute = Number(match[5]); + const second = Number(match[6]); + if (month < 1 || month > 12 || hour > 23 || minute > 59 || second > 60) return false; + let maximumDay = 31; + if (month === 2) maximumDay = year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0) ? 29 : 28; + else if (month === 4 || month === 6 || month === 9 || month === 11) maximumDay = 30; + if (day < 1 || day > maximumDay) return false; + let offsetMinutes = 0; + if (match[7] !== undefined) { + const offsetHour = Number(match[8]); + const offsetMinute = Number(match[9]); + if (offsetHour > 23 || offsetMinute > 59) return false; + offsetMinutes = (match[7] === "+" ? 1 : -1) * (offsetHour * 60 + offsetMinute); + } + if (second < 60) return true; + const utc = new Date(0); + utc.setUTCFullYear(year, month - 1, day); + utc.setUTCHours(hour, minute - offsetMinutes, 0, 0); + return utc.getUTCHours() === 23 && utc.getUTCMinutes() === 59 && + isKnownUtcLeapSecondDate(utc.getUTCFullYear(), utc.getUTCMonth() + 1, utc.getUTCDate()); +} + +function watchEvent(value: unknown, eventId: string | undefined): WatchEvent { + const event = record(value); + if ( + event === undefined || + typeof event.type !== "string" || + typeof event.cursor !== "string" || + event.cursor === "" || + !hasAtMostCodePoints(event.cursor, 512) || + !CURSOR_PATTERN.test(event.cursor) || + (eventId !== undefined && eventId !== event.cursor) + ) throw protocolError(502, "T4 API returned an invalid watch event"); + if ( + event.type === "heartbeat" && + hasOnlyKeys(event, { type: true, cursor: true, observedAt: true }) && + typeof event.observedAt === "string" && validRfc3339DateTime(event.observedAt) + ) return event as WatchEvent; + if ( + event.type === "session" && + hasOnlyKeys(event, { type: true, cursor: true, state: true, revision: true }) && + typeof event.state === "string" && + SESSION_STATES[event.state as components["schemas"]["SessionState"]] === true && + Number.isSafeInteger(event.revision) && Number(event.revision) >= 1 + ) return event as WatchEvent; + if ( + event.type === "command" && + hasOnlyKeys(event, { type: true, cursor: true, commandId: true, state: true }) && + typeof event.commandId === "string" && + event.commandId !== "" && hasAtMostCodePoints(event.commandId, 128) && + /^[A-Za-z0-9][A-Za-z0-9._~-]*$/u.test(event.commandId) && + typeof event.state === "string" && + COMMAND_STATES[event.state as components["schemas"]["CommandState"]] === true + ) return event as WatchEvent; + throw protocolError(502, "T4 API returned an invalid watch event"); +} + +function decodeSseFrame(frame: string): WatchEvent | undefined { + let eventId: string | undefined; + const data: string[] = []; + for (const rawLine of frame.split(/\r\n|\r|\n/u)) { + const line = rawLine; + if (line === "" || line.startsWith(":")) continue; + const separator = line.indexOf(":"); + const field = separator < 0 ? line : line.slice(0, separator); + const rawValue = separator < 0 ? "" : line.slice(separator + 1); + const value = rawValue.startsWith(" ") ? rawValue.slice(1) : rawValue; + if (field === "id") eventId = value; + else if (field === "data") data.push(value); + } + if (data.length === 0) return undefined; + try { + return watchEvent(JSON.parse(data.join("\n")), eventId); + } catch (error) { + if (error instanceof T4ApiError) throw error; + throw protocolError(502, "T4 API returned malformed SSE data"); + } +} + +class SseFrameParser { + readonly #buffer = new Uint8Array(MAX_EVENT_BYTES + 4); + #length = 0; + #lineStart = 0; + #pendingCarriageReturn = -1; + + push(chunk: Uint8Array): Uint8Array[] { + const frames: Uint8Array[] = []; + for (const byte of chunk) this.#pushByte(byte, frames); + return frames; + } + + finish(): Uint8Array[] { + const frames: Uint8Array[] = []; + if (this.#pendingCarriageReturn >= 0) { + this.#finishLine(this.#pendingCarriageReturn, frames); + this.#pendingCarriageReturn = -1; + } + if (this.#length > MAX_EVENT_BYTES) this.#oversized(); + this.#reset(); + return frames; + } + + #pushByte(byte: number, frames: Uint8Array[]): void { + if (this.#pendingCarriageReturn >= 0) { + if (byte === 10) { + this.#append(byte); + this.#finishLine(this.#pendingCarriageReturn, frames); + this.#pendingCarriageReturn = -1; + return; + } + this.#finishLine(this.#pendingCarriageReturn, frames); + this.#pendingCarriageReturn = -1; + } + this.#append(byte); + if (byte === 13) this.#pendingCarriageReturn = this.#length - 1; + else if (byte === 10) this.#finishLine(this.#length - 1, frames); + } + + #append(byte: number): void { + if (this.#length >= this.#buffer.byteLength) this.#oversized(); + this.#buffer[this.#length++] = byte; + } + + #finishLine(newlineStart: number, frames: Uint8Array[]): void { + if (newlineStart === this.#lineStart) { + if (this.#lineStart > MAX_EVENT_BYTES) this.#oversized(); + frames.push(this.#buffer.slice(0, this.#lineStart)); + this.#reset(); + return; + } + this.#lineStart = this.#length; + if (this.#lineStart > MAX_EVENT_BYTES) this.#oversized(); + } + + #reset(): void { + this.#length = 0; + this.#lineStart = 0; + } + + #oversized(): never { + throw protocolError(502, "T4 API watch event exceeds the client bound"); + } +} + +function decodedFrames(parser: SseFrameParser, chunk: Uint8Array | undefined): WatchEvent[] { + const frames = chunk === undefined ? parser.finish() : parser.push(chunk); + return frames.flatMap((bytes) => { + let text: string; + try { text = new TextDecoder("utf-8", { fatal: true }).decode(bytes); } catch { + throw protocolError(502, "T4 API returned malformed SSE data"); + } + const event = decodeSseFrame(text); + return event === undefined ? [] : [event]; + }); +} + +async function retryDelay(milliseconds: number, signal: AbortSignal): Promise { + if (signal.aborted) return false; + if (milliseconds === 0) return true; + return await new Promise((resolve) => { + const timeout = setTimeout(() => { + signal.removeEventListener("abort", abort); + resolve(true); + }, milliseconds); + const abort = (): void => { + clearTimeout(timeout); + resolve(false); + }; + signal.addEventListener("abort", abort, { once: true }); + }); +} + +async function* watch( + baseUrl: string, + credential: string, + majorVersion: string, + fetchImpl: typeof globalThis.fetch, + sessionIdValue: string, + options: WatchSessionOptions, +): AsyncGenerator { + const sessionId = requiredSessionId(sessionIdValue); + const maxEvents = boundedInteger(options.maxEvents, 100, 1, 1000, "maxEvents"); + const heartbeatSeconds = boundedInteger(options.heartbeatSeconds, 15, 5, 60, "heartbeatSeconds"); + const maxReconnectAttempts = boundedInteger(options.maxReconnectAttempts, 3, 0, 10, "maxReconnectAttempts"); + const retryBackoffMs = boundedInteger(options.retryBackoffMs, 250, 0, 30_000, "retryBackoffMs"); + if (options.cursor !== undefined && (options.cursor.length < 1 || options.cursor.length > 512 || !CURSOR_PATTERN.test(options.cursor))) { + throw new TypeError("cursor must be a header-safe token containing between 1 and 512 characters"); + } + const controller = new AbortController(); + const abort = (): void => controller.abort(options.signal?.reason); + if (options.signal?.aborted === true) abort(); + else options.signal?.addEventListener("abort", abort, { once: true }); + let delivered = 0; + let reconnectAttempts = 0; + let cursor = options.cursor; + const deliveredCursors = new Set(); + if (cursor !== undefined) deliveredCursors.add(cursor); + try { + while (delivered < maxEvents && !controller.signal.aborted) { + const deliveredAtAttemptStart = delivered; + let reader: ReadableStreamDefaultReader | undefined; + let transientFailure: unknown; + let retryAfterMs = 0; + try { + const url = new URL(`${baseUrl}/v1/sessions/${encodeURIComponent(sessionId)}/events`); + url.searchParams.set("maxEvents", String(maxEvents - delivered)); + url.searchParams.set("heartbeatSeconds", String(heartbeatSeconds)); + if (cursor !== undefined) url.searchParams.set("cursor", cursor); + const headers = new Headers({ + Accept: "text/event-stream", + Authorization: `Bearer ${credential}`, + "Cache-Control": "no-store", + "T4-API-Version": majorVersion, + }); + if (cursor !== undefined) headers.set("Last-Event-ID", cursor); + const response = await fetchImpl(url, { method: "GET", headers, signal: controller.signal }); + if (!response.ok) { + if (!EVENTS_RESPONSE.errors.includes(response.status)) { + void response.body?.cancel().catch(() => {}); + throw protocolError(502, "T4 API returned an undeclared watch error status"); + } + if (response.status !== 401 && !validSelectedVersion(response)) { + void response.body?.cancel().catch(() => {}); + throw protocolError(502, "T4 API returned a missing or invalid selected version"); + } + const error = await parsedError(response); + throw error ?? protocolError(502, "T4 API returned an invalid or oversized error envelope"); + } + if (response.status !== 200) { + void response.body?.cancel().catch(() => {}); + throw protocolError(502, "T4 API returned an undeclared watch success status"); + } + if (!validSelectedVersion(response, majorVersion)) { + void response.body?.cancel().catch(() => {}); + throw protocolError(502, "T4 API returned a missing or invalid selected version"); + } + if (response.headers.get("Cache-Control") !== "no-store") { + void response.body?.cancel().catch(() => {}); + throw protocolError(502, "T4 API watch did not return Cache-Control: no-store"); + } + if (response.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase() !== "text/event-stream") { + void response.body?.cancel().catch(() => {}); + throw protocolError(502, "T4 API watch did not return text/event-stream"); + } + if (response.body === null) throw protocolError(502, "T4 API watch response body is unavailable"); + reader = response.body.getReader(); + const parser = new SseFrameParser(); + while (delivered < maxEvents && !controller.signal.aborted) { + const chunk = await reader.read(); + const events = chunk.done ? decodedFrames(parser, undefined) : decodedFrames(parser, chunk.value); + for (const event of events) { + if (event.type === "heartbeat") { + if (cursor === undefined) { + cursor = event.cursor; + deliveredCursors.add(event.cursor); + } else if (event.cursor !== cursor) { + throw protocolError(502, "T4 API watch heartbeat advanced the durable event cursor"); + } + } else { + if (deliveredCursors.has(event.cursor)) throw protocolError(502, "T4 API watch repeated a durable event cursor"); + deliveredCursors.add(event.cursor); + cursor = event.cursor; + } + delivered += 1; + reconnectAttempts = 0; + yield event; + if (delivered >= maxEvents || controller.signal.aborted) break; + } + if (chunk.done) break; + } + if (delivered >= maxEvents || controller.signal.aborted) return; + transientFailure = new TypeError("T4 API watch ended before the requested event bound"); + } catch (error) { + if (controller.signal.aborted) return; + if (error instanceof T4ApiError) { + const reconnectable = error.retryable && error.status === 503 && (error.code === "unavailable" || error.code === "indeterminate"); + if (!reconnectable) throw error; + retryAfterMs = error.retryAfterMs ?? 0; + } + transientFailure = error; + } finally { + if (reader !== undefined) { + try { await reader.cancel(); } catch { /* cancellation is best effort */ } + reader.releaseLock(); + } + } + const madeProgress = delivered > deliveredAtAttemptStart; + if (!madeProgress && reconnectAttempts >= maxReconnectAttempts) { + throw new T4ApiError(502, { code: "indeterminate", message: "T4 API watch reconnect attempts exhausted", requestId: "unavailable", retryable: true }, { cause: transientFailure }); + } + const delay = Math.min(30_000, Math.max(retryAfterMs, retryBackoffMs * (2 ** reconnectAttempts))); + if (!madeProgress) reconnectAttempts += 1; + if (!await retryDelay(delay, controller.signal)) return; + } + } finally { + options.signal?.removeEventListener("abort", abort); + controller.abort(); + } +} + +export function createT4ApiClient(options: T4ApiClientOptions): T4ApiClient { + const baseUrl = normalizedBaseUrl(options.baseUrl); + const credential = requiredCredential(options.credential); + const majorVersion = requiredMajor(options.majorVersion); + const fetchImpl = options.fetch ?? globalThis.fetch; + const authenticatedFetch: typeof globalThis.fetch = async (input, init) => { + const request = new Request(input, init); + request.headers.set("Authorization", `Bearer ${credential}`); + request.headers.set("T4-API-Version", majorVersion); + const path = relativeApiPath(request, baseUrl); + const contract = path === undefined ? undefined : responseContract(request.method, path); + request.headers.set("Accept", contract?.success[200] === "event-stream" ? "text/event-stream" : "application/json"); + const response = await fetchImpl(request); + await validateResponse(request, response, baseUrl); + return response; + }; + const http = createClient({ baseUrl, fetch: authenticatedFetch }); + return Object.freeze({ + http, + watchSession: (sessionId: string, watchOptions: WatchSessionOptions = {}) => + watch(baseUrl, credential, majorVersion, fetchImpl, sessionId, watchOptions), + }); +} diff --git a/packages/t4-api-client/tsconfig.json b/packages/t4-api-client/tsconfig.json new file mode 100644 index 00000000..824d57c3 --- /dev/null +++ b/packages/t4-api-client/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "types": ["node"] + }, + "include": ["src/**/*.ts"] +} diff --git a/packages/t4-api-contract/openapi.json b/packages/t4-api-contract/openapi.json new file mode 100644 index 00000000..66d77c91 --- /dev/null +++ b/packages/t4-api-contract/openapi.json @@ -0,0 +1,861 @@ +{ + "openapi": "3.1.0", + "info": { + "title": "T4 API", + "version": "1.0.0", + "summary": "Authenticated public API for bounded T4 workspace and session lifecycle", + "description": "The public boundary for T4 clients. The service is not assumed to be publicly exposed. Every operation requires an opaque bearer credential and deny-by-default server-side scope authorization. Kubernetes, PostgreSQL, controller, runtime, and private OMP representations are never exposed." + }, + "servers": [{ "url": "https://t4.example.invalid", "description": "Operator-configured private HTTPS endpoint" }], + "security": [{ "BearerAuth": [] }], + "tags": [ + { "name": "Discovery" }, + { "name": "Workspaces" }, + { "name": "Sessions" }, + { "name": "Watch" } + ], + "paths": { + "/v1": { + "get": { + "operationId": "discoverV1", + "tags": ["Discovery"], + "summary": "Negotiate T4 API v1 and discover capabilities and bounds", + "parameters": [{ "$ref": "#/components/parameters/ApiVersion" }], + "responses": { + "200": { "$ref": "#/components/responses/Discovery" }, + "401": {"$ref": "#/components/responses/Error401"}, + "403": {"$ref": "#/components/responses/Error403"}, + "406": {"$ref": "#/components/responses/Error406"}, + "503": {"$ref": "#/components/responses/Error503"} + } + } + }, + "/v1/workspaces": { + "get": { + "operationId": "listWorkspaces", + "tags": ["Workspaces"], + "parameters": [ + { "$ref": "#/components/parameters/ApiVersion" }, + { "$ref": "#/components/parameters/PageSize" }, + { "$ref": "#/components/parameters/PageCursor" } + ], + "responses": { + "200": { "$ref": "#/components/responses/WorkspacePage" }, + "400": {"$ref": "#/components/responses/Error400"}, + "401": {"$ref": "#/components/responses/Error401"}, + "403": {"$ref": "#/components/responses/Error403"}, + "406": {"$ref": "#/components/responses/Error406"}, + "422": {"$ref": "#/components/responses/Error422"}, + "503": {"$ref": "#/components/responses/Error503"} + } + }, + "post": { + "operationId": "createWorkspace", + "tags": ["Workspaces"], + "parameters": [ + { "$ref": "#/components/parameters/ApiVersion" }, + { "$ref": "#/components/parameters/IdempotencyKey" } + ], + "requestBody": { + "required": true, + "content": { "application/json": { "schema": { "$ref": "#/components/schemas/WorkspaceCreate" } } } + }, + "responses": { + "200": { "$ref": "#/components/responses/WorkspaceReplay" }, + "202": { "$ref": "#/components/responses/WorkspaceAccepted" }, + "400": {"$ref": "#/components/responses/Error400"}, + "401": {"$ref": "#/components/responses/Error401"}, + "403": {"$ref": "#/components/responses/Error403"}, + "406": {"$ref": "#/components/responses/Error406"}, + "409": {"$ref": "#/components/responses/Error409"}, + "422": {"$ref": "#/components/responses/Error422"}, + "503": {"$ref": "#/components/responses/Error503"} + } + } + }, + "/v1/workspaces/{workspaceId}": { + "parameters": [{ "$ref": "#/components/parameters/WorkspaceId" }], + "get": { + "operationId": "getWorkspace", + "tags": ["Workspaces"], + "parameters": [{ "$ref": "#/components/parameters/ApiVersion" }], + "responses": { + "200": { "$ref": "#/components/responses/Workspace" }, + "401": {"$ref": "#/components/responses/Error401"}, + "403": {"$ref": "#/components/responses/Error403"}, + "404": {"$ref": "#/components/responses/Error404"}, + "406": {"$ref": "#/components/responses/Error406"}, + "503": {"$ref": "#/components/responses/Error503"} + } + }, + "patch": { + "operationId": "mutateWorkspace", + "tags": ["Workspaces"], + "parameters": [ + { "$ref": "#/components/parameters/ApiVersion" }, + { "$ref": "#/components/parameters/IfMatch" }, + { "$ref": "#/components/parameters/IdempotencyKey" } + ], + "requestBody": { + "required": true, + "content": { "application/json": { "schema": { "$ref": "#/components/schemas/WorkspaceMutation" } } } + }, + "responses": { + "400": { "$ref": "#/components/responses/Error400" }, + "200": { "$ref": "#/components/responses/WorkspaceReplay" }, + "401": {"$ref": "#/components/responses/Error401"}, + "403": {"$ref": "#/components/responses/Error403"}, + "404": {"$ref": "#/components/responses/Error404"}, + "406": {"$ref": "#/components/responses/Error406"}, + "409": {"$ref": "#/components/responses/Error409"}, + "422": {"$ref": "#/components/responses/Error422"}, + "503": {"$ref": "#/components/responses/Error503"} + } + }, + "delete": { + "operationId": "deleteWorkspace", + "tags": ["Workspaces"], + "parameters": [ + { "$ref": "#/components/parameters/ApiVersion" }, + { "$ref": "#/components/parameters/IdempotencyKey" } + ], + "responses": { + "204": { "$ref": "#/components/responses/Deleted" }, + "400": { "$ref": "#/components/responses/Error400" }, + "401": {"$ref": "#/components/responses/Error401"}, + "403": {"$ref": "#/components/responses/Error403"}, + "404": {"$ref": "#/components/responses/Error404"}, + "406": {"$ref": "#/components/responses/Error406"}, + "409": {"$ref": "#/components/responses/Error409"}, + "503": {"$ref": "#/components/responses/Error503"} + } + } + }, + "/v1/workspaces/{workspaceId}/sessions": { + "parameters": [{ "$ref": "#/components/parameters/WorkspaceId" }], + "get": { + "operationId": "listSessions", + "tags": ["Sessions"], + "parameters": [ + { "$ref": "#/components/parameters/ApiVersion" }, + { "$ref": "#/components/parameters/PageSize" }, + { "$ref": "#/components/parameters/PageCursor" } + ], + "responses": { + "200": { "$ref": "#/components/responses/SessionPage" }, + "401": {"$ref": "#/components/responses/Error401"}, + "403": {"$ref": "#/components/responses/Error403"}, + "404": {"$ref": "#/components/responses/Error404"}, + "406": {"$ref": "#/components/responses/Error406"}, + "422": {"$ref": "#/components/responses/Error422"}, + "503": {"$ref": "#/components/responses/Error503"} + } + }, + "post": { + "operationId": "spawnSession", + "tags": ["Sessions"], + "parameters": [ + { "$ref": "#/components/parameters/ApiVersion" }, + { "$ref": "#/components/parameters/IdempotencyKey" } + ], + "requestBody": { + "required": true, + "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SessionCreate" } } } + }, + "responses": { + "200": { "$ref": "#/components/responses/SessionReplay" }, + "202": { "$ref": "#/components/responses/SessionAccepted" }, + "400": { "$ref": "#/components/responses/Error400" }, + "401": {"$ref": "#/components/responses/Error401"}, + "403": {"$ref": "#/components/responses/Error403"}, + "404": {"$ref": "#/components/responses/Error404"}, + "406": {"$ref": "#/components/responses/Error406"}, + "409": {"$ref": "#/components/responses/Error409"}, + "422": {"$ref": "#/components/responses/Error422"}, + "503": {"$ref": "#/components/responses/Error503"} + } + } + }, + "/v1/sessions/{sessionId}": { + "parameters": [{ "$ref": "#/components/parameters/SessionId" }], + "get": { + "operationId": "getSession", + "tags": ["Sessions"], + "parameters": [{ "$ref": "#/components/parameters/ApiVersion" }], + "responses": { + "200": { "$ref": "#/components/responses/Session" }, + "401": {"$ref": "#/components/responses/Error401"}, + "403": {"$ref": "#/components/responses/Error403"}, + "404": {"$ref": "#/components/responses/Error404"}, + "406": {"$ref": "#/components/responses/Error406"}, + "503": {"$ref": "#/components/responses/Error503"} + } + }, + "patch": { + "operationId": "mutateSession", + "tags": ["Sessions"], + "parameters": [ + { "$ref": "#/components/parameters/ApiVersion" }, + { "$ref": "#/components/parameters/IfMatch" }, + { "$ref": "#/components/parameters/IdempotencyKey" } + ], + "requestBody": { + "required": true, + "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SessionMutation" } } } + }, + "responses": { + "400": { "$ref": "#/components/responses/Error400" }, + "200": { "$ref": "#/components/responses/SessionReplay" }, + "401": {"$ref": "#/components/responses/Error401"}, + "403": {"$ref": "#/components/responses/Error403"}, + "404": {"$ref": "#/components/responses/Error404"}, + "406": {"$ref": "#/components/responses/Error406"}, + "409": {"$ref": "#/components/responses/Error409"}, + "422": {"$ref": "#/components/responses/Error422"}, + "503": {"$ref": "#/components/responses/Error503"} + } + }, + "delete": { + "operationId": "deleteSession", + "tags": ["Sessions"], + "parameters": [ + { "$ref": "#/components/parameters/ApiVersion" }, + { "$ref": "#/components/parameters/IdempotencyKey" } + ], + "responses": { + "204": { "$ref": "#/components/responses/Deleted" }, + "400": { "$ref": "#/components/responses/Error400" }, + "401": {"$ref": "#/components/responses/Error401"}, + "403": {"$ref": "#/components/responses/Error403"}, + "404": {"$ref": "#/components/responses/Error404"}, + "406": {"$ref": "#/components/responses/Error406"}, + "409": {"$ref": "#/components/responses/Error409"}, + "503": {"$ref": "#/components/responses/Error503"} + } + } + }, + "/v1/sessions/{sessionId}/cancel": { + "post": { + "operationId": "cancelSession", + "tags": ["Sessions"], + "parameters": [ + { "$ref": "#/components/parameters/SessionId" }, + { "$ref": "#/components/parameters/ApiVersion" }, + { "$ref": "#/components/parameters/IdempotencyKey" } + ], + "responses": { + "200": { "$ref": "#/components/responses/SessionReplay" }, + "400": { "$ref": "#/components/responses/Error400" }, + "202": { "$ref": "#/components/responses/SessionAccepted" }, + "401": {"$ref": "#/components/responses/Error401"}, + "403": {"$ref": "#/components/responses/Error403"}, + "404": {"$ref": "#/components/responses/Error404"}, + "406": {"$ref": "#/components/responses/Error406"}, + "409": {"$ref": "#/components/responses/Error409"}, + "503": {"$ref": "#/components/responses/Error503"} + } + } + }, + "/v1/sessions/{sessionId}/commands": { + "post": { + "operationId": "submitCommand", + "tags": ["Sessions"], + "parameters": [ + { "$ref": "#/components/parameters/SessionId" }, + { "$ref": "#/components/parameters/ApiVersion" }, + { "$ref": "#/components/parameters/IdempotencyKey" } + ], + "requestBody": { + "required": true, + "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CommandCreate" } } } + }, + "responses": { + "200": { "$ref": "#/components/responses/CommandReplay" }, + "202": { "$ref": "#/components/responses/CommandAccepted" }, + "400": { "$ref": "#/components/responses/Error400" }, + "401": {"$ref": "#/components/responses/Error401"}, + "403": {"$ref": "#/components/responses/Error403"}, + "404": {"$ref": "#/components/responses/Error404"}, + "406": {"$ref": "#/components/responses/Error406"}, + "409": {"$ref": "#/components/responses/Error409"}, + "422": {"$ref": "#/components/responses/Error422"}, + "503": {"$ref": "#/components/responses/Error503"} + } + } + }, + "/v1/sessions/{sessionId}/snapshot": { + "get": { + "operationId": "getSessionSnapshot", + "tags": ["Watch"], + "parameters": [ + { "$ref": "#/components/parameters/SessionId" }, + { "$ref": "#/components/parameters/ApiVersion" } + ], + "responses": { + "200": { "$ref": "#/components/responses/Snapshot" }, + "401": {"$ref": "#/components/responses/Error401"}, + "403": {"$ref": "#/components/responses/Error403"}, + "404": {"$ref": "#/components/responses/Error404"}, + "406": {"$ref": "#/components/responses/Error406"}, + "503": {"$ref": "#/components/responses/Error503"} + } + } + }, + "/v1/sessions/{sessionId}/events": { + "get": { + "operationId": "watchSessionEvents", + "tags": ["Watch"], + "summary": "Watch a bounded Server-Sent Events stream over HTTPS", + "description": "The response is standard Server-Sent Events over HTTPS. Clients reconnect with the last event cursor in the cursor query parameter or Last-Event-ID header. Heartbeat events bound idle detection. Clients explicitly cancel by aborting the HTTPS request. A retained cursor expiry returns a typed 410 response with a snapshot resync target.", + "parameters": [ + { "$ref": "#/components/parameters/SessionId" }, + { "$ref": "#/components/parameters/ApiVersion" }, + { "$ref": "#/components/parameters/WatchCursor" }, + { "$ref": "#/components/parameters/LastEventId" }, + { "$ref": "#/components/parameters/MaxEvents" }, + { "$ref": "#/components/parameters/HeartbeatSeconds" } + ], + "responses": { + "200": { + "description": "Bounded SSE event stream. Every non-empty SSE data field is one JSON value conforming to WatchEvent; clients MUST reject unknown fields and schema-invalid event payloads before delivery. Transport chunk boundaries do not delimit frames.", + "headers": { + "Cache-Control": { "required": true, "schema": { "type": "string", "const": "no-store" } }, + "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } + }, + "content": { + "text/event-stream": { + "schema": { "type": "string", "description": "Server-Sent Events framing whose data values conform to #/components/schemas/WatchEvent", "x-t4-sse-data-schema": "#/components/schemas/WatchEvent" }, + "example": "id: cur_01J...\\nevent: heartbeat\\ndata: {\"type\":\"heartbeat\",\"cursor\":\"cur_01J...\",\"observedAt\":\"2026-07-21T00:00:00Z\"}\\n\\n" + } + } + }, + "400": {"$ref": "#/components/responses/Error400"}, + "401": {"$ref": "#/components/responses/Error401"}, + "403": {"$ref": "#/components/responses/Error403"}, + "404": {"$ref": "#/components/responses/Error404"}, + "406": {"$ref": "#/components/responses/Error406"}, + "410": {"$ref": "#/components/responses/Error410"}, + "422": {"$ref": "#/components/responses/Error422"}, + "503": {"$ref": "#/components/responses/Error503"} + } + } + } + }, + "components": { + "securitySchemes": { + "BearerAuth": { + "type": "http", + "scheme": "bearer", + "description": "Opaque bearer credential. The server maps it to deny-by-default tenant and resource scopes; clients must not inspect or persist credential contents in API resources." + } + }, + "parameters": { + "ApiVersion": { + "name": "T4-API-Version", + "in": "header", + "required": true, + "description": "Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading.", + "schema": { "type": "string", "pattern": "^[1-9][0-9]{0,3}(?:\\.[0-9]+)?$", "maxLength": 16 } + }, + "IdempotencyKey": { + "name": "Idempotency-Key", + "in": "header", + "required": true, + "description": "Opaque caller-generated key scoped by the server to the authenticated principal, operation ID, and target resource IDs. Idempotency is evaluated only after path, query, header, and request-schema validation and application of schema defaults. Request identity is the operation ID, target IDs, relevant precondition headers, and the RFC 8785 JSON Canonicalization Scheme (JCS) bytes of the validated JSON body. JCS object member order is insignificant, array order is significant, omitted fields remain distinct unless the request schema defaulted them, and no Unicode normalization is performed beyond RFC 8785. Reusing a key with identical identity replays the original terminal response and advertises Idempotency-Replayed; reusing it with a different identity returns 409 idempotency_conflict.", + "schema": { "type": "string", "minLength": 16, "maxLength": 128, "pattern": "^[A-Za-z0-9._~-]+$" } + }, + "IfMatch": { + "name": "If-Match", + "in": "header", + "required": true, + "description": "Decimal resource revision for optimistic mutation.", + "schema": { "type": "string", "pattern": "^[1-9][0-9]{0,18}$" } + }, + "WorkspaceId": { + "name": "workspaceId", + "in": "path", + "required": true, + "schema": { "$ref": "#/components/schemas/ResourceId" } + }, + "SessionId": { + "name": "sessionId", + "in": "path", + "required": true, + "schema": { "$ref": "#/components/schemas/ResourceId" } + }, + "PageSize": { + "name": "pageSize", + "in": "query", + "schema": { "type": "integer", "minimum": 1, "maximum": 100, "default": 25 } + }, + "PageCursor": { + "name": "cursor", + "in": "query", + "description": "Opaque page cursor, valid only for the same identity and list operation.", + "schema": { "$ref": "#/components/schemas/Cursor" } + }, + "WatchCursor": { + "name": "cursor", + "in": "query", + "description": "Opaque last-consumed watch event cursor.", + "schema": { "$ref": "#/components/schemas/Cursor" } + }, + "LastEventId": { + "name": "Last-Event-ID", + "in": "header", + "description": "Standard SSE reconnect header. Must agree with cursor when both are present.", + "schema": { "$ref": "#/components/schemas/Cursor" } + }, + "MaxEvents": { + "name": "maxEvents", + "in": "query", + "schema": { "type": "integer", "minimum": 1, "maximum": 1000, "default": 100 } + }, + "HeartbeatSeconds": { + "name": "heartbeatSeconds", + "in": "query", + "schema": { "type": "integer", "minimum": 5, "maximum": 60, "default": 15 } + } + }, + "headers": { + "SelectedVersion": { + "description": "Selected compatible T4 API minor version.", + "required": true, + "schema": { "type": "string", "pattern": "^1\\.[0-9]+$", "maxLength": 16 } + }, + "IdempotencyReplayed": { + "description": "True when this response replays a prior identical request.", + "required": true, + "schema": { "type": "string", "enum": ["true", "false"] } + }, + "EventCursor": { + "description": "Durable event cursor committed with the accepted mutation and preserved by replay.", + "required": true, + "schema": { "$ref": "#/components/schemas/Cursor" } + } + }, + "schemas": { + "ResourceId": { "type": "string", "minLength": 1, "maxLength": 128, "pattern": "^[A-Za-z0-9][A-Za-z0-9._~-]*$" }, + "Cursor": { "type": "string", "minLength": 1, "maxLength": 512, "pattern": "^[A-Za-z0-9._~-]+$", "description": "Opaque server-issued header-safe SSE cursor." }, + "Revision": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 }, + "CommandState": { "type": "string", "enum": ["accepted", "projected", "dispatching", "running", "succeeded", "failed", "cancelling", "cancelled", "rejected", "unavailable", "indeterminate"] }, + "WorkspaceState": { "type": "string", "enum": ["accepted", "provisioning", "ready", "deleting", "deleted", "failed", "unavailable", "indeterminate"] }, + "SessionState": { "type": "string", "enum": ["accepted", "provisioning", "ready", "cancelling", "cancelled", "failed", "unavailable", "indeterminate"] }, + "ServerBuild": { + "type": "object", + "additionalProperties": false, + "required": ["version", "revision"], + "properties": { + "version": { "type": "string", "minLength": 1, "maxLength": 128 }, + "revision": { "type": "string", "minLength": 1, "maxLength": 128 } + } + }, + "CapabilityDeprecation": { + "type": "object", + "additionalProperties": false, + "required": ["message"], + "properties": { + "message": { "type": "string", "minLength": 1, "maxLength": 1024 }, + "sinceVersion": { "type": "string", "minLength": 1, "maxLength": 128 }, + "sunsetAt": { "type": "string", "format": "date-time", "maxLength": 64 }, + "replacement": { "type": "string", "minLength": 1, "maxLength": 128, "pattern": "^[a-z][a-z0-9.-]*$" } + } + }, + "CapabilityStatus": { + "type": "object", + "additionalProperties": false, + "required": ["supported", "enabled", "authorized", "available"], + "properties": { + "supported": { "type": "boolean" }, + "enabled": { "type": "boolean" }, + "authorized": { "type": "boolean" }, + "available": { "type": "boolean" }, + "deprecation": { "$ref": "#/components/schemas/CapabilityDeprecation" } + } + }, + "Capabilities": { + "type": "object", + "maxProperties": 128, + "propertyNames": { "minLength": 1, "maxLength": 128, "pattern": "^[a-z][a-z0-9.-]*$" }, + "additionalProperties": { "$ref": "#/components/schemas/CapabilityStatus" } + }, + "Discovery": { + "type": "object", + "additionalProperties": false, + "required": ["apiVersion", "serverBuild", "supportedMajors", "capabilities", "limits"], + "properties": { + "apiVersion": { "type": "string", "pattern": "^1\\.[0-9]+$" }, + "serverBuild": { "$ref": "#/components/schemas/ServerBuild" }, + "supportedMajors": { "type": "array", "minItems": 1, "maxItems": 8, "uniqueItems": true, "items": { "type": "integer", "minimum": 1 } }, + "capabilities": { "$ref": "#/components/schemas/Capabilities" }, + "limits": { + "type": "object", + "additionalProperties": false, + "required": ["pageSizeDefault", "pageSizeMax", "commandBytesMax", "commandRequestBytesMax", "commandMetadataValueBytesMax", "watchEventsMax", "heartbeatSeconds"], + "properties": { + "pageSizeDefault": { "type": "integer", "minimum": 1, "maximum": 100 }, + "pageSizeMax": { "type": "integer", "minimum": 1, "maximum": 100 }, + "commandBytesMax": { "type": "integer", "minimum": 1, "maximum": 262144 }, + "commandRequestBytesMax": { "type": "integer", "minimum": 1, "maximum": 1048576 }, + "commandMetadataValueBytesMax": { "type": "integer", "minimum": 1, "maximum": 262144 }, + "watchEventsMax": { "type": "integer", "minimum": 1, "maximum": 1000 }, + "heartbeatSeconds": { "type": "integer", "minimum": 5, "maximum": 60 } + } + } + } + }, + "WorkspaceCreate": { + "type": "object", + "additionalProperties": false, + "required": ["name"], + "properties": { + "name": { "type": "string", "minLength": 1, "maxLength": 128 }, + "labels": { "$ref": "#/components/schemas/Labels" } + } + }, + "WorkspaceMutation": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "name": { "type": "string", "minLength": 1, "maxLength": 128 }, + "labels": { "$ref": "#/components/schemas/Labels" } + } + }, + "Workspace": { + "type": "object", + "additionalProperties": false, + "required": ["id", "name", "state", "revision"], + "properties": { + "id": { "$ref": "#/components/schemas/ResourceId" }, + "name": { "type": "string", "minLength": 1, "maxLength": 128 }, + "state": { "$ref": "#/components/schemas/WorkspaceState" }, + "revision": { "$ref": "#/components/schemas/Revision" }, + "labels": { "$ref": "#/components/schemas/Labels" } + } + }, + "WorkspacePage": { + "type": "object", + "additionalProperties": false, + "required": ["items"], + "properties": { + "items": { "type": "array", "maxItems": 100, "items": { "$ref": "#/components/schemas/Workspace" } }, + "nextCursor": { "$ref": "#/components/schemas/Cursor" } + } + }, + "SessionCreate": { + "type": "object", + "additionalProperties": false, + "required": ["title"], + "properties": { + "title": { "type": "string", "minLength": 1, "maxLength": 128 }, + "labels": { "$ref": "#/components/schemas/Labels" } + } + }, + "SessionMutation": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "title": { "type": "string", "minLength": 1, "maxLength": 128 }, + "labels": { "$ref": "#/components/schemas/Labels" } + } + }, + "Session": { + "type": "object", + "additionalProperties": false, + "required": ["id", "workspaceId", "title", "state", "revision"], + "properties": { + "id": { "$ref": "#/components/schemas/ResourceId" }, + "workspaceId": { "$ref": "#/components/schemas/ResourceId" }, + "title": { "type": "string", "minLength": 1, "maxLength": 128 }, + "state": { "$ref": "#/components/schemas/SessionState" }, + "revision": { "$ref": "#/components/schemas/Revision" }, + "labels": { "$ref": "#/components/schemas/Labels" } + } + }, + "SessionPage": { + "type": "object", + "additionalProperties": false, + "required": ["items"], + "properties": { + "items": { "type": "array", "maxItems": 100, "items": { "$ref": "#/components/schemas/Session" } }, + "nextCursor": { "$ref": "#/components/schemas/Cursor" } + } + }, + "Labels": { + "type": "object", + "maxProperties": 32, + "propertyNames": { "pattern": "^[a-z][a-z0-9.-]{0,62}$" }, + "additionalProperties": { "type": "string", "maxLength": 128 } + }, + "CommandCreate": { + "type": "object", + "additionalProperties": false, + "required": ["command"], + "x-t4-maxUtf8Bytes": 1048576, + "properties": { + "command": { "type": "string", "minLength": 1, "maxLength": 262144, "x-t4-maxUtf8Bytes": 262144 }, + "metadata": { + "type": "object", + "default": {}, + "maxProperties": 32, + "propertyNames": { "pattern": "^[a-z][a-z0-9.-]{0,62}$" }, + "additionalProperties": { + "oneOf": [ + { "type": "string", "maxLength": 262144, "x-t4-maxUtf8Bytes": 262144 }, + { "type": "integer", "minimum": -9007199254740991, "maximum": 9007199254740991 }, + { "type": "boolean" }, + { "type": "null" } + ] + } + } + } + }, + "CommandResult": { + "type": "object", + "additionalProperties": false, + "required": ["commandId", "state"], + "properties": { + "commandId": { "$ref": "#/components/schemas/ResourceId" }, + "state": { "$ref": "#/components/schemas/CommandState" } + } + }, + "SnapshotEntry": { + "type": "object", + "additionalProperties": false, + "required": ["sequence", "kind", "text"], + "properties": { + "sequence": { "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, + "kind": { "type": "string", "enum": ["input", "output", "status"] }, + "text": { "type": "string", "maxLength": 1048576 } + } + }, + "SessionSnapshot": { + "type": "object", + "additionalProperties": false, + "x-t4-maxUtf8Bytes": 16777216, + "required": ["sessionId", "cursor", "state", "entries"], + "properties": { + "sessionId": { "$ref": "#/components/schemas/ResourceId" }, + "cursor": { "$ref": "#/components/schemas/Cursor" }, + "state": { "$ref": "#/components/schemas/SessionState" }, + "entries": { "type": "array", "maxItems": 1000, "items": { "$ref": "#/components/schemas/SnapshotEntry" } } + } + }, + "HeartbeatWatchEvent": { + "type": "object", + "additionalProperties": false, + "required": ["type", "cursor", "observedAt"], + "properties": { + "type": { "type": "string", "const": "heartbeat" }, + "cursor": { "$ref": "#/components/schemas/Cursor" }, + "observedAt": { "type": "string", "format": "date-time", "maxLength": 64 } + } + }, + "SessionWatchEvent": { + "type": "object", + "additionalProperties": false, + "required": ["type", "cursor", "state", "revision"], + "properties": { + "type": { "type": "string", "const": "session" }, + "cursor": { "$ref": "#/components/schemas/Cursor" }, + "state": { "$ref": "#/components/schemas/SessionState" }, + "revision": { "$ref": "#/components/schemas/Revision" } + } + }, + "CommandWatchEvent": { + "type": "object", + "additionalProperties": false, + "required": ["type", "cursor", "commandId", "state"], + "properties": { + "type": { "type": "string", "const": "command" }, + "cursor": { "$ref": "#/components/schemas/Cursor" }, + "commandId": { "$ref": "#/components/schemas/ResourceId" }, + "state": { "$ref": "#/components/schemas/CommandState" } + } + }, + "WatchEvent": { + "oneOf": [ + { "$ref": "#/components/schemas/HeartbeatWatchEvent" }, + { "$ref": "#/components/schemas/SessionWatchEvent" }, + { "$ref": "#/components/schemas/CommandWatchEvent" } + ] + }, + "FieldViolation": { + "type": "object", + "additionalProperties": false, + "required": ["field", "rule", "message"], + "properties": { + "field": { "type": "string", "minLength": 1, "maxLength": 256 }, + "rule": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[A-Za-z][A-Za-z0-9._-]*$" }, + "message": { "type": "string", "minLength": 1, "maxLength": 512 } + } + }, + "Resync": { + "type": "object", + "additionalProperties": false, + "required": ["snapshotUrl", "cursor"], + "properties": { + "snapshotUrl": { "type": "string", "pattern": "^/v1/sessions/[A-Za-z0-9][A-Za-z0-9._~-]{0,127}/snapshot$", "maxLength": 150 }, + "cursor": { "$ref": "#/components/schemas/Cursor" } + } + }, + "ApiError": { + "type": "object", + "additionalProperties": false, + "required": ["code", "message", "requestId", "retryable"], + "properties": { + "code": { + "type": "string", + "enum": ["invalid_request", "unauthenticated", "forbidden", "not_found", "idempotency_key_required", "idempotency_conflict", "revision_conflict", "incompatible_version", "cursor_expired", "unavailable", "indeterminate", "invalid_origin", "https_required"] + }, + "message": { "type": "string", "minLength": 1, "maxLength": 1024 }, + "requestId": { "type": "string", "minLength": 1, "maxLength": 128 }, + "retryable": { "type": "boolean" }, + "violations": { "type": "array", "maxItems": 64, "items": { "$ref": "#/components/schemas/FieldViolation" } }, + "supportedMajors": { "type": "array", "maxItems": 8, "uniqueItems": true, "items": { "type": "integer", "minimum": 1 } }, + "resync": { "$ref": "#/components/schemas/Resync" } + } + }, + "BadRequestApiError": { + "allOf": [ + { "$ref": "#/components/schemas/ApiError" }, + { "type": "object", "properties": { "code": { "enum": ["invalid_request", "idempotency_key_required", "invalid_origin", "https_required"] } } } + ] + }, + "UnauthenticatedApiError": { + "allOf": [ + { "$ref": "#/components/schemas/ApiError" }, + { "type": "object", "properties": { "code": { "const": "unauthenticated" } } } + ] + }, + "ForbiddenApiError": { + "allOf": [ + { "$ref": "#/components/schemas/ApiError" }, + { "type": "object", "properties": { "code": { "const": "forbidden" } } } + ] + }, + "NotFoundApiError": { + "allOf": [ + { "$ref": "#/components/schemas/ApiError" }, + { "type": "object", "properties": { "code": { "const": "not_found" } } } + ] + }, + "ConflictApiError": { + "allOf": [ + { "$ref": "#/components/schemas/ApiError" }, + { "type": "object", "properties": { "code": { "enum": ["idempotency_conflict", "revision_conflict"] } } } + ] + }, + "UnavailableApiError": { + "allOf": [ + { "$ref": "#/components/schemas/ApiError" }, + { "type": "object", "properties": { "code": { "enum": ["unavailable", "indeterminate"] } } } + ] + }, + "IncompatibleVersionApiError": { + "allOf": [ + { "$ref": "#/components/schemas/ApiError" }, + { "type": "object", "required": ["supportedMajors"], "properties": { "code": { "const": "incompatible_version" }, "supportedMajors": { "type": "array", "minItems": 1, "maxItems": 8, "uniqueItems": true, "items": { "type": "integer", "minimum": 1 } } } } + ] + }, + "CursorExpiredApiError": { + "allOf": [ + { "$ref": "#/components/schemas/ApiError" }, + { "type": "object", "required": ["resync"], "properties": { "code": { "const": "cursor_expired" }, "resync": { "$ref": "#/components/schemas/Resync" } } } + ] + }, + "InvalidRequestApiError": { + "allOf": [ + { "$ref": "#/components/schemas/ApiError" }, + { "type": "object", "required": ["violations"], "properties": { "code": { "const": "invalid_request" }, "violations": { "type": "array", "minItems": 1, "maxItems": 64, "items": { "$ref": "#/components/schemas/FieldViolation" } } } } + ] + }, + "ErrorEnvelope": { + "type": "object", + "additionalProperties": false, + "required": ["error"], + "properties": { "error": { "$ref": "#/components/schemas/ApiError" } } + }, + "BadRequestErrorEnvelope": { + "type": "object", + "additionalProperties": false, + "required": ["error"], + "properties": { "error": { "$ref": "#/components/schemas/BadRequestApiError" } } + }, + "UnauthenticatedErrorEnvelope": { + "type": "object", + "additionalProperties": false, + "required": ["error"], + "properties": { "error": { "$ref": "#/components/schemas/UnauthenticatedApiError" } } + }, + "ForbiddenErrorEnvelope": { + "type": "object", + "additionalProperties": false, + "required": ["error"], + "properties": { "error": { "$ref": "#/components/schemas/ForbiddenApiError" } } + }, + "NotFoundErrorEnvelope": { + "type": "object", + "additionalProperties": false, + "required": ["error"], + "properties": { "error": { "$ref": "#/components/schemas/NotFoundApiError" } } + }, + "ConflictErrorEnvelope": { + "type": "object", + "additionalProperties": false, + "required": ["error"], + "properties": { "error": { "$ref": "#/components/schemas/ConflictApiError" } } + }, + "UnavailableErrorEnvelope": { + "type": "object", + "additionalProperties": false, + "required": ["error"], + "properties": { "error": { "$ref": "#/components/schemas/UnavailableApiError" } } + }, + "IncompatibleVersionErrorEnvelope": { + "type": "object", + "additionalProperties": false, + "required": ["error"], + "properties": { "error": { "$ref": "#/components/schemas/IncompatibleVersionApiError" } } + }, + "CursorExpiredErrorEnvelope": { + "type": "object", + "additionalProperties": false, + "required": ["error"], + "properties": { "error": { "$ref": "#/components/schemas/CursorExpiredApiError" } } + }, + "InvalidRequestErrorEnvelope": { + "type": "object", + "additionalProperties": false, + "required": ["error"], + "properties": { "error": { "$ref": "#/components/schemas/InvalidRequestApiError" } } + } + }, + "responses": { + "Discovery": { "description": "Negotiated v1 discovery", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Discovery" } } } }, + "Workspace": { "description": "Workspace", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Workspace" } } } }, + "WorkspaceAccepted": { "description": "Workspace intent accepted", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" }, "T4-Event-Cursor": { "$ref": "#/components/headers/EventCursor" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Workspace" } } } }, + "WorkspaceReplay": { "description": "Identical workspace request replay", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" }, "T4-Event-Cursor": { "$ref": "#/components/headers/EventCursor" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Workspace" } } } }, + "WorkspacePage": { "description": "Bounded workspace page", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/WorkspacePage" } } } }, + "Session": { "description": "Session", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Session" } } } }, + "SessionAccepted": { "description": "Session intent accepted", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" }, "T4-Event-Cursor": { "$ref": "#/components/headers/EventCursor" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Session" } } } }, + "SessionReplay": { "description": "Identical session mutation replay", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" }, "T4-Event-Cursor": { "$ref": "#/components/headers/EventCursor" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Session" } } } }, + "SessionPage": { "description": "Bounded session page", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SessionPage" } } } }, + "CommandAccepted": { "description": "Command intent accepted with a stable outcome state", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" }, "T4-Event-Cursor": { "$ref": "#/components/headers/EventCursor" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CommandResult" } } } }, + "CommandReplay": { "description": "Identical command request replay", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" }, "T4-Event-Cursor": { "$ref": "#/components/headers/EventCursor" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CommandResult" } } } }, + "Snapshot": { "description": "Bounded session snapshot and reconnect cursor", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SessionSnapshot" } } } }, + "Deleted": { "description": "Idempotent deletion accepted or resource already absent", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" }, "T4-Event-Cursor": { "$ref": "#/components/headers/EventCursor" } } }, + "Error400": { "description": "Malformed request or missing idempotency key", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/BadRequestErrorEnvelope" } } } }, + "Error401": { "description": "Missing or invalid opaque bearer credential", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/UnauthenticatedErrorEnvelope" } } } }, + "Error403": { "description": "Credential lacks the deny-by-default operation or resource scope", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ForbiddenErrorEnvelope" } } } }, + "Error404": { "description": "Resource absent or outside caller scope (scope existence is not disclosed)", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/NotFoundErrorEnvelope" } } } }, + "Error406": { "description": "Requested major is incompatible; no silent downgrade", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/IncompatibleVersionErrorEnvelope" } } } }, + "Error409": { "description": "Idempotency or resource revision conflict", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ConflictErrorEnvelope" } } } }, + "Error410": { "description": "Watch cursor expired; resync from the typed snapshot target", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CursorExpiredErrorEnvelope" } } } }, + "Error422": { "description": "Bounded semantic validation failure with stable field violations", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/InvalidRequestErrorEnvelope" } } } }, + "Error503": { "description": "Service temporarily unavailable or outcome indeterminate. Retryable watch failures may advertise Retry-After; clients honor it with a 30 second ceiling.", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Retry-After": { "description": "RFC 9110 delay in seconds or HTTP date; clients bound the applied delay.", "schema": { "type": "string", "minLength": 1, "maxLength": 128 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/UnavailableErrorEnvelope" } } } } + } + } +} diff --git a/packages/t4-api-contract/package.json b/packages/t4-api-contract/package.json new file mode 100644 index 00000000..641b913d --- /dev/null +++ b/packages/t4-api-contract/package.json @@ -0,0 +1,22 @@ +{ + "name": "@t4-code/t4-api-contract", + "version": "0.1.30", + "private": true, + "type": "module", + "exports": { + "./openapi.json": "./openapi.json" + }, + "scripts": { + "build": "pnpm validate && pnpm check:generated", + "typecheck": "pnpm validate && pnpm check:generated", + "test": "pnpm validate && pnpm check:generated", + "validate": "node scripts/validate.mjs", + "generate": "node scripts/generate.mjs --write", + "generate:ci": "node scripts/generate.mjs --ci-artifact", + "check:generated": "node scripts/generate.mjs --check" + }, + "devDependencies": { + "@readme/openapi-parser": "6.3.0", + "openapi-typescript": "7.13.0" + } +} diff --git a/packages/t4-api-contract/scripts/generate.mjs b/packages/t4-api-contract/scripts/generate.mjs new file mode 100644 index 00000000..e216f71a --- /dev/null +++ b/packages/t4-api-contract/scripts/generate.mjs @@ -0,0 +1,44 @@ +import { createHash } from "node:crypto"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import openapiTS, { astToString } from "openapi-typescript"; + +const contractRoot = fileURLToPath(new URL("..", import.meta.url)); +const schemaUrl = new URL("../openapi.json", import.meta.url); +const target = resolve(contractRoot, "../t4-api-client/src/generated/schema.ts"); +const mode = process.argv[2]; +if (!new Set(["--write", "--check", "--ci-artifact"]).has(mode)) { + throw new Error("usage: node scripts/generate.mjs --write|--check|--ci-artifact"); +} + +const generated = `${astToString(await openapiTS(schemaUrl, { alphabetize: true, defaultNonNullable: false }))}\n`; +const digest = createHash("sha256").update(generated).digest("hex"); + +if (mode === "--write") { + await mkdir(dirname(target), { recursive: true }); + await writeFile(target, generated); + console.log(`generated ${target} sha256:${digest}`); + process.exit(0); +} + +if (mode === "--ci-artifact") { + const artifact = resolve(contractRoot, "../../artifacts/t4-api/generated/schema.ts"); + await mkdir(dirname(artifact), { recursive: true }); + await writeFile(artifact, generated); + console.log(`T4_API_GENERATED_SHA256=${digest}`); + console.log(`T4_API_GENERATED_BASE64_BEGIN\n${Buffer.from(generated).toString("base64")}\nT4_API_GENERATED_BASE64_END`); +} + +let checkedIn; +try { + checkedIn = await readFile(target, "utf8"); +} catch (error) { + if (mode === "--ci-artifact") throw new Error(`generated client is absent; recover the CI artifact at artifacts/t4-api/generated/schema.ts (sha256:${digest})`, { cause: error }); + throw error; +} +if (checkedIn !== generated) { + throw new Error(`generated client drift: expected sha256:${digest}; regenerate only with the authorized CI artifact`); +} +console.log(`generated client is deterministic (sha256:${digest})`); diff --git a/packages/t4-api-contract/scripts/validate.mjs b/packages/t4-api-contract/scripts/validate.mjs new file mode 100644 index 00000000..b7eb134a --- /dev/null +++ b/packages/t4-api-contract/scripts/validate.mjs @@ -0,0 +1,75 @@ +import { readFile } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; + +import { compileErrors, validate } from "@readme/openapi-parser"; + +const source = new URL("../openapi.json", import.meta.url); +const document = JSON.parse(await readFile(source, "utf8")); +const validation = await validate(fileURLToPath(source)); +if (!validation.valid || validation.warnings.length > 0) throw new Error(compileErrors(validation)); + +if (document.openapi !== "3.1.0") throw new Error("T4 API contract must remain OpenAPI 3.1.0"); +if (!Array.isArray(document.servers) || document.servers.length === 0) throw new Error("T4 API contract requires an HTTPS server"); +for (const server of document.servers) { + if (new URL(server.url).protocol !== "https:") throw new Error("T4 API contract permits only HTTPS servers"); +} +if (document.security?.[0]?.BearerAuth === undefined) throw new Error("T4 API contract must default to bearer authentication"); +const sseSchema = document.paths?.["/v1/sessions/{sessionId}/events"]?.get?.responses?.["200"]?.content?.["text/event-stream"]?.schema; +if (sseSchema?.["x-t4-sse-data-schema"] !== "#/components/schemas/WatchEvent") throw new Error("watch SSE data must remain linked to WatchEvent"); +const schemas = document.components?.schemas ?? {}; +const commandCreate = schemas.CommandCreate; +if (commandCreate?.["x-t4-maxUtf8Bytes"] !== 1048576) throw new Error("CommandCreate must retain its UTF-8 request-byte bound"); +if (commandCreate?.properties?.command?.["x-t4-maxUtf8Bytes"] !== 262144) throw new Error("command must retain its UTF-8 byte bound"); +const metadataString = commandCreate?.properties?.metadata?.additionalProperties?.oneOf?.find((item) => item?.type === "string"); +if (metadataString?.["x-t4-maxUtf8Bytes"] !== 262144) throw new Error("command metadata strings must retain their UTF-8 byte bound"); +if (schemas.SessionSnapshot?.["x-t4-maxUtf8Bytes"] !== 16777216) throw new Error("SessionSnapshot must retain its aggregate UTF-8 response-byte bound"); + +for (const [schema, property] of [["Discovery", "supportedMajors"], ["ApiError", "supportedMajors"]]) { + if (schemas[schema]?.properties?.[property]?.uniqueItems !== true) throw new Error(`${schema}.${property} must retain uniqueItems`); +} +if (schemas.Discovery?.required?.includes("serverBuild") !== true || schemas.Discovery?.properties?.capabilities?.$ref !== "#/components/schemas/Capabilities") throw new Error("Discovery must require structured server build and capabilities"); +if (schemas.Capabilities?.maxProperties !== 128 || schemas.Capabilities?.additionalProperties?.$ref !== "#/components/schemas/CapabilityStatus") throw new Error("Capabilities must remain a bounded status map"); + +const selectedVersionHeader = document.components?.headers?.SelectedVersion; +const replayHeaderObject = document.components?.headers?.IdempotencyReplayed; +const eventCursorHeader = document.components?.headers?.EventCursor; +if (selectedVersionHeader?.required !== true) throw new Error("SelectedVersion must remain required wherever referenced"); +if (replayHeaderObject?.required !== true) throw new Error("IdempotencyReplayed must remain required wherever referenced"); +if (eventCursorHeader?.required !== true || eventCursorHeader?.schema?.$ref !== "#/components/schemas/Cursor") throw new Error("EventCursor must remain a required header-safe cursor"); +const selectedVersion = document.components?.headers?.SelectedVersion?.schema; +if (selectedVersion?.pattern !== "^1\\.[0-9]+$" || selectedVersion?.maxLength !== 16) throw new Error("SelectedVersion must remain a bounded v1 minor"); +const replayHeader = document.components?.headers?.IdempotencyReplayed?.schema; +if (replayHeader?.type !== "string" || JSON.stringify(replayHeader.enum) !== '["true","false"]') throw new Error("IdempotencyReplayed must remain the exact true|false string enum"); +if (schemas.HeartbeatWatchEvent?.properties?.observedAt?.maxLength !== 64) throw new Error("HeartbeatWatchEvent.observedAt must remain bounded to 64 characters"); +const selectedVersionRef = "#/components/headers/SelectedVersion"; +const replayRef = "#/components/headers/IdempotencyReplayed"; +const selectedVersionResponses = ["Discovery", "Workspace", "WorkspaceAccepted", "WorkspaceReplay", "WorkspacePage", "Session", "SessionAccepted", "SessionReplay", "SessionPage", "CommandAccepted", "CommandReplay", "Snapshot", "Deleted", "Error400", "Error403", "Error404", "Error406", "Error409", "Error410", "Error422", "Error503"]; +for (const responseName of selectedVersionResponses) { + if (document.components?.responses?.[responseName]?.headers?.["T4-API-Version"]?.$ref !== selectedVersionRef) throw new Error(`${responseName} must declare SelectedVersion`); +} +const mutationResponses = ["WorkspaceAccepted", "WorkspaceReplay", "SessionAccepted", "SessionReplay", "CommandAccepted", "CommandReplay", "Deleted"]; +for (const responseName of mutationResponses) { + if (document.components?.responses?.[responseName]?.headers?.["Idempotency-Replayed"]?.$ref !== replayRef) throw new Error(`${responseName} must declare IdempotencyReplayed`); + if (document.components?.responses?.[responseName]?.headers?.["T4-Event-Cursor"]?.$ref !== "#/components/headers/EventCursor") throw new Error(`${responseName} must declare EventCursor`); +} +if (document.paths?.["/v1/sessions/{sessionId}/events"]?.get?.responses?.["200"]?.headers?.["T4-API-Version"]?.$ref !== selectedVersionRef) throw new Error("watch success must declare SelectedVersion"); +const watchCacheControl = document.paths?.["/v1/sessions/{sessionId}/events"]?.get?.responses?.["200"]?.headers?.["Cache-Control"]; +if (watchCacheControl?.required !== true || watchCacheControl?.schema?.const !== "no-store") throw new Error("watch success must require Cache-Control: no-store"); +if (document.components?.responses?.Error401?.headers?.["T4-API-Version"] !== undefined) throw new Error("Error401 must omit SelectedVersion"); +if (document.paths?.["/v1/workspaces/{workspaceId}"]?.patch?.responses?.["200"]?.$ref !== "#/components/responses/WorkspaceReplay") throw new Error("workspace PATCH success must declare replay headers"); +if (document.paths?.["/v1/sessions/{sessionId}"]?.patch?.responses?.["200"]?.$ref !== "#/components/responses/SessionReplay") throw new Error("session PATCH success must declare replay headers"); +const errorResponses = { 400: "Error400", 401: "Error401", 403: "Error403", 404: "Error404", 406: "Error406", 409: "Error409", 410: "Error410", 422: "Error422", 503: "Error503" }; +for (const path of Object.values(document.paths ?? {})) { + for (const operation of Object.values(path)) { + if (operation === null || typeof operation !== "object" || operation.responses === undefined) continue; + for (const [status, component] of Object.entries(errorResponses)) { + const response = operation.responses[status]; + if (response !== undefined && response.$ref !== `#/components/responses/${component}`) throw new Error(`${status} responses must use ${component}`); + } + } +} +const serializedSchemas = JSON.stringify(document.components?.schemas ?? {}).toLowerCase(); +for (const privateType of ["kubernetes.io", "v1alpha1", "postgresql", "ompserver", "podspec"]) { + if (serializedSchemas.includes(privateType)) throw new Error(`public schema leaks private type ${privateType}`); +} +console.log("T4 API OpenAPI 3.1 contract is valid"); diff --git a/packages/t4-api-contract/tsconfig.json b/packages/t4-api-contract/tsconfig.json new file mode 100644 index 00000000..9e7ef5aa --- /dev/null +++ b/packages/t4-api-contract/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "types": ["node"] + }, + "include": [] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 36b7b3d1..4ca7ae94 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -261,28 +261,6 @@ importers: specifier: 'catalog:' version: 0.2.2(@types/node@26.1.1)(jiti@2.7.0)(typescript@6.0.3) - packages/cluster-server: - dependencies: - '@t4-code/host-service': - specifier: workspace:* - version: link:../host-service - '@t4-code/host-wire': - specifier: workspace:* - version: link:../host-wire - devDependencies: - '@types/bun': - specifier: 1.3.5 - version: 1.3.5 - '@types/node': - specifier: 24.12.4 - version: 24.12.4 - typescript: - specifier: ~6.0.3 - version: 6.0.3 - vite-plus: - specifier: 0.2.2 - version: 0.2.2(@types/node@24.12.4)(jiti@2.7.0)(typescript@6.0.3) - packages/client: dependencies: '@t4-code/protocol': @@ -295,6 +273,9 @@ importers: '@t4-code/host-wire': specifier: workspace:* version: link:../host-wire + '@t4-code/t4-api-client': + specifier: workspace:* + version: link:../t4-api-client '@types/node': specifier: 'catalog:' version: 24.12.4 @@ -308,6 +289,28 @@ importers: specifier: 8.18.3 version: 8.18.3 + packages/cluster-server: + dependencies: + '@t4-code/host-service': + specifier: workspace:* + version: link:../host-service + '@t4-code/host-wire': + specifier: workspace:* + version: link:../host-wire + devDependencies: + '@types/bun': + specifier: 1.3.5 + version: 1.3.5 + '@types/node': + specifier: 24.12.4 + version: 24.12.4 + typescript: + specifier: ~6.0.3 + version: 6.0.3 + vite-plus: + specifier: 0.2.2 + version: 0.2.2(@types/node@24.12.4)(jiti@2.7.0)(typescript@6.0.3) + packages/fixture-server: dependencies: '@t4-code/protocol': @@ -421,6 +424,31 @@ importers: specifier: 'catalog:' version: 0.2.2(@types/node@24.12.4)(jiti@2.7.0)(typescript@6.0.3) + packages/t4-api-client: + dependencies: + openapi-fetch: + specifier: 0.17.0 + version: 0.17.0 + devDependencies: + '@types/node': + specifier: 'catalog:' + version: 24.12.4 + typescript: + specifier: 'catalog:' + version: 6.0.3 + vite-plus: + specifier: 'catalog:' + version: 0.2.2(@types/node@24.12.4)(jiti@2.7.0)(typescript@6.0.3) + + packages/t4-api-contract: + devDependencies: + '@readme/openapi-parser': + specifier: 6.3.0 + version: 6.3.0(openapi-types@12.1.3) + openapi-typescript: + specifier: 7.13.0 + version: 7.13.0(typescript@6.0.3) + packages/ui: dependencies: '@base-ui/react': @@ -465,6 +493,12 @@ packages: peerDependencies: zod: ^3.25.0 || ^4.0.0 + '@apidevtools/json-schema-ref-parser@14.2.1': + resolution: {integrity: sha512-HmdFw9CDYqM6B25pqGBpNeLCKvGPlIx1EbLrVL0zPvj50CJQUHyBNBw45Muk0kEIkogo1VZvOKHajdMuAzSxRg==} + engines: {node: '>= 20'} + peerDependencies: + '@types/json-schema': ^7.0.15 + '@babel/code-frame@7.29.7': resolution: {integrity: sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==} engines: {node: '>=6.9.0'} @@ -587,6 +621,10 @@ packages: '@fontsource/jetbrains-mono@5.2.8': resolution: {integrity: sha512-6w8/SG4kqvIMu7xd7wt6x3idn1Qux3p9N62s6G3rfldOUYHpWcc2FKrqf+Vo44jRvqWj2oAtTHrZXEP23oSKwQ==} + '@humanwhocodes/momoa@2.0.4': + resolution: {integrity: sha512-RE815I4arJFtt+FVeU1Tgp9/Xvecacji8w/V6XtXsWWH/wz/eNkNbhb+ny/+PlVZjV0rxQpRSQKNKE3lcktHEA==} + engines: {node: '>=10.10.0'} + '@ionic/cli-framework-output@2.2.8': resolution: {integrity: sha512-TshtaFQsovB4NWRBydbNFawql6yul7d5bMiW1WYYf17hd99V6xdDdk3vtF51bw6sLkxON3bDQpWsnUc9/hVo3g==} engines: {node: '>=16.0.0'} @@ -1051,6 +1089,32 @@ packages: '@preact/signals-core@1.14.4': resolution: {integrity: sha512-HNB6HYeYKhQbJ1aKl+YRjrS4+QWHLKX6qKoUsfS/m0vqzsVaEBiZiaKbG/e+NKk2ch5ALQr/ihWaMHxiCuuWHA==} + '@readme/better-ajv-errors@2.4.0': + resolution: {integrity: sha512-9WODaOAKSl/mU+MYNZ2aHCrkoRSvmQ+1YkLj589OEqqjOAhbn8j7Z+ilYoiTu/he6X63/clsxxAB4qny9/dDzg==} + engines: {node: '>=18'} + peerDependencies: + ajv: 4.11.8 - 8 + + '@readme/openapi-parser@6.3.0': + resolution: {integrity: sha512-pLsspfvN/m6C4BuZ91ZppK2uo86IT4EDhXu6B+giNLehLkuZbtg47OJvOktwDHBmheULPmXg5pzM7065AAHeyQ==} + engines: {node: '>=20'} + peerDependencies: + openapi-types: '>=7' + + '@readme/openapi-schemas@4.0.0': + resolution: {integrity: sha512-hrG//9/+RpOZTrUDirU4OzfMqaxCdY5BMpr3YuLf3Rje9rfNsydQJsH9iCPTie9ZRu9xe+0OUD4fHe0JmGwa2w==} + engines: {node: '>=20'} + + '@redocly/ajv@8.11.2': + resolution: {integrity: sha512-io1JpnwtIcvojV7QKDUSIuMN/ikdOUd1ReEnUnMKGfDVridQZ31J0MmIuqwuRjWDZfmvr+Q0MqCcfHM2gTivOg==} + + '@redocly/config@0.22.0': + resolution: {integrity: sha512-gAy93Ddo01Z3bHuVdPWfCwzgfaYgMdaZPcfL7JZ7hWJoK9V0lXDbigTWkhiPFAaLWzbOJ+kbUQG1+XwIm0KRGQ==} + + '@redocly/openapi-core@1.34.17': + resolution: {integrity: sha512-wsV2keCt6B806XpSdezbWZ9aFJYf14YVh+XQf0ESt7M90yqVuxH9//PxvtC70sgj9OCkRM3nRaLfu4MsGQZRig==} + engines: {node: '>=18.17.0', npm: '>=9.5.0'} + '@rolldown/pluginutils@1.0.1': resolution: {integrity: sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==} @@ -1229,6 +1293,9 @@ packages: '@types/http-cache-semantics@4.2.0': resolution: {integrity: sha512-L3LgimLHXtGkWikKnsPg0/VFx9OGZaC+eN1u4r+OB1XRqH3meBIAVC2zr1WdMH+RHmnRkqliQAOHNJ/E0j/e0Q==} + '@types/json-schema@7.0.15': + resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} + '@types/keyv@3.1.4': resolution: {integrity: sha512-BQ5aZNSCpj7D6K2ksrRCTmKRLEpnPvWDiLPfoGyhZ++8YtiK9d/3DBKPJgry359X/P1PfruyYwvnvwFjuEiEIg==} @@ -1499,6 +1566,14 @@ packages: resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} engines: {node: '>= 14'} + ajv-draft-04@1.0.0: + resolution: {integrity: sha512-mv00Te6nmYbRp5DCwclxtt7yV/joXJPGS7nM+97GdxvuttCOfgI3K4U25zboyeX0O+myI8ERluxQe5wljMmVIw==} + peerDependencies: + ajv: ^8.5.0 + peerDependenciesMeta: + ajv: + optional: true + ajv-formats@2.1.1: resolution: {integrity: sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==} peerDependencies: @@ -1510,6 +1585,10 @@ packages: ajv@8.20.0: resolution: {integrity: sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==} + ansi-colors@4.1.3: + resolution: {integrity: sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==} + engines: {node: '>=6'} + ansi-regex@5.0.1: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} engines: {node: '>=8'} @@ -1650,6 +1729,9 @@ packages: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} engines: {node: '>=10'} + change-case@5.4.4: + resolution: {integrity: sha512-HRQyTk2/YPEkt9TnUPbOpr64Uw3KOicFWPVBb+xiHvd6eBx/qPr9xqfBFDT8P2vWsvvz4jbEkfDe71W3VyNu2w==} + character-entities-html4@2.1.0: resolution: {integrity: sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==} @@ -1698,6 +1780,9 @@ packages: color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + colorette@1.4.0: + resolution: {integrity: sha512-Y2oEozpomLn7Q3HFP7dpww7AtMJplbM9lGZP6RDfHqmbeRjiwRg4n6VM6j4KLmRke85uWEI7JqF17f3pqdRA0g==} + combined-stream@1.0.8: resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} engines: {node: '>= 0.8'} @@ -2120,6 +2205,10 @@ packages: resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} engines: {node: '>= 14'} + index-to-position@1.2.0: + resolution: {integrity: sha512-Yg7+ztRkqslMAS2iFaU+Oa4KTSidr63OsFGlOrJoW981kIYO3CGCS3wA95P1mUi/IVSJkn0D479KTJpVpvFNuw==} + engines: {node: '>=18'} + inflight@1.0.6: resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} deprecated: This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful. @@ -2205,9 +2294,17 @@ packages: resolution: {integrity: sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==} hasBin: true + js-levenshtein@1.1.6: + resolution: {integrity: sha512-X2BB11YZtrRqY4EnQcLX5Rh373zbK4alC1FW7D7MBhL2gtcC17cTnr6DmfHZeS0s2rTHjUTMMHfG7gO8SSdw+g==} + engines: {node: '>=0.10.0'} + js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} + js-yaml@4.2.0: + resolution: {integrity: sha512-ePWsvanv0DWuDRsW8dnt+R4jQ31SCRCQ7hhNcPXZPsoBZiemuZNYGf7adZdqX2D86j6rvKp3RpCxVTSb8WQlOw==} + hasBin: true + js-yaml@4.3.0: resolution: {integrity: sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q==} hasBin: true @@ -2235,6 +2332,10 @@ packages: jsonfile@6.2.1: resolution: {integrity: sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q==} + jsonpointer@5.0.1: + resolution: {integrity: sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==} + engines: {node: '>=0.10.0'} + keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} @@ -2249,6 +2350,10 @@ packages: lazy-val@1.0.5: resolution: {integrity: sha512-0/BnGCCfyUMkBpeDgWihanIAF9JmZhHBgUhEqzvf+adhNGLoP6TaiI5oF8oyb3I45P+PcnrqihSf01M0l0G5+Q==} + leven@3.1.0: + resolution: {integrity: sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==} + engines: {node: '>=6'} + lexical@0.41.0: resolution: {integrity: sha512-pNIm5+n+hVnJHB9gYPDYsIO5Y59dNaDU9rJmPPsfqQhP2ojKFnUoPbcRnrI9FJLXB14sSumcY8LUw7Sq70TZqA==} @@ -2634,6 +2739,21 @@ packages: resolution: {integrity: sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==} engines: {node: '>=12'} + openapi-fetch@0.17.0: + resolution: {integrity: sha512-PsbZR1wAPcG91eEthKhN+Zn92FMHxv+/faECIwjXdxfTODGSGegYv0sc1Olz+HYPvKOuoXfp+0pA2XVt2cI0Ig==} + + openapi-types@12.1.3: + resolution: {integrity: sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==} + + openapi-typescript-helpers@0.1.0: + resolution: {integrity: sha512-OKTGPthhivLw/fHz6c3OPtg72vi86qaMlqbJuVJ23qOvQ+53uw1n7HdmkJFibloF7QEjDrDkzJiOJuockM/ljw==} + + openapi-typescript@7.13.0: + resolution: {integrity: sha512-EFP392gcqXS7ntPvbhBzbF8TyBA+baIYEm791Hy5YkjDYKTnk/Tn5OQeKm5BIZvJihpp8Zzr4hzx0Irde1LNGQ==} + hasBin: true + peerDependencies: + typescript: ^5.x + oxfmt@0.57.0: resolution: {integrity: sha512-ZB7Bi+rGDSqmVIo9jwcLyFgjxXvQhDdU+jx+ZrVy6VRiVXK2+CHc4hO3J4dUQjHe7V0ymHB+MDuv5z+NhK07HA==} engines: {node: ^20.19.0 || >=22.12.0} @@ -2690,6 +2810,10 @@ packages: parse-entities@4.0.2: resolution: {integrity: sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==} + parse-json@8.3.0: + resolution: {integrity: sha512-ybiGyvspI+fAoRQbIPRddCcSTV9/LsJbf0e/S85VLowVGzRmokfneg2kwVW/KU5rOXrPSbF1qAKPMgNTqqROQQ==} + engines: {node: '>=18'} + parse5@7.3.0: resolution: {integrity: sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==} @@ -2748,6 +2872,10 @@ packages: resolution: {integrity: sha512-uysumyrvkUX0rX/dEVqt8gC3sTBzd4zoWfLeS29nb53imdaXVvLINYXTI2GNqzaMuvacNx4uJQ8+b3zXR0pkgQ==} engines: {node: '>=10.4.0'} + pluralize@8.0.0: + resolution: {integrity: sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==} + engines: {node: '>=4'} + pngjs@7.0.0: resolution: {integrity: sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow==} engines: {node: '>=14.19.0'} @@ -3042,6 +3170,10 @@ packages: resolution: {integrity: sha512-MvjXzkz/BOfyVDkG0oFOtBxHX2u3gKbMHIF/dXblZsgD3BWOFLmHovIpZY7BykJdAjcqRCBi1WYBNdEC9yI7vg==} engines: {node: '>= 8.0'} + supports-color@10.2.2: + resolution: {integrity: sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==} + engines: {node: '>=18'} + supports-color@7.2.0: resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} engines: {node: '>=8'} @@ -3133,6 +3265,10 @@ packages: resolution: {integrity: sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==} engines: {node: '>=12.20'} + type-fest@4.41.0: + resolution: {integrity: sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==} + engines: {node: '>=16'} + typescript@6.0.3: resolution: {integrity: sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==} engines: {node: '>=14.17'} @@ -3181,6 +3317,9 @@ packages: unzipper@0.12.5: resolution: {integrity: sha512-tXYOi9R57Uj/2Z25SOs5RRSzq886MBQj2gY8dPL+xl/kv6s6SvByoKfAtvfVeEuhntWDgjd2o9p2lb4TVPAz0A==} + uri-js-replace@1.0.1: + resolution: {integrity: sha512-W+C9NWNLFOoBI2QWDp4UT9pv65r2w5Cx+3sTYFvtMdDBxkKt1syCqsUdSFAChbEe1uK5TfS04wt/nGwmaeIQ0g==} + use-sync-external-store@1.6.0: resolution: {integrity: sha512-Pp6GSwGP/NrPIrxVFAIkOQeyw8lFenOHijQWkUTrDvrF4ALqylP2C/KCkeS9dpUM3KvYRQhna5vt7IL95+ZQ9w==} peerDependencies: @@ -3335,6 +3474,9 @@ packages: resolution: {integrity: sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==} engines: {node: '>=18'} + yaml-ast-parser@0.0.43: + resolution: {integrity: sha512-2PTINUwsRqSd+s8XxKaJWQlUuEMHJQyEuh2edBbW8KNJz0SJPwUSD2zRWqezFEdN7IzAgeuYHFUCF7o8zRdZ0A==} + yargs-parser@21.1.1: resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} engines: {node: '>=12'} @@ -3384,6 +3526,11 @@ snapshots: dependencies: zod: 4.4.3 + '@apidevtools/json-schema-ref-parser@14.2.1(@types/json-schema@7.0.15)': + dependencies: + '@types/json-schema': 7.0.15 + js-yaml: 4.3.0 + '@babel/code-frame@7.29.7': dependencies: '@babel/helper-validator-identifier': 7.29.7 @@ -3429,7 +3576,7 @@ snapshots: '@ionic/utils-subprocess': 3.0.1 '@ionic/utils-terminal': 2.3.5 commander: 12.1.0 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) env-paths: 2.2.1 fs-extra: 11.3.6 kleur: 4.1.5 @@ -3463,7 +3610,7 @@ snapshots: '@electron/get@2.0.3': dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) env-paths: 2.2.1 fs-extra: 8.1.0 got: 11.8.6 @@ -3477,7 +3624,7 @@ snapshots: '@electron/get@3.1.0': dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) env-paths: 2.2.1 fs-extra: 8.1.0 got: 11.8.6 @@ -3491,7 +3638,7 @@ snapshots: '@electron/notarize@2.5.0': dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) fs-extra: 9.1.0 promise-retry: 2.0.1 transitivePeerDependencies: @@ -3500,7 +3647,7 @@ snapshots: '@electron/osx-sign@1.3.3': dependencies: compare-version: 0.1.2 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) fs-extra: 10.1.0 isbinaryfile: 4.0.10 minimist: 1.2.8 @@ -3511,7 +3658,7 @@ snapshots: '@electron/rebuild@4.2.0': dependencies: '@malept/cross-spawn-promise': 2.0.0 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) node-abi: 4.33.0 node-api-version: 0.2.1 node-gyp: 12.4.0 @@ -3523,7 +3670,7 @@ snapshots: dependencies: '@electron/asar': 3.4.1 '@malept/cross-spawn-promise': 2.0.0 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) dir-compare: 4.2.0 fs-extra: 11.3.6 minimatch: 9.0.9 @@ -3534,7 +3681,7 @@ snapshots: '@electron/windows-sign@1.2.2': dependencies: cross-dirname: 0.1.0 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) fs-extra: 11.3.6 minimist: 1.2.8 postject: 1.0.0-alpha.6 @@ -3571,17 +3718,19 @@ snapshots: '@fontsource/jetbrains-mono@5.2.8': {} + '@humanwhocodes/momoa@2.0.4': {} + '@ionic/cli-framework-output@2.2.8': dependencies: '@ionic/utils-terminal': 2.3.5 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) tslib: 2.8.1 transitivePeerDependencies: - supports-color '@ionic/utils-array@2.1.6': dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) tslib: 2.8.1 transitivePeerDependencies: - supports-color @@ -3589,7 +3738,7 @@ snapshots: '@ionic/utils-fs@3.1.7': dependencies: '@types/fs-extra': 8.1.5 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) fs-extra: 9.1.0 tslib: 2.8.1 transitivePeerDependencies: @@ -3597,7 +3746,7 @@ snapshots: '@ionic/utils-object@2.1.6': dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) tslib: 2.8.1 transitivePeerDependencies: - supports-color @@ -3606,7 +3755,7 @@ snapshots: dependencies: '@ionic/utils-object': 2.1.6 '@ionic/utils-terminal': 2.3.5 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) signal-exit: 3.0.7 tree-kill: 1.2.2 tslib: 2.8.1 @@ -3615,7 +3764,7 @@ snapshots: '@ionic/utils-stream@3.1.7': dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) tslib: 2.8.1 transitivePeerDependencies: - supports-color @@ -3628,7 +3777,7 @@ snapshots: '@ionic/utils-stream': 3.1.7 '@ionic/utils-terminal': 2.3.5 cross-spawn: 7.0.6 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) tslib: 2.8.1 transitivePeerDependencies: - supports-color @@ -3636,7 +3785,7 @@ snapshots: '@ionic/utils-terminal@2.3.5': dependencies: '@types/slice-ansi': 4.0.0 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) signal-exit: 3.0.7 slice-ansi: 4.0.0 string-width: 4.2.3 @@ -3842,7 +3991,7 @@ snapshots: '@malept/flatpak-bundler@0.4.0': dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) fs-extra: 9.1.0 lodash: 4.18.1 tmp-promise: 3.0.3 @@ -4021,6 +4170,53 @@ snapshots: '@preact/signals-core@1.14.4': {} + '@readme/better-ajv-errors@2.4.0(ajv@8.20.0)': + dependencies: + '@babel/code-frame': 7.29.7 + '@babel/runtime': 7.29.7 + '@humanwhocodes/momoa': 2.0.4 + ajv: 8.20.0 + jsonpointer: 5.0.1 + leven: 3.1.0 + picocolors: 1.1.1 + + '@readme/openapi-parser@6.3.0(openapi-types@12.1.3)': + dependencies: + '@apidevtools/json-schema-ref-parser': 14.2.1(@types/json-schema@7.0.15) + '@readme/better-ajv-errors': 2.4.0(ajv@8.20.0) + '@readme/openapi-schemas': 4.0.0 + '@types/json-schema': 7.0.15 + ajv: 8.20.0 + ajv-draft-04: 1.0.0(ajv@8.20.0) + openapi-types: 12.1.3 + + '@readme/openapi-schemas@4.0.0': + dependencies: + '@types/json-schema': 7.0.15 + + '@redocly/ajv@8.11.2': + dependencies: + fast-deep-equal: 3.1.3 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 + uri-js-replace: 1.0.1 + + '@redocly/config@0.22.0': {} + + '@redocly/openapi-core@1.34.17(supports-color@10.2.2)': + dependencies: + '@redocly/ajv': 8.11.2 + '@redocly/config': 0.22.0 + colorette: 1.4.0 + https-proxy-agent: 7.0.6(supports-color@10.2.2) + js-levenshtein: 1.1.6 + js-yaml: 4.2.0 + minimatch: 5.1.9 + pluralize: 8.0.0 + yaml-ast-parser: 0.0.43 + transitivePeerDependencies: + - supports-color + '@rolldown/pluginutils@1.0.1': {} '@sindresorhus/is@4.6.0': {} @@ -4185,6 +4381,8 @@ snapshots: '@types/http-cache-semantics@4.2.0': {} + '@types/json-schema@7.0.15': {} + '@types/keyv@3.1.4': dependencies: '@types/node': 24.12.4 @@ -4433,6 +4631,10 @@ snapshots: agent-base@7.1.4: {} + ajv-draft-04@1.0.0(ajv@8.20.0): + optionalDependencies: + ajv: 8.20.0 + ajv-formats@2.1.1(ajv@8.20.0): optionalDependencies: ajv: 8.20.0 @@ -4444,6 +4646,8 @@ snapshots: json-schema-traverse: 1.0.0 require-from-string: 2.0.2 + ansi-colors@4.1.3: {} + ansi-regex@5.0.1: {} ansi-styles@4.3.0: @@ -4472,7 +4676,7 @@ snapshots: builder-util-runtime: 9.7.0 chromium-pickle-js: 0.2.0 ci-info: 4.3.1 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) dmg-builder: 26.15.6(electron-builder-squirrel-windows@26.15.6) dotenv: 16.6.1 dotenv-expand: 11.0.7 @@ -4566,7 +4770,7 @@ snapshots: builder-util-runtime@9.7.0: dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) sax: 1.6.0 transitivePeerDependencies: - supports-color @@ -4577,10 +4781,10 @@ snapshots: builder-util-runtime: 9.7.0 chalk: 4.1.2 cross-spawn: 7.0.6 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) fs-extra: 10.1.0 http-proxy-agent: 7.0.2 - https-proxy-agent: 7.0.6 + https-proxy-agent: 7.0.6(supports-color@10.2.2) js-yaml: 4.3.0 sanitize-filename: 1.6.4 source-map-support: 0.5.21 @@ -4622,6 +4826,8 @@ snapshots: ansi-styles: 4.3.0 supports-color: 7.2.0 + change-case@5.4.4: {} + character-entities-html4@2.1.0: {} character-entities-legacy@3.0.0: {} @@ -4660,6 +4866,8 @@ snapshots: color-name@1.1.4: {} + colorette@1.4.0: {} + combined-stream@1.0.8: dependencies: delayed-stream: 1.0.0 @@ -4711,9 +4919,11 @@ snapshots: dependencies: mimic-fn: 3.1.0 - debug@4.4.3: + debug@4.4.3(supports-color@10.2.2): dependencies: ms: 2.1.3 + optionalDependencies: + supports-color: 10.2.2 decode-named-character-reference@1.3.0: dependencies: @@ -4855,7 +5065,7 @@ snapshots: electron-winstaller@5.4.0: dependencies: '@electron/asar': 3.4.1 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) fs-extra: 7.0.1 lodash: 4.18.1 temp: 0.9.4 @@ -4934,7 +5144,7 @@ snapshots: extract-zip@2.0.1: dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) get-stream: 5.2.0 yauzl: 2.10.0 optionalDependencies: @@ -5199,7 +5409,7 @@ snapshots: http-proxy-agent@7.0.2: dependencies: agent-base: 7.1.4 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) transitivePeerDependencies: - supports-color @@ -5208,13 +5418,15 @@ snapshots: quick-lru: 5.1.1 resolve-alpn: 1.2.1 - https-proxy-agent@7.0.6: + https-proxy-agent@7.0.6(supports-color@10.2.2): dependencies: agent-base: 7.1.4 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) transitivePeerDependencies: - supports-color + index-to-position@1.2.0: {} + inflight@1.0.6: dependencies: once: 1.4.0 @@ -5273,8 +5485,14 @@ snapshots: jiti@2.7.0: {} + js-levenshtein@1.1.6: {} + js-tokens@4.0.0: {} + js-yaml@4.2.0: + dependencies: + argparse: 2.0.1 + js-yaml@4.3.0: dependencies: argparse: 2.0.1 @@ -5300,6 +5518,8 @@ snapshots: optionalDependencies: graceful-fs: 4.2.11 + jsonpointer@5.0.1: {} + keyv@4.5.4: dependencies: json-buffer: 3.0.1 @@ -5310,6 +5530,8 @@ snapshots: lazy-val@1.0.5: {} + leven@3.1.0: {} + lexical@0.41.0: {} lib0@0.2.117: @@ -5735,7 +5957,7 @@ snapshots: micromark@4.0.2: dependencies: '@types/debug': 4.1.13 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) decode-named-character-reference: 1.3.0 devlop: 1.1.0 micromark-core-commonmark: 2.0.3 @@ -5809,7 +6031,7 @@ snapshots: '@ionic/utils-fs': 3.1.7 '@ionic/utils-terminal': 2.3.5 bplist-parser: 0.3.2 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) elementtree: 0.1.7 ini: 4.1.3 plist: 3.1.0 @@ -5868,6 +6090,24 @@ snapshots: is-docker: 2.2.1 is-wsl: 2.2.0 + openapi-fetch@0.17.0: + dependencies: + openapi-typescript-helpers: 0.1.0 + + openapi-types@12.1.3: {} + + openapi-typescript-helpers@0.1.0: {} + + openapi-typescript@7.13.0(typescript@6.0.3): + dependencies: + '@redocly/openapi-core': 1.34.17(supports-color@10.2.2) + ansi-colors: 4.1.3 + change-case: 5.4.4 + parse-json: 8.3.0 + supports-color: 10.2.2 + typescript: 6.0.3 + yargs-parser: 21.1.1 + oxfmt@0.57.0(vite-plus@0.2.2(@types/node@24.12.4)(jiti@2.7.0)(typescript@6.0.3)): dependencies: tinypool: 2.1.0 @@ -6003,6 +6243,12 @@ snapshots: is-decimal: 2.0.1 is-hexadecimal: 2.0.1 + parse-json@8.3.0: + dependencies: + '@babel/code-frame': 7.29.7 + index-to-position: 1.2.0 + type-fest: 4.41.0 + parse5@7.3.0: dependencies: entities: 6.0.1 @@ -6055,6 +6301,8 @@ snapshots: base64-js: 1.5.1 xmlbuilder: 15.1.1 + pluralize@8.0.0: {} + pngjs@7.0.0: {} postcss@8.5.16: @@ -6150,7 +6398,7 @@ snapshots: read-binary-file-arch@1.0.6: dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) transitivePeerDependencies: - supports-color @@ -6371,10 +6619,12 @@ snapshots: sumchecker@3.0.1: dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) transitivePeerDependencies: - supports-color + supports-color@10.2.2: {} + supports-color@7.2.0: dependencies: has-flag: 4.0.0 @@ -6453,6 +6703,8 @@ snapshots: type-fest@2.19.0: {} + type-fest@4.41.0: {} + typescript@6.0.3: {} undici-types@7.16.0: {} @@ -6508,6 +6760,8 @@ snapshots: graceful-fs: 4.2.11 node-int64: 0.4.0 + uri-js-replace@1.0.1: {} + use-sync-external-store@1.6.0(react@19.2.6): dependencies: react: 19.2.6 @@ -6757,6 +7011,8 @@ snapshots: yallist@5.0.0: {} + yaml-ast-parser@0.0.43: {} + yargs-parser@21.1.1: {} yargs@17.7.3: diff --git a/scripts/check-release-consistency.mjs b/scripts/check-release-consistency.mjs index 7694fc50..53b1f09d 100644 --- a/scripts/check-release-consistency.mjs +++ b/scripts/check-release-consistency.mjs @@ -968,8 +968,9 @@ export function collectReleaseConsistencyErrors(files, releaseTag) { "test -x apps/flutter/build/macos/Build/Products/Debug/t4code.app/Contents/Resources/runtime/t4-host", "name: verify", "if: ${{ always() }}", - "needs: [changes, core, legacy-bridge-continuity, official-omp-gate0, cluster, tooling, android-debug, flutter, flutter-android, flutter-apple]", + "needs: [changes, t4-api-generation, core, legacy-bridge-continuity, official-omp-gate0, cluster, tooling, android-debug, flutter, flutter-android, flutter-apple]", 'test "$CHANGES_RESULT" = success', + 'test "$T4_API_GENERATION_RESULT" = success', 'test "$CORE_RESULT" = success', "for result in \\", "success|skipped) ;;", diff --git a/scripts/check-release-consistency.test.mjs b/scripts/check-release-consistency.test.mjs index b049aba9..c3df7de9 100644 --- a/scripts/check-release-consistency.test.mjs +++ b/scripts/check-release-consistency.test.mjs @@ -275,7 +275,7 @@ test("rejects updater channel, stable manifest, and publication-contract drift", ".github/workflows/ci.yml", (text) => text.replace( - "needs: [changes, core, legacy-bridge-continuity, official-omp-gate0, cluster, tooling, android-debug, flutter, flutter-android, flutter-apple]", + "needs: [changes, t4-api-generation, core, legacy-bridge-continuity, official-omp-gate0, cluster, tooling, android-debug, flutter, flutter-android, flutter-apple]", "needs: [changes, core, tooling, android-debug]", ), ], @@ -580,10 +580,11 @@ test("deploys release site source only after artifact publication", () => { assert.ok(ciWorkflow.includes("if: ${{ always() }}")); assert.ok( ciWorkflow.includes( - "needs: [changes, core, legacy-bridge-continuity, official-omp-gate0, cluster, tooling, android-debug, flutter, flutter-android, flutter-apple]", + "needs: [changes, t4-api-generation, core, legacy-bridge-continuity, official-omp-gate0, cluster, tooling, android-debug, flutter, flutter-android, flutter-apple]", ), ); assert.ok(ciWorkflow.includes('test "$CHANGES_RESULT" = success')); + assert.ok(ciWorkflow.includes('test "$T4_API_GENERATION_RESULT" = success')); assert.ok(ciWorkflow.includes('test "$CORE_RESULT" = success')); assert.ok(ciWorkflow.includes("for result in \\")); assert.ok(ciWorkflow.includes("success|skipped) ;;")); diff --git a/scripts/cluster-ci/cluster-ci-contract.test.mjs b/scripts/cluster-ci/cluster-ci-contract.test.mjs index 4c240a34..ccbf21b5 100644 --- a/scripts/cluster-ci/cluster-ci-contract.test.mjs +++ b/scripts/cluster-ci/cluster-ci-contract.test.mjs @@ -309,9 +309,9 @@ test("Woodpecker keeps upstream gates and serializes bounded cluster publication ); assert.ok(steps["cluster-ci-contracts"].commands.includes("pnpm test:cluster:ci")); assert.deepEqual(steps["cluster-operator-tests"].commands, [ - "GOMAXPROCS=1 GOFLAGS=-p=1 go test ./api/... ./controllers/... ./cmd/...", + "GOMAXPROCS=1 GOFLAGS='-mod=readonly -p=1' go test ./api/... ./controllers/... ./cmd/...", "mkdir -p ../../artifacts/cluster-proof", - "CGO_ENABLED=0 GOMAXPROCS=1 GOFLAGS=-p=1 go test -c ./charttests -o ../../artifacts/cluster-proof/chart-contract.test", + "CGO_ENABLED=0 GOMAXPROCS=1 GOFLAGS='-mod=readonly -p=1' go test -c ./charttests -o ../../artifacts/cluster-proof/chart-contract.test", ]); assert.equal( steps["cluster-operator-tests"].backend_options.kubernetes.resources.limits.memory, diff --git a/scripts/cluster-ci/crd-lifecycle.sh b/scripts/cluster-ci/crd-lifecycle.sh new file mode 100755 index 00000000..d698c7b3 --- /dev/null +++ b/scripts/cluster-ci/crd-lifecycle.sh @@ -0,0 +1,139 @@ +#!/bin/sh +set -eu + +usage() { + cat >&2 <<'EOF' +usage: crd-lifecycle.sh install|upgrade -- helm install|upgrade ... --skip-crds + +Environment: + KUBECTL kubectl executable (default: kubectl) + T4_CRD_DIRECTORY reviewed CRD directory + T4_COMPAT_DIRECTORY old-object compatibility fixture directory + T4_VALIDATION_NAMESPACE existing namespace used for server dry-runs (default: default) + T4_CRD_VALIDATOR proposed-schema validator executable (default: Go validator) + T4_DISCOVERY_OBSERVATIONS consecutive matching OpenAPI observations required (default: 3) +EOF + exit 64 +} + +[ "$#" -ge 4 ] || usage +mode=$1 +shift +case "$mode" in + install|upgrade) ;; + *) usage ;; +esac +[ "${1:-}" = "--" ] || usage +shift +[ "${1:-}" = "helm" ] || usage +[ "${2:-}" = "$mode" ] || usage + +has_skip_crds=false +for argument in "$@"; do + case "$argument" in + --skip-crds) has_skip_crds=true ;; + --force|--force=*|--force-conflicts|replace) + echo "force replacement is prohibited by the T4 CRD lifecycle" >&2 + exit 64 + ;; + esac +done +[ "$has_skip_crds" = true ] || { + echo "the workload command must include --skip-crds" >&2 + exit 64 +} + +repo_root=$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd) +kubectl=${KUBECTL:-kubectl} +crd_directory=${T4_CRD_DIRECTORY:-$repo_root/deploy/charts/t4-cluster/crds} +compat_directory=${T4_COMPAT_DIRECTORY:-$repo_root/packages/cluster-operator/api/v1alpha1/testdata/compat} +validation_namespace=${T4_VALIDATION_NAMESPACE:-default} +field_manager=t4-crd-lifecycle +crds="crd/t4clusterhosts.cluster.t4.dev crd/t4workspaces.cluster.t4.dev crd/t4sessions.cluster.t4.dev" +live_resources="t4clusterhosts.cluster.t4.dev t4workspaces.cluster.t4.dev t4sessions.cluster.t4.dev" +discovery_observations=${T4_DISCOVERY_OBSERVATIONS:-3} + +case "$discovery_observations" in + ''|*[!0-9]*|0) + echo "T4_DISCOVERY_OBSERVATIONS must be a positive integer" >&2 + exit 64 + ;; +esac + +run_validator() { + if [ -n "${T4_CRD_VALIDATOR:-}" ]; then + "$T4_CRD_VALIDATOR" "$@" + else + (cd "$repo_root/packages/cluster-operator" && go run ./cmd/crd-preflight "$@") + fi +} + +# Validate the persisted compatibility instances, including status and CEL, +# directly against the proposed structural schemas. This is intentionally the +# first operation: current-cluster admission cannot prove candidate compatibility. +run_validator fixtures "$crd_directory" "$compat_directory" + +# Curated fixtures cannot prove that every value currently persisted in the +# cluster remains valid. Enumerate only the three namespaced T4 resources and +# run each complete live object, including status, through the same proposed- +# schema engine before a CRD or workload can be mutated. A denied or malformed +# list fails closed under set -e. +live_objects=$(mktemp) +trap 'rm -f "$live_objects"' EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM +for resource in $live_resources; do + installed_crd=$("$kubectl" get "crd/$resource" --ignore-not-found -o name) + if [ -z "$installed_crd" ]; then + continue + fi + "$kubectl" get "$resource" --all-namespaces -o json >"$live_objects" + run_validator objects "$crd_directory" <"$live_objects" +done +rm -f "$live_objects" +trap - EXIT HUP INT TERM + +# Every kubectl operation before the first non-dry-run apply is read-only. +"$kubectl" apply --server-side --dry-run=server --validate=strict \ + --field-manager="$field_manager" -f "$crd_directory" >/dev/null + +"$kubectl" apply --server-side --validate=strict \ + --field-manager="$field_manager" -f "$crd_directory" +# Discovery and admission must converge before compatibility validation or any +# workload rollout uses the new schema. +# shellcheck disable=SC2086 # The fixed CRD words are intentional argv entries. +"$kubectl" wait --for=condition=Established --timeout=120s $crds + +# Established can remain True across an update. Require several independently +# fetched discovery documents to expose exactly the candidate OpenAPI semantics +# before trusting admission or starting a workload rollout. +openapi_document=$(mktemp) +trap 'rm -f "$openapi_document"' EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM +observation=0 +while [ "$observation" -lt "$discovery_observations" ]; do + "$kubectl" get --raw /openapi/v3/apis/cluster.t4.dev/v1alpha1 >"$openapi_document" + run_validator served "$crd_directory" <"$openapi_document" + observation=$((observation + 1)) +done + +"$kubectl" apply --dry-run=server --validate=strict \ + --namespace "$validation_namespace" -f "$compat_directory" >/dev/null + +for crd in $crds; do + stored_versions=$("$kubectl" get "$crd" -o 'jsonpath={.status.storedVersions[*]}') + if [ "$stored_versions" != v1alpha1 ]; then + echo "$crd status.storedVersions is '$stored_versions'; expected exactly 'v1alpha1'" >&2 + exit 65 + fi +done + +# Helm is deliberately last and must not manage CRDs. If any earlier gate fails, +# the existing controller, server, session workloads, and custom resources are +# untouched by this runner. +rm -f "$openapi_document" +trap - EXIT HUP INT TERM +exec "$@" diff --git a/scripts/cluster-ci/crd-lifecycle.test.mjs b/scripts/cluster-ci/crd-lifecycle.test.mjs new file mode 100644 index 00000000..543f7775 --- /dev/null +++ b/scripts/cluster-ci/crd-lifecycle.test.mjs @@ -0,0 +1,311 @@ +import assert from "node:assert/strict"; +import { chmod, mkdtemp, mkdir, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { spawn } from "node:child_process"; +import test from "node:test"; + +const repoRoot = resolve(import.meta.dirname, "../.."); +const lifecycle = resolve(import.meta.dirname, "crd-lifecycle.sh"); + +async function fixture() { + const root = await mkdtemp(join(tmpdir(), "t4-crd-lifecycle-")); + const bin = join(root, "bin"); + const log = join(root, "commands.log"); + await mkdir(bin); + await writeFile( + join(bin, "kubectl"), + `#!/bin/sh +set -eu +printf 'kubectl' >>"$COMMAND_LOG" +for argument in "$@"; do printf '\\t%s' "$argument" >>"$COMMAND_LOG"; done +printf '\\n' >>"$COMMAND_LOG" +for argument in "$@"; do + if [ "$argument" = "--dry-run=server" ] && [ "\${FAIL_DRY_RUN:-0}" = 1 ]; then + exit 42 + fi +done +if [ "\${1:-}" = get ]; then + case "\${2:-}" in + crd/*) + for argument in "$@"; do + if [ "$argument" = "--ignore-not-found" ]; then + if [ "\${FAIL_CRD_READ:-}" = "\${2:-}" ]; then exit 45; fi + if [ "\${MISSING_LIVE_CRDS:-0}" = 1 ]; then exit 0; fi + printf '%s' "\${2:-}" + exit 0 + fi + done + ;; + esac + case "\${2:-}" in + t4clusterhosts.cluster.t4.dev|t4workspaces.cluster.t4.dev|t4sessions.cluster.t4.dev) + if [ "\${FAIL_LIVE_LIST:-}" = "\${2:-}" ]; then exit 44; fi + if [ "\${TERMINATE_DURING_LIVE_READ:-}" = "\${2:-}" ]; then + kill -TERM "$PPID" + sleep 1 + fi + printf '%s' '{"apiVersion":"v1","kind":"List","items":[]}' + exit 0 + ;; + esac + printf '%s' "\${STORED_VERSIONS:-v1alpha1}" +fi +`, + ); + await writeFile( + join(bin, "crd-preflight"), + `#!/bin/sh +set -eu +printf 'validator' >>"$COMMAND_LOG" +for argument in "$@"; do printf '\\t%s' "$argument" >>"$COMMAND_LOG"; done +printf '\\n' >>"$COMMAND_LOG" +case "\${1:-}:\${FAIL_PROPOSED_VALIDATION:-}" in + fixtures:spec|fixtures:status|objects:live|served:stale) exit 43 ;; +esac +cat >/dev/null +`, + ); + await writeFile( + join(bin, "helm"), + `#!/bin/sh +set -eu +printf 'helm' >>"$COMMAND_LOG" +for argument in "$@"; do printf '\\t%s' "$argument" >>"$COMMAND_LOG"; done +printf '\\n' >>"$COMMAND_LOG" +`, + ); + await chmod(join(bin, "kubectl"), 0o755); + await chmod(join(bin, "helm"), 0o755); + await chmod(join(bin, "crd-preflight"), 0o755); + return { + root, + log, + env: { ...process.env, PATH: `${bin}:${process.env.PATH}`, COMMAND_LOG: log, T4_CRD_VALIDATOR: join(bin, "crd-preflight") }, + }; +} + +async function runLifecycle(args, env = {}) { + const result = await new Promise((resolveResult, reject) => { + const child = spawn(lifecycle, args, { + cwd: repoRoot, + env, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.on("error", reject); + child.on("close", (code, signal) => resolveResult({ code, signal, stdout, stderr })); + }); + return result; +} + +async function commands(log) { + return (await readFile(log, "utf8")).trim().split("\n").filter(Boolean); +} + +function findCommand(log, predicate, description) { + const index = log.findIndex(predicate); + assert.notEqual(index, -1, `missing ${description}:\n${log.join("\n")}`); + return index; +} + +test("upgrade validates proposed schemas, proves served convergence, verifies storage, then upgrades workloads", async () => { + const value = await fixture(); + const result = await runLifecycle( + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "deploy/charts/t4-cluster", "--namespace", "t4-system", "--skip-crds"], + value.env, + ); + assert.equal(result.code, 0, `${result.stdout}\n${result.stderr}`); + const log = await commands(value.log); + const proposedPreflight = findCommand(log, (line) => line.startsWith("validator\tfixtures\t"), "local proposed-schema fixture preflight"); + const crdReads = log.map((line, index) => ({ line, index })).filter(({ line }) => /^kubectl\tget\tcrd\/t4(?:clusterhosts|workspaces|sessions)\.cluster\.t4\.dev\t--ignore-not-found\t-o\tname$/u.test(line)); + const liveLists = log.map((line, index) => ({ line, index })).filter(({ line }) => /^kubectl\tget\tt4(?:clusterhosts|workspaces|sessions)\.cluster\.t4\.dev\t--all-namespaces\t-o\tjson$/u.test(line)); + const liveValidations = log.map((line, index) => ({ line, index })).filter(({ line }) => line.startsWith("validator\tobjects\t")); + const crdPreflight = findCommand(log, (line) => line.includes("apply") && line.includes("--server-side") && line.includes("--dry-run=server") && line.includes("deploy/charts/t4-cluster/crds"), "server-side CRD preflight"); + const crdApply = findCommand(log, (line) => line.includes("apply") && line.includes("--server-side") && !line.includes("--dry-run=server"), "CRD apply"); + const established = findCommand(log, (line) => line.includes("wait") && line.includes("condition=Established") && line.includes("t4clusterhosts.cluster.t4.dev") && line.includes("t4workspaces.cluster.t4.dev") && line.includes("t4sessions.cluster.t4.dev"), "Established wait"); + const servedChecks = log.map((line, index) => ({ line, index })).filter(({ line }) => line.startsWith("validator\tserved\t")); + const admissionPreflight = findCommand(log, (line) => line.includes("apply") && line.includes("--dry-run=server") && line.includes("testdata/compat"), "converged admission preflight"); + const storageChecks = log.map((line, index) => ({ line, index })).filter(({ line }) => line.startsWith("kubectl\tget\tcrd/") && line.includes("status.storedVersions")); + assert.deepEqual(storageChecks.length, 3); + assert.equal(servedChecks.length, 3); + assert.equal(crdReads.length, 3); + assert.equal(liveLists.length, 3); + assert.equal(liveValidations.length, 3); + const workload = findCommand(log, (line) => line.startsWith("helm\tupgrade\t"), "Helm workload upgrade"); + assert.ok(proposedPreflight < crdPreflight); + assert.ok(proposedPreflight < crdReads[0].index); + assert.ok(crdReads[0].index < liveLists[0].index); + assert.ok(liveLists[0].index < liveValidations[0].index); + assert.ok(liveValidations[0].index < crdReads[1].index); + assert.ok(crdReads[1].index < liveLists[1].index); + assert.ok(liveLists[1].index < liveValidations[1].index); + assert.ok(liveValidations[1].index < crdReads[2].index); + assert.ok(crdReads[2].index < liveLists[2].index); + assert.ok(liveLists[2].index < liveValidations[2].index); + assert.ok(liveValidations[2].index < crdPreflight); + assert.ok(crdPreflight < crdApply); + assert.ok(crdApply < established); + assert.ok(established < servedChecks[0].index); + assert.ok(servedChecks.at(-1).index < admissionPreflight); + assert.ok(admissionPreflight < storageChecks[0].index); + assert.ok(storageChecks.every(({ index }) => index < workload)); + assert.ok(log.every((line) => !line.includes("--force") && !line.includes("replace") && !line.includes("delete\tcrd")), log.join("\n")); +}); + +test("fresh install establishes and validates CRDs before Helm installs with CRD handling disabled", async () => { + const value = await fixture(); + const result = await runLifecycle( + ["install", "--", "helm", "install", "t4-cluster", "deploy/charts/t4-cluster", "--namespace", "t4-system", "--skip-crds"], + { ...value.env, MISSING_LIVE_CRDS: "1" }, + ); + assert.equal(result.code, 0, `${result.stdout}\n${result.stderr}`); + const log = await commands(value.log); + assert.ok(log.every((line) => !line.startsWith("validator\tobjects\t") && !/kubectl\tget\tt4(?:clusterhosts|workspaces|sessions)\.cluster\.t4\.dev\t--all-namespaces/u.test(line)), log.join("\n")); + const established = findCommand(log, (line) => line.includes("wait") && line.includes("condition=Established"), "Established wait"); + const fixtureValidation = findCommand(log, (line) => line.includes("--dry-run=server") && line.includes("testdata/compat"), "fixture validation"); + const storage = findCommand(log, (line) => line.includes("status.storedVersions"), "stored-version check"); + const workload = findCommand(log, (line) => line.startsWith("helm\tinstall\t"), "Helm install"); + assert.ok(established < fixtureValidation && fixtureValidation < storage && storage < workload); +}); + +test("candidate schema tightening fails locally before any cluster or workload mutation", async () => { + const value = await fixture(); + const result = await runLifecycle( + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "chart", "--skip-crds"], + { ...value.env, FAIL_PROPOSED_VALIDATION: "spec" }, + ); + assert.notEqual(result.code, 0); + assert.deepEqual(await commands(value.log), [ + `validator\tfixtures\t${join(repoRoot, "deploy/charts/t4-cluster/crds")}\t${join(repoRoot, "packages/cluster-operator/api/v1alpha1/testdata/compat")}`, + ]); +}); + +test("persisted status is validated against the proposed status schema before mutation", async () => { + const value = await fixture(); + const result = await runLifecycle( + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "chart", "--skip-crds"], + { ...value.env, FAIL_PROPOSED_VALIDATION: "status" }, + ); + assert.notEqual(result.code, 0); + assert.deepEqual(await commands(value.log), [ + `validator\tfixtures\t${join(repoRoot, "deploy/charts/t4-cluster/crds")}\t${join(repoRoot, "packages/cluster-operator/api/v1alpha1/testdata/compat")}`, + ]); +}); + +test("live object incompatibility fails before non-dry-run CRD, object, or Helm mutation", async () => { + const value = await fixture(); + const result = await runLifecycle( + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "chart", "--skip-crds"], + { ...value.env, FAIL_PROPOSED_VALIDATION: "live" }, + ); + assert.notEqual(result.code, 0); + const log = await commands(value.log); + assert.match(log.at(-1), /^validator\tobjects\t/u); + assert.ok(log.some((line) => /^kubectl\tget\tt4clusterhosts\.cluster\.t4\.dev\t--all-namespaces\t-o\tjson$/u.test(line)), log.join("\n")); + assert.ok(log.every((line) => !line.startsWith("helm\t") && !/kubectl\tapply(?!.*--dry-run=server)/u.test(line)), log.join("\n")); +}); + +test("live object definition read or list denial fails closed before non-dry-run mutation", async () => { + const deniedResource = "t4workspaces.cluster.t4.dev"; + for (const scenario of [ + { env: { FAIL_CRD_READ: `crd/${deniedResource}` }, expected: `kubectl\tget\tcrd/${deniedResource}\t--ignore-not-found\t-o\tname` }, + { env: { FAIL_LIVE_LIST: deniedResource }, expected: `kubectl\tget\t${deniedResource}\t--all-namespaces\t-o\tjson` }, + ]) { + const value = await fixture(); + const result = await runLifecycle( + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "chart", "--skip-crds"], + { ...value.env, ...scenario.env }, + ); + assert.notEqual(result.code, 0); + const log = await commands(value.log); + assert.ok(log.some((line) => line.startsWith(scenario.expected)), log.join("\n")); + assert.ok(log.every((line) => !line.startsWith("helm\t") && !/kubectl\tapply(?!.*--dry-run=server)/u.test(line)), log.join("\n")); + } +}); + +test("cancellation during live enumeration cannot continue to mutation", async () => { + const value = await fixture(); + const result = await runLifecycle( + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "chart", "--skip-crds"], + { ...value.env, TERMINATE_DURING_LIVE_READ: "t4clusterhosts.cluster.t4.dev" }, + ); + assert.notEqual(result.code, 0); + const log = await commands(value.log); + assert.match(log.at(-1), /^kubectl\tget\tt4clusterhosts\.cluster\.t4\.dev\t--all-namespaces\t-o\tjson$/u); + assert.ok(log.every((line) => !line.startsWith("helm\t") && !/kubectl\tapply(?!.*--dry-run=server)/u.test(line)), log.join("\n")); +}); + +test("retained Established cannot pass readiness while served OpenAPI is stale", async () => { + const value = await fixture(); + const result = await runLifecycle( + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "chart", "--skip-crds"], + { ...value.env, FAIL_PROPOSED_VALIDATION: "stale" }, + ); + assert.notEqual(result.code, 0); + const log = await commands(value.log); + const apply = findCommand(log, (line) => line.includes("kubectl\tapply") && !line.includes("--dry-run=server"), "non-dry-run CRD apply"); + const established = findCommand(log, (line) => line.includes("condition=Established"), "Established wait"); + const served = findCommand(log, (line) => line.startsWith("validator\tserved\t"), "served-schema semantic verification"); + assert.ok(apply < established && established < served, log.join("\n")); + assert.ok(log.every((line) => !line.startsWith("helm\t")), log.join("\n")); +}); + +test("failed server preflight leaves CRDs and workloads untouched", async () => { + const value = await fixture(); + const result = await runLifecycle( + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "deploy/charts/t4-cluster", "--namespace", "t4-system", "--skip-crds"], + { ...value.env, FAIL_DRY_RUN: "1" }, + ); + assert.notEqual(result.code, 0); + const log = await commands(value.log); + assert.equal(log.length, 11, log.join("\n")); + assert.match(log[0], /^validator\tfixtures\t/u); + assert.match(log.at(-1), /apply.*--server-side.*--dry-run=server/u); + assert.ok(log.every((line) => !line.startsWith("helm\t") && !/kubectl\tapply(?!.*--dry-run=server)/u.test(line)), log.join("\n")); +}); + +test("an unexpected stored version stops workload rollout", async () => { + const value = await fixture(); + const result = await runLifecycle( + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "deploy/charts/t4-cluster", "--namespace", "t4-system", "--skip-crds"], + { ...value.env, STORED_VERSIONS: "v1alpha1,v1beta1" }, + ); + assert.notEqual(result.code, 0); + const log = await commands(value.log); + assert.ok(log.some((line) => line.includes("status.storedVersions"))); + assert.ok(log.every((line) => !line.startsWith("helm\t")), log.join("\n")); +}); + +test("force replacement and implicit Helm CRD handling are rejected before cluster access", async () => { + const value = await fixture(); + for (const args of [ + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "chart", "--skip-crds", "--force"], + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "chart"], + ]) { + const result = await runLifecycle(args, value.env); + assert.equal(result.code, 64, `${result.stdout}\n${result.stderr}`); + } +}); + +test("future storage migration explicitly retires the old stored version only after rewrite and dual-version reads", async () => { + const docs = await readFile(join(repoRoot, "docs/CLUSTER_OPERATOR.md"), "utf8"); + const migration = docs.slice(docs.indexOf("### Future `v1beta1`"), docs.indexOf("### Workload rollback")); + const storageFlip = migration.indexOf("`v1beta1` storage to true"); + const rewrite = migration.indexOf("rewrite every object"); + const verifyReads = migration.indexOf("read every rewritten object through both served versions"); + const statusUpdate = migration.indexOf("/status"); + const exactAssertion = migration.indexOf("exactly `[v1beta1]`"); + const oldStillServed = migration.indexOf("Keep `v1alpha1` served"); + assert.ok(storageFlip >= 0 && storageFlip < rewrite, migration); + assert.ok(rewrite < verifyReads && verifyReads < statusUpdate, migration); + assert.ok(statusUpdate < exactAssertion && exactAssertion < oldStillServed, migration); + assert.match(migration, /patch customresourcedefinition[^\n]*--subresource=status/u); +});