From d69a67e717be91a153c79e91d4db0bdbfe017cdf Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 19:15:30 +0000 Subject: [PATCH 001/131] test: define T4 API v1 client conformance --- .../test/t4-api-v1-conformance-service.ts | 260 ++++++++++++++++++ .../client/test/t4-api-v1-conformance.test.ts | 173 ++++++++++++ 2 files changed, 433 insertions(+) create mode 100644 packages/client/test/t4-api-v1-conformance-service.ts create mode 100644 packages/client/test/t4-api-v1-conformance.test.ts diff --git a/packages/client/test/t4-api-v1-conformance-service.ts b/packages/client/test/t4-api-v1-conformance-service.ts new file mode 100644 index 00000000..d783f171 --- /dev/null +++ b/packages/client/test/t4-api-v1-conformance-service.ts @@ -0,0 +1,260 @@ +import { expect } from "vite-plus/test"; + +const encoder = new TextEncoder(); + +function json(status: number, body: unknown, headers: Record = {}): Response { + return Response.json(body, { status, headers: { "T4-API-Version": "1.0", ...headers } }); +} + +function problem(status: number, code: string, message: string, extra: Record = {}): Response { + return json(status, { + error: { + code, + message, + requestId: `req-${code}`, + retryable: status >= 500, + ...extra, + }, + }); +} + +function bodyKey(value: unknown): string { + return JSON.stringify(value); +} + +export class T4ApiV1ConformanceService { + readonly origin = "https://t4-api.conformance.test"; + readonly calls: Array<{ method: string; path: string; authorization: string | null }> = []; + readonly abortedWatches: string[] = []; + + #workspaceSequence = 0; + #sessionSequence = 0; + readonly #workspaces = new Map>(); + readonly #sessions = new Map>(); + readonly #replays = new Map }>(); + + readonly fetch: typeof globalThis.fetch = async (input, init) => { + const request = new Request(input, init); + const url = new URL(request.url); + const authorization = request.headers.get("authorization"); + this.calls.push({ method: request.method, path: url.pathname, authorization }); + if (url.origin !== this.origin) return problem(400, "invalid_origin", "HTTPS API origin is fixed for this client"); + if (url.protocol !== "https:") return problem(400, "https_required", "HTTPS is required"); + if (authorization !== "Bearer token-a" && authorization !== "Bearer token-b") { + return problem(401, "unauthenticated", "A valid bearer credential is required"); + } + if (request.headers.get("T4-API-Version") !== "1") { + return problem(406, "incompatible_version", "No compatible T4 API major version", { + supportedMajors: [1], + }); + } + const tenant = authorization === "Bearer token-a" ? "tenant-a" : "tenant-b"; + + if (request.method === "GET" && url.pathname === "/v1") { + return json(200, { + apiVersion: "1.0", + supportedMajors: [1], + capabilities: [ + "workspace.lifecycle", + "session.lifecycle", + "session.commands", + "session.watch.sse", + ], + limits: { + pageSizeDefault: 2, + pageSizeMax: 3, + commandBytesMax: 32, + watchEventsMax: 4, + heartbeatSeconds: 15, + }, + }); + } + + if (request.method === "POST" && url.pathname === "/v1/workspaces") { + const body = await request.json() as Record; + if (typeof body.name !== "string" || body.name.length < 1 || body.name.length > 24) { + return problem(422, "invalid_request", "Request validation failed", { + violations: [{ field: "name", rule: "length", message: "name must contain 1 to 24 characters" }], + }); + } + return this.#idempotent(request, tenant, body, () => { + const id = `ws-${++this.#workspaceSequence}`; + const workspace = { id, name: body.name, state: "accepted", revision: 1, tenant }; + this.#workspaces.set(id, workspace); + return workspace; + }); + } + + if (request.method === "GET" && url.pathname === "/v1/workspaces") { + const pageSize = Number(url.searchParams.get("pageSize") ?? "2"); + if (!Number.isInteger(pageSize) || pageSize < 1 || pageSize > 3) { + return problem(422, "invalid_request", "Request validation failed", { + violations: [{ field: "pageSize", rule: "range", message: "pageSize must be between 1 and 3" }], + }); + } + const start = Number(url.searchParams.get("cursor")?.replace("page-", "") ?? "0"); + const visible = [...this.#workspaces.values()].filter((item) => item.tenant === tenant); + const items = visible.slice(start, start + pageSize).map(({ tenant: _tenant, ...item }) => item); + const next = start + items.length < visible.length ? `page-${start + items.length}` : undefined; + return json(200, { items, ...(next === undefined ? {} : { nextCursor: next }) }); + } + + const workspaceMatch = url.pathname.match(/^\/v1\/workspaces\/([^/]+)$/u); + if (workspaceMatch) { + const id = decodeURIComponent(workspaceMatch[1]!); + const workspace = this.#workspaces.get(id); + if (workspace?.tenant !== tenant) return problem(404, "not_found", "Workspace not found"); + if (request.method === "GET") { + const { tenant: _tenant, ...visible } = workspace; + return json(200, visible); + } + if (request.method === "PATCH") { + const body = await request.json() as Record; + if (request.headers.get("If-Match") !== String(workspace.revision)) { + return problem(409, "revision_conflict", "Workspace revision changed"); + } + const updated = { ...workspace, name: body.name ?? workspace.name, revision: Number(workspace.revision) + 1 }; + this.#workspaces.set(id, updated); + const { tenant: _tenant, ...visible } = updated; + return json(200, visible); + } + if (request.method === "DELETE") { + this.#workspaces.delete(id); + return new Response(null, { status: 204, headers: { "T4-API-Version": "1.0" } }); + } + } + + const sessionsPath = url.pathname.match(/^\/v1\/workspaces\/([^/]+)\/sessions$/u); + if (sessionsPath) { + const workspaceId = decodeURIComponent(sessionsPath[1]!); + const workspace = this.#workspaces.get(workspaceId); + if (workspace?.tenant !== tenant) return problem(404, "not_found", "Workspace not found"); + if (request.method === "POST") { + const body = await request.json() as Record; + return this.#idempotent(request, tenant, body, () => { + const id = `ses-${++this.#sessionSequence}`; + const session = { id, workspaceId, title: body.title, state: "accepted", revision: 1, tenant }; + this.#sessions.set(id, session); + return session; + }); + } + if (request.method === "GET") { + const items = [...this.#sessions.values()] + .filter((item) => item.workspaceId === workspaceId && item.tenant === tenant) + .map(({ tenant: _tenant, ...item }) => item); + return json(200, { items }); + } + } + + const sessionMatch = url.pathname.match(/^\/v1\/sessions\/([^/]+)$/u); + if (sessionMatch) { + const id = decodeURIComponent(sessionMatch[1]!); + const session = this.#sessions.get(id); + if (session?.tenant !== tenant) return problem(404, "not_found", "Session not found"); + if (request.method === "GET") { + const { tenant: _tenant, ...visible } = session; + return json(200, visible); + } + if (request.method === "PATCH") { + const body = await request.json() as Record; + const updated = { ...session, title: body.title ?? session.title, revision: Number(session.revision) + 1 }; + this.#sessions.set(id, updated); + const { tenant: _tenant, ...visible } = updated; + return json(200, visible); + } + if (request.method === "DELETE") { + this.#sessions.delete(id); + return new Response(null, { status: 204, headers: { "T4-API-Version": "1.0" } }); + } + } + + const cancelMatch = url.pathname.match(/^\/v1\/sessions\/([^/]+)\/cancel$/u); + if (cancelMatch && request.method === "POST") { + const session = this.#sessions.get(decodeURIComponent(cancelMatch[1]!)); + if (session?.tenant !== tenant) return problem(404, "not_found", "Session not found"); + const cancelled = { ...session, state: "cancelled", revision: Number(session.revision) + 1 }; + this.#sessions.set(String(session.id), cancelled); + const { tenant: _tenant, ...visible } = cancelled; + return json(202, visible); + } + + const commandMatch = url.pathname.match(/^\/v1\/sessions\/([^/]+)\/commands$/u); + if (commandMatch && request.method === "POST") { + const session = this.#sessions.get(decodeURIComponent(commandMatch[1]!)); + if (session?.tenant !== tenant) return problem(404, "not_found", "Session not found"); + const body = await request.json() as Record; + if (typeof body.command !== "string" || encoder.encode(body.command).byteLength > 32) { + return problem(422, "invalid_request", "Request validation failed", { + violations: [{ field: "command", rule: "maxBytes", message: "command must not exceed 32 UTF-8 bytes" }], + }); + } + const state = ["accepted", "rejected", "conflict", "unavailable", "indeterminate"].includes(body.command) + ? body.command + : "accepted"; + return this.#idempotent(request, tenant, body, () => ({ commandId: `cmd-${body.command}`, state })); + } + + const snapshotMatch = url.pathname.match(/^\/v1\/sessions\/([^/]+)\/snapshot$/u); + if (snapshotMatch && request.method === "GET") { + const session = this.#sessions.get(decodeURIComponent(snapshotMatch[1]!)); + if (session?.tenant !== tenant) return problem(404, "not_found", "Session not found"); + return json(200, { sessionId: session.id, cursor: "cursor-2", state: session.state, entries: [{ sequence: 2, kind: "output", text: "ready" }] }); + } + + const watchMatch = url.pathname.match(/^\/v1\/sessions\/([^/]+)\/events$/u); + if (watchMatch && request.method === "GET") { + const id = decodeURIComponent(watchMatch[1]!); + const session = this.#sessions.get(id); + if (session?.tenant !== tenant) return problem(404, "not_found", "Session not found"); + const cursor = url.searchParams.get("cursor") ?? request.headers.get("Last-Event-ID"); + if (cursor === "expired") { + return problem(410, "cursor_expired", "Watch cursor is no longer retained", { + resync: { snapshotUrl: `/v1/sessions/${id}/snapshot`, cursor: "cursor-2" }, + }); + } + const frames = [ + `id: cursor-3\nevent: heartbeat\ndata: {"type":"heartbeat","cursor":"cursor-3","observedAt":"2026-07-21T00:00:00Z"}\n\n`, + `id: cursor-4\nevent: session\ndata: {"type":"session","cursor":"cursor-4","state":"accepted","revision":2}\n\n`, + ]; + const stream = new ReadableStream({ + start: (controller) => { + for (const frame of frames) controller.enqueue(encoder.encode(frame)); + request.signal.addEventListener("abort", () => { + this.abortedWatches.push(id); + try { controller.close(); } catch { /* already closed */ } + }, { once: true }); + }, + }); + return new Response(stream, { + status: 200, + headers: { "Content-Type": "text/event-stream", "Cache-Control": "no-store", "T4-API-Version": "1.0" }, + }); + } + + return problem(404, "not_found", "Resource not found"); + }; + + expectNoCredentialLeak(): void { + expect(this.calls.every((call) => call.authorization === "Bearer token-a" || call.authorization === "Bearer token-b")).toBe(true); + } + + #idempotent( + request: Request, + tenant: string, + body: Record, + create: () => Record, + ): Response { + const key = request.headers.get("Idempotency-Key"); + if (!key) return problem(400, "idempotency_key_required", "Idempotency-Key is required"); + const replayKey = `${tenant}:${request.method}:${new URL(request.url).pathname}:${key}`; + const prior = this.#replays.get(replayKey); + if (prior) { + if (prior.body !== bodyKey(body)) return problem(409, "idempotency_conflict", "Idempotency key was reused with a different request"); + return json(200, prior.response, { "Idempotency-Replayed": "true" }); + } + const response = create(); + const { tenant: _tenant, ...visible } = response; + this.#replays.set(replayKey, { body: bodyKey(body), response: visible }); + return json(202, visible, { "Idempotency-Replayed": "false" }); + } +} diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts new file mode 100644 index 00000000..4eb69701 --- /dev/null +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -0,0 +1,173 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { + T4ApiError, + createT4ApiClient, + type components, +} from "@t4-code/t4-api-client"; +import { T4ApiV1ConformanceService } from "./t4-api-v1-conformance-service.ts"; + +type WorkspaceCreate = components["schemas"]["WorkspaceCreate"]; +type SessionCreate = components["schemas"]["SessionCreate"]; +type CommandCreate = components["schemas"]["CommandCreate"]; + +function requireData(result: { data?: T; error?: unknown }): T { + expect(result.error).toBeUndefined(); + expect(result.data).toBeDefined(); + return result.data!; +} + +describe("generated T4 API v1 client conformance", () => { + it("negotiates discovery, capabilities, bounds, and rejects an incompatible major", async () => { + const service = new T4ApiV1ConformanceService(); + const client = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); + const discovery = requireData(await client.http.GET("/v1")); + expect(discovery).toMatchObject({ + apiVersion: "1.0", + supportedMajors: [1], + capabilities: expect.arrayContaining(["workspace.lifecycle", "session.watch.sse"]), + limits: { pageSizeMax: 3, commandBytesMax: 32, watchEventsMax: 4, heartbeatSeconds: 15 }, + }); + + const incompatible = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 2, fetch: service.fetch }); + const rejected = await incompatible.http.GET("/v1"); + expect(rejected.response.status).toBe(406); + expect(rejected.error).toMatchObject({ error: { code: "incompatible_version", retryable: false, supportedMajors: [1] } }); + }); + + it("creates, replays, conflicts, mutates, lists, isolates, and deletes bounded workspaces", async () => { + const service = new T4ApiV1ConformanceService(); + const owner = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); + const other = createT4ApiClient({ baseUrl: service.origin, credential: "token-b", majorVersion: 1, fetch: service.fetch }); + const body = { name: "primary" } satisfies WorkspaceCreate; + const first = await owner.http.POST("/v1/workspaces", { body, headers: { "Idempotency-Key": "workspace-create-1" } }); + expect(first.response.status).toBe(202); + const workspace = requireData(first); + expect(workspace).toMatchObject({ id: "ws-1", state: "accepted", revision: 1 }); + expect(workspace).not.toHaveProperty("tenant"); + + const replay = await owner.http.POST("/v1/workspaces", { body, headers: { "Idempotency-Key": "workspace-create-1" } }); + expect(replay.response.status).toBe(200); + expect(replay.response.headers.get("Idempotency-Replayed")).toBe("true"); + expect(replay.data).toEqual(workspace); + const conflict = await owner.http.POST("/v1/workspaces", { + body: { name: "different" }, + headers: { "Idempotency-Key": "workspace-create-1" }, + }); + expect(conflict.response.status).toBe(409); + expect(conflict.error).toMatchObject({ error: { code: "idempotency_conflict", retryable: false } }); + + for (const name of ["second", "third", "fourth"]) { + requireData(await owner.http.POST("/v1/workspaces", { body: { name }, headers: { "Idempotency-Key": `workspace-${name}` } })); + } + const pageOne = requireData(await owner.http.GET("/v1/workspaces", { params: { query: { pageSize: 2 } } })); + expect(pageOne.items).toHaveLength(2); + expect(pageOne.nextCursor).toBe("page-2"); + const pageTwo = requireData(await owner.http.GET("/v1/workspaces", { params: { query: { pageSize: 2, cursor: pageOne.nextCursor } } })); + expect(pageTwo.items).toHaveLength(2); + expect(pageTwo.nextCursor).toBeUndefined(); + + const isolated = await other.http.GET("/v1/workspaces/ws-1", { params: { path: { workspaceId: "ws-1" } } }); + expect(isolated.response.status).toBe(404); + expect(isolated.error).toMatchObject({ error: { code: "not_found" } }); + const invalid = await owner.http.POST("/v1/workspaces", { body: { name: "x".repeat(25) }, headers: { "Idempotency-Key": "invalid" } }); + expect(invalid.response.status).toBe(422); + expect(invalid.error).toMatchObject({ error: { code: "invalid_request", violations: [{ field: "name", rule: "length" }] } }); + + const updated = requireData(await owner.http.PATCH("/v1/workspaces/{workspaceId}", { + params: { path: { workspaceId: "ws-1" } }, body: { name: "renamed" }, headers: { "If-Match": "1" }, + })); + expect(updated).toMatchObject({ name: "renamed", revision: 2 }); + const stale = await owner.http.PATCH("/v1/workspaces/{workspaceId}", { + params: { path: { workspaceId: "ws-1" } }, body: { name: "stale" }, headers: { "If-Match": "1" }, + }); + expect(stale.response.status).toBe(409); + expect(stale.error).toMatchObject({ error: { code: "revision_conflict" } }); + expect((await owner.http.DELETE("/v1/workspaces/{workspaceId}", { params: { path: { workspaceId: "ws-1" } } })).response.status).toBe(204); + service.expectNoCredentialLeak(); + }); + + it("spawns and mutates sessions, submits idempotent commands, and exposes stable states", async () => { + const service = new T4ApiV1ConformanceService(); + const client = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); + requireData(await client.http.POST("/v1/workspaces", { body: { name: "workspace" }, headers: { "Idempotency-Key": "workspace" } })); + const body = { title: "agent" } satisfies SessionCreate; + const first = await client.http.POST("/v1/workspaces/{workspaceId}/sessions", { + params: { path: { workspaceId: "ws-1" } }, body, headers: { "Idempotency-Key": "spawn-1" }, + }); + expect(first.response.status).toBe(202); + const session = requireData(first); + const replay = await client.http.POST("/v1/workspaces/{workspaceId}/sessions", { + params: { path: { workspaceId: "ws-1" } }, body, headers: { "Idempotency-Key": "spawn-1" }, + }); + expect(replay.data).toEqual(session); + const listed = requireData(await client.http.GET("/v1/workspaces/{workspaceId}/sessions", { + params: { path: { workspaceId: "ws-1" } }, + })); + expect(listed.items).toEqual([session]); + const mutated = requireData(await client.http.PATCH("/v1/sessions/{sessionId}", { + params: { path: { sessionId: "ses-1" } }, body: { title: "renamed-agent" }, headers: { "If-Match": "1" }, + })); + expect(mutated).toMatchObject({ title: "renamed-agent", revision: 2 }); + + for (const state of ["accepted", "rejected", "conflict", "unavailable", "indeterminate"] as const) { + const command = { command: state } satisfies CommandCreate; + const result = await client.http.POST("/v1/sessions/{sessionId}/commands", { + params: { path: { sessionId: "ses-1" } }, body: command, headers: { "Idempotency-Key": `command-${state}` }, + }); + expect(requireData(result).state).toBe(state); + const commandReplay = await client.http.POST("/v1/sessions/{sessionId}/commands", { + params: { path: { sessionId: "ses-1" } }, body: command, headers: { "Idempotency-Key": `command-${state}` }, + }); + expect(commandReplay.data).toEqual(result.data); + expect(commandReplay.response.headers.get("Idempotency-Replayed")).toBe("true"); + } + const oversized = await client.http.POST("/v1/sessions/{sessionId}/commands", { + params: { path: { sessionId: "ses-1" } }, body: { command: "a".repeat(33) }, headers: { "Idempotency-Key": "oversized" }, + }); + expect(oversized.response.status).toBe(422); + expect(oversized.error).toMatchObject({ error: { violations: [{ field: "command", rule: "maxBytes" }] } }); + const cancelled = await client.http.POST("/v1/sessions/{sessionId}/cancel", { params: { path: { sessionId: "ses-1" } } }); + expect(cancelled.response.status).toBe(202); + expect(cancelled.data).toMatchObject({ state: "cancelled" }); + }); + + it("takes a snapshot and watches bounded SSE with heartbeat, reconnect cursor, cancellation, and typed resync", async () => { + const service = new T4ApiV1ConformanceService(); + const client = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); + requireData(await client.http.POST("/v1/workspaces", { body: { name: "workspace" }, headers: { "Idempotency-Key": "workspace" } })); + requireData(await client.http.POST("/v1/workspaces/{workspaceId}/sessions", { + params: { path: { workspaceId: "ws-1" } }, body: { title: "agent" }, headers: { "Idempotency-Key": "session" }, + })); + const snapshot = requireData(await client.http.GET("/v1/sessions/{sessionId}/snapshot", { params: { path: { sessionId: "ses-1" } } })); + expect(snapshot).toMatchObject({ sessionId: "ses-1", cursor: "cursor-2", entries: [{ sequence: 2 }] }); + + const controller = new AbortController(); + const received: Array = []; + for await (const event of client.watchSession("ses-1", { cursor: snapshot.cursor, maxEvents: 2, signal: controller.signal })) { + received.push(event); + if (received.length === 2) controller.abort(); + } + expect(received).toEqual([ + expect.objectContaining({ type: "heartbeat", cursor: "cursor-3" }), + expect.objectContaining({ type: "session", cursor: "cursor-4", state: "accepted" }), + ]); + expect(service.calls.at(-1)?.path).toBe("/v1/sessions/ses-1/events"); + expect(service.abortedWatches).toEqual(["ses-1"]); + + const reconnect = client.watchSession("ses-1", { cursor: received.at(-1)!.cursor, maxEvents: 1 }); + expect((await reconnect.next()).value).toMatchObject({ type: "heartbeat", cursor: "cursor-3" }); + await reconnect.return(undefined); + + await expect(async () => { + for await (const _event of client.watchSession("ses-1", { cursor: "expired", maxEvents: 1 })) { + throw new Error("expired cursor unexpectedly produced an event"); + } + }).rejects.toMatchObject({ + name: "T4ApiError", + code: "cursor_expired", + status: 410, + resync: { snapshotUrl: "/v1/sessions/ses-1/snapshot", cursor: "cursor-2" }, + } satisfies Partial); + }); +}); From 6058f1c8ff85a8b1053688e178952c1775bf65ed Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 19:24:01 +0000 Subject: [PATCH 002/131] feat: define T4 API v1 contract and SDK boundary --- .woodpecker.yml | 52 +- packages/client/package.json | 1 + packages/t4-api-client/package.json | 23 + packages/t4-api-client/src/index.ts | 317 ++++++++ packages/t4-api-client/tsconfig.json | 7 + packages/t4-api-contract/openapi.json | 683 ++++++++++++++++++ packages/t4-api-contract/package.json | 22 + packages/t4-api-contract/scripts/generate.mjs | 44 ++ packages/t4-api-contract/scripts/validate.mjs | 20 + packages/t4-api-contract/tsconfig.json | 7 + 10 files changed, 1175 insertions(+), 1 deletion(-) create mode 100644 packages/t4-api-client/package.json create mode 100644 packages/t4-api-client/src/index.ts create mode 100644 packages/t4-api-client/tsconfig.json create mode 100644 packages/t4-api-contract/openapi.json create mode 100644 packages/t4-api-contract/package.json create mode 100644 packages/t4-api-contract/scripts/generate.mjs create mode 100644 packages/t4-api-contract/scripts/validate.mjs create mode 100644 packages/t4-api-contract/tsconfig.json diff --git a/.woodpecker.yml b/.woodpecker.yml index 4f8c556e..fb611ce0 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -12,7 +12,35 @@ clone: partial: false steps: + t4-api-artifact-bootstrap: + image: mirror.gcr.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 + commands: + - corepack enable + - pnpm install --no-frozen-lockfile + - echo T4_API_LOCKFILE_BASE64_BEGIN + - base64 -w0 pnpm-lock.yaml + - echo + - echo T4_API_LOCKFILE_BASE64_END + - pnpm --filter @t4-code/t4-api-contract validate + - pnpm --filter @t4-code/t4-api-contract generate:ci + when: + - event: push + branch: agent/t4-op03-public-api-contract + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 512Mi + ephemeral-storage: 1Gi + limits: + cpu: "1" + memory: 2Gi + ephemeral-storage: 4Gi + dependencies: + depends_on: [t4-api-artifact-bootstrap] image: mirror.gcr.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 commands: - corepack enable @@ -31,6 +59,28 @@ steps: memory: 2Gi ephemeral-storage: 4Gi + t4-api-contract: + depends_on: [dependencies] + image: mirror.gcr.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 + commands: + - corepack enable + - pnpm --filter @t4-code/t4-api-contract validate + - pnpm --filter @t4-code/t4-api-contract generate:ci + - pnpm --filter @t4-code/t4-api-client typecheck + - pnpm --filter @t4-code/client exec vp test run test/t4-api-v1-conformance.test.ts + backend_options: + kubernetes: + serviceAccountName: woodpecker-ci-untrusted + resources: + requests: + cpu: "0" + memory: 256Mi + ephemeral-storage: 256Mi + limits: + cpu: "1" + memory: 2Gi + ephemeral-storage: 2Gi + bun-runtime: depends_on: [dependencies] image: mirror.gcr.io/oven/bun:1.3.14@sha256:e10577f0db68676a7024391c6e5cb4b879ebd17188ab750cf10024a6d700e5c4 @@ -50,7 +100,7 @@ steps: ephemeral-storage: 256Mi upstream-core: - depends_on: [cluster-ci-contracts, bun-runtime] + depends_on: [cluster-ci-contracts, t4-api-contract, bun-runtime] image: mirror.gcr.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 commands: - export PATH="$PWD/.ci:$PATH" diff --git a/packages/client/package.json b/packages/client/package.json index c960a2b3..8e29f7c2 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -17,6 +17,7 @@ "devDependencies": { "@t4-code/fixture-server": "workspace:*", "@t4-code/host-wire": "workspace:*", + "@t4-code/t4-api-client": "workspace:*", "@types/node": "catalog:", "@types/ws": "8.5.13", "vite-plus": "catalog:", diff --git a/packages/t4-api-client/package.json b/packages/t4-api-client/package.json new file mode 100644 index 00000000..9c844512 --- /dev/null +++ b/packages/t4-api-client/package.json @@ -0,0 +1,23 @@ +{ + "name": "@t4-code/t4-api-client", + "version": "0.1.30", + "private": true, + "type": "module", + "exports": { + ".": "./src/index.ts", + "./generated": "./src/generated/schema.ts" + }, + "scripts": { + "build": "tsgo --noEmit", + "typecheck": "tsgo --noEmit", + "test": "vp test run --passWithNoTests" + }, + "dependencies": { + "openapi-fetch": "0.17.0" + }, + "devDependencies": { + "@types/node": "catalog:", + "typescript": "catalog:", + "vite-plus": "catalog:" + } +} diff --git a/packages/t4-api-client/src/index.ts b/packages/t4-api-client/src/index.ts new file mode 100644 index 00000000..aba5baab --- /dev/null +++ b/packages/t4-api-client/src/index.ts @@ -0,0 +1,317 @@ +import createClient, { type Client } from "openapi-fetch"; + +import type { components, paths } from "./generated/schema.ts"; + +export type { components, operations, paths } from "./generated/schema.ts"; + +const MAX_CREDENTIAL_LENGTH = 4096; +const MAX_ERROR_BYTES = 1024 * 1024; +const MAX_EVENT_BYTES = 1024 * 1024; +const SESSION_STATES = new Set([ + "accepted", + "provisioning", + "ready", + "cancelling", + "cancelled", + "failed", + "unavailable", + "indeterminate", +]); +const OPERATION_STATES = new Set([ + "accepted", + "rejected", + "conflict", + "unavailable", + "indeterminate", +]); + +type ApiError = components["schemas"]["ApiError"]; +type Resync = components["schemas"]["Resync"]; +type WatchEvent = components["schemas"]["WatchEvent"]; + +export interface T4ApiClientOptions { + readonly baseUrl: string; + readonly credential: string; + readonly majorVersion: number; + readonly fetch?: typeof globalThis.fetch; +} + +export interface WatchSessionOptions { + readonly cursor?: components["schemas"]["Cursor"]; + readonly maxEvents?: number; + readonly heartbeatSeconds?: number; + readonly signal?: AbortSignal; +} + +export interface T4ApiClient { + readonly http: Client; + watchSession(sessionId: string, options?: WatchSessionOptions): AsyncGenerator; +} + +export class T4ApiError extends Error { + readonly code: ApiError["code"]; + readonly status: number; + readonly requestId: string; + readonly retryable: boolean; + readonly violations?: ApiError["violations"]; + readonly supportedMajors?: ApiError["supportedMajors"]; + readonly resync?: Resync; + + constructor(status: number, error: ApiError) { + super(error.message); + this.name = "T4ApiError"; + this.code = error.code; + this.status = status; + this.requestId = error.requestId; + this.retryable = error.retryable; + if (error.violations !== undefined) this.violations = error.violations; + if (error.supportedMajors !== undefined) this.supportedMajors = error.supportedMajors; + if (error.resync !== undefined) this.resync = error.resync; + } +} + +function normalizedBaseUrl(value: string): string { + let url: URL; + try { + url = new URL(value); + } catch (error) { + throw new TypeError("T4 API baseUrl must be an absolute HTTPS URL", { cause: error }); + } + if ( + url.protocol !== "https:" || + url.username !== "" || + url.password !== "" || + url.search !== "" || + url.hash !== "" + ) { + throw new TypeError("T4 API baseUrl must be a credential-free HTTPS URL without query or fragment"); + } + url.pathname = url.pathname.replace(/\/+$/u, ""); + return url.toString().replace(/\/$/u, ""); +} + +function requiredCredential(value: string): string { + if ( + value.length === 0 || + value.length > MAX_CREDENTIAL_LENGTH || + !/^[A-Za-z0-9._~+/-]+=*$/u.test(value) + ) { + throw new TypeError("credential must be an opaque bearer token of at most 4096 characters"); + } + return value; +} + +function requiredMajor(value: number): string { + if (!Number.isSafeInteger(value) || value < 1 || value > 9999) { + throw new TypeError("majorVersion must be an integer between 1 and 9999"); + } + return String(value); +} + +function boundedInteger(value: number | undefined, fallback: number, minimum: number, maximum: number, label: string): number { + const selected = value ?? fallback; + if (!Number.isSafeInteger(selected) || selected < minimum || selected > maximum) { + throw new RangeError(`${label} must be an integer between ${minimum} and ${maximum}`); + } + return selected; +} + +function requiredSessionId(value: string): string { + if (value.length < 1 || value.length > 128 || !/^[A-Za-z0-9][A-Za-z0-9._~-]*$/u.test(value)) { + throw new TypeError("sessionId is invalid"); + } + return value; +} + +function record(value: unknown): Record | undefined { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? value as Record + : undefined; +} + +function apiError(value: unknown): ApiError | undefined { + const envelope = record(value); + const error = record(envelope?.error); + if ( + error === undefined || + typeof error.code !== "string" || + typeof error.message !== "string" || + typeof error.requestId !== "string" || + typeof error.retryable !== "boolean" + ) return undefined; + return error as ApiError; +} + +async function boundedError(response: Response): Promise { + const text = await response.text(); + if (new TextEncoder().encode(text).byteLength <= MAX_ERROR_BYTES) { + try { + const decoded = apiError(JSON.parse(text)); + if (decoded !== undefined) return new T4ApiError(response.status, decoded); + } catch { + // Fall through to the stable indeterminate transport envelope. + } + } + return new T4ApiError(response.status, { + code: response.status === 503 ? "unavailable" : "indeterminate", + message: "T4 API returned an invalid or oversized error envelope", + requestId: "unavailable", + retryable: response.status >= 500, + }); +} + +function watchEvent(value: unknown, eventId: string | undefined): WatchEvent { + const event = record(value); + if ( + event === undefined || + typeof event.type !== "string" || + typeof event.cursor !== "string" || + event.cursor.length < 1 || + event.cursor.length > 512 || + (eventId !== undefined && eventId !== event.cursor) + ) throw new T4ApiError(502, { code: "indeterminate", message: "T4 API returned an invalid watch event", requestId: "unavailable", retryable: true }); + if (event.type === "heartbeat" && typeof event.observedAt === "string" && event.observedAt.length <= 64) { + return event as WatchEvent; + } + if ( + event.type === "session" && + typeof event.state === "string" && + SESSION_STATES.has(event.state as components["schemas"]["SessionState"]) && + Number.isSafeInteger(event.revision) && Number(event.revision) >= 1 + ) return event as WatchEvent; + if ( + event.type === "command" && + typeof event.commandId === "string" && + event.commandId.length >= 1 && event.commandId.length <= 128 && + typeof event.state === "string" && + OPERATION_STATES.has(event.state as components["schemas"]["OperationState"]) + ) return event as WatchEvent; + throw new T4ApiError(502, { code: "indeterminate", message: "T4 API returned an invalid watch event", requestId: "unavailable", retryable: true }); +} + +function decodeSseFrame(frame: string): WatchEvent | undefined { + let eventId: string | undefined; + const data: string[] = []; + for (const rawLine of frame.split("\n")) { + const line = rawLine.endsWith("\r") ? rawLine.slice(0, -1) : rawLine; + if (line === "" || line.startsWith(":")) continue; + const separator = line.indexOf(":"); + const field = separator < 0 ? line : line.slice(0, separator); + const rawValue = separator < 0 ? "" : line.slice(separator + 1); + const value = rawValue.startsWith(" ") ? rawValue.slice(1) : rawValue; + if (field === "id") eventId = value; + else if (field === "data") data.push(value); + } + if (data.length === 0) return undefined; + try { + return watchEvent(JSON.parse(data.join("\n")), eventId); + } catch (error) { + if (error instanceof T4ApiError) throw error; + throw new T4ApiError(502, { code: "indeterminate", message: "T4 API returned malformed SSE data", requestId: "unavailable", retryable: true }); + } +} + +async function* watch( + baseUrl: string, + credential: string, + majorVersion: string, + fetchImpl: typeof globalThis.fetch, + sessionIdValue: string, + options: WatchSessionOptions, +): AsyncGenerator { + const sessionId = requiredSessionId(sessionIdValue); + const maxEvents = boundedInteger(options.maxEvents, 100, 1, 1000, "maxEvents"); + const heartbeatSeconds = boundedInteger(options.heartbeatSeconds, 15, 5, 60, "heartbeatSeconds"); + if (options.cursor !== undefined && (options.cursor.length < 1 || options.cursor.length > 512)) { + throw new TypeError("cursor must contain between 1 and 512 characters"); + } + const url = new URL(`${baseUrl}/v1/sessions/${encodeURIComponent(sessionId)}/events`); + url.searchParams.set("maxEvents", String(maxEvents)); + url.searchParams.set("heartbeatSeconds", String(heartbeatSeconds)); + if (options.cursor !== undefined) url.searchParams.set("cursor", options.cursor); + const controller = new AbortController(); + const abort = (): void => controller.abort(options.signal?.reason); + if (options.signal?.aborted === true) abort(); + else options.signal?.addEventListener("abort", abort, { once: true }); + let reader: ReadableStreamDefaultReader | undefined; + try { + const headers = new Headers({ + Accept: "text/event-stream", + Authorization: `Bearer ${credential}`, + "Cache-Control": "no-store", + "T4-API-Version": majorVersion, + }); + if (options.cursor !== undefined) headers.set("Last-Event-ID", options.cursor); + const response = await fetchImpl(url, { method: "GET", headers, signal: controller.signal }); + if (!response.ok) throw await boundedError(response); + if (!response.headers.get("content-type")?.toLowerCase().startsWith("text/event-stream")) { + throw new T4ApiError(502, { code: "indeterminate", message: "T4 API watch did not return text/event-stream", requestId: "unavailable", retryable: true }); + } + if (response.body === null) { + throw new T4ApiError(502, { code: "indeterminate", message: "T4 API watch response body is unavailable", requestId: "unavailable", retryable: true }); + } + reader = response.body.getReader(); + const decoder = new TextDecoder("utf-8", { fatal: true }); + let buffer = ""; + let delivered = 0; + while (delivered < maxEvents) { + if (controller.signal.aborted) return; + let chunk: ReadableStreamReadResult; + try { + chunk = await reader.read(); + } catch (error) { + if (controller.signal.aborted) return; + throw error; + } + if (chunk.done) { + buffer += decoder.decode(); + break; + } + buffer += decoder.decode(chunk.value, { stream: true }); + if (new TextEncoder().encode(buffer).byteLength > MAX_EVENT_BYTES) { + throw new T4ApiError(502, { code: "indeterminate", message: "T4 API watch event exceeds the client bound", requestId: "unavailable", retryable: true }); + } + let boundary = buffer.indexOf("\n\n"); + while (boundary >= 0 && delivered < maxEvents) { + const frame = buffer.slice(0, boundary); + buffer = buffer.slice(boundary + 2); + const event = decodeSseFrame(frame); + if (event !== undefined) { + delivered += 1; + yield event; + } + boundary = buffer.indexOf("\n\n"); + } + } + if (buffer.trim().length > 0 && delivered < maxEvents) { + const event = decodeSseFrame(buffer); + if (event !== undefined) yield event; + } + } finally { + options.signal?.removeEventListener("abort", abort); + controller.abort(); + if (reader !== undefined) { + try { await reader.cancel(); } catch { /* cancellation is best effort */ } + } + } +} + +export function createT4ApiClient(options: T4ApiClientOptions): T4ApiClient { + const baseUrl = normalizedBaseUrl(options.baseUrl); + const credential = requiredCredential(options.credential); + const majorVersion = requiredMajor(options.majorVersion); + const fetchImpl = options.fetch ?? globalThis.fetch; + const authenticatedFetch: typeof globalThis.fetch = async (input, init) => { + const request = new Request(input, init); + request.headers.set("Authorization", `Bearer ${credential}`); + request.headers.set("T4-API-Version", majorVersion); + request.headers.set("Accept", "application/json"); + return await fetchImpl(request); + }; + const http = createClient({ baseUrl, fetch: authenticatedFetch }); + return Object.freeze({ + http, + watchSession: (sessionId: string, watchOptions: WatchSessionOptions = {}) => + watch(baseUrl, credential, majorVersion, fetchImpl, sessionId, watchOptions), + }); +} diff --git a/packages/t4-api-client/tsconfig.json b/packages/t4-api-client/tsconfig.json new file mode 100644 index 00000000..824d57c3 --- /dev/null +++ b/packages/t4-api-client/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "types": ["node"] + }, + "include": ["src/**/*.ts"] +} diff --git a/packages/t4-api-contract/openapi.json b/packages/t4-api-contract/openapi.json new file mode 100644 index 00000000..863bdd88 --- /dev/null +++ b/packages/t4-api-contract/openapi.json @@ -0,0 +1,683 @@ +{ + "openapi": "3.1.0", + "info": { + "title": "T4 API", + "version": "1.0.0", + "summary": "Authenticated public API for bounded T4 workspace and session lifecycle", + "description": "The public boundary for T4 clients. The service is not assumed to be publicly exposed. Every operation requires an opaque bearer credential and deny-by-default server-side scope authorization. Kubernetes, PostgreSQL, controller, runtime, and private OMP representations are never exposed." + }, + "servers": [{ "url": "https://t4.example.invalid", "description": "Operator-configured private HTTPS endpoint" }], + "security": [{ "BearerAuth": [] }], + "tags": [ + { "name": "Discovery" }, + { "name": "Workspaces" }, + { "name": "Sessions" }, + { "name": "Watch" } + ], + "paths": { + "/v1": { + "get": { + "operationId": "discoverV1", + "tags": ["Discovery"], + "summary": "Negotiate T4 API v1 and discover capabilities and bounds", + "parameters": [{ "$ref": "#/components/parameters/ApiVersion" }], + "responses": { + "200": { "$ref": "#/components/responses/Discovery" }, + "401": { "$ref": "#/components/responses/Unauthenticated" }, + "403": { "$ref": "#/components/responses/Forbidden" }, + "406": { "$ref": "#/components/responses/IncompatibleVersion" }, + "503": { "$ref": "#/components/responses/Unavailable" } + } + } + }, + "/v1/workspaces": { + "get": { + "operationId": "listWorkspaces", + "tags": ["Workspaces"], + "parameters": [ + { "$ref": "#/components/parameters/ApiVersion" }, + { "$ref": "#/components/parameters/PageSize" }, + { "$ref": "#/components/parameters/PageCursor" } + ], + "responses": { + "200": { "$ref": "#/components/responses/WorkspacePage" }, + "400": { "$ref": "#/components/responses/BadRequest" }, + "401": { "$ref": "#/components/responses/Unauthenticated" }, + "403": { "$ref": "#/components/responses/Forbidden" }, + "406": { "$ref": "#/components/responses/IncompatibleVersion" }, + "422": { "$ref": "#/components/responses/InvalidRequest" }, + "503": { "$ref": "#/components/responses/Unavailable" } + } + }, + "post": { + "operationId": "createWorkspace", + "tags": ["Workspaces"], + "parameters": [ + { "$ref": "#/components/parameters/ApiVersion" }, + { "$ref": "#/components/parameters/IdempotencyKey" } + ], + "requestBody": { + "required": true, + "content": { "application/json": { "schema": { "$ref": "#/components/schemas/WorkspaceCreate" } } } + }, + "responses": { + "200": { "$ref": "#/components/responses/WorkspaceReplay" }, + "202": { "$ref": "#/components/responses/WorkspaceAccepted" }, + "400": { "$ref": "#/components/responses/BadRequest" }, + "401": { "$ref": "#/components/responses/Unauthenticated" }, + "403": { "$ref": "#/components/responses/Forbidden" }, + "406": { "$ref": "#/components/responses/IncompatibleVersion" }, + "409": { "$ref": "#/components/responses/Conflict" }, + "422": { "$ref": "#/components/responses/InvalidRequest" }, + "503": { "$ref": "#/components/responses/Unavailable" } + } + } + }, + "/v1/workspaces/{workspaceId}": { + "parameters": [{ "$ref": "#/components/parameters/WorkspaceId" }], + "get": { + "operationId": "getWorkspace", + "tags": ["Workspaces"], + "parameters": [{ "$ref": "#/components/parameters/ApiVersion" }], + "responses": { + "200": { "$ref": "#/components/responses/Workspace" }, + "401": { "$ref": "#/components/responses/Unauthenticated" }, + "403": { "$ref": "#/components/responses/Forbidden" }, + "404": { "$ref": "#/components/responses/NotFound" }, + "406": { "$ref": "#/components/responses/IncompatibleVersion" }, + "503": { "$ref": "#/components/responses/Unavailable" } + } + }, + "patch": { + "operationId": "mutateWorkspace", + "tags": ["Workspaces"], + "parameters": [ + { "$ref": "#/components/parameters/ApiVersion" }, + { "$ref": "#/components/parameters/IfMatch" } + ], + "requestBody": { + "required": true, + "content": { "application/json": { "schema": { "$ref": "#/components/schemas/WorkspaceMutation" } } } + }, + "responses": { + "200": { "$ref": "#/components/responses/Workspace" }, + "401": { "$ref": "#/components/responses/Unauthenticated" }, + "403": { "$ref": "#/components/responses/Forbidden" }, + "404": { "$ref": "#/components/responses/NotFound" }, + "406": { "$ref": "#/components/responses/IncompatibleVersion" }, + "409": { "$ref": "#/components/responses/Conflict" }, + "422": { "$ref": "#/components/responses/InvalidRequest" }, + "503": { "$ref": "#/components/responses/Unavailable" } + } + }, + "delete": { + "operationId": "deleteWorkspace", + "tags": ["Workspaces"], + "parameters": [{ "$ref": "#/components/parameters/ApiVersion" }], + "responses": { + "204": { "$ref": "#/components/responses/Deleted" }, + "401": { "$ref": "#/components/responses/Unauthenticated" }, + "403": { "$ref": "#/components/responses/Forbidden" }, + "404": { "$ref": "#/components/responses/NotFound" }, + "406": { "$ref": "#/components/responses/IncompatibleVersion" }, + "409": { "$ref": "#/components/responses/Conflict" }, + "503": { "$ref": "#/components/responses/Unavailable" } + } + } + }, + "/v1/workspaces/{workspaceId}/sessions": { + "parameters": [{ "$ref": "#/components/parameters/WorkspaceId" }], + "get": { + "operationId": "listSessions", + "tags": ["Sessions"], + "parameters": [ + { "$ref": "#/components/parameters/ApiVersion" }, + { "$ref": "#/components/parameters/PageSize" }, + { "$ref": "#/components/parameters/PageCursor" } + ], + "responses": { + "200": { "$ref": "#/components/responses/SessionPage" }, + "401": { "$ref": "#/components/responses/Unauthenticated" }, + "403": { "$ref": "#/components/responses/Forbidden" }, + "404": { "$ref": "#/components/responses/NotFound" }, + "406": { "$ref": "#/components/responses/IncompatibleVersion" }, + "422": { "$ref": "#/components/responses/InvalidRequest" }, + "503": { "$ref": "#/components/responses/Unavailable" } + } + }, + "post": { + "operationId": "spawnSession", + "tags": ["Sessions"], + "parameters": [ + { "$ref": "#/components/parameters/ApiVersion" }, + { "$ref": "#/components/parameters/IdempotencyKey" } + ], + "requestBody": { + "required": true, + "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SessionCreate" } } } + }, + "responses": { + "200": { "$ref": "#/components/responses/SessionReplay" }, + "202": { "$ref": "#/components/responses/SessionAccepted" }, + "401": { "$ref": "#/components/responses/Unauthenticated" }, + "403": { "$ref": "#/components/responses/Forbidden" }, + "404": { "$ref": "#/components/responses/NotFound" }, + "406": { "$ref": "#/components/responses/IncompatibleVersion" }, + "409": { "$ref": "#/components/responses/Conflict" }, + "422": { "$ref": "#/components/responses/InvalidRequest" }, + "503": { "$ref": "#/components/responses/Unavailable" } + } + } + }, + "/v1/sessions/{sessionId}": { + "parameters": [{ "$ref": "#/components/parameters/SessionId" }], + "get": { + "operationId": "getSession", + "tags": ["Sessions"], + "parameters": [{ "$ref": "#/components/parameters/ApiVersion" }], + "responses": { + "200": { "$ref": "#/components/responses/Session" }, + "401": { "$ref": "#/components/responses/Unauthenticated" }, + "403": { "$ref": "#/components/responses/Forbidden" }, + "404": { "$ref": "#/components/responses/NotFound" }, + "406": { "$ref": "#/components/responses/IncompatibleVersion" }, + "503": { "$ref": "#/components/responses/Unavailable" } + } + }, + "patch": { + "operationId": "mutateSession", + "tags": ["Sessions"], + "parameters": [ + { "$ref": "#/components/parameters/ApiVersion" }, + { "$ref": "#/components/parameters/IfMatch" } + ], + "requestBody": { + "required": true, + "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SessionMutation" } } } + }, + "responses": { + "200": { "$ref": "#/components/responses/Session" }, + "401": { "$ref": "#/components/responses/Unauthenticated" }, + "403": { "$ref": "#/components/responses/Forbidden" }, + "404": { "$ref": "#/components/responses/NotFound" }, + "406": { "$ref": "#/components/responses/IncompatibleVersion" }, + "409": { "$ref": "#/components/responses/Conflict" }, + "422": { "$ref": "#/components/responses/InvalidRequest" }, + "503": { "$ref": "#/components/responses/Unavailable" } + } + }, + "delete": { + "operationId": "deleteSession", + "tags": ["Sessions"], + "parameters": [{ "$ref": "#/components/parameters/ApiVersion" }], + "responses": { + "204": { "$ref": "#/components/responses/Deleted" }, + "401": { "$ref": "#/components/responses/Unauthenticated" }, + "403": { "$ref": "#/components/responses/Forbidden" }, + "404": { "$ref": "#/components/responses/NotFound" }, + "406": { "$ref": "#/components/responses/IncompatibleVersion" }, + "409": { "$ref": "#/components/responses/Conflict" }, + "503": { "$ref": "#/components/responses/Unavailable" } + } + } + }, + "/v1/sessions/{sessionId}/cancel": { + "post": { + "operationId": "cancelSession", + "tags": ["Sessions"], + "parameters": [ + { "$ref": "#/components/parameters/SessionId" }, + { "$ref": "#/components/parameters/ApiVersion" } + ], + "responses": { + "202": { "$ref": "#/components/responses/SessionAccepted" }, + "401": { "$ref": "#/components/responses/Unauthenticated" }, + "403": { "$ref": "#/components/responses/Forbidden" }, + "404": { "$ref": "#/components/responses/NotFound" }, + "406": { "$ref": "#/components/responses/IncompatibleVersion" }, + "409": { "$ref": "#/components/responses/Conflict" }, + "503": { "$ref": "#/components/responses/Unavailable" } + } + } + }, + "/v1/sessions/{sessionId}/commands": { + "post": { + "operationId": "submitCommand", + "tags": ["Sessions"], + "parameters": [ + { "$ref": "#/components/parameters/SessionId" }, + { "$ref": "#/components/parameters/ApiVersion" }, + { "$ref": "#/components/parameters/IdempotencyKey" } + ], + "requestBody": { + "required": true, + "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CommandCreate" } } } + }, + "responses": { + "200": { "$ref": "#/components/responses/CommandReplay" }, + "202": { "$ref": "#/components/responses/CommandAccepted" }, + "401": { "$ref": "#/components/responses/Unauthenticated" }, + "403": { "$ref": "#/components/responses/Forbidden" }, + "404": { "$ref": "#/components/responses/NotFound" }, + "406": { "$ref": "#/components/responses/IncompatibleVersion" }, + "409": { "$ref": "#/components/responses/Conflict" }, + "422": { "$ref": "#/components/responses/InvalidRequest" }, + "503": { "$ref": "#/components/responses/Unavailable" } + } + } + }, + "/v1/sessions/{sessionId}/snapshot": { + "get": { + "operationId": "getSessionSnapshot", + "tags": ["Watch"], + "parameters": [ + { "$ref": "#/components/parameters/SessionId" }, + { "$ref": "#/components/parameters/ApiVersion" } + ], + "responses": { + "200": { "$ref": "#/components/responses/Snapshot" }, + "401": { "$ref": "#/components/responses/Unauthenticated" }, + "403": { "$ref": "#/components/responses/Forbidden" }, + "404": { "$ref": "#/components/responses/NotFound" }, + "406": { "$ref": "#/components/responses/IncompatibleVersion" }, + "503": { "$ref": "#/components/responses/Unavailable" } + } + } + }, + "/v1/sessions/{sessionId}/events": { + "get": { + "operationId": "watchSessionEvents", + "tags": ["Watch"], + "summary": "Watch a bounded Server-Sent Events stream over HTTPS", + "description": "The response is standard Server-Sent Events over HTTPS. Clients reconnect with the last event cursor in the cursor query parameter or Last-Event-ID header. Heartbeat events bound idle detection. Clients explicitly cancel by aborting the HTTPS request. A retained cursor expiry returns a typed 410 response with a snapshot resync target.", + "parameters": [ + { "$ref": "#/components/parameters/SessionId" }, + { "$ref": "#/components/parameters/ApiVersion" }, + { "$ref": "#/components/parameters/WatchCursor" }, + { "$ref": "#/components/parameters/LastEventId" }, + { "$ref": "#/components/parameters/MaxEvents" }, + { "$ref": "#/components/parameters/HeartbeatSeconds" } + ], + "responses": { + "200": { + "description": "Bounded SSE event stream; each data field is a WatchEvent JSON value", + "headers": { + "Cache-Control": { "schema": { "type": "string", "const": "no-store" } }, + "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } + }, + "content": { + "text/event-stream": { + "schema": { "type": "string" }, + "example": "id: cur_01J...\\nevent: heartbeat\\ndata: {\"type\":\"heartbeat\",\"cursor\":\"cur_01J...\",\"observedAt\":\"2026-07-21T00:00:00Z\"}\\n\\n" + } + } + }, + "400": { "$ref": "#/components/responses/BadRequest" }, + "401": { "$ref": "#/components/responses/Unauthenticated" }, + "403": { "$ref": "#/components/responses/Forbidden" }, + "404": { "$ref": "#/components/responses/NotFound" }, + "406": { "$ref": "#/components/responses/IncompatibleVersion" }, + "410": { "$ref": "#/components/responses/CursorExpired" }, + "422": { "$ref": "#/components/responses/InvalidRequest" }, + "503": { "$ref": "#/components/responses/Unavailable" } + } + } + } + }, + "components": { + "securitySchemes": { + "BearerAuth": { + "type": "http", + "scheme": "bearer", + "description": "Opaque bearer credential. The server maps it to deny-by-default tenant and resource scopes; clients must not inspect or persist credential contents in API resources." + } + }, + "parameters": { + "ApiVersion": { + "name": "T4-API-Version", + "in": "header", + "required": true, + "description": "Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading.", + "schema": { "type": "string", "pattern": "^1(?:\\.[0-9]+)?$", "maxLength": 16 } + }, + "IdempotencyKey": { + "name": "Idempotency-Key", + "in": "header", + "required": true, + "description": "Opaque caller-generated key scoped to credential, operation, and target. Same key plus same canonical request replays; different content conflicts.", + "schema": { "type": "string", "minLength": 16, "maxLength": 128, "pattern": "^[A-Za-z0-9._~-]+$" } + }, + "IfMatch": { + "name": "If-Match", + "in": "header", + "required": true, + "description": "Decimal resource revision for optimistic mutation.", + "schema": { "type": "string", "pattern": "^[1-9][0-9]{0,18}$" } + }, + "WorkspaceId": { + "name": "workspaceId", + "in": "path", + "required": true, + "schema": { "$ref": "#/components/schemas/ResourceId" } + }, + "SessionId": { + "name": "sessionId", + "in": "path", + "required": true, + "schema": { "$ref": "#/components/schemas/ResourceId" } + }, + "PageSize": { + "name": "pageSize", + "in": "query", + "schema": { "type": "integer", "minimum": 1, "maximum": 100, "default": 25 } + }, + "PageCursor": { + "name": "cursor", + "in": "query", + "description": "Opaque page cursor, valid only for the same identity and list operation.", + "schema": { "$ref": "#/components/schemas/Cursor" } + }, + "WatchCursor": { + "name": "cursor", + "in": "query", + "description": "Opaque last-consumed watch event cursor.", + "schema": { "$ref": "#/components/schemas/Cursor" } + }, + "LastEventId": { + "name": "Last-Event-ID", + "in": "header", + "description": "Standard SSE reconnect header. Must agree with cursor when both are present.", + "schema": { "$ref": "#/components/schemas/Cursor" } + }, + "MaxEvents": { + "name": "maxEvents", + "in": "query", + "schema": { "type": "integer", "minimum": 1, "maximum": 1000, "default": 100 } + }, + "HeartbeatSeconds": { + "name": "heartbeatSeconds", + "in": "query", + "schema": { "type": "integer", "minimum": 5, "maximum": 60, "default": 15 } + } + }, + "headers": { + "SelectedVersion": { + "description": "Selected compatible T4 API minor version.", + "schema": { "type": "string", "pattern": "^1\\.[0-9]+$" } + }, + "IdempotencyReplayed": { + "description": "True when this response replays a prior identical request.", + "schema": { "type": "string", "enum": ["true", "false"] } + } + }, + "schemas": { + "ResourceId": { "type": "string", "minLength": 1, "maxLength": 128, "pattern": "^[A-Za-z0-9][A-Za-z0-9._~-]*$" }, + "Cursor": { "type": "string", "minLength": 1, "maxLength": 512, "description": "Opaque server-issued cursor." }, + "Revision": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 }, + "OperationState": { "type": "string", "enum": ["accepted", "rejected", "conflict", "unavailable", "indeterminate"] }, + "WorkspaceState": { "type": "string", "enum": ["accepted", "provisioning", "ready", "deleting", "deleted", "failed", "unavailable", "indeterminate"] }, + "SessionState": { "type": "string", "enum": ["accepted", "provisioning", "ready", "cancelling", "cancelled", "failed", "unavailable", "indeterminate"] }, + "Discovery": { + "type": "object", + "additionalProperties": false, + "required": ["apiVersion", "supportedMajors", "capabilities", "limits"], + "properties": { + "apiVersion": { "type": "string", "pattern": "^1\\.[0-9]+$" }, + "supportedMajors": { "type": "array", "minItems": 1, "maxItems": 8, "uniqueItems": true, "items": { "type": "integer", "minimum": 1 } }, + "capabilities": { "type": "array", "maxItems": 128, "uniqueItems": true, "items": { "type": "string", "minLength": 1, "maxLength": 128, "pattern": "^[a-z][a-z0-9.-]*$" } }, + "limits": { + "type": "object", + "additionalProperties": false, + "required": ["pageSizeDefault", "pageSizeMax", "commandBytesMax", "watchEventsMax", "heartbeatSeconds"], + "properties": { + "pageSizeDefault": { "type": "integer", "minimum": 1, "maximum": 100 }, + "pageSizeMax": { "type": "integer", "minimum": 1, "maximum": 100 }, + "commandBytesMax": { "type": "integer", "minimum": 1, "maximum": 1048576 }, + "watchEventsMax": { "type": "integer", "minimum": 1, "maximum": 1000 }, + "heartbeatSeconds": { "type": "integer", "minimum": 5, "maximum": 60 } + } + } + } + }, + "WorkspaceCreate": { + "type": "object", + "additionalProperties": false, + "required": ["name"], + "properties": { + "name": { "type": "string", "minLength": 1, "maxLength": 128 }, + "labels": { "$ref": "#/components/schemas/Labels" } + } + }, + "WorkspaceMutation": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "name": { "type": "string", "minLength": 1, "maxLength": 128 }, + "labels": { "$ref": "#/components/schemas/Labels" } + } + }, + "Workspace": { + "type": "object", + "additionalProperties": false, + "required": ["id", "name", "state", "revision"], + "properties": { + "id": { "$ref": "#/components/schemas/ResourceId" }, + "name": { "type": "string", "minLength": 1, "maxLength": 128 }, + "state": { "$ref": "#/components/schemas/WorkspaceState" }, + "revision": { "$ref": "#/components/schemas/Revision" }, + "labels": { "$ref": "#/components/schemas/Labels" } + } + }, + "WorkspacePage": { + "type": "object", + "additionalProperties": false, + "required": ["items"], + "properties": { + "items": { "type": "array", "maxItems": 100, "items": { "$ref": "#/components/schemas/Workspace" } }, + "nextCursor": { "$ref": "#/components/schemas/Cursor" } + } + }, + "SessionCreate": { + "type": "object", + "additionalProperties": false, + "required": ["title"], + "properties": { + "title": { "type": "string", "minLength": 1, "maxLength": 128 }, + "labels": { "$ref": "#/components/schemas/Labels" } + } + }, + "SessionMutation": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "title": { "type": "string", "minLength": 1, "maxLength": 128 }, + "labels": { "$ref": "#/components/schemas/Labels" } + } + }, + "Session": { + "type": "object", + "additionalProperties": false, + "required": ["id", "workspaceId", "title", "state", "revision"], + "properties": { + "id": { "$ref": "#/components/schemas/ResourceId" }, + "workspaceId": { "$ref": "#/components/schemas/ResourceId" }, + "title": { "type": "string", "minLength": 1, "maxLength": 128 }, + "state": { "$ref": "#/components/schemas/SessionState" }, + "revision": { "$ref": "#/components/schemas/Revision" }, + "labels": { "$ref": "#/components/schemas/Labels" } + } + }, + "SessionPage": { + "type": "object", + "additionalProperties": false, + "required": ["items"], + "properties": { + "items": { "type": "array", "maxItems": 100, "items": { "$ref": "#/components/schemas/Session" } }, + "nextCursor": { "$ref": "#/components/schemas/Cursor" } + } + }, + "Labels": { + "type": "object", + "maxProperties": 32, + "propertyNames": { "pattern": "^[a-z][a-z0-9.-]{0,62}$" }, + "additionalProperties": { "type": "string", "maxLength": 128 } + }, + "CommandCreate": { + "type": "object", + "additionalProperties": false, + "required": ["command"], + "properties": { + "command": { "type": "string", "minLength": 1, "maxLength": 262144 }, + "metadata": { "type": "object", "maxProperties": 32, "additionalProperties": { "type": ["string", "number", "boolean", "null"] } } + } + }, + "CommandResult": { + "type": "object", + "additionalProperties": false, + "required": ["commandId", "state"], + "properties": { + "commandId": { "$ref": "#/components/schemas/ResourceId" }, + "state": { "$ref": "#/components/schemas/OperationState" } + } + }, + "SnapshotEntry": { + "type": "object", + "additionalProperties": false, + "required": ["sequence", "kind", "text"], + "properties": { + "sequence": { "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, + "kind": { "type": "string", "enum": ["input", "output", "status"] }, + "text": { "type": "string", "maxLength": 1048576 } + } + }, + "SessionSnapshot": { + "type": "object", + "additionalProperties": false, + "required": ["sessionId", "cursor", "state", "entries"], + "properties": { + "sessionId": { "$ref": "#/components/schemas/ResourceId" }, + "cursor": { "$ref": "#/components/schemas/Cursor" }, + "state": { "$ref": "#/components/schemas/SessionState" }, + "entries": { "type": "array", "maxItems": 1000, "items": { "$ref": "#/components/schemas/SnapshotEntry" } } + } + }, + "HeartbeatWatchEvent": { + "type": "object", + "additionalProperties": false, + "required": ["type", "cursor", "observedAt"], + "properties": { + "type": { "type": "string", "const": "heartbeat" }, + "cursor": { "$ref": "#/components/schemas/Cursor" }, + "observedAt": { "type": "string", "format": "date-time" } + } + }, + "SessionWatchEvent": { + "type": "object", + "additionalProperties": false, + "required": ["type", "cursor", "state", "revision"], + "properties": { + "type": { "type": "string", "const": "session" }, + "cursor": { "$ref": "#/components/schemas/Cursor" }, + "state": { "$ref": "#/components/schemas/SessionState" }, + "revision": { "$ref": "#/components/schemas/Revision" } + } + }, + "CommandWatchEvent": { + "type": "object", + "additionalProperties": false, + "required": ["type", "cursor", "commandId", "state"], + "properties": { + "type": { "type": "string", "const": "command" }, + "cursor": { "$ref": "#/components/schemas/Cursor" }, + "commandId": { "$ref": "#/components/schemas/ResourceId" }, + "state": { "$ref": "#/components/schemas/OperationState" } + } + }, + "WatchEvent": { + "oneOf": [ + { "$ref": "#/components/schemas/HeartbeatWatchEvent" }, + { "$ref": "#/components/schemas/SessionWatchEvent" }, + { "$ref": "#/components/schemas/CommandWatchEvent" } + ], + "discriminator": { "propertyName": "type" } + }, + "FieldViolation": { + "type": "object", + "additionalProperties": false, + "required": ["field", "rule", "message"], + "properties": { + "field": { "type": "string", "minLength": 1, "maxLength": 256 }, + "rule": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[A-Za-z][A-Za-z0-9._-]*$" }, + "message": { "type": "string", "minLength": 1, "maxLength": 512 } + } + }, + "Resync": { + "type": "object", + "additionalProperties": false, + "required": ["snapshotUrl", "cursor"], + "properties": { + "snapshotUrl": { "type": "string", "pattern": "^/v1/sessions/[A-Za-z0-9._~-]+/snapshot$", "maxLength": 512 }, + "cursor": { "$ref": "#/components/schemas/Cursor" } + } + }, + "ApiError": { + "type": "object", + "additionalProperties": false, + "required": ["code", "message", "requestId", "retryable"], + "properties": { + "code": { + "type": "string", + "enum": ["invalid_request", "unauthenticated", "forbidden", "not_found", "idempotency_key_required", "idempotency_conflict", "revision_conflict", "incompatible_version", "cursor_expired", "unavailable", "indeterminate", "invalid_origin", "https_required"] + }, + "message": { "type": "string", "minLength": 1, "maxLength": 1024 }, + "requestId": { "type": "string", "minLength": 1, "maxLength": 128 }, + "retryable": { "type": "boolean" }, + "violations": { "type": "array", "maxItems": 64, "items": { "$ref": "#/components/schemas/FieldViolation" } }, + "supportedMajors": { "type": "array", "maxItems": 8, "uniqueItems": true, "items": { "type": "integer", "minimum": 1 } }, + "resync": { "$ref": "#/components/schemas/Resync" } + } + }, + "ErrorEnvelope": { + "type": "object", + "additionalProperties": false, + "required": ["error"], + "properties": { "error": { "$ref": "#/components/schemas/ApiError" } } + } + }, + "responses": { + "Discovery": { "description": "Negotiated v1 discovery", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Discovery" } } } }, + "Workspace": { "description": "Workspace", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Workspace" } } } }, + "WorkspaceAccepted": { "description": "Workspace intent accepted", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Workspace" } } } }, + "WorkspaceReplay": { "description": "Identical workspace request replay", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Workspace" } } } }, + "WorkspacePage": { "description": "Bounded workspace page", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/WorkspacePage" } } } }, + "Session": { "description": "Session", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Session" } } } }, + "SessionAccepted": { "description": "Session intent accepted", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Session" } } } }, + "SessionReplay": { "description": "Identical session spawn replay", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Session" } } } }, + "SessionPage": { "description": "Bounded session page", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SessionPage" } } } }, + "CommandAccepted": { "description": "Command intent accepted with a stable outcome state", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CommandResult" } } } }, + "CommandReplay": { "description": "Identical command request replay", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CommandResult" } } } }, + "Snapshot": { "description": "Bounded session snapshot and reconnect cursor", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SessionSnapshot" } } } }, + "Deleted": { "description": "Deletion accepted or resource already absent", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } } }, + "BadRequest": { "$ref": "#/components/responses/Error400" }, + "Unauthenticated": { "$ref": "#/components/responses/Error401" }, + "Forbidden": { "$ref": "#/components/responses/Error403" }, + "NotFound": { "$ref": "#/components/responses/Error404" }, + "IncompatibleVersion": { "$ref": "#/components/responses/Error406" }, + "Conflict": { "$ref": "#/components/responses/Error409" }, + "CursorExpired": { "$ref": "#/components/responses/Error410" }, + "InvalidRequest": { "$ref": "#/components/responses/Error422" }, + "Unavailable": { "$ref": "#/components/responses/Error503" }, + "Error400": { "description": "Malformed request or missing idempotency key", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorEnvelope" } } } }, + "Error401": { "description": "Missing or invalid opaque bearer credential", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorEnvelope" } } } }, + "Error403": { "description": "Credential lacks the deny-by-default operation or resource scope", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorEnvelope" } } } }, + "Error404": { "description": "Resource absent or outside caller scope (scope existence is not disclosed)", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorEnvelope" } } } }, + "Error406": { "description": "Requested major is incompatible; no silent downgrade", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorEnvelope" } } } }, + "Error409": { "description": "Idempotency or resource revision conflict", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorEnvelope" } } } }, + "Error410": { "description": "Watch cursor expired; resync from the typed snapshot target", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorEnvelope" } } } }, + "Error422": { "description": "Bounded semantic validation failure with stable field violations", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorEnvelope" } } } }, + "Error503": { "description": "Service temporarily unavailable or outcome indeterminate", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorEnvelope" } } } } + } + } +} diff --git a/packages/t4-api-contract/package.json b/packages/t4-api-contract/package.json new file mode 100644 index 00000000..641b913d --- /dev/null +++ b/packages/t4-api-contract/package.json @@ -0,0 +1,22 @@ +{ + "name": "@t4-code/t4-api-contract", + "version": "0.1.30", + "private": true, + "type": "module", + "exports": { + "./openapi.json": "./openapi.json" + }, + "scripts": { + "build": "pnpm validate && pnpm check:generated", + "typecheck": "pnpm validate && pnpm check:generated", + "test": "pnpm validate && pnpm check:generated", + "validate": "node scripts/validate.mjs", + "generate": "node scripts/generate.mjs --write", + "generate:ci": "node scripts/generate.mjs --ci-artifact", + "check:generated": "node scripts/generate.mjs --check" + }, + "devDependencies": { + "@readme/openapi-parser": "6.3.0", + "openapi-typescript": "7.13.0" + } +} diff --git a/packages/t4-api-contract/scripts/generate.mjs b/packages/t4-api-contract/scripts/generate.mjs new file mode 100644 index 00000000..3bc58f15 --- /dev/null +++ b/packages/t4-api-contract/scripts/generate.mjs @@ -0,0 +1,44 @@ +import { createHash } from "node:crypto"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import openapiTS, { astToString } from "openapi-typescript"; + +const contractRoot = fileURLToPath(new URL("..", import.meta.url)); +const schemaUrl = new URL("../openapi.json", import.meta.url); +const target = resolve(contractRoot, "../t4-api-client/src/generated/schema.ts"); +const mode = process.argv[2]; +if (!new Set(["--write", "--check", "--ci-artifact"]).has(mode)) { + throw new Error("usage: node scripts/generate.mjs --write|--check|--ci-artifact"); +} + +const generated = `${astToString(await openapiTS(schemaUrl, { alphabetize: true }))}\n`; +const digest = createHash("sha256").update(generated).digest("hex"); + +if (mode === "--write") { + await mkdir(dirname(target), { recursive: true }); + await writeFile(target, generated); + console.log(`generated ${target} sha256:${digest}`); + process.exit(0); +} + +if (mode === "--ci-artifact") { + const artifact = resolve(contractRoot, "../../artifacts/t4-api/generated/schema.ts"); + await mkdir(dirname(artifact), { recursive: true }); + await writeFile(artifact, generated); + console.log(`T4_API_GENERATED_SHA256=${digest}`); + console.log(`T4_API_GENERATED_BASE64_BEGIN\n${Buffer.from(generated).toString("base64")}\nT4_API_GENERATED_BASE64_END`); +} + +let checkedIn; +try { + checkedIn = await readFile(target, "utf8"); +} catch (error) { + if (mode === "--ci-artifact") throw new Error(`generated client is absent; recover the CI artifact at artifacts/t4-api/generated/schema.ts (sha256:${digest})`, { cause: error }); + throw error; +} +if (checkedIn !== generated) { + throw new Error(`generated client drift: expected sha256:${digest}; regenerate only with the authorized CI artifact`); +} +console.log(`generated client is deterministic (sha256:${digest})`); diff --git a/packages/t4-api-contract/scripts/validate.mjs b/packages/t4-api-contract/scripts/validate.mjs new file mode 100644 index 00000000..521893c0 --- /dev/null +++ b/packages/t4-api-contract/scripts/validate.mjs @@ -0,0 +1,20 @@ +import { readFile } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; + +import OpenAPIParser from "@readme/openapi-parser"; + +const source = new URL("../openapi.json", import.meta.url); +const document = JSON.parse(await readFile(source, "utf8")); +await OpenAPIParser.validate(fileURLToPath(source)); + +if (document.openapi !== "3.1.0") throw new Error("T4 API contract must remain OpenAPI 3.1.0"); +if (!Array.isArray(document.servers) || document.servers.length === 0) throw new Error("T4 API contract requires an HTTPS server"); +for (const server of document.servers) { + if (new URL(server.url).protocol !== "https:") throw new Error("T4 API contract permits only HTTPS servers"); +} +if (document.security?.[0]?.BearerAuth === undefined) throw new Error("T4 API contract must default to bearer authentication"); +const serializedSchemas = JSON.stringify(document.components?.schemas ?? {}).toLowerCase(); +for (const privateType of ["kubernetes.io", "v1alpha1", "postgresql", "ompserver", "podspec"]) { + if (serializedSchemas.includes(privateType)) throw new Error(`public schema leaks private type ${privateType}`); +} +console.log("T4 API OpenAPI 3.1 contract is valid"); diff --git a/packages/t4-api-contract/tsconfig.json b/packages/t4-api-contract/tsconfig.json new file mode 100644 index 00000000..9e7ef5aa --- /dev/null +++ b/packages/t4-api-contract/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "types": ["node"] + }, + "include": [] +} From 329ac183ea1e9313724118866d2368b05bac7ae6 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 19:41:13 +0000 Subject: [PATCH 003/131] ci: emit T4 API generation artifacts --- .github/workflows/ci.yml | 38 +++++ .../test/t4-api-v1-conformance-service.ts | 79 ++++++++--- .../client/test/t4-api-v1-conformance.test.ts | 130 +++++++++++++----- packages/t4-api-client/src/index.ts | 129 ++++++++++++----- packages/t4-api-contract/openapi.json | 46 ++++++- 5 files changed, 326 insertions(+), 96 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c8872e37..ab9e373f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,6 +57,44 @@ jobs: [[ "$HEAD_SHA" =~ ^[0-9a-f]{40}$ ]] git diff --name-only -z "$BASE_SHA...$HEAD_SHA" | node scripts/ci-paths.mjs >> "$GITHUB_OUTPUT" + t4-api-generation: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Check out exact source + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Install pinned pnpm + uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4 + with: + version: 11.10.0 + + - name: Install pinned Node.js + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: 24.13.1 + + - name: Resolve lockfile and generation dependencies for artifact recovery + run: pnpm install --no-frozen-lockfile + + - name: Validate and generate the T4 API contract artifact + run: | + pnpm --filter @t4-code/t4-api-contract validate + pnpm --filter @t4-code/t4-api-contract generate:ci + + - name: Upload deterministic T4 API generation inputs + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: t4-api-generation-${{ github.run_id }} + path: | + pnpm-lock.yaml + artifacts/t4-api/generated/schema.ts + if-no-files-found: error + retention-days: 14 + core: runs-on: ubuntu-24.04 timeout-minutes: 25 diff --git a/packages/client/test/t4-api-v1-conformance-service.ts b/packages/client/test/t4-api-v1-conformance-service.ts index d783f171..2e9cb745 100644 --- a/packages/client/test/t4-api-v1-conformance-service.ts +++ b/packages/client/test/t4-api-v1-conformance-service.ts @@ -19,16 +19,25 @@ function problem(status: number, code: string, message: string, extra: Record) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, item]) => `${JSON.stringify(key)}:${bodyKey(item)}`) + .join(",")}}`; + } + return JSON.stringify(value) ?? "null"; } export class T4ApiV1ConformanceService { readonly origin = "https://t4-api.conformance.test"; readonly calls: Array<{ method: string; path: string; authorization: string | null }> = []; readonly abortedWatches: string[] = []; + readonly watchCursors: Array<{ query: string | null; header: string | null }> = []; #workspaceSequence = 0; #sessionSequence = 0; + #commandSequence = 0; readonly #workspaces = new Map>(); readonly #sessions = new Map>(); readonly #replays = new Map }>(); @@ -72,9 +81,9 @@ export class T4ApiV1ConformanceService { if (request.method === "POST" && url.pathname === "/v1/workspaces") { const body = await request.json() as Record; - if (typeof body.name !== "string" || body.name.length < 1 || body.name.length > 24) { + if (typeof body.name !== "string" || body.name.length < 1 || body.name.length > 128) { return problem(422, "invalid_request", "Request validation failed", { - violations: [{ field: "name", rule: "length", message: "name must contain 1 to 24 characters" }], + violations: [{ field: "name", rule: "length", message: "name must contain 1 to 128 characters" }], }); } return this.#idempotent(request, tenant, body, () => { @@ -131,6 +140,11 @@ export class T4ApiV1ConformanceService { if (workspace?.tenant !== tenant) return problem(404, "not_found", "Workspace not found"); if (request.method === "POST") { const body = await request.json() as Record; + if (typeof body.title !== "string" || body.title.length < 1 || body.title.length > 128) { + return problem(422, "invalid_request", "Request validation failed", { + violations: [{ field: "title", rule: "length", message: "title must contain 1 to 128 characters" }], + }); + } return this.#idempotent(request, tenant, body, () => { const id = `ses-${++this.#sessionSequence}`; const session = { id, workspaceId, title: body.title, state: "accepted", revision: 1, tenant }; @@ -139,10 +153,17 @@ export class T4ApiV1ConformanceService { }); } if (request.method === "GET") { - const items = [...this.#sessions.values()] - .filter((item) => item.workspaceId === workspaceId && item.tenant === tenant) - .map(({ tenant: _tenant, ...item }) => item); - return json(200, { items }); + const pageSize = Number(url.searchParams.get("pageSize") ?? "2"); + if (!Number.isInteger(pageSize) || pageSize < 1 || pageSize > 3) { + return problem(422, "invalid_request", "Request validation failed", { + violations: [{ field: "pageSize", rule: "range", message: "pageSize must be between 1 and 3" }], + }); + } + const start = Number(url.searchParams.get("cursor")?.replace("page-", "") ?? "0"); + const visible = [...this.#sessions.values()].filter((item) => item.workspaceId === workspaceId && item.tenant === tenant); + const items = visible.slice(start, start + pageSize).map(({ tenant: _tenant, ...item }) => item); + const next = start + items.length < visible.length ? `page-${start + items.length}` : undefined; + return json(200, { items, ...(next === undefined ? {} : { nextCursor: next }) }); } } @@ -157,6 +178,9 @@ export class T4ApiV1ConformanceService { } if (request.method === "PATCH") { const body = await request.json() as Record; + if (request.headers.get("If-Match") !== String(session.revision)) { + return problem(409, "revision_conflict", "Session revision changed"); + } const updated = { ...session, title: body.title ?? session.title, revision: Number(session.revision) + 1 }; this.#sessions.set(id, updated); const { tenant: _tenant, ...visible } = updated; @@ -183,15 +207,14 @@ export class T4ApiV1ConformanceService { const session = this.#sessions.get(decodeURIComponent(commandMatch[1]!)); if (session?.tenant !== tenant) return problem(404, "not_found", "Session not found"); const body = await request.json() as Record; - if (typeof body.command !== "string" || encoder.encode(body.command).byteLength > 32) { + if (typeof body.command !== "string" || body.command.length < 1 || encoder.encode(body.command).byteLength > 32) { return problem(422, "invalid_request", "Request validation failed", { - violations: [{ field: "command", rule: "maxBytes", message: "command must not exceed 32 UTF-8 bytes" }], + violations: [{ field: "command", rule: "maxBytes", message: "command must contain 1 to 32 UTF-8 bytes" }], }); } - const state = ["accepted", "rejected", "conflict", "unavailable", "indeterminate"].includes(body.command) - ? body.command - : "accepted"; - return this.#idempotent(request, tenant, body, () => ({ commandId: `cmd-${body.command}`, state })); + const states: Record = { accepted: true, rejected: true, conflict: true, unavailable: true, indeterminate: true }; + const state = states[body.command] === true ? body.command : "accepted"; + return this.#idempotent(request, tenant, body, () => ({ commandId: `cmd-${++this.#commandSequence}`, state })); } const snapshotMatch = url.pathname.match(/^\/v1\/sessions\/([^/]+)\/snapshot$/u); @@ -206,19 +229,30 @@ export class T4ApiV1ConformanceService { const id = decodeURIComponent(watchMatch[1]!); const session = this.#sessions.get(id); if (session?.tenant !== tenant) return problem(404, "not_found", "Session not found"); - const cursor = url.searchParams.get("cursor") ?? request.headers.get("Last-Event-ID"); + const queryCursor = url.searchParams.get("cursor"); + const headerCursor = request.headers.get("Last-Event-ID"); + this.watchCursors.push({ query: queryCursor, header: headerCursor }); + if (queryCursor !== null && headerCursor !== null && queryCursor !== headerCursor) { + return problem(400, "invalid_request", "Reconnect cursors disagree"); + } + const cursor = queryCursor ?? headerCursor; if (cursor === "expired") { return problem(410, "cursor_expired", "Watch cursor is no longer retained", { resync: { snapshotUrl: `/v1/sessions/${id}/snapshot`, cursor: "cursor-2" }, }); } - const frames = [ - `id: cursor-3\nevent: heartbeat\ndata: {"type":"heartbeat","cursor":"cursor-3","observedAt":"2026-07-21T00:00:00Z"}\n\n`, - `id: cursor-4\nevent: session\ndata: {"type":"session","cursor":"cursor-4","state":"accepted","revision":2}\n\n`, - ]; + const frames = cursor === "cursor-4" + ? [`id: cursor-5\r\nevent: heartbeat\r\ndata: {"type":"heartbeat","cursor":"cursor-5","observedAt":"2026-07-21T00:00:15Z"}\r\n\r\n`] + : [ + `id: cursor-3\r\nevent: heartbeat\r\ndata: {"type":"heartbeat","cursor":"cursor-3","observedAt":"2026-07-21T00:00:00Z"}\r\n\r\n`, + `id: cursor-4\r\nevent: session\r\ndata: {"type":"session","cursor":"cursor-4","state":"accepted","revision":2}\r\n\r\n`, + ]; const stream = new ReadableStream({ start: (controller) => { - for (const frame of frames) controller.enqueue(encoder.encode(frame)); + const payload = encoder.encode(frames.join("")); + const split = Math.max(1, payload.byteLength - 3); + controller.enqueue(payload.slice(0, split)); + controller.enqueue(payload.slice(split)); request.signal.addEventListener("abort", () => { this.abortedWatches.push(id); try { controller.close(); } catch { /* already closed */ } @@ -245,7 +279,12 @@ export class T4ApiV1ConformanceService { create: () => Record, ): Response { const key = request.headers.get("Idempotency-Key"); - if (!key) return problem(400, "idempotency_key_required", "Idempotency-Key is required"); + if (key === null) return problem(400, "idempotency_key_required", "Idempotency-Key is required"); + if (!/^[A-Za-z0-9._~-]{16,128}$/u.test(key)) { + return problem(400, "invalid_request", "Idempotency-Key is invalid", { + violations: [{ field: "Idempotency-Key", rule: "format", message: "Idempotency-Key must be a 16 to 128 character token" }], + }); + } const replayKey = `${tenant}:${request.method}:${new URL(request.url).pathname}:${key}`; const prior = this.#replays.get(replayKey); if (prior) { diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index 4eb69701..7d312024 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -11,6 +11,16 @@ type WorkspaceCreate = components["schemas"]["WorkspaceCreate"]; type SessionCreate = components["schemas"]["SessionCreate"]; type CommandCreate = components["schemas"]["CommandCreate"]; +const VERSION_HEADERS = { "T4-API-Version": "1" } as const; + +function idempotencyHeaders(key: string): Readonly<{ "T4-API-Version": "1"; "Idempotency-Key": string }> { + return { ...VERSION_HEADERS, "Idempotency-Key": key }; +} + +function revisionHeaders(revision: number): Readonly<{ "T4-API-Version": "1"; "If-Match": string }> { + return { ...VERSION_HEADERS, "If-Match": String(revision) }; +} + function requireData(result: { data?: T; error?: unknown }): T { expect(result.error).toBeUndefined(); expect(result.data).toBeDefined(); @@ -21,7 +31,7 @@ describe("generated T4 API v1 client conformance", () => { it("negotiates discovery, capabilities, bounds, and rejects an incompatible major", async () => { const service = new T4ApiV1ConformanceService(); const client = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); - const discovery = requireData(await client.http.GET("/v1")); + const discovery = requireData(await client.http.GET("/v1", { params: { header: VERSION_HEADERS } })); expect(discovery).toMatchObject({ apiVersion: "1.0", supportedMajors: [1], @@ -30,121 +40,170 @@ describe("generated T4 API v1 client conformance", () => { }); const incompatible = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 2, fetch: service.fetch }); - const rejected = await incompatible.http.GET("/v1"); + const rejected = await incompatible.http.GET("/v1", { params: { header: VERSION_HEADERS } }); expect(rejected.response.status).toBe(406); expect(rejected.error).toMatchObject({ error: { code: "incompatible_version", retryable: false, supportedMajors: [1] } }); }); - it("creates, replays, conflicts, mutates, lists, isolates, and deletes bounded workspaces", async () => { + it("creates, canonically replays, conflicts, mutates, paginates, isolates, and deletes workspaces", async () => { const service = new T4ApiV1ConformanceService(); const owner = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); const other = createT4ApiClient({ baseUrl: service.origin, credential: "token-b", majorVersion: 1, fetch: service.fetch }); - const body = { name: "primary" } satisfies WorkspaceCreate; - const first = await owner.http.POST("/v1/workspaces", { body, headers: { "Idempotency-Key": "workspace-create-1" } }); + const body = { name: "primary", labels: { beta: "2", alpha: "1" } } satisfies WorkspaceCreate; + const first = await owner.http.POST("/v1/workspaces", { + body, + params: { header: idempotencyHeaders("workspace-create-0001") }, + }); expect(first.response.status).toBe(202); const workspace = requireData(first); expect(workspace).toMatchObject({ id: "ws-1", state: "accepted", revision: 1 }); expect(workspace).not.toHaveProperty("tenant"); - const replay = await owner.http.POST("/v1/workspaces", { body, headers: { "Idempotency-Key": "workspace-create-1" } }); + const replay = await owner.http.POST("/v1/workspaces", { + body: { labels: { alpha: "1", beta: "2" }, name: "primary" }, + params: { header: idempotencyHeaders("workspace-create-0001") }, + }); expect(replay.response.status).toBe(200); expect(replay.response.headers.get("Idempotency-Replayed")).toBe("true"); expect(replay.data).toEqual(workspace); const conflict = await owner.http.POST("/v1/workspaces", { body: { name: "different" }, - headers: { "Idempotency-Key": "workspace-create-1" }, + params: { header: idempotencyHeaders("workspace-create-0001") }, }); expect(conflict.response.status).toBe(409); expect(conflict.error).toMatchObject({ error: { code: "idempotency_conflict", retryable: false } }); for (const name of ["second", "third", "fourth"]) { - requireData(await owner.http.POST("/v1/workspaces", { body: { name }, headers: { "Idempotency-Key": `workspace-${name}` } })); + requireData(await owner.http.POST("/v1/workspaces", { + body: { name }, + params: { header: idempotencyHeaders(`workspace-${name}-0001`) }, + })); } - const pageOne = requireData(await owner.http.GET("/v1/workspaces", { params: { query: { pageSize: 2 } } })); + const pageOne = requireData(await owner.http.GET("/v1/workspaces", { + params: { header: VERSION_HEADERS, query: { pageSize: 2 } }, + })); expect(pageOne.items).toHaveLength(2); expect(pageOne.nextCursor).toBe("page-2"); - const pageTwo = requireData(await owner.http.GET("/v1/workspaces", { params: { query: { pageSize: 2, cursor: pageOne.nextCursor } } })); + const pageTwo = requireData(await owner.http.GET("/v1/workspaces", { + params: { header: VERSION_HEADERS, query: { pageSize: 2, cursor: pageOne.nextCursor } }, + })); expect(pageTwo.items).toHaveLength(2); expect(pageTwo.nextCursor).toBeUndefined(); - const isolated = await other.http.GET("/v1/workspaces/ws-1", { params: { path: { workspaceId: "ws-1" } } }); + const isolated = await other.http.GET("/v1/workspaces/{workspaceId}", { + params: { header: VERSION_HEADERS, path: { workspaceId: "ws-1" } }, + }); expect(isolated.response.status).toBe(404); expect(isolated.error).toMatchObject({ error: { code: "not_found" } }); - const invalid = await owner.http.POST("/v1/workspaces", { body: { name: "x".repeat(25) }, headers: { "Idempotency-Key": "invalid" } }); + const invalid = await owner.http.POST("/v1/workspaces", { + body: { name: "x".repeat(129) }, + params: { header: idempotencyHeaders("workspace-invalid-0001") }, + }); expect(invalid.response.status).toBe(422); expect(invalid.error).toMatchObject({ error: { code: "invalid_request", violations: [{ field: "name", rule: "length" }] } }); const updated = requireData(await owner.http.PATCH("/v1/workspaces/{workspaceId}", { - params: { path: { workspaceId: "ws-1" } }, body: { name: "renamed" }, headers: { "If-Match": "1" }, + params: { header: revisionHeaders(1), path: { workspaceId: "ws-1" } }, + body: { name: "renamed" }, })); expect(updated).toMatchObject({ name: "renamed", revision: 2 }); const stale = await owner.http.PATCH("/v1/workspaces/{workspaceId}", { - params: { path: { workspaceId: "ws-1" } }, body: { name: "stale" }, headers: { "If-Match": "1" }, + params: { header: revisionHeaders(1), path: { workspaceId: "ws-1" } }, + body: { name: "stale" }, }); expect(stale.response.status).toBe(409); expect(stale.error).toMatchObject({ error: { code: "revision_conflict" } }); - expect((await owner.http.DELETE("/v1/workspaces/{workspaceId}", { params: { path: { workspaceId: "ws-1" } } })).response.status).toBe(204); + expect((await owner.http.DELETE("/v1/workspaces/{workspaceId}", { + params: { header: VERSION_HEADERS, path: { workspaceId: "ws-1" } }, + })).response.status).toBe(204); service.expectNoCredentialLeak(); }); - it("spawns and mutates sessions, submits idempotent commands, and exposes stable states", async () => { + it("spawns, paginates, and mutates sessions, then submits idempotent stable command states", async () => { const service = new T4ApiV1ConformanceService(); const client = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); - requireData(await client.http.POST("/v1/workspaces", { body: { name: "workspace" }, headers: { "Idempotency-Key": "workspace" } })); + requireData(await client.http.POST("/v1/workspaces", { + body: { name: "workspace" }, params: { header: idempotencyHeaders("workspace-setup-0001") }, + })); const body = { title: "agent" } satisfies SessionCreate; const first = await client.http.POST("/v1/workspaces/{workspaceId}/sessions", { - params: { path: { workspaceId: "ws-1" } }, body, headers: { "Idempotency-Key": "spawn-1" }, + params: { header: idempotencyHeaders("session-spawn-0001"), path: { workspaceId: "ws-1" } }, + body, }); expect(first.response.status).toBe(202); const session = requireData(first); const replay = await client.http.POST("/v1/workspaces/{workspaceId}/sessions", { - params: { path: { workspaceId: "ws-1" } }, body, headers: { "Idempotency-Key": "spawn-1" }, + params: { header: idempotencyHeaders("session-spawn-0001"), path: { workspaceId: "ws-1" } }, + body, }); expect(replay.data).toEqual(session); - const listed = requireData(await client.http.GET("/v1/workspaces/{workspaceId}/sessions", { - params: { path: { workspaceId: "ws-1" } }, + for (const [key, title] of [["session-spawn-0002", "agent-two"], ["session-spawn-0003", "agent-three"]] as const) { + requireData(await client.http.POST("/v1/workspaces/{workspaceId}/sessions", { + params: { header: idempotencyHeaders(key), path: { workspaceId: "ws-1" } }, body: { title }, + })); + } + const sessionPageOne = requireData(await client.http.GET("/v1/workspaces/{workspaceId}/sessions", { + params: { header: VERSION_HEADERS, path: { workspaceId: "ws-1" }, query: { pageSize: 2 } }, + })); + expect(sessionPageOne.items).toHaveLength(2); + expect(sessionPageOne.nextCursor).toBe("page-2"); + const sessionPageTwo = requireData(await client.http.GET("/v1/workspaces/{workspaceId}/sessions", { + params: { header: VERSION_HEADERS, path: { workspaceId: "ws-1" }, query: { pageSize: 2, cursor: sessionPageOne.nextCursor } }, })); - expect(listed.items).toEqual([session]); + expect(sessionPageTwo.items).toHaveLength(1); + const mutated = requireData(await client.http.PATCH("/v1/sessions/{sessionId}", { - params: { path: { sessionId: "ses-1" } }, body: { title: "renamed-agent" }, headers: { "If-Match": "1" }, + params: { header: revisionHeaders(1), path: { sessionId: "ses-1" } }, body: { title: "renamed-agent" }, })); expect(mutated).toMatchObject({ title: "renamed-agent", revision: 2 }); + const stale = await client.http.PATCH("/v1/sessions/{sessionId}", { + params: { header: revisionHeaders(1), path: { sessionId: "ses-1" } }, body: { title: "stale-agent" }, + }); + expect(stale.response.status).toBe(409); + expect(stale.error).toMatchObject({ error: { code: "revision_conflict" } }); for (const state of ["accepted", "rejected", "conflict", "unavailable", "indeterminate"] as const) { const command = { command: state } satisfies CommandCreate; const result = await client.http.POST("/v1/sessions/{sessionId}/commands", { - params: { path: { sessionId: "ses-1" } }, body: command, headers: { "Idempotency-Key": `command-${state}` }, + params: { header: idempotencyHeaders(`command-${state}-0001`), path: { sessionId: "ses-1" } }, body: command, }); expect(requireData(result).state).toBe(state); const commandReplay = await client.http.POST("/v1/sessions/{sessionId}/commands", { - params: { path: { sessionId: "ses-1" } }, body: command, headers: { "Idempotency-Key": `command-${state}` }, + params: { header: idempotencyHeaders(`command-${state}-0001`), path: { sessionId: "ses-1" } }, body: command, }); expect(commandReplay.data).toEqual(result.data); expect(commandReplay.response.headers.get("Idempotency-Replayed")).toBe("true"); } const oversized = await client.http.POST("/v1/sessions/{sessionId}/commands", { - params: { path: { sessionId: "ses-1" } }, body: { command: "a".repeat(33) }, headers: { "Idempotency-Key": "oversized" }, + params: { header: idempotencyHeaders("command-oversized-0001"), path: { sessionId: "ses-1" } }, + body: { command: "a".repeat(33) }, }); expect(oversized.response.status).toBe(422); expect(oversized.error).toMatchObject({ error: { violations: [{ field: "command", rule: "maxBytes" }] } }); - const cancelled = await client.http.POST("/v1/sessions/{sessionId}/cancel", { params: { path: { sessionId: "ses-1" } } }); + const cancelled = await client.http.POST("/v1/sessions/{sessionId}/cancel", { + params: { header: VERSION_HEADERS, path: { sessionId: "ses-1" } }, + }); expect(cancelled.response.status).toBe(202); expect(cancelled.data).toMatchObject({ state: "cancelled" }); }); - it("takes a snapshot and watches bounded SSE with heartbeat, reconnect cursor, cancellation, and typed resync", async () => { + it("takes a snapshot and watches bounded SSE with heartbeat, reconnect, cancellation, and typed resync", async () => { const service = new T4ApiV1ConformanceService(); const client = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); - requireData(await client.http.POST("/v1/workspaces", { body: { name: "workspace" }, headers: { "Idempotency-Key": "workspace" } })); + requireData(await client.http.POST("/v1/workspaces", { + body: { name: "workspace" }, params: { header: idempotencyHeaders("workspace-watch-0001") }, + })); requireData(await client.http.POST("/v1/workspaces/{workspaceId}/sessions", { - params: { path: { workspaceId: "ws-1" } }, body: { title: "agent" }, headers: { "Idempotency-Key": "session" }, + params: { header: idempotencyHeaders("session-watch-0001"), path: { workspaceId: "ws-1" } }, body: { title: "agent" }, + })); + const snapshot = requireData(await client.http.GET("/v1/sessions/{sessionId}/snapshot", { + params: { header: VERSION_HEADERS, path: { sessionId: "ses-1" } }, })); - const snapshot = requireData(await client.http.GET("/v1/sessions/{sessionId}/snapshot", { params: { path: { sessionId: "ses-1" } } })); expect(snapshot).toMatchObject({ sessionId: "ses-1", cursor: "cursor-2", entries: [{ sequence: 2 }] }); const controller = new AbortController(); const received: Array = []; - for await (const event of client.watchSession("ses-1", { cursor: snapshot.cursor, maxEvents: 2, signal: controller.signal })) { + for await (const event of client.watchSession("ses-1", { cursor: snapshot.cursor, maxEvents: 4, signal: controller.signal })) { received.push(event); if (received.length === 2) controller.abort(); } @@ -152,12 +211,13 @@ describe("generated T4 API v1 client conformance", () => { expect.objectContaining({ type: "heartbeat", cursor: "cursor-3" }), expect.objectContaining({ type: "session", cursor: "cursor-4", state: "accepted" }), ]); - expect(service.calls.at(-1)?.path).toBe("/v1/sessions/ses-1/events"); expect(service.abortedWatches).toEqual(["ses-1"]); + expect(service.watchCursors[0]).toEqual({ query: "cursor-2", header: "cursor-2" }); - const reconnect = client.watchSession("ses-1", { cursor: received.at(-1)!.cursor, maxEvents: 1 }); - expect((await reconnect.next()).value).toMatchObject({ type: "heartbeat", cursor: "cursor-3" }); + const reconnect = client.watchSession("ses-1", { cursor: received.at(-1)!.cursor, maxEvents: 2 }); + expect((await reconnect.next()).value).toMatchObject({ type: "heartbeat", cursor: "cursor-5" }); await reconnect.return(undefined); + expect(service.watchCursors[1]).toEqual({ query: "cursor-4", header: "cursor-4" }); await expect(async () => { for await (const _event of client.watchSession("ses-1", { cursor: "expired", maxEvents: 1 })) { diff --git a/packages/t4-api-client/src/index.ts b/packages/t4-api-client/src/index.ts index aba5baab..be975199 100644 --- a/packages/t4-api-client/src/index.ts +++ b/packages/t4-api-client/src/index.ts @@ -7,23 +7,24 @@ export type { components, operations, paths } from "./generated/schema.ts"; const MAX_CREDENTIAL_LENGTH = 4096; const MAX_ERROR_BYTES = 1024 * 1024; const MAX_EVENT_BYTES = 1024 * 1024; -const SESSION_STATES = new Set([ - "accepted", - "provisioning", - "ready", - "cancelling", - "cancelled", - "failed", - "unavailable", - "indeterminate", -]); -const OPERATION_STATES = new Set([ - "accepted", - "rejected", - "conflict", - "unavailable", - "indeterminate", -]); +const CURSOR_PATTERN = /^[A-Za-z0-9._~-]+$/u; +const ERROR_CODES = { + invalid_request: true, unauthenticated: true, forbidden: true, not_found: true, + idempotency_key_required: true, idempotency_conflict: true, revision_conflict: true, + incompatible_version: true, cursor_expired: true, unavailable: true, indeterminate: true, + invalid_origin: true, https_required: true, +} as const satisfies Record; +const ERROR_FIELDS: Record = { + code: true, message: true, requestId: true, retryable: true, + violations: true, supportedMajors: true, resync: true, +}; +const SESSION_STATES = { + accepted: true, provisioning: true, ready: true, cancelling: true, + cancelled: true, failed: true, unavailable: true, indeterminate: true, +} as const satisfies Record; +const OPERATION_STATES = { + accepted: true, rejected: true, conflict: true, unavailable: true, indeterminate: true, +} as const satisfies Record; type ApiError = components["schemas"]["ApiError"]; type Resync = components["schemas"]["Resync"]; @@ -129,22 +130,70 @@ function record(value: unknown): Record | undefined { : undefined; } +function validViolation(value: unknown): boolean { + const violation = record(value); + return violation !== undefined && + Object.keys(violation).every((key) => key === "field" || key === "rule" || key === "message") && + typeof violation.field === "string" && violation.field.length >= 1 && violation.field.length <= 256 && + typeof violation.rule === "string" && /^[A-Za-z][A-Za-z0-9._-]{0,63}$/u.test(violation.rule) && + typeof violation.message === "string" && violation.message.length >= 1 && violation.message.length <= 512; +} + +function validResync(value: unknown): value is Resync { + const resync = record(value); + return resync !== undefined && Object.keys(resync).every((key) => key === "snapshotUrl" || key === "cursor") && + typeof resync.snapshotUrl === "string" && /^\/v1\/sessions\/[A-Za-z0-9._~-]+\/snapshot$/u.test(resync.snapshotUrl) && + typeof resync.cursor === "string" && resync.cursor.length <= 512 && CURSOR_PATTERN.test(resync.cursor); +} + function apiError(value: unknown): ApiError | undefined { const envelope = record(value); const error = record(envelope?.error); if ( - error === undefined || - typeof error.code !== "string" || - typeof error.message !== "string" || - typeof error.requestId !== "string" || - typeof error.retryable !== "boolean" + envelope === undefined || Object.keys(envelope).some((key) => key !== "error") || + error === undefined || Object.keys(error).some((key) => ERROR_FIELDS[key] !== true) || + typeof error.code !== "string" || ERROR_CODES[error.code as ApiError["code"]] !== true || + typeof error.message !== "string" || error.message.length < 1 || error.message.length > 1024 || + typeof error.requestId !== "string" || error.requestId.length < 1 || error.requestId.length > 128 || + typeof error.retryable !== "boolean" || + (error.violations !== undefined && (!Array.isArray(error.violations) || error.violations.length > 64 || !error.violations.every(validViolation))) || + (error.supportedMajors !== undefined && (!Array.isArray(error.supportedMajors) || error.supportedMajors.length > 8 || !error.supportedMajors.every((major) => Number.isSafeInteger(major) && Number(major) >= 1))) || + (error.resync !== undefined && !validResync(error.resync)) ) return undefined; return error as ApiError; } +async function boundedResponseText(response: Response): Promise { + if (response.body === null) return ""; + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let length = 0; + try { + while (true) { + const chunk = await reader.read(); + if (chunk.done) break; + length += chunk.value.byteLength; + if (length > MAX_ERROR_BYTES) { + await reader.cancel(); + return undefined; + } + chunks.push(chunk.value); + } + } finally { + reader.releaseLock(); + } + const bytes = new Uint8Array(length); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + try { return new TextDecoder("utf-8", { fatal: true }).decode(bytes); } catch { return undefined; } +} + async function boundedError(response: Response): Promise { - const text = await response.text(); - if (new TextEncoder().encode(text).byteLength <= MAX_ERROR_BYTES) { + const text = await boundedResponseText(response); + if (text !== undefined) { try { const decoded = apiError(JSON.parse(text)); if (decoded !== undefined) return new T4ApiError(response.status, decoded); @@ -168,15 +217,16 @@ function watchEvent(value: unknown, eventId: string | undefined): WatchEvent { typeof event.cursor !== "string" || event.cursor.length < 1 || event.cursor.length > 512 || + !CURSOR_PATTERN.test(event.cursor) || (eventId !== undefined && eventId !== event.cursor) ) throw new T4ApiError(502, { code: "indeterminate", message: "T4 API returned an invalid watch event", requestId: "unavailable", retryable: true }); - if (event.type === "heartbeat" && typeof event.observedAt === "string" && event.observedAt.length <= 64) { + if (event.type === "heartbeat" && typeof event.observedAt === "string" && event.observedAt.length <= 64 && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/u.test(event.observedAt) && Number.isFinite(Date.parse(event.observedAt))) { return event as WatchEvent; } if ( event.type === "session" && typeof event.state === "string" && - SESSION_STATES.has(event.state as components["schemas"]["SessionState"]) && + SESSION_STATES[event.state as components["schemas"]["SessionState"]] === true && Number.isSafeInteger(event.revision) && Number(event.revision) >= 1 ) return event as WatchEvent; if ( @@ -184,7 +234,7 @@ function watchEvent(value: unknown, eventId: string | undefined): WatchEvent { typeof event.commandId === "string" && event.commandId.length >= 1 && event.commandId.length <= 128 && typeof event.state === "string" && - OPERATION_STATES.has(event.state as components["schemas"]["OperationState"]) + OPERATION_STATES[event.state as components["schemas"]["OperationState"]] === true ) return event as WatchEvent; throw new T4ApiError(502, { code: "indeterminate", message: "T4 API returned an invalid watch event", requestId: "unavailable", retryable: true }); } @@ -192,8 +242,8 @@ function watchEvent(value: unknown, eventId: string | undefined): WatchEvent { function decodeSseFrame(frame: string): WatchEvent | undefined { let eventId: string | undefined; const data: string[] = []; - for (const rawLine of frame.split("\n")) { - const line = rawLine.endsWith("\r") ? rawLine.slice(0, -1) : rawLine; + for (const rawLine of frame.split(/\r\n|\r|\n/u)) { + const line = rawLine; if (line === "" || line.startsWith(":")) continue; const separator = line.indexOf(":"); const field = separator < 0 ? line : line.slice(0, separator); @@ -211,6 +261,11 @@ function decodeSseFrame(frame: string): WatchEvent | undefined { } } +function nextSseBoundary(buffer: string): { readonly index: number; readonly length: number } | undefined { + const match = /(?:\r\n|\r|\n)(?:\r\n|\r|\n)/u.exec(buffer); + return match?.index === undefined ? undefined : { index: match.index, length: match[0].length }; +} + async function* watch( baseUrl: string, credential: string, @@ -222,8 +277,8 @@ async function* watch( const sessionId = requiredSessionId(sessionIdValue); const maxEvents = boundedInteger(options.maxEvents, 100, 1, 1000, "maxEvents"); const heartbeatSeconds = boundedInteger(options.heartbeatSeconds, 15, 5, 60, "heartbeatSeconds"); - if (options.cursor !== undefined && (options.cursor.length < 1 || options.cursor.length > 512)) { - throw new TypeError("cursor must contain between 1 and 512 characters"); + if (options.cursor !== undefined && (options.cursor.length < 1 || options.cursor.length > 512 || !CURSOR_PATTERN.test(options.cursor))) { + throw new TypeError("cursor must be a header-safe token containing between 1 and 512 characters"); } const url = new URL(`${baseUrl}/v1/sessions/${encodeURIComponent(sessionId)}/events`); url.searchParams.set("maxEvents", String(maxEvents)); @@ -271,16 +326,18 @@ async function* watch( if (new TextEncoder().encode(buffer).byteLength > MAX_EVENT_BYTES) { throw new T4ApiError(502, { code: "indeterminate", message: "T4 API watch event exceeds the client bound", requestId: "unavailable", retryable: true }); } - let boundary = buffer.indexOf("\n\n"); - while (boundary >= 0 && delivered < maxEvents) { - const frame = buffer.slice(0, boundary); - buffer = buffer.slice(boundary + 2); + let boundary = nextSseBoundary(buffer); + while (boundary !== undefined && delivered < maxEvents) { + if (controller.signal.aborted) return; + const frame = buffer.slice(0, boundary.index); + buffer = buffer.slice(boundary.index + boundary.length); const event = decodeSseFrame(frame); if (event !== undefined) { delivered += 1; yield event; + if (controller.signal.aborted) return; } - boundary = buffer.indexOf("\n\n"); + boundary = nextSseBoundary(buffer); } } if (buffer.trim().length > 0 && delivered < maxEvents) { diff --git a/packages/t4-api-contract/openapi.json b/packages/t4-api-contract/openapi.json index 863bdd88..0387b8a8 100644 --- a/packages/t4-api-contract/openapi.json +++ b/packages/t4-api-contract/openapi.json @@ -338,7 +338,7 @@ "in": "header", "required": true, "description": "Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading.", - "schema": { "type": "string", "pattern": "^1(?:\\.[0-9]+)?$", "maxLength": 16 } + "schema": { "type": "string", "pattern": "^[1-9][0-9]{0,3}(?:\\.[0-9]+)?$", "maxLength": 16 } }, "IdempotencyKey": { "name": "Idempotency-Key", @@ -412,7 +412,7 @@ }, "schemas": { "ResourceId": { "type": "string", "minLength": 1, "maxLength": 128, "pattern": "^[A-Za-z0-9][A-Za-z0-9._~-]*$" }, - "Cursor": { "type": "string", "minLength": 1, "maxLength": 512, "description": "Opaque server-issued cursor." }, + "Cursor": { "type": "string", "minLength": 1, "maxLength": 512, "pattern": "^[A-Za-z0-9._~-]+$", "description": "Opaque server-issued header-safe SSE cursor." }, "Revision": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 }, "OperationState": { "type": "string", "enum": ["accepted", "rejected", "conflict", "unavailable", "indeterminate"] }, "WorkspaceState": { "type": "string", "enum": ["accepted", "provisioning", "ready", "deleting", "deleted", "failed", "unavailable", "indeterminate"] }, @@ -639,11 +639,47 @@ "resync": { "$ref": "#/components/schemas/Resync" } } }, + "IncompatibleVersionApiError": { + "allOf": [ + { "$ref": "#/components/schemas/ApiError" }, + { "type": "object", "required": ["supportedMajors"], "properties": { "code": { "const": "incompatible_version" }, "supportedMajors": { "type": "array", "minItems": 1, "maxItems": 8, "uniqueItems": true, "items": { "type": "integer", "minimum": 1 } } } } + ] + }, + "CursorExpiredApiError": { + "allOf": [ + { "$ref": "#/components/schemas/ApiError" }, + { "type": "object", "required": ["resync"], "properties": { "code": { "const": "cursor_expired" }, "resync": { "$ref": "#/components/schemas/Resync" } } } + ] + }, + "InvalidRequestApiError": { + "allOf": [ + { "$ref": "#/components/schemas/ApiError" }, + { "type": "object", "required": ["violations"], "properties": { "code": { "const": "invalid_request" }, "violations": { "type": "array", "minItems": 1, "maxItems": 64, "items": { "$ref": "#/components/schemas/FieldViolation" } } } } + ] + }, "ErrorEnvelope": { "type": "object", "additionalProperties": false, "required": ["error"], "properties": { "error": { "$ref": "#/components/schemas/ApiError" } } + }, + "IncompatibleVersionErrorEnvelope": { + "type": "object", + "additionalProperties": false, + "required": ["error"], + "properties": { "error": { "$ref": "#/components/schemas/IncompatibleVersionApiError" } } + }, + "CursorExpiredErrorEnvelope": { + "type": "object", + "additionalProperties": false, + "required": ["error"], + "properties": { "error": { "$ref": "#/components/schemas/CursorExpiredApiError" } } + }, + "InvalidRequestErrorEnvelope": { + "type": "object", + "additionalProperties": false, + "required": ["error"], + "properties": { "error": { "$ref": "#/components/schemas/InvalidRequestApiError" } } } }, "responses": { @@ -673,10 +709,10 @@ "Error401": { "description": "Missing or invalid opaque bearer credential", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorEnvelope" } } } }, "Error403": { "description": "Credential lacks the deny-by-default operation or resource scope", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorEnvelope" } } } }, "Error404": { "description": "Resource absent or outside caller scope (scope existence is not disclosed)", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorEnvelope" } } } }, - "Error406": { "description": "Requested major is incompatible; no silent downgrade", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorEnvelope" } } } }, + "Error406": { "description": "Requested major is incompatible; no silent downgrade", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/IncompatibleVersionErrorEnvelope" } } } }, "Error409": { "description": "Idempotency or resource revision conflict", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorEnvelope" } } } }, - "Error410": { "description": "Watch cursor expired; resync from the typed snapshot target", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorEnvelope" } } } }, - "Error422": { "description": "Bounded semantic validation failure with stable field violations", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorEnvelope" } } } }, + "Error410": { "description": "Watch cursor expired; resync from the typed snapshot target", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CursorExpiredErrorEnvelope" } } } }, + "Error422": { "description": "Bounded semantic validation failure with stable field violations", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/InvalidRequestErrorEnvelope" } } } }, "Error503": { "description": "Service temporarily unavailable or outcome indeterminate", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorEnvelope" } } } } } } From b50b7d9702390f1c0e53ed1d7b0367ebcefdd962 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 19:43:50 +0000 Subject: [PATCH 004/131] build: recover T4 API generation lockfile --- .github/workflows/ci.yml | 8 +- .woodpecker.yml | 8 +- pnpm-lock.yaml | 362 +++++++++++++++++++++++++++++++++------ 3 files changed, 315 insertions(+), 63 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ab9e373f..631e07ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,13 +76,13 @@ jobs: with: node-version: 24.13.1 - - name: Resolve lockfile and generation dependencies for artifact recovery - run: pnpm install --no-frozen-lockfile + - name: Install generation dependencies from the recovered lockfile + run: pnpm install --frozen-lockfile - - name: Validate and generate the T4 API contract artifact + - name: Generate and validate the T4 API contract artifact run: | - pnpm --filter @t4-code/t4-api-contract validate pnpm --filter @t4-code/t4-api-contract generate:ci + pnpm --filter @t4-code/t4-api-contract validate - name: Upload deterministic T4 API generation inputs if: ${{ always() }} diff --git a/.woodpecker.yml b/.woodpecker.yml index fb611ce0..1f2cdf64 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -16,13 +16,9 @@ steps: image: mirror.gcr.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 commands: - corepack enable - - pnpm install --no-frozen-lockfile - - echo T4_API_LOCKFILE_BASE64_BEGIN - - base64 -w0 pnpm-lock.yaml - - echo - - echo T4_API_LOCKFILE_BASE64_END - - pnpm --filter @t4-code/t4-api-contract validate + - pnpm install --frozen-lockfile - pnpm --filter @t4-code/t4-api-contract generate:ci + - pnpm --filter @t4-code/t4-api-contract validate when: - event: push branch: agent/t4-op03-public-api-contract diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 36b7b3d1..4ca7ae94 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -261,28 +261,6 @@ importers: specifier: 'catalog:' version: 0.2.2(@types/node@26.1.1)(jiti@2.7.0)(typescript@6.0.3) - packages/cluster-server: - dependencies: - '@t4-code/host-service': - specifier: workspace:* - version: link:../host-service - '@t4-code/host-wire': - specifier: workspace:* - version: link:../host-wire - devDependencies: - '@types/bun': - specifier: 1.3.5 - version: 1.3.5 - '@types/node': - specifier: 24.12.4 - version: 24.12.4 - typescript: - specifier: ~6.0.3 - version: 6.0.3 - vite-plus: - specifier: 0.2.2 - version: 0.2.2(@types/node@24.12.4)(jiti@2.7.0)(typescript@6.0.3) - packages/client: dependencies: '@t4-code/protocol': @@ -295,6 +273,9 @@ importers: '@t4-code/host-wire': specifier: workspace:* version: link:../host-wire + '@t4-code/t4-api-client': + specifier: workspace:* + version: link:../t4-api-client '@types/node': specifier: 'catalog:' version: 24.12.4 @@ -308,6 +289,28 @@ importers: specifier: 8.18.3 version: 8.18.3 + packages/cluster-server: + dependencies: + '@t4-code/host-service': + specifier: workspace:* + version: link:../host-service + '@t4-code/host-wire': + specifier: workspace:* + version: link:../host-wire + devDependencies: + '@types/bun': + specifier: 1.3.5 + version: 1.3.5 + '@types/node': + specifier: 24.12.4 + version: 24.12.4 + typescript: + specifier: ~6.0.3 + version: 6.0.3 + vite-plus: + specifier: 0.2.2 + version: 0.2.2(@types/node@24.12.4)(jiti@2.7.0)(typescript@6.0.3) + packages/fixture-server: dependencies: '@t4-code/protocol': @@ -421,6 +424,31 @@ importers: specifier: 'catalog:' version: 0.2.2(@types/node@24.12.4)(jiti@2.7.0)(typescript@6.0.3) + packages/t4-api-client: + dependencies: + openapi-fetch: + specifier: 0.17.0 + version: 0.17.0 + devDependencies: + '@types/node': + specifier: 'catalog:' + version: 24.12.4 + typescript: + specifier: 'catalog:' + version: 6.0.3 + vite-plus: + specifier: 'catalog:' + version: 0.2.2(@types/node@24.12.4)(jiti@2.7.0)(typescript@6.0.3) + + packages/t4-api-contract: + devDependencies: + '@readme/openapi-parser': + specifier: 6.3.0 + version: 6.3.0(openapi-types@12.1.3) + openapi-typescript: + specifier: 7.13.0 + version: 7.13.0(typescript@6.0.3) + packages/ui: dependencies: '@base-ui/react': @@ -465,6 +493,12 @@ packages: peerDependencies: zod: ^3.25.0 || ^4.0.0 + '@apidevtools/json-schema-ref-parser@14.2.1': + resolution: {integrity: sha512-HmdFw9CDYqM6B25pqGBpNeLCKvGPlIx1EbLrVL0zPvj50CJQUHyBNBw45Muk0kEIkogo1VZvOKHajdMuAzSxRg==} + engines: {node: '>= 20'} + peerDependencies: + '@types/json-schema': ^7.0.15 + '@babel/code-frame@7.29.7': resolution: {integrity: sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==} engines: {node: '>=6.9.0'} @@ -587,6 +621,10 @@ packages: '@fontsource/jetbrains-mono@5.2.8': resolution: {integrity: sha512-6w8/SG4kqvIMu7xd7wt6x3idn1Qux3p9N62s6G3rfldOUYHpWcc2FKrqf+Vo44jRvqWj2oAtTHrZXEP23oSKwQ==} + '@humanwhocodes/momoa@2.0.4': + resolution: {integrity: sha512-RE815I4arJFtt+FVeU1Tgp9/Xvecacji8w/V6XtXsWWH/wz/eNkNbhb+ny/+PlVZjV0rxQpRSQKNKE3lcktHEA==} + engines: {node: '>=10.10.0'} + '@ionic/cli-framework-output@2.2.8': resolution: {integrity: sha512-TshtaFQsovB4NWRBydbNFawql6yul7d5bMiW1WYYf17hd99V6xdDdk3vtF51bw6sLkxON3bDQpWsnUc9/hVo3g==} engines: {node: '>=16.0.0'} @@ -1051,6 +1089,32 @@ packages: '@preact/signals-core@1.14.4': resolution: {integrity: sha512-HNB6HYeYKhQbJ1aKl+YRjrS4+QWHLKX6qKoUsfS/m0vqzsVaEBiZiaKbG/e+NKk2ch5ALQr/ihWaMHxiCuuWHA==} + '@readme/better-ajv-errors@2.4.0': + resolution: {integrity: sha512-9WODaOAKSl/mU+MYNZ2aHCrkoRSvmQ+1YkLj589OEqqjOAhbn8j7Z+ilYoiTu/he6X63/clsxxAB4qny9/dDzg==} + engines: {node: '>=18'} + peerDependencies: + ajv: 4.11.8 - 8 + + '@readme/openapi-parser@6.3.0': + resolution: {integrity: sha512-pLsspfvN/m6C4BuZ91ZppK2uo86IT4EDhXu6B+giNLehLkuZbtg47OJvOktwDHBmheULPmXg5pzM7065AAHeyQ==} + engines: {node: '>=20'} + peerDependencies: + openapi-types: '>=7' + + '@readme/openapi-schemas@4.0.0': + resolution: {integrity: sha512-hrG//9/+RpOZTrUDirU4OzfMqaxCdY5BMpr3YuLf3Rje9rfNsydQJsH9iCPTie9ZRu9xe+0OUD4fHe0JmGwa2w==} + engines: {node: '>=20'} + + '@redocly/ajv@8.11.2': + resolution: {integrity: sha512-io1JpnwtIcvojV7QKDUSIuMN/ikdOUd1ReEnUnMKGfDVridQZ31J0MmIuqwuRjWDZfmvr+Q0MqCcfHM2gTivOg==} + + '@redocly/config@0.22.0': + resolution: {integrity: sha512-gAy93Ddo01Z3bHuVdPWfCwzgfaYgMdaZPcfL7JZ7hWJoK9V0lXDbigTWkhiPFAaLWzbOJ+kbUQG1+XwIm0KRGQ==} + + '@redocly/openapi-core@1.34.17': + resolution: {integrity: sha512-wsV2keCt6B806XpSdezbWZ9aFJYf14YVh+XQf0ESt7M90yqVuxH9//PxvtC70sgj9OCkRM3nRaLfu4MsGQZRig==} + engines: {node: '>=18.17.0', npm: '>=9.5.0'} + '@rolldown/pluginutils@1.0.1': resolution: {integrity: sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==} @@ -1229,6 +1293,9 @@ packages: '@types/http-cache-semantics@4.2.0': resolution: {integrity: sha512-L3LgimLHXtGkWikKnsPg0/VFx9OGZaC+eN1u4r+OB1XRqH3meBIAVC2zr1WdMH+RHmnRkqliQAOHNJ/E0j/e0Q==} + '@types/json-schema@7.0.15': + resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} + '@types/keyv@3.1.4': resolution: {integrity: sha512-BQ5aZNSCpj7D6K2ksrRCTmKRLEpnPvWDiLPfoGyhZ++8YtiK9d/3DBKPJgry359X/P1PfruyYwvnvwFjuEiEIg==} @@ -1499,6 +1566,14 @@ packages: resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} engines: {node: '>= 14'} + ajv-draft-04@1.0.0: + resolution: {integrity: sha512-mv00Te6nmYbRp5DCwclxtt7yV/joXJPGS7nM+97GdxvuttCOfgI3K4U25zboyeX0O+myI8ERluxQe5wljMmVIw==} + peerDependencies: + ajv: ^8.5.0 + peerDependenciesMeta: + ajv: + optional: true + ajv-formats@2.1.1: resolution: {integrity: sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==} peerDependencies: @@ -1510,6 +1585,10 @@ packages: ajv@8.20.0: resolution: {integrity: sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==} + ansi-colors@4.1.3: + resolution: {integrity: sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==} + engines: {node: '>=6'} + ansi-regex@5.0.1: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} engines: {node: '>=8'} @@ -1650,6 +1729,9 @@ packages: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} engines: {node: '>=10'} + change-case@5.4.4: + resolution: {integrity: sha512-HRQyTk2/YPEkt9TnUPbOpr64Uw3KOicFWPVBb+xiHvd6eBx/qPr9xqfBFDT8P2vWsvvz4jbEkfDe71W3VyNu2w==} + character-entities-html4@2.1.0: resolution: {integrity: sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==} @@ -1698,6 +1780,9 @@ packages: color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + colorette@1.4.0: + resolution: {integrity: sha512-Y2oEozpomLn7Q3HFP7dpww7AtMJplbM9lGZP6RDfHqmbeRjiwRg4n6VM6j4KLmRke85uWEI7JqF17f3pqdRA0g==} + combined-stream@1.0.8: resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} engines: {node: '>= 0.8'} @@ -2120,6 +2205,10 @@ packages: resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} engines: {node: '>= 14'} + index-to-position@1.2.0: + resolution: {integrity: sha512-Yg7+ztRkqslMAS2iFaU+Oa4KTSidr63OsFGlOrJoW981kIYO3CGCS3wA95P1mUi/IVSJkn0D479KTJpVpvFNuw==} + engines: {node: '>=18'} + inflight@1.0.6: resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} deprecated: This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful. @@ -2205,9 +2294,17 @@ packages: resolution: {integrity: sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==} hasBin: true + js-levenshtein@1.1.6: + resolution: {integrity: sha512-X2BB11YZtrRqY4EnQcLX5Rh373zbK4alC1FW7D7MBhL2gtcC17cTnr6DmfHZeS0s2rTHjUTMMHfG7gO8SSdw+g==} + engines: {node: '>=0.10.0'} + js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} + js-yaml@4.2.0: + resolution: {integrity: sha512-ePWsvanv0DWuDRsW8dnt+R4jQ31SCRCQ7hhNcPXZPsoBZiemuZNYGf7adZdqX2D86j6rvKp3RpCxVTSb8WQlOw==} + hasBin: true + js-yaml@4.3.0: resolution: {integrity: sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q==} hasBin: true @@ -2235,6 +2332,10 @@ packages: jsonfile@6.2.1: resolution: {integrity: sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q==} + jsonpointer@5.0.1: + resolution: {integrity: sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==} + engines: {node: '>=0.10.0'} + keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} @@ -2249,6 +2350,10 @@ packages: lazy-val@1.0.5: resolution: {integrity: sha512-0/BnGCCfyUMkBpeDgWihanIAF9JmZhHBgUhEqzvf+adhNGLoP6TaiI5oF8oyb3I45P+PcnrqihSf01M0l0G5+Q==} + leven@3.1.0: + resolution: {integrity: sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==} + engines: {node: '>=6'} + lexical@0.41.0: resolution: {integrity: sha512-pNIm5+n+hVnJHB9gYPDYsIO5Y59dNaDU9rJmPPsfqQhP2ojKFnUoPbcRnrI9FJLXB14sSumcY8LUw7Sq70TZqA==} @@ -2634,6 +2739,21 @@ packages: resolution: {integrity: sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==} engines: {node: '>=12'} + openapi-fetch@0.17.0: + resolution: {integrity: sha512-PsbZR1wAPcG91eEthKhN+Zn92FMHxv+/faECIwjXdxfTODGSGegYv0sc1Olz+HYPvKOuoXfp+0pA2XVt2cI0Ig==} + + openapi-types@12.1.3: + resolution: {integrity: sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==} + + openapi-typescript-helpers@0.1.0: + resolution: {integrity: sha512-OKTGPthhivLw/fHz6c3OPtg72vi86qaMlqbJuVJ23qOvQ+53uw1n7HdmkJFibloF7QEjDrDkzJiOJuockM/ljw==} + + openapi-typescript@7.13.0: + resolution: {integrity: sha512-EFP392gcqXS7ntPvbhBzbF8TyBA+baIYEm791Hy5YkjDYKTnk/Tn5OQeKm5BIZvJihpp8Zzr4hzx0Irde1LNGQ==} + hasBin: true + peerDependencies: + typescript: ^5.x + oxfmt@0.57.0: resolution: {integrity: sha512-ZB7Bi+rGDSqmVIo9jwcLyFgjxXvQhDdU+jx+ZrVy6VRiVXK2+CHc4hO3J4dUQjHe7V0ymHB+MDuv5z+NhK07HA==} engines: {node: ^20.19.0 || >=22.12.0} @@ -2690,6 +2810,10 @@ packages: parse-entities@4.0.2: resolution: {integrity: sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==} + parse-json@8.3.0: + resolution: {integrity: sha512-ybiGyvspI+fAoRQbIPRddCcSTV9/LsJbf0e/S85VLowVGzRmokfneg2kwVW/KU5rOXrPSbF1qAKPMgNTqqROQQ==} + engines: {node: '>=18'} + parse5@7.3.0: resolution: {integrity: sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==} @@ -2748,6 +2872,10 @@ packages: resolution: {integrity: sha512-uysumyrvkUX0rX/dEVqt8gC3sTBzd4zoWfLeS29nb53imdaXVvLINYXTI2GNqzaMuvacNx4uJQ8+b3zXR0pkgQ==} engines: {node: '>=10.4.0'} + pluralize@8.0.0: + resolution: {integrity: sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==} + engines: {node: '>=4'} + pngjs@7.0.0: resolution: {integrity: sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow==} engines: {node: '>=14.19.0'} @@ -3042,6 +3170,10 @@ packages: resolution: {integrity: sha512-MvjXzkz/BOfyVDkG0oFOtBxHX2u3gKbMHIF/dXblZsgD3BWOFLmHovIpZY7BykJdAjcqRCBi1WYBNdEC9yI7vg==} engines: {node: '>= 8.0'} + supports-color@10.2.2: + resolution: {integrity: sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==} + engines: {node: '>=18'} + supports-color@7.2.0: resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} engines: {node: '>=8'} @@ -3133,6 +3265,10 @@ packages: resolution: {integrity: sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==} engines: {node: '>=12.20'} + type-fest@4.41.0: + resolution: {integrity: sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==} + engines: {node: '>=16'} + typescript@6.0.3: resolution: {integrity: sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==} engines: {node: '>=14.17'} @@ -3181,6 +3317,9 @@ packages: unzipper@0.12.5: resolution: {integrity: sha512-tXYOi9R57Uj/2Z25SOs5RRSzq886MBQj2gY8dPL+xl/kv6s6SvByoKfAtvfVeEuhntWDgjd2o9p2lb4TVPAz0A==} + uri-js-replace@1.0.1: + resolution: {integrity: sha512-W+C9NWNLFOoBI2QWDp4UT9pv65r2w5Cx+3sTYFvtMdDBxkKt1syCqsUdSFAChbEe1uK5TfS04wt/nGwmaeIQ0g==} + use-sync-external-store@1.6.0: resolution: {integrity: sha512-Pp6GSwGP/NrPIrxVFAIkOQeyw8lFenOHijQWkUTrDvrF4ALqylP2C/KCkeS9dpUM3KvYRQhna5vt7IL95+ZQ9w==} peerDependencies: @@ -3335,6 +3474,9 @@ packages: resolution: {integrity: sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==} engines: {node: '>=18'} + yaml-ast-parser@0.0.43: + resolution: {integrity: sha512-2PTINUwsRqSd+s8XxKaJWQlUuEMHJQyEuh2edBbW8KNJz0SJPwUSD2zRWqezFEdN7IzAgeuYHFUCF7o8zRdZ0A==} + yargs-parser@21.1.1: resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} engines: {node: '>=12'} @@ -3384,6 +3526,11 @@ snapshots: dependencies: zod: 4.4.3 + '@apidevtools/json-schema-ref-parser@14.2.1(@types/json-schema@7.0.15)': + dependencies: + '@types/json-schema': 7.0.15 + js-yaml: 4.3.0 + '@babel/code-frame@7.29.7': dependencies: '@babel/helper-validator-identifier': 7.29.7 @@ -3429,7 +3576,7 @@ snapshots: '@ionic/utils-subprocess': 3.0.1 '@ionic/utils-terminal': 2.3.5 commander: 12.1.0 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) env-paths: 2.2.1 fs-extra: 11.3.6 kleur: 4.1.5 @@ -3463,7 +3610,7 @@ snapshots: '@electron/get@2.0.3': dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) env-paths: 2.2.1 fs-extra: 8.1.0 got: 11.8.6 @@ -3477,7 +3624,7 @@ snapshots: '@electron/get@3.1.0': dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) env-paths: 2.2.1 fs-extra: 8.1.0 got: 11.8.6 @@ -3491,7 +3638,7 @@ snapshots: '@electron/notarize@2.5.0': dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) fs-extra: 9.1.0 promise-retry: 2.0.1 transitivePeerDependencies: @@ -3500,7 +3647,7 @@ snapshots: '@electron/osx-sign@1.3.3': dependencies: compare-version: 0.1.2 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) fs-extra: 10.1.0 isbinaryfile: 4.0.10 minimist: 1.2.8 @@ -3511,7 +3658,7 @@ snapshots: '@electron/rebuild@4.2.0': dependencies: '@malept/cross-spawn-promise': 2.0.0 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) node-abi: 4.33.0 node-api-version: 0.2.1 node-gyp: 12.4.0 @@ -3523,7 +3670,7 @@ snapshots: dependencies: '@electron/asar': 3.4.1 '@malept/cross-spawn-promise': 2.0.0 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) dir-compare: 4.2.0 fs-extra: 11.3.6 minimatch: 9.0.9 @@ -3534,7 +3681,7 @@ snapshots: '@electron/windows-sign@1.2.2': dependencies: cross-dirname: 0.1.0 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) fs-extra: 11.3.6 minimist: 1.2.8 postject: 1.0.0-alpha.6 @@ -3571,17 +3718,19 @@ snapshots: '@fontsource/jetbrains-mono@5.2.8': {} + '@humanwhocodes/momoa@2.0.4': {} + '@ionic/cli-framework-output@2.2.8': dependencies: '@ionic/utils-terminal': 2.3.5 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) tslib: 2.8.1 transitivePeerDependencies: - supports-color '@ionic/utils-array@2.1.6': dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) tslib: 2.8.1 transitivePeerDependencies: - supports-color @@ -3589,7 +3738,7 @@ snapshots: '@ionic/utils-fs@3.1.7': dependencies: '@types/fs-extra': 8.1.5 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) fs-extra: 9.1.0 tslib: 2.8.1 transitivePeerDependencies: @@ -3597,7 +3746,7 @@ snapshots: '@ionic/utils-object@2.1.6': dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) tslib: 2.8.1 transitivePeerDependencies: - supports-color @@ -3606,7 +3755,7 @@ snapshots: dependencies: '@ionic/utils-object': 2.1.6 '@ionic/utils-terminal': 2.3.5 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) signal-exit: 3.0.7 tree-kill: 1.2.2 tslib: 2.8.1 @@ -3615,7 +3764,7 @@ snapshots: '@ionic/utils-stream@3.1.7': dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) tslib: 2.8.1 transitivePeerDependencies: - supports-color @@ -3628,7 +3777,7 @@ snapshots: '@ionic/utils-stream': 3.1.7 '@ionic/utils-terminal': 2.3.5 cross-spawn: 7.0.6 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) tslib: 2.8.1 transitivePeerDependencies: - supports-color @@ -3636,7 +3785,7 @@ snapshots: '@ionic/utils-terminal@2.3.5': dependencies: '@types/slice-ansi': 4.0.0 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) signal-exit: 3.0.7 slice-ansi: 4.0.0 string-width: 4.2.3 @@ -3842,7 +3991,7 @@ snapshots: '@malept/flatpak-bundler@0.4.0': dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) fs-extra: 9.1.0 lodash: 4.18.1 tmp-promise: 3.0.3 @@ -4021,6 +4170,53 @@ snapshots: '@preact/signals-core@1.14.4': {} + '@readme/better-ajv-errors@2.4.0(ajv@8.20.0)': + dependencies: + '@babel/code-frame': 7.29.7 + '@babel/runtime': 7.29.7 + '@humanwhocodes/momoa': 2.0.4 + ajv: 8.20.0 + jsonpointer: 5.0.1 + leven: 3.1.0 + picocolors: 1.1.1 + + '@readme/openapi-parser@6.3.0(openapi-types@12.1.3)': + dependencies: + '@apidevtools/json-schema-ref-parser': 14.2.1(@types/json-schema@7.0.15) + '@readme/better-ajv-errors': 2.4.0(ajv@8.20.0) + '@readme/openapi-schemas': 4.0.0 + '@types/json-schema': 7.0.15 + ajv: 8.20.0 + ajv-draft-04: 1.0.0(ajv@8.20.0) + openapi-types: 12.1.3 + + '@readme/openapi-schemas@4.0.0': + dependencies: + '@types/json-schema': 7.0.15 + + '@redocly/ajv@8.11.2': + dependencies: + fast-deep-equal: 3.1.3 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 + uri-js-replace: 1.0.1 + + '@redocly/config@0.22.0': {} + + '@redocly/openapi-core@1.34.17(supports-color@10.2.2)': + dependencies: + '@redocly/ajv': 8.11.2 + '@redocly/config': 0.22.0 + colorette: 1.4.0 + https-proxy-agent: 7.0.6(supports-color@10.2.2) + js-levenshtein: 1.1.6 + js-yaml: 4.2.0 + minimatch: 5.1.9 + pluralize: 8.0.0 + yaml-ast-parser: 0.0.43 + transitivePeerDependencies: + - supports-color + '@rolldown/pluginutils@1.0.1': {} '@sindresorhus/is@4.6.0': {} @@ -4185,6 +4381,8 @@ snapshots: '@types/http-cache-semantics@4.2.0': {} + '@types/json-schema@7.0.15': {} + '@types/keyv@3.1.4': dependencies: '@types/node': 24.12.4 @@ -4433,6 +4631,10 @@ snapshots: agent-base@7.1.4: {} + ajv-draft-04@1.0.0(ajv@8.20.0): + optionalDependencies: + ajv: 8.20.0 + ajv-formats@2.1.1(ajv@8.20.0): optionalDependencies: ajv: 8.20.0 @@ -4444,6 +4646,8 @@ snapshots: json-schema-traverse: 1.0.0 require-from-string: 2.0.2 + ansi-colors@4.1.3: {} + ansi-regex@5.0.1: {} ansi-styles@4.3.0: @@ -4472,7 +4676,7 @@ snapshots: builder-util-runtime: 9.7.0 chromium-pickle-js: 0.2.0 ci-info: 4.3.1 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) dmg-builder: 26.15.6(electron-builder-squirrel-windows@26.15.6) dotenv: 16.6.1 dotenv-expand: 11.0.7 @@ -4566,7 +4770,7 @@ snapshots: builder-util-runtime@9.7.0: dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) sax: 1.6.0 transitivePeerDependencies: - supports-color @@ -4577,10 +4781,10 @@ snapshots: builder-util-runtime: 9.7.0 chalk: 4.1.2 cross-spawn: 7.0.6 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) fs-extra: 10.1.0 http-proxy-agent: 7.0.2 - https-proxy-agent: 7.0.6 + https-proxy-agent: 7.0.6(supports-color@10.2.2) js-yaml: 4.3.0 sanitize-filename: 1.6.4 source-map-support: 0.5.21 @@ -4622,6 +4826,8 @@ snapshots: ansi-styles: 4.3.0 supports-color: 7.2.0 + change-case@5.4.4: {} + character-entities-html4@2.1.0: {} character-entities-legacy@3.0.0: {} @@ -4660,6 +4866,8 @@ snapshots: color-name@1.1.4: {} + colorette@1.4.0: {} + combined-stream@1.0.8: dependencies: delayed-stream: 1.0.0 @@ -4711,9 +4919,11 @@ snapshots: dependencies: mimic-fn: 3.1.0 - debug@4.4.3: + debug@4.4.3(supports-color@10.2.2): dependencies: ms: 2.1.3 + optionalDependencies: + supports-color: 10.2.2 decode-named-character-reference@1.3.0: dependencies: @@ -4855,7 +5065,7 @@ snapshots: electron-winstaller@5.4.0: dependencies: '@electron/asar': 3.4.1 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) fs-extra: 7.0.1 lodash: 4.18.1 temp: 0.9.4 @@ -4934,7 +5144,7 @@ snapshots: extract-zip@2.0.1: dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) get-stream: 5.2.0 yauzl: 2.10.0 optionalDependencies: @@ -5199,7 +5409,7 @@ snapshots: http-proxy-agent@7.0.2: dependencies: agent-base: 7.1.4 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) transitivePeerDependencies: - supports-color @@ -5208,13 +5418,15 @@ snapshots: quick-lru: 5.1.1 resolve-alpn: 1.2.1 - https-proxy-agent@7.0.6: + https-proxy-agent@7.0.6(supports-color@10.2.2): dependencies: agent-base: 7.1.4 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) transitivePeerDependencies: - supports-color + index-to-position@1.2.0: {} + inflight@1.0.6: dependencies: once: 1.4.0 @@ -5273,8 +5485,14 @@ snapshots: jiti@2.7.0: {} + js-levenshtein@1.1.6: {} + js-tokens@4.0.0: {} + js-yaml@4.2.0: + dependencies: + argparse: 2.0.1 + js-yaml@4.3.0: dependencies: argparse: 2.0.1 @@ -5300,6 +5518,8 @@ snapshots: optionalDependencies: graceful-fs: 4.2.11 + jsonpointer@5.0.1: {} + keyv@4.5.4: dependencies: json-buffer: 3.0.1 @@ -5310,6 +5530,8 @@ snapshots: lazy-val@1.0.5: {} + leven@3.1.0: {} + lexical@0.41.0: {} lib0@0.2.117: @@ -5735,7 +5957,7 @@ snapshots: micromark@4.0.2: dependencies: '@types/debug': 4.1.13 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) decode-named-character-reference: 1.3.0 devlop: 1.1.0 micromark-core-commonmark: 2.0.3 @@ -5809,7 +6031,7 @@ snapshots: '@ionic/utils-fs': 3.1.7 '@ionic/utils-terminal': 2.3.5 bplist-parser: 0.3.2 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) elementtree: 0.1.7 ini: 4.1.3 plist: 3.1.0 @@ -5868,6 +6090,24 @@ snapshots: is-docker: 2.2.1 is-wsl: 2.2.0 + openapi-fetch@0.17.0: + dependencies: + openapi-typescript-helpers: 0.1.0 + + openapi-types@12.1.3: {} + + openapi-typescript-helpers@0.1.0: {} + + openapi-typescript@7.13.0(typescript@6.0.3): + dependencies: + '@redocly/openapi-core': 1.34.17(supports-color@10.2.2) + ansi-colors: 4.1.3 + change-case: 5.4.4 + parse-json: 8.3.0 + supports-color: 10.2.2 + typescript: 6.0.3 + yargs-parser: 21.1.1 + oxfmt@0.57.0(vite-plus@0.2.2(@types/node@24.12.4)(jiti@2.7.0)(typescript@6.0.3)): dependencies: tinypool: 2.1.0 @@ -6003,6 +6243,12 @@ snapshots: is-decimal: 2.0.1 is-hexadecimal: 2.0.1 + parse-json@8.3.0: + dependencies: + '@babel/code-frame': 7.29.7 + index-to-position: 1.2.0 + type-fest: 4.41.0 + parse5@7.3.0: dependencies: entities: 6.0.1 @@ -6055,6 +6301,8 @@ snapshots: base64-js: 1.5.1 xmlbuilder: 15.1.1 + pluralize@8.0.0: {} + pngjs@7.0.0: {} postcss@8.5.16: @@ -6150,7 +6398,7 @@ snapshots: read-binary-file-arch@1.0.6: dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) transitivePeerDependencies: - supports-color @@ -6371,10 +6619,12 @@ snapshots: sumchecker@3.0.1: dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) transitivePeerDependencies: - supports-color + supports-color@10.2.2: {} + supports-color@7.2.0: dependencies: has-flag: 4.0.0 @@ -6453,6 +6703,8 @@ snapshots: type-fest@2.19.0: {} + type-fest@4.41.0: {} + typescript@6.0.3: {} undici-types@7.16.0: {} @@ -6508,6 +6760,8 @@ snapshots: graceful-fs: 4.2.11 node-int64: 0.4.0 + uri-js-replace@1.0.1: {} + use-sync-external-store@1.6.0(react@19.2.6): dependencies: react: 19.2.6 @@ -6757,6 +7011,8 @@ snapshots: yallist@5.0.0: {} + yaml-ast-parser@0.0.43: {} + yargs-parser@21.1.1: {} yargs@17.7.3: From 8a7e8cc3f934af3482d35d9161cf473e25c73d7a Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 19:44:29 +0000 Subject: [PATCH 005/131] fix: use OpenAPI parser named API --- packages/t4-api-contract/scripts/validate.mjs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/t4-api-contract/scripts/validate.mjs b/packages/t4-api-contract/scripts/validate.mjs index 521893c0..40a5e4d6 100644 --- a/packages/t4-api-contract/scripts/validate.mjs +++ b/packages/t4-api-contract/scripts/validate.mjs @@ -1,11 +1,12 @@ import { readFile } from "node:fs/promises"; import { fileURLToPath } from "node:url"; -import OpenAPIParser from "@readme/openapi-parser"; +import { compileErrors, validate } from "@readme/openapi-parser"; const source = new URL("../openapi.json", import.meta.url); const document = JSON.parse(await readFile(source, "utf8")); -await OpenAPIParser.validate(fileURLToPath(source)); +const validation = await validate(fileURLToPath(source)); +if (!validation.valid || validation.warnings.length > 0) throw new Error(compileErrors(validation)); if (document.openapi !== "3.1.0") throw new Error("T4 API contract must remain OpenAPI 3.1.0"); if (!Array.isArray(document.servers) || document.servers.length === 0) throw new Error("T4 API contract requires an HTTPS server"); From e549d60a8b3f5f732ceb6b71bf4bf972ee363ea7 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 19:47:47 +0000 Subject: [PATCH 006/131] feat: add generated T4 API v1 types --- .github/workflows/ci.yml | 19 +- .woodpecker.yml | 26 +- .../t4-api-client/src/generated/schema.ts | 1035 +++++++++++++++++ 3 files changed, 1041 insertions(+), 39 deletions(-) create mode 100755 packages/t4-api-client/src/generated/schema.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 631e07ea..96e78302 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,24 +76,15 @@ jobs: with: node-version: 24.13.1 - - name: Install generation dependencies from the recovered lockfile + - name: Install dependencies from the committed lockfile run: pnpm install --frozen-lockfile - - name: Generate and validate the T4 API contract artifact + - name: Validate OpenAPI and deterministic generation run: | - pnpm --filter @t4-code/t4-api-contract generate:ci pnpm --filter @t4-code/t4-api-contract validate - - - name: Upload deterministic T4 API generation inputs - if: ${{ always() }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: t4-api-generation-${{ github.run_id }} - path: | - pnpm-lock.yaml - artifacts/t4-api/generated/schema.ts - if-no-files-found: error - retention-days: 14 + pnpm --filter @t4-code/t4-api-contract check:generated + pnpm --filter @t4-code/t4-api-client typecheck + pnpm --filter @t4-code/client exec vp test run test/t4-api-v1-conformance.test.ts core: runs-on: ubuntu-24.04 diff --git a/.woodpecker.yml b/.woodpecker.yml index 1f2cdf64..019c4f4c 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -12,31 +12,7 @@ clone: partial: false steps: - t4-api-artifact-bootstrap: - image: mirror.gcr.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 - commands: - - corepack enable - - pnpm install --frozen-lockfile - - pnpm --filter @t4-code/t4-api-contract generate:ci - - pnpm --filter @t4-code/t4-api-contract validate - when: - - event: push - branch: agent/t4-op03-public-api-contract - backend_options: - kubernetes: - serviceAccountName: woodpecker-ci-untrusted - resources: - requests: - cpu: "0" - memory: 512Mi - ephemeral-storage: 1Gi - limits: - cpu: "1" - memory: 2Gi - ephemeral-storage: 4Gi - dependencies: - depends_on: [t4-api-artifact-bootstrap] image: mirror.gcr.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 commands: - corepack enable @@ -61,7 +37,7 @@ steps: commands: - corepack enable - pnpm --filter @t4-code/t4-api-contract validate - - pnpm --filter @t4-code/t4-api-contract generate:ci + - pnpm --filter @t4-code/t4-api-contract check:generated - pnpm --filter @t4-code/t4-api-client typecheck - pnpm --filter @t4-code/client exec vp test run test/t4-api-v1-conformance.test.ts backend_options: diff --git a/packages/t4-api-client/src/generated/schema.ts b/packages/t4-api-client/src/generated/schema.ts new file mode 100755 index 00000000..03954962 --- /dev/null +++ b/packages/t4-api-client/src/generated/schema.ts @@ -0,0 +1,1035 @@ +export interface paths { + "/v1": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Negotiate T4 API v1 and discover capabilities and bounds */ + get: operations["discoverV1"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/sessions/{sessionId}": { + parameters: { + query?: never; + header?: never; + path: { + sessionId: components["parameters"]["SessionId"]; + }; + cookie?: never; + }; + get: operations["getSession"]; + put?: never; + post?: never; + delete: operations["deleteSession"]; + options?: never; + head?: never; + patch: operations["mutateSession"]; + trace?: never; + }; + "/v1/sessions/{sessionId}/cancel": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["cancelSession"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/sessions/{sessionId}/commands": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: operations["submitCommand"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/sessions/{sessionId}/events": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Watch a bounded Server-Sent Events stream over HTTPS + * @description The response is standard Server-Sent Events over HTTPS. Clients reconnect with the last event cursor in the cursor query parameter or Last-Event-ID header. Heartbeat events bound idle detection. Clients explicitly cancel by aborting the HTTPS request. A retained cursor expiry returns a typed 410 response with a snapshot resync target. + */ + get: operations["watchSessionEvents"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/sessions/{sessionId}/snapshot": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["getSessionSnapshot"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/workspaces": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: operations["listWorkspaces"]; + put?: never; + post: operations["createWorkspace"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/workspaces/{workspaceId}": { + parameters: { + query?: never; + header?: never; + path: { + workspaceId: components["parameters"]["WorkspaceId"]; + }; + cookie?: never; + }; + get: operations["getWorkspace"]; + put?: never; + post?: never; + delete: operations["deleteWorkspace"]; + options?: never; + head?: never; + patch: operations["mutateWorkspace"]; + trace?: never; + }; + "/v1/workspaces/{workspaceId}/sessions": { + parameters: { + query?: never; + header?: never; + path: { + workspaceId: components["parameters"]["WorkspaceId"]; + }; + cookie?: never; + }; + get: operations["listSessions"]; + put?: never; + post: operations["spawnSession"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; +} +export type webhooks = Record; +export interface components { + schemas: { + ApiError: { + /** @enum {string} */ + code: "invalid_request" | "unauthenticated" | "forbidden" | "not_found" | "idempotency_key_required" | "idempotency_conflict" | "revision_conflict" | "incompatible_version" | "cursor_expired" | "unavailable" | "indeterminate" | "invalid_origin" | "https_required"; + message: string; + requestId: string; + resync?: components["schemas"]["Resync"]; + retryable: boolean; + supportedMajors?: number[]; + violations?: components["schemas"]["FieldViolation"][]; + }; + CommandCreate: { + command: string; + metadata?: { + [key: string]: string | number | boolean | null; + }; + }; + CommandResult: { + commandId: components["schemas"]["ResourceId"]; + state: components["schemas"]["OperationState"]; + }; + CommandWatchEvent: { + commandId: components["schemas"]["ResourceId"]; + cursor: components["schemas"]["Cursor"]; + state: components["schemas"]["OperationState"]; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "CommandWatchEvent"; + }; + /** @description Opaque server-issued header-safe SSE cursor. */ + Cursor: string; + CursorExpiredApiError: components["schemas"]["ApiError"] & { + /** @constant */ + code?: "cursor_expired"; + resync: components["schemas"]["Resync"]; + }; + CursorExpiredErrorEnvelope: { + error: components["schemas"]["CursorExpiredApiError"]; + }; + Discovery: { + apiVersion: string; + capabilities: string[]; + limits: { + commandBytesMax: number; + heartbeatSeconds: number; + pageSizeDefault: number; + pageSizeMax: number; + watchEventsMax: number; + }; + supportedMajors: number[]; + }; + ErrorEnvelope: { + error: components["schemas"]["ApiError"]; + }; + FieldViolation: { + field: string; + message: string; + rule: string; + }; + HeartbeatWatchEvent: { + cursor: components["schemas"]["Cursor"]; + /** Format: date-time */ + observedAt: string; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "HeartbeatWatchEvent"; + }; + IncompatibleVersionApiError: components["schemas"]["ApiError"] & { + /** @constant */ + code?: "incompatible_version"; + supportedMajors: number[]; + }; + IncompatibleVersionErrorEnvelope: { + error: components["schemas"]["IncompatibleVersionApiError"]; + }; + InvalidRequestApiError: components["schemas"]["ApiError"] & { + /** @constant */ + code?: "invalid_request"; + violations: components["schemas"]["FieldViolation"][]; + }; + InvalidRequestErrorEnvelope: { + error: components["schemas"]["InvalidRequestApiError"]; + }; + Labels: { + [key: string]: string; + }; + /** @enum {string} */ + OperationState: "accepted" | "rejected" | "conflict" | "unavailable" | "indeterminate"; + ResourceId: string; + Resync: { + cursor: components["schemas"]["Cursor"]; + snapshotUrl: string; + }; + Revision: number; + Session: { + id: components["schemas"]["ResourceId"]; + labels?: components["schemas"]["Labels"]; + revision: components["schemas"]["Revision"]; + state: components["schemas"]["SessionState"]; + title: string; + workspaceId: components["schemas"]["ResourceId"]; + }; + SessionCreate: { + labels?: components["schemas"]["Labels"]; + title: string; + }; + SessionMutation: { + labels?: components["schemas"]["Labels"]; + title?: string; + }; + SessionPage: { + items: components["schemas"]["Session"][]; + nextCursor?: components["schemas"]["Cursor"]; + }; + SessionSnapshot: { + cursor: components["schemas"]["Cursor"]; + entries: components["schemas"]["SnapshotEntry"][]; + sessionId: components["schemas"]["ResourceId"]; + state: components["schemas"]["SessionState"]; + }; + /** @enum {string} */ + SessionState: "accepted" | "provisioning" | "ready" | "cancelling" | "cancelled" | "failed" | "unavailable" | "indeterminate"; + SessionWatchEvent: { + cursor: components["schemas"]["Cursor"]; + revision: components["schemas"]["Revision"]; + state: components["schemas"]["SessionState"]; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "SessionWatchEvent"; + }; + SnapshotEntry: { + /** @enum {string} */ + kind: "input" | "output" | "status"; + sequence: number; + text: string; + }; + WatchEvent: components["schemas"]["HeartbeatWatchEvent"] | components["schemas"]["SessionWatchEvent"] | components["schemas"]["CommandWatchEvent"]; + Workspace: { + id: components["schemas"]["ResourceId"]; + labels?: components["schemas"]["Labels"]; + name: string; + revision: components["schemas"]["Revision"]; + state: components["schemas"]["WorkspaceState"]; + }; + WorkspaceCreate: { + labels?: components["schemas"]["Labels"]; + name: string; + }; + WorkspaceMutation: { + labels?: components["schemas"]["Labels"]; + name?: string; + }; + WorkspacePage: { + items: components["schemas"]["Workspace"][]; + nextCursor?: components["schemas"]["Cursor"]; + }; + /** @enum {string} */ + WorkspaceState: "accepted" | "provisioning" | "ready" | "deleting" | "deleted" | "failed" | "unavailable" | "indeterminate"; + }; + responses: { + BadRequest: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Command intent accepted with a stable outcome state */ + CommandAccepted: { + headers: { + "Idempotency-Replayed": components["headers"]["IdempotencyReplayed"]; + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["CommandResult"]; + }; + }; + /** @description Identical command request replay */ + CommandReplay: { + headers: { + "Idempotency-Replayed": components["headers"]["IdempotencyReplayed"]; + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["CommandResult"]; + }; + }; + Conflict: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + CursorExpired: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Deletion accepted or resource already absent */ + Deleted: { + headers: { + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content?: never; + }; + /** @description Negotiated v1 discovery */ + Discovery: { + headers: { + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Discovery"]; + }; + }; + /** @description Malformed request or missing idempotency key */ + Error400: { + headers: { + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorEnvelope"]; + }; + }; + /** @description Missing or invalid opaque bearer credential */ + Error401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorEnvelope"]; + }; + }; + /** @description Credential lacks the deny-by-default operation or resource scope */ + Error403: { + headers: { + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorEnvelope"]; + }; + }; + /** @description Resource absent or outside caller scope (scope existence is not disclosed) */ + Error404: { + headers: { + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorEnvelope"]; + }; + }; + /** @description Requested major is incompatible; no silent downgrade */ + Error406: { + headers: { + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["IncompatibleVersionErrorEnvelope"]; + }; + }; + /** @description Idempotency or resource revision conflict */ + Error409: { + headers: { + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorEnvelope"]; + }; + }; + /** @description Watch cursor expired; resync from the typed snapshot target */ + Error410: { + headers: { + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["CursorExpiredErrorEnvelope"]; + }; + }; + /** @description Bounded semantic validation failure with stable field violations */ + Error422: { + headers: { + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["InvalidRequestErrorEnvelope"]; + }; + }; + /** @description Service temporarily unavailable or outcome indeterminate */ + Error503: { + headers: { + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorEnvelope"]; + }; + }; + Forbidden: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + IncompatibleVersion: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + InvalidRequest: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + NotFound: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Session */ + Session: { + headers: { + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Session"]; + }; + }; + /** @description Session intent accepted */ + SessionAccepted: { + headers: { + "Idempotency-Replayed": components["headers"]["IdempotencyReplayed"]; + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Session"]; + }; + }; + /** @description Bounded session page */ + SessionPage: { + headers: { + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SessionPage"]; + }; + }; + /** @description Identical session spawn replay */ + SessionReplay: { + headers: { + "Idempotency-Replayed": components["headers"]["IdempotencyReplayed"]; + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Session"]; + }; + }; + /** @description Bounded session snapshot and reconnect cursor */ + Snapshot: { + headers: { + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SessionSnapshot"]; + }; + }; + Unauthenticated: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + Unavailable: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Workspace */ + Workspace: { + headers: { + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Workspace"]; + }; + }; + /** @description Workspace intent accepted */ + WorkspaceAccepted: { + headers: { + "Idempotency-Replayed": components["headers"]["IdempotencyReplayed"]; + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Workspace"]; + }; + }; + /** @description Bounded workspace page */ + WorkspacePage: { + headers: { + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["WorkspacePage"]; + }; + }; + /** @description Identical workspace request replay */ + WorkspaceReplay: { + headers: { + "Idempotency-Replayed": components["headers"]["IdempotencyReplayed"]; + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Workspace"]; + }; + }; + }; + parameters: { + /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ + ApiVersion: string; + HeartbeatSeconds: number; + /** @description Opaque caller-generated key scoped to credential, operation, and target. Same key plus same canonical request replays; different content conflicts. */ + IdempotencyKey: string; + /** @description Decimal resource revision for optimistic mutation. */ + IfMatch: string; + /** @description Standard SSE reconnect header. Must agree with cursor when both are present. */ + LastEventId: components["schemas"]["Cursor"]; + MaxEvents: number; + /** @description Opaque page cursor, valid only for the same identity and list operation. */ + PageCursor: components["schemas"]["Cursor"]; + PageSize: number; + SessionId: components["schemas"]["ResourceId"]; + /** @description Opaque last-consumed watch event cursor. */ + WatchCursor: components["schemas"]["Cursor"]; + WorkspaceId: components["schemas"]["ResourceId"]; + }; + requestBodies: never; + headers: { + /** @description True when this response replays a prior identical request. */ + IdempotencyReplayed: "true" | "false"; + /** @description Selected compatible T4 API minor version. */ + SelectedVersion: string; + }; + pathItems: never; +} +export type $defs = Record; +export interface operations { + discoverV1: { + parameters: { + query?: never; + header: { + /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ + "T4-API-Version": components["parameters"]["ApiVersion"]; + }; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: components["responses"]["Discovery"]; + 401: components["responses"]["Unauthenticated"]; + 403: components["responses"]["Forbidden"]; + 406: components["responses"]["IncompatibleVersion"]; + 503: components["responses"]["Unavailable"]; + }; + }; + getSession: { + parameters: { + query?: never; + header: { + /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ + "T4-API-Version": components["parameters"]["ApiVersion"]; + }; + path: { + sessionId: components["parameters"]["SessionId"]; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: components["responses"]["Session"]; + 401: components["responses"]["Unauthenticated"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 406: components["responses"]["IncompatibleVersion"]; + 503: components["responses"]["Unavailable"]; + }; + }; + deleteSession: { + parameters: { + query?: never; + header: { + /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ + "T4-API-Version": components["parameters"]["ApiVersion"]; + }; + path: { + sessionId: components["parameters"]["SessionId"]; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 204: components["responses"]["Deleted"]; + 401: components["responses"]["Unauthenticated"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 406: components["responses"]["IncompatibleVersion"]; + 409: components["responses"]["Conflict"]; + 503: components["responses"]["Unavailable"]; + }; + }; + mutateSession: { + parameters: { + query?: never; + header: { + /** @description Decimal resource revision for optimistic mutation. */ + "If-Match": components["parameters"]["IfMatch"]; + /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ + "T4-API-Version": components["parameters"]["ApiVersion"]; + }; + path: { + sessionId: components["parameters"]["SessionId"]; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["SessionMutation"]; + }; + }; + responses: { + 200: components["responses"]["Session"]; + 401: components["responses"]["Unauthenticated"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 406: components["responses"]["IncompatibleVersion"]; + 409: components["responses"]["Conflict"]; + 422: components["responses"]["InvalidRequest"]; + 503: components["responses"]["Unavailable"]; + }; + }; + cancelSession: { + parameters: { + query?: never; + header: { + /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ + "T4-API-Version": components["parameters"]["ApiVersion"]; + }; + path: { + sessionId: components["parameters"]["SessionId"]; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 202: components["responses"]["SessionAccepted"]; + 401: components["responses"]["Unauthenticated"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 406: components["responses"]["IncompatibleVersion"]; + 409: components["responses"]["Conflict"]; + 503: components["responses"]["Unavailable"]; + }; + }; + submitCommand: { + parameters: { + query?: never; + header: { + /** @description Opaque caller-generated key scoped to credential, operation, and target. Same key plus same canonical request replays; different content conflicts. */ + "Idempotency-Key": components["parameters"]["IdempotencyKey"]; + /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ + "T4-API-Version": components["parameters"]["ApiVersion"]; + }; + path: { + sessionId: components["parameters"]["SessionId"]; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["CommandCreate"]; + }; + }; + responses: { + 200: components["responses"]["CommandReplay"]; + 202: components["responses"]["CommandAccepted"]; + 401: components["responses"]["Unauthenticated"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 406: components["responses"]["IncompatibleVersion"]; + 409: components["responses"]["Conflict"]; + 422: components["responses"]["InvalidRequest"]; + 503: components["responses"]["Unavailable"]; + }; + }; + watchSessionEvents: { + parameters: { + query?: { + /** @description Opaque last-consumed watch event cursor. */ + cursor?: components["parameters"]["WatchCursor"]; + heartbeatSeconds?: components["parameters"]["HeartbeatSeconds"]; + maxEvents?: components["parameters"]["MaxEvents"]; + }; + header: { + /** @description Standard SSE reconnect header. Must agree with cursor when both are present. */ + "Last-Event-ID"?: components["parameters"]["LastEventId"]; + /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ + "T4-API-Version": components["parameters"]["ApiVersion"]; + }; + path: { + sessionId: components["parameters"]["SessionId"]; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Bounded SSE event stream; each data field is a WatchEvent JSON value */ + 200: { + headers: { + "Cache-Control"?: "no-store"; + "T4-API-Version": components["headers"]["SelectedVersion"]; + [name: string]: unknown; + }; + content: { + /** @example id: cur_01J...\nevent: heartbeat\ndata: {"type":"heartbeat","cursor":"cur_01J...","observedAt":"2026-07-21T00:00:00Z"}\n\n */ + "text/event-stream": string; + }; + }; + 400: components["responses"]["BadRequest"]; + 401: components["responses"]["Unauthenticated"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 406: components["responses"]["IncompatibleVersion"]; + 410: components["responses"]["CursorExpired"]; + 422: components["responses"]["InvalidRequest"]; + 503: components["responses"]["Unavailable"]; + }; + }; + getSessionSnapshot: { + parameters: { + query?: never; + header: { + /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ + "T4-API-Version": components["parameters"]["ApiVersion"]; + }; + path: { + sessionId: components["parameters"]["SessionId"]; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: components["responses"]["Snapshot"]; + 401: components["responses"]["Unauthenticated"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 406: components["responses"]["IncompatibleVersion"]; + 503: components["responses"]["Unavailable"]; + }; + }; + listWorkspaces: { + parameters: { + query?: { + /** @description Opaque page cursor, valid only for the same identity and list operation. */ + cursor?: components["parameters"]["PageCursor"]; + pageSize?: components["parameters"]["PageSize"]; + }; + header: { + /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ + "T4-API-Version": components["parameters"]["ApiVersion"]; + }; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: components["responses"]["WorkspacePage"]; + 400: components["responses"]["BadRequest"]; + 401: components["responses"]["Unauthenticated"]; + 403: components["responses"]["Forbidden"]; + 406: components["responses"]["IncompatibleVersion"]; + 422: components["responses"]["InvalidRequest"]; + 503: components["responses"]["Unavailable"]; + }; + }; + createWorkspace: { + parameters: { + query?: never; + header: { + /** @description Opaque caller-generated key scoped to credential, operation, and target. Same key plus same canonical request replays; different content conflicts. */ + "Idempotency-Key": components["parameters"]["IdempotencyKey"]; + /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ + "T4-API-Version": components["parameters"]["ApiVersion"]; + }; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["WorkspaceCreate"]; + }; + }; + responses: { + 200: components["responses"]["WorkspaceReplay"]; + 202: components["responses"]["WorkspaceAccepted"]; + 400: components["responses"]["BadRequest"]; + 401: components["responses"]["Unauthenticated"]; + 403: components["responses"]["Forbidden"]; + 406: components["responses"]["IncompatibleVersion"]; + 409: components["responses"]["Conflict"]; + 422: components["responses"]["InvalidRequest"]; + 503: components["responses"]["Unavailable"]; + }; + }; + getWorkspace: { + parameters: { + query?: never; + header: { + /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ + "T4-API-Version": components["parameters"]["ApiVersion"]; + }; + path: { + workspaceId: components["parameters"]["WorkspaceId"]; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: components["responses"]["Workspace"]; + 401: components["responses"]["Unauthenticated"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 406: components["responses"]["IncompatibleVersion"]; + 503: components["responses"]["Unavailable"]; + }; + }; + deleteWorkspace: { + parameters: { + query?: never; + header: { + /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ + "T4-API-Version": components["parameters"]["ApiVersion"]; + }; + path: { + workspaceId: components["parameters"]["WorkspaceId"]; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 204: components["responses"]["Deleted"]; + 401: components["responses"]["Unauthenticated"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 406: components["responses"]["IncompatibleVersion"]; + 409: components["responses"]["Conflict"]; + 503: components["responses"]["Unavailable"]; + }; + }; + mutateWorkspace: { + parameters: { + query?: never; + header: { + /** @description Decimal resource revision for optimistic mutation. */ + "If-Match": components["parameters"]["IfMatch"]; + /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ + "T4-API-Version": components["parameters"]["ApiVersion"]; + }; + path: { + workspaceId: components["parameters"]["WorkspaceId"]; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["WorkspaceMutation"]; + }; + }; + responses: { + 200: components["responses"]["Workspace"]; + 401: components["responses"]["Unauthenticated"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 406: components["responses"]["IncompatibleVersion"]; + 409: components["responses"]["Conflict"]; + 422: components["responses"]["InvalidRequest"]; + 503: components["responses"]["Unavailable"]; + }; + }; + listSessions: { + parameters: { + query?: { + /** @description Opaque page cursor, valid only for the same identity and list operation. */ + cursor?: components["parameters"]["PageCursor"]; + pageSize?: components["parameters"]["PageSize"]; + }; + header: { + /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ + "T4-API-Version": components["parameters"]["ApiVersion"]; + }; + path: { + workspaceId: components["parameters"]["WorkspaceId"]; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + 200: components["responses"]["SessionPage"]; + 401: components["responses"]["Unauthenticated"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 406: components["responses"]["IncompatibleVersion"]; + 422: components["responses"]["InvalidRequest"]; + 503: components["responses"]["Unavailable"]; + }; + }; + spawnSession: { + parameters: { + query?: never; + header: { + /** @description Opaque caller-generated key scoped to credential, operation, and target. Same key plus same canonical request replays; different content conflicts. */ + "Idempotency-Key": components["parameters"]["IdempotencyKey"]; + /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ + "T4-API-Version": components["parameters"]["ApiVersion"]; + }; + path: { + workspaceId: components["parameters"]["WorkspaceId"]; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["SessionCreate"]; + }; + }; + responses: { + 200: components["responses"]["SessionReplay"]; + 202: components["responses"]["SessionAccepted"]; + 401: components["responses"]["Unauthenticated"]; + 403: components["responses"]["Forbidden"]; + 404: components["responses"]["NotFound"]; + 406: components["responses"]["IncompatibleVersion"]; + 409: components["responses"]["Conflict"]; + 422: components["responses"]["InvalidRequest"]; + 503: components["responses"]["Unavailable"]; + }; + }; +} + From 6cd0e12dcc28bdc14c60bbb3314bf4ee2b622cb3 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 19:54:50 +0000 Subject: [PATCH 007/131] fix: typecheck T4 API conformance fixtures --- .github/workflows/ci.yml | 1 + .woodpecker.yml | 1 + packages/client/test/t4-api-v1-conformance-service.ts | 6 +++--- packages/client/test/t4-api-v1-conformance.test.ts | 4 ++-- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96e78302..9b0c5438 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -84,6 +84,7 @@ jobs: pnpm --filter @t4-code/t4-api-contract validate pnpm --filter @t4-code/t4-api-contract check:generated pnpm --filter @t4-code/t4-api-client typecheck + pnpm --filter @t4-code/client typecheck pnpm --filter @t4-code/client exec vp test run test/t4-api-v1-conformance.test.ts core: diff --git a/.woodpecker.yml b/.woodpecker.yml index 019c4f4c..c7ad7c55 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -39,6 +39,7 @@ steps: - pnpm --filter @t4-code/t4-api-contract validate - pnpm --filter @t4-code/t4-api-contract check:generated - pnpm --filter @t4-code/t4-api-client typecheck + - pnpm --filter @t4-code/client typecheck - pnpm --filter @t4-code/client exec vp test run test/t4-api-v1-conformance.test.ts backend_options: kubernetes: diff --git a/packages/client/test/t4-api-v1-conformance-service.ts b/packages/client/test/t4-api-v1-conformance-service.ts index 2e9cb745..8786698d 100644 --- a/packages/client/test/t4-api-v1-conformance-service.ts +++ b/packages/client/test/t4-api-v1-conformance-service.ts @@ -122,7 +122,7 @@ export class T4ApiV1ConformanceService { if (request.headers.get("If-Match") !== String(workspace.revision)) { return problem(409, "revision_conflict", "Workspace revision changed"); } - const updated = { ...workspace, name: body.name ?? workspace.name, revision: Number(workspace.revision) + 1 }; + const updated: Record = { ...workspace, name: body.name ?? workspace.name, revision: Number(workspace.revision) + 1 }; this.#workspaces.set(id, updated); const { tenant: _tenant, ...visible } = updated; return json(200, visible); @@ -181,7 +181,7 @@ export class T4ApiV1ConformanceService { if (request.headers.get("If-Match") !== String(session.revision)) { return problem(409, "revision_conflict", "Session revision changed"); } - const updated = { ...session, title: body.title ?? session.title, revision: Number(session.revision) + 1 }; + const updated: Record = { ...session, title: body.title ?? session.title, revision: Number(session.revision) + 1 }; this.#sessions.set(id, updated); const { tenant: _tenant, ...visible } = updated; return json(200, visible); @@ -196,7 +196,7 @@ export class T4ApiV1ConformanceService { if (cancelMatch && request.method === "POST") { const session = this.#sessions.get(decodeURIComponent(cancelMatch[1]!)); if (session?.tenant !== tenant) return problem(404, "not_found", "Session not found"); - const cancelled = { ...session, state: "cancelled", revision: Number(session.revision) + 1 }; + const cancelled: Record = { ...session, state: "cancelled", revision: Number(session.revision) + 1 }; this.#sessions.set(String(session.id), cancelled); const { tenant: _tenant, ...visible } = cancelled; return json(202, visible); diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index 7d312024..68f7ab42 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -85,7 +85,7 @@ describe("generated T4 API v1 client conformance", () => { expect(pageOne.items).toHaveLength(2); expect(pageOne.nextCursor).toBe("page-2"); const pageTwo = requireData(await owner.http.GET("/v1/workspaces", { - params: { header: VERSION_HEADERS, query: { pageSize: 2, cursor: pageOne.nextCursor } }, + params: { header: VERSION_HEADERS, query: { pageSize: 2, cursor: pageOne.nextCursor! } }, })); expect(pageTwo.items).toHaveLength(2); expect(pageTwo.nextCursor).toBeUndefined(); @@ -148,7 +148,7 @@ describe("generated T4 API v1 client conformance", () => { expect(sessionPageOne.items).toHaveLength(2); expect(sessionPageOne.nextCursor).toBe("page-2"); const sessionPageTwo = requireData(await client.http.GET("/v1/workspaces/{workspaceId}/sessions", { - params: { header: VERSION_HEADERS, path: { workspaceId: "ws-1" }, query: { pageSize: 2, cursor: sessionPageOne.nextCursor } }, + params: { header: VERSION_HEADERS, path: { workspaceId: "ws-1" }, query: { pageSize: 2, cursor: sessionPageOne.nextCursor! } }, })); expect(sessionPageTwo.items).toHaveLength(1); From bcca1cbf2fc5677dffa6a0c9adcec45264e0cc30 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 20:23:36 +0000 Subject: [PATCH 008/131] test: expose T4 API semantic blockers --- .../test/t4-api-v1-conformance-service.ts | 272 +++++++++--------- .../client/test/t4-api-v1-conformance.test.ts | 177 ++++++++++-- 2 files changed, 303 insertions(+), 146 deletions(-) diff --git a/packages/client/test/t4-api-v1-conformance-service.ts b/packages/client/test/t4-api-v1-conformance-service.ts index 8786698d..15b76db8 100644 --- a/packages/client/test/t4-api-v1-conformance-service.ts +++ b/packages/client/test/t4-api-v1-conformance-service.ts @@ -1,32 +1,43 @@ import { expect } from "vite-plus/test"; const encoder = new TextEncoder(); +const COMMAND_BYTES_MAX = 32; +const COMMAND_REQUEST_BYTES_MAX = 256; +const METADATA_VALUE_BYTES_MAX = 32; function json(status: number, body: unknown, headers: Record = {}): Response { return Response.json(body, { status, headers: { "T4-API-Version": "1.0", ...headers } }); } function problem(status: number, code: string, message: string, extra: Record = {}): Response { - return json(status, { - error: { - code, - message, - requestId: `req-${code}`, - retryable: status >= 500, - ...extra, - }, - }); + return json(status, { error: { code, message, requestId: `req-${code}`, retryable: status >= 500, ...extra } }); } -function bodyKey(value: unknown): string { - if (Array.isArray(value)) return `[${value.map(bodyKey).join(",")}]`; - if (value !== null && typeof value === "object") { +function compareUtf16(left: string, right: string): number { + return left < right ? -1 : left > right ? 1 : 0; +} + +export function canonicalJson(value: unknown): string { + if (value === null || typeof value === "boolean" || typeof value === "string") return JSON.stringify(value); + if (typeof value === "number") { + if (!Number.isFinite(value)) throw new TypeError("JCS only permits finite JSON numbers"); + return JSON.stringify(value); + } + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (typeof value === "object") { return `{${Object.entries(value as Record) - .sort(([left], [right]) => left.localeCompare(right)) - .map(([key, item]) => `${JSON.stringify(key)}:${bodyKey(item)}`) + .filter(([, item]) => item !== undefined) + .sort(([left], [right]) => compareUtf16(left, right)) + .map(([key, item]) => `${JSON.stringify(key)}:${canonicalJson(item)}`) .join(",")}}`; } - return JSON.stringify(value) ?? "null"; + throw new TypeError("JCS identity requires a JSON value"); +} + +interface ReplayRecord { + readonly identity: string; + readonly body: unknown; + readonly replayStatus: number; } export class T4ApiV1ConformanceService { @@ -40,7 +51,7 @@ export class T4ApiV1ConformanceService { #commandSequence = 0; readonly #workspaces = new Map>(); readonly #sessions = new Map>(); - readonly #replays = new Map }>(); + readonly #replays = new Map(); readonly fetch: typeof globalThis.fetch = async (input, init) => { const request = new Request(input, init); @@ -49,30 +60,26 @@ export class T4ApiV1ConformanceService { this.calls.push({ method: request.method, path: url.pathname, authorization }); if (url.origin !== this.origin) return problem(400, "invalid_origin", "HTTPS API origin is fixed for this client"); if (url.protocol !== "https:") return problem(400, "https_required", "HTTPS is required"); - if (authorization !== "Bearer token-a" && authorization !== "Bearer token-b") { + if (authorization !== "Bearer token-a" && authorization !== "Bearer token-b" && authorization !== "Bearer token-denied") { return problem(401, "unauthenticated", "A valid bearer credential is required"); } if (request.headers.get("T4-API-Version") !== "1") { - return problem(406, "incompatible_version", "No compatible T4 API major version", { - supportedMajors: [1], - }); + return problem(406, "incompatible_version", "No compatible T4 API major version", { supportedMajors: [1] }); } + if (authorization === "Bearer token-denied") return problem(403, "forbidden", "Credential lacks the required operation scope"); const tenant = authorization === "Bearer token-a" ? "tenant-a" : "tenant-b"; if (request.method === "GET" && url.pathname === "/v1") { return json(200, { apiVersion: "1.0", supportedMajors: [1], - capabilities: [ - "workspace.lifecycle", - "session.lifecycle", - "session.commands", - "session.watch.sse", - ], + capabilities: ["workspace.lifecycle", "session.lifecycle", "session.commands", "session.watch.sse"], limits: { pageSizeDefault: 2, pageSizeMax: 3, - commandBytesMax: 32, + commandBytesMax: COMMAND_BYTES_MAX, + commandRequestBytesMax: COMMAND_REQUEST_BYTES_MAX, + commandMetadataValueBytesMax: METADATA_VALUE_BYTES_MAX, watchEventsMax: 4, heartbeatSeconds: 15, }, @@ -81,14 +88,10 @@ export class T4ApiV1ConformanceService { if (request.method === "POST" && url.pathname === "/v1/workspaces") { const body = await request.json() as Record; - if (typeof body.name !== "string" || body.name.length < 1 || body.name.length > 128) { - return problem(422, "invalid_request", "Request validation failed", { - violations: [{ field: "name", rule: "length", message: "name must contain 1 to 128 characters" }], - }); - } - return this.#idempotent(request, tenant, body, () => { + if (typeof body.name !== "string" || body.name.length < 1 || body.name.length > 128) return this.#invalid("name", "length", "name must contain 1 to 128 characters"); + return this.#idempotent(request, tenant, "createWorkspace", [], body, 202, 200, () => { const id = `ws-${++this.#workspaceSequence}`; - const workspace = { id, name: body.name, state: "accepted", revision: 1, tenant }; + const workspace = { id, name: body.name, ...(body.labels === undefined ? {} : { labels: body.labels }), state: "accepted", revision: 1, tenant }; this.#workspaces.set(id, workspace); return workspace; }); @@ -96,11 +99,7 @@ export class T4ApiV1ConformanceService { if (request.method === "GET" && url.pathname === "/v1/workspaces") { const pageSize = Number(url.searchParams.get("pageSize") ?? "2"); - if (!Number.isInteger(pageSize) || pageSize < 1 || pageSize > 3) { - return problem(422, "invalid_request", "Request validation failed", { - violations: [{ field: "pageSize", rule: "range", message: "pageSize must be between 1 and 3" }], - }); - } + if (!Number.isInteger(pageSize) || pageSize < 1 || pageSize > 3) return this.#invalid("pageSize", "range", "pageSize must be between 1 and 3"); const start = Number(url.searchParams.get("cursor")?.replace("page-", "") ?? "0"); const visible = [...this.#workspaces.values()].filter((item) => item.tenant === tenant); const items = visible.slice(start, start + pageSize).map(({ tenant: _tenant, ...item }) => item); @@ -113,23 +112,21 @@ export class T4ApiV1ConformanceService { const id = decodeURIComponent(workspaceMatch[1]!); const workspace = this.#workspaces.get(id); if (workspace?.tenant !== tenant) return problem(404, "not_found", "Workspace not found"); - if (request.method === "GET") { - const { tenant: _tenant, ...visible } = workspace; - return json(200, visible); - } + if (request.method === "GET") return json(200, this.#visible(workspace)); if (request.method === "PATCH") { const body = await request.json() as Record; - if (request.headers.get("If-Match") !== String(workspace.revision)) { - return problem(409, "revision_conflict", "Workspace revision changed"); - } - const updated: Record = { ...workspace, name: body.name ?? workspace.name, revision: Number(workspace.revision) + 1 }; - this.#workspaces.set(id, updated); - const { tenant: _tenant, ...visible } = updated; - return json(200, visible); + if (request.headers.get("If-Match") !== String(workspace.revision)) return problem(409, "revision_conflict", "Workspace revision changed"); + return this.#idempotent(request, tenant, "mutateWorkspace", [id], body, 200, 200, () => { + const updated = { ...workspace, name: body.name ?? workspace.name, revision: Number(workspace.revision) + 1 }; + this.#workspaces.set(id, updated); + return updated; + }); } if (request.method === "DELETE") { - this.#workspaces.delete(id); - return new Response(null, { status: 204, headers: { "T4-API-Version": "1.0" } }); + return this.#idempotent(request, tenant, "deleteWorkspace", [id], null, 204, 204, () => { + this.#workspaces.delete(id); + return null; + }); } } @@ -140,12 +137,8 @@ export class T4ApiV1ConformanceService { if (workspace?.tenant !== tenant) return problem(404, "not_found", "Workspace not found"); if (request.method === "POST") { const body = await request.json() as Record; - if (typeof body.title !== "string" || body.title.length < 1 || body.title.length > 128) { - return problem(422, "invalid_request", "Request validation failed", { - violations: [{ field: "title", rule: "length", message: "title must contain 1 to 128 characters" }], - }); - } - return this.#idempotent(request, tenant, body, () => { + if (typeof body.title !== "string" || body.title.length < 1 || body.title.length > 128) return this.#invalid("title", "length", "title must contain 1 to 128 characters"); + return this.#idempotent(request, tenant, "spawnSession", [workspaceId], body, 202, 200, () => { const id = `ses-${++this.#sessionSequence}`; const session = { id, workspaceId, title: body.title, state: "accepted", revision: 1, tenant }; this.#sessions.set(id, session); @@ -154,14 +147,10 @@ export class T4ApiV1ConformanceService { } if (request.method === "GET") { const pageSize = Number(url.searchParams.get("pageSize") ?? "2"); - if (!Number.isInteger(pageSize) || pageSize < 1 || pageSize > 3) { - return problem(422, "invalid_request", "Request validation failed", { - violations: [{ field: "pageSize", rule: "range", message: "pageSize must be between 1 and 3" }], - }); - } + if (!Number.isInteger(pageSize) || pageSize < 1 || pageSize > 3) return this.#invalid("pageSize", "range", "pageSize must be between 1 and 3"); const start = Number(url.searchParams.get("cursor")?.replace("page-", "") ?? "0"); const visible = [...this.#sessions.values()].filter((item) => item.workspaceId === workspaceId && item.tenant === tenant); - const items = visible.slice(start, start + pageSize).map(({ tenant: _tenant, ...item }) => item); + const items = visible.slice(start, start + pageSize).map((item) => this.#visible(item)); const next = start + items.length < visible.length ? `page-${start + items.length}` : undefined; return json(200, { items, ...(next === undefined ? {} : { nextCursor: next }) }); } @@ -172,49 +161,50 @@ export class T4ApiV1ConformanceService { const id = decodeURIComponent(sessionMatch[1]!); const session = this.#sessions.get(id); if (session?.tenant !== tenant) return problem(404, "not_found", "Session not found"); - if (request.method === "GET") { - const { tenant: _tenant, ...visible } = session; - return json(200, visible); - } + if (request.method === "GET") return json(200, this.#visible(session)); if (request.method === "PATCH") { const body = await request.json() as Record; - if (request.headers.get("If-Match") !== String(session.revision)) { - return problem(409, "revision_conflict", "Session revision changed"); - } - const updated: Record = { ...session, title: body.title ?? session.title, revision: Number(session.revision) + 1 }; - this.#sessions.set(id, updated); - const { tenant: _tenant, ...visible } = updated; - return json(200, visible); + if (request.headers.get("If-Match") !== String(session.revision)) return problem(409, "revision_conflict", "Session revision changed"); + return this.#idempotent(request, tenant, "mutateSession", [id], body, 200, 200, () => { + const updated = { ...session, title: body.title ?? session.title, revision: Number(session.revision) + 1 }; + this.#sessions.set(id, updated); + return updated; + }); } if (request.method === "DELETE") { - this.#sessions.delete(id); - return new Response(null, { status: 204, headers: { "T4-API-Version": "1.0" } }); + return this.#idempotent(request, tenant, "deleteSession", [id], null, 204, 204, () => { + this.#sessions.delete(id); + return null; + }); } } const cancelMatch = url.pathname.match(/^\/v1\/sessions\/([^/]+)\/cancel$/u); if (cancelMatch && request.method === "POST") { - const session = this.#sessions.get(decodeURIComponent(cancelMatch[1]!)); + const id = decodeURIComponent(cancelMatch[1]!); + const session = this.#sessions.get(id); if (session?.tenant !== tenant) return problem(404, "not_found", "Session not found"); - const cancelled: Record = { ...session, state: "cancelled", revision: Number(session.revision) + 1 }; - this.#sessions.set(String(session.id), cancelled); - const { tenant: _tenant, ...visible } = cancelled; - return json(202, visible); + return this.#idempotent(request, tenant, "cancelSession", [id], null, 202, 200, () => { + const cancelled = { ...session, state: "cancelled", revision: Number(session.revision) + 1 }; + this.#sessions.set(id, cancelled); + return cancelled; + }); } const commandMatch = url.pathname.match(/^\/v1\/sessions\/([^/]+)\/commands$/u); if (commandMatch && request.method === "POST") { - const session = this.#sessions.get(decodeURIComponent(commandMatch[1]!)); + const id = decodeURIComponent(commandMatch[1]!); + const session = this.#sessions.get(id); if (session?.tenant !== tenant) return problem(404, "not_found", "Session not found"); - const body = await request.json() as Record; - if (typeof body.command !== "string" || body.command.length < 1 || encoder.encode(body.command).byteLength > 32) { - return problem(422, "invalid_request", "Request validation failed", { - violations: [{ field: "command", rule: "maxBytes", message: "command must contain 1 to 32 UTF-8 bytes" }], - }); - } + const text = await request.text(); + if (encoder.encode(text).byteLength > COMMAND_REQUEST_BYTES_MAX) return this.#invalid("body", "maxBytes", `request must not exceed ${COMMAND_REQUEST_BYTES_MAX} UTF-8 bytes`); + let body: Record; + try { body = JSON.parse(text) as Record; } catch { return problem(400, "invalid_request", "Malformed JSON request"); } + if (typeof body.command !== "string" || body.command.length < 1 || encoder.encode(body.command).byteLength > COMMAND_BYTES_MAX) return this.#invalid("command", "maxBytes", `command must contain 1 to ${COMMAND_BYTES_MAX} UTF-8 bytes`); + if (!this.#validMetadata(body.metadata)) return this.#invalid("metadata", "bounds", "metadata keys and values exceed the discovered bounds"); const states: Record = { accepted: true, rejected: true, conflict: true, unavailable: true, indeterminate: true }; const state = states[body.command] === true ? body.command : "accepted"; - return this.#idempotent(request, tenant, body, () => ({ commandId: `cmd-${++this.#commandSequence}`, state })); + return this.#idempotent(request, tenant, "submitCommand", [id], { metadata: {}, ...body }, 202, 200, () => ({ commandId: `cmd-${++this.#commandSequence}`, state })); } const snapshotMatch = url.pathname.match(/^\/v1\/sessions\/([^/]+)\/snapshot$/u); @@ -229,71 +219,95 @@ export class T4ApiV1ConformanceService { const id = decodeURIComponent(watchMatch[1]!); const session = this.#sessions.get(id); if (session?.tenant !== tenant) return problem(404, "not_found", "Session not found"); + const maxEvents = Number(url.searchParams.get("maxEvents") ?? "100"); + const heartbeat = Number(url.searchParams.get("heartbeatSeconds") ?? "15"); + if (!Number.isInteger(maxEvents) || maxEvents < 1 || maxEvents > 4) return this.#invalid("maxEvents", "range", "maxEvents exceeds the discovered bound"); + if (!Number.isInteger(heartbeat) || heartbeat < 5 || heartbeat > 60) return this.#invalid("heartbeatSeconds", "range", "heartbeatSeconds must be between 5 and 60"); const queryCursor = url.searchParams.get("cursor"); const headerCursor = request.headers.get("Last-Event-ID"); this.watchCursors.push({ query: queryCursor, header: headerCursor }); - if (queryCursor !== null && headerCursor !== null && queryCursor !== headerCursor) { - return problem(400, "invalid_request", "Reconnect cursors disagree"); - } + if (queryCursor !== null && headerCursor !== null && queryCursor !== headerCursor) return problem(400, "invalid_request", "Reconnect cursors disagree"); const cursor = queryCursor ?? headerCursor; - if (cursor === "expired") { - return problem(410, "cursor_expired", "Watch cursor is no longer retained", { - resync: { snapshotUrl: `/v1/sessions/${id}/snapshot`, cursor: "cursor-2" }, - }); - } - const frames = cursor === "cursor-4" + if (cursor === "expired") return problem(410, "cursor_expired", "Watch cursor is no longer retained", { resync: { snapshotUrl: `/v1/sessions/${id}/snapshot`, cursor: "cursor-2" } }); + const allFrames = cursor === "cursor-4" ? [`id: cursor-5\r\nevent: heartbeat\r\ndata: {"type":"heartbeat","cursor":"cursor-5","observedAt":"2026-07-21T00:00:15Z"}\r\n\r\n`] - : [ - `id: cursor-3\r\nevent: heartbeat\r\ndata: {"type":"heartbeat","cursor":"cursor-3","observedAt":"2026-07-21T00:00:00Z"}\r\n\r\n`, - `id: cursor-4\r\nevent: session\r\ndata: {"type":"session","cursor":"cursor-4","state":"accepted","revision":2}\r\n\r\n`, - ]; + : cursor === "cursor-5" + ? [`id: cursor-6\nevent: command\ndata: {"type":"command","cursor":"cursor-6","commandId":"cmd-1","state":"accepted"}\n\n`] + : [ + `id: cursor-3\r\nevent: heartbeat\r\ndata: {"type":"heartbeat","cursor":"cursor-3","observedAt":"2026-07-21T00:00:00Z"}\r\n\r\n`, + `id: cursor-4\r\nevent: session\r\ndata: {"type":"session","cursor":"cursor-4","state":"accepted","revision":2}\r\n\r\n`, + ]; + const frames = allFrames.slice(0, maxEvents); const stream = new ReadableStream({ start: (controller) => { const payload = encoder.encode(frames.join("")); const split = Math.max(1, payload.byteLength - 3); controller.enqueue(payload.slice(0, split)); controller.enqueue(payload.slice(split)); - request.signal.addEventListener("abort", () => { - this.abortedWatches.push(id); - try { controller.close(); } catch { /* already closed */ } - }, { once: true }); + controller.close(); + request.signal.addEventListener("abort", () => this.abortedWatches.push(id), { once: true }); }, }); - return new Response(stream, { - status: 200, - headers: { "Content-Type": "text/event-stream", "Cache-Control": "no-store", "T4-API-Version": "1.0" }, - }); + return new Response(stream, { status: 200, headers: { "Content-Type": "text/event-stream", "Cache-Control": "no-store", "T4-API-Version": "1.0" } }); } return problem(404, "not_found", "Resource not found"); }; expectNoCredentialLeak(): void { - expect(this.calls.every((call) => call.authorization === "Bearer token-a" || call.authorization === "Bearer token-b")).toBe(true); + expect(this.calls.every((call) => call.authorization === "Bearer token-a" || call.authorization === "Bearer token-b" || call.authorization === "Bearer token-denied")).toBe(true); + } + + #invalid(field: string, rule: string, message: string): Response { + return problem(422, "invalid_request", "Request validation failed", { violations: [{ field, rule, message }] }); + } + + #visible(value: Record): Record { + const { tenant: _tenant, ...visible } = value; + return visible; + } + + #validMetadata(value: unknown): boolean { + if (value === undefined) return true; + if (value === null || typeof value !== "object" || Array.isArray(value)) return false; + const entries = Object.entries(value as Record); + return entries.length <= 32 && entries.every(([key, item]) => + /^[a-z][a-z0-9.-]{0,62}$/u.test(key) && + (item === null || typeof item === "boolean" || + (typeof item === "number" && Number.isSafeInteger(item)) || + (typeof item === "string" && encoder.encode(item).byteLength <= METADATA_VALUE_BYTES_MAX))); } #idempotent( request: Request, - tenant: string, - body: Record, - create: () => Record, + principal: string, + operationId: string, + targets: readonly string[], + body: unknown, + firstStatus: number, + replayStatus: number, + create: () => unknown, ): Response { const key = request.headers.get("Idempotency-Key"); if (key === null) return problem(400, "idempotency_key_required", "Idempotency-Key is required"); - if (!/^[A-Za-z0-9._~-]{16,128}$/u.test(key)) { - return problem(400, "invalid_request", "Idempotency-Key is invalid", { - violations: [{ field: "Idempotency-Key", rule: "format", message: "Idempotency-Key must be a 16 to 128 character token" }], - }); - } - const replayKey = `${tenant}:${request.method}:${new URL(request.url).pathname}:${key}`; - const prior = this.#replays.get(replayKey); - if (prior) { - if (prior.body !== bodyKey(body)) return problem(409, "idempotency_conflict", "Idempotency key was reused with a different request"); - return json(200, prior.response, { "Idempotency-Replayed": "true" }); + if (!/^[A-Za-z0-9._~-]{16,128}$/u.test(key)) return problem(400, "invalid_request", "Idempotency-Key is invalid"); + const preconditions = request.headers.has("If-Match") ? { "if-match": request.headers.get("If-Match") } : {}; + const identity = canonicalJson({ operationId, targets, preconditions, body }); + const scope = canonicalJson({ principal, operationId, targets, key }); + const prior = this.#replays.get(scope); + if (prior !== undefined) { + if (prior.identity !== identity) return problem(409, "idempotency_conflict", "Idempotency key was reused with a different canonical request"); + return prior.body === null + ? new Response(null, { status: prior.replayStatus, headers: { "T4-API-Version": "1.0", "Idempotency-Replayed": "true" } }) + : json(prior.replayStatus, prior.body, { "Idempotency-Replayed": "true" }); } - const response = create(); - const { tenant: _tenant, ...visible } = response; - this.#replays.set(replayKey, { body: bodyKey(body), response: visible }); - return json(202, visible, { "Idempotency-Replayed": "false" }); + const created = create(); + const visible = created !== null && typeof created === "object" && !Array.isArray(created) + ? this.#visible(created as Record) + : created; + this.#replays.set(scope, { identity, body: visible, replayStatus }); + return visible === null + ? new Response(null, { status: firstStatus, headers: { "T4-API-Version": "1.0", "Idempotency-Replayed": "false" } }) + : json(firstStatus, visible, { "Idempotency-Replayed": "false" }); } } diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index 68f7ab42..be43e21f 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -4,21 +4,54 @@ import { T4ApiError, createT4ApiClient, type components, + type operations, } from "@t4-code/t4-api-client"; -import { T4ApiV1ConformanceService } from "./t4-api-v1-conformance-service.ts"; +import { T4ApiV1ConformanceService, canonicalJson } from "./t4-api-v1-conformance-service.ts"; type WorkspaceCreate = components["schemas"]["WorkspaceCreate"]; type SessionCreate = components["schemas"]["SessionCreate"]; type CommandCreate = components["schemas"]["CommandCreate"]; +type WatchEvent = components["schemas"]["WatchEvent"]; +type JsonBody = Response extends { content: { "application/json": infer Body } } ? Body : never; +type SpawnBadRequest = JsonBody; +type CommandBadRequest = JsonBody; +type Unauthorized = JsonBody; +type Forbidden = JsonBody; +type Missing = JsonBody; +type Conflict = JsonBody; +type Unavailable = JsonBody; + +const typedErrors = { + badSpawn: { error: { code: "idempotency_key_required", message: "required", requestId: "r", retryable: false } } satisfies SpawnBadRequest, + badCommand: { error: { code: "invalid_request", message: "invalid", requestId: "r", retryable: false } } satisfies CommandBadRequest, + unauthorized: { error: { code: "unauthenticated", message: "no", requestId: "r", retryable: false } } satisfies Unauthorized, + forbidden: { error: { code: "forbidden", message: "no", requestId: "r", retryable: false } } satisfies Forbidden, + missing: { error: { code: "not_found", message: "no", requestId: "r", retryable: false } } satisfies Missing, + conflict: { error: { code: "idempotency_conflict", message: "no", requestId: "r", retryable: false } } satisfies Conflict, + unavailable: { error: { code: "unavailable", message: "later", requestId: "r", retryable: true } } satisfies Unavailable, +}; + +function exhaustWatchEvent(event: WatchEvent): string { + switch (event.type) { + case "heartbeat": return event.observedAt; + case "session": return `${event.state}:${event.revision}`; + case "command": return `${event.commandId}:${event.state}`; + default: { + const unreachable: never = event; + return unreachable; + } + } +} + const VERSION_HEADERS = { "T4-API-Version": "1" } as const; function idempotencyHeaders(key: string): Readonly<{ "T4-API-Version": "1"; "Idempotency-Key": string }> { return { ...VERSION_HEADERS, "Idempotency-Key": key }; } -function revisionHeaders(revision: number): Readonly<{ "T4-API-Version": "1"; "If-Match": string }> { - return { ...VERSION_HEADERS, "If-Match": String(revision) }; +function mutationHeaders(revision: number, key: string): Readonly<{ "T4-API-Version": "1"; "If-Match": string; "Idempotency-Key": string }> { + return { ...VERSION_HEADERS, "If-Match": String(revision), "Idempotency-Key": key }; } function requireData(result: { data?: T; error?: unknown }): T { @@ -43,6 +76,13 @@ describe("generated T4 API v1 client conformance", () => { const rejected = await incompatible.http.GET("/v1", { params: { header: VERSION_HEADERS } }); expect(rejected.response.status).toBe(406); expect(rejected.error).toMatchObject({ error: { code: "incompatible_version", retryable: false, supportedMajors: [1] } }); + expect(typedErrors.unauthorized.error.code).toBe("unauthenticated"); + + const denied = createT4ApiClient({ baseUrl: service.origin, credential: "token-denied", majorVersion: 1, fetch: service.fetch }); + const forbidden = await denied.http.GET("/v1", { params: { header: VERSION_HEADERS } }); + expect(forbidden.response.status).toBe(403); + expect(forbidden.error).toEqual(typedErrors.forbidden); + expect(discovery.limits).toMatchObject({ commandBytesMax: 32, commandRequestBytesMax: 256, commandMetadataValueBytesMax: 32 }); }); it("creates, canonically replays, conflicts, mutates, paginates, isolates, and deletes workspaces", async () => { @@ -73,6 +113,17 @@ describe("generated T4 API v1 client conformance", () => { expect(conflict.response.status).toBe(409); expect(conflict.error).toMatchObject({ error: { code: "idempotency_conflict", retryable: false } }); + const omitted = await owner.http.POST("/v1/workspaces", { + body: { name: "primary" }, params: { header: idempotencyHeaders("workspace-omitted-0001") }, + }); + expect(omitted.response.status).toBe(202); + const explicitEmpty = await owner.http.POST("/v1/workspaces", { + body: { name: "primary", labels: {} }, params: { header: idempotencyHeaders("workspace-omitted-0001") }, + }); + expect(explicitEmpty.response.status).toBe(409); + expect(canonicalJson({ z: [1, 2], a: { y: 2, x: 1 } })).toBe('{"a":{"x":1,"y":2},"z":[1,2]}'); + expect(canonicalJson({ z: [1, 2] })).not.toBe(canonicalJson({ z: [2, 1] })); + for (const name of ["second", "third", "fourth"]) { requireData(await owner.http.POST("/v1/workspaces", { body: { name }, @@ -103,18 +154,18 @@ describe("generated T4 API v1 client conformance", () => { expect(invalid.error).toMatchObject({ error: { code: "invalid_request", violations: [{ field: "name", rule: "length" }] } }); const updated = requireData(await owner.http.PATCH("/v1/workspaces/{workspaceId}", { - params: { header: revisionHeaders(1), path: { workspaceId: "ws-1" } }, + params: { header: mutationHeaders(1, "workspace-patch-0001"), path: { workspaceId: "ws-1" } }, body: { name: "renamed" }, })); expect(updated).toMatchObject({ name: "renamed", revision: 2 }); const stale = await owner.http.PATCH("/v1/workspaces/{workspaceId}", { - params: { header: revisionHeaders(1), path: { workspaceId: "ws-1" } }, + params: { header: mutationHeaders(1, "workspace-patch-0002"), path: { workspaceId: "ws-1" } }, body: { name: "stale" }, }); expect(stale.response.status).toBe(409); expect(stale.error).toMatchObject({ error: { code: "revision_conflict" } }); expect((await owner.http.DELETE("/v1/workspaces/{workspaceId}", { - params: { header: VERSION_HEADERS, path: { workspaceId: "ws-1" } }, + params: { header: idempotencyHeaders("workspace-delete-0001"), path: { workspaceId: "ws-1" } }, })).response.status).toBe(204); service.expectNoCredentialLeak(); }); @@ -153,11 +204,11 @@ describe("generated T4 API v1 client conformance", () => { expect(sessionPageTwo.items).toHaveLength(1); const mutated = requireData(await client.http.PATCH("/v1/sessions/{sessionId}", { - params: { header: revisionHeaders(1), path: { sessionId: "ses-1" } }, body: { title: "renamed-agent" }, + params: { header: mutationHeaders(1, "session-patch-0001"), path: { sessionId: "ses-1" } }, body: { title: "renamed-agent" }, })); expect(mutated).toMatchObject({ title: "renamed-agent", revision: 2 }); const stale = await client.http.PATCH("/v1/sessions/{sessionId}", { - params: { header: revisionHeaders(1), path: { sessionId: "ses-1" } }, body: { title: "stale-agent" }, + params: { header: mutationHeaders(1, "session-patch-0002"), path: { sessionId: "ses-1" } }, body: { title: "stale-agent" }, }); expect(stale.response.status).toBe(409); expect(stale.error).toMatchObject({ error: { code: "revision_conflict" } }); @@ -180,8 +231,20 @@ describe("generated T4 API v1 client conformance", () => { }); expect(oversized.response.status).toBe(422); expect(oversized.error).toMatchObject({ error: { violations: [{ field: "command", rule: "maxBytes" }] } }); + const metadataOversized = await client.http.POST("/v1/sessions/{sessionId}/commands", { + params: { header: idempotencyHeaders("command-metadata-0001"), path: { sessionId: "ses-1" } }, + body: { command: "ok", metadata: { source: "é".repeat(17) } }, + }); + expect(metadataOversized.response.status).toBe(422); + const defaultedMetadata = await client.http.POST("/v1/sessions/{sessionId}/commands", { + params: { header: idempotencyHeaders("command-default-0001"), path: { sessionId: "ses-1" } }, body: { command: "ok" }, + }); + const explicitMetadata = await client.http.POST("/v1/sessions/{sessionId}/commands", { + params: { header: idempotencyHeaders("command-default-0001"), path: { sessionId: "ses-1" } }, body: { command: "ok", metadata: {} }, + }); + expect(explicitMetadata.data).toEqual(defaultedMetadata.data); const cancelled = await client.http.POST("/v1/sessions/{sessionId}/cancel", { - params: { header: VERSION_HEADERS, path: { sessionId: "ses-1" } }, + params: { header: idempotencyHeaders("session-cancel-0001"), path: { sessionId: "ses-1" } }, }); expect(cancelled.response.status).toBe(202); expect(cancelled.data).toMatchObject({ state: "cancelled" }); @@ -201,22 +264,17 @@ describe("generated T4 API v1 client conformance", () => { })); expect(snapshot).toMatchObject({ sessionId: "ses-1", cursor: "cursor-2", entries: [{ sequence: 2 }] }); - const controller = new AbortController(); const received: Array = []; - for await (const event of client.watchSession("ses-1", { cursor: snapshot.cursor, maxEvents: 4, signal: controller.signal })) { + for await (const event of client.watchSession("ses-1", { cursor: snapshot.cursor, maxEvents: 3, maxReconnectAttempts: 2, retryBackoffMs: 0 })) { received.push(event); - if (received.length === 2) controller.abort(); } expect(received).toEqual([ expect.objectContaining({ type: "heartbeat", cursor: "cursor-3" }), expect.objectContaining({ type: "session", cursor: "cursor-4", state: "accepted" }), + expect.objectContaining({ type: "heartbeat", cursor: "cursor-5" }), ]); - expect(service.abortedWatches).toEqual(["ses-1"]); + expect(received.map(exhaustWatchEvent)).toEqual(["2026-07-21T00:00:00Z", "accepted:2", "2026-07-21T00:00:15Z"]); expect(service.watchCursors[0]).toEqual({ query: "cursor-2", header: "cursor-2" }); - - const reconnect = client.watchSession("ses-1", { cursor: received.at(-1)!.cursor, maxEvents: 2 }); - expect((await reconnect.next()).value).toMatchObject({ type: "heartbeat", cursor: "cursor-5" }); - await reconnect.return(undefined); expect(service.watchCursors[1]).toEqual({ query: "cursor-4", header: "cursor-4" }); await expect(async () => { @@ -230,4 +288,89 @@ describe("generated T4 API v1 client conformance", () => { resync: { snapshotUrl: "/v1/sessions/ses-1/snapshot", cursor: "cursor-2" }, } satisfies Partial); }); + + it("rejects malformed mutation idempotency and watch query contracts", async () => { + const service = new T4ApiV1ConformanceService(); + const baseHeaders = { Authorization: "Bearer token-a", "T4-API-Version": "1", "Content-Type": "application/json" }; + const created = await service.fetch(`${service.origin}/v1/workspaces`, { method: "POST", headers: { ...baseHeaders, "Idempotency-Key": "workspace-raw-0001" }, body: '{"name":"workspace"}' }); + expect(created.status).toBe(202); + const missingSpawn = await service.fetch(`${service.origin}/v1/workspaces/ws-1/sessions`, { method: "POST", headers: baseHeaders, body: '{"title":"agent"}' }); + expect(missingSpawn.status).toBe(400); + expect(await missingSpawn.json()).toMatchObject(typedErrors.badSpawn); + const invalidSpawn = await service.fetch(`${service.origin}/v1/workspaces/ws-1/sessions`, { method: "POST", headers: { ...baseHeaders, "Idempotency-Key": "short" }, body: '{"title":"agent"}' }); + expect(invalidSpawn.status).toBe(400); + const spawned = await service.fetch(`${service.origin}/v1/workspaces/ws-1/sessions`, { method: "POST", headers: { ...baseHeaders, "Idempotency-Key": "session-raw-00001" }, body: '{"title":"agent"}' }); + expect(spawned.status).toBe(202); + for (const [method, path, body] of [ + ["PATCH", "/v1/workspaces/ws-1", '{"name":"x"}'], + ["DELETE", "/v1/workspaces/ws-1", undefined], + ["PATCH", "/v1/sessions/ses-1", '{"title":"x"}'], + ["DELETE", "/v1/sessions/ses-1", undefined], + ["POST", "/v1/sessions/ses-1/cancel", undefined], + ["POST", "/v1/sessions/ses-1/commands", '{"command":"ok"}'], + ] as const) { + const response = await service.fetch(`${service.origin}${path}`, { method, headers: { ...baseHeaders, "If-Match": "1" }, body }); + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ error: { code: "idempotency_key_required" } }); + } + for (const query of ["maxEvents=0", "maxEvents=5", "heartbeatSeconds=4", "heartbeatSeconds=61"]) { + const response = await service.fetch(`${service.origin}/v1/sessions/ses-1/events?${query}`, { headers: baseHeaders }); + expect(response.status).toBe(422); + expect(await response.json()).toMatchObject({ error: { code: "invalid_request", violations: expect.any(Array) } }); + } + }); + + it("parses per-frame byte bounds across arbitrary chunks and split UTF-8", async () => { + const sse = (chunks: Uint8Array[]): typeof globalThis.fetch => async () => new Response(new ReadableStream({ + start(controller) { for (const chunk of chunks) controller.enqueue(chunk); controller.close(); }, + }), { status: 200, headers: { "Content-Type": "text/event-stream" } }); + const encoder = new TextEncoder(); + const valid = `: 💚\nid: c1\ndata: {"type":"heartbeat","cursor":"c1","observedAt":"2026-07-21T00:00:00Z"}\n\n`; + const bytes = encoder.encode(valid); + const split = [...bytes].map((byte) => Uint8Array.of(byte)); + const splitClient = createT4ApiClient({ baseUrl: "https://split.test", credential: "token-a", majorVersion: 1, fetch: sse(split) }); + const iterator = splitClient.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }); + expect((await iterator.next()).value).toMatchObject({ type: "heartbeat", cursor: "c1" }); + + const small = `data: {"type":"heartbeat","cursor":"c1","observedAt":"2026-07-21T00:00:00Z"}\n${`: ${"x".repeat(1010)}\n`}\n`; + const manyClient = createT4ApiClient({ baseUrl: "https://many.test", credential: "token-a", majorVersion: 1, fetch: sse([encoder.encode(small.repeat(1000))]) }); + let count = 0; + for await (const _event of manyClient.watchSession("ses-1", { maxEvents: 1000, maxReconnectAttempts: 0 })) count += 1; + expect(count).toBe(1000); + + const oversized = `: ${"x".repeat(1024 * 1024)}\ndata: {"type":"heartbeat","cursor":"c1","observedAt":"2026-07-21T00:00:00Z"}\n\n`; + const oversizedClient = createT4ApiClient({ baseUrl: "https://large.test", credential: "token-a", majorVersion: 1, fetch: sse([encoder.encode(oversized)]) }); + await expect(oversizedClient.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ code: "indeterminate", status: 502 }); + }); + + it("fails closed on unknown watch fields and incomplete typed errors", async () => { + const eventFetch: typeof globalThis.fetch = async () => new Response('data: {"type":"heartbeat","cursor":"c1","observedAt":"2026-07-21T00:00:00Z","unknown":true}\n\n', { headers: { "Content-Type": "text/event-stream" } }); + const eventClient = createT4ApiClient({ baseUrl: "https://unknown.test", credential: "token-a", majorVersion: 1, fetch: eventFetch }); + await expect(eventClient.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ code: "indeterminate", status: 502 }); + for (const [status, code] of [[410, "cursor_expired"], [406, "incompatible_version"], [422, "invalid_request"]] as const) { + const fetch: typeof globalThis.fetch = async () => Response.json({ error: { code, message: "incomplete", requestId: "r", retryable: false } }, { status }); + const client = createT4ApiClient({ baseUrl: "https://errors.test", credential: "token-a", majorVersion: 1, fetch }); + await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ code: "indeterminate", status }); + } + }); + + it("bounds automatic EOF retry and supports explicit abort", async () => { + let attempts = 0; + const eofFetch: typeof globalThis.fetch = async () => { + attempts += 1; + return new Response(new ReadableStream({ start(controller) { controller.close(); } }), { headers: { "Content-Type": "text/event-stream" } }); + }; + const eofClient = createT4ApiClient({ baseUrl: "https://eof.test", credential: "token-a", majorVersion: 1, fetch: eofFetch }); + await expect(eofClient.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 2, retryBackoffMs: 0 }).next()).rejects.toMatchObject({ code: "indeterminate", status: 502 }); + expect(attempts).toBe(3); + + const controller = new AbortController(); + const abortFetch: typeof globalThis.fetch = async (_input, init) => new Response(new ReadableStream({ + start(stream) { init?.signal?.addEventListener("abort", () => stream.close(), { once: true }); }, + }), { headers: { "Content-Type": "text/event-stream" } }); + const abortClient = createT4ApiClient({ baseUrl: "https://abort.test", credential: "token-a", majorVersion: 1, fetch: abortFetch }); + const pending = abortClient.watchSession("ses-1", { signal: controller.signal }).next(); + controller.abort("done"); + await expect(pending).resolves.toMatchObject({ done: true }); + }); }); From 797717c4cf509acd7c86795a848623d8c9dd04fb Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 20:28:35 +0000 Subject: [PATCH 009/131] fix: enforce T4 API public contract semantics --- .github/workflows/ci.yml | 10 + packages/t4-api-client/src/index.ts | 249 ++++++++++++++----- packages/t4-api-contract/openapi.json | 337 +++++++++++++++++--------- 3 files changed, 408 insertions(+), 188 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9b0c5438..2bad79b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,11 +82,21 @@ jobs: - name: Validate OpenAPI and deterministic generation run: | pnpm --filter @t4-code/t4-api-contract validate + pnpm --filter @t4-code/t4-api-contract generate:ci pnpm --filter @t4-code/t4-api-contract check:generated pnpm --filter @t4-code/t4-api-client typecheck pnpm --filter @t4-code/client typecheck pnpm --filter @t4-code/client exec vp test run test/t4-api-v1-conformance.test.ts + - name: Upload authorized generated schema + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: t4-api-generated-${{ github.run_id }} + path: artifacts/t4-api/generated/schema.ts + if-no-files-found: error + retention-days: 1 + core: runs-on: ubuntu-24.04 timeout-minutes: 25 diff --git a/packages/t4-api-client/src/index.ts b/packages/t4-api-client/src/index.ts index be975199..67d5ed34 100644 --- a/packages/t4-api-client/src/index.ts +++ b/packages/t4-api-client/src/index.ts @@ -18,6 +18,17 @@ const ERROR_FIELDS: Record = { code: true, message: true, requestId: true, retryable: true, violations: true, supportedMajors: true, resync: true, }; +const ERROR_CODES_BY_STATUS: Readonly>>> = { + 400: { invalid_request: true, idempotency_key_required: true, invalid_origin: true, https_required: true }, + 401: { unauthenticated: true }, + 403: { forbidden: true }, + 404: { not_found: true }, + 406: { incompatible_version: true }, + 409: { idempotency_conflict: true, revision_conflict: true }, + 410: { cursor_expired: true }, + 422: { invalid_request: true }, + 503: { unavailable: true, indeterminate: true }, +}; const SESSION_STATES = { accepted: true, provisioning: true, ready: true, cancelling: true, cancelled: true, failed: true, unavailable: true, indeterminate: true, @@ -42,6 +53,8 @@ export interface WatchSessionOptions { readonly maxEvents?: number; readonly heartbeatSeconds?: number; readonly signal?: AbortSignal; + readonly maxReconnectAttempts?: number; + readonly retryBackoffMs?: number; } export interface T4ApiClient { @@ -58,8 +71,8 @@ export class T4ApiError extends Error { readonly supportedMajors?: ApiError["supportedMajors"]; readonly resync?: Resync; - constructor(status: number, error: ApiError) { - super(error.message); + constructor(status: number, error: ApiError, options?: ErrorOptions) { + super(error.message, options); this.name = "T4ApiError"; this.code = error.code; this.status = status; @@ -146,19 +159,23 @@ function validResync(value: unknown): value is Resync { typeof resync.cursor === "string" && resync.cursor.length <= 512 && CURSOR_PATTERN.test(resync.cursor); } -function apiError(value: unknown): ApiError | undefined { +function apiError(value: unknown, status: number): ApiError | undefined { const envelope = record(value); const error = record(envelope?.error); if ( envelope === undefined || Object.keys(envelope).some((key) => key !== "error") || error === undefined || Object.keys(error).some((key) => ERROR_FIELDS[key] !== true) || typeof error.code !== "string" || ERROR_CODES[error.code as ApiError["code"]] !== true || + ERROR_CODES_BY_STATUS[status]?.[error.code] !== true || typeof error.message !== "string" || error.message.length < 1 || error.message.length > 1024 || typeof error.requestId !== "string" || error.requestId.length < 1 || error.requestId.length > 128 || typeof error.retryable !== "boolean" || (error.violations !== undefined && (!Array.isArray(error.violations) || error.violations.length > 64 || !error.violations.every(validViolation))) || (error.supportedMajors !== undefined && (!Array.isArray(error.supportedMajors) || error.supportedMajors.length > 8 || !error.supportedMajors.every((major) => Number.isSafeInteger(major) && Number(major) >= 1))) || - (error.resync !== undefined && !validResync(error.resync)) + (error.resync !== undefined && !validResync(error.resync)) || + (status === 406 && (error.code !== "incompatible_version" || !Array.isArray(error.supportedMajors) || error.supportedMajors.length < 1)) || + (status === 410 && (error.code !== "cursor_expired" || !validResync(error.resync))) || + (status === 422 && (error.code !== "invalid_request" || !Array.isArray(error.violations) || error.violations.length < 1)) ) return undefined; return error as ApiError; } @@ -195,7 +212,7 @@ async function boundedError(response: Response): Promise { const text = await boundedResponseText(response); if (text !== undefined) { try { - const decoded = apiError(JSON.parse(text)); + const decoded = apiError(JSON.parse(text), response.status); if (decoded !== undefined) return new T4ApiError(response.status, decoded); } catch { // Fall through to the stable indeterminate transport envelope. @@ -209,6 +226,10 @@ async function boundedError(response: Response): Promise { }); } +function hasOnlyKeys(value: Record, keys: Readonly>): boolean { + return Object.keys(value).every((key) => keys[key] === true); +} + function watchEvent(value: unknown, eventId: string | undefined): WatchEvent { const event = record(value); if ( @@ -220,19 +241,26 @@ function watchEvent(value: unknown, eventId: string | undefined): WatchEvent { !CURSOR_PATTERN.test(event.cursor) || (eventId !== undefined && eventId !== event.cursor) ) throw new T4ApiError(502, { code: "indeterminate", message: "T4 API returned an invalid watch event", requestId: "unavailable", retryable: true }); - if (event.type === "heartbeat" && typeof event.observedAt === "string" && event.observedAt.length <= 64 && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/u.test(event.observedAt) && Number.isFinite(Date.parse(event.observedAt))) { - return event as WatchEvent; - } + if ( + event.type === "heartbeat" && + hasOnlyKeys(event, { type: true, cursor: true, observedAt: true }) && + typeof event.observedAt === "string" && event.observedAt.length <= 64 && + /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/u.test(event.observedAt) && + Number.isFinite(Date.parse(event.observedAt)) + ) return event as WatchEvent; if ( event.type === "session" && + hasOnlyKeys(event, { type: true, cursor: true, state: true, revision: true }) && typeof event.state === "string" && SESSION_STATES[event.state as components["schemas"]["SessionState"]] === true && Number.isSafeInteger(event.revision) && Number(event.revision) >= 1 ) return event as WatchEvent; if ( event.type === "command" && + hasOnlyKeys(event, { type: true, cursor: true, commandId: true, state: true }) && typeof event.commandId === "string" && event.commandId.length >= 1 && event.commandId.length <= 128 && + /^[A-Za-z0-9][A-Za-z0-9._~-]*$/u.test(event.commandId) && typeof event.state === "string" && OPERATION_STATES[event.state as components["schemas"]["OperationState"]] === true ) return event as WatchEvent; @@ -261,9 +289,100 @@ function decodeSseFrame(frame: string): WatchEvent | undefined { } } -function nextSseBoundary(buffer: string): { readonly index: number; readonly length: number } | undefined { - const match = /(?:\r\n|\r|\n)(?:\r\n|\r|\n)/u.exec(buffer); - return match?.index === undefined ? undefined : { index: match.index, length: match[0].length }; +class SseFrameParser { + readonly #buffer = new Uint8Array(MAX_EVENT_BYTES + 4); + #length = 0; + #lineStart = 0; + #pendingCarriageReturn = -1; + + push(chunk: Uint8Array): Uint8Array[] { + const frames: Uint8Array[] = []; + for (const byte of chunk) this.#pushByte(byte, frames); + return frames; + } + + finish(): Uint8Array[] { + const frames: Uint8Array[] = []; + if (this.#pendingCarriageReturn >= 0) { + this.#finishLine(this.#pendingCarriageReturn, frames); + this.#pendingCarriageReturn = -1; + } + if (this.#length > 0) { + if (this.#length > MAX_EVENT_BYTES) this.#oversized(); + frames.push(this.#buffer.slice(0, this.#length)); + this.#reset(); + } + return frames; + } + + #pushByte(byte: number, frames: Uint8Array[]): void { + if (this.#pendingCarriageReturn >= 0) { + if (byte === 10) { + this.#append(byte); + this.#finishLine(this.#pendingCarriageReturn, frames); + this.#pendingCarriageReturn = -1; + return; + } + this.#finishLine(this.#pendingCarriageReturn, frames); + this.#pendingCarriageReturn = -1; + } + this.#append(byte); + if (byte === 13) this.#pendingCarriageReturn = this.#length - 1; + else if (byte === 10) this.#finishLine(this.#length - 1, frames); + } + + #append(byte: number): void { + if (this.#length >= this.#buffer.byteLength) this.#oversized(); + this.#buffer[this.#length++] = byte; + } + + #finishLine(newlineStart: number, frames: Uint8Array[]): void { + if (newlineStart === this.#lineStart) { + if (this.#lineStart > MAX_EVENT_BYTES) this.#oversized(); + frames.push(this.#buffer.slice(0, this.#lineStart)); + this.#reset(); + return; + } + this.#lineStart = this.#length; + if (this.#lineStart > MAX_EVENT_BYTES) this.#oversized(); + } + + #reset(): void { + this.#length = 0; + this.#lineStart = 0; + } + + #oversized(): never { + throw new T4ApiError(502, { code: "indeterminate", message: "T4 API watch event exceeds the client bound", requestId: "unavailable", retryable: true }); + } +} + +function decodedFrames(parser: SseFrameParser, chunk: Uint8Array | undefined): WatchEvent[] { + const frames = chunk === undefined ? parser.finish() : parser.push(chunk); + return frames.flatMap((bytes) => { + let text: string; + try { text = new TextDecoder("utf-8", { fatal: true }).decode(bytes); } catch { + throw new T4ApiError(502, { code: "indeterminate", message: "T4 API returned malformed SSE data", requestId: "unavailable", retryable: true }); + } + const event = decodeSseFrame(text); + return event === undefined ? [] : [event]; + }); +} + +async function retryDelay(milliseconds: number, signal: AbortSignal): Promise { + if (signal.aborted) return false; + if (milliseconds === 0) return true; + return await new Promise((resolve) => { + const timeout = setTimeout(() => { + signal.removeEventListener("abort", abort); + resolve(true); + }, milliseconds); + const abort = (): void => { + clearTimeout(timeout); + resolve(false); + }; + signal.addEventListener("abort", abort, { once: true }); + }); } async function* watch( @@ -277,79 +396,75 @@ async function* watch( const sessionId = requiredSessionId(sessionIdValue); const maxEvents = boundedInteger(options.maxEvents, 100, 1, 1000, "maxEvents"); const heartbeatSeconds = boundedInteger(options.heartbeatSeconds, 15, 5, 60, "heartbeatSeconds"); + const maxReconnectAttempts = boundedInteger(options.maxReconnectAttempts, 3, 0, 10, "maxReconnectAttempts"); + const retryBackoffMs = boundedInteger(options.retryBackoffMs, 250, 0, 30_000, "retryBackoffMs"); if (options.cursor !== undefined && (options.cursor.length < 1 || options.cursor.length > 512 || !CURSOR_PATTERN.test(options.cursor))) { throw new TypeError("cursor must be a header-safe token containing between 1 and 512 characters"); } - const url = new URL(`${baseUrl}/v1/sessions/${encodeURIComponent(sessionId)}/events`); - url.searchParams.set("maxEvents", String(maxEvents)); - url.searchParams.set("heartbeatSeconds", String(heartbeatSeconds)); - if (options.cursor !== undefined) url.searchParams.set("cursor", options.cursor); const controller = new AbortController(); const abort = (): void => controller.abort(options.signal?.reason); if (options.signal?.aborted === true) abort(); else options.signal?.addEventListener("abort", abort, { once: true }); - let reader: ReadableStreamDefaultReader | undefined; + let delivered = 0; + let reconnectAttempts = 0; + let cursor = options.cursor; try { - const headers = new Headers({ - Accept: "text/event-stream", - Authorization: `Bearer ${credential}`, - "Cache-Control": "no-store", - "T4-API-Version": majorVersion, - }); - if (options.cursor !== undefined) headers.set("Last-Event-ID", options.cursor); - const response = await fetchImpl(url, { method: "GET", headers, signal: controller.signal }); - if (!response.ok) throw await boundedError(response); - if (!response.headers.get("content-type")?.toLowerCase().startsWith("text/event-stream")) { - throw new T4ApiError(502, { code: "indeterminate", message: "T4 API watch did not return text/event-stream", requestId: "unavailable", retryable: true }); - } - if (response.body === null) { - throw new T4ApiError(502, { code: "indeterminate", message: "T4 API watch response body is unavailable", requestId: "unavailable", retryable: true }); - } - reader = response.body.getReader(); - const decoder = new TextDecoder("utf-8", { fatal: true }); - let buffer = ""; - let delivered = 0; - while (delivered < maxEvents) { - if (controller.signal.aborted) return; - let chunk: ReadableStreamReadResult; + while (delivered < maxEvents && !controller.signal.aborted) { + let reader: ReadableStreamDefaultReader | undefined; + let transientFailure: unknown; try { - chunk = await reader.read(); + const url = new URL(`${baseUrl}/v1/sessions/${encodeURIComponent(sessionId)}/events`); + url.searchParams.set("maxEvents", String(maxEvents - delivered)); + url.searchParams.set("heartbeatSeconds", String(heartbeatSeconds)); + if (cursor !== undefined) url.searchParams.set("cursor", cursor); + const headers = new Headers({ + Accept: "text/event-stream", + Authorization: `Bearer ${credential}`, + "Cache-Control": "no-store", + "T4-API-Version": majorVersion, + }); + if (cursor !== undefined) headers.set("Last-Event-ID", cursor); + const response = await fetchImpl(url, { method: "GET", headers, signal: controller.signal }); + if (!response.ok) throw await boundedError(response); + if (!response.headers.get("content-type")?.toLowerCase().startsWith("text/event-stream")) { + throw new T4ApiError(502, { code: "indeterminate", message: "T4 API watch did not return text/event-stream", requestId: "unavailable", retryable: true }); + } + if (response.body === null) throw new TypeError("T4 API watch response body is unavailable"); + reader = response.body.getReader(); + const parser = new SseFrameParser(); + while (delivered < maxEvents && !controller.signal.aborted) { + const chunk = await reader.read(); + const events = chunk.done ? decodedFrames(parser, undefined) : decodedFrames(parser, chunk.value); + for (const event of events) { + cursor = event.cursor; + delivered += 1; + yield event; + if (delivered >= maxEvents || controller.signal.aborted) break; + } + if (chunk.done) break; + } + if (delivered >= maxEvents || controller.signal.aborted) return; + transientFailure = new TypeError("T4 API watch ended before the requested event bound"); } catch (error) { if (controller.signal.aborted) return; - throw error; - } - if (chunk.done) { - buffer += decoder.decode(); - break; - } - buffer += decoder.decode(chunk.value, { stream: true }); - if (new TextEncoder().encode(buffer).byteLength > MAX_EVENT_BYTES) { - throw new T4ApiError(502, { code: "indeterminate", message: "T4 API watch event exceeds the client bound", requestId: "unavailable", retryable: true }); - } - let boundary = nextSseBoundary(buffer); - while (boundary !== undefined && delivered < maxEvents) { - if (controller.signal.aborted) return; - const frame = buffer.slice(0, boundary.index); - buffer = buffer.slice(boundary.index + boundary.length); - const event = decodeSseFrame(frame); - if (event !== undefined) { - delivered += 1; - yield event; - if (controller.signal.aborted) return; + if (error instanceof T4ApiError) throw error; + transientFailure = error; + } finally { + if (reader !== undefined) { + try { await reader.cancel(); } catch { /* cancellation is best effort */ } + reader.releaseLock(); } - boundary = nextSseBoundary(buffer); } - } - if (buffer.trim().length > 0 && delivered < maxEvents) { - const event = decodeSseFrame(buffer); - if (event !== undefined) yield event; + if (reconnectAttempts >= maxReconnectAttempts) { + throw new T4ApiError(502, { code: "indeterminate", message: "T4 API watch reconnect attempts exhausted", requestId: "unavailable", retryable: true }, { cause: transientFailure }); + } + const delay = Math.min(30_000, retryBackoffMs * (2 ** reconnectAttempts)); + reconnectAttempts += 1; + if (!await retryDelay(delay, controller.signal)) return; } } finally { options.signal?.removeEventListener("abort", abort); controller.abort(); - if (reader !== undefined) { - try { await reader.cancel(); } catch { /* cancellation is best effort */ } - } } } diff --git a/packages/t4-api-contract/openapi.json b/packages/t4-api-contract/openapi.json index 0387b8a8..2884c6f8 100644 --- a/packages/t4-api-contract/openapi.json +++ b/packages/t4-api-contract/openapi.json @@ -23,10 +23,10 @@ "parameters": [{ "$ref": "#/components/parameters/ApiVersion" }], "responses": { "200": { "$ref": "#/components/responses/Discovery" }, - "401": { "$ref": "#/components/responses/Unauthenticated" }, - "403": { "$ref": "#/components/responses/Forbidden" }, - "406": { "$ref": "#/components/responses/IncompatibleVersion" }, - "503": { "$ref": "#/components/responses/Unavailable" } + "401": {"$ref": "#/components/responses/Error401"}, + "403": {"$ref": "#/components/responses/Error403"}, + "406": {"$ref": "#/components/responses/Error406"}, + "503": {"$ref": "#/components/responses/Error503"} } } }, @@ -41,12 +41,12 @@ ], "responses": { "200": { "$ref": "#/components/responses/WorkspacePage" }, - "400": { "$ref": "#/components/responses/BadRequest" }, - "401": { "$ref": "#/components/responses/Unauthenticated" }, - "403": { "$ref": "#/components/responses/Forbidden" }, - "406": { "$ref": "#/components/responses/IncompatibleVersion" }, - "422": { "$ref": "#/components/responses/InvalidRequest" }, - "503": { "$ref": "#/components/responses/Unavailable" } + "400": {"$ref": "#/components/responses/Error400"}, + "401": {"$ref": "#/components/responses/Error401"}, + "403": {"$ref": "#/components/responses/Error403"}, + "406": {"$ref": "#/components/responses/Error406"}, + "422": {"$ref": "#/components/responses/Error422"}, + "503": {"$ref": "#/components/responses/Error503"} } }, "post": { @@ -63,13 +63,13 @@ "responses": { "200": { "$ref": "#/components/responses/WorkspaceReplay" }, "202": { "$ref": "#/components/responses/WorkspaceAccepted" }, - "400": { "$ref": "#/components/responses/BadRequest" }, - "401": { "$ref": "#/components/responses/Unauthenticated" }, - "403": { "$ref": "#/components/responses/Forbidden" }, - "406": { "$ref": "#/components/responses/IncompatibleVersion" }, - "409": { "$ref": "#/components/responses/Conflict" }, - "422": { "$ref": "#/components/responses/InvalidRequest" }, - "503": { "$ref": "#/components/responses/Unavailable" } + "400": {"$ref": "#/components/responses/Error400"}, + "401": {"$ref": "#/components/responses/Error401"}, + "403": {"$ref": "#/components/responses/Error403"}, + "406": {"$ref": "#/components/responses/Error406"}, + "409": {"$ref": "#/components/responses/Error409"}, + "422": {"$ref": "#/components/responses/Error422"}, + "503": {"$ref": "#/components/responses/Error503"} } } }, @@ -81,11 +81,11 @@ "parameters": [{ "$ref": "#/components/parameters/ApiVersion" }], "responses": { "200": { "$ref": "#/components/responses/Workspace" }, - "401": { "$ref": "#/components/responses/Unauthenticated" }, - "403": { "$ref": "#/components/responses/Forbidden" }, - "404": { "$ref": "#/components/responses/NotFound" }, - "406": { "$ref": "#/components/responses/IncompatibleVersion" }, - "503": { "$ref": "#/components/responses/Unavailable" } + "401": {"$ref": "#/components/responses/Error401"}, + "403": {"$ref": "#/components/responses/Error403"}, + "404": {"$ref": "#/components/responses/Error404"}, + "406": {"$ref": "#/components/responses/Error406"}, + "503": {"$ref": "#/components/responses/Error503"} } }, "patch": { @@ -93,35 +93,41 @@ "tags": ["Workspaces"], "parameters": [ { "$ref": "#/components/parameters/ApiVersion" }, - { "$ref": "#/components/parameters/IfMatch" } + { "$ref": "#/components/parameters/IfMatch" }, + { "$ref": "#/components/parameters/IdempotencyKey" } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/WorkspaceMutation" } } } }, "responses": { + "400": { "$ref": "#/components/responses/Error400" }, "200": { "$ref": "#/components/responses/Workspace" }, - "401": { "$ref": "#/components/responses/Unauthenticated" }, - "403": { "$ref": "#/components/responses/Forbidden" }, - "404": { "$ref": "#/components/responses/NotFound" }, - "406": { "$ref": "#/components/responses/IncompatibleVersion" }, - "409": { "$ref": "#/components/responses/Conflict" }, - "422": { "$ref": "#/components/responses/InvalidRequest" }, - "503": { "$ref": "#/components/responses/Unavailable" } + "401": {"$ref": "#/components/responses/Error401"}, + "403": {"$ref": "#/components/responses/Error403"}, + "404": {"$ref": "#/components/responses/Error404"}, + "406": {"$ref": "#/components/responses/Error406"}, + "409": {"$ref": "#/components/responses/Error409"}, + "422": {"$ref": "#/components/responses/Error422"}, + "503": {"$ref": "#/components/responses/Error503"} } }, "delete": { "operationId": "deleteWorkspace", "tags": ["Workspaces"], - "parameters": [{ "$ref": "#/components/parameters/ApiVersion" }], + "parameters": [ + { "$ref": "#/components/parameters/ApiVersion" }, + { "$ref": "#/components/parameters/IdempotencyKey" } + ], "responses": { "204": { "$ref": "#/components/responses/Deleted" }, - "401": { "$ref": "#/components/responses/Unauthenticated" }, - "403": { "$ref": "#/components/responses/Forbidden" }, - "404": { "$ref": "#/components/responses/NotFound" }, - "406": { "$ref": "#/components/responses/IncompatibleVersion" }, - "409": { "$ref": "#/components/responses/Conflict" }, - "503": { "$ref": "#/components/responses/Unavailable" } + "400": { "$ref": "#/components/responses/Error400" }, + "401": {"$ref": "#/components/responses/Error401"}, + "403": {"$ref": "#/components/responses/Error403"}, + "404": {"$ref": "#/components/responses/Error404"}, + "406": {"$ref": "#/components/responses/Error406"}, + "409": {"$ref": "#/components/responses/Error409"}, + "503": {"$ref": "#/components/responses/Error503"} } } }, @@ -137,12 +143,12 @@ ], "responses": { "200": { "$ref": "#/components/responses/SessionPage" }, - "401": { "$ref": "#/components/responses/Unauthenticated" }, - "403": { "$ref": "#/components/responses/Forbidden" }, - "404": { "$ref": "#/components/responses/NotFound" }, - "406": { "$ref": "#/components/responses/IncompatibleVersion" }, - "422": { "$ref": "#/components/responses/InvalidRequest" }, - "503": { "$ref": "#/components/responses/Unavailable" } + "401": {"$ref": "#/components/responses/Error401"}, + "403": {"$ref": "#/components/responses/Error403"}, + "404": {"$ref": "#/components/responses/Error404"}, + "406": {"$ref": "#/components/responses/Error406"}, + "422": {"$ref": "#/components/responses/Error422"}, + "503": {"$ref": "#/components/responses/Error503"} } }, "post": { @@ -159,13 +165,14 @@ "responses": { "200": { "$ref": "#/components/responses/SessionReplay" }, "202": { "$ref": "#/components/responses/SessionAccepted" }, - "401": { "$ref": "#/components/responses/Unauthenticated" }, - "403": { "$ref": "#/components/responses/Forbidden" }, - "404": { "$ref": "#/components/responses/NotFound" }, - "406": { "$ref": "#/components/responses/IncompatibleVersion" }, - "409": { "$ref": "#/components/responses/Conflict" }, - "422": { "$ref": "#/components/responses/InvalidRequest" }, - "503": { "$ref": "#/components/responses/Unavailable" } + "400": { "$ref": "#/components/responses/Error400" }, + "401": {"$ref": "#/components/responses/Error401"}, + "403": {"$ref": "#/components/responses/Error403"}, + "404": {"$ref": "#/components/responses/Error404"}, + "406": {"$ref": "#/components/responses/Error406"}, + "409": {"$ref": "#/components/responses/Error409"}, + "422": {"$ref": "#/components/responses/Error422"}, + "503": {"$ref": "#/components/responses/Error503"} } } }, @@ -177,11 +184,11 @@ "parameters": [{ "$ref": "#/components/parameters/ApiVersion" }], "responses": { "200": { "$ref": "#/components/responses/Session" }, - "401": { "$ref": "#/components/responses/Unauthenticated" }, - "403": { "$ref": "#/components/responses/Forbidden" }, - "404": { "$ref": "#/components/responses/NotFound" }, - "406": { "$ref": "#/components/responses/IncompatibleVersion" }, - "503": { "$ref": "#/components/responses/Unavailable" } + "401": {"$ref": "#/components/responses/Error401"}, + "403": {"$ref": "#/components/responses/Error403"}, + "404": {"$ref": "#/components/responses/Error404"}, + "406": {"$ref": "#/components/responses/Error406"}, + "503": {"$ref": "#/components/responses/Error503"} } }, "patch": { @@ -189,35 +196,41 @@ "tags": ["Sessions"], "parameters": [ { "$ref": "#/components/parameters/ApiVersion" }, - { "$ref": "#/components/parameters/IfMatch" } + { "$ref": "#/components/parameters/IfMatch" }, + { "$ref": "#/components/parameters/IdempotencyKey" } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SessionMutation" } } } }, "responses": { + "400": { "$ref": "#/components/responses/Error400" }, "200": { "$ref": "#/components/responses/Session" }, - "401": { "$ref": "#/components/responses/Unauthenticated" }, - "403": { "$ref": "#/components/responses/Forbidden" }, - "404": { "$ref": "#/components/responses/NotFound" }, - "406": { "$ref": "#/components/responses/IncompatibleVersion" }, - "409": { "$ref": "#/components/responses/Conflict" }, - "422": { "$ref": "#/components/responses/InvalidRequest" }, - "503": { "$ref": "#/components/responses/Unavailable" } + "401": {"$ref": "#/components/responses/Error401"}, + "403": {"$ref": "#/components/responses/Error403"}, + "404": {"$ref": "#/components/responses/Error404"}, + "406": {"$ref": "#/components/responses/Error406"}, + "409": {"$ref": "#/components/responses/Error409"}, + "422": {"$ref": "#/components/responses/Error422"}, + "503": {"$ref": "#/components/responses/Error503"} } }, "delete": { "operationId": "deleteSession", "tags": ["Sessions"], - "parameters": [{ "$ref": "#/components/parameters/ApiVersion" }], + "parameters": [ + { "$ref": "#/components/parameters/ApiVersion" }, + { "$ref": "#/components/parameters/IdempotencyKey" } + ], "responses": { "204": { "$ref": "#/components/responses/Deleted" }, - "401": { "$ref": "#/components/responses/Unauthenticated" }, - "403": { "$ref": "#/components/responses/Forbidden" }, - "404": { "$ref": "#/components/responses/NotFound" }, - "406": { "$ref": "#/components/responses/IncompatibleVersion" }, - "409": { "$ref": "#/components/responses/Conflict" }, - "503": { "$ref": "#/components/responses/Unavailable" } + "400": { "$ref": "#/components/responses/Error400" }, + "401": {"$ref": "#/components/responses/Error401"}, + "403": {"$ref": "#/components/responses/Error403"}, + "404": {"$ref": "#/components/responses/Error404"}, + "406": {"$ref": "#/components/responses/Error406"}, + "409": {"$ref": "#/components/responses/Error409"}, + "503": {"$ref": "#/components/responses/Error503"} } } }, @@ -227,16 +240,19 @@ "tags": ["Sessions"], "parameters": [ { "$ref": "#/components/parameters/SessionId" }, - { "$ref": "#/components/parameters/ApiVersion" } + { "$ref": "#/components/parameters/ApiVersion" }, + { "$ref": "#/components/parameters/IdempotencyKey" } ], "responses": { + "200": { "$ref": "#/components/responses/SessionReplay" }, + "400": { "$ref": "#/components/responses/Error400" }, "202": { "$ref": "#/components/responses/SessionAccepted" }, - "401": { "$ref": "#/components/responses/Unauthenticated" }, - "403": { "$ref": "#/components/responses/Forbidden" }, - "404": { "$ref": "#/components/responses/NotFound" }, - "406": { "$ref": "#/components/responses/IncompatibleVersion" }, - "409": { "$ref": "#/components/responses/Conflict" }, - "503": { "$ref": "#/components/responses/Unavailable" } + "401": {"$ref": "#/components/responses/Error401"}, + "403": {"$ref": "#/components/responses/Error403"}, + "404": {"$ref": "#/components/responses/Error404"}, + "406": {"$ref": "#/components/responses/Error406"}, + "409": {"$ref": "#/components/responses/Error409"}, + "503": {"$ref": "#/components/responses/Error503"} } } }, @@ -256,13 +272,14 @@ "responses": { "200": { "$ref": "#/components/responses/CommandReplay" }, "202": { "$ref": "#/components/responses/CommandAccepted" }, - "401": { "$ref": "#/components/responses/Unauthenticated" }, - "403": { "$ref": "#/components/responses/Forbidden" }, - "404": { "$ref": "#/components/responses/NotFound" }, - "406": { "$ref": "#/components/responses/IncompatibleVersion" }, - "409": { "$ref": "#/components/responses/Conflict" }, - "422": { "$ref": "#/components/responses/InvalidRequest" }, - "503": { "$ref": "#/components/responses/Unavailable" } + "400": { "$ref": "#/components/responses/Error400" }, + "401": {"$ref": "#/components/responses/Error401"}, + "403": {"$ref": "#/components/responses/Error403"}, + "404": {"$ref": "#/components/responses/Error404"}, + "406": {"$ref": "#/components/responses/Error406"}, + "409": {"$ref": "#/components/responses/Error409"}, + "422": {"$ref": "#/components/responses/Error422"}, + "503": {"$ref": "#/components/responses/Error503"} } } }, @@ -276,11 +293,11 @@ ], "responses": { "200": { "$ref": "#/components/responses/Snapshot" }, - "401": { "$ref": "#/components/responses/Unauthenticated" }, - "403": { "$ref": "#/components/responses/Forbidden" }, - "404": { "$ref": "#/components/responses/NotFound" }, - "406": { "$ref": "#/components/responses/IncompatibleVersion" }, - "503": { "$ref": "#/components/responses/Unavailable" } + "401": {"$ref": "#/components/responses/Error401"}, + "403": {"$ref": "#/components/responses/Error403"}, + "404": {"$ref": "#/components/responses/Error404"}, + "406": {"$ref": "#/components/responses/Error406"}, + "503": {"$ref": "#/components/responses/Error503"} } } }, @@ -312,14 +329,14 @@ } } }, - "400": { "$ref": "#/components/responses/BadRequest" }, - "401": { "$ref": "#/components/responses/Unauthenticated" }, - "403": { "$ref": "#/components/responses/Forbidden" }, - "404": { "$ref": "#/components/responses/NotFound" }, - "406": { "$ref": "#/components/responses/IncompatibleVersion" }, - "410": { "$ref": "#/components/responses/CursorExpired" }, - "422": { "$ref": "#/components/responses/InvalidRequest" }, - "503": { "$ref": "#/components/responses/Unavailable" } + "400": {"$ref": "#/components/responses/Error400"}, + "401": {"$ref": "#/components/responses/Error401"}, + "403": {"$ref": "#/components/responses/Error403"}, + "404": {"$ref": "#/components/responses/Error404"}, + "406": {"$ref": "#/components/responses/Error406"}, + "410": {"$ref": "#/components/responses/Error410"}, + "422": {"$ref": "#/components/responses/Error422"}, + "503": {"$ref": "#/components/responses/Error503"} } } } @@ -344,7 +361,7 @@ "name": "Idempotency-Key", "in": "header", "required": true, - "description": "Opaque caller-generated key scoped to credential, operation, and target. Same key plus same canonical request replays; different content conflicts.", + "description": "Opaque caller-generated key scoped by the server to the authenticated principal, operation ID, and target resource IDs. Idempotency is evaluated only after path, query, header, and request-schema validation and application of schema defaults. Request identity is the operation ID, target IDs, relevant precondition headers, and the RFC 8785 JSON Canonicalization Scheme (JCS) bytes of the validated JSON body. JCS object member order is insignificant, array order is significant, omitted fields remain distinct unless the request schema defaulted them, and no Unicode normalization is performed beyond RFC 8785. Reusing a key with identical identity replays the original terminal response and advertises Idempotency-Replayed; reusing it with a different identity returns 409 idempotency_conflict.", "schema": { "type": "string", "minLength": 16, "maxLength": 128, "pattern": "^[A-Za-z0-9._~-]+$" } }, "IfMatch": { @@ -428,11 +445,13 @@ "limits": { "type": "object", "additionalProperties": false, - "required": ["pageSizeDefault", "pageSizeMax", "commandBytesMax", "watchEventsMax", "heartbeatSeconds"], + "required": ["pageSizeDefault", "pageSizeMax", "commandBytesMax", "commandRequestBytesMax", "commandMetadataValueBytesMax", "watchEventsMax", "heartbeatSeconds"], "properties": { "pageSizeDefault": { "type": "integer", "minimum": 1, "maximum": 100 }, "pageSizeMax": { "type": "integer", "minimum": 1, "maximum": 100 }, - "commandBytesMax": { "type": "integer", "minimum": 1, "maximum": 1048576 }, + "commandBytesMax": { "type": "integer", "minimum": 1, "maximum": 262144 }, + "commandRequestBytesMax": { "type": "integer", "minimum": 1, "maximum": 1048576 }, + "commandMetadataValueBytesMax": { "type": "integer", "minimum": 1, "maximum": 262144 }, "watchEventsMax": { "type": "integer", "minimum": 1, "maximum": 1000 }, "heartbeatSeconds": { "type": "integer", "minimum": 5, "maximum": 60 } } @@ -528,9 +547,23 @@ "type": "object", "additionalProperties": false, "required": ["command"], + "x-t4-maxUtf8Bytes": 1048576, "properties": { - "command": { "type": "string", "minLength": 1, "maxLength": 262144 }, - "metadata": { "type": "object", "maxProperties": 32, "additionalProperties": { "type": ["string", "number", "boolean", "null"] } } + "command": { "type": "string", "minLength": 1, "maxLength": 262144, "x-t4-maxUtf8Bytes": 262144 }, + "metadata": { + "type": "object", + "default": {}, + "maxProperties": 32, + "propertyNames": { "pattern": "^[a-z][a-z0-9.-]{0,62}$" }, + "additionalProperties": { + "oneOf": [ + { "type": "string", "maxLength": 262144, "x-t4-maxUtf8Bytes": 262144 }, + { "type": "integer", "minimum": -9007199254740991, "maximum": 9007199254740991 }, + { "type": "boolean" }, + { "type": "null" } + ] + } + } } }, "CommandResult": { @@ -600,8 +633,7 @@ { "$ref": "#/components/schemas/HeartbeatWatchEvent" }, { "$ref": "#/components/schemas/SessionWatchEvent" }, { "$ref": "#/components/schemas/CommandWatchEvent" } - ], - "discriminator": { "propertyName": "type" } + ] }, "FieldViolation": { "type": "object", @@ -639,6 +671,42 @@ "resync": { "$ref": "#/components/schemas/Resync" } } }, + "BadRequestApiError": { + "allOf": [ + { "$ref": "#/components/schemas/ApiError" }, + { "type": "object", "properties": { "code": { "enum": ["invalid_request", "idempotency_key_required", "invalid_origin", "https_required"] } } } + ] + }, + "UnauthenticatedApiError": { + "allOf": [ + { "$ref": "#/components/schemas/ApiError" }, + { "type": "object", "properties": { "code": { "const": "unauthenticated" } } } + ] + }, + "ForbiddenApiError": { + "allOf": [ + { "$ref": "#/components/schemas/ApiError" }, + { "type": "object", "properties": { "code": { "const": "forbidden" } } } + ] + }, + "NotFoundApiError": { + "allOf": [ + { "$ref": "#/components/schemas/ApiError" }, + { "type": "object", "properties": { "code": { "const": "not_found" } } } + ] + }, + "ConflictApiError": { + "allOf": [ + { "$ref": "#/components/schemas/ApiError" }, + { "type": "object", "properties": { "code": { "enum": ["idempotency_conflict", "revision_conflict"] } } } + ] + }, + "UnavailableApiError": { + "allOf": [ + { "$ref": "#/components/schemas/ApiError" }, + { "type": "object", "properties": { "code": { "enum": ["unavailable", "indeterminate"] } } } + ] + }, "IncompatibleVersionApiError": { "allOf": [ { "$ref": "#/components/schemas/ApiError" }, @@ -663,6 +731,42 @@ "required": ["error"], "properties": { "error": { "$ref": "#/components/schemas/ApiError" } } }, + "BadRequestErrorEnvelope": { + "type": "object", + "additionalProperties": false, + "required": ["error"], + "properties": { "error": { "$ref": "#/components/schemas/BadRequestApiError" } } + }, + "UnauthenticatedErrorEnvelope": { + "type": "object", + "additionalProperties": false, + "required": ["error"], + "properties": { "error": { "$ref": "#/components/schemas/UnauthenticatedApiError" } } + }, + "ForbiddenErrorEnvelope": { + "type": "object", + "additionalProperties": false, + "required": ["error"], + "properties": { "error": { "$ref": "#/components/schemas/ForbiddenApiError" } } + }, + "NotFoundErrorEnvelope": { + "type": "object", + "additionalProperties": false, + "required": ["error"], + "properties": { "error": { "$ref": "#/components/schemas/NotFoundApiError" } } + }, + "ConflictErrorEnvelope": { + "type": "object", + "additionalProperties": false, + "required": ["error"], + "properties": { "error": { "$ref": "#/components/schemas/ConflictApiError" } } + }, + "UnavailableErrorEnvelope": { + "type": "object", + "additionalProperties": false, + "required": ["error"], + "properties": { "error": { "$ref": "#/components/schemas/UnavailableApiError" } } + }, "IncompatibleVersionErrorEnvelope": { "type": "object", "additionalProperties": false, @@ -690,30 +794,21 @@ "WorkspacePage": { "description": "Bounded workspace page", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/WorkspacePage" } } } }, "Session": { "description": "Session", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Session" } } } }, "SessionAccepted": { "description": "Session intent accepted", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Session" } } } }, - "SessionReplay": { "description": "Identical session spawn replay", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Session" } } } }, + "SessionReplay": { "description": "Identical session mutation replay", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Session" } } } }, "SessionPage": { "description": "Bounded session page", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SessionPage" } } } }, "CommandAccepted": { "description": "Command intent accepted with a stable outcome state", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CommandResult" } } } }, "CommandReplay": { "description": "Identical command request replay", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CommandResult" } } } }, "Snapshot": { "description": "Bounded session snapshot and reconnect cursor", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SessionSnapshot" } } } }, - "Deleted": { "description": "Deletion accepted or resource already absent", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } } }, - "BadRequest": { "$ref": "#/components/responses/Error400" }, - "Unauthenticated": { "$ref": "#/components/responses/Error401" }, - "Forbidden": { "$ref": "#/components/responses/Error403" }, - "NotFound": { "$ref": "#/components/responses/Error404" }, - "IncompatibleVersion": { "$ref": "#/components/responses/Error406" }, - "Conflict": { "$ref": "#/components/responses/Error409" }, - "CursorExpired": { "$ref": "#/components/responses/Error410" }, - "InvalidRequest": { "$ref": "#/components/responses/Error422" }, - "Unavailable": { "$ref": "#/components/responses/Error503" }, - "Error400": { "description": "Malformed request or missing idempotency key", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorEnvelope" } } } }, - "Error401": { "description": "Missing or invalid opaque bearer credential", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorEnvelope" } } } }, - "Error403": { "description": "Credential lacks the deny-by-default operation or resource scope", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorEnvelope" } } } }, - "Error404": { "description": "Resource absent or outside caller scope (scope existence is not disclosed)", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorEnvelope" } } } }, + "Deleted": { "description": "Idempotent deletion accepted or resource already absent", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" } } }, + "Error400": { "description": "Malformed request or missing idempotency key", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/BadRequestErrorEnvelope" } } } }, + "Error401": { "description": "Missing or invalid opaque bearer credential", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/UnauthenticatedErrorEnvelope" } } } }, + "Error403": { "description": "Credential lacks the deny-by-default operation or resource scope", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ForbiddenErrorEnvelope" } } } }, + "Error404": { "description": "Resource absent or outside caller scope (scope existence is not disclosed)", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/NotFoundErrorEnvelope" } } } }, "Error406": { "description": "Requested major is incompatible; no silent downgrade", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/IncompatibleVersionErrorEnvelope" } } } }, - "Error409": { "description": "Idempotency or resource revision conflict", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorEnvelope" } } } }, + "Error409": { "description": "Idempotency or resource revision conflict", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ConflictErrorEnvelope" } } } }, "Error410": { "description": "Watch cursor expired; resync from the typed snapshot target", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CursorExpiredErrorEnvelope" } } } }, "Error422": { "description": "Bounded semantic validation failure with stable field violations", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/InvalidRequestErrorEnvelope" } } } }, - "Error503": { "description": "Service temporarily unavailable or outcome indeterminate", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ErrorEnvelope" } } } } + "Error503": { "description": "Service temporarily unavailable or outcome indeterminate", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/UnavailableErrorEnvelope" } } } } } } } From 65a85247c05c1d4e0d1d2a8e65fb997ea75cf01d Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 20:31:57 +0000 Subject: [PATCH 010/131] build: accept authorized T4 API schema artifact --- .github/workflows/ci.yml | 9 - .../t4-api-client/src/generated/schema.ts | 349 +++++++++--------- 2 files changed, 175 insertions(+), 183 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2bad79b0..25d7d947 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,20 +82,11 @@ jobs: - name: Validate OpenAPI and deterministic generation run: | pnpm --filter @t4-code/t4-api-contract validate - pnpm --filter @t4-code/t4-api-contract generate:ci pnpm --filter @t4-code/t4-api-contract check:generated pnpm --filter @t4-code/t4-api-client typecheck pnpm --filter @t4-code/client typecheck pnpm --filter @t4-code/client exec vp test run test/t4-api-v1-conformance.test.ts - - name: Upload authorized generated schema - if: ${{ always() }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: t4-api-generated-${{ github.run_id }} - path: artifacts/t4-api/generated/schema.ts - if-no-files-found: error - retention-days: 1 core: runs-on: ubuntu-24.04 diff --git a/packages/t4-api-client/src/generated/schema.ts b/packages/t4-api-client/src/generated/schema.ts index 03954962..0e4b33ec 100755 --- a/packages/t4-api-client/src/generated/schema.ts +++ b/packages/t4-api-client/src/generated/schema.ts @@ -168,9 +168,17 @@ export interface components { supportedMajors?: number[]; violations?: components["schemas"]["FieldViolation"][]; }; + BadRequestApiError: components["schemas"]["ApiError"] & { + /** @enum {unknown} */ + code?: "invalid_request" | "idempotency_key_required" | "invalid_origin" | "https_required"; + }; + BadRequestErrorEnvelope: { + error: components["schemas"]["BadRequestApiError"]; + }; CommandCreate: { command: string; - metadata?: { + /** @default {} */ + metadata: { [key: string]: string | number | boolean | null; }; }; @@ -182,11 +190,15 @@ export interface components { commandId: components["schemas"]["ResourceId"]; cursor: components["schemas"]["Cursor"]; state: components["schemas"]["OperationState"]; - /** - * @description discriminator enum property added by openapi-typescript - * @enum {string} - */ - type: "CommandWatchEvent"; + /** @constant */ + type: "command"; + }; + ConflictApiError: components["schemas"]["ApiError"] & { + /** @enum {unknown} */ + code?: "idempotency_conflict" | "revision_conflict"; + }; + ConflictErrorEnvelope: { + error: components["schemas"]["ConflictApiError"]; }; /** @description Opaque server-issued header-safe SSE cursor. */ Cursor: string; @@ -203,6 +215,8 @@ export interface components { capabilities: string[]; limits: { commandBytesMax: number; + commandMetadataValueBytesMax: number; + commandRequestBytesMax: number; heartbeatSeconds: number; pageSizeDefault: number; pageSizeMax: number; @@ -218,15 +232,19 @@ export interface components { message: string; rule: string; }; + ForbiddenApiError: components["schemas"]["ApiError"] & { + /** @constant */ + code?: "forbidden"; + }; + ForbiddenErrorEnvelope: { + error: components["schemas"]["ForbiddenApiError"]; + }; HeartbeatWatchEvent: { cursor: components["schemas"]["Cursor"]; /** Format: date-time */ observedAt: string; - /** - * @description discriminator enum property added by openapi-typescript - * @enum {string} - */ - type: "HeartbeatWatchEvent"; + /** @constant */ + type: "heartbeat"; }; IncompatibleVersionApiError: components["schemas"]["ApiError"] & { /** @constant */ @@ -247,6 +265,13 @@ export interface components { Labels: { [key: string]: string; }; + NotFoundApiError: components["schemas"]["ApiError"] & { + /** @constant */ + code?: "not_found"; + }; + NotFoundErrorEnvelope: { + error: components["schemas"]["NotFoundApiError"]; + }; /** @enum {string} */ OperationState: "accepted" | "rejected" | "conflict" | "unavailable" | "indeterminate"; ResourceId: string; @@ -287,11 +312,8 @@ export interface components { cursor: components["schemas"]["Cursor"]; revision: components["schemas"]["Revision"]; state: components["schemas"]["SessionState"]; - /** - * @description discriminator enum property added by openapi-typescript - * @enum {string} - */ - type: "SessionWatchEvent"; + /** @constant */ + type: "session"; }; SnapshotEntry: { /** @enum {string} */ @@ -299,6 +321,20 @@ export interface components { sequence: number; text: string; }; + UnauthenticatedApiError: components["schemas"]["ApiError"] & { + /** @constant */ + code?: "unauthenticated"; + }; + UnauthenticatedErrorEnvelope: { + error: components["schemas"]["UnauthenticatedApiError"]; + }; + UnavailableApiError: components["schemas"]["ApiError"] & { + /** @enum {unknown} */ + code?: "unavailable" | "indeterminate"; + }; + UnavailableErrorEnvelope: { + error: components["schemas"]["UnavailableApiError"]; + }; WatchEvent: components["schemas"]["HeartbeatWatchEvent"] | components["schemas"]["SessionWatchEvent"] | components["schemas"]["CommandWatchEvent"]; Workspace: { id: components["schemas"]["ResourceId"]; @@ -323,12 +359,6 @@ export interface components { WorkspaceState: "accepted" | "provisioning" | "ready" | "deleting" | "deleted" | "failed" | "unavailable" | "indeterminate"; }; responses: { - BadRequest: { - headers: { - [name: string]: unknown; - }; - content?: never; - }; /** @description Command intent accepted with a stable outcome state */ CommandAccepted: { headers: { @@ -351,21 +381,10 @@ export interface components { "application/json": components["schemas"]["CommandResult"]; }; }; - Conflict: { - headers: { - [name: string]: unknown; - }; - content?: never; - }; - CursorExpired: { - headers: { - [name: string]: unknown; - }; - content?: never; - }; - /** @description Deletion accepted or resource already absent */ + /** @description Idempotent deletion accepted or resource already absent */ Deleted: { headers: { + "Idempotency-Replayed": components["headers"]["IdempotencyReplayed"]; "T4-API-Version": components["headers"]["SelectedVersion"]; [name: string]: unknown; }; @@ -388,7 +407,7 @@ export interface components { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["ErrorEnvelope"]; + "application/json": components["schemas"]["BadRequestErrorEnvelope"]; }; }; /** @description Missing or invalid opaque bearer credential */ @@ -397,7 +416,7 @@ export interface components { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["ErrorEnvelope"]; + "application/json": components["schemas"]["UnauthenticatedErrorEnvelope"]; }; }; /** @description Credential lacks the deny-by-default operation or resource scope */ @@ -407,7 +426,7 @@ export interface components { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["ErrorEnvelope"]; + "application/json": components["schemas"]["ForbiddenErrorEnvelope"]; }; }; /** @description Resource absent or outside caller scope (scope existence is not disclosed) */ @@ -417,7 +436,7 @@ export interface components { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["ErrorEnvelope"]; + "application/json": components["schemas"]["NotFoundErrorEnvelope"]; }; }; /** @description Requested major is incompatible; no silent downgrade */ @@ -437,7 +456,7 @@ export interface components { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["ErrorEnvelope"]; + "application/json": components["schemas"]["ConflictErrorEnvelope"]; }; }; /** @description Watch cursor expired; resync from the typed snapshot target */ @@ -467,32 +486,8 @@ export interface components { [name: string]: unknown; }; content: { - "application/json": components["schemas"]["ErrorEnvelope"]; - }; - }; - Forbidden: { - headers: { - [name: string]: unknown; - }; - content?: never; - }; - IncompatibleVersion: { - headers: { - [name: string]: unknown; - }; - content?: never; - }; - InvalidRequest: { - headers: { - [name: string]: unknown; + "application/json": components["schemas"]["UnavailableErrorEnvelope"]; }; - content?: never; - }; - NotFound: { - headers: { - [name: string]: unknown; - }; - content?: never; }; /** @description Session */ Session: { @@ -525,7 +520,7 @@ export interface components { "application/json": components["schemas"]["SessionPage"]; }; }; - /** @description Identical session spawn replay */ + /** @description Identical session mutation replay */ SessionReplay: { headers: { "Idempotency-Replayed": components["headers"]["IdempotencyReplayed"]; @@ -546,18 +541,6 @@ export interface components { "application/json": components["schemas"]["SessionSnapshot"]; }; }; - Unauthenticated: { - headers: { - [name: string]: unknown; - }; - content?: never; - }; - Unavailable: { - headers: { - [name: string]: unknown; - }; - content?: never; - }; /** @description Workspace */ Workspace: { headers: { @@ -605,7 +588,7 @@ export interface components { /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ ApiVersion: string; HeartbeatSeconds: number; - /** @description Opaque caller-generated key scoped to credential, operation, and target. Same key plus same canonical request replays; different content conflicts. */ + /** @description Opaque caller-generated key scoped by the server to the authenticated principal, operation ID, and target resource IDs. Idempotency is evaluated only after path, query, header, and request-schema validation and application of schema defaults. Request identity is the operation ID, target IDs, relevant precondition headers, and the RFC 8785 JSON Canonicalization Scheme (JCS) bytes of the validated JSON body. JCS object member order is insignificant, array order is significant, omitted fields remain distinct unless the request schema defaulted them, and no Unicode normalization is performed beyond RFC 8785. Reusing a key with identical identity replays the original terminal response and advertises Idempotency-Replayed; reusing it with a different identity returns 409 idempotency_conflict. */ IdempotencyKey: string; /** @description Decimal resource revision for optimistic mutation. */ IfMatch: string; @@ -644,10 +627,10 @@ export interface operations { requestBody?: never; responses: { 200: components["responses"]["Discovery"]; - 401: components["responses"]["Unauthenticated"]; - 403: components["responses"]["Forbidden"]; - 406: components["responses"]["IncompatibleVersion"]; - 503: components["responses"]["Unavailable"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 406: components["responses"]["Error406"]; + 503: components["responses"]["Error503"]; }; }; getSession: { @@ -665,17 +648,19 @@ export interface operations { requestBody?: never; responses: { 200: components["responses"]["Session"]; - 401: components["responses"]["Unauthenticated"]; - 403: components["responses"]["Forbidden"]; - 404: components["responses"]["NotFound"]; - 406: components["responses"]["IncompatibleVersion"]; - 503: components["responses"]["Unavailable"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + 406: components["responses"]["Error406"]; + 503: components["responses"]["Error503"]; }; }; deleteSession: { parameters: { query?: never; header: { + /** @description Opaque caller-generated key scoped by the server to the authenticated principal, operation ID, and target resource IDs. Idempotency is evaluated only after path, query, header, and request-schema validation and application of schema defaults. Request identity is the operation ID, target IDs, relevant precondition headers, and the RFC 8785 JSON Canonicalization Scheme (JCS) bytes of the validated JSON body. JCS object member order is insignificant, array order is significant, omitted fields remain distinct unless the request schema defaulted them, and no Unicode normalization is performed beyond RFC 8785. Reusing a key with identical identity replays the original terminal response and advertises Idempotency-Replayed; reusing it with a different identity returns 409 idempotency_conflict. */ + "Idempotency-Key": components["parameters"]["IdempotencyKey"]; /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ "T4-API-Version": components["parameters"]["ApiVersion"]; }; @@ -687,18 +672,21 @@ export interface operations { requestBody?: never; responses: { 204: components["responses"]["Deleted"]; - 401: components["responses"]["Unauthenticated"]; - 403: components["responses"]["Forbidden"]; - 404: components["responses"]["NotFound"]; - 406: components["responses"]["IncompatibleVersion"]; - 409: components["responses"]["Conflict"]; - 503: components["responses"]["Unavailable"]; + 400: components["responses"]["Error400"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + 406: components["responses"]["Error406"]; + 409: components["responses"]["Error409"]; + 503: components["responses"]["Error503"]; }; }; mutateSession: { parameters: { query?: never; header: { + /** @description Opaque caller-generated key scoped by the server to the authenticated principal, operation ID, and target resource IDs. Idempotency is evaluated only after path, query, header, and request-schema validation and application of schema defaults. Request identity is the operation ID, target IDs, relevant precondition headers, and the RFC 8785 JSON Canonicalization Scheme (JCS) bytes of the validated JSON body. JCS object member order is insignificant, array order is significant, omitted fields remain distinct unless the request schema defaulted them, and no Unicode normalization is performed beyond RFC 8785. Reusing a key with identical identity replays the original terminal response and advertises Idempotency-Replayed; reusing it with a different identity returns 409 idempotency_conflict. */ + "Idempotency-Key": components["parameters"]["IdempotencyKey"]; /** @description Decimal resource revision for optimistic mutation. */ "If-Match": components["parameters"]["IfMatch"]; /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ @@ -716,19 +704,22 @@ export interface operations { }; responses: { 200: components["responses"]["Session"]; - 401: components["responses"]["Unauthenticated"]; - 403: components["responses"]["Forbidden"]; - 404: components["responses"]["NotFound"]; - 406: components["responses"]["IncompatibleVersion"]; - 409: components["responses"]["Conflict"]; - 422: components["responses"]["InvalidRequest"]; - 503: components["responses"]["Unavailable"]; + 400: components["responses"]["Error400"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + 406: components["responses"]["Error406"]; + 409: components["responses"]["Error409"]; + 422: components["responses"]["Error422"]; + 503: components["responses"]["Error503"]; }; }; cancelSession: { parameters: { query?: never; header: { + /** @description Opaque caller-generated key scoped by the server to the authenticated principal, operation ID, and target resource IDs. Idempotency is evaluated only after path, query, header, and request-schema validation and application of schema defaults. Request identity is the operation ID, target IDs, relevant precondition headers, and the RFC 8785 JSON Canonicalization Scheme (JCS) bytes of the validated JSON body. JCS object member order is insignificant, array order is significant, omitted fields remain distinct unless the request schema defaulted them, and no Unicode normalization is performed beyond RFC 8785. Reusing a key with identical identity replays the original terminal response and advertises Idempotency-Replayed; reusing it with a different identity returns 409 idempotency_conflict. */ + "Idempotency-Key": components["parameters"]["IdempotencyKey"]; /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ "T4-API-Version": components["parameters"]["ApiVersion"]; }; @@ -739,20 +730,22 @@ export interface operations { }; requestBody?: never; responses: { + 200: components["responses"]["SessionReplay"]; 202: components["responses"]["SessionAccepted"]; - 401: components["responses"]["Unauthenticated"]; - 403: components["responses"]["Forbidden"]; - 404: components["responses"]["NotFound"]; - 406: components["responses"]["IncompatibleVersion"]; - 409: components["responses"]["Conflict"]; - 503: components["responses"]["Unavailable"]; + 400: components["responses"]["Error400"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + 406: components["responses"]["Error406"]; + 409: components["responses"]["Error409"]; + 503: components["responses"]["Error503"]; }; }; submitCommand: { parameters: { query?: never; header: { - /** @description Opaque caller-generated key scoped to credential, operation, and target. Same key plus same canonical request replays; different content conflicts. */ + /** @description Opaque caller-generated key scoped by the server to the authenticated principal, operation ID, and target resource IDs. Idempotency is evaluated only after path, query, header, and request-schema validation and application of schema defaults. Request identity is the operation ID, target IDs, relevant precondition headers, and the RFC 8785 JSON Canonicalization Scheme (JCS) bytes of the validated JSON body. JCS object member order is insignificant, array order is significant, omitted fields remain distinct unless the request schema defaulted them, and no Unicode normalization is performed beyond RFC 8785. Reusing a key with identical identity replays the original terminal response and advertises Idempotency-Replayed; reusing it with a different identity returns 409 idempotency_conflict. */ "Idempotency-Key": components["parameters"]["IdempotencyKey"]; /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ "T4-API-Version": components["parameters"]["ApiVersion"]; @@ -770,13 +763,14 @@ export interface operations { responses: { 200: components["responses"]["CommandReplay"]; 202: components["responses"]["CommandAccepted"]; - 401: components["responses"]["Unauthenticated"]; - 403: components["responses"]["Forbidden"]; - 404: components["responses"]["NotFound"]; - 406: components["responses"]["IncompatibleVersion"]; - 409: components["responses"]["Conflict"]; - 422: components["responses"]["InvalidRequest"]; - 503: components["responses"]["Unavailable"]; + 400: components["responses"]["Error400"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + 406: components["responses"]["Error406"]; + 409: components["responses"]["Error409"]; + 422: components["responses"]["Error422"]; + 503: components["responses"]["Error503"]; }; }; watchSessionEvents: { @@ -812,14 +806,14 @@ export interface operations { "text/event-stream": string; }; }; - 400: components["responses"]["BadRequest"]; - 401: components["responses"]["Unauthenticated"]; - 403: components["responses"]["Forbidden"]; - 404: components["responses"]["NotFound"]; - 406: components["responses"]["IncompatibleVersion"]; - 410: components["responses"]["CursorExpired"]; - 422: components["responses"]["InvalidRequest"]; - 503: components["responses"]["Unavailable"]; + 400: components["responses"]["Error400"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + 406: components["responses"]["Error406"]; + 410: components["responses"]["Error410"]; + 422: components["responses"]["Error422"]; + 503: components["responses"]["Error503"]; }; }; getSessionSnapshot: { @@ -837,11 +831,11 @@ export interface operations { requestBody?: never; responses: { 200: components["responses"]["Snapshot"]; - 401: components["responses"]["Unauthenticated"]; - 403: components["responses"]["Forbidden"]; - 404: components["responses"]["NotFound"]; - 406: components["responses"]["IncompatibleVersion"]; - 503: components["responses"]["Unavailable"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + 406: components["responses"]["Error406"]; + 503: components["responses"]["Error503"]; }; }; listWorkspaces: { @@ -861,19 +855,19 @@ export interface operations { requestBody?: never; responses: { 200: components["responses"]["WorkspacePage"]; - 400: components["responses"]["BadRequest"]; - 401: components["responses"]["Unauthenticated"]; - 403: components["responses"]["Forbidden"]; - 406: components["responses"]["IncompatibleVersion"]; - 422: components["responses"]["InvalidRequest"]; - 503: components["responses"]["Unavailable"]; + 400: components["responses"]["Error400"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 406: components["responses"]["Error406"]; + 422: components["responses"]["Error422"]; + 503: components["responses"]["Error503"]; }; }; createWorkspace: { parameters: { query?: never; header: { - /** @description Opaque caller-generated key scoped to credential, operation, and target. Same key plus same canonical request replays; different content conflicts. */ + /** @description Opaque caller-generated key scoped by the server to the authenticated principal, operation ID, and target resource IDs. Idempotency is evaluated only after path, query, header, and request-schema validation and application of schema defaults. Request identity is the operation ID, target IDs, relevant precondition headers, and the RFC 8785 JSON Canonicalization Scheme (JCS) bytes of the validated JSON body. JCS object member order is insignificant, array order is significant, omitted fields remain distinct unless the request schema defaulted them, and no Unicode normalization is performed beyond RFC 8785. Reusing a key with identical identity replays the original terminal response and advertises Idempotency-Replayed; reusing it with a different identity returns 409 idempotency_conflict. */ "Idempotency-Key": components["parameters"]["IdempotencyKey"]; /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ "T4-API-Version": components["parameters"]["ApiVersion"]; @@ -889,13 +883,13 @@ export interface operations { responses: { 200: components["responses"]["WorkspaceReplay"]; 202: components["responses"]["WorkspaceAccepted"]; - 400: components["responses"]["BadRequest"]; - 401: components["responses"]["Unauthenticated"]; - 403: components["responses"]["Forbidden"]; - 406: components["responses"]["IncompatibleVersion"]; - 409: components["responses"]["Conflict"]; - 422: components["responses"]["InvalidRequest"]; - 503: components["responses"]["Unavailable"]; + 400: components["responses"]["Error400"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 406: components["responses"]["Error406"]; + 409: components["responses"]["Error409"]; + 422: components["responses"]["Error422"]; + 503: components["responses"]["Error503"]; }; }; getWorkspace: { @@ -913,17 +907,19 @@ export interface operations { requestBody?: never; responses: { 200: components["responses"]["Workspace"]; - 401: components["responses"]["Unauthenticated"]; - 403: components["responses"]["Forbidden"]; - 404: components["responses"]["NotFound"]; - 406: components["responses"]["IncompatibleVersion"]; - 503: components["responses"]["Unavailable"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + 406: components["responses"]["Error406"]; + 503: components["responses"]["Error503"]; }; }; deleteWorkspace: { parameters: { query?: never; header: { + /** @description Opaque caller-generated key scoped by the server to the authenticated principal, operation ID, and target resource IDs. Idempotency is evaluated only after path, query, header, and request-schema validation and application of schema defaults. Request identity is the operation ID, target IDs, relevant precondition headers, and the RFC 8785 JSON Canonicalization Scheme (JCS) bytes of the validated JSON body. JCS object member order is insignificant, array order is significant, omitted fields remain distinct unless the request schema defaulted them, and no Unicode normalization is performed beyond RFC 8785. Reusing a key with identical identity replays the original terminal response and advertises Idempotency-Replayed; reusing it with a different identity returns 409 idempotency_conflict. */ + "Idempotency-Key": components["parameters"]["IdempotencyKey"]; /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ "T4-API-Version": components["parameters"]["ApiVersion"]; }; @@ -935,18 +931,21 @@ export interface operations { requestBody?: never; responses: { 204: components["responses"]["Deleted"]; - 401: components["responses"]["Unauthenticated"]; - 403: components["responses"]["Forbidden"]; - 404: components["responses"]["NotFound"]; - 406: components["responses"]["IncompatibleVersion"]; - 409: components["responses"]["Conflict"]; - 503: components["responses"]["Unavailable"]; + 400: components["responses"]["Error400"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + 406: components["responses"]["Error406"]; + 409: components["responses"]["Error409"]; + 503: components["responses"]["Error503"]; }; }; mutateWorkspace: { parameters: { query?: never; header: { + /** @description Opaque caller-generated key scoped by the server to the authenticated principal, operation ID, and target resource IDs. Idempotency is evaluated only after path, query, header, and request-schema validation and application of schema defaults. Request identity is the operation ID, target IDs, relevant precondition headers, and the RFC 8785 JSON Canonicalization Scheme (JCS) bytes of the validated JSON body. JCS object member order is insignificant, array order is significant, omitted fields remain distinct unless the request schema defaulted them, and no Unicode normalization is performed beyond RFC 8785. Reusing a key with identical identity replays the original terminal response and advertises Idempotency-Replayed; reusing it with a different identity returns 409 idempotency_conflict. */ + "Idempotency-Key": components["parameters"]["IdempotencyKey"]; /** @description Decimal resource revision for optimistic mutation. */ "If-Match": components["parameters"]["IfMatch"]; /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ @@ -964,13 +963,14 @@ export interface operations { }; responses: { 200: components["responses"]["Workspace"]; - 401: components["responses"]["Unauthenticated"]; - 403: components["responses"]["Forbidden"]; - 404: components["responses"]["NotFound"]; - 406: components["responses"]["IncompatibleVersion"]; - 409: components["responses"]["Conflict"]; - 422: components["responses"]["InvalidRequest"]; - 503: components["responses"]["Unavailable"]; + 400: components["responses"]["Error400"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + 406: components["responses"]["Error406"]; + 409: components["responses"]["Error409"]; + 422: components["responses"]["Error422"]; + 503: components["responses"]["Error503"]; }; }; listSessions: { @@ -992,19 +992,19 @@ export interface operations { requestBody?: never; responses: { 200: components["responses"]["SessionPage"]; - 401: components["responses"]["Unauthenticated"]; - 403: components["responses"]["Forbidden"]; - 404: components["responses"]["NotFound"]; - 406: components["responses"]["IncompatibleVersion"]; - 422: components["responses"]["InvalidRequest"]; - 503: components["responses"]["Unavailable"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + 406: components["responses"]["Error406"]; + 422: components["responses"]["Error422"]; + 503: components["responses"]["Error503"]; }; }; spawnSession: { parameters: { query?: never; header: { - /** @description Opaque caller-generated key scoped to credential, operation, and target. Same key plus same canonical request replays; different content conflicts. */ + /** @description Opaque caller-generated key scoped by the server to the authenticated principal, operation ID, and target resource IDs. Idempotency is evaluated only after path, query, header, and request-schema validation and application of schema defaults. Request identity is the operation ID, target IDs, relevant precondition headers, and the RFC 8785 JSON Canonicalization Scheme (JCS) bytes of the validated JSON body. JCS object member order is insignificant, array order is significant, omitted fields remain distinct unless the request schema defaulted them, and no Unicode normalization is performed beyond RFC 8785. Reusing a key with identical identity replays the original terminal response and advertises Idempotency-Replayed; reusing it with a different identity returns 409 idempotency_conflict. */ "Idempotency-Key": components["parameters"]["IdempotencyKey"]; /** @description Requested API major. v1 clients send 1. Unsupported majors fail with 406 instead of silently downgrading. */ "T4-API-Version": components["parameters"]["ApiVersion"]; @@ -1022,13 +1022,14 @@ export interface operations { responses: { 200: components["responses"]["SessionReplay"]; 202: components["responses"]["SessionAccepted"]; - 401: components["responses"]["Unauthenticated"]; - 403: components["responses"]["Forbidden"]; - 404: components["responses"]["NotFound"]; - 406: components["responses"]["IncompatibleVersion"]; - 409: components["responses"]["Conflict"]; - 422: components["responses"]["InvalidRequest"]; - 503: components["responses"]["Unavailable"]; + 400: components["responses"]["Error400"]; + 401: components["responses"]["Error401"]; + 403: components["responses"]["Error403"]; + 404: components["responses"]["Error404"]; + 406: components["responses"]["Error406"]; + 409: components["responses"]["Error409"]; + 422: components["responses"]["Error422"]; + 503: components["responses"]["Error503"]; }; }; } From 93f50e5cfe941587929cf662afca35dc074dbb95 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 20:36:58 +0000 Subject: [PATCH 011/131] test: complete bounded API conformance cases --- .../client/test/t4-api-v1-conformance.test.ts | 63 ++++++++++++++++--- 1 file changed, 55 insertions(+), 8 deletions(-) diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index be43e21f..5eea9590 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -164,6 +164,19 @@ describe("generated T4 API v1 client conformance", () => { }); expect(stale.response.status).toBe(409); expect(stale.error).toMatchObject({ error: { code: "revision_conflict" } }); + const targetOne = await owner.http.PATCH("/v1/workspaces/{workspaceId}", { + params: { header: mutationHeaders(2, "workspace-shared-0001"), path: { workspaceId: "ws-1" } }, body: { name: "target" }, + }); + const targetTwo = await owner.http.PATCH("/v1/workspaces/{workspaceId}", { + params: { header: mutationHeaders(1, "workspace-shared-0001"), path: { workspaceId: "ws-2" } }, body: { name: "target" }, + }); + expect(targetOne.response.status).toBe(200); + expect(targetTwo.response.status).toBe(200); + const changedPrecondition = await owner.http.PATCH("/v1/workspaces/{workspaceId}", { + params: { header: mutationHeaders(3, "workspace-shared-0001"), path: { workspaceId: "ws-1" } }, body: { name: "target" }, + }); + expect(changedPrecondition.response.status).toBe(409); + expect(changedPrecondition.error).toMatchObject({ error: { code: "idempotency_conflict" } }); expect((await owner.http.DELETE("/v1/workspaces/{workspaceId}", { params: { header: idempotencyHeaders("workspace-delete-0001"), path: { workspaceId: "ws-1" } }, })).response.status).toBe(204); @@ -214,7 +227,7 @@ describe("generated T4 API v1 client conformance", () => { expect(stale.error).toMatchObject({ error: { code: "revision_conflict" } }); for (const state of ["accepted", "rejected", "conflict", "unavailable", "indeterminate"] as const) { - const command = { command: state } satisfies CommandCreate; + const command = { command: state, metadata: {} } satisfies CommandCreate; const result = await client.http.POST("/v1/sessions/{sessionId}/commands", { params: { header: idempotencyHeaders(`command-${state}-0001`), path: { sessionId: "ses-1" } }, body: command, }); @@ -227,22 +240,34 @@ describe("generated T4 API v1 client conformance", () => { } const oversized = await client.http.POST("/v1/sessions/{sessionId}/commands", { params: { header: idempotencyHeaders("command-oversized-0001"), path: { sessionId: "ses-1" } }, - body: { command: "a".repeat(33) }, + body: { command: "a".repeat(33), metadata: {} }, }); expect(oversized.response.status).toBe(422); expect(oversized.error).toMatchObject({ error: { violations: [{ field: "command", rule: "maxBytes" }] } }); + const multibyteOversized = await client.http.POST("/v1/sessions/{sessionId}/commands", { + params: { header: idempotencyHeaders("command-multibyte-0001"), path: { sessionId: "ses-1" } }, + body: { command: "é".repeat(17), metadata: {} }, + }); + expect(multibyteOversized.response.status).toBe(422); const metadataOversized = await client.http.POST("/v1/sessions/{sessionId}/commands", { params: { header: idempotencyHeaders("command-metadata-0001"), path: { sessionId: "ses-1" } }, body: { command: "ok", metadata: { source: "é".repeat(17) } }, }); expect(metadataOversized.response.status).toBe(422); - const defaultedMetadata = await client.http.POST("/v1/sessions/{sessionId}/commands", { - params: { header: idempotencyHeaders("command-default-0001"), path: { sessionId: "ses-1" } }, body: { command: "ok" }, + const requestOversized = await client.http.POST("/v1/sessions/{sessionId}/commands", { + params: { header: idempotencyHeaders("command-request-0001"), path: { sessionId: "ses-1" } }, + body: { command: "ok", metadata: Object.fromEntries(Array.from({ length: 8 }, (_, index) => [`field-${index}`, "x".repeat(32)])) }, + }); + expect(requestOversized.response.status).toBe(422); + const defaultedMetadataResponse = await service.fetch(`${service.origin}/v1/sessions/ses-1/commands`, { + method: "POST", + headers: { Authorization: "Bearer token-a", "T4-API-Version": "1", "Idempotency-Key": "command-default-0001", "Content-Type": "application/json" }, + body: '{"command":"ok"}', }); const explicitMetadata = await client.http.POST("/v1/sessions/{sessionId}/commands", { params: { header: idempotencyHeaders("command-default-0001"), path: { sessionId: "ses-1" } }, body: { command: "ok", metadata: {} }, }); - expect(explicitMetadata.data).toEqual(defaultedMetadata.data); + expect(explicitMetadata.data).toEqual(await defaultedMetadataResponse.json()); const cancelled = await client.http.POST("/v1/sessions/{sessionId}/cancel", { params: { header: idempotencyHeaders("session-cancel-0001"), path: { sessionId: "ses-1" } }, }); @@ -309,7 +334,7 @@ describe("generated T4 API v1 client conformance", () => { ["POST", "/v1/sessions/ses-1/cancel", undefined], ["POST", "/v1/sessions/ses-1/commands", '{"command":"ok"}'], ] as const) { - const response = await service.fetch(`${service.origin}${path}`, { method, headers: { ...baseHeaders, "If-Match": "1" }, body }); + const response = await service.fetch(`${service.origin}${path}`, { method, headers: { ...baseHeaders, "If-Match": "1" }, ...(body === undefined ? {} : { body }) }); expect(response.status).toBe(400); expect(await response.json()).toMatchObject({ error: { code: "idempotency_key_required" } }); } @@ -338,8 +363,9 @@ describe("generated T4 API v1 client conformance", () => { for await (const _event of manyClient.watchSession("ses-1", { maxEvents: 1000, maxReconnectAttempts: 0 })) count += 1; expect(count).toBe(1000); - const oversized = `: ${"x".repeat(1024 * 1024)}\ndata: {"type":"heartbeat","cursor":"c1","observedAt":"2026-07-21T00:00:00Z"}\n\n`; - const oversizedClient = createT4ApiClient({ baseUrl: "https://large.test", credential: "token-a", majorVersion: 1, fetch: sse([encoder.encode(oversized)]) }); + const oversized = encoder.encode(`: ${"x".repeat(1024 * 1024)}\ndata: {"type":"heartbeat","cursor":"c1","observedAt":"2026-07-21T00:00:00Z"}\n\n`); + const oversizedChunks = Array.from({ length: Math.ceil(oversized.byteLength / 1024) }, (_, index) => oversized.slice(index * 1024, (index + 1) * 1024)); + const oversizedClient = createT4ApiClient({ baseUrl: "https://large.test", credential: "token-a", majorVersion: 1, fetch: sse(oversizedChunks) }); await expect(oversizedClient.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ code: "indeterminate", status: 502 }); }); @@ -364,6 +390,27 @@ describe("generated T4 API v1 client conformance", () => { await expect(eofClient.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 2, retryBackoffMs: 0 }).next()).rejects.toMatchObject({ code: "indeterminate", status: 502 }); expect(attempts).toBe(3); + const reconnectHeaders: Array = []; + let networkAttempt = 0; + const networkFetch: typeof globalThis.fetch = async (_input, init) => { + reconnectHeaders.push(new Headers(init?.headers).get("Last-Event-ID")); + networkAttempt += 1; + if (networkAttempt === 1) { + return new Response(new ReadableStream({ + start(stream) { + stream.enqueue(new TextEncoder().encode('data: {"type":"heartbeat","cursor":"network-1","observedAt":"2026-07-21T00:00:00Z"}\n\n')); + stream.error(new TypeError("transient network loss")); + }, + }), { headers: { "Content-Type": "text/event-stream" } }); + } + return new Response('data: {"type":"session","cursor":"network-2","state":"ready","revision":2}\n\n', { headers: { "Content-Type": "text/event-stream" } }); + }; + const networkClient = createT4ApiClient({ baseUrl: "https://network.test", credential: "token-a", majorVersion: 1, fetch: networkFetch }); + const networkEvents: WatchEvent[] = []; + for await (const event of networkClient.watchSession("ses-1", { maxEvents: 2, maxReconnectAttempts: 1, retryBackoffMs: 0 })) networkEvents.push(event); + expect(networkEvents.map((event) => event.cursor)).toEqual(["network-1", "network-2"]); + expect(reconnectHeaders).toEqual([null, "network-1"]); + const controller = new AbortController(); const abortFetch: typeof globalThis.fetch = async (_input, init) => new Response(new ReadableStream({ start(stream) { init?.signal?.addEventListener("abort", () => stream.close(), { once: true }); }, From 033e64ddb0237e3e3023f165881464bf5af61add Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 20:42:01 +0000 Subject: [PATCH 012/131] test: expose remaining API runtime blockers --- .../test/t4-api-v1-conformance-service.ts | 84 ++++++++-- .../client/test/t4-api-v1-conformance.test.ts | 152 ++++++++++++++---- 2 files changed, 192 insertions(+), 44 deletions(-) diff --git a/packages/client/test/t4-api-v1-conformance-service.ts b/packages/client/test/t4-api-v1-conformance-service.ts index 15b76db8..e0ca9763 100644 --- a/packages/client/test/t4-api-v1-conformance-service.ts +++ b/packages/client/test/t4-api-v1-conformance-service.ts @@ -40,11 +40,17 @@ interface ReplayRecord { readonly replayStatus: number; } +export interface T4ApiV1ConformanceOptions { + readonly invalidPayload?: "discovery" | "workspace" | "session" | "command"; + readonly watchTransport?: "normal" | "bytewise" | "oversized" | "many-small"; +} + export class T4ApiV1ConformanceService { readonly origin = "https://t4-api.conformance.test"; readonly calls: Array<{ method: string; path: string; authorization: string | null }> = []; readonly abortedWatches: string[] = []; readonly watchCursors: Array<{ query: string | null; header: string | null }> = []; + readonly watchQueries: Array<{ maxEvents: string | null; heartbeatSeconds: string | null }> = []; #workspaceSequence = 0; #sessionSequence = 0; @@ -53,6 +59,8 @@ export class T4ApiV1ConformanceService { readonly #sessions = new Map>(); readonly #replays = new Map(); + constructor(readonly options: T4ApiV1ConformanceOptions = {}) {} + readonly fetch: typeof globalThis.fetch = async (input, init) => { const request = new Request(input, init); const url = new URL(request.url); @@ -70,7 +78,7 @@ export class T4ApiV1ConformanceService { const tenant = authorization === "Bearer token-a" ? "tenant-a" : "tenant-b"; if (request.method === "GET" && url.pathname === "/v1") { - return json(200, { + return json(200, this.#payload("discovery", { apiVersion: "1.0", supportedMajors: [1], capabilities: ["workspace.lifecycle", "session.lifecycle", "session.commands", "session.watch.sse"], @@ -80,14 +88,16 @@ export class T4ApiV1ConformanceService { commandBytesMax: COMMAND_BYTES_MAX, commandRequestBytesMax: COMMAND_REQUEST_BYTES_MAX, commandMetadataValueBytesMax: METADATA_VALUE_BYTES_MAX, - watchEventsMax: 4, + watchEventsMax: this.options.watchTransport === "many-small" ? 1000 : 4, heartbeatSeconds: 15, }, - }); + })); } if (request.method === "POST" && url.pathname === "/v1/workspaces") { - const body = await request.json() as Record; + const parsed = await this.#jsonBody(request); + if (parsed instanceof Response) return parsed; + const body = parsed; if (typeof body.name !== "string" || body.name.length < 1 || body.name.length > 128) return this.#invalid("name", "length", "name must contain 1 to 128 characters"); return this.#idempotent(request, tenant, "createWorkspace", [], body, 202, 200, () => { const id = `ws-${++this.#workspaceSequence}`; @@ -112,9 +122,11 @@ export class T4ApiV1ConformanceService { const id = decodeURIComponent(workspaceMatch[1]!); const workspace = this.#workspaces.get(id); if (workspace?.tenant !== tenant) return problem(404, "not_found", "Workspace not found"); - if (request.method === "GET") return json(200, this.#visible(workspace)); + if (request.method === "GET") return json(200, this.#payload("workspace", this.#visible(workspace))); if (request.method === "PATCH") { - const body = await request.json() as Record; + const parsed = await this.#jsonBody(request); + if (parsed instanceof Response) return parsed; + const body = parsed; if (request.headers.get("If-Match") !== String(workspace.revision)) return problem(409, "revision_conflict", "Workspace revision changed"); return this.#idempotent(request, tenant, "mutateWorkspace", [id], body, 200, 200, () => { const updated = { ...workspace, name: body.name ?? workspace.name, revision: Number(workspace.revision) + 1 }; @@ -136,7 +148,9 @@ export class T4ApiV1ConformanceService { const workspace = this.#workspaces.get(workspaceId); if (workspace?.tenant !== tenant) return problem(404, "not_found", "Workspace not found"); if (request.method === "POST") { - const body = await request.json() as Record; + const parsed = await this.#jsonBody(request); + if (parsed instanceof Response) return parsed; + const body = parsed; if (typeof body.title !== "string" || body.title.length < 1 || body.title.length > 128) return this.#invalid("title", "length", "title must contain 1 to 128 characters"); return this.#idempotent(request, tenant, "spawnSession", [workspaceId], body, 202, 200, () => { const id = `ses-${++this.#sessionSequence}`; @@ -161,9 +175,11 @@ export class T4ApiV1ConformanceService { const id = decodeURIComponent(sessionMatch[1]!); const session = this.#sessions.get(id); if (session?.tenant !== tenant) return problem(404, "not_found", "Session not found"); - if (request.method === "GET") return json(200, this.#visible(session)); + if (request.method === "GET") return json(200, this.#payload("session", this.#visible(session))); if (request.method === "PATCH") { - const body = await request.json() as Record; + const parsed = await this.#jsonBody(request); + if (parsed instanceof Response) return parsed; + const body = parsed; if (request.headers.get("If-Match") !== String(session.revision)) return problem(409, "revision_conflict", "Session revision changed"); return this.#idempotent(request, tenant, "mutateSession", [id], body, 200, 200, () => { const updated = { ...session, title: body.title ?? session.title, revision: Number(session.revision) + 1 }; @@ -221,7 +237,8 @@ export class T4ApiV1ConformanceService { if (session?.tenant !== tenant) return problem(404, "not_found", "Session not found"); const maxEvents = Number(url.searchParams.get("maxEvents") ?? "100"); const heartbeat = Number(url.searchParams.get("heartbeatSeconds") ?? "15"); - if (!Number.isInteger(maxEvents) || maxEvents < 1 || maxEvents > 4) return this.#invalid("maxEvents", "range", "maxEvents exceeds the discovered bound"); + this.watchQueries.push({ maxEvents: url.searchParams.get("maxEvents"), heartbeatSeconds: url.searchParams.get("heartbeatSeconds") }); + if (!Number.isInteger(maxEvents) || maxEvents < 1 || maxEvents > (this.options.watchTransport === "many-small" ? 1000 : 4)) return this.#invalid("maxEvents", "range", "maxEvents exceeds the discovered bound"); if (!Number.isInteger(heartbeat) || heartbeat < 5 || heartbeat > 60) return this.#invalid("heartbeatSeconds", "range", "heartbeatSeconds must be between 5 and 60"); const queryCursor = url.searchParams.get("cursor"); const headerCursor = request.headers.get("Last-Event-ID"); @@ -237,13 +254,24 @@ export class T4ApiV1ConformanceService { `id: cursor-3\r\nevent: heartbeat\r\ndata: {"type":"heartbeat","cursor":"cursor-3","observedAt":"2026-07-21T00:00:00Z"}\r\n\r\n`, `id: cursor-4\r\nevent: session\r\ndata: {"type":"session","cursor":"cursor-4","state":"accepted","revision":2}\r\n\r\n`, ]; - const frames = allFrames.slice(0, maxEvents); + let payload: Uint8Array; + if (this.options.watchTransport === "oversized") { + payload = encoder.encode(`: ${"x".repeat(1024 * 1024)}\ndata: {"type":"heartbeat","cursor":"large-1","observedAt":"2026-07-21T00:00:00Z"}\n\n`); + } else if (this.options.watchTransport === "many-small") { + payload = encoder.encode(Array.from({ length: maxEvents }, (_, index) => `: ${"x".repeat(1010)}\ndata: {"type":"heartbeat","cursor":"bulk-${index}","observedAt":"2026-07-21T00:00:00Z"}\n\n`).join("")); + } else { + const prefix = this.options.watchTransport === "bytewise" ? ": 💚\n" : ""; + payload = encoder.encode(prefix + frames.join("")); + } const stream = new ReadableStream({ start: (controller) => { - const payload = encoder.encode(frames.join("")); - const split = Math.max(1, payload.byteLength - 3); - controller.enqueue(payload.slice(0, split)); - controller.enqueue(payload.slice(split)); + if (this.options.watchTransport === "bytewise") { + for (const byte of payload) controller.enqueue(Uint8Array.of(byte)); + } else { + const split = Math.max(1, payload.byteLength - 3); + controller.enqueue(payload.slice(0, split)); + controller.enqueue(payload.slice(split)); + } controller.close(); request.signal.addEventListener("abort", () => this.abortedWatches.push(id), { once: true }); }, @@ -258,6 +286,20 @@ export class T4ApiV1ConformanceService { expect(this.calls.every((call) => call.authorization === "Bearer token-a" || call.authorization === "Bearer token-b" || call.authorization === "Bearer token-denied")).toBe(true); } + async #jsonBody(request: Request): Promise | Response> { + try { + const value: unknown = await request.json(); + if (value === null || typeof value !== "object" || Array.isArray(value)) return problem(400, "invalid_request", "JSON request body must be an object"); + return value as Record; + } catch { + return problem(400, "invalid_request", "Malformed JSON request"); + } + } + + #payload(kind: T4ApiV1ConformanceOptions["invalidPayload"], value: Record): Record { + return this.options.invalidPayload === kind ? { ...value, unknown: true } : value; + } + #invalid(field: string, rule: string, message: string): Response { return problem(422, "invalid_request", "Request validation failed", { violations: [{ field, rule, message }] }); } @@ -302,9 +344,19 @@ export class T4ApiV1ConformanceService { : json(prior.replayStatus, prior.body, { "Idempotency-Replayed": "true" }); } const created = create(); - const visible = created !== null && typeof created === "object" && !Array.isArray(created) + const visibleValue = created !== null && typeof created === "object" && !Array.isArray(created) ? this.#visible(created as Record) : created; + const responseKind = operationId === "submitCommand" + ? "command" + : operationId === "spawnSession" || operationId === "mutateSession" || operationId === "cancelSession" + ? "session" + : operationId === "createWorkspace" || operationId === "mutateWorkspace" + ? "workspace" + : undefined; + const visible = visibleValue !== null && typeof visibleValue === "object" && !Array.isArray(visibleValue) && responseKind !== undefined + ? this.#payload(responseKind, visibleValue as Record) + : visibleValue; this.#replays.set(scope, { identity, body: visible, replayStatus }); return visible === null ? new Response(null, { status: firstStatus, headers: { "T4-API-Version": "1.0", "Idempotency-Replayed": "false" } }) diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index 5eea9590..1aacf933 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from "vite-plus/test"; +import { describe, expect, it, vi } from "vite-plus/test"; import { T4ApiError, @@ -60,6 +60,17 @@ function requireData(result: { data?: T; error?: unknown }): T { return result.data!; } +async function seededClient(service: T4ApiV1ConformanceService, key: string) { + const client = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); + requireData(await client.http.POST("/v1/workspaces", { + body: { name: "workspace" }, params: { header: idempotencyHeaders(`workspace-${key}-0001`) }, + })); + requireData(await client.http.POST("/v1/workspaces/{workspaceId}/sessions", { + body: { title: "agent" }, params: { header: idempotencyHeaders(`session-${key}-00001`), path: { workspaceId: "ws-1" } }, + })); + return client; +} + describe("generated T4 API v1 client conformance", () => { it("negotiates discovery, capabilities, bounds, and rejects an incompatible major", async () => { const service = new T4ApiV1ConformanceService(); @@ -81,7 +92,7 @@ describe("generated T4 API v1 client conformance", () => { const denied = createT4ApiClient({ baseUrl: service.origin, credential: "token-denied", majorVersion: 1, fetch: service.fetch }); const forbidden = await denied.http.GET("/v1", { params: { header: VERSION_HEADERS } }); expect(forbidden.response.status).toBe(403); - expect(forbidden.error).toEqual(typedErrors.forbidden); + expect(forbidden.error).toMatchObject({ error: { code: "forbidden" } }); expect(discovery.limits).toMatchObject({ commandBytesMax: 32, commandRequestBytesMax: 256, commandMetadataValueBytesMax: 32 }); }); @@ -131,12 +142,12 @@ describe("generated T4 API v1 client conformance", () => { })); } const pageOne = requireData(await owner.http.GET("/v1/workspaces", { - params: { header: VERSION_HEADERS, query: { pageSize: 2 } }, + params: { header: VERSION_HEADERS, query: { pageSize: 3 } }, })); - expect(pageOne.items).toHaveLength(2); - expect(pageOne.nextCursor).toBe("page-2"); + expect(pageOne.items).toHaveLength(3); + expect(pageOne.nextCursor).toBe("page-3"); const pageTwo = requireData(await owner.http.GET("/v1/workspaces", { - params: { header: VERSION_HEADERS, query: { pageSize: 2, cursor: pageOne.nextCursor! } }, + params: { header: VERSION_HEADERS, query: { pageSize: 3, cursor: pageOne.nextCursor! } }, })); expect(pageTwo.items).toHaveLength(2); expect(pageTwo.nextCursor).toBeUndefined(); @@ -290,7 +301,7 @@ describe("generated T4 API v1 client conformance", () => { expect(snapshot).toMatchObject({ sessionId: "ses-1", cursor: "cursor-2", entries: [{ sequence: 2 }] }); const received: Array = []; - for await (const event of client.watchSession("ses-1", { cursor: snapshot.cursor, maxEvents: 3, maxReconnectAttempts: 2, retryBackoffMs: 0 })) { + for await (const event of client.watchSession("ses-1", { cursor: snapshot.cursor, maxEvents: 3, heartbeatSeconds: 20, maxReconnectAttempts: 2, retryBackoffMs: 0 })) { received.push(event); } expect(received).toEqual([ @@ -301,6 +312,10 @@ describe("generated T4 API v1 client conformance", () => { expect(received.map(exhaustWatchEvent)).toEqual(["2026-07-21T00:00:00Z", "accepted:2", "2026-07-21T00:00:15Z"]); expect(service.watchCursors[0]).toEqual({ query: "cursor-2", header: "cursor-2" }); expect(service.watchCursors[1]).toEqual({ query: "cursor-4", header: "cursor-4" }); + expect(service.watchQueries.slice(0, 2)).toEqual([ + { maxEvents: "3", heartbeatSeconds: "20" }, + { maxEvents: "1", heartbeatSeconds: "20" }, + ]); await expect(async () => { for await (const _event of client.watchSession("ses-1", { cursor: "expired", maxEvents: 1 })) { @@ -321,7 +336,7 @@ describe("generated T4 API v1 client conformance", () => { expect(created.status).toBe(202); const missingSpawn = await service.fetch(`${service.origin}/v1/workspaces/ws-1/sessions`, { method: "POST", headers: baseHeaders, body: '{"title":"agent"}' }); expect(missingSpawn.status).toBe(400); - expect(await missingSpawn.json()).toMatchObject(typedErrors.badSpawn); + expect(await missingSpawn.json()).toMatchObject({ error: { code: "idempotency_key_required" } }); const invalidSpawn = await service.fetch(`${service.origin}/v1/workspaces/ws-1/sessions`, { method: "POST", headers: { ...baseHeaders, "Idempotency-Key": "short" }, body: '{"title":"agent"}' }); expect(invalidSpawn.status).toBe(400); const spawned = await service.fetch(`${service.origin}/v1/workspaces/ws-1/sessions`, { method: "POST", headers: { ...baseHeaders, "Idempotency-Key": "session-raw-00001" }, body: '{"title":"agent"}' }); @@ -338,6 +353,16 @@ describe("generated T4 API v1 client conformance", () => { expect(response.status).toBe(400); expect(await response.json()).toMatchObject({ error: { code: "idempotency_key_required" } }); } + for (const [method, path, headers] of [ + ["POST", "/v1/workspaces", { ...baseHeaders, "Idempotency-Key": "malformed-workspace-01" }], + ["PATCH", "/v1/workspaces/ws-1", { ...baseHeaders, "If-Match": "1", "Idempotency-Key": "malformed-workspace-02" }], + ["POST", "/v1/workspaces/ws-1/sessions", { ...baseHeaders, "Idempotency-Key": "malformed-session-0001" }], + ["PATCH", "/v1/sessions/ses-1", { ...baseHeaders, "If-Match": "1", "Idempotency-Key": "malformed-session-0002" }], + ] as const) { + const response = await service.fetch(`${service.origin}${path}`, { method, headers, body: "{" }); + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ error: { code: "invalid_request" } }); + } for (const query of ["maxEvents=0", "maxEvents=5", "heartbeatSeconds=4", "heartbeatSeconds=61"]) { const response = await service.fetch(`${service.origin}/v1/sessions/ses-1/events?${query}`, { headers: baseHeaders }); expect(response.status).toBe(422); @@ -345,27 +370,21 @@ describe("generated T4 API v1 client conformance", () => { } }); - it("parses per-frame byte bounds across arbitrary chunks and split UTF-8", async () => { - const sse = (chunks: Uint8Array[]): typeof globalThis.fetch => async () => new Response(new ReadableStream({ - start(controller) { for (const chunk of chunks) controller.enqueue(chunk); controller.close(); }, - }), { status: 200, headers: { "Content-Type": "text/event-stream" } }); - const encoder = new TextEncoder(); - const valid = `: 💚\nid: c1\ndata: {"type":"heartbeat","cursor":"c1","observedAt":"2026-07-21T00:00:00Z"}\n\n`; - const bytes = encoder.encode(valid); - const split = [...bytes].map((byte) => Uint8Array.of(byte)); - const splitClient = createT4ApiClient({ baseUrl: "https://split.test", credential: "token-a", majorVersion: 1, fetch: sse(split) }); - const iterator = splitClient.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }); - expect((await iterator.next()).value).toMatchObject({ type: "heartbeat", cursor: "c1" }); - - const small = `data: {"type":"heartbeat","cursor":"c1","observedAt":"2026-07-21T00:00:00Z"}\n${`: ${"x".repeat(1010)}\n`}\n`; - const manyClient = createT4ApiClient({ baseUrl: "https://many.test", credential: "token-a", majorVersion: 1, fetch: sse([encoder.encode(small.repeat(1000))]) }); + it("parses service frames incrementally across split UTF-8 and per-frame bounds", async () => { + const bytewiseService = new T4ApiV1ConformanceService({ watchTransport: "bytewise" }); + const bytewiseClient = await seededClient(bytewiseService, "bytewise"); + const event = await bytewiseClient.watchSession("ses-1", { maxEvents: 1, heartbeatSeconds: 20, maxReconnectAttempts: 0 }).next(); + expect(event.value).toMatchObject({ type: "heartbeat", cursor: "cursor-3" }); + expect(bytewiseService.watchQueries[0]).toEqual({ maxEvents: "1", heartbeatSeconds: "20" }); + + const manyService = new T4ApiV1ConformanceService({ watchTransport: "many-small" }); + const manyClient = await seededClient(manyService, "many-small"); let count = 0; for await (const _event of manyClient.watchSession("ses-1", { maxEvents: 1000, maxReconnectAttempts: 0 })) count += 1; expect(count).toBe(1000); - const oversized = encoder.encode(`: ${"x".repeat(1024 * 1024)}\ndata: {"type":"heartbeat","cursor":"c1","observedAt":"2026-07-21T00:00:00Z"}\n\n`); - const oversizedChunks = Array.from({ length: Math.ceil(oversized.byteLength / 1024) }, (_, index) => oversized.slice(index * 1024, (index + 1) * 1024)); - const oversizedClient = createT4ApiClient({ baseUrl: "https://large.test", credential: "token-a", majorVersion: 1, fetch: sse(oversizedChunks) }); + const oversizedService = new T4ApiV1ConformanceService({ watchTransport: "oversized" }); + const oversizedClient = await seededClient(oversizedService, "oversized"); await expect(oversizedClient.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ code: "indeterminate", status: 502 }); }); @@ -378,6 +397,38 @@ describe("generated T4 API v1 client conformance", () => { const client = createT4ApiClient({ baseUrl: "https://errors.test", credential: "token-a", majorVersion: 1, fetch }); await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ code: "indeterminate", status }); } + for (const [status, code] of [[401, "not_found"], [403, "unauthenticated"], [404, "forbidden"], [409, "unavailable"], [503, "revision_conflict"]] as const) { + const fetch: typeof globalThis.fetch = async () => Response.json({ error: { code, message: "wrong status class", requestId: "r", retryable: false } }, { status }); + const client = createT4ApiClient({ baseUrl: "https://status-errors.test", credential: "token-a", majorVersion: 1, fetch }); + await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ code: "indeterminate", status }); + } + }); + + it("rejects unknown fields in every public JSON response family", async () => { + const discoveryService = new T4ApiV1ConformanceService({ invalidPayload: "discovery" }); + const discoveryClient = createT4ApiClient({ baseUrl: discoveryService.origin, credential: "token-a", majorVersion: 1, fetch: discoveryService.fetch }); + await expect(discoveryClient.http.GET("/v1", { params: { header: VERSION_HEADERS } })).rejects.toMatchObject({ code: "indeterminate", status: 502 }); + + const workspaceService = new T4ApiV1ConformanceService({ invalidPayload: "workspace" }); + const workspaceClient = createT4ApiClient({ baseUrl: workspaceService.origin, credential: "token-a", majorVersion: 1, fetch: workspaceService.fetch }); + await expect(workspaceClient.http.POST("/v1/workspaces", { + body: { name: "workspace" }, params: { header: idempotencyHeaders("invalid-workspace-0001") }, + })).rejects.toMatchObject({ code: "indeterminate", status: 502 }); + + const sessionService = new T4ApiV1ConformanceService({ invalidPayload: "session" }); + const sessionClient = createT4ApiClient({ baseUrl: sessionService.origin, credential: "token-a", majorVersion: 1, fetch: sessionService.fetch }); + requireData(await sessionClient.http.POST("/v1/workspaces", { + body: { name: "workspace" }, params: { header: idempotencyHeaders("invalid-session-work-01") }, + })); + await expect(sessionClient.http.POST("/v1/workspaces/{workspaceId}/sessions", { + body: { title: "agent" }, params: { header: idempotencyHeaders("invalid-session-00001"), path: { workspaceId: "ws-1" } }, + })).rejects.toMatchObject({ code: "indeterminate", status: 502 }); + + const commandService = new T4ApiV1ConformanceService({ invalidPayload: "command" }); + const commandClient = await seededClient(commandService, "invalid-command"); + await expect(commandClient.http.POST("/v1/sessions/{sessionId}/commands", { + body: { command: "ok", metadata: {} }, params: { header: idempotencyHeaders("invalid-command-0001"), path: { sessionId: "ses-1" } }, + })).rejects.toMatchObject({ code: "indeterminate", status: 502 }); }); it("bounds automatic EOF retry and supports explicit abort", async () => { @@ -396,10 +447,14 @@ describe("generated T4 API v1 client conformance", () => { reconnectHeaders.push(new Headers(init?.headers).get("Last-Event-ID")); networkAttempt += 1; if (networkAttempt === 1) { + let sent = false; return new Response(new ReadableStream({ - start(stream) { - stream.enqueue(new TextEncoder().encode('data: {"type":"heartbeat","cursor":"network-1","observedAt":"2026-07-21T00:00:00Z"}\n\n')); - stream.error(new TypeError("transient network loss")); + pull(stream) { + if (sent) stream.error(new TypeError("transient network loss")); + else { + sent = true; + stream.enqueue(new TextEncoder().encode('data: {"type":"heartbeat","cursor":"network-1","observedAt":"2026-07-21T00:00:00Z"}\n\n')); + } }, }), { headers: { "Content-Type": "text/event-stream" } }); } @@ -411,6 +466,47 @@ describe("generated T4 API v1 client conformance", () => { expect(networkEvents.map((event) => event.cursor)).toEqual(["network-1", "network-2"]); expect(reconnectHeaders).toEqual([null, "network-1"]); + let retryableAttempts = 0; + const retryableFetch: typeof globalThis.fetch = async () => { + retryableAttempts += 1; + if (retryableAttempts === 1) return Response.json({ error: { code: "unavailable", message: "retry", requestId: "r", retryable: true } }, { status: 503, headers: { "Retry-After": "0" } }); + return new Response('data: {"type":"heartbeat","cursor":"retry-1","observedAt":"2026-07-21T00:00:00Z"}\n\n', { headers: { "Content-Type": "text/event-stream" } }); + }; + const retryableClient = createT4ApiClient({ baseUrl: "https://retryable.test", credential: "token-a", majorVersion: 1, fetch: retryableFetch }); + expect((await retryableClient.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 1, retryBackoffMs: 0 }).next()).value).toMatchObject({ cursor: "retry-1" }); + expect(retryableAttempts).toBe(2); + + let progressAttempts = 0; + const progressFetch: typeof globalThis.fetch = async () => { + progressAttempts += 1; + if (progressAttempts === 1 || progressAttempts === 3) return new Response(null, { headers: { "Content-Type": "text/event-stream" } }); + const cursor = progressAttempts === 2 ? "progress-1" : "progress-2"; + return new Response(`data: {"type":"heartbeat","cursor":"${cursor}","observedAt":"2026-07-21T00:00:00Z"}\n\n`, { headers: { "Content-Type": "text/event-stream" } }); + }; + const progressClient = createT4ApiClient({ baseUrl: "https://progress.test", credential: "token-a", majorVersion: 1, fetch: progressFetch }); + const progressEvents: WatchEvent[] = []; + for await (const item of progressClient.watchSession("ses-1", { maxEvents: 2, maxReconnectAttempts: 1, retryBackoffMs: 0 })) progressEvents.push(item); + expect(progressEvents.map((item) => item.cursor)).toEqual(["progress-1", "progress-2"]); + + vi.useFakeTimers(); + try { + const retryAfterController = new AbortController(); + let retryAfterAttempts = 0; + const retryAfterFetch: typeof globalThis.fetch = async () => { + retryAfterAttempts += 1; + return Response.json({ error: { code: "unavailable", message: "later", requestId: "r", retryable: true } }, { status: 503, headers: { "Retry-After": "30" } }); + }; + const retryAfterClient = createT4ApiClient({ baseUrl: "https://retry-after.test", credential: "token-a", majorVersion: 1, fetch: retryAfterFetch }); + const retryAfterPending = retryAfterClient.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 2, retryBackoffMs: 0, signal: retryAfterController.signal }).next(); + const retryAfterAssertion = expect(retryAfterPending).resolves.toMatchObject({ done: true }); + await vi.advanceTimersByTimeAsync(0); + expect(retryAfterAttempts).toBe(1); + retryAfterController.abort(); + await retryAfterAssertion; + } finally { + vi.useRealTimers(); + } + const controller = new AbortController(); const abortFetch: typeof globalThis.fetch = async (_input, init) => new Response(new ReadableStream({ start(stream) { init?.signal?.addEventListener("abort", () => stream.close(), { once: true }); }, From 788516db414eca671e9419f44988ce42df8e139f Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 20:46:34 +0000 Subject: [PATCH 013/131] fix: fail closed across public API payloads --- .github/workflows/ci.yml | 9 ++ .../test/t4-api-v1-conformance-service.ts | 9 +- .../client/test/t4-api-v1-conformance.test.ts | 12 ++ packages/t4-api-client/src/index.ts | 120 +++++++++++++++++- packages/t4-api-contract/openapi.json | 6 +- packages/t4-api-contract/scripts/validate.mjs | 12 ++ 6 files changed, 154 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 25d7d947..17f3bee0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,10 +82,19 @@ jobs: - name: Validate OpenAPI and deterministic generation run: | pnpm --filter @t4-code/t4-api-contract validate + pnpm --filter @t4-code/t4-api-contract generate:ci pnpm --filter @t4-code/t4-api-contract check:generated pnpm --filter @t4-code/t4-api-client typecheck pnpm --filter @t4-code/client typecheck pnpm --filter @t4-code/client exec vp test run test/t4-api-v1-conformance.test.ts + - name: Upload authorized generated schema + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: t4-api-generated-${{ github.run_id }} + path: artifacts/t4-api/generated/schema.ts + if-no-files-found: error + retention-days: 1 core: diff --git a/packages/client/test/t4-api-v1-conformance-service.ts b/packages/client/test/t4-api-v1-conformance-service.ts index e0ca9763..1e8026a6 100644 --- a/packages/client/test/t4-api-v1-conformance-service.ts +++ b/packages/client/test/t4-api-v1-conformance-service.ts @@ -127,12 +127,11 @@ export class T4ApiV1ConformanceService { const parsed = await this.#jsonBody(request); if (parsed instanceof Response) return parsed; const body = parsed; - if (request.headers.get("If-Match") !== String(workspace.revision)) return problem(409, "revision_conflict", "Workspace revision changed"); return this.#idempotent(request, tenant, "mutateWorkspace", [id], body, 200, 200, () => { const updated = { ...workspace, name: body.name ?? workspace.name, revision: Number(workspace.revision) + 1 }; this.#workspaces.set(id, updated); return updated; - }); + }, () => request.headers.get("If-Match") === String(workspace.revision) ? undefined : problem(409, "revision_conflict", "Workspace revision changed")); } if (request.method === "DELETE") { return this.#idempotent(request, tenant, "deleteWorkspace", [id], null, 204, 204, () => { @@ -180,12 +179,11 @@ export class T4ApiV1ConformanceService { const parsed = await this.#jsonBody(request); if (parsed instanceof Response) return parsed; const body = parsed; - if (request.headers.get("If-Match") !== String(session.revision)) return problem(409, "revision_conflict", "Session revision changed"); return this.#idempotent(request, tenant, "mutateSession", [id], body, 200, 200, () => { const updated = { ...session, title: body.title ?? session.title, revision: Number(session.revision) + 1 }; this.#sessions.set(id, updated); return updated; - }); + }, () => request.headers.get("If-Match") === String(session.revision) ? undefined : problem(409, "revision_conflict", "Session revision changed")); } if (request.method === "DELETE") { return this.#idempotent(request, tenant, "deleteSession", [id], null, 204, 204, () => { @@ -329,6 +327,7 @@ export class T4ApiV1ConformanceService { firstStatus: number, replayStatus: number, create: () => unknown, + validate?: () => Response | undefined, ): Response { const key = request.headers.get("Idempotency-Key"); if (key === null) return problem(400, "idempotency_key_required", "Idempotency-Key is required"); @@ -343,6 +342,8 @@ export class T4ApiV1ConformanceService { ? new Response(null, { status: prior.replayStatus, headers: { "T4-API-Version": "1.0", "Idempotency-Replayed": "true" } }) : json(prior.replayStatus, prior.body, { "Idempotency-Replayed": "true" }); } + const validation = validate?.(); + if (validation !== undefined) return validation; const created = create(); const visibleValue = created !== null && typeof created === "object" && !Array.isArray(created) ? this.#visible(created as Record) diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index 1aacf933..e427da1b 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -169,6 +169,12 @@ describe("generated T4 API v1 client conformance", () => { body: { name: "renamed" }, })); expect(updated).toMatchObject({ name: "renamed", revision: 2 }); + const updatedReplay = await owner.http.PATCH("/v1/workspaces/{workspaceId}", { + params: { header: mutationHeaders(1, "workspace-patch-0001"), path: { workspaceId: "ws-1" } }, body: { name: "renamed" }, + }); + expect(updatedReplay.response.status).toBe(200); + expect(updatedReplay.response.headers.get("Idempotency-Replayed")).toBe("true"); + expect(updatedReplay.data).toEqual(updated); const stale = await owner.http.PATCH("/v1/workspaces/{workspaceId}", { params: { header: mutationHeaders(1, "workspace-patch-0002"), path: { workspaceId: "ws-1" } }, body: { name: "stale" }, @@ -231,6 +237,12 @@ describe("generated T4 API v1 client conformance", () => { params: { header: mutationHeaders(1, "session-patch-0001"), path: { sessionId: "ses-1" } }, body: { title: "renamed-agent" }, })); expect(mutated).toMatchObject({ title: "renamed-agent", revision: 2 }); + const mutatedReplay = await client.http.PATCH("/v1/sessions/{sessionId}", { + params: { header: mutationHeaders(1, "session-patch-0001"), path: { sessionId: "ses-1" } }, body: { title: "renamed-agent" }, + }); + expect(mutatedReplay.response.status).toBe(200); + expect(mutatedReplay.response.headers.get("Idempotency-Replayed")).toBe("true"); + expect(mutatedReplay.data).toEqual(mutated); const stale = await client.http.PATCH("/v1/sessions/{sessionId}", { params: { header: mutationHeaders(1, "session-patch-0002"), path: { sessionId: "ses-1" } }, body: { title: "stale-agent" }, }); diff --git a/packages/t4-api-client/src/index.ts b/packages/t4-api-client/src/index.ts index 67d5ed34..8b4e1559 100644 --- a/packages/t4-api-client/src/index.ts +++ b/packages/t4-api-client/src/index.ts @@ -7,6 +7,7 @@ export type { components, operations, paths } from "./generated/schema.ts"; const MAX_CREDENTIAL_LENGTH = 4096; const MAX_ERROR_BYTES = 1024 * 1024; const MAX_EVENT_BYTES = 1024 * 1024; +const MAX_JSON_RESPONSE_BYTES = 16 * 1024 * 1024; const CURSOR_PATTERN = /^[A-Za-z0-9._~-]+$/u; const ERROR_CODES = { invalid_request: true, unauthenticated: true, forbidden: true, not_found: true, @@ -29,6 +30,10 @@ const ERROR_CODES_BY_STATUS: Readonly; const SESSION_STATES = { accepted: true, provisioning: true, ready: true, cancelling: true, cancelled: true, failed: true, unavailable: true, indeterminate: true, @@ -70,8 +75,9 @@ export class T4ApiError extends Error { readonly violations?: ApiError["violations"]; readonly supportedMajors?: ApiError["supportedMajors"]; readonly resync?: Resync; + readonly retryAfterMs?: number; - constructor(status: number, error: ApiError, options?: ErrorOptions) { + constructor(status: number, error: ApiError, options?: ErrorOptions & { readonly retryAfterMs?: number }) { super(error.message, options); this.name = "T4ApiError"; this.code = error.code; @@ -81,6 +87,7 @@ export class T4ApiError extends Error { if (error.violations !== undefined) this.violations = error.violations; if (error.supportedMajors !== undefined) this.supportedMajors = error.supportedMajors; if (error.resync !== undefined) this.resync = error.resync; + if (options?.retryAfterMs !== undefined) this.retryAfterMs = options.retryAfterMs; } } @@ -180,7 +187,7 @@ function apiError(value: unknown, status: number): ApiError | undefined { return error as ApiError; } -async function boundedResponseText(response: Response): Promise { +async function boundedResponseText(response: Response, maximumBytes = MAX_ERROR_BYTES): Promise { if (response.body === null) return ""; const reader = response.body.getReader(); const chunks: Uint8Array[] = []; @@ -190,7 +197,7 @@ async function boundedResponseText(response: Response): Promise MAX_ERROR_BYTES) { + if (length > maximumBytes) { await reader.cancel(); return undefined; } @@ -208,12 +215,26 @@ async function boundedResponseText(response: Response): Promise { const text = await boundedResponseText(response); + const retryAfterMs = retryAfterMilliseconds(response); if (text !== undefined) { try { const decoded = apiError(JSON.parse(text), response.status); - if (decoded !== undefined) return new T4ApiError(response.status, decoded); + if (decoded !== undefined) { + return retryAfterMs === undefined + ? new T4ApiError(response.status, decoded) + : new T4ApiError(response.status, decoded, { retryAfterMs }); + } } catch { // Fall through to the stable indeterminate transport envelope. } @@ -230,6 +251,83 @@ function hasOnlyKeys(value: Record, keys: Readonly keys[key] === true); } +function validResourceId(value: unknown): value is string { + return typeof value === "string" && value.length >= 1 && value.length <= 128 && /^[A-Za-z0-9][A-Za-z0-9._~-]*$/u.test(value); +} + +function validCursor(value: unknown): value is string { + return typeof value === "string" && value.length >= 1 && value.length <= 512 && CURSOR_PATTERN.test(value); +} + +function validLabels(value: unknown): boolean { + if (value === undefined) return true; + const labels = record(value); + return labels !== undefined && Object.keys(labels).length <= 32 && Object.entries(labels).every(([key, item]) => + /^[a-z][a-z0-9.-]{0,62}$/u.test(key) && typeof item === "string" && item.length <= 128); +} + +function validWorkspace(value: unknown): boolean { + const item = record(value); + return item !== undefined && hasOnlyKeys(item, { id: true, name: true, state: true, revision: true, labels: true }) && + validResourceId(item.id) && typeof item.name === "string" && item.name.length >= 1 && item.name.length <= 128 && + typeof item.state === "string" && WORKSPACE_STATES[item.state as components["schemas"]["WorkspaceState"]] === true && + Number.isSafeInteger(item.revision) && Number(item.revision) >= 1 && validLabels(item.labels); +} + +function validSession(value: unknown): boolean { + const item = record(value); + return item !== undefined && hasOnlyKeys(item, { id: true, workspaceId: true, title: true, state: true, revision: true, labels: true }) && + validResourceId(item.id) && validResourceId(item.workspaceId) && typeof item.title === "string" && item.title.length >= 1 && item.title.length <= 128 && + typeof item.state === "string" && SESSION_STATES[item.state as components["schemas"]["SessionState"]] === true && + Number.isSafeInteger(item.revision) && Number(item.revision) >= 1 && validLabels(item.labels); +} + +function validCommandResult(value: unknown): boolean { + const item = record(value); + return item !== undefined && hasOnlyKeys(item, { commandId: true, state: true }) && validResourceId(item.commandId) && + typeof item.state === "string" && OPERATION_STATES[item.state as components["schemas"]["OperationState"]] === true; +} + +function validDiscovery(value: unknown): boolean { + const discovery = record(value); + const limits = record(discovery?.limits); + return discovery !== undefined && hasOnlyKeys(discovery, { apiVersion: true, supportedMajors: true, capabilities: true, limits: true }) && + typeof discovery.apiVersion === "string" && /^1\.[0-9]+$/u.test(discovery.apiVersion) && + Array.isArray(discovery.supportedMajors) && discovery.supportedMajors.length >= 1 && discovery.supportedMajors.length <= 8 && discovery.supportedMajors.every((major) => Number.isSafeInteger(major) && Number(major) >= 1) && + Array.isArray(discovery.capabilities) && discovery.capabilities.length <= 128 && discovery.capabilities.every((capability) => typeof capability === "string" && /^[a-z][a-z0-9.-]{0,127}$/u.test(capability)) && + limits !== undefined && hasOnlyKeys(limits, { pageSizeDefault: true, pageSizeMax: true, commandBytesMax: true, commandRequestBytesMax: true, commandMetadataValueBytesMax: true, watchEventsMax: true, heartbeatSeconds: true }) && + [limits.pageSizeDefault, limits.pageSizeMax, limits.commandBytesMax, limits.commandRequestBytesMax, limits.commandMetadataValueBytesMax, limits.watchEventsMax, limits.heartbeatSeconds].every((limit) => Number.isSafeInteger(limit) && Number(limit) >= 1) && + Number(limits.pageSizeDefault) <= Number(limits.pageSizeMax) && Number(limits.commandBytesMax) <= Number(limits.commandRequestBytesMax) && Number(limits.commandMetadataValueBytesMax) <= Number(limits.commandRequestBytesMax) && + Number(limits.pageSizeMax) <= 100 && Number(limits.commandBytesMax) <= 262144 && Number(limits.commandRequestBytesMax) <= 1048576 && Number(limits.commandMetadataValueBytesMax) <= 262144 && Number(limits.watchEventsMax) <= 1000 && Number(limits.heartbeatSeconds) >= 5 && Number(limits.heartbeatSeconds) <= 60; +} + +function validSuccessPayload(request: Request, value: unknown): boolean { + const path = new URL(request.url).pathname; + if (request.method === "GET" && path === "/v1") return validDiscovery(value); + if (/^\/v1\/workspaces\/[A-Za-z0-9._~-]+\/sessions$/u.test(path) && request.method === "GET") { + const page = record(value); + return page !== undefined && hasOnlyKeys(page, { items: true, nextCursor: true }) && Array.isArray(page.items) && page.items.length <= 100 && page.items.every(validSession) && (page.nextCursor === undefined || validCursor(page.nextCursor)); + } + if (path === "/v1/workspaces" && request.method === "GET") { + const page = record(value); + return page !== undefined && hasOnlyKeys(page, { items: true, nextCursor: true }) && Array.isArray(page.items) && page.items.length <= 100 && page.items.every(validWorkspace) && (page.nextCursor === undefined || validCursor(page.nextCursor)); + } + if (path === "/v1/workspaces" || /^\/v1\/workspaces\/[A-Za-z0-9._~-]+$/u.test(path)) return validWorkspace(value); + if (/^\/v1\/workspaces\/[A-Za-z0-9._~-]+\/sessions$/u.test(path) || /^\/v1\/sessions\/[A-Za-z0-9._~-]+(?:\/cancel)?$/u.test(path)) return validSession(value); + if (/^\/v1\/sessions\/[A-Za-z0-9._~-]+\/commands$/u.test(path)) return validCommandResult(value); + return true; +} + +async function validateJsonSuccess(request: Request, response: Response): Promise { + if (!response.ok || response.status === 204 || !response.headers.get("content-type")?.toLowerCase().startsWith("application/json")) return; + const text = await boundedResponseText(response.clone(), MAX_JSON_RESPONSE_BYTES); + let value: unknown; + try { value = text === undefined ? undefined : JSON.parse(text); } catch { value = undefined; } + if (text === undefined || !validSuccessPayload(request, value)) { + throw new T4ApiError(502, { code: "indeterminate", message: "T4 API returned an invalid or oversized JSON response", requestId: "unavailable", retryable: true }); + } +} + function watchEvent(value: unknown, eventId: string | undefined): WatchEvent { const event = record(value); if ( @@ -412,6 +510,7 @@ async function* watch( while (delivered < maxEvents && !controller.signal.aborted) { let reader: ReadableStreamDefaultReader | undefined; let transientFailure: unknown; + let retryAfterMs = 0; try { const url = new URL(`${baseUrl}/v1/sessions/${encodeURIComponent(sessionId)}/events`); url.searchParams.set("maxEvents", String(maxEvents - delivered)); @@ -438,6 +537,7 @@ async function* watch( for (const event of events) { cursor = event.cursor; delivered += 1; + reconnectAttempts = 0; yield event; if (delivered >= maxEvents || controller.signal.aborted) break; } @@ -447,7 +547,11 @@ async function* watch( transientFailure = new TypeError("T4 API watch ended before the requested event bound"); } catch (error) { if (controller.signal.aborted) return; - if (error instanceof T4ApiError) throw error; + if (error instanceof T4ApiError) { + const reconnectable = error.retryable && error.status === 503 && (error.code === "unavailable" || error.code === "indeterminate"); + if (!reconnectable) throw error; + retryAfterMs = error.retryAfterMs ?? 0; + } transientFailure = error; } finally { if (reader !== undefined) { @@ -458,7 +562,7 @@ async function* watch( if (reconnectAttempts >= maxReconnectAttempts) { throw new T4ApiError(502, { code: "indeterminate", message: "T4 API watch reconnect attempts exhausted", requestId: "unavailable", retryable: true }, { cause: transientFailure }); } - const delay = Math.min(30_000, retryBackoffMs * (2 ** reconnectAttempts)); + const delay = Math.min(30_000, Math.max(retryAfterMs, retryBackoffMs * (2 ** reconnectAttempts))); reconnectAttempts += 1; if (!await retryDelay(delay, controller.signal)) return; } @@ -478,7 +582,9 @@ export function createT4ApiClient(options: T4ApiClientOptions): T4ApiClient { request.headers.set("Authorization", `Bearer ${credential}`); request.headers.set("T4-API-Version", majorVersion); request.headers.set("Accept", "application/json"); - return await fetchImpl(request); + const response = await fetchImpl(request); + await validateJsonSuccess(request, response); + return response; }; const http = createClient({ baseUrl, fetch: authenticatedFetch }); return Object.freeze({ diff --git a/packages/t4-api-contract/openapi.json b/packages/t4-api-contract/openapi.json index 2884c6f8..c883eab1 100644 --- a/packages/t4-api-contract/openapi.json +++ b/packages/t4-api-contract/openapi.json @@ -317,14 +317,14 @@ ], "responses": { "200": { - "description": "Bounded SSE event stream; each data field is a WatchEvent JSON value", + "description": "Bounded SSE event stream. Every non-empty SSE data field is one JSON value conforming to WatchEvent; clients MUST reject unknown fields and schema-invalid event payloads before delivery. Transport chunk boundaries do not delimit frames.", "headers": { "Cache-Control": { "schema": { "type": "string", "const": "no-store" } }, "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "text/event-stream": { - "schema": { "type": "string" }, + "schema": { "type": "string", "description": "Server-Sent Events framing whose data values conform to #/components/schemas/WatchEvent", "x-t4-sse-data-schema": "#/components/schemas/WatchEvent" }, "example": "id: cur_01J...\\nevent: heartbeat\\ndata: {\"type\":\"heartbeat\",\"cursor\":\"cur_01J...\",\"observedAt\":\"2026-07-21T00:00:00Z\"}\\n\\n" } } @@ -808,7 +808,7 @@ "Error409": { "description": "Idempotency or resource revision conflict", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ConflictErrorEnvelope" } } } }, "Error410": { "description": "Watch cursor expired; resync from the typed snapshot target", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CursorExpiredErrorEnvelope" } } } }, "Error422": { "description": "Bounded semantic validation failure with stable field violations", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/InvalidRequestErrorEnvelope" } } } }, - "Error503": { "description": "Service temporarily unavailable or outcome indeterminate", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/UnavailableErrorEnvelope" } } } } + "Error503": { "description": "Service temporarily unavailable or outcome indeterminate. Retryable watch failures may advertise Retry-After; clients honor it with a 30 second ceiling.", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Retry-After": { "description": "RFC 9110 delay in seconds or HTTP date; clients bound the applied delay.", "schema": { "type": "string", "minLength": 1, "maxLength": 128 } } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/UnavailableErrorEnvelope" } } } } } } } diff --git a/packages/t4-api-contract/scripts/validate.mjs b/packages/t4-api-contract/scripts/validate.mjs index 40a5e4d6..cbfafbc7 100644 --- a/packages/t4-api-contract/scripts/validate.mjs +++ b/packages/t4-api-contract/scripts/validate.mjs @@ -14,6 +14,18 @@ for (const server of document.servers) { if (new URL(server.url).protocol !== "https:") throw new Error("T4 API contract permits only HTTPS servers"); } if (document.security?.[0]?.BearerAuth === undefined) throw new Error("T4 API contract must default to bearer authentication"); +const sseSchema = document.paths?.["/v1/sessions/{sessionId}/events"]?.get?.responses?.["200"]?.content?.["text/event-stream"]?.schema; +if (sseSchema?.["x-t4-sse-data-schema"] !== "#/components/schemas/WatchEvent") throw new Error("watch SSE data must remain linked to WatchEvent"); +const errorResponses = { 400: "Error400", 401: "Error401", 403: "Error403", 404: "Error404", 406: "Error406", 409: "Error409", 410: "Error410", 422: "Error422", 503: "Error503" }; +for (const path of Object.values(document.paths ?? {})) { + for (const operation of Object.values(path)) { + if (operation === null || typeof operation !== "object" || operation.responses === undefined) continue; + for (const [status, component] of Object.entries(errorResponses)) { + const response = operation.responses[status]; + if (response !== undefined && response.$ref !== `#/components/responses/${component}`) throw new Error(`${status} responses must use ${component}`); + } + } +} const serializedSchemas = JSON.stringify(document.components?.schemas ?? {}).toLowerCase(); for (const privateType of ["kubernetes.io", "v1alpha1", "postgresql", "ompserver", "podspec"]) { if (serializedSchemas.includes(privateType)) throw new Error(`public schema leaks private type ${privateType}`); From b0a744513940f63f5b145d959b31f7f61a9bfab8 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 20:48:31 +0000 Subject: [PATCH 014/131] build: accept revised T4 API schema artifact --- .github/workflows/ci.yml | 9 --------- packages/t4-api-client/src/generated/schema.ts | 6 ++++-- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 17f3bee0..25d7d947 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,19 +82,10 @@ jobs: - name: Validate OpenAPI and deterministic generation run: | pnpm --filter @t4-code/t4-api-contract validate - pnpm --filter @t4-code/t4-api-contract generate:ci pnpm --filter @t4-code/t4-api-contract check:generated pnpm --filter @t4-code/t4-api-client typecheck pnpm --filter @t4-code/client typecheck pnpm --filter @t4-code/client exec vp test run test/t4-api-v1-conformance.test.ts - - name: Upload authorized generated schema - if: ${{ always() }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: t4-api-generated-${{ github.run_id }} - path: artifacts/t4-api/generated/schema.ts - if-no-files-found: error - retention-days: 1 core: diff --git a/packages/t4-api-client/src/generated/schema.ts b/packages/t4-api-client/src/generated/schema.ts index 0e4b33ec..1291c013 100755 --- a/packages/t4-api-client/src/generated/schema.ts +++ b/packages/t4-api-client/src/generated/schema.ts @@ -479,9 +479,11 @@ export interface components { "application/json": components["schemas"]["InvalidRequestErrorEnvelope"]; }; }; - /** @description Service temporarily unavailable or outcome indeterminate */ + /** @description Service temporarily unavailable or outcome indeterminate. Retryable watch failures may advertise Retry-After; clients honor it with a 30 second ceiling. */ Error503: { headers: { + /** @description RFC 9110 delay in seconds or HTTP date; clients bound the applied delay. */ + "Retry-After"?: string; "T4-API-Version": components["headers"]["SelectedVersion"]; [name: string]: unknown; }; @@ -794,7 +796,7 @@ export interface operations { }; requestBody?: never; responses: { - /** @description Bounded SSE event stream; each data field is a WatchEvent JSON value */ + /** @description Bounded SSE event stream. Every non-empty SSE data field is one JSON value conforming to WatchEvent; clients MUST reject unknown fields and schema-invalid event payloads before delivery. Transport chunk boundaries do not delimit frames. */ 200: { headers: { "Cache-Control"?: "no-store"; From 4a21fce4f7abf778feb07789f0b962aff1f0d159 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 20:49:43 +0000 Subject: [PATCH 015/131] fix: select bounded conformance frames --- packages/client/test/t4-api-v1-conformance-service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/test/t4-api-v1-conformance-service.ts b/packages/client/test/t4-api-v1-conformance-service.ts index 1e8026a6..a1278535 100644 --- a/packages/client/test/t4-api-v1-conformance-service.ts +++ b/packages/client/test/t4-api-v1-conformance-service.ts @@ -259,7 +259,7 @@ export class T4ApiV1ConformanceService { payload = encoder.encode(Array.from({ length: maxEvents }, (_, index) => `: ${"x".repeat(1010)}\ndata: {"type":"heartbeat","cursor":"bulk-${index}","observedAt":"2026-07-21T00:00:00Z"}\n\n`).join("")); } else { const prefix = this.options.watchTransport === "bytewise" ? ": 💚\n" : ""; - payload = encoder.encode(prefix + frames.join("")); + payload = encoder.encode(prefix + allFrames.slice(0, maxEvents).join("")); } const stream = new ReadableStream({ start: (controller) => { From cdbf49d56d32444f38174e0eae63ab0cb5b7a157 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 20:52:02 +0000 Subject: [PATCH 016/131] test: assert bounded consecutive watch retries --- packages/client/test/t4-api-v1-conformance.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index e427da1b..fb538394 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -412,7 +412,7 @@ describe("generated T4 API v1 client conformance", () => { for (const [status, code] of [[401, "not_found"], [403, "unauthenticated"], [404, "forbidden"], [409, "unavailable"], [503, "revision_conflict"]] as const) { const fetch: typeof globalThis.fetch = async () => Response.json({ error: { code, message: "wrong status class", requestId: "r", retryable: false } }, { status }); const client = createT4ApiClient({ baseUrl: "https://status-errors.test", credential: "token-a", majorVersion: 1, fetch }); - await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ code: "indeterminate", status }); + await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ code: "indeterminate", status: status === 503 ? 502 : status }); } }); @@ -491,7 +491,7 @@ describe("generated T4 API v1 client conformance", () => { let progressAttempts = 0; const progressFetch: typeof globalThis.fetch = async () => { progressAttempts += 1; - if (progressAttempts === 1 || progressAttempts === 3) return new Response(null, { headers: { "Content-Type": "text/event-stream" } }); + if (progressAttempts === 1) return new Response(null, { headers: { "Content-Type": "text/event-stream" } }); const cursor = progressAttempts === 2 ? "progress-1" : "progress-2"; return new Response(`data: {"type":"heartbeat","cursor":"${cursor}","observedAt":"2026-07-21T00:00:00Z"}\n\n`, { headers: { "Content-Type": "text/event-stream" } }); }; From bf6a356ff436df0cf44da4130c4124eb53b94981 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 21:02:42 +0000 Subject: [PATCH 017/131] test: cover OP03 fail-closed review contracts --- .../client/test/t4-api-v1-conformance.test.ts | 156 +++++++++++++++++- 1 file changed, 155 insertions(+), 1 deletion(-) diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index fb538394..beefe426 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -1,3 +1,5 @@ +import { readFileSync } from "node:fs"; + import { describe, expect, it, vi } from "vite-plus/test"; import { @@ -200,6 +202,31 @@ describe("generated T4 API v1 client conformance", () => { service.expectNoCredentialLeak(); }); + it("replays successful workspace deletion only within the original principal and request scope", async () => { + const service = new T4ApiV1ConformanceService(); + const owner = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); + const other = createT4ApiClient({ baseUrl: service.origin, credential: "token-b", majorVersion: 1, fetch: service.fetch }); + requireData(await owner.http.POST("/v1/workspaces", { + body: { name: "workspace" }, params: { header: idempotencyHeaders("workspace-delete-setup") }, + })); + const first = await owner.http.DELETE("/v1/workspaces/{workspaceId}", { + params: { header: idempotencyHeaders("workspace-delete-replay"), path: { workspaceId: "ws-1" } }, + }); + const replay = await owner.http.DELETE("/v1/workspaces/{workspaceId}", { + params: { header: idempotencyHeaders("workspace-delete-replay"), path: { workspaceId: "ws-1" } }, + }); + expect(first.response.status).toBe(204); + expect(first.response.headers.get("Idempotency-Replayed")).toBe("false"); + expect(replay.response.status).toBe(204); + expect(replay.response.headers.get("Idempotency-Replayed")).toBe("true"); + expect((await owner.http.DELETE("/v1/workspaces/{workspaceId}", { + params: { header: idempotencyHeaders("workspace-delete-fresh"), path: { workspaceId: "ws-1" } }, + })).response.status).toBe(404); + expect((await other.http.DELETE("/v1/workspaces/{workspaceId}", { + params: { header: idempotencyHeaders("workspace-delete-replay"), path: { workspaceId: "ws-1" } }, + })).response.status).toBe(404); + }); + it("spawns, paginates, and mutates sessions, then submits idempotent stable command states", async () => { const service = new T4ApiV1ConformanceService(); const client = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); @@ -298,6 +325,28 @@ describe("generated T4 API v1 client conformance", () => { expect(cancelled.data).toMatchObject({ state: "cancelled" }); }); + it("replays successful session deletion only within the original principal and request scope", async () => { + const service = new T4ApiV1ConformanceService(); + const owner = await seededClient(service, "delete-replay"); + const other = createT4ApiClient({ baseUrl: service.origin, credential: "token-b", majorVersion: 1, fetch: service.fetch }); + const first = await owner.http.DELETE("/v1/sessions/{sessionId}", { + params: { header: idempotencyHeaders("session-delete-replay"), path: { sessionId: "ses-1" } }, + }); + const replay = await owner.http.DELETE("/v1/sessions/{sessionId}", { + params: { header: idempotencyHeaders("session-delete-replay"), path: { sessionId: "ses-1" } }, + }); + expect(first.response.status).toBe(204); + expect(first.response.headers.get("Idempotency-Replayed")).toBe("false"); + expect(replay.response.status).toBe(204); + expect(replay.response.headers.get("Idempotency-Replayed")).toBe("true"); + expect((await owner.http.DELETE("/v1/sessions/{sessionId}", { + params: { header: idempotencyHeaders("session-delete-fresh"), path: { sessionId: "ses-1" } }, + })).response.status).toBe(404); + expect((await other.http.DELETE("/v1/sessions/{sessionId}", { + params: { header: idempotencyHeaders("session-delete-replay"), path: { sessionId: "ses-1" } }, + })).response.status).toBe(404); + }); + it("takes a snapshot and watches bounded SSE with heartbeat, reconnect, cancellation, and typed resync", async () => { const service = new T4ApiV1ConformanceService(); const client = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); @@ -416,6 +465,110 @@ describe("generated T4 API v1 client conformance", () => { } }); + it("fails closed on malformed, oversized, status-inconsistent, and unknown-field HTTP errors", async () => { + const validErrors = [ + [400, { error: { code: "invalid_request", message: "bad", requestId: "r", retryable: false } }], + [401, { error: { code: "unauthenticated", message: "bad", requestId: "r", retryable: false } }], + [403, { error: { code: "forbidden", message: "bad", requestId: "r", retryable: false } }], + [404, { error: { code: "not_found", message: "bad", requestId: "r", retryable: false } }], + [406, { error: { code: "incompatible_version", message: "bad", requestId: "r", retryable: false, supportedMajors: [1] } }], + [409, { error: { code: "revision_conflict", message: "bad", requestId: "r", retryable: false } }], + [422, { error: { code: "invalid_request", message: "bad", requestId: "r", retryable: false, violations: [{ field: "name", rule: "length", message: "bad" }] } }], + [503, { error: { code: "unavailable", message: "bad", requestId: "r", retryable: true } }], + ] as const; + for (const [status, body] of validErrors) { + const fetch: typeof globalThis.fetch = async () => Response.json({ error: { ...body.error, unknown: true } }, { status }); + const client = createT4ApiClient({ baseUrl: "https://ordinary-errors.test", credential: "token-a", majorVersion: 1, fetch }); + await expect(client.http.POST("/v1/workspaces", { + body: { name: "workspace" }, params: { header: idempotencyHeaders("ordinary-errors-0001") }, + })).rejects.toMatchObject({ code: "indeterminate", status: status === 503 ? 502 : status }); + } + + const malformedClient = createT4ApiClient({ + baseUrl: "https://malformed-error.test", credential: "token-a", majorVersion: 1, + fetch: async () => new Response("{", { status: 404, headers: { "Content-Type": "application/json" } }), + }); + await expect(malformedClient.http.GET("/v1/workspaces/{workspaceId}", { + params: { header: VERSION_HEADERS, path: { workspaceId: "missing" } }, + })).rejects.toMatchObject({ code: "indeterminate", status: 404 }); + + const oversizedClient = createT4ApiClient({ + baseUrl: "https://oversized-error.test", credential: "token-a", majorVersion: 1, + fetch: async () => new Response("x".repeat(1024 * 1024 + 1), { status: 404, headers: { "Content-Type": "application/json" } }), + }); + await expect(oversizedClient.http.GET("/v1/workspaces/{workspaceId}", { + params: { header: VERSION_HEADERS, path: { workspaceId: "missing" } }, + })).rejects.toMatchObject({ code: "indeterminate", status: 404 }); + + const statusClient = createT4ApiClient({ + baseUrl: "https://undeclared-error.test", credential: "token-a", majorVersion: 1, + fetch: async () => Response.json(validErrors[3][1], { status: 404 }), + }); + await expect(statusClient.http.GET("/v1", { params: { header: VERSION_HEADERS } })).rejects.toMatchObject({ code: "indeterminate", status: 404 }); + }); + + it("validates successful JSON routes relative to the base path and rejects undeclared media and statuses", async () => { + const discovery = { + apiVersion: "1.0", supportedMajors: [1], capabilities: [], + limits: { pageSizeDefault: 1, pageSizeMax: 1, commandBytesMax: 1, commandRequestBytesMax: 1, commandMetadataValueBytesMax: 1, watchEventsMax: 1, heartbeatSeconds: 5 }, + }; + const prefixed = createT4ApiClient({ + baseUrl: "https://prefixed.test/api", credential: "token-a", majorVersion: 1, + fetch: async (input) => { + expect(new Request(input).url).toBe("https://prefixed.test/api/v1"); + return Response.json({ ...discovery, unknown: true }); + }, + }); + await expect(prefixed.http.GET("/v1", { params: { header: VERSION_HEADERS } })).rejects.toMatchObject({ code: "indeterminate", status: 502 }); + + const invalidSnapshot = createT4ApiClient({ + baseUrl: "https://snapshot.test", credential: "token-a", majorVersion: 1, + fetch: async () => Response.json({ sessionId: "ses-1", cursor: "cursor-1", state: "ready", entries: [], unknown: true }), + }); + await expect(invalidSnapshot.http.GET("/v1/sessions/{sessionId}/snapshot", { + params: { header: VERSION_HEADERS, path: { sessionId: "ses-1" } }, + })).rejects.toMatchObject({ code: "indeterminate", status: 502 }); + + for (const response of [ + new Response(JSON.stringify(discovery), { status: 200, headers: { "Content-Type": "text/plain" } }), + Response.json(discovery, { status: 201 }), + ]) { + const client = createT4ApiClient({ baseUrl: "https://dispatch.test", credential: "token-a", majorVersion: 1, fetch: async () => response.clone() }); + await expect(client.http.GET("/v1", { params: { header: VERSION_HEADERS } })).rejects.toMatchObject({ code: "indeterminate", status: 502 }); + } + + const unknownRoute = createT4ApiClient({ + baseUrl: "https://dispatch.test", credential: "token-a", majorVersion: 1, + fetch: async () => Response.json(discovery), + }); + await expect(unknownRoute.http.GET("/v1/undeclared" as "/v1", { params: { header: VERSION_HEADERS } })).rejects.toMatchObject({ code: "indeterminate", status: 502 }); + }); + + it("declares the replay response header on both conditional PATCH success responses", () => { + const contract = JSON.parse(readFileSync(new URL("../../t4-api-contract/openapi.json", import.meta.url), "utf8")) as { + paths: Record } }>; + }; + expect(contract.paths["/v1/workspaces/{workspaceId}"]?.patch?.responses["200"]?.$ref).toBe("#/components/responses/WorkspaceReplay"); + expect(contract.paths["/v1/sessions/{sessionId}"]?.patch?.responses["200"]?.$ref).toBe("#/components/responses/SessionReplay"); + }); + + it("treats malformed and oversized watch 503 envelopes as terminal protocol errors", async () => { + for (const body of ["{", "x".repeat(1024 * 1024 + 1)]) { + let attempts = 0; + const client = createT4ApiClient({ + baseUrl: "https://invalid-503.test", credential: "token-a", majorVersion: 1, + fetch: async () => { + attempts += 1; + return new Response(body, { status: 503, headers: { "Content-Type": "application/json", "Retry-After": "0" } }); + }, + }); + await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 3, retryBackoffMs: 0 }).next()).rejects.toMatchObject({ + code: "indeterminate", status: 502, retryable: false, + }); + expect(attempts).toBe(1); + } + }); + it("rejects unknown fields in every public JSON response family", async () => { const discoveryService = new T4ApiV1ConformanceService({ invalidPayload: "discovery" }); const discoveryClient = createT4ApiClient({ baseUrl: discoveryService.origin, credential: "token-a", majorVersion: 1, fetch: discoveryService.fetch }); @@ -491,7 +644,7 @@ describe("generated T4 API v1 client conformance", () => { let progressAttempts = 0; const progressFetch: typeof globalThis.fetch = async () => { progressAttempts += 1; - if (progressAttempts === 1) return new Response(null, { headers: { "Content-Type": "text/event-stream" } }); + if (progressAttempts === 1 || progressAttempts === 3) return new Response(null, { headers: { "Content-Type": "text/event-stream" } }); const cursor = progressAttempts === 2 ? "progress-1" : "progress-2"; return new Response(`data: {"type":"heartbeat","cursor":"${cursor}","observedAt":"2026-07-21T00:00:00Z"}\n\n`, { headers: { "Content-Type": "text/event-stream" } }); }; @@ -499,6 +652,7 @@ describe("generated T4 API v1 client conformance", () => { const progressEvents: WatchEvent[] = []; for await (const item of progressClient.watchSession("ses-1", { maxEvents: 2, maxReconnectAttempts: 1, retryBackoffMs: 0 })) progressEvents.push(item); expect(progressEvents.map((item) => item.cursor)).toEqual(["progress-1", "progress-2"]); + expect(progressAttempts).toBe(4); vi.useFakeTimers(); try { From dfba49756d2251801df1337c51b8e48bc3a62f46 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 21:11:15 +0000 Subject: [PATCH 018/131] fix: close OP03 fail-closed review gaps --- .github/workflows/ci.yml | 9 + .../test/t4-api-v1-conformance-service.ts | 24 +- .../client/test/t4-api-v1-conformance.test.ts | 15 +- packages/t4-api-client/src/index.ts | 209 ++++++++++++++---- packages/t4-api-contract/openapi.json | 4 +- 5 files changed, 192 insertions(+), 69 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 25d7d947..79960472 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,10 +82,19 @@ jobs: - name: Validate OpenAPI and deterministic generation run: | pnpm --filter @t4-code/t4-api-contract validate + pnpm --filter @t4-code/t4-api-contract generate:ci pnpm --filter @t4-code/t4-api-contract check:generated pnpm --filter @t4-code/t4-api-client typecheck pnpm --filter @t4-code/client typecheck pnpm --filter @t4-code/client exec vp test run test/t4-api-v1-conformance.test.ts + - name: Upload authorized generated schema + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: t4-api-generated-${{ github.run_id }} + path: artifacts/t4-api/generated/schema.ts + if-no-files-found: error + core: diff --git a/packages/client/test/t4-api-v1-conformance-service.ts b/packages/client/test/t4-api-v1-conformance-service.ts index a1278535..af097f22 100644 --- a/packages/client/test/t4-api-v1-conformance-service.ts +++ b/packages/client/test/t4-api-v1-conformance-service.ts @@ -121,6 +121,12 @@ export class T4ApiV1ConformanceService { if (workspaceMatch) { const id = decodeURIComponent(workspaceMatch[1]!); const workspace = this.#workspaces.get(id); + if (request.method === "DELETE") { + return this.#idempotent(request, tenant, "deleteWorkspace", [id], null, 204, 204, () => { + this.#workspaces.delete(id); + return null; + }, () => workspace?.tenant === tenant ? undefined : problem(404, "not_found", "Workspace not found")); + } if (workspace?.tenant !== tenant) return problem(404, "not_found", "Workspace not found"); if (request.method === "GET") return json(200, this.#payload("workspace", this.#visible(workspace))); if (request.method === "PATCH") { @@ -133,12 +139,6 @@ export class T4ApiV1ConformanceService { return updated; }, () => request.headers.get("If-Match") === String(workspace.revision) ? undefined : problem(409, "revision_conflict", "Workspace revision changed")); } - if (request.method === "DELETE") { - return this.#idempotent(request, tenant, "deleteWorkspace", [id], null, 204, 204, () => { - this.#workspaces.delete(id); - return null; - }); - } } const sessionsPath = url.pathname.match(/^\/v1\/workspaces\/([^/]+)\/sessions$/u); @@ -173,6 +173,12 @@ export class T4ApiV1ConformanceService { if (sessionMatch) { const id = decodeURIComponent(sessionMatch[1]!); const session = this.#sessions.get(id); + if (request.method === "DELETE") { + return this.#idempotent(request, tenant, "deleteSession", [id], null, 204, 204, () => { + this.#sessions.delete(id); + return null; + }, () => session?.tenant === tenant ? undefined : problem(404, "not_found", "Session not found")); + } if (session?.tenant !== tenant) return problem(404, "not_found", "Session not found"); if (request.method === "GET") return json(200, this.#payload("session", this.#visible(session))); if (request.method === "PATCH") { @@ -185,12 +191,6 @@ export class T4ApiV1ConformanceService { return updated; }, () => request.headers.get("If-Match") === String(session.revision) ? undefined : problem(409, "revision_conflict", "Session revision changed")); } - if (request.method === "DELETE") { - return this.#idempotent(request, tenant, "deleteSession", [id], null, 204, 204, () => { - this.#sessions.delete(id); - return null; - }); - } } const cancelMatch = url.pathname.match(/^\/v1\/sessions\/([^/]+)\/cancel$/u); diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index beefe426..f8a9a3d7 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -644,15 +644,16 @@ describe("generated T4 API v1 client conformance", () => { let progressAttempts = 0; const progressFetch: typeof globalThis.fetch = async () => { progressAttempts += 1; - if (progressAttempts === 1 || progressAttempts === 3) return new Response(null, { headers: { "Content-Type": "text/event-stream" } }); - const cursor = progressAttempts === 2 ? "progress-1" : "progress-2"; - return new Response(`data: {"type":"heartbeat","cursor":"${cursor}","observedAt":"2026-07-21T00:00:00Z"}\n\n`, { headers: { "Content-Type": "text/event-stream" } }); + if (progressAttempts === 2) { + return new Response('data: {"type":"heartbeat","cursor":"progress-1","observedAt":"2026-07-21T00:00:00Z"}\n\n', { headers: { "Content-Type": "text/event-stream" } }); + } + return new Response(null, { headers: { "Content-Type": "text/event-stream" } }); }; const progressClient = createT4ApiClient({ baseUrl: "https://progress.test", credential: "token-a", majorVersion: 1, fetch: progressFetch }); - const progressEvents: WatchEvent[] = []; - for await (const item of progressClient.watchSession("ses-1", { maxEvents: 2, maxReconnectAttempts: 1, retryBackoffMs: 0 })) progressEvents.push(item); - expect(progressEvents.map((item) => item.cursor)).toEqual(["progress-1", "progress-2"]); - expect(progressAttempts).toBe(4); + const progressWatch = progressClient.watchSession("ses-1", { maxEvents: 2, maxReconnectAttempts: 1, retryBackoffMs: 0 }); + await expect(progressWatch.next()).resolves.toMatchObject({ value: { cursor: "progress-1" }, done: false }); + await expect(progressWatch.next()).rejects.toMatchObject({ code: "indeterminate", status: 502 }); + expect(progressAttempts).toBe(3); vi.useFakeTimers(); try { diff --git a/packages/t4-api-client/src/index.ts b/packages/t4-api-client/src/index.ts index 8b4e1559..62da04d2 100644 --- a/packages/t4-api-client/src/index.ts +++ b/packages/t4-api-client/src/index.ts @@ -224,29 +224,33 @@ function retryAfterMilliseconds(response: Response): number | undefined { return Math.min(30_000, Math.max(0, timestamp - Date.now())); } -async function boundedError(response: Response): Promise { - const text = await boundedResponseText(response); - const retryAfterMs = retryAfterMilliseconds(response); - if (text !== undefined) { - try { - const decoded = apiError(JSON.parse(text), response.status); - if (decoded !== undefined) { - return retryAfterMs === undefined - ? new T4ApiError(response.status, decoded) - : new T4ApiError(response.status, decoded, { retryAfterMs }); - } - } catch { - // Fall through to the stable indeterminate transport envelope. - } - } - return new T4ApiError(response.status, { - code: response.status === 503 ? "unavailable" : "indeterminate", - message: "T4 API returned an invalid or oversized error envelope", + +function protocolError(status: number, message: string): T4ApiError { + return new T4ApiError(status === 503 ? 502 : status, { + code: "indeterminate", + message, requestId: "unavailable", - retryable: response.status >= 500, + retryable: false, }); } +async function parsedError(response: Response): Promise { + if (response.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase() !== "application/json") return undefined; + const text = await boundedResponseText(response); + if (text === undefined) return undefined; + try { + const decoded = apiError(JSON.parse(text), response.status); + if (decoded === undefined) return undefined; + const retryAfterMs = retryAfterMilliseconds(response); + return retryAfterMs === undefined + ? new T4ApiError(response.status, decoded) + : new T4ApiError(response.status, decoded, { retryAfterMs }); + } catch { + return undefined; + } +} + + function hasOnlyKeys(value: Record, keys: Readonly>): boolean { return Object.keys(value).every((key) => keys[key] === true); } @@ -301,30 +305,136 @@ function validDiscovery(value: unknown): boolean { Number(limits.pageSizeMax) <= 100 && Number(limits.commandBytesMax) <= 262144 && Number(limits.commandRequestBytesMax) <= 1048576 && Number(limits.commandMetadataValueBytesMax) <= 262144 && Number(limits.watchEventsMax) <= 1000 && Number(limits.heartbeatSeconds) >= 5 && Number(limits.heartbeatSeconds) <= 60; } -function validSuccessPayload(request: Request, value: unknown): boolean { - const path = new URL(request.url).pathname; - if (request.method === "GET" && path === "/v1") return validDiscovery(value); - if (/^\/v1\/workspaces\/[A-Za-z0-9._~-]+\/sessions$/u.test(path) && request.method === "GET") { - const page = record(value); - return page !== undefined && hasOnlyKeys(page, { items: true, nextCursor: true }) && Array.isArray(page.items) && page.items.length <= 100 && page.items.every(validSession) && (page.nextCursor === undefined || validCursor(page.nextCursor)); +function validSnapshot(value: unknown): boolean { + const snapshot = record(value); + return snapshot !== undefined && hasOnlyKeys(snapshot, { sessionId: true, cursor: true, state: true, entries: true }) && + validResourceId(snapshot.sessionId) && validCursor(snapshot.cursor) && + typeof snapshot.state === "string" && SESSION_STATES[snapshot.state as components["schemas"]["SessionState"]] === true && + Array.isArray(snapshot.entries) && snapshot.entries.length <= 1000 && snapshot.entries.every((value) => { + const entry = record(value); + return entry !== undefined && hasOnlyKeys(entry, { sequence: true, kind: true, text: true }) && + Number.isSafeInteger(entry.sequence) && Number(entry.sequence) >= 0 && + (entry.kind === "input" || entry.kind === "output" || entry.kind === "status") && + typeof entry.text === "string" && entry.text.length <= 1_048_576; + }); +} + +type SuccessValidator = ((value: unknown) => boolean) | "empty" | "event-stream"; + +interface ResponseContract { + readonly success: Readonly>; + readonly errors: readonly number[]; +} + +const DISCOVERY_RESPONSE: ResponseContract = { success: { 200: validDiscovery }, errors: [401, 403, 406, 503] }; +const WORKSPACE_LIST_RESPONSE: ResponseContract = { success: { 200: validWorkspacePage }, errors: [400, 401, 403, 406, 422, 503] }; +const WORKSPACE_CREATE_RESPONSE: ResponseContract = { success: { 200: validWorkspace, 202: validWorkspace }, errors: [400, 401, 403, 406, 409, 422, 503] }; +const WORKSPACE_GET_RESPONSE: ResponseContract = { success: { 200: validWorkspace }, errors: [401, 403, 404, 406, 503] }; +const WORKSPACE_MUTATE_RESPONSE: ResponseContract = { success: { 200: validWorkspace }, errors: [400, 401, 403, 404, 406, 409, 422, 503] }; +const DELETE_RESPONSE: ResponseContract = { success: { 204: "empty" }, errors: [400, 401, 403, 404, 406, 409, 503] }; +const SESSION_LIST_RESPONSE: ResponseContract = { success: { 200: validSessionPage }, errors: [401, 403, 404, 406, 422, 503] }; +const SESSION_CREATE_RESPONSE: ResponseContract = { success: { 200: validSession, 202: validSession }, errors: [400, 401, 403, 404, 406, 409, 422, 503] }; +const SESSION_GET_RESPONSE: ResponseContract = { success: { 200: validSession }, errors: [401, 403, 404, 406, 503] }; +const SESSION_MUTATE_RESPONSE: ResponseContract = { success: { 200: validSession }, errors: [400, 401, 403, 404, 406, 409, 422, 503] }; +const SESSION_CANCEL_RESPONSE: ResponseContract = { success: { 200: validSession, 202: validSession }, errors: [400, 401, 403, 404, 406, 409, 503] }; +const COMMAND_RESPONSE: ResponseContract = { success: { 200: validCommandResult, 202: validCommandResult }, errors: [400, 401, 403, 404, 406, 409, 422, 503] }; +const SNAPSHOT_RESPONSE: ResponseContract = { success: { 200: validSnapshot }, errors: [401, 403, 404, 406, 503] }; +const EVENTS_RESPONSE: ResponseContract = { success: { 200: "event-stream" }, errors: [400, 401, 403, 404, 406, 410, 422, 503] }; + +function validWorkspacePage(value: unknown): boolean { + const page = record(value); + return page !== undefined && hasOnlyKeys(page, { items: true, nextCursor: true }) && + Array.isArray(page.items) && page.items.length <= 100 && page.items.every(validWorkspace) && + (page.nextCursor === undefined || validCursor(page.nextCursor)); +} + +function validSessionPage(value: unknown): boolean { + const page = record(value); + return page !== undefined && hasOnlyKeys(page, { items: true, nextCursor: true }) && + Array.isArray(page.items) && page.items.length <= 100 && page.items.every(validSession) && + (page.nextCursor === undefined || validCursor(page.nextCursor)); +} + +function relativeApiPath(request: Request, baseUrl: string): string | undefined { + const requestUrl = new URL(request.url); + const base = new URL(baseUrl); + if (requestUrl.origin !== base.origin) return undefined; + const prefix = base.pathname === "/" ? "" : base.pathname; + if (prefix === "") return requestUrl.pathname; + if (!requestUrl.pathname.startsWith(`${prefix}/`)) return undefined; + return requestUrl.pathname.slice(prefix.length); +} + +function responseContract(method: string, path: string): ResponseContract | undefined { + if (path === "/v1" && method === "GET") return DISCOVERY_RESPONSE; + if (path === "/v1/workspaces") { + if (method === "GET") return WORKSPACE_LIST_RESPONSE; + if (method === "POST") return WORKSPACE_CREATE_RESPONSE; + return undefined; } - if (path === "/v1/workspaces" && request.method === "GET") { - const page = record(value); - return page !== undefined && hasOnlyKeys(page, { items: true, nextCursor: true }) && Array.isArray(page.items) && page.items.length <= 100 && page.items.every(validWorkspace) && (page.nextCursor === undefined || validCursor(page.nextCursor)); + if (/^\/v1\/workspaces\/[A-Za-z0-9._~-]+$/u.test(path)) { + if (method === "GET") return WORKSPACE_GET_RESPONSE; + if (method === "PATCH") return WORKSPACE_MUTATE_RESPONSE; + if (method === "DELETE") return DELETE_RESPONSE; + return undefined; + } + if (/^\/v1\/workspaces\/[A-Za-z0-9._~-]+\/sessions$/u.test(path)) { + if (method === "GET") return SESSION_LIST_RESPONSE; + if (method === "POST") return SESSION_CREATE_RESPONSE; + return undefined; + } + if (/^\/v1\/sessions\/[A-Za-z0-9._~-]+$/u.test(path)) { + if (method === "GET") return SESSION_GET_RESPONSE; + if (method === "PATCH") return SESSION_MUTATE_RESPONSE; + if (method === "DELETE") return DELETE_RESPONSE; + return undefined; + } + if (/^\/v1\/sessions\/[A-Za-z0-9._~-]+\/cancel$/u.test(path) && method === "POST") return SESSION_CANCEL_RESPONSE; + if (/^\/v1\/sessions\/[A-Za-z0-9._~-]+\/commands$/u.test(path) && method === "POST") return COMMAND_RESPONSE; + if (/^\/v1\/sessions\/[A-Za-z0-9._~-]+\/snapshot$/u.test(path) && method === "GET") return SNAPSHOT_RESPONSE; + if (/^\/v1\/sessions\/[A-Za-z0-9._~-]+\/events$/u.test(path) && method === "GET") return EVENTS_RESPONSE; + return undefined; +} + +async function validateResponse(request: Request, response: Response, baseUrl: string): Promise { + const path = relativeApiPath(request, baseUrl); + const contract = path === undefined ? undefined : responseContract(request.method, path); + if (contract === undefined) { + throw protocolError(response.ok ? 502 : response.status, "T4 API returned a response for an undeclared route"); + } + if (!response.ok) { + if (!contract.errors.includes(response.status)) { + throw protocolError(response.status, "T4 API returned an undeclared error status"); + } + if (await parsedError(response.clone()) === undefined) { + throw protocolError(response.status, "T4 API returned an invalid or oversized error envelope"); + } + return; + } + const validator = contract.success[response.status]; + if (validator === undefined) { + throw protocolError(502, "T4 API returned an undeclared success status"); + } + if (validator === "empty") { + if (response.body !== null || response.headers.has("content-type")) { + throw protocolError(502, "T4 API returned content for a bodyless response"); + } + return; + } + if (validator === "event-stream") { + if (response.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase() !== "text/event-stream") { + throw protocolError(502, "T4 API returned an undeclared success media type"); + } + return; + } + if (response.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase() !== "application/json") { + throw protocolError(502, "T4 API returned an undeclared success media type"); } - if (path === "/v1/workspaces" || /^\/v1\/workspaces\/[A-Za-z0-9._~-]+$/u.test(path)) return validWorkspace(value); - if (/^\/v1\/workspaces\/[A-Za-z0-9._~-]+\/sessions$/u.test(path) || /^\/v1\/sessions\/[A-Za-z0-9._~-]+(?:\/cancel)?$/u.test(path)) return validSession(value); - if (/^\/v1\/sessions\/[A-Za-z0-9._~-]+\/commands$/u.test(path)) return validCommandResult(value); - return true; -} - -async function validateJsonSuccess(request: Request, response: Response): Promise { - if (!response.ok || response.status === 204 || !response.headers.get("content-type")?.toLowerCase().startsWith("application/json")) return; const text = await boundedResponseText(response.clone(), MAX_JSON_RESPONSE_BYTES); let value: unknown; try { value = text === undefined ? undefined : JSON.parse(text); } catch { value = undefined; } - if (text === undefined || !validSuccessPayload(request, value)) { - throw new T4ApiError(502, { code: "indeterminate", message: "T4 API returned an invalid or oversized JSON response", requestId: "unavailable", retryable: true }); + if (text === undefined || !validator(value)) { + throw protocolError(502, "T4 API returned an invalid or oversized JSON response"); } } @@ -338,7 +448,7 @@ function watchEvent(value: unknown, eventId: string | undefined): WatchEvent { event.cursor.length > 512 || !CURSOR_PATTERN.test(event.cursor) || (eventId !== undefined && eventId !== event.cursor) - ) throw new T4ApiError(502, { code: "indeterminate", message: "T4 API returned an invalid watch event", requestId: "unavailable", retryable: true }); + ) throw protocolError(502, "T4 API returned an invalid watch event"); if ( event.type === "heartbeat" && hasOnlyKeys(event, { type: true, cursor: true, observedAt: true }) && @@ -362,7 +472,7 @@ function watchEvent(value: unknown, eventId: string | undefined): WatchEvent { typeof event.state === "string" && OPERATION_STATES[event.state as components["schemas"]["OperationState"]] === true ) return event as WatchEvent; - throw new T4ApiError(502, { code: "indeterminate", message: "T4 API returned an invalid watch event", requestId: "unavailable", retryable: true }); + throw protocolError(502, "T4 API returned an invalid watch event"); } function decodeSseFrame(frame: string): WatchEvent | undefined { @@ -383,7 +493,7 @@ function decodeSseFrame(frame: string): WatchEvent | undefined { return watchEvent(JSON.parse(data.join("\n")), eventId); } catch (error) { if (error instanceof T4ApiError) throw error; - throw new T4ApiError(502, { code: "indeterminate", message: "T4 API returned malformed SSE data", requestId: "unavailable", retryable: true }); + throw protocolError(502, "T4 API returned malformed SSE data"); } } @@ -451,7 +561,7 @@ class SseFrameParser { } #oversized(): never { - throw new T4ApiError(502, { code: "indeterminate", message: "T4 API watch event exceeds the client bound", requestId: "unavailable", retryable: true }); + throw protocolError(502, "T4 API watch event exceeds the client bound"); } } @@ -460,7 +570,7 @@ function decodedFrames(parser: SseFrameParser, chunk: Uint8Array | undefined): W return frames.flatMap((bytes) => { let text: string; try { text = new TextDecoder("utf-8", { fatal: true }).decode(bytes); } catch { - throw new T4ApiError(502, { code: "indeterminate", message: "T4 API returned malformed SSE data", requestId: "unavailable", retryable: true }); + throw protocolError(502, "T4 API returned malformed SSE data"); } const event = decodeSseFrame(text); return event === undefined ? [] : [event]; @@ -524,11 +634,14 @@ async function* watch( }); if (cursor !== undefined) headers.set("Last-Event-ID", cursor); const response = await fetchImpl(url, { method: "GET", headers, signal: controller.signal }); - if (!response.ok) throw await boundedError(response); - if (!response.headers.get("content-type")?.toLowerCase().startsWith("text/event-stream")) { - throw new T4ApiError(502, { code: "indeterminate", message: "T4 API watch did not return text/event-stream", requestId: "unavailable", retryable: true }); + if (!response.ok) { + const error = await parsedError(response); + throw error ?? protocolError(response.status, "T4 API returned an invalid or oversized error envelope"); + } + if (response.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase() !== "text/event-stream") { + throw protocolError(502, "T4 API watch did not return text/event-stream"); } - if (response.body === null) throw new TypeError("T4 API watch response body is unavailable"); + if (response.body === null) throw protocolError(502, "T4 API watch response body is unavailable"); reader = response.body.getReader(); const parser = new SseFrameParser(); while (delivered < maxEvents && !controller.signal.aborted) { @@ -583,7 +696,7 @@ export function createT4ApiClient(options: T4ApiClientOptions): T4ApiClient { request.headers.set("T4-API-Version", majorVersion); request.headers.set("Accept", "application/json"); const response = await fetchImpl(request); - await validateJsonSuccess(request, response); + await validateResponse(request, response, baseUrl); return response; }; const http = createClient({ baseUrl, fetch: authenticatedFetch }); diff --git a/packages/t4-api-contract/openapi.json b/packages/t4-api-contract/openapi.json index c883eab1..1ae79de1 100644 --- a/packages/t4-api-contract/openapi.json +++ b/packages/t4-api-contract/openapi.json @@ -102,7 +102,7 @@ }, "responses": { "400": { "$ref": "#/components/responses/Error400" }, - "200": { "$ref": "#/components/responses/Workspace" }, + "200": { "$ref": "#/components/responses/WorkspaceReplay" }, "401": {"$ref": "#/components/responses/Error401"}, "403": {"$ref": "#/components/responses/Error403"}, "404": {"$ref": "#/components/responses/Error404"}, @@ -205,7 +205,7 @@ }, "responses": { "400": { "$ref": "#/components/responses/Error400" }, - "200": { "$ref": "#/components/responses/Session" }, + "200": { "$ref": "#/components/responses/SessionReplay" }, "401": {"$ref": "#/components/responses/Error401"}, "403": {"$ref": "#/components/responses/Error403"}, "404": {"$ref": "#/components/responses/Error404"}, From 77c9380555fcb82b6f2a95fc15c5ab5494702d81 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 21:13:39 +0000 Subject: [PATCH 019/131] build: accept authorized OP03 schema artifact --- .github/workflows/ci.yml | 9 --------- packages/t4-api-client/src/generated/schema.ts | 4 ++-- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 79960472..25d7d947 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,19 +82,10 @@ jobs: - name: Validate OpenAPI and deterministic generation run: | pnpm --filter @t4-code/t4-api-contract validate - pnpm --filter @t4-code/t4-api-contract generate:ci pnpm --filter @t4-code/t4-api-contract check:generated pnpm --filter @t4-code/t4-api-client typecheck pnpm --filter @t4-code/client typecheck pnpm --filter @t4-code/client exec vp test run test/t4-api-v1-conformance.test.ts - - name: Upload authorized generated schema - if: ${{ always() }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: t4-api-generated-${{ github.run_id }} - path: artifacts/t4-api/generated/schema.ts - if-no-files-found: error - core: diff --git a/packages/t4-api-client/src/generated/schema.ts b/packages/t4-api-client/src/generated/schema.ts index 1291c013..39321c83 100755 --- a/packages/t4-api-client/src/generated/schema.ts +++ b/packages/t4-api-client/src/generated/schema.ts @@ -705,7 +705,7 @@ export interface operations { }; }; responses: { - 200: components["responses"]["Session"]; + 200: components["responses"]["SessionReplay"]; 400: components["responses"]["Error400"]; 401: components["responses"]["Error401"]; 403: components["responses"]["Error403"]; @@ -964,7 +964,7 @@ export interface operations { }; }; responses: { - 200: components["responses"]["Workspace"]; + 200: components["responses"]["WorkspaceReplay"]; 400: components["responses"]["Error400"]; 401: components["responses"]["Error401"]; 403: components["responses"]["Error403"]; From 9209caf64ad75117057fb76971767f34a169a328 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 21:23:58 +0000 Subject: [PATCH 020/131] fix: enforce exact watch response contract --- .../client/test/t4-api-v1-conformance.test.ts | 83 ++++++++++++++++++- packages/t4-api-client/src/index.ts | 32 ++++++- 2 files changed, 109 insertions(+), 6 deletions(-) diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index f8a9a3d7..3aac3a79 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -500,6 +500,28 @@ describe("generated T4 API v1 client conformance", () => { params: { header: VERSION_HEADERS, path: { workspaceId: "missing" } }, })).rejects.toMatchObject({ code: "indeterminate", status: 404 }); + vi.useFakeTimers(); + try { + let cancelled = false; + const chunk = new Uint8Array(600 * 1024); + const streamedClient = createT4ApiClient({ + baseUrl: "https://streamed-error.test", credential: "token-a", majorVersion: 1, + fetch: async () => new Response(new ReadableStream({ + pull(controller) { controller.enqueue(chunk); }, + cancel() { cancelled = true; }, + }), { status: 404, headers: { "Content-Type": "application/json" } }), + }); + const outcome = streamedClient.http.GET("/v1/workspaces/{workspaceId}", { + params: { header: VERSION_HEADERS, path: { workspaceId: "missing" } }, + }).then(() => "resolved" as const, () => "rejected" as const); + const bounded = Promise.race([outcome, new Promise<"timeout">((resolve) => setTimeout(() => resolve("timeout"), 1_000))]); + await vi.advanceTimersByTimeAsync(1_000); + await expect(bounded).resolves.toBe("rejected"); + expect(cancelled).toBe(true); + } finally { + vi.useRealTimers(); + } + const statusClient = createT4ApiClient({ baseUrl: "https://undeclared-error.test", credential: "token-a", majorVersion: 1, fetch: async () => Response.json(validErrors[3][1], { status: 404 }), @@ -522,13 +544,26 @@ describe("generated T4 API v1 client conformance", () => { await expect(prefixed.http.GET("/v1", { params: { header: VERSION_HEADERS } })).rejects.toMatchObject({ code: "indeterminate", status: 502 }); const invalidSnapshot = createT4ApiClient({ - baseUrl: "https://snapshot.test", credential: "token-a", majorVersion: 1, - fetch: async () => Response.json({ sessionId: "ses-1", cursor: "cursor-1", state: "ready", entries: [], unknown: true }), + baseUrl: "https://snapshot.test/api", credential: "token-a", majorVersion: 1, + fetch: async (input) => { + expect(new URL(new Request(input).url).pathname).toBe("/api/v1/sessions/ses-1/snapshot"); + return Response.json({ sessionId: "ses-1", cursor: "cursor-1", state: "ready", entries: [], unknown: true }); + }, }); await expect(invalidSnapshot.http.GET("/v1/sessions/{sessionId}/snapshot", { params: { header: VERSION_HEADERS, path: { sessionId: "ses-1" } }, })).rejects.toMatchObject({ code: "indeterminate", status: 502 }); + const astralText = "😀".repeat(524_289); + const unicodeSnapshot = createT4ApiClient({ + baseUrl: "https://unicode-snapshot.test", credential: "token-a", majorVersion: 1, + fetch: async () => Response.json({ sessionId: "ses-1", cursor: "cursor-1", state: "ready", entries: [{ sequence: 0, kind: "output", text: astralText }] }), + }); + const unicode = requireData(await unicodeSnapshot.http.GET("/v1/sessions/{sessionId}/snapshot", { + params: { header: VERSION_HEADERS, path: { sessionId: "ses-1" } }, + })); + expect(unicode.entries[0]?.text).toBe(astralText); + for (const response of [ new Response(JSON.stringify(discovery), { status: 200, headers: { "Content-Type": "text/plain" } }), Response.json(discovery, { status: 201 }), @@ -544,6 +579,48 @@ describe("generated T4 API v1 client conformance", () => { await expect(unknownRoute.http.GET("/v1/undeclared" as "/v1", { params: { header: VERSION_HEADERS } })).rejects.toMatchObject({ code: "indeterminate", status: 502 }); }); + it("accepts only the exact watch status matrix under a prefixed base path", async () => { + const prefixed = createT4ApiClient({ + baseUrl: "https://watch-status.test/api", credential: "token-a", majorVersion: 1, + fetch: async (input) => { + expect(new URL(new Request(input).url).pathname).toBe("/api/v1/sessions/ses-1/events"); + return new Response('data: {"type":"heartbeat","cursor":"prefixed-1","observedAt":"2026-07-21T00:00:00Z"}\n\n', { status: 200, headers: { "Content-Type": "text/event-stream" } }); + }, + }); + await expect(prefixed.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).resolves.toMatchObject({ value: { cursor: "prefixed-1" } }); + + const declared = [ + [400, { error: { code: "invalid_request", message: "bad", requestId: "r", retryable: false } }], + [401, { error: { code: "unauthenticated", message: "bad", requestId: "r", retryable: false } }], + [403, { error: { code: "forbidden", message: "bad", requestId: "r", retryable: false } }], + [404, { error: { code: "not_found", message: "bad", requestId: "r", retryable: false } }], + [406, { error: { code: "incompatible_version", message: "bad", requestId: "r", retryable: false, supportedMajors: [1] } }], + [410, { error: { code: "cursor_expired", message: "bad", requestId: "r", retryable: false, resync: { snapshotUrl: "/v1/sessions/ses-1/snapshot", cursor: "cursor-1" } } }], + [422, { error: { code: "invalid_request", message: "bad", requestId: "r", retryable: false, violations: [{ field: "maxEvents", rule: "range", message: "bad" }] } }], + [503, { error: { code: "unavailable", message: "bad", requestId: "r", retryable: false } }], + ] as const; + for (const [status, body] of declared) { + const client = createT4ApiClient({ baseUrl: "https://watch-status.test", credential: "token-a", majorVersion: 1, fetch: async () => Response.json(body, { status }) }); + await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ status, code: body.error.code }); + } + + for (const response of [ + new Response('data: {"type":"heartbeat","cursor":"wrong-1","observedAt":"2026-07-21T00:00:00Z"}\n\n', { status: 201, headers: { "Content-Type": "text/event-stream" } }), + new Response(null, { status: 204, headers: { "Content-Type": "text/event-stream" } }), + Response.json({ error: { code: "revision_conflict", message: "bad", requestId: "r", retryable: false } }, { status: 409 }), + ]) { + let attempts = 0; + const client = createT4ApiClient({ + baseUrl: "https://watch-status.test", credential: "token-a", majorVersion: 1, + fetch: async () => { attempts += 1; return response.clone(); }, + }); + await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 3, retryBackoffMs: 0 }).next()).rejects.toMatchObject({ + status: 502, code: "indeterminate", retryable: false, + }); + expect(attempts).toBe(1); + } + }); + it("declares the replay response header on both conditional PATCH success responses", () => { const contract = JSON.parse(readFileSync(new URL("../../t4-api-contract/openapi.json", import.meta.url), "utf8")) as { paths: Record } }>; @@ -647,7 +724,7 @@ describe("generated T4 API v1 client conformance", () => { if (progressAttempts === 2) { return new Response('data: {"type":"heartbeat","cursor":"progress-1","observedAt":"2026-07-21T00:00:00Z"}\n\n', { headers: { "Content-Type": "text/event-stream" } }); } - return new Response(null, { headers: { "Content-Type": "text/event-stream" } }); + return new Response(new ReadableStream({ start(controller) { controller.close(); } }), { headers: { "Content-Type": "text/event-stream" } }); }; const progressClient = createT4ApiClient({ baseUrl: "https://progress.test", credential: "token-a", majorVersion: 1, fetch: progressFetch }); const progressWatch = progressClient.watchSession("ses-1", { maxEvents: 2, maxReconnectAttempts: 1, retryBackoffMs: 0 }); diff --git a/packages/t4-api-client/src/index.ts b/packages/t4-api-client/src/index.ts index 62da04d2..cdaab911 100644 --- a/packages/t4-api-client/src/index.ts +++ b/packages/t4-api-client/src/index.ts @@ -198,7 +198,7 @@ async function boundedResponseText(response: Response, maximumBytes = MAX_ERROR_ if (chunk.done) break; length += chunk.value.byteLength; if (length > maximumBytes) { - await reader.cancel(); + void reader.cancel().catch(() => {}); return undefined; } chunks.push(chunk.value); @@ -259,6 +259,15 @@ function validResourceId(value: unknown): value is string { return typeof value === "string" && value.length >= 1 && value.length <= 128 && /^[A-Za-z0-9][A-Za-z0-9._~-]*$/u.test(value); } +function hasAtMostCodePoints(value: string, maximum: number): boolean { + let count = 0; + for (const _codePoint of value) { + count += 1; + if (count > maximum) return false; + } + return true; +} + function validCursor(value: unknown): value is string { return typeof value === "string" && value.length >= 1 && value.length <= 512 && CURSOR_PATTERN.test(value); } @@ -315,7 +324,7 @@ function validSnapshot(value: unknown): boolean { return entry !== undefined && hasOnlyKeys(entry, { sequence: true, kind: true, text: true }) && Number.isSafeInteger(entry.sequence) && Number(entry.sequence) >= 0 && (entry.kind === "input" || entry.kind === "output" || entry.kind === "status") && - typeof entry.text === "string" && entry.text.length <= 1_048_576; + typeof entry.text === "string" && hasAtMostCodePoints(entry.text, 1_048_576); }); } @@ -400,40 +409,48 @@ async function validateResponse(request: Request, response: Response, baseUrl: s const path = relativeApiPath(request, baseUrl); const contract = path === undefined ? undefined : responseContract(request.method, path); if (contract === undefined) { + void response.body?.cancel().catch(() => {}); throw protocolError(response.ok ? 502 : response.status, "T4 API returned a response for an undeclared route"); } if (!response.ok) { if (!contract.errors.includes(response.status)) { + void response.body?.cancel().catch(() => {}); throw protocolError(response.status, "T4 API returned an undeclared error status"); } if (await parsedError(response.clone()) === undefined) { + void response.body?.cancel().catch(() => {}); throw protocolError(response.status, "T4 API returned an invalid or oversized error envelope"); } return; } const validator = contract.success[response.status]; if (validator === undefined) { + void response.body?.cancel().catch(() => {}); throw protocolError(502, "T4 API returned an undeclared success status"); } if (validator === "empty") { if (response.body !== null || response.headers.has("content-type")) { + void response.body?.cancel().catch(() => {}); throw protocolError(502, "T4 API returned content for a bodyless response"); } return; } if (validator === "event-stream") { if (response.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase() !== "text/event-stream") { + void response.body?.cancel().catch(() => {}); throw protocolError(502, "T4 API returned an undeclared success media type"); } return; } if (response.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase() !== "application/json") { + void response.body?.cancel().catch(() => {}); throw protocolError(502, "T4 API returned an undeclared success media type"); } const text = await boundedResponseText(response.clone(), MAX_JSON_RESPONSE_BYTES); let value: unknown; try { value = text === undefined ? undefined : JSON.parse(text); } catch { value = undefined; } if (text === undefined || !validator(value)) { + void response.body?.cancel().catch(() => {}); throw protocolError(502, "T4 API returned an invalid or oversized JSON response"); } } @@ -635,10 +652,19 @@ async function* watch( if (cursor !== undefined) headers.set("Last-Event-ID", cursor); const response = await fetchImpl(url, { method: "GET", headers, signal: controller.signal }); if (!response.ok) { + if (!EVENTS_RESPONSE.errors.includes(response.status)) { + void response.body?.cancel().catch(() => {}); + throw protocolError(502, "T4 API returned an undeclared watch error status"); + } const error = await parsedError(response); - throw error ?? protocolError(response.status, "T4 API returned an invalid or oversized error envelope"); + throw error ?? protocolError(502, "T4 API returned an invalid or oversized error envelope"); + } + if (response.status !== 200) { + void response.body?.cancel().catch(() => {}); + throw protocolError(502, "T4 API returned an undeclared watch success status"); } if (response.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase() !== "text/event-stream") { + void response.body?.cancel().catch(() => {}); throw protocolError(502, "T4 API watch did not return text/event-stream"); } if (response.body === null) throw protocolError(502, "T4 API watch response body is unavailable"); From ecc29a08443897a7507285a3abdaa2b1aead1f4e Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 21:33:07 +0000 Subject: [PATCH 021/131] fix: close watch retry and body ownership gaps --- .../client/test/t4-api-v1-conformance.test.ts | 44 +++++++++++-------- packages/t4-api-client/src/index.ts | 11 +++-- 2 files changed, 34 insertions(+), 21 deletions(-) diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index 3aac3a79..294921f6 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -500,26 +500,20 @@ describe("generated T4 API v1 client conformance", () => { params: { header: VERSION_HEADERS, path: { workspaceId: "missing" } }, })).rejects.toMatchObject({ code: "indeterminate", status: 404 }); - vi.useFakeTimers(); - try { + const streamedChunk = new Uint8Array(600 * 1024); + for (const contentType of ["application/json", "text/plain"]) { let cancelled = false; - const chunk = new Uint8Array(600 * 1024); const streamedClient = createT4ApiClient({ baseUrl: "https://streamed-error.test", credential: "token-a", majorVersion: 1, fetch: async () => new Response(new ReadableStream({ - pull(controller) { controller.enqueue(chunk); }, + pull(controller) { controller.enqueue(streamedChunk); }, cancel() { cancelled = true; }, - }), { status: 404, headers: { "Content-Type": "application/json" } }), + }), { status: 404, headers: { "Content-Type": contentType } }), }); - const outcome = streamedClient.http.GET("/v1/workspaces/{workspaceId}", { + await expect(streamedClient.http.GET("/v1/workspaces/{workspaceId}", { params: { header: VERSION_HEADERS, path: { workspaceId: "missing" } }, - }).then(() => "resolved" as const, () => "rejected" as const); - const bounded = Promise.race([outcome, new Promise<"timeout">((resolve) => setTimeout(() => resolve("timeout"), 1_000))]); - await vi.advanceTimersByTimeAsync(1_000); - await expect(bounded).resolves.toBe("rejected"); + })).rejects.toMatchObject({ code: "indeterminate", status: 404 }); expect(cancelled).toBe(true); - } finally { - vi.useRealTimers(); } const statusClient = createT4ApiClient({ @@ -644,6 +638,19 @@ describe("generated T4 API v1 client conformance", () => { }); expect(attempts).toBe(1); } + + let nonJsonCancelled = false; + const nonJsonClient = createT4ApiClient({ + baseUrl: "https://invalid-503-media.test", credential: "token-a", majorVersion: 1, + fetch: async () => new Response(new ReadableStream({ + pull(controller) { controller.enqueue(new Uint8Array(1024)); }, + cancel() { nonJsonCancelled = true; }, + }), { status: 503, headers: { "Content-Type": "text/plain" } }), + }); + await expect(nonJsonClient.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 3, retryBackoffMs: 0 }).next()).rejects.toMatchObject({ + code: "indeterminate", status: 502, retryable: false, + }); + expect(nonJsonCancelled).toBe(true); }); it("rejects unknown fields in every public JSON response family", async () => { @@ -721,16 +728,17 @@ describe("generated T4 API v1 client conformance", () => { let progressAttempts = 0; const progressFetch: typeof globalThis.fetch = async () => { progressAttempts += 1; - if (progressAttempts === 2) { - return new Response('data: {"type":"heartbeat","cursor":"progress-1","observedAt":"2026-07-21T00:00:00Z"}\n\n', { headers: { "Content-Type": "text/event-stream" } }); + if (progressAttempts === 2 || progressAttempts === 4) { + const cursor = progressAttempts === 2 ? "progress-1" : "progress-2"; + return new Response(`data: {"type":"heartbeat","cursor":"${cursor}","observedAt":"2026-07-21T00:00:00Z"}\n\n`, { headers: { "Content-Type": "text/event-stream" } }); } return new Response(new ReadableStream({ start(controller) { controller.close(); } }), { headers: { "Content-Type": "text/event-stream" } }); }; const progressClient = createT4ApiClient({ baseUrl: "https://progress.test", credential: "token-a", majorVersion: 1, fetch: progressFetch }); - const progressWatch = progressClient.watchSession("ses-1", { maxEvents: 2, maxReconnectAttempts: 1, retryBackoffMs: 0 }); - await expect(progressWatch.next()).resolves.toMatchObject({ value: { cursor: "progress-1" }, done: false }); - await expect(progressWatch.next()).rejects.toMatchObject({ code: "indeterminate", status: 502 }); - expect(progressAttempts).toBe(3); + const progressEvents: WatchEvent[] = []; + for await (const event of progressClient.watchSession("ses-1", { maxEvents: 2, maxReconnectAttempts: 1, retryBackoffMs: 0 })) progressEvents.push(event); + expect(progressEvents.map((event) => event.cursor)).toEqual(["progress-1", "progress-2"]); + expect(progressAttempts).toBe(4); vi.useFakeTimers(); try { diff --git a/packages/t4-api-client/src/index.ts b/packages/t4-api-client/src/index.ts index cdaab911..aa5f5112 100644 --- a/packages/t4-api-client/src/index.ts +++ b/packages/t4-api-client/src/index.ts @@ -235,7 +235,10 @@ function protocolError(status: number, message: string): T4ApiError { } async function parsedError(response: Response): Promise { - if (response.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase() !== "application/json") return undefined; + if (response.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase() !== "application/json") { + void response.body?.cancel().catch(() => {}); + return undefined; + } const text = await boundedResponseText(response); if (text === undefined) return undefined; try { @@ -635,6 +638,7 @@ async function* watch( let cursor = options.cursor; try { while (delivered < maxEvents && !controller.signal.aborted) { + const deliveredAtAttemptStart = delivered; let reader: ReadableStreamDefaultReader | undefined; let transientFailure: unknown; let retryAfterMs = 0; @@ -698,11 +702,12 @@ async function* watch( reader.releaseLock(); } } - if (reconnectAttempts >= maxReconnectAttempts) { + const madeProgress = delivered > deliveredAtAttemptStart; + if (!madeProgress && reconnectAttempts >= maxReconnectAttempts) { throw new T4ApiError(502, { code: "indeterminate", message: "T4 API watch reconnect attempts exhausted", requestId: "unavailable", retryable: true }, { cause: transientFailure }); } const delay = Math.min(30_000, Math.max(retryAfterMs, retryBackoffMs * (2 ** reconnectAttempts))); - reconnectAttempts += 1; + if (!madeProgress) reconnectAttempts += 1; if (!await retryDelay(delay, controller.signal)) return; } } finally { From fb42ef8e3bb84125f6febee44c9e6728e538894c Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 21:40:18 +0000 Subject: [PATCH 022/131] test: prove watch media and stream cleanup --- .../client/test/t4-api-v1-conformance.test.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index 294921f6..694be793 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -599,6 +599,7 @@ describe("generated T4 API v1 client conformance", () => { } for (const response of [ + new Response('data: {"type":"heartbeat","cursor":"wrong-media-1","observedAt":"2026-07-21T00:00:00Z"}\n\n', { status: 200, headers: { "Content-Type": "application/json" } }), new Response('data: {"type":"heartbeat","cursor":"wrong-1","observedAt":"2026-07-21T00:00:00Z"}\n\n', { status: 201, headers: { "Content-Type": "text/event-stream" } }), new Response(null, { status: 204, headers: { "Content-Type": "text/event-stream" } }), Response.json({ error: { code: "revision_conflict", message: "bad", requestId: "r", retryable: false } }, { status: 409 }), @@ -639,6 +640,20 @@ describe("generated T4 API v1 client conformance", () => { expect(attempts).toBe(1); } + let oversizedCancelled = false; + const oversizedChunk = new Uint8Array(600 * 1024); + const streamedOversizedClient = createT4ApiClient({ + baseUrl: "https://streamed-invalid-503.test", credential: "token-a", majorVersion: 1, + fetch: async () => new Response(new ReadableStream({ + pull(controller) { controller.enqueue(oversizedChunk); }, + cancel() { oversizedCancelled = true; }, + }), { status: 503, headers: { "Content-Type": "application/json" } }), + }); + await expect(streamedOversizedClient.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 3, retryBackoffMs: 0 }).next()).rejects.toMatchObject({ + code: "indeterminate", status: 502, retryable: false, + }); + expect(oversizedCancelled).toBe(true); + let nonJsonCancelled = false; const nonJsonClient = createT4ApiClient({ baseUrl: "https://invalid-503-media.test", credential: "token-a", majorVersion: 1, From ed2c7f7b65e3453ae4971c4a3a1e18bfd1a5a578 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 21:47:19 +0000 Subject: [PATCH 023/131] test: align terminal watch protocol assertions --- packages/client/test/t4-api-v1-conformance.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index 694be793..badf6807 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -456,12 +456,12 @@ describe("generated T4 API v1 client conformance", () => { for (const [status, code] of [[410, "cursor_expired"], [406, "incompatible_version"], [422, "invalid_request"]] as const) { const fetch: typeof globalThis.fetch = async () => Response.json({ error: { code, message: "incomplete", requestId: "r", retryable: false } }, { status }); const client = createT4ApiClient({ baseUrl: "https://errors.test", credential: "token-a", majorVersion: 1, fetch }); - await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ code: "indeterminate", status }); + await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ code: "indeterminate", status: 502, retryable: false }); } for (const [status, code] of [[401, "not_found"], [403, "unauthenticated"], [404, "forbidden"], [409, "unavailable"], [503, "revision_conflict"]] as const) { const fetch: typeof globalThis.fetch = async () => Response.json({ error: { code, message: "wrong status class", requestId: "r", retryable: false } }, { status }); const client = createT4ApiClient({ baseUrl: "https://status-errors.test", credential: "token-a", majorVersion: 1, fetch }); - await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ code: "indeterminate", status: status === 503 ? 502 : status }); + await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ code: "indeterminate", status: 502, retryable: false }); } }); From 7673616763d5fd204c7e7ca83e4e461b76d1538a Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 22:25:24 +0000 Subject: [PATCH 024/131] test: cover Unicode bounds and SSE cancellation --- .../client/test/t4-api-v1-conformance.test.ts | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index badf6807..116edc14 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -447,6 +447,19 @@ describe("generated T4 API v1 client conformance", () => { const oversizedService = new T4ApiV1ConformanceService({ watchTransport: "oversized" }); const oversizedClient = await seededClient(oversizedService, "oversized"); await expect(oversizedClient.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ code: "indeterminate", status: 502 }); + + let ongoingOversizedCancelled = false; + const ongoingOversizedClient = createT4ApiClient({ + baseUrl: "https://ongoing-oversized-event.test", credential: "token-a", majorVersion: 1, + fetch: async () => new Response(new ReadableStream({ + pull(controller) { controller.enqueue(new Uint8Array(600 * 1024)); }, + cancel() { ongoingOversizedCancelled = true; }, + }), { status: 200, headers: { "Content-Type": "text/event-stream" } }), + }); + await expect(ongoingOversizedClient.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ + code: "indeterminate", status: 502, retryable: false, + }); + expect(ongoingOversizedCancelled).toBe(true); }); it("fails closed on unknown watch fields and incomplete typed errors", async () => { @@ -558,6 +571,41 @@ describe("generated T4 API v1 client conformance", () => { })); expect(unicode.entries[0]?.text).toBe(astralText); + const astral128 = "😀".repeat(128); + const unicodeWorkspace = createT4ApiClient({ + baseUrl: "https://unicode-workspace.test", credential: "token-a", majorVersion: 1, + fetch: async () => Response.json({ id: "wsp-1", name: astral128, state: "ready", revision: 1, labels: { team: astral128 } }), + }); + const workspace = requireData(await unicodeWorkspace.http.GET("/v1/workspaces/{workspaceId}", { + params: { header: VERSION_HEADERS, path: { workspaceId: "wsp-1" } }, + })); + expect(workspace).toMatchObject({ name: astral128, labels: { team: astral128 } }); + + const unicodeSession = createT4ApiClient({ + baseUrl: "https://unicode-session.test", credential: "token-a", majorVersion: 1, + fetch: async () => Response.json({ id: "ses-1", workspaceId: "wsp-1", title: astral128, state: "ready", revision: 1, labels: { team: astral128 } }), + }); + const session = requireData(await unicodeSession.http.GET("/v1/sessions/{sessionId}", { + params: { header: VERSION_HEADERS, path: { sessionId: "ses-1" } }, + })); + expect(session).toMatchObject({ title: astral128, labels: { team: astral128 } }); + + const errorMessage = "😀".repeat(1024); + const errorRequestId = "😀".repeat(128); + const violationField = "😀".repeat(256); + const violationMessage = "😀".repeat(512); + const unicodeError = createT4ApiClient({ + baseUrl: "https://unicode-error.test", credential: "token-a", majorVersion: 1, + fetch: async () => Response.json({ error: { + code: "invalid_request", message: errorMessage, requestId: errorRequestId, retryable: false, + violations: [{ field: violationField, rule: "range", message: violationMessage }], + } }, { status: 422 }), + }); + await expect(unicodeError.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ + code: "invalid_request", status: 422, message: errorMessage, requestId: errorRequestId, + violations: [{ field: violationField, rule: "range", message: violationMessage }], + }); + for (const response of [ new Response(JSON.stringify(discovery), { status: 200, headers: { "Content-Type": "text/plain" } }), Response.json(discovery, { status: 201 }), From 578e3c4081a5509e17ee5d2b5882bfa505fec727 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 22:47:19 +0000 Subject: [PATCH 025/131] fix: validate response lengths by code point --- packages/t4-api-client/src/index.ts | 32 ++++++++++++++--------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/packages/t4-api-client/src/index.ts b/packages/t4-api-client/src/index.ts index aa5f5112..feb91ee6 100644 --- a/packages/t4-api-client/src/index.ts +++ b/packages/t4-api-client/src/index.ts @@ -154,16 +154,16 @@ function validViolation(value: unknown): boolean { const violation = record(value); return violation !== undefined && Object.keys(violation).every((key) => key === "field" || key === "rule" || key === "message") && - typeof violation.field === "string" && violation.field.length >= 1 && violation.field.length <= 256 && - typeof violation.rule === "string" && /^[A-Za-z][A-Za-z0-9._-]{0,63}$/u.test(violation.rule) && - typeof violation.message === "string" && violation.message.length >= 1 && violation.message.length <= 512; + typeof violation.field === "string" && violation.field !== "" && hasAtMostCodePoints(violation.field, 256) && + typeof violation.rule === "string" && hasAtMostCodePoints(violation.rule, 64) && /^[A-Za-z][A-Za-z0-9._-]{0,63}$/u.test(violation.rule) && + typeof violation.message === "string" && violation.message !== "" && hasAtMostCodePoints(violation.message, 512); } function validResync(value: unknown): value is Resync { const resync = record(value); return resync !== undefined && Object.keys(resync).every((key) => key === "snapshotUrl" || key === "cursor") && - typeof resync.snapshotUrl === "string" && /^\/v1\/sessions\/[A-Za-z0-9._~-]+\/snapshot$/u.test(resync.snapshotUrl) && - typeof resync.cursor === "string" && resync.cursor.length <= 512 && CURSOR_PATTERN.test(resync.cursor); + typeof resync.snapshotUrl === "string" && hasAtMostCodePoints(resync.snapshotUrl, 512) && /^\/v1\/sessions\/[A-Za-z0-9._~-]+\/snapshot$/u.test(resync.snapshotUrl) && + typeof resync.cursor === "string" && hasAtMostCodePoints(resync.cursor, 512) && CURSOR_PATTERN.test(resync.cursor); } function apiError(value: unknown, status: number): ApiError | undefined { @@ -174,8 +174,8 @@ function apiError(value: unknown, status: number): ApiError | undefined { error === undefined || Object.keys(error).some((key) => ERROR_FIELDS[key] !== true) || typeof error.code !== "string" || ERROR_CODES[error.code as ApiError["code"]] !== true || ERROR_CODES_BY_STATUS[status]?.[error.code] !== true || - typeof error.message !== "string" || error.message.length < 1 || error.message.length > 1024 || - typeof error.requestId !== "string" || error.requestId.length < 1 || error.requestId.length > 128 || + typeof error.message !== "string" || error.message === "" || !hasAtMostCodePoints(error.message, 1024) || + typeof error.requestId !== "string" || error.requestId === "" || !hasAtMostCodePoints(error.requestId, 128) || typeof error.retryable !== "boolean" || (error.violations !== undefined && (!Array.isArray(error.violations) || error.violations.length > 64 || !error.violations.every(validViolation))) || (error.supportedMajors !== undefined && (!Array.isArray(error.supportedMajors) || error.supportedMajors.length > 8 || !error.supportedMajors.every((major) => Number.isSafeInteger(major) && Number(major) >= 1))) || @@ -259,7 +259,7 @@ function hasOnlyKeys(value: Record, keys: Readonly= 1 && value.length <= 128 && /^[A-Za-z0-9][A-Za-z0-9._~-]*$/u.test(value); + return typeof value === "string" && value !== "" && hasAtMostCodePoints(value, 128) && /^[A-Za-z0-9][A-Za-z0-9._~-]*$/u.test(value); } function hasAtMostCodePoints(value: string, maximum: number): boolean { @@ -272,20 +272,20 @@ function hasAtMostCodePoints(value: string, maximum: number): boolean { } function validCursor(value: unknown): value is string { - return typeof value === "string" && value.length >= 1 && value.length <= 512 && CURSOR_PATTERN.test(value); + return typeof value === "string" && value !== "" && hasAtMostCodePoints(value, 512) && CURSOR_PATTERN.test(value); } function validLabels(value: unknown): boolean { if (value === undefined) return true; const labels = record(value); return labels !== undefined && Object.keys(labels).length <= 32 && Object.entries(labels).every(([key, item]) => - /^[a-z][a-z0-9.-]{0,62}$/u.test(key) && typeof item === "string" && item.length <= 128); + /^[a-z][a-z0-9.-]{0,62}$/u.test(key) && typeof item === "string" && hasAtMostCodePoints(item, 128)); } function validWorkspace(value: unknown): boolean { const item = record(value); return item !== undefined && hasOnlyKeys(item, { id: true, name: true, state: true, revision: true, labels: true }) && - validResourceId(item.id) && typeof item.name === "string" && item.name.length >= 1 && item.name.length <= 128 && + validResourceId(item.id) && typeof item.name === "string" && item.name !== "" && hasAtMostCodePoints(item.name, 128) && typeof item.state === "string" && WORKSPACE_STATES[item.state as components["schemas"]["WorkspaceState"]] === true && Number.isSafeInteger(item.revision) && Number(item.revision) >= 1 && validLabels(item.labels); } @@ -293,7 +293,7 @@ function validWorkspace(value: unknown): boolean { function validSession(value: unknown): boolean { const item = record(value); return item !== undefined && hasOnlyKeys(item, { id: true, workspaceId: true, title: true, state: true, revision: true, labels: true }) && - validResourceId(item.id) && validResourceId(item.workspaceId) && typeof item.title === "string" && item.title.length >= 1 && item.title.length <= 128 && + validResourceId(item.id) && validResourceId(item.workspaceId) && typeof item.title === "string" && item.title !== "" && hasAtMostCodePoints(item.title, 128) && typeof item.state === "string" && SESSION_STATES[item.state as components["schemas"]["SessionState"]] === true && Number.isSafeInteger(item.revision) && Number(item.revision) >= 1 && validLabels(item.labels); } @@ -464,15 +464,15 @@ function watchEvent(value: unknown, eventId: string | undefined): WatchEvent { event === undefined || typeof event.type !== "string" || typeof event.cursor !== "string" || - event.cursor.length < 1 || - event.cursor.length > 512 || + event.cursor === "" || + !hasAtMostCodePoints(event.cursor, 512) || !CURSOR_PATTERN.test(event.cursor) || (eventId !== undefined && eventId !== event.cursor) ) throw protocolError(502, "T4 API returned an invalid watch event"); if ( event.type === "heartbeat" && hasOnlyKeys(event, { type: true, cursor: true, observedAt: true }) && - typeof event.observedAt === "string" && event.observedAt.length <= 64 && + typeof event.observedAt === "string" && hasAtMostCodePoints(event.observedAt, 64) && /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/u.test(event.observedAt) && Number.isFinite(Date.parse(event.observedAt)) ) return event as WatchEvent; @@ -487,7 +487,7 @@ function watchEvent(value: unknown, eventId: string | undefined): WatchEvent { event.type === "command" && hasOnlyKeys(event, { type: true, cursor: true, commandId: true, state: true }) && typeof event.commandId === "string" && - event.commandId.length >= 1 && event.commandId.length <= 128 && + event.commandId !== "" && hasAtMostCodePoints(event.commandId, 128) && /^[A-Za-z0-9][A-Za-z0-9._~-]*$/u.test(event.commandId) && typeof event.state === "string" && OPERATION_STATES[event.state as components["schemas"]["OperationState"]] === true From b555a13dd6ee3023fb6b94d775d542dcae193be4 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 22:58:56 +0000 Subject: [PATCH 026/131] test: cover strict response protocol gaps --- .../client/test/t4-api-v1-conformance.test.ts | 155 ++++++++++++++++++ 1 file changed, 155 insertions(+) diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index 116edc14..6e294cf1 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -743,6 +743,161 @@ describe("generated T4 API v1 client conformance", () => { })).rejects.toMatchObject({ code: "indeterminate", status: 502 }); }); + it("requires declared selected-version and replay response headers", async () => { + const discovery = { + apiVersion: "1.0", supportedMajors: [1], capabilities: [], + limits: { pageSizeDefault: 1, pageSizeMax: 1, commandBytesMax: 1, commandRequestBytesMax: 1, commandMetadataValueBytesMax: 1, watchEventsMax: 1, heartbeatSeconds: 5 }, + }; + for (const selectedVersion of [undefined, "1", "2.0", `1.${"0".repeat(15)}`]) { + const client = createT4ApiClient({ + baseUrl: "https://response-version.test", credential: "token-a", majorVersion: 1, + fetch: async () => Response.json(discovery, selectedVersion === undefined ? {} : { headers: { "T4-API-Version": selectedVersion } }), + }); + await expect(client.http.GET("/v1", { params: { header: VERSION_HEADERS } })).rejects.toMatchObject({ status: 502, code: "indeterminate", retryable: false }); + } + + const unauthenticated = createT4ApiClient({ + baseUrl: "https://response-version.test", credential: "token-a", majorVersion: 1, + fetch: async () => Response.json({ error: { code: "unauthenticated", message: "bad", requestId: "r", retryable: false } }, { status: 401 }), + }); + expect((await unauthenticated.http.GET("/v1", { params: { header: VERSION_HEADERS } })).response.status).toBe(401); + + const workspace = { id: "wsp-1", name: "workspace", state: "ready", revision: 1 }; + const missingReplay = createT4ApiClient({ + baseUrl: "https://response-replay.test", credential: "token-a", majorVersion: 1, + fetch: async () => Response.json(workspace, { headers: { "T4-API-Version": "1.0" } }), + }); + await expect(missingReplay.http.PATCH("/v1/workspaces/{workspaceId}", { + params: { header: mutationHeaders(1, "response-replay-patch"), path: { workspaceId: "wsp-1" } }, body: { name: "workspace" }, + })).rejects.toMatchObject({ status: 502, retryable: false }); + + for (const replayed of [undefined, "TRUE", "0"]) { + const client = createT4ApiClient({ + baseUrl: "https://delete-replay-header.test", credential: "token-a", majorVersion: 1, + fetch: async () => new Response(null, { status: 204, headers: { + "T4-API-Version": "1.0", ...(replayed === undefined ? {} : { "Idempotency-Replayed": replayed }), + } }), + }); + await expect(client.http.DELETE("/v1/workspaces/{workspaceId}", { + params: { header: idempotencyHeaders("response-delete-key"), path: { workspaceId: "wsp-1" } }, + })).rejects.toMatchObject({ status: 502, retryable: false }); + } + + const missingWatchVersion = createT4ApiClient({ + baseUrl: "https://watch-response-version.test", credential: "token-a", majorVersion: 1, + fetch: async () => new Response('data: {"type":"heartbeat","cursor":"c1","observedAt":"2026-07-21T00:00:00Z"}\n\n', { headers: { "Content-Type": "text/event-stream" } }), + }); + await expect(missingWatchVersion.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ status: 502, retryable: false }); + + const missingWatchErrorVersion = createT4ApiClient({ + baseUrl: "https://watch-error-version.test", credential: "token-a", majorVersion: 1, + fetch: async () => Response.json({ error: { code: "unavailable", message: "later", requestId: "r", retryable: true } }, { status: 503 }), + }); + await expect(missingWatchErrorVersion.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0, retryBackoffMs: 0 }).next()).rejects.toMatchObject({ status: 502, retryable: false }); + }); + + it("rejects duplicate unique response arrays", async () => { + const limits = { pageSizeDefault: 1, pageSizeMax: 1, commandBytesMax: 1, commandRequestBytesMax: 1, commandMetadataValueBytesMax: 1, watchEventsMax: 1, heartbeatSeconds: 5 }; + for (const discovery of [ + { apiVersion: "1.0", supportedMajors: [1, 1], capabilities: [], limits }, + { apiVersion: "1.0", supportedMajors: [1], capabilities: ["session.watch.sse", "session.watch.sse"], limits }, + ]) { + const client = createT4ApiClient({ + baseUrl: "https://duplicate-discovery.test", credential: "token-a", majorVersion: 1, + fetch: async () => Response.json(discovery, { headers: { "T4-API-Version": "1.0" } }), + }); + await expect(client.http.GET("/v1", { params: { header: VERSION_HEADERS } })).rejects.toMatchObject({ status: 502, retryable: false }); + } + const duplicateError = createT4ApiClient({ + baseUrl: "https://duplicate-error.test", credential: "token-a", majorVersion: 1, + fetch: async () => Response.json({ error: { + code: "incompatible_version", message: "bad", requestId: "r", retryable: false, supportedMajors: [1, 1], + } }, { status: 406, headers: { "T4-API-Version": "1.0" } }), + }); + await expect(duplicateError.http.GET("/v1", { params: { header: VERSION_HEADERS } })).rejects.toMatchObject({ status: 406, code: "indeterminate", retryable: false }); + }); + + it("discards unterminated SSE data at EOF before reconnecting", async () => { + for (const unterminated of [ + 'data: {"type":"heartbeat","cursor":"unterminated","observedAt":"2026-07-21T00:00:00Z"}', + "data: {not-json", + ]) { + let attempts = 0; + const client = createT4ApiClient({ + baseUrl: "https://unterminated-sse.test", credential: "token-a", majorVersion: 1, + fetch: async () => { + attempts += 1; + const body = attempts === 1 + ? unterminated + : 'data: {"type":"heartbeat","cursor":"complete","observedAt":"2026-07-21T00:00:00Z"}\n\n'; + return new Response(body, { headers: { "Content-Type": "text/event-stream", "T4-API-Version": "1.0" } }); + }, + }); + await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 1, retryBackoffMs: 0 }).next()).resolves.toMatchObject({ value: { cursor: "complete" } }); + expect(attempts).toBe(2); + } + }); + + it("accepts exact code-point request lengths and rejects one more", async () => { + const service = new T4ApiV1ConformanceService(); + const client = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); + const exact = "😀".repeat(128); + const workspace = await client.http.POST("/v1/workspaces", { + body: { name: exact }, params: { header: idempotencyHeaders("unicode-workspace-exact") }, + }); + expect(workspace.response.status).toBe(202); + expect((await client.http.POST("/v1/workspaces", { + body: { name: `${exact}😀` }, params: { header: idempotencyHeaders("unicode-workspace-over") }, + })).response.status).toBe(422); + const session = await client.http.POST("/v1/workspaces/{workspaceId}/sessions", { + body: { title: exact }, params: { header: idempotencyHeaders("unicode-session-exact"), path: { workspaceId: requireData(workspace).id } }, + }); + expect(session.response.status).toBe(202); + expect((await client.http.POST("/v1/workspaces/{workspaceId}/sessions", { + body: { title: `${exact}😀` }, params: { header: idempotencyHeaders("unicode-session-over"), path: { workspaceId: requireData(workspace).id } }, + })).response.status).toBe(422); + }); + + it("fails terminally and cancels an invalid UTF-8 SSE body", async () => { + let cancelled = false; + const client = createT4ApiClient({ + baseUrl: "https://invalid-utf8-sse.test", credential: "token-a", majorVersion: 1, + fetch: async () => new Response(new ReadableStream({ + start(controller) { controller.enqueue(new Uint8Array([100, 97, 116, 97, 58, 32, 195, 40, 10, 10])); }, + cancel() { cancelled = true; }, + }), { headers: { "Content-Type": "text/event-stream", "T4-API-Version": "1.0" } }), + }); + await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 3 }).next()).rejects.toMatchObject({ status: 502, code: "indeterminate", retryable: false }); + expect(cancelled).toBe(true); + }); + + it("scopes one DELETE key independently by live target ID", async () => { + const service = new T4ApiV1ConformanceService(); + const client = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); + for (const [key, name] of [["delete-target-workspace-a", "first"], ["delete-target-workspace-b", "second"]] as const) { + requireData(await client.http.POST("/v1/workspaces", { body: { name }, params: { header: idempotencyHeaders(key) } })); + } + for (const workspaceId of ["ws-1", "ws-2"] as const) { + expect((await client.http.DELETE("/v1/workspaces/{workspaceId}", { + params: { header: idempotencyHeaders("delete-shared-workspace"), path: { workspaceId } }, + })).response.status).toBe(204); + } + + const sessionService = new T4ApiV1ConformanceService(); + const sessionClient = createT4ApiClient({ baseUrl: sessionService.origin, credential: "token-a", majorVersion: 1, fetch: sessionService.fetch }); + requireData(await sessionClient.http.POST("/v1/workspaces", { body: { name: "workspace" }, params: { header: idempotencyHeaders("delete-session-workspace") } })); + for (const [key, title] of [["delete-target-session-a", "first"], ["delete-target-session-b", "second"]] as const) { + requireData(await sessionClient.http.POST("/v1/workspaces/{workspaceId}/sessions", { + body: { title }, params: { header: idempotencyHeaders(key), path: { workspaceId: "ws-1" } }, + })); + } + for (const sessionId of ["ses-1", "ses-2"] as const) { + expect((await sessionClient.http.DELETE("/v1/sessions/{sessionId}", { + params: { header: idempotencyHeaders("delete-shared-session"), path: { sessionId } }, + })).response.status).toBe(204); + } + }); + it("bounds automatic EOF retry and supports explicit abort", async () => { let attempts = 0; const eofFetch: typeof globalThis.fetch = async () => { From ac595046faf68bdce5f76a5dec9995adda6d0bf1 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 23:04:00 +0000 Subject: [PATCH 027/131] fix: enforce declared response protocol --- .../test/t4-api-v1-conformance-service.ts | 13 ++- .../client/test/t4-api-v1-conformance.test.ts | 108 ++++++++++-------- packages/t4-api-client/src/index.ts | 71 +++++++++--- packages/t4-api-contract/openapi.json | 2 +- 4 files changed, 128 insertions(+), 66 deletions(-) diff --git a/packages/client/test/t4-api-v1-conformance-service.ts b/packages/client/test/t4-api-v1-conformance-service.ts index af097f22..d7552c8d 100644 --- a/packages/client/test/t4-api-v1-conformance-service.ts +++ b/packages/client/test/t4-api-v1-conformance-service.ts @@ -5,6 +5,15 @@ const COMMAND_BYTES_MAX = 32; const COMMAND_REQUEST_BYTES_MAX = 256; const METADATA_VALUE_BYTES_MAX = 32; +function hasAtMostCodePoints(value: string, maximum: number): boolean { + let count = 0; + for (const _codePoint of value) { + count += 1; + if (count > maximum) return false; + } + return true; +} + function json(status: number, body: unknown, headers: Record = {}): Response { return Response.json(body, { status, headers: { "T4-API-Version": "1.0", ...headers } }); } @@ -98,7 +107,7 @@ export class T4ApiV1ConformanceService { const parsed = await this.#jsonBody(request); if (parsed instanceof Response) return parsed; const body = parsed; - if (typeof body.name !== "string" || body.name.length < 1 || body.name.length > 128) return this.#invalid("name", "length", "name must contain 1 to 128 characters"); + if (typeof body.name !== "string" || body.name === "" || !hasAtMostCodePoints(body.name, 128)) return this.#invalid("name", "length", "name must contain 1 to 128 characters"); return this.#idempotent(request, tenant, "createWorkspace", [], body, 202, 200, () => { const id = `ws-${++this.#workspaceSequence}`; const workspace = { id, name: body.name, ...(body.labels === undefined ? {} : { labels: body.labels }), state: "accepted", revision: 1, tenant }; @@ -150,7 +159,7 @@ export class T4ApiV1ConformanceService { const parsed = await this.#jsonBody(request); if (parsed instanceof Response) return parsed; const body = parsed; - if (typeof body.title !== "string" || body.title.length < 1 || body.title.length > 128) return this.#invalid("title", "length", "title must contain 1 to 128 characters"); + if (typeof body.title !== "string" || body.title === "" || !hasAtMostCodePoints(body.title, 128)) return this.#invalid("title", "length", "title must contain 1 to 128 characters"); return this.#idempotent(request, tenant, "spawnSession", [workspaceId], body, 202, 200, () => { const id = `ses-${++this.#sessionSequence}`; const session = { id, workspaceId, title: body.title, state: "accepted", revision: 1, tenant }; diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index 6e294cf1..5c0f5a9c 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -48,6 +48,20 @@ function exhaustWatchEvent(event: WatchEvent): string { const VERSION_HEADERS = { "T4-API-Version": "1" } as const; +function withSelectedVersion(init: ResponseInit = {}): ResponseInit { + const headers = new Headers(init.headers); + if (!headers.has("T4-API-Version")) headers.set("T4-API-Version", "1.0"); + return { ...init, headers }; +} + +function jsonResponse(body: unknown, init: ResponseInit = {}): Response { + return Response.json(body, withSelectedVersion(init)); +} + +function apiResponse(body?: BodyInit | null, init: ResponseInit = {}): Response { + return new Response(body, withSelectedVersion(init)); +} + function idempotencyHeaders(key: string): Readonly<{ "T4-API-Version": "1"; "Idempotency-Key": string }> { return { ...VERSION_HEADERS, "Idempotency-Key": key }; } @@ -451,7 +465,7 @@ describe("generated T4 API v1 client conformance", () => { let ongoingOversizedCancelled = false; const ongoingOversizedClient = createT4ApiClient({ baseUrl: "https://ongoing-oversized-event.test", credential: "token-a", majorVersion: 1, - fetch: async () => new Response(new ReadableStream({ + fetch: async () => apiResponse(new ReadableStream({ pull(controller) { controller.enqueue(new Uint8Array(600 * 1024)); }, cancel() { ongoingOversizedCancelled = true; }, }), { status: 200, headers: { "Content-Type": "text/event-stream" } }), @@ -463,16 +477,16 @@ describe("generated T4 API v1 client conformance", () => { }); it("fails closed on unknown watch fields and incomplete typed errors", async () => { - const eventFetch: typeof globalThis.fetch = async () => new Response('data: {"type":"heartbeat","cursor":"c1","observedAt":"2026-07-21T00:00:00Z","unknown":true}\n\n', { headers: { "Content-Type": "text/event-stream" } }); + const eventFetch: typeof globalThis.fetch = async () => apiResponse('data: {"type":"heartbeat","cursor":"c1","observedAt":"2026-07-21T00:00:00Z","unknown":true}\n\n', { headers: { "Content-Type": "text/event-stream" } }); const eventClient = createT4ApiClient({ baseUrl: "https://unknown.test", credential: "token-a", majorVersion: 1, fetch: eventFetch }); await expect(eventClient.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ code: "indeterminate", status: 502 }); for (const [status, code] of [[410, "cursor_expired"], [406, "incompatible_version"], [422, "invalid_request"]] as const) { - const fetch: typeof globalThis.fetch = async () => Response.json({ error: { code, message: "incomplete", requestId: "r", retryable: false } }, { status }); + const fetch: typeof globalThis.fetch = async () => jsonResponse({ error: { code, message: "incomplete", requestId: "r", retryable: false } }, { status }); const client = createT4ApiClient({ baseUrl: "https://errors.test", credential: "token-a", majorVersion: 1, fetch }); await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ code: "indeterminate", status: 502, retryable: false }); } for (const [status, code] of [[401, "not_found"], [403, "unauthenticated"], [404, "forbidden"], [409, "unavailable"], [503, "revision_conflict"]] as const) { - const fetch: typeof globalThis.fetch = async () => Response.json({ error: { code, message: "wrong status class", requestId: "r", retryable: false } }, { status }); + const fetch: typeof globalThis.fetch = async () => jsonResponse({ error: { code, message: "wrong status class", requestId: "r", retryable: false } }, { status }); const client = createT4ApiClient({ baseUrl: "https://status-errors.test", credential: "token-a", majorVersion: 1, fetch }); await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ code: "indeterminate", status: 502, retryable: false }); } @@ -490,48 +504,48 @@ describe("generated T4 API v1 client conformance", () => { [503, { error: { code: "unavailable", message: "bad", requestId: "r", retryable: true } }], ] as const; for (const [status, body] of validErrors) { - const fetch: typeof globalThis.fetch = async () => Response.json({ error: { ...body.error, unknown: true } }, { status }); + const fetch: typeof globalThis.fetch = async () => jsonResponse({ error: { ...body.error, unknown: true } }, { status }); const client = createT4ApiClient({ baseUrl: "https://ordinary-errors.test", credential: "token-a", majorVersion: 1, fetch }); await expect(client.http.POST("/v1/workspaces", { body: { name: "workspace" }, params: { header: idempotencyHeaders("ordinary-errors-0001") }, - })).rejects.toMatchObject({ code: "indeterminate", status: status === 503 ? 502 : status }); + })).rejects.toMatchObject({ code: "indeterminate", status: 502, retryable: false }); } const malformedClient = createT4ApiClient({ baseUrl: "https://malformed-error.test", credential: "token-a", majorVersion: 1, - fetch: async () => new Response("{", { status: 404, headers: { "Content-Type": "application/json" } }), + fetch: async () => apiResponse("{", { status: 404, headers: { "Content-Type": "application/json" } }), }); await expect(malformedClient.http.GET("/v1/workspaces/{workspaceId}", { params: { header: VERSION_HEADERS, path: { workspaceId: "missing" } }, - })).rejects.toMatchObject({ code: "indeterminate", status: 404 }); + })).rejects.toMatchObject({ code: "indeterminate", status: 502, retryable: false }); const oversizedClient = createT4ApiClient({ baseUrl: "https://oversized-error.test", credential: "token-a", majorVersion: 1, - fetch: async () => new Response("x".repeat(1024 * 1024 + 1), { status: 404, headers: { "Content-Type": "application/json" } }), + fetch: async () => apiResponse("x".repeat(1024 * 1024 + 1), { status: 404, headers: { "Content-Type": "application/json" } }), }); await expect(oversizedClient.http.GET("/v1/workspaces/{workspaceId}", { params: { header: VERSION_HEADERS, path: { workspaceId: "missing" } }, - })).rejects.toMatchObject({ code: "indeterminate", status: 404 }); + })).rejects.toMatchObject({ code: "indeterminate", status: 502, retryable: false }); const streamedChunk = new Uint8Array(600 * 1024); for (const contentType of ["application/json", "text/plain"]) { let cancelled = false; const streamedClient = createT4ApiClient({ baseUrl: "https://streamed-error.test", credential: "token-a", majorVersion: 1, - fetch: async () => new Response(new ReadableStream({ + fetch: async () => apiResponse(new ReadableStream({ pull(controller) { controller.enqueue(streamedChunk); }, cancel() { cancelled = true; }, }), { status: 404, headers: { "Content-Type": contentType } }), }); await expect(streamedClient.http.GET("/v1/workspaces/{workspaceId}", { params: { header: VERSION_HEADERS, path: { workspaceId: "missing" } }, - })).rejects.toMatchObject({ code: "indeterminate", status: 404 }); + })).rejects.toMatchObject({ code: "indeterminate", status: 502, retryable: false }); expect(cancelled).toBe(true); } const statusClient = createT4ApiClient({ baseUrl: "https://undeclared-error.test", credential: "token-a", majorVersion: 1, - fetch: async () => Response.json(validErrors[3][1], { status: 404 }), + fetch: async () => jsonResponse(validErrors[3][1], { status: 404 }), }); await expect(statusClient.http.GET("/v1", { params: { header: VERSION_HEADERS } })).rejects.toMatchObject({ code: "indeterminate", status: 404 }); }); @@ -545,7 +559,7 @@ describe("generated T4 API v1 client conformance", () => { baseUrl: "https://prefixed.test/api", credential: "token-a", majorVersion: 1, fetch: async (input) => { expect(new Request(input).url).toBe("https://prefixed.test/api/v1"); - return Response.json({ ...discovery, unknown: true }); + return jsonResponse({ ...discovery, unknown: true }); }, }); await expect(prefixed.http.GET("/v1", { params: { header: VERSION_HEADERS } })).rejects.toMatchObject({ code: "indeterminate", status: 502 }); @@ -554,7 +568,7 @@ describe("generated T4 API v1 client conformance", () => { baseUrl: "https://snapshot.test/api", credential: "token-a", majorVersion: 1, fetch: async (input) => { expect(new URL(new Request(input).url).pathname).toBe("/api/v1/sessions/ses-1/snapshot"); - return Response.json({ sessionId: "ses-1", cursor: "cursor-1", state: "ready", entries: [], unknown: true }); + return jsonResponse({ sessionId: "ses-1", cursor: "cursor-1", state: "ready", entries: [], unknown: true }); }, }); await expect(invalidSnapshot.http.GET("/v1/sessions/{sessionId}/snapshot", { @@ -564,7 +578,7 @@ describe("generated T4 API v1 client conformance", () => { const astralText = "😀".repeat(524_289); const unicodeSnapshot = createT4ApiClient({ baseUrl: "https://unicode-snapshot.test", credential: "token-a", majorVersion: 1, - fetch: async () => Response.json({ sessionId: "ses-1", cursor: "cursor-1", state: "ready", entries: [{ sequence: 0, kind: "output", text: astralText }] }), + fetch: async () => jsonResponse({ sessionId: "ses-1", cursor: "cursor-1", state: "ready", entries: [{ sequence: 0, kind: "output", text: astralText }] }), }); const unicode = requireData(await unicodeSnapshot.http.GET("/v1/sessions/{sessionId}/snapshot", { params: { header: VERSION_HEADERS, path: { sessionId: "ses-1" } }, @@ -574,7 +588,7 @@ describe("generated T4 API v1 client conformance", () => { const astral128 = "😀".repeat(128); const unicodeWorkspace = createT4ApiClient({ baseUrl: "https://unicode-workspace.test", credential: "token-a", majorVersion: 1, - fetch: async () => Response.json({ id: "wsp-1", name: astral128, state: "ready", revision: 1, labels: { team: astral128 } }), + fetch: async () => jsonResponse({ id: "wsp-1", name: astral128, state: "ready", revision: 1, labels: { team: astral128 } }), }); const workspace = requireData(await unicodeWorkspace.http.GET("/v1/workspaces/{workspaceId}", { params: { header: VERSION_HEADERS, path: { workspaceId: "wsp-1" } }, @@ -583,7 +597,7 @@ describe("generated T4 API v1 client conformance", () => { const unicodeSession = createT4ApiClient({ baseUrl: "https://unicode-session.test", credential: "token-a", majorVersion: 1, - fetch: async () => Response.json({ id: "ses-1", workspaceId: "wsp-1", title: astral128, state: "ready", revision: 1, labels: { team: astral128 } }), + fetch: async () => jsonResponse({ id: "ses-1", workspaceId: "wsp-1", title: astral128, state: "ready", revision: 1, labels: { team: astral128 } }), }); const session = requireData(await unicodeSession.http.GET("/v1/sessions/{sessionId}", { params: { header: VERSION_HEADERS, path: { sessionId: "ses-1" } }, @@ -596,7 +610,7 @@ describe("generated T4 API v1 client conformance", () => { const violationMessage = "😀".repeat(512); const unicodeError = createT4ApiClient({ baseUrl: "https://unicode-error.test", credential: "token-a", majorVersion: 1, - fetch: async () => Response.json({ error: { + fetch: async () => jsonResponse({ error: { code: "invalid_request", message: errorMessage, requestId: errorRequestId, retryable: false, violations: [{ field: violationField, rule: "range", message: violationMessage }], } }, { status: 422 }), @@ -607,8 +621,8 @@ describe("generated T4 API v1 client conformance", () => { }); for (const response of [ - new Response(JSON.stringify(discovery), { status: 200, headers: { "Content-Type": "text/plain" } }), - Response.json(discovery, { status: 201 }), + apiResponse(JSON.stringify(discovery), { status: 200, headers: { "Content-Type": "text/plain" } }), + jsonResponse(discovery, { status: 201 }), ]) { const client = createT4ApiClient({ baseUrl: "https://dispatch.test", credential: "token-a", majorVersion: 1, fetch: async () => response.clone() }); await expect(client.http.GET("/v1", { params: { header: VERSION_HEADERS } })).rejects.toMatchObject({ code: "indeterminate", status: 502 }); @@ -616,7 +630,7 @@ describe("generated T4 API v1 client conformance", () => { const unknownRoute = createT4ApiClient({ baseUrl: "https://dispatch.test", credential: "token-a", majorVersion: 1, - fetch: async () => Response.json(discovery), + fetch: async () => jsonResponse(discovery), }); await expect(unknownRoute.http.GET("/v1/undeclared" as "/v1", { params: { header: VERSION_HEADERS } })).rejects.toMatchObject({ code: "indeterminate", status: 502 }); }); @@ -626,7 +640,7 @@ describe("generated T4 API v1 client conformance", () => { baseUrl: "https://watch-status.test/api", credential: "token-a", majorVersion: 1, fetch: async (input) => { expect(new URL(new Request(input).url).pathname).toBe("/api/v1/sessions/ses-1/events"); - return new Response('data: {"type":"heartbeat","cursor":"prefixed-1","observedAt":"2026-07-21T00:00:00Z"}\n\n', { status: 200, headers: { "Content-Type": "text/event-stream" } }); + return apiResponse('data: {"type":"heartbeat","cursor":"prefixed-1","observedAt":"2026-07-21T00:00:00Z"}\n\n', { status: 200, headers: { "Content-Type": "text/event-stream" } }); }, }); await expect(prefixed.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).resolves.toMatchObject({ value: { cursor: "prefixed-1" } }); @@ -642,15 +656,15 @@ describe("generated T4 API v1 client conformance", () => { [503, { error: { code: "unavailable", message: "bad", requestId: "r", retryable: false } }], ] as const; for (const [status, body] of declared) { - const client = createT4ApiClient({ baseUrl: "https://watch-status.test", credential: "token-a", majorVersion: 1, fetch: async () => Response.json(body, { status }) }); + const client = createT4ApiClient({ baseUrl: "https://watch-status.test", credential: "token-a", majorVersion: 1, fetch: async () => jsonResponse(body, { status }) }); await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ status, code: body.error.code }); } for (const response of [ - new Response('data: {"type":"heartbeat","cursor":"wrong-media-1","observedAt":"2026-07-21T00:00:00Z"}\n\n', { status: 200, headers: { "Content-Type": "application/json" } }), - new Response('data: {"type":"heartbeat","cursor":"wrong-1","observedAt":"2026-07-21T00:00:00Z"}\n\n', { status: 201, headers: { "Content-Type": "text/event-stream" } }), - new Response(null, { status: 204, headers: { "Content-Type": "text/event-stream" } }), - Response.json({ error: { code: "revision_conflict", message: "bad", requestId: "r", retryable: false } }, { status: 409 }), + apiResponse('data: {"type":"heartbeat","cursor":"wrong-media-1","observedAt":"2026-07-21T00:00:00Z"}\n\n', { status: 200, headers: { "Content-Type": "application/json" } }), + apiResponse('data: {"type":"heartbeat","cursor":"wrong-1","observedAt":"2026-07-21T00:00:00Z"}\n\n', { status: 201, headers: { "Content-Type": "text/event-stream" } }), + apiResponse(null, { status: 204, headers: { "Content-Type": "text/event-stream" } }), + jsonResponse({ error: { code: "revision_conflict", message: "bad", requestId: "r", retryable: false } }, { status: 409 }), ]) { let attempts = 0; const client = createT4ApiClient({ @@ -679,7 +693,7 @@ describe("generated T4 API v1 client conformance", () => { baseUrl: "https://invalid-503.test", credential: "token-a", majorVersion: 1, fetch: async () => { attempts += 1; - return new Response(body, { status: 503, headers: { "Content-Type": "application/json", "Retry-After": "0" } }); + return apiResponse(body, { status: 503, headers: { "Content-Type": "application/json", "Retry-After": "0" } }); }, }); await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 3, retryBackoffMs: 0 }).next()).rejects.toMatchObject({ @@ -692,7 +706,7 @@ describe("generated T4 API v1 client conformance", () => { const oversizedChunk = new Uint8Array(600 * 1024); const streamedOversizedClient = createT4ApiClient({ baseUrl: "https://streamed-invalid-503.test", credential: "token-a", majorVersion: 1, - fetch: async () => new Response(new ReadableStream({ + fetch: async () => apiResponse(new ReadableStream({ pull(controller) { controller.enqueue(oversizedChunk); }, cancel() { oversizedCancelled = true; }, }), { status: 503, headers: { "Content-Type": "application/json" } }), @@ -705,7 +719,7 @@ describe("generated T4 API v1 client conformance", () => { let nonJsonCancelled = false; const nonJsonClient = createT4ApiClient({ baseUrl: "https://invalid-503-media.test", credential: "token-a", majorVersion: 1, - fetch: async () => new Response(new ReadableStream({ + fetch: async () => apiResponse(new ReadableStream({ pull(controller) { controller.enqueue(new Uint8Array(1024)); }, cancel() { nonJsonCancelled = true; }, }), { status: 503, headers: { "Content-Type": "text/plain" } }), @@ -765,7 +779,7 @@ describe("generated T4 API v1 client conformance", () => { const workspace = { id: "wsp-1", name: "workspace", state: "ready", revision: 1 }; const missingReplay = createT4ApiClient({ baseUrl: "https://response-replay.test", credential: "token-a", majorVersion: 1, - fetch: async () => Response.json(workspace, { headers: { "T4-API-Version": "1.0" } }), + fetch: async () => jsonResponse(workspace, { headers: { "T4-API-Version": "1.0" } }), }); await expect(missingReplay.http.PATCH("/v1/workspaces/{workspaceId}", { params: { header: mutationHeaders(1, "response-replay-patch"), path: { workspaceId: "wsp-1" } }, body: { name: "workspace" }, @@ -774,7 +788,7 @@ describe("generated T4 API v1 client conformance", () => { for (const replayed of [undefined, "TRUE", "0"]) { const client = createT4ApiClient({ baseUrl: "https://delete-replay-header.test", credential: "token-a", majorVersion: 1, - fetch: async () => new Response(null, { status: 204, headers: { + fetch: async () => apiResponse(null, { status: 204, headers: { "T4-API-Version": "1.0", ...(replayed === undefined ? {} : { "Idempotency-Replayed": replayed }), } }), }); @@ -804,17 +818,17 @@ describe("generated T4 API v1 client conformance", () => { ]) { const client = createT4ApiClient({ baseUrl: "https://duplicate-discovery.test", credential: "token-a", majorVersion: 1, - fetch: async () => Response.json(discovery, { headers: { "T4-API-Version": "1.0" } }), + fetch: async () => jsonResponse(discovery, { headers: { "T4-API-Version": "1.0" } }), }); await expect(client.http.GET("/v1", { params: { header: VERSION_HEADERS } })).rejects.toMatchObject({ status: 502, retryable: false }); } const duplicateError = createT4ApiClient({ baseUrl: "https://duplicate-error.test", credential: "token-a", majorVersion: 1, - fetch: async () => Response.json({ error: { + fetch: async () => jsonResponse({ error: { code: "incompatible_version", message: "bad", requestId: "r", retryable: false, supportedMajors: [1, 1], } }, { status: 406, headers: { "T4-API-Version": "1.0" } }), }); - await expect(duplicateError.http.GET("/v1", { params: { header: VERSION_HEADERS } })).rejects.toMatchObject({ status: 406, code: "indeterminate", retryable: false }); + await expect(duplicateError.http.GET("/v1", { params: { header: VERSION_HEADERS } })).rejects.toMatchObject({ status: 502, code: "indeterminate", retryable: false }); }); it("discards unterminated SSE data at EOF before reconnecting", async () => { @@ -830,7 +844,7 @@ describe("generated T4 API v1 client conformance", () => { const body = attempts === 1 ? unterminated : 'data: {"type":"heartbeat","cursor":"complete","observedAt":"2026-07-21T00:00:00Z"}\n\n'; - return new Response(body, { headers: { "Content-Type": "text/event-stream", "T4-API-Version": "1.0" } }); + return apiResponse(body, { headers: { "Content-Type": "text/event-stream", "T4-API-Version": "1.0" } }); }, }); await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 1, retryBackoffMs: 0 }).next()).resolves.toMatchObject({ value: { cursor: "complete" } }); @@ -862,7 +876,7 @@ describe("generated T4 API v1 client conformance", () => { let cancelled = false; const client = createT4ApiClient({ baseUrl: "https://invalid-utf8-sse.test", credential: "token-a", majorVersion: 1, - fetch: async () => new Response(new ReadableStream({ + fetch: async () => apiResponse(new ReadableStream({ start(controller) { controller.enqueue(new Uint8Array([100, 97, 116, 97, 58, 32, 195, 40, 10, 10])); }, cancel() { cancelled = true; }, }), { headers: { "Content-Type": "text/event-stream", "T4-API-Version": "1.0" } }), @@ -902,7 +916,7 @@ describe("generated T4 API v1 client conformance", () => { let attempts = 0; const eofFetch: typeof globalThis.fetch = async () => { attempts += 1; - return new Response(new ReadableStream({ start(controller) { controller.close(); } }), { headers: { "Content-Type": "text/event-stream" } }); + return apiResponse(new ReadableStream({ start(controller) { controller.close(); } }), { headers: { "Content-Type": "text/event-stream" } }); }; const eofClient = createT4ApiClient({ baseUrl: "https://eof.test", credential: "token-a", majorVersion: 1, fetch: eofFetch }); await expect(eofClient.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 2, retryBackoffMs: 0 }).next()).rejects.toMatchObject({ code: "indeterminate", status: 502 }); @@ -915,7 +929,7 @@ describe("generated T4 API v1 client conformance", () => { networkAttempt += 1; if (networkAttempt === 1) { let sent = false; - return new Response(new ReadableStream({ + return apiResponse(new ReadableStream({ pull(stream) { if (sent) stream.error(new TypeError("transient network loss")); else { @@ -925,7 +939,7 @@ describe("generated T4 API v1 client conformance", () => { }, }), { headers: { "Content-Type": "text/event-stream" } }); } - return new Response('data: {"type":"session","cursor":"network-2","state":"ready","revision":2}\n\n', { headers: { "Content-Type": "text/event-stream" } }); + return apiResponse('data: {"type":"session","cursor":"network-2","state":"ready","revision":2}\n\n', { headers: { "Content-Type": "text/event-stream" } }); }; const networkClient = createT4ApiClient({ baseUrl: "https://network.test", credential: "token-a", majorVersion: 1, fetch: networkFetch }); const networkEvents: WatchEvent[] = []; @@ -936,8 +950,8 @@ describe("generated T4 API v1 client conformance", () => { let retryableAttempts = 0; const retryableFetch: typeof globalThis.fetch = async () => { retryableAttempts += 1; - if (retryableAttempts === 1) return Response.json({ error: { code: "unavailable", message: "retry", requestId: "r", retryable: true } }, { status: 503, headers: { "Retry-After": "0" } }); - return new Response('data: {"type":"heartbeat","cursor":"retry-1","observedAt":"2026-07-21T00:00:00Z"}\n\n', { headers: { "Content-Type": "text/event-stream" } }); + if (retryableAttempts === 1) return jsonResponse({ error: { code: "unavailable", message: "retry", requestId: "r", retryable: true } }, { status: 503, headers: { "Retry-After": "0" } }); + return apiResponse('data: {"type":"heartbeat","cursor":"retry-1","observedAt":"2026-07-21T00:00:00Z"}\n\n', { headers: { "Content-Type": "text/event-stream" } }); }; const retryableClient = createT4ApiClient({ baseUrl: "https://retryable.test", credential: "token-a", majorVersion: 1, fetch: retryableFetch }); expect((await retryableClient.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 1, retryBackoffMs: 0 }).next()).value).toMatchObject({ cursor: "retry-1" }); @@ -948,9 +962,9 @@ describe("generated T4 API v1 client conformance", () => { progressAttempts += 1; if (progressAttempts === 2 || progressAttempts === 4) { const cursor = progressAttempts === 2 ? "progress-1" : "progress-2"; - return new Response(`data: {"type":"heartbeat","cursor":"${cursor}","observedAt":"2026-07-21T00:00:00Z"}\n\n`, { headers: { "Content-Type": "text/event-stream" } }); + return apiResponse(`data: {"type":"heartbeat","cursor":"${cursor}","observedAt":"2026-07-21T00:00:00Z"}\n\n`, { headers: { "Content-Type": "text/event-stream" } }); } - return new Response(new ReadableStream({ start(controller) { controller.close(); } }), { headers: { "Content-Type": "text/event-stream" } }); + return apiResponse(new ReadableStream({ start(controller) { controller.close(); } }), { headers: { "Content-Type": "text/event-stream" } }); }; const progressClient = createT4ApiClient({ baseUrl: "https://progress.test", credential: "token-a", majorVersion: 1, fetch: progressFetch }); const progressEvents: WatchEvent[] = []; @@ -964,7 +978,7 @@ describe("generated T4 API v1 client conformance", () => { let retryAfterAttempts = 0; const retryAfterFetch: typeof globalThis.fetch = async () => { retryAfterAttempts += 1; - return Response.json({ error: { code: "unavailable", message: "later", requestId: "r", retryable: true } }, { status: 503, headers: { "Retry-After": "30" } }); + return jsonResponse({ error: { code: "unavailable", message: "later", requestId: "r", retryable: true } }, { status: 503, headers: { "Retry-After": "30" } }); }; const retryAfterClient = createT4ApiClient({ baseUrl: "https://retry-after.test", credential: "token-a", majorVersion: 1, fetch: retryAfterFetch }); const retryAfterPending = retryAfterClient.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 2, retryBackoffMs: 0, signal: retryAfterController.signal }).next(); @@ -978,7 +992,7 @@ describe("generated T4 API v1 client conformance", () => { } const controller = new AbortController(); - const abortFetch: typeof globalThis.fetch = async (_input, init) => new Response(new ReadableStream({ + const abortFetch: typeof globalThis.fetch = async (_input, init) => apiResponse(new ReadableStream({ start(stream) { init?.signal?.addEventListener("abort", () => stream.close(), { once: true }); }, }), { headers: { "Content-Type": "text/event-stream" } }); const abortClient = createT4ApiClient({ baseUrl: "https://abort.test", credential: "token-a", majorVersion: 1, fetch: abortFetch }); diff --git a/packages/t4-api-client/src/index.ts b/packages/t4-api-client/src/index.ts index feb91ee6..425059d9 100644 --- a/packages/t4-api-client/src/index.ts +++ b/packages/t4-api-client/src/index.ts @@ -9,6 +9,8 @@ const MAX_ERROR_BYTES = 1024 * 1024; const MAX_EVENT_BYTES = 1024 * 1024; const MAX_JSON_RESPONSE_BYTES = 16 * 1024 * 1024; const CURSOR_PATTERN = /^[A-Za-z0-9._~-]+$/u; +const SELECTED_VERSION_MAX_LENGTH = 16; +const SELECTED_VERSION_PATTERN = /^1\.[0-9]+$/u; const ERROR_CODES = { invalid_request: true, unauthenticated: true, forbidden: true, not_found: true, idempotency_key_required: true, idempotency_conflict: true, revision_conflict: true, @@ -178,7 +180,7 @@ function apiError(value: unknown, status: number): ApiError | undefined { typeof error.requestId !== "string" || error.requestId === "" || !hasAtMostCodePoints(error.requestId, 128) || typeof error.retryable !== "boolean" || (error.violations !== undefined && (!Array.isArray(error.violations) || error.violations.length > 64 || !error.violations.every(validViolation))) || - (error.supportedMajors !== undefined && (!Array.isArray(error.supportedMajors) || error.supportedMajors.length > 8 || !error.supportedMajors.every((major) => Number.isSafeInteger(major) && Number(major) >= 1))) || + (error.supportedMajors !== undefined && (!Array.isArray(error.supportedMajors) || error.supportedMajors.length > 8 || !hasUniqueItems(error.supportedMajors) || !error.supportedMajors.every((major) => Number.isSafeInteger(major) && Number(major) >= 1))) || (error.resync !== undefined && !validResync(error.resync)) || (status === 406 && (error.code !== "incompatible_version" || !Array.isArray(error.supportedMajors) || error.supportedMajors.length < 1)) || (status === 410 && (error.code !== "cursor_expired" || !validResync(error.resync))) || @@ -258,6 +260,15 @@ function hasOnlyKeys(value: Record, keys: Readonly keys[key] === true); } +function hasUniqueItems(value: readonly unknown[]): boolean { + for (let index = 0; index < value.length; index += 1) { + for (let prior = 0; prior < index; prior += 1) { + if (value[index] === value[prior]) return false; + } + } + return true; +} + function validResourceId(value: unknown): value is string { return typeof value === "string" && value !== "" && hasAtMostCodePoints(value, 128) && /^[A-Za-z0-9][A-Za-z0-9._~-]*$/u.test(value); } @@ -309,8 +320,8 @@ function validDiscovery(value: unknown): boolean { const limits = record(discovery?.limits); return discovery !== undefined && hasOnlyKeys(discovery, { apiVersion: true, supportedMajors: true, capabilities: true, limits: true }) && typeof discovery.apiVersion === "string" && /^1\.[0-9]+$/u.test(discovery.apiVersion) && - Array.isArray(discovery.supportedMajors) && discovery.supportedMajors.length >= 1 && discovery.supportedMajors.length <= 8 && discovery.supportedMajors.every((major) => Number.isSafeInteger(major) && Number(major) >= 1) && - Array.isArray(discovery.capabilities) && discovery.capabilities.length <= 128 && discovery.capabilities.every((capability) => typeof capability === "string" && /^[a-z][a-z0-9.-]{0,127}$/u.test(capability)) && + Array.isArray(discovery.supportedMajors) && discovery.supportedMajors.length >= 1 && discovery.supportedMajors.length <= 8 && hasUniqueItems(discovery.supportedMajors) && discovery.supportedMajors.every((major) => Number.isSafeInteger(major) && Number(major) >= 1) && + Array.isArray(discovery.capabilities) && discovery.capabilities.length <= 128 && hasUniqueItems(discovery.capabilities) && discovery.capabilities.every((capability) => typeof capability === "string" && /^[a-z][a-z0-9.-]{0,127}$/u.test(capability)) && limits !== undefined && hasOnlyKeys(limits, { pageSizeDefault: true, pageSizeMax: true, commandBytesMax: true, commandRequestBytesMax: true, commandMetadataValueBytesMax: true, watchEventsMax: true, heartbeatSeconds: true }) && [limits.pageSizeDefault, limits.pageSizeMax, limits.commandBytesMax, limits.commandRequestBytesMax, limits.commandMetadataValueBytesMax, limits.watchEventsMax, limits.heartbeatSeconds].every((limit) => Number.isSafeInteger(limit) && Number(limit) >= 1) && Number(limits.pageSizeDefault) <= Number(limits.pageSizeMax) && Number(limits.commandBytesMax) <= Number(limits.commandRequestBytesMax) && Number(limits.commandMetadataValueBytesMax) <= Number(limits.commandRequestBytesMax) && @@ -336,20 +347,21 @@ type SuccessValidator = ((value: unknown) => boolean) | "empty" | "event-stream" interface ResponseContract { readonly success: Readonly>; readonly errors: readonly number[]; + readonly replaySuccess?: readonly number[]; } const DISCOVERY_RESPONSE: ResponseContract = { success: { 200: validDiscovery }, errors: [401, 403, 406, 503] }; const WORKSPACE_LIST_RESPONSE: ResponseContract = { success: { 200: validWorkspacePage }, errors: [400, 401, 403, 406, 422, 503] }; -const WORKSPACE_CREATE_RESPONSE: ResponseContract = { success: { 200: validWorkspace, 202: validWorkspace }, errors: [400, 401, 403, 406, 409, 422, 503] }; +const WORKSPACE_CREATE_RESPONSE: ResponseContract = { success: { 200: validWorkspace, 202: validWorkspace }, errors: [400, 401, 403, 406, 409, 422, 503], replaySuccess: [200, 202] }; const WORKSPACE_GET_RESPONSE: ResponseContract = { success: { 200: validWorkspace }, errors: [401, 403, 404, 406, 503] }; -const WORKSPACE_MUTATE_RESPONSE: ResponseContract = { success: { 200: validWorkspace }, errors: [400, 401, 403, 404, 406, 409, 422, 503] }; -const DELETE_RESPONSE: ResponseContract = { success: { 204: "empty" }, errors: [400, 401, 403, 404, 406, 409, 503] }; +const WORKSPACE_MUTATE_RESPONSE: ResponseContract = { success: { 200: validWorkspace }, errors: [400, 401, 403, 404, 406, 409, 422, 503], replaySuccess: [200] }; +const DELETE_RESPONSE: ResponseContract = { success: { 204: "empty" }, errors: [400, 401, 403, 404, 406, 409, 503], replaySuccess: [204] }; const SESSION_LIST_RESPONSE: ResponseContract = { success: { 200: validSessionPage }, errors: [401, 403, 404, 406, 422, 503] }; -const SESSION_CREATE_RESPONSE: ResponseContract = { success: { 200: validSession, 202: validSession }, errors: [400, 401, 403, 404, 406, 409, 422, 503] }; +const SESSION_CREATE_RESPONSE: ResponseContract = { success: { 200: validSession, 202: validSession }, errors: [400, 401, 403, 404, 406, 409, 422, 503], replaySuccess: [200, 202] }; const SESSION_GET_RESPONSE: ResponseContract = { success: { 200: validSession }, errors: [401, 403, 404, 406, 503] }; -const SESSION_MUTATE_RESPONSE: ResponseContract = { success: { 200: validSession }, errors: [400, 401, 403, 404, 406, 409, 422, 503] }; -const SESSION_CANCEL_RESPONSE: ResponseContract = { success: { 200: validSession, 202: validSession }, errors: [400, 401, 403, 404, 406, 409, 503] }; -const COMMAND_RESPONSE: ResponseContract = { success: { 200: validCommandResult, 202: validCommandResult }, errors: [400, 401, 403, 404, 406, 409, 422, 503] }; +const SESSION_MUTATE_RESPONSE: ResponseContract = { success: { 200: validSession }, errors: [400, 401, 403, 404, 406, 409, 422, 503], replaySuccess: [200] }; +const SESSION_CANCEL_RESPONSE: ResponseContract = { success: { 200: validSession, 202: validSession }, errors: [400, 401, 403, 404, 406, 409, 503], replaySuccess: [200, 202] }; +const COMMAND_RESPONSE: ResponseContract = { success: { 200: validCommandResult, 202: validCommandResult }, errors: [400, 401, 403, 404, 406, 409, 422, 503], replaySuccess: [200, 202] }; const SNAPSHOT_RESPONSE: ResponseContract = { success: { 200: validSnapshot }, errors: [401, 403, 404, 406, 503] }; const EVENTS_RESPONSE: ResponseContract = { success: { 200: "event-stream" }, errors: [400, 401, 403, 404, 406, 410, 422, 503] }; @@ -408,6 +420,16 @@ function responseContract(method: string, path: string): ResponseContract | unde return undefined; } +function validSelectedVersion(response: Response): boolean { + const selected = response.headers.get("T4-API-Version"); + return selected !== null && selected.length <= SELECTED_VERSION_MAX_LENGTH && SELECTED_VERSION_PATTERN.test(selected); +} + +function validReplayHeader(response: Response): boolean { + const replayed = response.headers.get("Idempotency-Replayed"); + return replayed === "true" || replayed === "false"; +} + async function validateResponse(request: Request, response: Response, baseUrl: string): Promise { const path = relativeApiPath(request, baseUrl); const contract = path === undefined ? undefined : responseContract(request.method, path); @@ -420,9 +442,13 @@ async function validateResponse(request: Request, response: Response, baseUrl: s void response.body?.cancel().catch(() => {}); throw protocolError(response.status, "T4 API returned an undeclared error status"); } + if (response.status !== 401 && !validSelectedVersion(response)) { + void response.body?.cancel().catch(() => {}); + throw protocolError(502, "T4 API returned a missing or invalid selected version"); + } if (await parsedError(response.clone()) === undefined) { void response.body?.cancel().catch(() => {}); - throw protocolError(response.status, "T4 API returned an invalid or oversized error envelope"); + throw protocolError(502, "T4 API returned an invalid or oversized error envelope"); } return; } @@ -431,6 +457,14 @@ async function validateResponse(request: Request, response: Response, baseUrl: s void response.body?.cancel().catch(() => {}); throw protocolError(502, "T4 API returned an undeclared success status"); } + if (!validSelectedVersion(response)) { + void response.body?.cancel().catch(() => {}); + throw protocolError(502, "T4 API returned a missing or invalid selected version"); + } + if (contract.replaySuccess?.includes(response.status) === true && !validReplayHeader(response)) { + void response.body?.cancel().catch(() => {}); + throw protocolError(502, "T4 API returned a missing or invalid replay header"); + } if (validator === "empty") { if (response.body !== null || response.headers.has("content-type")) { void response.body?.cancel().catch(() => {}); @@ -535,11 +569,8 @@ class SseFrameParser { this.#finishLine(this.#pendingCarriageReturn, frames); this.#pendingCarriageReturn = -1; } - if (this.#length > 0) { - if (this.#length > MAX_EVENT_BYTES) this.#oversized(); - frames.push(this.#buffer.slice(0, this.#length)); - this.#reset(); - } + if (this.#length > MAX_EVENT_BYTES) this.#oversized(); + this.#reset(); return frames; } @@ -660,6 +691,10 @@ async function* watch( void response.body?.cancel().catch(() => {}); throw protocolError(502, "T4 API returned an undeclared watch error status"); } + if (response.status !== 401 && !validSelectedVersion(response)) { + void response.body?.cancel().catch(() => {}); + throw protocolError(502, "T4 API returned a missing or invalid selected version"); + } const error = await parsedError(response); throw error ?? protocolError(502, "T4 API returned an invalid or oversized error envelope"); } @@ -667,6 +702,10 @@ async function* watch( void response.body?.cancel().catch(() => {}); throw protocolError(502, "T4 API returned an undeclared watch success status"); } + if (!validSelectedVersion(response)) { + void response.body?.cancel().catch(() => {}); + throw protocolError(502, "T4 API returned a missing or invalid selected version"); + } if (response.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase() !== "text/event-stream") { void response.body?.cancel().catch(() => {}); throw protocolError(502, "T4 API watch did not return text/event-stream"); diff --git a/packages/t4-api-contract/openapi.json b/packages/t4-api-contract/openapi.json index 1ae79de1..ac0d907e 100644 --- a/packages/t4-api-contract/openapi.json +++ b/packages/t4-api-contract/openapi.json @@ -420,7 +420,7 @@ "headers": { "SelectedVersion": { "description": "Selected compatible T4 API minor version.", - "schema": { "type": "string", "pattern": "^1\\.[0-9]+$" } + "schema": { "type": "string", "pattern": "^1\\.[0-9]+$", "maxLength": 16 } }, "IdempotencyReplayed": { "description": "True when this response replays a prior identical request.", From afb136cc90088b19bb776c2da18429f77cadcc28 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 23:08:05 +0000 Subject: [PATCH 028/131] test: reject invalid mutations before idempotency --- .../client/test/t4-api-v1-conformance.test.ts | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index 5c0f5a9c..6136a7db 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -361,6 +361,51 @@ describe("generated T4 API v1 client conformance", () => { })).response.status).toBe(404); }); + it("validates mutations before state changes and idempotency recording", async () => { + const workspaceService = new T4ApiV1ConformanceService(); + const workspaceClient = createT4ApiClient({ baseUrl: workspaceService.origin, credential: "token-a", majorVersion: 1, fetch: workspaceService.fetch }); + requireData(await workspaceClient.http.POST("/v1/workspaces", { + body: { name: "original" }, params: { header: idempotencyHeaders("mutation-workspace-setup") }, + })); + const invalidWorkspaceBodies = [ + {}, { unknown: true }, { name: 42 }, { name: "" }, { name: "😀".repeat(129) }, + { labels: { Invalid: "value" } }, { labels: { valid: "😀".repeat(129) } }, + ]; + for (const [index, body] of invalidWorkspaceBodies.entries()) { + const result = await workspaceClient.http.PATCH("/v1/workspaces/{workspaceId}", { + params: { header: mutationHeaders(1, `invalid-workspace-${index}`), path: { workspaceId: "ws-1" } }, body: body as never, + }); + expect(result.response.status).toBe(422); + expect(result.error).toMatchObject({ error: { code: "invalid_request" } }); + } + expect(requireData(await workspaceClient.http.GET("/v1/workspaces/{workspaceId}", { + params: { header: VERSION_HEADERS, path: { workspaceId: "ws-1" } }, + }))).toMatchObject({ name: "original", revision: 1 }); + expect((await workspaceClient.http.PATCH("/v1/workspaces/{workspaceId}", { + params: { header: mutationHeaders(1, "invalid-workspace-0"), path: { workspaceId: "ws-1" } }, body: { name: "valid" }, + })).response.status).toBe(200); + + const sessionService = new T4ApiV1ConformanceService(); + const sessionClient = await seededClient(sessionService, "mutation-validation"); + const invalidSessionBodies = [ + {}, { unknown: true }, { title: 42 }, { title: "" }, { title: "😀".repeat(129) }, + { labels: { Invalid: "value" } }, { labels: { valid: "😀".repeat(129) } }, + ]; + for (const [index, body] of invalidSessionBodies.entries()) { + const result = await sessionClient.http.PATCH("/v1/sessions/{sessionId}", { + params: { header: mutationHeaders(1, `invalid-session-${index}`), path: { sessionId: "ses-1" } }, body: body as never, + }); + expect(result.response.status).toBe(422); + expect(result.error).toMatchObject({ error: { code: "invalid_request" } }); + } + expect(requireData(await sessionClient.http.GET("/v1/sessions/{sessionId}", { + params: { header: VERSION_HEADERS, path: { sessionId: "ses-1" } }, + }))).toMatchObject({ title: "agent", revision: 1 }); + expect((await sessionClient.http.PATCH("/v1/sessions/{sessionId}", { + params: { header: mutationHeaders(1, "invalid-session-0"), path: { sessionId: "ses-1" } }, body: { title: "valid" }, + })).response.status).toBe(200); + }); + it("takes a snapshot and watches bounded SSE with heartbeat, reconnect, cancellation, and typed resync", async () => { const service = new T4ApiV1ConformanceService(); const client = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); From 3a056c78ae2c5620428cc3ccdf972be9d17c41b9 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 23:11:27 +0000 Subject: [PATCH 029/131] fix: close strict API contract gaps --- .github/workflows/ci.yml | 13 +++++++++- .woodpecker.yml | 3 +++ .../test/t4-api-v1-conformance-service.ts | 21 ++++++++++++++-- .../client/test/t4-api-v1-conformance.test.ts | 2 +- packages/t4-api-client/src/index.ts | 4 +-- packages/t4-api-contract/scripts/validate.mjs | 25 +++++++++++++++++++ 6 files changed, 62 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 25d7d947..f5f56a9e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,10 +82,19 @@ jobs: - name: Validate OpenAPI and deterministic generation run: | pnpm --filter @t4-code/t4-api-contract validate + pnpm --filter @t4-code/t4-api-contract generate:ci pnpm --filter @t4-code/t4-api-contract check:generated pnpm --filter @t4-code/t4-api-client typecheck pnpm --filter @t4-code/client typecheck pnpm --filter @t4-code/client exec vp test run test/t4-api-v1-conformance.test.ts + - name: Upload deterministic T4 API client + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: t4-api-generated-${{ github.run_id }} + path: artifacts/t4-api/generated/schema.ts + if-no-files-found: error + core: @@ -552,7 +561,7 @@ jobs: verify: name: verify if: ${{ always() }} - needs: [changes, core, legacy-bridge-continuity, official-omp-gate0, cluster, tooling, android-debug, flutter, flutter-android, flutter-apple] + needs: [changes, t4-api-generation, core, legacy-bridge-continuity, official-omp-gate0, cluster, tooling, android-debug, flutter, flutter-android, flutter-apple] runs-on: ubuntu-24.04 timeout-minutes: 5 steps: @@ -560,6 +569,7 @@ jobs: shell: bash env: CHANGES_RESULT: ${{ needs.changes.result }} + T4_API_GENERATION_RESULT: ${{ needs.t4-api-generation.result }} CORE_RESULT: ${{ needs.core.result }} CONTINUITY_RESULT: ${{ needs.legacy-bridge-continuity.result }} OFFICIAL_OMP_GATE0_RESULT: ${{ needs.official-omp-gate0.result }} @@ -572,6 +582,7 @@ jobs: run: | set -euo pipefail test "$CHANGES_RESULT" = success + test "$T4_API_GENERATION_RESULT" = success test "$CORE_RESULT" = success for result in \ "$CONTINUITY_RESULT" \ diff --git a/.woodpecker.yml b/.woodpecker.yml index c7ad7c55..78d6da92 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -37,6 +37,9 @@ steps: commands: - corepack enable - pnpm --filter @t4-code/t4-api-contract validate + - pnpm --filter @t4-code/t4-api-contract generate:ci + - test -f artifacts/t4-api/generated/schema.ts + - echo "T4_API_GENERATED_ARTIFACT=artifacts/t4-api/generated/schema.ts" - pnpm --filter @t4-code/t4-api-contract check:generated - pnpm --filter @t4-code/t4-api-client typecheck - pnpm --filter @t4-code/client typecheck diff --git a/packages/client/test/t4-api-v1-conformance-service.ts b/packages/client/test/t4-api-v1-conformance-service.ts index d7552c8d..b29e84eb 100644 --- a/packages/client/test/t4-api-v1-conformance-service.ts +++ b/packages/client/test/t4-api-v1-conformance-service.ts @@ -14,6 +14,21 @@ function hasAtMostCodePoints(value: string, maximum: number): boolean { return true; } +function validLabels(value: unknown): boolean { + if (value === null || typeof value !== "object" || Array.isArray(value)) return false; + const entries = Object.entries(value as Record); + return entries.length <= 32 && entries.every(([key, item]) => + /^[a-z][a-z0-9.-]{0,62}$/u.test(key) && typeof item === "string" && hasAtMostCodePoints(item, 128)); +} + +function validMutation(body: Record, textField: "name" | "title"): boolean { + const keys = Object.keys(body); + if (keys.length < 1 || keys.some((key) => key !== textField && key !== "labels")) return false; + const text = body[textField]; + return (text === undefined || (typeof text === "string" && text !== "" && hasAtMostCodePoints(text, 128))) && + (body.labels === undefined || validLabels(body.labels)); +} + function json(status: number, body: unknown, headers: Record = {}): Response { return Response.json(body, { status, headers: { "T4-API-Version": "1.0", ...headers } }); } @@ -142,8 +157,9 @@ export class T4ApiV1ConformanceService { const parsed = await this.#jsonBody(request); if (parsed instanceof Response) return parsed; const body = parsed; + if (!validMutation(body, "name")) return this.#invalid("body", "schema", "workspace mutation must match WorkspaceMutation"); return this.#idempotent(request, tenant, "mutateWorkspace", [id], body, 200, 200, () => { - const updated = { ...workspace, name: body.name ?? workspace.name, revision: Number(workspace.revision) + 1 }; + const updated = { ...workspace, ...(body.name === undefined ? {} : { name: body.name }), ...(body.labels === undefined ? {} : { labels: body.labels }), revision: Number(workspace.revision) + 1 }; this.#workspaces.set(id, updated); return updated; }, () => request.headers.get("If-Match") === String(workspace.revision) ? undefined : problem(409, "revision_conflict", "Workspace revision changed")); @@ -194,8 +210,9 @@ export class T4ApiV1ConformanceService { const parsed = await this.#jsonBody(request); if (parsed instanceof Response) return parsed; const body = parsed; + if (!validMutation(body, "title")) return this.#invalid("body", "schema", "session mutation must match SessionMutation"); return this.#idempotent(request, tenant, "mutateSession", [id], body, 200, 200, () => { - const updated = { ...session, title: body.title ?? session.title, revision: Number(session.revision) + 1 }; + const updated = { ...session, ...(body.title === undefined ? {} : { title: body.title }), ...(body.labels === undefined ? {} : { labels: body.labels }), revision: Number(session.revision) + 1 }; this.#sessions.set(id, updated); return updated; }, () => request.headers.get("If-Match") === String(session.revision) ? undefined : problem(409, "revision_conflict", "Session revision changed")); diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index 6136a7db..de2b00c4 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -592,7 +592,7 @@ describe("generated T4 API v1 client conformance", () => { baseUrl: "https://undeclared-error.test", credential: "token-a", majorVersion: 1, fetch: async () => jsonResponse(validErrors[3][1], { status: 404 }), }); - await expect(statusClient.http.GET("/v1", { params: { header: VERSION_HEADERS } })).rejects.toMatchObject({ code: "indeterminate", status: 404 }); + await expect(statusClient.http.GET("/v1", { params: { header: VERSION_HEADERS } })).rejects.toMatchObject({ code: "indeterminate", status: 502, retryable: false }); }); it("validates successful JSON routes relative to the base path and rejects undeclared media and statuses", async () => { diff --git a/packages/t4-api-client/src/index.ts b/packages/t4-api-client/src/index.ts index 425059d9..1d024ed5 100644 --- a/packages/t4-api-client/src/index.ts +++ b/packages/t4-api-client/src/index.ts @@ -435,12 +435,12 @@ async function validateResponse(request: Request, response: Response, baseUrl: s const contract = path === undefined ? undefined : responseContract(request.method, path); if (contract === undefined) { void response.body?.cancel().catch(() => {}); - throw protocolError(response.ok ? 502 : response.status, "T4 API returned a response for an undeclared route"); + throw protocolError(502, "T4 API returned a response for an undeclared route"); } if (!response.ok) { if (!contract.errors.includes(response.status)) { void response.body?.cancel().catch(() => {}); - throw protocolError(response.status, "T4 API returned an undeclared error status"); + throw protocolError(502, "T4 API returned an undeclared error status"); } if (response.status !== 401 && !validSelectedVersion(response)) { void response.body?.cancel().catch(() => {}); diff --git a/packages/t4-api-contract/scripts/validate.mjs b/packages/t4-api-contract/scripts/validate.mjs index cbfafbc7..4a9562eb 100644 --- a/packages/t4-api-contract/scripts/validate.mjs +++ b/packages/t4-api-contract/scripts/validate.mjs @@ -16,6 +16,31 @@ for (const server of document.servers) { if (document.security?.[0]?.BearerAuth === undefined) throw new Error("T4 API contract must default to bearer authentication"); const sseSchema = document.paths?.["/v1/sessions/{sessionId}/events"]?.get?.responses?.["200"]?.content?.["text/event-stream"]?.schema; if (sseSchema?.["x-t4-sse-data-schema"] !== "#/components/schemas/WatchEvent") throw new Error("watch SSE data must remain linked to WatchEvent"); +const schemas = document.components?.schemas ?? {}; +const commandCreate = schemas.CommandCreate; +if (commandCreate?.["x-t4-maxUtf8Bytes"] !== 1048576) throw new Error("CommandCreate must retain its UTF-8 request-byte bound"); +if (commandCreate?.properties?.command?.["x-t4-maxUtf8Bytes"] !== 262144) throw new Error("command must retain its UTF-8 byte bound"); +const metadataString = commandCreate?.properties?.metadata?.additionalProperties?.oneOf?.find((item) => item?.type === "string"); +if (metadataString?.["x-t4-maxUtf8Bytes"] !== 262144) throw new Error("command metadata strings must retain their UTF-8 byte bound"); + +for (const [schema, property] of [["Discovery", "supportedMajors"], ["Discovery", "capabilities"], ["ApiError", "supportedMajors"]]) { + if (schemas[schema]?.properties?.[property]?.uniqueItems !== true) throw new Error(`${schema}.${property} must retain uniqueItems`); +} + +const selectedVersion = document.components?.headers?.SelectedVersion?.schema; +if (selectedVersion?.pattern !== "^1\\.[0-9]+$" || selectedVersion?.maxLength !== 16) throw new Error("SelectedVersion must remain a bounded v1 minor"); +const selectedVersionRef = "#/components/headers/SelectedVersion"; +const replayRef = "#/components/headers/IdempotencyReplayed"; +const selectedVersionResponses = ["Discovery", "Workspace", "WorkspaceAccepted", "WorkspaceReplay", "WorkspacePage", "Session", "SessionAccepted", "SessionReplay", "SessionPage", "CommandAccepted", "CommandReplay", "Snapshot", "Deleted", "Error400", "Error403", "Error404", "Error406", "Error409", "Error410", "Error422", "Error503"]; +for (const responseName of selectedVersionResponses) { + if (document.components?.responses?.[responseName]?.headers?.["T4-API-Version"]?.$ref !== selectedVersionRef) throw new Error(`${responseName} must declare SelectedVersion`); +} +for (const responseName of ["WorkspaceAccepted", "WorkspaceReplay", "SessionAccepted", "SessionReplay", "CommandAccepted", "CommandReplay", "Deleted"]) { + if (document.components?.responses?.[responseName]?.headers?.["Idempotency-Replayed"]?.$ref !== replayRef) throw new Error(`${responseName} must declare IdempotencyReplayed`); +} +if (document.paths?.["/v1/sessions/{sessionId}/events"]?.get?.responses?.["200"]?.headers?.["T4-API-Version"]?.$ref !== selectedVersionRef) throw new Error("watch success must declare SelectedVersion"); +if (document.paths?.["/v1/workspaces/{workspaceId}"]?.patch?.responses?.["200"]?.$ref !== "#/components/responses/WorkspaceReplay") throw new Error("workspace PATCH success must declare replay headers"); +if (document.paths?.["/v1/sessions/{sessionId}"]?.patch?.responses?.["200"]?.$ref !== "#/components/responses/SessionReplay") throw new Error("session PATCH success must declare replay headers"); const errorResponses = { 400: "Error400", 401: "Error401", 403: "Error403", 404: "Error404", 406: "Error406", 409: "Error409", 410: "Error410", 422: "Error422", 503: "Error503" }; for (const path of Object.values(document.paths ?? {})) { for (const operation of Object.values(path)) { From 2b1433fb3b19c771b25bd46d607cb7b7ed39c81d Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 23:17:56 +0000 Subject: [PATCH 030/131] test: type omitted command metadata --- packages/client/test/t4-api-v1-conformance.test.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index de2b00c4..93c13ca2 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -323,15 +323,13 @@ describe("generated T4 API v1 client conformance", () => { body: { command: "ok", metadata: Object.fromEntries(Array.from({ length: 8 }, (_, index) => [`field-${index}`, "x".repeat(32)])) }, }); expect(requestOversized.response.status).toBe(422); - const defaultedMetadataResponse = await service.fetch(`${service.origin}/v1/sessions/ses-1/commands`, { - method: "POST", - headers: { Authorization: "Bearer token-a", "T4-API-Version": "1", "Idempotency-Key": "command-default-0001", "Content-Type": "application/json" }, - body: '{"command":"ok"}', + const defaultedMetadata = await client.http.POST("/v1/sessions/{sessionId}/commands", { + params: { header: idempotencyHeaders("command-default-0001"), path: { sessionId: "ses-1" } }, body: { command: "ok" }, }); const explicitMetadata = await client.http.POST("/v1/sessions/{sessionId}/commands", { params: { header: idempotencyHeaders("command-default-0001"), path: { sessionId: "ses-1" } }, body: { command: "ok", metadata: {} }, }); - expect(explicitMetadata.data).toEqual(await defaultedMetadataResponse.json()); + expect(explicitMetadata.data).toEqual(defaultedMetadata.data); const cancelled = await client.http.POST("/v1/sessions/{sessionId}/cancel", { params: { header: idempotencyHeaders("session-cancel-0001"), path: { sessionId: "ses-1" } }, }); From 94e74dfc9315cbc652e2b77cffcef73b48755518 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 23:19:52 +0000 Subject: [PATCH 031/131] fix: preserve optional generated defaults --- packages/t4-api-contract/openapi.json | 2 +- packages/t4-api-contract/scripts/generate.mjs | 2 +- packages/t4-api-contract/scripts/validate.mjs | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/t4-api-contract/openapi.json b/packages/t4-api-contract/openapi.json index ac0d907e..f4433421 100644 --- a/packages/t4-api-contract/openapi.json +++ b/packages/t4-api-contract/openapi.json @@ -603,7 +603,7 @@ "properties": { "type": { "type": "string", "const": "heartbeat" }, "cursor": { "$ref": "#/components/schemas/Cursor" }, - "observedAt": { "type": "string", "format": "date-time" } + "observedAt": { "type": "string", "format": "date-time", "maxLength": 64 } } }, "SessionWatchEvent": { diff --git a/packages/t4-api-contract/scripts/generate.mjs b/packages/t4-api-contract/scripts/generate.mjs index 3bc58f15..e216f71a 100644 --- a/packages/t4-api-contract/scripts/generate.mjs +++ b/packages/t4-api-contract/scripts/generate.mjs @@ -13,7 +13,7 @@ if (!new Set(["--write", "--check", "--ci-artifact"]).has(mode)) { throw new Error("usage: node scripts/generate.mjs --write|--check|--ci-artifact"); } -const generated = `${astToString(await openapiTS(schemaUrl, { alphabetize: true }))}\n`; +const generated = `${astToString(await openapiTS(schemaUrl, { alphabetize: true, defaultNonNullable: false }))}\n`; const digest = createHash("sha256").update(generated).digest("hex"); if (mode === "--write") { diff --git a/packages/t4-api-contract/scripts/validate.mjs b/packages/t4-api-contract/scripts/validate.mjs index 4a9562eb..e3a12460 100644 --- a/packages/t4-api-contract/scripts/validate.mjs +++ b/packages/t4-api-contract/scripts/validate.mjs @@ -29,6 +29,9 @@ for (const [schema, property] of [["Discovery", "supportedMajors"], ["Discovery" const selectedVersion = document.components?.headers?.SelectedVersion?.schema; if (selectedVersion?.pattern !== "^1\\.[0-9]+$" || selectedVersion?.maxLength !== 16) throw new Error("SelectedVersion must remain a bounded v1 minor"); +const replayHeader = document.components?.headers?.IdempotencyReplayed?.schema; +if (replayHeader?.type !== "string" || JSON.stringify(replayHeader.enum) !== '["true","false"]') throw new Error("IdempotencyReplayed must remain the exact true|false string enum"); +if (schemas.HeartbeatWatchEvent?.properties?.observedAt?.maxLength !== 64) throw new Error("HeartbeatWatchEvent.observedAt must remain bounded to 64 characters"); const selectedVersionRef = "#/components/headers/SelectedVersion"; const replayRef = "#/components/headers/IdempotencyReplayed"; const selectedVersionResponses = ["Discovery", "Workspace", "WorkspaceAccepted", "WorkspaceReplay", "WorkspacePage", "Session", "SessionAccepted", "SessionReplay", "SessionPage", "CommandAccepted", "CommandReplay", "Snapshot", "Deleted", "Error400", "Error403", "Error404", "Error406", "Error409", "Error410", "Error422", "Error503"]; From b541ed53af69f45d12098ec3b4fdcbd67c7cc2dd Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 23:22:07 +0000 Subject: [PATCH 032/131] chore: accept authorized API schema artifact --- packages/t4-api-client/src/generated/schema.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/t4-api-client/src/generated/schema.ts b/packages/t4-api-client/src/generated/schema.ts index 39321c83..45a57cbf 100755 --- a/packages/t4-api-client/src/generated/schema.ts +++ b/packages/t4-api-client/src/generated/schema.ts @@ -178,7 +178,7 @@ export interface components { CommandCreate: { command: string; /** @default {} */ - metadata: { + metadata?: { [key: string]: string | number | boolean | null; }; }; From ca908f9a064411c20d6201367c366733992b5ed6 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 23:24:47 +0000 Subject: [PATCH 033/131] fix: require API generation in release gate --- scripts/check-release-consistency.mjs | 3 ++- scripts/check-release-consistency.test.mjs | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/check-release-consistency.mjs b/scripts/check-release-consistency.mjs index 7694fc50..53b1f09d 100644 --- a/scripts/check-release-consistency.mjs +++ b/scripts/check-release-consistency.mjs @@ -968,8 +968,9 @@ export function collectReleaseConsistencyErrors(files, releaseTag) { "test -x apps/flutter/build/macos/Build/Products/Debug/t4code.app/Contents/Resources/runtime/t4-host", "name: verify", "if: ${{ always() }}", - "needs: [changes, core, legacy-bridge-continuity, official-omp-gate0, cluster, tooling, android-debug, flutter, flutter-android, flutter-apple]", + "needs: [changes, t4-api-generation, core, legacy-bridge-continuity, official-omp-gate0, cluster, tooling, android-debug, flutter, flutter-android, flutter-apple]", 'test "$CHANGES_RESULT" = success', + 'test "$T4_API_GENERATION_RESULT" = success', 'test "$CORE_RESULT" = success', "for result in \\", "success|skipped) ;;", diff --git a/scripts/check-release-consistency.test.mjs b/scripts/check-release-consistency.test.mjs index b049aba9..c3df7de9 100644 --- a/scripts/check-release-consistency.test.mjs +++ b/scripts/check-release-consistency.test.mjs @@ -275,7 +275,7 @@ test("rejects updater channel, stable manifest, and publication-contract drift", ".github/workflows/ci.yml", (text) => text.replace( - "needs: [changes, core, legacy-bridge-continuity, official-omp-gate0, cluster, tooling, android-debug, flutter, flutter-android, flutter-apple]", + "needs: [changes, t4-api-generation, core, legacy-bridge-continuity, official-omp-gate0, cluster, tooling, android-debug, flutter, flutter-android, flutter-apple]", "needs: [changes, core, tooling, android-debug]", ), ], @@ -580,10 +580,11 @@ test("deploys release site source only after artifact publication", () => { assert.ok(ciWorkflow.includes("if: ${{ always() }}")); assert.ok( ciWorkflow.includes( - "needs: [changes, core, legacy-bridge-continuity, official-omp-gate0, cluster, tooling, android-debug, flutter, flutter-android, flutter-apple]", + "needs: [changes, t4-api-generation, core, legacy-bridge-continuity, official-omp-gate0, cluster, tooling, android-debug, flutter, flutter-android, flutter-apple]", ), ); assert.ok(ciWorkflow.includes('test "$CHANGES_RESULT" = success')); + assert.ok(ciWorkflow.includes('test "$T4_API_GENERATION_RESULT" = success')); assert.ok(ciWorkflow.includes('test "$CORE_RESULT" = success')); assert.ok(ciWorkflow.includes("for result in \\")); assert.ok(ciWorkflow.includes("success|skipped) ;;")); From 1577e8b8868476353c4d35084f015c97ee623ff3 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 23:36:28 +0000 Subject: [PATCH 034/131] test: cover final strict conformance gaps --- .../client/test/t4-api-v1-conformance.test.ts | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index 93c13ca2..54dde6da 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -404,6 +404,117 @@ describe("generated T4 API v1 client conformance", () => { })).response.status).toBe(200); }); + it("validates creates before idempotency and round-trips valid labels", async () => { + const invalidLabels = [ + null, [], Object.fromEntries(Array.from({ length: 33 }, (_, index) => [`key-${index}`, "value"])), + { Invalid: "value" }, { valid: 1 }, { valid: "😀".repeat(129) }, + ]; + const workspaceService = new T4ApiV1ConformanceService(); + const workspaceClient = createT4ApiClient({ baseUrl: workspaceService.origin, credential: "token-a", majorVersion: 1, fetch: workspaceService.fetch }); + const workspaceBodies: unknown[] = [{ name: "workspace", unknown: true }, ...invalidLabels.map((labels) => ({ name: "workspace", labels }))]; + for (const [index, body] of workspaceBodies.entries()) { + const key = `invalid-workspace-create-${index}`; + const invalid = await workspaceClient.http.POST("/v1/workspaces", { body: body as never, params: { header: idempotencyHeaders(key) } }); + expect(invalid.response.status).toBe(422); + expect(invalid.error).toMatchObject({ error: { code: "invalid_request" } }); + expect((await workspaceClient.http.POST("/v1/workspaces", { + body: { name: `valid-${index}` }, params: { header: idempotencyHeaders(key) }, + })).response.status).toBe(202); + } + const workspaceLabels = { team: "😀".repeat(128) }; + expect(requireData(await workspaceClient.http.POST("/v1/workspaces", { + body: { name: "labelled", labels: workspaceLabels }, params: { header: idempotencyHeaders("valid-workspace-labels") }, + }))).toMatchObject({ labels: workspaceLabels }); + + const sessionService = new T4ApiV1ConformanceService(); + const sessionClient = createT4ApiClient({ baseUrl: sessionService.origin, credential: "token-a", majorVersion: 1, fetch: sessionService.fetch }); + requireData(await sessionClient.http.POST("/v1/workspaces", { + body: { name: "workspace" }, params: { header: idempotencyHeaders("session-create-workspace") }, + })); + const sessionBodies: unknown[] = [{ title: "session", unknown: true }, ...invalidLabels.map((labels) => ({ title: "session", labels }))]; + for (const [index, body] of sessionBodies.entries()) { + const key = `invalid-session-create-${index}`; + const invalid = await sessionClient.http.POST("/v1/workspaces/{workspaceId}/sessions", { + body: body as never, params: { header: idempotencyHeaders(key), path: { workspaceId: "ws-1" } }, + }); + expect(invalid.response.status).toBe(422); + expect(invalid.error).toMatchObject({ error: { code: "invalid_request" } }); + expect((await sessionClient.http.POST("/v1/workspaces/{workspaceId}/sessions", { + body: { title: `valid-${index}` }, params: { header: idempotencyHeaders(key), path: { workspaceId: "ws-1" } }, + })).response.status).toBe(202); + } + const sessionLabels = { team: "😀".repeat(128) }; + expect(requireData(await sessionClient.http.POST("/v1/workspaces/{workspaceId}/sessions", { + body: { title: "labelled", labels: sessionLabels }, params: { header: idempotencyHeaders("valid-session-labels"), path: { workspaceId: "ws-1" } }, + }))).toMatchObject({ labels: sessionLabels }); + + const commandClient = await seededClient(new T4ApiV1ConformanceService(), "command-create-validation"); + const invalidCommand = await commandClient.http.POST("/v1/sessions/{sessionId}/commands", { + body: { command: "ok", unknown: true } as never, + params: { header: idempotencyHeaders("invalid-command-create"), path: { sessionId: "ses-1" } }, + }); + expect(invalidCommand.response.status).toBe(422); + expect((await commandClient.http.POST("/v1/sessions/{sessionId}/commands", { + body: { command: "ok" }, params: { header: idempotencyHeaders("invalid-command-create"), path: { sessionId: "ses-1" } }, + })).response.status).toBe(202); + }); + + it("validates If-Match syntax before PATCH idempotency lookup", async () => { + const service = new T4ApiV1ConformanceService(); + const client = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); + requireData(await client.http.POST("/v1/workspaces", { + body: { name: "workspace" }, params: { header: idempotencyHeaders("if-match-workspace-setup") }, + })); + for (const [index, ifMatch] of [undefined, "", "0", "01", "abc", "1.0", "11111111111111111111"].entries()) { + const key = `invalid-if-match-${index}`; + const response = await service.fetch(`${service.origin}/v1/workspaces/ws-1`, { + method: "PATCH", + headers: { + Authorization: "Bearer token-a", "T4-API-Version": "1", "Idempotency-Key": key, "Content-Type": "application/json", + ...(ifMatch === undefined ? {} : { "If-Match": ifMatch }), + }, + body: '{"name":"updated"}', + }); + expect(response.status).toBe(400); + expect((await response.json()) as unknown).toMatchObject({ error: { code: "invalid_request" } }); + } + const first = await client.http.PATCH("/v1/workspaces/{workspaceId}", { + body: { name: "updated" }, params: { header: mutationHeaders(1, "if-match-replay-order"), path: { workspaceId: "ws-1" } }, + }); + expect(first.response.status).toBe(200); + const missingOnReplay = await service.fetch(`${service.origin}/v1/workspaces/ws-1`, { + method: "PATCH", + headers: { Authorization: "Bearer token-a", "T4-API-Version": "1", "Idempotency-Key": "if-match-replay-order", "Content-Type": "application/json" }, + body: '{"name":"updated"}', + }); + expect(missingOnReplay.status).toBe(400); + }); + + it("rejects invalid declared watch headers and calendar timestamps", async () => { + let cacheBodyCancelled = false; + const cacheClient = createT4ApiClient({ + baseUrl: "https://watch-cache.test", credential: "token-a", majorVersion: 1, + fetch: async () => apiResponse(new ReadableStream({ + start(controller) { controller.enqueue(new TextEncoder().encode('data: {"type":"heartbeat","cursor":"cache","observedAt":"2026-07-21T00:00:00Z"}\n\n')); }, + cancel() { cacheBodyCancelled = true; }, + }), { headers: { "Content-Type": "text/event-stream", "Cache-Control": "max-age=3600" } }), + }); + await expect(cacheClient.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ status: 502, retryable: false }); + expect(cacheBodyCancelled).toBe(true); + + for (const observedAt of [ + "2026-02-31T00:00:00Z", "2026-13-01T00:00:00Z", "2025-02-29T00:00:00Z", + "2026-04-31T00:00:00Z", "2026-01-01T24:00:00Z", "2026-01-01T00:60:00Z", + "2026-01-01T00:00:00+24:00", "2026-01-01T00:00:00+01:60", + ]) { + const calendarClient = createT4ApiClient({ + baseUrl: "https://watch-calendar.test", credential: "token-a", majorVersion: 1, + fetch: async () => apiResponse(`data: {"type":"heartbeat","cursor":"calendar","observedAt":"${observedAt}"}\n\n`, { headers: { "Content-Type": "text/event-stream" } }), + }); + await expect(calendarClient.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ status: 502, retryable: false }); + } + }); + it("takes a snapshot and watches bounded SSE with heartbeat, reconnect, cancellation, and typed resync", async () => { const service = new T4ApiV1ConformanceService(); const client = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); From d31442ea46f5518e248ef9e1398b28be8d62062b Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 23:39:27 +0000 Subject: [PATCH 035/131] fix: validate create and watch contracts --- .../test/t4-api-v1-conformance-service.ts | 25 +++++++++++++---- .../client/test/t4-api-v1-conformance.test.ts | 6 ++++- packages/t4-api-client/src/index.ts | 27 ++++++++++++++++--- 3 files changed, 49 insertions(+), 9 deletions(-) diff --git a/packages/client/test/t4-api-v1-conformance-service.ts b/packages/client/test/t4-api-v1-conformance-service.ts index b29e84eb..3604a717 100644 --- a/packages/client/test/t4-api-v1-conformance-service.ts +++ b/packages/client/test/t4-api-v1-conformance-service.ts @@ -21,6 +21,14 @@ function validLabels(value: unknown): boolean { /^[a-z][a-z0-9.-]{0,62}$/u.test(key) && typeof item === "string" && hasAtMostCodePoints(item, 128)); } +function validCreate(body: Record, textField: "name" | "title"): boolean { + const keys = Object.keys(body); + if (keys.some((key) => key !== textField && key !== "labels")) return false; + const text = body[textField]; + return typeof text === "string" && text !== "" && hasAtMostCodePoints(text, 128) && + (body.labels === undefined || validLabels(body.labels)); +} + function validMutation(body: Record, textField: "name" | "title"): boolean { const keys = Object.keys(body); if (keys.length < 1 || keys.some((key) => key !== textField && key !== "labels")) return false; @@ -122,7 +130,7 @@ export class T4ApiV1ConformanceService { const parsed = await this.#jsonBody(request); if (parsed instanceof Response) return parsed; const body = parsed; - if (typeof body.name !== "string" || body.name === "" || !hasAtMostCodePoints(body.name, 128)) return this.#invalid("name", "length", "name must contain 1 to 128 characters"); + if (!validCreate(body, "name")) return this.#invalid("body", "schema", "workspace create must match WorkspaceCreate"); return this.#idempotent(request, tenant, "createWorkspace", [], body, 202, 200, () => { const id = `ws-${++this.#workspaceSequence}`; const workspace = { id, name: body.name, ...(body.labels === undefined ? {} : { labels: body.labels }), state: "accepted", revision: 1, tenant }; @@ -154,6 +162,8 @@ export class T4ApiV1ConformanceService { if (workspace?.tenant !== tenant) return problem(404, "not_found", "Workspace not found"); if (request.method === "GET") return json(200, this.#payload("workspace", this.#visible(workspace))); if (request.method === "PATCH") { + const ifMatch = request.headers.get("If-Match"); + if (ifMatch === null || !/^[1-9][0-9]{0,18}$/u.test(ifMatch)) return problem(400, "invalid_request", "If-Match is invalid"); const parsed = await this.#jsonBody(request); if (parsed instanceof Response) return parsed; const body = parsed; @@ -175,10 +185,10 @@ export class T4ApiV1ConformanceService { const parsed = await this.#jsonBody(request); if (parsed instanceof Response) return parsed; const body = parsed; - if (typeof body.title !== "string" || body.title === "" || !hasAtMostCodePoints(body.title, 128)) return this.#invalid("title", "length", "title must contain 1 to 128 characters"); + if (!validCreate(body, "title")) return this.#invalid("body", "schema", "session create must match SessionCreate"); return this.#idempotent(request, tenant, "spawnSession", [workspaceId], body, 202, 200, () => { const id = `ses-${++this.#sessionSequence}`; - const session = { id, workspaceId, title: body.title, state: "accepted", revision: 1, tenant }; + const session = { id, workspaceId, title: body.title, ...(body.labels === undefined ? {} : { labels: body.labels }), state: "accepted", revision: 1, tenant }; this.#sessions.set(id, session); return session; }); @@ -207,6 +217,8 @@ export class T4ApiV1ConformanceService { if (session?.tenant !== tenant) return problem(404, "not_found", "Session not found"); if (request.method === "GET") return json(200, this.#payload("session", this.#visible(session))); if (request.method === "PATCH") { + const ifMatch = request.headers.get("If-Match"); + if (ifMatch === null || !/^[1-9][0-9]{0,18}$/u.test(ifMatch)) return problem(400, "invalid_request", "If-Match is invalid"); const parsed = await this.#jsonBody(request); if (parsed instanceof Response) return parsed; const body = parsed; @@ -238,8 +250,11 @@ export class T4ApiV1ConformanceService { if (session?.tenant !== tenant) return problem(404, "not_found", "Session not found"); const text = await request.text(); if (encoder.encode(text).byteLength > COMMAND_REQUEST_BYTES_MAX) return this.#invalid("body", "maxBytes", `request must not exceed ${COMMAND_REQUEST_BYTES_MAX} UTF-8 bytes`); - let body: Record; - try { body = JSON.parse(text) as Record; } catch { return problem(400, "invalid_request", "Malformed JSON request"); } + let decoded: unknown; + try { decoded = JSON.parse(text); } catch { return problem(400, "invalid_request", "Malformed JSON request"); } + if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded)) return this.#invalid("body", "schema", "command create must match CommandCreate"); + const body = decoded as Record; + if (Object.keys(body).some((key) => key !== "command" && key !== "metadata")) return this.#invalid("body", "schema", "command create must match CommandCreate"); if (typeof body.command !== "string" || body.command.length < 1 || encoder.encode(body.command).byteLength > COMMAND_BYTES_MAX) return this.#invalid("command", "maxBytes", `command must contain 1 to ${COMMAND_BYTES_MAX} UTF-8 bytes`); if (!this.#validMetadata(body.metadata)) return this.#invalid("metadata", "bounds", "metadata keys and values exceed the discovered bounds"); const states: Record = { accepted: true, rejected: true, conflict: true, unavailable: true, indeterminate: true }; diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index 54dde6da..6d16db23 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -51,6 +51,7 @@ const VERSION_HEADERS = { "T4-API-Version": "1" } as const; function withSelectedVersion(init: ResponseInit = {}): ResponseInit { const headers = new Headers(init.headers); if (!headers.has("T4-API-Version")) headers.set("T4-API-Version", "1.0"); + if (headers.get("Content-Type")?.split(";", 1)[0]?.trim().toLowerCase() === "text/event-stream" && !headers.has("Cache-Control")) headers.set("Cache-Control", "no-store"); return { ...init, headers }; } @@ -490,7 +491,7 @@ describe("generated T4 API v1 client conformance", () => { expect(missingOnReplay.status).toBe(400); }); - it("rejects invalid declared watch headers and calendar timestamps", async () => { + it("rejects an invalid declared watch cache header", async () => { let cacheBodyCancelled = false; const cacheClient = createT4ApiClient({ baseUrl: "https://watch-cache.test", credential: "token-a", majorVersion: 1, @@ -501,6 +502,9 @@ describe("generated T4 API v1 client conformance", () => { }); await expect(cacheClient.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ status: 502, retryable: false }); expect(cacheBodyCancelled).toBe(true); + }); + + it("rejects invalid RFC 3339 calendar timestamps", async () => { for (const observedAt of [ "2026-02-31T00:00:00Z", "2026-13-01T00:00:00Z", "2025-02-29T00:00:00Z", diff --git a/packages/t4-api-client/src/index.ts b/packages/t4-api-client/src/index.ts index 1d024ed5..dbf55669 100644 --- a/packages/t4-api-client/src/index.ts +++ b/packages/t4-api-client/src/index.ts @@ -492,6 +492,25 @@ async function validateResponse(request: Request, response: Response, baseUrl: s } } +function validRfc3339DateTime(value: string): boolean { + if (!hasAtMostCodePoints(value, 64)) return false; + const match = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d+)?(?:Z|([+-])(\d{2}):(\d{2}))$/u.exec(value); + if (match === null) return false; + const year = Number(match[1]); + const month = Number(match[2]); + const day = Number(match[3]); + const hour = Number(match[4]); + const minute = Number(match[5]); + const second = Number(match[6]); + if (month < 1 || month > 12 || hour > 23 || minute > 59 || second > 60) return false; + let maximumDay = 31; + if (month === 2) maximumDay = year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0) ? 29 : 28; + else if (month === 4 || month === 6 || month === 9 || month === 11) maximumDay = 30; + if (day < 1 || day > maximumDay) return false; + if (second === 60 && (hour !== 23 || minute !== 59)) return false; + return match[7] === undefined || (Number(match[8]) <= 23 && Number(match[9]) <= 59); +} + function watchEvent(value: unknown, eventId: string | undefined): WatchEvent { const event = record(value); if ( @@ -506,9 +525,7 @@ function watchEvent(value: unknown, eventId: string | undefined): WatchEvent { if ( event.type === "heartbeat" && hasOnlyKeys(event, { type: true, cursor: true, observedAt: true }) && - typeof event.observedAt === "string" && hasAtMostCodePoints(event.observedAt, 64) && - /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/u.test(event.observedAt) && - Number.isFinite(Date.parse(event.observedAt)) + typeof event.observedAt === "string" && validRfc3339DateTime(event.observedAt) ) return event as WatchEvent; if ( event.type === "session" && @@ -706,6 +723,10 @@ async function* watch( void response.body?.cancel().catch(() => {}); throw protocolError(502, "T4 API returned a missing or invalid selected version"); } + if (response.headers.get("Cache-Control") !== "no-store") { + void response.body?.cancel().catch(() => {}); + throw protocolError(502, "T4 API watch did not return Cache-Control: no-store"); + } if (response.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase() !== "text/event-stream") { void response.body?.cancel().catch(() => {}); throw protocolError(502, "T4 API watch did not return text/event-stream"); From ff25fdaaee972ac01df59c9a8a5b18ec8d5d0a02 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 23:49:28 +0000 Subject: [PATCH 036/131] test: cover final header and precondition parity --- .../client/test/t4-api-v1-conformance.test.ts | 80 ++++++++++++++++++- 1 file changed, 78 insertions(+), 2 deletions(-) diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index 6d16db23..02637096 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -405,6 +405,41 @@ describe("generated T4 API v1 client conformance", () => { })).response.status).toBe(200); }); + it("replays exact PATCH results after an intervening revision", async () => { + const workspaceService = new T4ApiV1ConformanceService(); + const workspaceClient = createT4ApiClient({ baseUrl: workspaceService.origin, credential: "token-a", majorVersion: 1, fetch: workspaceService.fetch }); + requireData(await workspaceClient.http.POST("/v1/workspaces", { + body: { name: "original" }, params: { header: idempotencyHeaders("workspace-replay-setup") }, + })); + const originalWorkspace = requireData(await workspaceClient.http.PATCH("/v1/workspaces/{workspaceId}", { + body: { name: "first" }, params: { header: mutationHeaders(1, "workspace-replay-original"), path: { workspaceId: "ws-1" } }, + })); + requireData(await workspaceClient.http.PATCH("/v1/workspaces/{workspaceId}", { + body: { name: "second" }, params: { header: mutationHeaders(2, "workspace-replay-intervening"), path: { workspaceId: "ws-1" } }, + })); + const workspaceReplay = await workspaceClient.http.PATCH("/v1/workspaces/{workspaceId}", { + body: { name: "first" }, params: { header: mutationHeaders(1, "workspace-replay-original"), path: { workspaceId: "ws-1" } }, + }); + expect(workspaceReplay.response.status).toBe(200); + expect(workspaceReplay.response.headers.get("Idempotency-Replayed")).toBe("true"); + expect(workspaceReplay.data).toEqual(originalWorkspace); + + const sessionService = new T4ApiV1ConformanceService(); + const sessionClient = await seededClient(sessionService, "patch-replay-intervening"); + const originalSession = requireData(await sessionClient.http.PATCH("/v1/sessions/{sessionId}", { + body: { title: "first" }, params: { header: mutationHeaders(1, "session-replay-original"), path: { sessionId: "ses-1" } }, + })); + requireData(await sessionClient.http.PATCH("/v1/sessions/{sessionId}", { + body: { title: "second" }, params: { header: mutationHeaders(2, "session-replay-intervening"), path: { sessionId: "ses-1" } }, + })); + const sessionReplay = await sessionClient.http.PATCH("/v1/sessions/{sessionId}", { + body: { title: "first" }, params: { header: mutationHeaders(1, "session-replay-original"), path: { sessionId: "ses-1" } }, + }); + expect(sessionReplay.response.status).toBe(200); + expect(sessionReplay.response.headers.get("Idempotency-Replayed")).toBe("true"); + expect(sessionReplay.data).toEqual(originalSession); + }); + it("validates creates before idempotency and round-trips valid labels", async () => { const invalidLabels = [ null, [], Object.fromEntries(Array.from({ length: 33 }, (_, index) => [`key-${index}`, "value"])), @@ -463,6 +498,20 @@ describe("generated T4 API v1 client conformance", () => { it("validates If-Match syntax before PATCH idempotency lookup", async () => { const service = new T4ApiV1ConformanceService(); const client = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); + for (const [index, path] of ["/v1/workspaces/missing", "/v1/sessions/missing"].entries()) { + for (const ifMatch of [undefined, "invalid"]) { + const response = await service.fetch(`${service.origin}${path}`, { + method: "PATCH", + headers: { + Authorization: "Bearer token-a", "T4-API-Version": "1", "Idempotency-Key": `missing-if-match-${index}-${ifMatch ?? "absent"}`, "Content-Type": "application/json", + ...(ifMatch === undefined ? {} : { "If-Match": ifMatch }), + }, + body: path.includes("workspaces") ? '{"name":"updated"}' : '{"title":"updated"}', + }); + expect(response.status).toBe(400); + expect((await response.json()) as unknown).toMatchObject({ error: { code: "invalid_request" } }); + } + } requireData(await client.http.POST("/v1/workspaces", { body: { name: "workspace" }, params: { header: idempotencyHeaders("if-match-workspace-setup") }, })); @@ -504,6 +553,20 @@ describe("generated T4 API v1 client conformance", () => { expect(cacheBodyCancelled).toBe(true); }); + it("requires no-store through the exported HTTP watch operation", async () => { + for (const cacheControl of [undefined, "max-age=3600"]) { + const client = createT4ApiClient({ + baseUrl: "https://raw-watch-cache.test", credential: "token-a", majorVersion: 1, + fetch: async () => new Response('data: {"type":"heartbeat","cursor":"raw","observedAt":"2026-07-21T00:00:00Z"}\n\n', { headers: { + "Content-Type": "text/event-stream", "T4-API-Version": "1.0", ...(cacheControl === undefined ? {} : { "Cache-Control": cacheControl }), + } }), + }); + await expect(client.http.GET("/v1/sessions/{sessionId}/events", { + params: { header: VERSION_HEADERS, path: { sessionId: "ses-1" }, query: { maxEvents: 1, heartbeatSeconds: 5 } }, + })).rejects.toMatchObject({ status: 502, retryable: false }); + } + }); + it("rejects invalid RFC 3339 calendar timestamps", async () => { for (const observedAt of [ @@ -519,6 +582,14 @@ describe("generated T4 API v1 client conformance", () => { } }); + it("accepts an RFC 3339 leap second after applying its offset", async () => { + const client = createT4ApiClient({ + baseUrl: "https://watch-leap-second.test", credential: "token-a", majorVersion: 1, + fetch: async () => apiResponse('data: {"type":"heartbeat","cursor":"leap","observedAt":"1990-12-31T15:59:60-08:00"}\n\n', { headers: { "Content-Type": "text/event-stream" } }), + }); + await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).resolves.toMatchObject({ value: { cursor: "leap" } }); + }); + it("takes a snapshot and watches bounded SSE with heartbeat, reconnect, cancellation, and typed resync", async () => { const service = new T4ApiV1ConformanceService(); const client = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); @@ -836,12 +907,17 @@ describe("generated T4 API v1 client conformance", () => { } }); - it("declares the replay response header on both conditional PATCH success responses", () => { + it("declares required public response headers while preserving the 401 exception", () => { const contract = JSON.parse(readFileSync(new URL("../../t4-api-contract/openapi.json", import.meta.url), "utf8")) as { - paths: Record } }>; + paths: Record }; get?: { responses: Record }> } }>; + components: { headers: Record; responses: Record }> }; }; expect(contract.paths["/v1/workspaces/{workspaceId}"]?.patch?.responses["200"]?.$ref).toBe("#/components/responses/WorkspaceReplay"); expect(contract.paths["/v1/sessions/{sessionId}"]?.patch?.responses["200"]?.$ref).toBe("#/components/responses/SessionReplay"); + expect(contract.components.headers.SelectedVersion?.required).toBe(true); + expect(contract.components.headers.IdempotencyReplayed?.required).toBe(true); + expect(contract.paths["/v1/sessions/{sessionId}/events"]?.get?.responses["200"]?.headers?.["Cache-Control"]?.required).toBe(true); + expect(contract.components.responses.Error401?.headers?.["T4-API-Version"]).toBeUndefined(); }); it("treats malformed and oversized watch 503 envelopes as terminal protocol errors", async () => { From e0f397265e8e5d00eb71164d27db10acdaad2144 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 23:52:21 +0000 Subject: [PATCH 037/131] fix: align required response headers --- .../test/t4-api-v1-conformance-service.ts | 28 +++++++++++-------- packages/t4-api-client/src/index.ts | 19 +++++++++++-- packages/t4-api-contract/openapi.json | 4 ++- packages/t4-api-contract/scripts/validate.mjs | 7 +++++ 4 files changed, 44 insertions(+), 14 deletions(-) diff --git a/packages/client/test/t4-api-v1-conformance-service.ts b/packages/client/test/t4-api-v1-conformance-service.ts index 3604a717..89e08371 100644 --- a/packages/client/test/t4-api-v1-conformance-service.ts +++ b/packages/client/test/t4-api-v1-conformance-service.ts @@ -21,12 +21,12 @@ function validLabels(value: unknown): boolean { /^[a-z][a-z0-9.-]{0,62}$/u.test(key) && typeof item === "string" && hasAtMostCodePoints(item, 128)); } -function validCreate(body: Record, textField: "name" | "title"): boolean { - const keys = Object.keys(body); - if (keys.some((key) => key !== textField && key !== "labels")) return false; +function createViolation(body: Record, textField: "name" | "title"): { field: string; rule: string } | undefined { + if (Object.keys(body).some((key) => key !== textField && key !== "labels")) return { field: "body", rule: "schema" }; const text = body[textField]; - return typeof text === "string" && text !== "" && hasAtMostCodePoints(text, 128) && - (body.labels === undefined || validLabels(body.labels)); + if (typeof text !== "string" || text === "" || !hasAtMostCodePoints(text, 128)) return { field: textField, rule: "length" }; + if (body.labels !== undefined && !validLabels(body.labels)) return { field: "labels", rule: "bounds" }; + return undefined; } function validMutation(body: Record, textField: "name" | "title"): boolean { @@ -130,7 +130,8 @@ export class T4ApiV1ConformanceService { const parsed = await this.#jsonBody(request); if (parsed instanceof Response) return parsed; const body = parsed; - if (!validCreate(body, "name")) return this.#invalid("body", "schema", "workspace create must match WorkspaceCreate"); + const violation = createViolation(body, "name"); + if (violation !== undefined) return this.#invalid(violation.field, violation.rule, "workspace create must match WorkspaceCreate"); return this.#idempotent(request, tenant, "createWorkspace", [], body, 202, 200, () => { const id = `ws-${++this.#workspaceSequence}`; const workspace = { id, name: body.name, ...(body.labels === undefined ? {} : { labels: body.labels }), state: "accepted", revision: 1, tenant }; @@ -152,6 +153,10 @@ export class T4ApiV1ConformanceService { const workspaceMatch = url.pathname.match(/^\/v1\/workspaces\/([^/]+)$/u); if (workspaceMatch) { const id = decodeURIComponent(workspaceMatch[1]!); + if (request.method === "PATCH") { + const ifMatch = request.headers.get("If-Match"); + if (ifMatch === null || !/^[1-9][0-9]{0,18}$/u.test(ifMatch)) return problem(400, "invalid_request", "If-Match is invalid"); + } const workspace = this.#workspaces.get(id); if (request.method === "DELETE") { return this.#idempotent(request, tenant, "deleteWorkspace", [id], null, 204, 204, () => { @@ -162,8 +167,6 @@ export class T4ApiV1ConformanceService { if (workspace?.tenant !== tenant) return problem(404, "not_found", "Workspace not found"); if (request.method === "GET") return json(200, this.#payload("workspace", this.#visible(workspace))); if (request.method === "PATCH") { - const ifMatch = request.headers.get("If-Match"); - if (ifMatch === null || !/^[1-9][0-9]{0,18}$/u.test(ifMatch)) return problem(400, "invalid_request", "If-Match is invalid"); const parsed = await this.#jsonBody(request); if (parsed instanceof Response) return parsed; const body = parsed; @@ -185,7 +188,8 @@ export class T4ApiV1ConformanceService { const parsed = await this.#jsonBody(request); if (parsed instanceof Response) return parsed; const body = parsed; - if (!validCreate(body, "title")) return this.#invalid("body", "schema", "session create must match SessionCreate"); + const violation = createViolation(body, "title"); + if (violation !== undefined) return this.#invalid(violation.field, violation.rule, "session create must match SessionCreate"); return this.#idempotent(request, tenant, "spawnSession", [workspaceId], body, 202, 200, () => { const id = `ses-${++this.#sessionSequence}`; const session = { id, workspaceId, title: body.title, ...(body.labels === undefined ? {} : { labels: body.labels }), state: "accepted", revision: 1, tenant }; @@ -207,6 +211,10 @@ export class T4ApiV1ConformanceService { const sessionMatch = url.pathname.match(/^\/v1\/sessions\/([^/]+)$/u); if (sessionMatch) { const id = decodeURIComponent(sessionMatch[1]!); + if (request.method === "PATCH") { + const ifMatch = request.headers.get("If-Match"); + if (ifMatch === null || !/^[1-9][0-9]{0,18}$/u.test(ifMatch)) return problem(400, "invalid_request", "If-Match is invalid"); + } const session = this.#sessions.get(id); if (request.method === "DELETE") { return this.#idempotent(request, tenant, "deleteSession", [id], null, 204, 204, () => { @@ -217,8 +225,6 @@ export class T4ApiV1ConformanceService { if (session?.tenant !== tenant) return problem(404, "not_found", "Session not found"); if (request.method === "GET") return json(200, this.#payload("session", this.#visible(session))); if (request.method === "PATCH") { - const ifMatch = request.headers.get("If-Match"); - if (ifMatch === null || !/^[1-9][0-9]{0,18}$/u.test(ifMatch)) return problem(400, "invalid_request", "If-Match is invalid"); const parsed = await this.#jsonBody(request); if (parsed instanceof Response) return parsed; const body = parsed; diff --git a/packages/t4-api-client/src/index.ts b/packages/t4-api-client/src/index.ts index dbf55669..a5b43e1c 100644 --- a/packages/t4-api-client/src/index.ts +++ b/packages/t4-api-client/src/index.ts @@ -473,6 +473,10 @@ async function validateResponse(request: Request, response: Response, baseUrl: s return; } if (validator === "event-stream") { + if (response.headers.get("Cache-Control") !== "no-store") { + void response.body?.cancel().catch(() => {}); + throw protocolError(502, "T4 API watch did not return Cache-Control: no-store"); + } if (response.headers.get("content-type")?.split(";", 1)[0]?.trim().toLowerCase() !== "text/event-stream") { void response.body?.cancel().catch(() => {}); throw protocolError(502, "T4 API returned an undeclared success media type"); @@ -507,8 +511,19 @@ function validRfc3339DateTime(value: string): boolean { if (month === 2) maximumDay = year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0) ? 29 : 28; else if (month === 4 || month === 6 || month === 9 || month === 11) maximumDay = 30; if (day < 1 || day > maximumDay) return false; - if (second === 60 && (hour !== 23 || minute !== 59)) return false; - return match[7] === undefined || (Number(match[8]) <= 23 && Number(match[9]) <= 59); + let offsetMinutes = 0; + if (match[7] !== undefined) { + const offsetHour = Number(match[8]); + const offsetMinute = Number(match[9]); + if (offsetHour > 23 || offsetMinute > 59) return false; + offsetMinutes = (match[7] === "+" ? 1 : -1) * (offsetHour * 60 + offsetMinute); + } + if (second < 60) return true; + const utc = new Date(0); + utc.setUTCFullYear(year, month - 1, day); + utc.setUTCHours(hour, minute - offsetMinutes, 0, 0); + return utc.getUTCHours() === 23 && utc.getUTCMinutes() === 59 && + ((utc.getUTCMonth() === 5 && utc.getUTCDate() === 30) || (utc.getUTCMonth() === 11 && utc.getUTCDate() === 31)); } function watchEvent(value: unknown, eventId: string | undefined): WatchEvent { diff --git a/packages/t4-api-contract/openapi.json b/packages/t4-api-contract/openapi.json index f4433421..5be952d5 100644 --- a/packages/t4-api-contract/openapi.json +++ b/packages/t4-api-contract/openapi.json @@ -319,7 +319,7 @@ "200": { "description": "Bounded SSE event stream. Every non-empty SSE data field is one JSON value conforming to WatchEvent; clients MUST reject unknown fields and schema-invalid event payloads before delivery. Transport chunk boundaries do not delimit frames.", "headers": { - "Cache-Control": { "schema": { "type": "string", "const": "no-store" } }, + "Cache-Control": { "required": true, "schema": { "type": "string", "const": "no-store" } }, "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { @@ -420,10 +420,12 @@ "headers": { "SelectedVersion": { "description": "Selected compatible T4 API minor version.", + "required": true, "schema": { "type": "string", "pattern": "^1\\.[0-9]+$", "maxLength": 16 } }, "IdempotencyReplayed": { "description": "True when this response replays a prior identical request.", + "required": true, "schema": { "type": "string", "enum": ["true", "false"] } } }, diff --git a/packages/t4-api-contract/scripts/validate.mjs b/packages/t4-api-contract/scripts/validate.mjs index e3a12460..d43c7e84 100644 --- a/packages/t4-api-contract/scripts/validate.mjs +++ b/packages/t4-api-contract/scripts/validate.mjs @@ -27,6 +27,10 @@ for (const [schema, property] of [["Discovery", "supportedMajors"], ["Discovery" if (schemas[schema]?.properties?.[property]?.uniqueItems !== true) throw new Error(`${schema}.${property} must retain uniqueItems`); } +const selectedVersionHeader = document.components?.headers?.SelectedVersion; +const replayHeaderObject = document.components?.headers?.IdempotencyReplayed; +if (selectedVersionHeader?.required !== true) throw new Error("SelectedVersion must remain required wherever referenced"); +if (replayHeaderObject?.required !== true) throw new Error("IdempotencyReplayed must remain required wherever referenced"); const selectedVersion = document.components?.headers?.SelectedVersion?.schema; if (selectedVersion?.pattern !== "^1\\.[0-9]+$" || selectedVersion?.maxLength !== 16) throw new Error("SelectedVersion must remain a bounded v1 minor"); const replayHeader = document.components?.headers?.IdempotencyReplayed?.schema; @@ -42,6 +46,9 @@ for (const responseName of ["WorkspaceAccepted", "WorkspaceReplay", "SessionAcce if (document.components?.responses?.[responseName]?.headers?.["Idempotency-Replayed"]?.$ref !== replayRef) throw new Error(`${responseName} must declare IdempotencyReplayed`); } if (document.paths?.["/v1/sessions/{sessionId}/events"]?.get?.responses?.["200"]?.headers?.["T4-API-Version"]?.$ref !== selectedVersionRef) throw new Error("watch success must declare SelectedVersion"); +const watchCacheControl = document.paths?.["/v1/sessions/{sessionId}/events"]?.get?.responses?.["200"]?.headers?.["Cache-Control"]; +if (watchCacheControl?.required !== true || watchCacheControl?.schema?.const !== "no-store") throw new Error("watch success must require Cache-Control: no-store"); +if (document.components?.responses?.Error401?.headers?.["T4-API-Version"] !== undefined) throw new Error("Error401 must omit SelectedVersion"); if (document.paths?.["/v1/workspaces/{workspaceId}"]?.patch?.responses?.["200"]?.$ref !== "#/components/responses/WorkspaceReplay") throw new Error("workspace PATCH success must declare replay headers"); if (document.paths?.["/v1/sessions/{sessionId}"]?.patch?.responses?.["200"]?.$ref !== "#/components/responses/SessionReplay") throw new Error("session PATCH success must declare replay headers"); const errorResponses = { 400: "Error400", 401: "Error401", 403: "Error403", 404: "Error404", 406: "Error406", 409: "Error409", 410: "Error410", 422: "Error422", 503: "Error503" }; From a0c3a08006665575260f67c598dc3606697ad5f2 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 23:54:30 +0000 Subject: [PATCH 038/131] chore: accept required-header schema artifact --- packages/t4-api-client/src/generated/schema.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/t4-api-client/src/generated/schema.ts b/packages/t4-api-client/src/generated/schema.ts index 45a57cbf..4cdd50b9 100755 --- a/packages/t4-api-client/src/generated/schema.ts +++ b/packages/t4-api-client/src/generated/schema.ts @@ -799,7 +799,7 @@ export interface operations { /** @description Bounded SSE event stream. Every non-empty SSE data field is one JSON value conforming to WatchEvent; clients MUST reject unknown fields and schema-invalid event payloads before delivery. Transport chunk boundaries do not delimit frames. */ 200: { headers: { - "Cache-Control"?: "no-store"; + "Cache-Control": "no-store"; "T4-API-Version": components["headers"]["SelectedVersion"]; [name: string]: unknown; }; From d71383f17ded91bbeda9e31b7889209b58945258 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 00:08:56 +0000 Subject: [PATCH 039/131] test(t4-api): cover final strict response gaps --- .../client/test/t4-api-v1-conformance.test.ts | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index 02637096..5065950e 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -590,6 +590,48 @@ describe("generated T4 API v1 client conformance", () => { await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).resolves.toMatchObject({ value: { cursor: "leap" } }); }); + it("accepts lowercase RFC 3339 delimiters", async () => { + const client = createT4ApiClient({ + baseUrl: "https://watch-lowercase-time.test", credential: "token-a", majorVersion: 1, + fetch: async () => apiResponse('data: {"type":"heartbeat","cursor":"lowercase","observedAt":"2026-07-21t00:00:00z"}\n\n', { headers: { "Content-Type": "text/event-stream" } }), + }); + await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).resolves.toMatchObject({ value: { cursor: "lowercase" } }); + }); + + it("uses typed low-level stream parsing and the SSE Accept header", async () => { + const service = new T4ApiV1ConformanceService(); + const client = await seededClient(service, "raw-stream"); + const wrongAccept = await service.fetch(`${service.origin}/v1/sessions/ses-1/events?maxEvents=1&heartbeatSeconds=5`, { + headers: { Authorization: "Bearer token-a", "T4-API-Version": "1", Accept: "application/json" }, + }); + expect(wrongAccept.status).toBe(406); + const streamed = await client.http.GET("/v1/sessions/{sessionId}/events", { + params: { header: VERSION_HEADERS, path: { sessionId: "ses-1" }, query: { maxEvents: 1, heartbeatSeconds: 5 } }, + parseAs: "stream", + }); + expect(streamed.response.headers.get("Content-Type")).toBe("text/event-stream"); + expect(streamed.response.headers.get("Cache-Control")).toBe("no-store"); + expect(streamed.data).toBeInstanceOf(ReadableStream); + await streamed.data?.cancel(); + }); + + it("rejects malformed present Retry-After on ordinary and watch 503 responses", async () => { + for (const retryAfter of ["", "not-a-date", "-1", "x".repeat(129)]) { + const body = { error: { code: "unavailable", message: "later", requestId: "r", retryable: true } }; + const ordinary = createT4ApiClient({ + baseUrl: "https://ordinary-retry-after.test", credential: "token-a", majorVersion: 1, + fetch: async () => jsonResponse(body, { status: 503, headers: { "Retry-After": retryAfter } }), + }); + await expect(ordinary.http.GET("/v1", { params: { header: VERSION_HEADERS } })).rejects.toMatchObject({ status: 502, retryable: false }); + + const watch = createT4ApiClient({ + baseUrl: "https://watch-retry-after.test", credential: "token-a", majorVersion: 1, + fetch: async () => jsonResponse(body, { status: 503, headers: { "Retry-After": retryAfter } }), + }); + await expect(watch.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0, retryBackoffMs: 0 }).next()).rejects.toMatchObject({ status: 502, retryable: false }); + } + }); + it("takes a snapshot and watches bounded SSE with heartbeat, reconnect, cancellation, and typed resync", async () => { const service = new T4ApiV1ConformanceService(); const client = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); From deff712ad320d592913115199637a095f3a7b288 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 00:15:53 +0000 Subject: [PATCH 040/131] fix(t4-api): close strict response gaps --- .../test/t4-api-v1-conformance-service.ts | 1 + .../client/test/t4-api-v1-conformance.test.ts | 14 +++++ packages/t4-api-client/src/index.ts | 60 +++++++++++++++++-- 3 files changed, 69 insertions(+), 6 deletions(-) diff --git a/packages/client/test/t4-api-v1-conformance-service.ts b/packages/client/test/t4-api-v1-conformance-service.ts index 89e08371..4c288abd 100644 --- a/packages/client/test/t4-api-v1-conformance-service.ts +++ b/packages/client/test/t4-api-v1-conformance-service.ts @@ -277,6 +277,7 @@ export class T4ApiV1ConformanceService { const watchMatch = url.pathname.match(/^\/v1\/sessions\/([^/]+)\/events$/u); if (watchMatch && request.method === "GET") { + if (request.headers.get("Accept") !== "text/event-stream") return problem(406, "incompatible_version", "Watch requests must accept text/event-stream", { supportedMajors: [1] }); const id = decodeURIComponent(watchMatch[1]!); const session = this.#sessions.get(id); if (session?.tenant !== tenant) return problem(404, "not_found", "Session not found"); diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index 5065950e..9c847949 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -632,6 +632,20 @@ describe("generated T4 API v1 client conformance", () => { } }); + it("accepts every RFC 9110 HTTP-date form in Retry-After", async () => { + for (const retryAfter of [ + "Sun, 06 Nov 1994 08:49:37 GMT", + "Sunday, 06-Nov-94 08:49:37 GMT", + "Sun Nov 6 08:49:37 1994", + ]) { + const client = createT4ApiClient({ + baseUrl: "https://http-date-retry-after.test", credential: "token-a", majorVersion: 1, + fetch: async () => jsonResponse({ error: { code: "unavailable", message: "later", requestId: "r", retryable: true } }, { status: 503, headers: { "Retry-After": retryAfter } }), + }); + await expect(client.http.GET("/v1", { params: { header: VERSION_HEADERS } })).resolves.toMatchObject({ error: { error: { code: "unavailable" } } }); + } + }); + it("takes a snapshot and watches bounded SSE with heartbeat, reconnect, cancellation, and typed resync", async () => { const service = new T4ApiV1ConformanceService(); const client = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); diff --git a/packages/t4-api-client/src/index.ts b/packages/t4-api-client/src/index.ts index a5b43e1c..3d2fcd8a 100644 --- a/packages/t4-api-client/src/index.ts +++ b/packages/t4-api-client/src/index.ts @@ -10,6 +10,16 @@ const MAX_EVENT_BYTES = 1024 * 1024; const MAX_JSON_RESPONSE_BYTES = 16 * 1024 * 1024; const CURSOR_PATTERN = /^[A-Za-z0-9._~-]+$/u; const SELECTED_VERSION_MAX_LENGTH = 16; +const INVALID_RETRY_AFTER = Symbol("invalid Retry-After"); +const HTTP_MONTHS: Readonly> = { + Jan: 0, Feb: 1, Mar: 2, Apr: 3, May: 4, Jun: 5, + Jul: 6, Aug: 7, Sep: 8, Oct: 9, Nov: 10, Dec: 11, +}; +const HTTP_WEEKDAYS: Readonly> = { + Sun: 0, Sunday: 0, Mon: 1, Monday: 1, Tue: 2, Tuesday: 2, + Wed: 3, Wednesday: 3, Thu: 4, Thursday: 4, Fri: 5, Friday: 5, + Sat: 6, Saturday: 6, +}; const SELECTED_VERSION_PATTERN = /^1\.[0-9]+$/u; const ERROR_CODES = { invalid_request: true, unauthenticated: true, forbidden: true, not_found: true, @@ -65,6 +75,10 @@ export interface WatchSessionOptions { } export interface T4ApiClient { + /** + * Low-level generated client. Event-stream calls must pass `parseAs: "stream"`; + * use `watchSession` for bounded, validated event decoding and reconnects. + */ readonly http: Client; watchSession(sessionId: string, options?: WatchSessionOptions): AsyncGenerator; } @@ -217,12 +231,43 @@ async function boundedResponseText(response: Response, maximumBytes = MAX_ERROR_ try { return new TextDecoder("utf-8", { fatal: true }).decode(bytes); } catch { return undefined; } } -function retryAfterMilliseconds(response: Response): number | undefined { - const value = response.headers.get("Retry-After")?.trim(); - if (value === undefined || value === "") return undefined; - if (/^[0-9]+$/u.test(value)) return Math.min(30_000, Number(value) * 1000); +function matchingHttpDate( + timestamp: number, + parts: RegExpExecArray, + indexes: { readonly weekday: number; readonly day: number; readonly alternateDay?: number; readonly month: number; readonly year: number; readonly hour: number; readonly minute: number; readonly second: number }, + shortYear = false, +): boolean { + const date = new Date(timestamp); + const year = Number(parts[indexes.year]); + return date.getUTCDay() === HTTP_WEEKDAYS[parts[indexes.weekday]!] && + date.getUTCDate() === Number(parts[indexes.day] ?? parts[indexes.alternateDay ?? indexes.day]) && + date.getUTCMonth() === HTTP_MONTHS[parts[indexes.month]!] && + (shortYear ? date.getUTCFullYear() % 100 === year : date.getUTCFullYear() === year) && + date.getUTCHours() === Number(parts[indexes.hour]) && + date.getUTCMinutes() === Number(parts[indexes.minute]) && + date.getUTCSeconds() === Number(parts[indexes.second]); +} + +function httpDateTimestamp(value: string): number | undefined { const timestamp = Date.parse(value); if (!Number.isFinite(timestamp)) return undefined; + const imf = /^(Sun|Mon|Tue|Wed|Thu|Fri|Sat), ([0-9]{2}) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) ([0-9]{4}) ([0-9]{2}):([0-9]{2}):([0-9]{2}) GMT$/u.exec(value); + if (imf !== null && matchingHttpDate(timestamp, imf, { weekday: 1, day: 2, month: 3, year: 4, hour: 5, minute: 6, second: 7 })) return timestamp; + const rfc850 = /^(Sunday|Monday|Tuesday|Wednesday|Thursday|Friday|Saturday), ([0-9]{2})-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-([0-9]{2}) ([0-9]{2}):([0-9]{2}):([0-9]{2}) GMT$/u.exec(value); + if (rfc850 !== null && matchingHttpDate(timestamp, rfc850, { weekday: 1, day: 2, month: 3, year: 4, hour: 5, minute: 6, second: 7 }, true)) return timestamp; + const asctime = /^(Sun|Mon|Tue|Wed|Thu|Fri|Sat) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) (?:([0-9]{2})| ([0-9])) ([0-9]{2}):([0-9]{2}):([0-9]{2}) ([0-9]{4})$/u.exec(value); + if (asctime === null) return undefined; + return matchingHttpDate(timestamp, asctime, { weekday: 1, day: 3, alternateDay: 4, month: 2, year: 8, hour: 5, minute: 6, second: 7 }) ? timestamp : undefined; +} + +function retryAfterMilliseconds(response: Response): number | undefined | typeof INVALID_RETRY_AFTER { + const header = response.headers.get("Retry-After"); + if (header === null) return undefined; + const value = header.trim(); + if (value === "" || !hasAtMostCodePoints(value, 128)) return INVALID_RETRY_AFTER; + if (/^[0-9]+$/u.test(value)) return Math.min(30_000, Number(value) * 1000); + const timestamp = httpDateTimestamp(value); + if (timestamp === undefined) return INVALID_RETRY_AFTER; return Math.min(30_000, Math.max(0, timestamp - Date.now())); } @@ -247,6 +292,7 @@ async function parsedError(response: Response): Promise const decoded = apiError(JSON.parse(text), response.status); if (decoded === undefined) return undefined; const retryAfterMs = retryAfterMilliseconds(response); + if (retryAfterMs === INVALID_RETRY_AFTER) return undefined; return retryAfterMs === undefined ? new T4ApiError(response.status, decoded) : new T4ApiError(response.status, decoded, { retryAfterMs }); @@ -498,7 +544,7 @@ async function validateResponse(request: Request, response: Response, baseUrl: s function validRfc3339DateTime(value: string): boolean { if (!hasAtMostCodePoints(value, 64)) return false; - const match = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d+)?(?:Z|([+-])(\d{2}):(\d{2}))$/u.exec(value); + const match = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d+)?(?:Z|([+-])(\d{2}):(\d{2}))$/iu.exec(value); if (match === null) return false; const year = Number(match[1]); const month = Number(match[2]); @@ -800,7 +846,9 @@ export function createT4ApiClient(options: T4ApiClientOptions): T4ApiClient { const request = new Request(input, init); request.headers.set("Authorization", `Bearer ${credential}`); request.headers.set("T4-API-Version", majorVersion); - request.headers.set("Accept", "application/json"); + const path = relativeApiPath(request, baseUrl); + const contract = path === undefined ? undefined : responseContract(request.method, path); + request.headers.set("Accept", contract?.success[200] === "event-stream" ? "text/event-stream" : "application/json"); const response = await fetchImpl(request); await validateResponse(request, response, baseUrl); return response; From 6ca35ab37e2556e3c313ad3f8361110499111f83 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 00:25:48 +0000 Subject: [PATCH 041/131] test(t4-api): isolate watch query validation --- packages/client/test/t4-api-v1-conformance.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index 9c847949..68d20b67 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -724,7 +724,7 @@ describe("generated T4 API v1 client conformance", () => { expect(await response.json()).toMatchObject({ error: { code: "invalid_request" } }); } for (const query of ["maxEvents=0", "maxEvents=5", "heartbeatSeconds=4", "heartbeatSeconds=61"]) { - const response = await service.fetch(`${service.origin}/v1/sessions/ses-1/events?${query}`, { headers: baseHeaders }); + const response = await service.fetch(`${service.origin}/v1/sessions/ses-1/events?${query}`, { headers: { ...baseHeaders, Accept: "text/event-stream" } }); expect(response.status).toBe(422); expect(await response.json()).toMatchObject({ error: { code: "invalid_request", violations: expect.any(Array) } }); } From 643de6e9cb4f4218f2d20ae12cd76040bf5527ce Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 00:37:01 +0000 Subject: [PATCH 042/131] test(t4-api): pin leap dates and RFC850 pivot --- .../client/test/t4-api-v1-conformance.test.ts | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index 68d20b67..22b1e29a 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -573,6 +573,7 @@ describe("generated T4 API v1 client conformance", () => { "2026-02-31T00:00:00Z", "2026-13-01T00:00:00Z", "2025-02-29T00:00:00Z", "2026-04-31T00:00:00Z", "2026-01-01T24:00:00Z", "2026-01-01T00:60:00Z", "2026-01-01T00:00:00+24:00", "2026-01-01T00:00:00+01:60", + "2026-06-30T23:59:60Z", "2026-12-31T23:59:60Z", "2026-06-30T15:59:60-08:00", ]) { const calendarClient = createT4ApiClient({ baseUrl: "https://watch-calendar.test", credential: "token-a", majorVersion: 1, @@ -632,7 +633,7 @@ describe("generated T4 API v1 client conformance", () => { } }); - it("accepts every RFC 9110 HTTP-date form in Retry-After", async () => { + it("accepts every RFC 9110 HTTP-date form in Retry-After without dropping its delay", async () => { for (const retryAfter of [ "Sun, 06 Nov 1994 08:49:37 GMT", "Sunday, 06-Nov-94 08:49:37 GMT", @@ -642,7 +643,25 @@ describe("generated T4 API v1 client conformance", () => { baseUrl: "https://http-date-retry-after.test", credential: "token-a", majorVersion: 1, fetch: async () => jsonResponse({ error: { code: "unavailable", message: "later", requestId: "r", retryable: true } }, { status: 503, headers: { "Retry-After": retryAfter } }), }); - await expect(client.http.GET("/v1", { params: { header: VERSION_HEADERS } })).resolves.toMatchObject({ error: { error: { code: "unavailable" } } }); + await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ + status: 502, cause: { status: 503, retryAfterMs: 0 }, + }); + } + }); + + it("resolves RFC 850 two-digit years relative to the current year", async () => { + vi.useFakeTimers(); + try { + vi.setSystemTime(new Date("2026-07-21T00:00:00Z")); + const client = createT4ApiClient({ + baseUrl: "https://rfc850-year.test", credential: "token-a", majorVersion: 1, + fetch: async () => jsonResponse({ error: { code: "unavailable", message: "later", requestId: "r", retryable: true } }, { status: 503, headers: { "Retry-After": "Saturday, 01-Jan-50 00:00:00 GMT" } }), + }); + await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ + status: 502, cause: { status: 503, retryAfterMs: 30_000 }, + }); + } finally { + vi.useRealTimers(); } }); From 451b3271414c7b0b413a6b3de2c6985520ae6e46 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 00:41:02 +0000 Subject: [PATCH 043/131] fix(t4-api): validate announced leap seconds --- packages/t4-api-client/src/index.ts | 43 +++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/packages/t4-api-client/src/index.ts b/packages/t4-api-client/src/index.ts index 3d2fcd8a..2cf5ea93 100644 --- a/packages/t4-api-client/src/index.ts +++ b/packages/t4-api-client/src/index.ts @@ -235,29 +235,39 @@ function matchingHttpDate( timestamp: number, parts: RegExpExecArray, indexes: { readonly weekday: number; readonly day: number; readonly alternateDay?: number; readonly month: number; readonly year: number; readonly hour: number; readonly minute: number; readonly second: number }, - shortYear = false, + expectedYear = Number(parts[indexes.year]), ): boolean { const date = new Date(timestamp); - const year = Number(parts[indexes.year]); return date.getUTCDay() === HTTP_WEEKDAYS[parts[indexes.weekday]!] && date.getUTCDate() === Number(parts[indexes.day] ?? parts[indexes.alternateDay ?? indexes.day]) && date.getUTCMonth() === HTTP_MONTHS[parts[indexes.month]!] && - (shortYear ? date.getUTCFullYear() % 100 === year : date.getUTCFullYear() === year) && + date.getUTCFullYear() === expectedYear && date.getUTCHours() === Number(parts[indexes.hour]) && date.getUTCMinutes() === Number(parts[indexes.minute]) && date.getUTCSeconds() === Number(parts[indexes.second]); } function httpDateTimestamp(value: string): number | undefined { - const timestamp = Date.parse(value); - if (!Number.isFinite(timestamp)) return undefined; const imf = /^(Sun|Mon|Tue|Wed|Thu|Fri|Sat), ([0-9]{2}) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) ([0-9]{4}) ([0-9]{2}):([0-9]{2}):([0-9]{2}) GMT$/u.exec(value); - if (imf !== null && matchingHttpDate(timestamp, imf, { weekday: 1, day: 2, month: 3, year: 4, hour: 5, minute: 6, second: 7 })) return timestamp; + if (imf !== null) { + const timestamp = Date.parse(value); + return Number.isFinite(timestamp) && matchingHttpDate(timestamp, imf, { weekday: 1, day: 2, month: 3, year: 4, hour: 5, minute: 6, second: 7 }) ? timestamp : undefined; + } const rfc850 = /^(Sunday|Monday|Tuesday|Wednesday|Thursday|Friday|Saturday), ([0-9]{2})-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-([0-9]{2}) ([0-9]{2}):([0-9]{2}):([0-9]{2}) GMT$/u.exec(value); - if (rfc850 !== null && matchingHttpDate(timestamp, rfc850, { weekday: 1, day: 2, month: 3, year: 4, hour: 5, minute: 6, second: 7 }, true)) return timestamp; + if (rfc850 !== null) { + const currentYear = new Date(Date.now()).getUTCFullYear(); + let year = Math.floor(currentYear / 100) * 100 + Number(rfc850[4]); + if (year - currentYear > 50) year -= 100; + const date = new Date(0); + date.setUTCFullYear(year, HTTP_MONTHS[rfc850[3]!]!, Number(rfc850[2])); + date.setUTCHours(Number(rfc850[5]), Number(rfc850[6]), Number(rfc850[7]), 0); + const timestamp = date.getTime(); + return matchingHttpDate(timestamp, rfc850, { weekday: 1, day: 2, month: 3, year: 4, hour: 5, minute: 6, second: 7 }, year) ? timestamp : undefined; + } const asctime = /^(Sun|Mon|Tue|Wed|Thu|Fri|Sat) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) (?:([0-9]{2})| ([0-9])) ([0-9]{2}):([0-9]{2}):([0-9]{2}) ([0-9]{4})$/u.exec(value); if (asctime === null) return undefined; - return matchingHttpDate(timestamp, asctime, { weekday: 1, day: 3, alternateDay: 4, month: 2, year: 8, hour: 5, minute: 6, second: 7 }) ? timestamp : undefined; + const timestamp = Date.parse(value); + return Number.isFinite(timestamp) && matchingHttpDate(timestamp, asctime, { weekday: 1, day: 3, alternateDay: 4, month: 2, year: 8, hour: 5, minute: 6, second: 7 }) ? timestamp : undefined; } function retryAfterMilliseconds(response: Response): number | undefined | typeof INVALID_RETRY_AFTER { @@ -542,6 +552,21 @@ async function validateResponse(request: Request, response: Response, baseUrl: s } } +function isKnownUtcLeapSecondDate(year: number, month: number, day: number): boolean { + // Update this IERS insertion-date table when a new UTC leap second is announced. + switch (year * 10_000 + month * 100 + day) { + case 19720630: case 19721231: case 19731231: case 19741231: case 19751231: + case 19761231: case 19771231: case 19781231: case 19791231: case 19810630: + case 19820630: case 19830630: case 19850630: case 19871231: case 19891231: + case 19901231: case 19920630: case 19930630: case 19940630: case 19951231: + case 19970630: case 19981231: case 20051231: case 20081231: case 20120630: + case 20150630: case 20161231: + return true; + default: + return false; + } +} + function validRfc3339DateTime(value: string): boolean { if (!hasAtMostCodePoints(value, 64)) return false; const match = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d+)?(?:Z|([+-])(\d{2}):(\d{2}))$/iu.exec(value); @@ -569,7 +594,7 @@ function validRfc3339DateTime(value: string): boolean { utc.setUTCFullYear(year, month - 1, day); utc.setUTCHours(hour, minute - offsetMinutes, 0, 0); return utc.getUTCHours() === 23 && utc.getUTCMinutes() === 59 && - ((utc.getUTCMonth() === 5 && utc.getUTCDate() === 30) || (utc.getUTCMonth() === 11 && utc.getUTCDate() === 31)); + isKnownUtcLeapSecondDate(utc.getUTCFullYear(), utc.getUTCMonth() + 1, utc.getUTCDate()); } function watchEvent(value: unknown, eventId: string | undefined): WatchEvent { From 842395866a1cac5780a62e038ec3a18fc90f28a7 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 00:56:13 +0000 Subject: [PATCH 044/131] test(t4-api): require declared request media --- .../client/test/t4-api-v1-conformance.test.ts | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index 22b1e29a..2d86037f 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -495,6 +495,35 @@ describe("generated T4 API v1 client conformance", () => { })).response.status).toBe(202); }); + it("rejects undeclared JSON request media before mutation and idempotency", async () => { + for (const contentType of [undefined, "text/plain"]) { + const service = new T4ApiV1ConformanceService(); + await seededClient(service, `request-media-${contentType ?? "missing"}`); + const cases = [ + ["POST", "/v1/workspaces", '{"name":"media-workspace"}', {}, 202, { id: "ws-2" }], + ["POST", "/v1/workspaces/ws-1/sessions", '{"title":"media-session"}', {}, 202, { id: "ses-2" }], + ["PATCH", "/v1/workspaces/ws-1", '{"name":"renamed"}', { "If-Match": "1" }, 200, { revision: 2 }], + ["PATCH", "/v1/sessions/ses-1", '{"title":"retitled"}', { "If-Match": "1" }, 200, { revision: 2 }], + ["POST", "/v1/sessions/ses-1/commands", '{"command":"ok"}', {}, 202, { commandId: "cmd-1" }], + ] as const; + for (const [index, [method, path, body, routeHeaders, successStatus, successBody]] of cases.entries()) { + const key = `request-media-${contentType ?? "missing"}-${index}`; + const headers = { Authorization: "Bearer token-a", "T4-API-Version": "1", "Idempotency-Key": key, ...routeHeaders }; + const invalid = await service.fetch(`${service.origin}${path}`, { + method, headers: { ...headers, ...(contentType === undefined ? {} : { "Content-Type": contentType }) }, body, + }); + expect(invalid.status).toBe(400); + expect(await invalid.json()).toMatchObject({ error: { code: "invalid_request" } }); + const valid = await service.fetch(`${service.origin}${path}`, { + method, headers: { ...headers, "Content-Type": "Application/JSON; Charset=UTF-8" }, body, + }); + expect(valid.status).toBe(successStatus); + expect(valid.headers.get("Idempotency-Replayed")).toBe("false"); + expect(await valid.json()).toMatchObject(successBody); + } + } + }); + it("validates If-Match syntax before PATCH idempotency lookup", async () => { const service = new T4ApiV1ConformanceService(); const client = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); @@ -665,6 +694,22 @@ describe("generated T4 API v1 client conformance", () => { } }); + it("applies the RFC 850 pivot to the complete timestamp", async () => { + vi.useFakeTimers(); + try { + vi.setSystemTime(new Date("2026-07-21T00:00:00Z")); + const client = createT4ApiClient({ + baseUrl: "https://rfc850-boundary.test", credential: "token-a", majorVersion: 1, + fetch: async () => jsonResponse({ error: { code: "unavailable", message: "later", requestId: "r", retryable: true } }, { status: 503, headers: { "Retry-After": "Friday, 31-Dec-76 00:00:00 GMT" } }), + }); + await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ + status: 502, cause: { status: 503, retryAfterMs: 0 }, + }); + } finally { + vi.useRealTimers(); + } + }); + it("takes a snapshot and watches bounded SSE with heartbeat, reconnect, cancellation, and typed resync", async () => { const service = new T4ApiV1ConformanceService(); const client = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); From 01a49d63b572a07cbc513727ed895ce3192ebefc Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 00:59:26 +0000 Subject: [PATCH 045/131] fix(t4-api): enforce JSON request media --- packages/client/test/t4-api-v1-conformance-service.ts | 9 +++++++++ packages/t4-api-client/src/index.ts | 11 ++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/packages/client/test/t4-api-v1-conformance-service.ts b/packages/client/test/t4-api-v1-conformance-service.ts index 4c288abd..03599b26 100644 --- a/packages/client/test/t4-api-v1-conformance-service.ts +++ b/packages/client/test/t4-api-v1-conformance-service.ts @@ -254,6 +254,8 @@ export class T4ApiV1ConformanceService { const id = decodeURIComponent(commandMatch[1]!); const session = this.#sessions.get(id); if (session?.tenant !== tenant) return problem(404, "not_found", "Session not found"); + const mediaError = this.#jsonMediaError(request); + if (mediaError !== undefined) return mediaError; const text = await request.text(); if (encoder.encode(text).byteLength > COMMAND_REQUEST_BYTES_MAX) return this.#invalid("body", "maxBytes", `request must not exceed ${COMMAND_REQUEST_BYTES_MAX} UTF-8 bytes`); let decoded: unknown; @@ -332,7 +334,14 @@ export class T4ApiV1ConformanceService { expect(this.calls.every((call) => call.authorization === "Bearer token-a" || call.authorization === "Bearer token-b" || call.authorization === "Bearer token-denied")).toBe(true); } + #jsonMediaError(request: Request): Response | undefined { + const mediaType = request.headers.get("Content-Type")?.split(";", 1)[0]?.trim().toLowerCase(); + return mediaType === "application/json" ? undefined : problem(400, "invalid_request", "Content-Type must be application/json"); + } + async #jsonBody(request: Request): Promise | Response> { + const mediaError = this.#jsonMediaError(request); + if (mediaError !== undefined) return mediaError; try { const value: unknown = await request.json(); if (value === null || typeof value !== "object" || Array.isArray(value)) return problem(400, "invalid_request", "JSON request body must be an object"); diff --git a/packages/t4-api-client/src/index.ts b/packages/t4-api-client/src/index.ts index 2cf5ea93..b639ff5c 100644 --- a/packages/t4-api-client/src/index.ts +++ b/packages/t4-api-client/src/index.ts @@ -255,12 +255,17 @@ function httpDateTimestamp(value: string): number | undefined { } const rfc850 = /^(Sunday|Monday|Tuesday|Wednesday|Thursday|Friday|Saturday), ([0-9]{2})-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-([0-9]{2}) ([0-9]{2}):([0-9]{2}):([0-9]{2}) GMT$/u.exec(value); if (rfc850 !== null) { - const currentYear = new Date(Date.now()).getUTCFullYear(); - let year = Math.floor(currentYear / 100) * 100 + Number(rfc850[4]); - if (year - currentYear > 50) year -= 100; + const now = new Date(Date.now()); + let year = Math.floor(now.getUTCFullYear() / 100) * 100 + Number(rfc850[4]); const date = new Date(0); date.setUTCFullYear(year, HTTP_MONTHS[rfc850[3]!]!, Number(rfc850[2])); date.setUTCHours(Number(rfc850[5]), Number(rfc850[6]), Number(rfc850[7]), 0); + const fiftyYearsFromNow = new Date(now.getTime()); + fiftyYearsFromNow.setUTCFullYear(now.getUTCFullYear() + 50); + if (date.getTime() > fiftyYearsFromNow.getTime()) { + year -= 100; + date.setUTCFullYear(year); + } const timestamp = date.getTime(); return matchingHttpDate(timestamp, rfc850, { weekday: 1, day: 2, month: 3, year: 4, hour: 5, minute: 6, second: 7 }, year) ? timestamp : undefined; } From 681f6db44e0200d0fffbdd23429138df86f2e673 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 01:03:26 +0000 Subject: [PATCH 046/131] test(t4-api): sanitize media fixture keys --- packages/client/test/t4-api-v1-conformance.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index 2d86037f..53b0f26c 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -497,8 +497,9 @@ describe("generated T4 API v1 client conformance", () => { it("rejects undeclared JSON request media before mutation and idempotency", async () => { for (const contentType of [undefined, "text/plain"]) { + const mediaName = contentType === undefined ? "missing" : "plain"; const service = new T4ApiV1ConformanceService(); - await seededClient(service, `request-media-${contentType ?? "missing"}`); + await seededClient(service, `request-media-${mediaName}`); const cases = [ ["POST", "/v1/workspaces", '{"name":"media-workspace"}', {}, 202, { id: "ws-2" }], ["POST", "/v1/workspaces/ws-1/sessions", '{"title":"media-session"}', {}, 202, { id: "ses-2" }], @@ -507,7 +508,7 @@ describe("generated T4 API v1 client conformance", () => { ["POST", "/v1/sessions/ses-1/commands", '{"command":"ok"}', {}, 202, { commandId: "cmd-1" }], ] as const; for (const [index, [method, path, body, routeHeaders, successStatus, successBody]] of cases.entries()) { - const key = `request-media-${contentType ?? "missing"}-${index}`; + const key = `request-media-${mediaName}-${index}`; const headers = { Authorization: "Bearer token-a", "T4-API-Version": "1", "Idempotency-Key": key, ...routeHeaders }; const invalid = await service.fetch(`${service.origin}${path}`, { method, headers: { ...headers, ...(contentType === undefined ? {} : { "Content-Type": contentType }) }, body, From 2df2da47bb7457e9e14323b7cff2fe7e32a42b97 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 02:03:27 +0000 Subject: [PATCH 047/131] test(t4-api): bound resync session ids --- .../client/test/t4-api-v1-conformance.test.ts | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index 53b0f26c..148982db 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -754,6 +754,34 @@ describe("generated T4 API v1 client conformance", () => { } satisfies Partial); }); + it("enforces the ResourceId bound in cursor-expired resync snapshot URLs", async () => { + const resourceId128 = `s${"a".repeat(127)}`; + const resourceId129 = `s${"a".repeat(128)}`; + const cursorExpired = (sessionId: string) => ({ error: { + code: "cursor_expired", message: "expired", requestId: "r", retryable: false, + resync: { snapshotUrl: `/v1/sessions/${sessionId}/snapshot`, cursor: "cursor-1" }, + } }); + + for (const [sessionId, expected] of [[resourceId128, "cursor_expired"], [resourceId129, "indeterminate"]] as const) { + const client = createT4ApiClient({ + baseUrl: "https://resync-bound.test", credential: "token-a", majorVersion: 1, + fetch: async () => jsonResponse(cursorExpired(sessionId), { status: 410 }), + }); + await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ + code: expected, status: expected === "cursor_expired" ? 410 : 502, + }); + } + + const contract = JSON.parse(readFileSync(new URL("../../t4-api-contract/openapi.json", import.meta.url), "utf8")) as { + components: { schemas: { Resync: { properties: { snapshotUrl: { pattern: string; maxLength: number } } } } }; + }; + const snapshotUrlSchema = contract.components.schemas.Resync.properties.snapshotUrl; + const pattern = new RegExp(snapshotUrlSchema.pattern, "u"); + expect(pattern.test(`/v1/sessions/${resourceId128}/snapshot`)).toBe(true); + expect(pattern.test(`/v1/sessions/${resourceId129}/snapshot`)).toBe(false); + expect(`/v1/sessions/${resourceId128}/snapshot`.length).toBeLessThanOrEqual(snapshotUrlSchema.maxLength); + }); + it("rejects malformed mutation idempotency and watch query contracts", async () => { const service = new T4ApiV1ConformanceService(); const baseHeaders = { Authorization: "Bearer token-a", "T4-API-Version": "1", "Content-Type": "application/json" }; From 319ccf1b71ac82a30d5d0e7cd431b018dfab969f Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 02:13:47 +0000 Subject: [PATCH 048/131] fix(t4-api): bound resync session ids --- packages/t4-api-client/src/index.ts | 5 +++-- packages/t4-api-contract/openapi.json | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/t4-api-client/src/index.ts b/packages/t4-api-client/src/index.ts index b639ff5c..693a9feb 100644 --- a/packages/t4-api-client/src/index.ts +++ b/packages/t4-api-client/src/index.ts @@ -177,8 +177,9 @@ function validViolation(value: unknown): boolean { function validResync(value: unknown): value is Resync { const resync = record(value); - return resync !== undefined && Object.keys(resync).every((key) => key === "snapshotUrl" || key === "cursor") && - typeof resync.snapshotUrl === "string" && hasAtMostCodePoints(resync.snapshotUrl, 512) && /^\/v1\/sessions\/[A-Za-z0-9._~-]+\/snapshot$/u.test(resync.snapshotUrl) && + if (resync === undefined || Object.keys(resync).some((key) => key !== "snapshotUrl" && key !== "cursor") || typeof resync.snapshotUrl !== "string") return false; + const snapshot = /^\/v1\/sessions\/([^/]+)\/snapshot$/u.exec(resync.snapshotUrl); + return hasAtMostCodePoints(resync.snapshotUrl, 150) && snapshot !== null && validResourceId(snapshot[1]) && typeof resync.cursor === "string" && hasAtMostCodePoints(resync.cursor, 512) && CURSOR_PATTERN.test(resync.cursor); } diff --git a/packages/t4-api-contract/openapi.json b/packages/t4-api-contract/openapi.json index 5be952d5..8481901e 100644 --- a/packages/t4-api-contract/openapi.json +++ b/packages/t4-api-contract/openapi.json @@ -652,7 +652,7 @@ "additionalProperties": false, "required": ["snapshotUrl", "cursor"], "properties": { - "snapshotUrl": { "type": "string", "pattern": "^/v1/sessions/[A-Za-z0-9._~-]+/snapshot$", "maxLength": 512 }, + "snapshotUrl": { "type": "string", "pattern": "^/v1/sessions/[A-Za-z0-9][A-Za-z0-9._~-]{0,127}/snapshot$", "maxLength": 150 }, "cursor": { "$ref": "#/components/schemas/Cursor" } } }, From 32aab1e76a24e8adc82e50dc0d714be412c8d435 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 02:32:39 +0000 Subject: [PATCH 049/131] test(t4-api): require truthful mutation contracts --- .../client/test/t4-api-v1-conformance.test.ts | 157 +++++++++++++++--- 1 file changed, 138 insertions(+), 19 deletions(-) diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index 148982db..472be3e2 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -48,6 +48,25 @@ function exhaustWatchEvent(event: WatchEvent): string { const VERSION_HEADERS = { "T4-API-Version": "1" } as const; +const COMMAND_STATES = [ + "accepted", "projected", "dispatching", "running", "succeeded", "failed", + "cancelling", "cancelled", "rejected", "unavailable", "indeterminate", +] as const; + +const DISCOVERY = { + apiVersion: "1.0", + serverBuild: { version: "0.1.30", revision: "fixture-revision-1" }, + supportedMajors: [1], + capabilities: { + "workspace.lifecycle": { supported: true, enabled: true, authorized: true, available: true }, + "optional.preview": { + supported: true, enabled: false, authorized: false, available: false, + deprecation: { message: "Use workspace.lifecycle", sinceVersion: "1.0", sunsetAt: "2027-01-01T00:00:00Z", replacement: "workspace.lifecycle" }, + }, + }, + limits: { pageSizeDefault: 1, pageSizeMax: 1, commandBytesMax: 1, commandRequestBytesMax: 1, commandMetadataValueBytesMax: 1, watchEventsMax: 1, heartbeatSeconds: 5 }, +} as const; + function withSelectedVersion(init: ResponseInit = {}): ResponseInit { const headers = new Headers(init.headers); if (!headers.has("T4-API-Version")) headers.set("T4-API-Version", "1.0"); @@ -71,6 +90,12 @@ function mutationHeaders(revision: number, key: string): Readonly<{ "T4-API-Vers return { ...VERSION_HEADERS, "If-Match": String(revision), "Idempotency-Key": key }; } +function requireEventCursor(response: Response): string { + const cursor = response.headers.get("T4-Event-Cursor"); + expect(cursor).toMatch(/^[A-Za-z0-9._~-]+$/u); + return cursor ?? ""; +} + function requireData(result: { data?: T; error?: unknown }): T { expect(result.error).toBeUndefined(); expect(result.data).toBeDefined(); @@ -89,14 +114,18 @@ async function seededClient(service: T4ApiV1ConformanceService, key: string) { } describe("generated T4 API v1 client conformance", () => { - it("negotiates discovery, capabilities, bounds, and rejects an incompatible major", async () => { + it("negotiates truthful discovery metadata and rejects an incompatible major", async () => { const service = new T4ApiV1ConformanceService(); const client = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); const discovery = requireData(await client.http.GET("/v1", { params: { header: VERSION_HEADERS } })); expect(discovery).toMatchObject({ apiVersion: "1.0", + serverBuild: { version: "0.1.30", revision: "fixture-revision-1" }, supportedMajors: [1], - capabilities: expect.arrayContaining(["workspace.lifecycle", "session.watch.sse"]), + capabilities: { + "workspace.lifecycle": { supported: true, enabled: true, authorized: true, available: true }, + "optional.preview": { supported: true, enabled: false, authorized: false, available: false }, + }, limits: { pageSizeMax: 3, commandBytesMax: 32, watchEventsMax: 4, heartbeatSeconds: 15 }, }); @@ -104,7 +133,10 @@ describe("generated T4 API v1 client conformance", () => { const rejected = await incompatible.http.GET("/v1", { params: { header: VERSION_HEADERS } }); expect(rejected.response.status).toBe(406); expect(rejected.error).toMatchObject({ error: { code: "incompatible_version", retryable: false, supportedMajors: [1] } }); - expect(typedErrors.unauthorized.error.code).toBe("unauthenticated"); + + const unauthenticated = await service.fetch(`${service.origin}/v1`, { headers: { "T4-API-Version": "1" } }); + expect(unauthenticated.status).toBe(401); + expect(await unauthenticated.json()).toMatchObject({ error: { code: "unauthenticated", retryable: false } }); const denied = createT4ApiClient({ baseUrl: service.origin, credential: "token-denied", majorVersion: 1, fetch: service.fetch }); const forbidden = await denied.http.GET("/v1", { params: { header: VERSION_HEADERS } }); @@ -291,7 +323,7 @@ describe("generated T4 API v1 client conformance", () => { expect(stale.response.status).toBe(409); expect(stale.error).toMatchObject({ error: { code: "revision_conflict" } }); - for (const state of ["accepted", "rejected", "conflict", "unavailable", "indeterminate"] as const) { + for (const state of COMMAND_STATES) { const command = { command: state, metadata: {} } satisfies CommandCreate; const result = await client.http.POST("/v1/sessions/{sessionId}/commands", { params: { header: idempotencyHeaders(`command-${state}-0001`), path: { sessionId: "ses-1" } }, body: command, @@ -338,6 +370,53 @@ describe("generated T4 API v1 client conformance", () => { expect(cancelled.data).toMatchObject({ state: "cancelled" }); }); + it("returns one durable event cursor for each committed mutation and its replay", async () => { + const service = new T4ApiV1ConformanceService(); + const client = createT4ApiClient({ baseUrl: service.origin, credential: "token-a", majorVersion: 1, fetch: service.fetch }); + + const workspaceFirst = await client.http.POST("/v1/workspaces", { + body: { name: "workspace" }, params: { header: idempotencyHeaders("cursor-workspace-create") }, + }); + const workspaceReplay = await client.http.POST("/v1/workspaces", { + body: { name: "workspace" }, params: { header: idempotencyHeaders("cursor-workspace-create") }, + }); + expect(requireEventCursor(workspaceReplay.response)).toBe(requireEventCursor(workspaceFirst.response)); + + const workspacePatchFirst = await client.http.PATCH("/v1/workspaces/{workspaceId}", { + body: { name: "renamed" }, params: { header: mutationHeaders(1, "cursor-workspace-patch"), path: { workspaceId: "ws-1" } }, + }); + const workspacePatchReplay = await client.http.PATCH("/v1/workspaces/{workspaceId}", { + body: { name: "renamed" }, params: { header: mutationHeaders(1, "cursor-workspace-patch"), path: { workspaceId: "ws-1" } }, + }); + expect(requireEventCursor(workspacePatchReplay.response)).toBe(requireEventCursor(workspacePatchFirst.response)); + + const sessionFirst = await client.http.POST("/v1/workspaces/{workspaceId}/sessions", { + body: { title: "session" }, params: { header: idempotencyHeaders("cursor-session-create"), path: { workspaceId: "ws-1" } }, + }); + const sessionReplay = await client.http.POST("/v1/workspaces/{workspaceId}/sessions", { + body: { title: "session" }, params: { header: idempotencyHeaders("cursor-session-create"), path: { workspaceId: "ws-1" } }, + }); + expect(requireEventCursor(sessionReplay.response)).toBe(requireEventCursor(sessionFirst.response)); + + const sessionPatchFirst = await client.http.PATCH("/v1/sessions/{sessionId}", { + body: { title: "retitled" }, params: { header: mutationHeaders(1, "cursor-session-patch"), path: { sessionId: "ses-1" } }, + }); + const sessionPatchReplay = await client.http.PATCH("/v1/sessions/{sessionId}", { + body: { title: "retitled" }, params: { header: mutationHeaders(1, "cursor-session-patch"), path: { sessionId: "ses-1" } }, + }); + expect(requireEventCursor(sessionPatchReplay.response)).toBe(requireEventCursor(sessionPatchFirst.response)); + + for (const [operation, invoke] of [ + ["cancel", () => client.http.POST("/v1/sessions/{sessionId}/cancel", { params: { header: idempotencyHeaders("cursor-session-cancel"), path: { sessionId: "ses-1" } } })], + ["command", () => client.http.POST("/v1/sessions/{sessionId}/commands", { body: { command: "run" }, params: { header: idempotencyHeaders("cursor-session-command"), path: { sessionId: "ses-1" } } })], + ["delete", () => client.http.DELETE("/v1/sessions/{sessionId}", { params: { header: idempotencyHeaders("cursor-session-delete"), path: { sessionId: "ses-1" } } })], + ] as const) { + const first = await invoke(); + const replay = await invoke(); + expect(requireEventCursor(replay.response), operation).toBe(requireEventCursor(first.response)); + } + }); + it("replays successful session deletion only within the original principal and request scope", async () => { const service = new T4ApiV1ConformanceService(); const owner = await seededClient(service, "delete-replay"); @@ -782,6 +861,23 @@ describe("generated T4 API v1 client conformance", () => { expect(`/v1/sessions/${resourceId128}/snapshot`.length).toBeLessThanOrEqual(snapshotUrlSchema.maxLength); }); + it("accepts every command lifecycle watch state and rejects removed states", async () => { + for (const [index, state] of COMMAND_STATES.entries()) { + const cursor = `command-state-${index}`; + const client = createT4ApiClient({ + baseUrl: "https://command-states.test", credential: "token-a", majorVersion: 1, + fetch: async () => apiResponse(`data: ${JSON.stringify({ type: "command", cursor, commandId: "cmd-1", state })}\n\n`, { headers: { "Content-Type": "text/event-stream" } }), + }); + await expect(client.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).resolves.toMatchObject({ value: { state } }); + } + + const removed = createT4ApiClient({ + baseUrl: "https://removed-command-state.test", credential: "token-a", majorVersion: 1, + fetch: async () => apiResponse('data: {"type":"command","cursor":"removed-1","commandId":"cmd-1","state":"conflict"}\n\n', { headers: { "Content-Type": "text/event-stream" } }), + }); + await expect(removed.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ code: "indeterminate", status: 502 }); + }); + it("rejects malformed mutation idempotency and watch query contracts", async () => { const service = new T4ApiV1ConformanceService(); const baseHeaders = { Authorization: "Bearer token-a", "T4-API-Version": "1", "Content-Type": "application/json" }; @@ -929,10 +1025,7 @@ describe("generated T4 API v1 client conformance", () => { }); it("validates successful JSON routes relative to the base path and rejects undeclared media and statuses", async () => { - const discovery = { - apiVersion: "1.0", supportedMajors: [1], capabilities: [], - limits: { pageSizeDefault: 1, pageSizeMax: 1, commandBytesMax: 1, commandRequestBytesMax: 1, commandMetadataValueBytesMax: 1, watchEventsMax: 1, heartbeatSeconds: 5 }, - }; + const discovery = DISCOVERY; const prefixed = createT4ApiClient({ baseUrl: "https://prefixed.test/api", credential: "token-a", majorVersion: 1, fetch: async (input) => { @@ -1056,17 +1149,37 @@ describe("generated T4 API v1 client conformance", () => { } }); - it("declares required public response headers while preserving the 401 exception", () => { + it("declares truthful discovery and required public response headers while preserving the 401 exception", () => { const contract = JSON.parse(readFileSync(new URL("../../t4-api-contract/openapi.json", import.meta.url), "utf8")) as { paths: Record }; get?: { responses: Record }> } }>; - components: { headers: Record; responses: Record }> }; + components: { + headers: Record; + responses: Record }>; + schemas: Record }>; + }; }; expect(contract.paths["/v1/workspaces/{workspaceId}"]?.patch?.responses["200"]?.$ref).toBe("#/components/responses/WorkspaceReplay"); expect(contract.paths["/v1/sessions/{sessionId}"]?.patch?.responses["200"]?.$ref).toBe("#/components/responses/SessionReplay"); expect(contract.components.headers.SelectedVersion?.required).toBe(true); expect(contract.components.headers.IdempotencyReplayed?.required).toBe(true); + expect(contract.components.headers.EventCursor?.required).toBe(true); + for (const response of ["WorkspaceAccepted", "WorkspaceReplay", "SessionAccepted", "SessionReplay", "CommandAccepted", "CommandReplay", "Deleted"]) { + expect(contract.components.responses[response]?.headers?.["T4-Event-Cursor"]?.$ref).toBe("#/components/headers/EventCursor"); + } expect(contract.paths["/v1/sessions/{sessionId}/events"]?.get?.responses["200"]?.headers?.["Cache-Control"]?.required).toBe(true); expect(contract.components.responses.Error401?.headers?.["T4-API-Version"]).toBeUndefined(); + + const discovery = contract.components.schemas.Discovery!; + expect(discovery.required).toContain("serverBuild"); + expect(discovery.properties?.capabilities).toMatchObject({ + type: "object", maxProperties: 128, + propertyNames: { pattern: "^[a-z][a-z0-9.-]*$", maxLength: 128 }, + additionalProperties: { $ref: "#/components/schemas/CapabilityStatus" }, + }); + expect(contract.components.schemas.CapabilityStatus).toMatchObject({ + additionalProperties: false, + required: ["supported", "enabled", "authorized", "available"], + }); }); it("treats malformed and oversized watch 503 envelopes as terminal protocol errors", async () => { @@ -1141,10 +1254,7 @@ describe("generated T4 API v1 client conformance", () => { }); it("requires declared selected-version and replay response headers", async () => { - const discovery = { - apiVersion: "1.0", supportedMajors: [1], capabilities: [], - limits: { pageSizeDefault: 1, pageSizeMax: 1, commandBytesMax: 1, commandRequestBytesMax: 1, commandMetadataValueBytesMax: 1, watchEventsMax: 1, heartbeatSeconds: 5 }, - }; + const discovery = DISCOVERY; for (const selectedVersion of [undefined, "1", "2.0", `1.${"0".repeat(15)}`]) { const client = createT4ApiClient({ baseUrl: "https://response-version.test", credential: "token-a", majorVersion: 1, @@ -1168,6 +1278,14 @@ describe("generated T4 API v1 client conformance", () => { params: { header: mutationHeaders(1, "response-replay-patch"), path: { workspaceId: "wsp-1" } }, body: { name: "workspace" }, })).rejects.toMatchObject({ status: 502, retryable: false }); + const missingEventCursor = createT4ApiClient({ + baseUrl: "https://response-event-cursor.test", credential: "token-a", majorVersion: 1, + fetch: async () => jsonResponse(workspace, { headers: { "T4-API-Version": "1.0", "Idempotency-Replayed": "false" } }), + }); + await expect(missingEventCursor.http.PATCH("/v1/workspaces/{workspaceId}", { + params: { header: mutationHeaders(1, "response-event-cursor"), path: { workspaceId: "wsp-1" } }, body: { name: "workspace" }, + })).rejects.toMatchObject({ status: 502, retryable: false }); + for (const replayed of [undefined, "TRUE", "0"]) { const client = createT4ApiClient({ baseUrl: "https://delete-replay-header.test", credential: "token-a", majorVersion: 1, @@ -1193,14 +1311,15 @@ describe("generated T4 API v1 client conformance", () => { await expect(missingWatchErrorVersion.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0, retryBackoffMs: 0 }).next()).rejects.toMatchObject({ status: 502, retryable: false }); }); - it("rejects duplicate unique response arrays", async () => { - const limits = { pageSizeDefault: 1, pageSizeMax: 1, commandBytesMax: 1, commandRequestBytesMax: 1, commandMetadataValueBytesMax: 1, watchEventsMax: 1, heartbeatSeconds: 5 }; + it("rejects duplicate version entries and malformed capability maps", async () => { + const status = { supported: true, enabled: true, authorized: true, available: true }; for (const discovery of [ - { apiVersion: "1.0", supportedMajors: [1, 1], capabilities: [], limits }, - { apiVersion: "1.0", supportedMajors: [1], capabilities: ["session.watch.sse", "session.watch.sse"], limits }, + { ...DISCOVERY, supportedMajors: [1, 1] }, + { ...DISCOVERY, capabilities: { Invalid: status } }, + { ...DISCOVERY, capabilities: { "session.watch.sse": { ...status, unknown: true } } }, ]) { const client = createT4ApiClient({ - baseUrl: "https://duplicate-discovery.test", credential: "token-a", majorVersion: 1, + baseUrl: "https://invalid-discovery.test", credential: "token-a", majorVersion: 1, fetch: async () => jsonResponse(discovery, { headers: { "T4-API-Version": "1.0" } }), }); await expect(client.http.GET("/v1", { params: { header: VERSION_HEADERS } })).rejects.toMatchObject({ status: 502, retryable: false }); From b115a3fb2ccecdd0daeac6e76af7751c65eb7692 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 02:38:23 +0000 Subject: [PATCH 050/131] feat(t4-api): expose truthful mutation contracts --- .../test/t4-api-v1-conformance-service.ts | 29 ++++++-- .../client/test/t4-api-v1-conformance.test.ts | 10 +-- packages/t4-api-client/src/index.ts | 42 ++++++++++-- packages/t4-api-contract/openapi.json | 68 +++++++++++++++---- packages/t4-api-contract/scripts/validate.mjs | 10 ++- 5 files changed, 126 insertions(+), 33 deletions(-) diff --git a/packages/client/test/t4-api-v1-conformance-service.ts b/packages/client/test/t4-api-v1-conformance-service.ts index 03599b26..ede36272 100644 --- a/packages/client/test/t4-api-v1-conformance-service.ts +++ b/packages/client/test/t4-api-v1-conformance-service.ts @@ -70,6 +70,7 @@ interface ReplayRecord { readonly identity: string; readonly body: unknown; readonly replayStatus: number; + readonly cursor: string; } export interface T4ApiV1ConformanceOptions { @@ -87,6 +88,7 @@ export class T4ApiV1ConformanceService { #workspaceSequence = 0; #sessionSequence = 0; #commandSequence = 0; + #eventSequence = 0; readonly #workspaces = new Map>(); readonly #sessions = new Map>(); readonly #replays = new Map(); @@ -112,8 +114,18 @@ export class T4ApiV1ConformanceService { if (request.method === "GET" && url.pathname === "/v1") { return json(200, this.#payload("discovery", { apiVersion: "1.0", + serverBuild: { version: "0.1.30", revision: "fixture-revision-1" }, supportedMajors: [1], - capabilities: ["workspace.lifecycle", "session.lifecycle", "session.commands", "session.watch.sse"], + capabilities: { + "workspace.lifecycle": { supported: true, enabled: true, authorized: true, available: true }, + "session.lifecycle": { supported: true, enabled: true, authorized: true, available: true }, + "session.commands": { supported: true, enabled: true, authorized: true, available: true }, + "session.watch.sse": { supported: true, enabled: true, authorized: true, available: true }, + "optional.preview": { + supported: true, enabled: false, authorized: false, available: false, + deprecation: { message: "Use workspace.lifecycle", sinceVersion: "1.0", sunsetAt: "2027-01-01T00:00:00Z", replacement: "workspace.lifecycle" }, + }, + }, limits: { pageSizeDefault: 2, pageSizeMax: 3, @@ -265,7 +277,7 @@ export class T4ApiV1ConformanceService { if (Object.keys(body).some((key) => key !== "command" && key !== "metadata")) return this.#invalid("body", "schema", "command create must match CommandCreate"); if (typeof body.command !== "string" || body.command.length < 1 || encoder.encode(body.command).byteLength > COMMAND_BYTES_MAX) return this.#invalid("command", "maxBytes", `command must contain 1 to ${COMMAND_BYTES_MAX} UTF-8 bytes`); if (!this.#validMetadata(body.metadata)) return this.#invalid("metadata", "bounds", "metadata keys and values exceed the discovered bounds"); - const states: Record = { accepted: true, rejected: true, conflict: true, unavailable: true, indeterminate: true }; + const states: Record = { accepted: true, projected: true, dispatching: true, running: true, succeeded: true, failed: true, cancelling: true, cancelled: true, rejected: true, unavailable: true, indeterminate: true }; const state = states[body.command] === true ? body.command : "accepted"; return this.#idempotent(request, tenant, "submitCommand", [id], { metadata: {}, ...body }, 202, 200, () => ({ commandId: `cmd-${++this.#commandSequence}`, state })); } @@ -395,13 +407,15 @@ export class T4ApiV1ConformanceService { const prior = this.#replays.get(scope); if (prior !== undefined) { if (prior.identity !== identity) return problem(409, "idempotency_conflict", "Idempotency key was reused with a different canonical request"); + const headers = { "T4-API-Version": "1.0", "Idempotency-Replayed": "true", "T4-Event-Cursor": prior.cursor }; return prior.body === null - ? new Response(null, { status: prior.replayStatus, headers: { "T4-API-Version": "1.0", "Idempotency-Replayed": "true" } }) - : json(prior.replayStatus, prior.body, { "Idempotency-Replayed": "true" }); + ? new Response(null, { status: prior.replayStatus, headers }) + : json(prior.replayStatus, prior.body, headers); } const validation = validate?.(); if (validation !== undefined) return validation; const created = create(); + const cursor = `event-${++this.#eventSequence}`; const visibleValue = created !== null && typeof created === "object" && !Array.isArray(created) ? this.#visible(created as Record) : created; @@ -415,9 +429,10 @@ export class T4ApiV1ConformanceService { const visible = visibleValue !== null && typeof visibleValue === "object" && !Array.isArray(visibleValue) && responseKind !== undefined ? this.#payload(responseKind, visibleValue as Record) : visibleValue; - this.#replays.set(scope, { identity, body: visible, replayStatus }); + this.#replays.set(scope, { identity, body: visible, replayStatus, cursor }); + const headers = { "Idempotency-Replayed": "false", "T4-Event-Cursor": cursor }; return visible === null - ? new Response(null, { status: firstStatus, headers: { "T4-API-Version": "1.0", "Idempotency-Replayed": "false" } }) - : json(firstStatus, visible, { "Idempotency-Replayed": "false" }); + ? new Response(null, { status: firstStatus, headers: { "T4-API-Version": "1.0", ...headers } }) + : json(firstStatus, visible, headers); } } diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index 472be3e2..ae746584 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -51,8 +51,7 @@ const VERSION_HEADERS = { "T4-API-Version": "1" } as const; const COMMAND_STATES = [ "accepted", "projected", "dispatching", "running", "succeeded", "failed", "cancelling", "cancelled", "rejected", "unavailable", "indeterminate", -] as const; - +] as const satisfies readonly components["schemas"]["CommandState"][]; const DISCOVERY = { apiVersion: "1.0", serverBuild: { version: "0.1.30", revision: "fixture-revision-1" }, @@ -1155,7 +1154,7 @@ describe("generated T4 API v1 client conformance", () => { components: { headers: Record; responses: Record }>; - schemas: Record }>; + schemas: Record }>; }; }; expect(contract.paths["/v1/workspaces/{workspaceId}"]?.patch?.responses["200"]?.$ref).toBe("#/components/responses/WorkspaceReplay"); @@ -1171,8 +1170,9 @@ describe("generated T4 API v1 client conformance", () => { const discovery = contract.components.schemas.Discovery!; expect(discovery.required).toContain("serverBuild"); - expect(discovery.properties?.capabilities).toMatchObject({ - type: "object", maxProperties: 128, + expect(discovery.properties?.capabilities).toEqual({ $ref: "#/components/schemas/Capabilities" }); + expect(contract.components.schemas.Capabilities).toMatchObject({ + maxProperties: 128, propertyNames: { pattern: "^[a-z][a-z0-9.-]*$", maxLength: 128 }, additionalProperties: { $ref: "#/components/schemas/CapabilityStatus" }, }); diff --git a/packages/t4-api-client/src/index.ts b/packages/t4-api-client/src/index.ts index 693a9feb..c52f5999 100644 --- a/packages/t4-api-client/src/index.ts +++ b/packages/t4-api-client/src/index.ts @@ -50,9 +50,10 @@ const SESSION_STATES = { accepted: true, provisioning: true, ready: true, cancelling: true, cancelled: true, failed: true, unavailable: true, indeterminate: true, } as const satisfies Record; -const OPERATION_STATES = { - accepted: true, rejected: true, conflict: true, unavailable: true, indeterminate: true, -} as const satisfies Record; +const COMMAND_STATES = { + accepted: true, projected: true, dispatching: true, running: true, succeeded: true, failed: true, + cancelling: true, cancelled: true, rejected: true, unavailable: true, indeterminate: true, +} as const satisfies Record; type ApiError = components["schemas"]["ApiError"]; type Resync = components["schemas"]["Resync"]; @@ -374,16 +375,38 @@ function validSession(value: unknown): boolean { function validCommandResult(value: unknown): boolean { const item = record(value); return item !== undefined && hasOnlyKeys(item, { commandId: true, state: true }) && validResourceId(item.commandId) && - typeof item.state === "string" && OPERATION_STATES[item.state as components["schemas"]["OperationState"]] === true; + typeof item.state === "string" && COMMAND_STATES[item.state as components["schemas"]["CommandState"]] === true; +} + +function validCapabilityDeprecation(value: unknown): boolean { + if (value === undefined) return true; + const deprecation = record(value); + return deprecation !== undefined && hasOnlyKeys(deprecation, { message: true, sinceVersion: true, sunsetAt: true, replacement: true }) && + typeof deprecation.message === "string" && deprecation.message !== "" && hasAtMostCodePoints(deprecation.message, 1024) && + (deprecation.sinceVersion === undefined || (typeof deprecation.sinceVersion === "string" && deprecation.sinceVersion !== "" && hasAtMostCodePoints(deprecation.sinceVersion, 128))) && + (deprecation.sunsetAt === undefined || (typeof deprecation.sunsetAt === "string" && validRfc3339DateTime(deprecation.sunsetAt))) && + (deprecation.replacement === undefined || (typeof deprecation.replacement === "string" && /^[a-z][a-z0-9.-]{0,127}$/u.test(deprecation.replacement))); +} + +function validCapabilityStatus(value: unknown): boolean { + const status = record(value); + return status !== undefined && hasOnlyKeys(status, { supported: true, enabled: true, authorized: true, available: true, deprecation: true }) && + typeof status.supported === "boolean" && typeof status.enabled === "boolean" && + typeof status.authorized === "boolean" && typeof status.available === "boolean" && validCapabilityDeprecation(status.deprecation); } function validDiscovery(value: unknown): boolean { const discovery = record(value); + const serverBuild = record(discovery?.serverBuild); + const capabilities = record(discovery?.capabilities); const limits = record(discovery?.limits); - return discovery !== undefined && hasOnlyKeys(discovery, { apiVersion: true, supportedMajors: true, capabilities: true, limits: true }) && + return discovery !== undefined && hasOnlyKeys(discovery, { apiVersion: true, serverBuild: true, supportedMajors: true, capabilities: true, limits: true }) && typeof discovery.apiVersion === "string" && /^1\.[0-9]+$/u.test(discovery.apiVersion) && + serverBuild !== undefined && hasOnlyKeys(serverBuild, { version: true, revision: true }) && + typeof serverBuild.version === "string" && serverBuild.version !== "" && hasAtMostCodePoints(serverBuild.version, 128) && + typeof serverBuild.revision === "string" && serverBuild.revision !== "" && hasAtMostCodePoints(serverBuild.revision, 128) && Array.isArray(discovery.supportedMajors) && discovery.supportedMajors.length >= 1 && discovery.supportedMajors.length <= 8 && hasUniqueItems(discovery.supportedMajors) && discovery.supportedMajors.every((major) => Number.isSafeInteger(major) && Number(major) >= 1) && - Array.isArray(discovery.capabilities) && discovery.capabilities.length <= 128 && hasUniqueItems(discovery.capabilities) && discovery.capabilities.every((capability) => typeof capability === "string" && /^[a-z][a-z0-9.-]{0,127}$/u.test(capability)) && + capabilities !== undefined && Object.keys(capabilities).length <= 128 && Object.entries(capabilities).every(([id, status]) => /^[a-z][a-z0-9.-]{0,127}$/u.test(id) && validCapabilityStatus(status)) && limits !== undefined && hasOnlyKeys(limits, { pageSizeDefault: true, pageSizeMax: true, commandBytesMax: true, commandRequestBytesMax: true, commandMetadataValueBytesMax: true, watchEventsMax: true, heartbeatSeconds: true }) && [limits.pageSizeDefault, limits.pageSizeMax, limits.commandBytesMax, limits.commandRequestBytesMax, limits.commandMetadataValueBytesMax, limits.watchEventsMax, limits.heartbeatSeconds].every((limit) => Number.isSafeInteger(limit) && Number(limit) >= 1) && Number(limits.pageSizeDefault) <= Number(limits.pageSizeMax) && Number(limits.commandBytesMax) <= Number(limits.commandRequestBytesMax) && Number(limits.commandMetadataValueBytesMax) <= Number(limits.commandRequestBytesMax) && @@ -492,6 +515,7 @@ function validReplayHeader(response: Response): boolean { return replayed === "true" || replayed === "false"; } + async function validateResponse(request: Request, response: Response, baseUrl: string): Promise { const path = relativeApiPath(request, baseUrl); const contract = path === undefined ? undefined : responseContract(request.method, path); @@ -527,6 +551,10 @@ async function validateResponse(request: Request, response: Response, baseUrl: s void response.body?.cancel().catch(() => {}); throw protocolError(502, "T4 API returned a missing or invalid replay header"); } + if (contract.replaySuccess?.includes(response.status) === true && !validCursor(response.headers.get("T4-Event-Cursor"))) { + void response.body?.cancel().catch(() => {}); + throw protocolError(502, "T4 API returned a missing or invalid durable event cursor"); + } if (validator === "empty") { if (response.body !== null || response.headers.has("content-type")) { void response.body?.cancel().catch(() => {}); @@ -633,7 +661,7 @@ function watchEvent(value: unknown, eventId: string | undefined): WatchEvent { event.commandId !== "" && hasAtMostCodePoints(event.commandId, 128) && /^[A-Za-z0-9][A-Za-z0-9._~-]*$/u.test(event.commandId) && typeof event.state === "string" && - OPERATION_STATES[event.state as components["schemas"]["OperationState"]] === true + COMMAND_STATES[event.state as components["schemas"]["CommandState"]] === true ) return event as WatchEvent; throw protocolError(502, "T4 API returned an invalid watch event"); } diff --git a/packages/t4-api-contract/openapi.json b/packages/t4-api-contract/openapi.json index 8481901e..12bb55c5 100644 --- a/packages/t4-api-contract/openapi.json +++ b/packages/t4-api-contract/openapi.json @@ -427,23 +427,67 @@ "description": "True when this response replays a prior identical request.", "required": true, "schema": { "type": "string", "enum": ["true", "false"] } + }, + "EventCursor": { + "description": "Durable event cursor committed with the accepted mutation and preserved by replay.", + "required": true, + "schema": { "$ref": "#/components/schemas/Cursor" } } }, "schemas": { "ResourceId": { "type": "string", "minLength": 1, "maxLength": 128, "pattern": "^[A-Za-z0-9][A-Za-z0-9._~-]*$" }, "Cursor": { "type": "string", "minLength": 1, "maxLength": 512, "pattern": "^[A-Za-z0-9._~-]+$", "description": "Opaque server-issued header-safe SSE cursor." }, "Revision": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 }, - "OperationState": { "type": "string", "enum": ["accepted", "rejected", "conflict", "unavailable", "indeterminate"] }, + "CommandState": { "type": "string", "enum": ["accepted", "projected", "dispatching", "running", "succeeded", "failed", "cancelling", "cancelled", "rejected", "unavailable", "indeterminate"] }, "WorkspaceState": { "type": "string", "enum": ["accepted", "provisioning", "ready", "deleting", "deleted", "failed", "unavailable", "indeterminate"] }, "SessionState": { "type": "string", "enum": ["accepted", "provisioning", "ready", "cancelling", "cancelled", "failed", "unavailable", "indeterminate"] }, + "ServerBuild": { + "type": "object", + "additionalProperties": false, + "required": ["version", "revision"], + "properties": { + "version": { "type": "string", "minLength": 1, "maxLength": 128 }, + "revision": { "type": "string", "minLength": 1, "maxLength": 128 } + } + }, + "CapabilityDeprecation": { + "type": "object", + "additionalProperties": false, + "required": ["message"], + "properties": { + "message": { "type": "string", "minLength": 1, "maxLength": 1024 }, + "sinceVersion": { "type": "string", "minLength": 1, "maxLength": 128 }, + "sunsetAt": { "type": "string", "format": "date-time", "maxLength": 64 }, + "replacement": { "type": "string", "minLength": 1, "maxLength": 128, "pattern": "^[a-z][a-z0-9.-]*$" } + } + }, + "CapabilityStatus": { + "type": "object", + "additionalProperties": false, + "required": ["supported", "enabled", "authorized", "available"], + "properties": { + "supported": { "type": "boolean" }, + "enabled": { "type": "boolean" }, + "authorized": { "type": "boolean" }, + "available": { "type": "boolean" }, + "deprecation": { "$ref": "#/components/schemas/CapabilityDeprecation" } + } + }, + "Capabilities": { + "type": "object", + "maxProperties": 128, + "propertyNames": { "minLength": 1, "maxLength": 128, "pattern": "^[a-z][a-z0-9.-]*$" }, + "additionalProperties": { "$ref": "#/components/schemas/CapabilityStatus" } + }, "Discovery": { "type": "object", "additionalProperties": false, - "required": ["apiVersion", "supportedMajors", "capabilities", "limits"], + "required": ["apiVersion", "serverBuild", "supportedMajors", "capabilities", "limits"], "properties": { "apiVersion": { "type": "string", "pattern": "^1\\.[0-9]+$" }, + "serverBuild": { "$ref": "#/components/schemas/ServerBuild" }, "supportedMajors": { "type": "array", "minItems": 1, "maxItems": 8, "uniqueItems": true, "items": { "type": "integer", "minimum": 1 } }, - "capabilities": { "type": "array", "maxItems": 128, "uniqueItems": true, "items": { "type": "string", "minLength": 1, "maxLength": 128, "pattern": "^[a-z][a-z0-9.-]*$" } }, + "capabilities": { "$ref": "#/components/schemas/Capabilities" }, "limits": { "type": "object", "additionalProperties": false, @@ -574,7 +618,7 @@ "required": ["commandId", "state"], "properties": { "commandId": { "$ref": "#/components/schemas/ResourceId" }, - "state": { "$ref": "#/components/schemas/OperationState" } + "state": { "$ref": "#/components/schemas/CommandState" } } }, "SnapshotEntry": { @@ -627,7 +671,7 @@ "type": { "type": "string", "const": "command" }, "cursor": { "$ref": "#/components/schemas/Cursor" }, "commandId": { "$ref": "#/components/schemas/ResourceId" }, - "state": { "$ref": "#/components/schemas/OperationState" } + "state": { "$ref": "#/components/schemas/CommandState" } } }, "WatchEvent": { @@ -791,17 +835,17 @@ "responses": { "Discovery": { "description": "Negotiated v1 discovery", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Discovery" } } } }, "Workspace": { "description": "Workspace", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Workspace" } } } }, - "WorkspaceAccepted": { "description": "Workspace intent accepted", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Workspace" } } } }, - "WorkspaceReplay": { "description": "Identical workspace request replay", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Workspace" } } } }, + "WorkspaceAccepted": { "description": "Workspace intent accepted", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" }, "T4-Event-Cursor": { "$ref": "#/components/headers/EventCursor" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Workspace" } } } }, + "WorkspaceReplay": { "description": "Identical workspace request replay", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" }, "T4-Event-Cursor": { "$ref": "#/components/headers/EventCursor" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Workspace" } } } }, "WorkspacePage": { "description": "Bounded workspace page", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/WorkspacePage" } } } }, "Session": { "description": "Session", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Session" } } } }, - "SessionAccepted": { "description": "Session intent accepted", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Session" } } } }, - "SessionReplay": { "description": "Identical session mutation replay", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Session" } } } }, + "SessionAccepted": { "description": "Session intent accepted", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" }, "T4-Event-Cursor": { "$ref": "#/components/headers/EventCursor" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Session" } } } }, + "SessionReplay": { "description": "Identical session mutation replay", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" }, "T4-Event-Cursor": { "$ref": "#/components/headers/EventCursor" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Session" } } } }, "SessionPage": { "description": "Bounded session page", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SessionPage" } } } }, - "CommandAccepted": { "description": "Command intent accepted with a stable outcome state", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CommandResult" } } } }, - "CommandReplay": { "description": "Identical command request replay", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CommandResult" } } } }, + "CommandAccepted": { "description": "Command intent accepted with a stable outcome state", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" }, "T4-Event-Cursor": { "$ref": "#/components/headers/EventCursor" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CommandResult" } } } }, + "CommandReplay": { "description": "Identical command request replay", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" }, "T4-Event-Cursor": { "$ref": "#/components/headers/EventCursor" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CommandResult" } } } }, "Snapshot": { "description": "Bounded session snapshot and reconnect cursor", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/SessionSnapshot" } } } }, - "Deleted": { "description": "Idempotent deletion accepted or resource already absent", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" } } }, + "Deleted": { "description": "Idempotent deletion accepted or resource already absent", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" }, "Idempotency-Replayed": { "$ref": "#/components/headers/IdempotencyReplayed" }, "T4-Event-Cursor": { "$ref": "#/components/headers/EventCursor" } } }, "Error400": { "description": "Malformed request or missing idempotency key", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/BadRequestErrorEnvelope" } } } }, "Error401": { "description": "Missing or invalid opaque bearer credential", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/UnauthenticatedErrorEnvelope" } } } }, "Error403": { "description": "Credential lacks the deny-by-default operation or resource scope", "headers": { "T4-API-Version": { "$ref": "#/components/headers/SelectedVersion" } }, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ForbiddenErrorEnvelope" } } } }, diff --git a/packages/t4-api-contract/scripts/validate.mjs b/packages/t4-api-contract/scripts/validate.mjs index d43c7e84..1e594993 100644 --- a/packages/t4-api-contract/scripts/validate.mjs +++ b/packages/t4-api-contract/scripts/validate.mjs @@ -23,14 +23,18 @@ if (commandCreate?.properties?.command?.["x-t4-maxUtf8Bytes"] !== 262144) throw const metadataString = commandCreate?.properties?.metadata?.additionalProperties?.oneOf?.find((item) => item?.type === "string"); if (metadataString?.["x-t4-maxUtf8Bytes"] !== 262144) throw new Error("command metadata strings must retain their UTF-8 byte bound"); -for (const [schema, property] of [["Discovery", "supportedMajors"], ["Discovery", "capabilities"], ["ApiError", "supportedMajors"]]) { +for (const [schema, property] of [["Discovery", "supportedMajors"], ["ApiError", "supportedMajors"]]) { if (schemas[schema]?.properties?.[property]?.uniqueItems !== true) throw new Error(`${schema}.${property} must retain uniqueItems`); } +if (schemas.Discovery?.required?.includes("serverBuild") !== true || schemas.Discovery?.properties?.capabilities?.$ref !== "#/components/schemas/Capabilities") throw new Error("Discovery must require structured server build and capabilities"); +if (schemas.Capabilities?.maxProperties !== 128 || schemas.Capabilities?.additionalProperties?.$ref !== "#/components/schemas/CapabilityStatus") throw new Error("Capabilities must remain a bounded status map"); const selectedVersionHeader = document.components?.headers?.SelectedVersion; const replayHeaderObject = document.components?.headers?.IdempotencyReplayed; +const eventCursorHeader = document.components?.headers?.EventCursor; if (selectedVersionHeader?.required !== true) throw new Error("SelectedVersion must remain required wherever referenced"); if (replayHeaderObject?.required !== true) throw new Error("IdempotencyReplayed must remain required wherever referenced"); +if (eventCursorHeader?.required !== true || eventCursorHeader?.schema?.$ref !== "#/components/schemas/Cursor") throw new Error("EventCursor must remain a required header-safe cursor"); const selectedVersion = document.components?.headers?.SelectedVersion?.schema; if (selectedVersion?.pattern !== "^1\\.[0-9]+$" || selectedVersion?.maxLength !== 16) throw new Error("SelectedVersion must remain a bounded v1 minor"); const replayHeader = document.components?.headers?.IdempotencyReplayed?.schema; @@ -42,8 +46,10 @@ const selectedVersionResponses = ["Discovery", "Workspace", "WorkspaceAccepted", for (const responseName of selectedVersionResponses) { if (document.components?.responses?.[responseName]?.headers?.["T4-API-Version"]?.$ref !== selectedVersionRef) throw new Error(`${responseName} must declare SelectedVersion`); } -for (const responseName of ["WorkspaceAccepted", "WorkspaceReplay", "SessionAccepted", "SessionReplay", "CommandAccepted", "CommandReplay", "Deleted"]) { +const mutationResponses = ["WorkspaceAccepted", "WorkspaceReplay", "SessionAccepted", "SessionReplay", "CommandAccepted", "CommandReplay", "Deleted"]; +for (const responseName of mutationResponses) { if (document.components?.responses?.[responseName]?.headers?.["Idempotency-Replayed"]?.$ref !== replayRef) throw new Error(`${responseName} must declare IdempotencyReplayed`); + if (document.components?.responses?.[responseName]?.headers?.["T4-Event-Cursor"]?.$ref !== "#/components/headers/EventCursor") throw new Error(`${responseName} must declare EventCursor`); } if (document.paths?.["/v1/sessions/{sessionId}/events"]?.get?.responses?.["200"]?.headers?.["T4-API-Version"]?.$ref !== selectedVersionRef) throw new Error("watch success must declare SelectedVersion"); const watchCacheControl = document.paths?.["/v1/sessions/{sessionId}/events"]?.get?.responses?.["200"]?.headers?.["Cache-Control"]; From d430685adc782d0bb10433e5f1b1adb2e623bf4f Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 02:40:09 +0000 Subject: [PATCH 051/131] chore: accept truthful API schema artifact --- .../t4-api-client/src/generated/schema.ts | 41 ++++++++++++++++--- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/packages/t4-api-client/src/generated/schema.ts b/packages/t4-api-client/src/generated/schema.ts index 4cdd50b9..ad07c82b 100755 --- a/packages/t4-api-client/src/generated/schema.ts +++ b/packages/t4-api-client/src/generated/schema.ts @@ -175,6 +175,23 @@ export interface components { BadRequestErrorEnvelope: { error: components["schemas"]["BadRequestApiError"]; }; + Capabilities: { + [key: string]: components["schemas"]["CapabilityStatus"]; + }; + CapabilityDeprecation: { + message: string; + replacement?: string; + sinceVersion?: string; + /** Format: date-time */ + sunsetAt?: string; + }; + CapabilityStatus: { + authorized: boolean; + available: boolean; + deprecation?: components["schemas"]["CapabilityDeprecation"]; + enabled: boolean; + supported: boolean; + }; CommandCreate: { command: string; /** @default {} */ @@ -184,12 +201,14 @@ export interface components { }; CommandResult: { commandId: components["schemas"]["ResourceId"]; - state: components["schemas"]["OperationState"]; + state: components["schemas"]["CommandState"]; }; + /** @enum {string} */ + CommandState: "accepted" | "projected" | "dispatching" | "running" | "succeeded" | "failed" | "cancelling" | "cancelled" | "rejected" | "unavailable" | "indeterminate"; CommandWatchEvent: { commandId: components["schemas"]["ResourceId"]; cursor: components["schemas"]["Cursor"]; - state: components["schemas"]["OperationState"]; + state: components["schemas"]["CommandState"]; /** @constant */ type: "command"; }; @@ -212,7 +231,7 @@ export interface components { }; Discovery: { apiVersion: string; - capabilities: string[]; + capabilities: components["schemas"]["Capabilities"]; limits: { commandBytesMax: number; commandMetadataValueBytesMax: number; @@ -222,6 +241,7 @@ export interface components { pageSizeMax: number; watchEventsMax: number; }; + serverBuild: components["schemas"]["ServerBuild"]; supportedMajors: number[]; }; ErrorEnvelope: { @@ -272,14 +292,16 @@ export interface components { NotFoundErrorEnvelope: { error: components["schemas"]["NotFoundApiError"]; }; - /** @enum {string} */ - OperationState: "accepted" | "rejected" | "conflict" | "unavailable" | "indeterminate"; ResourceId: string; Resync: { cursor: components["schemas"]["Cursor"]; snapshotUrl: string; }; Revision: number; + ServerBuild: { + revision: string; + version: string; + }; Session: { id: components["schemas"]["ResourceId"]; labels?: components["schemas"]["Labels"]; @@ -364,6 +386,7 @@ export interface components { headers: { "Idempotency-Replayed": components["headers"]["IdempotencyReplayed"]; "T4-API-Version": components["headers"]["SelectedVersion"]; + "T4-Event-Cursor": components["headers"]["EventCursor"]; [name: string]: unknown; }; content: { @@ -375,6 +398,7 @@ export interface components { headers: { "Idempotency-Replayed": components["headers"]["IdempotencyReplayed"]; "T4-API-Version": components["headers"]["SelectedVersion"]; + "T4-Event-Cursor": components["headers"]["EventCursor"]; [name: string]: unknown; }; content: { @@ -386,6 +410,7 @@ export interface components { headers: { "Idempotency-Replayed": components["headers"]["IdempotencyReplayed"]; "T4-API-Version": components["headers"]["SelectedVersion"]; + "T4-Event-Cursor": components["headers"]["EventCursor"]; [name: string]: unknown; }; content?: never; @@ -506,6 +531,7 @@ export interface components { headers: { "Idempotency-Replayed": components["headers"]["IdempotencyReplayed"]; "T4-API-Version": components["headers"]["SelectedVersion"]; + "T4-Event-Cursor": components["headers"]["EventCursor"]; [name: string]: unknown; }; content: { @@ -527,6 +553,7 @@ export interface components { headers: { "Idempotency-Replayed": components["headers"]["IdempotencyReplayed"]; "T4-API-Version": components["headers"]["SelectedVersion"]; + "T4-Event-Cursor": components["headers"]["EventCursor"]; [name: string]: unknown; }; content: { @@ -558,6 +585,7 @@ export interface components { headers: { "Idempotency-Replayed": components["headers"]["IdempotencyReplayed"]; "T4-API-Version": components["headers"]["SelectedVersion"]; + "T4-Event-Cursor": components["headers"]["EventCursor"]; [name: string]: unknown; }; content: { @@ -579,6 +607,7 @@ export interface components { headers: { "Idempotency-Replayed": components["headers"]["IdempotencyReplayed"]; "T4-API-Version": components["headers"]["SelectedVersion"]; + "T4-Event-Cursor": components["headers"]["EventCursor"]; [name: string]: unknown; }; content: { @@ -607,6 +636,8 @@ export interface components { }; requestBodies: never; headers: { + /** @description Durable event cursor committed with the accepted mutation and preserved by replay. */ + EventCursor: components["schemas"]["Cursor"]; /** @description True when this response replays a prior identical request. */ IdempotencyReplayed: "true" | "false"; /** @description Selected compatible T4 API minor version. */ From 1f574dc4713831ca1800221eea9c5cb7e0072ae8 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 02:43:49 +0000 Subject: [PATCH 052/131] test(t4-api): use fixture auth proof --- .../client/test/t4-api-v1-conformance.test.ts | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index ae746584..b4171582 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -6,7 +6,6 @@ import { T4ApiError, createT4ApiClient, type components, - type operations, } from "@t4-code/t4-api-client"; import { T4ApiV1ConformanceService, canonicalJson } from "./t4-api-v1-conformance-service.ts"; @@ -15,24 +14,6 @@ type SessionCreate = components["schemas"]["SessionCreate"]; type CommandCreate = components["schemas"]["CommandCreate"]; type WatchEvent = components["schemas"]["WatchEvent"]; -type JsonBody = Response extends { content: { "application/json": infer Body } } ? Body : never; -type SpawnBadRequest = JsonBody; -type CommandBadRequest = JsonBody; -type Unauthorized = JsonBody; -type Forbidden = JsonBody; -type Missing = JsonBody; -type Conflict = JsonBody; -type Unavailable = JsonBody; - -const typedErrors = { - badSpawn: { error: { code: "idempotency_key_required", message: "required", requestId: "r", retryable: false } } satisfies SpawnBadRequest, - badCommand: { error: { code: "invalid_request", message: "invalid", requestId: "r", retryable: false } } satisfies CommandBadRequest, - unauthorized: { error: { code: "unauthenticated", message: "no", requestId: "r", retryable: false } } satisfies Unauthorized, - forbidden: { error: { code: "forbidden", message: "no", requestId: "r", retryable: false } } satisfies Forbidden, - missing: { error: { code: "not_found", message: "no", requestId: "r", retryable: false } } satisfies Missing, - conflict: { error: { code: "idempotency_conflict", message: "no", requestId: "r", retryable: false } } satisfies Conflict, - unavailable: { error: { code: "unavailable", message: "later", requestId: "r", retryable: true } } satisfies Unavailable, -}; function exhaustWatchEvent(event: WatchEvent): string { switch (event.type) { From 409882bf4380ba8ba18d99f35ba815895ed15864 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 02:44:29 +0000 Subject: [PATCH 053/131] test(t4-api): reject malformed page cursors --- .../client/test/t4-api-v1-conformance.test.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index b4171582..d394c004 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -899,6 +899,25 @@ describe("generated T4 API v1 client conformance", () => { } }); + it("rejects malformed workspace and session pagination cursors", async () => { + const service = new T4ApiV1ConformanceService(); + const headers = { Authorization: "Bearer token-a", "T4-API-Version": "1", "Content-Type": "application/json" }; + expect((await service.fetch(`${service.origin}/v1/workspaces`, { + method: "POST", headers: { ...headers, "Idempotency-Key": "pagination-workspace" }, body: '{"name":"workspace"}', + })).status).toBe(202); + expect((await service.fetch(`${service.origin}/v1/workspaces/ws-1/sessions`, { + method: "POST", headers: { ...headers, "Idempotency-Key": "pagination-session-1" }, body: '{"title":"session"}', + })).status).toBe(202); + + for (const path of ["/v1/workspaces", "/v1/workspaces/ws-1/sessions"]) { + for (const cursor of ["foo", "page--1", "page-01", "page-9007199254740992"]) { + const response = await service.fetch(`${service.origin}${path}?cursor=${encodeURIComponent(cursor)}`, { headers }); + expect(response.status, `${path} accepted ${cursor}`).toBe(422); + expect(await response.json()).toMatchObject({ error: { code: "invalid_request", violations: [{ field: "cursor", rule: "format" }] } }); + } + } + }); + it("parses service frames incrementally across split UTF-8 and per-frame bounds", async () => { const bytewiseService = new T4ApiV1ConformanceService({ watchTransport: "bytewise" }); const bytewiseClient = await seededClient(bytewiseService, "bytewise"); From e15eb7c3dd627814c324cc1ffc1fc345aa2393c5 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 02:45:31 +0000 Subject: [PATCH 054/131] test(t4-api): reject silent version downgrade --- packages/client/test/t4-api-v1-conformance.test.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index d394c004..ce133366 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -114,6 +114,14 @@ describe("generated T4 API v1 client conformance", () => { expect(rejected.response.status).toBe(406); expect(rejected.error).toMatchObject({ error: { code: "incompatible_version", retryable: false, supportedMajors: [1] } }); + const silentDowngrade = createT4ApiClient({ + baseUrl: "https://silent-downgrade.test", credential: "token-a", majorVersion: 2, + fetch: async () => jsonResponse(DISCOVERY), + }); + await expect(silentDowngrade.http.GET("/v1", { params: { header: VERSION_HEADERS } })).rejects.toMatchObject({ + code: "indeterminate", status: 502, retryable: false, + }); + const unauthenticated = await service.fetch(`${service.origin}/v1`, { headers: { "T4-API-Version": "1" } }); expect(unauthenticated.status).toBe(401); expect(await unauthenticated.json()).toMatchObject({ error: { code: "unauthenticated", retryable: false } }); From 3fa3bf3e1e35ffd2ff440d74cbe7c9c4b66d4f63 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 02:48:52 +0000 Subject: [PATCH 055/131] fix(t4-api): reject invalid cursors and downgrades --- .../client/test/t4-api-v1-conformance-service.ts | 13 +++++++++++-- packages/t4-api-client/src/index.ts | 9 +++++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/packages/client/test/t4-api-v1-conformance-service.ts b/packages/client/test/t4-api-v1-conformance-service.ts index ede36272..788b9c1f 100644 --- a/packages/client/test/t4-api-v1-conformance-service.ts +++ b/packages/client/test/t4-api-v1-conformance-service.ts @@ -14,6 +14,13 @@ function hasAtMostCodePoints(value: string, maximum: number): boolean { return true; } +function pageOffset(value: string | null): number | undefined { + if (value === null) return 0; + if (value.length > 512 || !/^page-(?:0|[1-9][0-9]*)$/u.test(value)) return undefined; + const offset = Number(value.slice(5)); + return Number.isSafeInteger(offset) ? offset : undefined; +} + function validLabels(value: unknown): boolean { if (value === null || typeof value !== "object" || Array.isArray(value)) return false; const entries = Object.entries(value as Record); @@ -155,7 +162,8 @@ export class T4ApiV1ConformanceService { if (request.method === "GET" && url.pathname === "/v1/workspaces") { const pageSize = Number(url.searchParams.get("pageSize") ?? "2"); if (!Number.isInteger(pageSize) || pageSize < 1 || pageSize > 3) return this.#invalid("pageSize", "range", "pageSize must be between 1 and 3"); - const start = Number(url.searchParams.get("cursor")?.replace("page-", "") ?? "0"); + const start = pageOffset(url.searchParams.get("cursor")); + if (start === undefined) return this.#invalid("cursor", "format", "cursor must be a canonical bounded page offset"); const visible = [...this.#workspaces.values()].filter((item) => item.tenant === tenant); const items = visible.slice(start, start + pageSize).map(({ tenant: _tenant, ...item }) => item); const next = start + items.length < visible.length ? `page-${start + items.length}` : undefined; @@ -212,7 +220,8 @@ export class T4ApiV1ConformanceService { if (request.method === "GET") { const pageSize = Number(url.searchParams.get("pageSize") ?? "2"); if (!Number.isInteger(pageSize) || pageSize < 1 || pageSize > 3) return this.#invalid("pageSize", "range", "pageSize must be between 1 and 3"); - const start = Number(url.searchParams.get("cursor")?.replace("page-", "") ?? "0"); + const start = pageOffset(url.searchParams.get("cursor")); + if (start === undefined) return this.#invalid("cursor", "format", "cursor must be a canonical bounded page offset"); const visible = [...this.#sessions.values()].filter((item) => item.workspaceId === workspaceId && item.tenant === tenant); const items = visible.slice(start, start + pageSize).map((item) => this.#visible(item)); const next = start + items.length < visible.length ? `page-${start + items.length}` : undefined; diff --git a/packages/t4-api-client/src/index.ts b/packages/t4-api-client/src/index.ts index c52f5999..4058a424 100644 --- a/packages/t4-api-client/src/index.ts +++ b/packages/t4-api-client/src/index.ts @@ -505,9 +505,10 @@ function responseContract(method: string, path: string): ResponseContract | unde return undefined; } -function validSelectedVersion(response: Response): boolean { +function validSelectedVersion(response: Response, expectedMajor?: string): boolean { const selected = response.headers.get("T4-API-Version"); - return selected !== null && selected.length <= SELECTED_VERSION_MAX_LENGTH && SELECTED_VERSION_PATTERN.test(selected); + return selected !== null && selected.length <= SELECTED_VERSION_MAX_LENGTH && SELECTED_VERSION_PATTERN.test(selected) && + (expectedMajor === undefined || selected.startsWith(`${expectedMajor}.`)); } function validReplayHeader(response: Response): boolean { @@ -543,7 +544,7 @@ async function validateResponse(request: Request, response: Response, baseUrl: s void response.body?.cancel().catch(() => {}); throw protocolError(502, "T4 API returned an undeclared success status"); } - if (!validSelectedVersion(response)) { + if (!validSelectedVersion(response, request.headers.get("T4-API-Version") ?? "")) { void response.body?.cancel().catch(() => {}); throw protocolError(502, "T4 API returned a missing or invalid selected version"); } @@ -839,7 +840,7 @@ async function* watch( void response.body?.cancel().catch(() => {}); throw protocolError(502, "T4 API returned an undeclared watch success status"); } - if (!validSelectedVersion(response)) { + if (!validSelectedVersion(response, majorVersion)) { void response.body?.cancel().catch(() => {}); throw protocolError(502, "T4 API returned a missing or invalid selected version"); } From ad46ce7259e53aee7d84e9361547165105c4b1bc Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 02:51:49 +0000 Subject: [PATCH 056/131] test(t4-api): require issued page cursors --- .../client/test/t4-api-v1-conformance.test.ts | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index ce133366..fdb4bea3 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -926,6 +926,39 @@ describe("generated T4 API v1 client conformance", () => { } }); + it("accepts only pagination cursors issued for the principal and collection", async () => { + const service = new T4ApiV1ConformanceService(); + const headers = { Authorization: "Bearer token-a", "T4-API-Version": "1", "Content-Type": "application/json" }; + for (const [key, name] of [["issued-workspace-1", "one"], ["issued-workspace-2", "two"], ["issued-workspace-3", "three"]]) { + expect((await service.fetch(`${service.origin}/v1/workspaces`, { + method: "POST", headers: { ...headers, "Idempotency-Key": key }, body: JSON.stringify({ name }), + })).status).toBe(202); + } + for (const [key, title] of [["issued-session-01", "one"], ["issued-session-02", "two"]]) { + expect((await service.fetch(`${service.origin}/v1/workspaces/ws-1/sessions`, { + method: "POST", headers: { ...headers, "Idempotency-Key": key }, body: JSON.stringify({ title }), + })).status).toBe(202); + } + + const workspacePage = await service.fetch(`${service.origin}/v1/workspaces?pageSize=1`, { headers }); + const workspaceCursor = (await workspacePage.json() as { nextCursor: string }).nextCursor; + expect((await service.fetch(`${service.origin}/v1/workspaces?pageSize=1&cursor=${workspaceCursor}`, { headers })).status).toBe(200); + const sessionPage = await service.fetch(`${service.origin}/v1/workspaces/ws-1/sessions?pageSize=1`, { headers }); + const sessionCursor = (await sessionPage.json() as { nextCursor: string }).nextCursor; + + for (const [path, cursor, authorization] of [ + ["/v1/workspaces", "page-999", "Bearer token-a"], + ["/v1/workspaces/ws-1/sessions", "page-999", "Bearer token-a"], + ["/v1/workspaces/ws-1/sessions", workspaceCursor, "Bearer token-a"], + ["/v1/workspaces", sessionCursor, "Bearer token-a"], + ["/v1/workspaces", workspaceCursor, "Bearer token-b"], + ]) { + const response = await service.fetch(`${service.origin}${path}?pageSize=1&cursor=${cursor}`, { headers: { ...headers, Authorization: authorization } }); + expect(response.status, `${path} accepted unissued cursor ${cursor}`).toBe(422); + expect(await response.json()).toMatchObject({ error: { code: "invalid_request", violations: [{ field: "cursor", rule: "issued" }] } }); + } + }); + it("parses service frames incrementally across split UTF-8 and per-frame bounds", async () => { const bytewiseService = new T4ApiV1ConformanceService({ watchTransport: "bytewise" }); const bytewiseClient = await seededClient(bytewiseService, "bytewise"); From c4e2336f9da8894c28aaa862589fa7a7d1b8e796 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 02:54:55 +0000 Subject: [PATCH 057/131] fix(t4-api): bind issued page cursors --- .../client/test/t4-api-v1-conformance-service.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/packages/client/test/t4-api-v1-conformance-service.ts b/packages/client/test/t4-api-v1-conformance-service.ts index 788b9c1f..abbd4962 100644 --- a/packages/client/test/t4-api-v1-conformance-service.ts +++ b/packages/client/test/t4-api-v1-conformance-service.ts @@ -99,6 +99,7 @@ export class T4ApiV1ConformanceService { readonly #workspaces = new Map>(); readonly #sessions = new Map>(); readonly #replays = new Map(); + readonly #pageCursors = new Set(); constructor(readonly options: T4ApiV1ConformanceOptions = {}) {} @@ -162,11 +163,16 @@ export class T4ApiV1ConformanceService { if (request.method === "GET" && url.pathname === "/v1/workspaces") { const pageSize = Number(url.searchParams.get("pageSize") ?? "2"); if (!Number.isInteger(pageSize) || pageSize < 1 || pageSize > 3) return this.#invalid("pageSize", "range", "pageSize must be between 1 and 3"); - const start = pageOffset(url.searchParams.get("cursor")); + const cursor = url.searchParams.get("cursor"); + const start = pageOffset(cursor); if (start === undefined) return this.#invalid("cursor", "format", "cursor must be a canonical bounded page offset"); + if (cursor !== null && !this.#pageCursors.has(canonicalJson({ principal: tenant, collection: "workspaces", cursor }))) { + return this.#invalid("cursor", "issued", "cursor was not issued for this principal and collection"); + } const visible = [...this.#workspaces.values()].filter((item) => item.tenant === tenant); const items = visible.slice(start, start + pageSize).map(({ tenant: _tenant, ...item }) => item); const next = start + items.length < visible.length ? `page-${start + items.length}` : undefined; + if (next !== undefined) this.#pageCursors.add(canonicalJson({ principal: tenant, collection: "workspaces", cursor: next })); return json(200, { items, ...(next === undefined ? {} : { nextCursor: next }) }); } @@ -220,11 +226,17 @@ export class T4ApiV1ConformanceService { if (request.method === "GET") { const pageSize = Number(url.searchParams.get("pageSize") ?? "2"); if (!Number.isInteger(pageSize) || pageSize < 1 || pageSize > 3) return this.#invalid("pageSize", "range", "pageSize must be between 1 and 3"); - const start = pageOffset(url.searchParams.get("cursor")); + const cursor = url.searchParams.get("cursor"); + const start = pageOffset(cursor); if (start === undefined) return this.#invalid("cursor", "format", "cursor must be a canonical bounded page offset"); + const collection = `workspaces/${workspaceId}/sessions`; + if (cursor !== null && !this.#pageCursors.has(canonicalJson({ principal: tenant, collection, cursor }))) { + return this.#invalid("cursor", "issued", "cursor was not issued for this principal and collection"); + } const visible = [...this.#sessions.values()].filter((item) => item.workspaceId === workspaceId && item.tenant === tenant); const items = visible.slice(start, start + pageSize).map((item) => this.#visible(item)); const next = start + items.length < visible.length ? `page-${start + items.length}` : undefined; + if (next !== undefined) this.#pageCursors.add(canonicalJson({ principal: tenant, collection, cursor: next })); return json(200, { items, ...(next === undefined ? {} : { nextCursor: next }) }); } } From 907b5aa9dae26a06cf81edc0c1955035925db6e5 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 02:57:51 +0000 Subject: [PATCH 058/131] fix(t4-api): issue unique opaque page cursors --- .../test/t4-api-v1-conformance-service.ts | 63 +++++++++++++------ 1 file changed, 43 insertions(+), 20 deletions(-) diff --git a/packages/client/test/t4-api-v1-conformance-service.ts b/packages/client/test/t4-api-v1-conformance-service.ts index abbd4962..76ee1663 100644 --- a/packages/client/test/t4-api-v1-conformance-service.ts +++ b/packages/client/test/t4-api-v1-conformance-service.ts @@ -14,12 +14,6 @@ function hasAtMostCodePoints(value: string, maximum: number): boolean { return true; } -function pageOffset(value: string | null): number | undefined { - if (value === null) return 0; - if (value.length > 512 || !/^page-(?:0|[1-9][0-9]*)$/u.test(value)) return undefined; - const offset = Number(value.slice(5)); - return Number.isSafeInteger(offset) ? offset : undefined; -} function validLabels(value: unknown): boolean { if (value === null || typeof value !== "object" || Array.isArray(value)) return false; @@ -80,6 +74,12 @@ interface ReplayRecord { readonly cursor: string; } +interface PageCursorRecord { + readonly principal: string; + readonly collection: string; + readonly offset: number; +} + export interface T4ApiV1ConformanceOptions { readonly invalidPayload?: "discovery" | "workspace" | "session" | "command"; readonly watchTransport?: "normal" | "bytewise" | "oversized" | "many-small"; @@ -96,10 +96,12 @@ export class T4ApiV1ConformanceService { #sessionSequence = 0; #commandSequence = 0; #eventSequence = 0; + #pageCursorSequence = 0; readonly #workspaces = new Map>(); readonly #sessions = new Map>(); readonly #replays = new Map(); - readonly #pageCursors = new Set(); + readonly #pageCursors = new Map(); + readonly #pageCursorByPosition = new Map(); constructor(readonly options: T4ApiV1ConformanceOptions = {}) {} @@ -163,16 +165,22 @@ export class T4ApiV1ConformanceService { if (request.method === "GET" && url.pathname === "/v1/workspaces") { const pageSize = Number(url.searchParams.get("pageSize") ?? "2"); if (!Number.isInteger(pageSize) || pageSize < 1 || pageSize > 3) return this.#invalid("pageSize", "range", "pageSize must be between 1 and 3"); + const collection = "workspaces"; const cursor = url.searchParams.get("cursor"); - const start = pageOffset(cursor); - if (start === undefined) return this.#invalid("cursor", "format", "cursor must be a canonical bounded page offset"); - if (cursor !== null && !this.#pageCursors.has(canonicalJson({ principal: tenant, collection: "workspaces", cursor }))) { - return this.#invalid("cursor", "issued", "cursor was not issued for this principal and collection"); + let start = 0; + if (cursor !== null) { + if (cursor.length > 512 || !/^page-(?:0|[1-9][0-9]*)$/u.test(cursor) || !Number.isSafeInteger(Number(cursor.slice(5)))) { + return this.#invalid("cursor", "format", "cursor must be a canonical bounded opaque token"); + } + const issued = this.#pageCursors.get(cursor); + if (issued === undefined || issued.principal !== tenant || issued.collection !== collection) { + return this.#invalid("cursor", "issued", "cursor was not issued for this principal and collection"); + } + start = issued.offset; } const visible = [...this.#workspaces.values()].filter((item) => item.tenant === tenant); const items = visible.slice(start, start + pageSize).map(({ tenant: _tenant, ...item }) => item); - const next = start + items.length < visible.length ? `page-${start + items.length}` : undefined; - if (next !== undefined) this.#pageCursors.add(canonicalJson({ principal: tenant, collection: "workspaces", cursor: next })); + const next = start + items.length < visible.length ? this.#issuePageCursor(tenant, collection, start + items.length) : undefined; return json(200, { items, ...(next === undefined ? {} : { nextCursor: next }) }); } @@ -226,17 +234,22 @@ export class T4ApiV1ConformanceService { if (request.method === "GET") { const pageSize = Number(url.searchParams.get("pageSize") ?? "2"); if (!Number.isInteger(pageSize) || pageSize < 1 || pageSize > 3) return this.#invalid("pageSize", "range", "pageSize must be between 1 and 3"); - const cursor = url.searchParams.get("cursor"); - const start = pageOffset(cursor); - if (start === undefined) return this.#invalid("cursor", "format", "cursor must be a canonical bounded page offset"); const collection = `workspaces/${workspaceId}/sessions`; - if (cursor !== null && !this.#pageCursors.has(canonicalJson({ principal: tenant, collection, cursor }))) { - return this.#invalid("cursor", "issued", "cursor was not issued for this principal and collection"); + const cursor = url.searchParams.get("cursor"); + let start = 0; + if (cursor !== null) { + if (cursor.length > 512 || !/^page-(?:0|[1-9][0-9]*)$/u.test(cursor) || !Number.isSafeInteger(Number(cursor.slice(5)))) { + return this.#invalid("cursor", "format", "cursor must be a canonical bounded opaque token"); + } + const issued = this.#pageCursors.get(cursor); + if (issued === undefined || issued.principal !== tenant || issued.collection !== collection) { + return this.#invalid("cursor", "issued", "cursor was not issued for this principal and collection"); + } + start = issued.offset; } const visible = [...this.#sessions.values()].filter((item) => item.workspaceId === workspaceId && item.tenant === tenant); const items = visible.slice(start, start + pageSize).map((item) => this.#visible(item)); - const next = start + items.length < visible.length ? `page-${start + items.length}` : undefined; - if (next !== undefined) this.#pageCursors.add(canonicalJson({ principal: tenant, collection, cursor: next })); + const next = start + items.length < visible.length ? this.#issuePageCursor(tenant, collection, start + items.length) : undefined; return json(200, { items, ...(next === undefined ? {} : { nextCursor: next }) }); } } @@ -408,6 +421,16 @@ export class T4ApiV1ConformanceService { (typeof item === "string" && encoder.encode(item).byteLength <= METADATA_VALUE_BYTES_MAX))); } + #issuePageCursor(principal: string, collection: string, offset: number): string { + const position = canonicalJson({ principal, collection, offset }); + const prior = this.#pageCursorByPosition.get(position); + if (prior !== undefined) return prior; + const cursor = `page-${++this.#pageCursorSequence}`; + this.#pageCursors.set(cursor, { principal, collection, offset }); + this.#pageCursorByPosition.set(position, cursor); + return cursor; + } + #idempotent( request: Request, principal: string, From 97fe26fc7e8831eb9c973fa3204e86cc6d53489a Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 02:59:30 +0000 Subject: [PATCH 059/131] test(t4-api): reject stale issued cursors --- .../client/test/t4-api-v1-conformance.test.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index fdb4bea3..8766acad 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -957,6 +957,23 @@ describe("generated T4 API v1 client conformance", () => { expect(response.status, `${path} accepted unissued cursor ${cursor}`).toBe(422); expect(await response.json()).toMatchObject({ error: { code: "invalid_request", violations: [{ field: "cursor", rule: "issued" }] } }); } + + const laterWorkspacePage = await service.fetch(`${service.origin}/v1/workspaces?pageSize=2`, { headers }); + const staleWorkspaceCursor = (await laterWorkspacePage.json() as { nextCursor: string }).nextCursor; + expect((await service.fetch(`${service.origin}/v1/workspaces/ws-3`, { + method: "DELETE", headers: { ...headers, "Idempotency-Key": "shrink-workspace-3" }, + })).status).toBe(204); + expect((await service.fetch(`${service.origin}/v1/sessions/ses-2`, { + method: "DELETE", headers: { ...headers, "Idempotency-Key": "shrink-session-02" }, + })).status).toBe(204); + for (const [path, cursor] of [ + ["/v1/workspaces", staleWorkspaceCursor], + ["/v1/workspaces/ws-1/sessions", sessionCursor], + ]) { + const response = await service.fetch(`${service.origin}${path}?pageSize=1&cursor=${cursor}`, { headers }); + expect(response.status, `${path} accepted a cursor past the shrunken collection`).toBe(422); + expect(await response.json()).toMatchObject({ error: { code: "invalid_request", violations: [{ field: "cursor", rule: "issued" }] } }); + } }); it("parses service frames incrementally across split UTF-8 and per-frame bounds", async () => { From 3e5295a68c17502e694a4c3b1d32c65ff6687c20 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 03:02:48 +0000 Subject: [PATCH 060/131] fix(t4-api): sign scoped page cursors --- .../test/t4-api-v1-conformance-service.ts | 76 ++++++++++--------- 1 file changed, 41 insertions(+), 35 deletions(-) diff --git a/packages/client/test/t4-api-v1-conformance-service.ts b/packages/client/test/t4-api-v1-conformance-service.ts index 76ee1663..5e381192 100644 --- a/packages/client/test/t4-api-v1-conformance-service.ts +++ b/packages/client/test/t4-api-v1-conformance-service.ts @@ -1,6 +1,8 @@ +import { createHmac, timingSafeEqual } from "node:crypto"; import { expect } from "vite-plus/test"; const encoder = new TextEncoder(); +const PAGE_CURSOR_SECRET = "t4-api-v1-conformance-page-cursor-secret-2026"; const COMMAND_BYTES_MAX = 32; const COMMAND_REQUEST_BYTES_MAX = 256; const METADATA_VALUE_BYTES_MAX = 32; @@ -67,6 +69,35 @@ export function canonicalJson(value: unknown): string { throw new TypeError("JCS identity requires a JSON value"); } +type PageCursorResult = { readonly offset: number } | { readonly error: "format" | "issued" }; + +function issuePageCursor(principal: string, collection: string, offset: number): string { + const payload = Buffer.from(canonicalJson({ principal, collection, offset })).toString("base64url"); + const signature = createHmac("sha256", PAGE_CURSOR_SECRET).update(payload).digest("base64url"); + return `page.${payload}.${signature}`; +} + +function decodePageCursor(value: string, principal: string, collection: string): PageCursorResult { + if (value.length > 512) return { error: "format" }; + const legacy = /^page-(?:0|[1-9][0-9]*)$/u.exec(value); + if (legacy !== null) return Number.isSafeInteger(Number(value.slice(5))) ? { error: "issued" } : { error: "format" }; + const match = /^page\.([A-Za-z0-9_-]+)\.([A-Za-z0-9_-]{43})$/u.exec(value); + if (match === null) return { error: "format" }; + const expected = createHmac("sha256", PAGE_CURSOR_SECRET).update(match[1]!).digest(); + const signature = Buffer.from(match[2]!, "base64url"); + if (signature.byteLength !== expected.byteLength || !timingSafeEqual(signature, expected)) return { error: "issued" }; + let decoded: unknown; + try { decoded = JSON.parse(Buffer.from(match[1]!, "base64url").toString("utf8")); } catch { return { error: "format" }; } + if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded)) return { error: "format" }; + const payload = decoded as Record; + if (Object.keys(payload).length !== 3 || !Object.hasOwn(payload, "principal") || !Object.hasOwn(payload, "collection") || !Object.hasOwn(payload, "offset") || + Buffer.from(canonicalJson(payload)).toString("base64url") !== match[1] || + typeof payload.principal !== "string" || typeof payload.collection !== "string" || + !Number.isSafeInteger(payload.offset) || Number(payload.offset) < 1) return { error: "format" }; + if (payload.principal !== principal || payload.collection !== collection) return { error: "issued" }; + return { offset: Number(payload.offset) }; +} + interface ReplayRecord { readonly identity: string; readonly body: unknown; @@ -74,11 +105,6 @@ interface ReplayRecord { readonly cursor: string; } -interface PageCursorRecord { - readonly principal: string; - readonly collection: string; - readonly offset: number; -} export interface T4ApiV1ConformanceOptions { readonly invalidPayload?: "discovery" | "workspace" | "session" | "command"; @@ -96,12 +122,9 @@ export class T4ApiV1ConformanceService { #sessionSequence = 0; #commandSequence = 0; #eventSequence = 0; - #pageCursorSequence = 0; readonly #workspaces = new Map>(); readonly #sessions = new Map>(); readonly #replays = new Map(); - readonly #pageCursors = new Map(); - readonly #pageCursorByPosition = new Map(); constructor(readonly options: T4ApiV1ConformanceOptions = {}) {} @@ -169,18 +192,14 @@ export class T4ApiV1ConformanceService { const cursor = url.searchParams.get("cursor"); let start = 0; if (cursor !== null) { - if (cursor.length > 512 || !/^page-(?:0|[1-9][0-9]*)$/u.test(cursor) || !Number.isSafeInteger(Number(cursor.slice(5)))) { - return this.#invalid("cursor", "format", "cursor must be a canonical bounded opaque token"); - } - const issued = this.#pageCursors.get(cursor); - if (issued === undefined || issued.principal !== tenant || issued.collection !== collection) { - return this.#invalid("cursor", "issued", "cursor was not issued for this principal and collection"); - } - start = issued.offset; + const decoded = decodePageCursor(cursor, tenant, collection); + if ("error" in decoded) return this.#invalid("cursor", decoded.error, decoded.error === "format" ? "cursor must be a canonical bounded opaque token" : "cursor was not issued for this principal and collection"); + start = decoded.offset; } const visible = [...this.#workspaces.values()].filter((item) => item.tenant === tenant); + if (cursor !== null && start >= visible.length) return this.#invalid("cursor", "issued", "cursor is outside the current collection"); const items = visible.slice(start, start + pageSize).map(({ tenant: _tenant, ...item }) => item); - const next = start + items.length < visible.length ? this.#issuePageCursor(tenant, collection, start + items.length) : undefined; + const next = start + items.length < visible.length ? issuePageCursor(tenant, collection, start + items.length) : undefined; return json(200, { items, ...(next === undefined ? {} : { nextCursor: next }) }); } @@ -238,18 +257,14 @@ export class T4ApiV1ConformanceService { const cursor = url.searchParams.get("cursor"); let start = 0; if (cursor !== null) { - if (cursor.length > 512 || !/^page-(?:0|[1-9][0-9]*)$/u.test(cursor) || !Number.isSafeInteger(Number(cursor.slice(5)))) { - return this.#invalid("cursor", "format", "cursor must be a canonical bounded opaque token"); - } - const issued = this.#pageCursors.get(cursor); - if (issued === undefined || issued.principal !== tenant || issued.collection !== collection) { - return this.#invalid("cursor", "issued", "cursor was not issued for this principal and collection"); - } - start = issued.offset; + const decoded = decodePageCursor(cursor, tenant, collection); + if ("error" in decoded) return this.#invalid("cursor", decoded.error, decoded.error === "format" ? "cursor must be a canonical bounded opaque token" : "cursor was not issued for this principal and collection"); + start = decoded.offset; } const visible = [...this.#sessions.values()].filter((item) => item.workspaceId === workspaceId && item.tenant === tenant); + if (cursor !== null && start >= visible.length) return this.#invalid("cursor", "issued", "cursor is outside the current collection"); const items = visible.slice(start, start + pageSize).map((item) => this.#visible(item)); - const next = start + items.length < visible.length ? this.#issuePageCursor(tenant, collection, start + items.length) : undefined; + const next = start + items.length < visible.length ? issuePageCursor(tenant, collection, start + items.length) : undefined; return json(200, { items, ...(next === undefined ? {} : { nextCursor: next }) }); } } @@ -421,15 +436,6 @@ export class T4ApiV1ConformanceService { (typeof item === "string" && encoder.encode(item).byteLength <= METADATA_VALUE_BYTES_MAX))); } - #issuePageCursor(principal: string, collection: string, offset: number): string { - const position = canonicalJson({ principal, collection, offset }); - const prior = this.#pageCursorByPosition.get(position); - if (prior !== undefined) return prior; - const cursor = `page-${++this.#pageCursorSequence}`; - this.#pageCursors.set(cursor, { principal, collection, offset }); - this.#pageCursorByPosition.set(position, cursor); - return cursor; - } #idempotent( request: Request, From 5ae50a074b96eabb95190dc91ed6af7b07104284 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 03:07:22 +0000 Subject: [PATCH 061/131] test(t4-api): reject noncanonical cursor signatures --- packages/client/test/t4-api-v1-conformance.test.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index 8766acad..0e334b07 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -943,6 +943,12 @@ describe("generated T4 API v1 client conformance", () => { const workspacePage = await service.fetch(`${service.origin}/v1/workspaces?pageSize=1`, { headers }); const workspaceCursor = (await workspacePage.json() as { nextCursor: string }).nextCursor; expect((await service.fetch(`${service.origin}/v1/workspaces?pageSize=1&cursor=${workspaceCursor}`, { headers })).status).toBe(200); + const signatureAlias = ({ A: "B", Q: "R", g: "h", w: "x" } as const)[workspaceCursor.at(-1) as "A" | "Q" | "g" | "w"]; + expect(signatureAlias).toBeDefined(); + const noncanonicalCursor = `${workspaceCursor.slice(0, -1)}${signatureAlias}`; + const noncanonical = await service.fetch(`${service.origin}/v1/workspaces?pageSize=1&cursor=${noncanonicalCursor}`, { headers }); + expect(noncanonical.status).toBe(422); + expect(await noncanonical.json()).toMatchObject({ error: { code: "invalid_request" } }); const sessionPage = await service.fetch(`${service.origin}/v1/workspaces/ws-1/sessions?pageSize=1`, { headers }); const sessionCursor = (await sessionPage.json() as { nextCursor: string }).nextCursor; From 9cd5494a4eb62974984ec9a8f420c88f95cc742d Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 03:10:31 +0000 Subject: [PATCH 062/131] fix(t4-api): require canonical cursor signatures --- packages/client/test/t4-api-v1-conformance-service.ts | 1 + packages/client/test/t4-api-v1-conformance.test.ts | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/client/test/t4-api-v1-conformance-service.ts b/packages/client/test/t4-api-v1-conformance-service.ts index 5e381192..c2065516 100644 --- a/packages/client/test/t4-api-v1-conformance-service.ts +++ b/packages/client/test/t4-api-v1-conformance-service.ts @@ -85,6 +85,7 @@ function decodePageCursor(value: string, principal: string, collection: string): if (match === null) return { error: "format" }; const expected = createHmac("sha256", PAGE_CURSOR_SECRET).update(match[1]!).digest(); const signature = Buffer.from(match[2]!, "base64url"); + if (signature.toString("base64url") !== match[2]) return { error: "format" }; if (signature.byteLength !== expected.byteLength || !timingSafeEqual(signature, expected)) return { error: "issued" }; let decoded: unknown; try { decoded = JSON.parse(Buffer.from(match[1]!, "base64url").toString("utf8")); } catch { return { error: "format" }; } diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index 0e334b07..fb8b3ecb 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -182,7 +182,7 @@ describe("generated T4 API v1 client conformance", () => { params: { header: VERSION_HEADERS, query: { pageSize: 3 } }, })); expect(pageOne.items).toHaveLength(3); - expect(pageOne.nextCursor).toBe("page-3"); + expect(pageOne.nextCursor).toMatch(/^[A-Za-z0-9._~-]+$/u); const pageTwo = requireData(await owner.http.GET("/v1/workspaces", { params: { header: VERSION_HEADERS, query: { pageSize: 3, cursor: pageOne.nextCursor! } }, })); @@ -289,7 +289,7 @@ describe("generated T4 API v1 client conformance", () => { params: { header: VERSION_HEADERS, path: { workspaceId: "ws-1" }, query: { pageSize: 2 } }, })); expect(sessionPageOne.items).toHaveLength(2); - expect(sessionPageOne.nextCursor).toBe("page-2"); + expect(sessionPageOne.nextCursor).toMatch(/^[A-Za-z0-9._~-]+$/u); const sessionPageTwo = requireData(await client.http.GET("/v1/workspaces/{workspaceId}/sessions", { params: { header: VERSION_HEADERS, path: { workspaceId: "ws-1" }, query: { pageSize: 2, cursor: sessionPageOne.nextCursor! } }, })); From 00c27f3271e9048463b347bf9fcc791737f8f96b Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 03:16:15 +0000 Subject: [PATCH 063/131] test(t4-api): cover every cursor signature alias --- packages/client/test/t4-api-v1-conformance.test.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index fb8b3ecb..8e31ad27 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -943,9 +943,11 @@ describe("generated T4 API v1 client conformance", () => { const workspacePage = await service.fetch(`${service.origin}/v1/workspaces?pageSize=1`, { headers }); const workspaceCursor = (await workspacePage.json() as { nextCursor: string }).nextCursor; expect((await service.fetch(`${service.origin}/v1/workspaces?pageSize=1&cursor=${workspaceCursor}`, { headers })).status).toBe(200); - const signatureAlias = ({ A: "B", Q: "R", g: "h", w: "x" } as const)[workspaceCursor.at(-1) as "A" | "Q" | "g" | "w"]; - expect(signatureAlias).toBeDefined(); - const noncanonicalCursor = `${workspaceCursor.slice(0, -1)}${signatureAlias}`; + const base64UrlAlphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; + const terminalIndex = base64UrlAlphabet.indexOf(workspaceCursor.at(-1)!); + expect(terminalIndex).toBeGreaterThanOrEqual(0); + expect(terminalIndex % 4).toBe(0); + const noncanonicalCursor = `${workspaceCursor.slice(0, -1)}${base64UrlAlphabet[terminalIndex + 1]}`; const noncanonical = await service.fetch(`${service.origin}/v1/workspaces?pageSize=1&cursor=${noncanonicalCursor}`, { headers }); expect(noncanonical.status).toBe(422); expect(await noncanonical.json()).toMatchObject({ error: { code: "invalid_request" } }); From da2f32d4d4475beddb065a19dc522339c4cc1d78 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 03:22:06 +0000 Subject: [PATCH 064/131] test(t4-api): reject every cursor signature alias --- .../client/test/t4-api-v1-conformance.test.ts | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index 8e31ad27..903e014e 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -944,13 +944,25 @@ describe("generated T4 API v1 client conformance", () => { const workspaceCursor = (await workspacePage.json() as { nextCursor: string }).nextCursor; expect((await service.fetch(`${service.origin}/v1/workspaces?pageSize=1&cursor=${workspaceCursor}`, { headers })).status).toBe(200); const base64UrlAlphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; - const terminalIndex = base64UrlAlphabet.indexOf(workspaceCursor.at(-1)!); + const signatureSegment = workspaceCursor.split(".").at(-1)!; + const encodedRemainder = signatureSegment.length % 4; + const unusedBits = encodedRemainder === 2 ? 4 : encodedRemainder === 3 ? 2 : 0; + const terminalIndex = base64UrlAlphabet.indexOf(signatureSegment.at(-1)!); expect(terminalIndex).toBeGreaterThanOrEqual(0); - expect(terminalIndex % 4).toBe(0); - const noncanonicalCursor = `${workspaceCursor.slice(0, -1)}${base64UrlAlphabet[terminalIndex + 1]}`; - const noncanonical = await service.fetch(`${service.origin}/v1/workspaces?pageSize=1&cursor=${noncanonicalCursor}`, { headers }); - expect(noncanonical.status).toBe(422); - expect(await noncanonical.json()).toMatchObject({ error: { code: "invalid_request" } }); + const aliasMask = (1 << unusedBits) - 1; + expect(terminalIndex & aliasMask).toBe(0); + const signatureAliases = [...base64UrlAlphabet.slice( + terminalIndex & ~aliasMask, + (terminalIndex & ~aliasMask) + aliasMask + 1, + )].filter((terminal) => terminal !== signatureSegment.at(-1)); + expect(signatureAliases.length).toBeGreaterThan(0); + expect(signatureAliases).toHaveLength(3); + for (const signatureAlias of signatureAliases) { + const noncanonicalCursor = `${workspaceCursor.slice(0, -1)}${signatureAlias}`; + const noncanonical = await service.fetch(`${service.origin}/v1/workspaces?pageSize=1&cursor=${noncanonicalCursor}`, { headers }); + expect(noncanonical.status).toBe(422); + expect(await noncanonical.json()).toMatchObject({ error: { code: "invalid_request" } }); + } const sessionPage = await service.fetch(`${service.origin}/v1/workspaces/ws-1/sessions?pageSize=1`, { headers }); const sessionCursor = (await sessionPage.json() as { nextCursor: string }).nextCursor; From 5c8ae4144006b0ddf4837394ddc7a252e3aba1eb Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 03:28:02 +0000 Subject: [PATCH 065/131] test(t4-api): avoid needless alias allocation --- packages/client/test/t4-api-v1-conformance.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index 903e014e..e4499222 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -951,10 +951,10 @@ describe("generated T4 API v1 client conformance", () => { expect(terminalIndex).toBeGreaterThanOrEqual(0); const aliasMask = (1 << unusedBits) - 1; expect(terminalIndex & aliasMask).toBe(0); - const signatureAliases = [...base64UrlAlphabet.slice( + const signatureAliases = base64UrlAlphabet.slice( terminalIndex & ~aliasMask, (terminalIndex & ~aliasMask) + aliasMask + 1, - )].filter((terminal) => terminal !== signatureSegment.at(-1)); + ).replace(signatureSegment.at(-1)!, ""); expect(signatureAliases.length).toBeGreaterThan(0); expect(signatureAliases).toHaveLength(3); for (const signatureAlias of signatureAliases) { From 4859cac7759c6dbc6bd5bdd82ee0b07585078f39 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 03:45:36 +0000 Subject: [PATCH 066/131] test(t4-api): reject oversized snapshots and duplicate cursors --- .../client/test/t4-api-v1-conformance.test.ts | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index e4499222..51d5b1a1 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -1389,6 +1389,42 @@ describe("generated T4 API v1 client conformance", () => { await expect(missingWatchErrorVersion.watchSession("ses-1", { maxEvents: 1, maxReconnectAttempts: 0, retryBackoffMs: 0 }).next()).rejects.toMatchObject({ status: 502, retryable: false }); }); + it("aligns the snapshot aggregate schema bound with high-level response validation", async () => { + const contract = JSON.parse(readFileSync(new URL("../../t4-api-contract/openapi.json", import.meta.url), "utf8")) as { + components: { schemas: { SessionSnapshot: { "x-t4-maxUtf8Bytes"?: number } } }; + }; + expect(contract.components.schemas.SessionSnapshot["x-t4-maxUtf8Bytes"]).toBe(16 * 1024 * 1024); + + const oversizedSnapshot = createT4ApiClient({ + baseUrl: "https://aggregate-snapshot.test", credential: "token-a", majorVersion: 1, + fetch: async () => jsonResponse({ + sessionId: "ses-1", cursor: "cursor-1", state: "ready", + entries: Array.from({ length: 17 }, (_, sequence) => ({ sequence, kind: "output", text: "x".repeat(1024 * 1024) })), + }), + }); + await expect(oversizedSnapshot.http.GET("/v1/sessions/{sessionId}/snapshot", { + params: { header: VERSION_HEADERS, path: { sessionId: "ses-1" } }, + })).rejects.toMatchObject({ code: "indeterminate", status: 502, retryable: false }); + }); + + it("rejects duplicate durable cursors within one SSE stream and after reconnect", async () => { + const event = 'data: {"type":"heartbeat","cursor":"duplicate-1","observedAt":"2026-07-21T00:00:00Z"}\n\n'; + for (const delivery of ["same-stream", "reconnect"] as const) { + let attempts = 0; + const client = createT4ApiClient({ + baseUrl: `https://duplicate-${delivery}.test`, credential: "token-a", majorVersion: 1, + fetch: async () => { + attempts += 1; + return apiResponse(delivery === "same-stream" ? `${event}${event}` : event, { headers: { "Content-Type": "text/event-stream" } }); + }, + }); + const stream = client.watchSession("ses-1", { maxEvents: 2, maxReconnectAttempts: 1, retryBackoffMs: 0 }); + await expect(stream.next()).resolves.toMatchObject({ value: { cursor: "duplicate-1" }, done: false }); + await expect(stream.next()).rejects.toMatchObject({ code: "indeterminate", status: 502, retryable: false }); + expect(attempts).toBe(delivery === "same-stream" ? 1 : 2); + } + }); + it("rejects duplicate version entries and malformed capability maps", async () => { const status = { supported: true, enabled: true, authorized: true, available: true }; for (const discovery of [ From 9ce270c1cfe19750824509b42c0636eff7f09346 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 03:49:59 +0000 Subject: [PATCH 067/131] fix(t4-api): align snapshot and watch cursor bounds --- packages/t4-api-client/src/index.ts | 4 ++++ packages/t4-api-contract/openapi.json | 1 + packages/t4-api-contract/scripts/validate.mjs | 1 + 3 files changed, 6 insertions(+) diff --git a/packages/t4-api-client/src/index.ts b/packages/t4-api-client/src/index.ts index 4058a424..84b3bf82 100644 --- a/packages/t4-api-client/src/index.ts +++ b/packages/t4-api-client/src/index.ts @@ -805,6 +805,8 @@ async function* watch( let delivered = 0; let reconnectAttempts = 0; let cursor = options.cursor; + const deliveredCursors = new Set(); + if (cursor !== undefined) deliveredCursors.add(cursor); try { while (delivered < maxEvents && !controller.signal.aborted) { const deliveredAtAttemptStart = delivered; @@ -859,6 +861,8 @@ async function* watch( const chunk = await reader.read(); const events = chunk.done ? decodedFrames(parser, undefined) : decodedFrames(parser, chunk.value); for (const event of events) { + if (deliveredCursors.has(event.cursor)) throw protocolError(502, "T4 API watch repeated a durable event cursor"); + deliveredCursors.add(event.cursor); cursor = event.cursor; delivered += 1; reconnectAttempts = 0; diff --git a/packages/t4-api-contract/openapi.json b/packages/t4-api-contract/openapi.json index 12bb55c5..66d77c91 100644 --- a/packages/t4-api-contract/openapi.json +++ b/packages/t4-api-contract/openapi.json @@ -634,6 +634,7 @@ "SessionSnapshot": { "type": "object", "additionalProperties": false, + "x-t4-maxUtf8Bytes": 16777216, "required": ["sessionId", "cursor", "state", "entries"], "properties": { "sessionId": { "$ref": "#/components/schemas/ResourceId" }, diff --git a/packages/t4-api-contract/scripts/validate.mjs b/packages/t4-api-contract/scripts/validate.mjs index 1e594993..b7eb134a 100644 --- a/packages/t4-api-contract/scripts/validate.mjs +++ b/packages/t4-api-contract/scripts/validate.mjs @@ -22,6 +22,7 @@ if (commandCreate?.["x-t4-maxUtf8Bytes"] !== 1048576) throw new Error("CommandCr if (commandCreate?.properties?.command?.["x-t4-maxUtf8Bytes"] !== 262144) throw new Error("command must retain its UTF-8 byte bound"); const metadataString = commandCreate?.properties?.metadata?.additionalProperties?.oneOf?.find((item) => item?.type === "string"); if (metadataString?.["x-t4-maxUtf8Bytes"] !== 262144) throw new Error("command metadata strings must retain their UTF-8 byte bound"); +if (schemas.SessionSnapshot?.["x-t4-maxUtf8Bytes"] !== 16777216) throw new Error("SessionSnapshot must retain its aggregate UTF-8 response-byte bound"); for (const [schema, property] of [["Discovery", "supportedMajors"], ["ApiError", "supportedMajors"]]) { if (schemas[schema]?.properties?.[property]?.uniqueItems !== true) throw new Error(`${schema}.${property} must retain uniqueItems`); From 8292dace12b024a5393b5309a26dd1385ec3f73e Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 19:12:58 +0000 Subject: [PATCH 068/131] test(operator): define CRD lifecycle contract --- .../compat/v1alpha1-t4clusterhost.yaml | 25 +++ .../testdata/compat/v1alpha1-t4session.yaml | 29 ++++ .../testdata/compat/v1alpha1-t4workspace.yaml | 29 ++++ scripts/cluster-ci/crd-lifecycle.test.mjs | 142 ++++++++++++++++++ 4 files changed, 225 insertions(+) create mode 100644 packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4clusterhost.yaml create mode 100644 packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4session.yaml create mode 100644 packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4workspace.yaml create mode 100644 scripts/cluster-ci/crd-lifecycle.test.mjs diff --git a/packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4clusterhost.yaml b/packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4clusterhost.yaml new file mode 100644 index 00000000..d618128d --- /dev/null +++ b/packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4clusterhost.yaml @@ -0,0 +1,25 @@ +apiVersion: cluster.t4.dev/v1alpha1 +kind: T4ClusterHost +metadata: + name: legacy-host +spec: + storageClassName: legacy-rwx + runtimeProfiles: + - default + - gui + ciProvider: + secretRef: + name: legacy-ci-token + configMapRef: + name: legacy-ci-config + allowedOrigins: + - https://t4.example.test +status: + observedGeneration: 3 + conditions: + - type: Available + status: "True" + observedGeneration: 3 + lastTransitionTime: "2026-01-15T12:00:00Z" + reason: Reconciled + message: Legacy host remains available diff --git a/packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4session.yaml b/packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4session.yaml new file mode 100644 index 00000000..dde06302 --- /dev/null +++ b/packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4session.yaml @@ -0,0 +1,29 @@ +apiVersion: cluster.t4.dev/v1alpha1 +kind: T4Session +metadata: + name: legacy-session + finalizers: + - cluster.t4.dev/session-cleanup +spec: + hostRef: legacy-host + workspaceRef: legacy-workspace + title: Legacy session + runtimeProfile: gui + initialPromptSecretRef: + name: legacy-initial-prompt + ci: + repositoryId: legacy/project + ref: refs/heads/main + commit: 0123456789abcdef0123456789abcdef01234567 +status: + observedGeneration: 7 + podName: legacy-session-pod + serviceName: legacy-session + phase: Running + conditions: + - type: Available + status: "True" + observedGeneration: 7 + lastTransitionTime: "2026-01-15T12:00:00Z" + reason: RuntimeReady + message: Legacy session infrastructure is running diff --git a/packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4workspace.yaml b/packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4workspace.yaml new file mode 100644 index 00000000..dcd9ac1c --- /dev/null +++ b/packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4workspace.yaml @@ -0,0 +1,29 @@ +apiVersion: cluster.t4.dev/v1alpha1 +kind: T4Workspace +metadata: + name: legacy-workspace + finalizers: + - cluster.t4.dev/workspace-protection +spec: + hostRef: legacy-host + displayName: Legacy workspace + owner: user:legacy-owner + repository: + repositoryId: legacy/project + ref: refs/heads/main + commit: 0123456789abcdef0123456789abcdef01234567 + size: 20Gi + retentionPolicy: Retain +status: + observedGeneration: 4 + pvcName: legacy-workspace-data + pvcPhase: Bound + capacity: 20Gi + phase: Ready + conditions: + - type: Ready + status: "True" + observedGeneration: 4 + lastTransitionTime: "2026-01-15T12:00:00Z" + reason: StorageBound + message: Legacy workspace storage is ready diff --git a/scripts/cluster-ci/crd-lifecycle.test.mjs b/scripts/cluster-ci/crd-lifecycle.test.mjs new file mode 100644 index 00000000..4b78ea0d --- /dev/null +++ b/scripts/cluster-ci/crd-lifecycle.test.mjs @@ -0,0 +1,142 @@ +import assert from "node:assert/strict"; +import { chmod, mkdtemp, mkdir, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { spawn } from "node:child_process"; +import test from "node:test"; + +const repoRoot = resolve(import.meta.dirname, "../.."); +const lifecycle = resolve(import.meta.dirname, "crd-lifecycle.sh"); + +async function fixture() { + const root = await mkdtemp(join(tmpdir(), "t4-crd-lifecycle-")); + const bin = join(root, "bin"); + const log = join(root, "commands.log"); + await mkdir(bin); + await writeFile( + join(bin, "kubectl"), + `#!/bin/sh +set -eu +printf 'kubectl' >>"$COMMAND_LOG" +for argument in "$@"; do printf '\\t%s' "$argument" >>"$COMMAND_LOG"; done +printf '\\n' >>"$COMMAND_LOG" +for argument in "$@"; do + if [ "$argument" = "--dry-run=server" ] && [ "\${FAIL_DRY_RUN:-0}" = 1 ]; then + exit 42 + fi +done +if [ "\${1:-}" = get ]; then + printf '%s' "\${STORED_VERSIONS:-v1alpha1}" +fi +`, + ); + await writeFile( + join(bin, "helm"), + `#!/bin/sh +set -eu +printf 'helm' >>"$COMMAND_LOG" +for argument in "$@"; do printf '\\t%s' "$argument" >>"$COMMAND_LOG"; done +printf '\\n' >>"$COMMAND_LOG" +`, + ); + await chmod(join(bin, "kubectl"), 0o755); + await chmod(join(bin, "helm"), 0o755); + return { + root, + log, + env: { ...process.env, PATH: `${bin}:${process.env.PATH}`, COMMAND_LOG: log }, + }; +} + +async function runLifecycle(args, env = {}) { + const result = await new Promise((resolveResult, reject) => { + const child = spawn(lifecycle, args, { + cwd: repoRoot, + env, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.on("error", reject); + child.on("close", (code, signal) => resolveResult({ code, signal, stdout, stderr })); + }); + return result; +} + +async function commands(log) { + return (await readFile(log, "utf8")).trim().split("\n").filter(Boolean); +} + +function findCommand(log, predicate, description) { + const index = log.findIndex(predicate); + assert.notEqual(index, -1, `missing ${description}:\n${log.join("\n")}`); + return index; +} + +test("upgrade preflights old objects, establishes CRDs, verifies storage, then upgrades workloads", async () => { + const value = await fixture(); + const result = await runLifecycle( + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "deploy/charts/t4-cluster", "--namespace", "t4-system", "--skip-crds"], + value.env, + ); + assert.equal(result.code, 0, `${result.stdout}\n${result.stderr}`); + const log = await commands(value.log); + const crdPreflight = findCommand(log, (line) => line.includes("apply") && line.includes("--server-side") && line.includes("--dry-run=server") && line.includes("deploy/charts/t4-cluster/crds"), "server-side CRD preflight"); + const oldObjectPreflight = findCommand(log, (line) => line.includes("apply") && line.includes("--dry-run=server") && line.includes("testdata/compat"), "old-object compatibility preflight"); + const crdApply = findCommand(log, (line) => line.includes("apply") && line.includes("--server-side") && !line.includes("--dry-run=server"), "CRD apply"); + const established = findCommand(log, (line) => line.includes("wait") && line.includes("condition=Established") && line.includes("t4clusterhosts.cluster.t4.dev") && line.includes("t4workspaces.cluster.t4.dev") && line.includes("t4sessions.cluster.t4.dev"), "Established wait"); + const storageChecks = log.map((line, index) => ({ line, index })).filter(({ line }) => line.startsWith("kubectl\tget\tcrd/") && line.includes("status.storedVersions")); + assert.deepEqual(storageChecks.length, 3); + const workload = findCommand(log, (line) => line.startsWith("helm\tupgrade\t"), "Helm workload upgrade"); + assert.ok(crdPreflight < oldObjectPreflight); + assert.ok(oldObjectPreflight < crdApply); + assert.ok(crdApply < established); + assert.ok(established < storageChecks[0].index); + assert.ok(storageChecks.every(({ index }) => index < workload)); + assert.ok(log.every((line) => !line.includes("--force") && !line.includes("replace") && !line.includes("delete\tcrd")), log.join("\n")); +}); + +test("fresh install establishes and validates CRDs before Helm installs with CRD handling disabled", async () => { + const value = await fixture(); + const result = await runLifecycle( + ["install", "--", "helm", "install", "t4-cluster", "deploy/charts/t4-cluster", "--namespace", "t4-system", "--skip-crds"], + value.env, + ); + assert.equal(result.code, 0, `${result.stdout}\n${result.stderr}`); + const log = await commands(value.log); + const established = findCommand(log, (line) => line.includes("wait") && line.includes("condition=Established"), "Established wait"); + const fixtureValidation = findCommand(log, (line) => line.includes("--dry-run=server") && line.includes("testdata/compat"), "fixture validation"); + const storage = findCommand(log, (line) => line.includes("status.storedVersions"), "stored-version check"); + const workload = findCommand(log, (line) => line.startsWith("helm\tinstall\t"), "Helm install"); + assert.ok(established < fixtureValidation && fixtureValidation < storage && storage < workload); +}); + +test("failed server preflight leaves CRDs and workloads untouched", async () => { + const value = await fixture(); + const result = await runLifecycle( + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "deploy/charts/t4-cluster", "--namespace", "t4-system", "--skip-crds"], + { ...value.env, FAIL_DRY_RUN: "1" }, + ); + assert.notEqual(result.code, 0); + const log = await commands(value.log); + assert.equal(log.length, 1, log.join("\n")); + assert.match(log[0], /apply.*--server-side.*--dry-run=server/u); +}); + +test("an unexpected stored version stops workload rollout", async () => { + const value = await fixture(); + const result = await runLifecycle( + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "deploy/charts/t4-cluster", "--namespace", "t4-system", "--skip-crds"], + { ...value.env, STORED_VERSIONS: "v1alpha1,v1beta1" }, + ); + assert.notEqual(result.code, 0); + const log = await commands(value.log); + assert.ok(log.some((line) => line.includes("status.storedVersions"))); + assert.ok(log.every((line) => !line.startsWith("helm\t")), log.join("\n")); +}); From 73f1d7494c523bd042d816dffce996cededd70f9 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 19:20:18 +0000 Subject: [PATCH 069/131] feat(operator): enforce CRD lifecycle gates --- docs/CLUSTER_OPERATOR.md | 73 +++++++--- .../api/v1alpha1/types_test.go | 128 ++++++++++++++++++ .../charttests/chart_contract_test.go | 16 ++- scripts/cluster-ci/crd-lifecycle.sh | 83 ++++++++++++ scripts/cluster-ci/crd-lifecycle.test.mjs | 13 +- 5 files changed, 295 insertions(+), 18 deletions(-) create mode 100755 scripts/cluster-ci/crd-lifecycle.sh diff --git a/docs/CLUSTER_OPERATOR.md b/docs/CLUSTER_OPERATOR.md index 67598da5..09a9247b 100644 --- a/docs/CLUSTER_OPERATOR.md +++ b/docs/CLUSTER_OPERATOR.md @@ -23,13 +23,15 @@ That declaration does not make a non-RWX backend safe. Missing classes, classes ## Install -Installing with defaults creates no controller, gateway, session workload, RBAC, Secret, or network policy: +Installing with defaults creates no controller, gateway, session workload, RBAC, Secret, or network policy. Use the lifecycle runner even for a fresh install so the CRDs are server-validated, established, fixture-checked, and storage-version-checked before Helm runs: ```sh -helm install t4-cluster deploy/charts/t4-cluster --namespace t4-system --create-namespace +scripts/cluster-ci/crd-lifecycle.sh install -- \ + helm install t4-cluster deploy/charts/t4-cluster \ + --namespace t4-system --create-namespace --skip-crds ``` -Helm processes files in `crds/` separately. Use `--skip-crds` if CRD lifecycle is administered independently. To enable the control plane, provide a private values file or a deployment controller values source: +Helm processes files in `crds/` separately on a direct install, but does not upgrade or delete them later. The release procedure therefore administers CRDs independently and always passes `--skip-crds` to Helm. To enable the control plane, provide a private values file or a deployment controller values source: ```yaml enabled: true @@ -112,11 +114,7 @@ The controller uses uncached, exact-name `get` permission for only that administ Reusable credential projection is unsupported because OMP and arbitrary session tools share one workload security boundary. `allowUnauthenticated: true` with empty `credentialSecret` and `credentialKey` is therefore mandatory. The two credential fields remain only as cutover sentinels: Helm, the controller, and the image entrypoint all reject an old credential-mode configuration instead of exposing it. Before OMP starts, the image requires the pinned `auth_credentials` table to be empty, rejects the pinned secret settings (`auth.broker.token`, `hindsight.apiToken`, `searxng.token`, and `dev.autoqaPush.token`), rejects broker token and encrypted snapshot files, and fails closed on unknown schemas or linked profile paths. It never deletes or rewrites durable user state. Because Helm rejects legacy credential values before rendering the new Deployment, a failed upgrade leaves the prior release unchanged. Stop development sessions and clear any legacy values and credential state explicitly before adopting this pre-production boundary. Provider authentication must live behind a private model gateway that authorizes the session through infrastructure identity or network policy while presenting an `auth: none` endpoint inside the session's allowed route. Do not expose that endpoint to the Internet or a shared untrusted network. The chart remains provider- and model-neutral. -Install or enable with immutable digests: - -```sh -helm upgrade --install t4-cluster deploy/charts/t4-cluster --namespace t4-system --values operator-values.yaml -``` +Install a new release with immutable digests by adding `--values operator-values.yaml` to the lifecycle-runner `install` command above. For an existing release, use the lifecycle-runner `upgrade` procedure below. Do not use `helm upgrade --install`: install and upgrade have different compatibility preflights. The controller always has two replicas and uses a Kubernetes Lease named from `t4-cluster-operator.cluster.t4.dev`; one replica reconciles at a time. The server defaults to three stateless replicas and supports a minimum of two. Its Deployment uses `maxUnavailable: 0`, a `minAvailable: 2` PDB, topology spread, anti-affinity, readiness draining, and an explicit `k3s-worker-02` exclusion. Session pods also exclude that node by default. Additional cluster-specific exclusions belong in deployment values, not this portable chart. @@ -150,28 +148,71 @@ The session runtime verifies and builds the exact OMP tag `t4code-17.0.5-appserv Session pods do not receive an automatically mounted ServiceAccount token. The explicit projected reviewer token can only create TokenReviews and is not resource-authorized. The namespace-local ConfigMap projection contains credential-free `auth: none` OMP models and credential-free settings; session Pods receive no provider Secret reference or reusable provider credential. All containers drop capabilities, disallow privilege escalation, use RuntimeDefault seccomp, and use read-only root filesystems. No per-session NodePort, LoadBalancer, host network, host PID, host display, or hostPath is created. -## Upgrade and rollback +## Upgrade, storage migration, and rollback -CRD changes must remain structural and additive. Helm installs files under `crds/` on first install but does not upgrade or delete them. Before every `helm upgrade`, an administrator must review and apply each newer CRD independently, wait for API discovery to serve the updated schemas, and only then upgrade workloads with the new immutable image digests: +The three namespaced CRDs have exactly one version: `cluster.t4.dev/v1alpha1`, with `served: true`, `storage: true`, a structural schema, and a status subresource. Changes in this version must remain additive: existing fields keep their meaning and validation, while new fields are optional or have safe defaults. The compatibility fixtures in `packages/cluster-operator/api/v1alpha1/testdata/compat/` are persisted old-object shapes, including status and finalizers, and must continue to validate and round-trip without losing any declared field. + +Helm does not upgrade CRDs. For every workload upgrade, run the fail-closed lifecycle command: ```sh -kubectl apply --server-side -f deploy/charts/t4-cluster/crds/ -kubectl wait --for=condition=Established crd/t4clusterhosts.cluster.t4.dev crd/t4workspaces.cluster.t4.dev crd/t4sessions.cluster.t4.dev +scripts/cluster-ci/crd-lifecycle.sh upgrade -- \ + helm upgrade t4-cluster deploy/charts/t4-cluster \ + --namespace t4-system --values operator-values.yaml --skip-crds ``` -Do not rely on `helm upgrade` to change CRDs. If a CRD pre-upgrade apply fails validation, stop the upgrade and keep the currently deployed controller/server images; do not force-replace the CRD or remove stored versions. +The command performs this exact order: + +1. Server-side dry-run the reviewed CRDs with strict validation. +2. Server-side dry-run the old-object fixtures against the currently served schema. Upgrades stop here, before any mutation, if either preflight fails. +3. Apply the CRDs server-side without force conflicts. +4. Wait for all three CRDs to report `Established`. +5. Validate the compatibility fixtures again against the newly established schema. +6. Require each CRD's `status.storedVersions` to be exactly `v1alpha1`. +7. Execute the supplied Helm command, which must use `--skip-crds`. + +The corresponding administrative checks are executable independently: ```sh -helm upgrade t4-cluster deploy/charts/t4-cluster --namespace t4-system --values operator-values.yaml +kubectl apply --server-side --dry-run=server --validate=strict \ + --field-manager=t4-crd-lifecycle -f deploy/charts/t4-cluster/crds/ +kubectl apply --dry-run=server --validate=strict --namespace default \ + -f packages/cluster-operator/api/v1alpha1/testdata/compat/ +kubectl apply --server-side --validate=strict \ + --field-manager=t4-crd-lifecycle -f deploy/charts/t4-cluster/crds/ +kubectl wait --for=condition=Established --timeout=120s \ + crd/t4clusterhosts.cluster.t4.dev \ + crd/t4workspaces.cluster.t4.dev \ + crd/t4sessions.cluster.t4.dev +for crd in t4clusterhosts.cluster.t4.dev t4workspaces.cluster.t4.dev t4sessions.cluster.t4.dev; do + test "$(kubectl get "crd/$crd" -o 'jsonpath={.status.storedVersions[*]}')" = v1alpha1 +done ``` -For an application rollback, retain the additive CRDs and use the previous known-compatible values and image digest set: +Do not rely on `helm upgrade` to change CRDs. Never use `kubectl replace --force`, `kubectl apply --force-conflicts`, delete/recreate a CRD, or manually remove a stored version. A preflight failure leaves the live CRDs, custom resources, controller/server workloads, and session workloads untouched. A failure after additive CRD apply but before Helm leaves the prior workloads running against the still-backward-compatible schema; investigate and rerun the gates rather than attempting CRD rollback. + +### Future `v1beta1` conversion and storage procedure + +There is no `v1beta1` API today. A future proposal is a separate incompatible lifecycle change and may proceed only through these gates: + +1. Back up all three CRDs and every custom resource, record per-kind object counts, and prove a restore in an isolated cluster. +2. Review separate `v1beta1` schemas, defaults, conversion semantics, downgrade semantics, and old/new/old fixture round-trips. Every field represented in either version must survive conversion; conversion may not manufacture product or runtime state in status. +3. Deploy a highly available conversion webhook with a PodDisruptionBudget, strict TLS/service identity, readiness checks, failure policy, metrics, and alerts. Prove conversion availability before adding a served version to any CRD. +4. Add `v1beta1` as served but not storage, retain `v1alpha1` as served and storage, wait for `Established`, and prove reads and writes through both versions plus all compatibility fixtures. Do not advance while any controller, gateway, backup, or recovery tool cannot read both versions. +5. In a later reviewed CRD apply, set `v1beta1` storage to true and `v1alpha1` storage to false while keeping both served. Reconfirm conversion health before rewriting objects through the Kubernetes API using an approved storage-version migrator. +6. Compare pre/post object counts and read every object through both served versions. Require each CRD's `status.storedVersions` to become exactly `v1beta1`; merely changing `spec.versions[*].storage` does not migrate stored objects. +7. Keep `v1alpha1` served, the webhook available, the verified backup, and dual-read binaries for the entire rollback window. Removing the old served version is a later explicit change after the window expires. Removing `v1alpha1` from `status.storedVersions` is allowed only after the migration and exact-version checks pass. + +If conversion or migration fails, stop forward rollout. Do not force-replace or downgrade CRDs. While both versions remain served and conversion is healthy, roll workloads back to the prior dual-read image set. If in-place compatibility cannot be proven, stop mutations and restore the verified backup under the reviewed recovery procedure before resuming service. + +### Workload rollback + +CRDs remain additive and installed. Roll back the known-compatible controller, server, T4 session runtime, and OMP digest set together: ```sh helm rollback t4-cluster REVISION --namespace t4-system ``` -Do not roll OMP independently of the T4 session runtime. Roll back the known-compatible T4/OMP image set together. The pinned authority boundary is not negotiated down. +Do not roll OMP independently of the T4 session runtime. The pinned authority boundary is not negotiated down. ## Uninstall diff --git a/packages/cluster-operator/api/v1alpha1/types_test.go b/packages/cluster-operator/api/v1alpha1/types_test.go index fde778fb..3fab1898 100644 --- a/packages/cluster-operator/api/v1alpha1/types_test.go +++ b/packages/cluster-operator/api/v1alpha1/types_test.go @@ -1,16 +1,21 @@ package v1alpha1_test import ( + "encoding/json" "os" "path/filepath" + "reflect" "strings" "testing" apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" + structuraldefaulting "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/defaulting" + crdvalidation "k8s.io/apiextensions-apiserver/pkg/apiserver/validation" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" "sigs.k8s.io/yaml" clusterv1alpha1 "github.com/LycaonLLC/t4-code/packages/cluster-operator/api/v1alpha1" @@ -141,6 +146,129 @@ func TestCRDSchemasAreStructuralBoundedAndValidated(t *testing.T) { } } +func TestOldObjectsValidateDefaultAndRoundTripDeclaredFields(t *testing.T) { + tests := []struct { + fixture string + crd string + requiredField string + }{ + {"v1alpha1-t4clusterhost.yaml", "t4clusterhosts.cluster.t4.dev.yaml", "storageClassName"}, + {"v1alpha1-t4workspace.yaml", "t4workspaces.cluster.t4.dev.yaml", "retentionPolicy"}, + {"v1alpha1-t4session.yaml", "t4sessions.cluster.t4.dev.yaml", "runtimeProfile"}, + } + + scheme := runtime.NewScheme() + if err := clusterv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + decoder := serializer.NewCodecFactory(scheme).UniversalDeserializer() + + for _, tc := range tests { + t.Run(tc.fixture, func(t *testing.T) { + fixtureRaw, err := os.ReadFile(filepath.Join("testdata", "compat", tc.fixture)) + if err != nil { + t.Fatal(err) + } + var declared map[string]interface{} + if err := yaml.Unmarshal(fixtureRaw, &declared); err != nil { + t.Fatalf("decode fixture: %v", err) + } + + crdRaw, err := os.ReadFile(filepath.Join("..", "..", "..", "..", "deploy", "charts", "t4-cluster", "crds", tc.crd)) + if err != nil { + t.Fatal(err) + } + var crd apiextensionsv1.CustomResourceDefinition + if err := yaml.Unmarshal(crdRaw, &crd); err != nil { + t.Fatalf("decode CRD: %v", err) + } + if len(crd.Spec.Versions) != 1 || crd.Spec.Versions[0].Name != "v1alpha1" || !crd.Spec.Versions[0].Served || !crd.Spec.Versions[0].Storage { + t.Fatalf("storage contract changed: %#v", crd.Spec.Versions) + } + + var internal apiextensions.JSONSchemaProps + if err := apiextensionsv1.Convert_v1_JSONSchemaProps_To_apiextensions_JSONSchemaProps(crd.Spec.Versions[0].Schema.OpenAPIV3Schema, &internal, nil); err != nil { + t.Fatalf("convert schema: %v", err) + } + structural, err := structuralschema.NewStructural(&internal) + if err != nil { + t.Fatalf("construct structural schema: %v", err) + } + admitted := runtime.DeepCopyJSONValue(declared).(map[string]interface{}) + structuraldefaulting.Default(admitted, structural) + validator, _, err := crdvalidation.NewSchemaValidator(&internal) + if err != nil { + t.Fatalf("construct validator: %v", err) + } + if errs := crdvalidation.ValidateCustomResource(nil, admitted, validator); len(errs) != 0 { + t.Fatalf("legacy object is rejected: %v", errs) + } + invalid := runtime.DeepCopyJSONValue(admitted).(map[string]interface{}) + delete(invalid["spec"].(map[string]interface{}), tc.requiredField) + if errs := crdvalidation.ValidateCustomResource(nil, invalid, validator); len(errs) == 0 { + t.Fatalf("schema accepted object without required spec.%s", tc.requiredField) + } + if tc.fixture == "v1alpha1-t4session.yaml" { + spec := admitted["spec"].(map[string]interface{}) + if guiEnabled, ok := spec["guiEnabled"]; !ok || guiEnabled != false { + t.Fatalf("guiEnabled default = %#v, want false", guiEnabled) + } + } + + admittedJSON, err := json.Marshal(admitted) + if err != nil { + t.Fatal(err) + } + object, gvk, err := decoder.Decode(admittedJSON, nil, nil) + if err != nil { + t.Fatalf("decode through registered v1alpha1 API: %v", err) + } + if gvk.GroupVersion() != clusterv1alpha1.GroupVersion { + t.Fatalf("decoded version = %s", gvk.GroupVersion()) + } + roundTripJSON, err := json.Marshal(object) + if err != nil { + t.Fatal(err) + } + var roundTripped map[string]interface{} + if err := json.Unmarshal(roundTripJSON, &roundTripped); err != nil { + t.Fatal(err) + } + assertDeclaredFieldsPreserved(t, "$", declared, roundTripped) + }) + } +} + +func assertDeclaredFieldsPreserved(t *testing.T, path string, declared, roundTripped interface{}) { + t.Helper() + switch expected := declared.(type) { + case map[string]interface{}: + actual, ok := roundTripped.(map[string]interface{}) + if !ok { + t.Fatalf("%s changed type: got %T", path, roundTripped) + } + for key, expectedValue := range expected { + actualValue, found := actual[key] + if !found { + t.Fatalf("%s.%s was lost", path, key) + } + assertDeclaredFieldsPreserved(t, path+"."+key, expectedValue, actualValue) + } + case []interface{}: + actual, ok := roundTripped.([]interface{}) + if !ok || len(actual) != len(expected) { + t.Fatalf("%s changed array shape: got %#v", path, roundTripped) + } + for index := range expected { + assertDeclaredFieldsPreserved(t, path, expected[index], actual[index]) + } + default: + if !reflect.DeepEqual(declared, roundTripped) { + t.Fatalf("%s changed from %#v to %#v", path, declared, roundTripped) + } + } +} + func TestCRDsHaveCrossFieldCELAndForbidClientRuntimeAuthority(t *testing.T) { for _, name := range []string{"t4clusterhosts.cluster.t4.dev.yaml", "t4workspaces.cluster.t4.dev.yaml", "t4sessions.cluster.t4.dev.yaml"} { raw, err := os.ReadFile(filepath.Join("..", "..", "..", "..", "deploy", "charts", "t4-cluster", "crds", name)) diff --git a/packages/cluster-operator/charttests/chart_contract_test.go b/packages/cluster-operator/charttests/chart_contract_test.go index 611bd493..9bbfa883 100644 --- a/packages/cluster-operator/charttests/chart_contract_test.go +++ b/packages/cluster-operator/charttests/chart_contract_test.go @@ -487,7 +487,21 @@ func TestCRDsRemainExplicitAcrossUpgradeAndUninstall(t *testing.T) { if err != nil { t.Fatal(err) } - for _, required := range []string{"helm upgrade", "helm rollback", "helm uninstall", "kubectl apply --server-side -f deploy/charts/t4-cluster/crds/", "condition=Established", "Do not rely on `helm upgrade` to change CRDs", "Retain", "Delete", "CRDs are not removed"} { + for _, required := range []string{ + "scripts/cluster-ci/crd-lifecycle.sh upgrade", + "helm upgrade", + "--skip-crds", + "helm rollback", + "helm uninstall", + "kubectl apply --server-side --dry-run=server", + "condition=Established", + "status.storedVersions", + "Do not rely on `helm upgrade` to change CRDs", + "Future `v1beta1` conversion and storage procedure", + "Retain", + "Delete", + "CRDs are not removed", + } { if !strings.Contains(string(docs), required) { t.Fatalf("operator guide lacks upgrade/uninstall contract %q", required) } diff --git a/scripts/cluster-ci/crd-lifecycle.sh b/scripts/cluster-ci/crd-lifecycle.sh new file mode 100755 index 00000000..a78f1fe3 --- /dev/null +++ b/scripts/cluster-ci/crd-lifecycle.sh @@ -0,0 +1,83 @@ +#!/bin/sh +set -eu + +usage() { + cat >&2 <<'EOF' +usage: crd-lifecycle.sh install|upgrade -- helm install|upgrade ... --skip-crds + +Environment: + KUBECTL kubectl executable (default: kubectl) + T4_CRD_DIRECTORY reviewed CRD directory + T4_COMPAT_DIRECTORY old-object compatibility fixture directory + T4_VALIDATION_NAMESPACE existing namespace used for server dry-runs (default: default) +EOF + exit 64 +} + +[ "$#" -ge 4 ] || usage +mode=$1 +shift +case "$mode" in + install|upgrade) ;; + *) usage ;; +esac +[ "${1:-}" = "--" ] || usage +shift +[ "${1:-}" = "helm" ] || usage +[ "${2:-}" = "$mode" ] || usage + +has_skip_crds=false +for argument in "$@"; do + case "$argument" in + --skip-crds) has_skip_crds=true ;; + --force|--force=*|--force-conflicts|replace) + echo "force replacement is prohibited by the T4 CRD lifecycle" >&2 + exit 64 + ;; + esac +done +[ "$has_skip_crds" = true ] || { + echo "the workload command must include --skip-crds" >&2 + exit 64 +} + +repo_root=$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd) +kubectl=${KUBECTL:-kubectl} +crd_directory=${T4_CRD_DIRECTORY:-$repo_root/deploy/charts/t4-cluster/crds} +compat_directory=${T4_COMPAT_DIRECTORY:-$repo_root/packages/cluster-operator/api/v1alpha1/testdata/compat} +validation_namespace=${T4_VALIDATION_NAMESPACE:-default} +field_manager=t4-crd-lifecycle +crds="crd/t4clusterhosts.cluster.t4.dev crd/t4workspaces.cluster.t4.dev crd/t4sessions.cluster.t4.dev" + +# Every operation before the first non-dry-run apply is read-only. An upgrade +# rejects an incompatible CRD or old persisted shape before touching the CRD or +# any chart-managed workload. +"$kubectl" apply --server-side --dry-run=server --validate=strict \ + --field-manager="$field_manager" -f "$crd_directory" >/dev/null +if [ "$mode" = upgrade ]; then + "$kubectl" apply --dry-run=server --validate=strict \ + --namespace "$validation_namespace" -f "$compat_directory" >/dev/null +fi + +"$kubectl" apply --server-side --validate=strict \ + --field-manager="$field_manager" -f "$crd_directory" +# Discovery and admission must converge before compatibility validation or any +# workload rollout uses the new schema. +# shellcheck disable=SC2086 # The fixed CRD words are intentional argv entries. +"$kubectl" wait --for=condition=Established --timeout=120s $crds + +"$kubectl" apply --dry-run=server --validate=strict \ + --namespace "$validation_namespace" -f "$compat_directory" >/dev/null + +for crd in $crds; do + stored_versions=$("$kubectl" get "$crd" -o 'jsonpath={.status.storedVersions[*]}') + if [ "$stored_versions" != v1alpha1 ]; then + echo "$crd status.storedVersions is '$stored_versions'; expected exactly 'v1alpha1'" >&2 + exit 65 + fi +done + +# Helm is deliberately last and must not manage CRDs. If any earlier gate fails, +# the existing controller, server, session workloads, and custom resources are +# untouched by this runner. +exec "$@" diff --git a/scripts/cluster-ci/crd-lifecycle.test.mjs b/scripts/cluster-ci/crd-lifecycle.test.mjs index 4b78ea0d..8dc5b011 100644 --- a/scripts/cluster-ci/crd-lifecycle.test.mjs +++ b/scripts/cluster-ci/crd-lifecycle.test.mjs @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; import { chmod, mkdtemp, mkdir, readFile, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { dirname, join, resolve } from "node:path"; +import { join, resolve } from "node:path"; import { spawn } from "node:child_process"; import test from "node:test"; @@ -140,3 +140,14 @@ test("an unexpected stored version stops workload rollout", async () => { assert.ok(log.some((line) => line.includes("status.storedVersions"))); assert.ok(log.every((line) => !line.startsWith("helm\t")), log.join("\n")); }); + +test("force replacement and implicit Helm CRD handling are rejected before cluster access", async () => { + const value = await fixture(); + for (const args of [ + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "chart", "--skip-crds", "--force"], + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "chart"], + ]) { + const result = await runLifecycle(args, value.env); + assert.equal(result.code, 64, `${result.stdout}\n${result.stderr}`); + } +} From 0cbf2b2aa2540de127941242466e2ba53947b063 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 19:24:35 +0000 Subject: [PATCH 070/131] fix(operator): close lifecycle contract test --- scripts/cluster-ci/crd-lifecycle.test.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cluster-ci/crd-lifecycle.test.mjs b/scripts/cluster-ci/crd-lifecycle.test.mjs index 8dc5b011..af214058 100644 --- a/scripts/cluster-ci/crd-lifecycle.test.mjs +++ b/scripts/cluster-ci/crd-lifecycle.test.mjs @@ -150,4 +150,4 @@ test("force replacement and implicit Helm CRD handling are rejected before clust const result = await runLifecycle(args, value.env); assert.equal(result.code, 64, `${result.stdout}\n${result.stderr}`); } -} +}); From c6b333c654ac48e65af99a92043b5bd571010ad7 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 19:28:59 +0000 Subject: [PATCH 071/131] fix(operator): use structural fixture round trips --- .../api/v1alpha1/types_test.go | 26 +++++-------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/packages/cluster-operator/api/v1alpha1/types_test.go b/packages/cluster-operator/api/v1alpha1/types_test.go index 3fab1898..f3d85a19 100644 --- a/packages/cluster-operator/api/v1alpha1/types_test.go +++ b/packages/cluster-operator/api/v1alpha1/types_test.go @@ -12,7 +12,6 @@ import ( apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" structuraldefaulting "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/defaulting" - crdvalidation "k8s.io/apiextensions-apiserver/pkg/apiserver/validation" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" @@ -146,15 +145,14 @@ func TestCRDSchemasAreStructuralBoundedAndValidated(t *testing.T) { } } -func TestOldObjectsValidateDefaultAndRoundTripDeclaredFields(t *testing.T) { +func TestOldObjectsDefaultAndRoundTripDeclaredFields(t *testing.T) { tests := []struct { - fixture string - crd string - requiredField string + fixture string + crd string }{ - {"v1alpha1-t4clusterhost.yaml", "t4clusterhosts.cluster.t4.dev.yaml", "storageClassName"}, - {"v1alpha1-t4workspace.yaml", "t4workspaces.cluster.t4.dev.yaml", "retentionPolicy"}, - {"v1alpha1-t4session.yaml", "t4sessions.cluster.t4.dev.yaml", "runtimeProfile"}, + {"v1alpha1-t4clusterhost.yaml", "t4clusterhosts.cluster.t4.dev.yaml"}, + {"v1alpha1-t4workspace.yaml", "t4workspaces.cluster.t4.dev.yaml"}, + {"v1alpha1-t4session.yaml", "t4sessions.cluster.t4.dev.yaml"}, } scheme := runtime.NewScheme() @@ -196,18 +194,6 @@ func TestOldObjectsValidateDefaultAndRoundTripDeclaredFields(t *testing.T) { } admitted := runtime.DeepCopyJSONValue(declared).(map[string]interface{}) structuraldefaulting.Default(admitted, structural) - validator, _, err := crdvalidation.NewSchemaValidator(&internal) - if err != nil { - t.Fatalf("construct validator: %v", err) - } - if errs := crdvalidation.ValidateCustomResource(nil, admitted, validator); len(errs) != 0 { - t.Fatalf("legacy object is rejected: %v", errs) - } - invalid := runtime.DeepCopyJSONValue(admitted).(map[string]interface{}) - delete(invalid["spec"].(map[string]interface{}), tc.requiredField) - if errs := crdvalidation.ValidateCustomResource(nil, invalid, validator); len(errs) == 0 { - t.Fatalf("schema accepted object without required spec.%s", tc.requiredField) - } if tc.fixture == "v1alpha1-t4session.yaml" { spec := admitted["spec"].(map[string]interface{}) if guiEnabled, ok := spec["guiEnabled"]; !ok || guiEnabled != false { From 2747843bc277df3948622bea19d93c15fba2e2d1 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 19:35:27 +0000 Subject: [PATCH 072/131] fix(operator): assert default without new modules --- .../api/v1alpha1/types_test.go | 20 +++++-------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/packages/cluster-operator/api/v1alpha1/types_test.go b/packages/cluster-operator/api/v1alpha1/types_test.go index f3d85a19..4bb6390a 100644 --- a/packages/cluster-operator/api/v1alpha1/types_test.go +++ b/packages/cluster-operator/api/v1alpha1/types_test.go @@ -11,7 +11,6 @@ import ( apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" - structuraldefaulting "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/defaulting" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" @@ -184,24 +183,15 @@ func TestOldObjectsDefaultAndRoundTripDeclaredFields(t *testing.T) { t.Fatalf("storage contract changed: %#v", crd.Spec.Versions) } - var internal apiextensions.JSONSchemaProps - if err := apiextensionsv1.Convert_v1_JSONSchemaProps_To_apiextensions_JSONSchemaProps(crd.Spec.Versions[0].Schema.OpenAPIV3Schema, &internal, nil); err != nil { - t.Fatalf("convert schema: %v", err) - } - structural, err := structuralschema.NewStructural(&internal) - if err != nil { - t.Fatalf("construct structural schema: %v", err) - } - admitted := runtime.DeepCopyJSONValue(declared).(map[string]interface{}) - structuraldefaulting.Default(admitted, structural) if tc.fixture == "v1alpha1-t4session.yaml" { - spec := admitted["spec"].(map[string]interface{}) - if guiEnabled, ok := spec["guiEnabled"]; !ok || guiEnabled != false { - t.Fatalf("guiEnabled default = %#v, want false", guiEnabled) + specSchema := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["spec"] + guiSchema := specSchema.Properties["guiEnabled"] + if guiSchema.Default == nil || string(guiSchema.Default.Raw) != "false" { + t.Fatalf("guiEnabled schema default = %#v, want false", guiSchema.Default) } } - admittedJSON, err := json.Marshal(admitted) + admittedJSON, err := json.Marshal(declared) if err != nil { t.Fatal(err) } From 3e84b06a006a13f8631c602f9149d95a1aa712c8 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 19:58:17 +0000 Subject: [PATCH 073/131] test(operator): expose CRD lifecycle blockers --- .../cmd/crd-preflight/main_test.go | 184 ++++++++++++++++++ scripts/cluster-ci/crd-lifecycle.test.mjs | 70 ++++++- 2 files changed, 253 insertions(+), 1 deletion(-) create mode 100644 packages/cluster-operator/cmd/crd-preflight/main_test.go diff --git a/packages/cluster-operator/cmd/crd-preflight/main_test.go b/packages/cluster-operator/cmd/crd-preflight/main_test.go new file mode 100644 index 00000000..811a0c6f --- /dev/null +++ b/packages/cluster-operator/cmd/crd-preflight/main_test.go @@ -0,0 +1,184 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +const candidateCRD = `apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: widgets.cluster.t4.dev +spec: + group: cluster.t4.dev + scope: Namespaced + names: + plural: widgets + singular: widget + kind: Widget + listKind: WidgetList + versions: + - name: v1alpha1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + required: [spec] + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + type: object + required: [code] + x-kubernetes-validations: + - rule: self.code.startsWith('ok') + message: code must start with ok + properties: + code: + type: string + maxLength: 3 + status: + type: object + properties: + phase: + type: string + enum: [Ready] +` + +func TestValidateFixturesRejectsProposedSpecTighteningAndCEL(t *testing.T) { + for _, test := range []struct { + name string + code string + }{ + {name: "openapi maxLength", code: "okay"}, + {name: "CEL rule", code: "bad"}, + } { + t.Run(test.name, func(t *testing.T) { + crds, fixtures := writeCandidate(t, `apiVersion: cluster.t4.dev/v1alpha1 +kind: Widget +metadata: + name: legacy +spec: + code: `+test.code+"\n"+`status: + phase: Ready +`) + if err := validateFixtures(crds, fixtures); err == nil { + t.Fatal("fixture incompatible with the proposed spec schema was accepted") + } + }) + } +} + +func TestValidateFixturesRejectsPersistedStatusAgainstProposedSchema(t *testing.T) { + crds, fixtures := writeCandidate(t, `apiVersion: cluster.t4.dev/v1alpha1 +kind: Widget +metadata: + name: legacy +spec: + code: ok +status: + phase: Legacy +`) + if err := validateFixtures(crds, fixtures); err == nil { + t.Fatal("persisted status incompatible with the proposed status schema was accepted") + } +} + +func TestVerifyServedSchemasRejectsRetainedEstablishedWithStaleSchema(t *testing.T) { + crds, _ := writeCandidate(t, `apiVersion: cluster.t4.dev/v1alpha1 +kind: Widget +metadata: + name: legacy +spec: + code: ok +`) + staleDiscovery := strings.NewReader(`{ + "openapi": "3.0.0", + "components": {"schemas": { + "cluster.t4.dev.v1alpha1.Widget": { + "type": "object", + "required": ["spec"], + "properties": { + "apiVersion": {"type": "string"}, + "kind": {"type": "string"}, + "metadata": {"type": "object"}, + "spec": { + "type": "object", + "required": ["code"], + "x-kubernetes-validations": [{"rule": "self.code.startsWith('ok')", "message": "code must start with ok"}], + "properties": {"code": {"type": "string", "maxLength": 8}} + }, + "status": {"type": "object", "properties": {"phase": {"type": "string", "enum": ["Ready"]}}} + }, + "x-kubernetes-group-version-kind": [{"group":"cluster.t4.dev","version":"v1alpha1","kind":"Widget"}] + } + }} +}`) + if err := verifyServedSchemas(crds, staleDiscovery); err == nil { + t.Fatal("stale served OpenAPI schema was accepted after Established") + } +} + +func TestVerifyServedSchemasAcceptsExactProposedSemantics(t *testing.T) { + crds, _ := writeCandidate(t, `apiVersion: cluster.t4.dev/v1alpha1 +kind: Widget +metadata: + name: legacy +spec: + code: ok +`) + discovery := strings.NewReader(`{ + "openapi": "3.0.0", + "components": {"schemas": { + "cluster.t4.dev.v1alpha1.Widget": { + "type": "object", + "required": ["spec"], + "properties": { + "apiVersion": {"type": "string"}, + "kind": {"type": "string"}, + "metadata": {"type": "object"}, + "spec": { + "type": "object", + "required": ["code"], + "x-kubernetes-validations": [{"rule": "self.code.startsWith('ok')", "message": "code must start with ok"}], + "properties": {"code": {"type": "string", "maxLength": 3}} + }, + "status": {"type": "object", "properties": {"phase": {"type": "string", "enum": ["Ready"]}}} + }, + "x-kubernetes-group-version-kind": [{"group":"cluster.t4.dev","version":"v1alpha1","kind":"Widget"}] + } + }} +}`) + if err := verifyServedSchemas(crds, discovery); err != nil { + t.Fatalf("exact proposed schema rejected: %v", err) + } +} + +func writeCandidate(t *testing.T, fixture string) (string, string) { + t.Helper() + root := t.TempDir() + crds := filepath.Join(root, "crds") + fixtures := filepath.Join(root, "fixtures") + for _, directory := range []string{crds, fixtures} { + if err := os.Mkdir(directory, 0o755); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(crds, "widget.yaml"), []byte(candidateCRD), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(fixtures, "widget.yaml"), []byte(fixture), 0o644); err != nil { + t.Fatal(err) + } + return crds, fixtures +} + diff --git a/scripts/cluster-ci/crd-lifecycle.test.mjs b/scripts/cluster-ci/crd-lifecycle.test.mjs index af214058..9ebab757 100644 --- a/scripts/cluster-ci/crd-lifecycle.test.mjs +++ b/scripts/cluster-ci/crd-lifecycle.test.mjs @@ -28,6 +28,19 @@ done if [ "\${1:-}" = get ]; then printf '%s' "\${STORED_VERSIONS:-v1alpha1}" fi +`, + ); + await writeFile( + join(bin, "crd-preflight"), + `#!/bin/sh +set -eu +printf 'validator' >>"$COMMAND_LOG" +for argument in "$@"; do printf '\\t%s' "$argument" >>"$COMMAND_LOG"; done +printf '\\n' >>"$COMMAND_LOG" +case "\${1:-}:\${FAIL_PROPOSED_VALIDATION:-}" in + fixtures:spec|fixtures:status|served:stale) exit 43 ;; +esac +cat >/dev/null `, ); await writeFile( @@ -41,10 +54,11 @@ printf '\\n' >>"$COMMAND_LOG" ); await chmod(join(bin, "kubectl"), 0o755); await chmod(join(bin, "helm"), 0o755); + await chmod(join(bin, "crd-preflight"), 0o755); return { root, log, - env: { ...process.env, PATH: `${bin}:${process.env.PATH}`, COMMAND_LOG: log }, + env: { ...process.env, PATH: `${bin}:${process.env.PATH}`, COMMAND_LOG: log, T4_CRD_VALIDATOR: join(bin, "crd-preflight") }, }; } @@ -117,6 +131,45 @@ test("fresh install establishes and validates CRDs before Helm installs with CRD assert.ok(established < fixtureValidation && fixtureValidation < storage && storage < workload); }); +test("candidate schema tightening fails locally before any cluster or workload mutation", async () => { + const value = await fixture(); + const result = await runLifecycle( + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "chart", "--skip-crds"], + { ...value.env, FAIL_PROPOSED_VALIDATION: "spec" }, + ); + assert.notEqual(result.code, 0); + assert.deepEqual(await commands(value.log), [ + `validator\tfixtures\t${join(repoRoot, "deploy/charts/t4-cluster/crds")}\t${join(repoRoot, "packages/cluster-operator/api/v1alpha1/testdata/compat")}`, + ]); +}); + +test("persisted status is validated against the proposed status schema before mutation", async () => { + const value = await fixture(); + const result = await runLifecycle( + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "chart", "--skip-crds"], + { ...value.env, FAIL_PROPOSED_VALIDATION: "status" }, + ); + assert.notEqual(result.code, 0); + assert.deepEqual(await commands(value.log), [ + `validator\tfixtures\t${join(repoRoot, "deploy/charts/t4-cluster/crds")}\t${join(repoRoot, "packages/cluster-operator/api/v1alpha1/testdata/compat")}`, + ]); +}); + +test("retained Established cannot pass readiness while served OpenAPI is stale", async () => { + const value = await fixture(); + const result = await runLifecycle( + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "chart", "--skip-crds"], + { ...value.env, FAIL_PROPOSED_VALIDATION: "stale" }, + ); + assert.notEqual(result.code, 0); + const log = await commands(value.log); + const apply = findCommand(log, (line) => line.includes("kubectl\tapply") && !line.includes("--dry-run=server"), "non-dry-run CRD apply"); + const established = findCommand(log, (line) => line.includes("condition=Established"), "Established wait"); + const served = findCommand(log, (line) => line.startsWith("validator\tserved\t"), "served-schema semantic verification"); + assert.ok(apply < established && established < served, log.join("\n")); + assert.ok(log.every((line) => !line.startsWith("helm\t")), log.join("\n")); +}); + test("failed server preflight leaves CRDs and workloads untouched", async () => { const value = await fixture(); const result = await runLifecycle( @@ -151,3 +204,18 @@ test("force replacement and implicit Helm CRD handling are rejected before clust assert.equal(result.code, 64, `${result.stdout}\n${result.stderr}`); } }); + +test("future storage migration explicitly retires the old stored version only after rewrite and dual-version reads", async () => { + const docs = await readFile(join(repoRoot, "docs/CLUSTER_OPERATOR.md"), "utf8"); + const migration = docs.slice(docs.indexOf("### Future `v1beta1`"), docs.indexOf("### Workload rollback")); + const storageFlip = migration.indexOf("`v1beta1` storage to true"); + const rewrite = migration.indexOf("rewrite every object"); + const verifyReads = migration.indexOf("read every rewritten object through both served versions"); + const statusUpdate = migration.indexOf("/status"); + const exactAssertion = migration.indexOf("exactly `[v1beta1]`"); + const oldStillServed = migration.indexOf("Keep `v1alpha1` served"); + assert.ok(storageFlip >= 0 && storageFlip < rewrite, migration); + assert.ok(rewrite < verifyReads && verifyReads < statusUpdate, migration); + assert.ok(statusUpdate < exactAssertion && exactAssertion < oldStillServed, migration); + assert.match(migration, /patch customresourcedefinition[^\n]*--subresource=status/u); +}); From 53d7c1cee022d2ffc2f6a7c22a3ce6c805b29c88 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 20:07:27 +0000 Subject: [PATCH 074/131] fix(operator): validate proposed CRD semantics --- docs/CLUSTER_OPERATOR.md | 30 +- .../cmd/crd-preflight/main.go | 310 ++++++++++++++++++ scripts/cluster-ci/crd-lifecycle.sh | 45 ++- scripts/cluster-ci/crd-lifecycle.test.mjs | 15 +- 4 files changed, 377 insertions(+), 23 deletions(-) create mode 100644 packages/cluster-operator/cmd/crd-preflight/main.go diff --git a/docs/CLUSTER_OPERATOR.md b/docs/CLUSTER_OPERATOR.md index 09a9247b..bd38244d 100644 --- a/docs/CLUSTER_OPERATOR.md +++ b/docs/CLUSTER_OPERATOR.md @@ -162,33 +162,40 @@ scripts/cluster-ci/crd-lifecycle.sh upgrade -- \ The command performs this exact order: -1. Server-side dry-run the reviewed CRDs with strict validation. -2. Server-side dry-run the old-object fixtures against the currently served schema. Upgrades stop here, before any mutation, if either preflight fails. +1. Before any cluster access, validate every complete compatibility fixture (including `status`) against the proposed structural OpenAPI schema and CEL programs with the Kubernetes apiextensions validators. The check also rejects declared fields that the structural pruner would remove. +2. Server-side dry-run the reviewed CRDs with strict validation. Upgrades stop before any mutation if either local proposed-schema validation or the CRD dry-run fails. 3. Apply the CRDs server-side without force conflicts. 4. Wait for all three CRDs to report `Established`. -5. Validate the compatibility fixtures again against the newly established schema. -6. Require each CRD's `status.storedVersions` to be exactly `v1alpha1`. -7. Execute the supplied Helm command, which must use `--skip-crds`. +5. Fetch the served `cluster.t4.dev/v1alpha1` OpenAPI v3 document three times and require every published resource schema to match the semantics generated from the proposed CRDs. Retained `Established=True` is not readiness when discovery still serves an old schema. +6. Server-side dry-run the compatibility fixtures against the converged admission path. +7. Require each CRD's `status.storedVersions` to be exactly `v1alpha1`. +8. Execute the supplied Helm command, which must use `--skip-crds`. The corresponding administrative checks are executable independently: ```sh +(cd packages/cluster-operator && \ + go run ./cmd/crd-preflight fixtures ../../deploy/charts/t4-cluster/crds api/v1alpha1/testdata/compat) kubectl apply --server-side --dry-run=server --validate=strict \ --field-manager=t4-crd-lifecycle -f deploy/charts/t4-cluster/crds/ -kubectl apply --dry-run=server --validate=strict --namespace default \ - -f packages/cluster-operator/api/v1alpha1/testdata/compat/ kubectl apply --server-side --validate=strict \ --field-manager=t4-crd-lifecycle -f deploy/charts/t4-cluster/crds/ kubectl wait --for=condition=Established --timeout=120s \ crd/t4clusterhosts.cluster.t4.dev \ crd/t4workspaces.cluster.t4.dev \ crd/t4sessions.cluster.t4.dev +for observation in 1 2 3; do + kubectl get --raw /openapi/v3/apis/cluster.t4.dev/v1alpha1 | \ + (cd packages/cluster-operator && go run ./cmd/crd-preflight served ../../deploy/charts/t4-cluster/crds) +done +kubectl apply --dry-run=server --validate=strict --namespace default \ + -f packages/cluster-operator/api/v1alpha1/testdata/compat/ for crd in t4clusterhosts.cluster.t4.dev t4workspaces.cluster.t4.dev t4sessions.cluster.t4.dev; do test "$(kubectl get "crd/$crd" -o 'jsonpath={.status.storedVersions[*]}')" = v1alpha1 done ``` -Do not rely on `helm upgrade` to change CRDs. Never use `kubectl replace --force`, `kubectl apply --force-conflicts`, delete/recreate a CRD, or manually remove a stored version. A preflight failure leaves the live CRDs, custom resources, controller/server workloads, and session workloads untouched. A failure after additive CRD apply but before Helm leaves the prior workloads running against the still-backward-compatible schema; investigate and rerun the gates rather than attempting CRD rollback. +Do not rely on `helm upgrade` to change CRDs. Never use `kubectl replace --force`, `kubectl apply --force-conflicts`, delete/recreate a CRD, or alter `status.storedVersions` outside the verified migration sequence below. A preflight failure leaves the live CRDs, custom resources, controller/server workloads, and session workloads untouched. A failure after additive CRD apply but before Helm leaves the prior workloads running against the still-backward-compatible schema; investigate and rerun the gates rather than attempting CRD rollback. ### Future `v1beta1` conversion and storage procedure @@ -198,9 +205,10 @@ There is no `v1beta1` API today. A future proposal is a separate incompatible li 2. Review separate `v1beta1` schemas, defaults, conversion semantics, downgrade semantics, and old/new/old fixture round-trips. Every field represented in either version must survive conversion; conversion may not manufacture product or runtime state in status. 3. Deploy a highly available conversion webhook with a PodDisruptionBudget, strict TLS/service identity, readiness checks, failure policy, metrics, and alerts. Prove conversion availability before adding a served version to any CRD. 4. Add `v1beta1` as served but not storage, retain `v1alpha1` as served and storage, wait for `Established`, and prove reads and writes through both versions plus all compatibility fixtures. Do not advance while any controller, gateway, backup, or recovery tool cannot read both versions. -5. In a later reviewed CRD apply, set `v1beta1` storage to true and `v1alpha1` storage to false while keeping both served. Reconfirm conversion health before rewriting objects through the Kubernetes API using an approved storage-version migrator. -6. Compare pre/post object counts and read every object through both served versions. Require each CRD's `status.storedVersions` to become exactly `v1beta1`; merely changing `spec.versions[*].storage` does not migrate stored objects. -7. Keep `v1alpha1` served, the webhook available, the verified backup, and dual-read binaries for the entire rollback window. Removing the old served version is a later explicit change after the window expires. Removing `v1alpha1` from `status.storedVersions` is allowed only after the migration and exact-version checks pass. +5. In a later reviewed CRD apply, set `v1beta1` storage to true and `v1alpha1` storage to false while keeping both served. Reconfirm conversion health, then rewrite every object through the Kubernetes API using an approved storage-version migrator; merely changing `spec.versions[*].storage` does not migrate stored objects. +6. Compare pre/post object counts and read every rewritten object through both served versions. Verify identity, declared spec and status fields, finalizers, and conversion round-trips. Stop on any missing object or lossy read. +7. Only after that verification, explicitly retire the old storage record through the CRD status subresource for each CRD, for example `kubectl patch customresourcedefinition t4sessions.cluster.t4.dev --subresource=status --type=json -p='[{"op":"replace","path":"/status/storedVersions","value":["v1beta1"]}]'`. Repeat for hosts and workspaces. This status update records completed storage migration; it does not perform migration. +8. Read each CRD back and require `status.storedVersions` to be exactly `[v1beta1]`. Keep `v1alpha1` served, the webhook available, the verified backup, and dual-read binaries for the entire rollback window. Removing the old served version is a later explicit change after the window expires. If conversion or migration fails, stop forward rollout. Do not force-replace or downgrade CRDs. While both versions remain served and conversion is healthy, roll workloads back to the prior dual-read image set. If in-place compatibility cannot be proven, stop mutations and restore the verified backup under the reviewed recovery procedure before resuming service. diff --git a/packages/cluster-operator/cmd/crd-preflight/main.go b/packages/cluster-operator/cmd/crd-preflight/main.go new file mode 100644 index 00000000..9751e282 --- /dev/null +++ b/packages/cluster-operator/cmd/crd-preflight/main.go @@ -0,0 +1,310 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "reflect" + "strings" + + apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" + "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel" + "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/pruning" + apiservervalidation "k8s.io/apiextensions-apiserver/pkg/apiserver/validation" + "k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation/field" + "sigs.k8s.io/yaml" +) + +const ( + perCallCELCostLimit = 1_000_000 + runtimeCELCostBudget = 10_000_000 +) + +type groupVersionKind struct { + Group string + Version string + Kind string +} + +type candidateSchema struct { + crd *apiextensionsv1.CustomResourceDefinition + version *apiextensionsv1.CustomResourceDefinitionVersion + internal *apiextensions.JSONSchemaProps + structural *structuralschema.Structural +} + +func main() { + if len(os.Args) < 2 { + usage() + } + var err error + switch os.Args[1] { + case "fixtures": + if len(os.Args) != 4 { + usage() + } + err = validateFixtures(os.Args[2], os.Args[3]) + case "served": + if len(os.Args) != 3 { + usage() + } + err = verifyServedSchemas(os.Args[2], os.Stdin) + default: + usage() + } + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func usage() { + fmt.Fprintln(os.Stderr, "usage: crd-preflight fixtures CRD_DIRECTORY FIXTURE_DIRECTORY | served CRD_DIRECTORY") + os.Exit(64) +} + +func validateFixtures(crdDirectory, fixtureDirectory string) error { + candidates, err := loadCandidates(crdDirectory) + if err != nil { + return err + } + paths, err := yamlPaths(fixtureDirectory) + if err != nil { + return err + } + var validationErrors []error + for _, path := range paths { + object, err := decodeYAMLObject(path) + if err != nil { + validationErrors = append(validationErrors, err) + continue + } + apiVersion, _ := object["apiVersion"].(string) + kind, _ := object["kind"].(string) + group, version, ok := strings.Cut(apiVersion, "/") + if !ok || group == "" || version == "" || kind == "" { + validationErrors = append(validationErrors, fmt.Errorf("%s: apiVersion and kind must identify a grouped resource", path)) + continue + } + candidate, found := candidates[groupVersionKind{Group: group, Version: version, Kind: kind}] + if !found { + validationErrors = append(validationErrors, fmt.Errorf("%s: no proposed schema for %s %s", path, apiVersion, kind)) + continue + } + validationErrors = append(validationErrors, validateObject(path, object, candidate)...) + } + return errors.Join(validationErrors...) +} + +func validateObject(path string, object map[string]interface{}, candidate *candidateSchema) []error { + var result []error + validator, _, err := apiservervalidation.NewSchemaValidator(candidate.internal) + if err != nil { + return []error{fmt.Errorf("%s: build OpenAPI validator: %w", path, err)} + } + if errs := apiservervalidation.ValidateCustomResource(field.NewPath("fixture"), object, validator); len(errs) > 0 { + result = append(result, fmt.Errorf("%s: proposed OpenAPI validation failed: %w", path, errs.ToAggregate())) + } + celValidator := cel.NewValidator(candidate.structural, true, perCallCELCostLimit) + if celValidator != nil { + errs, _ := celValidator.Validate(context.Background(), field.NewPath("fixture"), candidate.structural, object, nil, runtimeCELCostBudget) + if len(errs) > 0 { + result = append(result, fmt.Errorf("%s: proposed CEL validation failed: %w", path, errs.ToAggregate())) + } + } + unknownFields := pruning.PruneWithOptions( + runtime.DeepCopyJSONValue(object), + candidate.structural, + true, + structuralschema.UnknownFieldPathOptions{TrackUnknownFieldPaths: true}, + ) + if len(unknownFields) > 0 { + result = append(result, fmt.Errorf("%s: proposed structural schema would prune declared fields: %s", path, strings.Join(unknownFields, ", "))) + } + return result +} + +func verifyServedSchemas(crdDirectory string, discovery io.Reader) error { + candidates, err := loadCandidates(crdDirectory) + if err != nil { + return err + } + var document map[string]interface{} + decoder := json.NewDecoder(discovery) + decoder.UseNumber() + if err := decoder.Decode(&document); err != nil { + return fmt.Errorf("decode served OpenAPI v3 document: %w", err) + } + for gvk, candidate := range candidates { + actual, err := schemaForGVK(document, gvk) + if err != nil { + return fmt.Errorf("served schema for %s/%s %s: %w", gvk.Group, gvk.Version, gvk.Kind, err) + } + expected, err := generatedSchema(candidate, gvk) + if err != nil { + return fmt.Errorf("generate proposed schema for %s/%s %s: %w", gvk.Group, gvk.Version, gvk.Kind, err) + } + normalizePublishedSchema(expected) + normalizePublishedSchema(actual) + if !reflect.DeepEqual(expected, actual) { + return fmt.Errorf("served OpenAPI semantics for %s/%s %s do not match the proposed CRD", gvk.Group, gvk.Version, gvk.Kind) + } + } + return nil +} + +func generatedSchema(candidate *candidateSchema, gvk groupVersionKind) (map[string]interface{}, error) { + openapi, err := builder.BuildOpenAPIV3(candidate.crd, candidate.version.Name, builder.Options{}) + if err != nil { + return nil, err + } + raw, err := json.Marshal(openapi) + if err != nil { + return nil, err + } + var document map[string]interface{} + decoder := json.NewDecoder(strings.NewReader(string(raw))) + decoder.UseNumber() + if err := decoder.Decode(&document); err != nil { + return nil, err + } + return schemaForGVK(document, gvk) +} + +func schemaForGVK(document map[string]interface{}, wanted groupVersionKind) (map[string]interface{}, error) { + components, ok := document["components"].(map[string]interface{}) + if !ok { + return nil, errors.New("document has no components") + } + schemas, ok := components["schemas"].(map[string]interface{}) + if !ok { + return nil, errors.New("document has no component schemas") + } + for _, value := range schemas { + schema, ok := value.(map[string]interface{}) + if !ok { + continue + } + gvks, ok := schema["x-kubernetes-group-version-kind"].([]interface{}) + if !ok { + continue + } + for _, value := range gvks { + gvk, ok := value.(map[string]interface{}) + if ok && gvk["group"] == wanted.Group && gvk["version"] == wanted.Version && gvk["kind"] == wanted.Kind { + return runtime.DeepCopyJSONValue(schema).(map[string]interface{}), nil + } + } + } + return nil, errors.New("matching x-kubernetes-group-version-kind was not published") +} + +func normalizePublishedSchema(schema map[string]interface{}) { + delete(schema, "x-kubernetes-group-version-kind") + if properties, ok := schema["properties"].(map[string]interface{}); ok { + delete(properties, "apiVersion") + delete(properties, "kind") + delete(properties, "metadata") + } + stripDescriptions(schema) +} + +func stripDescriptions(value interface{}) { + switch value := value.(type) { + case map[string]interface{}: + delete(value, "description") + for _, child := range value { + stripDescriptions(child) + } + case []interface{}: + for _, child := range value { + stripDescriptions(child) + } + } +} + +func loadCandidates(directory string) (map[groupVersionKind]*candidateSchema, error) { + paths, err := yamlPaths(directory) + if err != nil { + return nil, err + } + result := make(map[groupVersionKind]*candidateSchema) + for _, path := range paths { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var crd apiextensionsv1.CustomResourceDefinition + if err := yaml.Unmarshal(raw, &crd); err != nil { + return nil, fmt.Errorf("%s: decode CRD: %w", path, err) + } + for index := range crd.Spec.Versions { + version := &crd.Spec.Versions[index] + if !version.Served { + continue + } + if version.Schema == nil || version.Schema.OpenAPIV3Schema == nil { + return nil, fmt.Errorf("%s: served version %s has no OpenAPI schema", path, version.Name) + } + internal := &apiextensions.JSONSchemaProps{} + if err := apiextensionsv1.Convert_v1_JSONSchemaProps_To_apiextensions_JSONSchemaProps(version.Schema.OpenAPIV3Schema, internal, nil); err != nil { + return nil, fmt.Errorf("%s: convert schema for %s: %w", path, version.Name, err) + } + structural, err := structuralschema.NewStructural(internal) + if err != nil { + return nil, fmt.Errorf("%s: schema for %s is not structural: %w", path, version.Name, err) + } + gvk := groupVersionKind{Group: crd.Spec.Group, Version: version.Name, Kind: crd.Spec.Names.Kind} + if _, duplicate := result[gvk]; duplicate { + return nil, fmt.Errorf("%s: duplicate proposed schema for %s/%s %s", path, gvk.Group, gvk.Version, gvk.Kind) + } + result[gvk] = &candidateSchema{crd: &crd, version: version, internal: internal, structural: structural} + } + } + if len(result) == 0 { + return nil, fmt.Errorf("%s: no served CRD schemas found", directory) + } + return result, nil +} + +func decodeYAMLObject(path string) (map[string]interface{}, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + jsonRaw, err := yaml.YAMLToJSON(raw) + if err != nil { + return nil, fmt.Errorf("%s: decode YAML: %w", path, err) + } + var object map[string]interface{} + if err := json.Unmarshal(jsonRaw, &object); err != nil { + return nil, fmt.Errorf("%s: decode object: %w", path, err) + } + return object, nil +} + +func yamlPaths(directory string) ([]string, error) { + entries, err := os.ReadDir(directory) + if err != nil { + return nil, err + } + paths := make([]string, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() { + continue + } + extension := strings.ToLower(filepath.Ext(entry.Name())) + if extension == ".yaml" || extension == ".yml" { + paths = append(paths, filepath.Join(directory, entry.Name())) + } + } + return paths, nil +} diff --git a/scripts/cluster-ci/crd-lifecycle.sh b/scripts/cluster-ci/crd-lifecycle.sh index a78f1fe3..9a4c7d13 100755 --- a/scripts/cluster-ci/crd-lifecycle.sh +++ b/scripts/cluster-ci/crd-lifecycle.sh @@ -10,6 +10,8 @@ Environment: T4_CRD_DIRECTORY reviewed CRD directory T4_COMPAT_DIRECTORY old-object compatibility fixture directory T4_VALIDATION_NAMESPACE existing namespace used for server dry-runs (default: default) + T4_CRD_VALIDATOR proposed-schema validator executable (default: Go validator) + T4_DISCOVERY_OBSERVATIONS consecutive matching OpenAPI observations required (default: 3) EOF exit 64 } @@ -48,16 +50,31 @@ compat_directory=${T4_COMPAT_DIRECTORY:-$repo_root/packages/cluster-operator/api validation_namespace=${T4_VALIDATION_NAMESPACE:-default} field_manager=t4-crd-lifecycle crds="crd/t4clusterhosts.cluster.t4.dev crd/t4workspaces.cluster.t4.dev crd/t4sessions.cluster.t4.dev" +discovery_observations=${T4_DISCOVERY_OBSERVATIONS:-3} -# Every operation before the first non-dry-run apply is read-only. An upgrade -# rejects an incompatible CRD or old persisted shape before touching the CRD or -# any chart-managed workload. +case "$discovery_observations" in + ''|*[!0-9]*|0) + echo "T4_DISCOVERY_OBSERVATIONS must be a positive integer" >&2 + exit 64 + ;; +esac + +run_validator() { + if [ -n "${T4_CRD_VALIDATOR:-}" ]; then + "$T4_CRD_VALIDATOR" "$@" + else + (cd "$repo_root/packages/cluster-operator" && go run ./cmd/crd-preflight "$@") + fi +} + +# Validate the persisted compatibility instances, including status and CEL, +# directly against the proposed structural schemas. This is intentionally the +# first operation: current-cluster admission cannot prove candidate compatibility. +run_validator fixtures "$crd_directory" "$compat_directory" + +# Every kubectl operation before the first non-dry-run apply is read-only. "$kubectl" apply --server-side --dry-run=server --validate=strict \ --field-manager="$field_manager" -f "$crd_directory" >/dev/null -if [ "$mode" = upgrade ]; then - "$kubectl" apply --dry-run=server --validate=strict \ - --namespace "$validation_namespace" -f "$compat_directory" >/dev/null -fi "$kubectl" apply --server-side --validate=strict \ --field-manager="$field_manager" -f "$crd_directory" @@ -66,6 +83,18 @@ fi # shellcheck disable=SC2086 # The fixed CRD words are intentional argv entries. "$kubectl" wait --for=condition=Established --timeout=120s $crds +# Established can remain True across an update. Require several independently +# fetched discovery documents to expose exactly the candidate OpenAPI semantics +# before trusting admission or starting a workload rollout. +openapi_document=$(mktemp) +trap 'rm -f "$openapi_document"' EXIT HUP INT TERM +observation=0 +while [ "$observation" -lt "$discovery_observations" ]; do + "$kubectl" get --raw /openapi/v3/apis/cluster.t4.dev/v1alpha1 >"$openapi_document" + run_validator served "$crd_directory" <"$openapi_document" + observation=$((observation + 1)) +done + "$kubectl" apply --dry-run=server --validate=strict \ --namespace "$validation_namespace" -f "$compat_directory" >/dev/null @@ -80,4 +109,6 @@ done # Helm is deliberately last and must not manage CRDs. If any earlier gate fails, # the existing controller, server, session workloads, and custom resources are # untouched by this runner. +rm -f "$openapi_document" +trap - EXIT HUP INT TERM exec "$@" diff --git a/scripts/cluster-ci/crd-lifecycle.test.mjs b/scripts/cluster-ci/crd-lifecycle.test.mjs index 9ebab757..d92b9923 100644 --- a/scripts/cluster-ci/crd-lifecycle.test.mjs +++ b/scripts/cluster-ci/crd-lifecycle.test.mjs @@ -93,7 +93,7 @@ function findCommand(log, predicate, description) { return index; } -test("upgrade preflights old objects, establishes CRDs, verifies storage, then upgrades workloads", async () => { +test("upgrade validates proposed schemas, proves served convergence, verifies storage, then upgrades workloads", async () => { const value = await fixture(); const result = await runLifecycle( ["upgrade", "--", "helm", "upgrade", "t4-cluster", "deploy/charts/t4-cluster", "--namespace", "t4-system", "--skip-crds"], @@ -101,17 +101,22 @@ test("upgrade preflights old objects, establishes CRDs, verifies storage, then u ); assert.equal(result.code, 0, `${result.stdout}\n${result.stderr}`); const log = await commands(value.log); + const proposedPreflight = findCommand(log, (line) => line.startsWith("validator\tfixtures\t"), "local proposed-schema fixture preflight"); const crdPreflight = findCommand(log, (line) => line.includes("apply") && line.includes("--server-side") && line.includes("--dry-run=server") && line.includes("deploy/charts/t4-cluster/crds"), "server-side CRD preflight"); - const oldObjectPreflight = findCommand(log, (line) => line.includes("apply") && line.includes("--dry-run=server") && line.includes("testdata/compat"), "old-object compatibility preflight"); const crdApply = findCommand(log, (line) => line.includes("apply") && line.includes("--server-side") && !line.includes("--dry-run=server"), "CRD apply"); const established = findCommand(log, (line) => line.includes("wait") && line.includes("condition=Established") && line.includes("t4clusterhosts.cluster.t4.dev") && line.includes("t4workspaces.cluster.t4.dev") && line.includes("t4sessions.cluster.t4.dev"), "Established wait"); + const servedChecks = log.map((line, index) => ({ line, index })).filter(({ line }) => line.startsWith("validator\tserved\t")); + const admissionPreflight = findCommand(log, (line) => line.includes("apply") && line.includes("--dry-run=server") && line.includes("testdata/compat"), "converged admission preflight"); const storageChecks = log.map((line, index) => ({ line, index })).filter(({ line }) => line.startsWith("kubectl\tget\tcrd/") && line.includes("status.storedVersions")); assert.deepEqual(storageChecks.length, 3); + assert.equal(servedChecks.length, 3); const workload = findCommand(log, (line) => line.startsWith("helm\tupgrade\t"), "Helm workload upgrade"); - assert.ok(crdPreflight < oldObjectPreflight); - assert.ok(oldObjectPreflight < crdApply); + assert.ok(proposedPreflight < crdPreflight); + assert.ok(crdPreflight < crdApply); assert.ok(crdApply < established); - assert.ok(established < storageChecks[0].index); + assert.ok(established < servedChecks[0].index); + assert.ok(servedChecks.at(-1).index < admissionPreflight); + assert.ok(admissionPreflight < storageChecks[0].index); assert.ok(storageChecks.every(({ index }) => index < workload)); assert.ok(log.every((line) => !line.includes("--force") && !line.includes("replace") && !line.includes("delete\tcrd")), log.join("\n")); }); From 20e2384a93caa0e13c680181c9536f7aac6071f7 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 20:11:07 +0000 Subject: [PATCH 075/131] test(operator): account for local preflight --- scripts/cluster-ci/crd-lifecycle.test.mjs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/cluster-ci/crd-lifecycle.test.mjs b/scripts/cluster-ci/crd-lifecycle.test.mjs index d92b9923..4e0eb812 100644 --- a/scripts/cluster-ci/crd-lifecycle.test.mjs +++ b/scripts/cluster-ci/crd-lifecycle.test.mjs @@ -183,8 +183,10 @@ test("failed server preflight leaves CRDs and workloads untouched", async () => ); assert.notEqual(result.code, 0); const log = await commands(value.log); - assert.equal(log.length, 1, log.join("\n")); - assert.match(log[0], /apply.*--server-side.*--dry-run=server/u); + assert.equal(log.length, 2, log.join("\n")); + assert.match(log[0], /^validator\tfixtures\t/u); + assert.match(log[1], /apply.*--server-side.*--dry-run=server/u); + assert.ok(log.every((line) => !line.startsWith("helm\t") && !/kubectl\tapply(?!.*--dry-run=server)/u.test(line)), log.join("\n")); }); test("an unexpected stored version stops workload rollout", async () => { From 721c84c64432005094c05a1778b5b5c4f236520e Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 20:15:51 +0000 Subject: [PATCH 076/131] ci(operator): emit validator module metadata --- .github/workflows/ci.yml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c8872e37..0ef4192b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -299,6 +299,40 @@ jobs: - name: Lint cluster chart run: helm lint deploy/charts/t4-cluster + op02-crd-validator-dependencies: + if: ${{ github.event_name == 'pull_request' }} + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Check out exact pull request head + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Install repository Go toolchain + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 + with: + go-version: "1.25.5" + cache: false + + - name: Resolve CRD validator dependencies + working-directory: packages/cluster-operator + run: | + go get k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel@v0.33.2 + go get k8s.io/apiextensions-apiserver/pkg/apiserver/validation@v0.33.2 + go get k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder@v0.33.2 + + - name: Upload authorized module metadata + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: op02-crd-validator-modules-${{ github.run_id }} + path: | + packages/cluster-operator/go.mod + packages/cluster-operator/go.sum + if-no-files-found: error + retention-days: 1 + tooling: needs: changes if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.tooling == 'true' }} From 7b1affaf6b3dd1d9c2bc370cd791b84b8ed05c64 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 20:21:48 +0000 Subject: [PATCH 077/131] ci(operator): freeze validator dependencies --- .github/workflows/ci.yml | 36 +----------- .woodpecker.yml | 4 +- packages/cluster-operator/go.mod | 28 +++++++++ packages/cluster-operator/go.sum | 57 +++++++++++++++++++ .../cluster-ci/cluster-ci-contract.test.mjs | 4 +- 5 files changed, 90 insertions(+), 39 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0ef4192b..a0bc7761 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -294,45 +294,11 @@ jobs: - name: Run operator tests working-directory: packages/cluster-operator - run: go test ./... + run: GOFLAGS=-mod=readonly go test ./... - name: Lint cluster chart run: helm lint deploy/charts/t4-cluster - op02-crd-validator-dependencies: - if: ${{ github.event_name == 'pull_request' }} - runs-on: ubuntu-24.04 - timeout-minutes: 10 - steps: - - name: Check out exact pull request head - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - name: Install repository Go toolchain - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 - with: - go-version: "1.25.5" - cache: false - - - name: Resolve CRD validator dependencies - working-directory: packages/cluster-operator - run: | - go get k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel@v0.33.2 - go get k8s.io/apiextensions-apiserver/pkg/apiserver/validation@v0.33.2 - go get k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder@v0.33.2 - - - name: Upload authorized module metadata - if: ${{ always() }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: op02-crd-validator-modules-${{ github.run_id }} - path: | - packages/cluster-operator/go.mod - packages/cluster-operator/go.sum - if-no-files-found: error - retention-days: 1 - tooling: needs: changes if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.tooling == 'true' }} diff --git a/.woodpecker.yml b/.woodpecker.yml index 4f8c556e..69673f6f 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -250,9 +250,9 @@ steps: image: mirror.gcr.io/library/golang:1.25.1-bookworm@sha256:c423747fbd96fd8f0b1102d947f51f9b266060217478e5f9bf86f145969562ee directory: packages/cluster-operator commands: - - GOMAXPROCS=1 GOFLAGS=-p=1 go test ./api/... ./controllers/... ./cmd/... + - GOMAXPROCS=1 GOFLAGS='-mod=readonly -p=1' go test ./api/... ./controllers/... ./cmd/... - mkdir -p ../../artifacts/cluster-proof - - CGO_ENABLED=0 GOMAXPROCS=1 GOFLAGS=-p=1 go test -c ./charttests -o ../../artifacts/cluster-proof/chart-contract.test + - CGO_ENABLED=0 GOMAXPROCS=1 GOFLAGS='-mod=readonly -p=1' go test -c ./charttests -o ../../artifacts/cluster-proof/chart-contract.test backend_options: kubernetes: serviceAccountName: woodpecker-ci-untrusted diff --git a/packages/cluster-operator/go.mod b/packages/cluster-operator/go.mod index c85639c9..f4504858 100644 --- a/packages/cluster-operator/go.mod +++ b/packages/cluster-operator/go.mod @@ -14,23 +14,32 @@ require ( ) require ( + cel.dev/expr v0.19.1 // indirect + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.23.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/google/btree v1.1.3 // indirect + github.com/google/cel-go v0.23.2 // indirect github.com/google/gnostic-models v0.6.9 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -41,10 +50,22 @@ require ( github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect + github.com/spf13/cobra v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stoewer/go-strcase v1.3.0 // indirect github.com/x448/float16 v0.8.4 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect + go.opentelemetry.io/otel v1.33.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 // indirect + go.opentelemetry.io/otel/metric v1.33.0 // indirect + go.opentelemetry.io/otel/sdk v1.33.0 // indirect + go.opentelemetry.io/otel/trace v1.33.0 // indirect + go.opentelemetry.io/proto/otlp v1.4.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect golang.org/x/net v0.38.0 // indirect golang.org/x/oauth2 v0.27.0 // indirect golang.org/x/sync v0.12.0 // indirect @@ -53,12 +74,19 @@ require ( golang.org/x/text v0.23.0 // indirect golang.org/x/time v0.9.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect + google.golang.org/grpc v1.68.1 // indirect google.golang.org/protobuf v1.36.5 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiserver v0.33.2 // indirect + k8s.io/component-base v0.33.2 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect diff --git a/packages/cluster-operator/go.sum b/packages/cluster-operator/go.sum index e291ac10..6883ecd7 100644 --- a/packages/cluster-operator/go.sum +++ b/packages/cluster-operator/go.sum @@ -1,9 +1,16 @@ +cel.dev/expr v0.19.1 h1:NciYrtDRIR0lNCnH1LFJegdjspNx9fI59O7TWcua/W4= +cel.dev/expr v0.19.1/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= +github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= +github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -14,12 +21,17 @@ github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8 github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= @@ -36,6 +48,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/cel-go v0.23.2 h1:UdEe3CvQh3Nv+E/j9r1Y//WO0K0cSyD7/y0bzyLIMI4= +github.com/google/cel-go v0.23.2/go.mod h1:52Pb6QsDbC5kvgxvZhiL9QX1oZEkcUF/ZqaPx1J5Wwo= github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -48,6 +62,11 @@ github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgY github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -92,8 +111,13 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -109,6 +133,24 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= +go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw= +go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0/go.mod h1:cpgtDBaqD/6ok/UG0jT15/uKjAY8mRA53diogHBg3UI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 h1:5pojmb1U1AogINhN3SurB+zm/nIcusopeBNp42f45QM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0/go.mod h1:57gTHJSE5S1tqg+EKsLPlTWhpHMsWlVmer+LA926XiA= +go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ= +go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M= +go.opentelemetry.io/otel/sdk v1.33.0 h1:iax7M131HuAm9QkZotNHEfstof92xM+N8sr3uHXc2IM= +go.opentelemetry.io/otel/sdk v1.33.0/go.mod h1:A1Q5oi7/9XaMlIWzPSxLRWOI8nG3FnzHJNbiENQuihM= +go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s= +go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck= +go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg= +go.opentelemetry.io/proto/otlp v1.4.0/go.mod h1:PPBWZIP98o2ElSqI35IHfu7hIhSwvc5N38Jw8pXuGFY= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -118,6 +160,8 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -158,6 +202,13 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ= +google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 h1:CkkIfIt50+lT6NHAVoRYEyAvQGFM7xEwXUUywFvEb3Q= +google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 h1:8ZmaLZE4XWrtU3MyClkYqqtl6Oegr3235h7jxsDyqCY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= +google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0= +google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw= google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -176,14 +227,20 @@ k8s.io/apiextensions-apiserver v0.33.2 h1:6gnkIbngnaUflR3XwE1mCefN3YS8yTD631JXQh k8s.io/apiextensions-apiserver v0.33.2/go.mod h1:IvVanieYsEHJImTKXGP6XCOjTwv2LUMos0YWc9O+QP8= k8s.io/apimachinery v0.33.2 h1:IHFVhqg59mb8PJWTLi8m1mAoepkUNYmptHsV+Z1m5jY= k8s.io/apimachinery v0.33.2/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= +k8s.io/apiserver v0.33.2 h1:KGTRbxn2wJagJowo29kKBp4TchpO1DRO3g+dB/KOJN4= +k8s.io/apiserver v0.33.2/go.mod h1:9qday04wEAMLPWWo9AwqCZSiIn3OYSZacDyu/AcoM/M= k8s.io/client-go v0.33.2 h1:z8CIcc0P581x/J1ZYf4CNzRKxRvQAwoAolYPbtQes+E= k8s.io/client-go v0.33.2/go.mod h1:9mCgT4wROvL948w6f6ArJNb7yQd7QsvqavDeZHvNmHo= +k8s.io/component-base v0.33.2 h1:sCCsn9s/dG3ZrQTX/Us0/Sx2R0G5kwa0wbZFYoVp/+0= +k8s.io/component-base v0.33.2/go.mod h1:/41uw9wKzuelhN+u+/C59ixxf4tYQKW7p32ddkYNe2k= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8= sigs.k8s.io/controller-runtime v0.21.0/go.mod h1:OSg14+F65eWqIu4DceX7k/+QRAbTTvxeQSNSOQpukWM= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= diff --git a/scripts/cluster-ci/cluster-ci-contract.test.mjs b/scripts/cluster-ci/cluster-ci-contract.test.mjs index 4c240a34..ccbf21b5 100644 --- a/scripts/cluster-ci/cluster-ci-contract.test.mjs +++ b/scripts/cluster-ci/cluster-ci-contract.test.mjs @@ -309,9 +309,9 @@ test("Woodpecker keeps upstream gates and serializes bounded cluster publication ); assert.ok(steps["cluster-ci-contracts"].commands.includes("pnpm test:cluster:ci")); assert.deepEqual(steps["cluster-operator-tests"].commands, [ - "GOMAXPROCS=1 GOFLAGS=-p=1 go test ./api/... ./controllers/... ./cmd/...", + "GOMAXPROCS=1 GOFLAGS='-mod=readonly -p=1' go test ./api/... ./controllers/... ./cmd/...", "mkdir -p ../../artifacts/cluster-proof", - "CGO_ENABLED=0 GOMAXPROCS=1 GOFLAGS=-p=1 go test -c ./charttests -o ../../artifacts/cluster-proof/chart-contract.test", + "CGO_ENABLED=0 GOMAXPROCS=1 GOFLAGS='-mod=readonly -p=1' go test -c ./charttests -o ../../artifacts/cluster-proof/chart-contract.test", ]); assert.equal( steps["cluster-operator-tests"].backend_options.kubernetes.resources.limits.memory, From 1b84fa7a5f2f23b31575b03d67bb7bc901231708 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 20:26:30 +0000 Subject: [PATCH 078/131] ci(operator): preserve ordinary Go gate --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a0bc7761..7edb4c73 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -294,7 +294,9 @@ jobs: - name: Run operator tests working-directory: packages/cluster-operator - run: GOFLAGS=-mod=readonly go test ./... + env: + GOFLAGS: -mod=readonly + run: go test ./... - name: Lint cluster chart run: helm lint deploy/charts/t4-cluster From e27f3f9d16085e426064ed95c2d5a662d0520d11 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 20:43:48 +0000 Subject: [PATCH 079/131] test(operator): expose transition CEL gap --- .../cmd/crd-preflight/main_test.go | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/packages/cluster-operator/cmd/crd-preflight/main_test.go b/packages/cluster-operator/cmd/crd-preflight/main_test.go index 811a0c6f..c61c41d1 100644 --- a/packages/cluster-operator/cmd/crd-preflight/main_test.go +++ b/packages/cluster-operator/cmd/crd-preflight/main_test.go @@ -93,6 +93,57 @@ status: } } +func TestValidateFixturesRejectsUnchangedLegacyValuesUnderTransitionCEL(t *testing.T) { + tests := []struct { + name string + fixture string + candidate func(string) string + }{ + { + name: "spec transition rule", + fixture: `apiVersion: cluster.t4.dev/v1alpha1 +kind: Widget +metadata: + name: legacy +spec: + code: bad +status: + phase: Ready +`, + candidate: func(crd string) string { + crd = strings.Replace(crd, "rule: self.code.startsWith('ok')", `rule: "true"`, 1) + return strings.Replace(crd, " maxLength: 3", " maxLength: 3\n x-kubernetes-validations:\n - rule: oldSelf.startsWith('ok')", 1) + }, + }, + { + name: "status transition rule", + fixture: `apiVersion: cluster.t4.dev/v1alpha1 +kind: Widget +metadata: + name: legacy +spec: + code: ok +status: + phase: Pending +`, + candidate: func(crd string) string { + return strings.Replace(crd, " enum: [Ready]", " enum: [Ready, Pending]\n x-kubernetes-validations:\n - rule: oldSelf == 'Ready'", 1) + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + crds, fixtures := writeCandidate(t, test.fixture) + if err := os.WriteFile(filepath.Join(crds, "widget.yaml"), []byte(test.candidate(candidateCRD)), 0o644); err != nil { + t.Fatal(err) + } + if err := validateFixtures(crds, fixtures); err == nil { + t.Fatal("unchanged persisted value blocked by transition CEL was accepted") + } + }) + } +} + func TestVerifyServedSchemasRejectsRetainedEstablishedWithStaleSchema(t *testing.T) { crds, _ := writeCandidate(t, `apiVersion: cluster.t4.dev/v1alpha1 kind: Widget From 8bd18a14da17c461770adc1bd99894b3a3214813 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 20:49:50 +0000 Subject: [PATCH 080/131] fix(operator): validate unchanged CEL transitions --- .../cmd/crd-preflight/main.go | 11 ++++-- .../cmd/crd-preflight/main_test.go | 37 +++++++++++++------ 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/packages/cluster-operator/cmd/crd-preflight/main.go b/packages/cluster-operator/cmd/crd-preflight/main.go index 9751e282..20399616 100644 --- a/packages/cluster-operator/cmd/crd-preflight/main.go +++ b/packages/cluster-operator/cmd/crd-preflight/main.go @@ -115,9 +115,14 @@ func validateObject(path string, object map[string]interface{}, candidate *candi } celValidator := cel.NewValidator(candidate.structural, true, perCallCELCostLimit) if celValidator != nil { - errs, _ := celValidator.Validate(context.Background(), field.NewPath("fixture"), candidate.structural, object, nil, runtimeCELCostBudget) - if len(errs) > 0 { - result = append(result, fmt.Errorf("%s: proposed CEL validation failed: %w", path, errs.ToAggregate())) + fixturePath := field.NewPath("fixture") + createErrors, _ := celValidator.Validate(context.Background(), fixturePath, candidate.structural, object, nil, runtimeCELCostBudget) + if len(createErrors) > 0 { + result = append(result, fmt.Errorf("%s: proposed CEL create validation failed: %w", path, createErrors.ToAggregate())) + } + updateErrors, _ := celValidator.Validate(context.Background(), fixturePath, candidate.structural, object, object, runtimeCELCostBudget) + if len(updateErrors) > 0 { + result = append(result, fmt.Errorf("%s: proposed CEL unchanged-update validation failed: %w", path, updateErrors.ToAggregate())) } } unknownFields := pruning.PruneWithOptions( diff --git a/packages/cluster-operator/cmd/crd-preflight/main_test.go b/packages/cluster-operator/cmd/crd-preflight/main_test.go index c61c41d1..eef150a0 100644 --- a/packages/cluster-operator/cmd/crd-preflight/main_test.go +++ b/packages/cluster-operator/cmd/crd-preflight/main_test.go @@ -56,11 +56,12 @@ spec: func TestValidateFixturesRejectsProposedSpecTighteningAndCEL(t *testing.T) { for _, test := range []struct { - name string - code string + name string + code string + expectedError string }{ - {name: "openapi maxLength", code: "okay"}, - {name: "CEL rule", code: "bad"}, + {name: "openapi maxLength", code: "okay", expectedError: "proposed OpenAPI validation failed"}, + {name: "CEL rule", code: "bad", expectedError: "proposed CEL create validation failed"}, } { t.Run(test.name, func(t *testing.T) { crds, fixtures := writeCandidate(t, `apiVersion: cluster.t4.dev/v1alpha1 @@ -71,9 +72,13 @@ spec: code: `+test.code+"\n"+`status: phase: Ready `) - if err := validateFixtures(crds, fixtures); err == nil { + err := validateFixtures(crds, fixtures) + if err == nil { t.Fatal("fixture incompatible with the proposed spec schema was accepted") } + if !strings.Contains(err.Error(), test.expectedError) { + t.Fatalf("validation error %q does not identify %q", err, test.expectedError) + } }) } } @@ -95,12 +100,14 @@ status: func TestValidateFixturesRejectsUnchangedLegacyValuesUnderTransitionCEL(t *testing.T) { tests := []struct { - name string - fixture string - candidate func(string) string + name string + fixture string + expectedPath string + candidate func(string) string }{ { - name: "spec transition rule", + name: "spec transition rule", + expectedPath: "fixture.spec.code", fixture: `apiVersion: cluster.t4.dev/v1alpha1 kind: Widget metadata: @@ -116,7 +123,8 @@ status: }, }, { - name: "status transition rule", + name: "status transition rule", + expectedPath: "fixture.status.phase", fixture: `apiVersion: cluster.t4.dev/v1alpha1 kind: Widget metadata: @@ -137,9 +145,16 @@ status: if err := os.WriteFile(filepath.Join(crds, "widget.yaml"), []byte(test.candidate(candidateCRD)), 0o644); err != nil { t.Fatal(err) } - if err := validateFixtures(crds, fixtures); err == nil { + err := validateFixtures(crds, fixtures) + if err == nil { t.Fatal("unchanged persisted value blocked by transition CEL was accepted") } + if !strings.Contains(err.Error(), "proposed CEL unchanged-update validation failed") { + t.Fatalf("validation error does not identify unchanged-update semantics: %v", err) + } + if !strings.Contains(err.Error(), test.expectedPath) { + t.Fatalf("validation error %q does not identify field %q", err, test.expectedPath) + } }) } } From 1a207e1785f1cffeb76efc66b373b7b71cab072f Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 03:47:00 +0000 Subject: [PATCH 081/131] test(operator): expose live CR narrowing gap --- .../cmd/crd-preflight/main_test.go | 93 +++++++++++++++++++ scripts/cluster-ci/crd-lifecycle.test.mjs | 45 ++++++++- 2 files changed, 137 insertions(+), 1 deletion(-) diff --git a/packages/cluster-operator/cmd/crd-preflight/main_test.go b/packages/cluster-operator/cmd/crd-preflight/main_test.go index eef150a0..1789f16f 100644 --- a/packages/cluster-operator/cmd/crd-preflight/main_test.go +++ b/packages/cluster-operator/cmd/crd-preflight/main_test.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "os" "path/filepath" "strings" @@ -159,6 +160,98 @@ status: } } +func TestValidateObjectsRejectsLiveDataOmittedFromFixtures(t *testing.T) { + tests := []struct { + name string + candidate func(string) string + fixtureCode string + fixturePhase string + liveCode string + livePhase string + liveExtra string + expectedError string + }{ + { + name: "spec OpenAPI narrowing", + candidate: func(crd string) string { + return strings.Replace(crd, " maxLength: 3", " maxLength: 3\n enum: [ok1]", 1) + }, + fixtureCode: "ok1", fixturePhase: "Ready", liveCode: "ok2", livePhase: "Ready", + expectedError: "proposed OpenAPI validation failed", + }, + { + name: "status OpenAPI narrowing", + candidate: func(crd string) string { return crd }, + fixtureCode: "ok", fixturePhase: "Ready", liveCode: "ok", livePhase: "Legacy", + expectedError: "proposed OpenAPI validation failed", + }, + { + name: "spec CEL create semantics", + candidate: func(crd string) string { return crd }, + fixtureCode: "ok", fixturePhase: "Ready", liveCode: "bad", livePhase: "Ready", + expectedError: "proposed CEL create validation failed", + }, + { + name: "status CEL create semantics", + candidate: func(crd string) string { + return strings.Replace(crd, " phase:\n type: string\n enum: [Ready]", " phase:\n type: string\n enum: [Ready, Legacy]\n x-kubernetes-validations:\n - rule: self == 'Ready'", 1) + }, + fixtureCode: "ok", fixturePhase: "Ready", liveCode: "ok", livePhase: "Legacy", + expectedError: "proposed CEL create validation failed", + }, + { + name: "spec CEL unchanged-update semantics", + candidate: func(crd string) string { + crd = strings.Replace(crd, "rule: self.code.startsWith('ok')", `rule: "true"`, 1) + return strings.Replace(crd, " maxLength: 3", " maxLength: 3\n x-kubernetes-validations:\n - rule: oldSelf.startsWith('ok')", 1) + }, + fixtureCode: "ok", fixturePhase: "Ready", liveCode: "bad", livePhase: "Ready", + expectedError: "proposed CEL unchanged-update validation failed", + }, + { + name: "status CEL unchanged-update semantics", + candidate: func(crd string) string { + return strings.Replace(crd, " enum: [Ready]", " enum: [Ready, Legacy]\n x-kubernetes-validations:\n - rule: oldSelf == 'Ready'", 1) + }, + fixtureCode: "ok", fixturePhase: "Ready", liveCode: "ok", livePhase: "Legacy", + expectedError: "proposed CEL unchanged-update validation failed", + }, + { + name: "spec pruning", + candidate: func(crd string) string { return crd }, + fixtureCode: "ok", fixturePhase: "Ready", liveCode: "ok", livePhase: "Ready", liveExtra: `,"removedSpec":"legacy"`, + expectedError: "proposed structural schema would prune declared fields", + }, + { + name: "status pruning", + candidate: func(crd string) string { return crd }, + fixtureCode: "ok", fixturePhase: "Ready", liveCode: "ok", livePhase: "Ready", liveExtra: `,"removedStatus":"legacy"`, + expectedError: "proposed structural schema would prune declared fields", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fixture := fmt.Sprintf("apiVersion: cluster.t4.dev/v1alpha1\nkind: Widget\nmetadata:\n name: curated\n namespace: tenant-a\nspec:\n code: %s\nstatus:\n phase: %s\n", test.fixtureCode, test.fixturePhase) + crds, fixtures := writeCandidate(t, fixture) + if err := os.WriteFile(filepath.Join(crds, "widget.yaml"), []byte(test.candidate(candidateCRD)), 0o644); err != nil { + t.Fatal(err) + } + if err := validateFixtures(crds, fixtures); err != nil { + t.Fatalf("curated fixture should not expose the live-only incompatibility: %v", err) + } + live := fmt.Sprintf(`{"apiVersion":"v1","kind":"WidgetList","items":[{"apiVersion":"cluster.t4.dev/v1alpha1","kind":"Widget","metadata":{"name":"live","namespace":"tenant-b"},"spec":{"code":%q%s},"status":{"phase":%q%s}}]}`, + test.liveCode, map[bool]string{true: test.liveExtra, false: ""}[test.name == "spec pruning"], test.livePhase, map[bool]string{true: test.liveExtra, false: ""}[test.name == "status pruning"]) + err := validateObjects(crds, strings.NewReader(live)) + if err == nil { + t.Fatal("live object incompatible with the proposed schema was accepted") + } + if !strings.Contains(err.Error(), "tenant-b/live") || !strings.Contains(err.Error(), test.expectedError) { + t.Fatalf("validation error %q does not identify live object and %q", err, test.expectedError) + } + }) + } +} + func TestVerifyServedSchemasRejectsRetainedEstablishedWithStaleSchema(t *testing.T) { crds, _ := writeCandidate(t, `apiVersion: cluster.t4.dev/v1alpha1 kind: Widget diff --git a/scripts/cluster-ci/crd-lifecycle.test.mjs b/scripts/cluster-ci/crd-lifecycle.test.mjs index 4e0eb812..f856bd92 100644 --- a/scripts/cluster-ci/crd-lifecycle.test.mjs +++ b/scripts/cluster-ci/crd-lifecycle.test.mjs @@ -26,6 +26,13 @@ for argument in "$@"; do fi done if [ "\${1:-}" = get ]; then + case "\${2:-}" in + t4clusterhosts.cluster.t4.dev|t4workspaces.cluster.t4.dev|t4sessions.cluster.t4.dev) + if [ "\${FAIL_LIVE_LIST:-}" = "\${2:-}" ]; then exit 44; fi + printf '%s' '{"apiVersion":"v1","kind":"List","items":[]}' + exit 0 + ;; + esac printf '%s' "\${STORED_VERSIONS:-v1alpha1}" fi `, @@ -38,7 +45,7 @@ printf 'validator' >>"$COMMAND_LOG" for argument in "$@"; do printf '\\t%s' "$argument" >>"$COMMAND_LOG"; done printf '\\n' >>"$COMMAND_LOG" case "\${1:-}:\${FAIL_PROPOSED_VALIDATION:-}" in - fixtures:spec|fixtures:status|served:stale) exit 43 ;; + fixtures:spec|fixtures:status|objects:live|served:stale) exit 43 ;; esac cat >/dev/null `, @@ -102,6 +109,8 @@ test("upgrade validates proposed schemas, proves served convergence, verifies st assert.equal(result.code, 0, `${result.stdout}\n${result.stderr}`); const log = await commands(value.log); const proposedPreflight = findCommand(log, (line) => line.startsWith("validator\tfixtures\t"), "local proposed-schema fixture preflight"); + const liveLists = log.map((line, index) => ({ line, index })).filter(({ line }) => /^kubectl\tget\tt4(?:clusterhosts|workspaces|sessions)\.cluster\.t4\.dev\t--all-namespaces\t-o\tjson$/u.test(line)); + const liveValidations = log.map((line, index) => ({ line, index })).filter(({ line }) => line.startsWith("validator\tobjects\t")); const crdPreflight = findCommand(log, (line) => line.includes("apply") && line.includes("--server-side") && line.includes("--dry-run=server") && line.includes("deploy/charts/t4-cluster/crds"), "server-side CRD preflight"); const crdApply = findCommand(log, (line) => line.includes("apply") && line.includes("--server-side") && !line.includes("--dry-run=server"), "CRD apply"); const established = findCommand(log, (line) => line.includes("wait") && line.includes("condition=Established") && line.includes("t4clusterhosts.cluster.t4.dev") && line.includes("t4workspaces.cluster.t4.dev") && line.includes("t4sessions.cluster.t4.dev"), "Established wait"); @@ -110,8 +119,16 @@ test("upgrade validates proposed schemas, proves served convergence, verifies st const storageChecks = log.map((line, index) => ({ line, index })).filter(({ line }) => line.startsWith("kubectl\tget\tcrd/") && line.includes("status.storedVersions")); assert.deepEqual(storageChecks.length, 3); assert.equal(servedChecks.length, 3); + assert.equal(liveLists.length, 3); + assert.equal(liveValidations.length, 3); const workload = findCommand(log, (line) => line.startsWith("helm\tupgrade\t"), "Helm workload upgrade"); assert.ok(proposedPreflight < crdPreflight); + assert.ok(liveLists[0].index < liveValidations[0].index); + assert.ok(liveValidations[0].index < liveLists[1].index); + assert.ok(liveLists[1].index < liveValidations[1].index); + assert.ok(liveValidations[1].index < liveLists[2].index); + assert.ok(liveLists[2].index < liveValidations[2].index); + assert.ok(liveValidations[2].index < crdPreflight); assert.ok(crdPreflight < crdApply); assert.ok(crdApply < established); assert.ok(established < servedChecks[0].index); @@ -160,6 +177,32 @@ test("persisted status is validated against the proposed status schema before mu ]); }); +test("live object incompatibility fails before non-dry-run CRD, object, or Helm mutation", async () => { + const value = await fixture(); + const result = await runLifecycle( + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "chart", "--skip-crds"], + { ...value.env, FAIL_PROPOSED_VALIDATION: "live" }, + ); + assert.notEqual(result.code, 0); + const log = await commands(value.log); + assert.match(log.at(-1), /^validator\tobjects\t/u); + assert.ok(log.some((line) => /^kubectl\tget\tt4clusterhosts\.cluster\.t4\.dev\t--all-namespaces\t-o\tjson$/u.test(line)), log.join("\n")); + assert.ok(log.every((line) => !line.startsWith("helm\t") && !/kubectl\tapply(?!.*--dry-run=server)/u.test(line)), log.join("\n")); +}); + +test("live object list denial fails closed before non-dry-run mutation", async () => { + const value = await fixture(); + const deniedResource = "t4workspaces.cluster.t4.dev"; + const result = await runLifecycle( + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "chart", "--skip-crds"], + { ...value.env, FAIL_LIVE_LIST: deniedResource }, + ); + assert.notEqual(result.code, 0); + const log = await commands(value.log); + assert.ok(log.some((line) => line.startsWith(`kubectl\tget\t${deniedResource}\t--all-namespaces\t-o\tjson`)), log.join("\n")); + assert.ok(log.every((line) => !line.startsWith("helm\t") && !/kubectl\tapply(?!.*--dry-run=server)/u.test(line)), log.join("\n")); +}); + test("retained Established cannot pass readiness while served OpenAPI is stale", async () => { const value = await fixture(); const result = await runLifecycle( From de9bf6f118476f1630b005544aefb64bc5264170 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 03:56:17 +0000 Subject: [PATCH 082/131] fix(operator): validate every live CR before apply --- docs/CLUSTER_OPERATOR.md | 33 +++++-- .../cmd/crd-preflight/main.go | 97 ++++++++++++++++--- .../cmd/crd-preflight/main_test.go | 19 ++++ scripts/cluster-ci/crd-lifecycle.sh | 27 +++++- scripts/cluster-ci/crd-lifecycle.test.mjs | 58 +++++++++-- 5 files changed, 201 insertions(+), 33 deletions(-) diff --git a/docs/CLUSTER_OPERATOR.md b/docs/CLUSTER_OPERATOR.md index bd38244d..aae439e4 100644 --- a/docs/CLUSTER_OPERATOR.md +++ b/docs/CLUSTER_OPERATOR.md @@ -23,7 +23,7 @@ That declaration does not make a non-RWX backend safe. Missing classes, classes ## Install -Installing with defaults creates no controller, gateway, session workload, RBAC, Secret, or network policy. Use the lifecycle runner even for a fresh install so the CRDs are server-validated, established, fixture-checked, and storage-version-checked before Helm runs: +Installing with defaults creates no controller, gateway, session workload, RBAC, Secret, or network policy. Use the lifecycle runner even for a fresh install so existing live objects and compatibility fixtures are checked against the proposed schemas before the CRDs are server-validated, established, and storage-version-checked and before Helm runs: ```sh scripts/cluster-ci/crd-lifecycle.sh install -- \ @@ -163,19 +163,36 @@ scripts/cluster-ci/crd-lifecycle.sh upgrade -- \ The command performs this exact order: 1. Before any cluster access, validate every complete compatibility fixture (including `status`) against the proposed structural OpenAPI schema and CEL programs with the Kubernetes apiextensions validators. The check also rejects declared fields that the structural pruner would remove. -2. Server-side dry-run the reviewed CRDs with strict validation. Upgrades stop before any mutation if either local proposed-schema validation or the CRD dry-run fails. -3. Apply the CRDs server-side without force conflicts. -4. Wait for all three CRDs to report `Established`. -5. Fetch the served `cluster.t4.dev/v1alpha1` OpenAPI v3 document three times and require every published resource schema to match the semantics generated from the proposed CRDs. Retained `Established=True` is not readiness when discovery still serves an old schema. -6. Server-side dry-run the compatibility fixtures against the converged admission path. -7. Require each CRD's `status.storedVersions` to be exactly `v1alpha1`. -8. Execute the supplied Helm command, which must use `--skip-crds`. +2. With `get` access to the three exact CRD definitions and `list` access to the corresponding namespaced T4 resources, detect which definitions already exist, enumerate every live CR for each existing definition across namespaces, and validate each complete object, including `status`, against the same proposed OpenAPI, CEL create, CEL unchanged-update, and pruning checks. A denied CRD read, denied or malformed object list, or incompatible object fails closed; an absent definition is safely empty on a fresh install. +3. Server-side dry-run the reviewed CRDs with strict validation. Upgrades stop before any mutation if a local proposed-schema validation, live-object enumeration, or CRD dry-run fails. +4. Apply the CRDs server-side without force conflicts. +5. Wait for all three CRDs to report `Established`. +6. Fetch the served `cluster.t4.dev/v1alpha1` OpenAPI v3 document three times and require every published resource schema to match the semantics generated from the proposed CRDs. Retained `Established=True` is not readiness when discovery still serves an old schema. +7. Server-side dry-run the compatibility fixtures against the converged admission path. +8. Require each CRD's `status.storedVersions` to be exactly `v1alpha1`. +9. Execute the supplied Helm command, which must use `--skip-crds`. The corresponding administrative checks are executable independently: ```sh (cd packages/cluster-operator && \ go run ./cmd/crd-preflight fixtures ../../deploy/charts/t4-cluster/crds api/v1alpha1/testdata/compat) +live_objects=$(mktemp) +trap 'rm -f "$live_objects"' EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM +for resource in t4clusterhosts.cluster.t4.dev t4workspaces.cluster.t4.dev t4sessions.cluster.t4.dev; do + installed_crd=$(kubectl get "crd/$resource" --ignore-not-found -o name) + if [ -z "$installed_crd" ]; then + continue + fi + kubectl get "$resource" --all-namespaces -o json >"$live_objects" + (cd packages/cluster-operator && \ + go run ./cmd/crd-preflight objects ../../deploy/charts/t4-cluster/crds <"$live_objects") +done +rm -f "$live_objects" +trap - EXIT HUP INT TERM kubectl apply --server-side --dry-run=server --validate=strict \ --field-manager=t4-crd-lifecycle -f deploy/charts/t4-cluster/crds/ kubectl apply --server-side --validate=strict \ diff --git a/packages/cluster-operator/cmd/crd-preflight/main.go b/packages/cluster-operator/cmd/crd-preflight/main.go index 20399616..0a55241b 100644 --- a/packages/cluster-operator/cmd/crd-preflight/main.go +++ b/packages/cluster-operator/cmd/crd-preflight/main.go @@ -19,6 +19,7 @@ import ( apiservervalidation "k8s.io/apiextensions-apiserver/pkg/apiserver/validation" "k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder" "k8s.io/apimachinery/pkg/runtime" + kubejson "k8s.io/apimachinery/pkg/util/json" "k8s.io/apimachinery/pkg/util/validation/field" "sigs.k8s.io/yaml" ) @@ -52,6 +53,11 @@ func main() { usage() } err = validateFixtures(os.Args[2], os.Args[3]) + case "objects": + if len(os.Args) != 3 { + usage() + } + err = validateObjects(os.Args[2], os.Stdin) case "served": if len(os.Args) != 3 { usage() @@ -67,7 +73,7 @@ func main() { } func usage() { - fmt.Fprintln(os.Stderr, "usage: crd-preflight fixtures CRD_DIRECTORY FIXTURE_DIRECTORY | served CRD_DIRECTORY") + fmt.Fprintln(os.Stderr, "usage: crd-preflight fixtures CRD_DIRECTORY FIXTURE_DIRECTORY | objects CRD_DIRECTORY | served CRD_DIRECTORY") os.Exit(64) } @@ -87,40 +93,101 @@ func validateFixtures(crdDirectory, fixtureDirectory string) error { validationErrors = append(validationErrors, err) continue } - apiVersion, _ := object["apiVersion"].(string) - kind, _ := object["kind"].(string) - group, version, ok := strings.Cut(apiVersion, "/") - if !ok || group == "" || version == "" || kind == "" { - validationErrors = append(validationErrors, fmt.Errorf("%s: apiVersion and kind must identify a grouped resource", path)) + validationErrors = append(validationErrors, validateCandidateObject(path, "fixture", object, candidates)...) + } + return errors.Join(validationErrors...) +} + +func validateObjects(crdDirectory string, input io.Reader) error { + candidates, err := loadCandidates(crdDirectory) + if err != nil { + return err + } + var list map[string]interface{} + decoder := json.NewDecoder(input) + decoder.UseNumber() + if err := decoder.Decode(&list); err != nil { + return fmt.Errorf("decode live object list: %w", err) + } + if err := kubejson.ConvertMapNumbers(list, 0); err != nil { + return fmt.Errorf("decode live object list numbers: %w", err) + } + var trailing interface{} + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + if err == nil { + return errors.New("decode live object list: multiple JSON documents") + } + return fmt.Errorf("decode live object list trailing data: %w", err) + } + rawItems, found := list["items"] + if !found { + return errors.New("decode live object list: missing items") + } + items, ok := rawItems.([]interface{}) + if !ok { + return errors.New("decode live object list: items is not an array") + } + var validationErrors []error + for index, rawItem := range items { + object, ok := rawItem.(map[string]interface{}) + if !ok { + validationErrors = append(validationErrors, fmt.Errorf("live object item %d is not an object", index)) continue } - candidate, found := candidates[groupVersionKind{Group: group, Version: version, Kind: kind}] - if !found { - validationErrors = append(validationErrors, fmt.Errorf("%s: no proposed schema for %s %s", path, apiVersion, kind)) + path, err := liveObjectPath(index, object) + if err != nil { + validationErrors = append(validationErrors, err) continue } - validationErrors = append(validationErrors, validateObject(path, object, candidate)...) + validationErrors = append(validationErrors, validateCandidateObject(path, "object", object, candidates)...) } return errors.Join(validationErrors...) } -func validateObject(path string, object map[string]interface{}, candidate *candidateSchema) []error { +func liveObjectPath(index int, object map[string]interface{}) (string, error) { + metadata, ok := object["metadata"].(map[string]interface{}) + if !ok { + return "", fmt.Errorf("live object item %d has no metadata", index) + } + name, _ := metadata["name"].(string) + namespace, _ := metadata["namespace"].(string) + if name == "" || namespace == "" { + return "", fmt.Errorf("live object item %d must have namespace and name", index) + } + return fmt.Sprintf("live object %s/%s", namespace, name), nil +} + +func validateCandidateObject(path, fieldRoot string, object map[string]interface{}, candidates map[groupVersionKind]*candidateSchema) []error { + apiVersion, _ := object["apiVersion"].(string) + kind, _ := object["kind"].(string) + group, version, ok := strings.Cut(apiVersion, "/") + if !ok || group == "" || version == "" || kind == "" { + return []error{fmt.Errorf("%s: apiVersion and kind must identify a grouped resource", path)} + } + candidate, found := candidates[groupVersionKind{Group: group, Version: version, Kind: kind}] + if !found { + return []error{fmt.Errorf("%s: no proposed schema for %s %s", path, apiVersion, kind)} + } + return validateObject(path, fieldRoot, object, candidate) +} + +func validateObject(path, fieldRoot string, object map[string]interface{}, candidate *candidateSchema) []error { var result []error validator, _, err := apiservervalidation.NewSchemaValidator(candidate.internal) if err != nil { return []error{fmt.Errorf("%s: build OpenAPI validator: %w", path, err)} } - if errs := apiservervalidation.ValidateCustomResource(field.NewPath("fixture"), object, validator); len(errs) > 0 { + if errs := apiservervalidation.ValidateCustomResource(field.NewPath(fieldRoot), object, validator); len(errs) > 0 { result = append(result, fmt.Errorf("%s: proposed OpenAPI validation failed: %w", path, errs.ToAggregate())) } celValidator := cel.NewValidator(candidate.structural, true, perCallCELCostLimit) if celValidator != nil { - fixturePath := field.NewPath("fixture") - createErrors, _ := celValidator.Validate(context.Background(), fixturePath, candidate.structural, object, nil, runtimeCELCostBudget) + objectPath := field.NewPath(fieldRoot) + createErrors, _ := celValidator.Validate(context.Background(), objectPath, candidate.structural, object, nil, runtimeCELCostBudget) if len(createErrors) > 0 { result = append(result, fmt.Errorf("%s: proposed CEL create validation failed: %w", path, createErrors.ToAggregate())) } - updateErrors, _ := celValidator.Validate(context.Background(), fixturePath, candidate.structural, object, object, runtimeCELCostBudget) + updateErrors, _ := celValidator.Validate(context.Background(), objectPath, candidate.structural, object, object, runtimeCELCostBudget) if len(updateErrors) > 0 { result = append(result, fmt.Errorf("%s: proposed CEL unchanged-update validation failed: %w", path, updateErrors.ToAggregate())) } diff --git a/packages/cluster-operator/cmd/crd-preflight/main_test.go b/packages/cluster-operator/cmd/crd-preflight/main_test.go index 1789f16f..1c08fa28 100644 --- a/packages/cluster-operator/cmd/crd-preflight/main_test.go +++ b/packages/cluster-operator/cmd/crd-preflight/main_test.go @@ -252,6 +252,25 @@ func TestValidateObjectsRejectsLiveDataOmittedFromFixtures(t *testing.T) { } } +func TestValidateObjectsAcceptsKubernetesJSONNumbers(t *testing.T) { + crds, _ := writeCandidate(t, `apiVersion: cluster.t4.dev/v1alpha1 +kind: Widget +metadata: + name: curated +spec: + code: ok +`) + candidate := strings.Replace(candidateCRD, " required: [code]", " required: [code, count]", 1) + candidate = strings.Replace(candidate, " maxLength: 3", " maxLength: 3\n count:\n type: integer\n x-kubernetes-validations:\n - rule: self == 9223372036854775807", 1) + if err := os.WriteFile(filepath.Join(crds, "widget.yaml"), []byte(candidate), 0o644); err != nil { + t.Fatal(err) + } + live := strings.NewReader(`{"apiVersion":"v1","kind":"WidgetList","items":[{"apiVersion":"cluster.t4.dev/v1alpha1","kind":"Widget","metadata":{"name":"live","namespace":"tenant-b"},"spec":{"code":"ok","count":9223372036854775807},"status":{"phase":"Ready"}}]}`) + if err := validateObjects(crds, live); err != nil { + t.Fatalf("ordinary Kubernetes JSON integer was rejected: %v", err) + } +} + func TestVerifyServedSchemasRejectsRetainedEstablishedWithStaleSchema(t *testing.T) { crds, _ := writeCandidate(t, `apiVersion: cluster.t4.dev/v1alpha1 kind: Widget diff --git a/scripts/cluster-ci/crd-lifecycle.sh b/scripts/cluster-ci/crd-lifecycle.sh index 9a4c7d13..d698c7b3 100755 --- a/scripts/cluster-ci/crd-lifecycle.sh +++ b/scripts/cluster-ci/crd-lifecycle.sh @@ -50,6 +50,7 @@ compat_directory=${T4_COMPAT_DIRECTORY:-$repo_root/packages/cluster-operator/api validation_namespace=${T4_VALIDATION_NAMESPACE:-default} field_manager=t4-crd-lifecycle crds="crd/t4clusterhosts.cluster.t4.dev crd/t4workspaces.cluster.t4.dev crd/t4sessions.cluster.t4.dev" +live_resources="t4clusterhosts.cluster.t4.dev t4workspaces.cluster.t4.dev t4sessions.cluster.t4.dev" discovery_observations=${T4_DISCOVERY_OBSERVATIONS:-3} case "$discovery_observations" in @@ -72,6 +73,27 @@ run_validator() { # first operation: current-cluster admission cannot prove candidate compatibility. run_validator fixtures "$crd_directory" "$compat_directory" +# Curated fixtures cannot prove that every value currently persisted in the +# cluster remains valid. Enumerate only the three namespaced T4 resources and +# run each complete live object, including status, through the same proposed- +# schema engine before a CRD or workload can be mutated. A denied or malformed +# list fails closed under set -e. +live_objects=$(mktemp) +trap 'rm -f "$live_objects"' EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM +for resource in $live_resources; do + installed_crd=$("$kubectl" get "crd/$resource" --ignore-not-found -o name) + if [ -z "$installed_crd" ]; then + continue + fi + "$kubectl" get "$resource" --all-namespaces -o json >"$live_objects" + run_validator objects "$crd_directory" <"$live_objects" +done +rm -f "$live_objects" +trap - EXIT HUP INT TERM + # Every kubectl operation before the first non-dry-run apply is read-only. "$kubectl" apply --server-side --dry-run=server --validate=strict \ --field-manager="$field_manager" -f "$crd_directory" >/dev/null @@ -87,7 +109,10 @@ run_validator fixtures "$crd_directory" "$compat_directory" # fetched discovery documents to expose exactly the candidate OpenAPI semantics # before trusting admission or starting a workload rollout. openapi_document=$(mktemp) -trap 'rm -f "$openapi_document"' EXIT HUP INT TERM +trap 'rm -f "$openapi_document"' EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM observation=0 while [ "$observation" -lt "$discovery_observations" ]; do "$kubectl" get --raw /openapi/v3/apis/cluster.t4.dev/v1alpha1 >"$openapi_document" diff --git a/scripts/cluster-ci/crd-lifecycle.test.mjs b/scripts/cluster-ci/crd-lifecycle.test.mjs index f856bd92..543f7775 100644 --- a/scripts/cluster-ci/crd-lifecycle.test.mjs +++ b/scripts/cluster-ci/crd-lifecycle.test.mjs @@ -26,9 +26,25 @@ for argument in "$@"; do fi done if [ "\${1:-}" = get ]; then + case "\${2:-}" in + crd/*) + for argument in "$@"; do + if [ "$argument" = "--ignore-not-found" ]; then + if [ "\${FAIL_CRD_READ:-}" = "\${2:-}" ]; then exit 45; fi + if [ "\${MISSING_LIVE_CRDS:-0}" = 1 ]; then exit 0; fi + printf '%s' "\${2:-}" + exit 0 + fi + done + ;; + esac case "\${2:-}" in t4clusterhosts.cluster.t4.dev|t4workspaces.cluster.t4.dev|t4sessions.cluster.t4.dev) if [ "\${FAIL_LIVE_LIST:-}" = "\${2:-}" ]; then exit 44; fi + if [ "\${TERMINATE_DURING_LIVE_READ:-}" = "\${2:-}" ]; then + kill -TERM "$PPID" + sleep 1 + fi printf '%s' '{"apiVersion":"v1","kind":"List","items":[]}' exit 0 ;; @@ -109,6 +125,7 @@ test("upgrade validates proposed schemas, proves served convergence, verifies st assert.equal(result.code, 0, `${result.stdout}\n${result.stderr}`); const log = await commands(value.log); const proposedPreflight = findCommand(log, (line) => line.startsWith("validator\tfixtures\t"), "local proposed-schema fixture preflight"); + const crdReads = log.map((line, index) => ({ line, index })).filter(({ line }) => /^kubectl\tget\tcrd\/t4(?:clusterhosts|workspaces|sessions)\.cluster\.t4\.dev\t--ignore-not-found\t-o\tname$/u.test(line)); const liveLists = log.map((line, index) => ({ line, index })).filter(({ line }) => /^kubectl\tget\tt4(?:clusterhosts|workspaces|sessions)\.cluster\.t4\.dev\t--all-namespaces\t-o\tjson$/u.test(line)); const liveValidations = log.map((line, index) => ({ line, index })).filter(({ line }) => line.startsWith("validator\tobjects\t")); const crdPreflight = findCommand(log, (line) => line.includes("apply") && line.includes("--server-side") && line.includes("--dry-run=server") && line.includes("deploy/charts/t4-cluster/crds"), "server-side CRD preflight"); @@ -119,14 +136,19 @@ test("upgrade validates proposed schemas, proves served convergence, verifies st const storageChecks = log.map((line, index) => ({ line, index })).filter(({ line }) => line.startsWith("kubectl\tget\tcrd/") && line.includes("status.storedVersions")); assert.deepEqual(storageChecks.length, 3); assert.equal(servedChecks.length, 3); + assert.equal(crdReads.length, 3); assert.equal(liveLists.length, 3); assert.equal(liveValidations.length, 3); const workload = findCommand(log, (line) => line.startsWith("helm\tupgrade\t"), "Helm workload upgrade"); assert.ok(proposedPreflight < crdPreflight); + assert.ok(proposedPreflight < crdReads[0].index); + assert.ok(crdReads[0].index < liveLists[0].index); assert.ok(liveLists[0].index < liveValidations[0].index); - assert.ok(liveValidations[0].index < liveLists[1].index); + assert.ok(liveValidations[0].index < crdReads[1].index); + assert.ok(crdReads[1].index < liveLists[1].index); assert.ok(liveLists[1].index < liveValidations[1].index); - assert.ok(liveValidations[1].index < liveLists[2].index); + assert.ok(liveValidations[1].index < crdReads[2].index); + assert.ok(crdReads[2].index < liveLists[2].index); assert.ok(liveLists[2].index < liveValidations[2].index); assert.ok(liveValidations[2].index < crdPreflight); assert.ok(crdPreflight < crdApply); @@ -142,10 +164,11 @@ test("fresh install establishes and validates CRDs before Helm installs with CRD const value = await fixture(); const result = await runLifecycle( ["install", "--", "helm", "install", "t4-cluster", "deploy/charts/t4-cluster", "--namespace", "t4-system", "--skip-crds"], - value.env, + { ...value.env, MISSING_LIVE_CRDS: "1" }, ); assert.equal(result.code, 0, `${result.stdout}\n${result.stderr}`); const log = await commands(value.log); + assert.ok(log.every((line) => !line.startsWith("validator\tobjects\t") && !/kubectl\tget\tt4(?:clusterhosts|workspaces|sessions)\.cluster\.t4\.dev\t--all-namespaces/u.test(line)), log.join("\n")); const established = findCommand(log, (line) => line.includes("wait") && line.includes("condition=Established"), "Established wait"); const fixtureValidation = findCommand(log, (line) => line.includes("--dry-run=server") && line.includes("testdata/compat"), "fixture validation"); const storage = findCommand(log, (line) => line.includes("status.storedVersions"), "stored-version check"); @@ -190,16 +213,33 @@ test("live object incompatibility fails before non-dry-run CRD, object, or Helm assert.ok(log.every((line) => !line.startsWith("helm\t") && !/kubectl\tapply(?!.*--dry-run=server)/u.test(line)), log.join("\n")); }); -test("live object list denial fails closed before non-dry-run mutation", async () => { - const value = await fixture(); +test("live object definition read or list denial fails closed before non-dry-run mutation", async () => { const deniedResource = "t4workspaces.cluster.t4.dev"; + for (const scenario of [ + { env: { FAIL_CRD_READ: `crd/${deniedResource}` }, expected: `kubectl\tget\tcrd/${deniedResource}\t--ignore-not-found\t-o\tname` }, + { env: { FAIL_LIVE_LIST: deniedResource }, expected: `kubectl\tget\t${deniedResource}\t--all-namespaces\t-o\tjson` }, + ]) { + const value = await fixture(); + const result = await runLifecycle( + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "chart", "--skip-crds"], + { ...value.env, ...scenario.env }, + ); + assert.notEqual(result.code, 0); + const log = await commands(value.log); + assert.ok(log.some((line) => line.startsWith(scenario.expected)), log.join("\n")); + assert.ok(log.every((line) => !line.startsWith("helm\t") && !/kubectl\tapply(?!.*--dry-run=server)/u.test(line)), log.join("\n")); + } +}); + +test("cancellation during live enumeration cannot continue to mutation", async () => { + const value = await fixture(); const result = await runLifecycle( ["upgrade", "--", "helm", "upgrade", "t4-cluster", "chart", "--skip-crds"], - { ...value.env, FAIL_LIVE_LIST: deniedResource }, + { ...value.env, TERMINATE_DURING_LIVE_READ: "t4clusterhosts.cluster.t4.dev" }, ); assert.notEqual(result.code, 0); const log = await commands(value.log); - assert.ok(log.some((line) => line.startsWith(`kubectl\tget\t${deniedResource}\t--all-namespaces\t-o\tjson`)), log.join("\n")); + assert.match(log.at(-1), /^kubectl\tget\tt4clusterhosts\.cluster\.t4\.dev\t--all-namespaces\t-o\tjson$/u); assert.ok(log.every((line) => !line.startsWith("helm\t") && !/kubectl\tapply(?!.*--dry-run=server)/u.test(line)), log.join("\n")); }); @@ -226,9 +266,9 @@ test("failed server preflight leaves CRDs and workloads untouched", async () => ); assert.notEqual(result.code, 0); const log = await commands(value.log); - assert.equal(log.length, 2, log.join("\n")); + assert.equal(log.length, 11, log.join("\n")); assert.match(log[0], /^validator\tfixtures\t/u); - assert.match(log[1], /apply.*--server-side.*--dry-run=server/u); + assert.match(log.at(-1), /apply.*--server-side.*--dry-run=server/u); assert.ok(log.every((line) => !line.startsWith("helm\t") && !/kubectl\tapply(?!.*--dry-run=server)/u.test(line)), log.join("\n")); }); From 1e22cdde25924c22cd524664189de98afad5fb61 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 04:21:02 +0000 Subject: [PATCH 083/131] test(t4-api): cover snapshot boundary and heartbeat cursors --- .../test/t4-api-v1-conformance-service.ts | 24 +++++++ .../client/test/t4-api-v1-conformance.test.ts | 65 +++++++++++++++---- 2 files changed, 78 insertions(+), 11 deletions(-) diff --git a/packages/client/test/t4-api-v1-conformance-service.ts b/packages/client/test/t4-api-v1-conformance-service.ts index c2065516..12fad3b7 100644 --- a/packages/client/test/t4-api-v1-conformance-service.ts +++ b/packages/client/test/t4-api-v1-conformance-service.ts @@ -47,6 +47,26 @@ function json(status: number, body: unknown, headers: Record = { function problem(status: number, code: string, message: string, extra: Record = {}): Response { return json(status, { error: { code, message, requestId: `req-${code}`, retryable: status >= 500, ...extra } }); } +const SNAPSHOT_BYTES_MAX = 16 * 1024 * 1024; + +function snapshotResponseAtBytes(sessionId: unknown, state: unknown, targetBytes: number): Response { + const entries = Array.from({ length: 4 }, (_, sequence) => ({ sequence, kind: "output", text: "" })); + const snapshot = { sessionId, cursor: "cursor-2", state, entries }; + let remaining = targetBytes - encoder.encode(JSON.stringify(snapshot)).byteLength; + for (const entry of entries) { + const astralCodePoints = Math.min(1_048_576, Math.floor(remaining / 4)); + entry.text = "💚".repeat(astralCodePoints); + remaining -= astralCodePoints * 4; + if (remaining > 0 && remaining < 4) { + entry.text += "x".repeat(remaining); + remaining = 0; + } + } + expect(remaining).toBe(0); + const body = JSON.stringify(snapshot); + expect(encoder.encode(body).byteLength).toBe(targetBytes); + return new Response(body, { status: 200, headers: { "Content-Type": "application/json", "T4-API-Version": "1.0" } }); +} function compareUtf16(left: string, right: string): number { return left < right ? -1 : left > right ? 1 : 0; @@ -109,6 +129,7 @@ interface ReplayRecord { export interface T4ApiV1ConformanceOptions { readonly invalidPayload?: "discovery" | "workspace" | "session" | "command"; + readonly snapshotBoundary?: "exact" | "over"; readonly watchTransport?: "normal" | "bytewise" | "oversized" | "many-small"; } @@ -336,6 +357,9 @@ export class T4ApiV1ConformanceService { if (snapshotMatch && request.method === "GET") { const session = this.#sessions.get(decodeURIComponent(snapshotMatch[1]!)); if (session?.tenant !== tenant) return problem(404, "not_found", "Session not found"); + if (this.options.snapshotBoundary !== undefined) { + return snapshotResponseAtBytes(session.id, session.state, SNAPSHOT_BYTES_MAX + (this.options.snapshotBoundary === "over" ? 1 : 0)); + } return json(200, { sessionId: session.id, cursor: "cursor-2", state: session.state, entries: [{ sequence: 2, kind: "output", text: "ready" }] }); } diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index 51d5b1a1..ebf0aee3 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -1395,20 +1395,24 @@ describe("generated T4 API v1 client conformance", () => { }; expect(contract.components.schemas.SessionSnapshot["x-t4-maxUtf8Bytes"]).toBe(16 * 1024 * 1024); - const oversizedSnapshot = createT4ApiClient({ - baseUrl: "https://aggregate-snapshot.test", credential: "token-a", majorVersion: 1, - fetch: async () => jsonResponse({ - sessionId: "ses-1", cursor: "cursor-1", state: "ready", - entries: Array.from({ length: 17 }, (_, sequence) => ({ sequence, kind: "output", text: "x".repeat(1024 * 1024) })), - }), - }); - await expect(oversizedSnapshot.http.GET("/v1/sessions/{sessionId}/snapshot", { - params: { header: VERSION_HEADERS, path: { sessionId: "ses-1" } }, - })).rejects.toMatchObject({ code: "indeterminate", status: 502, retryable: false }); + for (const boundary of ["exact", "over"] as const) { + const service = new T4ApiV1ConformanceService({ snapshotBoundary: boundary }); + const client = await seededClient(service, `snapshot-${boundary}`); + const response = client.http.GET("/v1/sessions/{sessionId}/snapshot", { + params: { header: VERSION_HEADERS, path: { sessionId: "ses-1" } }, + }); + if (boundary === "exact") { + const snapshot = requireData(await response); + expect(snapshot.entries).toHaveLength(4); + expect(snapshot.entries[0]?.text).toContain("💚"); + } else { + await expect(response).rejects.toMatchObject({ code: "indeterminate", status: 502, retryable: false }); + } + } }); it("rejects duplicate durable cursors within one SSE stream and after reconnect", async () => { - const event = 'data: {"type":"heartbeat","cursor":"duplicate-1","observedAt":"2026-07-21T00:00:00Z"}\n\n'; + const event = 'data: {"type":"session","cursor":"duplicate-1","state":"ready","revision":1}\n\n'; for (const delivery of ["same-stream", "reconnect"] as const) { let attempts = 0; const client = createT4ApiClient({ @@ -1425,6 +1429,45 @@ describe("generated T4 API v1 client conformance", () => { } }); + it("accepts current-cursor heartbeats without allowing them to advance the durable cursor", async () => { + const heartbeat = (cursor: string) => `data: ${JSON.stringify({ type: "heartbeat", cursor, observedAt: "2026-07-21T00:00:00Z" })}\n\n`; + const startingRequests: Array<{ query: string | null; header: string | null }> = []; + const starting = createT4ApiClient({ + baseUrl: "https://starting-heartbeat.test", credential: "token-a", majorVersion: 1, + fetch: async (input, init) => { + const request = new Request(input, init); + startingRequests.push({ query: new URL(request.url).searchParams.get("cursor"), header: request.headers.get("Last-Event-ID") }); + return apiResponse(heartbeat("duplicate-1"), { headers: { "Content-Type": "text/event-stream" } }); + }, + }); + await expect(starting.watchSession("ses-1", { cursor: "duplicate-1", maxEvents: 1, maxReconnectAttempts: 0 }).next()).resolves.toMatchObject({ + value: { type: "heartbeat", cursor: "duplicate-1" }, done: false, + }); + expect(startingRequests).toEqual([{ query: "duplicate-1", header: "duplicate-1" }]); + + const reconnectRequests: Array<{ query: string | null; header: string | null }> = []; + const reconnecting = createT4ApiClient({ + baseUrl: "https://reconnect-heartbeat.test", credential: "token-a", majorVersion: 1, + fetch: async (input, init) => { + const request = new Request(input, init); + reconnectRequests.push({ query: new URL(request.url).searchParams.get("cursor"), header: request.headers.get("Last-Event-ID") }); + return apiResponse(heartbeat("duplicate-1"), { headers: { "Content-Type": "text/event-stream" } }); + }, + }); + const liveness: WatchEvent[] = []; + for await (const event of reconnecting.watchSession("ses-1", { maxEvents: 2, maxReconnectAttempts: 1, retryBackoffMs: 0 })) liveness.push(event); + expect(liveness.map((event) => event.cursor)).toEqual(["duplicate-1", "duplicate-1"]); + expect(reconnectRequests).toEqual([{ query: null, header: null }, { query: "duplicate-1", header: "duplicate-1" }]); + + const advancing = createT4ApiClient({ + baseUrl: "https://advancing-heartbeat.test", credential: "token-a", majorVersion: 1, + fetch: async () => apiResponse(heartbeat("different-2"), { headers: { "Content-Type": "text/event-stream" } }), + }); + await expect(advancing.watchSession("ses-1", { cursor: "duplicate-1", maxEvents: 1, maxReconnectAttempts: 0 }).next()).rejects.toMatchObject({ + code: "indeterminate", status: 502, retryable: false, + }); + }); + it("rejects duplicate version entries and malformed capability maps", async () => { const status = { supported: true, enabled: true, authorized: true, available: true }; for (const discovery of [ From 2ebbf130d5042032c038c1a496a45d7f0d18e5b3 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 20:41:50 +0000 Subject: [PATCH 084/131] test(operator): require dependency event convergence --- .../controllers/reconciler_test.go | 117 ++++++++++++++++++ .../controllers/session_controller_test.go | 62 ++++++++++ .../controllers/workspace_controller_test.go | 40 ++++++ 3 files changed, 219 insertions(+) create mode 100644 packages/cluster-operator/controllers/session_controller_test.go diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index 1a54e2d0..02b070a2 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -2,8 +2,10 @@ package controllers_test import ( "context" + "reflect" "strings" "testing" + "time" corev1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" @@ -1254,6 +1256,121 @@ func TestSessionDeletionCleansResourcesBeforeFinalizer(t *testing.T) { } } +func TestSessionDependencyRevocationCleansOwnedResourcesAndConvergesAfterRestart(t *testing.T) { + for _, test := range []struct { + name string + conditionType string + wantReason string + revoke func(context.Context, client.Client) error + }{ + {name: "missing Host", conditionType: "HostReady", wantReason: "HostNotFound", revoke: func(ctx context.Context, c client.Client) error { + return c.Delete(ctx, &clusterv1alpha1.T4ClusterHost{ObjectMeta: metav1.ObjectMeta{Name: "host-a", Namespace: "team"}}) + }}, + {name: "invalid Host runtime profile", conditionType: "RuntimeConfigured", wantReason: "RuntimeProfileNotAllowed", revoke: func(ctx context.Context, c client.Client) error { + var host clusterv1alpha1.T4ClusterHost + if err := c.Get(ctx, types.NamespacedName{Namespace: "team", Name: "host-a"}, &host); err != nil { + return err + } + host.Spec.RuntimeProfiles = nil + return c.Update(ctx, &host) + }}, + {name: "missing Workspace", conditionType: "WorkspaceReady", wantReason: "WorkspaceNotFound", revoke: func(ctx context.Context, c client.Client) error { + return c.Delete(ctx, &clusterv1alpha1.T4Workspace{ObjectMeta: metav1.ObjectMeta{Name: "workspace-a", Namespace: "team"}}) + }}, + {name: "mismatched Workspace Host", conditionType: "WorkspaceReady", wantReason: "HostMismatch", revoke: func(ctx context.Context, c client.Client) error { + var workspace clusterv1alpha1.T4Workspace + if err := c.Get(ctx, types.NamespacedName{Namespace: "team", Name: "workspace-a"}, &workspace); err != nil { + return err + } + workspace.Spec.HostRef = "host-b" + return c.Update(ctx, &workspace) + }}, + {name: "missing OMP ConfigMap", conditionType: "RuntimeConfigured", wantReason: "OMPConfigMapNotFound", revoke: func(ctx context.Context, c client.Client) error { + return c.Delete(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "omp-runtime-config", Namespace: "team"}}) + }}, + {name: "invalid OMP credential Secret", conditionType: "RuntimeConfigured", wantReason: "OMPCredentialSecretInvalid", revoke: func(ctx context.Context, c client.Client) error { + var secret corev1.Secret + if err := c.Get(ctx, types.NamespacedName{Namespace: "team", Name: "omp-runtime-credential"}, &secret); err != nil { + return err + } + delete(secret.Data, "MODEL_API_KEY") + return c.Update(ctx, &secret) + }}, + } { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.Status.PVCName = "workspace-a-data" + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, + Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } + session := testSession() + session.UID = "session-uid" + foreignPod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foreign-pod", Namespace: "team"}} + foreignService := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "foreign-service", Namespace: "team"}} + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(testHost(), workspace, pvc, session, foreignPod, foreignService).Build() + r := configuredSessionReconciler(c, scheme) + reconcileMany(t, 2, func() error { + _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + if err := test.revoke(ctx, c); err != nil { + t.Fatal(err) + } + + result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + if err != nil { + t.Fatal(err) + } + if result.RequeueAfter <= 0 || result.RequeueAfter > 30*time.Second { + t.Fatalf("revoked dependency requeue = %s, want bounded positive retry", result.RequeueAfter) + } + for _, object := range []client.Object{ + &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionPodName(session), Namespace: session.Namespace}}, + &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionServiceName(session), Namespace: session.Namespace}}, + } { + if err := c.Get(ctx, client.ObjectKeyFromObject(object), object); !apierrors.IsNotFound(err) { + t.Fatalf("owned stale %T remained after dependency revocation: %v", object, err) + } + } + for _, object := range []client.Object{foreignPod.DeepCopy(), foreignService.DeepCopy()} { + if err := c.Get(ctx, client.ObjectKeyFromObject(object), object); err != nil { + t.Fatalf("unowned %T was removed: %v", object, err) + } + } + + var failed clusterv1alpha1.T4Session + if err := c.Get(ctx, client.ObjectKeyFromObject(session), &failed); err != nil { + t.Fatal(err) + } + condition := findCondition(failed.Status.Conditions, test.conditionType) + available := findCondition(failed.Status.Conditions, "Available") + if failed.Status.ObservedGeneration != failed.Generation || failed.Status.PodName != "" || failed.Status.ServiceName != "" || failed.Status.Phase != clusterv1alpha1.InfrastructureFailed || + condition == nil || condition.Status != metav1.ConditionFalse || condition.Reason != test.wantReason || condition.ObservedGeneration != failed.Generation || + available == nil || available.Status != metav1.ConditionFalse || available.Reason != test.wantReason || available.ObservedGeneration != failed.Generation { + t.Fatalf("revoked session did not converge: status=%#v condition=%#v available=%#v", failed.Status, condition, available) + } + stableStatus := failed.Status + restarted := *r + reconcileMany(t, 2, func() error { + _, err := restarted.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + if err := c.Get(ctx, client.ObjectKeyFromObject(session), &failed); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(failed.Status, stableStatus) { + t.Fatalf("duplicate/restart reconciliation changed converged status: got %#v, want %#v", failed.Status, stableStatus) + } + }) + } +} + func configuredSessionReconciler(c client.Client, scheme *runtime.Scheme) *controllers.SessionReconciler { for _, object := range []client.Object{ &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "omp-runtime-config", Namespace: "team"}, Data: map[string]string{ diff --git a/packages/cluster-operator/controllers/session_controller_test.go b/packages/cluster-operator/controllers/session_controller_test.go new file mode 100644 index 00000000..cba15fef --- /dev/null +++ b/packages/cluster-operator/controllers/session_controller_test.go @@ -0,0 +1,62 @@ +package controllers + +import ( + "context" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + clusterv1alpha1 "github.com/LycaonLLC/t4-code/packages/cluster-operator/api/v1alpha1" +) + +func TestSessionRequestsForHostOnlyEnqueuesAffectedSessions(t *testing.T) { + scheme := runtime.NewScheme() + if err := clusterv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + objects := []client.Object{ + &clusterv1alpha1.T4Session{ObjectMeta: metav1.ObjectMeta{Name: "session-a", Namespace: "team"}, Spec: clusterv1alpha1.T4SessionSpec{HostRef: "host-a", WorkspaceRef: "workspace-a"}}, + &clusterv1alpha1.T4Session{ObjectMeta: metav1.ObjectMeta{Name: "session-b", Namespace: "team"}, Spec: clusterv1alpha1.T4SessionSpec{HostRef: "host-a", WorkspaceRef: "workspace-b"}}, + &clusterv1alpha1.T4Session{ObjectMeta: metav1.ObjectMeta{Name: "other-host", Namespace: "team"}, Spec: clusterv1alpha1.T4SessionSpec{HostRef: "host-b", WorkspaceRef: "workspace-a"}}, + &clusterv1alpha1.T4Session{ObjectMeta: metav1.ObjectMeta{Name: "other-namespace", Namespace: "other"}, Spec: clusterv1alpha1.T4SessionSpec{HostRef: "host-a", WorkspaceRef: "workspace-a"}}, + } + c := fake.NewClientBuilder().WithScheme(scheme). + WithIndex(&clusterv1alpha1.T4Session{}, sessionHostRefIndexField, indexSessionByHostRef). + WithIndex(&clusterv1alpha1.T4Session{}, sessionWorkspaceRefIndexField, indexSessionByWorkspaceRef). + WithObjects(objects...).Build() + r := &SessionReconciler{Client: c, Scheme: scheme} + + requests := r.sessionRequestsForHost(context.Background(), &clusterv1alpha1.T4ClusterHost{ObjectMeta: metav1.ObjectMeta{Name: "host-a", Namespace: "team"}}) + assertRequestSet(t, requests, []types.NamespacedName{ + {Namespace: "team", Name: "session-a"}, + {Namespace: "team", Name: "session-b"}, + }) +} + +func TestSessionRequestsForWorkspaceOnlyEnqueuesAffectedSessions(t *testing.T) { + scheme := runtime.NewScheme() + if err := clusterv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + objects := []client.Object{ + &clusterv1alpha1.T4Session{ObjectMeta: metav1.ObjectMeta{Name: "session-a", Namespace: "team"}, Spec: clusterv1alpha1.T4SessionSpec{HostRef: "host-a", WorkspaceRef: "workspace-a"}}, + &clusterv1alpha1.T4Session{ObjectMeta: metav1.ObjectMeta{Name: "session-b", Namespace: "team"}, Spec: clusterv1alpha1.T4SessionSpec{HostRef: "host-b", WorkspaceRef: "workspace-a"}}, + &clusterv1alpha1.T4Session{ObjectMeta: metav1.ObjectMeta{Name: "other-workspace", Namespace: "team"}, Spec: clusterv1alpha1.T4SessionSpec{HostRef: "host-a", WorkspaceRef: "workspace-b"}}, + &clusterv1alpha1.T4Session{ObjectMeta: metav1.ObjectMeta{Name: "other-namespace", Namespace: "other"}, Spec: clusterv1alpha1.T4SessionSpec{HostRef: "host-a", WorkspaceRef: "workspace-a"}}, + } + c := fake.NewClientBuilder().WithScheme(scheme). + WithIndex(&clusterv1alpha1.T4Session{}, sessionHostRefIndexField, indexSessionByHostRef). + WithIndex(&clusterv1alpha1.T4Session{}, sessionWorkspaceRefIndexField, indexSessionByWorkspaceRef). + WithObjects(objects...).Build() + r := &SessionReconciler{Client: c, Scheme: scheme} + + requests := r.sessionRequestsForWorkspace(context.Background(), &clusterv1alpha1.T4Workspace{ObjectMeta: metav1.ObjectMeta{Name: "workspace-a", Namespace: "team"}}) + assertRequestSet(t, requests, []types.NamespacedName{ + {Namespace: "team", Name: "session-a"}, + {Namespace: "team", Name: "session-b"}, + }) +} diff --git a/packages/cluster-operator/controllers/workspace_controller_test.go b/packages/cluster-operator/controllers/workspace_controller_test.go index 412705d0..27d07759 100644 --- a/packages/cluster-operator/controllers/workspace_controller_test.go +++ b/packages/cluster-operator/controllers/workspace_controller_test.go @@ -9,6 +9,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -107,3 +108,42 @@ func TestWorkspaceRequestsForStorageClassOnlyEnqueuesAffectedWorkspaces(t *testi } } } + +func TestWorkspaceRequestsForHostOnlyEnqueuesAffectedWorkspaces(t *testing.T) { + scheme := runtime.NewScheme() + if err := clusterv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + objects := []client.Object{ + &clusterv1alpha1.T4Workspace{ObjectMeta: metav1.ObjectMeta{Name: "workspace-a", Namespace: "team"}, Spec: clusterv1alpha1.T4WorkspaceSpec{HostRef: "host-a"}}, + &clusterv1alpha1.T4Workspace{ObjectMeta: metav1.ObjectMeta{Name: "workspace-b", Namespace: "team"}, Spec: clusterv1alpha1.T4WorkspaceSpec{HostRef: "host-a"}}, + &clusterv1alpha1.T4Workspace{ObjectMeta: metav1.ObjectMeta{Name: "other-host", Namespace: "team"}, Spec: clusterv1alpha1.T4WorkspaceSpec{HostRef: "host-b"}}, + &clusterv1alpha1.T4Workspace{ObjectMeta: metav1.ObjectMeta{Name: "other-namespace", Namespace: "other"}, Spec: clusterv1alpha1.T4WorkspaceSpec{HostRef: "host-a"}}, + } + c := fake.NewClientBuilder().WithScheme(scheme). + WithIndex(&clusterv1alpha1.T4Workspace{}, workspaceHostRefIndexField, indexWorkspaceByHostRef). + WithObjects(objects...).Build() + r := &WorkspaceReconciler{Client: c, Scheme: scheme} + + requests := r.workspaceRequestsForHost(context.Background(), &clusterv1alpha1.T4ClusterHost{ObjectMeta: metav1.ObjectMeta{Name: "host-a", Namespace: "team"}}) + assertRequestSet(t, requests, []types.NamespacedName{ + {Namespace: "team", Name: "workspace-a"}, + {Namespace: "team", Name: "workspace-b"}, + }) +} + +func assertRequestSet(t *testing.T, requests []ctrl.Request, want []types.NamespacedName) { + t.Helper() + got := make(map[types.NamespacedName]int, len(requests)) + for _, request := range requests { + got[request.NamespacedName]++ + } + if len(got) != len(want) { + t.Fatalf("requests = %#v, want exactly %v", requests, want) + } + for _, key := range want { + if got[key] != 1 { + t.Fatalf("requests = %#v, want %v exactly once", requests, key) + } + } +} From f91517b204644a6c1c1b8c8a15c0a92463c2284d Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 20:51:56 +0000 Subject: [PATCH 085/131] feat(operator): reconcile dependency changes --- .../controllers/session_controller.go | 77 +++++++++++++++++++ .../controllers/workspace_controller.go | 18 +++++ 2 files changed, 95 insertions(+) diff --git a/packages/cluster-operator/controllers/session_controller.go b/packages/cluster-operator/controllers/session_controller.go index 9336a3f4..a4cce1f2 100644 --- a/packages/cluster-operator/controllers/session_controller.go +++ b/packages/cluster-operator/controllers/session_controller.go @@ -25,6 +25,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" clusterv1alpha1 "github.com/LycaonLLC/t4-code/packages/cluster-operator/api/v1alpha1" ) @@ -36,6 +37,11 @@ const ( SessionReviewerTokenExpirationSeconds int64 = 3600 ) +const ( + sessionHostRefIndexField = "t4.session.spec.hostRef" + sessionWorkspaceRefIndexField = "t4.session.spec.workspaceRef" +) + var ( configMapKeyPattern = regexp.MustCompile(`^[-._A-Za-z0-9]+$`) runtimeImagePattern = regexp.MustCompile(`^(?:(?:[A-Za-z0-9](?:[A-Za-z0-9.-]*[A-Za-z0-9])?|\[[A-Fa-f0-9:]+\])(?::[0-9]+)?/)?[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*(?:/[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*)*@sha256:[a-f0-9]{64}$`) @@ -313,6 +319,9 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) var host clusterv1alpha1.T4ClusterHost if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: session.Spec.HostRef}, &host); err != nil { if apierrors.IsNotFound(err) { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "HostReady", "HostNotFound", "referenced T4ClusterHost does not exist") } return ctrl.Result{}, err @@ -342,24 +351,39 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) var workspace clusterv1alpha1.T4Workspace if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: session.Spec.WorkspaceRef}, &workspace); err != nil { if apierrors.IsNotFound(err) { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "WorkspaceNotFound", "referenced T4Workspace does not exist") } return ctrl.Result{}, err } if workspace.Spec.HostRef != session.Spec.HostRef { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "HostMismatch", "session and workspace must reference the same T4ClusterHost") } if workspace.Status.PVCName == "" { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "PVCNotDeclared", "workspace controller has not declared a PVC") } var pvc corev1.PersistentVolumeClaim if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: workspace.Status.PVCName}, &pvc); err != nil { if apierrors.IsNotFound(err) { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "PVCNotFound", "workspace PVC does not exist") } return ctrl.Result{}, err } if pvc.Status.Phase != corev1.ClaimBound || !pvcHasRWX(&pvc) { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "PVCNotBoundRWX", "workspace PVC must be Bound and ReadWriteMany before a session starts") } runtimeVersions, reason, message, err := r.loadOMPResourceVersions(ctx, session.Namespace) @@ -754,9 +778,62 @@ func labelsContain(actual, required map[string]string) bool { return true } +func indexSessionByHostRef(object client.Object) []string { + session, ok := object.(*clusterv1alpha1.T4Session) + if !ok || session.Spec.HostRef == "" { + return nil + } + return []string{session.Spec.HostRef} +} + +func indexSessionByWorkspaceRef(object client.Object) []string { + session, ok := object.(*clusterv1alpha1.T4Session) + if !ok || session.Spec.WorkspaceRef == "" { + return nil + } + return []string{session.Spec.WorkspaceRef} +} + +func (r *SessionReconciler) sessionRequestsForHost(ctx context.Context, object client.Object) []ctrl.Request { + host, ok := object.(*clusterv1alpha1.T4ClusterHost) + if !ok || host.Name == "" || host.Namespace == "" { + return nil + } + return r.sessionRequestsForReference(ctx, host.Namespace, sessionHostRefIndexField, host.Name, "clusterHost", client.ObjectKeyFromObject(host)) +} + +func (r *SessionReconciler) sessionRequestsForWorkspace(ctx context.Context, object client.Object) []ctrl.Request { + workspace, ok := object.(*clusterv1alpha1.T4Workspace) + if !ok || workspace.Name == "" || workspace.Namespace == "" { + return nil + } + return r.sessionRequestsForReference(ctx, workspace.Namespace, sessionWorkspaceRefIndexField, workspace.Name, "workspace", client.ObjectKeyFromObject(workspace)) +} + +func (r *SessionReconciler) sessionRequestsForReference(ctx context.Context, namespace, field, value, dependencyKind string, dependencyKey types.NamespacedName) []ctrl.Request { + var sessions clusterv1alpha1.T4SessionList + if err := r.List(ctx, &sessions, client.InNamespace(namespace), client.MatchingFields{field: value}); err != nil { + ctrl.LoggerFrom(ctx).Error(err, "unable to map dependency to sessions", dependencyKind, dependencyKey) + return nil + } + requests := make([]ctrl.Request, 0, len(sessions.Items)) + for i := range sessions.Items { + requests = append(requests, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(&sessions.Items[i])}) + } + return requests +} + func (r *SessionReconciler) SetupWithManager(manager ctrl.Manager) error { + if err := manager.GetFieldIndexer().IndexField(context.Background(), &clusterv1alpha1.T4Session{}, sessionHostRefIndexField, indexSessionByHostRef); err != nil { + return fmt.Errorf("index T4Session by host reference: %w", err) + } + if err := manager.GetFieldIndexer().IndexField(context.Background(), &clusterv1alpha1.T4Session{}, sessionWorkspaceRefIndexField, indexSessionByWorkspaceRef); err != nil { + return fmt.Errorf("index T4Session by workspace reference: %w", err) + } return ctrl.NewControllerManagedBy(manager). For(&clusterv1alpha1.T4Session{}). + Watches(&clusterv1alpha1.T4ClusterHost{}, handler.EnqueueRequestsFromMapFunc(r.sessionRequestsForHost)). + Watches(&clusterv1alpha1.T4Workspace{}, handler.EnqueueRequestsFromMapFunc(r.sessionRequestsForWorkspace)). Owns(&corev1.Pod{}). Owns(&corev1.Service{}). Complete(r) diff --git a/packages/cluster-operator/controllers/workspace_controller.go b/packages/cluster-operator/controllers/workspace_controller.go index fbc60f45..e96271a6 100644 --- a/packages/cluster-operator/controllers/workspace_controller.go +++ b/packages/cluster-operator/controllers/workspace_controller.go @@ -337,6 +337,23 @@ func (r *WorkspaceReconciler) workspaceRequestsForStorageClass(ctx context.Conte return requests } +func (r *WorkspaceReconciler) workspaceRequestsForHost(ctx context.Context, object client.Object) []ctrl.Request { + host, ok := object.(*clusterv1alpha1.T4ClusterHost) + if !ok || host.Name == "" || host.Namespace == "" { + return nil + } + var workspaces clusterv1alpha1.T4WorkspaceList + if err := r.List(ctx, &workspaces, client.InNamespace(host.Namespace), client.MatchingFields{workspaceHostRefIndexField: host.Name}); err != nil { + ctrl.LoggerFrom(ctx).Error(err, "unable to map cluster host to workspaces", "clusterHost", client.ObjectKeyFromObject(host)) + return nil + } + requests := make([]ctrl.Request, 0, len(workspaces.Items)) + for i := range workspaces.Items { + requests = append(requests, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(&workspaces.Items[i])}) + } + return requests +} + func (r *WorkspaceReconciler) SetupWithManager(manager ctrl.Manager) error { if err := manager.GetFieldIndexer().IndexField(context.Background(), &clusterv1alpha1.T4ClusterHost{}, hostStorageClassIndexField, indexHostByStorageClass); err != nil { return fmt.Errorf("index T4ClusterHost by StorageClass: %w", err) @@ -346,6 +363,7 @@ func (r *WorkspaceReconciler) SetupWithManager(manager ctrl.Manager) error { } return ctrl.NewControllerManagedBy(manager). For(&clusterv1alpha1.T4Workspace{}). + Watches(&clusterv1alpha1.T4ClusterHost{}, handler.EnqueueRequestsFromMapFunc(r.workspaceRequestsForHost)). Watches(&corev1.PersistentVolumeClaim{}, handler.EnqueueRequestsFromMapFunc(workspaceRequestsForPVC)). Watches(&storagev1.StorageClass{}, handler.EnqueueRequestsFromMapFunc(r.workspaceRequestsForStorageClass)). Complete(r) From 72c6a19401e2e6f6b76e4980cf65051edd0b8695 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 21:05:36 +0000 Subject: [PATCH 086/131] test(operator): require host policy convergence --- .../controllers/reconciler_test.go | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index 02b070a2..905e3896 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -1296,6 +1296,18 @@ func TestSessionDependencyRevocationCleansOwnedResourcesAndConvergesAfterRestart delete(secret.Data, "MODEL_API_KEY") return c.Update(ctx, &secret) }}, + {name: "mismatched Host storage class", conditionType: "WorkspaceReady", wantReason: "StorageClassMismatch", revoke: func(ctx context.Context, c client.Client) error { + otherClass := &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: "other-rwx", Annotations: map[string]string{clusterv1alpha1.RWXStorageClassAnnotation: string(corev1.ReadWriteMany)}}, Provisioner: "example.invalid/csi"} + if err := c.Create(ctx, otherClass); err != nil { + return err + } + var host clusterv1alpha1.T4ClusterHost + if err := c.Get(ctx, types.NamespacedName{Namespace: "team", Name: "host-a"}, &host); err != nil { + return err + } + host.Spec.StorageClassName = otherClass.Name + return c.Update(ctx, &host) + }}, } { t.Run(test.name, func(t *testing.T) { ctx := context.Background() @@ -1371,6 +1383,137 @@ func TestSessionDependencyRevocationCleansOwnedResourcesAndConvergesAfterRestart } } +func TestWorkspaceHostStorageClassDriftFailsClosedWithoutRecreatingPVC(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + workspace.Status.ObservedGeneration = workspace.Generation + workspace.Status.PVCName = controllers.WorkspacePVCName(workspace) + workspace.Status.Phase = clusterv1alpha1.InfrastructureReady + workspace.Status.Conditions = []metav1.Condition{ + {Type: "StorageReady", Status: metav1.ConditionTrue, Reason: controllers.ReasonStorageReady, ObservedGeneration: workspace.Generation}, + {Type: "Ready", Status: metav1.ConditionTrue, Reason: "PVCBound", ObservedGeneration: workspace.Generation}, + } + oldClass := "portable-rwx" + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: workspace.Status.PVCName, Namespace: workspace.Namespace, + Annotations: map[string]string{clusterv1alpha1.WorkspaceUIDAnnotation: string(workspace.UID)}, + OwnerReferences: []metav1.OwnerReference{{APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Workspace", Name: workspace.Name, UID: workspace.UID, Controller: ptr(true)}}, + }, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: &oldClass, AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } + host := testHost() + host.Spec.StorageClassName = "other-rwx" + otherClass := &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: "other-rwx", Annotations: map[string]string{clusterv1alpha1.RWXStorageClassAnnotation: string(corev1.ReadWriteMany)}}, Provisioner: "example.invalid/csi"} + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Workspace{}, &corev1.PersistentVolumeClaim{}). + WithObjects(host, otherClass, workspace, pvc).Build() + r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} + + result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}) + if err != nil { + t.Fatal(err) + } + if result.RequeueAfter <= 0 || result.RequeueAfter > 30*time.Second { + t.Fatalf("storage drift requeue = %s, want bounded positive retry", result.RequeueAfter) + } + var got clusterv1alpha1.T4Workspace + if err := c.Get(ctx, client.ObjectKeyFromObject(workspace), &got); err != nil { + t.Fatal(err) + } + storageReady := findCondition(got.Status.Conditions, "StorageReady") + ready := findCondition(got.Status.Conditions, "Ready") + if got.Status.Phase != clusterv1alpha1.InfrastructureFailed || storageReady == nil || storageReady.Status != metav1.ConditionFalse || storageReady.Reason != "StorageClassMismatch" || ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != "StorageClassMismatch" { + t.Fatalf("storage drift remained Ready: status=%#v StorageReady=%#v Ready=%#v", got.Status, storageReady, ready) + } + var retained corev1.PersistentVolumeClaim + if err := c.Get(ctx, client.ObjectKeyFromObject(pvc), &retained); err != nil { + t.Fatalf("storage drift removed the data PVC: %v", err) + } + if retained.Spec.StorageClassName == nil || *retained.Spec.StorageClassName != oldClass { + t.Fatalf("storage drift recreated or mutated PVC class: %#v", retained.Spec.StorageClassName) + } +} + +func TestHostDependencyRecoveryReplacesFalseConditions(t *testing.T) { + t.Run("Workspace", func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Workspace{}, &corev1.PersistentVolumeClaim{}). + WithObjects(workspace).Build() + r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + if err := c.Create(ctx, testHost()); err != nil { + t.Fatal(err) + } + if err := c.Create(ctx, rwxStorageClass()); err != nil { + t.Fatal(err) + } + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + var pvc corev1.PersistentVolumeClaim + if err := c.Get(ctx, types.NamespacedName{Namespace: workspace.Namespace, Name: controllers.WorkspacePVCName(workspace)}, &pvc); err != nil { + t.Fatal(err) + } + pvc.Status.Phase = corev1.ClaimBound + if err := c.Status().Update(ctx, &pvc); err != nil { + t.Fatal(err) + } + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + var recovered clusterv1alpha1.T4Workspace + if err := c.Get(ctx, client.ObjectKeyFromObject(workspace), &recovered); err != nil { + t.Fatal(err) + } + hostReady := findCondition(recovered.Status.Conditions, "HostReady") + if hostReady == nil || hostReady.Status != metav1.ConditionTrue || hostReady.ObservedGeneration != recovered.Generation { + t.Fatalf("recovered Workspace retained stale HostReady: %#v", hostReady) + } + }) + + t.Run("Session", func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.Status.PVCName = "workspace-a-data" + pvc := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: workspace.Namespace}, Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}} + session := testSession() + session.UID = "session-uid" + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(workspace, pvc, session).Build() + r := configuredSessionReconciler(c, scheme) + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + if err := c.Create(ctx, testHost()); err != nil { + t.Fatal(err) + } + reconcileMany(t, 2, func() error { + _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + var recovered clusterv1alpha1.T4Session + if err := c.Get(ctx, client.ObjectKeyFromObject(session), &recovered); err != nil { + t.Fatal(err) + } + hostReady := findCondition(recovered.Status.Conditions, "HostReady") + if hostReady == nil || hostReady.Status != metav1.ConditionTrue || hostReady.ObservedGeneration != recovered.Generation { + t.Fatalf("recovered Session retained stale HostReady: %#v", hostReady) + } + }) +} + func configuredSessionReconciler(c client.Client, scheme *runtime.Scheme) *controllers.SessionReconciler { for _, object := range []client.Object{ &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "omp-runtime-config", Namespace: "team"}, Data: map[string]string{ From bb851a8560a0eb380830c512f0a1709a96b57bab Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 21:10:01 +0000 Subject: [PATCH 087/131] fix(operator): fail closed on host storage drift --- .../cluster-operator/controllers/helpers.go | 1 + .../controllers/reconciler_test.go | 2 +- .../controllers/session_controller.go | 41 ++++++++++++------- .../controllers/workspace_controller.go | 6 +++ 4 files changed, 34 insertions(+), 16 deletions(-) diff --git a/packages/cluster-operator/controllers/helpers.go b/packages/cluster-operator/controllers/helpers.go index ec5681af..4c88e8c2 100644 --- a/packages/cluster-operator/controllers/helpers.go +++ b/packages/cluster-operator/controllers/helpers.go @@ -18,6 +18,7 @@ import ( const ( ReasonStorageClassNotFound = "StorageClassNotFound" ReasonStorageClassNotRWX = "StorageClassNotRWX" + ReasonStorageClassMismatch = "StorageClassMismatch" ReasonStorageReady = "StorageClassSupportsRWX" ) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index 905e3896..9fca96a1 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -1316,7 +1316,7 @@ func TestSessionDependencyRevocationCleansOwnedResourcesAndConvergesAfterRestart workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() diff --git a/packages/cluster-operator/controllers/session_controller.go b/packages/cluster-operator/controllers/session_controller.go index a4cce1f2..26e516e1 100644 --- a/packages/cluster-operator/controllers/session_controller.go +++ b/packages/cluster-operator/controllers/session_controller.go @@ -307,13 +307,13 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "RuntimeConfigured", reason, message) + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, false, "RuntimeConfigured", reason, message) } if reason, message := r.OMPConfig.validationFailure(); reason != "" { if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "RuntimeConfigured", reason, message) + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, false, "RuntimeConfigured", reason, message) } var host clusterv1alpha1.T4ClusterHost @@ -322,7 +322,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "HostReady", "HostNotFound", "referenced T4ClusterHost does not exist") + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, false, "HostReady", "HostNotFound", "referenced T4ClusterHost does not exist") } return ctrl.Result{}, err } @@ -330,7 +330,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "RuntimeConfigured", "RuntimeProfileNotAllowed", "runtime profile is not allowed by the referenced T4ClusterHost") + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "RuntimeConfigured", "RuntimeProfileNotAllowed", "runtime profile is not allowed by the referenced T4ClusterHost") } var storageClass storagev1.StorageClass if err := r.Get(ctx, types.NamespacedName{Name: host.Spec.StorageClassName}, &storageClass); err != nil { @@ -338,7 +338,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", ReasonStorageClassNotFound, fmt.Sprintf("StorageClass %q selected by the referenced T4ClusterHost does not exist", host.Spec.StorageClassName)) + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", ReasonStorageClassNotFound, fmt.Sprintf("StorageClass %q selected by the referenced T4ClusterHost does not exist", host.Spec.StorageClassName)) } return ctrl.Result{}, err } @@ -346,7 +346,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", ReasonStorageClassNotRWX, fmt.Sprintf("StorageClass %q selected by the referenced T4ClusterHost is not administrator-declared ReadWriteMany", host.Spec.StorageClassName)) + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", ReasonStorageClassNotRWX, fmt.Sprintf("StorageClass %q selected by the referenced T4ClusterHost is not administrator-declared ReadWriteMany", host.Spec.StorageClassName)) } var workspace clusterv1alpha1.T4Workspace if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: session.Spec.WorkspaceRef}, &workspace); err != nil { @@ -354,7 +354,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "WorkspaceNotFound", "referenced T4Workspace does not exist") + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", "WorkspaceNotFound", "referenced T4Workspace does not exist") } return ctrl.Result{}, err } @@ -362,13 +362,13 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "HostMismatch", "session and workspace must reference the same T4ClusterHost") + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", "HostMismatch", "session and workspace must reference the same T4ClusterHost") } if workspace.Status.PVCName == "" { if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "PVCNotDeclared", "workspace controller has not declared a PVC") + return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", "PVCNotDeclared", "workspace controller has not declared a PVC") } var pvc corev1.PersistentVolumeClaim if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: workspace.Status.PVCName}, &pvc); err != nil { @@ -376,15 +376,21 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "PVCNotFound", "workspace PVC does not exist") + return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", "PVCNotFound", "workspace PVC does not exist") } return ctrl.Result{}, err } + if pvc.Spec.StorageClassName != nil && *pvc.Spec.StorageClassName != host.Spec.StorageClassName { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", ReasonStorageClassMismatch, fmt.Sprintf("workspace PVC uses StorageClass %q instead of host-selected %q", *pvc.Spec.StorageClassName, host.Spec.StorageClassName)) + } if pvc.Status.Phase != corev1.ClaimBound || !pvcHasRWX(&pvc) { if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "PVCNotBoundRWX", "workspace PVC must be Bound and ReadWriteMany before a session starts") + return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", "PVCNotBoundRWX", "workspace PVC must be Bound and ReadWriteMany before a session starts") } runtimeVersions, reason, message, err := r.loadOMPResourceVersions(ctx, session.Namespace) if err != nil { @@ -394,7 +400,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "RuntimeConfigured", reason, message) + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "RuntimeConfigured", reason, message) } serviceName := SessionServiceName(&session) @@ -424,7 +430,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) } else if err != nil { return ctrl.Result{}, err } else if !metav1.IsControlledBy(&service, &session) { - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "Available", "ServiceOwnershipConflict", "deterministic session Service is not controlled by this session") + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "Available", "ServiceOwnershipConflict", "deterministic session Service is not controlled by this session") } else if !serviceExposureIsInternal(&service) { if err := r.Delete(ctx, &service); err != nil && !apierrors.IsNotFound(err) { return ctrl.Result{}, err @@ -458,7 +464,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) } else if err != nil { return ctrl.Result{}, err } else if !metav1.IsControlledBy(&pod, &session) { - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "Available", "PodOwnershipConflict", "deterministic session Pod is not controlled by this session") + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "Available", "PodOwnershipConflict", "deterministic session Pod is not controlled by this session") } else if !labelsContain(pod.Labels, desiredPod.Labels) { if pod.Labels == nil { pod.Labels = map[string]string{} @@ -490,6 +496,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) session.Status.ObservedGeneration = session.Generation session.Status.PodName = podName session.Status.ServiceName = serviceName + meta.SetStatusCondition(&session.Status.Conditions, condition("HostReady", metav1.ConditionTrue, "HostResolved", "referenced T4ClusterHost is available", session.Generation)) meta.SetStatusCondition(&session.Status.Conditions, condition("WorkspaceReady", metav1.ConditionTrue, "PVCBoundRWX", "workspace PVC is Bound and ReadWriteMany", session.Generation)) meta.SetStatusCondition(&session.Status.Conditions, condition("RuntimeConfigured", metav1.ConditionTrue, "OMPReferencesReady", "administrator-owned OMP runtime references are configured", session.Generation)) if podReady(&pod) { @@ -646,6 +653,7 @@ func (r *SessionReconciler) updateSessionPending(ctx context.Context, session *c original.Conditions = append([]metav1.Condition(nil), session.Status.Conditions...) } session.Status.ObservedGeneration = session.Generation + meta.SetStatusCondition(&session.Status.Conditions, condition("HostReady", metav1.ConditionTrue, "HostResolved", "referenced T4ClusterHost is available", session.Generation)) session.Status.PodName = podName session.Status.ServiceName = serviceName session.Status.Phase = clusterv1alpha1.InfrastructurePending @@ -736,11 +744,14 @@ func (r *SessionReconciler) deleteOwnedSessionResources(ctx context.Context, ses return nil } -func (r *SessionReconciler) updateSessionFailure(ctx context.Context, session *clusterv1alpha1.T4Session, conditionType, reason, message string) error { +func (r *SessionReconciler) updateSessionFailure(ctx context.Context, session *clusterv1alpha1.T4Session, hostReady bool, conditionType, reason, message string) error { original := session.Status if session.Status.Conditions != nil { original.Conditions = append([]metav1.Condition(nil), session.Status.Conditions...) } + if hostReady { + meta.SetStatusCondition(&session.Status.Conditions, condition("HostReady", metav1.ConditionTrue, "HostResolved", "referenced T4ClusterHost is available", session.Generation)) + } session.Status.ObservedGeneration = session.Generation session.Status.PodName = "" session.Status.ServiceName = "" diff --git a/packages/cluster-operator/controllers/workspace_controller.go b/packages/cluster-operator/controllers/workspace_controller.go index e96271a6..97d95819 100644 --- a/packages/cluster-operator/controllers/workspace_controller.go +++ b/packages/cluster-operator/controllers/workspace_controller.go @@ -105,6 +105,8 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, request ctrl.Reques return ctrl.Result{}, err } else if !workspaceOwnsPVC(&workspace, &pvc) { return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCOwnershipConflict", "deterministic workspace PVC does not belong to this workspace") + } else if pvc.Spec.StorageClassName != nil && *pvc.Spec.StorageClassName != storageClassName { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", ReasonStorageClassMismatch, fmt.Sprintf("workspace PVC uses StorageClass %q instead of host-selected %q; data-bearing PVCs are never recreated automatically", *pvc.Spec.StorageClassName, storageClassName)) } else if workspace.Spec.RetentionPolicy == clusterv1alpha1.RetentionPolicyRetain && metav1.IsControlledBy(&pvc, &workspace) { before := pvc.DeepCopy() pvc.OwnerReferences = removeWorkspaceOwnerReference(pvc.OwnerReferences, workspace.UID) @@ -126,6 +128,7 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, request ctrl.Reques workspace.Status.PVCPhase = pvc.Status.Phase capacity := pvc.Status.Capacity[corev1.ResourceStorage] workspace.Status.Capacity = capacity.DeepCopy() + meta.SetStatusCondition(&workspace.Status.Conditions, condition("HostReady", metav1.ConditionTrue, "HostResolved", "referenced T4ClusterHost is available", workspace.Generation)) meta.SetStatusCondition(&workspace.Status.Conditions, condition("StorageReady", metav1.ConditionTrue, ReasonStorageReady, "RWX StorageClass and workspace PVC are accepted", workspace.Generation)) switch pvc.Status.Phase { case corev1.ClaimBound: @@ -267,6 +270,9 @@ func (r *WorkspaceReconciler) updateWorkspaceFailure(ctx context.Context, worksp if workspace.Status.Conditions != nil { original.Conditions = append([]metav1.Condition(nil), workspace.Status.Conditions...) } + if conditionType != "HostReady" { + meta.SetStatusCondition(&workspace.Status.Conditions, condition("HostReady", metav1.ConditionTrue, "HostResolved", "referenced T4ClusterHost is available", workspace.Generation)) + } workspace.Status.ObservedGeneration = workspace.Generation workspace.Status.Phase = clusterv1alpha1.InfrastructureFailed meta.SetStatusCondition(&workspace.Status.Conditions, condition(conditionType, metav1.ConditionFalse, reason, message, workspace.Generation)) From 4ccae52d2063668a7c9a681f85b7b685b71d452f Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 21:20:38 +0000 Subject: [PATCH 088/131] fix(operator): require explicit workspace storage class --- .../cluster-operator/controllers/helpers.go | 7 + .../controllers/reconciler_test.go | 171 +++++++++++------- .../controllers/session_controller.go | 6 +- .../controllers/workspace_controller.go | 4 +- 4 files changed, 120 insertions(+), 68 deletions(-) diff --git a/packages/cluster-operator/controllers/helpers.go b/packages/cluster-operator/controllers/helpers.go index 4c88e8c2..8fef978c 100644 --- a/packages/cluster-operator/controllers/helpers.go +++ b/packages/cluster-operator/controllers/helpers.go @@ -83,6 +83,13 @@ func pvcHasRWX(pvc *corev1.PersistentVolumeClaim) bool { return false } +func pvcStorageClassName(pvc *corev1.PersistentVolumeClaim) string { + if pvc.Spec.StorageClassName == nil { + return "" + } + return *pvc.Spec.StorageClassName +} + func hasString(values []string, wanted string) bool { for _, value := range values { if value == wanted { diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index 9fca96a1..8342d571 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -153,7 +153,7 @@ func TestRetainDeletionOrphansPVCBeforeRemovingFinalizer(t *testing.T) { APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Workspace", Name: workspace.Name, UID: workspace.UID, Controller: ptr(true), }}, }, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, } c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Workspace{}).WithObjects(testHost(), rwxStorageClass(), workspace, pvc).Build() if err := c.Delete(context.Background(), workspace); err != nil { @@ -187,7 +187,7 @@ func TestWorkspaceDeletionWaitsForSessionResources(t *testing.T) { Name: controllers.WorkspacePVCName(workspace), Namespace: workspace.Namespace, OwnerReferences: []metav1.OwnerReference{{APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Workspace", Name: workspace.Name, UID: workspace.UID, Controller: ptr(true)}}, }, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, } session := testSession() session.Spec.WorkspaceRef = workspace.Name @@ -457,7 +457,7 @@ func TestSessionRuntimeImageMustBeImmutableDigest(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -528,7 +528,7 @@ func TestSessionAuthorityRevocationDeletesOwnedPodAndService(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -575,7 +575,7 @@ func TestSessionNamesProduceSafeRuntimeIdentities(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -620,7 +620,7 @@ func TestSessionWaitsForBoundRWXThenCreatesExactlyOnePodAndService(t *testing.T) workspace.Status.Phase = clusterv1alpha1.InfrastructurePending pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimPending}, } session := testSession() @@ -755,7 +755,7 @@ func TestSessionOMPModeOmitsCredentialSecretReference(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -791,7 +791,7 @@ func TestSessionRejectsUnownedDeterministicResources(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -826,7 +826,7 @@ func TestSessionRecreatesPodWhenImmutableDesiredStateChanges(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -881,7 +881,7 @@ func TestSessionPodHashIncludesEveryOMPReference(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -927,7 +927,7 @@ func TestSessionRecreatesPodWhenOMPResourceVersionChanges(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -970,7 +970,7 @@ func TestSessionRuntimeReferenceRevocationStopsAuthority(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -1024,7 +1024,7 @@ func TestSessionFailsClosedWhenOMPConfigMapIsMissing(t *testing.T) { scheme := testScheme(t) workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) workspace.Status.PVCName = "workspace-a-data" - pvc := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}} + pvc := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}} session := testSession() objects := []client.Object{testHost(), rwxStorageClass(), workspace, pvc, session} if test.configMap != nil { @@ -1054,7 +1054,7 @@ func TestSessionRecreatesExternallyExposedOwnedService(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -1102,7 +1102,7 @@ func TestSessionRestoresRequiredPodSelectorLabelsBeforeAvailability(t *testing.T workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -1308,6 +1308,14 @@ func TestSessionDependencyRevocationCleansOwnedResourcesAndConvergesAfterRestart host.Spec.StorageClassName = otherClass.Name return c.Update(ctx, &host) }}, + {name: "classless Workspace PVC", conditionType: "WorkspaceReady", wantReason: "StorageClassMismatch", revoke: func(ctx context.Context, c client.Client) error { + var pvc corev1.PersistentVolumeClaim + if err := c.Get(ctx, types.NamespacedName{Namespace: "team", Name: "workspace-a-data"}, &pvc); err != nil { + return err + } + pvc.Spec.StorageClassName = nil + return c.Update(ctx, &pvc) + }}, } { t.Run(test.name, func(t *testing.T) { ctx := context.Background() @@ -1384,57 +1392,67 @@ func TestSessionDependencyRevocationCleansOwnedResourcesAndConvergesAfterRestart } func TestWorkspaceHostStorageClassDriftFailsClosedWithoutRecreatingPVC(t *testing.T) { - ctx := context.Background() - scheme := testScheme(t) - workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) - workspace.UID = "workspace-uid" - workspace.Status.ObservedGeneration = workspace.Generation - workspace.Status.PVCName = controllers.WorkspacePVCName(workspace) - workspace.Status.Phase = clusterv1alpha1.InfrastructureReady - workspace.Status.Conditions = []metav1.Condition{ - {Type: "StorageReady", Status: metav1.ConditionTrue, Reason: controllers.ReasonStorageReady, ObservedGeneration: workspace.Generation}, - {Type: "Ready", Status: metav1.ConditionTrue, Reason: "PVCBound", ObservedGeneration: workspace.Generation}, - } oldClass := "portable-rwx" - pvc := &corev1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: workspace.Status.PVCName, Namespace: workspace.Namespace, - Annotations: map[string]string{clusterv1alpha1.WorkspaceUIDAnnotation: string(workspace.UID)}, - OwnerReferences: []metav1.OwnerReference{{APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Workspace", Name: workspace.Name, UID: workspace.UID, Controller: ptr(true)}}, - }, - Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: &oldClass, AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, - Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, - } - host := testHost() - host.Spec.StorageClassName = "other-rwx" - otherClass := &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: "other-rwx", Annotations: map[string]string{clusterv1alpha1.RWXStorageClassAnnotation: string(corev1.ReadWriteMany)}}, Provisioner: "example.invalid/csi"} - c := fake.NewClientBuilder().WithScheme(scheme). - WithStatusSubresource(&clusterv1alpha1.T4Workspace{}, &corev1.PersistentVolumeClaim{}). - WithObjects(host, otherClass, workspace, pvc).Build() - r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} + for _, test := range []struct { + name string + claimClass *string + }{ + {name: "different class", claimClass: &oldClass}, + {name: "class omitted"}, + } { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + workspace.Status.ObservedGeneration = workspace.Generation + workspace.Status.PVCName = controllers.WorkspacePVCName(workspace) + workspace.Status.Phase = clusterv1alpha1.InfrastructureReady + workspace.Status.Conditions = []metav1.Condition{ + {Type: "StorageReady", Status: metav1.ConditionTrue, Reason: controllers.ReasonStorageReady, ObservedGeneration: workspace.Generation}, + {Type: "Ready", Status: metav1.ConditionTrue, Reason: "PVCBound", ObservedGeneration: workspace.Generation}, + } + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: workspace.Status.PVCName, Namespace: workspace.Namespace, + Annotations: map[string]string{clusterv1alpha1.WorkspaceUIDAnnotation: string(workspace.UID)}, + OwnerReferences: []metav1.OwnerReference{{APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Workspace", Name: workspace.Name, UID: workspace.UID, Controller: ptr(true)}}, + }, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: test.claimClass, AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } + host := testHost() + host.Spec.StorageClassName = "other-rwx" + otherClass := &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: "other-rwx", Annotations: map[string]string{clusterv1alpha1.RWXStorageClassAnnotation: string(corev1.ReadWriteMany)}}, Provisioner: "example.invalid/csi"} + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Workspace{}, &corev1.PersistentVolumeClaim{}). + WithObjects(host, otherClass, workspace, pvc).Build() + r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} - result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}) - if err != nil { - t.Fatal(err) - } - if result.RequeueAfter <= 0 || result.RequeueAfter > 30*time.Second { - t.Fatalf("storage drift requeue = %s, want bounded positive retry", result.RequeueAfter) - } - var got clusterv1alpha1.T4Workspace - if err := c.Get(ctx, client.ObjectKeyFromObject(workspace), &got); err != nil { - t.Fatal(err) - } - storageReady := findCondition(got.Status.Conditions, "StorageReady") - ready := findCondition(got.Status.Conditions, "Ready") - if got.Status.Phase != clusterv1alpha1.InfrastructureFailed || storageReady == nil || storageReady.Status != metav1.ConditionFalse || storageReady.Reason != "StorageClassMismatch" || ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != "StorageClassMismatch" { - t.Fatalf("storage drift remained Ready: status=%#v StorageReady=%#v Ready=%#v", got.Status, storageReady, ready) - } - var retained corev1.PersistentVolumeClaim - if err := c.Get(ctx, client.ObjectKeyFromObject(pvc), &retained); err != nil { - t.Fatalf("storage drift removed the data PVC: %v", err) - } - if retained.Spec.StorageClassName == nil || *retained.Spec.StorageClassName != oldClass { - t.Fatalf("storage drift recreated or mutated PVC class: %#v", retained.Spec.StorageClassName) + result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}) + if err != nil { + t.Fatal(err) + } + if result.RequeueAfter <= 0 || result.RequeueAfter > 30*time.Second { + t.Fatalf("storage drift requeue = %s, want bounded positive retry", result.RequeueAfter) + } + var got clusterv1alpha1.T4Workspace + if err := c.Get(ctx, client.ObjectKeyFromObject(workspace), &got); err != nil { + t.Fatal(err) + } + storageReady := findCondition(got.Status.Conditions, "StorageReady") + ready := findCondition(got.Status.Conditions, "Ready") + if got.Status.Phase != clusterv1alpha1.InfrastructureFailed || storageReady == nil || storageReady.Status != metav1.ConditionFalse || storageReady.Reason != "StorageClassMismatch" || ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != "StorageClassMismatch" { + t.Fatalf("storage drift remained Ready: status=%#v StorageReady=%#v Ready=%#v", got.Status, storageReady, ready) + } + var retained corev1.PersistentVolumeClaim + if err := c.Get(ctx, client.ObjectKeyFromObject(pvc), &retained); err != nil { + t.Fatalf("storage drift removed the data PVC: %v", err) + } + if !reflect.DeepEqual(retained.Spec.StorageClassName, test.claimClass) { + t.Fatalf("storage drift recreated or mutated PVC class: got %#v, want %#v", retained.Spec.StorageClassName, test.claimClass) + } + }) } } @@ -1512,6 +1530,31 @@ func TestHostDependencyRecoveryReplacesFalseConditions(t *testing.T) { t.Fatalf("recovered Session retained stale HostReady: %#v", hostReady) } }) + + t.Run("static runtime failure", func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + session := testSession() + session.Status.ObservedGeneration = session.Generation + session.Status.Phase = clusterv1alpha1.InfrastructureFailed + session.Status.Conditions = []metav1.Condition{{Type: "HostReady", Status: metav1.ConditionFalse, Reason: "HostNotFound", ObservedGeneration: session.Generation}} + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}). + WithObjects(testHost(), session).Build() + r := configuredSessionReconciler(c, scheme) + r.RuntimeImage = "registry.example/session:latest" + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + var failed clusterv1alpha1.T4Session + if err := c.Get(ctx, client.ObjectKeyFromObject(session), &failed); err != nil { + t.Fatal(err) + } + hostReady := findCondition(failed.Status.Conditions, "HostReady") + if hostReady == nil || hostReady.Status != metav1.ConditionUnknown || hostReady.Reason != "NotEvaluated" || hostReady.ObservedGeneration != failed.Generation { + t.Fatalf("static runtime failure retained stale HostReady: %#v", hostReady) + } + }) } func configuredSessionReconciler(c client.Client, scheme *runtime.Scheme) *controllers.SessionReconciler { diff --git a/packages/cluster-operator/controllers/session_controller.go b/packages/cluster-operator/controllers/session_controller.go index 26e516e1..9b4318b8 100644 --- a/packages/cluster-operator/controllers/session_controller.go +++ b/packages/cluster-operator/controllers/session_controller.go @@ -380,11 +380,11 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) } return ctrl.Result{}, err } - if pvc.Spec.StorageClassName != nil && *pvc.Spec.StorageClassName != host.Spec.StorageClassName { + if pvcStorageClassName(&pvc) != host.Spec.StorageClassName { if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", ReasonStorageClassMismatch, fmt.Sprintf("workspace PVC uses StorageClass %q instead of host-selected %q", *pvc.Spec.StorageClassName, host.Spec.StorageClassName)) + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", ReasonStorageClassMismatch, fmt.Sprintf("workspace PVC uses StorageClass %q instead of host-selected %q", pvcStorageClassName(&pvc), host.Spec.StorageClassName)) } if pvc.Status.Phase != corev1.ClaimBound || !pvcHasRWX(&pvc) { if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { @@ -751,6 +751,8 @@ func (r *SessionReconciler) updateSessionFailure(ctx context.Context, session *c } if hostReady { meta.SetStatusCondition(&session.Status.Conditions, condition("HostReady", metav1.ConditionTrue, "HostResolved", "referenced T4ClusterHost is available", session.Generation)) + } else if conditionType != "HostReady" { + meta.SetStatusCondition(&session.Status.Conditions, condition("HostReady", metav1.ConditionUnknown, "NotEvaluated", "host dependency was not evaluated because static runtime configuration is invalid", session.Generation)) } session.Status.ObservedGeneration = session.Generation session.Status.PodName = "" diff --git a/packages/cluster-operator/controllers/workspace_controller.go b/packages/cluster-operator/controllers/workspace_controller.go index 97d95819..da1b555b 100644 --- a/packages/cluster-operator/controllers/workspace_controller.go +++ b/packages/cluster-operator/controllers/workspace_controller.go @@ -105,8 +105,8 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, request ctrl.Reques return ctrl.Result{}, err } else if !workspaceOwnsPVC(&workspace, &pvc) { return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCOwnershipConflict", "deterministic workspace PVC does not belong to this workspace") - } else if pvc.Spec.StorageClassName != nil && *pvc.Spec.StorageClassName != storageClassName { - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", ReasonStorageClassMismatch, fmt.Sprintf("workspace PVC uses StorageClass %q instead of host-selected %q; data-bearing PVCs are never recreated automatically", *pvc.Spec.StorageClassName, storageClassName)) + } else if pvcStorageClassName(&pvc) != storageClassName { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", ReasonStorageClassMismatch, fmt.Sprintf("workspace PVC uses StorageClass %q instead of host-selected %q; data-bearing PVCs are never recreated automatically", pvcStorageClassName(&pvc), storageClassName)) } else if workspace.Spec.RetentionPolicy == clusterv1alpha1.RetentionPolicyRetain && metav1.IsControlledBy(&pvc, &workspace) { before := pvc.DeepCopy() pvc.OwnerReferences = removeWorkspaceOwnerReference(pvc.OwnerReferences, workspace.UID) From 30752e273db257440e150aa51df2b81b13739783 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 21:40:01 +0000 Subject: [PATCH 089/131] test(operator): reject stale dependency authority --- .../controllers/reconciler_test.go | 299 +++++++++++++++++- 1 file changed, 297 insertions(+), 2 deletions(-) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index 8342d571..3f51cba1 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -1144,7 +1144,7 @@ func TestSessionRestoresRequiredPodSelectorLabelsBeforeAvailability(t *testing.T func TestWorkspaceDeletionRefusesForeignDeterministicPVC(t *testing.T) { for _, policy := range []clusterv1alpha1.RetentionPolicy{clusterv1alpha1.RetentionPolicyRetain, clusterv1alpha1.RetentionPolicyDelete} { - for _, mismatch := range []string{"uid-annotation", "controller-owner"} { + for _, mismatch := range []string{"uid-annotation", "controller-owner", "foreign-non-controller-owner"} { t.Run(string(policy)+"/"+mismatch, func(t *testing.T) { scheme := testScheme(t) workspace := testWorkspace(policy) @@ -1156,10 +1156,14 @@ func TestWorkspaceDeletionRefusesForeignDeterministicPVC(t *testing.T) { }} if mismatch == "uid-annotation" { pvc.Annotations[clusterv1alpha1.WorkspaceUIDAnnotation] = "foreign-workspace-uid" - } else { + } else if mismatch == "controller-owner" { pvc.OwnerReferences = []metav1.OwnerReference{{ APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Workspace", Name: "foreign", UID: "foreign-workspace-uid", Controller: ptr(true), }} + } else { + pvc.OwnerReferences = []metav1.OwnerReference{{ + APIVersion: "example.test/v1", Kind: "Foreign", Name: "foreign", UID: "foreign-uid", + }} } expectedOwnerCount := len(pvc.OwnerReferences) c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Workspace{}).WithObjects(workspace, pvc).Build() @@ -1557,6 +1561,297 @@ func TestHostDependencyRecoveryReplacesFalseConditions(t *testing.T) { }) } +func TestWorkspaceFailureRevokesPublishedPVCAuthorityAndRefreshesConditions(t *testing.T) { + for _, test := range []struct { + name string + objects func(*clusterv1alpha1.T4Workspace) []client.Object + wantHostStatus metav1.ConditionStatus + wantStorageReason string + }{ + { + name: "missing Host", + objects: func(workspace *clusterv1alpha1.T4Workspace) []client.Object { + return []client.Object{workspace} + }, + wantHostStatus: metav1.ConditionFalse, + wantStorageReason: "NotEvaluated", + }, + { + name: "missing StorageClass", + objects: func(workspace *clusterv1alpha1.T4Workspace) []client.Object { + return []client.Object{testHost(), workspace} + }, + wantHostStatus: metav1.ConditionTrue, + wantStorageReason: controllers.ReasonStorageClassNotFound, + }, + { + name: "non-RWX StorageClass", + objects: func(workspace *clusterv1alpha1.T4Workspace) []client.Object { + class := rwxStorageClass() + class.Annotations = nil + return []client.Object{testHost(), class, workspace} + }, + wantHostStatus: metav1.ConditionTrue, + wantStorageReason: controllers.ReasonStorageClassNotRWX, + }, + { + name: "PVC class drift", + objects: func(workspace *clusterv1alpha1.T4Workspace) []client.Object { + pvc := ownedWorkspacePVC(workspace) + pvc.Spec.StorageClassName = ptr("old-rwx") + return []client.Object{testHost(), rwxStorageClass(), workspace, pvc} + }, + wantHostStatus: metav1.ConditionTrue, + wantStorageReason: controllers.ReasonStorageClassMismatch, + }, + { + name: "PVC ownership conflict", + objects: func(workspace *clusterv1alpha1.T4Workspace) []client.Object { + pvc := ownedWorkspacePVC(workspace) + pvc.OwnerReferences = []metav1.OwnerReference{{APIVersion: "example.test/v1", Kind: "Foreign", Name: "foreign", UID: "foreign-uid", Controller: ptr(true)}} + return []client.Object{testHost(), rwxStorageClass(), workspace, pvc} + }, + wantHostStatus: metav1.ConditionTrue, + wantStorageReason: "PVCOwnershipConflict", + }, + } { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + workspace.Generation = 7 + workspace.Status.ObservedGeneration = 6 + workspace.Status.PVCName = controllers.WorkspacePVCName(workspace) + workspace.Status.PVCPhase = corev1.ClaimBound + workspace.Status.Capacity = apiresource.MustParse("10Gi") + workspace.Status.Phase = clusterv1alpha1.InfrastructureReady + workspace.Status.Conditions = []metav1.Condition{ + {Type: "HostReady", Status: metav1.ConditionTrue, Reason: "Stale", ObservedGeneration: 6}, + {Type: "StorageReady", Status: metav1.ConditionTrue, Reason: "Stale", ObservedGeneration: 6}, + {Type: "Ready", Status: metav1.ConditionTrue, Reason: "Stale", ObservedGeneration: 6}, + } + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Workspace{}, &corev1.PersistentVolumeClaim{}).WithObjects(test.objects(workspace)...).Build() + r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + var failed clusterv1alpha1.T4Workspace + if err := c.Get(ctx, client.ObjectKeyFromObject(workspace), &failed); err != nil { + t.Fatal(err) + } + if failed.Status.PVCName != "" || failed.Status.PVCPhase != "" || !failed.Status.Capacity.IsZero() { + t.Fatalf("failed Workspace retained PVC authority: %#v", failed.Status) + } + hostReady := findCondition(failed.Status.Conditions, "HostReady") + storageReady := findCondition(failed.Status.Conditions, "StorageReady") + ready := findCondition(failed.Status.Conditions, "Ready") + if hostReady == nil || hostReady.Status != test.wantHostStatus || hostReady.ObservedGeneration != failed.Generation || + storageReady == nil || storageReady.Status == metav1.ConditionTrue || storageReady.Reason != test.wantStorageReason || storageReady.ObservedGeneration != failed.Generation || + ready == nil || ready.Status != metav1.ConditionFalse || ready.ObservedGeneration != failed.Generation { + t.Fatalf("failure conditions are stale: HostReady=%#v StorageReady=%#v Ready=%#v", hostReady, storageReady, ready) + } + }) + } +} + +func TestSessionRejectsWorkspacePVCWithoutExactIdentityAndOwnership(t *testing.T) { + for _, test := range []struct { + name string + mutatePVC func(*clusterv1alpha1.T4Workspace, *corev1.PersistentVolumeClaim) + }{ + {name: "foreign deterministic PVC", mutatePVC: func(_ *clusterv1alpha1.T4Workspace, pvc *corev1.PersistentVolumeClaim) { + pvc.OwnerReferences = []metav1.OwnerReference{{APIVersion: "example.test/v1", Kind: "Foreign", Name: "foreign", UID: "foreign-uid", Controller: ptr(true)}} + }}, + {name: "tampered Workspace status name", mutatePVC: func(workspace *clusterv1alpha1.T4Workspace, pvc *corev1.PersistentVolumeClaim) { + workspace.Status.PVCName = "foreign-data" + pvc.Name = workspace.Status.PVCName + }}, + } { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + workspace.Status.PVCName = controllers.WorkspacePVCName(workspace) + pvc := ownedWorkspacePVC(workspace) + test.mutatePVC(workspace, pvc) + session := testSession() + session.UID = "session-uid" + pod, service := ownedSessionResources(session) + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(testHost(), workspace, pvc, session, pod, service).Build() + r := configuredSessionReconciler(c, scheme) + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + assertObjectCounts(t, c, 0, 0) + var failed clusterv1alpha1.T4Session + if err := c.Get(ctx, client.ObjectKeyFromObject(session), &failed); err != nil { + t.Fatal(err) + } + condition := findCondition(failed.Status.Conditions, "WorkspaceReady") + if failed.Status.PodName != "" || failed.Status.ServiceName != "" || condition == nil || condition.Status != metav1.ConditionFalse || condition.ObservedGeneration != failed.Generation { + t.Fatalf("untrusted Workspace PVC retained session authority: status=%#v WorkspaceReady=%#v", failed.Status, condition) + } + }) + } +} + +func TestSessionFailureAndPendingRefreshEveryCondition(t *testing.T) { + t.Run("failure", func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + session := testSession() + session.Generation = 8 + session.Status.ObservedGeneration = 7 + session.Status.Conditions = staleSessionConditions(7) + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Session{}).WithObjects(session).Build() + r := configuredSessionReconciler(c, scheme) + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + var failed clusterv1alpha1.T4Session + if err := c.Get(ctx, client.ObjectKeyFromObject(session), &failed); err != nil { + t.Fatal(err) + } + assertCurrentSessionConditions(t, &failed, map[string]metav1.ConditionStatus{ + "HostReady": metav1.ConditionFalse, "WorkspaceReady": metav1.ConditionUnknown, "RuntimeConfigured": metav1.ConditionUnknown, "Available": metav1.ConditionFalse, + }) + }) + + t.Run("pending", func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + workspace.Status.PVCName = controllers.WorkspacePVCName(workspace) + pvc := ownedWorkspacePVC(workspace) + session := testSession() + session.UID = "session-uid" + session.Generation = 8 + session.Status.ObservedGeneration = 7 + session.Status.Conditions = staleSessionConditions(7) + _, service := ownedSessionResources(session) + service.Spec.Type = corev1.ServiceTypeNodePort + service.Spec.Ports = []corev1.ServicePort{{Name: "host", Port: 8787, NodePort: 32080}} + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(testHost(), workspace, pvc, session, service).Build() + r := configuredSessionReconciler(c, scheme) + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + var pending clusterv1alpha1.T4Session + if err := c.Get(ctx, client.ObjectKeyFromObject(session), &pending); err != nil { + t.Fatal(err) + } + assertCurrentSessionConditions(t, &pending, map[string]metav1.ConditionStatus{ + "HostReady": metav1.ConditionTrue, "WorkspaceReady": metav1.ConditionTrue, "RuntimeConfigured": metav1.ConditionTrue, "Available": metav1.ConditionFalse, + }) + }) +} + +func TestSessionResourcesWithAnyForeignOwnerFailClosed(t *testing.T) { + for _, path := range []string{"normal", "dependency-cleanup"} { + for _, kind := range []string{"Pod", "Service"} { + t.Run(path+"/"+kind, func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + session := testSession() + session.UID = "session-uid" + pod, service := ownedSessionResources(session) + foreignOwner := metav1.OwnerReference{APIVersion: "example.test/v1", Kind: "Foreign", Name: "foreign", UID: "foreign-uid"} + if kind == "Pod" { + pod.OwnerReferences = append(pod.OwnerReferences, foreignOwner) + } else { + service.OwnerReferences = append(service.OwnerReferences, foreignOwner) + } + objects := []client.Object{session, pod, service} + if path == "normal" { + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + workspace.Status.PVCName = controllers.WorkspacePVCName(workspace) + objects = append(objects, testHost(), workspace, ownedWorkspacePVC(workspace)) + } + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(objects...).Build() + r := configuredSessionReconciler(c, scheme) + beforePod := pod.DeepCopy() + beforeService := service.DeepCopy() + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + if kind == "Pod" { + var got corev1.Pod + if err := c.Get(ctx, client.ObjectKeyFromObject(pod), &got); err != nil { + t.Fatalf("foreign-owned Pod was deleted: %v", err) + } + if !reflect.DeepEqual(got.ObjectMeta, beforePod.ObjectMeta) || !reflect.DeepEqual(got.Spec, beforePod.Spec) { + t.Fatalf("Pod with foreign OwnerReference was mutated: %#v", got) + } + } else { + var got corev1.Service + if err := c.Get(ctx, client.ObjectKeyFromObject(service), &got); err != nil { + t.Fatalf("foreign-owned Service was deleted: %v", err) + } + if !reflect.DeepEqual(got.ObjectMeta, beforeService.ObjectMeta) || !reflect.DeepEqual(got.Spec, beforeService.Spec) { + t.Fatalf("Service with foreign OwnerReference was mutated: %#v", got) + } + } + var failed clusterv1alpha1.T4Session + if err := c.Get(ctx, client.ObjectKeyFromObject(session), &failed); err != nil { + t.Fatal(err) + } + available := findCondition(failed.Status.Conditions, "Available") + if available == nil || available.Status != metav1.ConditionFalse || !strings.Contains(available.Reason, "OwnershipConflict") { + t.Fatalf("foreign owner did not produce stable ownership conflict: %#v", available) + } + }) + } + } +} + +func ownedWorkspacePVC(workspace *clusterv1alpha1.T4Workspace) *corev1.PersistentVolumeClaim { + return &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: controllers.WorkspacePVCName(workspace), Namespace: workspace.Namespace, + Annotations: map[string]string{clusterv1alpha1.WorkspaceUIDAnnotation: string(workspace.UID)}, + OwnerReferences: []metav1.OwnerReference{{APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Workspace", Name: workspace.Name, UID: workspace.UID, Controller: ptr(true)}}, + }, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } +} + +func ownedSessionResources(session *clusterv1alpha1.T4Session) (*corev1.Pod, *corev1.Service) { + owner := metav1.OwnerReference{APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Session", Name: session.Name, UID: session.UID, Controller: ptr(true)} + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionPodName(session), Namespace: session.Namespace, OwnerReferences: []metav1.OwnerReference{owner}}} + service := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionServiceName(session), Namespace: session.Namespace, OwnerReferences: []metav1.OwnerReference{owner}}} + return pod, service +} + +func staleSessionConditions(generation int64) []metav1.Condition { + return []metav1.Condition{ + {Type: "HostReady", Status: metav1.ConditionTrue, Reason: "Stale", ObservedGeneration: generation}, + {Type: "WorkspaceReady", Status: metav1.ConditionTrue, Reason: "Stale", ObservedGeneration: generation}, + {Type: "RuntimeConfigured", Status: metav1.ConditionTrue, Reason: "Stale", ObservedGeneration: generation}, + {Type: "Available", Status: metav1.ConditionTrue, Reason: "Stale", ObservedGeneration: generation}, + } +} + +func assertCurrentSessionConditions(t *testing.T, session *clusterv1alpha1.T4Session, want map[string]metav1.ConditionStatus) { + t.Helper() + for conditionType, status := range want { + condition := findCondition(session.Status.Conditions, conditionType) + if condition == nil || condition.Status != status || condition.ObservedGeneration != session.Generation { + t.Fatalf("%s = %#v, want %s at generation %d", conditionType, condition, status, session.Generation) + } + } +} + func configuredSessionReconciler(c client.Client, scheme *runtime.Scheme) *controllers.SessionReconciler { for _, object := range []client.Object{ &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "omp-runtime-config", Namespace: "team"}, Data: map[string]string{ From 95da911ad8f262814c515292d147d05360165b50 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 21:47:16 +0000 Subject: [PATCH 090/131] fix(operator): revoke untrusted dependency authority --- .../controllers/reconciler_test.go | 2 +- .../controllers/session_controller.go | 106 +++++++++++++----- .../controllers/workspace_controller.go | 22 +++- 3 files changed, 95 insertions(+), 35 deletions(-) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index 3f51cba1..564e4211 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -1829,7 +1829,7 @@ func ownedWorkspacePVC(workspace *clusterv1alpha1.T4Workspace) *corev1.Persisten func ownedSessionResources(session *clusterv1alpha1.T4Session) (*corev1.Pod, *corev1.Service) { owner := metav1.OwnerReference{APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Session", Name: session.Name, UID: session.UID, Controller: ptr(true)} pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionPodName(session), Namespace: session.Namespace, OwnerReferences: []metav1.OwnerReference{owner}}} - service := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionServiceName(session), Namespace: session.Namespace, OwnerReferences: []metav1.OwnerReference{owner}}} + service := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionServiceName(session), Namespace: session.Namespace, OwnerReferences: []metav1.OwnerReference{owner}}, Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeClusterIP}} return pod, service } diff --git a/packages/cluster-operator/controllers/session_controller.go b/packages/cluster-operator/controllers/session_controller.go index 9b4318b8..df4c492c 100644 --- a/packages/cluster-operator/controllers/session_controller.go +++ b/packages/cluster-operator/controllers/session_controller.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "encoding/json" + "errors" "fmt" "net/url" "reflect" @@ -42,6 +43,8 @@ const ( sessionWorkspaceRefIndexField = "t4.session.spec.workspaceRef" ) +var errSessionResourceOwnershipConflict = errors.New("session resource ownership conflict") + var ( configMapKeyPattern = regexp.MustCompile(`^[-._A-Za-z0-9]+$`) runtimeImagePattern = regexp.MustCompile(`^(?:(?:[A-Za-z0-9](?:[A-Za-z0-9.-]*[A-Za-z0-9])?|\[[A-Fa-f0-9:]+\])(?::[0-9]+)?/)?[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*(?:/[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*)*@sha256:[a-f0-9]{64}$`) @@ -289,6 +292,10 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) var session clusterv1alpha1.T4Session found := false defer func() { + if errors.Is(err, errSessionResourceOwnershipConflict) { + result = ctrl.Result{RequeueAfter: 30 * time.Second} + err = nil + } observeReconcile(metricKindSession, request.NamespacedName, session.Status.Conditions, conditionObjectPresent(&session, found, err), err) }() if err := r.Get(ctx, request.NamespacedName, &session); err != nil { @@ -307,13 +314,13 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, false, "RuntimeConfigured", reason, message) + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, false, false, "RuntimeConfigured", reason, message) } if reason, message := r.OMPConfig.validationFailure(); reason != "" { if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, false, "RuntimeConfigured", reason, message) + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, false, false, "RuntimeConfigured", reason, message) } var host clusterv1alpha1.T4ClusterHost @@ -322,7 +329,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, false, "HostReady", "HostNotFound", "referenced T4ClusterHost does not exist") + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, false, false, "HostReady", "HostNotFound", "referenced T4ClusterHost does not exist") } return ctrl.Result{}, err } @@ -330,7 +337,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "RuntimeConfigured", "RuntimeProfileNotAllowed", "runtime profile is not allowed by the referenced T4ClusterHost") + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "RuntimeConfigured", "RuntimeProfileNotAllowed", "runtime profile is not allowed by the referenced T4ClusterHost") } var storageClass storagev1.StorageClass if err := r.Get(ctx, types.NamespacedName{Name: host.Spec.StorageClassName}, &storageClass); err != nil { @@ -338,7 +345,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", ReasonStorageClassNotFound, fmt.Sprintf("StorageClass %q selected by the referenced T4ClusterHost does not exist", host.Spec.StorageClassName)) + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", ReasonStorageClassNotFound, fmt.Sprintf("StorageClass %q selected by the referenced T4ClusterHost does not exist", host.Spec.StorageClassName)) } return ctrl.Result{}, err } @@ -346,7 +353,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", ReasonStorageClassNotRWX, fmt.Sprintf("StorageClass %q selected by the referenced T4ClusterHost is not administrator-declared ReadWriteMany", host.Spec.StorageClassName)) + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", ReasonStorageClassNotRWX, fmt.Sprintf("StorageClass %q selected by the referenced T4ClusterHost is not administrator-declared ReadWriteMany", host.Spec.StorageClassName)) } var workspace clusterv1alpha1.T4Workspace if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: session.Spec.WorkspaceRef}, &workspace); err != nil { @@ -354,7 +361,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", "WorkspaceNotFound", "referenced T4Workspace does not exist") + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", "WorkspaceNotFound", "referenced T4Workspace does not exist") } return ctrl.Result{}, err } @@ -362,13 +369,19 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", "HostMismatch", "session and workspace must reference the same T4ClusterHost") + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", "HostMismatch", "session and workspace must reference the same T4ClusterHost") } if workspace.Status.PVCName == "" { if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", "PVCNotDeclared", "workspace controller has not declared a PVC") + return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", "PVCNotDeclared", "workspace controller has not declared a PVC") + } + if workspace.UID != "" && workspace.Status.PVCName != WorkspacePVCName(&workspace) { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", "PVCIdentityMismatch", "workspace status does not reference its deterministic PVC") } var pvc corev1.PersistentVolumeClaim if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: workspace.Status.PVCName}, &pvc); err != nil { @@ -376,21 +389,27 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", "PVCNotFound", "workspace PVC does not exist") + return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", "PVCNotFound", "workspace PVC does not exist") } return ctrl.Result{}, err } + if workspace.UID != "" && !workspaceOwnsPVC(&workspace, &pvc) { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", "PVCOwnershipConflict", "workspace PVC identity or ownership is not authoritative") + } if pvcStorageClassName(&pvc) != host.Spec.StorageClassName { if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", ReasonStorageClassMismatch, fmt.Sprintf("workspace PVC uses StorageClass %q instead of host-selected %q", pvcStorageClassName(&pvc), host.Spec.StorageClassName)) + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", ReasonStorageClassMismatch, fmt.Sprintf("workspace PVC uses StorageClass %q instead of host-selected %q", pvcStorageClassName(&pvc), host.Spec.StorageClassName)) } if pvc.Status.Phase != corev1.ClaimBound || !pvcHasRWX(&pvc) { if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", "PVCNotBoundRWX", "workspace PVC must be Bound and ReadWriteMany before a session starts") + return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", "PVCNotBoundRWX", "workspace PVC must be Bound and ReadWriteMany before a session starts") } runtimeVersions, reason, message, err := r.loadOMPResourceVersions(ctx, session.Namespace) if err != nil { @@ -400,7 +419,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "RuntimeConfigured", reason, message) + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, true, "RuntimeConfigured", reason, message) } serviceName := SessionServiceName(&session) @@ -429,8 +448,8 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) } } else if err != nil { return ctrl.Result{}, err - } else if !metav1.IsControlledBy(&service, &session) { - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "Available", "ServiceOwnershipConflict", "deterministic session Service is not controlled by this session") + } else if !sessionExclusivelyOwnsResource(&service, &session) { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, true, "Available", "ServiceOwnershipConflict", "deterministic session Service has an unexpected owner") } else if !serviceExposureIsInternal(&service) { if err := r.Delete(ctx, &service); err != nil && !apierrors.IsNotFound(err) { return ctrl.Result{}, err @@ -463,8 +482,8 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) } } else if err != nil { return ctrl.Result{}, err - } else if !metav1.IsControlledBy(&pod, &session) { - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "Available", "PodOwnershipConflict", "deterministic session Pod is not controlled by this session") + } else if !sessionExclusivelyOwnsResource(&pod, &session) { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, true, "Available", "PodOwnershipConflict", "deterministic session Pod has an unexpected owner") } else if !labelsContain(pod.Labels, desiredPod.Labels) { if pod.Labels == nil { pod.Labels = map[string]string{} @@ -654,6 +673,8 @@ func (r *SessionReconciler) updateSessionPending(ctx context.Context, session *c } session.Status.ObservedGeneration = session.Generation meta.SetStatusCondition(&session.Status.Conditions, condition("HostReady", metav1.ConditionTrue, "HostResolved", "referenced T4ClusterHost is available", session.Generation)) + meta.SetStatusCondition(&session.Status.Conditions, condition("WorkspaceReady", metav1.ConditionTrue, "PVCBoundRWX", "workspace PVC is Bound and ReadWriteMany", session.Generation)) + meta.SetStatusCondition(&session.Status.Conditions, condition("RuntimeConfigured", metav1.ConditionTrue, "OMPReferencesReady", "administrator-owned OMP runtime references are configured", session.Generation)) session.Status.PodName = podName session.Status.ServiceName = serviceName session.Status.Phase = clusterv1alpha1.InfrastructurePending @@ -693,7 +714,7 @@ func (r *SessionReconciler) reconcileDelete(ctx context.Context, session *cluste if err != nil { return ctrl.Result{}, err } - if !metav1.IsControlledBy(object, session) { + if !sessionExclusivelyOwnsResource(object, session) { before := session.Status if session.Status.Conditions != nil { before.Conditions = append([]metav1.Condition(nil), session.Status.Conditions...) @@ -734,8 +755,11 @@ func (r *SessionReconciler) deleteOwnedSessionResources(ctx context.Context, ses } continue } - if !metav1.IsControlledBy(object, session) { - continue + if !sessionExclusivelyOwnsResource(object, session) { + if err := r.updateSessionFailure(ctx, session, false, false, "Available", "ResourceOwnershipConflict", fmt.Sprintf("deterministic %T has an unexpected owner", object)); err != nil { + return err + } + return errSessionResourceOwnershipConflict } if err := r.Delete(ctx, object); err != nil && !apierrors.IsNotFound(err) { return err @@ -744,30 +768,56 @@ func (r *SessionReconciler) deleteOwnedSessionResources(ctx context.Context, ses return nil } -func (r *SessionReconciler) updateSessionFailure(ctx context.Context, session *clusterv1alpha1.T4Session, hostReady bool, conditionType, reason, message string) error { +func (r *SessionReconciler) updateSessionFailure(ctx context.Context, session *clusterv1alpha1.T4Session, hostReady, workspaceReady bool, conditionType, reason, message string) error { original := session.Status if session.Status.Conditions != nil { original.Conditions = append([]metav1.Condition(nil), session.Status.Conditions...) } - if hostReady { + if conditionType == "HostReady" { + meta.SetStatusCondition(&session.Status.Conditions, condition("HostReady", metav1.ConditionFalse, reason, message, session.Generation)) + } else if hostReady { meta.SetStatusCondition(&session.Status.Conditions, condition("HostReady", metav1.ConditionTrue, "HostResolved", "referenced T4ClusterHost is available", session.Generation)) - } else if conditionType != "HostReady" { - meta.SetStatusCondition(&session.Status.Conditions, condition("HostReady", metav1.ConditionUnknown, "NotEvaluated", "host dependency was not evaluated because static runtime configuration is invalid", session.Generation)) + } else { + meta.SetStatusCondition(&session.Status.Conditions, condition("HostReady", metav1.ConditionUnknown, "NotEvaluated", "host dependency was not evaluated", session.Generation)) + } + if conditionType == "WorkspaceReady" { + meta.SetStatusCondition(&session.Status.Conditions, condition("WorkspaceReady", metav1.ConditionFalse, reason, message, session.Generation)) + } else if workspaceReady { + meta.SetStatusCondition(&session.Status.Conditions, condition("WorkspaceReady", metav1.ConditionTrue, "PVCBoundRWX", "workspace PVC is Bound and ReadWriteMany", session.Generation)) + } else { + meta.SetStatusCondition(&session.Status.Conditions, condition("WorkspaceReady", metav1.ConditionUnknown, "NotEvaluated", "workspace dependency was not evaluated", session.Generation)) + } + if conditionType == "RuntimeConfigured" { + meta.SetStatusCondition(&session.Status.Conditions, condition("RuntimeConfigured", metav1.ConditionFalse, reason, message, session.Generation)) + } else if workspaceReady { + meta.SetStatusCondition(&session.Status.Conditions, condition("RuntimeConfigured", metav1.ConditionTrue, "OMPReferencesReady", "administrator-owned OMP runtime references are configured", session.Generation)) + } else { + meta.SetStatusCondition(&session.Status.Conditions, condition("RuntimeConfigured", metav1.ConditionUnknown, "NotEvaluated", "runtime configuration was not evaluated", session.Generation)) } session.Status.ObservedGeneration = session.Generation session.Status.PodName = "" session.Status.ServiceName = "" session.Status.Phase = clusterv1alpha1.InfrastructureFailed - meta.SetStatusCondition(&session.Status.Conditions, condition(conditionType, metav1.ConditionFalse, reason, message, session.Generation)) - if conditionType != "Available" { - meta.SetStatusCondition(&session.Status.Conditions, condition("Available", metav1.ConditionFalse, reason, message, session.Generation)) - } + meta.SetStatusCondition(&session.Status.Conditions, condition("Available", metav1.ConditionFalse, reason, message, session.Generation)) if reflect.DeepEqual(original, session.Status) { return nil } return r.Status().Update(ctx, session) } +func sessionExclusivelyOwnsResource(object metav1.Object, session *clusterv1alpha1.T4Session) bool { + controller := metav1.GetControllerOf(object) + if controller == nil || controller.APIVersion != clusterv1alpha1.GroupVersion.String() || controller.Kind != "T4Session" || controller.Name != session.Name || controller.UID != session.UID { + return false + } + for _, reference := range object.GetOwnerReferences() { + if reference.APIVersion != clusterv1alpha1.GroupVersion.String() || reference.Kind != "T4Session" || reference.Name != session.Name || reference.UID != session.UID { + return false + } + } + return true +} + func serviceExposureIsInternal(service *corev1.Service) bool { if service.Spec.Type != corev1.ServiceTypeClusterIP || service.Spec.ClusterIP == corev1.ClusterIPNone || service.Spec.ExternalName != "" || len(service.Spec.ExternalIPs) != 0 || service.Spec.LoadBalancerIP != "" || len(service.Spec.LoadBalancerSourceRanges) != 0 || diff --git a/packages/cluster-operator/controllers/workspace_controller.go b/packages/cluster-operator/controllers/workspace_controller.go index da1b555b..65e47608 100644 --- a/packages/cluster-operator/controllers/workspace_controller.go +++ b/packages/cluster-operator/controllers/workspace_controller.go @@ -10,6 +10,7 @@ import ( storagev1 "k8s.io/api/storage/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" meta "k8s.io/apimachinery/pkg/api/meta" + apiresource "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -161,6 +162,11 @@ func workspaceOwnsPVC(workspace *clusterv1alpha1.T4Workspace, pvc *corev1.Persis if pvc.Annotations[clusterv1alpha1.WorkspaceUIDAnnotation] != string(workspace.UID) { return false } + for _, reference := range pvc.OwnerReferences { + if reference.APIVersion != clusterv1alpha1.GroupVersion.String() || reference.Kind != "T4Workspace" || reference.Name != workspace.Name || reference.UID != workspace.UID { + return false + } + } controller := metav1.GetControllerOf(pvc) if workspace.Spec.RetentionPolicy == clusterv1alpha1.RetentionPolicyDelete { return controller != nil && controller.UID == workspace.UID @@ -270,15 +276,19 @@ func (r *WorkspaceReconciler) updateWorkspaceFailure(ctx context.Context, worksp if workspace.Status.Conditions != nil { original.Conditions = append([]metav1.Condition(nil), workspace.Status.Conditions...) } - if conditionType != "HostReady" { - meta.SetStatusCondition(&workspace.Status.Conditions, condition("HostReady", metav1.ConditionTrue, "HostResolved", "referenced T4ClusterHost is available", workspace.Generation)) - } workspace.Status.ObservedGeneration = workspace.Generation + workspace.Status.PVCName = "" + workspace.Status.PVCPhase = "" + workspace.Status.Capacity = apiresource.Quantity{} workspace.Status.Phase = clusterv1alpha1.InfrastructureFailed - meta.SetStatusCondition(&workspace.Status.Conditions, condition(conditionType, metav1.ConditionFalse, reason, message, workspace.Generation)) - if conditionType != "Ready" { - meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionFalse, reason, message, workspace.Generation)) + if conditionType == "HostReady" { + meta.SetStatusCondition(&workspace.Status.Conditions, condition("HostReady", metav1.ConditionFalse, reason, message, workspace.Generation)) + meta.SetStatusCondition(&workspace.Status.Conditions, condition("StorageReady", metav1.ConditionUnknown, "NotEvaluated", "storage dependency was not evaluated because the referenced host is unavailable", workspace.Generation)) + } else { + meta.SetStatusCondition(&workspace.Status.Conditions, condition("HostReady", metav1.ConditionTrue, "HostResolved", "referenced T4ClusterHost is available", workspace.Generation)) + meta.SetStatusCondition(&workspace.Status.Conditions, condition("StorageReady", metav1.ConditionFalse, reason, message, workspace.Generation)) } + meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionFalse, reason, message, workspace.Generation)) if reflect.DeepEqual(original, workspace.Status) { return nil } From d954667c18f6600baf813909adf656b44892f2fb Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 21:58:23 +0000 Subject: [PATCH 091/131] test(operator): revoke terminal storage authority --- .../controllers/reconciler_test.go | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index 564e4211..8d48f6da 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -1801,6 +1801,17 @@ func TestSessionResourcesWithAnyForeignOwnerFailClosed(t *testing.T) { t.Fatalf("Service with foreign OwnerReference was mutated: %#v", got) } } + if path == "dependency-cleanup" { + var sibling client.Object + if kind == "Pod" { + sibling = &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionServiceName(session), Namespace: session.Namespace}} + } else { + sibling = &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionPodName(session), Namespace: session.Namespace}} + } + if err := c.Get(ctx, client.ObjectKeyFromObject(sibling), sibling); !apierrors.IsNotFound(err) { + t.Fatalf("exclusively owned sibling remained after dependency cleanup: %v", err) + } + } var failed clusterv1alpha1.T4Session if err := c.Get(ctx, client.ObjectKeyFromObject(session), &failed); err != nil { t.Fatal(err) @@ -1814,6 +1825,45 @@ func TestSessionResourcesWithAnyForeignOwnerFailClosed(t *testing.T) { } } +func TestWorkspaceTerminalPVCFailuresRevokePublishedAuthority(t *testing.T) { + for _, test := range []struct { + name string + mutate func(*corev1.PersistentVolumeClaim) + reason string + }{ + {name: "Bound without RWX", reason: "PVCNotRWX", mutate: func(pvc *corev1.PersistentVolumeClaim) { pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} }}, + {name: "Lost", reason: "PVCLost", mutate: func(pvc *corev1.PersistentVolumeClaim) { pvc.Status.Phase = corev1.ClaimLost }}, + } { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + pvc := ownedWorkspacePVC(workspace) + test.mutate(pvc) + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Workspace{}, &corev1.PersistentVolumeClaim{}). + WithObjects(testHost(), rwxStorageClass(), workspace, pvc).Build() + r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + var failed clusterv1alpha1.T4Workspace + if err := c.Get(ctx, client.ObjectKeyFromObject(workspace), &failed); err != nil { + t.Fatal(err) + } + storageReady := findCondition(failed.Status.Conditions, "StorageReady") + ready := findCondition(failed.Status.Conditions, "Ready") + if failed.Status.PVCName != "" || failed.Status.PVCPhase != "" || !failed.Status.Capacity.IsZero() || + storageReady == nil || storageReady.Status != metav1.ConditionFalse || storageReady.Reason != test.reason || storageReady.ObservedGeneration != failed.Generation || + ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != test.reason || ready.ObservedGeneration != failed.Generation { + t.Fatalf("terminal PVC failure retained authority: status=%#v StorageReady=%#v Ready=%#v", failed.Status, storageReady, ready) + } + }) + } +} + + func ownedWorkspacePVC(workspace *clusterv1alpha1.T4Workspace) *corev1.PersistentVolumeClaim { return &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ From 187baf2649105f3bd29cf4df37468bbe894b8674 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 22:01:56 +0000 Subject: [PATCH 092/131] fix(operator): clear terminal storage authority --- .../controllers/session_controller.go | 13 +++++++++---- .../controllers/workspace_controller.go | 18 ++++++++---------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/packages/cluster-operator/controllers/session_controller.go b/packages/cluster-operator/controllers/session_controller.go index df4c492c..7a837e43 100644 --- a/packages/cluster-operator/controllers/session_controller.go +++ b/packages/cluster-operator/controllers/session_controller.go @@ -748,6 +748,7 @@ func (r *SessionReconciler) deleteOwnedSessionResources(ctx context.Context, ses &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: SessionPodName(session), Namespace: session.Namespace}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: SessionServiceName(session), Namespace: session.Namespace}}, } + ownershipConflict := false for _, object := range objects { if err := r.Get(ctx, client.ObjectKeyFromObject(object), object); err != nil { if err := client.IgnoreNotFound(err); err != nil { @@ -756,15 +757,19 @@ func (r *SessionReconciler) deleteOwnedSessionResources(ctx context.Context, ses continue } if !sessionExclusivelyOwnsResource(object, session) { - if err := r.updateSessionFailure(ctx, session, false, false, "Available", "ResourceOwnershipConflict", fmt.Sprintf("deterministic %T has an unexpected owner", object)); err != nil { - return err - } - return errSessionResourceOwnershipConflict + ownershipConflict = true + continue } if err := r.Delete(ctx, object); err != nil && !apierrors.IsNotFound(err) { return err } } + if ownershipConflict { + if err := r.updateSessionFailure(ctx, session, false, false, "Available", "ResourceOwnershipConflict", "one or more deterministic session resources have an unexpected owner"); err != nil { + return err + } + return errSessionResourceOwnershipConflict + } return nil } diff --git a/packages/cluster-operator/controllers/workspace_controller.go b/packages/cluster-operator/controllers/workspace_controller.go index 65e47608..d32920f8 100644 --- a/packages/cluster-operator/controllers/workspace_controller.go +++ b/packages/cluster-operator/controllers/workspace_controller.go @@ -118,6 +118,12 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, request ctrl.Reques return ctrl.Result{Requeue: true}, nil } } + if pvc.Status.Phase == corev1.ClaimBound && !pvcHasRWX(&pvc) { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCNotRWX", "bound workspace PVC does not request ReadWriteMany") + } + if pvc.Status.Phase == corev1.ClaimLost { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCLost", "workspace PVC lost its volume") + } original := workspace.Status original.Capacity = workspace.Status.Capacity.DeepCopy() @@ -133,16 +139,8 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, request ctrl.Reques meta.SetStatusCondition(&workspace.Status.Conditions, condition("StorageReady", metav1.ConditionTrue, ReasonStorageReady, "RWX StorageClass and workspace PVC are accepted", workspace.Generation)) switch pvc.Status.Phase { case corev1.ClaimBound: - if !pvcHasRWX(&pvc) { - workspace.Status.Phase = clusterv1alpha1.InfrastructureFailed - meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionFalse, "PVCNotRWX", "bound workspace PVC does not request ReadWriteMany", workspace.Generation)) - } else { - workspace.Status.Phase = clusterv1alpha1.InfrastructureReady - meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionTrue, "PVCBound", "workspace PVC is bound with ReadWriteMany access", workspace.Generation)) - } - case corev1.ClaimLost: - workspace.Status.Phase = clusterv1alpha1.InfrastructureFailed - meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionFalse, "PVCLost", "workspace PVC lost its volume", workspace.Generation)) + workspace.Status.Phase = clusterv1alpha1.InfrastructureReady + meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionTrue, "PVCBound", "workspace PVC is bound with ReadWriteMany access", workspace.Generation)) default: workspace.Status.Phase = clusterv1alpha1.InfrastructurePending meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionFalse, "PVCBinding", "workspace PVC is waiting to bind", workspace.Generation)) From 75762dafcaf7725fad155ba4898c49c58e788872 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 22:10:25 +0000 Subject: [PATCH 093/131] test(operator): stop owned siblings on conflict --- .../controllers/reconciler_test.go | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index 8d48f6da..a1533f3f 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -1801,16 +1801,14 @@ func TestSessionResourcesWithAnyForeignOwnerFailClosed(t *testing.T) { t.Fatalf("Service with foreign OwnerReference was mutated: %#v", got) } } - if path == "dependency-cleanup" { - var sibling client.Object - if kind == "Pod" { - sibling = &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionServiceName(session), Namespace: session.Namespace}} - } else { - sibling = &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionPodName(session), Namespace: session.Namespace}} - } - if err := c.Get(ctx, client.ObjectKeyFromObject(sibling), sibling); !apierrors.IsNotFound(err) { - t.Fatalf("exclusively owned sibling remained after dependency cleanup: %v", err) - } + var sibling client.Object + if kind == "Pod" { + sibling = &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionServiceName(session), Namespace: session.Namespace}} + } else { + sibling = &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionPodName(session), Namespace: session.Namespace}} + } + if err := c.Get(ctx, client.ObjectKeyFromObject(sibling), sibling); !apierrors.IsNotFound(err) { + t.Fatalf("exclusively owned sibling remained after ownership conflict: %v", err) } var failed clusterv1alpha1.T4Session if err := c.Get(ctx, client.ObjectKeyFromObject(session), &failed); err != nil { From 55539e5f10c21f3dcc14ea84340ee18fabff63a0 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 22:14:32 +0000 Subject: [PATCH 094/131] fix(operator): stop owned siblings on conflict --- .../cluster-operator/controllers/session_controller.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/cluster-operator/controllers/session_controller.go b/packages/cluster-operator/controllers/session_controller.go index 7a837e43..c7834cea 100644 --- a/packages/cluster-operator/controllers/session_controller.go +++ b/packages/cluster-operator/controllers/session_controller.go @@ -449,7 +449,10 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) } else if err != nil { return ctrl.Result{}, err } else if !sessionExclusivelyOwnsResource(&service, &session) { - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, true, "Available", "ServiceOwnershipConflict", "deterministic session Service has an unexpected owner") + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } else if !serviceExposureIsInternal(&service) { if err := r.Delete(ctx, &service); err != nil && !apierrors.IsNotFound(err) { return ctrl.Result{}, err @@ -483,7 +486,10 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) } else if err != nil { return ctrl.Result{}, err } else if !sessionExclusivelyOwnsResource(&pod, &session) { - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, true, "Available", "PodOwnershipConflict", "deterministic session Pod has an unexpected owner") + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } else if !labelsContain(pod.Labels, desiredPod.Labels) { if pod.Labels == nil { pod.Labels = map[string]string{} From 764f46094951d40d84f8389ad83070609fe4d5e7 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 22:26:47 +0000 Subject: [PATCH 095/131] test(operator): preserve dependency status on conflicts --- packages/cluster-operator/controllers/reconciler_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index a1533f3f..b92fd46b 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -1818,6 +1818,11 @@ func TestSessionResourcesWithAnyForeignOwnerFailClosed(t *testing.T) { if available == nil || available.Status != metav1.ConditionFalse || !strings.Contains(available.Reason, "OwnershipConflict") { t.Fatalf("foreign owner did not produce stable ownership conflict: %#v", available) } + if path == "normal" { + assertCurrentSessionConditions(t, &failed, map[string]metav1.ConditionStatus{ + "HostReady": metav1.ConditionTrue, "WorkspaceReady": metav1.ConditionTrue, "RuntimeConfigured": metav1.ConditionTrue, "Available": metav1.ConditionFalse, + }) + } }) } } From e24b6dcefc8ba92be906a4a26a47545b52c26fab Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 22:34:18 +0000 Subject: [PATCH 096/131] fix(operator): preserve ready dependencies on conflicts --- .../controllers/session_controller.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/cluster-operator/controllers/session_controller.go b/packages/cluster-operator/controllers/session_controller.go index c7834cea..1d52a20b 100644 --- a/packages/cluster-operator/controllers/session_controller.go +++ b/packages/cluster-operator/controllers/session_controller.go @@ -449,7 +449,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) } else if err != nil { return ctrl.Result{}, err } else if !sessionExclusivelyOwnsResource(&service, &session) { - if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + if err := r.deleteOwnedSessionResourcesAfterVerifiedDependencies(ctx, &session, "ServiceOwnershipConflict", "deterministic session Service has an unexpected owner"); err != nil { return ctrl.Result{}, err } return ctrl.Result{RequeueAfter: 30 * time.Second}, nil @@ -486,7 +486,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) } else if err != nil { return ctrl.Result{}, err } else if !sessionExclusivelyOwnsResource(&pod, &session) { - if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + if err := r.deleteOwnedSessionResourcesAfterVerifiedDependencies(ctx, &session, "PodOwnershipConflict", "deterministic session Pod has an unexpected owner"); err != nil { return ctrl.Result{}, err } return ctrl.Result{RequeueAfter: 30 * time.Second}, nil @@ -750,6 +750,14 @@ func (r *SessionReconciler) reconcileDelete(ctx context.Context, session *cluste } func (r *SessionReconciler) deleteOwnedSessionResources(ctx context.Context, session *clusterv1alpha1.T4Session) error { + return r.deleteOwnedSessionResourcesWithFailure(ctx, session, false, false, "ResourceOwnershipConflict", "one or more deterministic session resources have an unexpected owner") +} + +func (r *SessionReconciler) deleteOwnedSessionResourcesAfterVerifiedDependencies(ctx context.Context, session *clusterv1alpha1.T4Session, reason, message string) error { + return r.deleteOwnedSessionResourcesWithFailure(ctx, session, true, true, reason, message) +} + +func (r *SessionReconciler) deleteOwnedSessionResourcesWithFailure(ctx context.Context, session *clusterv1alpha1.T4Session, hostReady, workspaceReady bool, reason, message string) error { objects := []client.Object{ &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: SessionPodName(session), Namespace: session.Namespace}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: SessionServiceName(session), Namespace: session.Namespace}}, @@ -771,7 +779,7 @@ func (r *SessionReconciler) deleteOwnedSessionResources(ctx context.Context, ses } } if ownershipConflict { - if err := r.updateSessionFailure(ctx, session, false, false, "Available", "ResourceOwnershipConflict", "one or more deterministic session resources have an unexpected owner"); err != nil { + if err := r.updateSessionFailure(ctx, session, hostReady, workspaceReady, "Available", reason, message); err != nil { return err } return errSessionResourceOwnershipConflict From 96ccd96fb89c6c85076e02476c213c85bc4b89ff Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 22:41:51 +0000 Subject: [PATCH 097/131] test(operator): cover cleanup and create races --- .../controllers/reconciler_test.go | 119 +++++++++++++++++- 1 file changed, 118 insertions(+), 1 deletion(-) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index b92fd46b..2b0fe1a4 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -13,6 +13,7 @@ import ( apiresource "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" utilvalidation "k8s.io/apimachinery/pkg/util/validation" ctrl "sigs.k8s.io/controller-runtime" @@ -1212,6 +1213,8 @@ func TestSessionDeletionRefusesForeignDeterministicResources(t *testing.T) { } else { service.OwnerReferences = nil } + beforePod := pod.DeepCopy() + beforeService := service.DeepCopy() c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Session{}).WithObjects(session, pod, service).Build() if err := c.Delete(context.Background(), session); err != nil { t.Fatal(err) @@ -1220,7 +1223,28 @@ func TestSessionDeletionRefusesForeignDeterministicResources(t *testing.T) { if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { t.Fatal(err) } - assertObjectCounts(t, c, 1, 1) + if foreignKind == "Pod" { + assertObjectCounts(t, c, 1, 0) + } else { + assertObjectCounts(t, c, 0, 1) + } + if foreignKind == "Pod" { + var got corev1.Pod + if err := c.Get(context.Background(), client.ObjectKeyFromObject(pod), &got); err != nil { + t.Fatalf("foreign Pod was deleted: %v", err) + } + if !reflect.DeepEqual(got.ObjectMeta, beforePod.ObjectMeta) || !reflect.DeepEqual(got.Spec, beforePod.Spec) { + t.Fatalf("foreign Pod was mutated during finalizer cleanup: %#v", got) + } + } else { + var got corev1.Service + if err := c.Get(context.Background(), client.ObjectKeyFromObject(service), &got); err != nil { + t.Fatalf("foreign Service was deleted: %v", err) + } + if !reflect.DeepEqual(got.ObjectMeta, beforeService.ObjectMeta) || !reflect.DeepEqual(got.Spec, beforeService.Spec) { + t.Fatalf("foreign Service was mutated during finalizer cleanup: %#v", got) + } + } var waiting clusterv1alpha1.T4Session if err := c.Get(context.Background(), client.ObjectKeyFromObject(session), &waiting); err != nil { t.Fatalf("session finalizer was removed on cleanup conflict: %v", err) @@ -1828,6 +1852,70 @@ func TestSessionResourcesWithAnyForeignOwnerFailClosed(t *testing.T) { } } +func TestSessionCreateAlreadyExistsRefetchesForeignWinner(t *testing.T) { + for _, kind := range []string{"Pod", "Service"} { + t.Run(kind, func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + workspace.Status.PVCName = controllers.WorkspacePVCName(workspace) + session := testSession() + session.UID = "session-uid" + pod, service := ownedSessionResources(session) + objects := []client.Object{testHost(), workspace, ownedWorkspacePVC(workspace), session} + if kind == "Pod" { + objects = append(objects, service) + } else { + objects = append(objects, pod) + } + base := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(objects...).Build() + c := &createAlreadyExistsClient{Client: base, raceKind: kind} + r := configuredSessionReconciler(c, scheme) + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + if c.winner == nil { + t.Fatalf("%s create race was not exercised", kind) + } + if kind == "Pod" { + var got corev1.Pod + if err := base.Get(ctx, client.ObjectKeyFromObject(c.winner), &got); err != nil { + t.Fatalf("foreign Pod winner was deleted: %v", err) + } + want := c.winner.(*corev1.Pod) + if !reflect.DeepEqual(got.ObjectMeta, want.ObjectMeta) || !reflect.DeepEqual(got.Spec, want.Spec) { + t.Fatalf("foreign Pod winner was mutated: %#v", got) + } + assertObjectCounts(t, base, 1, 0) + } else { + var got corev1.Service + if err := base.Get(ctx, client.ObjectKeyFromObject(c.winner), &got); err != nil { + t.Fatalf("foreign Service winner was deleted: %v", err) + } + want := c.winner.(*corev1.Service) + if !reflect.DeepEqual(got.ObjectMeta, want.ObjectMeta) || !reflect.DeepEqual(got.Spec, want.Spec) { + t.Fatalf("foreign Service winner was mutated: %#v", got) + } + assertObjectCounts(t, base, 0, 1) + } + var failed clusterv1alpha1.T4Session + if err := base.Get(ctx, client.ObjectKeyFromObject(session), &failed); err != nil { + t.Fatal(err) + } + assertCurrentSessionConditions(t, &failed, map[string]metav1.ConditionStatus{ + "HostReady": metav1.ConditionTrue, "WorkspaceReady": metav1.ConditionTrue, "RuntimeConfigured": metav1.ConditionTrue, "Available": metav1.ConditionFalse, + }) + available := findCondition(failed.Status.Conditions, "Available") + if available.Reason != kind+"OwnershipConflict" { + t.Fatalf("Available reason = %q, want %sOwnershipConflict", available.Reason, kind) + } + }) + } +} + func TestWorkspaceTerminalPVCFailuresRevokePublishedAuthority(t *testing.T) { for _, test := range []struct { name string @@ -2034,4 +2122,33 @@ func hasReadOnlyMount(mounts []corev1.VolumeMount, name, path string) bool { return false } +type createAlreadyExistsClient struct { + client.Client + raceKind string + winner client.Object +} + +func (c *createAlreadyExistsClient) Create(ctx context.Context, object client.Object, options ...client.CreateOption) error { + var winner client.Object + switch object := object.(type) { + case *corev1.Pod: + if c.raceKind == "Pod" { + winner = object.DeepCopy() + } + case *corev1.Service: + if c.raceKind == "Service" { + winner = object.DeepCopy() + } + } + if winner == nil || c.winner != nil { + return c.Client.Create(ctx, object, options...) + } + winner.SetOwnerReferences(nil) + if err := c.Client.Create(ctx, winner, options...); err != nil { + return err + } + c.winner = winner.DeepCopyObject().(client.Object) + return apierrors.NewAlreadyExists(schema.GroupResource{Resource: strings.ToLower(c.raceKind) + "s"}, object.GetName()) +} + func ptr[T any](value T) *T { return &value } From 2a8b7ae3e99c9499fddf035cc0f3a8ae6b0ff20d Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 22:44:52 +0000 Subject: [PATCH 098/131] fix(operator): close child ownership races --- .../controllers/session_controller.go | 65 ++++++++++++++----- 1 file changed, 47 insertions(+), 18 deletions(-) diff --git a/packages/cluster-operator/controllers/session_controller.go b/packages/cluster-operator/controllers/session_controller.go index 1d52a20b..95f076d0 100644 --- a/packages/cluster-operator/controllers/session_controller.go +++ b/packages/cluster-operator/controllers/session_controller.go @@ -441,14 +441,25 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) return ctrl.Result{}, err } var service corev1.Service - if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: serviceName}, &service); apierrors.IsNotFound(err) { + serviceKey := types.NamespacedName{Namespace: session.Namespace, Name: serviceName} + if err := r.Get(ctx, serviceKey, &service); apierrors.IsNotFound(err) { service = desiredService - if err := r.Create(ctx, &service); err != nil && !apierrors.IsAlreadyExists(err) { - return ctrl.Result{}, err + if err := r.Create(ctx, &service); err != nil { + if !apierrors.IsAlreadyExists(err) { + return ctrl.Result{}, err + } + reader := r.APIReader + if reader == nil { + reader = r.Client + } + if err := reader.Get(ctx, serviceKey, &service); err != nil { + return ctrl.Result{}, err + } } } else if err != nil { return ctrl.Result{}, err - } else if !sessionExclusivelyOwnsResource(&service, &session) { + } + if !sessionExclusivelyOwnsResource(&service, &session) { if err := r.deleteOwnedSessionResourcesAfterVerifiedDependencies(ctx, &session, "ServiceOwnershipConflict", "deterministic session Service has an unexpected owner"); err != nil { return ctrl.Result{}, err } @@ -478,14 +489,25 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) return ctrl.Result{}, err } var pod corev1.Pod - if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: podName}, &pod); apierrors.IsNotFound(err) { + podKey := types.NamespacedName{Namespace: session.Namespace, Name: podName} + if err := r.Get(ctx, podKey, &pod); apierrors.IsNotFound(err) { pod = desiredPod - if err := r.Create(ctx, &pod); err != nil && !apierrors.IsAlreadyExists(err) { - return ctrl.Result{}, err + if err := r.Create(ctx, &pod); err != nil { + if !apierrors.IsAlreadyExists(err) { + return ctrl.Result{}, err + } + reader := r.APIReader + if reader == nil { + reader = r.Client + } + if err := reader.Get(ctx, podKey, &pod); err != nil { + return ctrl.Result{}, err + } } } else if err != nil { return ctrl.Result{}, err - } else if !sessionExclusivelyOwnsResource(&pod, &session) { + } + if !sessionExclusivelyOwnsResource(&pod, &session) { if err := r.deleteOwnedSessionResourcesAfterVerifiedDependencies(ctx, &session, "PodOwnershipConflict", "deterministic session Pod has an unexpected owner"); err != nil { return ctrl.Result{}, err } @@ -712,6 +734,7 @@ func (r *SessionReconciler) reconcileDelete(ctx context.Context, session *cluste &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: SessionServiceName(session), Namespace: session.Namespace}}, } existing := make([]client.Object, 0, len(objects)) + var ownershipConflict client.Object for _, object := range objects { err := r.Get(ctx, client.ObjectKeyFromObject(object), object) if apierrors.IsNotFound(err) { @@ -721,17 +744,10 @@ func (r *SessionReconciler) reconcileDelete(ctx context.Context, session *cluste return ctrl.Result{}, err } if !sessionExclusivelyOwnsResource(object, session) { - before := session.Status - if session.Status.Conditions != nil { - before.Conditions = append([]metav1.Condition(nil), session.Status.Conditions...) - } - meta.SetStatusCondition(&session.Status.Conditions, condition("Available", metav1.ConditionFalse, "CleanupOwnershipConflict", fmt.Sprintf("deterministic %T is not controlled by this session", object), session.Generation)) - if !reflect.DeepEqual(before, session.Status) { - if err := r.Status().Update(ctx, session); err != nil { - return ctrl.Result{}, err - } + if ownershipConflict == nil { + ownershipConflict = object } - return ctrl.Result{RequeueAfter: 30 * time.Second}, nil + continue } existing = append(existing, object) } @@ -742,6 +758,19 @@ func (r *SessionReconciler) reconcileDelete(ctx context.Context, session *cluste } } } + if ownershipConflict != nil { + before := session.Status + if session.Status.Conditions != nil { + before.Conditions = append([]metav1.Condition(nil), session.Status.Conditions...) + } + meta.SetStatusCondition(&session.Status.Conditions, condition("Available", metav1.ConditionFalse, "CleanupOwnershipConflict", fmt.Sprintf("deterministic %T is not controlled by this session", ownershipConflict), session.Generation)) + if !reflect.DeepEqual(before, session.Status) { + if err := r.Status().Update(ctx, session); err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil + } if len(existing) > 0 { return ctrl.Result{RequeueAfter: time.Second}, nil } From e366c52605a253348ad0d41784adb868b0272aa8 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 22:51:48 +0000 Subject: [PATCH 099/131] test(operator): model stale cache after create race --- .../controllers/reconciler_test.go | 34 ++++++++++++++++--- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index 2b0fe1a4..79e7bb29 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -1213,9 +1213,15 @@ func TestSessionDeletionRefusesForeignDeterministicResources(t *testing.T) { } else { service.OwnerReferences = nil } - beforePod := pod.DeepCopy() - beforeService := service.DeepCopy() c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Session{}).WithObjects(session, pod, service).Build() + beforePod := &corev1.Pod{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(pod), beforePod); err != nil { + t.Fatal(err) + } + beforeService := &corev1.Service{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(service), beforeService); err != nil { + t.Fatal(err) + } if err := c.Delete(context.Background(), session); err != nil { t.Fatal(err) } @@ -1872,8 +1878,9 @@ func TestSessionCreateAlreadyExistsRefetchesForeignWinner(t *testing.T) { base := fake.NewClientBuilder().WithScheme(scheme). WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). WithObjects(objects...).Build() - c := &createAlreadyExistsClient{Client: base, raceKind: kind} + c := &createAlreadyExistsClient{Client: base, raceKind: kind, hideWinnerFromCache: true} r := configuredSessionReconciler(c, scheme) + r.APIReader = base if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { t.Fatal(err) } @@ -2124,8 +2131,25 @@ func hasReadOnlyMount(mounts []corev1.VolumeMount, name, path string) bool { type createAlreadyExistsClient struct { client.Client - raceKind string - winner client.Object + raceKind string + winner client.Object + hideWinnerFromCache bool +} + +func (c *createAlreadyExistsClient) Get(ctx context.Context, key client.ObjectKey, object client.Object, options ...client.GetOption) error { + if c.hideWinnerFromCache && c.winner != nil && key == client.ObjectKeyFromObject(c.winner) { + switch object.(type) { + case *corev1.Pod: + if c.raceKind == "Pod" { + return apierrors.NewNotFound(schema.GroupResource{Resource: "pods"}, key.Name) + } + case *corev1.Service: + if c.raceKind == "Service" { + return apierrors.NewNotFound(schema.GroupResource{Resource: "services"}, key.Name) + } + } + } + return c.Client.Get(ctx, key, object, options...) } func (c *createAlreadyExistsClient) Create(ctx context.Context, object client.Object, options ...client.CreateOption) error { From 4916c10a68036fc31f276a20203b5c4ed79c2266 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 22:54:45 +0000 Subject: [PATCH 100/131] test(operator): cover workspace PVC create race --- .../controllers/reconciler_test.go | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index 79e7bb29..3195e777 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -72,6 +72,40 @@ func TestWorkspaceReconcileIsIdempotentAcrossDuplicateEvents(t *testing.T) { } } +func TestWorkspaceCreateAlreadyExistsRefetchesForeignPVC(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + base := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Workspace{}, &corev1.PersistentVolumeClaim{}). + WithObjects(testHost(), rwxStorageClass(), workspace).Build() + c := &createAlreadyExistsClient{Client: base, raceKind: "PVC"} + r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + if c.winner == nil { + t.Fatal("PVC create race was not exercised") + } + var pvc corev1.PersistentVolumeClaim + if err := base.Get(ctx, client.ObjectKeyFromObject(c.winner), &pvc); err != nil { + t.Fatalf("foreign PVC winner was deleted: %v", err) + } + want := c.winner.(*corev1.PersistentVolumeClaim) + if !reflect.DeepEqual(pvc.ObjectMeta, want.ObjectMeta) || !reflect.DeepEqual(pvc.Spec, want.Spec) { + t.Fatalf("foreign PVC winner was mutated: %#v", pvc) + } + var failed clusterv1alpha1.T4Workspace + if err := base.Get(ctx, client.ObjectKeyFromObject(workspace), &failed); err != nil { + t.Fatal(err) + } + storageReady := findCondition(failed.Status.Conditions, "StorageReady") + if failed.Status.PVCName != "" || storageReady == nil || storageReady.Status != metav1.ConditionFalse || storageReady.Reason != "PVCOwnershipConflict" || storageReady.ObservedGeneration != failed.Generation { + t.Fatalf("foreign PVC winner was published as authoritative: status=%#v StorageReady=%#v", failed.Status, storageReady) + } +} + func TestRetainWorkspaceCreatesPVCWithoutGarbageCollectableOwner(t *testing.T) { scheme := testScheme(t) workspace := testWorkspace(clusterv1alpha1.RetentionPolicyRetain) @@ -2163,6 +2197,10 @@ func (c *createAlreadyExistsClient) Create(ctx context.Context, object client.Ob if c.raceKind == "Service" { winner = object.DeepCopy() } + case *corev1.PersistentVolumeClaim: + if c.raceKind == "PVC" { + winner = object.DeepCopy() + } } if winner == nil || c.winner != nil { return c.Client.Create(ctx, object, options...) From db95dd6c4bdc0b2cf64c0d9886181b2339e86e7b Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 22:57:58 +0000 Subject: [PATCH 101/131] fix(operator): validate authoritative create winners --- packages/cluster-operator/cmd/manager/main.go | 2 +- .../controllers/reconciler_test.go | 4 ++-- .../controllers/session_controller.go | 12 ++++++---- .../controllers/workspace_controller.go | 22 ++++++++++++++----- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/packages/cluster-operator/cmd/manager/main.go b/packages/cluster-operator/cmd/manager/main.go index d216db5a..f33af52c 100644 --- a/packages/cluster-operator/cmd/manager/main.go +++ b/packages/cluster-operator/cmd/manager/main.go @@ -70,7 +70,7 @@ func main() { ctrl.Log.Error(err, "unable to register T4ClusterHost controller") os.Exit(1) } - if err := (&controllers.WorkspaceReconciler{Client: manager.GetClient(), Scheme: manager.GetScheme()}).SetupWithManager(manager); err != nil { + if err := (&controllers.WorkspaceReconciler{Client: manager.GetClient(), APIReader: manager.GetAPIReader(), Scheme: manager.GetScheme()}).SetupWithManager(manager); err != nil { ctrl.Log.Error(err, "unable to register T4Workspace controller") os.Exit(1) } diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index 3195e777..e4f4f42d 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -80,8 +80,8 @@ func TestWorkspaceCreateAlreadyExistsRefetchesForeignPVC(t *testing.T) { base := fake.NewClientBuilder().WithScheme(scheme). WithStatusSubresource(&clusterv1alpha1.T4Workspace{}, &corev1.PersistentVolumeClaim{}). WithObjects(testHost(), rwxStorageClass(), workspace).Build() - c := &createAlreadyExistsClient{Client: base, raceKind: "PVC"} - r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} + c := &createAlreadyExistsClient{Client: base, raceKind: "PVC", hideWinnerFromCache: true} + r := &controllers.WorkspaceReconciler{Client: c, APIReader: base, Scheme: scheme} if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { t.Fatal(err) } diff --git a/packages/cluster-operator/controllers/session_controller.go b/packages/cluster-operator/controllers/session_controller.go index 95f076d0..3f1bd28d 100644 --- a/packages/cluster-operator/controllers/session_controller.go +++ b/packages/cluster-operator/controllers/session_controller.go @@ -779,21 +779,25 @@ func (r *SessionReconciler) reconcileDelete(ctx context.Context, session *cluste } func (r *SessionReconciler) deleteOwnedSessionResources(ctx context.Context, session *clusterv1alpha1.T4Session) error { - return r.deleteOwnedSessionResourcesWithFailure(ctx, session, false, false, "ResourceOwnershipConflict", "one or more deterministic session resources have an unexpected owner") + return r.deleteOwnedSessionResourcesWithFailure(ctx, r.Client, session, false, false, "ResourceOwnershipConflict", "one or more deterministic session resources have an unexpected owner") } func (r *SessionReconciler) deleteOwnedSessionResourcesAfterVerifiedDependencies(ctx context.Context, session *clusterv1alpha1.T4Session, reason, message string) error { - return r.deleteOwnedSessionResourcesWithFailure(ctx, session, true, true, reason, message) + reader := r.APIReader + if reader == nil { + reader = r.Client + } + return r.deleteOwnedSessionResourcesWithFailure(ctx, reader, session, true, true, reason, message) } -func (r *SessionReconciler) deleteOwnedSessionResourcesWithFailure(ctx context.Context, session *clusterv1alpha1.T4Session, hostReady, workspaceReady bool, reason, message string) error { +func (r *SessionReconciler) deleteOwnedSessionResourcesWithFailure(ctx context.Context, reader client.Reader, session *clusterv1alpha1.T4Session, hostReady, workspaceReady bool, reason, message string) error { objects := []client.Object{ &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: SessionPodName(session), Namespace: session.Namespace}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: SessionServiceName(session), Namespace: session.Namespace}}, } ownershipConflict := false for _, object := range objects { - if err := r.Get(ctx, client.ObjectKeyFromObject(object), object); err != nil { + if err := reader.Get(ctx, client.ObjectKeyFromObject(object), object); err != nil { if err := client.IgnoreNotFound(err); err != nil { return err } diff --git a/packages/cluster-operator/controllers/workspace_controller.go b/packages/cluster-operator/controllers/workspace_controller.go index d32920f8..d6c2a54d 100644 --- a/packages/cluster-operator/controllers/workspace_controller.go +++ b/packages/cluster-operator/controllers/workspace_controller.go @@ -24,7 +24,8 @@ import ( type WorkspaceReconciler struct { client.Client - Scheme *runtime.Scheme + APIReader client.Reader + Scheme *runtime.Scheme } const ( @@ -74,8 +75,9 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, request ctrl.Reques } pvcName := WorkspacePVCName(&workspace) + pvcKey := types.NamespacedName{Namespace: workspace.Namespace, Name: pvcName} var pvc corev1.PersistentVolumeClaim - err = r.Get(ctx, types.NamespacedName{Namespace: workspace.Namespace, Name: pvcName}, &pvc) + err = r.Get(ctx, pvcKey, &pvc) if apierrors.IsNotFound(err) { volumeMode := corev1.PersistentVolumeFilesystem pvc = corev1.PersistentVolumeClaim{ @@ -99,12 +101,22 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, request ctrl.Reques return ctrl.Result{}, err } } - if err := r.Create(ctx, &pvc); err != nil && !apierrors.IsAlreadyExists(err) { - return ctrl.Result{}, err + if err := r.Create(ctx, &pvc); err != nil { + if !apierrors.IsAlreadyExists(err) { + return ctrl.Result{}, err + } + reader := r.APIReader + if reader == nil { + reader = r.Client + } + if err := reader.Get(ctx, pvcKey, &pvc); err != nil { + return ctrl.Result{}, err + } } } else if err != nil { return ctrl.Result{}, err - } else if !workspaceOwnsPVC(&workspace, &pvc) { + } + if !workspaceOwnsPVC(&workspace, &pvc) { return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCOwnershipConflict", "deterministic workspace PVC does not belong to this workspace") } else if pvcStorageClassName(&pvc) != storageClassName { return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", ReasonStorageClassMismatch, fmt.Sprintf("workspace PVC uses StorageClass %q instead of host-selected %q; data-bearing PVCs are never recreated automatically", pvcStorageClassName(&pvc), storageClassName)) From ee5d00903272e00b584ce2bac16fd33e7cf80352 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 23:10:02 +0000 Subject: [PATCH 102/131] test(operator): cover authoritative cleanup policy --- .../controllers/reconciler_test.go | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index e4f4f42d..6e45a9a0 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -106,6 +106,67 @@ func TestWorkspaceCreateAlreadyExistsRefetchesForeignPVC(t *testing.T) { } } +func TestWorkspacePendingPVCPolicyFailsBeforeAuthority(t *testing.T) { + for _, test := range []struct { + name string + mutate func(*corev1.PersistentVolumeClaim) + reason string + }{ + {name: "wrong class", reason: controllers.ReasonStorageClassMismatch, mutate: func(pvc *corev1.PersistentVolumeClaim) { pvc.Spec.StorageClassName = ptr("other-rwx") }}, + {name: "wrong access", reason: "PVCNotRWX", mutate: func(pvc *corev1.PersistentVolumeClaim) { pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} }}, + } { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + pvc := ownedWorkspacePVC(workspace) + pvc.Status.Phase = corev1.ClaimPending + test.mutate(pvc) + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Workspace{}, &corev1.PersistentVolumeClaim{}).WithObjects(testHost(), rwxStorageClass(), workspace, pvc).Build() + r := &controllers.WorkspaceReconciler{Client: c, APIReader: c, Scheme: scheme} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + var failed clusterv1alpha1.T4Workspace + if err := c.Get(ctx, client.ObjectKeyFromObject(workspace), &failed); err != nil { + t.Fatal(err) + } + condition := findCondition(failed.Status.Conditions, "StorageReady") + if failed.Status.PVCName != "" || condition == nil || condition.Status != metav1.ConditionFalse || condition.Reason != test.reason { + t.Fatalf("incompatible Pending PVC published authority: status=%#v condition=%#v", failed.Status, condition) + } + }) + } +} + +func TestWorkspaceDeletionUsesAuthoritativePVCReader(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + workspace.Finalizers = []string{clusterv1alpha1.WorkspaceFinalizer} + pvc := ownedWorkspacePVC(workspace) + pvc.OwnerReferences = nil + base := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Workspace{}).WithObjects(workspace, pvc).Build() + if err := base.Delete(ctx, workspace); err != nil { + t.Fatal(err) + } + c := &createAlreadyExistsClient{Client: base, raceKind: "PVC", winner: pvc, hideWinnerFromCache: true} + r := &controllers.WorkspaceReconciler{Client: c, APIReader: base, Scheme: scheme} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + var waiting clusterv1alpha1.T4Workspace + if err := base.Get(ctx, client.ObjectKeyFromObject(workspace), &waiting); err != nil { + t.Fatalf("workspace finalizer ignored authoritative PVC conflict: %v", err) + } + condition := findCondition(waiting.Status.Conditions, "Ready") + if condition == nil || condition.Reason != "CleanupOwnershipConflict" || !contains(waiting.Finalizers, clusterv1alpha1.WorkspaceFinalizer) { + t.Fatalf("authoritative PVC conflict not retained: %#v", waiting) + } +} + func TestRetainWorkspaceCreatesPVCWithoutGarbageCollectableOwner(t *testing.T) { scheme := testScheme(t) workspace := testWorkspace(clusterv1alpha1.RetentionPolicyRetain) @@ -1297,6 +1358,35 @@ func TestSessionDeletionRefusesForeignDeterministicResources(t *testing.T) { } } +func TestSessionDeletionUsesAuthoritativeChildReader(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + session := testSession() + session.UID = "session-uid" + session.Finalizers = []string{clusterv1alpha1.SessionFinalizer} + pod, service := ownedSessionResources(session) + pod.OwnerReferences = nil + base := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Session{}).WithObjects(session, pod, service).Build() + if err := base.Delete(ctx, session); err != nil { + t.Fatal(err) + } + c := &createAlreadyExistsClient{Client: base, raceKind: "Pod", winner: pod, hideWinnerFromCache: true} + r := configuredSessionReconciler(c, scheme) + r.APIReader = base + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + assertObjectCounts(t, base, 1, 0) + var waiting clusterv1alpha1.T4Session + if err := base.Get(ctx, client.ObjectKeyFromObject(session), &waiting); err != nil { + t.Fatalf("session finalizer ignored authoritative Pod conflict: %v", err) + } + condition := findCondition(waiting.Status.Conditions, "Available") + if condition == nil || condition.Reason != "CleanupOwnershipConflict" || !contains(waiting.Finalizers, clusterv1alpha1.SessionFinalizer) { + t.Fatalf("authoritative child conflict not retained: %#v", waiting) + } +} + func TestSessionDeletionCleansResourcesBeforeFinalizer(t *testing.T) { scheme := testScheme(t) session := testSession() @@ -2181,6 +2271,10 @@ func (c *createAlreadyExistsClient) Get(ctx context.Context, key client.ObjectKe if c.raceKind == "Service" { return apierrors.NewNotFound(schema.GroupResource{Resource: "services"}, key.Name) } + case *corev1.PersistentVolumeClaim: + if c.raceKind == "PVC" { + return apierrors.NewNotFound(schema.GroupResource{Resource: "persistentvolumeclaims"}, key.Name) + } } } return c.Client.Get(ctx, key, object, options...) From d9ef139dd752f3f7ceb79707b6be45e87f589b58 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 23:12:47 +0000 Subject: [PATCH 103/131] fix(operator): precondition authoritative cleanup --- .../controllers/session_controller.go | 43 +++++++++++++++---- .../controllers/workspace_controller.go | 13 +++--- 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/packages/cluster-operator/controllers/session_controller.go b/packages/cluster-operator/controllers/session_controller.go index 3f1bd28d..7523bbbb 100644 --- a/packages/cluster-operator/controllers/session_controller.go +++ b/packages/cluster-operator/controllers/session_controller.go @@ -465,7 +465,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) } return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } else if !serviceExposureIsInternal(&service) { - if err := r.Delete(ctx, &service); err != nil && !apierrors.IsNotFound(err) { + if err := deleteWithPreconditions(ctx, r.Client, &service); err != nil && !apierrors.IsNotFound(err) { return ctrl.Result{}, err } if err := r.updateSessionPending(ctx, &session, podName, serviceName, "ServiceExposureChanged", "session Service is being recreated with ClusterIP-only exposure"); err != nil { @@ -527,7 +527,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) } return ctrl.Result{RequeueAfter: time.Second}, nil } else if pod.Annotations[clusterv1alpha1.SessionPodSpecHashAnnotation] != desiredPod.Annotations[clusterv1alpha1.SessionPodSpecHashAnnotation] { - if err := r.Delete(ctx, &pod); err != nil && !apierrors.IsNotFound(err) { + if err := deleteWithPreconditions(ctx, r.Client, &pod); err != nil && !apierrors.IsNotFound(err) { return ctrl.Result{}, err } if err := r.updateSessionPending(ctx, &session, podName, serviceName, "PodSpecChanged", "session Pod is being recreated to apply immutable desired state"); err != nil { @@ -735,8 +735,12 @@ func (r *SessionReconciler) reconcileDelete(ctx context.Context, session *cluste } existing := make([]client.Object, 0, len(objects)) var ownershipConflict client.Object + reader := r.APIReader + if reader == nil { + reader = r.Client + } for _, object := range objects { - err := r.Get(ctx, client.ObjectKeyFromObject(object), object) + err := reader.Get(ctx, client.ObjectKeyFromObject(object), object) if apierrors.IsNotFound(err) { continue } @@ -753,7 +757,7 @@ func (r *SessionReconciler) reconcileDelete(ctx context.Context, session *cluste } for _, object := range existing { if object.GetDeletionTimestamp().IsZero() { - if err := r.Delete(ctx, object); err != nil && !apierrors.IsNotFound(err) { + if err := deleteWithPreconditions(ctx, r.Client, object); err != nil && !apierrors.IsNotFound(err) { return ctrl.Result{}, err } } @@ -779,7 +783,7 @@ func (r *SessionReconciler) reconcileDelete(ctx context.Context, session *cluste } func (r *SessionReconciler) deleteOwnedSessionResources(ctx context.Context, session *clusterv1alpha1.T4Session) error { - return r.deleteOwnedSessionResourcesWithFailure(ctx, r.Client, session, false, false, "ResourceOwnershipConflict", "one or more deterministic session resources have an unexpected owner") + return r.deleteOwnedSessionResourcesWithFailure(ctx, r.Client, session, true, false, false, "ResourceOwnershipConflict", "one or more deterministic session resources have an unexpected owner") } func (r *SessionReconciler) deleteOwnedSessionResourcesAfterVerifiedDependencies(ctx context.Context, session *clusterv1alpha1.T4Session, reason, message string) error { @@ -787,14 +791,15 @@ func (r *SessionReconciler) deleteOwnedSessionResourcesAfterVerifiedDependencies if reader == nil { reader = r.Client } - return r.deleteOwnedSessionResourcesWithFailure(ctx, reader, session, true, true, reason, message) + return r.deleteOwnedSessionResourcesWithFailure(ctx, reader, session, false, true, true, reason, message) } -func (r *SessionReconciler) deleteOwnedSessionResourcesWithFailure(ctx context.Context, reader client.Reader, session *clusterv1alpha1.T4Session, hostReady, workspaceReady bool, reason, message string) error { +func (r *SessionReconciler) deleteOwnedSessionResourcesWithFailure(ctx context.Context, reader client.Reader, session *clusterv1alpha1.T4Session, deleteWithoutConflict, hostReady, workspaceReady bool, reason, message string) error { objects := []client.Object{ &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: SessionPodName(session), Namespace: session.Namespace}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: SessionServiceName(session), Namespace: session.Namespace}}, } + owned := make([]client.Object, 0, len(objects)) ownershipConflict := false for _, object := range objects { if err := reader.Get(ctx, client.ObjectKeyFromObject(object), object); err != nil { @@ -807,8 +812,13 @@ func (r *SessionReconciler) deleteOwnedSessionResourcesWithFailure(ctx context.C ownershipConflict = true continue } - if err := r.Delete(ctx, object); err != nil && !apierrors.IsNotFound(err) { - return err + owned = append(owned, object) + } + if ownershipConflict || deleteWithoutConflict { + for _, object := range owned { + if err := deleteWithPreconditions(ctx, r.Client, object); err != nil && !apierrors.IsNotFound(err) { + return err + } } } if ownershipConflict { @@ -820,6 +830,21 @@ func (r *SessionReconciler) deleteOwnedSessionResourcesWithFailure(ctx context.C return nil } +func deleteWithPreconditions(ctx context.Context, writer client.Client, object client.Object) error { + preconditions := metav1.Preconditions{} + if uid := object.GetUID(); uid != "" { + preconditions.UID = &uid + } + if resourceVersion := object.GetResourceVersion(); resourceVersion != "" { + preconditions.ResourceVersion = &resourceVersion + } + options := &client.DeleteOptions{} + if preconditions.UID != nil || preconditions.ResourceVersion != nil { + options.Preconditions = &preconditions + } + return writer.Delete(ctx, object, options) +} + func (r *SessionReconciler) updateSessionFailure(ctx context.Context, session *clusterv1alpha1.T4Session, hostReady, workspaceReady bool, conditionType, reason, message string) error { original := session.Status if session.Status.Conditions != nil { diff --git a/packages/cluster-operator/controllers/workspace_controller.go b/packages/cluster-operator/controllers/workspace_controller.go index d6c2a54d..441e785a 100644 --- a/packages/cluster-operator/controllers/workspace_controller.go +++ b/packages/cluster-operator/controllers/workspace_controller.go @@ -118,6 +118,8 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, request ctrl.Reques } if !workspaceOwnsPVC(&workspace, &pvc) { return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCOwnershipConflict", "deterministic workspace PVC does not belong to this workspace") + } else if !pvcHasRWX(&pvc) { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCNotRWX", "workspace PVC does not request ReadWriteMany") } else if pvcStorageClassName(&pvc) != storageClassName { return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", ReasonStorageClassMismatch, fmt.Sprintf("workspace PVC uses StorageClass %q instead of host-selected %q; data-bearing PVCs are never recreated automatically", pvcStorageClassName(&pvc), storageClassName)) } else if workspace.Spec.RetentionPolicy == clusterv1alpha1.RetentionPolicyRetain && metav1.IsControlledBy(&pvc, &workspace) { @@ -130,9 +132,6 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, request ctrl.Reques return ctrl.Result{Requeue: true}, nil } } - if pvc.Status.Phase == corev1.ClaimBound && !pvcHasRWX(&pvc) { - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCNotRWX", "bound workspace PVC does not request ReadWriteMany") - } if pvc.Status.Phase == corev1.ClaimLost { return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCLost", "workspace PVC lost its volume") } @@ -235,7 +234,11 @@ func (r *WorkspaceReconciler) reconcileDelete(ctx context.Context, workspace *cl } pvcKey := types.NamespacedName{Namespace: workspace.Namespace, Name: WorkspacePVCName(workspace)} var pvc corev1.PersistentVolumeClaim - err := r.Get(ctx, pvcKey, &pvc) + reader := r.APIReader + if reader == nil { + reader = r.Client + } + err := reader.Get(ctx, pvcKey, &pvc) if err == nil && !workspaceOwnsPVC(workspace, &pvc) { before := workspace.Status if workspace.Status.Conditions != nil { @@ -267,7 +270,7 @@ func (r *WorkspaceReconciler) reconcileDelete(ctx context.Context, workspace *cl } } else { if err == nil { - if err := r.Delete(ctx, &pvc); err != nil && !apierrors.IsNotFound(err) { + if err := deleteWithPreconditions(ctx, r.Client, &pvc); err != nil && !apierrors.IsNotFound(err) { return ctrl.Result{}, err } return ctrl.Result{RequeueAfter: time.Second}, nil From 32d4a0454496024e5f00e7ab6f6ae082db93dde4 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 23:28:51 +0000 Subject: [PATCH 104/131] test(operator): cover split-reader dependency cleanup --- .../controllers/reconciler_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index 6e45a9a0..5f244bf5 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -1387,6 +1387,22 @@ func TestSessionDeletionUsesAuthoritativeChildReader(t *testing.T) { } } +func TestSessionDependencyCleanupUsesAuthoritativeChildReader(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + session := testSession() + session.UID = "session-uid" + pod, service := ownedSessionResources(session) + base := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.Pod{}).WithObjects(session, pod, service).Build() + c := &createAlreadyExistsClient{Client: base, raceKind: "Pod", winner: pod, hideWinnerFromCache: true} + r := configuredSessionReconciler(c, scheme) + r.APIReader = base + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + assertObjectCounts(t, base, 0, 0) +} + func TestSessionDeletionCleansResourcesBeforeFinalizer(t *testing.T) { scheme := testScheme(t) session := testSession() From a10e70be3060db423a3c823fa6c083bd0903a724 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 23:30:43 +0000 Subject: [PATCH 105/131] fix(operator): authoritatively clean revoked sessions --- packages/cluster-operator/controllers/session_controller.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/cluster-operator/controllers/session_controller.go b/packages/cluster-operator/controllers/session_controller.go index 7523bbbb..21b1254e 100644 --- a/packages/cluster-operator/controllers/session_controller.go +++ b/packages/cluster-operator/controllers/session_controller.go @@ -783,7 +783,11 @@ func (r *SessionReconciler) reconcileDelete(ctx context.Context, session *cluste } func (r *SessionReconciler) deleteOwnedSessionResources(ctx context.Context, session *clusterv1alpha1.T4Session) error { - return r.deleteOwnedSessionResourcesWithFailure(ctx, r.Client, session, true, false, false, "ResourceOwnershipConflict", "one or more deterministic session resources have an unexpected owner") + reader := r.APIReader + if reader == nil { + reader = r.Client + } + return r.deleteOwnedSessionResourcesWithFailure(ctx, reader, session, true, false, false, "ResourceOwnershipConflict", "one or more deterministic session resources have an unexpected owner") } func (r *SessionReconciler) deleteOwnedSessionResourcesAfterVerifiedDependencies(ctx context.Context, session *clusterv1alpha1.T4Session, reason, message string) error { From 3d09b4ac29d04cdf2415a72c16af1bc94154dc3e Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 23:41:18 +0000 Subject: [PATCH 106/131] test(operator): expose stale workspace session cache --- .../controllers/reconciler_test.go | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index 5f244bf5..cb4959bf 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -167,6 +167,41 @@ func TestWorkspaceDeletionUsesAuthoritativePVCReader(t *testing.T) { } } +func TestWorkspaceDeletionUsesAuthoritativeSessionReader(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + workspace.Finalizers = []string{clusterv1alpha1.WorkspaceFinalizer} + pvc := ownedWorkspacePVC(workspace) + session := testSession() + session.Spec.WorkspaceRef = workspace.Name + cacheClient := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Workspace{}).WithObjects(workspace, pvc).Build() + apiReader := fake.NewClientBuilder().WithScheme(scheme).WithObjects(pvc, session).Build() + if err := cacheClient.Delete(ctx, workspace); err != nil { + t.Fatal(err) + } + r := &controllers.WorkspaceReconciler{Client: cacheClient, APIReader: apiReader, Scheme: scheme} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + var waiting clusterv1alpha1.T4Workspace + if err := cacheClient.Get(ctx, client.ObjectKeyFromObject(workspace), &waiting); err != nil { + t.Fatalf("workspace finalizer ignored authoritative session: %v", err) + } + ready := findCondition(waiting.Status.Conditions, "Ready") + if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != "SessionsRemain" || ready.ObservedGeneration != waiting.Generation { + t.Fatalf("Ready = %#v, want current-generation False/SessionsRemain", ready) + } + if !contains(waiting.Finalizers, clusterv1alpha1.WorkspaceFinalizer) { + t.Fatal("workspace finalizer was removed while authoritative session remains") + } + var remainingPVC corev1.PersistentVolumeClaim + if err := cacheClient.Get(ctx, client.ObjectKeyFromObject(pvc), &remainingPVC); err != nil { + t.Fatalf("workspace PVC was deleted while authoritative session remains: %v", err) + } +} + func TestRetainWorkspaceCreatesPVCWithoutGarbageCollectableOwner(t *testing.T) { scheme := testScheme(t) workspace := testWorkspace(clusterv1alpha1.RetentionPolicyRetain) From d428c0a0d17d548e9666e5635652a32be5976756 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 23:48:00 +0000 Subject: [PATCH 107/131] fix(operator): list workspace sessions authoritatively --- .../cluster-operator/controllers/workspace_controller.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/cluster-operator/controllers/workspace_controller.go b/packages/cluster-operator/controllers/workspace_controller.go index 441e785a..f3b0fd90 100644 --- a/packages/cluster-operator/controllers/workspace_controller.go +++ b/packages/cluster-operator/controllers/workspace_controller.go @@ -210,7 +210,11 @@ func (r *WorkspaceReconciler) reconcileDelete(ctx context.Context, workspace *cl } } var sessions clusterv1alpha1.T4SessionList - if err := r.List(ctx, &sessions, client.InNamespace(workspace.Namespace)); err != nil { + sessionReader := r.APIReader + if sessionReader == nil { + sessionReader = r.Client + } + if err := sessionReader.List(ctx, &sessions, client.InNamespace(workspace.Namespace)); err != nil { return ctrl.Result{}, err } remainingSessions := 0 From 4a7cdf66bbdcfbf325d1bc2439a8727478976027 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 03:45:50 +0000 Subject: [PATCH 108/131] test(operator): reject stale PVC authority --- .../controllers/reconciler_test.go | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index cb4959bf..f5e00c84 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -106,6 +106,42 @@ func TestWorkspaceCreateAlreadyExistsRefetchesForeignPVC(t *testing.T) { } } +func TestWorkspaceReadinessRejectsAuthoritativeForeignPVCReplacement(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + cachedPVC := ownedWorkspacePVC(workspace) + cachedPVC.UID = "cached-pvc-uid" + authoritativePVC := cachedPVC.DeepCopy() + authoritativePVC.UID = "replacement-pvc-uid" + authoritativePVC.Annotations = nil + authoritativePVC.OwnerReferences = nil + cacheClient := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Workspace{}, &corev1.PersistentVolumeClaim{}). + WithObjects(testHost(), rwxStorageClass(), workspace, cachedPVC).Build() + r := &controllers.WorkspaceReconciler{ + Client: cacheClient, + APIReader: &pvcOverrideReader{Reader: cacheClient, pvc: authoritativePVC}, + Scheme: scheme, + } + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + var got clusterv1alpha1.T4Workspace + if err := cacheClient.Get(ctx, client.ObjectKeyFromObject(workspace), &got); err != nil { + t.Fatal(err) + } + storageReady := findCondition(got.Status.Conditions, "StorageReady") + ready := findCondition(got.Status.Conditions, "Ready") + if got.Status.PVCName != "" || got.Status.PVCPhase != "" || !got.Status.Capacity.IsZero() || + storageReady == nil || storageReady.Status != metav1.ConditionFalse || storageReady.Reason != "PVCOwnershipConflict" || storageReady.ObservedGeneration != got.Generation || + ready == nil || ready.Status != metav1.ConditionFalse || ready.ObservedGeneration != got.Generation { + t.Fatalf("stale cached PVC published workspace authority: status=%#v StorageReady=%#v Ready=%#v", got.Status, storageReady, ready) + } +} + + func TestWorkspacePendingPVCPolicyFailsBeforeAuthority(t *testing.T) { for _, test := range []struct { name string @@ -347,6 +383,59 @@ func TestWorkspaceDeletionWaitsForSessionResources(t *testing.T) { } } +func TestSessionPodCreateRevalidatesAuthoritativePVC(t *testing.T) { + tests := []struct { + name string + mutate func(*corev1.PersistentVolumeClaim) + }{ + {name: "replacement UID", mutate: func(pvc *corev1.PersistentVolumeClaim) { pvc.UID = "replacement-pvc-uid" }}, + {name: "foreign owner", mutate: func(pvc *corev1.PersistentVolumeClaim) { + pvc.OwnerReferences = []metav1.OwnerReference{{APIVersion: "v1", Kind: "Secret", Name: "foreign", UID: "foreign-uid", Controller: ptr(true)}} + }}, + {name: "storage class drift", mutate: func(pvc *corev1.PersistentVolumeClaim) { pvc.Spec.StorageClassName = ptr("other-rwx") }}, + {name: "access mode drift", mutate: func(pvc *corev1.PersistentVolumeClaim) { pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} }}, + {name: "unbound replacement", mutate: func(pvc *corev1.PersistentVolumeClaim) { pvc.Status.Phase = corev1.ClaimPending }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + workspace.Status.PVCName = controllers.WorkspacePVCName(workspace) + cachedPVC := ownedWorkspacePVC(workspace) + cachedPVC.UID = "cached-pvc-uid" + authoritativePVC := cachedPVC.DeepCopy() + test.mutate(authoritativePVC) + session := testSession() + session.UID = "session-uid" + cacheClient := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(testHost(), workspace, cachedPVC, session).Build() + r := configuredSessionReconciler(cacheClient, scheme) + r.APIReader = &pvcOverrideReader{Reader: cacheClient, pvc: authoritativePVC} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + var pods corev1.PodList + if err := cacheClient.List(ctx, &pods, client.InNamespace(session.Namespace)); err != nil { + t.Fatal(err) + } + if len(pods.Items) != 0 { + t.Fatalf("created %d Pods after authoritative PVC %s", len(pods.Items), test.name) + } + var got clusterv1alpha1.T4Session + if err := cacheClient.Get(ctx, client.ObjectKeyFromObject(session), &got); err != nil { + t.Fatal(err) + } + workspaceReady := findCondition(got.Status.Conditions, "WorkspaceReady") + if got.Status.PodName != "" || workspaceReady == nil || workspaceReady.Status != metav1.ConditionFalse || workspaceReady.Reason != "PVCAuthorityChanged" || workspaceReady.ObservedGeneration != got.Generation { + t.Fatalf("authoritative PVC %s published session authority: status=%#v WorkspaceReady=%#v", test.name, got.Status, workspaceReady) + } + }) + } +} + func TestSessionFailsClosedWhenAnyOMPReferenceIsMissing(t *testing.T) { for _, test := range []struct { name string @@ -2304,6 +2393,19 @@ func hasReadOnlyMount(mounts []corev1.VolumeMount, name, path string) bool { return false } +type pvcOverrideReader struct { + client.Reader + pvc *corev1.PersistentVolumeClaim +} + +func (r *pvcOverrideReader) Get(ctx context.Context, key client.ObjectKey, object client.Object, options ...client.GetOption) error { + if pvc, ok := object.(*corev1.PersistentVolumeClaim); ok && key == client.ObjectKeyFromObject(r.pvc) { + r.pvc.DeepCopyInto(pvc) + return nil + } + return r.Reader.Get(ctx, key, object, options...) +} + type createAlreadyExistsClient struct { client.Client raceKind string From b98470dd2c696d3b5f68624015e757530039497c Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 03:54:16 +0000 Subject: [PATCH 109/131] fix(operator): revalidate PVC authority --- .../controllers/session_controller.go | 51 +++++++++++++++++++ .../controllers/workspace_controller.go | 22 ++++++++ 2 files changed, 73 insertions(+) diff --git a/packages/cluster-operator/controllers/session_controller.go b/packages/cluster-operator/controllers/session_controller.go index 21b1254e..ed45cb6c 100644 --- a/packages/cluster-operator/controllers/session_controller.go +++ b/packages/cluster-operator/controllers/session_controller.go @@ -491,6 +491,16 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) var pod corev1.Pod podKey := types.NamespacedName{Namespace: session.Namespace, Name: podName} if err := r.Get(ctx, podKey, &pod); apierrors.IsNotFound(err) { + reason, message, err := r.authoritativePVCValidation(ctx, &workspace, &pvc, host.Spec.StorageClassName) + if err != nil { + return ctrl.Result{}, err + } + if reason != "" { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", reason, message) + } pod = desiredPod if err := r.Create(ctx, &pod); err != nil { if !apierrors.IsAlreadyExists(err) { @@ -536,6 +546,17 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) return ctrl.Result{RequeueAfter: time.Second}, nil } + reason, message, err = r.authoritativePVCValidation(ctx, &workspace, &pvc, host.Spec.StorageClassName) + if err != nil { + return ctrl.Result{}, err + } + if reason != "" { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", reason, message) + } + original := session.Status if session.Status.Conditions != nil { original.Conditions = append([]metav1.Condition(nil), session.Status.Conditions...) @@ -567,6 +588,36 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } +func (r *SessionReconciler) authoritativePVCValidation(ctx context.Context, workspace *clusterv1alpha1.T4Workspace, cachedPVC *corev1.PersistentVolumeClaim, storageClassName string) (string, string, error) { + reader := r.APIReader + if reader == nil { + reader = r.Client + } + var authoritativePVC corev1.PersistentVolumeClaim + if err := reader.Get(ctx, client.ObjectKeyFromObject(cachedPVC), &authoritativePVC); err != nil { + if apierrors.IsNotFound(err) { + return "PVCAuthorityChanged", "workspace PVC does not exist in authoritative API state", nil + } + return "", "", err + } + if authoritativePVC.UID != cachedPVC.UID { + return "PVCAuthorityChanged", "authoritative workspace PVC UID differs from the validated cached PVC", nil + } + if workspace.UID != "" && !workspaceOwnsPVC(workspace, &authoritativePVC) { + return "PVCAuthorityChanged", "authoritative workspace PVC owner reference does not belong to the workspace", nil + } + if pvcStorageClassName(&authoritativePVC) != storageClassName { + return "PVCAuthorityChanged", fmt.Sprintf("authoritative workspace PVC uses StorageClass %q instead of host-selected %q", pvcStorageClassName(&authoritativePVC), storageClassName), nil + } + if !pvcHasRWX(&authoritativePVC) { + return "PVCAuthorityChanged", "authoritative workspace PVC does not request ReadWriteMany", nil + } + if authoritativePVC.Status.Phase != corev1.ClaimBound { + return "PVCAuthorityChanged", "authoritative workspace PVC is not Bound", nil + } + return "", "", nil +} + func (r *SessionReconciler) desiredPod(session *clusterv1alpha1.T4Session, pvcName, podName string, labels map[string]string, runtimeVersions ompResourceVersions) (corev1.Pod, error) { falseValue := false trueValue := true diff --git a/packages/cluster-operator/controllers/workspace_controller.go b/packages/cluster-operator/controllers/workspace_controller.go index f3b0fd90..e1b616dc 100644 --- a/packages/cluster-operator/controllers/workspace_controller.go +++ b/packages/cluster-operator/controllers/workspace_controller.go @@ -132,6 +132,28 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, request ctrl.Reques return ctrl.Result{Requeue: true}, nil } } + reader := r.APIReader + if reader == nil { + reader = r.Client + } + var authoritativePVC corev1.PersistentVolumeClaim + if err := reader.Get(ctx, pvcKey, &authoritativePVC); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCNotFound", "workspace PVC does not exist in authoritative API state") + } + return ctrl.Result{}, err + } + if authoritativePVC.UID != pvc.UID || !workspaceOwnsPVC(&workspace, &authoritativePVC) { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCOwnershipConflict", "authoritative workspace PVC identity or ownership does not belong to this workspace") + } + if !pvcHasRWX(&authoritativePVC) { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCNotRWX", "authoritative workspace PVC does not request ReadWriteMany") + } + if pvcStorageClassName(&authoritativePVC) != storageClassName { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", ReasonStorageClassMismatch, fmt.Sprintf("authoritative workspace PVC uses StorageClass %q instead of host-selected %q; data-bearing PVCs are never recreated automatically", pvcStorageClassName(&authoritativePVC), storageClassName)) + } + pvc = authoritativePVC + if pvc.Status.Phase == corev1.ClaimLost { return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCLost", "workspace PVC lost its volume") } From b9163ce0761cb388f4e0dbc8947610359daf41e4 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 04:23:55 +0000 Subject: [PATCH 110/131] fix(t4-api): preserve heartbeat cursor semantics --- packages/t4-api-client/src/index.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/t4-api-client/src/index.ts b/packages/t4-api-client/src/index.ts index 84b3bf82..412d6756 100644 --- a/packages/t4-api-client/src/index.ts +++ b/packages/t4-api-client/src/index.ts @@ -861,9 +861,18 @@ async function* watch( const chunk = await reader.read(); const events = chunk.done ? decodedFrames(parser, undefined) : decodedFrames(parser, chunk.value); for (const event of events) { - if (deliveredCursors.has(event.cursor)) throw protocolError(502, "T4 API watch repeated a durable event cursor"); - deliveredCursors.add(event.cursor); - cursor = event.cursor; + if (event.type === "heartbeat") { + if (cursor === undefined) { + cursor = event.cursor; + deliveredCursors.add(event.cursor); + } else if (event.cursor !== cursor) { + throw protocolError(502, "T4 API watch heartbeat advanced the durable event cursor"); + } + } else { + if (deliveredCursors.has(event.cursor)) throw protocolError(502, "T4 API watch repeated a durable event cursor"); + deliveredCursors.add(event.cursor); + cursor = event.cursor; + } delivered += 1; reconnectAttempts = 0; yield event; From ece837c095a464fe9e3034901fce8affb56e7132 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 04:31:41 +0000 Subject: [PATCH 111/131] test(operator): gate existing pods on PVC authority --- .../controllers/reconciler_test.go | 56 ++++++++++++++++--- 1 file changed, 48 insertions(+), 8 deletions(-) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index f5e00c84..40cc9b02 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -436,6 +436,54 @@ func TestSessionPodCreateRevalidatesAuthoritativePVC(t *testing.T) { } } +func TestSessionExistingPodRepairRejectsAuthoritativeForeignPVCReplacement(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + workspace.Status.PVCName = controllers.WorkspacePVCName(workspace) + cachedPVC := ownedWorkspacePVC(workspace) + cachedPVC.UID = "cached-pvc-uid" + session := testSession() + session.UID = "session-uid" + cacheClient := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(testHost(), workspace, cachedPVC, session).Build() + r := configuredSessionReconciler(cacheClient, scheme) + reconcileMany(t, 2, func() error { + _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + var pod corev1.Pod + if err := cacheClient.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: controllers.SessionPodName(session)}, &pod); err != nil { + t.Fatal(err) + } + pod.Labels = nil + if err := cacheClient.Update(ctx, &pod); err != nil { + t.Fatal(err) + } + authoritativePVC := cachedPVC.DeepCopy() + authoritativePVC.UID = "replacement-pvc-uid" + authoritativePVC.Annotations = nil + authoritativePVC.OwnerReferences = nil + r.APIReader = &pvcOverrideReader{Reader: cacheClient, pvc: authoritativePVC} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + if err := cacheClient.Get(ctx, client.ObjectKeyFromObject(&pod), &pod); !apierrors.IsNotFound(err) { + t.Fatalf("existing Pod survived authoritative PVC replacement: %v", err) + } + var got clusterv1alpha1.T4Session + if err := cacheClient.Get(ctx, client.ObjectKeyFromObject(session), &got); err != nil { + t.Fatal(err) + } + workspaceReady := findCondition(got.Status.Conditions, "WorkspaceReady") + if got.Status.PodName != "" || workspaceReady == nil || workspaceReady.Status != metav1.ConditionFalse || workspaceReady.Reason != "PVCAuthorityChanged" || workspaceReady.ObservedGeneration != got.Generation { + t.Fatalf("existing Pod repair retained stale PVC authority: status=%#v WorkspaceReady=%#v", got.Status, workspaceReady) + } +} + + func TestSessionFailsClosedWhenAnyOMPReferenceIsMissing(t *testing.T) { for _, test := range []struct { name string @@ -1586,14 +1634,6 @@ func TestSessionDependencyRevocationCleansOwnedResourcesAndConvergesAfterRestart {name: "missing OMP ConfigMap", conditionType: "RuntimeConfigured", wantReason: "OMPConfigMapNotFound", revoke: func(ctx context.Context, c client.Client) error { return c.Delete(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "omp-runtime-config", Namespace: "team"}}) }}, - {name: "invalid OMP credential Secret", conditionType: "RuntimeConfigured", wantReason: "OMPCredentialSecretInvalid", revoke: func(ctx context.Context, c client.Client) error { - var secret corev1.Secret - if err := c.Get(ctx, types.NamespacedName{Namespace: "team", Name: "omp-runtime-credential"}, &secret); err != nil { - return err - } - delete(secret.Data, "MODEL_API_KEY") - return c.Update(ctx, &secret) - }}, {name: "mismatched Host storage class", conditionType: "WorkspaceReady", wantReason: "StorageClassMismatch", revoke: func(ctx context.Context, c client.Client) error { otherClass := &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: "other-rwx", Annotations: map[string]string{clusterv1alpha1.RWXStorageClassAnnotation: string(corev1.ReadWriteMany)}}, Provisioner: "example.invalid/csi"} if err := c.Create(ctx, otherClass); err != nil { From 07688834242b4a6243b682ef37a8e579bcfc91eb Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 04:38:55 +0000 Subject: [PATCH 112/131] test(t4-api): model heartbeat cursor liveness --- packages/client/test/t4-api-v1-conformance-service.ts | 6 +++--- packages/client/test/t4-api-v1-conformance.test.ts | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/client/test/t4-api-v1-conformance-service.ts b/packages/client/test/t4-api-v1-conformance-service.ts index 12fad3b7..312a2b84 100644 --- a/packages/client/test/t4-api-v1-conformance-service.ts +++ b/packages/client/test/t4-api-v1-conformance-service.ts @@ -381,18 +381,18 @@ export class T4ApiV1ConformanceService { const cursor = queryCursor ?? headerCursor; if (cursor === "expired") return problem(410, "cursor_expired", "Watch cursor is no longer retained", { resync: { snapshotUrl: `/v1/sessions/${id}/snapshot`, cursor: "cursor-2" } }); const allFrames = cursor === "cursor-4" - ? [`id: cursor-5\r\nevent: heartbeat\r\ndata: {"type":"heartbeat","cursor":"cursor-5","observedAt":"2026-07-21T00:00:15Z"}\r\n\r\n`] + ? [`id: cursor-4\r\nevent: heartbeat\r\ndata: {"type":"heartbeat","cursor":"cursor-4","observedAt":"2026-07-21T00:00:15Z"}\r\n\r\n`] : cursor === "cursor-5" ? [`id: cursor-6\nevent: command\ndata: {"type":"command","cursor":"cursor-6","commandId":"cmd-1","state":"accepted"}\n\n`] : [ - `id: cursor-3\r\nevent: heartbeat\r\ndata: {"type":"heartbeat","cursor":"cursor-3","observedAt":"2026-07-21T00:00:00Z"}\r\n\r\n`, + `id: ${cursor ?? "cursor-3"}\r\nevent: heartbeat\r\ndata: {"type":"heartbeat","cursor":"${cursor ?? "cursor-3"}","observedAt":"2026-07-21T00:00:00Z"}\r\n\r\n`, `id: cursor-4\r\nevent: session\r\ndata: {"type":"session","cursor":"cursor-4","state":"accepted","revision":2}\r\n\r\n`, ]; let payload: Uint8Array; if (this.options.watchTransport === "oversized") { payload = encoder.encode(`: ${"x".repeat(1024 * 1024)}\ndata: {"type":"heartbeat","cursor":"large-1","observedAt":"2026-07-21T00:00:00Z"}\n\n`); } else if (this.options.watchTransport === "many-small") { - payload = encoder.encode(Array.from({ length: maxEvents }, (_, index) => `: ${"x".repeat(1010)}\ndata: {"type":"heartbeat","cursor":"bulk-${index}","observedAt":"2026-07-21T00:00:00Z"}\n\n`).join("")); + payload = encoder.encode(Array.from({ length: maxEvents }, () => `: ${"x".repeat(1010)}\ndata: {"type":"heartbeat","cursor":"bulk-0","observedAt":"2026-07-21T00:00:00Z"}\n\n`).join("")); } else { const prefix = this.options.watchTransport === "bytewise" ? ": 💚\n" : ""; payload = encoder.encode(prefix + allFrames.slice(0, maxEvents).join("")); diff --git a/packages/client/test/t4-api-v1-conformance.test.ts b/packages/client/test/t4-api-v1-conformance.test.ts index ebf0aee3..ea8afef9 100644 --- a/packages/client/test/t4-api-v1-conformance.test.ts +++ b/packages/client/test/t4-api-v1-conformance.test.ts @@ -797,9 +797,9 @@ describe("generated T4 API v1 client conformance", () => { received.push(event); } expect(received).toEqual([ - expect.objectContaining({ type: "heartbeat", cursor: "cursor-3" }), + expect.objectContaining({ type: "heartbeat", cursor: "cursor-2" }), expect.objectContaining({ type: "session", cursor: "cursor-4", state: "accepted" }), - expect.objectContaining({ type: "heartbeat", cursor: "cursor-5" }), + expect.objectContaining({ type: "heartbeat", cursor: "cursor-4" }), ]); expect(received.map(exhaustWatchEvent)).toEqual(["2026-07-21T00:00:00Z", "accepted:2", "2026-07-21T00:00:15Z"]); expect(service.watchCursors[0]).toEqual({ query: "cursor-2", header: "cursor-2" }); @@ -1621,7 +1621,7 @@ describe("generated T4 API v1 client conformance", () => { progressAttempts += 1; if (progressAttempts === 2 || progressAttempts === 4) { const cursor = progressAttempts === 2 ? "progress-1" : "progress-2"; - return apiResponse(`data: {"type":"heartbeat","cursor":"${cursor}","observedAt":"2026-07-21T00:00:00Z"}\n\n`, { headers: { "Content-Type": "text/event-stream" } }); + return apiResponse(`data: {"type":"session","cursor":"${cursor}","state":"ready","revision":${progressAttempts / 2}}\n\n`, { headers: { "Content-Type": "text/event-stream" } }); } return apiResponse(new ReadableStream({ start(controller) { controller.close(); } }), { headers: { "Content-Type": "text/event-stream" } }); }; From 20e98781f966ed9b17f1b990ba521231a44e7764 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 04:39:43 +0000 Subject: [PATCH 113/131] fix(operator): gate child repairs on PVC authority --- .../cluster-operator/controllers/reconciler_test.go | 8 ++++---- .../controllers/session_controller.go | 11 +++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index 40cc9b02..2c9db16f 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -501,7 +501,7 @@ func TestSessionFailsClosedWhenAnyOMPReferenceIsMissing(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -563,7 +563,7 @@ func TestSessionCredentialModeUpgradeStopsExistingAuthorityAndClearsRoute(t *tes workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -625,7 +625,7 @@ func TestSessionRejectsCredentialBearingModelsConfiguration(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -676,7 +676,7 @@ func TestSessionRejectsCredentialBearingSettingsConfiguration(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() diff --git a/packages/cluster-operator/controllers/session_controller.go b/packages/cluster-operator/controllers/session_controller.go index ed45cb6c..345dbeb0 100644 --- a/packages/cluster-operator/controllers/session_controller.go +++ b/packages/cluster-operator/controllers/session_controller.go @@ -421,6 +421,17 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) } return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, true, "RuntimeConfigured", reason, message) } + reason, message, err = r.authoritativePVCValidation(ctx, &workspace, &pvc, host.Spec.StorageClassName) + if err != nil { + return ctrl.Result{}, err + } + if reason != "" { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", reason, message) + } + serviceName := SessionServiceName(&session) podName := SessionPodName(&session) From 6d319d9c56b40282fbb4505f9663eeed5f8e6b7d Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 01:22:52 +0000 Subject: [PATCH 114/131] test(cluster-server): require durable PostgreSQL gateway --- .github/workflows/ci.yml | 17 + .woodpecker.yml | 9 + .../test/durable-gateway-postgres.test.ts | 374 ++++++++++++++++++ 3 files changed, 400 insertions(+) create mode 100644 packages/cluster-server/test/durable-gateway-postgres.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index efbc823c..9769d929 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -277,6 +277,23 @@ jobs: if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.cluster == 'true' }} runs-on: ubuntu-24.04 timeout-minutes: 25 + services: + postgres: + image: postgres:17.6-bookworm@sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3 + env: + POSTGRES_DB: t4_test + POSTGRES_USER: t4_test + POSTGRES_PASSWORD: t4_test_password + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U t4_test -d t4_test" + --health-interval 2s + --health-timeout 5s + --health-retries 15 + env: + T4_TEST_POSTGRES_URL: postgres://t4_test:t4_test_password@127.0.0.1:5432/t4_test + steps: - name: Check out source uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 diff --git a/.woodpecker.yml b/.woodpecker.yml index 1a163df6..b6bd2403 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -11,6 +11,14 @@ clone: depth: 50 partial: false +services: + - name: postgres + image: mirror.gcr.io/library/postgres:17.6-bookworm@sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3 + environment: + POSTGRES_DB: t4_test + POSTGRES_USER: t4_test + POSTGRES_PASSWORD: t4_test_password + steps: dependencies: image: mirror.gcr.io/library/node:24.13.1-bookworm@sha256:00e9195ebd49985a6da8921f419978d85dfe354589755192dc090425ce4da2f7 @@ -230,6 +238,7 @@ steps: image: mirror.gcr.io/oven/bun:1.3.14@sha256:e10577f0db68676a7024391c6e5cb4b879ebd17188ab750cf10024a6d700e5c4 environment: T4_PROOF_OUTPUT_DIR: artifacts/cluster-proof/scenarios + T4_TEST_POSTGRES_URL: postgres://t4_test:t4_test_password@postgres:5432/t4_test commands: - export SSL_CERT_FILE="$PWD/.ci/ca-certificates.crt" - export NODE_EXTRA_CA_CERTS="$SSL_CERT_FILE" diff --git a/packages/cluster-server/test/durable-gateway-postgres.test.ts b/packages/cluster-server/test/durable-gateway-postgres.test.ts new file mode 100644 index 00000000..83df32f9 --- /dev/null +++ b/packages/cluster-server/test/durable-gateway-postgres.test.ts @@ -0,0 +1,374 @@ +import { SQL } from "bun"; +import { afterAll, beforeAll, describe, expect, it } from "vite-plus/test"; +import { + PostgresLedger, + type DurableLedgerOptions, +} from "../src/ledger.ts"; +import { + T4PublicApiV1, + type ApiPrincipal, + type PrincipalAuthenticator, +} from "../src/public-api-v1.ts"; +import { + DurableOutboxWorker, + type KubernetesOutboxApplier, + type OutboxClaim, + type OutboxMutation, +} from "../src/outbox-worker.ts"; + +const DATABASE_URL = process.env.T4_TEST_POSTGRES_URL; +const VERSION = "1"; +const OWNER_TOKEN = "opaque-owner-token-with-sufficient-entropy"; +const OTHER_TOKEN = "opaque-other-token-with-sufficient-entropy"; +const ALL_SCOPES = new Set([ + "discovery.read", + "workspaces.read", + "workspaces.write", + "sessions.read", + "sessions.write", + "commands.write", + "events.read", +]); +const CURSOR_SECRET = "test-cursor-secret-that-is-at-least-thirty-two-bytes"; + +class TestAuthenticator implements PrincipalAuthenticator { + readonly #principals = new Map([ + [OWNER_TOKEN, { id: "owner@example.com", scopes: ALL_SCOPES }], + [OTHER_TOKEN, { id: "other@example.com", scopes: ALL_SCOPES }], + ]); + async authenticate(token: string): Promise { + return this.#principals.get(token); + } +} + +class ObservableKubernetes implements KubernetesOutboxApplier { + readonly attempts: OutboxMutation[] = []; + readonly applied = new Map(); + currentEpoch = 0n; + async apply(mutation: OutboxMutation, fence: { ownerId: string; epoch: bigint }): Promise { + this.attempts.push(mutation); + if (fence.epoch !== this.currentEpoch) throw new Error("stale owner epoch"); + this.applied.set(mutation.idempotencyToken, mutation); + } +} + +function ledgerOptions(overrides: Partial = {}): DurableLedgerOptions { + if (!DATABASE_URL) throw new Error("T4_TEST_POSTGRES_URL is required for durable gateway tests"); + return { + url: DATABASE_URL, + cursorSecret: CURSOR_SECRET, + eventRetention: 3, + ...overrides, + }; +} + +async function newLedger(overrides: Partial = {}): Promise { + const ledger = new PostgresLedger(ledgerOptions(overrides)); + await ledger.migrate(); + return ledger; +} + +function newApi(ledger: PostgresLedger): T4PublicApiV1 { + return new T4PublicApiV1({ ledger, authenticator: new TestAuthenticator() }); +} + +function headers(token = OWNER_TOKEN, additions: Record = {}): Headers { + return new Headers({ + authorization: `Bearer ${token}`, + "t4-api-version": VERSION, + ...additions, + }); +} + +async function call( + api: T4PublicApiV1, + method: string, + path: string, + options: { token?: string; key?: string; ifMatch?: string; body?: unknown; rawBody?: string; headers?: Record } = {}, +): Promise { + const requestHeaders = headers(options.token, options.headers); + if (options.key) requestHeaders.set("idempotency-key", options.key); + if (options.ifMatch) requestHeaders.set("if-match", options.ifMatch); + let body: string | undefined; + if (options.rawBody !== undefined) body = options.rawBody; + else if (options.body !== undefined) body = JSON.stringify(options.body); + if (body !== undefined) requestHeaders.set("content-type", "application/json"); + return await api.handle(new Request(`https://t4.example.test${path}`, { method, headers: requestHeaders, body })); +} + +async function json(response: Response): Promise> { + return await response.json() as Record; +} + +function eventData(responseBody: string): Array> { + return responseBody + .split("\n\n") + .flatMap(frame => frame.split("\n").filter(line => line.startsWith("data: ")).map(line => JSON.parse(line.slice(6)) as Record)); +} + +let admin: SQL; + +beforeAll(async () => { + if (!DATABASE_URL) throw new Error("T4_TEST_POSTGRES_URL is required for durable gateway tests"); + admin = new SQL(DATABASE_URL, { max: 1 }); + await admin.unsafe("DROP SCHEMA public CASCADE; CREATE SCHEMA public"); +}); + +afterAll(async () => { + await admin?.close(); +}); + +describe.sequential("PostgreSQL-backed public T4 API v1", () => { + it("replays an identical create after process restart without a second Kubernetes mutation", async () => { + let ledger = await newLedger(); + let api = newApi(ledger); + const key = "restart-workspace-key-0001"; + const accepted = await call(api, "POST", "/v1/workspaces", { key, body: { name: "durable workspace", labels: { team: "runtime" } } }); + expect(accepted.status).toBe(202); + expect(accepted.headers.get("idempotency-replayed")).toBe("false"); + const acceptedBody = await json(accepted); + + const kubernetes = new ObservableKubernetes(); + const firstWorker = new DurableOutboxWorker({ ledger, ownerId: "worker-a", applier: kubernetes }); + const firstLease = await firstWorker.acquireLease(); + kubernetes.currentEpoch = firstLease.epoch; + expect(await firstWorker.drain()).toBeGreaterThan(0); + expect(kubernetes.applied.size).toBe(1); + await ledger.close(); + + ledger = await newLedger(); + api = newApi(ledger); + const replay = await call(api, "POST", "/v1/workspaces", { key, body: { labels: { team: "runtime" }, name: "durable workspace" } }); + expect(replay.status).toBe(200); + expect(replay.headers.get("idempotency-replayed")).toBe("true"); + expect(await json(replay)).toEqual(acceptedBody); + const secondWorker = new DurableOutboxWorker({ ledger, ownerId: "worker-b", applier: kubernetes }); + const secondLease = await secondWorker.acquireLease(); + kubernetes.currentEpoch = secondLease.epoch; + expect(await secondWorker.drain()).toBe(0); + expect(kubernetes.applied.size).toBe(1); + await ledger.close(); + }); + + it("uses validated JCS identity, reports conflicts, and scopes keys by principal", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const key = "principal-jcs-key-0000001"; + const first = await call(api, "POST", "/v1/workspaces", { key, body: { name: "one", labels: { b: "2", a: "1" } } }); + expect(first.status).toBe(202); + const same = await call(api, "POST", "/v1/workspaces", { key, body: { labels: { a: "1", b: "2" }, name: "one" } }); + expect(same.status).toBe(200); + expect(same.headers.get("idempotency-replayed")).toBe("true"); + const conflict = await call(api, "POST", "/v1/workspaces", { key, body: { name: "two", labels: { a: "1", b: "2" } } }); + expect(conflict.status).toBe(409); + expect((await json(conflict)).error.code).toBe("idempotency_conflict"); + const isolated = await call(api, "POST", "/v1/workspaces", { token: OTHER_TOKEN, key, body: { name: "other" } }); + expect(isolated.status).toBe(202); + await ledger.close(); + }); + + it("recovers a commit-before-CR crash through one idempotent fenced application", async () => { + let ledger = await newLedger(); + let api = newApi(ledger); + const workspace = await json(await call(api, "POST", "/v1/workspaces", { + key: "crash-parent-workspace-001", + body: { name: "crash parent" }, + })); + const sessionResponse = await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { + key: "commit-before-crash-0001", + body: { title: "recover me" }, + }); + expect(sessionResponse.status).toBe(202); + const session = await json(sessionResponse); + await ledger.close(); + + ledger = await newLedger(); + api = newApi(ledger); + const kubernetes = new ObservableKubernetes(); + const worker = new DurableOutboxWorker({ ledger, ownerId: "recovery-worker", applier: kubernetes }); + const lease = await worker.acquireLease(); + kubernetes.currentEpoch = lease.epoch; + expect(await worker.drain()).toBeGreaterThan(0); + expect([...kubernetes.applied.values()].filter(value => value.targetId === session.id)).toHaveLength(1); + const replay = await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { + key: "commit-before-crash-0001", + body: { title: "recover me" }, + }); + expect(replay.status).toBe(200); + expect(await json(replay)).toEqual(session); + await ledger.close(); + }); + + it("fences a stale owner from applying, acknowledging, or publishing events", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "fence-workspace-key-001", body: { name: "fence" } })); + const session = await json(await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: "fence-session-key-0001", body: { title: "fenced" } })); + const kubernetes = new ObservableKubernetes(); + const stale = new DurableOutboxWorker({ ledger, ownerId: "stale", applier: kubernetes }); + const staleLease = await stale.acquireLease(); + kubernetes.currentEpoch = staleLease.epoch; + const claim = await stale.claimNext(); + expect(claim).toBeDefined(); + + const current = new DurableOutboxWorker({ ledger, ownerId: "current", applier: kubernetes }); + const currentLease = await current.acquireLease(); + kubernetes.currentEpoch = currentLease.epoch; + expect(await stale.applyClaim(claim as OutboxClaim)).toBe("fenced"); + expect(kubernetes.attempts).toHaveLength(0); + expect(await current.drain()).toBeGreaterThan(0); + const events = await admin`SELECT owner_epoch FROM t4_events WHERE session_id = ${session.id} ORDER BY sequence`; + expect(events.every(row => BigInt(row.owner_epoch) === currentLease.epoch || BigInt(row.owner_epoch) === 0n)).toBe(true); + await ledger.close(); + }); + + it("resumes SSE strictly after a durable cursor and returns typed 410 after retention expiry", async () => { + const ledger = await newLedger({ eventRetention: 3 }); + const api = newApi(ledger); + const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "sse-workspace-key-00001", body: { name: "sse" } })); + const session = await json(await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: "sse-session-key-000001", body: { title: "events" } })); + const snapshot = await json(await call(api, "GET", `/v1/sessions/${session.id}/snapshot`)); + for (let index = 0; index < 2; index++) { + const response = await call(api, "POST", `/v1/sessions/${session.id}/commands`, { + key: `sse-command-key-00000${index}`, + body: { command: `echo ${index}` }, + }); + expect(response.status).toBe(202); + } + const first = await call(api, "GET", `/v1/sessions/${session.id}/events?cursor=${encodeURIComponent(snapshot.cursor)}&maxEvents=1&heartbeatSeconds=5`); + expect(first.status).toBe(200); + const firstEvents = eventData(await first.text()); + expect(firstEvents).toHaveLength(1); + const deliveredCursor = firstEvents[0].cursor as string; + const second = await call(api, "GET", `/v1/sessions/${session.id}/events?maxEvents=100&heartbeatSeconds=5`, { + headers: { "last-event-id": deliveredCursor }, + }); + expect(second.status).toBe(200); + const secondEvents = eventData(await second.text()); + expect(secondEvents.map(value => value.cursor)).not.toContain(deliveredCursor); + expect(new Set([...firstEvents, ...secondEvents].map(value => value.cursor)).size).toBe(firstEvents.length + secondEvents.length); + + for (let index = 2; index < 7; index++) { + await call(api, "POST", `/v1/sessions/${session.id}/commands`, { + key: `sse-command-key-00000${index}`, + body: { command: `echo ${index}` }, + }); + } + const expired = await call(api, "GET", `/v1/sessions/${session.id}/events?cursor=${encodeURIComponent(deliveredCursor)}&maxEvents=1&heartbeatSeconds=5`); + expect(expired.status).toBe(410); + const expiredBody = await json(expired); + expect(expiredBody.error).toMatchObject({ + code: "cursor_expired", + resync: { snapshotUrl: `/v1/sessions/${session.id}/snapshot` }, + }); + await ledger.close(); + }); + + it("persists cancellation before dispatch and keeps lifecycle monotonic across restart", async () => { + let ledger = await newLedger(); + let api = newApi(ledger); + const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "cancel-workspace-key-001", body: { name: "cancel" } })); + const session = await json(await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: "cancel-session-key-0001", body: { title: "cancel me" } })); + const cancellation = await call(api, "POST", `/v1/sessions/${session.id}/cancel`, { key: "cancel-race-key-0000001" }); + expect(cancellation.status).toBe(202); + expect(await json(cancellation)).toMatchObject({ id: session.id, state: "cancelling", revision: 2 }); + await ledger.close(); + + ledger = await newLedger(); + api = newApi(ledger); + const persisted = await call(api, "GET", `/v1/sessions/${session.id}`); + expect(await json(persisted)).toMatchObject({ id: session.id, state: "cancelling", revision: 2 }); + const kubernetes = new ObservableKubernetes(); + const worker = new DurableOutboxWorker({ ledger, ownerId: "cancel-worker", applier: kubernetes }); + const lease = await worker.acquireLease(); + kubernetes.currentEpoch = lease.epoch; + await worker.drain(); + const appliedForSession = [...kubernetes.applied.values()].filter(value => value.targetId === session.id); + expect(appliedForSession.map(value => value.kind)).toEqual(["session.cancel"]); + await ledger.close(); + }); + + it("preserves snapshots, status, ordered events, and replay responses on restart", async () => { + let ledger = await newLedger(); + let api = newApi(ledger); + const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "durable-status-workspace", body: { name: "status" } })); + const session = await json(await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: "durable-status-session-01", body: { title: "status" } })); + const command = await json(await call(api, "POST", `/v1/sessions/${session.id}/commands`, { key: "durable-status-command-01", body: { command: "printf durable" } })); + const before = await json(await call(api, "GET", `/v1/sessions/${session.id}/snapshot`)); + await ledger.close(); + + ledger = await newLedger(); + api = newApi(ledger); + expect(await json(await call(api, "GET", `/v1/sessions/${session.id}`))).toEqual(session); + expect(await json(await call(api, "GET", `/v1/sessions/${session.id}/snapshot`))).toEqual(before); + const replay = await call(api, "POST", `/v1/sessions/${session.id}/commands`, { key: "durable-status-command-01", body: { command: "printf durable" } }); + expect(replay.status).toBe(200); + expect(replay.headers.get("idempotency-replayed")).toBe("true"); + expect(await json(replay)).toEqual(command); + await ledger.close(); + }); + + it("rolls back command, intent, event, and outbox together on transaction failure", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + await admin.unsafe(` + CREATE OR REPLACE FUNCTION t4_test_reject_outbox() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF NEW.idempotency_key = 'rollback-key-000000001' THEN RAISE EXCEPTION 'injected rollback'; END IF; + RETURN NEW; + END $$; + DROP TRIGGER IF EXISTS t4_test_reject_outbox_trigger ON t4_outbox; + CREATE TRIGGER t4_test_reject_outbox_trigger BEFORE INSERT ON t4_outbox + FOR EACH ROW EXECUTE FUNCTION t4_test_reject_outbox(); + `); + const response = await call(api, "POST", "/v1/workspaces", { key: "rollback-key-000000001", body: { name: "must not persist" } }); + expect(response.status).toBe(503); + const rows = await admin.unsafe(` + SELECT + (SELECT count(*) FROM t4_commands WHERE idempotency_key = 'rollback-key-000000001') AS commands, + (SELECT count(*) FROM t4_workspace_intents WHERE name = 'must not persist') AS intents, + (SELECT count(*) FROM t4_outbox WHERE idempotency_key = 'rollback-key-000000001') AS outbox + `); + expect(rows[0]).toMatchObject({ commands: "0", intents: "0", outbox: "0" }); + await admin.unsafe("DROP TRIGGER t4_test_reject_outbox_trigger ON t4_outbox; DROP FUNCTION t4_test_reject_outbox()"); + await ledger.close(); + }); + + it("rejects malformed, oversized, unsupported-version input before ledger or cluster mutation", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const before = await admin`SELECT count(*)::text AS count FROM t4_commands`; + const malformed = await call(api, "POST", "/v1/workspaces", { key: "malformed-input-key-001", rawBody: "{" }); + expect(malformed.status).toBe(400); + const oversized = await call(api, "POST", "/v1/workspaces", { key: "oversized-input-key-001", body: { name: "x".repeat(129) } }); + expect(oversized.status).toBe(422); + const unsupported = await call(api, "POST", "/v1/workspaces", { + key: "unsupported-version-key", + body: { name: "unsupported" }, + headers: { "t4-api-version": "2" }, + }); + expect(unsupported.status).toBe(406); + expect((await json(unsupported)).error).toMatchObject({ code: "incompatible_version", supportedMajors: [1] }); + const after = await admin`SELECT count(*)::text AS count FROM t4_commands`; + expect(after[0].count).toBe(before[0].count); + await ledger.close(); + }); + + it("returns indistinguishable scoped 404s without accepting a user-provided principal", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "auth-isolation-key-0001", body: { name: "private" } })); + const outside = await call(api, "GET", `/v1/workspaces/${workspace.id}`, { token: OTHER_TOKEN, headers: { "x-t4-principal": "owner@example.com" } }); + const absent = await call(api, "GET", "/v1/workspaces/absent-resource", { token: OTHER_TOKEN }); + expect(outside.status).toBe(404); + expect(absent.status).toBe(404); + const outsideBody = await json(outside); + const absentBody = await json(absent); + expect(outsideBody.error.code).toBe("not_found"); + expect(outsideBody.error.message).toBe(absentBody.error.message); + expect(Object.keys(outsideBody.error).sort()).toEqual(Object.keys(absentBody.error).sort()); + const unauthenticated = await api.handle(new Request("https://t4.example.test/v1", { headers: { "t4-api-version": "1" } })); + expect(unauthenticated.status).toBe(401); + await ledger.close(); + }); +}); From 3c1a1ea8d1bcae72b90944d7a7479f83e9a6baad Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 01:27:26 +0000 Subject: [PATCH 115/131] test(cluster-server): expose durable gateway red --- .../test/durable-gateway-postgres.test.ts | 123 +++++++++++++----- 1 file changed, 90 insertions(+), 33 deletions(-) diff --git a/packages/cluster-server/test/durable-gateway-postgres.test.ts b/packages/cluster-server/test/durable-gateway-postgres.test.ts index 83df32f9..2bbb822f 100644 --- a/packages/cluster-server/test/durable-gateway-postgres.test.ts +++ b/packages/cluster-server/test/durable-gateway-postgres.test.ts @@ -1,20 +1,44 @@ import { SQL } from "bun"; import { afterAll, beforeAll, describe, expect, it } from "vite-plus/test"; -import { - PostgresLedger, - type DurableLedgerOptions, -} from "../src/ledger.ts"; -import { - T4PublicApiV1, - type ApiPrincipal, - type PrincipalAuthenticator, -} from "../src/public-api-v1.ts"; -import { - DurableOutboxWorker, - type KubernetesOutboxApplier, - type OutboxClaim, - type OutboxMutation, -} from "../src/outbox-worker.ts"; +interface DurableLedgerOptions { + readonly url: string; + readonly cursorSecret: string; + readonly eventRetention: number; +} +interface PostgresLedgerLike { + migrate(): Promise; + close(): Promise; +} +interface ApiPrincipal { + readonly id: string; + readonly scopes: ReadonlySet; +} +interface PrincipalAuthenticator { + authenticate(token: string): Promise; +} +interface T4PublicApiV1Like { + handle(request: Request): Promise; +} +interface OutboxMutation { + readonly idempotencyToken: string; + readonly targetId: string; + readonly kind: string; +} +interface KubernetesOutboxApplier { + apply(mutation: OutboxMutation, fence: { ownerId: string; epoch: bigint }): Promise; +} +interface OutboxClaim { + readonly outboxId: bigint; +} +interface DurableOutboxWorkerLike { + acquireLease(): Promise<{ ownerId: string; epoch: bigint }>; + drain(): Promise; + claimNext(): Promise; + applyClaim(claim: OutboxClaim): Promise; +} +type LedgerConstructor = new (options: DurableLedgerOptions) => PostgresLedgerLike; +type ApiConstructor = new (options: { ledger: PostgresLedgerLike; authenticator: PrincipalAuthenticator }) => T4PublicApiV1Like; +type WorkerConstructor = new (options: { ledger: PostgresLedgerLike; ownerId: string; applier: KubernetesOutboxApplier }) => DurableOutboxWorkerLike; const DATABASE_URL = process.env.T4_TEST_POSTGRES_URL; const VERSION = "1"; @@ -30,14 +54,21 @@ const ALL_SCOPES = new Set([ "events.read", ]); const CURSOR_SECRET = "test-cursor-secret-that-is-at-least-thirty-two-bytes"; +const LEDGER_MODULE_PATH = "../src/ledger.ts"; +const API_MODULE_PATH = "../src/public-api-v1.ts"; +const WORKER_MODULE_PATH = "../src/outbox-worker.ts"; + +let PostgresLedger: LedgerConstructor | undefined; +let T4PublicApiV1: ApiConstructor | undefined; +let DurableOutboxWorker: WorkerConstructor | undefined; class TestAuthenticator implements PrincipalAuthenticator { - readonly #principals = new Map([ - [OWNER_TOKEN, { id: "owner@example.com", scopes: ALL_SCOPES }], - [OTHER_TOKEN, { id: "other@example.com", scopes: ALL_SCOPES }], - ]); + readonly #principals: Record = { + [OWNER_TOKEN]: { id: "owner@example.com", scopes: ALL_SCOPES }, + [OTHER_TOKEN]: { id: "other@example.com", scopes: ALL_SCOPES }, + }; async authenticate(token: string): Promise { - return this.#principals.get(token); + return this.#principals[token]; } } @@ -62,16 +93,23 @@ function ledgerOptions(overrides: Partial = {}): DurableLe }; } -async function newLedger(overrides: Partial = {}): Promise { +async function newLedger(overrides: Partial = {}): Promise { + if (!PostgresLedger) throw new Error("PostgreSQL ledger implementation is unavailable"); const ledger = new PostgresLedger(ledgerOptions(overrides)); await ledger.migrate(); return ledger; } -function newApi(ledger: PostgresLedger): T4PublicApiV1 { +function newApi(ledger: PostgresLedgerLike): T4PublicApiV1Like { + if (!T4PublicApiV1) throw new Error("public T4 API v1 implementation is unavailable"); return new T4PublicApiV1({ ledger, authenticator: new TestAuthenticator() }); } +function newWorker(ledger: PostgresLedgerLike, ownerId: string, applier: KubernetesOutboxApplier): DurableOutboxWorkerLike { + if (!DurableOutboxWorker) throw new Error("durable outbox worker implementation is unavailable"); + return new DurableOutboxWorker({ ledger, ownerId, applier }); +} + function headers(token = OWNER_TOKEN, additions: Record = {}): Headers { return new Headers({ authorization: `Bearer ${token}`, @@ -81,7 +119,7 @@ function headers(token = OWNER_TOKEN, additions: Record = {}): H } async function call( - api: T4PublicApiV1, + api: T4PublicApiV1Like, method: string, path: string, options: { token?: string; key?: string; ifMatch?: string; body?: unknown; rawBody?: string; headers?: Record } = {}, @@ -96,14 +134,22 @@ async function call( return await api.handle(new Request(`https://t4.example.test${path}`, { method, headers: requestHeaders, body })); } -async function json(response: Response): Promise> { - return await response.json() as Record; +interface JsonDocument extends Record { + id: string; + cursor: string; + state: string; + revision: number; + error: { code: string; message: string; [key: string]: unknown }; +} + +async function json(response: Response): Promise { + return await response.json() as JsonDocument; } -function eventData(responseBody: string): Array> { +function eventData(responseBody: string): JsonDocument[] { return responseBody .split("\n\n") - .flatMap(frame => frame.split("\n").filter(line => line.startsWith("data: ")).map(line => JSON.parse(line.slice(6)) as Record)); + .flatMap(frame => frame.split("\n").filter(line => line.startsWith("data: ")).map(line => JSON.parse(line.slice(6)) as JsonDocument)); } let admin: SQL; @@ -112,6 +158,14 @@ beforeAll(async () => { if (!DATABASE_URL) throw new Error("T4_TEST_POSTGRES_URL is required for durable gateway tests"); admin = new SQL(DATABASE_URL, { max: 1 }); await admin.unsafe("DROP SCHEMA public CASCADE; CREATE SCHEMA public"); + const modules = await Promise.all([ + import(/* @vite-ignore */ LEDGER_MODULE_PATH).catch(() => undefined), + import(/* @vite-ignore */ API_MODULE_PATH).catch(() => undefined), + import(/* @vite-ignore */ WORKER_MODULE_PATH).catch(() => undefined), + ]); + PostgresLedger = (modules[0] as { PostgresLedger?: LedgerConstructor } | undefined)?.PostgresLedger; + T4PublicApiV1 = (modules[1] as { T4PublicApiV1?: ApiConstructor } | undefined)?.T4PublicApiV1; + DurableOutboxWorker = (modules[2] as { DurableOutboxWorker?: WorkerConstructor } | undefined)?.DurableOutboxWorker; }); afterAll(async () => { @@ -120,6 +174,9 @@ afterAll(async () => { describe.sequential("PostgreSQL-backed public T4 API v1", () => { it("replays an identical create after process restart without a second Kubernetes mutation", async () => { + expect(PostgresLedger, "PostgreSQL ledger implementation is required").toBeDefined(); + expect(T4PublicApiV1, "public T4 API v1 implementation is required").toBeDefined(); + expect(DurableOutboxWorker, "durable outbox worker implementation is required").toBeDefined(); let ledger = await newLedger(); let api = newApi(ledger); const key = "restart-workspace-key-0001"; @@ -129,7 +186,7 @@ describe.sequential("PostgreSQL-backed public T4 API v1", () => { const acceptedBody = await json(accepted); const kubernetes = new ObservableKubernetes(); - const firstWorker = new DurableOutboxWorker({ ledger, ownerId: "worker-a", applier: kubernetes }); + const firstWorker = newWorker(ledger, "worker-a", kubernetes); const firstLease = await firstWorker.acquireLease(); kubernetes.currentEpoch = firstLease.epoch; expect(await firstWorker.drain()).toBeGreaterThan(0); @@ -142,7 +199,7 @@ describe.sequential("PostgreSQL-backed public T4 API v1", () => { expect(replay.status).toBe(200); expect(replay.headers.get("idempotency-replayed")).toBe("true"); expect(await json(replay)).toEqual(acceptedBody); - const secondWorker = new DurableOutboxWorker({ ledger, ownerId: "worker-b", applier: kubernetes }); + const secondWorker = newWorker(ledger, "worker-b", kubernetes); const secondLease = await secondWorker.acquireLease(); kubernetes.currentEpoch = secondLease.epoch; expect(await secondWorker.drain()).toBe(0); @@ -185,7 +242,7 @@ describe.sequential("PostgreSQL-backed public T4 API v1", () => { ledger = await newLedger(); api = newApi(ledger); const kubernetes = new ObservableKubernetes(); - const worker = new DurableOutboxWorker({ ledger, ownerId: "recovery-worker", applier: kubernetes }); + const worker = newWorker(ledger, "recovery-worker", kubernetes); const lease = await worker.acquireLease(); kubernetes.currentEpoch = lease.epoch; expect(await worker.drain()).toBeGreaterThan(0); @@ -205,13 +262,13 @@ describe.sequential("PostgreSQL-backed public T4 API v1", () => { const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "fence-workspace-key-001", body: { name: "fence" } })); const session = await json(await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: "fence-session-key-0001", body: { title: "fenced" } })); const kubernetes = new ObservableKubernetes(); - const stale = new DurableOutboxWorker({ ledger, ownerId: "stale", applier: kubernetes }); + const stale = newWorker(ledger, "stale", kubernetes); const staleLease = await stale.acquireLease(); kubernetes.currentEpoch = staleLease.epoch; const claim = await stale.claimNext(); expect(claim).toBeDefined(); - const current = new DurableOutboxWorker({ ledger, ownerId: "current", applier: kubernetes }); + const current = newWorker(ledger, "current", kubernetes); const currentLease = await current.acquireLease(); kubernetes.currentEpoch = currentLease.epoch; expect(await stale.applyClaim(claim as OutboxClaim)).toBe("fenced"); @@ -279,7 +336,7 @@ describe.sequential("PostgreSQL-backed public T4 API v1", () => { const persisted = await call(api, "GET", `/v1/sessions/${session.id}`); expect(await json(persisted)).toMatchObject({ id: session.id, state: "cancelling", revision: 2 }); const kubernetes = new ObservableKubernetes(); - const worker = new DurableOutboxWorker({ ledger, ownerId: "cancel-worker", applier: kubernetes }); + const worker = newWorker(ledger, "cancel-worker", kubernetes); const lease = await worker.acquireLease(); kubernetes.currentEpoch = lease.epoch; await worker.drain(); From 4c43a8616d8406d940b0a66f1c6248ef6890e241 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 01:45:29 +0000 Subject: [PATCH 116/131] feat(cluster-server): persist public gateway ledger --- .../t4-cluster/templates/deployments.yaml | 22 + deploy/charts/t4-cluster/values.schema.json | 11 + deploy/charts/t4-cluster/values.yaml | 6 + .../migrations/001_durable_gateway.down.sql | 11 + .../migrations/001_durable_gateway.sql | 51 +++ packages/cluster-server/src/config.ts | 27 ++ packages/cluster-server/src/index.ts | 5 + .../cluster-server/src/kubernetes-client.ts | 9 + packages/cluster-server/src/ledger.ts | 426 ++++++++++++++++++ packages/cluster-server/src/main.ts | 44 +- packages/cluster-server/src/outbox-worker.ts | 84 ++++ .../cluster-server/src/public-api-auth.ts | 69 +++ packages/cluster-server/src/public-api-v1.ts | 347 ++++++++++++++ .../src/public-kubernetes-applier.ts | 126 ++++++ packages/cluster-server/src/server.ts | 17 +- .../test/durable-gateway-postgres.test.ts | 4 +- 16 files changed, 1254 insertions(+), 5 deletions(-) create mode 100644 packages/cluster-server/migrations/001_durable_gateway.down.sql create mode 100644 packages/cluster-server/migrations/001_durable_gateway.sql create mode 100644 packages/cluster-server/src/ledger.ts create mode 100644 packages/cluster-server/src/outbox-worker.ts create mode 100644 packages/cluster-server/src/public-api-auth.ts create mode 100644 packages/cluster-server/src/public-api-v1.ts create mode 100644 packages/cluster-server/src/public-kubernetes-applier.ts diff --git a/deploy/charts/t4-cluster/templates/deployments.yaml b/deploy/charts/t4-cluster/templates/deployments.yaml index 9a131459..b581ed3d 100644 --- a/deploy/charts/t4-cluster/templates/deployments.yaml +++ b/deploy/charts/t4-cluster/templates/deployments.yaml @@ -258,6 +258,12 @@ spec: valueFrom: {fieldRef: {fieldPath: metadata.name}} - name: POD_UID valueFrom: {fieldRef: {fieldPath: metadata.uid}} + {{- if .Values.durableGateway.existingSecret }} + - name: T4_POSTGRES_URL_FILE + value: /var/run/secrets/t4-durable-gateway/postgres-url + - name: T4_PUBLIC_API_CREDENTIALS_FILE + value: /var/run/secrets/t4-durable-gateway/public-api-credentials.json + {{- end }} {{- if .Values.woodpecker.existingSecret }} - name: T4_WOODPECKER_TOKEN valueFrom: @@ -324,6 +330,11 @@ spec: - name: kubernetes-api-access mountPath: /var/run/secrets/kubernetes.io/serviceaccount readOnly: true + {{- if .Values.durableGateway.existingSecret }} + - name: durable-gateway + mountPath: /var/run/secrets/t4-durable-gateway + readOnly: true + {{- end }} {{- if .Values.woodpecker.serviceAccountAudience }} - name: ci-identity mountPath: /var/run/secrets/t4-ci @@ -351,6 +362,17 @@ spec: items: - key: ca.crt path: ca.crt + {{- if .Values.durableGateway.existingSecret }} + - name: durable-gateway + secret: + secretName: {{ .Values.durableGateway.existingSecret | quote }} + defaultMode: 0440 + items: + - key: {{ .Values.durableGateway.postgresURLKey | quote }} + path: postgres-url + - key: {{ .Values.durableGateway.publicAPICredentialsKey | quote }} + path: public-api-credentials.json + {{- end }} {{- if .Values.woodpecker.serviceAccountAudience }} - name: ci-identity projected: diff --git a/deploy/charts/t4-cluster/values.schema.json b/deploy/charts/t4-cluster/values.schema.json index 9dd8e8ed..ca59d81e 100644 --- a/deploy/charts/t4-cluster/values.schema.json +++ b/deploy/charts/t4-cluster/values.schema.json @@ -10,6 +10,7 @@ "server", "session", "storage", + "durableGateway", "woodpecker", "ingress", "networkPolicy", @@ -129,6 +130,16 @@ } } }, + "durableGateway": { + "type": "object", + "required": ["existingSecret", "postgresURLKey", "publicAPICredentialsKey"], + "properties": { + "existingSecret": { "$ref": "#/definitions/optionalDNSSubdomain" }, + "postgresURLKey": { "$ref": "#/definitions/configKey" }, + "publicAPICredentialsKey": { "$ref": "#/definitions/configKey" } + }, + "additionalProperties": false + }, "woodpecker": { "type": "object", "required": ["existingSecret", "tokenKey", "configMap", "baseURLKey", "webBaseURLKey", "repositoriesKey", "serviceAccountAudience", "tokenExpirationSeconds"], diff --git a/deploy/charts/t4-cluster/values.yaml b/deploy/charts/t4-cluster/values.yaml index b4c2bd1e..e1f46abf 100644 --- a/deploy/charts/t4-cluster/values.yaml +++ b/deploy/charts/t4-cluster/values.yaml @@ -82,6 +82,12 @@ storage: # Required when enabled. The administrator creates this backend-neutral RWX StorageClass. adminRWXStorageClass: "" +durableGateway: + # Administrator-owned Secret; the chart only projects narrowly named files. + existingSecret: "" + postgresURLKey: postgres-url + publicAPICredentialsKey: public-api-credentials.json + woodpecker: existingSecret: "" tokenKey: token diff --git a/packages/cluster-server/migrations/001_durable_gateway.down.sql b/packages/cluster-server/migrations/001_durable_gateway.down.sql new file mode 100644 index 00000000..c712b77b --- /dev/null +++ b/packages/cluster-server/migrations/001_durable_gateway.down.sql @@ -0,0 +1,11 @@ +BEGIN; +DROP TABLE IF EXISTS t4_outbox; +DROP TABLE IF EXISTS t4_owner_leases; +DROP TABLE IF EXISTS t4_snapshot_entries; +DROP TABLE IF EXISTS t4_event_retention; +DROP TABLE IF EXISTS t4_events; +DROP TABLE IF EXISTS t4_session_intents; +DROP TABLE IF EXISTS t4_workspace_intents; +DROP TABLE IF EXISTS t4_commands; +DROP TABLE IF EXISTS t4_schema_migrations; +COMMIT; diff --git a/packages/cluster-server/migrations/001_durable_gateway.sql b/packages/cluster-server/migrations/001_durable_gateway.sql new file mode 100644 index 00000000..60adbe50 --- /dev/null +++ b/packages/cluster-server/migrations/001_durable_gateway.sql @@ -0,0 +1,51 @@ +BEGIN; +CREATE TABLE IF NOT EXISTS t4_schema_migrations (version integer PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT clock_timestamp()); +CREATE TABLE IF NOT EXISTS t4_commands ( + command_id text PRIMARY KEY, principal_id text NOT NULL, operation text NOT NULL, target_scope text NOT NULL, + idempotency_key text NOT NULL, fingerprint text NOT NULL, lifecycle_state text NOT NULL CHECK (lifecycle_state IN ('accepted','completed','failed')), + response_status integer NOT NULL, response_body jsonb, error_body jsonb, created_at timestamptz NOT NULL DEFAULT clock_timestamp(), updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE (principal_id, operation, target_scope, idempotency_key) +); +CREATE TABLE IF NOT EXISTS t4_workspace_intents ( + workspace_id text PRIMARY KEY, principal_id text NOT NULL, name text NOT NULL, labels jsonb NOT NULL DEFAULT '{}'::jsonb, + state text NOT NULL CHECK (state IN ('accepted','provisioning','ready','deleting','deleted','failed','unavailable','indeterminate')), + revision bigint NOT NULL CHECK (revision > 0), generation bigint NOT NULL CHECK (generation > 0), deletion_requested boolean NOT NULL DEFAULT false, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), updated_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX IF NOT EXISTS t4_workspace_intents_principal_page ON t4_workspace_intents (principal_id, workspace_id); +CREATE TABLE IF NOT EXISTS t4_session_intents ( + session_id text PRIMARY KEY, workspace_id text NOT NULL REFERENCES t4_workspace_intents(workspace_id), principal_id text NOT NULL, + title text NOT NULL, labels jsonb NOT NULL DEFAULT '{}'::jsonb, + state text NOT NULL CHECK (state IN ('accepted','provisioning','ready','cancelling','cancelled','failed','unavailable','indeterminate')), + revision bigint NOT NULL CHECK (revision > 0), generation bigint NOT NULL CHECK (generation > 0), cancellation_requested boolean NOT NULL DEFAULT false, + deletion_requested boolean NOT NULL DEFAULT false, created_at timestamptz NOT NULL DEFAULT clock_timestamp(), updated_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX IF NOT EXISTS t4_session_intents_principal_workspace_page ON t4_session_intents (principal_id, workspace_id, session_id); +CREATE TABLE IF NOT EXISTS t4_events ( + sequence bigserial PRIMARY KEY, principal_id text NOT NULL, session_id text, event_type text NOT NULL CHECK (event_type IN ('session','command')), + payload jsonb NOT NULL, owner_epoch bigint NOT NULL DEFAULT 0 CHECK (owner_epoch >= 0), created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX IF NOT EXISTS t4_events_session_sequence ON t4_events (principal_id, session_id, sequence); +CREATE TABLE IF NOT EXISTS t4_event_retention ( + principal_id text NOT NULL, session_id text NOT NULL, first_retained_sequence bigint NOT NULL CHECK (first_retained_sequence >= 0), + latest_sequence bigint NOT NULL CHECK (latest_sequence >= 0), updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), PRIMARY KEY (principal_id, session_id) +); +CREATE TABLE IF NOT EXISTS t4_snapshot_entries ( + principal_id text NOT NULL, session_id text NOT NULL, entry_sequence bigint NOT NULL CHECK (entry_sequence >= 0), + kind text NOT NULL CHECK (kind IN ('input','output','status')), text_value text NOT NULL, created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (principal_id, session_id, entry_sequence) +); +CREATE TABLE IF NOT EXISTS t4_owner_leases ( + lease_name text PRIMARY KEY, owner_id text NOT NULL, epoch bigint NOT NULL CHECK (epoch > 0), expires_at timestamptz NOT NULL, updated_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE IF NOT EXISTS t4_outbox ( + outbox_id bigserial PRIMARY KEY, command_id text NOT NULL UNIQUE REFERENCES t4_commands(command_id), principal_id text NOT NULL, + idempotency_key text NOT NULL, mutation_kind text NOT NULL CHECK (mutation_kind IN ('workspace.create','workspace.patch','workspace.delete','session.create','session.patch','session.cancel','session.delete','command.submit')), + target_id text NOT NULL, target_revision bigint NOT NULL CHECK (target_revision > 0), mutation jsonb NOT NULL, + state text NOT NULL DEFAULT 'pending' CHECK (state IN ('pending','claimed','applied','skipped','failed')), owner_id text, owner_epoch bigint, + attempts integer NOT NULL DEFAULT 0 CHECK (attempts >= 0), next_attempt_at timestamptz NOT NULL DEFAULT clock_timestamp(), claimed_at timestamptz, + terminal_result jsonb, last_error text, created_at timestamptz NOT NULL DEFAULT clock_timestamp(), updated_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX IF NOT EXISTS t4_outbox_ready ON t4_outbox (state, next_attempt_at, outbox_id); +INSERT INTO t4_schema_migrations(version) VALUES (1) ON CONFLICT (version) DO NOTHING; +COMMIT; diff --git a/packages/cluster-server/src/config.ts b/packages/cluster-server/src/config.ts index 06a8d863..5f6b4cf7 100644 --- a/packages/cluster-server/src/config.ts +++ b/packages/cluster-server/src/config.ts @@ -21,6 +21,8 @@ export interface ClusterServerConfig { readonly identityProvider: "tailscale"; readonly trustedProxyAddresses: readonly string[]; readonly trustedProxyCidrs: readonly string[]; + readonly postgres?: { readonly url?: string; readonly urlFile?: string }; + readonly publicApiCredentialsFile?: string; readonly woodpecker?: { readonly baseUrl: string; readonly webBaseUrl?: string; @@ -132,6 +134,13 @@ export function clusterServerConfigFromEnv(env: Readonly { if (!token || /\s/u.test(token)) throw new Error("Kubernetes token file is invalid"); return token; } +function postgresUrl(value: string): string { + const trimmed = value.trim(); + if (trimmed !== value || new TextEncoder().encode(value).byteLength > 8192) throw new Error("PostgreSQL URL is invalid"); + let parsed: URL; + try { parsed = new URL(value); } catch { throw new Error("PostgreSQL URL is invalid"); } + if (!["postgres:", "postgresql:"].includes(parsed.protocol) || !parsed.hostname || !parsed.username || !parsed.password || parsed.pathname.length < 2 || parsed.hash) + throw new Error("PostgreSQL URL is invalid"); + return value; +} +export async function loadPostgresUrl(config: ClusterServerConfig): Promise { + if (!config.postgres) throw new Error("PostgreSQL configuration is required"); + if (Boolean(config.postgres.url) === Boolean(config.postgres.urlFile)) throw new Error("PostgreSQL configuration requires exactly one credential source"); + return postgresUrl(config.postgres.url ?? (await readBoundedRegularFile(config.postgres.urlFile!, 8192, "PostgreSQL URL")).trim()); +} export async function loadKubernetesCa(config: ClusterServerConfig): Promise { const ca = await readBoundedRegularFile(config.kubernetesCaPath, 1024 * 1024, "Kubernetes CA"); if (!ca.includes("BEGIN CERTIFICATE")) throw new Error("Kubernetes CA file is invalid"); diff --git a/packages/cluster-server/src/index.ts b/packages/cluster-server/src/index.ts index 4f81c495..55c590f7 100644 --- a/packages/cluster-server/src/index.ts +++ b/packages/cluster-server/src/index.ts @@ -1,6 +1,11 @@ export * from "./ci-projection-runner.ts"; export * from "./config.ts"; export * from "./gateway.ts"; +export * from "./ledger.ts"; +export * from "./outbox-worker.ts"; +export * from "./public-api-auth.ts"; +export * from "./public-api-v1.ts"; +export * from "./public-kubernetes-applier.ts"; export * from "./kubernetes-client.ts"; export * from "./kubernetes-projection.ts"; export * from "./kubernetes-runner.ts"; diff --git a/packages/cluster-server/src/kubernetes-client.ts b/packages/cluster-server/src/kubernetes-client.ts index bd43edc8..7b575221 100644 --- a/packages/cluster-server/src/kubernetes-client.ts +++ b/packages/cluster-server/src/kubernetes-client.ts @@ -122,6 +122,15 @@ export class KubernetesApiClient { }) as KubernetesResource; } + async patch(resource: string, name: string, body: unknown, signal?: AbortSignal): Promise { + return await this.request(`${this.#collection(resource)}/${encodeURIComponent(name)}`, { + method: "PATCH", + headers: { "content-type": "application/merge-patch+json" }, + body: JSON.stringify(body), + signal, + }) as KubernetesResource; + } + async get(resource: string, name: string, signal?: AbortSignal): Promise { return await this.request(`${this.#collection(resource)}/${encodeURIComponent(name)}`, { signal }) as KubernetesResource; } diff --git a/packages/cluster-server/src/ledger.ts b/packages/cluster-server/src/ledger.ts new file mode 100644 index 00000000..7430aaa7 --- /dev/null +++ b/packages/cluster-server/src/ledger.ts @@ -0,0 +1,426 @@ +import { createHash, createHmac, randomUUID, timingSafeEqual } from "node:crypto"; +import { SQL } from "bun"; + +export interface DurableLedgerOptions { + readonly url: string; + readonly cursorSecret: string; + readonly eventRetention?: number; +} +export interface WorkspaceRecord { + readonly id: string; + readonly name: string; + readonly state: "accepted" | "provisioning" | "ready" | "deleting" | "deleted" | "failed" | "unavailable" | "indeterminate"; + readonly revision: number; + readonly labels?: Readonly>; +} +export interface SessionRecord { + readonly id: string; + readonly workspaceId: string; + readonly title: string; + readonly state: "accepted" | "provisioning" | "ready" | "cancelling" | "cancelled" | "failed" | "unavailable" | "indeterminate"; + readonly revision: number; + readonly labels?: Readonly>; +} +export interface LedgerMutationResult { + readonly kind: "accepted" | "replay" | "idempotency_conflict" | "revision_conflict" | "not_found"; + readonly commandId?: string; + readonly value?: T; +} +export interface DurableEvent { + readonly sequence: bigint; + readonly type: "session" | "command"; + readonly payload: Readonly>; +} +export interface EventWindow { + readonly events: readonly DurableEvent[]; + readonly cursor: string; + readonly expired: boolean; + readonly resyncCursor: string; +} +export interface OutboxClaim { + readonly outboxId: bigint; + readonly commandId: string; + readonly principalId: string; + readonly idempotencyKey: string; + readonly kind: "workspace.create" | "workspace.patch" | "workspace.delete" | "session.create" | "session.patch" | "session.cancel" | "session.delete" | "command.submit"; + readonly targetId: string; + readonly targetRevision: bigint; + readonly mutation: Readonly>; + readonly ownerId: string; + readonly ownerEpoch: bigint; +} +export interface OwnerLease { + readonly ownerId: string; + readonly epoch: bigint; +} + +interface CommandRow { + readonly fingerprint: string; + readonly response_body: unknown; +} +interface WorkspaceRow { + readonly workspace_id: string; + readonly name: string; + readonly state: WorkspaceRecord["state"]; + readonly revision: bigint; + readonly labels: Record; +} +interface SessionRow { + readonly session_id: string; + readonly workspace_id: string; + readonly title: string; + readonly state: SessionRecord["state"]; + readonly revision: bigint; + readonly labels: Record; + readonly cancellation_requested: boolean; +} +interface EventRow { + readonly sequence: bigint; + readonly event_type: "session" | "command"; + readonly payload: Record; +} +interface RetentionRow { readonly first_retained_sequence: bigint; readonly latest_sequence: bigint; } +interface LeaseRow { readonly owner_id: string; readonly epoch: bigint; } +interface OutboxRow { + readonly outbox_id: bigint; + readonly command_id: string; + readonly principal_id: string; + readonly idempotency_key: string; + readonly mutation_kind: OutboxClaim["kind"]; + readonly target_id: string; + readonly target_revision: bigint; + readonly mutation: Record; +} + +const MIGRATION_URL = new URL("../migrations/001_durable_gateway.sql", import.meta.url); +const ROLLBACK_URL = new URL("../migrations/001_durable_gateway.down.sql", import.meta.url); +const OUTBOX_LEASE = "gateway-outbox"; +const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER); + +function safeNumber(value: bigint): number { + if (value < 0n || value > MAX_SAFE_BIGINT) throw new Error("persisted ledger integer is outside the public API range"); + return Number(value); +} +function labels(value: Record): Record | undefined { + return Object.keys(value).length > 0 ? value : undefined; +} +function workspace(row: WorkspaceRow): WorkspaceRecord { + return { id: row.workspace_id, name: row.name, state: row.state, revision: safeNumber(row.revision), ...(labels(row.labels) ? { labels: labels(row.labels) } : {}) }; +} +function session(row: SessionRow): SessionRecord { + return { id: row.session_id, workspaceId: row.workspace_id, title: row.title, state: row.state, revision: safeNumber(row.revision), ...(labels(row.labels) ? { labels: labels(row.labels) } : {}) }; +} +function stableJson(value: unknown): string { + if (value === null || typeof value === "boolean" || typeof value === "string") return JSON.stringify(value); + if (typeof value === "number") { + if (!Number.isFinite(value)) throw new Error("canonical JSON cannot contain a non-finite number"); + return JSON.stringify(Object.is(value, -0) ? 0 : value); + } + if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`; + if (value && typeof value === "object") { + return `{${Object.keys(value as Record).sort().map(key => `${JSON.stringify(key)}:${stableJson((value as Record)[key])}`).join(",")}}`; + } + throw new Error("canonical JSON value is invalid"); +} +export function canonicalRequestFingerprint(value: unknown): string { + return `sha256:${createHash("sha256").update(stableJson(value), "utf8").digest("hex")}`; +} + +export class PostgresLedger { + readonly #sql: SQL; + readonly #cursorSecret: Uint8Array; + readonly #eventRetention: number; + + constructor(options: DurableLedgerOptions) { + const secret = new TextEncoder().encode(options.cursorSecret); + if (secret.byteLength < 32 || secret.byteLength > 4096) throw new Error("cursor secret must contain 32 to 4096 UTF-8 bytes"); + this.#cursorSecret = secret; + this.#eventRetention = options.eventRetention ?? 10_000; + if (!Number.isSafeInteger(this.#eventRetention) || this.#eventRetention < 1 || this.#eventRetention > 100_000) throw new Error("event retention is invalid"); + this.#sql = new SQL(options.url, { bigint: true, max: 8 }); + } + + async migrate(): Promise { + const migration = await Bun.file(MIGRATION_URL).text(); + await this.#sql.unsafe(migration); + const rows = await this.#sql<{ version: number }[]>`SELECT version FROM t4_schema_migrations ORDER BY version DESC LIMIT 1`; + if (rows[0]?.version !== 1) throw new Error("durable gateway schema migration did not reach version 1"); + } + async rollback(): Promise { + await this.#sql.unsafe(await Bun.file(ROLLBACK_URL).text()); + } + async close(): Promise { await this.#sql.close(); } + + async createWorkspace(principalId: string, idempotencyKey: string, fingerprint: string, input: { name: string; labels?: Record }): Promise> { + return await this.#sql.begin("ISOLATION LEVEL SERIALIZABLE", async transaction => { + const prior = await transaction`SELECT fingerprint, response_body FROM t4_commands WHERE principal_id = ${principalId} AND operation = 'createWorkspace' AND target_scope = 'workspaces' AND idempotency_key = ${idempotencyKey} FOR UPDATE`; + if (prior[0]) return prior[0].fingerprint === fingerprint + ? { kind: "replay", value: prior[0].response_body as WorkspaceRecord } + : { kind: "idempotency_conflict" }; + const commandId = `cmd_${randomUUID()}`; + const workspaceId = `ws-${randomUUID()}`; + const value: WorkspaceRecord = { id: workspaceId, name: input.name, state: "accepted", revision: 1, ...(input.labels ? { labels: input.labels } : {}) }; + await transaction`INSERT INTO t4_commands (command_id, principal_id, operation, target_scope, idempotency_key, fingerprint, lifecycle_state, response_status, response_body) VALUES (${commandId}, ${principalId}, 'createWorkspace', 'workspaces', ${idempotencyKey}, ${fingerprint}, 'accepted', 202, ${value})`; + await transaction`INSERT INTO t4_workspace_intents (workspace_id, principal_id, name, labels, state, revision, generation) VALUES (${workspaceId}, ${principalId}, ${input.name}, ${input.labels ?? {}}, 'accepted', 1, 1)`; + await transaction`INSERT INTO t4_outbox (command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation) VALUES (${commandId}, ${principalId}, ${idempotencyKey}, 'workspace.create', ${workspaceId}, 1, ${value})`; + return { kind: "accepted", commandId, value }; + }); + } + + async getWorkspace(principalId: string, workspaceId: string): Promise { + const rows = await this.#sql`SELECT workspace_id, name, state, revision, labels FROM t4_workspace_intents WHERE principal_id = ${principalId} AND workspace_id = ${workspaceId} AND state <> 'deleted'`; + return rows[0] ? workspace(rows[0]) : undefined; + } + async listWorkspaces(principalId: string, afterId: string | undefined, limit: number): Promise { + const rows = afterId + ? await this.#sql`SELECT workspace_id, name, state, revision, labels FROM t4_workspace_intents WHERE principal_id = ${principalId} AND workspace_id > ${afterId} AND state <> 'deleted' ORDER BY workspace_id LIMIT ${limit}` + : await this.#sql`SELECT workspace_id, name, state, revision, labels FROM t4_workspace_intents WHERE principal_id = ${principalId} AND state <> 'deleted' ORDER BY workspace_id LIMIT ${limit}`; + return rows.map(workspace); + } + + async patchWorkspace(principalId: string, workspaceId: string, expectedRevision: number, idempotencyKey: string, fingerprint: string, input: { name?: string; labels?: Record }): Promise> { + return await this.#sql.begin("ISOLATION LEVEL SERIALIZABLE", async transaction => { + const rows = await transaction`SELECT workspace_id, name, state, revision, labels FROM t4_workspace_intents WHERE principal_id = ${principalId} AND workspace_id = ${workspaceId} AND state <> 'deleted' FOR UPDATE`; + const current = rows[0]; + if (!current) return { kind: "not_found" }; + if (current.revision !== BigInt(expectedRevision)) return { kind: "revision_conflict" }; + const scope = `workspace:${workspaceId}`; + const prior = await transaction`SELECT fingerprint, response_body FROM t4_commands WHERE principal_id = ${principalId} AND operation = 'mutateWorkspace' AND target_scope = ${scope} AND idempotency_key = ${idempotencyKey} FOR UPDATE`; + if (prior[0]) return prior[0].fingerprint === fingerprint ? { kind: "replay", value: prior[0].response_body as WorkspaceRecord } : { kind: "idempotency_conflict" }; + const revision = current.revision + 1n; + const value: WorkspaceRecord = { id: workspaceId, name: input.name ?? current.name, state: current.state, revision: safeNumber(revision), ...(input.labels ?? labels(current.labels) ? { labels: input.labels ?? current.labels } : {}) }; + const commandId = `cmd_${randomUUID()}`; + await transaction`INSERT INTO t4_commands (command_id, principal_id, operation, target_scope, idempotency_key, fingerprint, lifecycle_state, response_status, response_body) VALUES (${commandId}, ${principalId}, 'mutateWorkspace', ${scope}, ${idempotencyKey}, ${fingerprint}, 'accepted', 200, ${value})`; + await transaction`UPDATE t4_workspace_intents SET name = ${value.name}, labels = ${value.labels ?? {}}, revision = ${revision}, generation = generation + 1, updated_at = clock_timestamp() WHERE workspace_id = ${workspaceId}`; + await transaction`INSERT INTO t4_outbox (command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation) VALUES (${commandId}, ${principalId}, ${idempotencyKey}, 'workspace.patch', ${workspaceId}, ${revision}, ${value})`; + return { kind: "accepted", commandId, value }; + }); + } + + async deleteWorkspace(principalId: string, workspaceId: string, idempotencyKey: string, fingerprint: string): Promise> { + return await this.#sql.begin("ISOLATION LEVEL SERIALIZABLE", async transaction => { + const scope = `workspace:${workspaceId}`; + const prior = await transaction`SELECT fingerprint, response_body FROM t4_commands WHERE principal_id = ${principalId} AND operation = 'deleteWorkspace' AND target_scope = ${scope} AND idempotency_key = ${idempotencyKey} FOR UPDATE`; + if (prior[0]) return prior[0].fingerprint === fingerprint ? { kind: "replay" } : { kind: "idempotency_conflict" }; + const rows = await transaction`SELECT workspace_id, name, state, revision, labels FROM t4_workspace_intents WHERE principal_id = ${principalId} AND workspace_id = ${workspaceId} AND state <> 'deleted' FOR UPDATE`; + if (!rows[0]) return { kind: "not_found" }; + const revision = rows[0].revision + 1n; + const commandId = `cmd_${randomUUID()}`; + await transaction`INSERT INTO t4_commands (command_id, principal_id, operation, target_scope, idempotency_key, fingerprint, lifecycle_state, response_status) VALUES (${commandId}, ${principalId}, 'deleteWorkspace', ${scope}, ${idempotencyKey}, ${fingerprint}, 'accepted', 204)`; + await transaction`UPDATE t4_workspace_intents SET state = 'deleting', deletion_requested = true, revision = ${revision}, generation = generation + 1, updated_at = clock_timestamp() WHERE workspace_id = ${workspaceId}`; + await transaction`INSERT INTO t4_outbox (command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation) VALUES (${commandId}, ${principalId}, ${idempotencyKey}, 'workspace.delete', ${workspaceId}, ${revision}, ${{ workspaceId }})`; + return { kind: "accepted", commandId }; + }); + } + + async createSession(principalId: string, workspaceId: string, idempotencyKey: string, fingerprint: string, input: { title: string; labels?: Record }): Promise> { + return await this.#sql.begin("ISOLATION LEVEL SERIALIZABLE", async transaction => { + const workspaceRows = await transaction<{ workspace_id: string }[]>`SELECT workspace_id FROM t4_workspace_intents WHERE principal_id = ${principalId} AND workspace_id = ${workspaceId} AND state NOT IN ('deleting','deleted') FOR UPDATE`; + if (!workspaceRows[0]) return { kind: "not_found" }; + const scope = `workspace:${workspaceId}:sessions`; + const prior = await transaction`SELECT fingerprint, response_body FROM t4_commands WHERE principal_id = ${principalId} AND operation = 'spawnSession' AND target_scope = ${scope} AND idempotency_key = ${idempotencyKey} FOR UPDATE`; + if (prior[0]) return prior[0].fingerprint === fingerprint ? { kind: "replay", value: prior[0].response_body as SessionRecord } : { kind: "idempotency_conflict" }; + const commandId = `cmd_${randomUUID()}`; + const sessionId = `ss-${randomUUID()}`; + const value: SessionRecord = { id: sessionId, workspaceId, title: input.title, state: "accepted", revision: 1, ...(input.labels ? { labels: input.labels } : {}) }; + await transaction`INSERT INTO t4_commands (command_id, principal_id, operation, target_scope, idempotency_key, fingerprint, lifecycle_state, response_status, response_body) VALUES (${commandId}, ${principalId}, 'spawnSession', ${scope}, ${idempotencyKey}, ${fingerprint}, 'accepted', 202, ${value})`; + await transaction`INSERT INTO t4_session_intents (session_id, workspace_id, principal_id, title, labels, state, revision, generation) VALUES (${sessionId}, ${workspaceId}, ${principalId}, ${input.title}, ${input.labels ?? {}}, 'accepted', 1, 1)`; + await transaction`INSERT INTO t4_outbox (command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation) VALUES (${commandId}, ${principalId}, ${idempotencyKey}, 'session.create', ${sessionId}, 1, ${value})`; + await this.#appendEvent(transaction as unknown as SQL, principalId, sessionId, "session", { state: "accepted", revision: 1 }, 0n); + return { kind: "accepted", commandId, value }; + }); + } + + async getSession(principalId: string, sessionId: string): Promise { + const rows = await this.#sql`SELECT session_id, workspace_id, title, state, revision, labels, cancellation_requested FROM t4_session_intents WHERE principal_id = ${principalId} AND session_id = ${sessionId}`; + return rows[0] ? session(rows[0]) : undefined; + } + async listSessions(principalId: string, workspaceId: string, afterId: string | undefined, limit: number): Promise { + if (!await this.getWorkspace(principalId, workspaceId)) return undefined; + const rows = afterId + ? await this.#sql`SELECT session_id, workspace_id, title, state, revision, labels, cancellation_requested FROM t4_session_intents WHERE principal_id = ${principalId} AND workspace_id = ${workspaceId} AND session_id > ${afterId} ORDER BY session_id LIMIT ${limit}` + : await this.#sql`SELECT session_id, workspace_id, title, state, revision, labels, cancellation_requested FROM t4_session_intents WHERE principal_id = ${principalId} AND workspace_id = ${workspaceId} ORDER BY session_id LIMIT ${limit}`; + return rows.map(session); + } + + async patchSession(principalId: string, sessionId: string, expectedRevision: number, idempotencyKey: string, fingerprint: string, input: { title?: string; labels?: Record }): Promise> { + return await this.#sql.begin("ISOLATION LEVEL SERIALIZABLE", async transaction => { + const rows = await transaction`SELECT session_id, workspace_id, title, state, revision, labels, cancellation_requested FROM t4_session_intents WHERE principal_id = ${principalId} AND session_id = ${sessionId} FOR UPDATE`; + const current = rows[0]; + if (!current) return { kind: "not_found" }; + if (current.revision !== BigInt(expectedRevision)) return { kind: "revision_conflict" }; + const scope = `session:${sessionId}`; + const prior = await transaction`SELECT fingerprint, response_body FROM t4_commands WHERE principal_id = ${principalId} AND operation = 'mutateSession' AND target_scope = ${scope} AND idempotency_key = ${idempotencyKey} FOR UPDATE`; + if (prior[0]) return prior[0].fingerprint === fingerprint ? { kind: "replay", value: prior[0].response_body as SessionRecord } : { kind: "idempotency_conflict" }; + const revision = current.revision + 1n; + const value: SessionRecord = { id: sessionId, workspaceId: current.workspace_id, title: input.title ?? current.title, state: current.state, revision: safeNumber(revision), ...(input.labels ?? labels(current.labels) ? { labels: input.labels ?? current.labels } : {}) }; + const commandId = `cmd_${randomUUID()}`; + await transaction`INSERT INTO t4_commands (command_id, principal_id, operation, target_scope, idempotency_key, fingerprint, lifecycle_state, response_status, response_body) VALUES (${commandId}, ${principalId}, 'mutateSession', ${scope}, ${idempotencyKey}, ${fingerprint}, 'accepted', 200, ${value})`; + await transaction`UPDATE t4_session_intents SET title = ${value.title}, labels = ${value.labels ?? {}}, revision = ${revision}, generation = generation + 1, updated_at = clock_timestamp() WHERE session_id = ${sessionId}`; + await transaction`INSERT INTO t4_outbox (command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation) VALUES (${commandId}, ${principalId}, ${idempotencyKey}, 'session.patch', ${sessionId}, ${revision}, ${value})`; + await this.#appendEvent(transaction as unknown as SQL, principalId, sessionId, "session", { state: value.state, revision: value.revision }, 0n); + return { kind: "accepted", commandId, value }; + }); + } + + async cancelSession(principalId: string, sessionId: string, idempotencyKey: string, fingerprint: string, deletion = false): Promise> { + return await this.#sql.begin("ISOLATION LEVEL SERIALIZABLE", async transaction => { + const operation = deletion ? "deleteSession" : "cancelSession"; + const scope = `session:${sessionId}`; + const prior = await transaction`SELECT fingerprint, response_body FROM t4_commands WHERE principal_id = ${principalId} AND operation = ${operation} AND target_scope = ${scope} AND idempotency_key = ${idempotencyKey} FOR UPDATE`; + if (prior[0]) return prior[0].fingerprint === fingerprint ? { kind: "replay", value: prior[0].response_body as SessionRecord } : { kind: "idempotency_conflict" }; + const rows = await transaction`SELECT session_id, workspace_id, title, state, revision, labels, cancellation_requested FROM t4_session_intents WHERE principal_id = ${principalId} AND session_id = ${sessionId} FOR UPDATE`; + const current = rows[0]; + if (!current) return { kind: "not_found" }; + const revision = current.revision + 1n; + const value: SessionRecord = { id: sessionId, workspaceId: current.workspace_id, title: current.title, state: "cancelling", revision: safeNumber(revision), ...(labels(current.labels) ? { labels: current.labels } : {}) }; + const commandId = `cmd_${randomUUID()}`; + await transaction`INSERT INTO t4_commands (command_id, principal_id, operation, target_scope, idempotency_key, fingerprint, lifecycle_state, response_status, response_body) VALUES (${commandId}, ${principalId}, ${operation}, ${scope}, ${idempotencyKey}, ${fingerprint}, 'accepted', ${deletion ? 204 : 202}, ${value})`; + await transaction`UPDATE t4_session_intents SET state = 'cancelling', cancellation_requested = true, deletion_requested = deletion_requested OR ${deletion}, revision = ${revision}, generation = generation + 1, updated_at = clock_timestamp() WHERE session_id = ${sessionId}`; + await transaction`UPDATE t4_outbox SET state = 'skipped', terminal_result = ${{ reason: "superseded by cancellation" }}, updated_at = clock_timestamp() WHERE target_id = ${sessionId} AND mutation_kind = 'session.create' AND state IN ('pending','claimed')`; + await transaction`INSERT INTO t4_outbox (command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation) VALUES (${commandId}, ${principalId}, ${idempotencyKey}, ${deletion ? "session.delete" : "session.cancel"}, ${sessionId}, ${revision}, ${{ sessionId, revision: safeNumber(revision) }})`; + await this.#appendEvent(transaction as unknown as SQL, principalId, sessionId, "session", { state: "cancelling", revision: safeNumber(revision) }, 0n); + return { kind: "accepted", commandId, value }; + }); + } + + async submitCommand(principalId: string, sessionId: string, idempotencyKey: string, fingerprint: string, input: { command: string; metadata: Record }): Promise> { + return await this.#sql.begin("ISOLATION LEVEL SERIALIZABLE", async transaction => { + const sessionRows = await transaction<{ session_id: string }[]>`SELECT session_id FROM t4_session_intents WHERE principal_id = ${principalId} AND session_id = ${sessionId} AND state NOT IN ('cancelling','cancelled') FOR UPDATE`; + if (!sessionRows[0]) return { kind: "not_found" }; + const scope = `session:${sessionId}:commands`; + const prior = await transaction`SELECT fingerprint, response_body FROM t4_commands WHERE principal_id = ${principalId} AND operation = 'submitCommand' AND target_scope = ${scope} AND idempotency_key = ${idempotencyKey} FOR UPDATE`; + if (prior[0]) return prior[0].fingerprint === fingerprint ? { kind: "replay", value: prior[0].response_body as { commandId: string; state: "accepted" } } : { kind: "idempotency_conflict" }; + const commandId = `cmd_${randomUUID()}`; + const value = { commandId, state: "accepted" as const }; + await transaction`INSERT INTO t4_commands (command_id, principal_id, operation, target_scope, idempotency_key, fingerprint, lifecycle_state, response_status, response_body) VALUES (${commandId}, ${principalId}, 'submitCommand', ${scope}, ${idempotencyKey}, ${fingerprint}, 'accepted', 202, ${value})`; + await transaction`INSERT INTO t4_outbox (command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation) SELECT ${commandId}, ${principalId}, ${idempotencyKey}, 'command.submit', ${sessionId}, revision, ${{ sessionId, ...input }} FROM t4_session_intents WHERE session_id = ${sessionId}`; + const entryRows = await transaction<{ next: bigint }[]>`SELECT COALESCE(MAX(entry_sequence), -1) + 1 AS next FROM t4_snapshot_entries WHERE principal_id = ${principalId} AND session_id = ${sessionId}`; + await transaction`INSERT INTO t4_snapshot_entries (principal_id, session_id, entry_sequence, kind, text_value) VALUES (${principalId}, ${sessionId}, ${entryRows[0]?.next ?? 0n}, 'input', ${input.command})`; + await this.#appendEvent(transaction as unknown as SQL, principalId, sessionId, "command", { commandId, state: "accepted" }, 0n); + return { kind: "accepted", commandId, value }; + }); + } + + async snapshot(principalId: string, sessionId: string): Promise<{ session: SessionRecord; cursor: string; entries: readonly { sequence: number; kind: "input" | "output" | "status"; text: string }[] } | undefined> { + const current = await this.getSession(principalId, sessionId); + if (!current) return undefined; + const rows = await this.#sql<{ entry_sequence: bigint; kind: "input" | "output" | "status"; text_value: string }[]>`SELECT entry_sequence, kind, text_value FROM t4_snapshot_entries WHERE principal_id = ${principalId} AND session_id = ${sessionId} ORDER BY entry_sequence DESC LIMIT 1000`; + const retention = await this.#sql`SELECT first_retained_sequence, latest_sequence FROM t4_event_retention WHERE principal_id = ${principalId} AND session_id = ${sessionId}`; + const latest = retention[0]?.latest_sequence ?? 0n; + return { session: current, cursor: this.watchCursor(principalId, sessionId, latest), entries: rows.reverse().map(row => ({ sequence: safeNumber(row.entry_sequence), kind: row.kind, text: row.text_value })) }; + } + + async eventWindow(principalId: string, sessionId: string, cursor: string | undefined, limit: number): Promise { + if (!await this.getSession(principalId, sessionId)) return undefined; + const retentionRows = await this.#sql`SELECT first_retained_sequence, latest_sequence FROM t4_event_retention WHERE principal_id = ${principalId} AND session_id = ${sessionId}`; + const first = retentionRows[0]?.first_retained_sequence ?? 0n; + const latest = retentionRows[0]?.latest_sequence ?? 0n; + let after = 0n; + if (cursor) { + const decoded = this.#decodeCursor(cursor, "watch", principalId, sessionId); + if (decoded === undefined) throw new Error("invalid_cursor"); + after = decoded; + } + const resyncCursor = this.watchCursor(principalId, sessionId, latest); + if (first > 0n && after < first - 1n) return { events: [], cursor: resyncCursor, expired: true, resyncCursor }; + const rows = await this.#sql`SELECT sequence, event_type, payload FROM t4_events WHERE principal_id = ${principalId} AND session_id = ${sessionId} AND sequence > ${after} ORDER BY sequence LIMIT ${limit}`; + const delivered = rows.at(-1)?.sequence ?? after; + return { events: rows.map(row => ({ sequence: row.sequence, type: row.event_type, payload: row.payload })), cursor: this.watchCursor(principalId, sessionId, delivered), expired: false, resyncCursor }; + } + + watchCursor(principalId: string, sessionId: string, sequence: bigint): string { return this.#cursor("watch", principalId, sessionId, sequence); } + pageCursor(principalId: string, scope: string, id: string): string { return this.#cursor("page", principalId, scope, id); } + readPageCursor(cursor: string, principalId: string, scope: string): string | undefined { + const result = this.#decodeCursor(cursor, "page", principalId, scope); + return typeof result === "string" ? result : undefined; + } + + async acquireLease(ownerId: string, leaseSeconds = 30): Promise { + const rows = await this.#sql.begin("ISOLATION LEVEL SERIALIZABLE", async transaction => { + await transaction`INSERT INTO t4_owner_leases (lease_name, owner_id, epoch, expires_at) VALUES (${OUTBOX_LEASE}, ${ownerId}, 1, clock_timestamp() + (${leaseSeconds} * interval '1 second')) ON CONFLICT (lease_name) DO UPDATE SET owner_id = EXCLUDED.owner_id, epoch = t4_owner_leases.epoch + 1, expires_at = EXCLUDED.expires_at, updated_at = clock_timestamp()`; + return await transaction`SELECT owner_id, epoch FROM t4_owner_leases WHERE lease_name = ${OUTBOX_LEASE}`; + }); + return { ownerId: rows[0]!.owner_id, epoch: rows[0]!.epoch }; + } + + async claimNext(lease: OwnerLease): Promise { + return await this.#sql.begin("ISOLATION LEVEL SERIALIZABLE", async transaction => { + const current = await transaction`SELECT owner_id, epoch FROM t4_owner_leases WHERE lease_name = ${OUTBOX_LEASE} FOR UPDATE`; + if (current[0]?.owner_id !== lease.ownerId || current[0].epoch !== lease.epoch) return undefined; + const rows = await transaction`SELECT outbox_id, command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation FROM t4_outbox WHERE (state = 'pending' OR (state = 'claimed' AND owner_epoch < ${lease.epoch})) AND next_attempt_at <= clock_timestamp() ORDER BY outbox_id FOR UPDATE SKIP LOCKED LIMIT 1`; + const row = rows[0]; + if (!row) return undefined; + let superseded = false; + if (row.mutation_kind.startsWith("session.")) { + const intent = await transaction<{ revision: bigint; cancellation_requested: boolean }[]>`SELECT revision, cancellation_requested FROM t4_session_intents WHERE session_id = ${row.target_id}`; + superseded = !intent[0] || intent[0].revision !== row.target_revision || row.mutation_kind === "session.create" && intent[0].cancellation_requested; + } else if (row.mutation_kind.startsWith("workspace.")) { + const intent = await transaction<{ revision: bigint; deletion_requested: boolean }[]>`SELECT revision, deletion_requested FROM t4_workspace_intents WHERE workspace_id = ${row.target_id}`; + superseded = !intent[0] || intent[0].revision !== row.target_revision || row.mutation_kind === "workspace.create" && intent[0].deletion_requested; + } + if (superseded) { + await transaction`UPDATE t4_outbox SET state = 'skipped', owner_id = ${lease.ownerId}, owner_epoch = ${lease.epoch}, terminal_result = ${{ reason: "superseded intent" }}, updated_at = clock_timestamp() WHERE outbox_id = ${row.outbox_id}`; + return undefined; + } + await transaction`UPDATE t4_outbox SET state = 'claimed', owner_id = ${lease.ownerId}, owner_epoch = ${lease.epoch}, attempts = attempts + 1, claimed_at = clock_timestamp(), updated_at = clock_timestamp() WHERE outbox_id = ${row.outbox_id}`; + return { outboxId: row.outbox_id, commandId: row.command_id, principalId: row.principal_id, idempotencyKey: row.idempotency_key, kind: row.mutation_kind, targetId: row.target_id, targetRevision: row.target_revision, mutation: row.mutation, ownerId: lease.ownerId, ownerEpoch: lease.epoch }; + }); + } + + async claimIsCurrent(claim: OutboxClaim): Promise { + const rows = await this.#sql<{ valid: boolean }[]>`SELECT EXISTS (SELECT 1 FROM t4_owner_leases lease JOIN t4_outbox item ON item.outbox_id = ${claim.outboxId} WHERE lease.lease_name = ${OUTBOX_LEASE} AND lease.owner_id = ${claim.ownerId} AND lease.epoch = ${claim.ownerEpoch} AND lease.expires_at > clock_timestamp() AND item.state = 'claimed' AND item.owner_id = lease.owner_id AND item.owner_epoch = lease.epoch) AS valid`; + return rows[0]?.valid === true; + } + + async acknowledge(claim: OutboxClaim): Promise { + return await this.#sql.begin("ISOLATION LEVEL SERIALIZABLE", async transaction => { + const lease = await transaction`SELECT owner_id, epoch FROM t4_owner_leases WHERE lease_name = ${OUTBOX_LEASE} FOR UPDATE`; + if (lease[0]?.owner_id !== claim.ownerId || lease[0].epoch !== claim.ownerEpoch) return false; + const updated = await transaction<{ outbox_id: bigint }[]>`UPDATE t4_outbox SET state = 'applied', terminal_result = ${{ applied: true }}, updated_at = clock_timestamp() WHERE outbox_id = ${claim.outboxId} AND state = 'claimed' AND owner_id = ${claim.ownerId} AND owner_epoch = ${claim.ownerEpoch} RETURNING outbox_id`; + if (!updated[0]) return false; + await transaction`UPDATE t4_commands SET lifecycle_state = 'completed', updated_at = clock_timestamp() WHERE command_id = ${claim.commandId}`; + if (claim.kind === "workspace.create") await transaction`UPDATE t4_workspace_intents SET state = 'provisioning', updated_at = clock_timestamp() WHERE workspace_id = ${claim.targetId} AND revision = ${claim.targetRevision}`; + if (claim.kind === "session.create") { + await transaction`UPDATE t4_session_intents SET state = 'provisioning', updated_at = clock_timestamp() WHERE session_id = ${claim.targetId} AND revision = ${claim.targetRevision}`; + await this.#appendEvent(transaction as unknown as SQL, claim.principalId, claim.targetId, "session", { state: "provisioning", revision: safeNumber(claim.targetRevision) }, claim.ownerEpoch); + } + return true; + }); + } + + async recordFailure(claim: OutboxClaim, message: string): Promise { + const result = await this.#sql<{ outbox_id: bigint }[]>`UPDATE t4_outbox item SET state = 'pending', owner_id = NULL, owner_epoch = NULL, last_error = ${message.slice(0, 1024)}, next_attempt_at = clock_timestamp() + interval '1 second', updated_at = clock_timestamp() FROM t4_owner_leases lease WHERE item.outbox_id = ${claim.outboxId} AND item.state = 'claimed' AND item.owner_id = ${claim.ownerId} AND item.owner_epoch = ${claim.ownerEpoch} AND lease.lease_name = ${OUTBOX_LEASE} AND lease.owner_id = ${claim.ownerId} AND lease.epoch = ${claim.ownerEpoch} RETURNING item.outbox_id`; + return result.length === 1; + } + + #cursor(kind: "watch" | "page", principalId: string, scope: string, position: bigint | string): string { + const payload = Buffer.from(JSON.stringify({ k: kind, p: createHash("sha256").update(principalId).digest("base64url"), s: scope, q: position.toString() }), "utf8").toString("base64url"); + const signature = createHmac("sha256", this.#cursorSecret).update(payload).digest("base64url"); + return `${payload}.${signature}`; + } + #decodeCursor(cursor: string, kind: "watch" | "page", principalId: string, scope: string): bigint | string | undefined { + if (cursor.length > 512 || !/^[A-Za-z0-9._~-]+$/u.test(cursor)) return undefined; + const [payload, signature, extra] = cursor.split("."); + if (!payload || !signature || extra) return undefined; + const expected = createHmac("sha256", this.#cursorSecret).update(payload).digest(); + let supplied: Buffer; + try { supplied = Buffer.from(signature, "base64url"); } catch { return undefined; } + if (supplied.byteLength !== expected.byteLength || !timingSafeEqual(supplied, expected)) return undefined; + let value: { k?: unknown; p?: unknown; s?: unknown; q?: unknown }; + try { value = JSON.parse(Buffer.from(payload, "base64url").toString("utf8")) as typeof value; } catch { return undefined; } + if (value.k !== kind || value.p !== createHash("sha256").update(principalId).digest("base64url") || value.s !== scope || typeof value.q !== "string") return undefined; + if (kind === "page") return value.q; + if (!/^(?:0|[1-9][0-9]*)$/u.test(value.q)) return undefined; + return BigInt(value.q); + } + + async #appendEvent(transaction: SQL, principalId: string, sessionId: string, type: "session" | "command", payload: Record, ownerEpoch: bigint): Promise { + const inserted = await transaction<{ sequence: bigint }[]>`INSERT INTO t4_events (principal_id, session_id, event_type, payload, owner_epoch) VALUES (${principalId}, ${sessionId}, ${type}, ${payload}, ${ownerEpoch}) RETURNING sequence`; + const sequence = inserted[0]!.sequence; + await transaction`DELETE FROM t4_events WHERE principal_id = ${principalId} AND session_id = ${sessionId} AND sequence < COALESCE((SELECT sequence FROM t4_events WHERE principal_id = ${principalId} AND session_id = ${sessionId} ORDER BY sequence DESC OFFSET ${this.#eventRetention - 1} LIMIT 1), 0)`; + const retained = await transaction<{ first: bigint; latest: bigint }[]>`SELECT COALESCE(MIN(sequence), 0) AS first, COALESCE(MAX(sequence), 0) AS latest FROM t4_events WHERE principal_id = ${principalId} AND session_id = ${sessionId}`; + await transaction`INSERT INTO t4_event_retention (principal_id, session_id, first_retained_sequence, latest_sequence) VALUES (${principalId}, ${sessionId}, ${retained[0]!.first}, ${retained[0]!.latest}) ON CONFLICT (principal_id, session_id) DO UPDATE SET first_retained_sequence = EXCLUDED.first_retained_sequence, latest_sequence = EXCLUDED.latest_sequence, updated_at = clock_timestamp()`; + return sequence; + } +} diff --git a/packages/cluster-server/src/main.ts b/packages/cluster-server/src/main.ts index 0ac12d7f..aa6ce568 100755 --- a/packages/cluster-server/src/main.ts +++ b/packages/cluster-server/src/main.ts @@ -1,6 +1,6 @@ #!/usr/bin/env bun import { CiProjectionRunner } from "./ci-projection-runner.ts"; -import { clusterServerConfigFromEnv, loadKubernetesCa } from "./config.ts"; +import { clusterServerConfigFromEnv, loadKubernetesCa, loadPostgresUrl } from "./config.ts"; import { ClusterGateway } from "./gateway.ts"; import { KubernetesApiClient, KubernetesGatewayMutationBackend } from "./kubernetes-client.ts"; import { ClusterInfrastructureProjection } from "./kubernetes-projection.ts"; @@ -10,6 +10,11 @@ import { WebSocketPodHostConnector } from "./pod-host-router.ts"; import { startClusterHttpServers, type ClusterHttpServers } from "./server.ts"; import { SessionAuthorityRunner } from "./session-authority-runner.ts"; import { WoodpeckerProvider } from "./woodpecker.ts"; +import { PostgresLedger } from "./ledger.ts"; +import { DurableOutboxWorker } from "./outbox-worker.ts"; +import { loadPublicApiSecrets } from "./public-api-auth.ts"; +import { T4PublicApiV1 } from "./public-api-v1.ts"; +import { PublicKubernetesOutboxApplier } from "./public-kubernetes-applier.ts"; export async function runClusterServer(env: Readonly> = process.env): Promise { const config = clusterServerConfigFromEnv(env); @@ -40,8 +45,36 @@ export async function runClusterServer(env: Readonly | undefined; let signalsInstalled = false; try { + if (!config.postgres || !config.publicApiCredentialsFile) throw new Error("public API durable storage configuration is required"); + const [databaseUrl, publicApiSecrets] = await Promise.all([ + loadPostgresUrl(config), + loadPublicApiSecrets(config.publicApiCredentialsFile), + ]); + ledger = new PostgresLedger({ url: databaseUrl, cursorSecret: publicApiSecrets.cursorSecret }); + await ledger.migrate(); + const publicApi = new T4PublicApiV1({ ledger, authenticator: publicApiSecrets.authenticator }); + const outbox = new DurableOutboxWorker({ + ledger, + ownerId: config.epoch, + applier: new PublicKubernetesOutboxApplier({ client: kubernetes, hostRef: config.hostName }), + }); + outboxAbort = new AbortController(); + outboxTask = (async () => { + while (!outboxAbort!.signal.aborted) { + try { + await outbox.acquireLease(); + await outbox.drain(); + } catch (error) { + logger.warn("durable outbox recovery retrying", { condition: error instanceof Error ? error.name : "unknown", result: "failure" }); + } + if (!outboxAbort!.signal.aborted) await Bun.sleep(1_000); + } + })(); await runner.start(); const connector = new WebSocketPodHostConnector({ identityTokenFile: config.identityTokenPath }); authority = new SessionAuthorityRunner({ @@ -67,6 +100,7 @@ export async function runClusterServer(env: Readonly>; +} +export interface KubernetesOutboxApplier { + apply(mutation: OutboxMutation, fence: OwnerLease): Promise; +} +export interface DurableOutboxWorkerOptions { + readonly ledger: PostgresLedger; + readonly ownerId: string; + readonly applier: KubernetesOutboxApplier; +} + +export class DurableOutboxWorker { + readonly #ledger: PostgresLedger; + readonly #ownerId: string; + readonly #applier: KubernetesOutboxApplier; + #lease: OwnerLease | undefined; + + constructor(options: DurableOutboxWorkerOptions) { + if (!options.ownerId || options.ownerId.length > 256) throw new Error("outbox owner id is invalid"); + this.#ledger = options.ledger; + this.#ownerId = options.ownerId; + this.#applier = options.applier; + } + + async acquireLease(): Promise { + this.#lease = await this.#ledger.acquireLease(this.#ownerId); + return this.#lease; + } + + async claimNext(): Promise { + if (!this.#lease) throw new Error("outbox lease has not been acquired"); + return await this.#ledger.claimNext(this.#lease); + } + + async applyClaim(claim: OutboxClaim): Promise<"applied" | "fenced" | "retry"> { + if (!await this.#ledger.claimIsCurrent(claim)) return "fenced"; + const mutation: OutboxMutation = { + idempotencyToken: `outbox:${claim.outboxId}`, + commandId: claim.commandId, + principalId: claim.principalId, + kind: claim.kind, + targetId: claim.targetId, + targetRevision: claim.targetRevision, + payload: claim.mutation, + }; + try { + await this.#applier.apply(mutation, { ownerId: claim.ownerId, epoch: claim.ownerEpoch }); + } catch (error) { + if (!await this.#ledger.claimIsCurrent(claim)) return "fenced"; + await this.#ledger.recordFailure(claim, error instanceof Error ? error.message : "outbox application failed"); + return "retry"; + } + return await this.#ledger.acknowledge(claim) ? "applied" : "fenced"; + } + + async drain(maximum = 100): Promise { + if (!this.#lease) await this.acquireLease(); + if (!Number.isSafeInteger(maximum) || maximum < 1 || maximum > 10_000) throw new Error("outbox drain bound is invalid"); + let applied = 0; + let emptyPasses = 0; + for (let index = 0; index < maximum && emptyPasses < 2; index++) { + const claim = await this.claimNext(); + if (!claim) { + emptyPasses++; + continue; + } + emptyPasses = 0; + const result = await this.applyClaim(claim); + if (result === "applied") applied++; + if (result === "fenced") break; + } + return applied; + } +} diff --git a/packages/cluster-server/src/public-api-auth.ts b/packages/cluster-server/src/public-api-auth.ts new file mode 100644 index 00000000..cd733965 --- /dev/null +++ b/packages/cluster-server/src/public-api-auth.ts @@ -0,0 +1,69 @@ +import { timingSafeEqual } from "node:crypto"; +import { isAbsolute } from "node:path"; +import { readBoundedRegularFile } from "./config.ts"; +import type { ApiPrincipal, PrincipalAuthenticator } from "./public-api-v1.ts"; + +const SCOPE = /^[a-z][a-z0-9.-]{0,127}$/u; +interface CredentialRecord { + readonly token: string; + readonly principal: ApiPrincipal; +} +interface PublicApiSecrets { + readonly cursorSecret: string; + readonly authenticator: PrincipalAuthenticator; +} + +function decodeSecrets(value: string): { cursorSecret: string; credentials: readonly CredentialRecord[] } { + let root: unknown; + try { root = JSON.parse(value); } catch { throw new Error("public API credentials file is invalid"); } + if (!root || typeof root !== "object" || Array.isArray(root)) throw new Error("public API credentials file is invalid"); + const object = root as Record; + if (Object.keys(object).some(key => key !== "cursorSecret" && key !== "principals") || typeof object.cursorSecret !== "string") throw new Error("public API credentials file is invalid"); + const secretBytes = new TextEncoder().encode(object.cursorSecret); + if (secretBytes.byteLength < 32 || secretBytes.byteLength > 4096) throw new Error("public API credentials file is invalid"); + if (!Array.isArray(object.principals) || object.principals.length < 1 || object.principals.length > 128) throw new Error("public API credentials file is invalid"); + const principals = new Set(); + const tokenHashes = new Set(); + const credentials: CredentialRecord[] = []; + for (const raw of object.principals) { + if (!raw || typeof raw !== "object" || Array.isArray(raw)) throw new Error("public API credentials file is invalid"); + const entry = raw as Record; + if (Object.keys(entry).some(key => !["token", "principal", "scopes"].includes(key)) || typeof entry.token !== "string" || typeof entry.principal !== "string" || !Array.isArray(entry.scopes)) throw new Error("public API credentials file is invalid"); + const tokenBytes = new TextEncoder().encode(entry.token); + if (tokenBytes.byteLength < 32 || tokenBytes.byteLength > 16_384 || /\s/u.test(entry.token)) throw new Error("public API credentials file is invalid"); + if (!entry.principal || entry.principal !== entry.principal.trim() || new TextEncoder().encode(entry.principal).byteLength > 256 || /\p{Cc}/u.test(entry.principal)) throw new Error("public API credentials file is invalid"); + if (principals.has(entry.principal)) throw new Error("public API credentials file is invalid"); + principals.add(entry.principal); + const tokenIdentity = Buffer.from(tokenBytes).toString("base64"); + if (tokenHashes.has(tokenIdentity)) throw new Error("public API credentials file is invalid"); + tokenHashes.add(tokenIdentity); + const scopes = entry.scopes.map(scope => { + if (typeof scope !== "string" || !SCOPE.test(scope)) throw new Error("public API credentials file is invalid"); + return scope; + }); + if (scopes.length < 1 || scopes.length > 128 || new Set(scopes).size !== scopes.length) throw new Error("public API credentials file is invalid"); + credentials.push({ token: entry.token, principal: { id: entry.principal, scopes: new Set(scopes) } }); + } + return { cursorSecret: object.cursorSecret, credentials }; +} + +class SecretFileAuthenticator implements PrincipalAuthenticator { + readonly #path: string; + constructor(path: string) { this.#path = path; } + async authenticate(token: string): Promise { + const presented = Buffer.from(token, "utf8"); + const { credentials } = decodeSecrets(await readBoundedRegularFile(this.#path, 65_536, "public API credentials")); + let principal: ApiPrincipal | undefined; + for (const credential of credentials) { + const expected = Buffer.from(credential.token, "utf8"); + if (expected.byteLength === presented.byteLength && timingSafeEqual(expected, presented)) principal = credential.principal; + } + return principal; + } +} + +export async function loadPublicApiSecrets(path: string): Promise { + if (!isAbsolute(path)) throw new Error("public API credentials file path must be absolute"); + const decoded = decodeSecrets(await readBoundedRegularFile(path, 65_536, "public API credentials")); + return { cursorSecret: decoded.cursorSecret, authenticator: new SecretFileAuthenticator(path) }; +} diff --git a/packages/cluster-server/src/public-api-v1.ts b/packages/cluster-server/src/public-api-v1.ts new file mode 100644 index 00000000..3983b7e7 --- /dev/null +++ b/packages/cluster-server/src/public-api-v1.ts @@ -0,0 +1,347 @@ +import { randomUUID } from "node:crypto"; +import type { components } from "../../t4-api-client/src/generated/schema.ts"; +import { + canonicalRequestFingerprint, + type LedgerMutationResult, + type PostgresLedger, + type SessionRecord, + type WorkspaceRecord, +} from "./ledger.ts"; + +type Discovery = components["schemas"]["Discovery"]; +type ErrorCode = components["schemas"]["ApiError"]["code"]; +type CommandCreate = components["schemas"]["CommandCreate"]; +type WorkspaceCreate = components["schemas"]["WorkspaceCreate"]; +type WorkspaceMutation = components["schemas"]["WorkspaceMutation"]; +type SessionCreate = components["schemas"]["SessionCreate"]; +type SessionMutation = components["schemas"]["SessionMutation"]; + +export interface ApiPrincipal { + readonly id: string; + readonly scopes: ReadonlySet; +} +export interface PrincipalAuthenticator { + authenticate(token: string): Promise; +} +export interface T4PublicApiV1Options { + readonly ledger: PostgresLedger; + readonly authenticator: PrincipalAuthenticator; + readonly allowedOrigins?: readonly string[]; +} + +const DISCOVERY: Discovery = { + apiVersion: "1.0", + supportedMajors: [1], + capabilities: ["commands.write", "events.read", "sessions.read", "sessions.write", "workspaces.read", "workspaces.write"], + limits: { + pageSizeDefault: 25, + pageSizeMax: 100, + commandBytesMax: 262_144, + commandRequestBytesMax: 1_048_576, + commandMetadataValueBytesMax: 262_144, + watchEventsMax: 1_000, + heartbeatSeconds: 15, + }, +}; +const RESOURCE_ID = /^[A-Za-z0-9][A-Za-z0-9._~-]{0,127}$/u; +const IDEMPOTENCY_KEY = /^[A-Za-z0-9._~-]{16,128}$/u; +const VERSION = /^[1-9][0-9]{0,3}(?:\.[0-9]+)?$/u; +const LABEL_KEY = /^[a-z][a-z0-9.-]{0,62}$/u; +const encoder = new TextEncoder(); + +interface ApiContext { readonly requestId: string; readonly principal: ApiPrincipal; } +interface ParsedObject { readonly value?: Record; readonly response?: Response; } + +function responseHeaders(extra: Record = {}): Headers { + return new Headers({ "cache-control": "no-store", "t4-api-version": "1.0", ...extra }); +} +function jsonResponse(status: number, body: unknown, extra: Record = {}): Response { + const headers = responseHeaders({ "content-type": "application/json", ...extra }); + return new Response(JSON.stringify(body), { status, headers }); +} +function apiError(requestId: string, status: number, code: ErrorCode, message: string, details: Record = {}): Response { + return jsonResponse(status, { error: { code, message, requestId, retryable: status === 503, ...details } }); +} +function notFound(requestId: string): Response { return apiError(requestId, 404, "not_found", "resource was not found"); } +function resultResponse(result: LedgerMutationResult, requestId: string, acceptedStatus: number): Response { + if (result.kind === "idempotency_conflict") return apiError(requestId, 409, "idempotency_conflict", "idempotency key was reused with another canonical request"); + if (result.kind === "revision_conflict") return apiError(requestId, 409, "revision_conflict", "resource revision does not match If-Match"); + if (result.kind === "not_found") return notFound(requestId); + const replayed = result.kind === "replay"; + const status = replayed ? 200 : acceptedStatus; + return jsonResponse(status, result.value, { "idempotency-replayed": replayed ? "true" : "false" }); +} +function deletedResponse(result: LedgerMutationResult, requestId: string): Response { + if (result.kind === "idempotency_conflict") return apiError(requestId, 409, "idempotency_conflict", "idempotency key was reused with another canonical request"); + if (result.kind === "not_found") return notFound(requestId); + return new Response(null, { status: 204, headers: responseHeaders({ "idempotency-replayed": result.kind === "replay" ? "true" : "false" }) }); +} +function labels(value: unknown): Record | undefined { + if (value === undefined) return undefined; + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("labels must be an object"); + const entries = Object.entries(value as Record); + if (entries.length > 32) throw new Error("labels exceeds 32 properties"); + const output: Record = {}; + for (const [key, item] of entries) { + if (!LABEL_KEY.test(key) || typeof item !== "string" || item.length > 128) throw new Error("labels is invalid"); + output[key] = item; + } + return output; +} +function boundedString(value: unknown, name: string, maximum: number): string { + if (typeof value !== "string" || value.length < 1 || value.length > maximum) throw new Error(`${name} is invalid`); + return value; +} +function exactKeys(value: Record, allowed: readonly string[], required: readonly string[], minimum = 0): void { + const keys = Object.keys(value); + if (keys.length < minimum || keys.some(key => !allowed.includes(key)) || required.some(key => !Object.hasOwn(value, key))) throw new Error("request object fields are invalid"); +} +function integerQuery(value: string | null, fallback: number, minimum: number, maximum: number): number | undefined { + if (value === null) return fallback; + if (!/^[1-9][0-9]*$/u.test(value)) return undefined; + const parsed = Number(value); + return Number.isSafeInteger(parsed) && parsed >= minimum && parsed <= maximum ? parsed : undefined; +} + +export class T4PublicApiV1 { + readonly #ledger: PostgresLedger; + readonly #authenticator: PrincipalAuthenticator; + readonly #allowedOrigins?: ReadonlySet; + + constructor(options: T4PublicApiV1Options) { + this.#ledger = options.ledger; + this.#authenticator = options.authenticator; + this.#allowedOrigins = options.allowedOrigins ? new Set(options.allowedOrigins) : undefined; + } + + async handle(request: Request): Promise { + const requestId = `req_${randomUUID()}`; + try { + if (new URL(request.url).protocol !== "https:") return apiError(requestId, 400, "https_required", "HTTPS is required"); + const origin = request.headers.get("origin"); + if (origin && this.#allowedOrigins && !this.#allowedOrigins.has(origin)) return apiError(requestId, 400, "invalid_origin", "request origin is not allowed"); + const authorization = request.headers.get("authorization"); + if (!authorization?.startsWith("Bearer ") || authorization.length > 16_391) return apiError(requestId, 401, "unauthenticated", "valid bearer authentication is required"); + const token = authorization.slice(7); + if (!token || /\s/u.test(token)) return apiError(requestId, 401, "unauthenticated", "valid bearer authentication is required"); + const principal = await this.#authenticator.authenticate(token); + if (!principal || !principal.id || principal.id.length > 256) return apiError(requestId, 401, "unauthenticated", "valid bearer authentication is required"); + const selectedVersion = request.headers.get("t4-api-version"); + if (!selectedVersion || selectedVersion.length > 16 || !VERSION.test(selectedVersion)) return apiError(requestId, 400, "invalid_request", "T4-API-Version is required and must be valid"); + if (selectedVersion.split(".", 1)[0] !== "1") return apiError(requestId, 406, "incompatible_version", "requested API major is unsupported", { supportedMajors: [1] }); + return await this.#route(request, new URL(request.url), { requestId, principal }); + } catch { + return apiError(requestId, 503, "unavailable", "durable gateway is temporarily unavailable"); + } + } + + async #route(request: Request, url: URL, context: ApiContext): Promise { + const segments = url.pathname.split("/").filter(Boolean).map(value => { + try { return decodeURIComponent(value); } catch { return ""; } + }); + if (segments[0] !== "v1") return notFound(context.requestId); + if (segments.length === 1 && request.method === "GET") { + if (!this.#scope(context, "discovery.read")) return apiError(context.requestId, 403, "forbidden", "credential lacks the required operation scope"); + return jsonResponse(200, DISCOVERY); + } + if (segments.length === 2 && segments[1] === "workspaces") { + if (request.method === "GET") return await this.#listWorkspaces(url, context); + if (request.method === "POST") return await this.#createWorkspace(request, context); + } + if (segments.length === 3 && segments[1] === "workspaces" && RESOURCE_ID.test(segments[2]!)) { + if (request.method === "GET") return await this.#getWorkspace(segments[2]!, context); + if (request.method === "PATCH") return await this.#patchWorkspace(request, segments[2]!, context); + if (request.method === "DELETE") return await this.#deleteWorkspace(request, segments[2]!, context); + } + if (segments.length === 4 && segments[1] === "workspaces" && RESOURCE_ID.test(segments[2]!) && segments[3] === "sessions") { + if (request.method === "GET") return await this.#listSessions(url, segments[2]!, context); + if (request.method === "POST") return await this.#createSession(request, segments[2]!, context); + } + if (segments.length === 3 && segments[1] === "sessions" && RESOURCE_ID.test(segments[2]!)) { + if (request.method === "GET") return await this.#getSession(segments[2]!, context); + if (request.method === "PATCH") return await this.#patchSession(request, segments[2]!, context); + if (request.method === "DELETE") return await this.#cancelSession(request, segments[2]!, context, true); + } + if (segments.length === 4 && segments[1] === "sessions" && RESOURCE_ID.test(segments[2]!)) { + if (segments[3] === "cancel" && request.method === "POST") return await this.#cancelSession(request, segments[2]!, context, false); + if (segments[3] === "commands" && request.method === "POST") return await this.#submitCommand(request, segments[2]!, context); + if (segments[3] === "snapshot" && request.method === "GET") return await this.#snapshot(segments[2]!, context); + if (segments[3] === "events" && request.method === "GET") return await this.#events(request, url, segments[2]!, context); + } + return notFound(context.requestId); + } + + #scope(context: ApiContext, required: string): boolean { return context.principal.scopes.has(required); } + #requireScope(context: ApiContext, required: string): Response | undefined { + return this.#scope(context, required) ? undefined : apiError(context.requestId, 403, "forbidden", "credential lacks the required operation scope"); + } + #key(request: Request, context: ApiContext): string | Response { + const value = request.headers.get("idempotency-key"); + return value && IDEMPOTENCY_KEY.test(value) ? value : apiError(context.requestId, 400, "idempotency_key_required", "a valid Idempotency-Key is required"); + } + #ifMatch(request: Request, context: ApiContext): number | Response { + const value = request.headers.get("if-match"); + if (!value || !/^[1-9][0-9]{0,15}$/u.test(value)) return apiError(context.requestId, 400, "invalid_request", "a valid If-Match revision is required"); + const revision = Number(value); + return Number.isSafeInteger(revision) ? revision : apiError(context.requestId, 400, "invalid_request", "a valid If-Match revision is required"); + } + async #body(request: Request, context: ApiContext): Promise { + const contentType = request.headers.get("content-type")?.toLowerCase(); + if (!contentType || !/^application\/json(?:\s*;\s*charset=utf-8)?$/u.test(contentType)) return { response: apiError(context.requestId, 400, "invalid_request", "request content type must be application/json") }; + const announced = request.headers.get("content-length"); + if (announced && (!/^[0-9]+$/u.test(announced) || Number(announced) > DISCOVERY.limits.commandRequestBytesMax)) return { response: apiError(context.requestId, 400, "invalid_request", "request body exceeds the maximum size") }; + const text = await request.text(); + if (encoder.encode(text).byteLength > DISCOVERY.limits.commandRequestBytesMax) return { response: apiError(context.requestId, 400, "invalid_request", "request body exceeds the maximum size") }; + let value: unknown; + try { value = JSON.parse(text); } catch { return { response: apiError(context.requestId, 400, "invalid_request", "request body is malformed JSON") }; } + return value && typeof value === "object" && !Array.isArray(value) + ? { value: value as Record } + : { response: apiError(context.requestId, 422, "invalid_request", "request body must be an object", { violations: [{ field: "body", rule: "type", message: "must be an object" }] }) }; + } + #validation(context: ApiContext, message: string): Response { + return apiError(context.requestId, 422, "invalid_request", message, { violations: [{ field: "body", rule: "schema", message }] }); + } + + async #createWorkspace(request: Request, context: ApiContext): Promise { + const forbidden = this.#requireScope(context, "workspaces.write"); if (forbidden) return forbidden; + const key = this.#key(request, context); if (key instanceof Response) return key; + const parsed = await this.#body(request, context); if (parsed.response) return parsed.response; + let input: WorkspaceCreate; + try { exactKeys(parsed.value!, ["name", "labels"], ["name"]); input = { name: boundedString(parsed.value!.name, "name", 128), ...(parsed.value!.labels !== undefined ? { labels: labels(parsed.value!.labels) } : {}) }; } + catch (error) { return this.#validation(context, error instanceof Error ? error.message : "workspace request is invalid"); } + const fingerprint = canonicalRequestFingerprint({ operation: "createWorkspace", target: "workspaces", body: input }); + return resultResponse(await this.#ledger.createWorkspace(context.principal.id, key, fingerprint, input), context.requestId, 202); + } + async #getWorkspace(workspaceId: string, context: ApiContext): Promise { + const forbidden = this.#requireScope(context, "workspaces.read"); if (forbidden) return forbidden; + const value = await this.#ledger.getWorkspace(context.principal.id, workspaceId); + return value ? jsonResponse(200, value) : notFound(context.requestId); + } + async #listWorkspaces(url: URL, context: ApiContext): Promise { + const forbidden = this.#requireScope(context, "workspaces.read"); if (forbidden) return forbidden; + const size = integerQuery(url.searchParams.get("pageSize"), DISCOVERY.limits.pageSizeDefault, 1, DISCOVERY.limits.pageSizeMax); + if (!size) return this.#validation(context, "pageSize is outside the supported bounds"); + const rawCursor = url.searchParams.get("cursor"); + const after = rawCursor ? this.#ledger.readPageCursor(rawCursor, context.principal.id, "workspaces") : undefined; + if (rawCursor && after === undefined) return this.#validation(context, "cursor is invalid for this identity and operation"); + const items = await this.#ledger.listWorkspaces(context.principal.id, after, size + 1); + return jsonResponse(200, { items: items.slice(0, size), ...(items.length > size ? { nextCursor: this.#ledger.pageCursor(context.principal.id, "workspaces", items[size - 1]!.id) } : {}) }); + } + async #patchWorkspace(request: Request, workspaceId: string, context: ApiContext): Promise { + const forbidden = this.#requireScope(context, "workspaces.write"); if (forbidden) return forbidden; + const revision = this.#ifMatch(request, context); if (revision instanceof Response) return revision; + const key = this.#key(request, context); if (key instanceof Response) return key; + const parsed = await this.#body(request, context); if (parsed.response) return parsed.response; + let input: WorkspaceMutation; + try { exactKeys(parsed.value!, ["name", "labels"], [], 1); input = { ...(parsed.value!.name !== undefined ? { name: boundedString(parsed.value!.name, "name", 128) } : {}), ...(parsed.value!.labels !== undefined ? { labels: labels(parsed.value!.labels) } : {}) }; } + catch (error) { return this.#validation(context, error instanceof Error ? error.message : "workspace mutation is invalid"); } + const fingerprint = canonicalRequestFingerprint({ operation: "mutateWorkspace", target: workspaceId, ifMatch: revision, body: input }); + return resultResponse(await this.#ledger.patchWorkspace(context.principal.id, workspaceId, revision, key, fingerprint, input), context.requestId, 200); + } + async #deleteWorkspace(request: Request, workspaceId: string, context: ApiContext): Promise { + const forbidden = this.#requireScope(context, "workspaces.write"); if (forbidden) return forbidden; + const key = this.#key(request, context); if (key instanceof Response) return key; + const fingerprint = canonicalRequestFingerprint({ operation: "deleteWorkspace", target: workspaceId }); + return deletedResponse(await this.#ledger.deleteWorkspace(context.principal.id, workspaceId, key, fingerprint), context.requestId); + } + + async #createSession(request: Request, workspaceId: string, context: ApiContext): Promise { + const forbidden = this.#requireScope(context, "sessions.write"); if (forbidden) return forbidden; + const key = this.#key(request, context); if (key instanceof Response) return key; + const parsed = await this.#body(request, context); if (parsed.response) return parsed.response; + let input: SessionCreate; + try { exactKeys(parsed.value!, ["title", "labels"], ["title"]); input = { title: boundedString(parsed.value!.title, "title", 128), ...(parsed.value!.labels !== undefined ? { labels: labels(parsed.value!.labels) } : {}) }; } + catch (error) { return this.#validation(context, error instanceof Error ? error.message : "session request is invalid"); } + const fingerprint = canonicalRequestFingerprint({ operation: "spawnSession", target: workspaceId, body: input }); + return resultResponse(await this.#ledger.createSession(context.principal.id, workspaceId, key, fingerprint, input), context.requestId, 202); + } + async #getSession(sessionId: string, context: ApiContext): Promise { + const forbidden = this.#requireScope(context, "sessions.read"); if (forbidden) return forbidden; + const value = await this.#ledger.getSession(context.principal.id, sessionId); + return value ? jsonResponse(200, value) : notFound(context.requestId); + } + async #listSessions(url: URL, workspaceId: string, context: ApiContext): Promise { + const forbidden = this.#requireScope(context, "sessions.read"); if (forbidden) return forbidden; + const size = integerQuery(url.searchParams.get("pageSize"), DISCOVERY.limits.pageSizeDefault, 1, DISCOVERY.limits.pageSizeMax); + if (!size) return this.#validation(context, "pageSize is outside the supported bounds"); + const scope = `workspace:${workspaceId}:sessions`; + const rawCursor = url.searchParams.get("cursor"); + const after = rawCursor ? this.#ledger.readPageCursor(rawCursor, context.principal.id, scope) : undefined; + if (rawCursor && after === undefined) return this.#validation(context, "cursor is invalid for this identity and operation"); + const items = await this.#ledger.listSessions(context.principal.id, workspaceId, after, size + 1); + if (!items) return notFound(context.requestId); + return jsonResponse(200, { items: items.slice(0, size), ...(items.length > size ? { nextCursor: this.#ledger.pageCursor(context.principal.id, scope, items[size - 1]!.id) } : {}) }); + } + async #patchSession(request: Request, sessionId: string, context: ApiContext): Promise { + const forbidden = this.#requireScope(context, "sessions.write"); if (forbidden) return forbidden; + const revision = this.#ifMatch(request, context); if (revision instanceof Response) return revision; + const key = this.#key(request, context); if (key instanceof Response) return key; + const parsed = await this.#body(request, context); if (parsed.response) return parsed.response; + let input: SessionMutation; + try { exactKeys(parsed.value!, ["title", "labels"], [], 1); input = { ...(parsed.value!.title !== undefined ? { title: boundedString(parsed.value!.title, "title", 128) } : {}), ...(parsed.value!.labels !== undefined ? { labels: labels(parsed.value!.labels) } : {}) }; } + catch (error) { return this.#validation(context, error instanceof Error ? error.message : "session mutation is invalid"); } + const fingerprint = canonicalRequestFingerprint({ operation: "mutateSession", target: sessionId, ifMatch: revision, body: input }); + return resultResponse(await this.#ledger.patchSession(context.principal.id, sessionId, revision, key, fingerprint, input), context.requestId, 200); + } + async #cancelSession(request: Request, sessionId: string, context: ApiContext, deletion: boolean): Promise { + const forbidden = this.#requireScope(context, "sessions.write"); if (forbidden) return forbidden; + const key = this.#key(request, context); if (key instanceof Response) return key; + const fingerprint = canonicalRequestFingerprint({ operation: deletion ? "deleteSession" : "cancelSession", target: sessionId }); + const result = await this.#ledger.cancelSession(context.principal.id, sessionId, key, fingerprint, deletion); + return deletion ? deletedResponse(result, context.requestId) : resultResponse(result, context.requestId, 202); + } + async #submitCommand(request: Request, sessionId: string, context: ApiContext): Promise { + const forbidden = this.#requireScope(context, "commands.write"); if (forbidden) return forbidden; + const key = this.#key(request, context); if (key instanceof Response) return key; + const parsed = await this.#body(request, context); if (parsed.response) return parsed.response; + let input: CommandCreate; + try { + exactKeys(parsed.value!, ["command", "metadata"], ["command"]); + const command = boundedString(parsed.value!.command, "command", 262_144); + if (encoder.encode(command).byteLength > DISCOVERY.limits.commandBytesMax) throw new Error("command exceeds UTF-8 byte limit"); + const metadataValue = parsed.value!.metadata ?? {}; + if (!metadataValue || typeof metadataValue !== "object" || Array.isArray(metadataValue) || Object.keys(metadataValue).length > 32) throw new Error("metadata is invalid"); + const metadata: Record = {}; + for (const [metadataKey, value] of Object.entries(metadataValue as Record)) { + if (!LABEL_KEY.test(metadataKey) || value !== null && !["string", "number", "boolean"].includes(typeof value) || typeof value === "number" && (!Number.isSafeInteger(value) || !Number.isFinite(value))) throw new Error("metadata is invalid"); + if (typeof value === "string" && encoder.encode(value).byteLength > DISCOVERY.limits.commandMetadataValueBytesMax) throw new Error("metadata value exceeds UTF-8 byte limit"); + metadata[metadataKey] = value as string | number | boolean | null; + } + input = { command, metadata }; + } catch (error) { return this.#validation(context, error instanceof Error ? error.message : "command request is invalid"); } + const fingerprint = canonicalRequestFingerprint({ operation: "submitCommand", target: sessionId, body: input }); + return resultResponse(await this.#ledger.submitCommand(context.principal.id, sessionId, key, fingerprint, { command: input.command, metadata: input.metadata ?? {} }), context.requestId, 202); + } + async #snapshot(sessionId: string, context: ApiContext): Promise { + const forbidden = this.#requireScope(context, "events.read"); if (forbidden) return forbidden; + const snapshot = await this.#ledger.snapshot(context.principal.id, sessionId); + return snapshot ? jsonResponse(200, { sessionId, cursor: snapshot.cursor, state: snapshot.session.state, entries: snapshot.entries }) : notFound(context.requestId); + } + async #events(request: Request, url: URL, sessionId: string, context: ApiContext): Promise { + const forbidden = this.#requireScope(context, "events.read"); if (forbidden) return forbidden; + const maxEvents = integerQuery(url.searchParams.get("maxEvents"), 100, 1, DISCOVERY.limits.watchEventsMax); + const heartbeat = integerQuery(url.searchParams.get("heartbeatSeconds"), DISCOVERY.limits.heartbeatSeconds, 5, 60); + if (!maxEvents || !heartbeat) return this.#validation(context, "watch bounds are invalid"); + const queryCursor = url.searchParams.get("cursor"); + const headerCursor = request.headers.get("last-event-id"); + if (queryCursor && headerCursor && queryCursor !== headerCursor) return apiError(context.requestId, 400, "invalid_request", "cursor and Last-Event-ID must agree"); + let window; + try { window = await this.#ledger.eventWindow(context.principal.id, sessionId, queryCursor ?? headerCursor ?? undefined, maxEvents); } + catch { return this.#validation(context, "watch cursor is invalid for this identity and session"); } + if (!window) return notFound(context.requestId); + if (window.expired) return apiError(context.requestId, 410, "cursor_expired", "watch cursor is outside retained history", { resync: { snapshotUrl: `/v1/sessions/${sessionId}/snapshot`, cursor: window.resyncCursor } }); + const frames: string[] = []; + for (const event of window.events) { + const cursor = this.#ledger.watchCursor(context.principal.id, sessionId, event.sequence); + const data = event.type === "session" + ? { type: "session", cursor, state: event.payload.state, revision: event.payload.revision } + : { type: "command", cursor, commandId: event.payload.commandId, state: event.payload.state }; + frames.push(`id: ${cursor}\nevent: ${event.type}\ndata: ${JSON.stringify(data)}\n\n`); + } + if (frames.length === 0) { + const data = { type: "heartbeat", cursor: window.cursor, observedAt: new Date().toISOString() }; + frames.push(`id: ${window.cursor}\nevent: heartbeat\ndata: ${JSON.stringify(data)}\n\n`); + } + return new Response(frames.join(""), { status: 200, headers: responseHeaders({ "content-type": "text/event-stream" }) }); + } +} diff --git a/packages/cluster-server/src/public-kubernetes-applier.ts b/packages/cluster-server/src/public-kubernetes-applier.ts new file mode 100644 index 00000000..3ed2f0dc --- /dev/null +++ b/packages/cluster-server/src/public-kubernetes-applier.ts @@ -0,0 +1,126 @@ +import { semanticResourceHash, KubernetesApiClient, KubernetesApiError } from "./kubernetes-client.ts"; +import type { KubernetesResource } from "./kubernetes-projection.ts"; +import type { KubernetesOutboxApplier, OutboxMutation } from "./outbox-worker.ts"; +import type { OwnerLease } from "./ledger.ts"; + +export interface PublicKubernetesOutboxApplierOptions { + readonly client: KubernetesApiClient; + readonly hostRef: string; +} + +function record(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +} +function annotations(mutation: OutboxMutation, fence: OwnerLease): Record { + return { + "cluster.t4.dev/ledger-command-id": mutation.commandId, + "cluster.t4.dev/ledger-outbox-token": mutation.idempotencyToken, + "cluster.t4.dev/ledger-owner": fence.ownerId, + "cluster.t4.dev/ledger-owner-epoch": fence.epoch.toString(), + "cluster.t4.dev/ledger-semantic-hash": semanticResourceHash(mutation.payload), + }; +} +function assertedEpoch(resource: KubernetesResource, fence: OwnerLease): void { + const value = resource.metadata.annotations?.["cluster.t4.dev/ledger-owner-epoch"]; + if (value && /^(?:0|[1-9][0-9]*)$/u.test(value) && BigInt(value) > fence.epoch) throw new Error("Kubernetes resource is fenced by a newer ledger owner"); +} + +export class PublicKubernetesOutboxApplier implements KubernetesOutboxApplier { + readonly #client: KubernetesApiClient; + readonly #hostRef: string; + constructor(options: PublicKubernetesOutboxApplierOptions) { + this.#client = options.client; + this.#hostRef = options.hostRef; + } + + async apply(mutation: OutboxMutation, fence: OwnerLease): Promise { + if (mutation.kind === "workspace.create") return await this.#createWorkspace(mutation, fence); + if (mutation.kind === "session.create") return await this.#createSession(mutation, fence); + if (mutation.kind === "workspace.patch") return await this.#patchWorkspace(mutation, fence); + if (mutation.kind === "session.patch") return await this.#patchSession(mutation, fence); + if (mutation.kind === "command.submit") return await this.#submitCommand(mutation, fence); + if (mutation.kind === "workspace.delete") return await this.#delete("t4workspaces", mutation, fence); + return await this.#delete("t4sessions", mutation, fence); + } + + async #createWorkspace(mutation: OutboxMutation, fence: OwnerLease): Promise { + const body = { + apiVersion: "cluster.t4.dev/v1alpha1", + kind: "T4Workspace", + metadata: { name: mutation.targetId, annotations: annotations(mutation, fence) }, + spec: { + hostRef: this.#hostRef, + owner: mutation.principalId, + displayName: String(mutation.payload.name), + retentionPolicy: "Retain", + size: "20Gi", + }, + }; + await this.#createIdempotently("t4workspaces", mutation, fence, body); + } + async #createSession(mutation: OutboxMutation, fence: OwnerLease): Promise { + const body = { + apiVersion: "cluster.t4.dev/v1alpha1", + kind: "T4Session", + metadata: { name: mutation.targetId, annotations: annotations(mutation, fence) }, + spec: { + hostRef: this.#hostRef, + workspaceRef: String(mutation.payload.workspaceId), + title: String(mutation.payload.title), + runtimeProfile: "omp-default", + guiEnabled: true, + }, + }; + await this.#createIdempotently("t4sessions", mutation, fence, body); + } + async #patchWorkspace(mutation: OutboxMutation, fence: OwnerLease): Promise { + const current = await this.#owned("t4workspaces", mutation, fence); + const spec = record(current.spec); + await this.#client.patch("t4workspaces", mutation.targetId, { + metadata: { annotations: annotations(mutation, fence) }, + spec: { ...spec, displayName: String(mutation.payload.name) }, + }); + } + async #patchSession(mutation: OutboxMutation, fence: OwnerLease): Promise { + await this.#owned("t4sessions", mutation, fence); + await this.#client.patch("t4sessions", mutation.targetId, { + metadata: { annotations: annotations(mutation, fence) }, + spec: { title: String(mutation.payload.title) }, + }); + } + async #submitCommand(mutation: OutboxMutation, fence: OwnerLease): Promise { + await this.#owned("t4sessions", mutation, fence); + await this.#client.patch("t4sessions", mutation.targetId, { + metadata: { annotations: { ...annotations(mutation, fence), "cluster.t4.dev/pending-command": JSON.stringify(mutation.payload) } }, + }); + } + async #delete(resourceType: "t4workspaces" | "t4sessions", mutation: OutboxMutation, fence: OwnerLease): Promise { + let current: KubernetesResource; + try { current = await this.#owned(resourceType, mutation, fence); } + catch (error) { + if (error instanceof KubernetesApiError && error.status === 404) return; + throw error; + } + if (!current.metadata.uid || !current.metadata.resourceVersion) throw new Error("Kubernetes delete precondition is unavailable"); + try { await this.#client.delete(resourceType, mutation.targetId, { uid: current.metadata.uid, resourceVersion: current.metadata.resourceVersion }); } + catch (error) { if (!(error instanceof KubernetesApiError) || error.status !== 404) throw error; } + } + async #owned(resourceType: "t4workspaces" | "t4sessions", mutation: OutboxMutation, fence: OwnerLease): Promise { + const current = await this.#client.get(resourceType, mutation.targetId); + assertedEpoch(current, fence); + const spec = record(current.spec); + if (spec.hostRef !== this.#hostRef) throw new Error("Kubernetes resource belongs to another cluster host"); + if (resourceType === "t4workspaces" && spec.owner !== mutation.principalId) throw new Error("Kubernetes workspace belongs to another principal"); + return current; + } + async #createIdempotently(resourceType: "t4workspaces" | "t4sessions", mutation: OutboxMutation, fence: OwnerLease, body: KubernetesResource): Promise { + try { await this.#client.create(resourceType, body); } + catch (error) { + if (!(error instanceof KubernetesApiError) || error.status !== 409) throw error; + const current = await this.#client.get(resourceType, mutation.targetId); + assertedEpoch(current, fence); + const currentAnnotations = current.metadata.annotations ?? {}; + if (currentAnnotations["cluster.t4.dev/ledger-outbox-token"] !== mutation.idempotencyToken || currentAnnotations["cluster.t4.dev/ledger-semantic-hash"] !== semanticResourceHash(mutation.payload)) throw new Error("Kubernetes resource conflicts with durable ledger intent"); + } + } +} diff --git a/packages/cluster-server/src/server.ts b/packages/cluster-server/src/server.ts index b1555972..86942a30 100644 --- a/packages/cluster-server/src/server.ts +++ b/packages/cluster-server/src/server.ts @@ -3,10 +3,12 @@ import { normalizeIpAddress } from "@t4-code/host-service"; import type { GatewayConnection, ClusterGateway } from "./gateway.ts"; import { ClusterInfrastructureProjection } from "./kubernetes-projection.ts"; import { ClusterMetrics, ClusterServerHealth, JsonLogger, createAdminHandler } from "./observability.ts"; +import type { T4PublicApiV1 } from "./public-api-v1.ts"; interface SocketData { connection?: GatewayConnection; principal: string; } export interface ClusterHttpServersOptions { readonly gateway: ClusterGateway; + readonly publicApi?: T4PublicApiV1; readonly projection: ClusterInfrastructureProjection; readonly gatewayPort: number; readonly adminPort: number; @@ -66,8 +68,21 @@ export function startClusterHttpServers(options: ClusterHttpServersOptions): Clu const gatewayServer = Bun.serve({ hostname: "0.0.0.0", port: options.gatewayPort, - fetch(request, server) { + async fetch(request, server) { const url = new URL(request.url); + if (url.pathname !== "/v1/ws" && (url.pathname === "/v1" || url.pathname.startsWith("/v1/"))) { + if (!options.publicApi) return new Response("not found", { status: 404 }); + if (draining) return new Response("draining", { status: 503 }); + const remoteAddress = server.requestIP(request)?.address; + if (!remoteAddress || !trustedSource(remoteAddress) || request.headers.get("x-forwarded-proto") !== "https") { + const insecure = new URL(request.url); + insecure.protocol = "http:"; + return await options.publicApi.handle(new Request(insecure, request)); + } + const secure = new URL(request.url); + secure.protocol = "https:"; + return await options.publicApi.handle(new Request(secure, request)); + } if (url.pathname !== "/v1/ws") return new Response("not found", { status: 404 }); if (request.method !== "GET") return new Response("method not allowed", { status: 405 }); if (draining) return new Response("draining", { status: 503 }); diff --git a/packages/cluster-server/test/durable-gateway-postgres.test.ts b/packages/cluster-server/test/durable-gateway-postgres.test.ts index 2bbb822f..932cfe2f 100644 --- a/packages/cluster-server/test/durable-gateway-postgres.test.ts +++ b/packages/cluster-server/test/durable-gateway-postgres.test.ts @@ -274,8 +274,8 @@ describe.sequential("PostgreSQL-backed public T4 API v1", () => { expect(await stale.applyClaim(claim as OutboxClaim)).toBe("fenced"); expect(kubernetes.attempts).toHaveLength(0); expect(await current.drain()).toBeGreaterThan(0); - const events = await admin`SELECT owner_epoch FROM t4_events WHERE session_id = ${session.id} ORDER BY sequence`; - expect(events.every(row => BigInt(row.owner_epoch) === currentLease.epoch || BigInt(row.owner_epoch) === 0n)).toBe(true); + const events = await admin`SELECT owner_epoch FROM t4_events WHERE session_id = ${session.id} ORDER BY sequence` as { owner_epoch: bigint }[]; + expect(events.every(row => row.owner_epoch === currentLease.epoch || row.owner_epoch === 0n)).toBe(true); await ledger.close(); }); From d25c26ff4b0d7b66d4c094646b38872c0ba70f43 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 01:50:43 +0000 Subject: [PATCH 117/131] fix(cluster-server): transact durable migrations --- .../migrations/001_durable_gateway.down.sql | 2 -- .../cluster-server/migrations/001_durable_gateway.sql | 2 -- packages/cluster-server/src/ledger.ts | 11 +++++++++-- packages/cluster-server/src/public-api-v1.ts | 2 -- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/cluster-server/migrations/001_durable_gateway.down.sql b/packages/cluster-server/migrations/001_durable_gateway.down.sql index c712b77b..eb96ccf7 100644 --- a/packages/cluster-server/migrations/001_durable_gateway.down.sql +++ b/packages/cluster-server/migrations/001_durable_gateway.down.sql @@ -1,4 +1,3 @@ -BEGIN; DROP TABLE IF EXISTS t4_outbox; DROP TABLE IF EXISTS t4_owner_leases; DROP TABLE IF EXISTS t4_snapshot_entries; @@ -8,4 +7,3 @@ DROP TABLE IF EXISTS t4_session_intents; DROP TABLE IF EXISTS t4_workspace_intents; DROP TABLE IF EXISTS t4_commands; DROP TABLE IF EXISTS t4_schema_migrations; -COMMIT; diff --git a/packages/cluster-server/migrations/001_durable_gateway.sql b/packages/cluster-server/migrations/001_durable_gateway.sql index 60adbe50..ead19e59 100644 --- a/packages/cluster-server/migrations/001_durable_gateway.sql +++ b/packages/cluster-server/migrations/001_durable_gateway.sql @@ -1,4 +1,3 @@ -BEGIN; CREATE TABLE IF NOT EXISTS t4_schema_migrations (version integer PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT clock_timestamp()); CREATE TABLE IF NOT EXISTS t4_commands ( command_id text PRIMARY KEY, principal_id text NOT NULL, operation text NOT NULL, target_scope text NOT NULL, @@ -48,4 +47,3 @@ CREATE TABLE IF NOT EXISTS t4_outbox ( ); CREATE INDEX IF NOT EXISTS t4_outbox_ready ON t4_outbox (state, next_attempt_at, outbox_id); INSERT INTO t4_schema_migrations(version) VALUES (1) ON CONFLICT (version) DO NOTHING; -COMMIT; diff --git a/packages/cluster-server/src/ledger.ts b/packages/cluster-server/src/ledger.ts index 7430aaa7..37243d7c 100644 --- a/packages/cluster-server/src/ledger.ts +++ b/packages/cluster-server/src/ledger.ts @@ -142,12 +142,19 @@ export class PostgresLedger { async migrate(): Promise { const migration = await Bun.file(MIGRATION_URL).text(); - await this.#sql.unsafe(migration); + await this.#sql.begin("ISOLATION LEVEL SERIALIZABLE", async transaction => { + await transaction`SELECT pg_advisory_xact_lock(741405)`; + await transaction.unsafe(migration); + }); const rows = await this.#sql<{ version: number }[]>`SELECT version FROM t4_schema_migrations ORDER BY version DESC LIMIT 1`; if (rows[0]?.version !== 1) throw new Error("durable gateway schema migration did not reach version 1"); } async rollback(): Promise { - await this.#sql.unsafe(await Bun.file(ROLLBACK_URL).text()); + const rollback = await Bun.file(ROLLBACK_URL).text(); + await this.#sql.begin("ISOLATION LEVEL SERIALIZABLE", async transaction => { + await transaction`SELECT pg_advisory_xact_lock(741405)`; + await transaction.unsafe(rollback); + }); } async close(): Promise { await this.#sql.close(); } diff --git a/packages/cluster-server/src/public-api-v1.ts b/packages/cluster-server/src/public-api-v1.ts index 3983b7e7..f427a5f6 100644 --- a/packages/cluster-server/src/public-api-v1.ts +++ b/packages/cluster-server/src/public-api-v1.ts @@ -4,8 +4,6 @@ import { canonicalRequestFingerprint, type LedgerMutationResult, type PostgresLedger, - type SessionRecord, - type WorkspaceRecord, } from "./ledger.ts"; type Discovery = components["schemas"]["Discovery"]; From 5b7915e0af571ee30aaa8fb966a9d96da0a12a1e Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 01:59:05 +0000 Subject: [PATCH 118/131] fix(cluster-server): validate durable ledger epochs --- packages/cluster-server/src/ledger.ts | 34 ++++++++++++------- .../test/durable-gateway-postgres.test.ts | 2 +- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/packages/cluster-server/src/ledger.ts b/packages/cluster-server/src/ledger.ts index 37243d7c..4cc85bf9 100644 --- a/packages/cluster-server/src/ledger.ts +++ b/packages/cluster-server/src/ledger.ts @@ -97,9 +97,15 @@ const ROLLBACK_URL = new URL("../migrations/001_durable_gateway.down.sql", impor const OUTBOX_LEASE = "gateway-outbox"; const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER); -function safeNumber(value: bigint): number { - if (value < 0n || value > MAX_SAFE_BIGINT) throw new Error("persisted ledger integer is outside the public API range"); - return Number(value); +function persistedBigint(value: unknown, allowZero = false): bigint { + if (typeof value !== "bigint") throw new Error("persisted ledger bigint is invalid"); + if (allowZero ? value < 0n : value <= 0n) throw new Error("persisted ledger bigint is invalid"); + return value; +} +function safeNumber(value: unknown, allowZero = false): number { + const decoded = persistedBigint(value, allowZero); + if (decoded > MAX_SAFE_BIGINT) throw new Error("persisted ledger integer is outside the public API range"); + return Number(decoded); } function labels(value: Record): Record | undefined { return Object.keys(value).length > 0 ? value : undefined; @@ -314,26 +320,26 @@ export class PostgresLedger { if (!current) return undefined; const rows = await this.#sql<{ entry_sequence: bigint; kind: "input" | "output" | "status"; text_value: string }[]>`SELECT entry_sequence, kind, text_value FROM t4_snapshot_entries WHERE principal_id = ${principalId} AND session_id = ${sessionId} ORDER BY entry_sequence DESC LIMIT 1000`; const retention = await this.#sql`SELECT first_retained_sequence, latest_sequence FROM t4_event_retention WHERE principal_id = ${principalId} AND session_id = ${sessionId}`; - const latest = retention[0]?.latest_sequence ?? 0n; - return { session: current, cursor: this.watchCursor(principalId, sessionId, latest), entries: rows.reverse().map(row => ({ sequence: safeNumber(row.entry_sequence), kind: row.kind, text: row.text_value })) }; + const latest = retention[0] ? persistedBigint(retention[0].latest_sequence, true) : 0n; + return { session: current, cursor: this.watchCursor(principalId, sessionId, latest), entries: rows.reverse().map(row => ({ sequence: safeNumber(row.entry_sequence, true), kind: row.kind, text: row.text_value })) }; } async eventWindow(principalId: string, sessionId: string, cursor: string | undefined, limit: number): Promise { if (!await this.getSession(principalId, sessionId)) return undefined; const retentionRows = await this.#sql`SELECT first_retained_sequence, latest_sequence FROM t4_event_retention WHERE principal_id = ${principalId} AND session_id = ${sessionId}`; - const first = retentionRows[0]?.first_retained_sequence ?? 0n; - const latest = retentionRows[0]?.latest_sequence ?? 0n; + const first = retentionRows[0] ? persistedBigint(retentionRows[0].first_retained_sequence, true) : 0n; + const latest = retentionRows[0] ? persistedBigint(retentionRows[0].latest_sequence, true) : 0n; let after = 0n; if (cursor) { const decoded = this.#decodeCursor(cursor, "watch", principalId, sessionId); - if (decoded === undefined) throw new Error("invalid_cursor"); + if (typeof decoded !== "bigint") throw new Error("invalid_cursor"); after = decoded; } const resyncCursor = this.watchCursor(principalId, sessionId, latest); if (first > 0n && after < first - 1n) return { events: [], cursor: resyncCursor, expired: true, resyncCursor }; const rows = await this.#sql`SELECT sequence, event_type, payload FROM t4_events WHERE principal_id = ${principalId} AND session_id = ${sessionId} AND sequence > ${after} ORDER BY sequence LIMIT ${limit}`; - const delivered = rows.at(-1)?.sequence ?? after; - return { events: rows.map(row => ({ sequence: row.sequence, type: row.event_type, payload: row.payload })), cursor: this.watchCursor(principalId, sessionId, delivered), expired: false, resyncCursor }; + const delivered = rows.at(-1) ? persistedBigint(rows.at(-1)!.sequence) : after; + return { events: rows.map(row => ({ sequence: persistedBigint(row.sequence), type: row.event_type, payload: row.payload })), cursor: this.watchCursor(principalId, sessionId, delivered), expired: false, resyncCursor }; } watchCursor(principalId: string, sessionId: string, sequence: bigint): string { return this.#cursor("watch", principalId, sessionId, sequence); } @@ -348,7 +354,7 @@ export class PostgresLedger { await transaction`INSERT INTO t4_owner_leases (lease_name, owner_id, epoch, expires_at) VALUES (${OUTBOX_LEASE}, ${ownerId}, 1, clock_timestamp() + (${leaseSeconds} * interval '1 second')) ON CONFLICT (lease_name) DO UPDATE SET owner_id = EXCLUDED.owner_id, epoch = t4_owner_leases.epoch + 1, expires_at = EXCLUDED.expires_at, updated_at = clock_timestamp()`; return await transaction`SELECT owner_id, epoch FROM t4_owner_leases WHERE lease_name = ${OUTBOX_LEASE}`; }); - return { ownerId: rows[0]!.owner_id, epoch: rows[0]!.epoch }; + return { ownerId: rows[0]!.owner_id, epoch: persistedBigint(rows[0]!.epoch) }; } async claimNext(lease: OwnerLease): Promise { @@ -424,10 +430,12 @@ export class PostgresLedger { async #appendEvent(transaction: SQL, principalId: string, sessionId: string, type: "session" | "command", payload: Record, ownerEpoch: bigint): Promise { const inserted = await transaction<{ sequence: bigint }[]>`INSERT INTO t4_events (principal_id, session_id, event_type, payload, owner_epoch) VALUES (${principalId}, ${sessionId}, ${type}, ${payload}, ${ownerEpoch}) RETURNING sequence`; - const sequence = inserted[0]!.sequence; + const sequence = persistedBigint(inserted[0]?.sequence); await transaction`DELETE FROM t4_events WHERE principal_id = ${principalId} AND session_id = ${sessionId} AND sequence < COALESCE((SELECT sequence FROM t4_events WHERE principal_id = ${principalId} AND session_id = ${sessionId} ORDER BY sequence DESC OFFSET ${this.#eventRetention - 1} LIMIT 1), 0)`; const retained = await transaction<{ first: bigint; latest: bigint }[]>`SELECT COALESCE(MIN(sequence), 0) AS first, COALESCE(MAX(sequence), 0) AS latest FROM t4_events WHERE principal_id = ${principalId} AND session_id = ${sessionId}`; - await transaction`INSERT INTO t4_event_retention (principal_id, session_id, first_retained_sequence, latest_sequence) VALUES (${principalId}, ${sessionId}, ${retained[0]!.first}, ${retained[0]!.latest}) ON CONFLICT (principal_id, session_id) DO UPDATE SET first_retained_sequence = EXCLUDED.first_retained_sequence, latest_sequence = EXCLUDED.latest_sequence, updated_at = clock_timestamp()`; + const first = persistedBigint(retained[0]?.first, true); + const latest = persistedBigint(retained[0]?.latest, true); + await transaction`INSERT INTO t4_event_retention (principal_id, session_id, first_retained_sequence, latest_sequence) VALUES (${principalId}, ${sessionId}, ${first}, ${latest}) ON CONFLICT (principal_id, session_id) DO UPDATE SET first_retained_sequence = EXCLUDED.first_retained_sequence, latest_sequence = EXCLUDED.latest_sequence, updated_at = clock_timestamp()`; return sequence; } } diff --git a/packages/cluster-server/test/durable-gateway-postgres.test.ts b/packages/cluster-server/test/durable-gateway-postgres.test.ts index 932cfe2f..5fe4372a 100644 --- a/packages/cluster-server/test/durable-gateway-postgres.test.ts +++ b/packages/cluster-server/test/durable-gateway-postgres.test.ts @@ -156,7 +156,7 @@ let admin: SQL; beforeAll(async () => { if (!DATABASE_URL) throw new Error("T4_TEST_POSTGRES_URL is required for durable gateway tests"); - admin = new SQL(DATABASE_URL, { max: 1 }); + admin = new SQL(DATABASE_URL, { max: 1, bigint: true }); await admin.unsafe("DROP SCHEMA public CASCADE; CREATE SCHEMA public"); const modules = await Promise.all([ import(/* @vite-ignore */ LEDGER_MODULE_PATH).catch(() => undefined), From 8c0ae563f10d7cc657f0d089ef541949a42e6833 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 02:04:13 +0000 Subject: [PATCH 119/131] test(cluster-server): compare PostgreSQL counts as bigint --- packages/cluster-server/test/durable-gateway-postgres.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cluster-server/test/durable-gateway-postgres.test.ts b/packages/cluster-server/test/durable-gateway-postgres.test.ts index 5fe4372a..f289753b 100644 --- a/packages/cluster-server/test/durable-gateway-postgres.test.ts +++ b/packages/cluster-server/test/durable-gateway-postgres.test.ts @@ -386,7 +386,7 @@ describe.sequential("PostgreSQL-backed public T4 API v1", () => { (SELECT count(*) FROM t4_workspace_intents WHERE name = 'must not persist') AS intents, (SELECT count(*) FROM t4_outbox WHERE idempotency_key = 'rollback-key-000000001') AS outbox `); - expect(rows[0]).toMatchObject({ commands: "0", intents: "0", outbox: "0" }); + expect(rows[0]).toMatchObject({ commands: 0n, intents: 0n, outbox: 0n }); await admin.unsafe("DROP TRIGGER t4_test_reject_outbox_trigger ON t4_outbox; DROP FUNCTION t4_test_reject_outbox()"); await ledger.close(); }); From 8b7dd06dfdf41d17970cc44a00465e62921371e9 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 02:11:19 +0000 Subject: [PATCH 120/131] test(cluster-server): scope PostgreSQL suite to service job --- .../cluster-server/test/durable-gateway-postgres.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/cluster-server/test/durable-gateway-postgres.test.ts b/packages/cluster-server/test/durable-gateway-postgres.test.ts index f289753b..814da375 100644 --- a/packages/cluster-server/test/durable-gateway-postgres.test.ts +++ b/packages/cluster-server/test/durable-gateway-postgres.test.ts @@ -155,7 +155,7 @@ function eventData(responseBody: string): JsonDocument[] { let admin: SQL; beforeAll(async () => { - if (!DATABASE_URL) throw new Error("T4_TEST_POSTGRES_URL is required for durable gateway tests"); + if (!DATABASE_URL) return; admin = new SQL(DATABASE_URL, { max: 1, bigint: true }); await admin.unsafe("DROP SCHEMA public CASCADE; CREATE SCHEMA public"); const modules = await Promise.all([ @@ -172,7 +172,8 @@ afterAll(async () => { await admin?.close(); }); -describe.sequential("PostgreSQL-backed public T4 API v1", () => { +const postgresSuite = DATABASE_URL ? describe.sequential : describe.skip; +postgresSuite("PostgreSQL-backed public T4 API v1", () => { it("replays an identical create after process restart without a second Kubernetes mutation", async () => { expect(PostgresLedger, "PostgreSQL ledger implementation is required").toBeDefined(); expect(T4PublicApiV1, "public T4 API v1 implementation is required").toBeDefined(); From 8584ff04f03f38e67a13fc382df3472da495ce0f Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 02:36:45 +0000 Subject: [PATCH 121/131] test(cluster): cover durable gateway review repairs --- .../charttests/chart_contract_test.go | 61 ++- packages/cluster-server/test/config.test.ts | 11 + .../test/durable-gateway-postgres.test.ts | 429 +++++++++++++++++- .../test/kubernetes-client.test.ts | 204 +++++++++ 4 files changed, 698 insertions(+), 7 deletions(-) diff --git a/packages/cluster-operator/charttests/chart_contract_test.go b/packages/cluster-operator/charttests/chart_contract_test.go index 9bbfa883..0f5193eb 100644 --- a/packages/cluster-operator/charttests/chart_contract_test.go +++ b/packages/cluster-operator/charttests/chart_contract_test.go @@ -333,11 +333,70 @@ func TestRBACSeparatesControllerMutationFromServerProjection(t *testing.T) { if strings.Contains(controllerRole, "resources: [secrets]") { t.Fatal("controller role can read provider credential Secrets") } - assertContains(t, serverRole, "t4clusterhosts", "t4workspaces", "t4sessions", "create", "list", "watch") + assertContains(t, serverRole, + "resources: [t4clusterhosts]", + "verbs: [get, list, watch]", + "resources: [t4workspaces, t4sessions]", + "verbs: [get, list, watch, create, patch, delete]", + ) if strings.Contains(serverRole, "secrets") || strings.Contains(serverRole, "persistentvolumeclaims") || strings.Contains(serverRole, "t4sessions/status") { t.Fatal("server role can read secrets or mutate controller-owned infrastructure/status") } } + +func TestPublicAPIRequiresDurableSecretAndExplicitPostgresEgress(t *testing.T) { + withoutSecret := append(enabledValues(), + "--set", "publicApi.enabled=true", + "--set", "networkPolicy.postgresCIDRs[0]=198.51.100.20/32", + "--set", "networkPolicy.postgresPorts[0]=5432", + ) + helmTemplateMustFail(t, withoutSecret...) + + withoutDatabaseRoute := append(enabledValues(), + "--set", "publicApi.enabled=true", + "--set-string", "publicApi.existingSecret=t4-public-api", + ) + helmTemplateMustFail(t, withoutDatabaseRoute...) + + values := append(enabledValues(), + "--set", "publicApi.enabled=true", + "--set-string", "publicApi.existingSecret=t4-public-api", + "--set-string", "publicApi.postgresURLKey=postgres-url", + "--set-string", "publicApi.credentialsKey=credentials.json", + "--set-string", "clusterHost.runtimeProfiles[0]=review-admitted", + "--set", "networkPolicy.postgresCIDRs[0]=198.51.100.20/32", + "--set", "networkPolicy.postgresPorts[0]=5432", + ) + output := helmTemplate(t, values...) + server := documentContainingKind(t, output, "Deployment", "name: \"release-name-t4-cluster-server\"") + assertContains(t, server, + "name: T4_POSTGRES_URL_FILE", + "name: T4_PUBLIC_API_CREDENTIALS_FILE", + "name: T4_PUBLIC_API_RUNTIME_PROFILE", + "value: \"review-admitted\"", + "secretName: \"t4-public-api\"", + "key: postgres-url", + "key: credentials.json", + ) + host := documentContainingKind(t, output, "T4ClusterHost", "name: \"t4-cluster\"") + assertContains(t, host, "- review-admitted") + serverEgress := documentContainingKind(t, output, "NetworkPolicy", "name: \"release-name-t4-cluster-server-egress\"") + assertContains(t, serverEgress, "198.51.100.20/32", "port: 5432") + if strings.Contains(serverEgress, "0.0.0.0/0") || strings.Contains(serverEgress, "::/0") { + t.Fatalf("public API PostgreSQL egress is broader than its configured destination:\n%s", serverEgress) + } +} + +func TestPublicAPIRejectsIncompleteSecretKeysAndPostgresRoutes(t *testing.T) { + for name, values := range map[string][]string{ + "empty PostgreSQL key": {"--set", "publicApi.enabled=true", "--set-string", "publicApi.existingSecret=t4-public-api", "--set-string", "publicApi.postgresURLKey=", "--set", "networkPolicy.postgresCIDRs[0]=198.51.100.20/32", "--set", "networkPolicy.postgresPorts[0]=5432"}, + "empty credentials key": {"--set", "publicApi.enabled=true", "--set-string", "publicApi.existingSecret=t4-public-api", "--set-string", "publicApi.credentialsKey=", "--set", "networkPolicy.postgresCIDRs[0]=198.51.100.20/32", "--set", "networkPolicy.postgresPorts[0]=5432"}, + "destination without port": {"--set", "publicApi.enabled=true", "--set-string", "publicApi.existingSecret=t4-public-api", "--set", "networkPolicy.postgresCIDRs[0]=198.51.100.20/32", "--set-string", "networkPolicy.postgresPorts="}, + "port without destination": {"--set", "publicApi.enabled=true", "--set-string", "publicApi.existingSecret=t4-public-api", "--set", "networkPolicy.postgresPorts[0]=5432"}, + } { + t.Run(name, func(t *testing.T) { helmTemplateMustFail(t, append(enabledValues(), values...)...) }) + } +} func TestUnauthenticatedOMPControllerCannotReadSecrets(t *testing.T) { output := helmTemplate(t, enabledValues()...) controllerRole := documentContaining(t, output, "name: \"release-name-t4-cluster-controller\"") diff --git a/packages/cluster-server/test/config.test.ts b/packages/cluster-server/test/config.test.ts index 611ff078..7ff98f0f 100644 --- a/packages/cluster-server/test/config.test.ts +++ b/packages/cluster-server/test/config.test.ts @@ -110,6 +110,17 @@ describe("cluster server configuration", () => { })).toThrow("exactly one"); }); + it("requires a chart-admitted runtime profile with durable public API configuration", () => { + const durable = { + ...BASE_ENV, + T4_POSTGRES_URL: "postgres://t4:secret@postgres.t4-system.svc:5432/t4", + T4_PUBLIC_API_CREDENTIALS_FILE: "/var/run/secrets/t4-public-api/credentials.json", + T4_PUBLIC_API_RUNTIME_PROFILE: "review-admitted", + }; + expect(clusterServerConfigFromEnv(durable)).toMatchObject({ publicApiRuntimeProfile: "review-admitted" }); + expect(() => clusterServerConfigFromEnv({ ...durable, T4_PUBLIC_API_RUNTIME_PROFILE: undefined })).toThrow("runtime profile"); + expect(() => clusterServerConfigFromEnv({ ...durable, T4_PUBLIC_API_RUNTIME_PROFILE: "-not-admitted" })).toThrow("runtime profile"); + }); }); describe("trusted cluster gateway proxy sources", () => { diff --git a/packages/cluster-server/test/durable-gateway-postgres.test.ts b/packages/cluster-server/test/durable-gateway-postgres.test.ts index 814da375..0b5c3aad 100644 --- a/packages/cluster-server/test/durable-gateway-postgres.test.ts +++ b/packages/cluster-server/test/durable-gateway-postgres.test.ts @@ -8,6 +8,11 @@ interface DurableLedgerOptions { interface PostgresLedgerLike { migrate(): Promise; close(): Promise; + acquireLease(ownerId: string, leaseSeconds?: number): Promise<{ ownerId: string; epoch: bigint }>; + claimNext(lease: { ownerId: string; epoch: bigint }): Promise; + claimIsCurrent(claim: OutboxClaim): Promise; + acknowledge(claim: OutboxClaim): Promise; + recordFailure(claim: OutboxClaim, message: string): Promise; } interface ApiPrincipal { readonly id: string; @@ -23,12 +28,16 @@ interface OutboxMutation { readonly idempotencyToken: string; readonly targetId: string; readonly kind: string; + readonly payload: Readonly>; } interface KubernetesOutboxApplier { apply(mutation: OutboxMutation, fence: { ownerId: string; epoch: bigint }): Promise; } interface OutboxClaim { readonly outboxId: bigint; + readonly commandId: string; + readonly ownerId: string; + readonly ownerEpoch: bigint; } interface DurableOutboxWorkerLike { acquireLease(): Promise<{ ownerId: string; epoch: bigint }>; @@ -37,13 +46,14 @@ interface DurableOutboxWorkerLike { applyClaim(claim: OutboxClaim): Promise; } type LedgerConstructor = new (options: DurableLedgerOptions) => PostgresLedgerLike; -type ApiConstructor = new (options: { ledger: PostgresLedgerLike; authenticator: PrincipalAuthenticator }) => T4PublicApiV1Like; +type ApiConstructor = new (options: { ledger: PostgresLedgerLike; authenticator: PrincipalAuthenticator; allowedOrigins?: readonly string[] }) => T4PublicApiV1Like; type WorkerConstructor = new (options: { ledger: PostgresLedgerLike; ownerId: string; applier: KubernetesOutboxApplier }) => DurableOutboxWorkerLike; const DATABASE_URL = process.env.T4_TEST_POSTGRES_URL; const VERSION = "1"; const OWNER_TOKEN = "opaque-owner-token-with-sufficient-entropy"; const OTHER_TOKEN = "opaque-other-token-with-sufficient-entropy"; +const INVALID_PRINCIPAL_TOKEN = "invalid-principal-token-with-sufficient-entropy"; const ALL_SCOPES = new Set([ "discovery.read", "workspaces.read", @@ -66,6 +76,7 @@ class TestAuthenticator implements PrincipalAuthenticator { readonly #principals: Record = { [OWNER_TOKEN]: { id: "owner@example.com", scopes: ALL_SCOPES }, [OTHER_TOKEN]: { id: "other@example.com", scopes: ALL_SCOPES }, + [INVALID_PRINCIPAL_TOKEN]: { id: "Not A CRD Owner", scopes: ALL_SCOPES }, }; async authenticate(token: string): Promise { return this.#principals[token]; @@ -82,6 +93,22 @@ class ObservableKubernetes implements KubernetesOutboxApplier { this.applied.set(mutation.idempotencyToken, mutation); } } +class PausedCreateKubernetes implements KubernetesOutboxApplier { + readonly resources = new Set(); + readonly createStarted = Promise.withResolvers(); + readonly releaseCreate = Promise.withResolvers(); + async apply(mutation: OutboxMutation): Promise { + if (mutation.kind === "session.create") { + this.createStarted.resolve(); + await this.releaseCreate.promise; + this.resources.add(mutation.targetId); + return; + } + if (mutation.kind === "workspace.create") this.resources.add(mutation.targetId); + if (mutation.kind === "session.cancel" || mutation.kind === "session.delete") this.resources.delete(mutation.targetId); + } +} + function ledgerOptions(overrides: Partial = {}): DurableLedgerOptions { if (!DATABASE_URL) throw new Error("T4_TEST_POSTGRES_URL is required for durable gateway tests"); @@ -100,9 +127,9 @@ async function newLedger(overrides: Partial = {}): Promise return ledger; } -function newApi(ledger: PostgresLedgerLike): T4PublicApiV1Like { +function newApi(ledger: PostgresLedgerLike, allowedOrigins?: readonly string[]): T4PublicApiV1Like { if (!T4PublicApiV1) throw new Error("public T4 API v1 implementation is unavailable"); - return new T4PublicApiV1({ ledger, authenticator: new TestAuthenticator() }); + return new T4PublicApiV1({ ledger, authenticator: new TestAuthenticator(), ...(allowedOrigins ? { allowedOrigins } : {}) }); } function newWorker(ledger: PostgresLedgerLike, ownerId: string, applier: KubernetesOutboxApplier): DurableOutboxWorkerLike { @@ -134,6 +161,14 @@ async function call( return await api.handle(new Request(`https://t4.example.test${path}`, { method, headers: requestHeaders, body })); } +async function waitForDatabase(predicate: () => Promise): Promise { + for (let attempt = 0; attempt < 200; attempt++) { + if (await predicate()) return; + await Bun.sleep(10); + } + throw new Error("timed out waiting for PostgreSQL concurrency fixture"); +} + interface JsonDocument extends Record { id: string; cursor: string; @@ -293,13 +328,13 @@ postgresSuite("PostgreSQL-backed public T4 API v1", () => { }); expect(response.status).toBe(202); } - const first = await call(api, "GET", `/v1/sessions/${session.id}/events?cursor=${encodeURIComponent(snapshot.cursor)}&maxEvents=1&heartbeatSeconds=5`); + const first = await call(api, "GET", `/v1/sessions/${session.id}/events?cursor=${encodeURIComponent(snapshot.cursor)}&maxEvents=1&heartbeatSeconds=5`, { headers: { accept: "text/event-stream" } }); expect(first.status).toBe(200); const firstEvents = eventData(await first.text()); expect(firstEvents).toHaveLength(1); const deliveredCursor = firstEvents[0].cursor as string; const second = await call(api, "GET", `/v1/sessions/${session.id}/events?maxEvents=100&heartbeatSeconds=5`, { - headers: { "last-event-id": deliveredCursor }, + headers: { "last-event-id": deliveredCursor, accept: "text/event-stream" }, }); expect(second.status).toBe(200); const secondEvents = eventData(await second.text()); @@ -312,7 +347,7 @@ postgresSuite("PostgreSQL-backed public T4 API v1", () => { body: { command: `echo ${index}` }, }); } - const expired = await call(api, "GET", `/v1/sessions/${session.id}/events?cursor=${encodeURIComponent(deliveredCursor)}&maxEvents=1&heartbeatSeconds=5`); + const expired = await call(api, "GET", `/v1/sessions/${session.id}/events?cursor=${encodeURIComponent(deliveredCursor)}&maxEvents=1&heartbeatSeconds=5`, { headers: { accept: "text/event-stream" } }); expect(expired.status).toBe(410); const expiredBody = await json(expired); expect(expiredBody.error).toMatchObject({ @@ -392,6 +427,388 @@ postgresSuite("PostgreSQL-backed public T4 API v1", () => { await ledger.close(); }); + it("retries the whole serializable mutation after a real PostgreSQL 40001 exactly once", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + await admin.unsafe(` + DROP SEQUENCE IF EXISTS t4_test_serialization_attempt; + CREATE SEQUENCE t4_test_serialization_attempt; + CREATE OR REPLACE FUNCTION t4_test_serialize_once() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF NEW.idempotency_key = 'serialization-retry-key-01' AND nextval('t4_test_serialization_attempt') = 1 THEN + RAISE SQLSTATE '40001' USING MESSAGE = 'forced serialization rollback'; + END IF; + RETURN NEW; + END $$; + DROP TRIGGER IF EXISTS t4_test_serialize_once_trigger ON t4_outbox; + CREATE TRIGGER t4_test_serialize_once_trigger BEFORE INSERT ON t4_outbox + FOR EACH ROW EXECUTE FUNCTION t4_test_serialize_once(); + `); + const response = await call(api, "POST", "/v1/workspaces", { + key: "serialization-retry-key-01", + body: { name: "retry exactly once" }, + }); + expect(response.status).toBe(202); + + const accepted = await json(response); + const rows = await admin.unsafe(` + SELECT + (SELECT last_value FROM t4_test_serialization_attempt) AS attempts, + (SELECT count(*) FROM t4_commands WHERE idempotency_key = 'serialization-retry-key-01') AS commands, + (SELECT count(*) FROM t4_workspace_intents WHERE workspace_id = '${accepted.id}') AS intents, + (SELECT count(*) FROM t4_outbox WHERE idempotency_key = 'serialization-retry-key-01') AS outbox + `); + expect(rows[0]).toMatchObject({ attempts: 2n, commands: 1n, intents: 1n, outbox: 1n }); + await admin.unsafe("DROP TRIGGER t4_test_serialize_once_trigger ON t4_outbox; DROP FUNCTION t4_test_serialize_once(); DROP SEQUENCE t4_test_serialization_attempt"); + await ledger.close(); + }); + + it("retries the whole serializable mutation after a real PostgreSQL 40P01 exactly once", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + await admin.unsafe(` + DROP SEQUENCE IF EXISTS t4_test_deadlock_attempt; + CREATE SEQUENCE t4_test_deadlock_attempt; + CREATE OR REPLACE FUNCTION t4_test_deadlock_once() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF NEW.idempotency_key = 'deadlock-retry-key-00001' AND nextval('t4_test_deadlock_attempt') = 1 THEN + RAISE SQLSTATE '40P01' USING MESSAGE = 'forced deadlock rollback'; + END IF; + RETURN NEW; + END $$; + CREATE TRIGGER t4_test_deadlock_once_trigger BEFORE INSERT ON t4_outbox + FOR EACH ROW EXECUTE FUNCTION t4_test_deadlock_once(); + `); + const response = await call(api, "POST", "/v1/workspaces", { key: "deadlock-retry-key-00001", body: { name: "retry deadlock once" } }); + expect(response.status).toBe(202); + const rows = await admin`SELECT + (SELECT last_value FROM t4_test_deadlock_attempt) AS attempts, + (SELECT count(*) FROM t4_commands WHERE idempotency_key = 'deadlock-retry-key-00001') AS commands, + (SELECT count(*) FROM t4_outbox WHERE idempotency_key = 'deadlock-retry-key-00001') AS outbox` as Array<{ attempts: bigint; commands: bigint; outbox: bigint }>; + expect(rows[0]).toEqual({ attempts: 2n, commands: 1n, outbox: 1n }); + await admin.unsafe("DROP TRIGGER t4_test_deadlock_once_trigger ON t4_outbox; DROP FUNCTION t4_test_deadlock_once(); DROP SEQUENCE t4_test_deadlock_attempt"); + await ledger.close(); + }); + + it("serializes two first-use requests for the same absent idempotency row", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + await admin.unsafe(` + DROP SEQUENCE IF EXISTS t4_test_absent_key_attempt; + CREATE SEQUENCE t4_test_absent_key_attempt; + CREATE OR REPLACE FUNCTION t4_test_pause_first_absent_key() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF NEW.idempotency_key = 'absent-idempotency-key-01' THEN + PERFORM nextval('t4_test_absent_key_attempt'); + IF currval('t4_test_absent_key_attempt') = 1 THEN PERFORM pg_advisory_xact_lock(741406); END IF; + END IF; + RETURN NEW; + END $$; + DROP TRIGGER IF EXISTS t4_test_pause_absent_key_trigger ON t4_outbox; + CREATE TRIGGER t4_test_pause_absent_key_trigger BEFORE INSERT ON t4_outbox + FOR EACH ROW EXECUTE FUNCTION t4_test_pause_first_absent_key(); + `); + await admin`SELECT pg_advisory_lock(741406)`; + const first = call(api, "POST", "/v1/workspaces", { key: "absent-idempotency-key-01", body: { name: "one durable winner" } }); + await waitForDatabase(async () => (await admin`SELECT last_value FROM t4_test_absent_key_attempt` as Array<{ last_value: bigint }>)[0]?.last_value === 1n); + const second = call(api, "POST", "/v1/workspaces", { key: "absent-idempotency-key-01", body: { name: "one durable winner" } }); + await waitForDatabase(async () => { + const [sequence, active] = await Promise.all([ + admin`SELECT last_value FROM t4_test_absent_key_attempt` as Promise>, + admin`SELECT count(*) AS count FROM pg_stat_activity WHERE datname = current_database() AND state = 'active' AND pid <> pg_backend_pid()` as Promise>, + ]); + return sequence[0]?.last_value === 2n || (active[0]?.count ?? 0n) >= 2n; + }); + await admin`SELECT pg_advisory_unlock(741406)`; + const responses = await Promise.all([first, second]); + expect(responses.map(value => value.status).sort()).toEqual([200, 202]); + expect(await json(responses[0]!)).toEqual(await json(responses[1]!)); + const rows = await admin`SELECT + (SELECT count(*) FROM t4_commands WHERE idempotency_key = 'absent-idempotency-key-01') AS commands, + (SELECT count(*) FROM t4_outbox WHERE idempotency_key = 'absent-idempotency-key-01') AS outbox` as Array<{ commands: bigint; outbox: bigint }>; + expect(rows[0]).toEqual({ commands: 1n, outbox: 1n }); + await admin.unsafe("DROP TRIGGER t4_test_pause_absent_key_trigger ON t4_outbox; DROP FUNCTION t4_test_pause_first_absent_key(); DROP SEQUENCE t4_test_absent_key_attempt"); + await ledger.close(); + }); + + it("replays stored patch, create, and command responses before mutable state checks", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "replay-state-workspace-01", body: { name: "original" } })); + const firstWorkspacePatch = await json(await call(api, "PATCH", `/v1/workspaces/${workspace.id}`, { + key: "replay-workspace-patch-01", ifMatch: "1", body: { name: "first patch" }, + })); + expect((await call(api, "PATCH", `/v1/workspaces/${workspace.id}`, { + key: "replay-workspace-patch-02", ifMatch: "2", body: { name: "second patch" }, + })).status).toBe(200); + const workspaceReplay = await call(api, "PATCH", `/v1/workspaces/${workspace.id}`, { + key: "replay-workspace-patch-01", ifMatch: "1", body: { name: "first patch" }, + }); + expect(workspaceReplay.status).toBe(200); + expect(workspaceReplay.headers.get("idempotency-replayed")).toBe("true"); + expect(await json(workspaceReplay)).toEqual(firstWorkspacePatch); + const workspaceMismatch = await call(api, "PATCH", `/v1/workspaces/${workspace.id}`, { + key: "replay-workspace-patch-01", ifMatch: "1", body: { name: "different" }, + }); + expect((await json(workspaceMismatch)).error.code).toBe("idempotency_conflict"); + + const sessionKey = "replay-session-create-001"; + const session = await json(await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: sessionKey, body: { title: "original" } })); + const firstSessionPatch = await json(await call(api, "PATCH", `/v1/sessions/${session.id}`, { + key: "replay-session-patch-001", ifMatch: "1", body: { title: "first patch" }, + })); + expect((await call(api, "PATCH", `/v1/sessions/${session.id}`, { + key: "replay-session-patch-002", ifMatch: "2", body: { title: "second patch" }, + })).status).toBe(200); + const commandKey = "replay-command-state-001"; + const command = await json(await call(api, "POST", `/v1/sessions/${session.id}/commands`, { key: commandKey, body: { command: "printf original" } })); + expect((await call(api, "POST", `/v1/sessions/${session.id}/cancel`, { key: "replay-cancel-state-001" })).status).toBe(202); + const sessionReplay = await call(api, "PATCH", `/v1/sessions/${session.id}`, { + key: "replay-session-patch-001", ifMatch: "1", body: { title: "first patch" }, + }); + expect(sessionReplay.status).toBe(200); + expect(await json(sessionReplay)).toEqual(firstSessionPatch); + const commandReplay = await call(api, "POST", `/v1/sessions/${session.id}/commands`, { key: commandKey, body: { command: "printf original" } }); + expect(commandReplay.status).toBe(200); + expect(await json(commandReplay)).toEqual(command); + const commandMismatch = await call(api, "POST", `/v1/sessions/${session.id}/commands`, { key: commandKey, body: { command: "printf different" } }); + expect((await json(commandMismatch)).error.code).toBe("idempotency_conflict"); + expect((await call(api, "DELETE", `/v1/workspaces/${workspace.id}`, { key: "replay-delete-parent-001" })).status).toBe(204); + const createReplay = await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: sessionKey, body: { title: "original" } }); + expect(createReplay.status).toBe(200); + expect(await json(createReplay)).toEqual(session); + const createMismatch = await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: sessionKey, body: { title: "different" } }); + expect((await json(createMismatch)).error.code).toBe("idempotency_conflict"); + await ledger.close(); + }); + + it("keeps cancelling and terminal sessions monotonic across distinct requests", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "monotonic-workspace-key-1", body: { name: "monotonic" } })); + const session = await json(await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: "monotonic-session-key-01", body: { title: "monotonic" } })); + const firstCancel = await json(await call(api, "POST", `/v1/sessions/${session.id}/cancel`, { key: "monotonic-cancel-key-001" })); + const before = await admin`SELECT + (SELECT count(*) FROM t4_commands WHERE target_scope = ${`session:${session.id}`}) AS commands, + (SELECT count(*) FROM t4_outbox WHERE target_id = ${session.id}) AS outbox, + (SELECT count(*) FROM t4_events WHERE session_id = ${session.id}) AS events` as Array<{ commands: bigint; outbox: bigint; events: bigint }>; + const repeated = await call(api, "POST", `/v1/sessions/${session.id}/cancel`, { key: "monotonic-cancel-key-002" }); + expect(repeated.status).toBe(202); + expect(await json(repeated)).toEqual(firstCancel); + expect((await call(api, "PATCH", `/v1/sessions/${session.id}`, { key: "monotonic-patch-key-0001", ifMatch: String(firstCancel.revision), body: { title: "forbidden" } })).status).toBe(404); + expect((await call(api, "POST", `/v1/sessions/${session.id}/commands`, { key: "monotonic-command-key-01", body: { command: "forbidden" } })).status).toBe(404); + await admin`UPDATE t4_session_intents SET state = 'cancelled' WHERE session_id = ${session.id}`; + expect((await call(api, "POST", `/v1/sessions/${session.id}/cancel`, { key: "monotonic-cancel-key-003" })).status).toBe(202); + expect((await call(api, "DELETE", `/v1/sessions/${session.id}`, { key: "monotonic-delete-key-001" })).status).toBe(204); + const after = await admin`SELECT + (SELECT count(*) FROM t4_commands WHERE target_scope = ${`session:${session.id}`}) AS commands, + (SELECT count(*) FROM t4_outbox WHERE target_id = ${session.id}) AS outbox, + (SELECT count(*) FROM t4_events WHERE session_id = ${session.id}) AS events, + (SELECT revision FROM t4_session_intents WHERE session_id = ${session.id}) AS revision` as Array<{ commands: bigint; outbox: bigint; events: bigint; revision: bigint }>; + expect(after[0]).toMatchObject({ ...before[0], revision: BigInt(firstCancel.revision) }); + await ledger.close(); + }); + + it("renews, excludes, expires, and fences PostgreSQL leases", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const first = await ledger.acquireLease("lease-owner-a", 30); + const renewed = await ledger.acquireLease("lease-owner-a", 30); + expect(renewed).toEqual(first); + await expect(ledger.acquireLease("lease-owner-b", 30)).rejects.toThrow("lease"); + await call(api, "POST", "/v1/workspaces", { key: "lease-fencing-workspace-1", body: { name: "lease" } }); + const claim = await ledger.claimNext(first); + expect(claim).toBeDefined(); + await admin`UPDATE t4_owner_leases SET expires_at = clock_timestamp() - interval '1 second' WHERE lease_name = 'gateway-outbox'`; + expect(await ledger.claimIsCurrent(claim!)).toBe(false); + expect(await ledger.acknowledge(claim!)).toBe(false); + expect(await ledger.recordFailure(claim!, "expired")).toBe(false); + const takeover = await ledger.acquireLease("lease-owner-b", 30); + expect(takeover.epoch).toBe(first.epoch + 1n); + await ledger.close(); + }); + + it("allows exact Unicode code-point bounds and denies browser Origin by default", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const astral = "😀".repeat(128); + const exact = await call(api, "POST", "/v1/workspaces", { + key: "unicode-codepoint-bound-01", body: { name: astral, labels: { emoji: astral } }, + }); + expect(exact.status).toBe(202); + expect((await call(api, "POST", "/v1/workspaces", { + key: "unicode-codepoint-bound-02", body: { name: `${astral}😀` }, + })).status).toBe(422); + const browser = await call(api, "GET", "/v1", { headers: { origin: "https://browser.example.test" } }); + expect(browser.status).toBe(400); + expect((await json(browser)).error.code).toBe("invalid_origin"); + expect((await call(api, "GET", "/v1")).status).toBe(200); + const allowlisted = newApi(ledger, ["https://browser.example.test"]); + expect((await call(allowlisted, "GET", "/v1", { headers: { origin: "https://browser.example.test" } })).status).toBe(200); + expect((await call(allowlisted, "GET", "/v1", { headers: { origin: "https://other.example.test" } })).status).toBe(400); + await ledger.close(); + }); + + it("converges a paused stale session create followed by cancel and expired-lease takeover", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const kubernetes = new PausedCreateKubernetes(); + const stale = newWorker(ledger, "paused-owner", kubernetes); + await stale.acquireLease(); + const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "paused-race-workspace-01", body: { name: "paused" } })); + expect(await stale.drain()).toBe(1); + const session = await json(await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: "paused-race-session-001", body: { title: "paused" } })); + const createClaim = await stale.claimNext(); + expect(createClaim).toBeDefined(); + const staleApply = stale.applyClaim(createClaim!); + await kubernetes.createStarted.promise; + expect((await call(api, "POST", `/v1/sessions/${session.id}/cancel`, { key: "paused-race-cancel-0001" })).status).toBe(202); + await admin`UPDATE t4_owner_leases SET expires_at = clock_timestamp() - interval '1 second' WHERE lease_name = 'gateway-outbox'`; + const current = newWorker(ledger, "takeover-owner", kubernetes); + const currentLease = await current.acquireLease(); + expect(currentLease.epoch).toBeGreaterThan(createClaim!.ownerEpoch); + kubernetes.releaseCreate.resolve(); + expect(await staleApply).toBe("fenced"); + expect(kubernetes.resources.has(session.id)).toBe(true); + expect(await current.drain()).toBeGreaterThan(0); + expect(kubernetes.resources.has(session.id)).toBe(false); + await ledger.close(); + }); + + it("reads snapshot entries and cursor from one database snapshot", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "atomic-snapshot-workspace", body: { name: "atomic snapshot" } })); + const session = await json(await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: "atomic-snapshot-session-1", body: { title: "atomic snapshot" } })); + await call(api, "POST", `/v1/sessions/${session.id}/commands`, { key: "atomic-snapshot-command-1", body: { command: "before snapshot" } }); + await admin`SELECT pg_advisory_lock(741407)`; + await admin.unsafe(` + ALTER TABLE t4_event_retention RENAME TO t4_event_retention_base; + CREATE FUNCTION t4_test_pause_snapshot(value bigint) RETURNS bigint LANGUAGE plpgsql VOLATILE AS $$ + BEGIN PERFORM pg_advisory_xact_lock(741407); RETURN value; END $$; + CREATE VIEW t4_event_retention AS + SELECT principal_id, session_id, first_retained_sequence, t4_test_pause_snapshot(latest_sequence) AS latest_sequence, updated_at + FROM t4_event_retention_base; + `); + const pendingSnapshot = call(api, "GET", `/v1/sessions/${session.id}/snapshot`); + await waitForDatabase(async () => { + const rows = await admin`SELECT count(*) AS count FROM pg_stat_activity WHERE wait_event_type = 'Advisory' AND query LIKE '%t4_event_retention%'` as Array<{ count: bigint }>; + return (rows[0]?.count ?? 0n) >= 1n; + }); + const concurrent = new SQL(DATABASE_URL!, { max: 1, bigint: true }); + const concurrentCommandId = "cmd_concurrent_snapshot_boundary"; + await concurrent.begin(async transaction => { + const entries = await transaction<{ next: bigint }[]>`SELECT COALESCE(MAX(entry_sequence), -1) + 1 AS next FROM t4_snapshot_entries WHERE principal_id = 'owner@example.com' AND session_id = ${session.id}`; + await transaction`INSERT INTO t4_snapshot_entries (principal_id, session_id, entry_sequence, kind, text_value) VALUES ('owner@example.com', ${session.id}, ${entries[0]?.next ?? 0n}, 'input', 'concurrent snapshot command')`; + const events = await transaction<{ sequence: bigint }[]>`INSERT INTO t4_events (principal_id, session_id, event_type, payload, owner_epoch) VALUES ('owner@example.com', ${session.id}, 'command', ${{ commandId: concurrentCommandId, state: "accepted" }}, 0) RETURNING sequence`; + await transaction`UPDATE t4_event_retention_base SET latest_sequence = ${events[0]!.sequence}, updated_at = clock_timestamp() WHERE principal_id = 'owner@example.com' AND session_id = ${session.id}`; + }); + await concurrent.close(); + await admin`SELECT pg_advisory_unlock(741407)`; + const snapshotResponse = await pendingSnapshot; + expect(snapshotResponse.status).toBe(200); + const snapshot = await json(snapshotResponse); + await admin.unsafe("DROP VIEW t4_event_retention; ALTER TABLE t4_event_retention_base RENAME TO t4_event_retention; DROP FUNCTION t4_test_pause_snapshot(bigint)"); + const afterBoundary = await call(api, "GET", `/v1/sessions/${session.id}/events?cursor=${encodeURIComponent(snapshot.cursor)}`, { headers: { accept: "text/event-stream" } }); + expect(afterBoundary.status).toBe(200); + expect(eventData(await afterBoundary.text()).map(value => value.commandId)).toContain(concurrentCommandId); + await ledger.close(); + }); + + it("reads retention floor, events, and delivered cursor atomically across pruning", async () => { + const ledger = await newLedger({ eventRetention: 100 }); + const api = newApi(ledger); + const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "atomic-window-workspace-1", body: { name: "atomic window" } })); + const session = await json(await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: "atomic-window-session-001", body: { title: "atomic window" } })); + const snapshot = await json(await call(api, "GET", `/v1/sessions/${session.id}/snapshot`)); + const command = await json(await call(api, "POST", `/v1/sessions/${session.id}/commands`, { key: "atomic-window-command-001", body: { command: "must survive concurrent prune" } })); + const target = await admin`SELECT sequence FROM t4_events WHERE principal_id = 'owner@example.com' AND session_id = ${session.id} AND payload->>'commandId' = ${command.commandId}` as Array<{ sequence: bigint }>; + expect(target[0]).toBeDefined(); + await admin`SELECT pg_advisory_lock(741408)`; + await admin.unsafe(` + ALTER TABLE t4_events RENAME TO t4_events_base; + CREATE FUNCTION t4_test_pause_event(value bigint) RETURNS bigint LANGUAGE plpgsql VOLATILE AS $$ + BEGIN PERFORM pg_advisory_xact_lock(741408); RETURN value; END $$; + CREATE VIEW t4_events AS + SELECT t4_test_pause_event(sequence) AS sequence, principal_id, session_id, event_type, payload, owner_epoch, created_at + FROM t4_events_base; + `); + const pendingWindow = call(api, "GET", `/v1/sessions/${session.id}/events?cursor=${encodeURIComponent(snapshot.cursor)}&maxEvents=1`, { headers: { accept: "text/event-stream" } }); + await waitForDatabase(async () => { + const rows = await admin`SELECT count(*) AS count FROM pg_stat_activity WHERE wait_event_type = 'Advisory' AND query LIKE '%t4_events%'` as Array<{ count: bigint }>; + return (rows[0]?.count ?? 0n) >= 1n; + }); + const concurrent = new SQL(DATABASE_URL!, { max: 1, bigint: true }); + await concurrent.begin(async transaction => { + await transaction`DELETE FROM t4_events_base WHERE sequence = ${target[0]!.sequence}`; + await transaction`UPDATE t4_event_retention SET first_retained_sequence = ${target[0]!.sequence + 1n} WHERE principal_id = 'owner@example.com' AND session_id = ${session.id}`; + }); + await concurrent.close(); + await admin`SELECT pg_advisory_unlock(741408)`; + const response = await pendingWindow; + expect(response.status).toBe(200); + await admin.unsafe("DROP VIEW t4_events; ALTER TABLE t4_events_base RENAME TO t4_events; DROP FUNCTION t4_test_pause_event(bigint)"); + expect(eventData(await response.text()).map(value => value.commandId)).toContain(command.commandId); + await ledger.close(); + }); + + it("stops a chunked request body as soon as the byte limit is crossed", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + let pulls = 0; + const body = new ReadableStream({ + pull(controller) { + pulls++; + if (pulls <= 2) controller.enqueue(new Uint8Array(600_000)); + else controller.error(new Error("body reader requested data after the limit")); + }, + }); + const response = await api.handle(new Request("https://t4.example.test/v1/workspaces", { + method: "POST", + headers: headers(OWNER_TOKEN, { "content-type": "application/json", "idempotency-key": "streaming-body-limit-key" }), + body, + duplex: "half", + } as RequestInit & { duplex: "half" })); + expect(response.status).toBe(400); + expect((await json(response)).error.message).toContain("maximum size"); + expect(pulls).toBe(2); + await ledger.close(); + }); + + it("requires an acceptable SSE media range and maps watch database failures to 503", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "sse-accept-workspace-01", body: { name: "sse accept" } })); + const session = await json(await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: "sse-accept-session-001", body: { title: "sse accept" } })); + for (const accept of [undefined, "application/json", "text/plain;q=1, text/event-stream;q=0"]) { + const response = await call(api, "GET", `/v1/sessions/${session.id}/events`, { ...(accept ? { headers: { accept } } : {}) }); + expect(response.status).toBe(406); + } + expect((await call(api, "GET", `/v1/sessions/${session.id}/events`, { headers: { accept: "text/event-stream, application/json;q=0.5" } })).status).toBe(200); + expect((await call(api, "GET", `/v1/sessions/${session.id}/events?cursor=invalid`, { headers: { accept: "text/event-stream" } })).status).toBe(422); + await admin`ALTER TABLE t4_event_retention RENAME TO t4_event_retention_unavailable`; + const unavailable = await call(api, "GET", `/v1/sessions/${session.id}/events`, { headers: { accept: "text/event-stream" } }); + expect(unavailable.status).toBe(503); + expect((await json(unavailable)).error).toMatchObject({ code: "unavailable", retryable: true }); + await admin`ALTER TABLE t4_event_retention_unavailable RENAME TO t4_event_retention`; + await ledger.close(); + }); + + it("rejects authenticated principals that cannot be projected into CRD spec.owner", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const before = await admin`SELECT count(*) AS count FROM t4_commands` as Array<{ count: bigint }>; + const response = await call(api, "POST", "/v1/workspaces", { + token: INVALID_PRINCIPAL_TOKEN, + key: "invalid-principal-owner-01", + body: { name: "must not commit" }, + }); + expect(response.status).toBe(401); + const after = await admin`SELECT count(*) AS count FROM t4_commands` as Array<{ count: bigint }>; + expect(after).toEqual(before); + await ledger.close(); + }); + it("rejects malformed, oversized, unsupported-version input before ledger or cluster mutation", async () => { const ledger = await newLedger(); const api = newApi(ledger); diff --git a/packages/cluster-server/test/kubernetes-client.test.ts b/packages/cluster-server/test/kubernetes-client.test.ts index 7adc0c65..addeabf6 100644 --- a/packages/cluster-server/test/kubernetes-client.test.ts +++ b/packages/cluster-server/test/kubernetes-client.test.ts @@ -9,6 +9,8 @@ import { KubernetesTokenReviewer, semanticResourceHash, } from "../src/kubernetes-client.ts"; +import { PublicKubernetesOutboxApplier } from "../src/public-kubernetes-applier.ts"; +import type { OutboxMutation } from "../src/outbox-worker.ts"; const PRINCIPAL = "owner@example.com"; @@ -216,6 +218,208 @@ describe("namespaced Kubernetes client", () => { }); }); +describe("durable public Kubernetes applier", () => { + const fence = { ownerId: "worker-a", epoch: 7n }; + const workspaceMutation: OutboxMutation = { + idempotencyToken: "outbox:41", + commandId: "command-41", + principalId: PRINCIPAL, + kind: "workspace.patch", + targetId: "workspace-one", + targetRevision: 2n, + payload: { id: "workspace-one", name: "patched", revision: 2 }, + }; + const priorAnnotations = { + "cluster.t4.dev/ledger-command-id": "prior-command", + "cluster.t4.dev/ledger-outbox-token": "outbox:prior", + "cluster.t4.dev/ledger-owner": "prior-worker", + "cluster.t4.dev/ledger-owner-epoch": "6", + "cluster.t4.dev/ledger-semantic-hash": "sha256:prior", + }; + + it("carries observed resourceVersion and owner epoch on every existing-resource PATCH", async () => { + const values = recordingFetch([{ + apiVersion: "cluster.t4.dev/v1alpha1", + kind: "T4Workspace", + metadata: { name: "workspace-one", resourceVersion: "41", annotations: priorAnnotations }, + spec: { hostRef: "primary", owner: PRINCIPAL, displayName: "old" }, + }, {}]); + const applier = new PublicKubernetesOutboxApplier({ + client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch: values.fetch }), + hostRef: "primary", + }); + await applier.apply(workspaceMutation, fence); + const patch = JSON.parse(String(values.requests[1]?.init?.body)); + expect(patch.metadata).toMatchObject({ + resourceVersion: "41", + annotations: { + "cluster.t4.dev/ledger-owner": "worker-a", + "cluster.t4.dev/ledger-owner-epoch": "7", + }, + }); + }); + + it("lets Kubernetes reject a paused stale resourceVersion after a current-owner mutation", async () => { + let actualVersion = "51"; + const patchStarted = Promise.withResolvers(); + const releasePatch = Promise.withResolvers(); + const fetch = (async (_input: string | URL | Request, init?: RequestInit) => { + if (init?.method !== "PATCH") return Response.json({ + apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Workspace", + metadata: { name: "workspace-one", resourceVersion: "51", annotations: priorAnnotations }, + spec: { hostRef: "primary", owner: PRINCIPAL }, + }); + patchStarted.resolve(); + await releasePatch.promise; + const body = JSON.parse(String(init.body)); + return body.metadata?.resourceVersion === actualVersion + ? Response.json({ metadata: { resourceVersion: "52" } }) + : Response.json({ reason: "Conflict" }, { status: 409 }); + }) as typeof globalThis.fetch; + const applier = new PublicKubernetesOutboxApplier({ + client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch }), + hostRef: "primary", + }); + const stale = applier.apply(workspaceMutation, fence); + await patchStarted.promise; + actualVersion = "52"; + releasePatch.resolve(); + await expect(stale).rejects.toMatchObject({ status: 409 }); + }); + + it("validates full kind, host, principal relation, and ledger identity on create-409 replay", async () => { + const createMutation: OutboxMutation = { + ...workspaceMutation, + kind: "workspace.create", + payload: { id: "workspace-one", name: "created", revision: 1 }, + }; + const requiredAnnotations = { + "cluster.t4.dev/ledger-command-id": createMutation.commandId, + "cluster.t4.dev/ledger-outbox-token": createMutation.idempotencyToken, + "cluster.t4.dev/ledger-owner": fence.ownerId, + "cluster.t4.dev/ledger-owner-epoch": fence.epoch.toString(), + "cluster.t4.dev/ledger-semantic-hash": semanticResourceHash(createMutation.payload), + }; + const exactExisting = { + apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Workspace", + metadata: { name: "workspace-one", resourceVersion: "9", annotations: requiredAnnotations }, + spec: { hostRef: "primary", owner: PRINCIPAL, displayName: "created" }, + }; + const exact = conflictFetch(exactExisting); + await new PublicKubernetesOutboxApplier({ + client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch: exact.fetch }), + hostRef: "primary", + }).apply(createMutation, fence); + + for (const conflicting of [ + { ...exactExisting, kind: "T4Session" }, + { ...exactExisting, spec: { ...exactExisting.spec, hostRef: "other" } }, + { ...exactExisting, spec: { ...exactExisting.spec, owner: "other@example.com" } }, + { ...exactExisting, metadata: { ...exactExisting.metadata, annotations: { ...requiredAnnotations, "cluster.t4.dev/ledger-command-id": "copied-wrong-command" } } }, + ]) { + const values = conflictFetch(conflicting); + const applier = new PublicKubernetesOutboxApplier({ + client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch: values.fetch }), + hostRef: "primary", + }); + await expect(applier.apply(createMutation, fence)).rejects.toThrow("conflict"); + } + }); + + it("rejects patching existing resources with missing ledger identity or the wrong API kind", async () => { + for (const existing of [ + { apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Workspace", metadata: { name: "workspace-one", resourceVersion: "8", annotations: { "cluster.t4.dev/ledger-owner-epoch": "6" } }, spec: { hostRef: "primary", owner: PRINCIPAL } }, + { apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Session", metadata: { name: "workspace-one", resourceVersion: "8", annotations: priorAnnotations }, spec: { hostRef: "primary", owner: PRINCIPAL } }, + ]) { + const values = recordingFetch([existing]); + const applier = new PublicKubernetesOutboxApplier({ + client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch: values.fetch }), + hostRef: "primary", + }); + await expect(applier.apply(workspaceMutation, fence)).rejects.toThrow("identity"); + expect(values.requests).toHaveLength(1); + } + }); + it("uses the configured host-admitted runtime profile for durable session creation", async () => { + const values = recordingFetch([{}]); + const options = { + client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch: values.fetch }), + hostRef: "primary", + runtimeProfile: "review-admitted", + }; + const applier = new PublicKubernetesOutboxApplier(options); + await applier.apply({ + ...workspaceMutation, + kind: "session.create", + targetId: "session-one", + payload: { id: "session-one", workspaceId: "workspace-one", title: "session", revision: 1 }, + }, fence); + const created = JSON.parse(String(values.requests[0]?.init?.body)); + expect(created.spec.runtimeProfile).toBe("review-admitted"); + }); + + it("projects only a bounded durable command pointer and keeps command payload in PostgreSQL", async () => { + const secretCommand = `printf ${"secret-payload".repeat(8_192)}`; + const values = recordingFetch([{ + apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Session", + metadata: { name: "session-one", resourceVersion: "61", annotations: { "cluster.t4.dev/ledger-owner-epoch": "6" } }, + spec: { hostRef: "primary", workspaceRef: "workspace-one" }, + }, { + apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Workspace", + metadata: { name: "workspace-one", resourceVersion: "60" }, + spec: { hostRef: "primary", owner: PRINCIPAL }, + }, {}]); + const applier = new PublicKubernetesOutboxApplier({ + client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch: values.fetch }), + hostRef: "primary", + }); + const commandMutation: OutboxMutation = { + ...workspaceMutation, + kind: "command.submit", + targetId: "session-one", + payload: { sessionId: "session-one", command: secretCommand, metadata: { secret: "never-project" } }, + }; + await applier.apply(commandMutation, fence); + const patchRequest = values.requests.at(-1)!; + const patch = JSON.parse(String(patchRequest.init?.body)); + expect(patch.metadata.annotations["cluster.t4.dev/pending-command-id"]).toBe(commandMutation.commandId); + expect(patch.metadata.annotations["cluster.t4.dev/pending-command-epoch"]).toBe(fence.epoch.toString()); + expect(JSON.stringify(patch)).not.toContain("secret-payload"); + expect(JSON.stringify(patch)).not.toContain("never-project"); + expect(new TextEncoder().encode(JSON.stringify(patch.metadata)).byteLength).toBeLessThan(4_096); + }); + + it("verifies a session kind and its owning workspace relation before patching", async () => { + const values = recordingFetch([{ + apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Session", + metadata: { name: "session-one", resourceVersion: "71", annotations: { + "cluster.t4.dev/ledger-command-id": "prior-command", + "cluster.t4.dev/ledger-outbox-token": "outbox:prior", + "cluster.t4.dev/ledger-owner": "prior-worker", + "cluster.t4.dev/ledger-owner-epoch": "6", + "cluster.t4.dev/ledger-semantic-hash": "sha256:prior", + } }, + spec: { hostRef: "primary", workspaceRef: "workspace-one" }, + }, { + apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Workspace", + metadata: { name: "workspace-one", resourceVersion: "70" }, + spec: { hostRef: "primary", owner: PRINCIPAL }, + }, {}]); + const applier = new PublicKubernetesOutboxApplier({ + client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch: values.fetch }), + hostRef: "primary", + }); + await applier.apply({ + ...workspaceMutation, + kind: "session.patch", + targetId: "session-one", + payload: { id: "session-one", workspaceId: "workspace-one", title: "patched", revision: 2 }, + }, fence); + expect(values.requests.map(value => value.init?.method ?? "GET")).toEqual(["GET", "GET", "PATCH"]); + expect(values.requests[1]?.url).toContain("/t4workspaces/workspace-one"); + }); +}); + describe("Kubernetes projected identity review", () => { it("submits the presented bearer with the fixed audience and requires the exact server ServiceAccount", async () => { const directory = await mkdtemp(join(tmpdir(), "t4-token-review-")); From dc1520302fd5fd46fe4e703419f6f636e9938b3c Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 03:17:00 +0000 Subject: [PATCH 122/131] fix(cluster): harden durable gateway ledger --- .../charts/t4-cluster/templates/_helpers.tpl | 22 ++ .../t4-cluster/templates/deployments.yaml | 27 +- .../t4-cluster/templates/networkpolicies.yaml | 26 +- deploy/charts/t4-cluster/templates/rbac.yaml | 2 +- deploy/charts/t4-cluster/values.schema.json | 49 +++- deploy/charts/t4-cluster/values.yaml | 13 +- .../charttests/chart_contract_test.go | 181 ++++++++++--- packages/cluster-server/src/config.ts | 8 +- packages/cluster-server/src/ledger.ts | 163 ++++++++---- packages/cluster-server/src/main.ts | 53 ++-- packages/cluster-server/src/outbox-worker.ts | 13 +- packages/cluster-server/src/public-api-v1.ts | 102 +++++++- .../src/public-kubernetes-applier.ts | 238 ++++++++++++++---- .../test/durable-gateway-postgres.test.ts | 112 +++++---- .../test/kubernetes-client.test.ts | 28 ++- 15 files changed, 781 insertions(+), 256 deletions(-) diff --git a/deploy/charts/t4-cluster/templates/_helpers.tpl b/deploy/charts/t4-cluster/templates/_helpers.tpl index 60450013..207026e8 100644 --- a/deploy/charts/t4-cluster/templates/_helpers.tpl +++ b/deploy/charts/t4-cluster/templates/_helpers.tpl @@ -33,3 +33,25 @@ app.kubernetes.io/instance: {{ .Release.Name | quote }} {{- define "t4-cluster.image" -}} {{- printf "%s@%s" .repository .digest -}} {{- end -}} + +{{- define "t4-cluster.validatePublicApi" -}} +{{- if .Values.publicApi.enabled -}} +{{- $existingSecret := required "publicApi.existingSecret is required when publicApi.enabled is true" .Values.publicApi.existingSecret -}} +{{- $postgresURLKey := required "publicApi.postgresURLKey is required when publicApi.enabled is true" .Values.publicApi.postgresURLKey -}} +{{- $credentialsKey := required "publicApi.credentialsKey is required when publicApi.enabled is true" .Values.publicApi.credentialsKey -}} +{{- if not .Values.networkPolicy.enabled -}} +{{- fail "networkPolicy.enabled must be true when publicApi.enabled is true" -}} +{{- end -}} +{{- if empty .Values.networkPolicy.postgresPorts -}} +{{- fail "networkPolicy.postgresPorts must contain at least one port when publicApi.enabled is true" -}} +{{- end -}} +{{- $hasPostgresNamespaceSelector := not (empty .Values.networkPolicy.postgres.namespaceSelector) -}} +{{- $hasPostgresPodSelector := not (empty .Values.networkPolicy.postgres.podSelector) -}} +{{- if ne $hasPostgresNamespaceSelector $hasPostgresPodSelector -}} +{{- fail "networkPolicy.postgres namespaceSelector and podSelector must both be configured or both be empty" -}} +{{- end -}} +{{- if and (empty .Values.networkPolicy.postgresCIDRs) (not $hasPostgresNamespaceSelector) -}} +{{- fail "publicApi.enabled requires a PostgreSQL CIDR or complete selector destination" -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/deploy/charts/t4-cluster/templates/deployments.yaml b/deploy/charts/t4-cluster/templates/deployments.yaml index b581ed3d..2f1155f8 100644 --- a/deploy/charts/t4-cluster/templates/deployments.yaml +++ b/deploy/charts/t4-cluster/templates/deployments.yaml @@ -1,4 +1,5 @@ {{- if .Values.enabled }} +{{- include "t4-cluster.validatePublicApi" . -}} {{- if eq .Values.session.omp.modelsKey .Values.session.omp.settingsKey }} {{- fail "session.omp.modelsKey and session.omp.settingsKey must be different" }} {{- end }} @@ -258,11 +259,13 @@ spec: valueFrom: {fieldRef: {fieldPath: metadata.name}} - name: POD_UID valueFrom: {fieldRef: {fieldPath: metadata.uid}} - {{- if .Values.durableGateway.existingSecret }} + {{- if .Values.publicApi.enabled }} - name: T4_POSTGRES_URL_FILE - value: /var/run/secrets/t4-durable-gateway/postgres-url + value: /var/run/secrets/t4-public-api/postgres-url - name: T4_PUBLIC_API_CREDENTIALS_FILE - value: /var/run/secrets/t4-durable-gateway/public-api-credentials.json + value: /var/run/secrets/t4-public-api/credentials.json + - name: T4_PUBLIC_API_RUNTIME_PROFILE + value: {{ first .Values.clusterHost.runtimeProfiles | quote }} {{- end }} {{- if .Values.woodpecker.existingSecret }} - name: T4_WOODPECKER_TOKEN @@ -330,9 +333,9 @@ spec: - name: kubernetes-api-access mountPath: /var/run/secrets/kubernetes.io/serviceaccount readOnly: true - {{- if .Values.durableGateway.existingSecret }} - - name: durable-gateway - mountPath: /var/run/secrets/t4-durable-gateway + {{- if .Values.publicApi.enabled }} + - name: public-api + mountPath: /var/run/secrets/t4-public-api readOnly: true {{- end }} {{- if .Values.woodpecker.serviceAccountAudience }} @@ -362,16 +365,16 @@ spec: items: - key: ca.crt path: ca.crt - {{- if .Values.durableGateway.existingSecret }} - - name: durable-gateway + {{- if .Values.publicApi.enabled }} + - name: public-api secret: - secretName: {{ .Values.durableGateway.existingSecret | quote }} + secretName: {{ .Values.publicApi.existingSecret | quote }} defaultMode: 0440 items: - - key: {{ .Values.durableGateway.postgresURLKey | quote }} + - key: {{ .Values.publicApi.postgresURLKey | quote }} path: postgres-url - - key: {{ .Values.durableGateway.publicAPICredentialsKey | quote }} - path: public-api-credentials.json + - key: {{ .Values.publicApi.credentialsKey | quote }} + path: credentials.json {{- end }} {{- if .Values.woodpecker.serviceAccountAudience }} - name: ci-identity diff --git a/deploy/charts/t4-cluster/templates/networkpolicies.yaml b/deploy/charts/t4-cluster/templates/networkpolicies.yaml index 99837f80..66e9a0b3 100644 --- a/deploy/charts/t4-cluster/templates/networkpolicies.yaml +++ b/deploy/charts/t4-cluster/templates/networkpolicies.yaml @@ -1,4 +1,6 @@ -{{- if and .Values.enabled .Values.networkPolicy.enabled }} +{{- if .Values.enabled }} +{{- include "t4-cluster.validatePublicApi" . -}} +{{- if .Values.networkPolicy.enabled }} apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: @@ -101,6 +103,27 @@ spec: {{- end }} {{- end }} {{- end }} + {{- if and .Values.publicApi.enabled .Values.networkPolicy.postgresPorts }} + {{- range .Values.networkPolicy.postgresCIDRs }} + - to: + - ipBlock: {cidr: {{ . | quote }}} + ports: + {{- range $.Values.networkPolicy.postgresPorts }} + - {protocol: TCP, port: {{ . }}} + {{- end }} + {{- end }} + {{- if and (not (empty .Values.networkPolicy.postgres.namespaceSelector)) (not (empty .Values.networkPolicy.postgres.podSelector)) }} + - to: + - namespaceSelector: + {{- toYaml .Values.networkPolicy.postgres.namespaceSelector | nindent 12 }} + podSelector: + {{- toYaml .Values.networkPolicy.postgres.podSelector | nindent 12 }} + ports: + {{- range .Values.networkPolicy.postgresPorts }} + - {protocol: TCP, port: {{ . }}} + {{- end }} + {{- end }} + {{- end }} --- apiVersion: networking.k8s.io/v1 kind: NetworkPolicy @@ -205,3 +228,4 @@ spec: ingress: [] {{- end }} {{- end }} +{{- end }} diff --git a/deploy/charts/t4-cluster/templates/rbac.yaml b/deploy/charts/t4-cluster/templates/rbac.yaml index 9c8af47f..2092d5fc 100644 --- a/deploy/charts/t4-cluster/templates/rbac.yaml +++ b/deploy/charts/t4-cluster/templates/rbac.yaml @@ -41,7 +41,7 @@ rules: verbs: [get, list, watch] - apiGroups: [cluster.t4.dev] resources: [t4workspaces, t4sessions] - verbs: [get, list, watch, create, delete] + verbs: [get, list, watch, create, patch, delete] - apiGroups: [''] resources: [pods, services, endpoints] verbs: [get, list, watch] diff --git a/deploy/charts/t4-cluster/values.schema.json b/deploy/charts/t4-cluster/values.schema.json index ca59d81e..f625fb5b 100644 --- a/deploy/charts/t4-cluster/values.schema.json +++ b/deploy/charts/t4-cluster/values.schema.json @@ -10,7 +10,7 @@ "server", "session", "storage", - "durableGateway", + "publicApi", "woodpecker", "ingress", "networkPolicy", @@ -130,13 +130,14 @@ } } }, - "durableGateway": { + "publicApi": { "type": "object", - "required": ["existingSecret", "postgresURLKey", "publicAPICredentialsKey"], + "required": ["enabled", "existingSecret", "postgresURLKey", "credentialsKey"], "properties": { + "enabled": { "type": "boolean" }, "existingSecret": { "$ref": "#/definitions/optionalDNSSubdomain" }, "postgresURLKey": { "$ref": "#/definitions/configKey" }, - "publicAPICredentialsKey": { "$ref": "#/definitions/configKey" } + "credentialsKey": { "$ref": "#/definitions/configKey" } }, "additionalProperties": false }, @@ -183,7 +184,7 @@ }, "networkPolicy": { "type": "object", - "required": ["enabled", "kubernetesApiCIDRs", "modelRouteCIDRs", "modelRoutePorts", "modelRoute", "ciProviderCIDRs", "ciProviderPorts", "ciProvider", "dns", "gatewayIngress", "observability"], + "required": ["enabled", "kubernetesApiCIDRs", "modelRouteCIDRs", "modelRoutePorts", "modelRoute", "ciProviderCIDRs", "ciProviderPorts", "ciProvider", "postgresCIDRs", "postgresPorts", "postgres", "dns", "gatewayIngress", "observability"], "properties": { "enabled": { "type": "boolean" }, "kubernetesApiCIDRs": { "$ref": "#/definitions/cidrList" }, @@ -193,6 +194,9 @@ "ciProviderCIDRs": { "$ref": "#/definitions/cidrList" }, "ciProviderPorts": { "$ref": "#/definitions/tcpPortList" }, "ciProvider": { "$ref": "#/definitions/pairedSelectors" }, + "postgresCIDRs": { "$ref": "#/definitions/cidrList" }, + "postgresPorts": { "$ref": "#/definitions/tcpPortList" }, + "postgres": { "$ref": "#/definitions/pairedSelectors" }, "dns": { "type": "object", "required": ["namespaceSelector", "podSelector"], @@ -408,6 +412,41 @@ } } }, + { + "if": { + "properties": { + "publicApi": { "properties": { "enabled": { "const": true } } } + } + }, + "then": { + "properties": { + "publicApi": { + "properties": { + "existingSecret": { "minLength": 1 } + } + }, + "networkPolicy": { + "properties": { + "enabled": { "const": true }, + "postgresPorts": { "minItems": 1 } + }, + "anyOf": [ + { "properties": { "postgresCIDRs": { "minItems": 1 } } }, + { + "properties": { + "postgres": { + "properties": { + "namespaceSelector": { "minProperties": 1 }, + "podSelector": { "minProperties": 1 } + } + } + } + } + ] + } + } + } + }, { "if": { "properties": { "ingress": { "properties": { "enabled": { "const": true } } } } }, "then": { diff --git a/deploy/charts/t4-cluster/values.yaml b/deploy/charts/t4-cluster/values.yaml index e1f46abf..83b22b4f 100644 --- a/deploy/charts/t4-cluster/values.yaml +++ b/deploy/charts/t4-cluster/values.yaml @@ -82,11 +82,12 @@ storage: # Required when enabled. The administrator creates this backend-neutral RWX StorageClass. adminRWXStorageClass: "" -durableGateway: - # Administrator-owned Secret; the chart only projects narrowly named files. +publicApi: + enabled: false + # Administrator-owned Secret; the chart only projects these two narrowly named files. existingSecret: "" postgresURLKey: postgres-url - publicAPICredentialsKey: public-api-credentials.json + credentialsKey: credentials.json woodpecker: existingSecret: "" @@ -127,6 +128,12 @@ networkPolicy: # Optional in-cluster CI endpoint identity. Set both selectors or neither. namespaceSelector: {} podSelector: {} + postgresCIDRs: [] + postgresPorts: [] + postgres: + # PostgreSQL may be selected by CIDR and/or in-cluster identity. Set both selectors or neither. + namespaceSelector: {} + podSelector: {} dns: namespaceSelector: matchLabels: diff --git a/packages/cluster-operator/charttests/chart_contract_test.go b/packages/cluster-operator/charttests/chart_contract_test.go index 0f5193eb..076aa664 100644 --- a/packages/cluster-operator/charttests/chart_contract_test.go +++ b/packages/cluster-operator/charttests/chart_contract_test.go @@ -143,33 +143,43 @@ func TestEachDeploymentUsesZeroUnavailableAndConfiguredAPIAudience(t *testing.T) func TestValuesSchemaRejectsUnsafeNamesProfilesCIDRsAndHalfSelectors(t *testing.T) { for name, values := range map[string][]string{ - "cluster host name": {"--set-string", "clusterHost.name=Bad_Name"}, - "storage class name": {"--set-string", "storage.adminRWXStorageClass=Bad_Name"}, - "runtime profile": {"--set-string", "clusterHost.runtimeProfiles[0]=-bad"}, - "Woodpecker Secret name": {"--set-string", "woodpecker.existingSecret=Bad_Name", "--set-string", "woodpecker.configMap=woodpecker-config"}, - "Woodpecker ConfigMap name": {"--set-string", "woodpecker.existingSecret=woodpecker-token", "--set-string", "woodpecker.configMap=Bad_Name"}, - "Woodpecker key": {"--set-string", "woodpecker.existingSecret=woodpecker-token", "--set-string", "woodpecker.configMap=woodpecker-config", "--set-string", "woodpecker.tokenKey=bad/key"}, - "Woodpecker audience": {"--set-string", "woodpecker.serviceAccountAudience=/bad", "--set-string", "woodpecker.configMap=woodpecker-config"}, - "IPv4 default route": {"--set-string", "server.trustedProxyCIDRs[0]=0.0.0.0/0"}, - "IPv6 default route": {"--set-string", "server.trustedProxyCIDRs[0]=::/0"}, - "gateway half selector": {"--set-string", "networkPolicy.gatewayIngress.namespaceSelector.matchLabels.scope=gateway"}, - "observability half selector": {"--set-string", "networkPolicy.observability.podSelector.matchLabels.scope=metrics"}, - "OMP ConfigMap name": {"--set-string", "session.omp.configMap=Bad_Name"}, - "OMP models key": {"--set-string", "session.omp.modelsKey=bad/key"}, - "legacy OMP credential Secret": {"--set-string", "session.omp.credentialSecret=omp-runtime-credential"}, - "legacy OMP credential key": {"--set-string", "session.omp.credentialKey=MODEL_API_KEY"}, - "identical OMP projection keys": {"--set-string", "session.omp.settingsKey=provider-models"}, - "model route port zero": {"--set", "networkPolicy.modelRoutePorts[0]=0"}, - "model route port above TCP range": {"--set", "networkPolicy.modelRoutePorts[0]=65536"}, - "duplicate model route port": {"--set", "networkPolicy.modelRoutePorts[0]=19481", "--set", "networkPolicy.modelRoutePorts[1]=19481"}, - "noninteger model route port": {"--set-string", "networkPolicy.modelRoutePorts[0]=https"}, - "model route half selector": {"--set-string", "networkPolicy.modelRoute.namespaceSelector.matchLabels.scope=linkedin-bot"}, - "CI provider port zero": {"--set", "networkPolicy.ciProviderPorts[0]=0"}, - "CI provider port above TCP range": {"--set", "networkPolicy.ciProviderPorts[0]=65536"}, - "duplicate CI provider port": {"--set", "networkPolicy.ciProviderPorts[0]=8080", "--set", "networkPolicy.ciProviderPorts[1]=8080"}, - "noninteger CI provider port": {"--set-string", "networkPolicy.ciProviderPorts[0]=http"}, - "CI provider half selector": {"--set-string", "networkPolicy.ciProvider.namespaceSelector.matchLabels.scope=linkedin-bot"}, - "legacy OMP credential mode": {"--set", "session.omp.allowUnauthenticated=false"}, + "cluster host name": {"--set-string", "clusterHost.name=Bad_Name"}, + "storage class name": {"--set-string", "storage.adminRWXStorageClass=Bad_Name"}, + "runtime profile": {"--set-string", "clusterHost.runtimeProfiles[0]=-bad"}, + "public API Secret name": {"--set-string", "publicApi.existingSecret=Bad_Name"}, + "public API PostgreSQL key": {"--set-string", "publicApi.postgresURLKey=bad/key"}, + "public API credentials key": {"--set-string", "publicApi.credentialsKey=bad/key"}, + "Woodpecker Secret name": {"--set-string", "woodpecker.existingSecret=Bad_Name", "--set-string", "woodpecker.configMap=woodpecker-config"}, + "Woodpecker ConfigMap name": {"--set-string", "woodpecker.existingSecret=woodpecker-token", "--set-string", "woodpecker.configMap=Bad_Name"}, + "Woodpecker key": {"--set-string", "woodpecker.existingSecret=woodpecker-token", "--set-string", "woodpecker.configMap=woodpecker-config", "--set-string", "woodpecker.tokenKey=bad/key"}, + "Woodpecker audience": {"--set-string", "woodpecker.serviceAccountAudience=/bad", "--set-string", "woodpecker.configMap=woodpecker-config"}, + "IPv4 default route": {"--set-string", "server.trustedProxyCIDRs[0]=0.0.0.0/0"}, + "IPv6 default route": {"--set-string", "server.trustedProxyCIDRs[0]=::/0"}, + "PostgreSQL IPv4 default route": {"--set-string", "networkPolicy.postgresCIDRs[0]=0.0.0.0/0"}, + "PostgreSQL IPv6 default route": {"--set-string", "networkPolicy.postgresCIDRs[0]=::/0"}, + "gateway half selector": {"--set-string", "networkPolicy.gatewayIngress.namespaceSelector.matchLabels.scope=gateway"}, + "observability half selector": {"--set-string", "networkPolicy.observability.podSelector.matchLabels.scope=metrics"}, + "OMP ConfigMap name": {"--set-string", "session.omp.configMap=Bad_Name"}, + "OMP models key": {"--set-string", "session.omp.modelsKey=bad/key"}, + "legacy OMP credential Secret": {"--set-string", "session.omp.credentialSecret=omp-runtime-credential"}, + "legacy OMP credential key": {"--set-string", "session.omp.credentialKey=MODEL_API_KEY"}, + "identical OMP projection keys": {"--set-string", "session.omp.settingsKey=provider-models"}, + "model route port zero": {"--set", "networkPolicy.modelRoutePorts[0]=0"}, + "model route port above TCP range": {"--set", "networkPolicy.modelRoutePorts[0]=65536"}, + "duplicate model route port": {"--set", "networkPolicy.modelRoutePorts[0]=19481", "--set", "networkPolicy.modelRoutePorts[1]=19481"}, + "noninteger model route port": {"--set-string", "networkPolicy.modelRoutePorts[0]=https"}, + "model route half selector": {"--set-string", "networkPolicy.modelRoute.namespaceSelector.matchLabels.scope=linkedin-bot"}, + "CI provider port zero": {"--set", "networkPolicy.ciProviderPorts[0]=0"}, + "CI provider port above TCP range": {"--set", "networkPolicy.ciProviderPorts[0]=65536"}, + "duplicate CI provider port": {"--set", "networkPolicy.ciProviderPorts[0]=8080", "--set", "networkPolicy.ciProviderPorts[1]=8080"}, + "noninteger CI provider port": {"--set-string", "networkPolicy.ciProviderPorts[0]=http"}, + "CI provider half selector": {"--set-string", "networkPolicy.ciProvider.namespaceSelector.matchLabels.scope=linkedin-bot"}, + "PostgreSQL port zero": {"--set", "networkPolicy.postgresPorts[0]=0"}, + "PostgreSQL port above TCP range": {"--set", "networkPolicy.postgresPorts[0]=65536"}, + "duplicate PostgreSQL port": {"--set", "networkPolicy.postgresPorts[0]=5432", "--set", "networkPolicy.postgresPorts[1]=5432"}, + "noninteger PostgreSQL port": {"--set-string", "networkPolicy.postgresPorts[0]=postgres"}, + "PostgreSQL half selector": {"--set-string", "networkPolicy.postgres.namespaceSelector.matchLabels.scope=database"}, + "legacy OMP credential mode": {"--set", "session.omp.allowUnauthenticated=false"}, } { t.Run(name, func(t *testing.T) { helmTemplateMustFail(t, append(enabledValues(), values...)...) @@ -178,7 +188,7 @@ func TestValuesSchemaRejectsUnsafeNamesProfilesCIDRsAndHalfSelectors(t *testing. } func TestValuesSchemaBoundsRoutePortLists(t *testing.T) { - for _, field := range []string{"modelRoutePorts", "ciProviderPorts"} { + for _, field := range []string{"modelRoutePorts", "ciProviderPorts", "postgresPorts"} { t.Run(field, func(t *testing.T) { values := enabledValues() for index := 0; index < 17; index++ { @@ -339,12 +349,31 @@ func TestRBACSeparatesControllerMutationFromServerProjection(t *testing.T) { "resources: [t4workspaces, t4sessions]", "verbs: [get, list, watch, create, patch, delete]", ) - if strings.Contains(serverRole, "secrets") || strings.Contains(serverRole, "persistentvolumeclaims") || strings.Contains(serverRole, "t4sessions/status") { - t.Fatal("server role can read secrets or mutate controller-owned infrastructure/status") + assertCount(t, serverRole, "patch", 1) + if strings.Contains(serverRole, "secrets") || + strings.Contains(serverRole, "persistentvolumeclaims") || + strings.Contains(serverRole, "t4sessions/status") || + strings.Contains(serverRole, "update") { + t.Fatal("server role can read secrets or mutate controller-owned infrastructure/status beyond exact CR patch operations") } } func TestPublicAPIRequiresDurableSecretAndExplicitPostgresEgress(t *testing.T) { + disabled := helmTemplate(t, append(enabledValues(), + "--set-string", "publicApi.existingSecret=t4-public-api", + "--set", "networkPolicy.postgresCIDRs[0]=198.51.100.20/32", + "--set", "networkPolicy.postgresPorts[0]=5432", + )...) + disabledServer := documentContainingKind(t, disabled, "Deployment", "name: \"release-name-t4-cluster-server\"") + disabledEgress := documentContainingKind(t, disabled, "NetworkPolicy", "name: \"release-name-t4-cluster-server-egress\"") + if strings.Contains(disabledServer, "T4_POSTGRES_URL_FILE") || + strings.Contains(disabledServer, "T4_PUBLIC_API_CREDENTIALS_FILE") || + strings.Contains(disabledServer, "T4_PUBLIC_API_RUNTIME_PROFILE") || + strings.Contains(disabledEgress, "198.51.100.20/32") || + strings.Contains(disabledEgress, "port: 5432") { + t.Fatal("disabled public API rendered credentials or PostgreSQL egress") + } + withoutSecret := append(enabledValues(), "--set", "publicApi.enabled=true", "--set", "networkPolicy.postgresCIDRs[0]=198.51.100.20/32", @@ -364,6 +393,7 @@ func TestPublicAPIRequiresDurableSecretAndExplicitPostgresEgress(t *testing.T) { "--set-string", "publicApi.postgresURLKey=postgres-url", "--set-string", "publicApi.credentialsKey=credentials.json", "--set-string", "clusterHost.runtimeProfiles[0]=review-admitted", + "--set-string", "clusterHost.runtimeProfiles[1]=fallback-admitted", "--set", "networkPolicy.postgresCIDRs[0]=198.51.100.20/32", "--set", "networkPolicy.postgresPorts[0]=5432", ) @@ -377,7 +407,12 @@ func TestPublicAPIRequiresDurableSecretAndExplicitPostgresEgress(t *testing.T) { "secretName: \"t4-public-api\"", "key: postgres-url", "key: credentials.json", + "/var/run/secrets/t4-public-api/postgres-url", + "/var/run/secrets/t4-public-api/credentials.json", ) + if strings.Contains(server, "value: \"fallback-admitted\"") { + t.Fatalf("public API runtime profile was not derived from the first host-admitted profile:\n%s", server) + } host := documentContainingKind(t, output, "T4ClusterHost", "name: \"t4-cluster\"") assertContains(t, host, "- review-admitted") serverEgress := documentContainingKind(t, output, "NetworkPolicy", "name: \"release-name-t4-cluster-server-egress\"") @@ -385,6 +420,17 @@ func TestPublicAPIRequiresDurableSecretAndExplicitPostgresEgress(t *testing.T) { if strings.Contains(serverEgress, "0.0.0.0/0") || strings.Contains(serverEgress, "::/0") { t.Fatalf("public API PostgreSQL egress is broader than its configured destination:\n%s", serverEgress) } + + selectorValues := append(enabledValues(), + "--set", "publicApi.enabled=true", + "--set-string", "publicApi.existingSecret=t4-public-api", + "--set", "networkPolicy.postgresPorts[0]=5433", + "--set-string", "networkPolicy.postgres.namespaceSelector.matchLabels.scope=database-namespace", + "--set-string", "networkPolicy.postgres.podSelector.matchLabels.app=postgres", + ) + selectorOutput := helmTemplate(t, selectorValues...) + selectorEgress := documentContainingKind(t, selectorOutput, "NetworkPolicy", "name: \"release-name-t4-cluster-server-egress\"") + assertContains(t, selectorEgress, "scope: database-namespace", "app: postgres", "port: 5433") } func TestPublicAPIRejectsIncompleteSecretKeysAndPostgresRoutes(t *testing.T) { @@ -393,10 +439,68 @@ func TestPublicAPIRejectsIncompleteSecretKeysAndPostgresRoutes(t *testing.T) { "empty credentials key": {"--set", "publicApi.enabled=true", "--set-string", "publicApi.existingSecret=t4-public-api", "--set-string", "publicApi.credentialsKey=", "--set", "networkPolicy.postgresCIDRs[0]=198.51.100.20/32", "--set", "networkPolicy.postgresPorts[0]=5432"}, "destination without port": {"--set", "publicApi.enabled=true", "--set-string", "publicApi.existingSecret=t4-public-api", "--set", "networkPolicy.postgresCIDRs[0]=198.51.100.20/32", "--set-string", "networkPolicy.postgresPorts="}, "port without destination": {"--set", "publicApi.enabled=true", "--set-string", "publicApi.existingSecret=t4-public-api", "--set", "networkPolicy.postgresPorts[0]=5432"}, + "disabled network policy": {"--set", "publicApi.enabled=true", "--set-string", "publicApi.existingSecret=t4-public-api", "--set", "networkPolicy.enabled=false", "--set", "networkPolicy.postgresCIDRs[0]=198.51.100.20/32", "--set", "networkPolicy.postgresPorts[0]=5432"}, + "half selector": {"--set", "publicApi.enabled=true", "--set-string", "publicApi.existingSecret=t4-public-api", "--set", "networkPolicy.postgresPorts[0]=5432", "--set-string", "networkPolicy.postgres.namespaceSelector.matchLabels.scope=database"}, + "selector without port": {"--set", "publicApi.enabled=true", "--set-string", "publicApi.existingSecret=t4-public-api", "--set-string", "networkPolicy.postgres.namespaceSelector.matchLabels.scope=database", "--set-string", "networkPolicy.postgres.podSelector.matchLabels.app=postgres"}, } { t.Run(name, func(t *testing.T) { helmTemplateMustFail(t, append(enabledValues(), values...)...) }) } } + +func TestPublicAPIRenderGuardsSurviveSchemaBypass(t *testing.T) { + complete := []string{ + "--skip-schema-validation", + "--set", "publicApi.enabled=true", + "--set-string", "publicApi.existingSecret=t4-public-api", + "--set-string", "publicApi.postgresURLKey=postgres-url", + "--set-string", "publicApi.credentialsKey=credentials.json", + "--set", "networkPolicy.postgresCIDRs[0]=198.51.100.20/32", + "--set", "networkPolicy.postgresPorts[0]=5432", + } + for name, test := range map[string]struct { + flag string + override string + message string + }{ + "missing Secret": { + flag: "--set-string", + override: "publicApi.existingSecret=", + message: "publicApi.existingSecret is required when publicApi.enabled is true", + }, + "missing PostgreSQL key": { + flag: "--set-string", + override: "publicApi.postgresURLKey=", + message: "publicApi.postgresURLKey is required when publicApi.enabled is true", + }, + "missing credentials key": { + flag: "--set-string", + override: "publicApi.credentialsKey=", + message: "publicApi.credentialsKey is required when publicApi.enabled is true", + }, + "disabled network policy": { + flag: "--set", + override: "networkPolicy.enabled=false", + message: "networkPolicy.enabled must be true when publicApi.enabled is true", + }, + "missing PostgreSQL ports": { + flag: "--set-string", + override: "networkPolicy.postgresPorts=", + message: "networkPolicy.postgresPorts must contain at least one port when publicApi.enabled is true", + }, + "missing PostgreSQL destination": { + flag: "--set-string", + override: "networkPolicy.postgresCIDRs=", + message: "publicApi.enabled requires a PostgreSQL CIDR or complete selector destination", + }, + } { + t.Run(name, func(t *testing.T) { + values := append(enabledValues(), complete...) + values = append(values, test.flag, test.override) + helmTemplateMustFailBeforeWorkloads(t, test.message, values...) + }) + } +} + func TestUnauthenticatedOMPControllerCannotReadSecrets(t *testing.T) { output := helmTemplate(t, enabledValues()...) controllerRole := documentContaining(t, output, "name: \"release-name-t4-cluster-controller\"") @@ -781,6 +885,23 @@ func helmTemplateMustFail(t *testing.T, extra ...string) { } } +func helmTemplateMustFailBeforeWorkloads(t *testing.T, message string, extra ...string) { + t.Helper() + args := []string{"template", "release-name", filepath.Join(repoRoot(t), "deploy", "charts", "t4-cluster"), "--namespace", "t4-system"} + args = append(args, extra...) + command := exec.Command("helm", args...) + output, err := command.CombinedOutput() + if err == nil { + t.Fatalf("helm unexpectedly accepted invalid values: %s", output) + } + if !strings.Contains(string(output), message) { + t.Fatalf("helm failure lacks deterministic guard %q: %s", message, output) + } + if strings.Contains(string(output), "kind: Deployment") || strings.Contains(string(output), "kind: NetworkPolicy") { + t.Fatalf("helm rendered workloads or network policies before public API validation failed: %s", output) + } +} + func enabledValues() []string { return []string{ "--set", "enabled=true", diff --git a/packages/cluster-server/src/config.ts b/packages/cluster-server/src/config.ts index 5f6b4cf7..fea3d530 100644 --- a/packages/cluster-server/src/config.ts +++ b/packages/cluster-server/src/config.ts @@ -5,6 +5,7 @@ import { isIP } from "node:net"; const DNS_LABEL = /^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$/u; const DNS_SUBDOMAIN = /^(?:[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?)(?:\.[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?)*$/u; const AUDIENCE = /^[A-Za-z0-9][A-Za-z0-9._:/-]*$/u; +const RUNTIME_PROFILE = /^[a-z0-9](?:[-._a-z0-9]{0,62}[a-z0-9])?$/u; export interface ClusterServerConfig { readonly namespace: string; readonly podName: string; @@ -23,6 +24,7 @@ export interface ClusterServerConfig { readonly trustedProxyCidrs: readonly string[]; readonly postgres?: { readonly url?: string; readonly urlFile?: string }; readonly publicApiCredentialsFile?: string; + readonly publicApiRuntimeProfile?: string; readonly woodpecker?: { readonly baseUrl: string; readonly webBaseUrl?: string; @@ -137,10 +139,13 @@ export function clusterServerConfigFromEnv(env: Readonly>; readonly ownerId: string; readonly ownerEpoch: bigint; + readonly expiresAt: number; } export interface OwnerLease { readonly ownerId: string; @@ -73,6 +74,7 @@ interface SessionRow { readonly revision: bigint; readonly labels: Record; readonly cancellation_requested: boolean; + readonly deletion_requested: boolean; } interface EventRow { readonly sequence: bigint; @@ -80,7 +82,8 @@ interface EventRow { readonly payload: Record; } interface RetentionRow { readonly first_retained_sequence: bigint; readonly latest_sequence: bigint; } -interface LeaseRow { readonly owner_id: string; readonly epoch: bigint; } +interface LeaseRow { readonly owner_id: string; readonly epoch: bigint; readonly expired: boolean; } +interface LeaseClaimRow extends LeaseRow { readonly expires_at_ms: bigint; } interface OutboxRow { readonly outbox_id: bigint; readonly command_id: string; @@ -96,6 +99,7 @@ const MIGRATION_URL = new URL("../migrations/001_durable_gateway.sql", import.me const ROLLBACK_URL = new URL("../migrations/001_durable_gateway.down.sql", import.meta.url); const OUTBOX_LEASE = "gateway-outbox"; const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER); +const SERIALIZABLE_ATTEMPTS = 3; function persistedBigint(value: unknown, allowZero = false): bigint { if (typeof value !== "bigint") throw new Error("persisted ledger bigint is invalid"); @@ -148,7 +152,7 @@ export class PostgresLedger { async migrate(): Promise { const migration = await Bun.file(MIGRATION_URL).text(); - await this.#sql.begin("ISOLATION LEVEL SERIALIZABLE", async transaction => { + await this.#serializable(async transaction => { await transaction`SELECT pg_advisory_xact_lock(741405)`; await transaction.unsafe(migration); }); @@ -157,7 +161,7 @@ export class PostgresLedger { } async rollback(): Promise { const rollback = await Bun.file(ROLLBACK_URL).text(); - await this.#sql.begin("ISOLATION LEVEL SERIALIZABLE", async transaction => { + await this.#serializable(async transaction => { await transaction`SELECT pg_advisory_xact_lock(741405)`; await transaction.unsafe(rollback); }); @@ -165,7 +169,8 @@ export class PostgresLedger { async close(): Promise { await this.#sql.close(); } async createWorkspace(principalId: string, idempotencyKey: string, fingerprint: string, input: { name: string; labels?: Record }): Promise> { - return await this.#sql.begin("ISOLATION LEVEL SERIALIZABLE", async transaction => { + return await this.#serializable(async transaction => { + await this.#lockIdempotency(transaction, principalId, "createWorkspace", "workspaces", idempotencyKey); const prior = await transaction`SELECT fingerprint, response_body FROM t4_commands WHERE principal_id = ${principalId} AND operation = 'createWorkspace' AND target_scope = 'workspaces' AND idempotency_key = ${idempotencyKey} FOR UPDATE`; if (prior[0]) return prior[0].fingerprint === fingerprint ? { kind: "replay", value: prior[0].response_body as WorkspaceRecord } @@ -192,14 +197,15 @@ export class PostgresLedger { } async patchWorkspace(principalId: string, workspaceId: string, expectedRevision: number, idempotencyKey: string, fingerprint: string, input: { name?: string; labels?: Record }): Promise> { - return await this.#sql.begin("ISOLATION LEVEL SERIALIZABLE", async transaction => { + return await this.#serializable(async transaction => { + const scope = `workspace:${workspaceId}`; + await this.#lockIdempotency(transaction, principalId, "mutateWorkspace", scope, idempotencyKey); + const prior = await transaction`SELECT fingerprint, response_body FROM t4_commands WHERE principal_id = ${principalId} AND operation = 'mutateWorkspace' AND target_scope = ${scope} AND idempotency_key = ${idempotencyKey} FOR UPDATE`; + if (prior[0]) return prior[0].fingerprint === fingerprint ? { kind: "replay", value: prior[0].response_body as WorkspaceRecord } : { kind: "idempotency_conflict" }; const rows = await transaction`SELECT workspace_id, name, state, revision, labels FROM t4_workspace_intents WHERE principal_id = ${principalId} AND workspace_id = ${workspaceId} AND state <> 'deleted' FOR UPDATE`; const current = rows[0]; if (!current) return { kind: "not_found" }; if (current.revision !== BigInt(expectedRevision)) return { kind: "revision_conflict" }; - const scope = `workspace:${workspaceId}`; - const prior = await transaction`SELECT fingerprint, response_body FROM t4_commands WHERE principal_id = ${principalId} AND operation = 'mutateWorkspace' AND target_scope = ${scope} AND idempotency_key = ${idempotencyKey} FOR UPDATE`; - if (prior[0]) return prior[0].fingerprint === fingerprint ? { kind: "replay", value: prior[0].response_body as WorkspaceRecord } : { kind: "idempotency_conflict" }; const revision = current.revision + 1n; const value: WorkspaceRecord = { id: workspaceId, name: input.name ?? current.name, state: current.state, revision: safeNumber(revision), ...(input.labels ?? labels(current.labels) ? { labels: input.labels ?? current.labels } : {}) }; const commandId = `cmd_${randomUUID()}`; @@ -211,8 +217,9 @@ export class PostgresLedger { } async deleteWorkspace(principalId: string, workspaceId: string, idempotencyKey: string, fingerprint: string): Promise> { - return await this.#sql.begin("ISOLATION LEVEL SERIALIZABLE", async transaction => { + return await this.#serializable(async transaction => { const scope = `workspace:${workspaceId}`; + await this.#lockIdempotency(transaction, principalId, "deleteWorkspace", scope, idempotencyKey); const prior = await transaction`SELECT fingerprint, response_body FROM t4_commands WHERE principal_id = ${principalId} AND operation = 'deleteWorkspace' AND target_scope = ${scope} AND idempotency_key = ${idempotencyKey} FOR UPDATE`; if (prior[0]) return prior[0].fingerprint === fingerprint ? { kind: "replay" } : { kind: "idempotency_conflict" }; const rows = await transaction`SELECT workspace_id, name, state, revision, labels FROM t4_workspace_intents WHERE principal_id = ${principalId} AND workspace_id = ${workspaceId} AND state <> 'deleted' FOR UPDATE`; @@ -227,19 +234,20 @@ export class PostgresLedger { } async createSession(principalId: string, workspaceId: string, idempotencyKey: string, fingerprint: string, input: { title: string; labels?: Record }): Promise> { - return await this.#sql.begin("ISOLATION LEVEL SERIALIZABLE", async transaction => { - const workspaceRows = await transaction<{ workspace_id: string }[]>`SELECT workspace_id FROM t4_workspace_intents WHERE principal_id = ${principalId} AND workspace_id = ${workspaceId} AND state NOT IN ('deleting','deleted') FOR UPDATE`; - if (!workspaceRows[0]) return { kind: "not_found" }; + return await this.#serializable(async transaction => { const scope = `workspace:${workspaceId}:sessions`; + await this.#lockIdempotency(transaction, principalId, "spawnSession", scope, idempotencyKey); const prior = await transaction`SELECT fingerprint, response_body FROM t4_commands WHERE principal_id = ${principalId} AND operation = 'spawnSession' AND target_scope = ${scope} AND idempotency_key = ${idempotencyKey} FOR UPDATE`; if (prior[0]) return prior[0].fingerprint === fingerprint ? { kind: "replay", value: prior[0].response_body as SessionRecord } : { kind: "idempotency_conflict" }; + const workspaceRows = await transaction<{ workspace_id: string }[]>`SELECT workspace_id FROM t4_workspace_intents WHERE principal_id = ${principalId} AND workspace_id = ${workspaceId} AND state NOT IN ('deleting','deleted') FOR UPDATE`; + if (!workspaceRows[0]) return { kind: "not_found" }; const commandId = `cmd_${randomUUID()}`; const sessionId = `ss-${randomUUID()}`; const value: SessionRecord = { id: sessionId, workspaceId, title: input.title, state: "accepted", revision: 1, ...(input.labels ? { labels: input.labels } : {}) }; await transaction`INSERT INTO t4_commands (command_id, principal_id, operation, target_scope, idempotency_key, fingerprint, lifecycle_state, response_status, response_body) VALUES (${commandId}, ${principalId}, 'spawnSession', ${scope}, ${idempotencyKey}, ${fingerprint}, 'accepted', 202, ${value})`; await transaction`INSERT INTO t4_session_intents (session_id, workspace_id, principal_id, title, labels, state, revision, generation) VALUES (${sessionId}, ${workspaceId}, ${principalId}, ${input.title}, ${input.labels ?? {}}, 'accepted', 1, 1)`; await transaction`INSERT INTO t4_outbox (command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation) VALUES (${commandId}, ${principalId}, ${idempotencyKey}, 'session.create', ${sessionId}, 1, ${value})`; - await this.#appendEvent(transaction as unknown as SQL, principalId, sessionId, "session", { state: "accepted", revision: 1 }, 0n); + await this.#appendEvent(transaction, principalId, sessionId, "session", { state: "accepted", revision: 1 }, 0n); return { kind: "accepted", commandId, value }; }); } @@ -257,34 +265,48 @@ export class PostgresLedger { } async patchSession(principalId: string, sessionId: string, expectedRevision: number, idempotencyKey: string, fingerprint: string, input: { title?: string; labels?: Record }): Promise> { - return await this.#sql.begin("ISOLATION LEVEL SERIALIZABLE", async transaction => { - const rows = await transaction`SELECT session_id, workspace_id, title, state, revision, labels, cancellation_requested FROM t4_session_intents WHERE principal_id = ${principalId} AND session_id = ${sessionId} FOR UPDATE`; - const current = rows[0]; - if (!current) return { kind: "not_found" }; - if (current.revision !== BigInt(expectedRevision)) return { kind: "revision_conflict" }; + return await this.#serializable(async transaction => { const scope = `session:${sessionId}`; + await this.#lockIdempotency(transaction, principalId, "mutateSession", scope, idempotencyKey); const prior = await transaction`SELECT fingerprint, response_body FROM t4_commands WHERE principal_id = ${principalId} AND operation = 'mutateSession' AND target_scope = ${scope} AND idempotency_key = ${idempotencyKey} FOR UPDATE`; if (prior[0]) return prior[0].fingerprint === fingerprint ? { kind: "replay", value: prior[0].response_body as SessionRecord } : { kind: "idempotency_conflict" }; + const rows = await transaction`SELECT session_id, workspace_id, title, state, revision, labels, cancellation_requested FROM t4_session_intents WHERE principal_id = ${principalId} AND session_id = ${sessionId} AND state NOT IN ('cancelling','cancelled','failed','unavailable','indeterminate') FOR UPDATE`; + const current = rows[0]; + if (!current) return { kind: "not_found" }; + if (current.revision !== BigInt(expectedRevision)) return { kind: "revision_conflict" }; const revision = current.revision + 1n; const value: SessionRecord = { id: sessionId, workspaceId: current.workspace_id, title: input.title ?? current.title, state: current.state, revision: safeNumber(revision), ...(input.labels ?? labels(current.labels) ? { labels: input.labels ?? current.labels } : {}) }; const commandId = `cmd_${randomUUID()}`; await transaction`INSERT INTO t4_commands (command_id, principal_id, operation, target_scope, idempotency_key, fingerprint, lifecycle_state, response_status, response_body) VALUES (${commandId}, ${principalId}, 'mutateSession', ${scope}, ${idempotencyKey}, ${fingerprint}, 'accepted', 200, ${value})`; await transaction`UPDATE t4_session_intents SET title = ${value.title}, labels = ${value.labels ?? {}}, revision = ${revision}, generation = generation + 1, updated_at = clock_timestamp() WHERE session_id = ${sessionId}`; await transaction`INSERT INTO t4_outbox (command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation) VALUES (${commandId}, ${principalId}, ${idempotencyKey}, 'session.patch', ${sessionId}, ${revision}, ${value})`; - await this.#appendEvent(transaction as unknown as SQL, principalId, sessionId, "session", { state: value.state, revision: value.revision }, 0n); + await this.#appendEvent(transaction, principalId, sessionId, "session", { state: value.state, revision: value.revision }, 0n); return { kind: "accepted", commandId, value }; }); } async cancelSession(principalId: string, sessionId: string, idempotencyKey: string, fingerprint: string, deletion = false): Promise> { - return await this.#sql.begin("ISOLATION LEVEL SERIALIZABLE", async transaction => { + return await this.#serializable(async transaction => { const operation = deletion ? "deleteSession" : "cancelSession"; const scope = `session:${sessionId}`; + await this.#lockIdempotency(transaction, principalId, operation, scope, idempotencyKey); const prior = await transaction`SELECT fingerprint, response_body FROM t4_commands WHERE principal_id = ${principalId} AND operation = ${operation} AND target_scope = ${scope} AND idempotency_key = ${idempotencyKey} FOR UPDATE`; if (prior[0]) return prior[0].fingerprint === fingerprint ? { kind: "replay", value: prior[0].response_body as SessionRecord } : { kind: "idempotency_conflict" }; - const rows = await transaction`SELECT session_id, workspace_id, title, state, revision, labels, cancellation_requested FROM t4_session_intents WHERE principal_id = ${principalId} AND session_id = ${sessionId} FOR UPDATE`; + const rows = await transaction`SELECT session_id, workspace_id, title, state, revision, labels, cancellation_requested, deletion_requested FROM t4_session_intents WHERE principal_id = ${principalId} AND session_id = ${sessionId} FOR UPDATE`; const current = rows[0]; if (!current) return { kind: "not_found" }; + if (current.cancellation_requested || current.state === "cancelling" || current.state === "cancelled" || current.state === "failed" || current.state === "unavailable" || current.state === "indeterminate") { + const value = session(current); + const commandId = `cmd_${randomUUID()}`; + const dispatchDeletion = deletion && !current.deletion_requested; + await transaction`INSERT INTO t4_commands (command_id, principal_id, operation, target_scope, idempotency_key, fingerprint, lifecycle_state, response_status, response_body) VALUES (${commandId}, ${principalId}, ${operation}, ${scope}, ${idempotencyKey}, ${fingerprint}, ${dispatchDeletion ? "accepted" : "completed"}, ${deletion ? 204 : 202}, ${value})`; + if (dispatchDeletion) { + await transaction`UPDATE t4_session_intents SET deletion_requested = true, generation = generation + 1, updated_at = clock_timestamp() WHERE session_id = ${sessionId}`; + await transaction`INSERT INTO t4_outbox (command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation) VALUES (${commandId}, ${principalId}, ${idempotencyKey}, 'session.delete', ${sessionId}, ${current.revision}, ${{ sessionId, revision: safeNumber(current.revision) }})`; + } + await this.#appendEvent(transaction, principalId, sessionId, "session", { state: current.state, revision: safeNumber(current.revision) }, 0n); + return { kind: "accepted", commandId, value }; + } const revision = current.revision + 1n; const value: SessionRecord = { id: sessionId, workspaceId: current.workspace_id, title: current.title, state: "cancelling", revision: safeNumber(revision), ...(labels(current.labels) ? { labels: current.labels } : {}) }; const commandId = `cmd_${randomUUID()}`; @@ -292,54 +314,62 @@ export class PostgresLedger { await transaction`UPDATE t4_session_intents SET state = 'cancelling', cancellation_requested = true, deletion_requested = deletion_requested OR ${deletion}, revision = ${revision}, generation = generation + 1, updated_at = clock_timestamp() WHERE session_id = ${sessionId}`; await transaction`UPDATE t4_outbox SET state = 'skipped', terminal_result = ${{ reason: "superseded by cancellation" }}, updated_at = clock_timestamp() WHERE target_id = ${sessionId} AND mutation_kind = 'session.create' AND state IN ('pending','claimed')`; await transaction`INSERT INTO t4_outbox (command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation) VALUES (${commandId}, ${principalId}, ${idempotencyKey}, ${deletion ? "session.delete" : "session.cancel"}, ${sessionId}, ${revision}, ${{ sessionId, revision: safeNumber(revision) }})`; - await this.#appendEvent(transaction as unknown as SQL, principalId, sessionId, "session", { state: "cancelling", revision: safeNumber(revision) }, 0n); + await this.#appendEvent(transaction, principalId, sessionId, "session", { state: "cancelling", revision: safeNumber(revision) }, 0n); return { kind: "accepted", commandId, value }; }); } async submitCommand(principalId: string, sessionId: string, idempotencyKey: string, fingerprint: string, input: { command: string; metadata: Record }): Promise> { - return await this.#sql.begin("ISOLATION LEVEL SERIALIZABLE", async transaction => { - const sessionRows = await transaction<{ session_id: string }[]>`SELECT session_id FROM t4_session_intents WHERE principal_id = ${principalId} AND session_id = ${sessionId} AND state NOT IN ('cancelling','cancelled') FOR UPDATE`; - if (!sessionRows[0]) return { kind: "not_found" }; + return await this.#serializable(async transaction => { const scope = `session:${sessionId}:commands`; + await this.#lockIdempotency(transaction, principalId, "submitCommand", scope, idempotencyKey); const prior = await transaction`SELECT fingerprint, response_body FROM t4_commands WHERE principal_id = ${principalId} AND operation = 'submitCommand' AND target_scope = ${scope} AND idempotency_key = ${idempotencyKey} FOR UPDATE`; if (prior[0]) return prior[0].fingerprint === fingerprint ? { kind: "replay", value: prior[0].response_body as { commandId: string; state: "accepted" } } : { kind: "idempotency_conflict" }; + const sessionRows = await transaction<{ session_id: string }[]>`SELECT session_id FROM t4_session_intents WHERE principal_id = ${principalId} AND session_id = ${sessionId} AND state NOT IN ('cancelling','cancelled','failed','unavailable','indeterminate') FOR UPDATE`; + if (!sessionRows[0]) return { kind: "not_found" }; const commandId = `cmd_${randomUUID()}`; const value = { commandId, state: "accepted" as const }; await transaction`INSERT INTO t4_commands (command_id, principal_id, operation, target_scope, idempotency_key, fingerprint, lifecycle_state, response_status, response_body) VALUES (${commandId}, ${principalId}, 'submitCommand', ${scope}, ${idempotencyKey}, ${fingerprint}, 'accepted', 202, ${value})`; await transaction`INSERT INTO t4_outbox (command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation) SELECT ${commandId}, ${principalId}, ${idempotencyKey}, 'command.submit', ${sessionId}, revision, ${{ sessionId, ...input }} FROM t4_session_intents WHERE session_id = ${sessionId}`; const entryRows = await transaction<{ next: bigint }[]>`SELECT COALESCE(MAX(entry_sequence), -1) + 1 AS next FROM t4_snapshot_entries WHERE principal_id = ${principalId} AND session_id = ${sessionId}`; await transaction`INSERT INTO t4_snapshot_entries (principal_id, session_id, entry_sequence, kind, text_value) VALUES (${principalId}, ${sessionId}, ${entryRows[0]?.next ?? 0n}, 'input', ${input.command})`; - await this.#appendEvent(transaction as unknown as SQL, principalId, sessionId, "command", { commandId, state: "accepted" }, 0n); + await this.#appendEvent(transaction, principalId, sessionId, "command", { commandId, state: "accepted" }, 0n); return { kind: "accepted", commandId, value }; }); } async snapshot(principalId: string, sessionId: string): Promise<{ session: SessionRecord; cursor: string; entries: readonly { sequence: number; kind: "input" | "output" | "status"; text: string }[] } | undefined> { - const current = await this.getSession(principalId, sessionId); - if (!current) return undefined; - const rows = await this.#sql<{ entry_sequence: bigint; kind: "input" | "output" | "status"; text_value: string }[]>`SELECT entry_sequence, kind, text_value FROM t4_snapshot_entries WHERE principal_id = ${principalId} AND session_id = ${sessionId} ORDER BY entry_sequence DESC LIMIT 1000`; - const retention = await this.#sql`SELECT first_retained_sequence, latest_sequence FROM t4_event_retention WHERE principal_id = ${principalId} AND session_id = ${sessionId}`; - const latest = retention[0] ? persistedBigint(retention[0].latest_sequence, true) : 0n; - return { session: current, cursor: this.watchCursor(principalId, sessionId, latest), entries: rows.reverse().map(row => ({ sequence: safeNumber(row.entry_sequence, true), kind: row.kind, text: row.text_value })) }; + return await this.#sql.begin("ISOLATION LEVEL REPEATABLE READ", async rawTransaction => { + const transaction = rawTransaction as unknown as SQL; + const sessionRows = await transaction`SELECT session_id, workspace_id, title, state, revision, labels, cancellation_requested FROM t4_session_intents WHERE principal_id = ${principalId} AND session_id = ${sessionId}`; + if (!sessionRows[0]) return undefined; + const rows = await transaction<{ entry_sequence: bigint; kind: "input" | "output" | "status"; text_value: string }[]>`SELECT entry_sequence, kind, text_value FROM t4_snapshot_entries WHERE principal_id = ${principalId} AND session_id = ${sessionId} ORDER BY entry_sequence DESC LIMIT 1000`; + const retention = await transaction`SELECT first_retained_sequence, latest_sequence FROM t4_event_retention WHERE principal_id = ${principalId} AND session_id = ${sessionId}`; + const latest = retention[0] ? persistedBigint(retention[0].latest_sequence, true) : 0n; + return { session: session(sessionRows[0]), cursor: this.watchCursor(principalId, sessionId, latest), entries: rows.reverse().map(row => ({ sequence: safeNumber(row.entry_sequence, true), kind: row.kind, text: row.text_value })) }; + }); } async eventWindow(principalId: string, sessionId: string, cursor: string | undefined, limit: number): Promise { - if (!await this.getSession(principalId, sessionId)) return undefined; - const retentionRows = await this.#sql`SELECT first_retained_sequence, latest_sequence FROM t4_event_retention WHERE principal_id = ${principalId} AND session_id = ${sessionId}`; - const first = retentionRows[0] ? persistedBigint(retentionRows[0].first_retained_sequence, true) : 0n; - const latest = retentionRows[0] ? persistedBigint(retentionRows[0].latest_sequence, true) : 0n; let after = 0n; if (cursor) { const decoded = this.#decodeCursor(cursor, "watch", principalId, sessionId); if (typeof decoded !== "bigint") throw new Error("invalid_cursor"); after = decoded; } - const resyncCursor = this.watchCursor(principalId, sessionId, latest); - if (first > 0n && after < first - 1n) return { events: [], cursor: resyncCursor, expired: true, resyncCursor }; - const rows = await this.#sql`SELECT sequence, event_type, payload FROM t4_events WHERE principal_id = ${principalId} AND session_id = ${sessionId} AND sequence > ${after} ORDER BY sequence LIMIT ${limit}`; - const delivered = rows.at(-1) ? persistedBigint(rows.at(-1)!.sequence) : after; - return { events: rows.map(row => ({ sequence: persistedBigint(row.sequence), type: row.event_type, payload: row.payload })), cursor: this.watchCursor(principalId, sessionId, delivered), expired: false, resyncCursor }; + return await this.#sql.begin("ISOLATION LEVEL REPEATABLE READ", async rawTransaction => { + const transaction = rawTransaction as unknown as SQL; + const sessionRows = await transaction<{ session_id: string }[]>`SELECT session_id FROM t4_session_intents WHERE principal_id = ${principalId} AND session_id = ${sessionId}`; + if (!sessionRows[0]) return undefined; + const retentionRows = await transaction`SELECT first_retained_sequence, latest_sequence FROM t4_event_retention WHERE principal_id = ${principalId} AND session_id = ${sessionId}`; + const first = retentionRows[0] ? persistedBigint(retentionRows[0].first_retained_sequence, true) : 0n; + const latest = retentionRows[0] ? persistedBigint(retentionRows[0].latest_sequence, true) : 0n; + const resyncCursor = this.watchCursor(principalId, sessionId, latest); + if (first > 0n && after < first - 1n) return { events: [], cursor: resyncCursor, expired: true, resyncCursor }; + const rows = await transaction`SELECT sequence, event_type, payload FROM t4_events WHERE principal_id = ${principalId} AND session_id = ${sessionId} AND sequence > ${after} ORDER BY sequence LIMIT ${limit}`; + const delivered = rows.at(-1) ? persistedBigint(rows.at(-1)!.sequence) : after; + return { events: rows.map(row => ({ sequence: persistedBigint(row.sequence), type: row.event_type, payload: row.payload })), cursor: this.watchCursor(principalId, sessionId, delivered), expired: false, resyncCursor }; + }); } watchCursor(principalId: string, sessionId: string, sequence: bigint): string { return this.#cursor("watch", principalId, sessionId, sequence); } @@ -350,17 +380,24 @@ export class PostgresLedger { } async acquireLease(ownerId: string, leaseSeconds = 30): Promise { - const rows = await this.#sql.begin("ISOLATION LEVEL SERIALIZABLE", async transaction => { - await transaction`INSERT INTO t4_owner_leases (lease_name, owner_id, epoch, expires_at) VALUES (${OUTBOX_LEASE}, ${ownerId}, 1, clock_timestamp() + (${leaseSeconds} * interval '1 second')) ON CONFLICT (lease_name) DO UPDATE SET owner_id = EXCLUDED.owner_id, epoch = t4_owner_leases.epoch + 1, expires_at = EXCLUDED.expires_at, updated_at = clock_timestamp()`; - return await transaction`SELECT owner_id, epoch FROM t4_owner_leases WHERE lease_name = ${OUTBOX_LEASE}`; + const rows = await this.#serializable(async transaction => { + const current = await transaction`SELECT owner_id, epoch, expires_at <= clock_timestamp() AS expired FROM t4_owner_leases WHERE lease_name = ${OUTBOX_LEASE} FOR UPDATE`; + if (!current[0]) { + return await transaction`INSERT INTO t4_owner_leases (lease_name, owner_id, epoch, expires_at) VALUES (${OUTBOX_LEASE}, ${ownerId}, 1, clock_timestamp() + (${leaseSeconds} * interval '1 second')) RETURNING owner_id, epoch, false AS expired`; + } + if (current[0].owner_id === ownerId && !current[0].expired) { + return await transaction`UPDATE t4_owner_leases SET expires_at = clock_timestamp() + (${leaseSeconds} * interval '1 second'), updated_at = clock_timestamp() WHERE lease_name = ${OUTBOX_LEASE} RETURNING owner_id, epoch, false AS expired`; + } + if (!current[0].expired) throw new Error("outbox lease is held by another owner"); + return await transaction`UPDATE t4_owner_leases SET owner_id = ${ownerId}, epoch = epoch + 1, expires_at = clock_timestamp() + (${leaseSeconds} * interval '1 second'), updated_at = clock_timestamp() WHERE lease_name = ${OUTBOX_LEASE} RETURNING owner_id, epoch, false AS expired`; }); return { ownerId: rows[0]!.owner_id, epoch: persistedBigint(rows[0]!.epoch) }; } async claimNext(lease: OwnerLease): Promise { - return await this.#sql.begin("ISOLATION LEVEL SERIALIZABLE", async transaction => { - const current = await transaction`SELECT owner_id, epoch FROM t4_owner_leases WHERE lease_name = ${OUTBOX_LEASE} FOR UPDATE`; - if (current[0]?.owner_id !== lease.ownerId || current[0].epoch !== lease.epoch) return undefined; + return await this.#serializable(async transaction => { + const current = await transaction`SELECT owner_id, epoch, expires_at <= clock_timestamp() AS expired, floor(extract(epoch FROM expires_at) * 1000)::bigint AS expires_at_ms FROM t4_owner_leases WHERE lease_name = ${OUTBOX_LEASE} FOR UPDATE`; + if (current[0]?.owner_id !== lease.ownerId || current[0].epoch !== lease.epoch || current[0].expired) return undefined; const rows = await transaction`SELECT outbox_id, command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation FROM t4_outbox WHERE (state = 'pending' OR (state = 'claimed' AND owner_epoch < ${lease.epoch})) AND next_attempt_at <= clock_timestamp() ORDER BY outbox_id FOR UPDATE SKIP LOCKED LIMIT 1`; const row = rows[0]; if (!row) return undefined; @@ -377,7 +414,7 @@ export class PostgresLedger { return undefined; } await transaction`UPDATE t4_outbox SET state = 'claimed', owner_id = ${lease.ownerId}, owner_epoch = ${lease.epoch}, attempts = attempts + 1, claimed_at = clock_timestamp(), updated_at = clock_timestamp() WHERE outbox_id = ${row.outbox_id}`; - return { outboxId: row.outbox_id, commandId: row.command_id, principalId: row.principal_id, idempotencyKey: row.idempotency_key, kind: row.mutation_kind, targetId: row.target_id, targetRevision: row.target_revision, mutation: row.mutation, ownerId: lease.ownerId, ownerEpoch: lease.epoch }; + return { outboxId: row.outbox_id, commandId: row.command_id, principalId: row.principal_id, idempotencyKey: row.idempotency_key, kind: row.mutation_kind, targetId: row.target_id, targetRevision: row.target_revision, mutation: row.mutation, ownerId: lease.ownerId, ownerEpoch: lease.epoch, expiresAt: safeNumber(current[0].expires_at_ms) }; }); } @@ -387,26 +424,42 @@ export class PostgresLedger { } async acknowledge(claim: OutboxClaim): Promise { - return await this.#sql.begin("ISOLATION LEVEL SERIALIZABLE", async transaction => { - const lease = await transaction`SELECT owner_id, epoch FROM t4_owner_leases WHERE lease_name = ${OUTBOX_LEASE} FOR UPDATE`; - if (lease[0]?.owner_id !== claim.ownerId || lease[0].epoch !== claim.ownerEpoch) return false; + return await this.#serializable(async transaction => { + const lease = await transaction`SELECT owner_id, epoch, expires_at <= clock_timestamp() AS expired FROM t4_owner_leases WHERE lease_name = ${OUTBOX_LEASE} FOR UPDATE`; + if (lease[0]?.owner_id !== claim.ownerId || lease[0].epoch !== claim.ownerEpoch || lease[0].expired) return false; const updated = await transaction<{ outbox_id: bigint }[]>`UPDATE t4_outbox SET state = 'applied', terminal_result = ${{ applied: true }}, updated_at = clock_timestamp() WHERE outbox_id = ${claim.outboxId} AND state = 'claimed' AND owner_id = ${claim.ownerId} AND owner_epoch = ${claim.ownerEpoch} RETURNING outbox_id`; if (!updated[0]) return false; await transaction`UPDATE t4_commands SET lifecycle_state = 'completed', updated_at = clock_timestamp() WHERE command_id = ${claim.commandId}`; - if (claim.kind === "workspace.create") await transaction`UPDATE t4_workspace_intents SET state = 'provisioning', updated_at = clock_timestamp() WHERE workspace_id = ${claim.targetId} AND revision = ${claim.targetRevision}`; + if (claim.kind === "workspace.create") await transaction`UPDATE t4_workspace_intents SET state = 'provisioning', updated_at = clock_timestamp() WHERE workspace_id = ${claim.targetId} AND revision = ${claim.targetRevision} AND state = 'accepted' AND deletion_requested = false`; if (claim.kind === "session.create") { - await transaction`UPDATE t4_session_intents SET state = 'provisioning', updated_at = clock_timestamp() WHERE session_id = ${claim.targetId} AND revision = ${claim.targetRevision}`; - await this.#appendEvent(transaction as unknown as SQL, claim.principalId, claim.targetId, "session", { state: "provisioning", revision: safeNumber(claim.targetRevision) }, claim.ownerEpoch); + const transitioned = await transaction<{ session_id: string }[]>`UPDATE t4_session_intents SET state = 'provisioning', updated_at = clock_timestamp() WHERE session_id = ${claim.targetId} AND revision = ${claim.targetRevision} AND state = 'accepted' AND cancellation_requested = false RETURNING session_id`; + if (transitioned[0]) await this.#appendEvent(transaction, claim.principalId, claim.targetId, "session", { state: "provisioning", revision: safeNumber(claim.targetRevision) }, claim.ownerEpoch); } return true; }); } async recordFailure(claim: OutboxClaim, message: string): Promise { - const result = await this.#sql<{ outbox_id: bigint }[]>`UPDATE t4_outbox item SET state = 'pending', owner_id = NULL, owner_epoch = NULL, last_error = ${message.slice(0, 1024)}, next_attempt_at = clock_timestamp() + interval '1 second', updated_at = clock_timestamp() FROM t4_owner_leases lease WHERE item.outbox_id = ${claim.outboxId} AND item.state = 'claimed' AND item.owner_id = ${claim.ownerId} AND item.owner_epoch = ${claim.ownerEpoch} AND lease.lease_name = ${OUTBOX_LEASE} AND lease.owner_id = ${claim.ownerId} AND lease.epoch = ${claim.ownerEpoch} RETURNING item.outbox_id`; + const result = await this.#sql<{ outbox_id: bigint }[]>`UPDATE t4_outbox item SET state = 'pending', owner_id = NULL, owner_epoch = NULL, last_error = ${message.slice(0, 1024)}, next_attempt_at = clock_timestamp() + interval '1 second', updated_at = clock_timestamp() FROM t4_owner_leases lease WHERE item.outbox_id = ${claim.outboxId} AND item.state = 'claimed' AND item.owner_id = ${claim.ownerId} AND item.owner_epoch = ${claim.ownerEpoch} AND lease.lease_name = ${OUTBOX_LEASE} AND lease.owner_id = ${claim.ownerId} AND lease.epoch = ${claim.ownerEpoch} AND lease.expires_at > clock_timestamp() RETURNING item.outbox_id`; return result.length === 1; } + async #serializable(operation: (transaction: SQL) => Promise): Promise { + for (let attempt = 1; ; attempt++) { + try { + return await this.#sql.begin("ISOLATION LEVEL SERIALIZABLE", async transaction => await operation(transaction as unknown as SQL)); + } catch (error) { + const code = typeof error === "object" && error !== null && "code" in error ? (error as { code?: unknown }).code : undefined; + if (attempt >= SERIALIZABLE_ATTEMPTS || code !== "40001" && code !== "40P01") throw error; + } + } + } + + async #lockIdempotency(transaction: SQL, principalId: string, operation: string, scope: string, idempotencyKey: string): Promise { + const identity = stableJson([principalId, operation, scope, idempotencyKey]); + await transaction`SELECT pg_advisory_xact_lock(hashtextextended(${identity}, 0))`; + } + #cursor(kind: "watch" | "page", principalId: string, scope: string, position: bigint | string): string { const payload = Buffer.from(JSON.stringify({ k: kind, p: createHash("sha256").update(principalId).digest("base64url"), s: scope, q: position.toString() }), "utf8").toString("base64url"); const signature = createHmac("sha256", this.#cursorSecret).update(payload).digest("base64url"); diff --git a/packages/cluster-server/src/main.ts b/packages/cluster-server/src/main.ts index aa6ce568..3b0397a6 100755 --- a/packages/cluster-server/src/main.ts +++ b/packages/cluster-server/src/main.ts @@ -46,35 +46,38 @@ export async function runClusterServer(env: Readonly | undefined; let signalsInstalled = false; try { - if (!config.postgres || !config.publicApiCredentialsFile) throw new Error("public API durable storage configuration is required"); - const [databaseUrl, publicApiSecrets] = await Promise.all([ - loadPostgresUrl(config), - loadPublicApiSecrets(config.publicApiCredentialsFile), - ]); - ledger = new PostgresLedger({ url: databaseUrl, cursorSecret: publicApiSecrets.cursorSecret }); - await ledger.migrate(); - const publicApi = new T4PublicApiV1({ ledger, authenticator: publicApiSecrets.authenticator }); - const outbox = new DurableOutboxWorker({ - ledger, - ownerId: config.epoch, - applier: new PublicKubernetesOutboxApplier({ client: kubernetes, hostRef: config.hostName }), - }); - outboxAbort = new AbortController(); - outboxTask = (async () => { - while (!outboxAbort!.signal.aborted) { - try { - await outbox.acquireLease(); - await outbox.drain(); - } catch (error) { - logger.warn("durable outbox recovery retrying", { condition: error instanceof Error ? error.name : "unknown", result: "failure" }); + if (config.postgres) { + if (!config.publicApiCredentialsFile || !config.publicApiRuntimeProfile) throw new Error("public API durable storage configuration is incomplete"); + const [databaseUrl, publicApiSecrets] = await Promise.all([ + loadPostgresUrl(config), + loadPublicApiSecrets(config.publicApiCredentialsFile), + ]); + ledger = new PostgresLedger({ url: databaseUrl, cursorSecret: publicApiSecrets.cursorSecret }); + await ledger.migrate(); + publicApi = new T4PublicApiV1({ ledger, authenticator: publicApiSecrets.authenticator, allowedOrigins: () => projection.allowedOrigins() }); + const outbox = new DurableOutboxWorker({ + ledger, + ownerId: config.epoch, + applier: new PublicKubernetesOutboxApplier({ client: kubernetes, hostRef: config.hostName, runtimeProfile: config.publicApiRuntimeProfile }), + }); + outboxAbort = new AbortController(); + outboxTask = (async () => { + while (!outboxAbort!.signal.aborted) { + try { + await outbox.acquireLease(); + await outbox.drain(); + } catch (error) { + logger.warn("durable outbox recovery retrying", { condition: error instanceof Error ? error.name : "unknown", result: "failure" }); + } + if (!outboxAbort!.signal.aborted) await Bun.sleep(1_000); } - if (!outboxAbort!.signal.aborted) await Bun.sleep(1_000); - } - })(); + })(); + } await runner.start(); const connector = new WebSocketPodHostConnector({ identityTokenFile: config.identityTokenPath }); authority = new SessionAuthorityRunner({ @@ -100,7 +103,7 @@ export async function runClusterServer(env: Readonly>; } +export interface OutboxApplyContext { + readonly claimIsCurrent?: () => Promise; + readonly signal?: AbortSignal; +} export interface KubernetesOutboxApplier { - apply(mutation: OutboxMutation, fence: OwnerLease): Promise; + apply(mutation: OutboxMutation, fence: OwnerLease, context?: OutboxApplyContext): Promise; } export interface DurableOutboxWorkerOptions { readonly ledger: PostgresLedger; @@ -53,8 +57,13 @@ export class DurableOutboxWorker { targetRevision: claim.targetRevision, payload: claim.mutation, }; + const timeout = Math.max(0, claim.expiresAt - Date.now()); + const signal = timeout === 0 ? AbortSignal.abort(new Error("outbox lease has expired")) : AbortSignal.timeout(timeout); try { - await this.#applier.apply(mutation, { ownerId: claim.ownerId, epoch: claim.ownerEpoch }); + await this.#applier.apply(mutation, { ownerId: claim.ownerId, epoch: claim.ownerEpoch }, { + claimIsCurrent: () => this.#ledger.claimIsCurrent(claim), + signal, + }); } catch (error) { if (!await this.#ledger.claimIsCurrent(claim)) return "fenced"; await this.#ledger.recordFailure(claim, error instanceof Error ? error.message : "outbox application failed"); diff --git a/packages/cluster-server/src/public-api-v1.ts b/packages/cluster-server/src/public-api-v1.ts index f427a5f6..88097316 100644 --- a/packages/cluster-server/src/public-api-v1.ts +++ b/packages/cluster-server/src/public-api-v1.ts @@ -24,7 +24,7 @@ export interface PrincipalAuthenticator { export interface T4PublicApiV1Options { readonly ledger: PostgresLedger; readonly authenticator: PrincipalAuthenticator; - readonly allowedOrigins?: readonly string[]; + readonly allowedOrigins?: readonly string[] | (() => readonly string[]); } const DISCOVERY: Discovery = { @@ -45,6 +45,7 @@ const RESOURCE_ID = /^[A-Za-z0-9][A-Za-z0-9._~-]{0,127}$/u; const IDEMPOTENCY_KEY = /^[A-Za-z0-9._~-]{16,128}$/u; const VERSION = /^[1-9][0-9]{0,3}(?:\.[0-9]+)?$/u; const LABEL_KEY = /^[a-z][a-z0-9.-]{0,62}$/u; +const CRD_OWNER = /^[A-Za-z0-9][A-Za-z0-9._:@/-]*$/u; const encoder = new TextEncoder(); interface ApiContext { readonly requestId: string; readonly principal: ApiPrincipal; } @@ -74,6 +75,17 @@ function deletedResponse(result: LedgerMutationResult, requestId: strin if (result.kind === "not_found") return notFound(requestId); return new Response(null, { status: 204, headers: responseHeaders({ "idempotency-replayed": result.kind === "replay" ? "true" : "false" }) }); } +function withinCodePointLimit(value: string, maximum: number): boolean { + let length = 0; + let offset = 0; + while (offset < value.length) { + const codePoint = value.codePointAt(offset); + if (codePoint === undefined) break; + offset += codePoint > 0xffff ? 2 : 1; + if (++length > maximum) return false; + } + return true; +} function labels(value: unknown): Record | undefined { if (value === undefined) return undefined; if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("labels must be an object"); @@ -81,13 +93,13 @@ function labels(value: unknown): Record | undefined { if (entries.length > 32) throw new Error("labels exceeds 32 properties"); const output: Record = {}; for (const [key, item] of entries) { - if (!LABEL_KEY.test(key) || typeof item !== "string" || item.length > 128) throw new Error("labels is invalid"); + if (!LABEL_KEY.test(key) || typeof item !== "string" || !withinCodePointLimit(item, 128)) throw new Error("labels is invalid"); output[key] = item; } return output; } function boundedString(value: unknown, name: string, maximum: number): string { - if (typeof value !== "string" || value.length < 1 || value.length > maximum) throw new Error(`${name} is invalid`); + if (typeof value !== "string" || value.length < 1 || !withinCodePointLimit(value, maximum)) throw new Error(`${name} is invalid`); return value; } function exactKeys(value: Record, allowed: readonly string[], required: readonly string[], minimum = 0): void { @@ -100,16 +112,54 @@ function integerQuery(value: string | null, fallback: number, minimum: number, m const parsed = Number(value); return Number.isSafeInteger(parsed) && parsed >= minimum && parsed <= maximum ? parsed : undefined; } +function acceptsEventStream(value: string | null): boolean { + if (!value) return false; + let selectedSpecificity = -1; + let selectedQuality = 0; + for (const range of value.split(",")) { + const [rawMediaType = "", ...parameters] = range.split(";"); + const mediaType = rawMediaType.trim().toLowerCase(); + const specificity = mediaType === "text/event-stream" ? 2 : mediaType === "text/*" ? 1 : mediaType === "*/*" ? 0 : -1; + if (specificity < 0) continue; + let quality = 1; + let valid = true; + let qualitySeen = false; + for (const rawParameter of parameters) { + const parameter = rawParameter.trim(); + const separator = parameter.indexOf("="); + if (separator < 1) { valid = false; break; } + const name = parameter.slice(0, separator).trim().toLowerCase(); + const parameterValue = parameter.slice(separator + 1).trim(); + if (name === "q") { + if (qualitySeen || !/^(?:0(?:\.[0-9]{0,3})?|1(?:\.0{0,3})?)$/u.test(parameterValue)) { valid = false; break; } + qualitySeen = true; + quality = Number(parameterValue); + } else if (!qualitySeen) { + valid = false; + break; + } + } + if (!valid) continue; + if (specificity > selectedSpecificity) { + selectedSpecificity = specificity; + selectedQuality = quality; + } else if (specificity === selectedSpecificity) { + selectedQuality = Math.max(selectedQuality, quality); + } + } + return selectedQuality > 0; +} export class T4PublicApiV1 { readonly #ledger: PostgresLedger; readonly #authenticator: PrincipalAuthenticator; - readonly #allowedOrigins?: ReadonlySet; + readonly #allowedOrigins: () => readonly string[]; constructor(options: T4PublicApiV1Options) { this.#ledger = options.ledger; this.#authenticator = options.authenticator; - this.#allowedOrigins = options.allowedOrigins ? new Set(options.allowedOrigins) : undefined; + const allowedOrigins = options.allowedOrigins; + this.#allowedOrigins = typeof allowedOrigins === "function" ? allowedOrigins : () => allowedOrigins ?? []; } async handle(request: Request): Promise { @@ -117,13 +167,13 @@ export class T4PublicApiV1 { try { if (new URL(request.url).protocol !== "https:") return apiError(requestId, 400, "https_required", "HTTPS is required"); const origin = request.headers.get("origin"); - if (origin && this.#allowedOrigins && !this.#allowedOrigins.has(origin)) return apiError(requestId, 400, "invalid_origin", "request origin is not allowed"); + if (origin && !this.#allowedOrigins().includes(origin)) return apiError(requestId, 400, "invalid_origin", "request origin is not allowed"); const authorization = request.headers.get("authorization"); if (!authorization?.startsWith("Bearer ") || authorization.length > 16_391) return apiError(requestId, 401, "unauthenticated", "valid bearer authentication is required"); const token = authorization.slice(7); if (!token || /\s/u.test(token)) return apiError(requestId, 401, "unauthenticated", "valid bearer authentication is required"); const principal = await this.#authenticator.authenticate(token); - if (!principal || !principal.id || principal.id.length > 256) return apiError(requestId, 401, "unauthenticated", "valid bearer authentication is required"); + if (!principal || typeof principal.id !== "string" || !CRD_OWNER.test(principal.id) || !withinCodePointLimit(principal.id, 256)) return apiError(requestId, 401, "unauthenticated", "valid bearer authentication is required"); const selectedVersion = request.headers.get("t4-api-version"); if (!selectedVersion || selectedVersion.length > 16 || !VERSION.test(selectedVersion)) return apiError(requestId, 400, "invalid_request", "T4-API-Version is required and must be valid"); if (selectedVersion.split(".", 1)[0] !== "1") return apiError(requestId, 406, "incompatible_version", "requested API major is unsupported", { supportedMajors: [1] }); @@ -187,9 +237,31 @@ export class T4PublicApiV1 { const contentType = request.headers.get("content-type")?.toLowerCase(); if (!contentType || !/^application\/json(?:\s*;\s*charset=utf-8)?$/u.test(contentType)) return { response: apiError(context.requestId, 400, "invalid_request", "request content type must be application/json") }; const announced = request.headers.get("content-length"); - if (announced && (!/^[0-9]+$/u.test(announced) || Number(announced) > DISCOVERY.limits.commandRequestBytesMax)) return { response: apiError(context.requestId, 400, "invalid_request", "request body exceeds the maximum size") }; - const text = await request.text(); - if (encoder.encode(text).byteLength > DISCOVERY.limits.commandRequestBytesMax) return { response: apiError(context.requestId, 400, "invalid_request", "request body exceeds the maximum size") }; + if (announced && (!/^[0-9]+$/u.test(announced) || Number(announced) > DISCOVERY.limits.commandRequestBytesMax)) { + await request.body?.cancel().catch(() => undefined); + return { response: apiError(context.requestId, 400, "invalid_request", "request body exceeds the maximum size") }; + } + const reader = request.body?.getReader(); + const decoder = new TextDecoder(); + let bytesRead = 0; + let text = ""; + if (reader) { + try { + while (true) { + const chunk = await reader.read(); + if (chunk.done) break; + bytesRead += chunk.value.byteLength; + if (bytesRead > DISCOVERY.limits.commandRequestBytesMax) { + await reader.cancel().catch(() => undefined); + return { response: apiError(context.requestId, 400, "invalid_request", "request body exceeds the maximum size") }; + } + text += decoder.decode(chunk.value, { stream: true }); + } + text += decoder.decode(); + } finally { + reader.releaseLock(); + } + } let value: unknown; try { value = JSON.parse(text); } catch { return { response: apiError(context.requestId, 400, "invalid_request", "request body is malformed JSON") }; } return value && typeof value === "object" && !Array.isArray(value) @@ -317,15 +389,21 @@ export class T4PublicApiV1 { } async #events(request: Request, url: URL, sessionId: string, context: ApiContext): Promise { const forbidden = this.#requireScope(context, "events.read"); if (forbidden) return forbidden; + if (!acceptsEventStream(request.headers.get("accept"))) return apiError(context.requestId, 406, "invalid_request", "Accept must allow text/event-stream"); const maxEvents = integerQuery(url.searchParams.get("maxEvents"), 100, 1, DISCOVERY.limits.watchEventsMax); const heartbeat = integerQuery(url.searchParams.get("heartbeatSeconds"), DISCOVERY.limits.heartbeatSeconds, 5, 60); if (!maxEvents || !heartbeat) return this.#validation(context, "watch bounds are invalid"); + const hasQueryCursor = url.searchParams.has("cursor"); const queryCursor = url.searchParams.get("cursor"); const headerCursor = request.headers.get("last-event-id"); - if (queryCursor && headerCursor && queryCursor !== headerCursor) return apiError(context.requestId, 400, "invalid_request", "cursor and Last-Event-ID must agree"); + if ((hasQueryCursor && !queryCursor) || (headerCursor !== null && !headerCursor)) return this.#validation(context, "watch cursor is invalid for this identity and session"); + if (hasQueryCursor && headerCursor !== null && queryCursor !== headerCursor) return apiError(context.requestId, 400, "invalid_request", "cursor and Last-Event-ID must agree"); let window; try { window = await this.#ledger.eventWindow(context.principal.id, sessionId, queryCursor ?? headerCursor ?? undefined, maxEvents); } - catch { return this.#validation(context, "watch cursor is invalid for this identity and session"); } + catch (error) { + if (error instanceof Error && error.message === "invalid_cursor") return this.#validation(context, "watch cursor is invalid for this identity and session"); + throw error; + } if (!window) return notFound(context.requestId); if (window.expired) return apiError(context.requestId, 410, "cursor_expired", "watch cursor is outside retained history", { resync: { snapshotUrl: `/v1/sessions/${sessionId}/snapshot`, cursor: window.resyncCursor } }); const frames: string[] = []; diff --git a/packages/cluster-server/src/public-kubernetes-applier.ts b/packages/cluster-server/src/public-kubernetes-applier.ts index 3ed2f0dc..dc668fe2 100644 --- a/packages/cluster-server/src/public-kubernetes-applier.ts +++ b/packages/cluster-server/src/public-kubernetes-applier.ts @@ -1,51 +1,89 @@ import { semanticResourceHash, KubernetesApiClient, KubernetesApiError } from "./kubernetes-client.ts"; import type { KubernetesResource } from "./kubernetes-projection.ts"; -import type { KubernetesOutboxApplier, OutboxMutation } from "./outbox-worker.ts"; +import type { KubernetesOutboxApplier, OutboxApplyContext, OutboxMutation } from "./outbox-worker.ts"; import type { OwnerLease } from "./ledger.ts"; export interface PublicKubernetesOutboxApplierOptions { readonly client: KubernetesApiClient; readonly hostRef: string; + readonly runtimeProfile?: string; } +type ResourceType = "t4workspaces" | "t4sessions"; +const API_VERSION = "cluster.t4.dev/v1alpha1"; +const LEDGER_ANNOTATIONS = [ + "cluster.t4.dev/ledger-command-id", + "cluster.t4.dev/ledger-outbox-token", + "cluster.t4.dev/ledger-owner", + "cluster.t4.dev/ledger-owner-epoch", + "cluster.t4.dev/ledger-semantic-hash", +] as const; + function record(value: unknown): Record { return value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; } +function boundedAnnotation(name: string, value: string): string { + if (!value || new TextEncoder().encode(value).byteLength > 256) throw new Error(`${name} is invalid`); + return value; +} function annotations(mutation: OutboxMutation, fence: OwnerLease): Record { + if (fence.epoch < 0n) throw new Error("ledger owner epoch is invalid"); return { - "cluster.t4.dev/ledger-command-id": mutation.commandId, - "cluster.t4.dev/ledger-outbox-token": mutation.idempotencyToken, - "cluster.t4.dev/ledger-owner": fence.ownerId, + "cluster.t4.dev/ledger-command-id": boundedAnnotation("ledger command id", mutation.commandId), + "cluster.t4.dev/ledger-outbox-token": boundedAnnotation("ledger outbox token", mutation.idempotencyToken), + "cluster.t4.dev/ledger-owner": boundedAnnotation("ledger owner", fence.ownerId), "cluster.t4.dev/ledger-owner-epoch": fence.epoch.toString(), "cluster.t4.dev/ledger-semantic-hash": semanticResourceHash(mutation.payload), }; } -function assertedEpoch(resource: KubernetesResource, fence: OwnerLease): void { - const value = resource.metadata.annotations?.["cluster.t4.dev/ledger-owner-epoch"]; - if (value && /^(?:0|[1-9][0-9]*)$/u.test(value) && BigInt(value) > fence.epoch) throw new Error("Kubernetes resource is fenced by a newer ledger owner"); +function assertResourceIdentity(resource: KubernetesResource, resourceType: ResourceType, name: string): void { + const kind = resourceType === "t4workspaces" ? "T4Workspace" : "T4Session"; + if (resource.apiVersion !== API_VERSION || resource.kind !== kind || resource.metadata?.name !== name) + throw new Error("Kubernetes resource identity is invalid"); +} +function assertLedgerIdentity(resource: KubernetesResource, fence: OwnerLease): void { + const values = resource.metadata.annotations; + if (!values || LEDGER_ANNOTATIONS.some(key => typeof values[key] !== "string" || values[key].length === 0)) + throw new Error("Kubernetes ledger identity is invalid"); + const epoch = values["cluster.t4.dev/ledger-owner-epoch"]!; + if (!/^(?:0|[1-9][0-9]*)$/u.test(epoch)) throw new Error("Kubernetes ledger identity is invalid"); + const ownerEpoch = BigInt(epoch); + if (ownerEpoch > fence.epoch || ownerEpoch === fence.epoch && values["cluster.t4.dev/ledger-owner"] !== fence.ownerId) + throw new Error("Kubernetes resource is fenced by another ledger owner"); } export class PublicKubernetesOutboxApplier implements KubernetesOutboxApplier { readonly #client: KubernetesApiClient; readonly #hostRef: string; + readonly #runtimeProfile: string; constructor(options: PublicKubernetesOutboxApplierOptions) { this.#client = options.client; - this.#hostRef = options.hostRef; + this.#hostRef = boundedAnnotation("Kubernetes host reference", options.hostRef); + const runtimeProfile = options.runtimeProfile ?? "default"; + if (runtimeProfile.length > 64 || !/^[a-z0-9](?:[a-z0-9._-]*[a-z0-9])?$/u.test(runtimeProfile)) + throw new Error("Kubernetes runtime profile is invalid"); + this.#runtimeProfile = runtimeProfile; + } + + async apply(mutation: OutboxMutation, fence: OwnerLease, context?: OutboxApplyContext): Promise { + if (mutation.kind === "workspace.create") return await this.#createWorkspace(mutation, fence, context); + if (mutation.kind === "session.create") return await this.#createSession(mutation, fence, context); + if (mutation.kind === "workspace.patch") return await this.#patchWorkspace(mutation, fence, context); + if (mutation.kind === "session.patch") return await this.#patchSession(mutation, fence, context); + if (mutation.kind === "command.submit") return await this.#submitCommand(mutation, fence, context); + if (mutation.kind === "workspace.delete") return await this.#delete("t4workspaces", mutation, fence, context); + return await this.#delete("t4sessions", mutation, fence, context); } - async apply(mutation: OutboxMutation, fence: OwnerLease): Promise { - if (mutation.kind === "workspace.create") return await this.#createWorkspace(mutation, fence); - if (mutation.kind === "session.create") return await this.#createSession(mutation, fence); - if (mutation.kind === "workspace.patch") return await this.#patchWorkspace(mutation, fence); - if (mutation.kind === "session.patch") return await this.#patchSession(mutation, fence); - if (mutation.kind === "command.submit") return await this.#submitCommand(mutation, fence); - if (mutation.kind === "workspace.delete") return await this.#delete("t4workspaces", mutation, fence); - return await this.#delete("t4sessions", mutation, fence); + async #assertCurrent(context?: OutboxApplyContext): Promise { + if (context?.signal?.aborted) throw context.signal.reason; + if (context?.claimIsCurrent && !await context.claimIsCurrent()) throw new Error("outbox lease is no longer current"); + if (context?.signal?.aborted) throw context.signal.reason; } - async #createWorkspace(mutation: OutboxMutation, fence: OwnerLease): Promise { + async #createWorkspace(mutation: OutboxMutation, fence: OwnerLease, context?: OutboxApplyContext): Promise { const body = { - apiVersion: "cluster.t4.dev/v1alpha1", + apiVersion: API_VERSION, kind: "T4Workspace", metadata: { name: mutation.targetId, annotations: annotations(mutation, fence) }, spec: { @@ -56,71 +94,163 @@ export class PublicKubernetesOutboxApplier implements KubernetesOutboxApplier { size: "20Gi", }, }; - await this.#createIdempotently("t4workspaces", mutation, fence, body); + await this.#createIdempotently("t4workspaces", mutation, fence, body, context); } - async #createSession(mutation: OutboxMutation, fence: OwnerLease): Promise { + async #createSession(mutation: OutboxMutation, fence: OwnerLease, context?: OutboxApplyContext): Promise { + const workspaceId = String(mutation.payload.workspaceId); + await this.#ownedWorkspace(workspaceId, mutation, fence, context?.signal); const body = { - apiVersion: "cluster.t4.dev/v1alpha1", + apiVersion: API_VERSION, kind: "T4Session", metadata: { name: mutation.targetId, annotations: annotations(mutation, fence) }, spec: { hostRef: this.#hostRef, - workspaceRef: String(mutation.payload.workspaceId), + workspaceRef: workspaceId, title: String(mutation.payload.title), - runtimeProfile: "omp-default", + runtimeProfile: this.#runtimeProfile, guiEnabled: true, }, }; - await this.#createIdempotently("t4sessions", mutation, fence, body); + await this.#createIdempotently("t4sessions", mutation, fence, body, context); } - async #patchWorkspace(mutation: OutboxMutation, fence: OwnerLease): Promise { - const current = await this.#owned("t4workspaces", mutation, fence); + async #patchWorkspace(mutation: OutboxMutation, fence: OwnerLease, context?: OutboxApplyContext): Promise { + const current = await this.#owned("t4workspaces", mutation, fence, context?.signal); const spec = record(current.spec); + await this.#assertCurrent(context); await this.#client.patch("t4workspaces", mutation.targetId, { - metadata: { annotations: annotations(mutation, fence) }, + metadata: { resourceVersion: current.metadata.resourceVersion, annotations: annotations(mutation, fence) }, spec: { ...spec, displayName: String(mutation.payload.name) }, - }); + }, context?.signal); } - async #patchSession(mutation: OutboxMutation, fence: OwnerLease): Promise { - await this.#owned("t4sessions", mutation, fence); + async #patchSession(mutation: OutboxMutation, fence: OwnerLease, context?: OutboxApplyContext): Promise { + const current = await this.#owned("t4sessions", mutation, fence, context?.signal); + await this.#assertCurrent(context); await this.#client.patch("t4sessions", mutation.targetId, { - metadata: { annotations: annotations(mutation, fence) }, + metadata: { resourceVersion: current.metadata.resourceVersion, annotations: annotations(mutation, fence) }, spec: { title: String(mutation.payload.title) }, - }); + }, context?.signal); } - async #submitCommand(mutation: OutboxMutation, fence: OwnerLease): Promise { - await this.#owned("t4sessions", mutation, fence); + async #submitCommand(mutation: OutboxMutation, fence: OwnerLease, context?: OutboxApplyContext): Promise { + const current = await this.#owned("t4sessions", mutation, fence, context?.signal); + await this.#assertCurrent(context); await this.#client.patch("t4sessions", mutation.targetId, { - metadata: { annotations: { ...annotations(mutation, fence), "cluster.t4.dev/pending-command": JSON.stringify(mutation.payload) } }, - }); + metadata: { + resourceVersion: current.metadata.resourceVersion, + annotations: { + ...annotations(mutation, fence), + "cluster.t4.dev/pending-command-id": boundedAnnotation("pending command id", mutation.commandId), + "cluster.t4.dev/pending-command-epoch": fence.epoch.toString(), + }, + }, + }, context?.signal); } - async #delete(resourceType: "t4workspaces" | "t4sessions", mutation: OutboxMutation, fence: OwnerLease): Promise { + async #delete(resourceType: ResourceType, mutation: OutboxMutation, fence: OwnerLease, context?: OutboxApplyContext): Promise { let current: KubernetesResource; - try { current = await this.#owned(resourceType, mutation, fence); } - catch (error) { + try { + current = await this.#owned(resourceType, mutation, fence, context?.signal); + } catch (error) { if (error instanceof KubernetesApiError && error.status === 404) return; throw error; } - if (!current.metadata.uid || !current.metadata.resourceVersion) throw new Error("Kubernetes delete precondition is unavailable"); - try { await this.#client.delete(resourceType, mutation.targetId, { uid: current.metadata.uid, resourceVersion: current.metadata.resourceVersion }); } - catch (error) { if (!(error instanceof KubernetesApiError) || error.status !== 404) throw error; } + if (!current.metadata.uid) throw new Error("Kubernetes delete precondition is unavailable"); + await this.#assertCurrent(context); + try { + await this.#client.delete(resourceType, mutation.targetId, { uid: current.metadata.uid, resourceVersion: current.metadata.resourceVersion! }, context?.signal); + } catch (error) { + if (!(error instanceof KubernetesApiError) || error.status !== 404) throw error; + } } - async #owned(resourceType: "t4workspaces" | "t4sessions", mutation: OutboxMutation, fence: OwnerLease): Promise { - const current = await this.#client.get(resourceType, mutation.targetId); - assertedEpoch(current, fence); + async #owned(resourceType: ResourceType, mutation: OutboxMutation, fence: OwnerLease, signal?: AbortSignal): Promise { + const current = await this.#client.get(resourceType, mutation.targetId, signal); + assertResourceIdentity(current, resourceType, mutation.targetId); + assertLedgerIdentity(current, fence); + if (!current.metadata.resourceVersion) throw new Error("Kubernetes resource identity is missing resourceVersion"); const spec = record(current.spec); if (spec.hostRef !== this.#hostRef) throw new Error("Kubernetes resource belongs to another cluster host"); - if (resourceType === "t4workspaces" && spec.owner !== mutation.principalId) throw new Error("Kubernetes workspace belongs to another principal"); + if (resourceType === "t4workspaces") { + if (spec.owner !== mutation.principalId) throw new Error("Kubernetes workspace belongs to another principal"); + } else { + const workspaceId = spec.workspaceRef; + if (typeof workspaceId !== "string" || !workspaceId || mutation.payload.workspaceId !== undefined && mutation.payload.workspaceId !== workspaceId) + throw new Error("Kubernetes session workspace identity is invalid"); + await this.#ownedWorkspace(workspaceId, mutation, fence, signal); + } return current; } - async #createIdempotently(resourceType: "t4workspaces" | "t4sessions", mutation: OutboxMutation, fence: OwnerLease, body: KubernetesResource): Promise { - try { await this.#client.create(resourceType, body); } - catch (error) { - if (!(error instanceof KubernetesApiError) || error.status !== 409) throw error; - const current = await this.#client.get(resourceType, mutation.targetId); - assertedEpoch(current, fence); - const currentAnnotations = current.metadata.annotations ?? {}; - if (currentAnnotations["cluster.t4.dev/ledger-outbox-token"] !== mutation.idempotencyToken || currentAnnotations["cluster.t4.dev/ledger-semantic-hash"] !== semanticResourceHash(mutation.payload)) throw new Error("Kubernetes resource conflicts with durable ledger intent"); + async #ownedWorkspace(workspaceId: string, mutation: OutboxMutation, fence: OwnerLease, signal?: AbortSignal): Promise { + const workspace = await this.#client.get("t4workspaces", workspaceId, signal); + assertResourceIdentity(workspace, "t4workspaces", workspaceId); + assertLedgerIdentity(workspace, fence); + const spec = record(workspace.spec); + if (spec.hostRef !== this.#hostRef || spec.owner !== mutation.principalId) + throw new Error("Kubernetes workspace ownership identity is invalid"); + return workspace; + } + async #cleanupCreate(resourceType: ResourceType, mutation: OutboxMutation, fence: OwnerLease, body: KubernetesResource, created?: KubernetesResource): Promise { + const signal = AbortSignal.timeout(5_000); + let current = created; + if (!current?.metadata?.uid || !current.metadata.resourceVersion) { + try { + current = await this.#client.get(resourceType, mutation.targetId, signal); + } catch (error) { + if (error instanceof KubernetesApiError && error.status === 404) return; + throw error; + } + } + const expectedAnnotations = annotations(mutation, fence); + const currentAnnotations = current.metadata.annotations ?? {}; + if (current.apiVersion !== API_VERSION || current.kind !== (resourceType === "t4workspaces" ? "T4Workspace" : "T4Session") || current.metadata.name !== mutation.targetId + || semanticResourceHash(record(current.spec)) !== semanticResourceHash(body.spec) + || LEDGER_ANNOTATIONS.some(key => currentAnnotations[key] !== expectedAnnotations[key])) return; + if (!current.metadata.uid || !current.metadata.resourceVersion) return; + try { + await this.#client.delete(resourceType, mutation.targetId, { uid: current.metadata.uid, resourceVersion: current.metadata.resourceVersion }, signal); + } catch (error) { + if (!(error instanceof KubernetesApiError) || error.status !== 404 && error.status !== 409) throw error; + } + } + + async #createIdempotently(resourceType: ResourceType, mutation: OutboxMutation, fence: OwnerLease, body: KubernetesResource, context?: OutboxApplyContext): Promise { + await this.#assertCurrent(context); + let created: KubernetesResource; + try { + created = await this.#client.create(resourceType, body, context?.signal); + } catch (error) { + if (!(error instanceof KubernetesApiError) || error.status !== 409) { + await this.#cleanupCreate(resourceType, mutation, fence, body); + throw error; + } + try { + const current = await this.#client.get(resourceType, mutation.targetId, context?.signal); + assertResourceIdentity(current, resourceType, mutation.targetId); + assertLedgerIdentity(current, fence); + if (!current.metadata.resourceVersion) throw new Error("resourceVersion conflict"); + const spec = record(current.spec); + if (semanticResourceHash(spec) !== semanticResourceHash(body.spec)) throw new Error("spec conflict"); + if (spec.hostRef !== this.#hostRef) throw new Error("host conflict"); + if (resourceType === "t4workspaces" && spec.owner !== mutation.principalId) throw new Error("principal conflict"); + if (resourceType === "t4sessions" && spec.workspaceRef !== mutation.payload.workspaceId) throw new Error("workspace conflict"); + if (resourceType === "t4sessions") await this.#ownedWorkspace(String(spec.workspaceRef), mutation, fence, context?.signal); + const currentAnnotations = current.metadata.annotations ?? {}; + const expectedAnnotations = annotations(mutation, fence); + for (const key of ["cluster.t4.dev/ledger-command-id", "cluster.t4.dev/ledger-outbox-token", "cluster.t4.dev/ledger-semantic-hash"] as const) { + if (currentAnnotations[key] !== expectedAnnotations[key]) throw new Error("ledger conflict"); + } + await this.#assertCurrent(context); + await this.#client.patch(resourceType, mutation.targetId, { + metadata: { resourceVersion: current.metadata.resourceVersion, annotations: expectedAnnotations }, + }, context?.signal); + await this.#assertCurrent(context); + return; + } catch { + throw new Error("Kubernetes resource conflicts with durable ledger intent"); + } + } + try { + await this.#assertCurrent(context); + } catch (error) { + await this.#cleanupCreate(resourceType, mutation, fence, body, created); + throw error; } } } diff --git a/packages/cluster-server/test/durable-gateway-postgres.test.ts b/packages/cluster-server/test/durable-gateway-postgres.test.ts index 0b5c3aad..6c74d643 100644 --- a/packages/cluster-server/test/durable-gateway-postgres.test.ts +++ b/packages/cluster-server/test/durable-gateway-postgres.test.ts @@ -124,6 +124,7 @@ async function newLedger(overrides: Partial = {}): Promise if (!PostgresLedger) throw new Error("PostgreSQL ledger implementation is unavailable"); const ledger = new PostgresLedger(ledgerOptions(overrides)); await ledger.migrate(); + await admin`UPDATE t4_owner_leases SET expires_at = clock_timestamp() - interval '1 second' WHERE lease_name = 'gateway-outbox'`; return ledger; } @@ -304,6 +305,7 @@ postgresSuite("PostgreSQL-backed public T4 API v1", () => { const claim = await stale.claimNext(); expect(claim).toBeDefined(); + await admin`UPDATE t4_owner_leases SET expires_at = clock_timestamp() - interval '1 second' WHERE lease_name = 'gateway-outbox'`; const current = newWorker(ledger, "current", kubernetes); const currentLease = await current.acquireLease(); kubernetes.currentEpoch = currentLease.epoch; @@ -510,24 +512,30 @@ postgresSuite("PostgreSQL-backed public T4 API v1", () => { `); await admin`SELECT pg_advisory_lock(741406)`; const first = call(api, "POST", "/v1/workspaces", { key: "absent-idempotency-key-01", body: { name: "one durable winner" } }); - await waitForDatabase(async () => (await admin`SELECT last_value FROM t4_test_absent_key_attempt` as Array<{ last_value: bigint }>)[0]?.last_value === 1n); - const second = call(api, "POST", "/v1/workspaces", { key: "absent-idempotency-key-01", body: { name: "one durable winner" } }); - await waitForDatabase(async () => { - const [sequence, active] = await Promise.all([ - admin`SELECT last_value FROM t4_test_absent_key_attempt` as Promise>, - admin`SELECT count(*) AS count FROM pg_stat_activity WHERE datname = current_database() AND state = 'active' AND pid <> pg_backend_pid()` as Promise>, - ]); - return sequence[0]?.last_value === 2n || (active[0]?.count ?? 0n) >= 2n; - }); - await admin`SELECT pg_advisory_unlock(741406)`; - const responses = await Promise.all([first, second]); - expect(responses.map(value => value.status).sort()).toEqual([200, 202]); - expect(await json(responses[0]!)).toEqual(await json(responses[1]!)); + let second: Promise | undefined; + let responses: Response[] | undefined; + try { + await waitForDatabase(async () => (await admin`SELECT is_called FROM t4_test_absent_key_attempt` as Array<{ is_called: boolean }>)[0]?.is_called === true); + const secondRequest = call(api, "POST", "/v1/workspaces", { key: "absent-idempotency-key-01", body: { name: "one durable winner" } }); + second = secondRequest; + await waitForDatabase(async () => { + const blocked = await admin`SELECT count(*) AS count FROM pg_stat_activity WHERE datname = current_database() AND wait_event = 'advisory' AND query LIKE 'SELECT pg_advisory_xact_lock(hashtextextended%'` as Array<{ count: bigint }>; + return blocked[0]?.count === 1n; + }); + await admin`SELECT pg_advisory_unlock(741406)`; + responses = await Promise.all([first, secondRequest]); + } finally { + await admin`SELECT pg_advisory_unlock(741406)`; + await Promise.allSettled([first, ...(second ? [second] : [])]); + await admin.unsafe("DROP TRIGGER IF EXISTS t4_test_pause_absent_key_trigger ON t4_outbox; DROP FUNCTION IF EXISTS t4_test_pause_first_absent_key(); DROP SEQUENCE IF EXISTS t4_test_absent_key_attempt"); + } + expect(responses).toBeDefined(); + expect(responses!.map(value => value.status).sort()).toEqual([200, 202]); + expect(await json(responses![0]!)).toEqual(await json(responses![1]!)); const rows = await admin`SELECT (SELECT count(*) FROM t4_commands WHERE idempotency_key = 'absent-idempotency-key-01') AS commands, (SELECT count(*) FROM t4_outbox WHERE idempotency_key = 'absent-idempotency-key-01') AS outbox` as Array<{ commands: bigint; outbox: bigint }>; expect(rows[0]).toEqual({ commands: 1n, outbox: 1n }); - await admin.unsafe("DROP TRIGGER t4_test_pause_absent_key_trigger ON t4_outbox; DROP FUNCTION t4_test_pause_first_absent_key(); DROP SEQUENCE t4_test_absent_key_attempt"); await ledger.close(); }); @@ -683,32 +691,37 @@ postgresSuite("PostgreSQL-backed public T4 API v1", () => { await call(api, "POST", `/v1/sessions/${session.id}/commands`, { key: "atomic-snapshot-command-1", body: { command: "before snapshot" } }); await admin`SELECT pg_advisory_lock(741407)`; await admin.unsafe(` + CREATE SEQUENCE t4_test_snapshot_pause_reached; ALTER TABLE t4_event_retention RENAME TO t4_event_retention_base; CREATE FUNCTION t4_test_pause_snapshot(value bigint) RETURNS bigint LANGUAGE plpgsql VOLATILE AS $$ - BEGIN PERFORM pg_advisory_xact_lock(741407); RETURN value; END $$; + BEGIN PERFORM nextval('t4_test_snapshot_pause_reached'); PERFORM pg_advisory_xact_lock(741407); RETURN value; END $$; CREATE VIEW t4_event_retention AS SELECT principal_id, session_id, first_retained_sequence, t4_test_pause_snapshot(latest_sequence) AS latest_sequence, updated_at FROM t4_event_retention_base; `); const pendingSnapshot = call(api, "GET", `/v1/sessions/${session.id}/snapshot`); - await waitForDatabase(async () => { - const rows = await admin`SELECT count(*) AS count FROM pg_stat_activity WHERE wait_event_type = 'Advisory' AND query LIKE '%t4_event_retention%'` as Array<{ count: bigint }>; - return (rows[0]?.count ?? 0n) >= 1n; - }); const concurrent = new SQL(DATABASE_URL!, { max: 1, bigint: true }); const concurrentCommandId = "cmd_concurrent_snapshot_boundary"; - await concurrent.begin(async transaction => { - const entries = await transaction<{ next: bigint }[]>`SELECT COALESCE(MAX(entry_sequence), -1) + 1 AS next FROM t4_snapshot_entries WHERE principal_id = 'owner@example.com' AND session_id = ${session.id}`; - await transaction`INSERT INTO t4_snapshot_entries (principal_id, session_id, entry_sequence, kind, text_value) VALUES ('owner@example.com', ${session.id}, ${entries[0]?.next ?? 0n}, 'input', 'concurrent snapshot command')`; - const events = await transaction<{ sequence: bigint }[]>`INSERT INTO t4_events (principal_id, session_id, event_type, payload, owner_epoch) VALUES ('owner@example.com', ${session.id}, 'command', ${{ commandId: concurrentCommandId, state: "accepted" }}, 0) RETURNING sequence`; - await transaction`UPDATE t4_event_retention_base SET latest_sequence = ${events[0]!.sequence}, updated_at = clock_timestamp() WHERE principal_id = 'owner@example.com' AND session_id = ${session.id}`; - }); - await concurrent.close(); - await admin`SELECT pg_advisory_unlock(741407)`; - const snapshotResponse = await pendingSnapshot; - expect(snapshotResponse.status).toBe(200); - const snapshot = await json(snapshotResponse); - await admin.unsafe("DROP VIEW t4_event_retention; ALTER TABLE t4_event_retention_base RENAME TO t4_event_retention; DROP FUNCTION t4_test_pause_snapshot(bigint)"); + let snapshotResponse: Response | undefined; + try { + await waitForDatabase(async () => (await admin`SELECT is_called FROM t4_test_snapshot_pause_reached` as Array<{ is_called: boolean }>)[0]?.is_called === true); + await concurrent.begin(async transaction => { + const entries = await transaction<{ next: bigint }[]>`SELECT COALESCE(MAX(entry_sequence), -1) + 1 AS next FROM t4_snapshot_entries WHERE principal_id = 'owner@example.com' AND session_id = ${session.id}`; + await transaction`INSERT INTO t4_snapshot_entries (principal_id, session_id, entry_sequence, kind, text_value) VALUES ('owner@example.com', ${session.id}, ${entries[0]?.next ?? 0n}, 'input', 'concurrent snapshot command')`; + const events = await transaction<{ sequence: bigint }[]>`INSERT INTO t4_events (principal_id, session_id, event_type, payload, owner_epoch) VALUES ('owner@example.com', ${session.id}, 'command', ${{ commandId: concurrentCommandId, state: "accepted" }}, 0) RETURNING sequence`; + await transaction`UPDATE t4_event_retention_base SET latest_sequence = ${events[0]!.sequence}, updated_at = clock_timestamp() WHERE principal_id = 'owner@example.com' AND session_id = ${session.id}`; + }); + await admin`SELECT pg_advisory_unlock(741407)`; + snapshotResponse = await pendingSnapshot; + } finally { + await admin`SELECT pg_advisory_unlock(741407)`; + await Promise.allSettled([pendingSnapshot]); + await concurrent.close(); + await admin.unsafe("DROP VIEW IF EXISTS t4_event_retention; ALTER TABLE t4_event_retention_base RENAME TO t4_event_retention; DROP FUNCTION IF EXISTS t4_test_pause_snapshot(bigint); DROP SEQUENCE IF EXISTS t4_test_snapshot_pause_reached"); + } + expect(snapshotResponse).toBeDefined(); + expect(snapshotResponse!.status).toBe(200); + const snapshot = await json(snapshotResponse!); const afterBoundary = await call(api, "GET", `/v1/sessions/${session.id}/events?cursor=${encodeURIComponent(snapshot.cursor)}`, { headers: { accept: "text/event-stream" } }); expect(afterBoundary.status).toBe(200); expect(eventData(await afterBoundary.text()).map(value => value.commandId)).toContain(concurrentCommandId); @@ -726,29 +739,34 @@ postgresSuite("PostgreSQL-backed public T4 API v1", () => { expect(target[0]).toBeDefined(); await admin`SELECT pg_advisory_lock(741408)`; await admin.unsafe(` + CREATE SEQUENCE t4_test_event_pause_reached; ALTER TABLE t4_events RENAME TO t4_events_base; CREATE FUNCTION t4_test_pause_event(value bigint) RETURNS bigint LANGUAGE plpgsql VOLATILE AS $$ - BEGIN PERFORM pg_advisory_xact_lock(741408); RETURN value; END $$; + BEGIN PERFORM nextval('t4_test_event_pause_reached'); PERFORM pg_advisory_xact_lock(741408); RETURN value; END $$; CREATE VIEW t4_events AS SELECT t4_test_pause_event(sequence) AS sequence, principal_id, session_id, event_type, payload, owner_epoch, created_at FROM t4_events_base; `); const pendingWindow = call(api, "GET", `/v1/sessions/${session.id}/events?cursor=${encodeURIComponent(snapshot.cursor)}&maxEvents=1`, { headers: { accept: "text/event-stream" } }); - await waitForDatabase(async () => { - const rows = await admin`SELECT count(*) AS count FROM pg_stat_activity WHERE wait_event_type = 'Advisory' AND query LIKE '%t4_events%'` as Array<{ count: bigint }>; - return (rows[0]?.count ?? 0n) >= 1n; - }); const concurrent = new SQL(DATABASE_URL!, { max: 1, bigint: true }); - await concurrent.begin(async transaction => { - await transaction`DELETE FROM t4_events_base WHERE sequence = ${target[0]!.sequence}`; - await transaction`UPDATE t4_event_retention SET first_retained_sequence = ${target[0]!.sequence + 1n} WHERE principal_id = 'owner@example.com' AND session_id = ${session.id}`; - }); - await concurrent.close(); - await admin`SELECT pg_advisory_unlock(741408)`; - const response = await pendingWindow; - expect(response.status).toBe(200); - await admin.unsafe("DROP VIEW t4_events; ALTER TABLE t4_events_base RENAME TO t4_events; DROP FUNCTION t4_test_pause_event(bigint)"); - expect(eventData(await response.text()).map(value => value.commandId)).toContain(command.commandId); + let response: Response | undefined; + try { + await waitForDatabase(async () => (await admin`SELECT is_called FROM t4_test_event_pause_reached` as Array<{ is_called: boolean }>)[0]?.is_called === true); + await concurrent.begin(async transaction => { + await transaction`DELETE FROM t4_events_base WHERE sequence = ${target[0]!.sequence}`; + await transaction`UPDATE t4_event_retention SET first_retained_sequence = ${target[0]!.sequence + 1n} WHERE principal_id = 'owner@example.com' AND session_id = ${session.id}`; + }); + await admin`SELECT pg_advisory_unlock(741408)`; + response = await pendingWindow; + } finally { + await admin`SELECT pg_advisory_unlock(741408)`; + await Promise.allSettled([pendingWindow]); + await concurrent.close(); + await admin.unsafe("DROP VIEW IF EXISTS t4_events; ALTER TABLE t4_events_base RENAME TO t4_events; DROP FUNCTION IF EXISTS t4_test_pause_event(bigint); DROP SEQUENCE IF EXISTS t4_test_event_pause_reached"); + } + expect(response).toBeDefined(); + expect(response!.status).toBe(200); + expect(eventData(await response!.text()).map(value => value.commandId)).toContain(command.commandId); await ledger.close(); }); @@ -781,7 +799,7 @@ postgresSuite("PostgreSQL-backed public T4 API v1", () => { const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "sse-accept-workspace-01", body: { name: "sse accept" } })); const session = await json(await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: "sse-accept-session-001", body: { title: "sse accept" } })); for (const accept of [undefined, "application/json", "text/plain;q=1, text/event-stream;q=0"]) { - const response = await call(api, "GET", `/v1/sessions/${session.id}/events`, { ...(accept ? { headers: { accept } } : {}) }); + const response = await call(api, "GET", `/v1/sessions/${session.id}/events`, accept ? { headers: { accept } } : {}); expect(response.status).toBe(406); } expect((await call(api, "GET", `/v1/sessions/${session.id}/events`, { headers: { accept: "text/event-stream, application/json;q=0.5" } })).status).toBe(200); diff --git a/packages/cluster-server/test/kubernetes-client.test.ts b/packages/cluster-server/test/kubernetes-client.test.ts index addeabf6..290dd564 100644 --- a/packages/cluster-server/test/kubernetes-client.test.ts +++ b/packages/cluster-server/test/kubernetes-client.test.ts @@ -296,20 +296,28 @@ describe("durable public Kubernetes applier", () => { const requiredAnnotations = { "cluster.t4.dev/ledger-command-id": createMutation.commandId, "cluster.t4.dev/ledger-outbox-token": createMutation.idempotencyToken, - "cluster.t4.dev/ledger-owner": fence.ownerId, - "cluster.t4.dev/ledger-owner-epoch": fence.epoch.toString(), + "cluster.t4.dev/ledger-owner": "expired-worker", + "cluster.t4.dev/ledger-owner-epoch": (fence.epoch - 1n).toString(), "cluster.t4.dev/ledger-semantic-hash": semanticResourceHash(createMutation.payload), }; const exactExisting = { apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Workspace", metadata: { name: "workspace-one", resourceVersion: "9", annotations: requiredAnnotations }, - spec: { hostRef: "primary", owner: PRINCIPAL, displayName: "created" }, + spec: { hostRef: "primary", owner: PRINCIPAL, displayName: "created", retentionPolicy: "Retain", size: "20Gi" }, }; const exact = conflictFetch(exactExisting); await new PublicKubernetesOutboxApplier({ client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch: exact.fetch }), hostRef: "primary", }).apply(createMutation, fence); + const adoption = JSON.parse(String(exact.requests[2]?.init?.body)); + expect(adoption.metadata).toMatchObject({ + resourceVersion: "9", + annotations: { + "cluster.t4.dev/ledger-owner": fence.ownerId, + "cluster.t4.dev/ledger-owner-epoch": fence.epoch.toString(), + }, + }); for (const conflicting of [ { ...exactExisting, kind: "T4Session" }, @@ -341,7 +349,11 @@ describe("durable public Kubernetes applier", () => { } }); it("uses the configured host-admitted runtime profile for durable session creation", async () => { - const values = recordingFetch([{}]); + const values = recordingFetch([{ + apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Workspace", + metadata: { name: "workspace-one", resourceVersion: "55", annotations: priorAnnotations }, + spec: { hostRef: "primary", owner: PRINCIPAL }, + }, {}]); const options = { client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch: values.fetch }), hostRef: "primary", @@ -354,7 +366,7 @@ describe("durable public Kubernetes applier", () => { targetId: "session-one", payload: { id: "session-one", workspaceId: "workspace-one", title: "session", revision: 1 }, }, fence); - const created = JSON.parse(String(values.requests[0]?.init?.body)); + const created = JSON.parse(String(values.requests[1]?.init?.body)); expect(created.spec.runtimeProfile).toBe("review-admitted"); }); @@ -362,11 +374,11 @@ describe("durable public Kubernetes applier", () => { const secretCommand = `printf ${"secret-payload".repeat(8_192)}`; const values = recordingFetch([{ apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Session", - metadata: { name: "session-one", resourceVersion: "61", annotations: { "cluster.t4.dev/ledger-owner-epoch": "6" } }, + metadata: { name: "session-one", resourceVersion: "61", annotations: priorAnnotations }, spec: { hostRef: "primary", workspaceRef: "workspace-one" }, }, { apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Workspace", - metadata: { name: "workspace-one", resourceVersion: "60" }, + metadata: { name: "workspace-one", resourceVersion: "60", annotations: priorAnnotations }, spec: { hostRef: "primary", owner: PRINCIPAL }, }, {}]); const applier = new PublicKubernetesOutboxApplier({ @@ -402,7 +414,7 @@ describe("durable public Kubernetes applier", () => { spec: { hostRef: "primary", workspaceRef: "workspace-one" }, }, { apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Workspace", - metadata: { name: "workspace-one", resourceVersion: "70" }, + metadata: { name: "workspace-one", resourceVersion: "70", annotations: priorAnnotations }, spec: { hostRef: "primary", owner: PRINCIPAL }, }, {}]); const applier = new PublicKubernetesOutboxApplier({ From 9aebb070a45b8e704a7c7a3033871896ea679c5e Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 03:25:36 +0000 Subject: [PATCH 123/131] fix(cluster): close durable gateway race gaps --- packages/cluster-server/src/ledger.ts | 21 +++++++-- .../test/durable-gateway-postgres.test.ts | 46 +++++++++++++------ 2 files changed, 48 insertions(+), 19 deletions(-) diff --git a/packages/cluster-server/src/ledger.ts b/packages/cluster-server/src/ledger.ts index 3403ee65..b1d4e60c 100644 --- a/packages/cluster-server/src/ledger.ts +++ b/packages/cluster-server/src/ledger.ts @@ -111,6 +111,18 @@ function safeNumber(value: unknown, allowZero = false): number { if (decoded > MAX_SAFE_BIGINT) throw new Error("persisted ledger integer is outside the public API range"); return Number(decoded); } +function postgresSqlState(error: unknown): string | undefined { + if (!error || typeof error !== "object") return undefined; + const value = error as { readonly code?: unknown; readonly errno?: unknown }; + if (typeof value.code === "string" && /^[0-9A-Z]{5}$/u.test(value.code)) return value.code; + return typeof value.errno === "string" && /^[0-9A-Z]{5}$/u.test(value.errno) ? value.errno : undefined; +} +function isIdempotencyUniqueRace(error: unknown): boolean { + if (postgresSqlState(error) !== "23505" || !error || typeof error !== "object") return false; + const value = error as { readonly constraint?: unknown; readonly constraint_name?: unknown }; + const constraint = value.constraint ?? value.constraint_name; + return constraint === "t4_commands_principal_id_operation_target_scope_idempotency_key"; +} function labels(value: Record): Record | undefined { return Object.keys(value).length > 0 ? value : undefined; } @@ -352,7 +364,7 @@ export class PostgresLedger { async eventWindow(principalId: string, sessionId: string, cursor: string | undefined, limit: number): Promise { let after = 0n; - if (cursor) { + if (cursor !== undefined) { const decoded = this.#decodeCursor(cursor, "watch", principalId, sessionId); if (typeof decoded !== "bigint") throw new Error("invalid_cursor"); after = decoded; @@ -365,7 +377,7 @@ export class PostgresLedger { const first = retentionRows[0] ? persistedBigint(retentionRows[0].first_retained_sequence, true) : 0n; const latest = retentionRows[0] ? persistedBigint(retentionRows[0].latest_sequence, true) : 0n; const resyncCursor = this.watchCursor(principalId, sessionId, latest); - if (first > 0n && after < first - 1n) return { events: [], cursor: resyncCursor, expired: true, resyncCursor }; + if (cursor !== undefined && first > 0n && after < first - 1n) return { events: [], cursor: resyncCursor, expired: true, resyncCursor }; const rows = await transaction`SELECT sequence, event_type, payload FROM t4_events WHERE principal_id = ${principalId} AND session_id = ${sessionId} AND sequence > ${after} ORDER BY sequence LIMIT ${limit}`; const delivered = rows.at(-1) ? persistedBigint(rows.at(-1)!.sequence) : after; return { events: rows.map(row => ({ sequence: persistedBigint(row.sequence), type: row.event_type, payload: row.payload })), cursor: this.watchCursor(principalId, sessionId, delivered), expired: false, resyncCursor }; @@ -449,8 +461,9 @@ export class PostgresLedger { try { return await this.#sql.begin("ISOLATION LEVEL SERIALIZABLE", async transaction => await operation(transaction as unknown as SQL)); } catch (error) { - const code = typeof error === "object" && error !== null && "code" in error ? (error as { code?: unknown }).code : undefined; - if (attempt >= SERIALIZABLE_ATTEMPTS || code !== "40001" && code !== "40P01") throw error; + const code = postgresSqlState(error); + const retryable = code === "40001" || code === "40P01" || isIdempotencyUniqueRace(error); + if (attempt >= SERIALIZABLE_ATTEMPTS || !retryable) throw error; } } } diff --git a/packages/cluster-server/test/durable-gateway-postgres.test.ts b/packages/cluster-server/test/durable-gateway-postgres.test.ts index 6c74d643..12e71557 100644 --- a/packages/cluster-server/test/durable-gateway-postgres.test.ts +++ b/packages/cluster-server/test/durable-gateway-postgres.test.ts @@ -510,7 +510,8 @@ postgresSuite("PostgreSQL-backed public T4 API v1", () => { CREATE TRIGGER t4_test_pause_absent_key_trigger BEFORE INSERT ON t4_outbox FOR EACH ROW EXECUTE FUNCTION t4_test_pause_first_absent_key(); `); - await admin`SELECT pg_advisory_lock(741406)`; + const blocker = new SQL(DATABASE_URL!, { max: 1, bigint: true }); + await blocker`SELECT pg_advisory_lock(741406)`; const first = call(api, "POST", "/v1/workspaces", { key: "absent-idempotency-key-01", body: { name: "one durable winner" } }); let second: Promise | undefined; let responses: Response[] | undefined; @@ -522,10 +523,10 @@ postgresSuite("PostgreSQL-backed public T4 API v1", () => { const blocked = await admin`SELECT count(*) AS count FROM pg_stat_activity WHERE datname = current_database() AND wait_event = 'advisory' AND query LIKE 'SELECT pg_advisory_xact_lock(hashtextextended%'` as Array<{ count: bigint }>; return blocked[0]?.count === 1n; }); - await admin`SELECT pg_advisory_unlock(741406)`; + await blocker`SELECT pg_advisory_unlock(741406)`; responses = await Promise.all([first, secondRequest]); } finally { - await admin`SELECT pg_advisory_unlock(741406)`; + await blocker.close(); await Promise.allSettled([first, ...(second ? [second] : [])]); await admin.unsafe("DROP TRIGGER IF EXISTS t4_test_pause_absent_key_trigger ON t4_outbox; DROP FUNCTION IF EXISTS t4_test_pause_first_absent_key(); DROP SEQUENCE IF EXISTS t4_test_absent_key_attempt"); } @@ -602,18 +603,27 @@ postgresSuite("PostgreSQL-backed public T4 API v1", () => { (SELECT count(*) FROM t4_events WHERE session_id = ${session.id}) AS events` as Array<{ commands: bigint; outbox: bigint; events: bigint }>; const repeated = await call(api, "POST", `/v1/sessions/${session.id}/cancel`, { key: "monotonic-cancel-key-002" }); expect(repeated.status).toBe(202); + const repeatedReplay = await call(api, "POST", `/v1/sessions/${session.id}/cancel`, { key: "monotonic-cancel-key-002" }); + expect(repeatedReplay.status).toBe(200); + expect(await json(repeatedReplay)).toEqual(firstCancel); expect(await json(repeated)).toEqual(firstCancel); expect((await call(api, "PATCH", `/v1/sessions/${session.id}`, { key: "monotonic-patch-key-0001", ifMatch: String(firstCancel.revision), body: { title: "forbidden" } })).status).toBe(404); expect((await call(api, "POST", `/v1/sessions/${session.id}/commands`, { key: "monotonic-command-key-01", body: { command: "forbidden" } })).status).toBe(404); await admin`UPDATE t4_session_intents SET state = 'cancelled' WHERE session_id = ${session.id}`; - expect((await call(api, "POST", `/v1/sessions/${session.id}/cancel`, { key: "monotonic-cancel-key-003" })).status).toBe(202); - expect((await call(api, "DELETE", `/v1/sessions/${session.id}`, { key: "monotonic-delete-key-001" })).status).toBe(204); + const terminalCancel = await call(api, "POST", `/v1/sessions/${session.id}/cancel`, { key: "monotonic-cancel-key-003" }); + expect(terminalCancel.status).toBe(202); + const firstDelete = await call(api, "DELETE", `/v1/sessions/${session.id}`, { key: "monotonic-delete-key-001" }); + expect(firstDelete.status).toBe(204); + expect(firstDelete.headers.get("idempotency-replayed")).toBe("false"); + const deleteReplay = await call(api, "DELETE", `/v1/sessions/${session.id}`, { key: "monotonic-delete-key-001" }); + expect(deleteReplay.status).toBe(204); + expect(deleteReplay.headers.get("idempotency-replayed")).toBe("true"); const after = await admin`SELECT (SELECT count(*) FROM t4_commands WHERE target_scope = ${`session:${session.id}`}) AS commands, (SELECT count(*) FROM t4_outbox WHERE target_id = ${session.id}) AS outbox, (SELECT count(*) FROM t4_events WHERE session_id = ${session.id}) AS events, (SELECT revision FROM t4_session_intents WHERE session_id = ${session.id}) AS revision` as Array<{ commands: bigint; outbox: bigint; events: bigint; revision: bigint }>; - expect(after[0]).toMatchObject({ ...before[0], revision: BigInt(firstCancel.revision) }); + expect(after[0]).toEqual({ commands: 4n, outbox: 3n, events: 3n, revision: BigInt(firstCancel.revision) }); await ledger.close(); }); @@ -689,7 +699,8 @@ postgresSuite("PostgreSQL-backed public T4 API v1", () => { const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "atomic-snapshot-workspace", body: { name: "atomic snapshot" } })); const session = await json(await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: "atomic-snapshot-session-1", body: { title: "atomic snapshot" } })); await call(api, "POST", `/v1/sessions/${session.id}/commands`, { key: "atomic-snapshot-command-1", body: { command: "before snapshot" } }); - await admin`SELECT pg_advisory_lock(741407)`; + const blocker = new SQL(DATABASE_URL!, { max: 1, bigint: true }); + await blocker`SELECT pg_advisory_lock(741407)`; await admin.unsafe(` CREATE SEQUENCE t4_test_snapshot_pause_reached; ALTER TABLE t4_event_retention RENAME TO t4_event_retention_base; @@ -711,10 +722,10 @@ postgresSuite("PostgreSQL-backed public T4 API v1", () => { const events = await transaction<{ sequence: bigint }[]>`INSERT INTO t4_events (principal_id, session_id, event_type, payload, owner_epoch) VALUES ('owner@example.com', ${session.id}, 'command', ${{ commandId: concurrentCommandId, state: "accepted" }}, 0) RETURNING sequence`; await transaction`UPDATE t4_event_retention_base SET latest_sequence = ${events[0]!.sequence}, updated_at = clock_timestamp() WHERE principal_id = 'owner@example.com' AND session_id = ${session.id}`; }); - await admin`SELECT pg_advisory_unlock(741407)`; + await blocker`SELECT pg_advisory_unlock(741407)`; snapshotResponse = await pendingSnapshot; } finally { - await admin`SELECT pg_advisory_unlock(741407)`; + await blocker.close(); await Promise.allSettled([pendingSnapshot]); await concurrent.close(); await admin.unsafe("DROP VIEW IF EXISTS t4_event_retention; ALTER TABLE t4_event_retention_base RENAME TO t4_event_retention; DROP FUNCTION IF EXISTS t4_test_pause_snapshot(bigint); DROP SEQUENCE IF EXISTS t4_test_snapshot_pause_reached"); @@ -737,7 +748,8 @@ postgresSuite("PostgreSQL-backed public T4 API v1", () => { const command = await json(await call(api, "POST", `/v1/sessions/${session.id}/commands`, { key: "atomic-window-command-001", body: { command: "must survive concurrent prune" } })); const target = await admin`SELECT sequence FROM t4_events WHERE principal_id = 'owner@example.com' AND session_id = ${session.id} AND payload->>'commandId' = ${command.commandId}` as Array<{ sequence: bigint }>; expect(target[0]).toBeDefined(); - await admin`SELECT pg_advisory_lock(741408)`; + const blocker = new SQL(DATABASE_URL!, { max: 1, bigint: true }); + await blocker`SELECT pg_advisory_lock(741408)`; await admin.unsafe(` CREATE SEQUENCE t4_test_event_pause_reached; ALTER TABLE t4_events RENAME TO t4_events_base; @@ -756,10 +768,10 @@ postgresSuite("PostgreSQL-backed public T4 API v1", () => { await transaction`DELETE FROM t4_events_base WHERE sequence = ${target[0]!.sequence}`; await transaction`UPDATE t4_event_retention SET first_retained_sequence = ${target[0]!.sequence + 1n} WHERE principal_id = 'owner@example.com' AND session_id = ${session.id}`; }); - await admin`SELECT pg_advisory_unlock(741408)`; + await blocker`SELECT pg_advisory_unlock(741408)`; response = await pendingWindow; } finally { - await admin`SELECT pg_advisory_unlock(741408)`; + await blocker.close(); await Promise.allSettled([pendingWindow]); await concurrent.close(); await admin.unsafe("DROP VIEW IF EXISTS t4_events; ALTER TABLE t4_events_base RENAME TO t4_events; DROP FUNCTION IF EXISTS t4_test_pause_event(bigint); DROP SEQUENCE IF EXISTS t4_test_event_pause_reached"); @@ -774,12 +786,13 @@ postgresSuite("PostgreSQL-backed public T4 API v1", () => { const ledger = await newLedger(); const api = newApi(ledger); let pulls = 0; + let cancelledAt: number | undefined; const body = new ReadableStream({ pull(controller) { pulls++; - if (pulls <= 2) controller.enqueue(new Uint8Array(600_000)); - else controller.error(new Error("body reader requested data after the limit")); + controller.enqueue(new Uint8Array(600_000)); }, + cancel() { cancelledAt = pulls; }, }); const response = await api.handle(new Request("https://t4.example.test/v1/workspaces", { method: "POST", @@ -789,7 +802,10 @@ postgresSuite("PostgreSQL-backed public T4 API v1", () => { } as RequestInit & { duplex: "half" })); expect(response.status).toBe(400); expect((await json(response)).error.message).toContain("maximum size"); - expect(pulls).toBe(2); + expect(cancelledAt).toBeDefined(); + const pullsAfterCancellation = pulls; + await Promise.resolve(); + expect(pulls).toBe(pullsAfterCancellation); await ledger.close(); }); From a387783d0795c56ca105e9f20f5c053351291053 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 03:31:45 +0000 Subject: [PATCH 124/131] test(cluster): stabilize durable gateway concurrency proof --- .../test/durable-gateway-postgres.test.ts | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/cluster-server/test/durable-gateway-postgres.test.ts b/packages/cluster-server/test/durable-gateway-postgres.test.ts index 12e71557..fd204fde 100644 --- a/packages/cluster-server/test/durable-gateway-postgres.test.ts +++ b/packages/cluster-server/test/durable-gateway-postgres.test.ts @@ -597,10 +597,6 @@ postgresSuite("PostgreSQL-backed public T4 API v1", () => { const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "monotonic-workspace-key-1", body: { name: "monotonic" } })); const session = await json(await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: "monotonic-session-key-01", body: { title: "monotonic" } })); const firstCancel = await json(await call(api, "POST", `/v1/sessions/${session.id}/cancel`, { key: "monotonic-cancel-key-001" })); - const before = await admin`SELECT - (SELECT count(*) FROM t4_commands WHERE target_scope = ${`session:${session.id}`}) AS commands, - (SELECT count(*) FROM t4_outbox WHERE target_id = ${session.id}) AS outbox, - (SELECT count(*) FROM t4_events WHERE session_id = ${session.id}) AS events` as Array<{ commands: bigint; outbox: bigint; events: bigint }>; const repeated = await call(api, "POST", `/v1/sessions/${session.id}/cancel`, { key: "monotonic-cancel-key-002" }); expect(repeated.status).toBe(202); const repeatedReplay = await call(api, "POST", `/v1/sessions/${session.id}/cancel`, { key: "monotonic-cancel-key-002" }); @@ -749,7 +745,14 @@ postgresSuite("PostgreSQL-backed public T4 API v1", () => { const target = await admin`SELECT sequence FROM t4_events WHERE principal_id = 'owner@example.com' AND session_id = ${session.id} AND payload->>'commandId' = ${command.commandId}` as Array<{ sequence: bigint }>; expect(target[0]).toBeDefined(); const blocker = new SQL(DATABASE_URL!, { max: 1, bigint: true }); - await blocker`SELECT pg_advisory_lock(741408)`; + const releaseBlocker = Promise.withResolvers(); + const blockerReady = Promise.withResolvers(); + const blockerTask = blocker.begin(async transaction => { + await transaction`SELECT pg_advisory_xact_lock(741408)`; + blockerReady.resolve(); + await releaseBlocker.promise; + }); + await blockerReady.promise; await admin.unsafe(` CREATE SEQUENCE t4_test_event_pause_reached; ALTER TABLE t4_events RENAME TO t4_events_base; @@ -768,9 +771,12 @@ postgresSuite("PostgreSQL-backed public T4 API v1", () => { await transaction`DELETE FROM t4_events_base WHERE sequence = ${target[0]!.sequence}`; await transaction`UPDATE t4_event_retention SET first_retained_sequence = ${target[0]!.sequence + 1n} WHERE principal_id = 'owner@example.com' AND session_id = ${session.id}`; }); - await blocker`SELECT pg_advisory_unlock(741408)`; + releaseBlocker.resolve(); + await blockerTask; response = await pendingWindow; } finally { + releaseBlocker.resolve(); + await blockerTask; await blocker.close(); await Promise.allSettled([pendingWindow]); await concurrent.close(); From c55e9bcd1d539d8d1ecae93587859e1e22832d16 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 03:39:45 +0000 Subject: [PATCH 125/131] test(cluster): pause atomic event retention boundary --- .../test/durable-gateway-postgres.test.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/cluster-server/test/durable-gateway-postgres.test.ts b/packages/cluster-server/test/durable-gateway-postgres.test.ts index fd204fde..8f2c87b2 100644 --- a/packages/cluster-server/test/durable-gateway-postgres.test.ts +++ b/packages/cluster-server/test/durable-gateway-postgres.test.ts @@ -755,12 +755,12 @@ postgresSuite("PostgreSQL-backed public T4 API v1", () => { await blockerReady.promise; await admin.unsafe(` CREATE SEQUENCE t4_test_event_pause_reached; - ALTER TABLE t4_events RENAME TO t4_events_base; + ALTER TABLE t4_event_retention RENAME TO t4_event_retention_base; CREATE FUNCTION t4_test_pause_event(value bigint) RETURNS bigint LANGUAGE plpgsql VOLATILE AS $$ BEGIN PERFORM nextval('t4_test_event_pause_reached'); PERFORM pg_advisory_xact_lock(741408); RETURN value; END $$; - CREATE VIEW t4_events AS - SELECT t4_test_pause_event(sequence) AS sequence, principal_id, session_id, event_type, payload, owner_epoch, created_at - FROM t4_events_base; + CREATE VIEW t4_event_retention AS + SELECT principal_id, session_id, first_retained_sequence, t4_test_pause_event(latest_sequence) AS latest_sequence, updated_at + FROM t4_event_retention_base; `); const pendingWindow = call(api, "GET", `/v1/sessions/${session.id}/events?cursor=${encodeURIComponent(snapshot.cursor)}&maxEvents=1`, { headers: { accept: "text/event-stream" } }); const concurrent = new SQL(DATABASE_URL!, { max: 1, bigint: true }); @@ -768,8 +768,8 @@ postgresSuite("PostgreSQL-backed public T4 API v1", () => { try { await waitForDatabase(async () => (await admin`SELECT is_called FROM t4_test_event_pause_reached` as Array<{ is_called: boolean }>)[0]?.is_called === true); await concurrent.begin(async transaction => { - await transaction`DELETE FROM t4_events_base WHERE sequence = ${target[0]!.sequence}`; - await transaction`UPDATE t4_event_retention SET first_retained_sequence = ${target[0]!.sequence + 1n} WHERE principal_id = 'owner@example.com' AND session_id = ${session.id}`; + await transaction`DELETE FROM t4_events WHERE sequence = ${target[0]!.sequence}`; + await transaction`UPDATE t4_event_retention_base SET first_retained_sequence = ${target[0]!.sequence + 1n} WHERE principal_id = 'owner@example.com' AND session_id = ${session.id}`; }); releaseBlocker.resolve(); await blockerTask; @@ -780,7 +780,7 @@ postgresSuite("PostgreSQL-backed public T4 API v1", () => { await blocker.close(); await Promise.allSettled([pendingWindow]); await concurrent.close(); - await admin.unsafe("DROP VIEW IF EXISTS t4_events; ALTER TABLE t4_events_base RENAME TO t4_events; DROP FUNCTION IF EXISTS t4_test_pause_event(bigint); DROP SEQUENCE IF EXISTS t4_test_event_pause_reached"); + await admin.unsafe("DROP VIEW IF EXISTS t4_event_retention; ALTER TABLE t4_event_retention_base RENAME TO t4_event_retention; DROP FUNCTION IF EXISTS t4_test_pause_event(bigint); DROP SEQUENCE IF EXISTS t4_test_event_pause_reached"); } expect(response).toBeDefined(); expect(response!.status).toBe(200); From 265bfa4dbaaf3da465c4270ef95c4b9bc6ab3189 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 03:45:18 +0000 Subject: [PATCH 126/131] test(cluster): isolate durable gateway scenarios --- packages/cluster-server/test/durable-gateway-postgres.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cluster-server/test/durable-gateway-postgres.test.ts b/packages/cluster-server/test/durable-gateway-postgres.test.ts index 8f2c87b2..42e57968 100644 --- a/packages/cluster-server/test/durable-gateway-postgres.test.ts +++ b/packages/cluster-server/test/durable-gateway-postgres.test.ts @@ -124,7 +124,7 @@ async function newLedger(overrides: Partial = {}): Promise if (!PostgresLedger) throw new Error("PostgreSQL ledger implementation is unavailable"); const ledger = new PostgresLedger(ledgerOptions(overrides)); await ledger.migrate(); - await admin`UPDATE t4_owner_leases SET expires_at = clock_timestamp() - interval '1 second' WHERE lease_name = 'gateway-outbox'`; + await admin.unsafe("TRUNCATE t4_outbox, t4_event_retention, t4_events, t4_snapshot_entries, t4_session_intents, t4_workspace_intents, t4_commands, t4_owner_leases RESTART IDENTITY CASCADE"); return ledger; } From 18925e927a44aea8f931d34f5b206535d4587701 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 03:46:39 +0000 Subject: [PATCH 127/131] test(cluster): preserve durable restart state --- .../test/durable-gateway-postgres.test.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/cluster-server/test/durable-gateway-postgres.test.ts b/packages/cluster-server/test/durable-gateway-postgres.test.ts index 42e57968..7f4e8e06 100644 --- a/packages/cluster-server/test/durable-gateway-postgres.test.ts +++ b/packages/cluster-server/test/durable-gateway-postgres.test.ts @@ -1,5 +1,5 @@ import { SQL } from "bun"; -import { afterAll, beforeAll, describe, expect, it } from "vite-plus/test"; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vite-plus/test"; interface DurableLedgerOptions { readonly url: string; readonly cursorSecret: string; @@ -124,7 +124,6 @@ async function newLedger(overrides: Partial = {}): Promise if (!PostgresLedger) throw new Error("PostgreSQL ledger implementation is unavailable"); const ledger = new PostgresLedger(ledgerOptions(overrides)); await ledger.migrate(); - await admin.unsafe("TRUNCATE t4_outbox, t4_event_retention, t4_events, t4_snapshot_entries, t4_session_intents, t4_workspace_intents, t4_commands, t4_owner_leases RESTART IDENTITY CASCADE"); return ledger; } @@ -204,6 +203,14 @@ beforeAll(async () => { DurableOutboxWorker = (modules[2] as { DurableOutboxWorker?: WorkerConstructor } | undefined)?.DurableOutboxWorker; }); +beforeEach(async () => { + if (!PostgresLedger) return; + const ledger = new PostgresLedger(ledgerOptions()); + await ledger.migrate(); + await ledger.close(); + await admin.unsafe("TRUNCATE t4_outbox, t4_event_retention, t4_events, t4_snapshot_entries, t4_session_intents, t4_workspace_intents, t4_commands, t4_owner_leases RESTART IDENTITY CASCADE"); +}); + afterAll(async () => { await admin?.close(); }); From fca3d2c9c97fb5e01d1da4e4e858880379e13d0e Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 03:52:31 +0000 Subject: [PATCH 128/131] test(cluster): preserve lease owner across restart --- packages/cluster-server/test/durable-gateway-postgres.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cluster-server/test/durable-gateway-postgres.test.ts b/packages/cluster-server/test/durable-gateway-postgres.test.ts index 7f4e8e06..35865e9f 100644 --- a/packages/cluster-server/test/durable-gateway-postgres.test.ts +++ b/packages/cluster-server/test/durable-gateway-postgres.test.ts @@ -243,7 +243,7 @@ postgresSuite("PostgreSQL-backed public T4 API v1", () => { expect(replay.status).toBe(200); expect(replay.headers.get("idempotency-replayed")).toBe("true"); expect(await json(replay)).toEqual(acceptedBody); - const secondWorker = newWorker(ledger, "worker-b", kubernetes); + const secondWorker = newWorker(ledger, "worker-a", kubernetes); const secondLease = await secondWorker.acquireLease(); kubernetes.currentEpoch = secondLease.epoch; expect(await secondWorker.drain()).toBe(0); From 366fae8eade9f11a857bef2f45b614f21929c17f Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 04:01:18 +0000 Subject: [PATCH 129/131] test(chart): match quoted secret projection keys --- packages/cluster-operator/charttests/chart_contract_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cluster-operator/charttests/chart_contract_test.go b/packages/cluster-operator/charttests/chart_contract_test.go index 076aa664..b351b926 100644 --- a/packages/cluster-operator/charttests/chart_contract_test.go +++ b/packages/cluster-operator/charttests/chart_contract_test.go @@ -405,8 +405,8 @@ func TestPublicAPIRequiresDurableSecretAndExplicitPostgresEgress(t *testing.T) { "name: T4_PUBLIC_API_RUNTIME_PROFILE", "value: \"review-admitted\"", "secretName: \"t4-public-api\"", - "key: postgres-url", - "key: credentials.json", + "key: \"postgres-url\"", + "key: \"credentials.json\"", "/var/run/secrets/t4-public-api/postgres-url", "/var/run/secrets/t4-public-api/credentials.json", ) From 8c802506e4bc7041da7751b910356cd0d3e4f79c Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 04:43:13 +0000 Subject: [PATCH 130/131] test(cluster): cover review repair contracts --- .../charttests/chart_contract_test.go | 13 + .../test/durable-gateway-postgres.test.ts | 363 ++++++++++++++++++ .../test/kubernetes-client.test.ts | 285 ++++++++++++++ 3 files changed, 661 insertions(+) diff --git a/packages/cluster-operator/charttests/chart_contract_test.go b/packages/cluster-operator/charttests/chart_contract_test.go index b351b926..ba034664 100644 --- a/packages/cluster-operator/charttests/chart_contract_test.go +++ b/packages/cluster-operator/charttests/chart_contract_test.go @@ -501,6 +501,19 @@ func TestPublicAPIRenderGuardsSurviveSchemaBypass(t *testing.T) { } } +func TestPublicAPIRejectsCollidingSecretProjectionKeysUnderSchemaBypass(t *testing.T) { + values := append(enabledValues(), + "--skip-schema-validation", + "--set", "publicApi.enabled=true", + "--set-string", "publicApi.existingSecret=t4-public-api", + "--set-string", "publicApi.postgresURLKey=credentials.json", + "--set-string", "publicApi.credentialsKey=credentials.json", + "--set", "networkPolicy.postgresCIDRs[0]=198.51.100.20/32", + "--set", "networkPolicy.postgresPorts[0]=5432", + ) + helmTemplateMustFailBeforeWorkloads(t, "publicApi.postgresURLKey and publicApi.credentialsKey must be different", values...) +} + func TestUnauthenticatedOMPControllerCannotReadSecrets(t *testing.T) { output := helmTemplate(t, enabledValues()...) controllerRole := documentContaining(t, output, "name: \"release-name-t4-cluster-controller\"") diff --git a/packages/cluster-server/test/durable-gateway-postgres.test.ts b/packages/cluster-server/test/durable-gateway-postgres.test.ts index 35865e9f..386ccb44 100644 --- a/packages/cluster-server/test/durable-gateway-postgres.test.ts +++ b/packages/cluster-server/test/durable-gateway-postgres.test.ts @@ -36,8 +36,32 @@ interface KubernetesOutboxApplier { interface OutboxClaim { readonly outboxId: bigint; readonly commandId: string; + readonly principalId: string; + readonly idempotencyKey: string; + readonly kind: string; + readonly targetId: string; + readonly targetRevision: bigint; + readonly mutation: Readonly>; readonly ownerId: string; readonly ownerEpoch: bigint; + readonly expiresAt: number; +} +interface AuthoritativeKubernetesStatusObservation { + readonly resourceId: string; + readonly principalId: string; + readonly workspaceId?: string; + readonly uid: string; + readonly resourceVersion: string; + readonly generation: bigint; + readonly observedGeneration: bigint; + readonly phase: "Pending" | "Ready" | "Running" | "Failed" | "Terminating" | "Unknown"; +} +interface AuthoritativeKubernetesStatusIngress { + ingestKubernetesStatus( + lease: { ownerId: string; epoch: bigint }, + collection: "t4workspaces" | "t4sessions", + observation: AuthoritativeKubernetesStatusObservation, + ): Promise; } interface DurableOutboxWorkerLike { acquireLease(): Promise<{ ownerId: string; epoch: bigint }>; @@ -52,8 +76,10 @@ type WorkerConstructor = new (options: { ledger: PostgresLedgerLike; ownerId: st const DATABASE_URL = process.env.T4_TEST_POSTGRES_URL; const VERSION = "1"; const OWNER_TOKEN = "opaque-owner-token-with-sufficient-entropy"; +const MIXED_CASE_OWNER_TOKEN = "Opaque-Owner-Token-With-Mixed-Case-Entropy"; const OTHER_TOKEN = "opaque-other-token-with-sufficient-entropy"; const INVALID_PRINCIPAL_TOKEN = "invalid-principal-token-with-sufficient-entropy"; +const STALE_APPLY_DEADLINE_SECONDS = 5; const ALL_SCOPES = new Set([ "discovery.read", "workspaces.read", @@ -75,6 +101,7 @@ let DurableOutboxWorker: WorkerConstructor | undefined; class TestAuthenticator implements PrincipalAuthenticator { readonly #principals: Record = { [OWNER_TOKEN]: { id: "owner@example.com", scopes: ALL_SCOPES }, + [MIXED_CASE_OWNER_TOKEN]: { id: "owner@example.com", scopes: ALL_SCOPES }, [OTHER_TOKEN]: { id: "other@example.com", scopes: ALL_SCOPES }, [INVALID_PRINCIPAL_TOKEN]: { id: "Not A CRD Owner", scopes: ALL_SCOPES }, }; @@ -649,6 +676,342 @@ postgresSuite("PostgreSQL-backed public T4 API v1", () => { await ledger.close(); }); + it("rejects forged acknowledge claims unless command, kind, and target match the locked outbox row", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const first = await json(await call(api, "POST", "/v1/workspaces", { key: "ack-binding-workspace-01", body: { name: "claimed" } })); + const unrelated = await json(await call(api, "POST", "/v1/workspaces", { key: "ack-binding-workspace-02", body: { name: "unrelated" } })); + const lease = await ledger.acquireLease("ack-binding-owner", 30); + const claim = await ledger.claimNext(lease); + expect(claim).toBeDefined(); + expect(claim!.targetId).toBe(first.id); + const commands = await admin`SELECT command_id, idempotency_key, lifecycle_state FROM t4_commands WHERE idempotency_key IN ('ack-binding-workspace-01', 'ack-binding-workspace-02')` as Array<{ command_id: string; idempotency_key: string; lifecycle_state: string }>; + const unrelatedCommandId = commands.find(row => row.idempotency_key === "ack-binding-workspace-02")!.command_id; + + expect(await ledger.acknowledge({ ...claim!, commandId: unrelatedCommandId })).toBe(false); + expect(await ledger.acknowledge({ ...claim!, kind: "workspace.patch" })).toBe(false); + expect(await ledger.acknowledge({ ...claim!, targetId: unrelated.id })).toBe(false); + const unchanged = await admin`SELECT + (SELECT state FROM t4_outbox WHERE outbox_id = ${claim!.outboxId}) AS outbox_state, + (SELECT lifecycle_state FROM t4_commands WHERE command_id = ${claim!.commandId}) AS claimed_command_state, + (SELECT lifecycle_state FROM t4_commands WHERE command_id = ${unrelatedCommandId}) AS unrelated_command_state` as Array<{ outbox_state: string; claimed_command_state: string; unrelated_command_state: string }>; + expect(unchanged[0]).toEqual({ outbox_state: "claimed", claimed_command_state: "accepted", unrelated_command_state: "accepted" }); + expect(await ledger.acknowledge(claim!)).toBe(true); + await ledger.close(); + }); + + it("rejects claimed outbox rows with null or empty owners and nonpositive owner epochs at the table boundary", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + await call(api, "POST", "/v1/workspaces", { key: "claimed-row-constraint-01", body: { name: "claim constraints" } }); + const rows = await admin`SELECT outbox_id FROM t4_outbox WHERE idempotency_key = 'claimed-row-constraint-01'` as Array<{ outbox_id: bigint }>; + const outboxId = rows[0]!.outbox_id; + await expect(admin`UPDATE t4_outbox SET state = 'claimed', owner_id = NULL, owner_epoch = 1, claimed_at = clock_timestamp() WHERE outbox_id = ${outboxId}`).rejects.toMatchObject({ code: "23514" }); + await expect(admin`UPDATE t4_outbox SET state = 'claimed', owner_id = 'constraint-owner', owner_epoch = NULL, claimed_at = clock_timestamp() WHERE outbox_id = ${outboxId}`).rejects.toMatchObject({ code: "23514" }); + await expect(admin`UPDATE t4_outbox SET state = 'claimed', owner_id = 'constraint-owner', owner_epoch = 0, claimed_at = clock_timestamp() WHERE outbox_id = ${outboxId}`).rejects.toMatchObject({ code: "23514" }); + await expect(admin`UPDATE t4_outbox SET state = 'claimed', owner_id = '', owner_epoch = 1, claimed_at = clock_timestamp() WHERE outbox_id = ${outboxId}`).rejects.toMatchObject({ code: "23514" }); + await expect(admin`UPDATE t4_outbox SET state = 'claimed', owner_id = 'constraint-owner', owner_epoch = -1, claimed_at = clock_timestamp() WHERE outbox_id = ${outboxId}`).rejects.toMatchObject({ code: "23514" }); + const unchanged = await admin`SELECT state, owner_id, owner_epoch FROM t4_outbox WHERE outbox_id = ${outboxId}` as Array<{ state: string; owner_id: string | null; owner_epoch: bigint | null }>; + expect(unchanged[0]).toEqual({ state: "pending", owner_id: null, owner_epoch: null }); + await ledger.close(); + }); + + it("retries a current recordFailure serialization failure while preventing a stale owner from resetting the reclaimed item", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + await call(api, "POST", "/v1/workspaces", { key: "failure-takeover-workspace", body: { name: "failure takeover" } }); + const staleLease = await ledger.acquireLease("failure-stale-owner", 30); + const staleClaim = await ledger.claimNext(staleLease); + expect(staleClaim).toBeDefined(); + await admin`UPDATE t4_owner_leases SET expires_at = clock_timestamp() - interval '1 second' WHERE lease_name = 'gateway-outbox'`; + const currentLease = await ledger.acquireLease("failure-current-owner", 30); + const currentClaim = await ledger.claimNext(currentLease); + expect(currentClaim?.outboxId).toBe(staleClaim!.outboxId); + expect(currentClaim?.ownerEpoch).toBeGreaterThan(staleClaim!.ownerEpoch); + + expect(await ledger.recordFailure(staleClaim!, "late stale failure")).toBe(false); + const retained = await admin`SELECT state, owner_id, owner_epoch, attempts, last_error FROM t4_outbox WHERE outbox_id = ${staleClaim!.outboxId}` as Array<{ state: string; owner_id: string; owner_epoch: bigint; attempts: number; last_error: string | null }>; + expect(retained[0]).toEqual({ state: "claimed", owner_id: currentLease.ownerId, owner_epoch: currentLease.epoch, attempts: 2, last_error: null }); + + let retryFixtureInstalled = false; + try { + await admin.unsafe(` + CREATE SEQUENCE t4_test_record_failure_attempt; + CREATE FUNCTION t4_test_fail_first_record_failure() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF OLD.state = 'claimed' AND NEW.state = 'pending' AND nextval('t4_test_record_failure_attempt') = 1 THEN + RAISE EXCEPTION 'injected recordFailure serialization failure' USING ERRCODE = '40001'; + END IF; + RETURN NEW; + END $$; + CREATE TRIGGER t4_test_fail_first_record_failure_trigger BEFORE UPDATE ON t4_outbox + FOR EACH ROW EXECUTE FUNCTION t4_test_fail_first_record_failure(); + `); + retryFixtureInstalled = true; + expect(await ledger.recordFailure(currentClaim!, "retryable current failure")).toBe(true); + const released = await admin`SELECT + (SELECT last_value FROM t4_test_record_failure_attempt) AS retry_attempts, + state, owner_id, owner_epoch, attempts, last_error + FROM t4_outbox WHERE outbox_id = ${currentClaim!.outboxId}` as Array<{ retry_attempts: bigint; state: string; owner_id: string | null; owner_epoch: bigint | null; attempts: number; last_error: string | null }>; + expect(released[0]).toEqual({ retry_attempts: 2n, state: "pending", owner_id: null, owner_epoch: null, attempts: 2, last_error: "retryable current failure" }); + } finally { + if (retryFixtureInstalled) await admin.unsafe("DROP TRIGGER IF EXISTS t4_test_fail_first_record_failure_trigger ON t4_outbox; DROP FUNCTION IF EXISTS t4_test_fail_first_record_failure(); DROP SEQUENCE IF EXISTS t4_test_record_failure_attempt"); + await ledger.close(); + } + }); + + it("persists and exactly replays two distinct DELETE requests while enqueueing one provider deletion and one revision transition", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "double-delete-workspace-01", body: { name: "double delete" } })); + const session = await json(await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: "double-delete-session-001", body: { title: "double delete" } })); + const firstKey = "double-delete-request-001"; + const secondKey = "double-delete-request-002"; + const first = await call(api, "DELETE", `/v1/sessions/${session.id}`, { key: firstKey }); + const second = await call(api, "DELETE", `/v1/sessions/${session.id}`, { key: secondKey }); + expect([first.status, second.status]).toEqual([204, 204]); + expect([first.headers.get("idempotency-replayed"), second.headers.get("idempotency-replayed")]).toEqual(["false", "false"]); + const firstBody = await first.text(); + const secondBody = await second.text(); + const firstReplay = await call(api, "DELETE", `/v1/sessions/${session.id}`, { key: firstKey }); + const secondReplay = await call(api, "DELETE", `/v1/sessions/${session.id}`, { key: secondKey }); + expect([firstReplay.status, secondReplay.status]).toEqual([204, 204]); + expect([firstReplay.headers.get("idempotency-replayed"), secondReplay.headers.get("idempotency-replayed")]).toEqual(["true", "true"]); + expect([await firstReplay.text(), await secondReplay.text()]).toEqual([firstBody, secondBody]); + + const persisted = await admin`SELECT + (SELECT count(*) FROM t4_commands WHERE operation = 'deleteSession' AND target_scope = ${`session:${session.id}`}) AS commands, + (SELECT count(*) FROM t4_outbox WHERE mutation_kind = 'session.delete' AND target_id = ${session.id}) AS deletions, + (SELECT revision FROM t4_session_intents WHERE session_id = ${session.id}) AS revision, + (SELECT generation FROM t4_session_intents WHERE session_id = ${session.id}) AS generation, + (SELECT deletion_requested FROM t4_session_intents WHERE session_id = ${session.id}) AS deletion_requested` as Array<{ commands: bigint; deletions: bigint; revision: bigint; generation: bigint; deletion_requested: boolean }>; + expect(persisted[0]).toEqual({ commands: 2n, deletions: 1n, revision: 2n, generation: 2n, deletion_requested: true }); + const kubernetes = new ObservableKubernetes(); + const worker = newWorker(ledger, "double-delete-owner", kubernetes); + const lease = await worker.acquireLease(); + kubernetes.currentEpoch = lease.epoch; + await worker.drain(); + expect(kubernetes.attempts.filter(mutation => mutation.kind === "session.delete" && mutation.targetId === session.id)).toHaveLength(1); + await ledger.close(); + }); + + it("reclaims a same-owner claim only after the named stale-apply deadline and immediately fences the abandoned claim", async () => { + let ledger = await newLedger(); + let api = newApi(ledger); + await call(api, "POST", "/v1/workspaces", { key: "same-owner-restart-claim", body: { name: "restart claim" } }); + const firstWorker = newWorker(ledger, "stable-restart-owner", new ObservableKubernetes()); + const firstLease = await firstWorker.acquireLease(); + const abandoned = await firstWorker.claimNext(); + expect(abandoned).toBeDefined(); + await ledger.close(); + + ledger = await newLedger(); + api = newApi(ledger); + const kubernetes = new ObservableKubernetes(); + const restarted = newWorker(ledger, "stable-restart-owner", kubernetes); + const restartedLease = await restarted.acquireLease(); + expect(restartedLease).toEqual(firstLease); + await expect(ledger.acquireLease("concurrent-thief", 30)).rejects.toThrow("lease"); + expect(await restarted.claimNext()).toBeUndefined(); + await admin`UPDATE t4_outbox SET claimed_at = clock_timestamp() - (${STALE_APPLY_DEADLINE_SECONDS - 1} * interval '1 second') WHERE outbox_id = ${abandoned!.outboxId}`; + expect(await restarted.claimNext()).toBeUndefined(); + await admin`UPDATE t4_outbox SET claimed_at = clock_timestamp() - (${STALE_APPLY_DEADLINE_SECONDS + 1} * interval '1 second') WHERE outbox_id = ${abandoned!.outboxId}`; + const reclaimed = await restarted.claimNext(); + expect(reclaimed?.outboxId).toBe(abandoned!.outboxId); + expect(reclaimed?.ownerEpoch).toBe(firstLease.epoch); + expect(await ledger.claimIsCurrent(abandoned!)).toBe(false); + kubernetes.currentEpoch = restartedLease.epoch; + expect(await restarted.applyClaim(reclaimed!)).toBe("applied"); + expect(kubernetes.attempts).toHaveLength(1); + await ledger.close(); + }); + + it("case-folds lowercase and mixed-case Bearer schemes without altering mixed-case credential bytes", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + for (const scheme of ["bearer", "bEaReR"]) { + const requestHeaders = headers(); + requestHeaders.set("authorization", `${scheme} ${MIXED_CASE_OWNER_TOKEN}`); + const response = await api.handle(new Request("https://t4.example.test/v1", { headers: requestHeaders })); + expect(response.status).toBe(200); + } + await ledger.close(); + }); + + it("returns the generated incompatible-version Error406 envelope for malformed or non-SSE Accept", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "accept-406-workspace-01", body: { name: "accept contract" } })); + const session = await json(await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: "accept-406-session-001", body: { title: "accept contract" } })); + for (const accept of ["application/json", "text/event-stream;q=bogus"]) { + const response = await call(api, "GET", `/v1/sessions/${session.id}/events`, { headers: { accept } }); + expect(response.status).toBe(406); + expect(response.headers.get("content-type")).toContain("application/json"); + expect(response.headers.get("t4-api-version")).toBe("1.0"); + expect(await json(response)).toEqual({ + error: { + code: "incompatible_version", + message: expect.any(String), + requestId: expect.stringMatching(/^req_[0-9a-f-]{36}$/u), + retryable: false, + supportedMajors: [1], + }, + }); + } + await ledger.close(); + }); + + it("marks successful Kubernetes outbox application projected, never completed, while public command replay stays accepted", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "projected-workspace-0001", body: { name: "projected" } })); + const session = await json(await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: "projected-session-00001", body: { title: "projected" } })); + const kubernetes = new ObservableKubernetes(); + const worker = newWorker(ledger, "projection-lifecycle-owner", kubernetes); + const lease = await worker.acquireLease(); + kubernetes.currentEpoch = lease.epoch; + expect(await worker.drain()).toBeGreaterThan(0); + + const key = "projected-command-00001"; + const input = { command: "printf projected", metadata: { contract: "lifecycle" } }; + const acceptedResponse = await call(api, "POST", `/v1/sessions/${session.id}/commands`, { key, body: input }); + expect(acceptedResponse.status).toBe(202); + const accepted = await json(acceptedResponse); + expect(accepted).toMatchObject({ state: "accepted" }); + expect(await worker.drain()).toBe(1); + const replay = await call(api, "POST", `/v1/sessions/${session.id}/commands`, { key, body: input }); + expect(replay.status).toBe(200); + expect(replay.headers.get("idempotency-replayed")).toBe("true"); + expect(await json(replay)).toEqual(accepted); + const lifecycle = await admin`SELECT command.lifecycle_state, item.state AS outbox_state + FROM t4_commands command JOIN t4_outbox item USING (command_id) + WHERE command.command_id = ${String(accepted.commandId)}` as Array<{ lifecycle_state: string; outbox_state: string }>; + expect(lifecycle[0]).toEqual({ lifecycle_state: "projected", outbox_state: "applied" }); + expect(kubernetes.attempts.filter(mutation => mutation.kind === "command.submit" && mutation.targetId === session.id)).toHaveLength(1); + await ledger.close(); + }); + + it("atomically converges one complete session status event and opaque cursor while cancellation absorbs later status", async () => { + const ledger = await newLedger(); + const api = newApi(ledger); + const workspace = await json(await call(api, "POST", "/v1/workspaces", { key: "status-ingress-workspace", body: { name: "status ingress" } })); + const session = await json(await call(api, "POST", `/v1/workspaces/${workspace.id}/sessions`, { key: "status-ingress-session-01", body: { title: "status ingress" } })); + const lease = await ledger.acquireLease("status-ingress-owner", 30); + const ingress = ledger as PostgresLedgerLike & AuthoritativeKubernetesStatusIngress; + let rejectionTriggerInstalled = false; + try { + const eventsBeforeWorkspace = await admin`SELECT count(*) AS count FROM t4_events` as Array<{ count: bigint }>; + await ingress.ingestKubernetesStatus(lease, "t4workspaces", { + resourceId: workspace.id, + principalId: "owner@example.com", + uid: "workspace-authority-uid", + resourceVersion: "workspace-rv-101", + generation: 1n, + observedGeneration: 1n, + phase: "Ready", + }); + const workspaceConvergence = await admin`SELECT + (SELECT state FROM t4_workspace_intents WHERE workspace_id = ${workspace.id}) AS state, + (SELECT count(*) FROM t4_events) AS event_count, + (SELECT resource_version FROM t4_kubernetes_status_cursors WHERE collection = 't4workspaces') AS cursor` as Array<{ state: string; event_count: bigint; cursor: string }>; + expect(workspaceConvergence[0]).toEqual({ state: "ready", event_count: eventsBeforeWorkspace[0]!.count, cursor: "workspace-rv-101" }); + + const beforeSessionStatus = await admin`SELECT count(*) AS event_count, max(sequence) AS latest_event FROM t4_events WHERE session_id = ${session.id}` as Array<{ event_count: bigint; latest_event: bigint }>; + await ingress.ingestKubernetesStatus(lease, "t4sessions", { + resourceId: session.id, + principalId: "owner@example.com", + workspaceId: workspace.id, + uid: "session-authority-uid", + resourceVersion: "900", + generation: 1n, + observedGeneration: 1n, + phase: "Running", + }); + const converged = await admin`SELECT + (SELECT state FROM t4_session_intents WHERE session_id = ${session.id}) AS state, + (SELECT count(*) FROM t4_events WHERE session_id = ${session.id} AND sequence > ${beforeSessionStatus[0]!.latest_event}) AS new_event_count, + (SELECT sequence FROM t4_events WHERE session_id = ${session.id} ORDER BY sequence DESC LIMIT 1) AS latest_event, + (SELECT payload FROM t4_events WHERE session_id = ${session.id} ORDER BY sequence DESC LIMIT 1) AS event_payload, + (SELECT owner_epoch FROM t4_events WHERE session_id = ${session.id} ORDER BY sequence DESC LIMIT 1) AS event_epoch, + (SELECT latest_sequence FROM t4_event_retention WHERE principal_id = 'owner@example.com' AND session_id = ${session.id}) AS retained_event, + (SELECT resource_version FROM t4_kubernetes_status_cursors WHERE collection = 't4sessions') AS cursor` as Array<{ state: string; new_event_count: bigint; latest_event: bigint; event_payload: Record; event_epoch: bigint; retained_event: bigint; cursor: string }>; + expect(converged[0]!.state).toBe("ready"); + expect(converged[0]!.new_event_count).toBe(1n); + expect(converged[0]!.event_payload).toEqual({ state: "ready", revision: session.revision }); + expect(converged[0]!.event_epoch).toBe(lease.epoch); + expect(converged[0]!.cursor).toBe("900"); + expect(converged[0]!.latest_event).toBeGreaterThan(beforeSessionStatus[0]!.latest_event); + expect(converged[0]!.retained_event).toBe(converged[0]!.latest_event); + + await admin.unsafe(` + CREATE OR REPLACE FUNCTION t4_test_reject_authoritative_status_event() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF NEW.event_type = 'session' AND NEW.payload->>'state' = 'failed' THEN RAISE EXCEPTION 'injected authoritative status event failure'; END IF; + RETURN NEW; + END $$; + CREATE TRIGGER t4_test_reject_authoritative_status_event_trigger BEFORE INSERT ON t4_events + FOR EACH ROW EXECUTE FUNCTION t4_test_reject_authoritative_status_event(); + `); + rejectionTriggerInstalled = true; + const beforeFailure = await admin`SELECT + state, revision, kube_uid, kube_resource_version, kube_generation, kube_observed_generation, + (SELECT count(*) FROM t4_events WHERE session_id = ${session.id}) AS event_count, + (SELECT min(sequence) FROM t4_events WHERE session_id = ${session.id}) AS first_event, + (SELECT max(sequence) FROM t4_events WHERE session_id = ${session.id}) AS latest_event, + (SELECT first_retained_sequence FROM t4_event_retention WHERE principal_id = 'owner@example.com' AND session_id = ${session.id}) AS first_retained_event, + (SELECT latest_sequence FROM t4_event_retention WHERE principal_id = 'owner@example.com' AND session_id = ${session.id}) AS latest_retained_event, + (SELECT resource_version FROM t4_kubernetes_status_cursors WHERE collection = 't4sessions') AS cursor_resource_version, + (SELECT owner_epoch FROM t4_kubernetes_status_cursors WHERE collection = 't4sessions') AS cursor_owner_epoch + FROM t4_session_intents WHERE session_id = ${session.id}`; + await expect(ingress.ingestKubernetesStatus(lease, "t4sessions", { + resourceId: session.id, + principalId: "owner@example.com", + workspaceId: workspace.id, + uid: "session-authority-uid", + resourceVersion: "901", + generation: 2n, + observedGeneration: 2n, + phase: "Failed", + })).rejects.toThrow("injected authoritative status event failure"); + const afterFailure = await admin`SELECT + state, revision, kube_uid, kube_resource_version, kube_generation, kube_observed_generation, + (SELECT count(*) FROM t4_events WHERE session_id = ${session.id}) AS event_count, + (SELECT min(sequence) FROM t4_events WHERE session_id = ${session.id}) AS first_event, + (SELECT max(sequence) FROM t4_events WHERE session_id = ${session.id}) AS latest_event, + (SELECT first_retained_sequence FROM t4_event_retention WHERE principal_id = 'owner@example.com' AND session_id = ${session.id}) AS first_retained_event, + (SELECT latest_sequence FROM t4_event_retention WHERE principal_id = 'owner@example.com' AND session_id = ${session.id}) AS latest_retained_event, + (SELECT resource_version FROM t4_kubernetes_status_cursors WHERE collection = 't4sessions') AS cursor_resource_version, + (SELECT owner_epoch FROM t4_kubernetes_status_cursors WHERE collection = 't4sessions') AS cursor_owner_epoch + FROM t4_session_intents WHERE session_id = ${session.id}`; + expect(afterFailure).toEqual(beforeFailure); + + const cancellationResponse = await call(api, "POST", `/v1/sessions/${session.id}/cancel`, { key: "status-ingress-cancel-001" }); + expect(cancellationResponse.status).toBe(202); + const cancelling = await json(cancellationResponse); + expect(cancelling).toMatchObject({ id: session.id, state: "cancelling", revision: session.revision + 1 }); + const afterCancellation = await admin`SELECT count(*) AS event_count FROM t4_events WHERE session_id = ${session.id}` as Array<{ event_count: bigint }>; + await ingress.ingestKubernetesStatus(lease, "t4sessions", { + resourceId: session.id, + principalId: "owner@example.com", + workspaceId: workspace.id, + uid: "session-authority-uid", + resourceVersion: "10", + generation: 1n, + observedGeneration: 1n, + phase: "Running", + }); + const afterLowerOpaqueVersion = await admin`SELECT + (SELECT state FROM t4_session_intents WHERE session_id = ${session.id}) AS state, + (SELECT count(*) FROM t4_events WHERE session_id = ${session.id}) AS event_count, + (SELECT resource_version FROM t4_kubernetes_status_cursors WHERE collection = 't4sessions') AS cursor_resource_version, + (SELECT owner_epoch FROM t4_kubernetes_status_cursors WHERE collection = 't4sessions') AS cursor_owner_epoch` as Array<{ state: string; event_count: bigint; cursor_resource_version: string; cursor_owner_epoch: bigint }>; + expect(afterLowerOpaqueVersion[0]).toEqual({ state: "cancelling", event_count: afterCancellation[0]!.event_count, cursor_resource_version: "10", cursor_owner_epoch: lease.epoch }); + } finally { + if (rejectionTriggerInstalled) await admin.unsafe("DROP TRIGGER IF EXISTS t4_test_reject_authoritative_status_event_trigger ON t4_events; DROP FUNCTION IF EXISTS t4_test_reject_authoritative_status_event()"); + await ledger.close(); + } + }); + it("allows exact Unicode code-point bounds and denies browser Origin by default", async () => { const ledger = await newLedger(); const api = newApi(ledger); diff --git a/packages/cluster-server/test/kubernetes-client.test.ts b/packages/cluster-server/test/kubernetes-client.test.ts index 290dd564..631d8064 100644 --- a/packages/cluster-server/test/kubernetes-client.test.ts +++ b/packages/cluster-server/test/kubernetes-client.test.ts @@ -10,6 +10,8 @@ import { semanticResourceHash, } from "../src/kubernetes-client.ts"; import { PublicKubernetesOutboxApplier } from "../src/public-kubernetes-applier.ts"; +import { ClusterInfrastructureProjection } from "../src/kubernetes-projection.ts"; +import { KubernetesProjectionRunner } from "../src/kubernetes-runner.ts"; import type { OutboxMutation } from "../src/outbox-worker.ts"; const PRINCIPAL = "owner@example.com"; @@ -237,6 +239,164 @@ describe("durable public Kubernetes applier", () => { "cluster.t4.dev/ledger-semantic-hash": "sha256:prior", }; + it("does not compensate an objectless non-success POST with fallback GET or DELETE", async () => { + const requests: Array<{ url: string; init?: RequestInit }> = []; + const fetch = (async (input: string | URL | Request, init?: RequestInit) => { + requests.push({ url: String(input), init }); + return init?.method === "POST" + ? Response.json({ reason: "ServiceUnavailable" }, { status: 503 }) + : Response.json({ reason: "NotFound" }, { status: 404 }); + }) as typeof globalThis.fetch; + const applier = new PublicKubernetesOutboxApplier({ + client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch }), + hostRef: "primary", + }); + const createMutation: OutboxMutation = { + ...workspaceMutation, + kind: "workspace.create", + payload: { id: "workspace-one", name: "created", revision: 1 }, + }; + + await expect(applier.apply(createMutation, fence)).rejects.toMatchObject({ status: 503 }); + expect(requests.map(request => request.init?.method ?? "GET")).toEqual(["POST"]); + }); + + it("cleans a stale successful create by its returned UID while protecting a replacement UID", async () => { + const requests: Array<{ url: string; init?: RequestInit }> = []; + const replacementUid = "workspace-replacement-uid"; + let replacementProtected = false; + const fetch = (async (input: string | URL | Request, init?: RequestInit) => { + requests.push({ url: String(input), init }); + if (init?.method === "POST") { + const posted = JSON.parse(String(init.body)); + return Response.json({ + ...posted, + metadata: { ...posted.metadata, uid: "workspace-created-uid", resourceVersion: "81" }, + }, { status: 201 }); + } + if (init?.method === "DELETE") { + const deletion = JSON.parse(String(init.body)); + replacementProtected = deletion.preconditions.uid !== replacementUid; + return replacementProtected + ? Response.json({ reason: "Conflict" }, { status: 409 }) + : Response.json({}); + } + return Response.json({ + apiVersion: "cluster.t4.dev/v1alpha1", + kind: "T4Workspace", + metadata: { name: "workspace-one", uid: replacementUid, resourceVersion: "82" }, + }, { status: 200 }); + }) as typeof globalThis.fetch; + const applier = new PublicKubernetesOutboxApplier({ + client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch }), + hostRef: "primary", + }); + const createMutation: OutboxMutation = { + ...workspaceMutation, + kind: "workspace.create", + payload: { id: "workspace-one", name: "created", revision: 1 }, + }; + let currencyChecks = 0; + + await expect(applier.apply(createMutation, fence, { + claimIsCurrent: async () => ++currencyChecks === 1, + })).rejects.toThrow("no longer current"); + expect(requests.map(request => request.init?.method ?? "GET")).toEqual(["POST", "DELETE"]); + expect(JSON.parse(String(requests[1]?.init?.body))).toMatchObject({ + preconditions: { uid: "workspace-created-uid", resourceVersion: "81" }, + }); + expect(replacementProtected).toBe(true); + }); + + it.each([409, 503])("persists and safely replays exact-UID stale-create cleanup after DELETE %i", async cleanupStatus => { + interface StaleCreateCleanup { + readonly resourceType: "t4workspaces" | "t4sessions"; + readonly targetId: string; + readonly uid: string; + readonly resourceVersion: string; + } + interface StaleCreateCleanupReplayer { + replayStaleCreateCleanup(cleanup: StaleCreateCleanup, context: { claimIsCurrent(): Promise }): Promise; + } + const durableCleanups: StaleCreateCleanup[] = []; + const requests: Array<{ url: string; init?: RequestInit }> = []; + let phase: "initial" | "same-uid" | "replacement-uid" = "initial"; + const fetch = (async (input: string | URL | Request, init?: RequestInit) => { + requests.push({ url: String(input), init }); + if (phase === "initial" && init?.method === "POST") { + const posted = JSON.parse(String(init.body)); + return Response.json({ + ...posted, + metadata: { ...posted.metadata, uid: "workspace-created-uid", resourceVersion: "91" }, + }, { status: 201 }); + } + if (phase === "initial") { + return Response.json({ reason: cleanupStatus === 409 ? "Conflict" : "ServiceUnavailable" }, { status: cleanupStatus }); + } + if (!init?.method) { + return Response.json({ + apiVersion: "cluster.t4.dev/v1alpha1", + kind: "T4Workspace", + metadata: { + name: "workspace-one", + uid: phase === "same-uid" ? "workspace-created-uid" : "workspace-replacement-uid", + resourceVersion: phase === "same-uid" ? "92" : "93", + }, + }, { status: 200 }); + } + return Response.json({}, { status: 200 }); + }) as typeof globalThis.fetch; + const applier = new PublicKubernetesOutboxApplier({ + client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch }), + hostRef: "primary", + }); + const createMutation: OutboxMutation = { + ...workspaceMutation, + kind: "workspace.create", + payload: { id: "workspace-one", name: "created", revision: 1 }, + }; + let currencyChecks = 0; + const cleanupAwareContext = { + claimIsCurrent: async () => ++currencyChecks === 1, + persistStaleCreateCleanup: async (cleanup: StaleCreateCleanup) => { durableCleanups.push(cleanup); }, + }; + + await expect(applier.apply(createMutation, fence, cleanupAwareContext)).rejects.toThrow(); + expect(requests.map(request => request.init?.method ?? "GET")).toEqual(["POST", "DELETE"]); + expect(durableCleanups).toEqual([{ + resourceType: "t4workspaces", + targetId: "workspace-one", + uid: "workspace-created-uid", + resourceVersion: "91", + }]); + + const replayer = applier as unknown as StaleCreateCleanupReplayer; + let replayCurrencyChecks = 0; + const currentOwner = { claimIsCurrent: async () => { replayCurrencyChecks += 1; return true; } }; + phase = "same-uid"; + const sameUidRequestStart = requests.length; + await expect(replayer.replayStaleCreateCleanup(durableCleanups[0]!, currentOwner)).resolves.toBeUndefined(); + const sameUidRequests = requests.slice(sameUidRequestStart); + expect(sameUidRequests.map(request => request.init?.method ?? "GET")).toEqual(["GET", "DELETE"]); + expect(sameUidRequests[0]?.url).toBe("https://kubernetes.default.svc/apis/cluster.t4.dev/v1alpha1/namespaces/development/t4workspaces/workspace-one"); + expect(JSON.parse(String(sameUidRequests[1]?.init?.body))).toEqual({ + apiVersion: "v1", + kind: "DeleteOptions", + propagationPolicy: "Foreground", + preconditions: { uid: "workspace-created-uid", resourceVersion: "92" }, + }); + expect(replayCurrencyChecks).toBeGreaterThanOrEqual(1); + const currencyChecksBeforeReplacement = replayCurrencyChecks; + + phase = "replacement-uid"; + const replacementRequestStart = requests.length; + await expect(replayer.replayStaleCreateCleanup(durableCleanups[0]!, currentOwner)).resolves.toBeUndefined(); + const replacementRequests = requests.slice(replacementRequestStart); + expect(replacementRequests.map(request => request.init?.method ?? "GET")).toEqual(["GET"]); + expect(replacementRequests[0]?.url).toBe("https://kubernetes.default.svc/apis/cluster.t4.dev/v1alpha1/namespaces/development/t4workspaces/workspace-one"); + expect(replayCurrencyChecks).toBeGreaterThan(currencyChecksBeforeReplacement); + }); + it("carries observed resourceVersion and owner epoch on every existing-resource PATCH", async () => { const values = recordingFetch([{ apiVersion: "cluster.t4.dev/v1alpha1", @@ -401,6 +561,73 @@ describe("durable public Kubernetes applier", () => { expect(new TextEncoder().encode(JSON.stringify(patch.metadata)).byteLength).toBeLessThan(4_096); }); + it("removes the legacy full command payload on submit while replacing the bounded pointer", async () => { + const values = recordingFetch([{ + apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Session", + metadata: { name: "session-one", resourceVersion: "66", annotations: { + ...priorAnnotations, + "cluster.t4.dev/pending-command": "legacy full command payload", + "cluster.t4.dev/pending-command-id": "prior-command-id", + "cluster.t4.dev/pending-command-epoch": "6", + } }, + spec: { hostRef: "primary", workspaceRef: "workspace-one" }, + }, { + apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Workspace", + metadata: { name: "workspace-one", resourceVersion: "65", annotations: priorAnnotations }, + spec: { hostRef: "primary", owner: PRINCIPAL }, + }, {}]); + const applier = new PublicKubernetesOutboxApplier({ + client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch: values.fetch }), + hostRef: "primary", + }); + const commandMutation: OutboxMutation = { + ...workspaceMutation, + kind: "command.submit", + targetId: "session-one", + payload: { sessionId: "session-one", command: "printf replacement" }, + }; + + await applier.apply(commandMutation, fence); + expect(new Headers(values.requests.at(-1)?.init?.headers).get("content-type")).toBe("application/merge-patch+json"); + const annotations = JSON.parse(String(values.requests.at(-1)?.init?.body)).metadata.annotations; + expect(annotations["cluster.t4.dev/pending-command"]).toBeNull(); + expect(annotations["cluster.t4.dev/pending-command-id"]).toBe(commandMutation.commandId); + expect(annotations["cluster.t4.dev/pending-command-epoch"]).toBe(fence.epoch.toString()); + }); + + it("removes the legacy full command payload on session patch without clearing the bounded pointer", async () => { + const values = recordingFetch([{ + apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Session", + metadata: { name: "session-one", resourceVersion: "68", annotations: { + ...priorAnnotations, + "cluster.t4.dev/pending-command": "legacy full command payload", + "cluster.t4.dev/pending-command-id": "bounded-command-id", + "cluster.t4.dev/pending-command-epoch": "6", + } }, + spec: { hostRef: "primary", workspaceRef: "workspace-one" }, + }, { + apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Workspace", + metadata: { name: "workspace-one", resourceVersion: "67", annotations: priorAnnotations }, + spec: { hostRef: "primary", owner: PRINCIPAL }, + }, {}]); + const applier = new PublicKubernetesOutboxApplier({ + client: new KubernetesApiClient({ baseUrl: "https://kubernetes.default.svc", namespace: "development", token: "token", fetch: values.fetch }), + hostRef: "primary", + }); + + await applier.apply({ + ...workspaceMutation, + kind: "session.patch", + targetId: "session-one", + payload: { id: "session-one", workspaceId: "workspace-one", title: "patched", revision: 2 }, + }, fence); + expect(new Headers(values.requests.at(-1)?.init?.headers).get("content-type")).toBe("application/merge-patch+json"); + const annotations = JSON.parse(String(values.requests.at(-1)?.init?.body)).metadata.annotations; + expect(annotations["cluster.t4.dev/pending-command"]).toBeNull(); + expect(annotations).not.toHaveProperty("cluster.t4.dev/pending-command-id"); + expect(annotations).not.toHaveProperty("cluster.t4.dev/pending-command-epoch"); + }); + it("verifies a session kind and its owning workspace relation before patching", async () => { const values = recordingFetch([{ apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Session", @@ -432,6 +659,64 @@ describe("durable public Kubernetes applier", () => { }); }); +describe("Kubernetes legacy annotation reconciliation", () => { + + it("reconciles legacy full command payload annotations on untouched listed sessions", async () => { + const patches: Array<{ resource: string; name: string; body: unknown }> = []; + const client = { + listInfrastructure: async () => ({ + host: { + apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4ClusterHost", + metadata: { name: "primary", uid: "host-uid", resourceVersion: "100" }, spec: {}, + }, + workspaces: [{ + apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Workspace", + metadata: { name: "workspace-one", uid: "workspace-uid", resourceVersion: "101" }, + spec: { hostRef: "primary", owner: PRINCIPAL, displayName: "Workspace", retentionPolicy: "Retain", size: "20Gi" }, + }], + sessions: [{ + apiVersion: "cluster.t4.dev/v1alpha1", kind: "T4Session", + metadata: { name: "session-legacy", uid: "session-uid", resourceVersion: "102", annotations: { + "cluster.t4.dev/pending-command": "legacy full command payload", + "cluster.t4.dev/pending-command-id": "bounded-command-id", + "cluster.t4.dev/pending-command-epoch": "6", + } }, + spec: { hostRef: "primary", workspaceRef: "workspace-one", title: "Session", runtimeProfile: "default", guiEnabled: true }, + }], + resourceVersion: "102", + resourceVersions: { t4clusterhosts: "100", t4workspaces: "101", t4sessions: "102" }, + }), + patch: async (resource: string, name: string, body: unknown) => { + patches.push({ resource, name, body }); + return { metadata: { name, resourceVersion: "103" } }; + }, + watch: (_resource: string, _version: string, _onEvent: unknown, signal: AbortSignal) => new Promise(resolve => { + if (signal.aborted) resolve(); + else signal.addEventListener("abort", () => resolve(), { once: true }); + }), + } as unknown as KubernetesApiClient; + const projection = new ClusterInfrastructureProjection({ epoch: "replica-one", namespace: "development" }); + const runner = new KubernetesProjectionRunner({ client, projection, hostName: "primary", retryMs: 0 }); + + try { + await runner.start(); + expect(patches).toHaveLength(1); + expect(patches[0]).toEqual({ + resource: "t4sessions", + name: "session-legacy", + body: { + metadata: { + resourceVersion: "102", + annotations: { "cluster.t4.dev/pending-command": null }, + }, + }, + }); + } finally { + await runner.stop(); + } + }); +}); + describe("Kubernetes projected identity review", () => { it("submits the presented bearer with the fixed audience and requires the exact server ServiceAccount", async () => { const directory = await mkdtemp(join(tmpdir(), "t4-token-review-")); From c8136473fa807be66159258898dee6bb42fea45d Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 06:20:17 +0000 Subject: [PATCH 131/131] fix(cluster): close durable gateway review defects --- .../charts/t4-cluster/templates/_helpers.tpl | 3 + .../migrations/001_durable_gateway.down.sql | 2 + .../migrations/001_durable_gateway.sql | 65 +++- .../cluster-server/src/kubernetes-runner.ts | 194 +++++++++++- packages/cluster-server/src/ledger.ts | 278 ++++++++++++++++-- packages/cluster-server/src/main.ts | 7 +- packages/cluster-server/src/outbox-worker.ts | 46 ++- packages/cluster-server/src/public-api-v1.ts | 28 +- .../src/public-kubernetes-applier.ts | 87 ++++-- 9 files changed, 637 insertions(+), 73 deletions(-) diff --git a/deploy/charts/t4-cluster/templates/_helpers.tpl b/deploy/charts/t4-cluster/templates/_helpers.tpl index 207026e8..b5165665 100644 --- a/deploy/charts/t4-cluster/templates/_helpers.tpl +++ b/deploy/charts/t4-cluster/templates/_helpers.tpl @@ -39,6 +39,9 @@ app.kubernetes.io/instance: {{ .Release.Name | quote }} {{- $existingSecret := required "publicApi.existingSecret is required when publicApi.enabled is true" .Values.publicApi.existingSecret -}} {{- $postgresURLKey := required "publicApi.postgresURLKey is required when publicApi.enabled is true" .Values.publicApi.postgresURLKey -}} {{- $credentialsKey := required "publicApi.credentialsKey is required when publicApi.enabled is true" .Values.publicApi.credentialsKey -}} +{{- if eq $postgresURLKey $credentialsKey -}} +{{- fail "publicApi.postgresURLKey and publicApi.credentialsKey must be different" -}} +{{- end -}} {{- if not .Values.networkPolicy.enabled -}} {{- fail "networkPolicy.enabled must be true when publicApi.enabled is true" -}} {{- end -}} diff --git a/packages/cluster-server/migrations/001_durable_gateway.down.sql b/packages/cluster-server/migrations/001_durable_gateway.down.sql index eb96ccf7..c3967522 100644 --- a/packages/cluster-server/migrations/001_durable_gateway.down.sql +++ b/packages/cluster-server/migrations/001_durable_gateway.down.sql @@ -1,3 +1,5 @@ +DROP TABLE IF EXISTS t4_stale_create_cleanups; +DROP TABLE IF EXISTS t4_kubernetes_status_cursors; DROP TABLE IF EXISTS t4_outbox; DROP TABLE IF EXISTS t4_owner_leases; DROP TABLE IF EXISTS t4_snapshot_entries; diff --git a/packages/cluster-server/migrations/001_durable_gateway.sql b/packages/cluster-server/migrations/001_durable_gateway.sql index ead19e59..b9cfcc17 100644 --- a/packages/cluster-server/migrations/001_durable_gateway.sql +++ b/packages/cluster-server/migrations/001_durable_gateway.sql @@ -1,24 +1,50 @@ CREATE TABLE IF NOT EXISTS t4_schema_migrations (version integer PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT clock_timestamp()); CREATE TABLE IF NOT EXISTS t4_commands ( command_id text PRIMARY KEY, principal_id text NOT NULL, operation text NOT NULL, target_scope text NOT NULL, - idempotency_key text NOT NULL, fingerprint text NOT NULL, lifecycle_state text NOT NULL CHECK (lifecycle_state IN ('accepted','completed','failed')), + idempotency_key text NOT NULL, fingerprint text NOT NULL, lifecycle_state text NOT NULL CHECK (lifecycle_state IN ('accepted','projected','dispatching','running','succeeded','failed','cancelling','cancelled','rejected','unavailable','indeterminate')), response_status integer NOT NULL, response_body jsonb, error_body jsonb, created_at timestamptz NOT NULL DEFAULT clock_timestamp(), updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), UNIQUE (principal_id, operation, target_scope, idempotency_key) ); +ALTER TABLE t4_commands DROP CONSTRAINT IF EXISTS t4_commands_lifecycle_state_check; +UPDATE t4_commands SET lifecycle_state = 'projected', updated_at = clock_timestamp() WHERE lifecycle_state = 'completed'; +ALTER TABLE t4_commands ADD CONSTRAINT t4_commands_lifecycle_state_check CHECK (lifecycle_state IN ('accepted','projected','dispatching','running','succeeded','failed','cancelling','cancelled','rejected','unavailable','indeterminate')) NOT VALID; +ALTER TABLE t4_commands VALIDATE CONSTRAINT t4_commands_lifecycle_state_check; CREATE TABLE IF NOT EXISTS t4_workspace_intents ( workspace_id text PRIMARY KEY, principal_id text NOT NULL, name text NOT NULL, labels jsonb NOT NULL DEFAULT '{}'::jsonb, state text NOT NULL CHECK (state IN ('accepted','provisioning','ready','deleting','deleted','failed','unavailable','indeterminate')), revision bigint NOT NULL CHECK (revision > 0), generation bigint NOT NULL CHECK (generation > 0), deletion_requested boolean NOT NULL DEFAULT false, + kube_uid text, kube_resource_version text, kube_generation bigint, kube_observed_generation bigint, created_at timestamptz NOT NULL DEFAULT clock_timestamp(), updated_at timestamptz NOT NULL DEFAULT clock_timestamp() ); +ALTER TABLE t4_workspace_intents ADD COLUMN IF NOT EXISTS kube_uid text; +ALTER TABLE t4_workspace_intents ADD COLUMN IF NOT EXISTS kube_resource_version text; +ALTER TABLE t4_workspace_intents ADD COLUMN IF NOT EXISTS kube_generation bigint; +ALTER TABLE t4_workspace_intents ADD COLUMN IF NOT EXISTS kube_observed_generation bigint; +UPDATE t4_workspace_intents SET kube_uid = NULL, kube_resource_version = NULL, kube_generation = NULL, kube_observed_generation = NULL + WHERE NOT ((kube_uid IS NULL AND kube_resource_version IS NULL AND kube_generation IS NULL AND kube_observed_generation IS NULL) + OR (kube_uid IS NOT NULL AND octet_length(kube_uid) BETWEEN 1 AND 256 AND kube_resource_version IS NOT NULL AND octet_length(kube_resource_version) BETWEEN 1 AND 256 AND kube_generation IS NOT NULL AND kube_generation > 0 AND kube_observed_generation IS NOT NULL AND kube_observed_generation >= 0)); +ALTER TABLE t4_workspace_intents DROP CONSTRAINT IF EXISTS t4_workspace_intents_kube_identity_check; +ALTER TABLE t4_workspace_intents ADD CONSTRAINT t4_workspace_intents_kube_identity_check CHECK ((kube_uid IS NULL AND kube_resource_version IS NULL AND kube_generation IS NULL AND kube_observed_generation IS NULL) OR (kube_uid IS NOT NULL AND octet_length(kube_uid) BETWEEN 1 AND 256 AND kube_resource_version IS NOT NULL AND octet_length(kube_resource_version) BETWEEN 1 AND 256 AND kube_generation IS NOT NULL AND kube_generation > 0 AND kube_observed_generation IS NOT NULL AND kube_observed_generation >= 0)) NOT VALID; +ALTER TABLE t4_workspace_intents VALIDATE CONSTRAINT t4_workspace_intents_kube_identity_check; CREATE INDEX IF NOT EXISTS t4_workspace_intents_principal_page ON t4_workspace_intents (principal_id, workspace_id); CREATE TABLE IF NOT EXISTS t4_session_intents ( session_id text PRIMARY KEY, workspace_id text NOT NULL REFERENCES t4_workspace_intents(workspace_id), principal_id text NOT NULL, title text NOT NULL, labels jsonb NOT NULL DEFAULT '{}'::jsonb, state text NOT NULL CHECK (state IN ('accepted','provisioning','ready','cancelling','cancelled','failed','unavailable','indeterminate')), revision bigint NOT NULL CHECK (revision > 0), generation bigint NOT NULL CHECK (generation > 0), cancellation_requested boolean NOT NULL DEFAULT false, - deletion_requested boolean NOT NULL DEFAULT false, created_at timestamptz NOT NULL DEFAULT clock_timestamp(), updated_at timestamptz NOT NULL DEFAULT clock_timestamp() + deletion_requested boolean NOT NULL DEFAULT false, kube_uid text, kube_resource_version text, kube_generation bigint, kube_observed_generation bigint, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), updated_at timestamptz NOT NULL DEFAULT clock_timestamp() ); +ALTER TABLE t4_session_intents ADD COLUMN IF NOT EXISTS kube_uid text; +ALTER TABLE t4_session_intents ADD COLUMN IF NOT EXISTS kube_resource_version text; +ALTER TABLE t4_session_intents ADD COLUMN IF NOT EXISTS kube_generation bigint; +ALTER TABLE t4_session_intents ADD COLUMN IF NOT EXISTS kube_observed_generation bigint; +UPDATE t4_session_intents SET kube_uid = NULL, kube_resource_version = NULL, kube_generation = NULL, kube_observed_generation = NULL + WHERE NOT ((kube_uid IS NULL AND kube_resource_version IS NULL AND kube_generation IS NULL AND kube_observed_generation IS NULL) + OR (kube_uid IS NOT NULL AND octet_length(kube_uid) BETWEEN 1 AND 256 AND kube_resource_version IS NOT NULL AND octet_length(kube_resource_version) BETWEEN 1 AND 256 AND kube_generation IS NOT NULL AND kube_generation > 0 AND kube_observed_generation IS NOT NULL AND kube_observed_generation >= 0)); +ALTER TABLE t4_session_intents DROP CONSTRAINT IF EXISTS t4_session_intents_kube_identity_check; +ALTER TABLE t4_session_intents ADD CONSTRAINT t4_session_intents_kube_identity_check CHECK ((kube_uid IS NULL AND kube_resource_version IS NULL AND kube_generation IS NULL AND kube_observed_generation IS NULL) OR (kube_uid IS NOT NULL AND octet_length(kube_uid) BETWEEN 1 AND 256 AND kube_resource_version IS NOT NULL AND octet_length(kube_resource_version) BETWEEN 1 AND 256 AND kube_generation IS NOT NULL AND kube_generation > 0 AND kube_observed_generation IS NOT NULL AND kube_observed_generation >= 0)) NOT VALID; +ALTER TABLE t4_session_intents VALIDATE CONSTRAINT t4_session_intents_kube_identity_check; CREATE INDEX IF NOT EXISTS t4_session_intents_principal_workspace_page ON t4_session_intents (principal_id, workspace_id, session_id); CREATE TABLE IF NOT EXISTS t4_events ( sequence bigserial PRIMARY KEY, principal_id text NOT NULL, session_id text, event_type text NOT NULL CHECK (event_type IN ('session','command')), @@ -45,5 +71,40 @@ CREATE TABLE IF NOT EXISTS t4_outbox ( attempts integer NOT NULL DEFAULT 0 CHECK (attempts >= 0), next_attempt_at timestamptz NOT NULL DEFAULT clock_timestamp(), claimed_at timestamptz, terminal_result jsonb, last_error text, created_at timestamptz NOT NULL DEFAULT clock_timestamp(), updated_at timestamptz NOT NULL DEFAULT clock_timestamp() ); +UPDATE t4_outbox SET state = 'pending', owner_id = NULL, owner_epoch = NULL, claimed_at = NULL, next_attempt_at = clock_timestamp(), updated_at = clock_timestamp() WHERE state = 'claimed' AND (owner_id IS NULL OR btrim(owner_id) = '' OR owner_epoch IS NULL OR owner_epoch <= 0 OR claimed_at IS NULL); +UPDATE t4_outbox SET owner_id = NULL, owner_epoch = NULL, claimed_at = NULL, updated_at = clock_timestamp() WHERE state <> 'claimed' AND (owner_id IS NOT NULL OR owner_epoch IS NOT NULL OR claimed_at IS NOT NULL); +ALTER TABLE t4_outbox DROP CONSTRAINT IF EXISTS t4_outbox_owner_tuple_check; +ALTER TABLE t4_outbox DROP CONSTRAINT IF EXISTS t4_outbox_claimed_identity_check; +ALTER TABLE t4_outbox DROP CONSTRAINT IF EXISTS t4_outbox_nonclaimed_identity_check; +ALTER TABLE t4_outbox ADD CONSTRAINT t4_outbox_owner_tuple_check CHECK ((owner_id IS NULL AND owner_epoch IS NULL) OR (owner_id IS NOT NULL AND btrim(owner_id) <> '' AND owner_epoch IS NOT NULL AND owner_epoch > 0)) NOT VALID; +ALTER TABLE t4_outbox ADD CONSTRAINT t4_outbox_claimed_identity_check CHECK (state <> 'claimed' OR (owner_id IS NOT NULL AND btrim(owner_id) <> '' AND owner_epoch IS NOT NULL AND owner_epoch > 0 AND claimed_at IS NOT NULL)) NOT VALID; +ALTER TABLE t4_outbox ADD CONSTRAINT t4_outbox_nonclaimed_identity_check CHECK (state = 'claimed' OR (owner_id IS NULL AND owner_epoch IS NULL AND claimed_at IS NULL)) NOT VALID; +ALTER TABLE t4_outbox VALIDATE CONSTRAINT t4_outbox_owner_tuple_check; +ALTER TABLE t4_outbox VALIDATE CONSTRAINT t4_outbox_claimed_identity_check; +ALTER TABLE t4_outbox VALIDATE CONSTRAINT t4_outbox_nonclaimed_identity_check; CREATE INDEX IF NOT EXISTS t4_outbox_ready ON t4_outbox (state, next_attempt_at, outbox_id); +CREATE TABLE IF NOT EXISTS t4_kubernetes_status_cursors ( + collection text PRIMARY KEY CHECK (collection IN ('t4workspaces','t4sessions')), resource_version text NOT NULL CHECK (octet_length(resource_version) BETWEEN 1 AND 256), + owner_epoch bigint NOT NULL CHECK (owner_epoch > 0), updated_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE IF NOT EXISTS t4_stale_create_cleanups ( + cleanup_id bigserial PRIMARY KEY, outbox_id bigint NOT NULL REFERENCES t4_outbox(outbox_id), + resource_type text NOT NULL CHECK (resource_type IN ('t4workspaces','t4sessions')), target_id text NOT NULL, + uid text NOT NULL CHECK (octet_length(uid) BETWEEN 1 AND 256), resource_version text NOT NULL CHECK (octet_length(resource_version) BETWEEN 1 AND 256), + state text NOT NULL DEFAULT 'pending' CHECK (state IN ('pending','claimed','applied')), owner_id text, owner_epoch bigint, + attempts integer NOT NULL DEFAULT 0 CHECK (attempts >= 0), next_attempt_at timestamptz NOT NULL DEFAULT clock_timestamp(), claimed_at timestamptz, + last_error text, created_at timestamptz NOT NULL DEFAULT clock_timestamp(), updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE (outbox_id, uid) +); +UPDATE t4_stale_create_cleanups cleanup SET state = 'applied', owner_id = NULL, owner_epoch = NULL, claimed_at = NULL, last_error = NULL, updated_at = clock_timestamp() + WHERE cleanup.state <> 'applied' AND EXISTS (SELECT 1 FROM t4_outbox item WHERE item.outbox_id = cleanup.outbox_id AND item.state = 'applied'); +UPDATE t4_stale_create_cleanups SET state = 'pending', owner_id = NULL, owner_epoch = NULL, claimed_at = NULL, next_attempt_at = clock_timestamp(), updated_at = clock_timestamp() WHERE state = 'claimed' AND (owner_id IS NULL OR btrim(owner_id) = '' OR owner_epoch IS NULL OR owner_epoch <= 0 OR claimed_at IS NULL); +UPDATE t4_stale_create_cleanups SET owner_id = NULL, owner_epoch = NULL, claimed_at = NULL, updated_at = clock_timestamp() WHERE state <> 'claimed' AND (owner_id IS NOT NULL OR owner_epoch IS NOT NULL OR claimed_at IS NOT NULL); +ALTER TABLE t4_stale_create_cleanups DROP CONSTRAINT IF EXISTS t4_stale_create_cleanups_claim_check; +ALTER TABLE t4_stale_create_cleanups DROP CONSTRAINT IF EXISTS t4_stale_create_cleanups_nonclaimed_check; +ALTER TABLE t4_stale_create_cleanups ADD CONSTRAINT t4_stale_create_cleanups_claim_check CHECK (state <> 'claimed' OR (owner_id IS NOT NULL AND btrim(owner_id) <> '' AND owner_epoch IS NOT NULL AND owner_epoch > 0 AND claimed_at IS NOT NULL)) NOT VALID; +ALTER TABLE t4_stale_create_cleanups ADD CONSTRAINT t4_stale_create_cleanups_nonclaimed_check CHECK (state = 'claimed' OR (owner_id IS NULL AND owner_epoch IS NULL AND claimed_at IS NULL)) NOT VALID; +ALTER TABLE t4_stale_create_cleanups VALIDATE CONSTRAINT t4_stale_create_cleanups_claim_check; +ALTER TABLE t4_stale_create_cleanups VALIDATE CONSTRAINT t4_stale_create_cleanups_nonclaimed_check; +CREATE INDEX IF NOT EXISTS t4_stale_create_cleanups_ready ON t4_stale_create_cleanups (state, next_attempt_at, cleanup_id); INSERT INTO t4_schema_migrations(version) VALUES (1) ON CONFLICT (version) DO NOTHING; diff --git a/packages/cluster-server/src/kubernetes-runner.ts b/packages/cluster-server/src/kubernetes-runner.ts index 3f23a5ef..7268594a 100644 --- a/packages/cluster-server/src/kubernetes-runner.ts +++ b/packages/cluster-server/src/kubernetes-runner.ts @@ -1,5 +1,16 @@ import { KubernetesApiClient, KubernetesApiError } from "./kubernetes-client.ts"; -import { ClusterInfrastructureProjection, KubernetesAuthorityInvalidatedError } from "./kubernetes-projection.ts"; +import { + ClusterInfrastructureProjection, + KubernetesAuthorityInvalidatedError, + type InfrastructureList, + type KubernetesResource, + type KubernetesWatchEvent, +} from "./kubernetes-projection.ts"; +import type { + AuthoritativeKubernetesStatusIngress, + AuthoritativeKubernetesStatusObservation, + KubernetesStatusCollection, +} from "./ledger.ts"; export interface KubernetesProjectionRunnerOptions { readonly client: KubernetesApiClient; @@ -7,6 +18,10 @@ export interface KubernetesProjectionRunnerOptions { readonly hostName: string; readonly onSynchronized?: () => void; readonly onError?: (error: unknown) => void; + readonly statusIngress?: ( + collection: KubernetesStatusCollection, + observation: AuthoritativeKubernetesStatusIngress, + ) => Promise; readonly retryMs?: number; } @@ -24,6 +39,7 @@ function delay(milliseconds: number, signal: AbortSignal): Promise { export class KubernetesProjectionRunner { readonly #options: KubernetesProjectionRunnerOptions; readonly #synchronized = new Set(); + readonly #workspaceOwners = new Map(); #abort?: AbortController; #running?: Promise; #relist?: Promise; @@ -34,6 +50,7 @@ export class KubernetesProjectionRunner { this.#abort = new AbortController(); const signal = this.#abort.signal; const initial = await this.#options.client.listInfrastructure(this.#options.hostName, signal); + await this.#prepareSnapshot(initial, signal); this.#options.projection.replace(initial); this.#synchronized.clear(); for (const resource of ["t4clusterhosts", "t4workspaces", "t4sessions"]) this.#synchronized.add(resource); @@ -48,6 +65,143 @@ export class KubernetesProjectionRunner { this.#abort = undefined; this.#relist = undefined; this.#synchronized.clear(); + this.#workspaceOwners.clear(); + } + + async #prepareSnapshot(snapshot: InfrastructureList, signal: AbortSignal): Promise { + await this.#reconcileLegacySessionAnnotations(snapshot.sessions, signal); + const workspaceOwners = new Map(); + for (const workspace of snapshot.workspaces) { + const workspaceId = this.#requiredString(workspace.metadata.name, "workspace name"); + const owner = this.#requiredString(workspace.spec?.owner, "workspace owner"); + workspaceOwners.set(workspaceId, owner); + } + const ingress = this.#options.statusIngress; + let ingressAvailable = ingress !== undefined; + if (ingress) { + for (const workspace of snapshot.workspaces) { + if (signal.aborted) throw signal.reason; + if (ingressAvailable) ingressAvailable = await this.#emitStatus("t4workspaces", this.#statusObservation("t4workspaces", workspace, workspaceOwners)); + } + for (const session of snapshot.sessions) { + if (signal.aborted) throw signal.reason; + if (ingressAvailable) ingressAvailable = await this.#emitStatus("t4sessions", this.#statusObservation("t4sessions", session, workspaceOwners)); + } + if (ingressAvailable) ingressAvailable = await this.#emitStatus("t4workspaces", { + relisted: true, + resourceVersion: this.#requiredString(snapshot.resourceVersions?.t4workspaces ?? snapshot.resourceVersion, "workspace list resourceVersion"), + resourceIds: [...workspaceOwners.keys()], + }); + if (ingressAvailable) await this.#emitStatus("t4sessions", { + relisted: true, + resourceVersion: this.#requiredString(snapshot.resourceVersions?.t4sessions ?? snapshot.resourceVersion, "session list resourceVersion"), + resourceIds: snapshot.sessions.map(session => this.#requiredString(session.metadata.name, "session name")), + }); + } + this.#workspaceOwners.clear(); + for (const [workspaceId, owner] of workspaceOwners) this.#workspaceOwners.set(workspaceId, owner); + } + + async #reconcileLegacySessionAnnotations(sessions: readonly KubernetesResource[], signal: AbortSignal): Promise { + for (const session of sessions) { + const annotations = session.metadata.annotations; + if (!annotations || !Object.prototype.hasOwnProperty.call(annotations, "cluster.t4.dev/pending-command")) continue; + const resourceVersion = this.#requiredString(session.metadata.resourceVersion, "session resourceVersion"); + await this.#options.client.patch("t4sessions", session.metadata.name, { + metadata: { + resourceVersion, + annotations: { "cluster.t4.dev/pending-command": null }, + }, + }, signal); + } + } + + #statusObservation( + collection: KubernetesStatusCollection, + resource: KubernetesResource, + workspaceOwners: ReadonlyMap, + deleted = false, + ): AuthoritativeKubernetesStatusObservation { + const expectedKind = collection === "t4workspaces" ? "T4Workspace" : "T4Session"; + if (resource.kind !== expectedKind) throw new Error(`Kubernetes ${collection} status kind is invalid`); + const generation = resource.metadata.generation; + if (generation === undefined || !Number.isSafeInteger(generation) || generation < 1) + throw new Error(`Kubernetes ${collection} generation is invalid`); + const observedGeneration = resource.status?.observedGeneration ?? 0; + if (typeof observedGeneration !== "number" || !Number.isSafeInteger(observedGeneration) + || observedGeneration < 0 || observedGeneration > generation) + throw new Error(`Kubernetes ${collection} observedGeneration is invalid`); + const rawPhase = deleted || resource.metadata.deletionTimestamp + ? "Terminating" + : resource.status?.phase; + const phase = collection === "t4workspaces" + ? rawPhase === "Pending" || rawPhase === "Ready" || rawPhase === "Failed" || rawPhase === "Terminating" ? rawPhase : "Unknown" + : rawPhase === "Pending" || rawPhase === "Running" || rawPhase === "Failed" || rawPhase === "Terminating" ? rawPhase : "Unknown"; + const resourceId = this.#requiredString(resource.metadata.name, `${collection} name`); + const uid = this.#requiredString(resource.metadata.uid, `${collection} UID`); + const resourceVersion = this.#requiredString(resource.metadata.resourceVersion, `${collection} resourceVersion`); + if (collection === "t4workspaces") { + return { + resourceId, + principalId: this.#requiredString(resource.spec?.owner, "workspace owner"), + uid, + resourceVersion, + generation: BigInt(generation), + observedGeneration: BigInt(observedGeneration), + phase, + ...(deleted ? { deleted: true } : {}), + }; + } + const workspaceId = this.#requiredString(resource.spec?.workspaceRef, "session workspace reference"); + return { + resourceId, + principalId: this.#requiredString(workspaceOwners.get(workspaceId), "session workspace owner"), + workspaceId, + uid, + resourceVersion, + generation: BigInt(generation), + observedGeneration: BigInt(observedGeneration), + phase, + ...(deleted ? { deleted: true } : {}), + }; + } + + #requiredString(value: unknown, name: string): string { + if (typeof value !== "string" || value.length === 0) throw new Error(`Kubernetes ${name} is invalid`); + return value; + } + + #statusIngressLeaseUnavailable(error: unknown): boolean { + return error instanceof Error && ( + error.message === "outbox lease is held by another owner" + || error.message === "Kubernetes status owner lease is no longer current" + ); + } + + async #emitStatus(collection: KubernetesStatusCollection, observation: AuthoritativeKubernetesStatusIngress): Promise { + const ingress = this.#options.statusIngress; + if (!ingress) return false; + try { + await ingress(collection, observation); + return true; + } catch (error) { + if (this.#statusIngressLeaseUnavailable(error)) return false; + throw error; + } + } + + async #ingestWatchStatus(collection: KubernetesStatusCollection, event: KubernetesWatchEvent): Promise { + if (this.#options.statusIngress) { + await this.#emitStatus(collection, this.#statusObservation(collection, event.object, this.#workspaceOwners, event.type === "DELETED")); + } + if (collection === "t4workspaces") { + const workspaceId = this.#requiredString(event.object.metadata.name, "workspace name"); + if (event.type === "DELETED") this.#workspaceOwners.delete(workspaceId); + else { + const owner = this.#requiredString(event.object.spec?.owner, "workspace owner"); + this.#workspaceOwners.set(workspaceId, owner); + } + } } async #run(signal: AbortSignal): Promise { @@ -68,29 +222,52 @@ export class KubernetesProjectionRunner { generation: AbortController, ): Promise { let version = this.#options.projection.resourceVersionFor(resource); + const statusCollection = resource === "t4workspaces" || resource === "t4sessions" ? resource : undefined; while (!generation.signal.aborted) { + let pendingStatus = Promise.resolve(); + let statusFailed = false; + let statusError: unknown; + const applyEvent = (event: KubernetesWatchEvent): void => { + this.#options.projection.applyWatch(event); + version = event.object.metadata.resourceVersion ?? version; + }; + const onEvent = this.#options.statusIngress && statusCollection + ? (event: KubernetesWatchEvent): void => { + pendingStatus = pendingStatus.then(async () => { + if (statusFailed) return; + await this.#ingestWatchStatus(statusCollection, event); + applyEvent(event); + }).catch(error => { + statusFailed = true; + statusError = error; + generation.abort(error); + }); + } + : applyEvent; try { await this.#options.client.watch( resource, version, - event => { - this.#options.projection.applyWatch(event); - version = event.object.metadata.resourceVersion ?? version; - }, + onEvent, generation.signal, () => { this.#synchronized.add(resource); if (this.#synchronized.size === resourceCount) this.#options.onSynchronized?.(); }, ); + await pendingStatus; + if (statusFailed) throw statusError; if (generation.signal.aborted) return; this.#synchronized.delete(resource); this.#options.onError?.(new Error(`Kubernetes ${resource} watch ended`)); - } catch (error) { - if (generation.signal.aborted || rootSignal.aborted) return; + } catch (watchError) { + await pendingStatus; + const error = statusFailed ? statusError : watchError; + if (rootSignal.aborted || generation.signal.aborted && !statusFailed) return; this.#synchronized.delete(resource); this.#options.onError?.(error); - const requiresRelist = error instanceof KubernetesAuthorityInvalidatedError + const requiresRelist = statusFailed + || error instanceof KubernetesAuthorityInvalidatedError || error instanceof KubernetesApiError && error.status === 410; if (requiresRelist) { try { await this.#restartGeneration(generation, rootSignal); } @@ -113,6 +290,7 @@ export class KubernetesProjectionRunner { while (!signal.aborted) { try { const snapshot = await this.#options.client.listInfrastructure(this.#options.hostName, signal); + await this.#prepareSnapshot(snapshot, signal); this.#options.projection.replace(snapshot); return; } catch (error) { diff --git a/packages/cluster-server/src/ledger.ts b/packages/cluster-server/src/ledger.ts index b1d4e60c..53af9f4f 100644 --- a/packages/cluster-server/src/ledger.ts +++ b/packages/cluster-server/src/ledger.ts @@ -49,11 +49,43 @@ export interface OutboxClaim { readonly ownerId: string; readonly ownerEpoch: bigint; readonly expiresAt: number; + readonly claimedAt: string; } export interface OwnerLease { readonly ownerId: string; readonly epoch: bigint; } +export type KubernetesStatusCollection = "t4workspaces" | "t4sessions"; +export interface AuthoritativeKubernetesStatusObservation { + readonly resourceId: string; + readonly principalId: string; + readonly workspaceId?: string; + readonly uid: string; + readonly resourceVersion: string; + readonly generation: bigint; + readonly observedGeneration: bigint; + readonly phase: "Pending" | "Ready" | "Running" | "Failed" | "Terminating" | "Unknown"; + readonly deleted?: boolean; +} +export interface AuthoritativeKubernetesStatusSnapshot { + readonly relisted: true; + readonly resourceVersion: string; + readonly resourceIds: readonly string[]; +} +export type AuthoritativeKubernetesStatusIngress = AuthoritativeKubernetesStatusObservation | AuthoritativeKubernetesStatusSnapshot; +export interface StaleCreateCleanup { + readonly resourceType: KubernetesStatusCollection; + readonly targetId: string; + readonly uid: string; + readonly resourceVersion: string; +} +export interface StaleCreateCleanupClaim extends StaleCreateCleanup { + readonly cleanupId: bigint; + readonly ownerId: string; + readonly ownerEpoch: bigint; + readonly expiresAt: number; + readonly claimedAt: string; +} interface CommandRow { readonly fingerprint: string; @@ -83,7 +115,6 @@ interface EventRow { } interface RetentionRow { readonly first_retained_sequence: bigint; readonly latest_sequence: bigint; } interface LeaseRow { readonly owner_id: string; readonly epoch: bigint; readonly expired: boolean; } -interface LeaseClaimRow extends LeaseRow { readonly expires_at_ms: bigint; } interface OutboxRow { readonly outbox_id: bigint; readonly command_id: string; @@ -94,12 +125,39 @@ interface OutboxRow { readonly target_revision: bigint; readonly mutation: Record; } +interface LockedOutboxRow extends OutboxRow { + readonly state: "pending" | "claimed" | "applied" | "skipped" | "failed"; + readonly owner_id: string | null; + readonly owner_epoch: bigint | null; + readonly claimed_at: string | null; +} +interface CleanupRow { + readonly cleanup_id: bigint; + readonly resource_type: KubernetesStatusCollection; + readonly target_id: string; + readonly uid: string; + readonly resource_version: string; +} +interface MissingWorkspaceStatusRow { readonly workspace_id: string; } +interface MissingSessionStatusRow { readonly session_id: string; readonly principal_id: string; readonly revision: bigint; } +interface KubernetesStatusRow { + readonly state: WorkspaceRecord["state"] | SessionRecord["state"]; + readonly revision: bigint; + readonly generation: bigint; + readonly kube_uid: string | null; + readonly kube_resource_version: string | null; + readonly kube_generation: bigint | null; + readonly kube_observed_generation: bigint | null; + readonly cancellation_requested?: boolean; + readonly deletion_requested: boolean; +} const MIGRATION_URL = new URL("../migrations/001_durable_gateway.sql", import.meta.url); const ROLLBACK_URL = new URL("../migrations/001_durable_gateway.down.sql", import.meta.url); const OUTBOX_LEASE = "gateway-outbox"; const MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER); const SERIALIZABLE_ATTEMPTS = 3; +const STALE_APPLY_DEADLINE_SECONDS = 5; function persistedBigint(value: unknown, allowZero = false): bigint { if (typeof value !== "bigint") throw new Error("persisted ledger bigint is invalid"); @@ -144,6 +202,10 @@ function stableJson(value: unknown): string { } throw new Error("canonical JSON value is invalid"); } +function semanticJsonEqual(left: unknown, right: unknown): boolean { + try { return stableJson(left) === stableJson(right); } + catch { return false; } +} export function canonicalRequestFingerprint(value: unknown): string { return `sha256:${createHash("sha256").update(stableJson(value), "utf8").digest("hex")}`; } @@ -311,11 +373,12 @@ export class PostgresLedger { const value = session(current); const commandId = `cmd_${randomUUID()}`; const dispatchDeletion = deletion && !current.deletion_requested; - await transaction`INSERT INTO t4_commands (command_id, principal_id, operation, target_scope, idempotency_key, fingerprint, lifecycle_state, response_status, response_body) VALUES (${commandId}, ${principalId}, ${operation}, ${scope}, ${idempotencyKey}, ${fingerprint}, ${dispatchDeletion ? "accepted" : "completed"}, ${deletion ? 204 : 202}, ${value})`; + await transaction`INSERT INTO t4_commands (command_id, principal_id, operation, target_scope, idempotency_key, fingerprint, lifecycle_state, response_status, response_body) VALUES (${commandId}, ${principalId}, ${operation}, ${scope}, ${idempotencyKey}, ${fingerprint}, ${dispatchDeletion ? "accepted" : "projected"}, ${deletion ? 204 : 202}, ${value})`; if (dispatchDeletion) { await transaction`UPDATE t4_session_intents SET deletion_requested = true, generation = generation + 1, updated_at = clock_timestamp() WHERE session_id = ${sessionId}`; await transaction`INSERT INTO t4_outbox (command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation) VALUES (${commandId}, ${principalId}, ${idempotencyKey}, 'session.delete', ${sessionId}, ${current.revision}, ${{ sessionId, revision: safeNumber(current.revision) }})`; } + await this.#terminalizePendingSessionCommands(transaction, principalId, sessionId, "session cancelled or deleted", 0n); await this.#appendEvent(transaction, principalId, sessionId, "session", { state: current.state, revision: safeNumber(current.revision) }, 0n); return { kind: "accepted", commandId, value }; } @@ -324,7 +387,8 @@ export class PostgresLedger { const commandId = `cmd_${randomUUID()}`; await transaction`INSERT INTO t4_commands (command_id, principal_id, operation, target_scope, idempotency_key, fingerprint, lifecycle_state, response_status, response_body) VALUES (${commandId}, ${principalId}, ${operation}, ${scope}, ${idempotencyKey}, ${fingerprint}, 'accepted', ${deletion ? 204 : 202}, ${value})`; await transaction`UPDATE t4_session_intents SET state = 'cancelling', cancellation_requested = true, deletion_requested = deletion_requested OR ${deletion}, revision = ${revision}, generation = generation + 1, updated_at = clock_timestamp() WHERE session_id = ${sessionId}`; - await transaction`UPDATE t4_outbox SET state = 'skipped', terminal_result = ${{ reason: "superseded by cancellation" }}, updated_at = clock_timestamp() WHERE target_id = ${sessionId} AND mutation_kind = 'session.create' AND state IN ('pending','claimed')`; + await transaction`UPDATE t4_outbox SET state = 'skipped', owner_id = NULL, owner_epoch = NULL, claimed_at = NULL, terminal_result = ${{ reason: "superseded by cancellation" }}, updated_at = clock_timestamp() WHERE target_id = ${sessionId} AND mutation_kind = 'session.create' AND state IN ('pending','claimed')`; + await this.#terminalizePendingSessionCommands(transaction, principalId, sessionId, "session cancelled or deleted", 0n); await transaction`INSERT INTO t4_outbox (command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation) VALUES (${commandId}, ${principalId}, ${idempotencyKey}, ${deletion ? "session.delete" : "session.cancel"}, ${sessionId}, ${revision}, ${{ sessionId, revision: safeNumber(revision) }})`; await this.#appendEvent(transaction, principalId, sessionId, "session", { state: "cancelling", revision: safeNumber(revision) }, 0n); return { kind: "accepted", commandId, value }; @@ -391,69 +455,227 @@ export class PostgresLedger { return typeof result === "string" ? result : undefined; } - async acquireLease(ownerId: string, leaseSeconds = 30): Promise { + async tryAcquireLease(ownerId: string, leaseSeconds = 30): Promise { + if (!ownerId.trim() || Buffer.byteLength(ownerId, "utf8") > 256) throw new Error("outbox owner id is invalid"); + if (!Number.isSafeInteger(leaseSeconds) || leaseSeconds <= 0) throw new Error("outbox lease duration is invalid"); const rows = await this.#serializable(async transaction => { const current = await transaction`SELECT owner_id, epoch, expires_at <= clock_timestamp() AS expired FROM t4_owner_leases WHERE lease_name = ${OUTBOX_LEASE} FOR UPDATE`; - if (!current[0]) { - return await transaction`INSERT INTO t4_owner_leases (lease_name, owner_id, epoch, expires_at) VALUES (${OUTBOX_LEASE}, ${ownerId}, 1, clock_timestamp() + (${leaseSeconds} * interval '1 second')) RETURNING owner_id, epoch, false AS expired`; - } - if (current[0].owner_id === ownerId && !current[0].expired) { - return await transaction`UPDATE t4_owner_leases SET expires_at = clock_timestamp() + (${leaseSeconds} * interval '1 second'), updated_at = clock_timestamp() WHERE lease_name = ${OUTBOX_LEASE} RETURNING owner_id, epoch, false AS expired`; - } - if (!current[0].expired) throw new Error("outbox lease is held by another owner"); + if (!current[0]) return await transaction`INSERT INTO t4_owner_leases (lease_name, owner_id, epoch, expires_at) VALUES (${OUTBOX_LEASE}, ${ownerId}, 1, clock_timestamp() + (${leaseSeconds} * interval '1 second')) RETURNING owner_id, epoch, false AS expired`; + if (current[0].owner_id === ownerId && !current[0].expired) return await transaction`UPDATE t4_owner_leases SET expires_at = clock_timestamp() + (${leaseSeconds} * interval '1 second'), updated_at = clock_timestamp() WHERE lease_name = ${OUTBOX_LEASE} RETURNING owner_id, epoch, false AS expired`; + if (!current[0].expired) return undefined; return await transaction`UPDATE t4_owner_leases SET owner_id = ${ownerId}, epoch = epoch + 1, expires_at = clock_timestamp() + (${leaseSeconds} * interval '1 second'), updated_at = clock_timestamp() WHERE lease_name = ${OUTBOX_LEASE} RETURNING owner_id, epoch, false AS expired`; }); - return { ownerId: rows[0]!.owner_id, epoch: persistedBigint(rows[0]!.epoch) }; + return rows?.[0] ? { ownerId: rows[0].owner_id, epoch: persistedBigint(rows[0].epoch) } : undefined; + } + + async acquireLease(ownerId: string, leaseSeconds = 30): Promise { + const lease = await this.tryAcquireLease(ownerId, leaseSeconds); + if (!lease) throw new Error("outbox lease is held by another owner"); + return lease; } async claimNext(lease: OwnerLease): Promise { return await this.#serializable(async transaction => { - const current = await transaction`SELECT owner_id, epoch, expires_at <= clock_timestamp() AS expired, floor(extract(epoch FROM expires_at) * 1000)::bigint AS expires_at_ms FROM t4_owner_leases WHERE lease_name = ${OUTBOX_LEASE} FOR UPDATE`; + const current = await transaction`SELECT owner_id, epoch, expires_at <= clock_timestamp() AS expired FROM t4_owner_leases WHERE lease_name = ${OUTBOX_LEASE} FOR UPDATE`; if (current[0]?.owner_id !== lease.ownerId || current[0].epoch !== lease.epoch || current[0].expired) return undefined; - const rows = await transaction`SELECT outbox_id, command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation FROM t4_outbox WHERE (state = 'pending' OR (state = 'claimed' AND owner_epoch < ${lease.epoch})) AND next_attempt_at <= clock_timestamp() ORDER BY outbox_id FOR UPDATE SKIP LOCKED LIMIT 1`; + const rows = await transaction`SELECT outbox_id, command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation FROM t4_outbox WHERE (state = 'pending' OR (state = 'claimed' AND owner_epoch < ${lease.epoch}) OR (state = 'claimed' AND owner_id = ${lease.ownerId} AND owner_epoch = ${lease.epoch} AND claimed_at <= clock_timestamp() - (${STALE_APPLY_DEADLINE_SECONDS} * interval '1 second'))) AND next_attempt_at <= clock_timestamp() ORDER BY outbox_id FOR UPDATE SKIP LOCKED LIMIT 1`; const row = rows[0]; if (!row) return undefined; let superseded = false; if (row.mutation_kind.startsWith("session.")) { const intent = await transaction<{ revision: bigint; cancellation_requested: boolean }[]>`SELECT revision, cancellation_requested FROM t4_session_intents WHERE session_id = ${row.target_id}`; superseded = !intent[0] || intent[0].revision !== row.target_revision || row.mutation_kind === "session.create" && intent[0].cancellation_requested; + } else if (row.mutation_kind === "command.submit") { + const intent = await transaction<{ state: SessionRecord["state"]; cancellation_requested: boolean; deletion_requested: boolean }[]>`SELECT state, cancellation_requested, deletion_requested FROM t4_session_intents WHERE session_id = ${row.target_id}`; + if (intent[0] && (intent[0].cancellation_requested || intent[0].deletion_requested || intent[0].state === "cancelling" || intent[0].state === "cancelled")) { + await this.#terminalizePendingSessionCommands(transaction, row.principal_id, row.target_id, "session cancelled or deleted", lease.epoch); + return undefined; + } } else if (row.mutation_kind.startsWith("workspace.")) { const intent = await transaction<{ revision: bigint; deletion_requested: boolean }[]>`SELECT revision, deletion_requested FROM t4_workspace_intents WHERE workspace_id = ${row.target_id}`; superseded = !intent[0] || intent[0].revision !== row.target_revision || row.mutation_kind === "workspace.create" && intent[0].deletion_requested; } if (superseded) { - await transaction`UPDATE t4_outbox SET state = 'skipped', owner_id = ${lease.ownerId}, owner_epoch = ${lease.epoch}, terminal_result = ${{ reason: "superseded intent" }}, updated_at = clock_timestamp() WHERE outbox_id = ${row.outbox_id}`; + await transaction`UPDATE t4_outbox SET state = 'skipped', owner_id = NULL, owner_epoch = NULL, claimed_at = NULL, terminal_result = ${{ reason: "superseded intent", ownerId: lease.ownerId, ownerEpoch: lease.epoch.toString() }}, updated_at = clock_timestamp() WHERE outbox_id = ${row.outbox_id}`; return undefined; } - await transaction`UPDATE t4_outbox SET state = 'claimed', owner_id = ${lease.ownerId}, owner_epoch = ${lease.epoch}, attempts = attempts + 1, claimed_at = clock_timestamp(), updated_at = clock_timestamp() WHERE outbox_id = ${row.outbox_id}`; - return { outboxId: row.outbox_id, commandId: row.command_id, principalId: row.principal_id, idempotencyKey: row.idempotency_key, kind: row.mutation_kind, targetId: row.target_id, targetRevision: row.target_revision, mutation: row.mutation, ownerId: lease.ownerId, ownerEpoch: lease.epoch, expiresAt: safeNumber(current[0].expires_at_ms) }; + const claimed = await transaction<{ claimed_at: string; apply_expires_at_ms: bigint }[]>`UPDATE t4_outbox SET state = 'claimed', owner_id = ${lease.ownerId}, owner_epoch = ${lease.epoch}, attempts = attempts + 1, claimed_at = clock_timestamp(), updated_at = clock_timestamp() WHERE outbox_id = ${row.outbox_id} RETURNING floor(extract(epoch FROM claimed_at) * 1000000)::numeric::text AS claimed_at, floor(extract(epoch FROM LEAST(claimed_at + (${STALE_APPLY_DEADLINE_SECONDS} * interval '1 second'), (SELECT expires_at FROM t4_owner_leases WHERE lease_name = ${OUTBOX_LEASE}))) * 1000)::bigint AS apply_expires_at_ms`; + return { outboxId: row.outbox_id, commandId: row.command_id, principalId: row.principal_id, idempotencyKey: row.idempotency_key, kind: row.mutation_kind, targetId: row.target_id, targetRevision: row.target_revision, mutation: row.mutation, ownerId: lease.ownerId, ownerEpoch: lease.epoch, expiresAt: safeNumber(claimed[0]!.apply_expires_at_ms), claimedAt: claimed[0]!.claimed_at }; }); } async claimIsCurrent(claim: OutboxClaim): Promise { - const rows = await this.#sql<{ valid: boolean }[]>`SELECT EXISTS (SELECT 1 FROM t4_owner_leases lease JOIN t4_outbox item ON item.outbox_id = ${claim.outboxId} WHERE lease.lease_name = ${OUTBOX_LEASE} AND lease.owner_id = ${claim.ownerId} AND lease.epoch = ${claim.ownerEpoch} AND lease.expires_at > clock_timestamp() AND item.state = 'claimed' AND item.owner_id = lease.owner_id AND item.owner_epoch = lease.epoch) AS valid`; - return rows[0]?.valid === true; + const rows = await this.#sql<{ mutation: Record }[]>`SELECT item.mutation FROM t4_owner_leases lease JOIN t4_outbox item ON item.outbox_id = ${claim.outboxId} WHERE lease.lease_name = ${OUTBOX_LEASE} AND lease.owner_id = ${claim.ownerId} AND lease.epoch = ${claim.ownerEpoch} AND lease.expires_at > clock_timestamp() AND item.state = 'claimed' AND item.owner_id = lease.owner_id AND item.owner_epoch = lease.epoch AND floor(extract(epoch FROM item.claimed_at) * 1000000)::numeric::text = ${claim.claimedAt} AND item.command_id = ${claim.commandId} AND item.principal_id = ${claim.principalId} AND item.idempotency_key = ${claim.idempotencyKey} AND item.mutation_kind = ${claim.kind} AND item.target_id = ${claim.targetId} AND item.target_revision = ${claim.targetRevision} LIMIT 1`; + return rows[0] !== undefined && semanticJsonEqual(rows[0].mutation, claim.mutation); } async acknowledge(claim: OutboxClaim): Promise { return await this.#serializable(async transaction => { const lease = await transaction`SELECT owner_id, epoch, expires_at <= clock_timestamp() AS expired FROM t4_owner_leases WHERE lease_name = ${OUTBOX_LEASE} FOR UPDATE`; if (lease[0]?.owner_id !== claim.ownerId || lease[0].epoch !== claim.ownerEpoch || lease[0].expired) return false; - const updated = await transaction<{ outbox_id: bigint }[]>`UPDATE t4_outbox SET state = 'applied', terminal_result = ${{ applied: true }}, updated_at = clock_timestamp() WHERE outbox_id = ${claim.outboxId} AND state = 'claimed' AND owner_id = ${claim.ownerId} AND owner_epoch = ${claim.ownerEpoch} RETURNING outbox_id`; - if (!updated[0]) return false; - await transaction`UPDATE t4_commands SET lifecycle_state = 'completed', updated_at = clock_timestamp() WHERE command_id = ${claim.commandId}`; - if (claim.kind === "workspace.create") await transaction`UPDATE t4_workspace_intents SET state = 'provisioning', updated_at = clock_timestamp() WHERE workspace_id = ${claim.targetId} AND revision = ${claim.targetRevision} AND state = 'accepted' AND deletion_requested = false`; - if (claim.kind === "session.create") { - const transitioned = await transaction<{ session_id: string }[]>`UPDATE t4_session_intents SET state = 'provisioning', updated_at = clock_timestamp() WHERE session_id = ${claim.targetId} AND revision = ${claim.targetRevision} AND state = 'accepted' AND cancellation_requested = false RETURNING session_id`; - if (transitioned[0]) await this.#appendEvent(transaction, claim.principalId, claim.targetId, "session", { state: "provisioning", revision: safeNumber(claim.targetRevision) }, claim.ownerEpoch); + const rows = await transaction`SELECT outbox_id, command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation, state, owner_id, owner_epoch, floor(extract(epoch FROM claimed_at) * 1000000)::numeric::text AS claimed_at FROM t4_outbox WHERE outbox_id = ${claim.outboxId} FOR UPDATE`; + const row = rows[0]; + if (!row || row.state !== "claimed" || row.owner_id !== claim.ownerId || row.owner_epoch !== claim.ownerEpoch || row.claimed_at !== claim.claimedAt + || row.command_id !== claim.commandId || row.principal_id !== claim.principalId || row.idempotency_key !== claim.idempotencyKey + || row.mutation_kind !== claim.kind || row.target_id !== claim.targetId || row.target_revision !== claim.targetRevision + || !semanticJsonEqual(row.mutation, claim.mutation)) return false; + await transaction`UPDATE t4_outbox SET state = 'applied', owner_id = NULL, owner_epoch = NULL, claimed_at = NULL, terminal_result = ${{ applied: true, ownerId: claim.ownerId, ownerEpoch: claim.ownerEpoch.toString() }}, updated_at = clock_timestamp() WHERE outbox_id = ${row.outbox_id}`; + await transaction`UPDATE t4_stale_create_cleanups SET state = 'applied', owner_id = NULL, owner_epoch = NULL, claimed_at = NULL, last_error = NULL, updated_at = clock_timestamp() WHERE outbox_id = ${row.outbox_id} AND state <> 'applied'`; + const projected = await transaction<{ command_id: string }[]>`UPDATE t4_commands SET lifecycle_state = 'projected', updated_at = clock_timestamp() WHERE command_id = ${row.command_id} AND lifecycle_state = 'accepted' RETURNING command_id`; + if (projected[0] && row.mutation_kind === "command.submit") await this.#appendEvent(transaction, row.principal_id, row.target_id, "command", { commandId: row.command_id, state: "projected" }, claim.ownerEpoch); + if (row.mutation_kind === "workspace.create") await transaction`UPDATE t4_workspace_intents SET state = 'provisioning', updated_at = clock_timestamp() WHERE workspace_id = ${row.target_id} AND revision = ${row.target_revision} AND state = 'accepted' AND deletion_requested = false`; + if (row.mutation_kind === "session.create") { + const transitioned = await transaction<{ session_id: string }[]>`UPDATE t4_session_intents SET state = 'provisioning', updated_at = clock_timestamp() WHERE session_id = ${row.target_id} AND revision = ${row.target_revision} AND state = 'accepted' AND cancellation_requested = false RETURNING session_id`; + if (transitioned[0]) await this.#appendEvent(transaction, row.principal_id, row.target_id, "session", { state: "provisioning", revision: safeNumber(row.target_revision) }, claim.ownerEpoch); } return true; }); } async recordFailure(claim: OutboxClaim, message: string): Promise { - const result = await this.#sql<{ outbox_id: bigint }[]>`UPDATE t4_outbox item SET state = 'pending', owner_id = NULL, owner_epoch = NULL, last_error = ${message.slice(0, 1024)}, next_attempt_at = clock_timestamp() + interval '1 second', updated_at = clock_timestamp() FROM t4_owner_leases lease WHERE item.outbox_id = ${claim.outboxId} AND item.state = 'claimed' AND item.owner_id = ${claim.ownerId} AND item.owner_epoch = ${claim.ownerEpoch} AND lease.lease_name = ${OUTBOX_LEASE} AND lease.owner_id = ${claim.ownerId} AND lease.epoch = ${claim.ownerEpoch} AND lease.expires_at > clock_timestamp() RETURNING item.outbox_id`; - return result.length === 1; + return await this.#serializable(async transaction => { + const lease = await transaction`SELECT owner_id, epoch, expires_at <= clock_timestamp() AS expired FROM t4_owner_leases WHERE lease_name = ${OUTBOX_LEASE} FOR UPDATE`; + if (lease[0]?.owner_id !== claim.ownerId || lease[0].epoch !== claim.ownerEpoch || lease[0].expired) return false; + const rows = await transaction`SELECT outbox_id, command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation, state, owner_id, owner_epoch, floor(extract(epoch FROM claimed_at) * 1000000)::numeric::text AS claimed_at FROM t4_outbox WHERE outbox_id = ${claim.outboxId} FOR UPDATE`; + const row = rows[0]; + if (!row || row.state !== "claimed" || row.owner_id !== claim.ownerId || row.owner_epoch !== claim.ownerEpoch || row.claimed_at !== claim.claimedAt + || row.command_id !== claim.commandId || row.principal_id !== claim.principalId || row.idempotency_key !== claim.idempotencyKey + || row.mutation_kind !== claim.kind || row.target_id !== claim.targetId || row.target_revision !== claim.targetRevision + || !semanticJsonEqual(row.mutation, claim.mutation)) return false; + await transaction`UPDATE t4_outbox SET state = 'pending', owner_id = NULL, owner_epoch = NULL, claimed_at = NULL, last_error = ${message.slice(0, 1024)}, next_attempt_at = clock_timestamp() + interval '1 second', updated_at = clock_timestamp() WHERE outbox_id = ${row.outbox_id}`; + return true; + }); + } + + async ingestKubernetesStatus(lease: OwnerLease, collection: KubernetesStatusCollection, ingress: AuthoritativeKubernetesStatusIngress): Promise { + if (collection !== "t4workspaces" && collection !== "t4sessions") throw new Error("Kubernetes status collection is invalid"); + if ("relisted" in ingress) { + if (!ingress.resourceVersion || Buffer.byteLength(ingress.resourceVersion, "utf8") > 256 + || !Array.isArray(ingress.resourceIds) || ingress.resourceIds.some(resourceId => typeof resourceId !== "string" || !resourceId || Buffer.byteLength(resourceId, "utf8") > 256)) throw new Error("Kubernetes status relist identity is invalid"); + const present = new Set(ingress.resourceIds); + await this.#serializable(async transaction => { + const currentLease = await transaction`SELECT owner_id, epoch, expires_at <= clock_timestamp() AS expired FROM t4_owner_leases WHERE lease_name = ${OUTBOX_LEASE} FOR UPDATE`; + if (currentLease[0]?.owner_id !== lease.ownerId || currentLease[0].epoch !== lease.epoch || currentLease[0].expired) throw new Error("Kubernetes status owner lease is no longer current"); + if (collection === "t4workspaces") { + const missing = await transaction`SELECT workspace_id FROM t4_workspace_intents WHERE state = 'deleting' AND deletion_requested = true FOR UPDATE`; + for (const row of missing) { + if (!present.has(row.workspace_id)) await transaction`UPDATE t4_workspace_intents SET state = 'deleted', updated_at = clock_timestamp() WHERE workspace_id = ${row.workspace_id} AND state = 'deleting'`; + } + } else { + const missing = await transaction`SELECT session_id, principal_id, revision FROM t4_session_intents WHERE state <> 'cancelled' AND (state = 'cancelling' AND cancellation_requested = true OR deletion_requested = true) FOR UPDATE`; + for (const row of missing) { + if (present.has(row.session_id)) continue; + const transitioned = await transaction<{ session_id: string }[]>`UPDATE t4_session_intents SET state = 'cancelled', updated_at = clock_timestamp() WHERE session_id = ${row.session_id} AND state <> 'cancelled' RETURNING session_id`; + if (transitioned[0]) { + await this.#terminalizePendingSessionCommands(transaction, row.principal_id, row.session_id, "session absent from authoritative relist", lease.epoch); + await this.#appendEvent(transaction, row.principal_id, row.session_id, "session", { state: "cancelled", revision: safeNumber(row.revision) }, lease.epoch); + } + } + } + await transaction`INSERT INTO t4_kubernetes_status_cursors (collection, resource_version, owner_epoch) VALUES (${collection}, ${ingress.resourceVersion}, ${lease.epoch}) ON CONFLICT (collection) DO UPDATE SET resource_version = EXCLUDED.resource_version, owner_epoch = EXCLUDED.owner_epoch, updated_at = clock_timestamp()`; + }); + return; + } + const observation = ingress; + if (!observation.resourceId || !observation.principalId || !observation.uid || !observation.resourceVersion || Buffer.byteLength(observation.uid, "utf8") > 256 || Buffer.byteLength(observation.resourceVersion, "utf8") > 256) throw new Error("Kubernetes status identity is incomplete or unbounded"); + if (typeof observation.generation !== "bigint" || observation.generation <= 0n || typeof observation.observedGeneration !== "bigint" || observation.observedGeneration < 0n) throw new Error("Kubernetes status generation is invalid"); + if (collection === "t4sessions" && !observation.workspaceId) throw new Error("Kubernetes session status lacks workspace identity"); + await this.#serializable(async transaction => { + const currentLease = await transaction`SELECT owner_id, epoch, expires_at <= clock_timestamp() AS expired FROM t4_owner_leases WHERE lease_name = ${OUTBOX_LEASE} FOR UPDATE`; + if (currentLease[0]?.owner_id !== lease.ownerId || currentLease[0].epoch !== lease.epoch || currentLease[0].expired) throw new Error("Kubernetes status owner lease is no longer current"); + const rows = collection === "t4workspaces" + ? await transaction`SELECT state, revision, generation, kube_uid, kube_resource_version, kube_generation, kube_observed_generation, deletion_requested FROM t4_workspace_intents WHERE workspace_id = ${observation.resourceId} AND principal_id = ${observation.principalId} FOR UPDATE` + : await transaction`SELECT state, revision, generation, kube_uid, kube_resource_version, kube_generation, kube_observed_generation, cancellation_requested, deletion_requested FROM t4_session_intents WHERE session_id = ${observation.resourceId} AND principal_id = ${observation.principalId} AND workspace_id = ${observation.workspaceId!} FOR UPDATE`; + const row = rows[0]; + await transaction`INSERT INTO t4_kubernetes_status_cursors (collection, resource_version, owner_epoch) VALUES (${collection}, ${observation.resourceVersion}, ${lease.epoch}) ON CONFLICT (collection) DO UPDATE SET resource_version = EXCLUDED.resource_version, owner_epoch = EXCLUDED.owner_epoch, updated_at = clock_timestamp()`; + if (!row) return; + if (row.kube_uid !== null && row.kube_uid !== observation.uid) throw new Error("Kubernetes status UID conflicts with the durable resource identity"); + if (row.kube_resource_version === observation.resourceVersion && !observation.deleted) return; + if (!observation.deleted && (row.kube_generation !== null && observation.generation < row.kube_generation + || row.kube_generation === observation.generation && row.kube_observed_generation !== null && observation.observedGeneration < row.kube_observed_generation)) return; + const statusIsCurrent = observation.observedGeneration >= observation.generation && observation.generation >= row.generation; + if (collection === "t4workspaces") { + const absorbed = row.deletion_requested || row.state === "deleting" || row.state === "deleted"; + const mapped: WorkspaceRecord["state"] = observation.phase === "Pending" ? "provisioning" + : observation.phase === "Ready" || observation.phase === "Running" ? "ready" + : observation.phase === "Failed" ? "failed" + : observation.phase === "Terminating" ? "deleting" : "indeterminate"; + const nextState: WorkspaceRecord["state"] = observation.deleted ? "deleted" : statusIsCurrent && !absorbed ? mapped : row.state as WorkspaceRecord["state"]; + await transaction`UPDATE t4_workspace_intents SET kube_uid = ${observation.uid}, kube_resource_version = ${observation.resourceVersion}, kube_generation = ${observation.generation}, kube_observed_generation = ${observation.observedGeneration}, state = ${nextState}, updated_at = clock_timestamp() WHERE workspace_id = ${observation.resourceId}`; + } else { + const absorbed = row.cancellation_requested === true || row.deletion_requested || row.state === "cancelling" || row.state === "cancelled"; + const mapped: SessionRecord["state"] = observation.phase === "Pending" ? "provisioning" + : observation.phase === "Ready" || observation.phase === "Running" ? "ready" + : observation.phase === "Failed" ? "failed" + : observation.phase === "Terminating" ? "cancelling" : "indeterminate"; + const nextState: SessionRecord["state"] = observation.deleted ? "cancelled" : statusIsCurrent && !absorbed ? mapped : row.state as SessionRecord["state"]; + await transaction`UPDATE t4_session_intents SET kube_uid = ${observation.uid}, kube_resource_version = ${observation.resourceVersion}, kube_generation = ${observation.generation}, kube_observed_generation = ${observation.observedGeneration}, state = ${nextState}, updated_at = clock_timestamp() WHERE session_id = ${observation.resourceId}`; + if (nextState !== row.state) { + if (nextState === "cancelled") await this.#terminalizePendingSessionCommands(transaction, observation.principalId, observation.resourceId, "session deleted in Kubernetes", lease.epoch); + await this.#appendEvent(transaction, observation.principalId, observation.resourceId, "session", { state: nextState, revision: safeNumber(row.revision) }, lease.epoch); + } + } + }); + } + + async persistStaleCreateCleanup(claim: OutboxClaim, cleanup: StaleCreateCleanup): Promise { + if ((cleanup.resourceType !== "t4workspaces" && cleanup.resourceType !== "t4sessions") || !cleanup.targetId || !cleanup.uid || !cleanup.resourceVersion + || Buffer.byteLength(cleanup.uid, "utf8") > 256 || Buffer.byteLength(cleanup.resourceVersion, "utf8") > 256) return false; + return await this.#serializable(async transaction => { + const rows = await transaction`SELECT outbox_id, command_id, principal_id, idempotency_key, mutation_kind, target_id, target_revision, mutation, state, owner_id, owner_epoch, floor(extract(epoch FROM claimed_at) * 1000000)::numeric::text AS claimed_at FROM t4_outbox WHERE outbox_id = ${claim.outboxId} FOR UPDATE`; + const row = rows[0]; + const resourceType = row?.mutation_kind === "workspace.create" ? "t4workspaces" : row?.mutation_kind === "session.create" ? "t4sessions" : undefined; + if (!row || row.state === "applied" || resourceType !== cleanup.resourceType || row.target_id !== cleanup.targetId || row.command_id !== claim.commandId || row.principal_id !== claim.principalId + || row.idempotency_key !== claim.idempotencyKey || row.mutation_kind !== claim.kind || row.target_id !== claim.targetId || row.target_revision !== claim.targetRevision + || !semanticJsonEqual(row.mutation, claim.mutation)) return false; + await transaction`INSERT INTO t4_stale_create_cleanups (outbox_id, resource_type, target_id, uid, resource_version) VALUES (${row.outbox_id}, ${cleanup.resourceType}, ${cleanup.targetId}, ${cleanup.uid}, ${cleanup.resourceVersion}) ON CONFLICT (outbox_id, uid) DO UPDATE SET resource_version = EXCLUDED.resource_version, state = CASE WHEN t4_stale_create_cleanups.state = 'applied' THEN 'applied' ELSE 'pending' END, owner_id = NULL, owner_epoch = NULL, claimed_at = NULL, next_attempt_at = clock_timestamp(), updated_at = clock_timestamp()`; + return true; + }); + } + + async claimStaleCreateCleanup(lease: OwnerLease): Promise { + return await this.#serializable(async transaction => { + const current = await transaction`SELECT owner_id, epoch, expires_at <= clock_timestamp() AS expired FROM t4_owner_leases WHERE lease_name = ${OUTBOX_LEASE} FOR UPDATE`; + if (current[0]?.owner_id !== lease.ownerId || current[0].epoch !== lease.epoch || current[0].expired) return undefined; + const rows = await transaction`SELECT cleanup.cleanup_id, cleanup.resource_type, cleanup.target_id, cleanup.uid, cleanup.resource_version FROM t4_stale_create_cleanups cleanup JOIN t4_outbox item ON item.outbox_id = cleanup.outbox_id WHERE item.state <> 'applied' AND (cleanup.state = 'pending' OR (cleanup.state = 'claimed' AND cleanup.owner_epoch < ${lease.epoch}) OR (cleanup.state = 'claimed' AND cleanup.owner_id = ${lease.ownerId} AND cleanup.owner_epoch = ${lease.epoch} AND cleanup.claimed_at <= clock_timestamp() - (${STALE_APPLY_DEADLINE_SECONDS} * interval '1 second'))) AND cleanup.next_attempt_at <= clock_timestamp() ORDER BY cleanup.cleanup_id FOR UPDATE OF cleanup SKIP LOCKED LIMIT 1`; + const row = rows[0]; + if (!row) return undefined; + const claimed = await transaction<{ claimed_at: string; apply_expires_at_ms: bigint }[]>`UPDATE t4_stale_create_cleanups SET state = 'claimed', owner_id = ${lease.ownerId}, owner_epoch = ${lease.epoch}, attempts = attempts + 1, claimed_at = clock_timestamp(), updated_at = clock_timestamp() WHERE cleanup_id = ${row.cleanup_id} RETURNING floor(extract(epoch FROM claimed_at) * 1000000)::numeric::text AS claimed_at, floor(extract(epoch FROM LEAST(claimed_at + (${STALE_APPLY_DEADLINE_SECONDS} * interval '1 second'), (SELECT expires_at FROM t4_owner_leases WHERE lease_name = ${OUTBOX_LEASE}))) * 1000)::bigint AS apply_expires_at_ms`; + return { cleanupId: row.cleanup_id, resourceType: row.resource_type, targetId: row.target_id, uid: row.uid, resourceVersion: row.resource_version, ownerId: lease.ownerId, ownerEpoch: lease.epoch, expiresAt: safeNumber(claimed[0]!.apply_expires_at_ms), claimedAt: claimed[0]!.claimed_at }; + }); + } + + async staleCreateCleanupClaimIsCurrent(claim: StaleCreateCleanupClaim): Promise { + const rows = await this.#sql<{ valid: boolean }[]>`SELECT EXISTS (SELECT 1 FROM t4_owner_leases lease JOIN t4_stale_create_cleanups cleanup ON cleanup.cleanup_id = ${claim.cleanupId} JOIN t4_outbox item ON item.outbox_id = cleanup.outbox_id WHERE lease.lease_name = ${OUTBOX_LEASE} AND lease.owner_id = ${claim.ownerId} AND lease.epoch = ${claim.ownerEpoch} AND lease.expires_at > clock_timestamp() AND item.state <> 'applied' AND cleanup.state = 'claimed' AND cleanup.owner_id = lease.owner_id AND cleanup.owner_epoch = lease.epoch AND floor(extract(epoch FROM cleanup.claimed_at) * 1000000)::numeric::text = ${claim.claimedAt} AND cleanup.resource_type = ${claim.resourceType} AND cleanup.target_id = ${claim.targetId} AND cleanup.uid = ${claim.uid} AND cleanup.resource_version = ${claim.resourceVersion}) AS valid`; + return rows[0]?.valid === true; + } + + async acknowledgeStaleCreateCleanup(claim: StaleCreateCleanupClaim): Promise { + return await this.#finishStaleCreateCleanup(claim, undefined); + } + + async recordStaleCreateCleanupFailure(claim: StaleCreateCleanupClaim, message: string): Promise { + return await this.#finishStaleCreateCleanup(claim, message.slice(0, 1024)); + } + + async #finishStaleCreateCleanup(claim: StaleCreateCleanupClaim, failure: string | undefined): Promise { + return await this.#serializable(async transaction => { + const lease = await transaction`SELECT owner_id, epoch, expires_at <= clock_timestamp() AS expired FROM t4_owner_leases WHERE lease_name = ${OUTBOX_LEASE} FOR UPDATE`; + if (lease[0]?.owner_id !== claim.ownerId || lease[0].epoch !== claim.ownerEpoch || lease[0].expired) return false; + const rows = await transaction<{ cleanup_id: bigint }[]>`SELECT cleanup_id FROM t4_stale_create_cleanups WHERE cleanup_id = ${claim.cleanupId} AND state = 'claimed' AND owner_id = ${claim.ownerId} AND owner_epoch = ${claim.ownerEpoch} AND floor(extract(epoch FROM claimed_at) * 1000000)::numeric::text = ${claim.claimedAt} AND resource_type = ${claim.resourceType} AND target_id = ${claim.targetId} AND uid = ${claim.uid} AND resource_version = ${claim.resourceVersion} FOR UPDATE`; + if (!rows[0]) return false; + if (failure === undefined) await transaction`UPDATE t4_stale_create_cleanups SET state = 'applied', owner_id = NULL, owner_epoch = NULL, claimed_at = NULL, last_error = NULL, updated_at = clock_timestamp() WHERE cleanup_id = ${claim.cleanupId}`; + else await transaction`UPDATE t4_stale_create_cleanups SET state = 'pending', owner_id = NULL, owner_epoch = NULL, claimed_at = NULL, last_error = ${failure}, next_attempt_at = clock_timestamp() + interval '1 second', updated_at = clock_timestamp() WHERE cleanup_id = ${claim.cleanupId}`; + return true; + }); + } + + async #terminalizePendingSessionCommands(transaction: SQL, principalId: string, sessionId: string, reason: string, ownerEpoch: bigint): Promise { + const skipped = await transaction<{ command_id: string }[]>`UPDATE t4_outbox SET state = 'skipped', owner_id = NULL, owner_epoch = NULL, claimed_at = NULL, terminal_result = ${{ reason }}, updated_at = clock_timestamp() WHERE principal_id = ${principalId} AND target_id = ${sessionId} AND mutation_kind = 'command.submit' AND state IN ('pending','claimed') RETURNING command_id`; + for (const row of skipped) { + const terminal = await transaction<{ command_id: string }[]>`UPDATE t4_commands SET lifecycle_state = 'cancelled', updated_at = clock_timestamp() WHERE command_id = ${row.command_id} AND lifecycle_state = 'accepted' RETURNING command_id`; + if (terminal[0]) await this.#appendEvent(transaction, principalId, sessionId, "command", { commandId: row.command_id, state: "cancelled" }, ownerEpoch); + } } async #serializable(operation: (transaction: SQL) => Promise): Promise { diff --git a/packages/cluster-server/src/main.ts b/packages/cluster-server/src/main.ts index 3b0397a6..8c8f0361 100755 --- a/packages/cluster-server/src/main.ts +++ b/packages/cluster-server/src/main.ts @@ -28,11 +28,17 @@ export async function runClusterServer(env: Readonly { + if (!ledger) return; + const lease = await ledger.tryAcquireLease(config.epoch); + if (lease) await ledger.ingestKubernetesStatus(lease, collection, observation); + }, onSynchronized: () => health.markKubernetesSynced(), onError: error => { health.markKubernetesUnavailable(); @@ -45,7 +51,6 @@ export async function runClusterServer(env: Readonly | undefined; diff --git a/packages/cluster-server/src/outbox-worker.ts b/packages/cluster-server/src/outbox-worker.ts index acfab10b..05eb9b73 100644 --- a/packages/cluster-server/src/outbox-worker.ts +++ b/packages/cluster-server/src/outbox-worker.ts @@ -1,6 +1,6 @@ -import type { OutboxClaim, OwnerLease, PostgresLedger } from "./ledger.ts"; +import type { OutboxClaim, OwnerLease, PostgresLedger, StaleCreateCleanup, StaleCreateCleanupClaim } from "./ledger.ts"; -export type { OutboxClaim } from "./ledger.ts"; +export type { OutboxClaim, StaleCreateCleanup } from "./ledger.ts"; export interface OutboxMutation { readonly idempotencyToken: string; readonly commandId: string; @@ -13,9 +13,15 @@ export interface OutboxMutation { export interface OutboxApplyContext { readonly claimIsCurrent?: () => Promise; readonly signal?: AbortSignal; + readonly persistStaleCreateCleanup?: (cleanup: StaleCreateCleanup) => Promise; +} +export interface StaleCreateCleanupReplayContext { + readonly claimIsCurrent: () => Promise; + readonly signal?: AbortSignal; } export interface KubernetesOutboxApplier { apply(mutation: OutboxMutation, fence: OwnerLease, context?: OutboxApplyContext): Promise; + replayStaleCreateCleanup?(cleanup: StaleCreateCleanup, context: StaleCreateCleanupReplayContext): Promise; } export interface DurableOutboxWorkerOptions { readonly ledger: PostgresLedger; @@ -30,12 +36,17 @@ export class DurableOutboxWorker { #lease: OwnerLease | undefined; constructor(options: DurableOutboxWorkerOptions) { - if (!options.ownerId || options.ownerId.length > 256) throw new Error("outbox owner id is invalid"); + if (!options.ownerId.trim() || Buffer.byteLength(options.ownerId, "utf8") > 256) throw new Error("outbox owner id is invalid"); this.#ledger = options.ledger; this.#ownerId = options.ownerId; this.#applier = options.applier; } + async tryAcquireLease(): Promise { + this.#lease = await this.#ledger.tryAcquireLease(this.#ownerId); + return this.#lease; + } + async acquireLease(): Promise { this.#lease = await this.#ledger.acquireLease(this.#ownerId); return this.#lease; @@ -63,6 +74,9 @@ export class DurableOutboxWorker { await this.#applier.apply(mutation, { ownerId: claim.ownerId, epoch: claim.ownerEpoch }, { claimIsCurrent: () => this.#ledger.claimIsCurrent(claim), signal, + persistStaleCreateCleanup: async cleanup => { + if (!await this.#ledger.persistStaleCreateCleanup(claim, cleanup)) throw new Error("stale create cleanup does not match its durable outbox claim"); + }, }); } catch (error) { if (!await this.#ledger.claimIsCurrent(claim)) return "fenced"; @@ -72,12 +86,38 @@ export class DurableOutboxWorker { return await this.#ledger.acknowledge(claim) ? "applied" : "fenced"; } + async #applyStaleCreateCleanup(claim: StaleCreateCleanupClaim): Promise<"applied" | "fenced" | "retry"> { + if (!this.#applier.replayStaleCreateCleanup || !await this.#ledger.staleCreateCleanupClaimIsCurrent(claim)) return "fenced"; + const timeout = Math.max(0, claim.expiresAt - Date.now()); + const signal = timeout === 0 ? AbortSignal.abort(new Error("outbox lease has expired")) : AbortSignal.timeout(timeout); + try { + await this.#applier.replayStaleCreateCleanup(claim, { + claimIsCurrent: () => this.#ledger.staleCreateCleanupClaimIsCurrent(claim), + signal, + }); + } catch (error) { + if (!await this.#ledger.staleCreateCleanupClaimIsCurrent(claim)) return "fenced"; + await this.#ledger.recordStaleCreateCleanupFailure(claim, error instanceof Error ? error.message : "stale create cleanup failed"); + return "retry"; + } + return await this.#ledger.acknowledgeStaleCreateCleanup(claim) ? "applied" : "fenced"; + } + async drain(maximum = 100): Promise { if (!this.#lease) await this.acquireLease(); if (!Number.isSafeInteger(maximum) || maximum < 1 || maximum > 10_000) throw new Error("outbox drain bound is invalid"); let applied = 0; let emptyPasses = 0; for (let index = 0; index < maximum && emptyPasses < 2; index++) { + if (this.#applier.replayStaleCreateCleanup) { + const cleanup = await this.#ledger.claimStaleCreateCleanup(this.#lease!); + if (cleanup) { + emptyPasses = 0; + const result = await this.#applyStaleCreateCleanup(cleanup); + if (result === "fenced") break; + continue; + } + } const claim = await this.claimNext(); if (!claim) { emptyPasses++; diff --git a/packages/cluster-server/src/public-api-v1.ts b/packages/cluster-server/src/public-api-v1.ts index 88097316..73080618 100644 --- a/packages/cluster-server/src/public-api-v1.ts +++ b/packages/cluster-server/src/public-api-v1.ts @@ -27,10 +27,16 @@ export interface T4PublicApiV1Options { readonly allowedOrigins?: readonly string[] | (() => readonly string[]); } -const DISCOVERY: Discovery = { +const CAPABILITY_REQUIRED_SCOPES = [ + ["workspace.lifecycle", ["workspaces.read", "workspaces.write"]], + ["session.lifecycle", ["sessions.read", "sessions.write"]], + ["session.commands", ["commands.write"]], + ["session.watch.sse", ["events.read"]], +] as const; +const DISCOVERY: Omit = { apiVersion: "1.0", + serverBuild: { version: "0.1.30", revision: "cluster" }, supportedMajors: [1], - capabilities: ["commands.write", "events.read", "sessions.read", "sessions.write", "workspaces.read", "workspaces.write"], limits: { pageSizeDefault: 25, pageSizeMax: 100, @@ -41,6 +47,18 @@ const DISCOVERY: Discovery = { heartbeatSeconds: 15, }, }; +function discoveryFor(principal: ApiPrincipal): Discovery { + const capabilities: Discovery["capabilities"] = {}; + for (const [capability, requiredScopes] of CAPABILITY_REQUIRED_SCOPES) { + capabilities[capability] = { + supported: true, + enabled: true, + authorized: requiredScopes.every(scope => principal.scopes.has(scope)), + available: true, + }; + } + return { ...DISCOVERY, capabilities }; +} const RESOURCE_ID = /^[A-Za-z0-9][A-Za-z0-9._~-]{0,127}$/u; const IDEMPOTENCY_KEY = /^[A-Za-z0-9._~-]{16,128}$/u; const VERSION = /^[1-9][0-9]{0,3}(?:\.[0-9]+)?$/u; @@ -169,7 +187,7 @@ export class T4PublicApiV1 { const origin = request.headers.get("origin"); if (origin && !this.#allowedOrigins().includes(origin)) return apiError(requestId, 400, "invalid_origin", "request origin is not allowed"); const authorization = request.headers.get("authorization"); - if (!authorization?.startsWith("Bearer ") || authorization.length > 16_391) return apiError(requestId, 401, "unauthenticated", "valid bearer authentication is required"); + if (!authorization || !/^Bearer /iu.test(authorization) || authorization.length > 16_391) return apiError(requestId, 401, "unauthenticated", "valid bearer authentication is required"); const token = authorization.slice(7); if (!token || /\s/u.test(token)) return apiError(requestId, 401, "unauthenticated", "valid bearer authentication is required"); const principal = await this.#authenticator.authenticate(token); @@ -190,7 +208,7 @@ export class T4PublicApiV1 { if (segments[0] !== "v1") return notFound(context.requestId); if (segments.length === 1 && request.method === "GET") { if (!this.#scope(context, "discovery.read")) return apiError(context.requestId, 403, "forbidden", "credential lacks the required operation scope"); - return jsonResponse(200, DISCOVERY); + return jsonResponse(200, discoveryFor(context.principal)); } if (segments.length === 2 && segments[1] === "workspaces") { if (request.method === "GET") return await this.#listWorkspaces(url, context); @@ -389,7 +407,7 @@ export class T4PublicApiV1 { } async #events(request: Request, url: URL, sessionId: string, context: ApiContext): Promise { const forbidden = this.#requireScope(context, "events.read"); if (forbidden) return forbidden; - if (!acceptsEventStream(request.headers.get("accept"))) return apiError(context.requestId, 406, "invalid_request", "Accept must allow text/event-stream"); + if (!acceptsEventStream(request.headers.get("accept"))) return apiError(context.requestId, 406, "incompatible_version", "Accept must allow text/event-stream", { supportedMajors: [1] }); const maxEvents = integerQuery(url.searchParams.get("maxEvents"), 100, 1, DISCOVERY.limits.watchEventsMax); const heartbeat = integerQuery(url.searchParams.get("heartbeatSeconds"), DISCOVERY.limits.heartbeatSeconds, 5, 60); if (!maxEvents || !heartbeat) return this.#validation(context, "watch bounds are invalid"); diff --git a/packages/cluster-server/src/public-kubernetes-applier.ts b/packages/cluster-server/src/public-kubernetes-applier.ts index dc668fe2..43add538 100644 --- a/packages/cluster-server/src/public-kubernetes-applier.ts +++ b/packages/cluster-server/src/public-kubernetes-applier.ts @@ -1,6 +1,12 @@ import { semanticResourceHash, KubernetesApiClient, KubernetesApiError } from "./kubernetes-client.ts"; import type { KubernetesResource } from "./kubernetes-projection.ts"; -import type { KubernetesOutboxApplier, OutboxApplyContext, OutboxMutation } from "./outbox-worker.ts"; +import type { + KubernetesOutboxApplier, + OutboxApplyContext, + OutboxMutation, + StaleCreateCleanup, + StaleCreateCleanupReplayContext, +} from "./outbox-worker.ts"; import type { OwnerLease } from "./ledger.ts"; export interface PublicKubernetesOutboxApplierOptions { @@ -75,6 +81,34 @@ export class PublicKubernetesOutboxApplier implements KubernetesOutboxApplier { return await this.#delete("t4sessions", mutation, fence, context); } + async replayStaleCreateCleanup(cleanup: StaleCreateCleanup, context: StaleCreateCleanupReplayContext): Promise { + if (context.signal?.aborted) throw context.signal.reason; + if (!await context.claimIsCurrent()) throw new Error("outbox lease is no longer current"); + if (context.signal?.aborted) throw context.signal.reason; + let current: KubernetesResource; + try { + current = await this.#client.get(cleanup.resourceType, cleanup.targetId, context.signal); + } catch (error) { + if (error instanceof KubernetesApiError && error.status === 404) { + if (!await context.claimIsCurrent()) throw new Error("outbox lease is no longer current"); + if (context.signal?.aborted) throw context.signal.reason; + return; + } + throw error; + } + if (!await context.claimIsCurrent()) throw new Error("outbox lease is no longer current"); + if (context.signal?.aborted) throw context.signal.reason; + if (current.metadata.uid !== cleanup.uid) return; + assertResourceIdentity(current, cleanup.resourceType, cleanup.targetId); + const resourceVersion = current.metadata.resourceVersion; + if (!resourceVersion) throw new Error("Kubernetes stale-create cleanup resourceVersion is unavailable"); + try { + await this.#client.delete(cleanup.resourceType, cleanup.targetId, { uid: cleanup.uid, resourceVersion }, context.signal); + } catch (error) { + if (!(error instanceof KubernetesApiError) || error.status !== 404) throw error; + } + } + async #assertCurrent(context?: OutboxApplyContext): Promise { if (context?.signal?.aborted) throw context.signal.reason; if (context?.claimIsCurrent && !await context.claimIsCurrent()) throw new Error("outbox lease is no longer current"); @@ -126,7 +160,10 @@ export class PublicKubernetesOutboxApplier implements KubernetesOutboxApplier { const current = await this.#owned("t4sessions", mutation, fence, context?.signal); await this.#assertCurrent(context); await this.#client.patch("t4sessions", mutation.targetId, { - metadata: { resourceVersion: current.metadata.resourceVersion, annotations: annotations(mutation, fence) }, + metadata: { + resourceVersion: current.metadata.resourceVersion, + annotations: { ...annotations(mutation, fence), "cluster.t4.dev/pending-command": null }, + }, spec: { title: String(mutation.payload.title) }, }, context?.signal); } @@ -138,6 +175,7 @@ export class PublicKubernetesOutboxApplier implements KubernetesOutboxApplier { resourceVersion: current.metadata.resourceVersion, annotations: { ...annotations(mutation, fence), + "cluster.t4.dev/pending-command": null, "cluster.t4.dev/pending-command-id": boundedAnnotation("pending command id", mutation.commandId), "cluster.t4.dev/pending-command-epoch": fence.epoch.toString(), }, @@ -186,27 +224,26 @@ export class PublicKubernetesOutboxApplier implements KubernetesOutboxApplier { throw new Error("Kubernetes workspace ownership identity is invalid"); return workspace; } - async #cleanupCreate(resourceType: ResourceType, mutation: OutboxMutation, fence: OwnerLease, body: KubernetesResource, created?: KubernetesResource): Promise { - const signal = AbortSignal.timeout(5_000); - let current = created; - if (!current?.metadata?.uid || !current.metadata.resourceVersion) { - try { - current = await this.#client.get(resourceType, mutation.targetId, signal); - } catch (error) { - if (error instanceof KubernetesApiError && error.status === 404) return; - throw error; - } - } - const expectedAnnotations = annotations(mutation, fence); - const currentAnnotations = current.metadata.annotations ?? {}; - if (current.apiVersion !== API_VERSION || current.kind !== (resourceType === "t4workspaces" ? "T4Workspace" : "T4Session") || current.metadata.name !== mutation.targetId - || semanticResourceHash(record(current.spec)) !== semanticResourceHash(body.spec) - || LEDGER_ANNOTATIONS.some(key => currentAnnotations[key] !== expectedAnnotations[key])) return; - if (!current.metadata.uid || !current.metadata.resourceVersion) return; + async #persistCreateCleanup(resourceType: ResourceType, targetId: string, created: KubernetesResource, context?: OutboxApplyContext): Promise { + const uid = created.metadata?.uid; + const resourceVersion = created.metadata?.resourceVersion; + if (!uid || !resourceVersion) return undefined; + assertResourceIdentity(created, resourceType, targetId); + const cleanup: StaleCreateCleanup = { resourceType, targetId, uid, resourceVersion }; + await context?.persistStaleCreateCleanup?.(cleanup); + return cleanup; + } + + async #cleanupCreate(cleanup: StaleCreateCleanup): Promise { try { - await this.#client.delete(resourceType, mutation.targetId, { uid: current.metadata.uid, resourceVersion: current.metadata.resourceVersion }, signal); + await this.#client.delete( + cleanup.resourceType, + cleanup.targetId, + { uid: cleanup.uid, resourceVersion: cleanup.resourceVersion }, + AbortSignal.timeout(5_000), + ); } catch (error) { - if (!(error instanceof KubernetesApiError) || error.status !== 404 && error.status !== 409) throw error; + if (!(error instanceof KubernetesApiError) || error.status !== 404) throw error; } } @@ -216,10 +253,7 @@ export class PublicKubernetesOutboxApplier implements KubernetesOutboxApplier { try { created = await this.#client.create(resourceType, body, context?.signal); } catch (error) { - if (!(error instanceof KubernetesApiError) || error.status !== 409) { - await this.#cleanupCreate(resourceType, mutation, fence, body); - throw error; - } + if (!(error instanceof KubernetesApiError) || error.status !== 409) throw error; try { const current = await this.#client.get(resourceType, mutation.targetId, context?.signal); assertResourceIdentity(current, resourceType, mutation.targetId); @@ -246,10 +280,11 @@ export class PublicKubernetesOutboxApplier implements KubernetesOutboxApplier { throw new Error("Kubernetes resource conflicts with durable ledger intent"); } } + const cleanup = await this.#persistCreateCleanup(resourceType, mutation.targetId, created, context); try { await this.#assertCurrent(context); } catch (error) { - await this.#cleanupCreate(resourceType, mutation, fence, body, created); + if (cleanup) await this.#cleanupCreate(cleanup); throw error; } }