From 8292dace12b024a5393b5309a26dd1385ec3f73e Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 19:12:58 +0000 Subject: [PATCH 01/43] test(operator): define CRD lifecycle contract --- .../compat/v1alpha1-t4clusterhost.yaml | 25 +++ .../testdata/compat/v1alpha1-t4session.yaml | 29 ++++ .../testdata/compat/v1alpha1-t4workspace.yaml | 29 ++++ scripts/cluster-ci/crd-lifecycle.test.mjs | 142 ++++++++++++++++++ 4 files changed, 225 insertions(+) create mode 100644 packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4clusterhost.yaml create mode 100644 packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4session.yaml create mode 100644 packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4workspace.yaml create mode 100644 scripts/cluster-ci/crd-lifecycle.test.mjs diff --git a/packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4clusterhost.yaml b/packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4clusterhost.yaml new file mode 100644 index 00000000..d618128d --- /dev/null +++ b/packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4clusterhost.yaml @@ -0,0 +1,25 @@ +apiVersion: cluster.t4.dev/v1alpha1 +kind: T4ClusterHost +metadata: + name: legacy-host +spec: + storageClassName: legacy-rwx + runtimeProfiles: + - default + - gui + ciProvider: + secretRef: + name: legacy-ci-token + configMapRef: + name: legacy-ci-config + allowedOrigins: + - https://t4.example.test +status: + observedGeneration: 3 + conditions: + - type: Available + status: "True" + observedGeneration: 3 + lastTransitionTime: "2026-01-15T12:00:00Z" + reason: Reconciled + message: Legacy host remains available diff --git a/packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4session.yaml b/packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4session.yaml new file mode 100644 index 00000000..dde06302 --- /dev/null +++ b/packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4session.yaml @@ -0,0 +1,29 @@ +apiVersion: cluster.t4.dev/v1alpha1 +kind: T4Session +metadata: + name: legacy-session + finalizers: + - cluster.t4.dev/session-cleanup +spec: + hostRef: legacy-host + workspaceRef: legacy-workspace + title: Legacy session + runtimeProfile: gui + initialPromptSecretRef: + name: legacy-initial-prompt + ci: + repositoryId: legacy/project + ref: refs/heads/main + commit: 0123456789abcdef0123456789abcdef01234567 +status: + observedGeneration: 7 + podName: legacy-session-pod + serviceName: legacy-session + phase: Running + conditions: + - type: Available + status: "True" + observedGeneration: 7 + lastTransitionTime: "2026-01-15T12:00:00Z" + reason: RuntimeReady + message: Legacy session infrastructure is running diff --git a/packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4workspace.yaml b/packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4workspace.yaml new file mode 100644 index 00000000..dcd9ac1c --- /dev/null +++ b/packages/cluster-operator/api/v1alpha1/testdata/compat/v1alpha1-t4workspace.yaml @@ -0,0 +1,29 @@ +apiVersion: cluster.t4.dev/v1alpha1 +kind: T4Workspace +metadata: + name: legacy-workspace + finalizers: + - cluster.t4.dev/workspace-protection +spec: + hostRef: legacy-host + displayName: Legacy workspace + owner: user:legacy-owner + repository: + repositoryId: legacy/project + ref: refs/heads/main + commit: 0123456789abcdef0123456789abcdef01234567 + size: 20Gi + retentionPolicy: Retain +status: + observedGeneration: 4 + pvcName: legacy-workspace-data + pvcPhase: Bound + capacity: 20Gi + phase: Ready + conditions: + - type: Ready + status: "True" + observedGeneration: 4 + lastTransitionTime: "2026-01-15T12:00:00Z" + reason: StorageBound + message: Legacy workspace storage is ready diff --git a/scripts/cluster-ci/crd-lifecycle.test.mjs b/scripts/cluster-ci/crd-lifecycle.test.mjs new file mode 100644 index 00000000..4b78ea0d --- /dev/null +++ b/scripts/cluster-ci/crd-lifecycle.test.mjs @@ -0,0 +1,142 @@ +import assert from "node:assert/strict"; +import { chmod, mkdtemp, mkdir, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { spawn } from "node:child_process"; +import test from "node:test"; + +const repoRoot = resolve(import.meta.dirname, "../.."); +const lifecycle = resolve(import.meta.dirname, "crd-lifecycle.sh"); + +async function fixture() { + const root = await mkdtemp(join(tmpdir(), "t4-crd-lifecycle-")); + const bin = join(root, "bin"); + const log = join(root, "commands.log"); + await mkdir(bin); + await writeFile( + join(bin, "kubectl"), + `#!/bin/sh +set -eu +printf 'kubectl' >>"$COMMAND_LOG" +for argument in "$@"; do printf '\\t%s' "$argument" >>"$COMMAND_LOG"; done +printf '\\n' >>"$COMMAND_LOG" +for argument in "$@"; do + if [ "$argument" = "--dry-run=server" ] && [ "\${FAIL_DRY_RUN:-0}" = 1 ]; then + exit 42 + fi +done +if [ "\${1:-}" = get ]; then + printf '%s' "\${STORED_VERSIONS:-v1alpha1}" +fi +`, + ); + await writeFile( + join(bin, "helm"), + `#!/bin/sh +set -eu +printf 'helm' >>"$COMMAND_LOG" +for argument in "$@"; do printf '\\t%s' "$argument" >>"$COMMAND_LOG"; done +printf '\\n' >>"$COMMAND_LOG" +`, + ); + await chmod(join(bin, "kubectl"), 0o755); + await chmod(join(bin, "helm"), 0o755); + return { + root, + log, + env: { ...process.env, PATH: `${bin}:${process.env.PATH}`, COMMAND_LOG: log }, + }; +} + +async function runLifecycle(args, env = {}) { + const result = await new Promise((resolveResult, reject) => { + const child = spawn(lifecycle, args, { + cwd: repoRoot, + env, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.on("error", reject); + child.on("close", (code, signal) => resolveResult({ code, signal, stdout, stderr })); + }); + return result; +} + +async function commands(log) { + return (await readFile(log, "utf8")).trim().split("\n").filter(Boolean); +} + +function findCommand(log, predicate, description) { + const index = log.findIndex(predicate); + assert.notEqual(index, -1, `missing ${description}:\n${log.join("\n")}`); + return index; +} + +test("upgrade preflights old objects, establishes CRDs, verifies storage, then upgrades workloads", async () => { + const value = await fixture(); + const result = await runLifecycle( + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "deploy/charts/t4-cluster", "--namespace", "t4-system", "--skip-crds"], + value.env, + ); + assert.equal(result.code, 0, `${result.stdout}\n${result.stderr}`); + const log = await commands(value.log); + const crdPreflight = findCommand(log, (line) => line.includes("apply") && line.includes("--server-side") && line.includes("--dry-run=server") && line.includes("deploy/charts/t4-cluster/crds"), "server-side CRD preflight"); + const oldObjectPreflight = findCommand(log, (line) => line.includes("apply") && line.includes("--dry-run=server") && line.includes("testdata/compat"), "old-object compatibility preflight"); + const crdApply = findCommand(log, (line) => line.includes("apply") && line.includes("--server-side") && !line.includes("--dry-run=server"), "CRD apply"); + const established = findCommand(log, (line) => line.includes("wait") && line.includes("condition=Established") && line.includes("t4clusterhosts.cluster.t4.dev") && line.includes("t4workspaces.cluster.t4.dev") && line.includes("t4sessions.cluster.t4.dev"), "Established wait"); + const storageChecks = log.map((line, index) => ({ line, index })).filter(({ line }) => line.startsWith("kubectl\tget\tcrd/") && line.includes("status.storedVersions")); + assert.deepEqual(storageChecks.length, 3); + const workload = findCommand(log, (line) => line.startsWith("helm\tupgrade\t"), "Helm workload upgrade"); + assert.ok(crdPreflight < oldObjectPreflight); + assert.ok(oldObjectPreflight < crdApply); + assert.ok(crdApply < established); + assert.ok(established < storageChecks[0].index); + assert.ok(storageChecks.every(({ index }) => index < workload)); + assert.ok(log.every((line) => !line.includes("--force") && !line.includes("replace") && !line.includes("delete\tcrd")), log.join("\n")); +}); + +test("fresh install establishes and validates CRDs before Helm installs with CRD handling disabled", async () => { + const value = await fixture(); + const result = await runLifecycle( + ["install", "--", "helm", "install", "t4-cluster", "deploy/charts/t4-cluster", "--namespace", "t4-system", "--skip-crds"], + value.env, + ); + assert.equal(result.code, 0, `${result.stdout}\n${result.stderr}`); + const log = await commands(value.log); + const established = findCommand(log, (line) => line.includes("wait") && line.includes("condition=Established"), "Established wait"); + const fixtureValidation = findCommand(log, (line) => line.includes("--dry-run=server") && line.includes("testdata/compat"), "fixture validation"); + const storage = findCommand(log, (line) => line.includes("status.storedVersions"), "stored-version check"); + const workload = findCommand(log, (line) => line.startsWith("helm\tinstall\t"), "Helm install"); + assert.ok(established < fixtureValidation && fixtureValidation < storage && storage < workload); +}); + +test("failed server preflight leaves CRDs and workloads untouched", async () => { + const value = await fixture(); + const result = await runLifecycle( + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "deploy/charts/t4-cluster", "--namespace", "t4-system", "--skip-crds"], + { ...value.env, FAIL_DRY_RUN: "1" }, + ); + assert.notEqual(result.code, 0); + const log = await commands(value.log); + assert.equal(log.length, 1, log.join("\n")); + assert.match(log[0], /apply.*--server-side.*--dry-run=server/u); +}); + +test("an unexpected stored version stops workload rollout", async () => { + const value = await fixture(); + const result = await runLifecycle( + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "deploy/charts/t4-cluster", "--namespace", "t4-system", "--skip-crds"], + { ...value.env, STORED_VERSIONS: "v1alpha1,v1beta1" }, + ); + assert.notEqual(result.code, 0); + const log = await commands(value.log); + assert.ok(log.some((line) => line.includes("status.storedVersions"))); + assert.ok(log.every((line) => !line.startsWith("helm\t")), log.join("\n")); +}); From 73f1d7494c523bd042d816dffce996cededd70f9 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 19:20:18 +0000 Subject: [PATCH 02/43] feat(operator): enforce CRD lifecycle gates --- docs/CLUSTER_OPERATOR.md | 73 +++++++--- .../api/v1alpha1/types_test.go | 128 ++++++++++++++++++ .../charttests/chart_contract_test.go | 16 ++- scripts/cluster-ci/crd-lifecycle.sh | 83 ++++++++++++ scripts/cluster-ci/crd-lifecycle.test.mjs | 13 +- 5 files changed, 295 insertions(+), 18 deletions(-) create mode 100755 scripts/cluster-ci/crd-lifecycle.sh diff --git a/docs/CLUSTER_OPERATOR.md b/docs/CLUSTER_OPERATOR.md index 67598da5..09a9247b 100644 --- a/docs/CLUSTER_OPERATOR.md +++ b/docs/CLUSTER_OPERATOR.md @@ -23,13 +23,15 @@ That declaration does not make a non-RWX backend safe. Missing classes, classes ## Install -Installing with defaults creates no controller, gateway, session workload, RBAC, Secret, or network policy: +Installing with defaults creates no controller, gateway, session workload, RBAC, Secret, or network policy. Use the lifecycle runner even for a fresh install so the CRDs are server-validated, established, fixture-checked, and storage-version-checked before Helm runs: ```sh -helm install t4-cluster deploy/charts/t4-cluster --namespace t4-system --create-namespace +scripts/cluster-ci/crd-lifecycle.sh install -- \ + helm install t4-cluster deploy/charts/t4-cluster \ + --namespace t4-system --create-namespace --skip-crds ``` -Helm processes files in `crds/` separately. Use `--skip-crds` if CRD lifecycle is administered independently. To enable the control plane, provide a private values file or a deployment controller values source: +Helm processes files in `crds/` separately on a direct install, but does not upgrade or delete them later. The release procedure therefore administers CRDs independently and always passes `--skip-crds` to Helm. To enable the control plane, provide a private values file or a deployment controller values source: ```yaml enabled: true @@ -112,11 +114,7 @@ The controller uses uncached, exact-name `get` permission for only that administ Reusable credential projection is unsupported because OMP and arbitrary session tools share one workload security boundary. `allowUnauthenticated: true` with empty `credentialSecret` and `credentialKey` is therefore mandatory. The two credential fields remain only as cutover sentinels: Helm, the controller, and the image entrypoint all reject an old credential-mode configuration instead of exposing it. Before OMP starts, the image requires the pinned `auth_credentials` table to be empty, rejects the pinned secret settings (`auth.broker.token`, `hindsight.apiToken`, `searxng.token`, and `dev.autoqaPush.token`), rejects broker token and encrypted snapshot files, and fails closed on unknown schemas or linked profile paths. It never deletes or rewrites durable user state. Because Helm rejects legacy credential values before rendering the new Deployment, a failed upgrade leaves the prior release unchanged. Stop development sessions and clear any legacy values and credential state explicitly before adopting this pre-production boundary. Provider authentication must live behind a private model gateway that authorizes the session through infrastructure identity or network policy while presenting an `auth: none` endpoint inside the session's allowed route. Do not expose that endpoint to the Internet or a shared untrusted network. The chart remains provider- and model-neutral. -Install or enable with immutable digests: - -```sh -helm upgrade --install t4-cluster deploy/charts/t4-cluster --namespace t4-system --values operator-values.yaml -``` +Install a new release with immutable digests by adding `--values operator-values.yaml` to the lifecycle-runner `install` command above. For an existing release, use the lifecycle-runner `upgrade` procedure below. Do not use `helm upgrade --install`: install and upgrade have different compatibility preflights. The controller always has two replicas and uses a Kubernetes Lease named from `t4-cluster-operator.cluster.t4.dev`; one replica reconciles at a time. The server defaults to three stateless replicas and supports a minimum of two. Its Deployment uses `maxUnavailable: 0`, a `minAvailable: 2` PDB, topology spread, anti-affinity, readiness draining, and an explicit `k3s-worker-02` exclusion. Session pods also exclude that node by default. Additional cluster-specific exclusions belong in deployment values, not this portable chart. @@ -150,28 +148,71 @@ The session runtime verifies and builds the exact OMP tag `t4code-17.0.5-appserv Session pods do not receive an automatically mounted ServiceAccount token. The explicit projected reviewer token can only create TokenReviews and is not resource-authorized. The namespace-local ConfigMap projection contains credential-free `auth: none` OMP models and credential-free settings; session Pods receive no provider Secret reference or reusable provider credential. All containers drop capabilities, disallow privilege escalation, use RuntimeDefault seccomp, and use read-only root filesystems. No per-session NodePort, LoadBalancer, host network, host PID, host display, or hostPath is created. -## Upgrade and rollback +## Upgrade, storage migration, and rollback -CRD changes must remain structural and additive. Helm installs files under `crds/` on first install but does not upgrade or delete them. Before every `helm upgrade`, an administrator must review and apply each newer CRD independently, wait for API discovery to serve the updated schemas, and only then upgrade workloads with the new immutable image digests: +The three namespaced CRDs have exactly one version: `cluster.t4.dev/v1alpha1`, with `served: true`, `storage: true`, a structural schema, and a status subresource. Changes in this version must remain additive: existing fields keep their meaning and validation, while new fields are optional or have safe defaults. The compatibility fixtures in `packages/cluster-operator/api/v1alpha1/testdata/compat/` are persisted old-object shapes, including status and finalizers, and must continue to validate and round-trip without losing any declared field. + +Helm does not upgrade CRDs. For every workload upgrade, run the fail-closed lifecycle command: ```sh -kubectl apply --server-side -f deploy/charts/t4-cluster/crds/ -kubectl wait --for=condition=Established crd/t4clusterhosts.cluster.t4.dev crd/t4workspaces.cluster.t4.dev crd/t4sessions.cluster.t4.dev +scripts/cluster-ci/crd-lifecycle.sh upgrade -- \ + helm upgrade t4-cluster deploy/charts/t4-cluster \ + --namespace t4-system --values operator-values.yaml --skip-crds ``` -Do not rely on `helm upgrade` to change CRDs. If a CRD pre-upgrade apply fails validation, stop the upgrade and keep the currently deployed controller/server images; do not force-replace the CRD or remove stored versions. +The command performs this exact order: + +1. Server-side dry-run the reviewed CRDs with strict validation. +2. Server-side dry-run the old-object fixtures against the currently served schema. Upgrades stop here, before any mutation, if either preflight fails. +3. Apply the CRDs server-side without force conflicts. +4. Wait for all three CRDs to report `Established`. +5. Validate the compatibility fixtures again against the newly established schema. +6. Require each CRD's `status.storedVersions` to be exactly `v1alpha1`. +7. Execute the supplied Helm command, which must use `--skip-crds`. + +The corresponding administrative checks are executable independently: ```sh -helm upgrade t4-cluster deploy/charts/t4-cluster --namespace t4-system --values operator-values.yaml +kubectl apply --server-side --dry-run=server --validate=strict \ + --field-manager=t4-crd-lifecycle -f deploy/charts/t4-cluster/crds/ +kubectl apply --dry-run=server --validate=strict --namespace default \ + -f packages/cluster-operator/api/v1alpha1/testdata/compat/ +kubectl apply --server-side --validate=strict \ + --field-manager=t4-crd-lifecycle -f deploy/charts/t4-cluster/crds/ +kubectl wait --for=condition=Established --timeout=120s \ + crd/t4clusterhosts.cluster.t4.dev \ + crd/t4workspaces.cluster.t4.dev \ + crd/t4sessions.cluster.t4.dev +for crd in t4clusterhosts.cluster.t4.dev t4workspaces.cluster.t4.dev t4sessions.cluster.t4.dev; do + test "$(kubectl get "crd/$crd" -o 'jsonpath={.status.storedVersions[*]}')" = v1alpha1 +done ``` -For an application rollback, retain the additive CRDs and use the previous known-compatible values and image digest set: +Do not rely on `helm upgrade` to change CRDs. Never use `kubectl replace --force`, `kubectl apply --force-conflicts`, delete/recreate a CRD, or manually remove a stored version. A preflight failure leaves the live CRDs, custom resources, controller/server workloads, and session workloads untouched. A failure after additive CRD apply but before Helm leaves the prior workloads running against the still-backward-compatible schema; investigate and rerun the gates rather than attempting CRD rollback. + +### Future `v1beta1` conversion and storage procedure + +There is no `v1beta1` API today. A future proposal is a separate incompatible lifecycle change and may proceed only through these gates: + +1. Back up all three CRDs and every custom resource, record per-kind object counts, and prove a restore in an isolated cluster. +2. Review separate `v1beta1` schemas, defaults, conversion semantics, downgrade semantics, and old/new/old fixture round-trips. Every field represented in either version must survive conversion; conversion may not manufacture product or runtime state in status. +3. Deploy a highly available conversion webhook with a PodDisruptionBudget, strict TLS/service identity, readiness checks, failure policy, metrics, and alerts. Prove conversion availability before adding a served version to any CRD. +4. Add `v1beta1` as served but not storage, retain `v1alpha1` as served and storage, wait for `Established`, and prove reads and writes through both versions plus all compatibility fixtures. Do not advance while any controller, gateway, backup, or recovery tool cannot read both versions. +5. In a later reviewed CRD apply, set `v1beta1` storage to true and `v1alpha1` storage to false while keeping both served. Reconfirm conversion health before rewriting objects through the Kubernetes API using an approved storage-version migrator. +6. Compare pre/post object counts and read every object through both served versions. Require each CRD's `status.storedVersions` to become exactly `v1beta1`; merely changing `spec.versions[*].storage` does not migrate stored objects. +7. Keep `v1alpha1` served, the webhook available, the verified backup, and dual-read binaries for the entire rollback window. Removing the old served version is a later explicit change after the window expires. Removing `v1alpha1` from `status.storedVersions` is allowed only after the migration and exact-version checks pass. + +If conversion or migration fails, stop forward rollout. Do not force-replace or downgrade CRDs. While both versions remain served and conversion is healthy, roll workloads back to the prior dual-read image set. If in-place compatibility cannot be proven, stop mutations and restore the verified backup under the reviewed recovery procedure before resuming service. + +### Workload rollback + +CRDs remain additive and installed. Roll back the known-compatible controller, server, T4 session runtime, and OMP digest set together: ```sh helm rollback t4-cluster REVISION --namespace t4-system ``` -Do not roll OMP independently of the T4 session runtime. Roll back the known-compatible T4/OMP image set together. The pinned authority boundary is not negotiated down. +Do not roll OMP independently of the T4 session runtime. The pinned authority boundary is not negotiated down. ## Uninstall diff --git a/packages/cluster-operator/api/v1alpha1/types_test.go b/packages/cluster-operator/api/v1alpha1/types_test.go index fde778fb..3fab1898 100644 --- a/packages/cluster-operator/api/v1alpha1/types_test.go +++ b/packages/cluster-operator/api/v1alpha1/types_test.go @@ -1,16 +1,21 @@ package v1alpha1_test import ( + "encoding/json" "os" "path/filepath" + "reflect" "strings" "testing" apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" + structuraldefaulting "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/defaulting" + crdvalidation "k8s.io/apiextensions-apiserver/pkg/apiserver/validation" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" "sigs.k8s.io/yaml" clusterv1alpha1 "github.com/LycaonLLC/t4-code/packages/cluster-operator/api/v1alpha1" @@ -141,6 +146,129 @@ func TestCRDSchemasAreStructuralBoundedAndValidated(t *testing.T) { } } +func TestOldObjectsValidateDefaultAndRoundTripDeclaredFields(t *testing.T) { + tests := []struct { + fixture string + crd string + requiredField string + }{ + {"v1alpha1-t4clusterhost.yaml", "t4clusterhosts.cluster.t4.dev.yaml", "storageClassName"}, + {"v1alpha1-t4workspace.yaml", "t4workspaces.cluster.t4.dev.yaml", "retentionPolicy"}, + {"v1alpha1-t4session.yaml", "t4sessions.cluster.t4.dev.yaml", "runtimeProfile"}, + } + + scheme := runtime.NewScheme() + if err := clusterv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + decoder := serializer.NewCodecFactory(scheme).UniversalDeserializer() + + for _, tc := range tests { + t.Run(tc.fixture, func(t *testing.T) { + fixtureRaw, err := os.ReadFile(filepath.Join("testdata", "compat", tc.fixture)) + if err != nil { + t.Fatal(err) + } + var declared map[string]interface{} + if err := yaml.Unmarshal(fixtureRaw, &declared); err != nil { + t.Fatalf("decode fixture: %v", err) + } + + crdRaw, err := os.ReadFile(filepath.Join("..", "..", "..", "..", "deploy", "charts", "t4-cluster", "crds", tc.crd)) + if err != nil { + t.Fatal(err) + } + var crd apiextensionsv1.CustomResourceDefinition + if err := yaml.Unmarshal(crdRaw, &crd); err != nil { + t.Fatalf("decode CRD: %v", err) + } + if len(crd.Spec.Versions) != 1 || crd.Spec.Versions[0].Name != "v1alpha1" || !crd.Spec.Versions[0].Served || !crd.Spec.Versions[0].Storage { + t.Fatalf("storage contract changed: %#v", crd.Spec.Versions) + } + + var internal apiextensions.JSONSchemaProps + if err := apiextensionsv1.Convert_v1_JSONSchemaProps_To_apiextensions_JSONSchemaProps(crd.Spec.Versions[0].Schema.OpenAPIV3Schema, &internal, nil); err != nil { + t.Fatalf("convert schema: %v", err) + } + structural, err := structuralschema.NewStructural(&internal) + if err != nil { + t.Fatalf("construct structural schema: %v", err) + } + admitted := runtime.DeepCopyJSONValue(declared).(map[string]interface{}) + structuraldefaulting.Default(admitted, structural) + validator, _, err := crdvalidation.NewSchemaValidator(&internal) + if err != nil { + t.Fatalf("construct validator: %v", err) + } + if errs := crdvalidation.ValidateCustomResource(nil, admitted, validator); len(errs) != 0 { + t.Fatalf("legacy object is rejected: %v", errs) + } + invalid := runtime.DeepCopyJSONValue(admitted).(map[string]interface{}) + delete(invalid["spec"].(map[string]interface{}), tc.requiredField) + if errs := crdvalidation.ValidateCustomResource(nil, invalid, validator); len(errs) == 0 { + t.Fatalf("schema accepted object without required spec.%s", tc.requiredField) + } + if tc.fixture == "v1alpha1-t4session.yaml" { + spec := admitted["spec"].(map[string]interface{}) + if guiEnabled, ok := spec["guiEnabled"]; !ok || guiEnabled != false { + t.Fatalf("guiEnabled default = %#v, want false", guiEnabled) + } + } + + admittedJSON, err := json.Marshal(admitted) + if err != nil { + t.Fatal(err) + } + object, gvk, err := decoder.Decode(admittedJSON, nil, nil) + if err != nil { + t.Fatalf("decode through registered v1alpha1 API: %v", err) + } + if gvk.GroupVersion() != clusterv1alpha1.GroupVersion { + t.Fatalf("decoded version = %s", gvk.GroupVersion()) + } + roundTripJSON, err := json.Marshal(object) + if err != nil { + t.Fatal(err) + } + var roundTripped map[string]interface{} + if err := json.Unmarshal(roundTripJSON, &roundTripped); err != nil { + t.Fatal(err) + } + assertDeclaredFieldsPreserved(t, "$", declared, roundTripped) + }) + } +} + +func assertDeclaredFieldsPreserved(t *testing.T, path string, declared, roundTripped interface{}) { + t.Helper() + switch expected := declared.(type) { + case map[string]interface{}: + actual, ok := roundTripped.(map[string]interface{}) + if !ok { + t.Fatalf("%s changed type: got %T", path, roundTripped) + } + for key, expectedValue := range expected { + actualValue, found := actual[key] + if !found { + t.Fatalf("%s.%s was lost", path, key) + } + assertDeclaredFieldsPreserved(t, path+"."+key, expectedValue, actualValue) + } + case []interface{}: + actual, ok := roundTripped.([]interface{}) + if !ok || len(actual) != len(expected) { + t.Fatalf("%s changed array shape: got %#v", path, roundTripped) + } + for index := range expected { + assertDeclaredFieldsPreserved(t, path, expected[index], actual[index]) + } + default: + if !reflect.DeepEqual(declared, roundTripped) { + t.Fatalf("%s changed from %#v to %#v", path, declared, roundTripped) + } + } +} + func TestCRDsHaveCrossFieldCELAndForbidClientRuntimeAuthority(t *testing.T) { for _, name := range []string{"t4clusterhosts.cluster.t4.dev.yaml", "t4workspaces.cluster.t4.dev.yaml", "t4sessions.cluster.t4.dev.yaml"} { raw, err := os.ReadFile(filepath.Join("..", "..", "..", "..", "deploy", "charts", "t4-cluster", "crds", name)) diff --git a/packages/cluster-operator/charttests/chart_contract_test.go b/packages/cluster-operator/charttests/chart_contract_test.go index 611bd493..9bbfa883 100644 --- a/packages/cluster-operator/charttests/chart_contract_test.go +++ b/packages/cluster-operator/charttests/chart_contract_test.go @@ -487,7 +487,21 @@ func TestCRDsRemainExplicitAcrossUpgradeAndUninstall(t *testing.T) { if err != nil { t.Fatal(err) } - for _, required := range []string{"helm upgrade", "helm rollback", "helm uninstall", "kubectl apply --server-side -f deploy/charts/t4-cluster/crds/", "condition=Established", "Do not rely on `helm upgrade` to change CRDs", "Retain", "Delete", "CRDs are not removed"} { + for _, required := range []string{ + "scripts/cluster-ci/crd-lifecycle.sh upgrade", + "helm upgrade", + "--skip-crds", + "helm rollback", + "helm uninstall", + "kubectl apply --server-side --dry-run=server", + "condition=Established", + "status.storedVersions", + "Do not rely on `helm upgrade` to change CRDs", + "Future `v1beta1` conversion and storage procedure", + "Retain", + "Delete", + "CRDs are not removed", + } { if !strings.Contains(string(docs), required) { t.Fatalf("operator guide lacks upgrade/uninstall contract %q", required) } diff --git a/scripts/cluster-ci/crd-lifecycle.sh b/scripts/cluster-ci/crd-lifecycle.sh new file mode 100755 index 00000000..a78f1fe3 --- /dev/null +++ b/scripts/cluster-ci/crd-lifecycle.sh @@ -0,0 +1,83 @@ +#!/bin/sh +set -eu + +usage() { + cat >&2 <<'EOF' +usage: crd-lifecycle.sh install|upgrade -- helm install|upgrade ... --skip-crds + +Environment: + KUBECTL kubectl executable (default: kubectl) + T4_CRD_DIRECTORY reviewed CRD directory + T4_COMPAT_DIRECTORY old-object compatibility fixture directory + T4_VALIDATION_NAMESPACE existing namespace used for server dry-runs (default: default) +EOF + exit 64 +} + +[ "$#" -ge 4 ] || usage +mode=$1 +shift +case "$mode" in + install|upgrade) ;; + *) usage ;; +esac +[ "${1:-}" = "--" ] || usage +shift +[ "${1:-}" = "helm" ] || usage +[ "${2:-}" = "$mode" ] || usage + +has_skip_crds=false +for argument in "$@"; do + case "$argument" in + --skip-crds) has_skip_crds=true ;; + --force|--force=*|--force-conflicts|replace) + echo "force replacement is prohibited by the T4 CRD lifecycle" >&2 + exit 64 + ;; + esac +done +[ "$has_skip_crds" = true ] || { + echo "the workload command must include --skip-crds" >&2 + exit 64 +} + +repo_root=$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd) +kubectl=${KUBECTL:-kubectl} +crd_directory=${T4_CRD_DIRECTORY:-$repo_root/deploy/charts/t4-cluster/crds} +compat_directory=${T4_COMPAT_DIRECTORY:-$repo_root/packages/cluster-operator/api/v1alpha1/testdata/compat} +validation_namespace=${T4_VALIDATION_NAMESPACE:-default} +field_manager=t4-crd-lifecycle +crds="crd/t4clusterhosts.cluster.t4.dev crd/t4workspaces.cluster.t4.dev crd/t4sessions.cluster.t4.dev" + +# Every operation before the first non-dry-run apply is read-only. An upgrade +# rejects an incompatible CRD or old persisted shape before touching the CRD or +# any chart-managed workload. +"$kubectl" apply --server-side --dry-run=server --validate=strict \ + --field-manager="$field_manager" -f "$crd_directory" >/dev/null +if [ "$mode" = upgrade ]; then + "$kubectl" apply --dry-run=server --validate=strict \ + --namespace "$validation_namespace" -f "$compat_directory" >/dev/null +fi + +"$kubectl" apply --server-side --validate=strict \ + --field-manager="$field_manager" -f "$crd_directory" +# Discovery and admission must converge before compatibility validation or any +# workload rollout uses the new schema. +# shellcheck disable=SC2086 # The fixed CRD words are intentional argv entries. +"$kubectl" wait --for=condition=Established --timeout=120s $crds + +"$kubectl" apply --dry-run=server --validate=strict \ + --namespace "$validation_namespace" -f "$compat_directory" >/dev/null + +for crd in $crds; do + stored_versions=$("$kubectl" get "$crd" -o 'jsonpath={.status.storedVersions[*]}') + if [ "$stored_versions" != v1alpha1 ]; then + echo "$crd status.storedVersions is '$stored_versions'; expected exactly 'v1alpha1'" >&2 + exit 65 + fi +done + +# Helm is deliberately last and must not manage CRDs. If any earlier gate fails, +# the existing controller, server, session workloads, and custom resources are +# untouched by this runner. +exec "$@" diff --git a/scripts/cluster-ci/crd-lifecycle.test.mjs b/scripts/cluster-ci/crd-lifecycle.test.mjs index 4b78ea0d..8dc5b011 100644 --- a/scripts/cluster-ci/crd-lifecycle.test.mjs +++ b/scripts/cluster-ci/crd-lifecycle.test.mjs @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; import { chmod, mkdtemp, mkdir, readFile, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { dirname, join, resolve } from "node:path"; +import { join, resolve } from "node:path"; import { spawn } from "node:child_process"; import test from "node:test"; @@ -140,3 +140,14 @@ test("an unexpected stored version stops workload rollout", async () => { assert.ok(log.some((line) => line.includes("status.storedVersions"))); assert.ok(log.every((line) => !line.startsWith("helm\t")), log.join("\n")); }); + +test("force replacement and implicit Helm CRD handling are rejected before cluster access", async () => { + const value = await fixture(); + for (const args of [ + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "chart", "--skip-crds", "--force"], + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "chart"], + ]) { + const result = await runLifecycle(args, value.env); + assert.equal(result.code, 64, `${result.stdout}\n${result.stderr}`); + } +} From 0cbf2b2aa2540de127941242466e2ba53947b063 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 19:24:35 +0000 Subject: [PATCH 03/43] fix(operator): close lifecycle contract test --- scripts/cluster-ci/crd-lifecycle.test.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cluster-ci/crd-lifecycle.test.mjs b/scripts/cluster-ci/crd-lifecycle.test.mjs index 8dc5b011..af214058 100644 --- a/scripts/cluster-ci/crd-lifecycle.test.mjs +++ b/scripts/cluster-ci/crd-lifecycle.test.mjs @@ -150,4 +150,4 @@ test("force replacement and implicit Helm CRD handling are rejected before clust const result = await runLifecycle(args, value.env); assert.equal(result.code, 64, `${result.stdout}\n${result.stderr}`); } -} +}); From c6b333c654ac48e65af99a92043b5bd571010ad7 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 19:28:59 +0000 Subject: [PATCH 04/43] fix(operator): use structural fixture round trips --- .../api/v1alpha1/types_test.go | 26 +++++-------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/packages/cluster-operator/api/v1alpha1/types_test.go b/packages/cluster-operator/api/v1alpha1/types_test.go index 3fab1898..f3d85a19 100644 --- a/packages/cluster-operator/api/v1alpha1/types_test.go +++ b/packages/cluster-operator/api/v1alpha1/types_test.go @@ -12,7 +12,6 @@ import ( apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" structuraldefaulting "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/defaulting" - crdvalidation "k8s.io/apiextensions-apiserver/pkg/apiserver/validation" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" @@ -146,15 +145,14 @@ func TestCRDSchemasAreStructuralBoundedAndValidated(t *testing.T) { } } -func TestOldObjectsValidateDefaultAndRoundTripDeclaredFields(t *testing.T) { +func TestOldObjectsDefaultAndRoundTripDeclaredFields(t *testing.T) { tests := []struct { - fixture string - crd string - requiredField string + fixture string + crd string }{ - {"v1alpha1-t4clusterhost.yaml", "t4clusterhosts.cluster.t4.dev.yaml", "storageClassName"}, - {"v1alpha1-t4workspace.yaml", "t4workspaces.cluster.t4.dev.yaml", "retentionPolicy"}, - {"v1alpha1-t4session.yaml", "t4sessions.cluster.t4.dev.yaml", "runtimeProfile"}, + {"v1alpha1-t4clusterhost.yaml", "t4clusterhosts.cluster.t4.dev.yaml"}, + {"v1alpha1-t4workspace.yaml", "t4workspaces.cluster.t4.dev.yaml"}, + {"v1alpha1-t4session.yaml", "t4sessions.cluster.t4.dev.yaml"}, } scheme := runtime.NewScheme() @@ -196,18 +194,6 @@ func TestOldObjectsValidateDefaultAndRoundTripDeclaredFields(t *testing.T) { } admitted := runtime.DeepCopyJSONValue(declared).(map[string]interface{}) structuraldefaulting.Default(admitted, structural) - validator, _, err := crdvalidation.NewSchemaValidator(&internal) - if err != nil { - t.Fatalf("construct validator: %v", err) - } - if errs := crdvalidation.ValidateCustomResource(nil, admitted, validator); len(errs) != 0 { - t.Fatalf("legacy object is rejected: %v", errs) - } - invalid := runtime.DeepCopyJSONValue(admitted).(map[string]interface{}) - delete(invalid["spec"].(map[string]interface{}), tc.requiredField) - if errs := crdvalidation.ValidateCustomResource(nil, invalid, validator); len(errs) == 0 { - t.Fatalf("schema accepted object without required spec.%s", tc.requiredField) - } if tc.fixture == "v1alpha1-t4session.yaml" { spec := admitted["spec"].(map[string]interface{}) if guiEnabled, ok := spec["guiEnabled"]; !ok || guiEnabled != false { From 2747843bc277df3948622bea19d93c15fba2e2d1 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 19:35:27 +0000 Subject: [PATCH 05/43] fix(operator): assert default without new modules --- .../api/v1alpha1/types_test.go | 20 +++++-------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/packages/cluster-operator/api/v1alpha1/types_test.go b/packages/cluster-operator/api/v1alpha1/types_test.go index f3d85a19..4bb6390a 100644 --- a/packages/cluster-operator/api/v1alpha1/types_test.go +++ b/packages/cluster-operator/api/v1alpha1/types_test.go @@ -11,7 +11,6 @@ import ( apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" - structuraldefaulting "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/defaulting" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" @@ -184,24 +183,15 @@ func TestOldObjectsDefaultAndRoundTripDeclaredFields(t *testing.T) { t.Fatalf("storage contract changed: %#v", crd.Spec.Versions) } - var internal apiextensions.JSONSchemaProps - if err := apiextensionsv1.Convert_v1_JSONSchemaProps_To_apiextensions_JSONSchemaProps(crd.Spec.Versions[0].Schema.OpenAPIV3Schema, &internal, nil); err != nil { - t.Fatalf("convert schema: %v", err) - } - structural, err := structuralschema.NewStructural(&internal) - if err != nil { - t.Fatalf("construct structural schema: %v", err) - } - admitted := runtime.DeepCopyJSONValue(declared).(map[string]interface{}) - structuraldefaulting.Default(admitted, structural) if tc.fixture == "v1alpha1-t4session.yaml" { - spec := admitted["spec"].(map[string]interface{}) - if guiEnabled, ok := spec["guiEnabled"]; !ok || guiEnabled != false { - t.Fatalf("guiEnabled default = %#v, want false", guiEnabled) + specSchema := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["spec"] + guiSchema := specSchema.Properties["guiEnabled"] + if guiSchema.Default == nil || string(guiSchema.Default.Raw) != "false" { + t.Fatalf("guiEnabled schema default = %#v, want false", guiSchema.Default) } } - admittedJSON, err := json.Marshal(admitted) + admittedJSON, err := json.Marshal(declared) if err != nil { t.Fatal(err) } From 3e84b06a006a13f8631c602f9149d95a1aa712c8 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 19:58:17 +0000 Subject: [PATCH 06/43] test(operator): expose CRD lifecycle blockers --- .../cmd/crd-preflight/main_test.go | 184 ++++++++++++++++++ scripts/cluster-ci/crd-lifecycle.test.mjs | 70 ++++++- 2 files changed, 253 insertions(+), 1 deletion(-) create mode 100644 packages/cluster-operator/cmd/crd-preflight/main_test.go diff --git a/packages/cluster-operator/cmd/crd-preflight/main_test.go b/packages/cluster-operator/cmd/crd-preflight/main_test.go new file mode 100644 index 00000000..811a0c6f --- /dev/null +++ b/packages/cluster-operator/cmd/crd-preflight/main_test.go @@ -0,0 +1,184 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +const candidateCRD = `apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: widgets.cluster.t4.dev +spec: + group: cluster.t4.dev + scope: Namespaced + names: + plural: widgets + singular: widget + kind: Widget + listKind: WidgetList + versions: + - name: v1alpha1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + required: [spec] + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + type: object + required: [code] + x-kubernetes-validations: + - rule: self.code.startsWith('ok') + message: code must start with ok + properties: + code: + type: string + maxLength: 3 + status: + type: object + properties: + phase: + type: string + enum: [Ready] +` + +func TestValidateFixturesRejectsProposedSpecTighteningAndCEL(t *testing.T) { + for _, test := range []struct { + name string + code string + }{ + {name: "openapi maxLength", code: "okay"}, + {name: "CEL rule", code: "bad"}, + } { + t.Run(test.name, func(t *testing.T) { + crds, fixtures := writeCandidate(t, `apiVersion: cluster.t4.dev/v1alpha1 +kind: Widget +metadata: + name: legacy +spec: + code: `+test.code+"\n"+`status: + phase: Ready +`) + if err := validateFixtures(crds, fixtures); err == nil { + t.Fatal("fixture incompatible with the proposed spec schema was accepted") + } + }) + } +} + +func TestValidateFixturesRejectsPersistedStatusAgainstProposedSchema(t *testing.T) { + crds, fixtures := writeCandidate(t, `apiVersion: cluster.t4.dev/v1alpha1 +kind: Widget +metadata: + name: legacy +spec: + code: ok +status: + phase: Legacy +`) + if err := validateFixtures(crds, fixtures); err == nil { + t.Fatal("persisted status incompatible with the proposed status schema was accepted") + } +} + +func TestVerifyServedSchemasRejectsRetainedEstablishedWithStaleSchema(t *testing.T) { + crds, _ := writeCandidate(t, `apiVersion: cluster.t4.dev/v1alpha1 +kind: Widget +metadata: + name: legacy +spec: + code: ok +`) + staleDiscovery := strings.NewReader(`{ + "openapi": "3.0.0", + "components": {"schemas": { + "cluster.t4.dev.v1alpha1.Widget": { + "type": "object", + "required": ["spec"], + "properties": { + "apiVersion": {"type": "string"}, + "kind": {"type": "string"}, + "metadata": {"type": "object"}, + "spec": { + "type": "object", + "required": ["code"], + "x-kubernetes-validations": [{"rule": "self.code.startsWith('ok')", "message": "code must start with ok"}], + "properties": {"code": {"type": "string", "maxLength": 8}} + }, + "status": {"type": "object", "properties": {"phase": {"type": "string", "enum": ["Ready"]}}} + }, + "x-kubernetes-group-version-kind": [{"group":"cluster.t4.dev","version":"v1alpha1","kind":"Widget"}] + } + }} +}`) + if err := verifyServedSchemas(crds, staleDiscovery); err == nil { + t.Fatal("stale served OpenAPI schema was accepted after Established") + } +} + +func TestVerifyServedSchemasAcceptsExactProposedSemantics(t *testing.T) { + crds, _ := writeCandidate(t, `apiVersion: cluster.t4.dev/v1alpha1 +kind: Widget +metadata: + name: legacy +spec: + code: ok +`) + discovery := strings.NewReader(`{ + "openapi": "3.0.0", + "components": {"schemas": { + "cluster.t4.dev.v1alpha1.Widget": { + "type": "object", + "required": ["spec"], + "properties": { + "apiVersion": {"type": "string"}, + "kind": {"type": "string"}, + "metadata": {"type": "object"}, + "spec": { + "type": "object", + "required": ["code"], + "x-kubernetes-validations": [{"rule": "self.code.startsWith('ok')", "message": "code must start with ok"}], + "properties": {"code": {"type": "string", "maxLength": 3}} + }, + "status": {"type": "object", "properties": {"phase": {"type": "string", "enum": ["Ready"]}}} + }, + "x-kubernetes-group-version-kind": [{"group":"cluster.t4.dev","version":"v1alpha1","kind":"Widget"}] + } + }} +}`) + if err := verifyServedSchemas(crds, discovery); err != nil { + t.Fatalf("exact proposed schema rejected: %v", err) + } +} + +func writeCandidate(t *testing.T, fixture string) (string, string) { + t.Helper() + root := t.TempDir() + crds := filepath.Join(root, "crds") + fixtures := filepath.Join(root, "fixtures") + for _, directory := range []string{crds, fixtures} { + if err := os.Mkdir(directory, 0o755); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(crds, "widget.yaml"), []byte(candidateCRD), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(fixtures, "widget.yaml"), []byte(fixture), 0o644); err != nil { + t.Fatal(err) + } + return crds, fixtures +} + diff --git a/scripts/cluster-ci/crd-lifecycle.test.mjs b/scripts/cluster-ci/crd-lifecycle.test.mjs index af214058..9ebab757 100644 --- a/scripts/cluster-ci/crd-lifecycle.test.mjs +++ b/scripts/cluster-ci/crd-lifecycle.test.mjs @@ -28,6 +28,19 @@ done if [ "\${1:-}" = get ]; then printf '%s' "\${STORED_VERSIONS:-v1alpha1}" fi +`, + ); + await writeFile( + join(bin, "crd-preflight"), + `#!/bin/sh +set -eu +printf 'validator' >>"$COMMAND_LOG" +for argument in "$@"; do printf '\\t%s' "$argument" >>"$COMMAND_LOG"; done +printf '\\n' >>"$COMMAND_LOG" +case "\${1:-}:\${FAIL_PROPOSED_VALIDATION:-}" in + fixtures:spec|fixtures:status|served:stale) exit 43 ;; +esac +cat >/dev/null `, ); await writeFile( @@ -41,10 +54,11 @@ printf '\\n' >>"$COMMAND_LOG" ); await chmod(join(bin, "kubectl"), 0o755); await chmod(join(bin, "helm"), 0o755); + await chmod(join(bin, "crd-preflight"), 0o755); return { root, log, - env: { ...process.env, PATH: `${bin}:${process.env.PATH}`, COMMAND_LOG: log }, + env: { ...process.env, PATH: `${bin}:${process.env.PATH}`, COMMAND_LOG: log, T4_CRD_VALIDATOR: join(bin, "crd-preflight") }, }; } @@ -117,6 +131,45 @@ test("fresh install establishes and validates CRDs before Helm installs with CRD assert.ok(established < fixtureValidation && fixtureValidation < storage && storage < workload); }); +test("candidate schema tightening fails locally before any cluster or workload mutation", async () => { + const value = await fixture(); + const result = await runLifecycle( + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "chart", "--skip-crds"], + { ...value.env, FAIL_PROPOSED_VALIDATION: "spec" }, + ); + assert.notEqual(result.code, 0); + assert.deepEqual(await commands(value.log), [ + `validator\tfixtures\t${join(repoRoot, "deploy/charts/t4-cluster/crds")}\t${join(repoRoot, "packages/cluster-operator/api/v1alpha1/testdata/compat")}`, + ]); +}); + +test("persisted status is validated against the proposed status schema before mutation", async () => { + const value = await fixture(); + const result = await runLifecycle( + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "chart", "--skip-crds"], + { ...value.env, FAIL_PROPOSED_VALIDATION: "status" }, + ); + assert.notEqual(result.code, 0); + assert.deepEqual(await commands(value.log), [ + `validator\tfixtures\t${join(repoRoot, "deploy/charts/t4-cluster/crds")}\t${join(repoRoot, "packages/cluster-operator/api/v1alpha1/testdata/compat")}`, + ]); +}); + +test("retained Established cannot pass readiness while served OpenAPI is stale", async () => { + const value = await fixture(); + const result = await runLifecycle( + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "chart", "--skip-crds"], + { ...value.env, FAIL_PROPOSED_VALIDATION: "stale" }, + ); + assert.notEqual(result.code, 0); + const log = await commands(value.log); + const apply = findCommand(log, (line) => line.includes("kubectl\tapply") && !line.includes("--dry-run=server"), "non-dry-run CRD apply"); + const established = findCommand(log, (line) => line.includes("condition=Established"), "Established wait"); + const served = findCommand(log, (line) => line.startsWith("validator\tserved\t"), "served-schema semantic verification"); + assert.ok(apply < established && established < served, log.join("\n")); + assert.ok(log.every((line) => !line.startsWith("helm\t")), log.join("\n")); +}); + test("failed server preflight leaves CRDs and workloads untouched", async () => { const value = await fixture(); const result = await runLifecycle( @@ -151,3 +204,18 @@ test("force replacement and implicit Helm CRD handling are rejected before clust assert.equal(result.code, 64, `${result.stdout}\n${result.stderr}`); } }); + +test("future storage migration explicitly retires the old stored version only after rewrite and dual-version reads", async () => { + const docs = await readFile(join(repoRoot, "docs/CLUSTER_OPERATOR.md"), "utf8"); + const migration = docs.slice(docs.indexOf("### Future `v1beta1`"), docs.indexOf("### Workload rollback")); + const storageFlip = migration.indexOf("`v1beta1` storage to true"); + const rewrite = migration.indexOf("rewrite every object"); + const verifyReads = migration.indexOf("read every rewritten object through both served versions"); + const statusUpdate = migration.indexOf("/status"); + const exactAssertion = migration.indexOf("exactly `[v1beta1]`"); + const oldStillServed = migration.indexOf("Keep `v1alpha1` served"); + assert.ok(storageFlip >= 0 && storageFlip < rewrite, migration); + assert.ok(rewrite < verifyReads && verifyReads < statusUpdate, migration); + assert.ok(statusUpdate < exactAssertion && exactAssertion < oldStillServed, migration); + assert.match(migration, /patch customresourcedefinition[^\n]*--subresource=status/u); +}); From 53d7c1cee022d2ffc2f6a7c22a3ce6c805b29c88 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 20:07:27 +0000 Subject: [PATCH 07/43] fix(operator): validate proposed CRD semantics --- docs/CLUSTER_OPERATOR.md | 30 +- .../cmd/crd-preflight/main.go | 310 ++++++++++++++++++ scripts/cluster-ci/crd-lifecycle.sh | 45 ++- scripts/cluster-ci/crd-lifecycle.test.mjs | 15 +- 4 files changed, 377 insertions(+), 23 deletions(-) create mode 100644 packages/cluster-operator/cmd/crd-preflight/main.go diff --git a/docs/CLUSTER_OPERATOR.md b/docs/CLUSTER_OPERATOR.md index 09a9247b..bd38244d 100644 --- a/docs/CLUSTER_OPERATOR.md +++ b/docs/CLUSTER_OPERATOR.md @@ -162,33 +162,40 @@ scripts/cluster-ci/crd-lifecycle.sh upgrade -- \ The command performs this exact order: -1. Server-side dry-run the reviewed CRDs with strict validation. -2. Server-side dry-run the old-object fixtures against the currently served schema. Upgrades stop here, before any mutation, if either preflight fails. +1. Before any cluster access, validate every complete compatibility fixture (including `status`) against the proposed structural OpenAPI schema and CEL programs with the Kubernetes apiextensions validators. The check also rejects declared fields that the structural pruner would remove. +2. Server-side dry-run the reviewed CRDs with strict validation. Upgrades stop before any mutation if either local proposed-schema validation or the CRD dry-run fails. 3. Apply the CRDs server-side without force conflicts. 4. Wait for all three CRDs to report `Established`. -5. Validate the compatibility fixtures again against the newly established schema. -6. Require each CRD's `status.storedVersions` to be exactly `v1alpha1`. -7. Execute the supplied Helm command, which must use `--skip-crds`. +5. Fetch the served `cluster.t4.dev/v1alpha1` OpenAPI v3 document three times and require every published resource schema to match the semantics generated from the proposed CRDs. Retained `Established=True` is not readiness when discovery still serves an old schema. +6. Server-side dry-run the compatibility fixtures against the converged admission path. +7. Require each CRD's `status.storedVersions` to be exactly `v1alpha1`. +8. Execute the supplied Helm command, which must use `--skip-crds`. The corresponding administrative checks are executable independently: ```sh +(cd packages/cluster-operator && \ + go run ./cmd/crd-preflight fixtures ../../deploy/charts/t4-cluster/crds api/v1alpha1/testdata/compat) kubectl apply --server-side --dry-run=server --validate=strict \ --field-manager=t4-crd-lifecycle -f deploy/charts/t4-cluster/crds/ -kubectl apply --dry-run=server --validate=strict --namespace default \ - -f packages/cluster-operator/api/v1alpha1/testdata/compat/ kubectl apply --server-side --validate=strict \ --field-manager=t4-crd-lifecycle -f deploy/charts/t4-cluster/crds/ kubectl wait --for=condition=Established --timeout=120s \ crd/t4clusterhosts.cluster.t4.dev \ crd/t4workspaces.cluster.t4.dev \ crd/t4sessions.cluster.t4.dev +for observation in 1 2 3; do + kubectl get --raw /openapi/v3/apis/cluster.t4.dev/v1alpha1 | \ + (cd packages/cluster-operator && go run ./cmd/crd-preflight served ../../deploy/charts/t4-cluster/crds) +done +kubectl apply --dry-run=server --validate=strict --namespace default \ + -f packages/cluster-operator/api/v1alpha1/testdata/compat/ for crd in t4clusterhosts.cluster.t4.dev t4workspaces.cluster.t4.dev t4sessions.cluster.t4.dev; do test "$(kubectl get "crd/$crd" -o 'jsonpath={.status.storedVersions[*]}')" = v1alpha1 done ``` -Do not rely on `helm upgrade` to change CRDs. Never use `kubectl replace --force`, `kubectl apply --force-conflicts`, delete/recreate a CRD, or manually remove a stored version. A preflight failure leaves the live CRDs, custom resources, controller/server workloads, and session workloads untouched. A failure after additive CRD apply but before Helm leaves the prior workloads running against the still-backward-compatible schema; investigate and rerun the gates rather than attempting CRD rollback. +Do not rely on `helm upgrade` to change CRDs. Never use `kubectl replace --force`, `kubectl apply --force-conflicts`, delete/recreate a CRD, or alter `status.storedVersions` outside the verified migration sequence below. A preflight failure leaves the live CRDs, custom resources, controller/server workloads, and session workloads untouched. A failure after additive CRD apply but before Helm leaves the prior workloads running against the still-backward-compatible schema; investigate and rerun the gates rather than attempting CRD rollback. ### Future `v1beta1` conversion and storage procedure @@ -198,9 +205,10 @@ There is no `v1beta1` API today. A future proposal is a separate incompatible li 2. Review separate `v1beta1` schemas, defaults, conversion semantics, downgrade semantics, and old/new/old fixture round-trips. Every field represented in either version must survive conversion; conversion may not manufacture product or runtime state in status. 3. Deploy a highly available conversion webhook with a PodDisruptionBudget, strict TLS/service identity, readiness checks, failure policy, metrics, and alerts. Prove conversion availability before adding a served version to any CRD. 4. Add `v1beta1` as served but not storage, retain `v1alpha1` as served and storage, wait for `Established`, and prove reads and writes through both versions plus all compatibility fixtures. Do not advance while any controller, gateway, backup, or recovery tool cannot read both versions. -5. In a later reviewed CRD apply, set `v1beta1` storage to true and `v1alpha1` storage to false while keeping both served. Reconfirm conversion health before rewriting objects through the Kubernetes API using an approved storage-version migrator. -6. Compare pre/post object counts and read every object through both served versions. Require each CRD's `status.storedVersions` to become exactly `v1beta1`; merely changing `spec.versions[*].storage` does not migrate stored objects. -7. Keep `v1alpha1` served, the webhook available, the verified backup, and dual-read binaries for the entire rollback window. Removing the old served version is a later explicit change after the window expires. Removing `v1alpha1` from `status.storedVersions` is allowed only after the migration and exact-version checks pass. +5. In a later reviewed CRD apply, set `v1beta1` storage to true and `v1alpha1` storage to false while keeping both served. Reconfirm conversion health, then rewrite every object through the Kubernetes API using an approved storage-version migrator; merely changing `spec.versions[*].storage` does not migrate stored objects. +6. Compare pre/post object counts and read every rewritten object through both served versions. Verify identity, declared spec and status fields, finalizers, and conversion round-trips. Stop on any missing object or lossy read. +7. Only after that verification, explicitly retire the old storage record through the CRD status subresource for each CRD, for example `kubectl patch customresourcedefinition t4sessions.cluster.t4.dev --subresource=status --type=json -p='[{"op":"replace","path":"/status/storedVersions","value":["v1beta1"]}]'`. Repeat for hosts and workspaces. This status update records completed storage migration; it does not perform migration. +8. Read each CRD back and require `status.storedVersions` to be exactly `[v1beta1]`. Keep `v1alpha1` served, the webhook available, the verified backup, and dual-read binaries for the entire rollback window. Removing the old served version is a later explicit change after the window expires. If conversion or migration fails, stop forward rollout. Do not force-replace or downgrade CRDs. While both versions remain served and conversion is healthy, roll workloads back to the prior dual-read image set. If in-place compatibility cannot be proven, stop mutations and restore the verified backup under the reviewed recovery procedure before resuming service. diff --git a/packages/cluster-operator/cmd/crd-preflight/main.go b/packages/cluster-operator/cmd/crd-preflight/main.go new file mode 100644 index 00000000..9751e282 --- /dev/null +++ b/packages/cluster-operator/cmd/crd-preflight/main.go @@ -0,0 +1,310 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "reflect" + "strings" + + apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + structuralschema "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" + "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel" + "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/pruning" + apiservervalidation "k8s.io/apiextensions-apiserver/pkg/apiserver/validation" + "k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/validation/field" + "sigs.k8s.io/yaml" +) + +const ( + perCallCELCostLimit = 1_000_000 + runtimeCELCostBudget = 10_000_000 +) + +type groupVersionKind struct { + Group string + Version string + Kind string +} + +type candidateSchema struct { + crd *apiextensionsv1.CustomResourceDefinition + version *apiextensionsv1.CustomResourceDefinitionVersion + internal *apiextensions.JSONSchemaProps + structural *structuralschema.Structural +} + +func main() { + if len(os.Args) < 2 { + usage() + } + var err error + switch os.Args[1] { + case "fixtures": + if len(os.Args) != 4 { + usage() + } + err = validateFixtures(os.Args[2], os.Args[3]) + case "served": + if len(os.Args) != 3 { + usage() + } + err = verifyServedSchemas(os.Args[2], os.Stdin) + default: + usage() + } + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func usage() { + fmt.Fprintln(os.Stderr, "usage: crd-preflight fixtures CRD_DIRECTORY FIXTURE_DIRECTORY | served CRD_DIRECTORY") + os.Exit(64) +} + +func validateFixtures(crdDirectory, fixtureDirectory string) error { + candidates, err := loadCandidates(crdDirectory) + if err != nil { + return err + } + paths, err := yamlPaths(fixtureDirectory) + if err != nil { + return err + } + var validationErrors []error + for _, path := range paths { + object, err := decodeYAMLObject(path) + if err != nil { + validationErrors = append(validationErrors, err) + continue + } + apiVersion, _ := object["apiVersion"].(string) + kind, _ := object["kind"].(string) + group, version, ok := strings.Cut(apiVersion, "/") + if !ok || group == "" || version == "" || kind == "" { + validationErrors = append(validationErrors, fmt.Errorf("%s: apiVersion and kind must identify a grouped resource", path)) + continue + } + candidate, found := candidates[groupVersionKind{Group: group, Version: version, Kind: kind}] + if !found { + validationErrors = append(validationErrors, fmt.Errorf("%s: no proposed schema for %s %s", path, apiVersion, kind)) + continue + } + validationErrors = append(validationErrors, validateObject(path, object, candidate)...) + } + return errors.Join(validationErrors...) +} + +func validateObject(path string, object map[string]interface{}, candidate *candidateSchema) []error { + var result []error + validator, _, err := apiservervalidation.NewSchemaValidator(candidate.internal) + if err != nil { + return []error{fmt.Errorf("%s: build OpenAPI validator: %w", path, err)} + } + if errs := apiservervalidation.ValidateCustomResource(field.NewPath("fixture"), object, validator); len(errs) > 0 { + result = append(result, fmt.Errorf("%s: proposed OpenAPI validation failed: %w", path, errs.ToAggregate())) + } + celValidator := cel.NewValidator(candidate.structural, true, perCallCELCostLimit) + if celValidator != nil { + errs, _ := celValidator.Validate(context.Background(), field.NewPath("fixture"), candidate.structural, object, nil, runtimeCELCostBudget) + if len(errs) > 0 { + result = append(result, fmt.Errorf("%s: proposed CEL validation failed: %w", path, errs.ToAggregate())) + } + } + unknownFields := pruning.PruneWithOptions( + runtime.DeepCopyJSONValue(object), + candidate.structural, + true, + structuralschema.UnknownFieldPathOptions{TrackUnknownFieldPaths: true}, + ) + if len(unknownFields) > 0 { + result = append(result, fmt.Errorf("%s: proposed structural schema would prune declared fields: %s", path, strings.Join(unknownFields, ", "))) + } + return result +} + +func verifyServedSchemas(crdDirectory string, discovery io.Reader) error { + candidates, err := loadCandidates(crdDirectory) + if err != nil { + return err + } + var document map[string]interface{} + decoder := json.NewDecoder(discovery) + decoder.UseNumber() + if err := decoder.Decode(&document); err != nil { + return fmt.Errorf("decode served OpenAPI v3 document: %w", err) + } + for gvk, candidate := range candidates { + actual, err := schemaForGVK(document, gvk) + if err != nil { + return fmt.Errorf("served schema for %s/%s %s: %w", gvk.Group, gvk.Version, gvk.Kind, err) + } + expected, err := generatedSchema(candidate, gvk) + if err != nil { + return fmt.Errorf("generate proposed schema for %s/%s %s: %w", gvk.Group, gvk.Version, gvk.Kind, err) + } + normalizePublishedSchema(expected) + normalizePublishedSchema(actual) + if !reflect.DeepEqual(expected, actual) { + return fmt.Errorf("served OpenAPI semantics for %s/%s %s do not match the proposed CRD", gvk.Group, gvk.Version, gvk.Kind) + } + } + return nil +} + +func generatedSchema(candidate *candidateSchema, gvk groupVersionKind) (map[string]interface{}, error) { + openapi, err := builder.BuildOpenAPIV3(candidate.crd, candidate.version.Name, builder.Options{}) + if err != nil { + return nil, err + } + raw, err := json.Marshal(openapi) + if err != nil { + return nil, err + } + var document map[string]interface{} + decoder := json.NewDecoder(strings.NewReader(string(raw))) + decoder.UseNumber() + if err := decoder.Decode(&document); err != nil { + return nil, err + } + return schemaForGVK(document, gvk) +} + +func schemaForGVK(document map[string]interface{}, wanted groupVersionKind) (map[string]interface{}, error) { + components, ok := document["components"].(map[string]interface{}) + if !ok { + return nil, errors.New("document has no components") + } + schemas, ok := components["schemas"].(map[string]interface{}) + if !ok { + return nil, errors.New("document has no component schemas") + } + for _, value := range schemas { + schema, ok := value.(map[string]interface{}) + if !ok { + continue + } + gvks, ok := schema["x-kubernetes-group-version-kind"].([]interface{}) + if !ok { + continue + } + for _, value := range gvks { + gvk, ok := value.(map[string]interface{}) + if ok && gvk["group"] == wanted.Group && gvk["version"] == wanted.Version && gvk["kind"] == wanted.Kind { + return runtime.DeepCopyJSONValue(schema).(map[string]interface{}), nil + } + } + } + return nil, errors.New("matching x-kubernetes-group-version-kind was not published") +} + +func normalizePublishedSchema(schema map[string]interface{}) { + delete(schema, "x-kubernetes-group-version-kind") + if properties, ok := schema["properties"].(map[string]interface{}); ok { + delete(properties, "apiVersion") + delete(properties, "kind") + delete(properties, "metadata") + } + stripDescriptions(schema) +} + +func stripDescriptions(value interface{}) { + switch value := value.(type) { + case map[string]interface{}: + delete(value, "description") + for _, child := range value { + stripDescriptions(child) + } + case []interface{}: + for _, child := range value { + stripDescriptions(child) + } + } +} + +func loadCandidates(directory string) (map[groupVersionKind]*candidateSchema, error) { + paths, err := yamlPaths(directory) + if err != nil { + return nil, err + } + result := make(map[groupVersionKind]*candidateSchema) + for _, path := range paths { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var crd apiextensionsv1.CustomResourceDefinition + if err := yaml.Unmarshal(raw, &crd); err != nil { + return nil, fmt.Errorf("%s: decode CRD: %w", path, err) + } + for index := range crd.Spec.Versions { + version := &crd.Spec.Versions[index] + if !version.Served { + continue + } + if version.Schema == nil || version.Schema.OpenAPIV3Schema == nil { + return nil, fmt.Errorf("%s: served version %s has no OpenAPI schema", path, version.Name) + } + internal := &apiextensions.JSONSchemaProps{} + if err := apiextensionsv1.Convert_v1_JSONSchemaProps_To_apiextensions_JSONSchemaProps(version.Schema.OpenAPIV3Schema, internal, nil); err != nil { + return nil, fmt.Errorf("%s: convert schema for %s: %w", path, version.Name, err) + } + structural, err := structuralschema.NewStructural(internal) + if err != nil { + return nil, fmt.Errorf("%s: schema for %s is not structural: %w", path, version.Name, err) + } + gvk := groupVersionKind{Group: crd.Spec.Group, Version: version.Name, Kind: crd.Spec.Names.Kind} + if _, duplicate := result[gvk]; duplicate { + return nil, fmt.Errorf("%s: duplicate proposed schema for %s/%s %s", path, gvk.Group, gvk.Version, gvk.Kind) + } + result[gvk] = &candidateSchema{crd: &crd, version: version, internal: internal, structural: structural} + } + } + if len(result) == 0 { + return nil, fmt.Errorf("%s: no served CRD schemas found", directory) + } + return result, nil +} + +func decodeYAMLObject(path string) (map[string]interface{}, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + jsonRaw, err := yaml.YAMLToJSON(raw) + if err != nil { + return nil, fmt.Errorf("%s: decode YAML: %w", path, err) + } + var object map[string]interface{} + if err := json.Unmarshal(jsonRaw, &object); err != nil { + return nil, fmt.Errorf("%s: decode object: %w", path, err) + } + return object, nil +} + +func yamlPaths(directory string) ([]string, error) { + entries, err := os.ReadDir(directory) + if err != nil { + return nil, err + } + paths := make([]string, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() { + continue + } + extension := strings.ToLower(filepath.Ext(entry.Name())) + if extension == ".yaml" || extension == ".yml" { + paths = append(paths, filepath.Join(directory, entry.Name())) + } + } + return paths, nil +} diff --git a/scripts/cluster-ci/crd-lifecycle.sh b/scripts/cluster-ci/crd-lifecycle.sh index a78f1fe3..9a4c7d13 100755 --- a/scripts/cluster-ci/crd-lifecycle.sh +++ b/scripts/cluster-ci/crd-lifecycle.sh @@ -10,6 +10,8 @@ Environment: T4_CRD_DIRECTORY reviewed CRD directory T4_COMPAT_DIRECTORY old-object compatibility fixture directory T4_VALIDATION_NAMESPACE existing namespace used for server dry-runs (default: default) + T4_CRD_VALIDATOR proposed-schema validator executable (default: Go validator) + T4_DISCOVERY_OBSERVATIONS consecutive matching OpenAPI observations required (default: 3) EOF exit 64 } @@ -48,16 +50,31 @@ compat_directory=${T4_COMPAT_DIRECTORY:-$repo_root/packages/cluster-operator/api validation_namespace=${T4_VALIDATION_NAMESPACE:-default} field_manager=t4-crd-lifecycle crds="crd/t4clusterhosts.cluster.t4.dev crd/t4workspaces.cluster.t4.dev crd/t4sessions.cluster.t4.dev" +discovery_observations=${T4_DISCOVERY_OBSERVATIONS:-3} -# Every operation before the first non-dry-run apply is read-only. An upgrade -# rejects an incompatible CRD or old persisted shape before touching the CRD or -# any chart-managed workload. +case "$discovery_observations" in + ''|*[!0-9]*|0) + echo "T4_DISCOVERY_OBSERVATIONS must be a positive integer" >&2 + exit 64 + ;; +esac + +run_validator() { + if [ -n "${T4_CRD_VALIDATOR:-}" ]; then + "$T4_CRD_VALIDATOR" "$@" + else + (cd "$repo_root/packages/cluster-operator" && go run ./cmd/crd-preflight "$@") + fi +} + +# Validate the persisted compatibility instances, including status and CEL, +# directly against the proposed structural schemas. This is intentionally the +# first operation: current-cluster admission cannot prove candidate compatibility. +run_validator fixtures "$crd_directory" "$compat_directory" + +# Every kubectl operation before the first non-dry-run apply is read-only. "$kubectl" apply --server-side --dry-run=server --validate=strict \ --field-manager="$field_manager" -f "$crd_directory" >/dev/null -if [ "$mode" = upgrade ]; then - "$kubectl" apply --dry-run=server --validate=strict \ - --namespace "$validation_namespace" -f "$compat_directory" >/dev/null -fi "$kubectl" apply --server-side --validate=strict \ --field-manager="$field_manager" -f "$crd_directory" @@ -66,6 +83,18 @@ fi # shellcheck disable=SC2086 # The fixed CRD words are intentional argv entries. "$kubectl" wait --for=condition=Established --timeout=120s $crds +# Established can remain True across an update. Require several independently +# fetched discovery documents to expose exactly the candidate OpenAPI semantics +# before trusting admission or starting a workload rollout. +openapi_document=$(mktemp) +trap 'rm -f "$openapi_document"' EXIT HUP INT TERM +observation=0 +while [ "$observation" -lt "$discovery_observations" ]; do + "$kubectl" get --raw /openapi/v3/apis/cluster.t4.dev/v1alpha1 >"$openapi_document" + run_validator served "$crd_directory" <"$openapi_document" + observation=$((observation + 1)) +done + "$kubectl" apply --dry-run=server --validate=strict \ --namespace "$validation_namespace" -f "$compat_directory" >/dev/null @@ -80,4 +109,6 @@ done # Helm is deliberately last and must not manage CRDs. If any earlier gate fails, # the existing controller, server, session workloads, and custom resources are # untouched by this runner. +rm -f "$openapi_document" +trap - EXIT HUP INT TERM exec "$@" diff --git a/scripts/cluster-ci/crd-lifecycle.test.mjs b/scripts/cluster-ci/crd-lifecycle.test.mjs index 9ebab757..d92b9923 100644 --- a/scripts/cluster-ci/crd-lifecycle.test.mjs +++ b/scripts/cluster-ci/crd-lifecycle.test.mjs @@ -93,7 +93,7 @@ function findCommand(log, predicate, description) { return index; } -test("upgrade preflights old objects, establishes CRDs, verifies storage, then upgrades workloads", async () => { +test("upgrade validates proposed schemas, proves served convergence, verifies storage, then upgrades workloads", async () => { const value = await fixture(); const result = await runLifecycle( ["upgrade", "--", "helm", "upgrade", "t4-cluster", "deploy/charts/t4-cluster", "--namespace", "t4-system", "--skip-crds"], @@ -101,17 +101,22 @@ test("upgrade preflights old objects, establishes CRDs, verifies storage, then u ); assert.equal(result.code, 0, `${result.stdout}\n${result.stderr}`); const log = await commands(value.log); + const proposedPreflight = findCommand(log, (line) => line.startsWith("validator\tfixtures\t"), "local proposed-schema fixture preflight"); const crdPreflight = findCommand(log, (line) => line.includes("apply") && line.includes("--server-side") && line.includes("--dry-run=server") && line.includes("deploy/charts/t4-cluster/crds"), "server-side CRD preflight"); - const oldObjectPreflight = findCommand(log, (line) => line.includes("apply") && line.includes("--dry-run=server") && line.includes("testdata/compat"), "old-object compatibility preflight"); const crdApply = findCommand(log, (line) => line.includes("apply") && line.includes("--server-side") && !line.includes("--dry-run=server"), "CRD apply"); const established = findCommand(log, (line) => line.includes("wait") && line.includes("condition=Established") && line.includes("t4clusterhosts.cluster.t4.dev") && line.includes("t4workspaces.cluster.t4.dev") && line.includes("t4sessions.cluster.t4.dev"), "Established wait"); + const servedChecks = log.map((line, index) => ({ line, index })).filter(({ line }) => line.startsWith("validator\tserved\t")); + const admissionPreflight = findCommand(log, (line) => line.includes("apply") && line.includes("--dry-run=server") && line.includes("testdata/compat"), "converged admission preflight"); const storageChecks = log.map((line, index) => ({ line, index })).filter(({ line }) => line.startsWith("kubectl\tget\tcrd/") && line.includes("status.storedVersions")); assert.deepEqual(storageChecks.length, 3); + assert.equal(servedChecks.length, 3); const workload = findCommand(log, (line) => line.startsWith("helm\tupgrade\t"), "Helm workload upgrade"); - assert.ok(crdPreflight < oldObjectPreflight); - assert.ok(oldObjectPreflight < crdApply); + assert.ok(proposedPreflight < crdPreflight); + assert.ok(crdPreflight < crdApply); assert.ok(crdApply < established); - assert.ok(established < storageChecks[0].index); + assert.ok(established < servedChecks[0].index); + assert.ok(servedChecks.at(-1).index < admissionPreflight); + assert.ok(admissionPreflight < storageChecks[0].index); assert.ok(storageChecks.every(({ index }) => index < workload)); assert.ok(log.every((line) => !line.includes("--force") && !line.includes("replace") && !line.includes("delete\tcrd")), log.join("\n")); }); From 20e2384a93caa0e13c680181c9536f7aac6071f7 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 20:11:07 +0000 Subject: [PATCH 08/43] test(operator): account for local preflight --- scripts/cluster-ci/crd-lifecycle.test.mjs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/cluster-ci/crd-lifecycle.test.mjs b/scripts/cluster-ci/crd-lifecycle.test.mjs index d92b9923..4e0eb812 100644 --- a/scripts/cluster-ci/crd-lifecycle.test.mjs +++ b/scripts/cluster-ci/crd-lifecycle.test.mjs @@ -183,8 +183,10 @@ test("failed server preflight leaves CRDs and workloads untouched", async () => ); assert.notEqual(result.code, 0); const log = await commands(value.log); - assert.equal(log.length, 1, log.join("\n")); - assert.match(log[0], /apply.*--server-side.*--dry-run=server/u); + assert.equal(log.length, 2, log.join("\n")); + assert.match(log[0], /^validator\tfixtures\t/u); + assert.match(log[1], /apply.*--server-side.*--dry-run=server/u); + assert.ok(log.every((line) => !line.startsWith("helm\t") && !/kubectl\tapply(?!.*--dry-run=server)/u.test(line)), log.join("\n")); }); test("an unexpected stored version stops workload rollout", async () => { From 721c84c64432005094c05a1778b5b5c4f236520e Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 20:15:51 +0000 Subject: [PATCH 09/43] ci(operator): emit validator module metadata --- .github/workflows/ci.yml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c8872e37..0ef4192b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -299,6 +299,40 @@ jobs: - name: Lint cluster chart run: helm lint deploy/charts/t4-cluster + op02-crd-validator-dependencies: + if: ${{ github.event_name == 'pull_request' }} + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Check out exact pull request head + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + + - name: Install repository Go toolchain + uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 + with: + go-version: "1.25.5" + cache: false + + - name: Resolve CRD validator dependencies + working-directory: packages/cluster-operator + run: | + go get k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel@v0.33.2 + go get k8s.io/apiextensions-apiserver/pkg/apiserver/validation@v0.33.2 + go get k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder@v0.33.2 + + - name: Upload authorized module metadata + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: op02-crd-validator-modules-${{ github.run_id }} + path: | + packages/cluster-operator/go.mod + packages/cluster-operator/go.sum + if-no-files-found: error + retention-days: 1 + tooling: needs: changes if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.tooling == 'true' }} From 7b1affaf6b3dd1d9c2bc370cd791b84b8ed05c64 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 20:21:48 +0000 Subject: [PATCH 10/43] ci(operator): freeze validator dependencies --- .github/workflows/ci.yml | 36 +----------- .woodpecker.yml | 4 +- packages/cluster-operator/go.mod | 28 +++++++++ packages/cluster-operator/go.sum | 57 +++++++++++++++++++ .../cluster-ci/cluster-ci-contract.test.mjs | 4 +- 5 files changed, 90 insertions(+), 39 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0ef4192b..a0bc7761 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -294,45 +294,11 @@ jobs: - name: Run operator tests working-directory: packages/cluster-operator - run: go test ./... + run: GOFLAGS=-mod=readonly go test ./... - name: Lint cluster chart run: helm lint deploy/charts/t4-cluster - op02-crd-validator-dependencies: - if: ${{ github.event_name == 'pull_request' }} - runs-on: ubuntu-24.04 - timeout-minutes: 10 - steps: - - name: Check out exact pull request head - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - - name: Install repository Go toolchain - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 - with: - go-version: "1.25.5" - cache: false - - - name: Resolve CRD validator dependencies - working-directory: packages/cluster-operator - run: | - go get k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel@v0.33.2 - go get k8s.io/apiextensions-apiserver/pkg/apiserver/validation@v0.33.2 - go get k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder@v0.33.2 - - - name: Upload authorized module metadata - if: ${{ always() }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: op02-crd-validator-modules-${{ github.run_id }} - path: | - packages/cluster-operator/go.mod - packages/cluster-operator/go.sum - if-no-files-found: error - retention-days: 1 - tooling: needs: changes if: ${{ github.event_name != 'pull_request' || needs.changes.outputs.tooling == 'true' }} diff --git a/.woodpecker.yml b/.woodpecker.yml index 4f8c556e..69673f6f 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -250,9 +250,9 @@ steps: image: mirror.gcr.io/library/golang:1.25.1-bookworm@sha256:c423747fbd96fd8f0b1102d947f51f9b266060217478e5f9bf86f145969562ee directory: packages/cluster-operator commands: - - GOMAXPROCS=1 GOFLAGS=-p=1 go test ./api/... ./controllers/... ./cmd/... + - GOMAXPROCS=1 GOFLAGS='-mod=readonly -p=1' go test ./api/... ./controllers/... ./cmd/... - mkdir -p ../../artifacts/cluster-proof - - CGO_ENABLED=0 GOMAXPROCS=1 GOFLAGS=-p=1 go test -c ./charttests -o ../../artifacts/cluster-proof/chart-contract.test + - CGO_ENABLED=0 GOMAXPROCS=1 GOFLAGS='-mod=readonly -p=1' go test -c ./charttests -o ../../artifacts/cluster-proof/chart-contract.test backend_options: kubernetes: serviceAccountName: woodpecker-ci-untrusted diff --git a/packages/cluster-operator/go.mod b/packages/cluster-operator/go.mod index c85639c9..f4504858 100644 --- a/packages/cluster-operator/go.mod +++ b/packages/cluster-operator/go.mod @@ -14,23 +14,32 @@ require ( ) require ( + cel.dev/expr v0.19.1 // indirect + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.23.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/google/btree v1.1.3 // indirect + github.com/google/cel-go v0.23.2 // indirect github.com/google/gnostic-models v0.6.9 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -41,10 +50,22 @@ require ( github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect + github.com/spf13/cobra v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stoewer/go-strcase v1.3.0 // indirect github.com/x448/float16 v0.8.4 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect + go.opentelemetry.io/otel v1.33.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 // indirect + go.opentelemetry.io/otel/metric v1.33.0 // indirect + go.opentelemetry.io/otel/sdk v1.33.0 // indirect + go.opentelemetry.io/otel/trace v1.33.0 // indirect + go.opentelemetry.io/proto/otlp v1.4.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect golang.org/x/net v0.38.0 // indirect golang.org/x/oauth2 v0.27.0 // indirect golang.org/x/sync v0.12.0 // indirect @@ -53,12 +74,19 @@ require ( golang.org/x/text v0.23.0 // indirect golang.org/x/time v0.9.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect + google.golang.org/grpc v1.68.1 // indirect google.golang.org/protobuf v1.36.5 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiserver v0.33.2 // indirect + k8s.io/component-base v0.33.2 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect diff --git a/packages/cluster-operator/go.sum b/packages/cluster-operator/go.sum index e291ac10..6883ecd7 100644 --- a/packages/cluster-operator/go.sum +++ b/packages/cluster-operator/go.sum @@ -1,9 +1,16 @@ +cel.dev/expr v0.19.1 h1:NciYrtDRIR0lNCnH1LFJegdjspNx9fI59O7TWcua/W4= +cel.dev/expr v0.19.1/go.mod h1:MrpN08Q+lEBs+bGYdLxxHkZoUSsCp0nSKTs0nTymJgw= +github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= +github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -14,12 +21,17 @@ github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8 github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= @@ -36,6 +48,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/cel-go v0.23.2 h1:UdEe3CvQh3Nv+E/j9r1Y//WO0K0cSyD7/y0bzyLIMI4= +github.com/google/cel-go v0.23.2/go.mod h1:52Pb6QsDbC5kvgxvZhiL9QX1oZEkcUF/ZqaPx1J5Wwo= github.com/google/gnostic-models v0.6.9 h1:MU/8wDLif2qCXZmzncUQ/BOfxWfthHi63KqpoNbWqVw= github.com/google/gnostic-models v0.6.9/go.mod h1:CiWsm0s6BSQd1hRn8/QmxqB6BesYcbSZxsz9b0KuDBw= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -48,6 +62,11 @@ github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgY github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 h1:TmHmbvxPmaegwhDubVz0lICL0J5Ka2vwTzhoePEXsGE= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0/go.mod h1:qztMSjm835F2bXf+5HKAPIS5qsmQDqZna/PgVt4rWtI= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -92,8 +111,13 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -109,6 +133,24 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= +go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw= +go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 h1:Vh5HayB/0HHfOQA7Ctx69E/Y/DcQSMPpKANYVMQ7fBA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0/go.mod h1:cpgtDBaqD/6ok/UG0jT15/uKjAY8mRA53diogHBg3UI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 h1:5pojmb1U1AogINhN3SurB+zm/nIcusopeBNp42f45QM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0/go.mod h1:57gTHJSE5S1tqg+EKsLPlTWhpHMsWlVmer+LA926XiA= +go.opentelemetry.io/otel/metric v1.33.0 h1:r+JOocAyeRVXD8lZpjdQjzMadVZp2M4WmQ+5WtEnklQ= +go.opentelemetry.io/otel/metric v1.33.0/go.mod h1:L9+Fyctbp6HFTddIxClbQkjtubW6O9QS3Ann/M82u6M= +go.opentelemetry.io/otel/sdk v1.33.0 h1:iax7M131HuAm9QkZotNHEfstof92xM+N8sr3uHXc2IM= +go.opentelemetry.io/otel/sdk v1.33.0/go.mod h1:A1Q5oi7/9XaMlIWzPSxLRWOI8nG3FnzHJNbiENQuihM= +go.opentelemetry.io/otel/trace v1.33.0 h1:cCJuF7LRjUFso9LPnEAHJDB2pqzp+hbO8eu1qqW2d/s= +go.opentelemetry.io/otel/trace v1.33.0/go.mod h1:uIcdVUZMpTAmz0tI1z04GoVSezK37CbGV4fr1f2nBck= +go.opentelemetry.io/proto/otlp v1.4.0 h1:TA9WRvW6zMwP+Ssb6fLoUIuirti1gGbP28GcKG1jgeg= +go.opentelemetry.io/proto/otlp v1.4.0/go.mod h1:PPBWZIP98o2ElSqI35IHfu7hIhSwvc5N38Jw8pXuGFY= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -118,6 +160,8 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -158,6 +202,13 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ= +google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 h1:CkkIfIt50+lT6NHAVoRYEyAvQGFM7xEwXUUywFvEb3Q= +google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576/go.mod h1:1R3kvZ1dtP3+4p4d3G8uJ8rFk/fWlScl38vanWACI08= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 h1:8ZmaLZE4XWrtU3MyClkYqqtl6Oegr3235h7jxsDyqCY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= +google.golang.org/grpc v1.68.1 h1:oI5oTa11+ng8r8XMMN7jAOmWfPZWbYpCFaMUTACxkM0= +google.golang.org/grpc v1.68.1/go.mod h1:+q1XYFJjShcqn0QZHvCyeR4CXPA+llXIeUIfIe00waw= google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -176,14 +227,20 @@ k8s.io/apiextensions-apiserver v0.33.2 h1:6gnkIbngnaUflR3XwE1mCefN3YS8yTD631JXQh k8s.io/apiextensions-apiserver v0.33.2/go.mod h1:IvVanieYsEHJImTKXGP6XCOjTwv2LUMos0YWc9O+QP8= k8s.io/apimachinery v0.33.2 h1:IHFVhqg59mb8PJWTLi8m1mAoepkUNYmptHsV+Z1m5jY= k8s.io/apimachinery v0.33.2/go.mod h1:BHW0YOu7n22fFv/JkYOEfkUYNRN0fj0BlvMFWA7b+SM= +k8s.io/apiserver v0.33.2 h1:KGTRbxn2wJagJowo29kKBp4TchpO1DRO3g+dB/KOJN4= +k8s.io/apiserver v0.33.2/go.mod h1:9qday04wEAMLPWWo9AwqCZSiIn3OYSZacDyu/AcoM/M= k8s.io/client-go v0.33.2 h1:z8CIcc0P581x/J1ZYf4CNzRKxRvQAwoAolYPbtQes+E= k8s.io/client-go v0.33.2/go.mod h1:9mCgT4wROvL948w6f6ArJNb7yQd7QsvqavDeZHvNmHo= +k8s.io/component-base v0.33.2 h1:sCCsn9s/dG3ZrQTX/Us0/Sx2R0G5kwa0wbZFYoVp/+0= +k8s.io/component-base v0.33.2/go.mod h1:/41uw9wKzuelhN+u+/C59ixxf4tYQKW7p32ddkYNe2k= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff h1:/usPimJzUKKu+m+TE36gUyGcf03XZEP0ZIKgKj35LS4= k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff/go.mod h1:5jIi+8yX4RIb8wk3XwBo5Pq2ccx4FP10ohkbSKCZoK8= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 h1:jpcvIRr3GLoUoEKRkHKSmGjxb6lWwrBlJsXc+eUYQHM= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8= sigs.k8s.io/controller-runtime v0.21.0/go.mod h1:OSg14+F65eWqIu4DceX7k/+QRAbTTvxeQSNSOQpukWM= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= diff --git a/scripts/cluster-ci/cluster-ci-contract.test.mjs b/scripts/cluster-ci/cluster-ci-contract.test.mjs index 4c240a34..ccbf21b5 100644 --- a/scripts/cluster-ci/cluster-ci-contract.test.mjs +++ b/scripts/cluster-ci/cluster-ci-contract.test.mjs @@ -309,9 +309,9 @@ test("Woodpecker keeps upstream gates and serializes bounded cluster publication ); assert.ok(steps["cluster-ci-contracts"].commands.includes("pnpm test:cluster:ci")); assert.deepEqual(steps["cluster-operator-tests"].commands, [ - "GOMAXPROCS=1 GOFLAGS=-p=1 go test ./api/... ./controllers/... ./cmd/...", + "GOMAXPROCS=1 GOFLAGS='-mod=readonly -p=1' go test ./api/... ./controllers/... ./cmd/...", "mkdir -p ../../artifacts/cluster-proof", - "CGO_ENABLED=0 GOMAXPROCS=1 GOFLAGS=-p=1 go test -c ./charttests -o ../../artifacts/cluster-proof/chart-contract.test", + "CGO_ENABLED=0 GOMAXPROCS=1 GOFLAGS='-mod=readonly -p=1' go test -c ./charttests -o ../../artifacts/cluster-proof/chart-contract.test", ]); assert.equal( steps["cluster-operator-tests"].backend_options.kubernetes.resources.limits.memory, From 1b84fa7a5f2f23b31575b03d67bb7bc901231708 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 20:26:30 +0000 Subject: [PATCH 11/43] ci(operator): preserve ordinary Go gate --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a0bc7761..7edb4c73 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -294,7 +294,9 @@ jobs: - name: Run operator tests working-directory: packages/cluster-operator - run: GOFLAGS=-mod=readonly go test ./... + env: + GOFLAGS: -mod=readonly + run: go test ./... - name: Lint cluster chart run: helm lint deploy/charts/t4-cluster From e27f3f9d16085e426064ed95c2d5a662d0520d11 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 20:43:48 +0000 Subject: [PATCH 12/43] test(operator): expose transition CEL gap --- .../cmd/crd-preflight/main_test.go | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/packages/cluster-operator/cmd/crd-preflight/main_test.go b/packages/cluster-operator/cmd/crd-preflight/main_test.go index 811a0c6f..c61c41d1 100644 --- a/packages/cluster-operator/cmd/crd-preflight/main_test.go +++ b/packages/cluster-operator/cmd/crd-preflight/main_test.go @@ -93,6 +93,57 @@ status: } } +func TestValidateFixturesRejectsUnchangedLegacyValuesUnderTransitionCEL(t *testing.T) { + tests := []struct { + name string + fixture string + candidate func(string) string + }{ + { + name: "spec transition rule", + fixture: `apiVersion: cluster.t4.dev/v1alpha1 +kind: Widget +metadata: + name: legacy +spec: + code: bad +status: + phase: Ready +`, + candidate: func(crd string) string { + crd = strings.Replace(crd, "rule: self.code.startsWith('ok')", `rule: "true"`, 1) + return strings.Replace(crd, " maxLength: 3", " maxLength: 3\n x-kubernetes-validations:\n - rule: oldSelf.startsWith('ok')", 1) + }, + }, + { + name: "status transition rule", + fixture: `apiVersion: cluster.t4.dev/v1alpha1 +kind: Widget +metadata: + name: legacy +spec: + code: ok +status: + phase: Pending +`, + candidate: func(crd string) string { + return strings.Replace(crd, " enum: [Ready]", " enum: [Ready, Pending]\n x-kubernetes-validations:\n - rule: oldSelf == 'Ready'", 1) + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + crds, fixtures := writeCandidate(t, test.fixture) + if err := os.WriteFile(filepath.Join(crds, "widget.yaml"), []byte(test.candidate(candidateCRD)), 0o644); err != nil { + t.Fatal(err) + } + if err := validateFixtures(crds, fixtures); err == nil { + t.Fatal("unchanged persisted value blocked by transition CEL was accepted") + } + }) + } +} + func TestVerifyServedSchemasRejectsRetainedEstablishedWithStaleSchema(t *testing.T) { crds, _ := writeCandidate(t, `apiVersion: cluster.t4.dev/v1alpha1 kind: Widget From 8bd18a14da17c461770adc1bd99894b3a3214813 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 20:49:50 +0000 Subject: [PATCH 13/43] fix(operator): validate unchanged CEL transitions --- .../cmd/crd-preflight/main.go | 11 ++++-- .../cmd/crd-preflight/main_test.go | 37 +++++++++++++------ 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/packages/cluster-operator/cmd/crd-preflight/main.go b/packages/cluster-operator/cmd/crd-preflight/main.go index 9751e282..20399616 100644 --- a/packages/cluster-operator/cmd/crd-preflight/main.go +++ b/packages/cluster-operator/cmd/crd-preflight/main.go @@ -115,9 +115,14 @@ func validateObject(path string, object map[string]interface{}, candidate *candi } celValidator := cel.NewValidator(candidate.structural, true, perCallCELCostLimit) if celValidator != nil { - errs, _ := celValidator.Validate(context.Background(), field.NewPath("fixture"), candidate.structural, object, nil, runtimeCELCostBudget) - if len(errs) > 0 { - result = append(result, fmt.Errorf("%s: proposed CEL validation failed: %w", path, errs.ToAggregate())) + fixturePath := field.NewPath("fixture") + createErrors, _ := celValidator.Validate(context.Background(), fixturePath, candidate.structural, object, nil, runtimeCELCostBudget) + if len(createErrors) > 0 { + result = append(result, fmt.Errorf("%s: proposed CEL create validation failed: %w", path, createErrors.ToAggregate())) + } + updateErrors, _ := celValidator.Validate(context.Background(), fixturePath, candidate.structural, object, object, runtimeCELCostBudget) + if len(updateErrors) > 0 { + result = append(result, fmt.Errorf("%s: proposed CEL unchanged-update validation failed: %w", path, updateErrors.ToAggregate())) } } unknownFields := pruning.PruneWithOptions( diff --git a/packages/cluster-operator/cmd/crd-preflight/main_test.go b/packages/cluster-operator/cmd/crd-preflight/main_test.go index c61c41d1..eef150a0 100644 --- a/packages/cluster-operator/cmd/crd-preflight/main_test.go +++ b/packages/cluster-operator/cmd/crd-preflight/main_test.go @@ -56,11 +56,12 @@ spec: func TestValidateFixturesRejectsProposedSpecTighteningAndCEL(t *testing.T) { for _, test := range []struct { - name string - code string + name string + code string + expectedError string }{ - {name: "openapi maxLength", code: "okay"}, - {name: "CEL rule", code: "bad"}, + {name: "openapi maxLength", code: "okay", expectedError: "proposed OpenAPI validation failed"}, + {name: "CEL rule", code: "bad", expectedError: "proposed CEL create validation failed"}, } { t.Run(test.name, func(t *testing.T) { crds, fixtures := writeCandidate(t, `apiVersion: cluster.t4.dev/v1alpha1 @@ -71,9 +72,13 @@ spec: code: `+test.code+"\n"+`status: phase: Ready `) - if err := validateFixtures(crds, fixtures); err == nil { + err := validateFixtures(crds, fixtures) + if err == nil { t.Fatal("fixture incompatible with the proposed spec schema was accepted") } + if !strings.Contains(err.Error(), test.expectedError) { + t.Fatalf("validation error %q does not identify %q", err, test.expectedError) + } }) } } @@ -95,12 +100,14 @@ status: func TestValidateFixturesRejectsUnchangedLegacyValuesUnderTransitionCEL(t *testing.T) { tests := []struct { - name string - fixture string - candidate func(string) string + name string + fixture string + expectedPath string + candidate func(string) string }{ { - name: "spec transition rule", + name: "spec transition rule", + expectedPath: "fixture.spec.code", fixture: `apiVersion: cluster.t4.dev/v1alpha1 kind: Widget metadata: @@ -116,7 +123,8 @@ status: }, }, { - name: "status transition rule", + name: "status transition rule", + expectedPath: "fixture.status.phase", fixture: `apiVersion: cluster.t4.dev/v1alpha1 kind: Widget metadata: @@ -137,9 +145,16 @@ status: if err := os.WriteFile(filepath.Join(crds, "widget.yaml"), []byte(test.candidate(candidateCRD)), 0o644); err != nil { t.Fatal(err) } - if err := validateFixtures(crds, fixtures); err == nil { + err := validateFixtures(crds, fixtures) + if err == nil { t.Fatal("unchanged persisted value blocked by transition CEL was accepted") } + if !strings.Contains(err.Error(), "proposed CEL unchanged-update validation failed") { + t.Fatalf("validation error does not identify unchanged-update semantics: %v", err) + } + if !strings.Contains(err.Error(), test.expectedPath) { + t.Fatalf("validation error %q does not identify field %q", err, test.expectedPath) + } }) } } From 1a207e1785f1cffeb76efc66b373b7b71cab072f Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 03:47:00 +0000 Subject: [PATCH 14/43] test(operator): expose live CR narrowing gap --- .../cmd/crd-preflight/main_test.go | 93 +++++++++++++++++++ scripts/cluster-ci/crd-lifecycle.test.mjs | 45 ++++++++- 2 files changed, 137 insertions(+), 1 deletion(-) diff --git a/packages/cluster-operator/cmd/crd-preflight/main_test.go b/packages/cluster-operator/cmd/crd-preflight/main_test.go index eef150a0..1789f16f 100644 --- a/packages/cluster-operator/cmd/crd-preflight/main_test.go +++ b/packages/cluster-operator/cmd/crd-preflight/main_test.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "os" "path/filepath" "strings" @@ -159,6 +160,98 @@ status: } } +func TestValidateObjectsRejectsLiveDataOmittedFromFixtures(t *testing.T) { + tests := []struct { + name string + candidate func(string) string + fixtureCode string + fixturePhase string + liveCode string + livePhase string + liveExtra string + expectedError string + }{ + { + name: "spec OpenAPI narrowing", + candidate: func(crd string) string { + return strings.Replace(crd, " maxLength: 3", " maxLength: 3\n enum: [ok1]", 1) + }, + fixtureCode: "ok1", fixturePhase: "Ready", liveCode: "ok2", livePhase: "Ready", + expectedError: "proposed OpenAPI validation failed", + }, + { + name: "status OpenAPI narrowing", + candidate: func(crd string) string { return crd }, + fixtureCode: "ok", fixturePhase: "Ready", liveCode: "ok", livePhase: "Legacy", + expectedError: "proposed OpenAPI validation failed", + }, + { + name: "spec CEL create semantics", + candidate: func(crd string) string { return crd }, + fixtureCode: "ok", fixturePhase: "Ready", liveCode: "bad", livePhase: "Ready", + expectedError: "proposed CEL create validation failed", + }, + { + name: "status CEL create semantics", + candidate: func(crd string) string { + return strings.Replace(crd, " phase:\n type: string\n enum: [Ready]", " phase:\n type: string\n enum: [Ready, Legacy]\n x-kubernetes-validations:\n - rule: self == 'Ready'", 1) + }, + fixtureCode: "ok", fixturePhase: "Ready", liveCode: "ok", livePhase: "Legacy", + expectedError: "proposed CEL create validation failed", + }, + { + name: "spec CEL unchanged-update semantics", + candidate: func(crd string) string { + crd = strings.Replace(crd, "rule: self.code.startsWith('ok')", `rule: "true"`, 1) + return strings.Replace(crd, " maxLength: 3", " maxLength: 3\n x-kubernetes-validations:\n - rule: oldSelf.startsWith('ok')", 1) + }, + fixtureCode: "ok", fixturePhase: "Ready", liveCode: "bad", livePhase: "Ready", + expectedError: "proposed CEL unchanged-update validation failed", + }, + { + name: "status CEL unchanged-update semantics", + candidate: func(crd string) string { + return strings.Replace(crd, " enum: [Ready]", " enum: [Ready, Legacy]\n x-kubernetes-validations:\n - rule: oldSelf == 'Ready'", 1) + }, + fixtureCode: "ok", fixturePhase: "Ready", liveCode: "ok", livePhase: "Legacy", + expectedError: "proposed CEL unchanged-update validation failed", + }, + { + name: "spec pruning", + candidate: func(crd string) string { return crd }, + fixtureCode: "ok", fixturePhase: "Ready", liveCode: "ok", livePhase: "Ready", liveExtra: `,"removedSpec":"legacy"`, + expectedError: "proposed structural schema would prune declared fields", + }, + { + name: "status pruning", + candidate: func(crd string) string { return crd }, + fixtureCode: "ok", fixturePhase: "Ready", liveCode: "ok", livePhase: "Ready", liveExtra: `,"removedStatus":"legacy"`, + expectedError: "proposed structural schema would prune declared fields", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fixture := fmt.Sprintf("apiVersion: cluster.t4.dev/v1alpha1\nkind: Widget\nmetadata:\n name: curated\n namespace: tenant-a\nspec:\n code: %s\nstatus:\n phase: %s\n", test.fixtureCode, test.fixturePhase) + crds, fixtures := writeCandidate(t, fixture) + if err := os.WriteFile(filepath.Join(crds, "widget.yaml"), []byte(test.candidate(candidateCRD)), 0o644); err != nil { + t.Fatal(err) + } + if err := validateFixtures(crds, fixtures); err != nil { + t.Fatalf("curated fixture should not expose the live-only incompatibility: %v", err) + } + live := fmt.Sprintf(`{"apiVersion":"v1","kind":"WidgetList","items":[{"apiVersion":"cluster.t4.dev/v1alpha1","kind":"Widget","metadata":{"name":"live","namespace":"tenant-b"},"spec":{"code":%q%s},"status":{"phase":%q%s}}]}`, + test.liveCode, map[bool]string{true: test.liveExtra, false: ""}[test.name == "spec pruning"], test.livePhase, map[bool]string{true: test.liveExtra, false: ""}[test.name == "status pruning"]) + err := validateObjects(crds, strings.NewReader(live)) + if err == nil { + t.Fatal("live object incompatible with the proposed schema was accepted") + } + if !strings.Contains(err.Error(), "tenant-b/live") || !strings.Contains(err.Error(), test.expectedError) { + t.Fatalf("validation error %q does not identify live object and %q", err, test.expectedError) + } + }) + } +} + func TestVerifyServedSchemasRejectsRetainedEstablishedWithStaleSchema(t *testing.T) { crds, _ := writeCandidate(t, `apiVersion: cluster.t4.dev/v1alpha1 kind: Widget diff --git a/scripts/cluster-ci/crd-lifecycle.test.mjs b/scripts/cluster-ci/crd-lifecycle.test.mjs index 4e0eb812..f856bd92 100644 --- a/scripts/cluster-ci/crd-lifecycle.test.mjs +++ b/scripts/cluster-ci/crd-lifecycle.test.mjs @@ -26,6 +26,13 @@ for argument in "$@"; do fi done if [ "\${1:-}" = get ]; then + case "\${2:-}" in + t4clusterhosts.cluster.t4.dev|t4workspaces.cluster.t4.dev|t4sessions.cluster.t4.dev) + if [ "\${FAIL_LIVE_LIST:-}" = "\${2:-}" ]; then exit 44; fi + printf '%s' '{"apiVersion":"v1","kind":"List","items":[]}' + exit 0 + ;; + esac printf '%s' "\${STORED_VERSIONS:-v1alpha1}" fi `, @@ -38,7 +45,7 @@ printf 'validator' >>"$COMMAND_LOG" for argument in "$@"; do printf '\\t%s' "$argument" >>"$COMMAND_LOG"; done printf '\\n' >>"$COMMAND_LOG" case "\${1:-}:\${FAIL_PROPOSED_VALIDATION:-}" in - fixtures:spec|fixtures:status|served:stale) exit 43 ;; + fixtures:spec|fixtures:status|objects:live|served:stale) exit 43 ;; esac cat >/dev/null `, @@ -102,6 +109,8 @@ test("upgrade validates proposed schemas, proves served convergence, verifies st assert.equal(result.code, 0, `${result.stdout}\n${result.stderr}`); const log = await commands(value.log); const proposedPreflight = findCommand(log, (line) => line.startsWith("validator\tfixtures\t"), "local proposed-schema fixture preflight"); + const liveLists = log.map((line, index) => ({ line, index })).filter(({ line }) => /^kubectl\tget\tt4(?:clusterhosts|workspaces|sessions)\.cluster\.t4\.dev\t--all-namespaces\t-o\tjson$/u.test(line)); + const liveValidations = log.map((line, index) => ({ line, index })).filter(({ line }) => line.startsWith("validator\tobjects\t")); const crdPreflight = findCommand(log, (line) => line.includes("apply") && line.includes("--server-side") && line.includes("--dry-run=server") && line.includes("deploy/charts/t4-cluster/crds"), "server-side CRD preflight"); const crdApply = findCommand(log, (line) => line.includes("apply") && line.includes("--server-side") && !line.includes("--dry-run=server"), "CRD apply"); const established = findCommand(log, (line) => line.includes("wait") && line.includes("condition=Established") && line.includes("t4clusterhosts.cluster.t4.dev") && line.includes("t4workspaces.cluster.t4.dev") && line.includes("t4sessions.cluster.t4.dev"), "Established wait"); @@ -110,8 +119,16 @@ test("upgrade validates proposed schemas, proves served convergence, verifies st const storageChecks = log.map((line, index) => ({ line, index })).filter(({ line }) => line.startsWith("kubectl\tget\tcrd/") && line.includes("status.storedVersions")); assert.deepEqual(storageChecks.length, 3); assert.equal(servedChecks.length, 3); + assert.equal(liveLists.length, 3); + assert.equal(liveValidations.length, 3); const workload = findCommand(log, (line) => line.startsWith("helm\tupgrade\t"), "Helm workload upgrade"); assert.ok(proposedPreflight < crdPreflight); + assert.ok(liveLists[0].index < liveValidations[0].index); + assert.ok(liveValidations[0].index < liveLists[1].index); + assert.ok(liveLists[1].index < liveValidations[1].index); + assert.ok(liveValidations[1].index < liveLists[2].index); + assert.ok(liveLists[2].index < liveValidations[2].index); + assert.ok(liveValidations[2].index < crdPreflight); assert.ok(crdPreflight < crdApply); assert.ok(crdApply < established); assert.ok(established < servedChecks[0].index); @@ -160,6 +177,32 @@ test("persisted status is validated against the proposed status schema before mu ]); }); +test("live object incompatibility fails before non-dry-run CRD, object, or Helm mutation", async () => { + const value = await fixture(); + const result = await runLifecycle( + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "chart", "--skip-crds"], + { ...value.env, FAIL_PROPOSED_VALIDATION: "live" }, + ); + assert.notEqual(result.code, 0); + const log = await commands(value.log); + assert.match(log.at(-1), /^validator\tobjects\t/u); + assert.ok(log.some((line) => /^kubectl\tget\tt4clusterhosts\.cluster\.t4\.dev\t--all-namespaces\t-o\tjson$/u.test(line)), log.join("\n")); + assert.ok(log.every((line) => !line.startsWith("helm\t") && !/kubectl\tapply(?!.*--dry-run=server)/u.test(line)), log.join("\n")); +}); + +test("live object list denial fails closed before non-dry-run mutation", async () => { + const value = await fixture(); + const deniedResource = "t4workspaces.cluster.t4.dev"; + const result = await runLifecycle( + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "chart", "--skip-crds"], + { ...value.env, FAIL_LIVE_LIST: deniedResource }, + ); + assert.notEqual(result.code, 0); + const log = await commands(value.log); + assert.ok(log.some((line) => line.startsWith(`kubectl\tget\t${deniedResource}\t--all-namespaces\t-o\tjson`)), log.join("\n")); + assert.ok(log.every((line) => !line.startsWith("helm\t") && !/kubectl\tapply(?!.*--dry-run=server)/u.test(line)), log.join("\n")); +}); + test("retained Established cannot pass readiness while served OpenAPI is stale", async () => { const value = await fixture(); const result = await runLifecycle( From de9bf6f118476f1630b005544aefb64bc5264170 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 03:56:17 +0000 Subject: [PATCH 15/43] fix(operator): validate every live CR before apply --- docs/CLUSTER_OPERATOR.md | 33 +++++-- .../cmd/crd-preflight/main.go | 97 ++++++++++++++++--- .../cmd/crd-preflight/main_test.go | 19 ++++ scripts/cluster-ci/crd-lifecycle.sh | 27 +++++- scripts/cluster-ci/crd-lifecycle.test.mjs | 58 +++++++++-- 5 files changed, 201 insertions(+), 33 deletions(-) diff --git a/docs/CLUSTER_OPERATOR.md b/docs/CLUSTER_OPERATOR.md index bd38244d..aae439e4 100644 --- a/docs/CLUSTER_OPERATOR.md +++ b/docs/CLUSTER_OPERATOR.md @@ -23,7 +23,7 @@ That declaration does not make a non-RWX backend safe. Missing classes, classes ## Install -Installing with defaults creates no controller, gateway, session workload, RBAC, Secret, or network policy. Use the lifecycle runner even for a fresh install so the CRDs are server-validated, established, fixture-checked, and storage-version-checked before Helm runs: +Installing with defaults creates no controller, gateway, session workload, RBAC, Secret, or network policy. Use the lifecycle runner even for a fresh install so existing live objects and compatibility fixtures are checked against the proposed schemas before the CRDs are server-validated, established, and storage-version-checked and before Helm runs: ```sh scripts/cluster-ci/crd-lifecycle.sh install -- \ @@ -163,19 +163,36 @@ scripts/cluster-ci/crd-lifecycle.sh upgrade -- \ The command performs this exact order: 1. Before any cluster access, validate every complete compatibility fixture (including `status`) against the proposed structural OpenAPI schema and CEL programs with the Kubernetes apiextensions validators. The check also rejects declared fields that the structural pruner would remove. -2. Server-side dry-run the reviewed CRDs with strict validation. Upgrades stop before any mutation if either local proposed-schema validation or the CRD dry-run fails. -3. Apply the CRDs server-side without force conflicts. -4. Wait for all three CRDs to report `Established`. -5. Fetch the served `cluster.t4.dev/v1alpha1` OpenAPI v3 document three times and require every published resource schema to match the semantics generated from the proposed CRDs. Retained `Established=True` is not readiness when discovery still serves an old schema. -6. Server-side dry-run the compatibility fixtures against the converged admission path. -7. Require each CRD's `status.storedVersions` to be exactly `v1alpha1`. -8. Execute the supplied Helm command, which must use `--skip-crds`. +2. With `get` access to the three exact CRD definitions and `list` access to the corresponding namespaced T4 resources, detect which definitions already exist, enumerate every live CR for each existing definition across namespaces, and validate each complete object, including `status`, against the same proposed OpenAPI, CEL create, CEL unchanged-update, and pruning checks. A denied CRD read, denied or malformed object list, or incompatible object fails closed; an absent definition is safely empty on a fresh install. +3. Server-side dry-run the reviewed CRDs with strict validation. Upgrades stop before any mutation if a local proposed-schema validation, live-object enumeration, or CRD dry-run fails. +4. Apply the CRDs server-side without force conflicts. +5. Wait for all three CRDs to report `Established`. +6. Fetch the served `cluster.t4.dev/v1alpha1` OpenAPI v3 document three times and require every published resource schema to match the semantics generated from the proposed CRDs. Retained `Established=True` is not readiness when discovery still serves an old schema. +7. Server-side dry-run the compatibility fixtures against the converged admission path. +8. Require each CRD's `status.storedVersions` to be exactly `v1alpha1`. +9. Execute the supplied Helm command, which must use `--skip-crds`. The corresponding administrative checks are executable independently: ```sh (cd packages/cluster-operator && \ go run ./cmd/crd-preflight fixtures ../../deploy/charts/t4-cluster/crds api/v1alpha1/testdata/compat) +live_objects=$(mktemp) +trap 'rm -f "$live_objects"' EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM +for resource in t4clusterhosts.cluster.t4.dev t4workspaces.cluster.t4.dev t4sessions.cluster.t4.dev; do + installed_crd=$(kubectl get "crd/$resource" --ignore-not-found -o name) + if [ -z "$installed_crd" ]; then + continue + fi + kubectl get "$resource" --all-namespaces -o json >"$live_objects" + (cd packages/cluster-operator && \ + go run ./cmd/crd-preflight objects ../../deploy/charts/t4-cluster/crds <"$live_objects") +done +rm -f "$live_objects" +trap - EXIT HUP INT TERM kubectl apply --server-side --dry-run=server --validate=strict \ --field-manager=t4-crd-lifecycle -f deploy/charts/t4-cluster/crds/ kubectl apply --server-side --validate=strict \ diff --git a/packages/cluster-operator/cmd/crd-preflight/main.go b/packages/cluster-operator/cmd/crd-preflight/main.go index 20399616..0a55241b 100644 --- a/packages/cluster-operator/cmd/crd-preflight/main.go +++ b/packages/cluster-operator/cmd/crd-preflight/main.go @@ -19,6 +19,7 @@ import ( apiservervalidation "k8s.io/apiextensions-apiserver/pkg/apiserver/validation" "k8s.io/apiextensions-apiserver/pkg/controller/openapi/builder" "k8s.io/apimachinery/pkg/runtime" + kubejson "k8s.io/apimachinery/pkg/util/json" "k8s.io/apimachinery/pkg/util/validation/field" "sigs.k8s.io/yaml" ) @@ -52,6 +53,11 @@ func main() { usage() } err = validateFixtures(os.Args[2], os.Args[3]) + case "objects": + if len(os.Args) != 3 { + usage() + } + err = validateObjects(os.Args[2], os.Stdin) case "served": if len(os.Args) != 3 { usage() @@ -67,7 +73,7 @@ func main() { } func usage() { - fmt.Fprintln(os.Stderr, "usage: crd-preflight fixtures CRD_DIRECTORY FIXTURE_DIRECTORY | served CRD_DIRECTORY") + fmt.Fprintln(os.Stderr, "usage: crd-preflight fixtures CRD_DIRECTORY FIXTURE_DIRECTORY | objects CRD_DIRECTORY | served CRD_DIRECTORY") os.Exit(64) } @@ -87,40 +93,101 @@ func validateFixtures(crdDirectory, fixtureDirectory string) error { validationErrors = append(validationErrors, err) continue } - apiVersion, _ := object["apiVersion"].(string) - kind, _ := object["kind"].(string) - group, version, ok := strings.Cut(apiVersion, "/") - if !ok || group == "" || version == "" || kind == "" { - validationErrors = append(validationErrors, fmt.Errorf("%s: apiVersion and kind must identify a grouped resource", path)) + validationErrors = append(validationErrors, validateCandidateObject(path, "fixture", object, candidates)...) + } + return errors.Join(validationErrors...) +} + +func validateObjects(crdDirectory string, input io.Reader) error { + candidates, err := loadCandidates(crdDirectory) + if err != nil { + return err + } + var list map[string]interface{} + decoder := json.NewDecoder(input) + decoder.UseNumber() + if err := decoder.Decode(&list); err != nil { + return fmt.Errorf("decode live object list: %w", err) + } + if err := kubejson.ConvertMapNumbers(list, 0); err != nil { + return fmt.Errorf("decode live object list numbers: %w", err) + } + var trailing interface{} + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + if err == nil { + return errors.New("decode live object list: multiple JSON documents") + } + return fmt.Errorf("decode live object list trailing data: %w", err) + } + rawItems, found := list["items"] + if !found { + return errors.New("decode live object list: missing items") + } + items, ok := rawItems.([]interface{}) + if !ok { + return errors.New("decode live object list: items is not an array") + } + var validationErrors []error + for index, rawItem := range items { + object, ok := rawItem.(map[string]interface{}) + if !ok { + validationErrors = append(validationErrors, fmt.Errorf("live object item %d is not an object", index)) continue } - candidate, found := candidates[groupVersionKind{Group: group, Version: version, Kind: kind}] - if !found { - validationErrors = append(validationErrors, fmt.Errorf("%s: no proposed schema for %s %s", path, apiVersion, kind)) + path, err := liveObjectPath(index, object) + if err != nil { + validationErrors = append(validationErrors, err) continue } - validationErrors = append(validationErrors, validateObject(path, object, candidate)...) + validationErrors = append(validationErrors, validateCandidateObject(path, "object", object, candidates)...) } return errors.Join(validationErrors...) } -func validateObject(path string, object map[string]interface{}, candidate *candidateSchema) []error { +func liveObjectPath(index int, object map[string]interface{}) (string, error) { + metadata, ok := object["metadata"].(map[string]interface{}) + if !ok { + return "", fmt.Errorf("live object item %d has no metadata", index) + } + name, _ := metadata["name"].(string) + namespace, _ := metadata["namespace"].(string) + if name == "" || namespace == "" { + return "", fmt.Errorf("live object item %d must have namespace and name", index) + } + return fmt.Sprintf("live object %s/%s", namespace, name), nil +} + +func validateCandidateObject(path, fieldRoot string, object map[string]interface{}, candidates map[groupVersionKind]*candidateSchema) []error { + apiVersion, _ := object["apiVersion"].(string) + kind, _ := object["kind"].(string) + group, version, ok := strings.Cut(apiVersion, "/") + if !ok || group == "" || version == "" || kind == "" { + return []error{fmt.Errorf("%s: apiVersion and kind must identify a grouped resource", path)} + } + candidate, found := candidates[groupVersionKind{Group: group, Version: version, Kind: kind}] + if !found { + return []error{fmt.Errorf("%s: no proposed schema for %s %s", path, apiVersion, kind)} + } + return validateObject(path, fieldRoot, object, candidate) +} + +func validateObject(path, fieldRoot string, object map[string]interface{}, candidate *candidateSchema) []error { var result []error validator, _, err := apiservervalidation.NewSchemaValidator(candidate.internal) if err != nil { return []error{fmt.Errorf("%s: build OpenAPI validator: %w", path, err)} } - if errs := apiservervalidation.ValidateCustomResource(field.NewPath("fixture"), object, validator); len(errs) > 0 { + if errs := apiservervalidation.ValidateCustomResource(field.NewPath(fieldRoot), object, validator); len(errs) > 0 { result = append(result, fmt.Errorf("%s: proposed OpenAPI validation failed: %w", path, errs.ToAggregate())) } celValidator := cel.NewValidator(candidate.structural, true, perCallCELCostLimit) if celValidator != nil { - fixturePath := field.NewPath("fixture") - createErrors, _ := celValidator.Validate(context.Background(), fixturePath, candidate.structural, object, nil, runtimeCELCostBudget) + objectPath := field.NewPath(fieldRoot) + createErrors, _ := celValidator.Validate(context.Background(), objectPath, candidate.structural, object, nil, runtimeCELCostBudget) if len(createErrors) > 0 { result = append(result, fmt.Errorf("%s: proposed CEL create validation failed: %w", path, createErrors.ToAggregate())) } - updateErrors, _ := celValidator.Validate(context.Background(), fixturePath, candidate.structural, object, object, runtimeCELCostBudget) + updateErrors, _ := celValidator.Validate(context.Background(), objectPath, candidate.structural, object, object, runtimeCELCostBudget) if len(updateErrors) > 0 { result = append(result, fmt.Errorf("%s: proposed CEL unchanged-update validation failed: %w", path, updateErrors.ToAggregate())) } diff --git a/packages/cluster-operator/cmd/crd-preflight/main_test.go b/packages/cluster-operator/cmd/crd-preflight/main_test.go index 1789f16f..1c08fa28 100644 --- a/packages/cluster-operator/cmd/crd-preflight/main_test.go +++ b/packages/cluster-operator/cmd/crd-preflight/main_test.go @@ -252,6 +252,25 @@ func TestValidateObjectsRejectsLiveDataOmittedFromFixtures(t *testing.T) { } } +func TestValidateObjectsAcceptsKubernetesJSONNumbers(t *testing.T) { + crds, _ := writeCandidate(t, `apiVersion: cluster.t4.dev/v1alpha1 +kind: Widget +metadata: + name: curated +spec: + code: ok +`) + candidate := strings.Replace(candidateCRD, " required: [code]", " required: [code, count]", 1) + candidate = strings.Replace(candidate, " maxLength: 3", " maxLength: 3\n count:\n type: integer\n x-kubernetes-validations:\n - rule: self == 9223372036854775807", 1) + if err := os.WriteFile(filepath.Join(crds, "widget.yaml"), []byte(candidate), 0o644); err != nil { + t.Fatal(err) + } + live := strings.NewReader(`{"apiVersion":"v1","kind":"WidgetList","items":[{"apiVersion":"cluster.t4.dev/v1alpha1","kind":"Widget","metadata":{"name":"live","namespace":"tenant-b"},"spec":{"code":"ok","count":9223372036854775807},"status":{"phase":"Ready"}}]}`) + if err := validateObjects(crds, live); err != nil { + t.Fatalf("ordinary Kubernetes JSON integer was rejected: %v", err) + } +} + func TestVerifyServedSchemasRejectsRetainedEstablishedWithStaleSchema(t *testing.T) { crds, _ := writeCandidate(t, `apiVersion: cluster.t4.dev/v1alpha1 kind: Widget diff --git a/scripts/cluster-ci/crd-lifecycle.sh b/scripts/cluster-ci/crd-lifecycle.sh index 9a4c7d13..d698c7b3 100755 --- a/scripts/cluster-ci/crd-lifecycle.sh +++ b/scripts/cluster-ci/crd-lifecycle.sh @@ -50,6 +50,7 @@ compat_directory=${T4_COMPAT_DIRECTORY:-$repo_root/packages/cluster-operator/api validation_namespace=${T4_VALIDATION_NAMESPACE:-default} field_manager=t4-crd-lifecycle crds="crd/t4clusterhosts.cluster.t4.dev crd/t4workspaces.cluster.t4.dev crd/t4sessions.cluster.t4.dev" +live_resources="t4clusterhosts.cluster.t4.dev t4workspaces.cluster.t4.dev t4sessions.cluster.t4.dev" discovery_observations=${T4_DISCOVERY_OBSERVATIONS:-3} case "$discovery_observations" in @@ -72,6 +73,27 @@ run_validator() { # first operation: current-cluster admission cannot prove candidate compatibility. run_validator fixtures "$crd_directory" "$compat_directory" +# Curated fixtures cannot prove that every value currently persisted in the +# cluster remains valid. Enumerate only the three namespaced T4 resources and +# run each complete live object, including status, through the same proposed- +# schema engine before a CRD or workload can be mutated. A denied or malformed +# list fails closed under set -e. +live_objects=$(mktemp) +trap 'rm -f "$live_objects"' EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM +for resource in $live_resources; do + installed_crd=$("$kubectl" get "crd/$resource" --ignore-not-found -o name) + if [ -z "$installed_crd" ]; then + continue + fi + "$kubectl" get "$resource" --all-namespaces -o json >"$live_objects" + run_validator objects "$crd_directory" <"$live_objects" +done +rm -f "$live_objects" +trap - EXIT HUP INT TERM + # Every kubectl operation before the first non-dry-run apply is read-only. "$kubectl" apply --server-side --dry-run=server --validate=strict \ --field-manager="$field_manager" -f "$crd_directory" >/dev/null @@ -87,7 +109,10 @@ run_validator fixtures "$crd_directory" "$compat_directory" # fetched discovery documents to expose exactly the candidate OpenAPI semantics # before trusting admission or starting a workload rollout. openapi_document=$(mktemp) -trap 'rm -f "$openapi_document"' EXIT HUP INT TERM +trap 'rm -f "$openapi_document"' EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM observation=0 while [ "$observation" -lt "$discovery_observations" ]; do "$kubectl" get --raw /openapi/v3/apis/cluster.t4.dev/v1alpha1 >"$openapi_document" diff --git a/scripts/cluster-ci/crd-lifecycle.test.mjs b/scripts/cluster-ci/crd-lifecycle.test.mjs index f856bd92..543f7775 100644 --- a/scripts/cluster-ci/crd-lifecycle.test.mjs +++ b/scripts/cluster-ci/crd-lifecycle.test.mjs @@ -26,9 +26,25 @@ for argument in "$@"; do fi done if [ "\${1:-}" = get ]; then + case "\${2:-}" in + crd/*) + for argument in "$@"; do + if [ "$argument" = "--ignore-not-found" ]; then + if [ "\${FAIL_CRD_READ:-}" = "\${2:-}" ]; then exit 45; fi + if [ "\${MISSING_LIVE_CRDS:-0}" = 1 ]; then exit 0; fi + printf '%s' "\${2:-}" + exit 0 + fi + done + ;; + esac case "\${2:-}" in t4clusterhosts.cluster.t4.dev|t4workspaces.cluster.t4.dev|t4sessions.cluster.t4.dev) if [ "\${FAIL_LIVE_LIST:-}" = "\${2:-}" ]; then exit 44; fi + if [ "\${TERMINATE_DURING_LIVE_READ:-}" = "\${2:-}" ]; then + kill -TERM "$PPID" + sleep 1 + fi printf '%s' '{"apiVersion":"v1","kind":"List","items":[]}' exit 0 ;; @@ -109,6 +125,7 @@ test("upgrade validates proposed schemas, proves served convergence, verifies st assert.equal(result.code, 0, `${result.stdout}\n${result.stderr}`); const log = await commands(value.log); const proposedPreflight = findCommand(log, (line) => line.startsWith("validator\tfixtures\t"), "local proposed-schema fixture preflight"); + const crdReads = log.map((line, index) => ({ line, index })).filter(({ line }) => /^kubectl\tget\tcrd\/t4(?:clusterhosts|workspaces|sessions)\.cluster\.t4\.dev\t--ignore-not-found\t-o\tname$/u.test(line)); const liveLists = log.map((line, index) => ({ line, index })).filter(({ line }) => /^kubectl\tget\tt4(?:clusterhosts|workspaces|sessions)\.cluster\.t4\.dev\t--all-namespaces\t-o\tjson$/u.test(line)); const liveValidations = log.map((line, index) => ({ line, index })).filter(({ line }) => line.startsWith("validator\tobjects\t")); const crdPreflight = findCommand(log, (line) => line.includes("apply") && line.includes("--server-side") && line.includes("--dry-run=server") && line.includes("deploy/charts/t4-cluster/crds"), "server-side CRD preflight"); @@ -119,14 +136,19 @@ test("upgrade validates proposed schemas, proves served convergence, verifies st const storageChecks = log.map((line, index) => ({ line, index })).filter(({ line }) => line.startsWith("kubectl\tget\tcrd/") && line.includes("status.storedVersions")); assert.deepEqual(storageChecks.length, 3); assert.equal(servedChecks.length, 3); + assert.equal(crdReads.length, 3); assert.equal(liveLists.length, 3); assert.equal(liveValidations.length, 3); const workload = findCommand(log, (line) => line.startsWith("helm\tupgrade\t"), "Helm workload upgrade"); assert.ok(proposedPreflight < crdPreflight); + assert.ok(proposedPreflight < crdReads[0].index); + assert.ok(crdReads[0].index < liveLists[0].index); assert.ok(liveLists[0].index < liveValidations[0].index); - assert.ok(liveValidations[0].index < liveLists[1].index); + assert.ok(liveValidations[0].index < crdReads[1].index); + assert.ok(crdReads[1].index < liveLists[1].index); assert.ok(liveLists[1].index < liveValidations[1].index); - assert.ok(liveValidations[1].index < liveLists[2].index); + assert.ok(liveValidations[1].index < crdReads[2].index); + assert.ok(crdReads[2].index < liveLists[2].index); assert.ok(liveLists[2].index < liveValidations[2].index); assert.ok(liveValidations[2].index < crdPreflight); assert.ok(crdPreflight < crdApply); @@ -142,10 +164,11 @@ test("fresh install establishes and validates CRDs before Helm installs with CRD const value = await fixture(); const result = await runLifecycle( ["install", "--", "helm", "install", "t4-cluster", "deploy/charts/t4-cluster", "--namespace", "t4-system", "--skip-crds"], - value.env, + { ...value.env, MISSING_LIVE_CRDS: "1" }, ); assert.equal(result.code, 0, `${result.stdout}\n${result.stderr}`); const log = await commands(value.log); + assert.ok(log.every((line) => !line.startsWith("validator\tobjects\t") && !/kubectl\tget\tt4(?:clusterhosts|workspaces|sessions)\.cluster\.t4\.dev\t--all-namespaces/u.test(line)), log.join("\n")); const established = findCommand(log, (line) => line.includes("wait") && line.includes("condition=Established"), "Established wait"); const fixtureValidation = findCommand(log, (line) => line.includes("--dry-run=server") && line.includes("testdata/compat"), "fixture validation"); const storage = findCommand(log, (line) => line.includes("status.storedVersions"), "stored-version check"); @@ -190,16 +213,33 @@ test("live object incompatibility fails before non-dry-run CRD, object, or Helm assert.ok(log.every((line) => !line.startsWith("helm\t") && !/kubectl\tapply(?!.*--dry-run=server)/u.test(line)), log.join("\n")); }); -test("live object list denial fails closed before non-dry-run mutation", async () => { - const value = await fixture(); +test("live object definition read or list denial fails closed before non-dry-run mutation", async () => { const deniedResource = "t4workspaces.cluster.t4.dev"; + for (const scenario of [ + { env: { FAIL_CRD_READ: `crd/${deniedResource}` }, expected: `kubectl\tget\tcrd/${deniedResource}\t--ignore-not-found\t-o\tname` }, + { env: { FAIL_LIVE_LIST: deniedResource }, expected: `kubectl\tget\t${deniedResource}\t--all-namespaces\t-o\tjson` }, + ]) { + const value = await fixture(); + const result = await runLifecycle( + ["upgrade", "--", "helm", "upgrade", "t4-cluster", "chart", "--skip-crds"], + { ...value.env, ...scenario.env }, + ); + assert.notEqual(result.code, 0); + const log = await commands(value.log); + assert.ok(log.some((line) => line.startsWith(scenario.expected)), log.join("\n")); + assert.ok(log.every((line) => !line.startsWith("helm\t") && !/kubectl\tapply(?!.*--dry-run=server)/u.test(line)), log.join("\n")); + } +}); + +test("cancellation during live enumeration cannot continue to mutation", async () => { + const value = await fixture(); const result = await runLifecycle( ["upgrade", "--", "helm", "upgrade", "t4-cluster", "chart", "--skip-crds"], - { ...value.env, FAIL_LIVE_LIST: deniedResource }, + { ...value.env, TERMINATE_DURING_LIVE_READ: "t4clusterhosts.cluster.t4.dev" }, ); assert.notEqual(result.code, 0); const log = await commands(value.log); - assert.ok(log.some((line) => line.startsWith(`kubectl\tget\t${deniedResource}\t--all-namespaces\t-o\tjson`)), log.join("\n")); + assert.match(log.at(-1), /^kubectl\tget\tt4clusterhosts\.cluster\.t4\.dev\t--all-namespaces\t-o\tjson$/u); assert.ok(log.every((line) => !line.startsWith("helm\t") && !/kubectl\tapply(?!.*--dry-run=server)/u.test(line)), log.join("\n")); }); @@ -226,9 +266,9 @@ test("failed server preflight leaves CRDs and workloads untouched", async () => ); assert.notEqual(result.code, 0); const log = await commands(value.log); - assert.equal(log.length, 2, log.join("\n")); + assert.equal(log.length, 11, log.join("\n")); assert.match(log[0], /^validator\tfixtures\t/u); - assert.match(log[1], /apply.*--server-side.*--dry-run=server/u); + assert.match(log.at(-1), /apply.*--server-side.*--dry-run=server/u); assert.ok(log.every((line) => !line.startsWith("helm\t") && !/kubectl\tapply(?!.*--dry-run=server)/u.test(line)), log.join("\n")); }); From 2ebbf130d5042032c038c1a496a45d7f0d18e5b3 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 20:41:50 +0000 Subject: [PATCH 16/43] test(operator): require dependency event convergence --- .../controllers/reconciler_test.go | 117 ++++++++++++++++++ .../controllers/session_controller_test.go | 62 ++++++++++ .../controllers/workspace_controller_test.go | 40 ++++++ 3 files changed, 219 insertions(+) create mode 100644 packages/cluster-operator/controllers/session_controller_test.go diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index 1a54e2d0..02b070a2 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -2,8 +2,10 @@ package controllers_test import ( "context" + "reflect" "strings" "testing" + "time" corev1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" @@ -1254,6 +1256,121 @@ func TestSessionDeletionCleansResourcesBeforeFinalizer(t *testing.T) { } } +func TestSessionDependencyRevocationCleansOwnedResourcesAndConvergesAfterRestart(t *testing.T) { + for _, test := range []struct { + name string + conditionType string + wantReason string + revoke func(context.Context, client.Client) error + }{ + {name: "missing Host", conditionType: "HostReady", wantReason: "HostNotFound", revoke: func(ctx context.Context, c client.Client) error { + return c.Delete(ctx, &clusterv1alpha1.T4ClusterHost{ObjectMeta: metav1.ObjectMeta{Name: "host-a", Namespace: "team"}}) + }}, + {name: "invalid Host runtime profile", conditionType: "RuntimeConfigured", wantReason: "RuntimeProfileNotAllowed", revoke: func(ctx context.Context, c client.Client) error { + var host clusterv1alpha1.T4ClusterHost + if err := c.Get(ctx, types.NamespacedName{Namespace: "team", Name: "host-a"}, &host); err != nil { + return err + } + host.Spec.RuntimeProfiles = nil + return c.Update(ctx, &host) + }}, + {name: "missing Workspace", conditionType: "WorkspaceReady", wantReason: "WorkspaceNotFound", revoke: func(ctx context.Context, c client.Client) error { + return c.Delete(ctx, &clusterv1alpha1.T4Workspace{ObjectMeta: metav1.ObjectMeta{Name: "workspace-a", Namespace: "team"}}) + }}, + {name: "mismatched Workspace Host", conditionType: "WorkspaceReady", wantReason: "HostMismatch", revoke: func(ctx context.Context, c client.Client) error { + var workspace clusterv1alpha1.T4Workspace + if err := c.Get(ctx, types.NamespacedName{Namespace: "team", Name: "workspace-a"}, &workspace); err != nil { + return err + } + workspace.Spec.HostRef = "host-b" + return c.Update(ctx, &workspace) + }}, + {name: "missing OMP ConfigMap", conditionType: "RuntimeConfigured", wantReason: "OMPConfigMapNotFound", revoke: func(ctx context.Context, c client.Client) error { + return c.Delete(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "omp-runtime-config", Namespace: "team"}}) + }}, + {name: "invalid OMP credential Secret", conditionType: "RuntimeConfigured", wantReason: "OMPCredentialSecretInvalid", revoke: func(ctx context.Context, c client.Client) error { + var secret corev1.Secret + if err := c.Get(ctx, types.NamespacedName{Namespace: "team", Name: "omp-runtime-credential"}, &secret); err != nil { + return err + } + delete(secret.Data, "MODEL_API_KEY") + return c.Update(ctx, &secret) + }}, + } { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.Status.PVCName = "workspace-a-data" + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, + Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } + session := testSession() + session.UID = "session-uid" + foreignPod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foreign-pod", Namespace: "team"}} + foreignService := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "foreign-service", Namespace: "team"}} + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(testHost(), workspace, pvc, session, foreignPod, foreignService).Build() + r := configuredSessionReconciler(c, scheme) + reconcileMany(t, 2, func() error { + _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + if err := test.revoke(ctx, c); err != nil { + t.Fatal(err) + } + + result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + if err != nil { + t.Fatal(err) + } + if result.RequeueAfter <= 0 || result.RequeueAfter > 30*time.Second { + t.Fatalf("revoked dependency requeue = %s, want bounded positive retry", result.RequeueAfter) + } + for _, object := range []client.Object{ + &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionPodName(session), Namespace: session.Namespace}}, + &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionServiceName(session), Namespace: session.Namespace}}, + } { + if err := c.Get(ctx, client.ObjectKeyFromObject(object), object); !apierrors.IsNotFound(err) { + t.Fatalf("owned stale %T remained after dependency revocation: %v", object, err) + } + } + for _, object := range []client.Object{foreignPod.DeepCopy(), foreignService.DeepCopy()} { + if err := c.Get(ctx, client.ObjectKeyFromObject(object), object); err != nil { + t.Fatalf("unowned %T was removed: %v", object, err) + } + } + + var failed clusterv1alpha1.T4Session + if err := c.Get(ctx, client.ObjectKeyFromObject(session), &failed); err != nil { + t.Fatal(err) + } + condition := findCondition(failed.Status.Conditions, test.conditionType) + available := findCondition(failed.Status.Conditions, "Available") + if failed.Status.ObservedGeneration != failed.Generation || failed.Status.PodName != "" || failed.Status.ServiceName != "" || failed.Status.Phase != clusterv1alpha1.InfrastructureFailed || + condition == nil || condition.Status != metav1.ConditionFalse || condition.Reason != test.wantReason || condition.ObservedGeneration != failed.Generation || + available == nil || available.Status != metav1.ConditionFalse || available.Reason != test.wantReason || available.ObservedGeneration != failed.Generation { + t.Fatalf("revoked session did not converge: status=%#v condition=%#v available=%#v", failed.Status, condition, available) + } + stableStatus := failed.Status + restarted := *r + reconcileMany(t, 2, func() error { + _, err := restarted.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + if err := c.Get(ctx, client.ObjectKeyFromObject(session), &failed); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(failed.Status, stableStatus) { + t.Fatalf("duplicate/restart reconciliation changed converged status: got %#v, want %#v", failed.Status, stableStatus) + } + }) + } +} + func configuredSessionReconciler(c client.Client, scheme *runtime.Scheme) *controllers.SessionReconciler { for _, object := range []client.Object{ &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "omp-runtime-config", Namespace: "team"}, Data: map[string]string{ diff --git a/packages/cluster-operator/controllers/session_controller_test.go b/packages/cluster-operator/controllers/session_controller_test.go new file mode 100644 index 00000000..cba15fef --- /dev/null +++ b/packages/cluster-operator/controllers/session_controller_test.go @@ -0,0 +1,62 @@ +package controllers + +import ( + "context" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + clusterv1alpha1 "github.com/LycaonLLC/t4-code/packages/cluster-operator/api/v1alpha1" +) + +func TestSessionRequestsForHostOnlyEnqueuesAffectedSessions(t *testing.T) { + scheme := runtime.NewScheme() + if err := clusterv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + objects := []client.Object{ + &clusterv1alpha1.T4Session{ObjectMeta: metav1.ObjectMeta{Name: "session-a", Namespace: "team"}, Spec: clusterv1alpha1.T4SessionSpec{HostRef: "host-a", WorkspaceRef: "workspace-a"}}, + &clusterv1alpha1.T4Session{ObjectMeta: metav1.ObjectMeta{Name: "session-b", Namespace: "team"}, Spec: clusterv1alpha1.T4SessionSpec{HostRef: "host-a", WorkspaceRef: "workspace-b"}}, + &clusterv1alpha1.T4Session{ObjectMeta: metav1.ObjectMeta{Name: "other-host", Namespace: "team"}, Spec: clusterv1alpha1.T4SessionSpec{HostRef: "host-b", WorkspaceRef: "workspace-a"}}, + &clusterv1alpha1.T4Session{ObjectMeta: metav1.ObjectMeta{Name: "other-namespace", Namespace: "other"}, Spec: clusterv1alpha1.T4SessionSpec{HostRef: "host-a", WorkspaceRef: "workspace-a"}}, + } + c := fake.NewClientBuilder().WithScheme(scheme). + WithIndex(&clusterv1alpha1.T4Session{}, sessionHostRefIndexField, indexSessionByHostRef). + WithIndex(&clusterv1alpha1.T4Session{}, sessionWorkspaceRefIndexField, indexSessionByWorkspaceRef). + WithObjects(objects...).Build() + r := &SessionReconciler{Client: c, Scheme: scheme} + + requests := r.sessionRequestsForHost(context.Background(), &clusterv1alpha1.T4ClusterHost{ObjectMeta: metav1.ObjectMeta{Name: "host-a", Namespace: "team"}}) + assertRequestSet(t, requests, []types.NamespacedName{ + {Namespace: "team", Name: "session-a"}, + {Namespace: "team", Name: "session-b"}, + }) +} + +func TestSessionRequestsForWorkspaceOnlyEnqueuesAffectedSessions(t *testing.T) { + scheme := runtime.NewScheme() + if err := clusterv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + objects := []client.Object{ + &clusterv1alpha1.T4Session{ObjectMeta: metav1.ObjectMeta{Name: "session-a", Namespace: "team"}, Spec: clusterv1alpha1.T4SessionSpec{HostRef: "host-a", WorkspaceRef: "workspace-a"}}, + &clusterv1alpha1.T4Session{ObjectMeta: metav1.ObjectMeta{Name: "session-b", Namespace: "team"}, Spec: clusterv1alpha1.T4SessionSpec{HostRef: "host-b", WorkspaceRef: "workspace-a"}}, + &clusterv1alpha1.T4Session{ObjectMeta: metav1.ObjectMeta{Name: "other-workspace", Namespace: "team"}, Spec: clusterv1alpha1.T4SessionSpec{HostRef: "host-a", WorkspaceRef: "workspace-b"}}, + &clusterv1alpha1.T4Session{ObjectMeta: metav1.ObjectMeta{Name: "other-namespace", Namespace: "other"}, Spec: clusterv1alpha1.T4SessionSpec{HostRef: "host-a", WorkspaceRef: "workspace-a"}}, + } + c := fake.NewClientBuilder().WithScheme(scheme). + WithIndex(&clusterv1alpha1.T4Session{}, sessionHostRefIndexField, indexSessionByHostRef). + WithIndex(&clusterv1alpha1.T4Session{}, sessionWorkspaceRefIndexField, indexSessionByWorkspaceRef). + WithObjects(objects...).Build() + r := &SessionReconciler{Client: c, Scheme: scheme} + + requests := r.sessionRequestsForWorkspace(context.Background(), &clusterv1alpha1.T4Workspace{ObjectMeta: metav1.ObjectMeta{Name: "workspace-a", Namespace: "team"}}) + assertRequestSet(t, requests, []types.NamespacedName{ + {Namespace: "team", Name: "session-a"}, + {Namespace: "team", Name: "session-b"}, + }) +} diff --git a/packages/cluster-operator/controllers/workspace_controller_test.go b/packages/cluster-operator/controllers/workspace_controller_test.go index 412705d0..27d07759 100644 --- a/packages/cluster-operator/controllers/workspace_controller_test.go +++ b/packages/cluster-operator/controllers/workspace_controller_test.go @@ -9,6 +9,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -107,3 +108,42 @@ func TestWorkspaceRequestsForStorageClassOnlyEnqueuesAffectedWorkspaces(t *testi } } } + +func TestWorkspaceRequestsForHostOnlyEnqueuesAffectedWorkspaces(t *testing.T) { + scheme := runtime.NewScheme() + if err := clusterv1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + objects := []client.Object{ + &clusterv1alpha1.T4Workspace{ObjectMeta: metav1.ObjectMeta{Name: "workspace-a", Namespace: "team"}, Spec: clusterv1alpha1.T4WorkspaceSpec{HostRef: "host-a"}}, + &clusterv1alpha1.T4Workspace{ObjectMeta: metav1.ObjectMeta{Name: "workspace-b", Namespace: "team"}, Spec: clusterv1alpha1.T4WorkspaceSpec{HostRef: "host-a"}}, + &clusterv1alpha1.T4Workspace{ObjectMeta: metav1.ObjectMeta{Name: "other-host", Namespace: "team"}, Spec: clusterv1alpha1.T4WorkspaceSpec{HostRef: "host-b"}}, + &clusterv1alpha1.T4Workspace{ObjectMeta: metav1.ObjectMeta{Name: "other-namespace", Namespace: "other"}, Spec: clusterv1alpha1.T4WorkspaceSpec{HostRef: "host-a"}}, + } + c := fake.NewClientBuilder().WithScheme(scheme). + WithIndex(&clusterv1alpha1.T4Workspace{}, workspaceHostRefIndexField, indexWorkspaceByHostRef). + WithObjects(objects...).Build() + r := &WorkspaceReconciler{Client: c, Scheme: scheme} + + requests := r.workspaceRequestsForHost(context.Background(), &clusterv1alpha1.T4ClusterHost{ObjectMeta: metav1.ObjectMeta{Name: "host-a", Namespace: "team"}}) + assertRequestSet(t, requests, []types.NamespacedName{ + {Namespace: "team", Name: "workspace-a"}, + {Namespace: "team", Name: "workspace-b"}, + }) +} + +func assertRequestSet(t *testing.T, requests []ctrl.Request, want []types.NamespacedName) { + t.Helper() + got := make(map[types.NamespacedName]int, len(requests)) + for _, request := range requests { + got[request.NamespacedName]++ + } + if len(got) != len(want) { + t.Fatalf("requests = %#v, want exactly %v", requests, want) + } + for _, key := range want { + if got[key] != 1 { + t.Fatalf("requests = %#v, want %v exactly once", requests, key) + } + } +} From f91517b204644a6c1c1b8c8a15c0a92463c2284d Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 20:51:56 +0000 Subject: [PATCH 17/43] feat(operator): reconcile dependency changes --- .../controllers/session_controller.go | 77 +++++++++++++++++++ .../controllers/workspace_controller.go | 18 +++++ 2 files changed, 95 insertions(+) diff --git a/packages/cluster-operator/controllers/session_controller.go b/packages/cluster-operator/controllers/session_controller.go index 9336a3f4..a4cce1f2 100644 --- a/packages/cluster-operator/controllers/session_controller.go +++ b/packages/cluster-operator/controllers/session_controller.go @@ -25,6 +25,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" clusterv1alpha1 "github.com/LycaonLLC/t4-code/packages/cluster-operator/api/v1alpha1" ) @@ -36,6 +37,11 @@ const ( SessionReviewerTokenExpirationSeconds int64 = 3600 ) +const ( + sessionHostRefIndexField = "t4.session.spec.hostRef" + sessionWorkspaceRefIndexField = "t4.session.spec.workspaceRef" +) + var ( configMapKeyPattern = regexp.MustCompile(`^[-._A-Za-z0-9]+$`) runtimeImagePattern = regexp.MustCompile(`^(?:(?:[A-Za-z0-9](?:[A-Za-z0-9.-]*[A-Za-z0-9])?|\[[A-Fa-f0-9:]+\])(?::[0-9]+)?/)?[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*(?:/[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*)*@sha256:[a-f0-9]{64}$`) @@ -313,6 +319,9 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) var host clusterv1alpha1.T4ClusterHost if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: session.Spec.HostRef}, &host); err != nil { if apierrors.IsNotFound(err) { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "HostReady", "HostNotFound", "referenced T4ClusterHost does not exist") } return ctrl.Result{}, err @@ -342,24 +351,39 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) var workspace clusterv1alpha1.T4Workspace if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: session.Spec.WorkspaceRef}, &workspace); err != nil { if apierrors.IsNotFound(err) { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "WorkspaceNotFound", "referenced T4Workspace does not exist") } return ctrl.Result{}, err } if workspace.Spec.HostRef != session.Spec.HostRef { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "HostMismatch", "session and workspace must reference the same T4ClusterHost") } if workspace.Status.PVCName == "" { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "PVCNotDeclared", "workspace controller has not declared a PVC") } var pvc corev1.PersistentVolumeClaim if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: workspace.Status.PVCName}, &pvc); err != nil { if apierrors.IsNotFound(err) { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "PVCNotFound", "workspace PVC does not exist") } return ctrl.Result{}, err } if pvc.Status.Phase != corev1.ClaimBound || !pvcHasRWX(&pvc) { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "PVCNotBoundRWX", "workspace PVC must be Bound and ReadWriteMany before a session starts") } runtimeVersions, reason, message, err := r.loadOMPResourceVersions(ctx, session.Namespace) @@ -754,9 +778,62 @@ func labelsContain(actual, required map[string]string) bool { return true } +func indexSessionByHostRef(object client.Object) []string { + session, ok := object.(*clusterv1alpha1.T4Session) + if !ok || session.Spec.HostRef == "" { + return nil + } + return []string{session.Spec.HostRef} +} + +func indexSessionByWorkspaceRef(object client.Object) []string { + session, ok := object.(*clusterv1alpha1.T4Session) + if !ok || session.Spec.WorkspaceRef == "" { + return nil + } + return []string{session.Spec.WorkspaceRef} +} + +func (r *SessionReconciler) sessionRequestsForHost(ctx context.Context, object client.Object) []ctrl.Request { + host, ok := object.(*clusterv1alpha1.T4ClusterHost) + if !ok || host.Name == "" || host.Namespace == "" { + return nil + } + return r.sessionRequestsForReference(ctx, host.Namespace, sessionHostRefIndexField, host.Name, "clusterHost", client.ObjectKeyFromObject(host)) +} + +func (r *SessionReconciler) sessionRequestsForWorkspace(ctx context.Context, object client.Object) []ctrl.Request { + workspace, ok := object.(*clusterv1alpha1.T4Workspace) + if !ok || workspace.Name == "" || workspace.Namespace == "" { + return nil + } + return r.sessionRequestsForReference(ctx, workspace.Namespace, sessionWorkspaceRefIndexField, workspace.Name, "workspace", client.ObjectKeyFromObject(workspace)) +} + +func (r *SessionReconciler) sessionRequestsForReference(ctx context.Context, namespace, field, value, dependencyKind string, dependencyKey types.NamespacedName) []ctrl.Request { + var sessions clusterv1alpha1.T4SessionList + if err := r.List(ctx, &sessions, client.InNamespace(namespace), client.MatchingFields{field: value}); err != nil { + ctrl.LoggerFrom(ctx).Error(err, "unable to map dependency to sessions", dependencyKind, dependencyKey) + return nil + } + requests := make([]ctrl.Request, 0, len(sessions.Items)) + for i := range sessions.Items { + requests = append(requests, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(&sessions.Items[i])}) + } + return requests +} + func (r *SessionReconciler) SetupWithManager(manager ctrl.Manager) error { + if err := manager.GetFieldIndexer().IndexField(context.Background(), &clusterv1alpha1.T4Session{}, sessionHostRefIndexField, indexSessionByHostRef); err != nil { + return fmt.Errorf("index T4Session by host reference: %w", err) + } + if err := manager.GetFieldIndexer().IndexField(context.Background(), &clusterv1alpha1.T4Session{}, sessionWorkspaceRefIndexField, indexSessionByWorkspaceRef); err != nil { + return fmt.Errorf("index T4Session by workspace reference: %w", err) + } return ctrl.NewControllerManagedBy(manager). For(&clusterv1alpha1.T4Session{}). + Watches(&clusterv1alpha1.T4ClusterHost{}, handler.EnqueueRequestsFromMapFunc(r.sessionRequestsForHost)). + Watches(&clusterv1alpha1.T4Workspace{}, handler.EnqueueRequestsFromMapFunc(r.sessionRequestsForWorkspace)). Owns(&corev1.Pod{}). Owns(&corev1.Service{}). Complete(r) diff --git a/packages/cluster-operator/controllers/workspace_controller.go b/packages/cluster-operator/controllers/workspace_controller.go index fbc60f45..e96271a6 100644 --- a/packages/cluster-operator/controllers/workspace_controller.go +++ b/packages/cluster-operator/controllers/workspace_controller.go @@ -337,6 +337,23 @@ func (r *WorkspaceReconciler) workspaceRequestsForStorageClass(ctx context.Conte return requests } +func (r *WorkspaceReconciler) workspaceRequestsForHost(ctx context.Context, object client.Object) []ctrl.Request { + host, ok := object.(*clusterv1alpha1.T4ClusterHost) + if !ok || host.Name == "" || host.Namespace == "" { + return nil + } + var workspaces clusterv1alpha1.T4WorkspaceList + if err := r.List(ctx, &workspaces, client.InNamespace(host.Namespace), client.MatchingFields{workspaceHostRefIndexField: host.Name}); err != nil { + ctrl.LoggerFrom(ctx).Error(err, "unable to map cluster host to workspaces", "clusterHost", client.ObjectKeyFromObject(host)) + return nil + } + requests := make([]ctrl.Request, 0, len(workspaces.Items)) + for i := range workspaces.Items { + requests = append(requests, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(&workspaces.Items[i])}) + } + return requests +} + func (r *WorkspaceReconciler) SetupWithManager(manager ctrl.Manager) error { if err := manager.GetFieldIndexer().IndexField(context.Background(), &clusterv1alpha1.T4ClusterHost{}, hostStorageClassIndexField, indexHostByStorageClass); err != nil { return fmt.Errorf("index T4ClusterHost by StorageClass: %w", err) @@ -346,6 +363,7 @@ func (r *WorkspaceReconciler) SetupWithManager(manager ctrl.Manager) error { } return ctrl.NewControllerManagedBy(manager). For(&clusterv1alpha1.T4Workspace{}). + Watches(&clusterv1alpha1.T4ClusterHost{}, handler.EnqueueRequestsFromMapFunc(r.workspaceRequestsForHost)). Watches(&corev1.PersistentVolumeClaim{}, handler.EnqueueRequestsFromMapFunc(workspaceRequestsForPVC)). Watches(&storagev1.StorageClass{}, handler.EnqueueRequestsFromMapFunc(r.workspaceRequestsForStorageClass)). Complete(r) From 72c6a19401e2e6f6b76e4980cf65051edd0b8695 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 21:05:36 +0000 Subject: [PATCH 18/43] test(operator): require host policy convergence --- .../controllers/reconciler_test.go | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index 02b070a2..905e3896 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -1296,6 +1296,18 @@ func TestSessionDependencyRevocationCleansOwnedResourcesAndConvergesAfterRestart delete(secret.Data, "MODEL_API_KEY") return c.Update(ctx, &secret) }}, + {name: "mismatched Host storage class", conditionType: "WorkspaceReady", wantReason: "StorageClassMismatch", revoke: func(ctx context.Context, c client.Client) error { + otherClass := &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: "other-rwx", Annotations: map[string]string{clusterv1alpha1.RWXStorageClassAnnotation: string(corev1.ReadWriteMany)}}, Provisioner: "example.invalid/csi"} + if err := c.Create(ctx, otherClass); err != nil { + return err + } + var host clusterv1alpha1.T4ClusterHost + if err := c.Get(ctx, types.NamespacedName{Namespace: "team", Name: "host-a"}, &host); err != nil { + return err + } + host.Spec.StorageClassName = otherClass.Name + return c.Update(ctx, &host) + }}, } { t.Run(test.name, func(t *testing.T) { ctx := context.Background() @@ -1371,6 +1383,137 @@ func TestSessionDependencyRevocationCleansOwnedResourcesAndConvergesAfterRestart } } +func TestWorkspaceHostStorageClassDriftFailsClosedWithoutRecreatingPVC(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + workspace.Status.ObservedGeneration = workspace.Generation + workspace.Status.PVCName = controllers.WorkspacePVCName(workspace) + workspace.Status.Phase = clusterv1alpha1.InfrastructureReady + workspace.Status.Conditions = []metav1.Condition{ + {Type: "StorageReady", Status: metav1.ConditionTrue, Reason: controllers.ReasonStorageReady, ObservedGeneration: workspace.Generation}, + {Type: "Ready", Status: metav1.ConditionTrue, Reason: "PVCBound", ObservedGeneration: workspace.Generation}, + } + oldClass := "portable-rwx" + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: workspace.Status.PVCName, Namespace: workspace.Namespace, + Annotations: map[string]string{clusterv1alpha1.WorkspaceUIDAnnotation: string(workspace.UID)}, + OwnerReferences: []metav1.OwnerReference{{APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Workspace", Name: workspace.Name, UID: workspace.UID, Controller: ptr(true)}}, + }, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: &oldClass, AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } + host := testHost() + host.Spec.StorageClassName = "other-rwx" + otherClass := &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: "other-rwx", Annotations: map[string]string{clusterv1alpha1.RWXStorageClassAnnotation: string(corev1.ReadWriteMany)}}, Provisioner: "example.invalid/csi"} + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Workspace{}, &corev1.PersistentVolumeClaim{}). + WithObjects(host, otherClass, workspace, pvc).Build() + r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} + + result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}) + if err != nil { + t.Fatal(err) + } + if result.RequeueAfter <= 0 || result.RequeueAfter > 30*time.Second { + t.Fatalf("storage drift requeue = %s, want bounded positive retry", result.RequeueAfter) + } + var got clusterv1alpha1.T4Workspace + if err := c.Get(ctx, client.ObjectKeyFromObject(workspace), &got); err != nil { + t.Fatal(err) + } + storageReady := findCondition(got.Status.Conditions, "StorageReady") + ready := findCondition(got.Status.Conditions, "Ready") + if got.Status.Phase != clusterv1alpha1.InfrastructureFailed || storageReady == nil || storageReady.Status != metav1.ConditionFalse || storageReady.Reason != "StorageClassMismatch" || ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != "StorageClassMismatch" { + t.Fatalf("storage drift remained Ready: status=%#v StorageReady=%#v Ready=%#v", got.Status, storageReady, ready) + } + var retained corev1.PersistentVolumeClaim + if err := c.Get(ctx, client.ObjectKeyFromObject(pvc), &retained); err != nil { + t.Fatalf("storage drift removed the data PVC: %v", err) + } + if retained.Spec.StorageClassName == nil || *retained.Spec.StorageClassName != oldClass { + t.Fatalf("storage drift recreated or mutated PVC class: %#v", retained.Spec.StorageClassName) + } +} + +func TestHostDependencyRecoveryReplacesFalseConditions(t *testing.T) { + t.Run("Workspace", func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Workspace{}, &corev1.PersistentVolumeClaim{}). + WithObjects(workspace).Build() + r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + if err := c.Create(ctx, testHost()); err != nil { + t.Fatal(err) + } + if err := c.Create(ctx, rwxStorageClass()); err != nil { + t.Fatal(err) + } + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + var pvc corev1.PersistentVolumeClaim + if err := c.Get(ctx, types.NamespacedName{Namespace: workspace.Namespace, Name: controllers.WorkspacePVCName(workspace)}, &pvc); err != nil { + t.Fatal(err) + } + pvc.Status.Phase = corev1.ClaimBound + if err := c.Status().Update(ctx, &pvc); err != nil { + t.Fatal(err) + } + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + var recovered clusterv1alpha1.T4Workspace + if err := c.Get(ctx, client.ObjectKeyFromObject(workspace), &recovered); err != nil { + t.Fatal(err) + } + hostReady := findCondition(recovered.Status.Conditions, "HostReady") + if hostReady == nil || hostReady.Status != metav1.ConditionTrue || hostReady.ObservedGeneration != recovered.Generation { + t.Fatalf("recovered Workspace retained stale HostReady: %#v", hostReady) + } + }) + + t.Run("Session", func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.Status.PVCName = "workspace-a-data" + pvc := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: workspace.Namespace}, Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}} + session := testSession() + session.UID = "session-uid" + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(workspace, pvc, session).Build() + r := configuredSessionReconciler(c, scheme) + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + if err := c.Create(ctx, testHost()); err != nil { + t.Fatal(err) + } + reconcileMany(t, 2, func() error { + _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + var recovered clusterv1alpha1.T4Session + if err := c.Get(ctx, client.ObjectKeyFromObject(session), &recovered); err != nil { + t.Fatal(err) + } + hostReady := findCondition(recovered.Status.Conditions, "HostReady") + if hostReady == nil || hostReady.Status != metav1.ConditionTrue || hostReady.ObservedGeneration != recovered.Generation { + t.Fatalf("recovered Session retained stale HostReady: %#v", hostReady) + } + }) +} + func configuredSessionReconciler(c client.Client, scheme *runtime.Scheme) *controllers.SessionReconciler { for _, object := range []client.Object{ &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "omp-runtime-config", Namespace: "team"}, Data: map[string]string{ From bb851a8560a0eb380830c512f0a1709a96b57bab Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 21:10:01 +0000 Subject: [PATCH 19/43] fix(operator): fail closed on host storage drift --- .../cluster-operator/controllers/helpers.go | 1 + .../controllers/reconciler_test.go | 2 +- .../controllers/session_controller.go | 41 ++++++++++++------- .../controllers/workspace_controller.go | 6 +++ 4 files changed, 34 insertions(+), 16 deletions(-) diff --git a/packages/cluster-operator/controllers/helpers.go b/packages/cluster-operator/controllers/helpers.go index ec5681af..4c88e8c2 100644 --- a/packages/cluster-operator/controllers/helpers.go +++ b/packages/cluster-operator/controllers/helpers.go @@ -18,6 +18,7 @@ import ( const ( ReasonStorageClassNotFound = "StorageClassNotFound" ReasonStorageClassNotRWX = "StorageClassNotRWX" + ReasonStorageClassMismatch = "StorageClassMismatch" ReasonStorageReady = "StorageClassSupportsRWX" ) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index 905e3896..9fca96a1 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -1316,7 +1316,7 @@ func TestSessionDependencyRevocationCleansOwnedResourcesAndConvergesAfterRestart workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() diff --git a/packages/cluster-operator/controllers/session_controller.go b/packages/cluster-operator/controllers/session_controller.go index a4cce1f2..26e516e1 100644 --- a/packages/cluster-operator/controllers/session_controller.go +++ b/packages/cluster-operator/controllers/session_controller.go @@ -307,13 +307,13 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "RuntimeConfigured", reason, message) + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, false, "RuntimeConfigured", reason, message) } if reason, message := r.OMPConfig.validationFailure(); reason != "" { if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "RuntimeConfigured", reason, message) + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, false, "RuntimeConfigured", reason, message) } var host clusterv1alpha1.T4ClusterHost @@ -322,7 +322,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "HostReady", "HostNotFound", "referenced T4ClusterHost does not exist") + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, false, "HostReady", "HostNotFound", "referenced T4ClusterHost does not exist") } return ctrl.Result{}, err } @@ -330,7 +330,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "RuntimeConfigured", "RuntimeProfileNotAllowed", "runtime profile is not allowed by the referenced T4ClusterHost") + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "RuntimeConfigured", "RuntimeProfileNotAllowed", "runtime profile is not allowed by the referenced T4ClusterHost") } var storageClass storagev1.StorageClass if err := r.Get(ctx, types.NamespacedName{Name: host.Spec.StorageClassName}, &storageClass); err != nil { @@ -338,7 +338,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", ReasonStorageClassNotFound, fmt.Sprintf("StorageClass %q selected by the referenced T4ClusterHost does not exist", host.Spec.StorageClassName)) + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", ReasonStorageClassNotFound, fmt.Sprintf("StorageClass %q selected by the referenced T4ClusterHost does not exist", host.Spec.StorageClassName)) } return ctrl.Result{}, err } @@ -346,7 +346,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", ReasonStorageClassNotRWX, fmt.Sprintf("StorageClass %q selected by the referenced T4ClusterHost is not administrator-declared ReadWriteMany", host.Spec.StorageClassName)) + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", ReasonStorageClassNotRWX, fmt.Sprintf("StorageClass %q selected by the referenced T4ClusterHost is not administrator-declared ReadWriteMany", host.Spec.StorageClassName)) } var workspace clusterv1alpha1.T4Workspace if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: session.Spec.WorkspaceRef}, &workspace); err != nil { @@ -354,7 +354,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "WorkspaceNotFound", "referenced T4Workspace does not exist") + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", "WorkspaceNotFound", "referenced T4Workspace does not exist") } return ctrl.Result{}, err } @@ -362,13 +362,13 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "HostMismatch", "session and workspace must reference the same T4ClusterHost") + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", "HostMismatch", "session and workspace must reference the same T4ClusterHost") } if workspace.Status.PVCName == "" { if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "PVCNotDeclared", "workspace controller has not declared a PVC") + return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", "PVCNotDeclared", "workspace controller has not declared a PVC") } var pvc corev1.PersistentVolumeClaim if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: workspace.Status.PVCName}, &pvc); err != nil { @@ -376,15 +376,21 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "PVCNotFound", "workspace PVC does not exist") + return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", "PVCNotFound", "workspace PVC does not exist") } return ctrl.Result{}, err } + if pvc.Spec.StorageClassName != nil && *pvc.Spec.StorageClassName != host.Spec.StorageClassName { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", ReasonStorageClassMismatch, fmt.Sprintf("workspace PVC uses StorageClass %q instead of host-selected %q", *pvc.Spec.StorageClassName, host.Spec.StorageClassName)) + } if pvc.Status.Phase != corev1.ClaimBound || !pvcHasRWX(&pvc) { if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, "WorkspaceReady", "PVCNotBoundRWX", "workspace PVC must be Bound and ReadWriteMany before a session starts") + return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", "PVCNotBoundRWX", "workspace PVC must be Bound and ReadWriteMany before a session starts") } runtimeVersions, reason, message, err := r.loadOMPResourceVersions(ctx, session.Namespace) if err != nil { @@ -394,7 +400,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "RuntimeConfigured", reason, message) + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "RuntimeConfigured", reason, message) } serviceName := SessionServiceName(&session) @@ -424,7 +430,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) } else if err != nil { return ctrl.Result{}, err } else if !metav1.IsControlledBy(&service, &session) { - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "Available", "ServiceOwnershipConflict", "deterministic session Service is not controlled by this session") + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "Available", "ServiceOwnershipConflict", "deterministic session Service is not controlled by this session") } else if !serviceExposureIsInternal(&service) { if err := r.Delete(ctx, &service); err != nil && !apierrors.IsNotFound(err) { return ctrl.Result{}, err @@ -458,7 +464,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) } else if err != nil { return ctrl.Result{}, err } else if !metav1.IsControlledBy(&pod, &session) { - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, "Available", "PodOwnershipConflict", "deterministic session Pod is not controlled by this session") + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "Available", "PodOwnershipConflict", "deterministic session Pod is not controlled by this session") } else if !labelsContain(pod.Labels, desiredPod.Labels) { if pod.Labels == nil { pod.Labels = map[string]string{} @@ -490,6 +496,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) session.Status.ObservedGeneration = session.Generation session.Status.PodName = podName session.Status.ServiceName = serviceName + meta.SetStatusCondition(&session.Status.Conditions, condition("HostReady", metav1.ConditionTrue, "HostResolved", "referenced T4ClusterHost is available", session.Generation)) meta.SetStatusCondition(&session.Status.Conditions, condition("WorkspaceReady", metav1.ConditionTrue, "PVCBoundRWX", "workspace PVC is Bound and ReadWriteMany", session.Generation)) meta.SetStatusCondition(&session.Status.Conditions, condition("RuntimeConfigured", metav1.ConditionTrue, "OMPReferencesReady", "administrator-owned OMP runtime references are configured", session.Generation)) if podReady(&pod) { @@ -646,6 +653,7 @@ func (r *SessionReconciler) updateSessionPending(ctx context.Context, session *c original.Conditions = append([]metav1.Condition(nil), session.Status.Conditions...) } session.Status.ObservedGeneration = session.Generation + meta.SetStatusCondition(&session.Status.Conditions, condition("HostReady", metav1.ConditionTrue, "HostResolved", "referenced T4ClusterHost is available", session.Generation)) session.Status.PodName = podName session.Status.ServiceName = serviceName session.Status.Phase = clusterv1alpha1.InfrastructurePending @@ -736,11 +744,14 @@ func (r *SessionReconciler) deleteOwnedSessionResources(ctx context.Context, ses return nil } -func (r *SessionReconciler) updateSessionFailure(ctx context.Context, session *clusterv1alpha1.T4Session, conditionType, reason, message string) error { +func (r *SessionReconciler) updateSessionFailure(ctx context.Context, session *clusterv1alpha1.T4Session, hostReady bool, conditionType, reason, message string) error { original := session.Status if session.Status.Conditions != nil { original.Conditions = append([]metav1.Condition(nil), session.Status.Conditions...) } + if hostReady { + meta.SetStatusCondition(&session.Status.Conditions, condition("HostReady", metav1.ConditionTrue, "HostResolved", "referenced T4ClusterHost is available", session.Generation)) + } session.Status.ObservedGeneration = session.Generation session.Status.PodName = "" session.Status.ServiceName = "" diff --git a/packages/cluster-operator/controllers/workspace_controller.go b/packages/cluster-operator/controllers/workspace_controller.go index e96271a6..97d95819 100644 --- a/packages/cluster-operator/controllers/workspace_controller.go +++ b/packages/cluster-operator/controllers/workspace_controller.go @@ -105,6 +105,8 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, request ctrl.Reques return ctrl.Result{}, err } else if !workspaceOwnsPVC(&workspace, &pvc) { return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCOwnershipConflict", "deterministic workspace PVC does not belong to this workspace") + } else if pvc.Spec.StorageClassName != nil && *pvc.Spec.StorageClassName != storageClassName { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", ReasonStorageClassMismatch, fmt.Sprintf("workspace PVC uses StorageClass %q instead of host-selected %q; data-bearing PVCs are never recreated automatically", *pvc.Spec.StorageClassName, storageClassName)) } else if workspace.Spec.RetentionPolicy == clusterv1alpha1.RetentionPolicyRetain && metav1.IsControlledBy(&pvc, &workspace) { before := pvc.DeepCopy() pvc.OwnerReferences = removeWorkspaceOwnerReference(pvc.OwnerReferences, workspace.UID) @@ -126,6 +128,7 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, request ctrl.Reques workspace.Status.PVCPhase = pvc.Status.Phase capacity := pvc.Status.Capacity[corev1.ResourceStorage] workspace.Status.Capacity = capacity.DeepCopy() + meta.SetStatusCondition(&workspace.Status.Conditions, condition("HostReady", metav1.ConditionTrue, "HostResolved", "referenced T4ClusterHost is available", workspace.Generation)) meta.SetStatusCondition(&workspace.Status.Conditions, condition("StorageReady", metav1.ConditionTrue, ReasonStorageReady, "RWX StorageClass and workspace PVC are accepted", workspace.Generation)) switch pvc.Status.Phase { case corev1.ClaimBound: @@ -267,6 +270,9 @@ func (r *WorkspaceReconciler) updateWorkspaceFailure(ctx context.Context, worksp if workspace.Status.Conditions != nil { original.Conditions = append([]metav1.Condition(nil), workspace.Status.Conditions...) } + if conditionType != "HostReady" { + meta.SetStatusCondition(&workspace.Status.Conditions, condition("HostReady", metav1.ConditionTrue, "HostResolved", "referenced T4ClusterHost is available", workspace.Generation)) + } workspace.Status.ObservedGeneration = workspace.Generation workspace.Status.Phase = clusterv1alpha1.InfrastructureFailed meta.SetStatusCondition(&workspace.Status.Conditions, condition(conditionType, metav1.ConditionFalse, reason, message, workspace.Generation)) From 4ccae52d2063668a7c9a681f85b7b685b71d452f Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 21:20:38 +0000 Subject: [PATCH 20/43] fix(operator): require explicit workspace storage class --- .../cluster-operator/controllers/helpers.go | 7 + .../controllers/reconciler_test.go | 171 +++++++++++------- .../controllers/session_controller.go | 6 +- .../controllers/workspace_controller.go | 4 +- 4 files changed, 120 insertions(+), 68 deletions(-) diff --git a/packages/cluster-operator/controllers/helpers.go b/packages/cluster-operator/controllers/helpers.go index 4c88e8c2..8fef978c 100644 --- a/packages/cluster-operator/controllers/helpers.go +++ b/packages/cluster-operator/controllers/helpers.go @@ -83,6 +83,13 @@ func pvcHasRWX(pvc *corev1.PersistentVolumeClaim) bool { return false } +func pvcStorageClassName(pvc *corev1.PersistentVolumeClaim) string { + if pvc.Spec.StorageClassName == nil { + return "" + } + return *pvc.Spec.StorageClassName +} + func hasString(values []string, wanted string) bool { for _, value := range values { if value == wanted { diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index 9fca96a1..8342d571 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -153,7 +153,7 @@ func TestRetainDeletionOrphansPVCBeforeRemovingFinalizer(t *testing.T) { APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Workspace", Name: workspace.Name, UID: workspace.UID, Controller: ptr(true), }}, }, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, } c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Workspace{}).WithObjects(testHost(), rwxStorageClass(), workspace, pvc).Build() if err := c.Delete(context.Background(), workspace); err != nil { @@ -187,7 +187,7 @@ func TestWorkspaceDeletionWaitsForSessionResources(t *testing.T) { Name: controllers.WorkspacePVCName(workspace), Namespace: workspace.Namespace, OwnerReferences: []metav1.OwnerReference{{APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Workspace", Name: workspace.Name, UID: workspace.UID, Controller: ptr(true)}}, }, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, } session := testSession() session.Spec.WorkspaceRef = workspace.Name @@ -457,7 +457,7 @@ func TestSessionRuntimeImageMustBeImmutableDigest(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -528,7 +528,7 @@ func TestSessionAuthorityRevocationDeletesOwnedPodAndService(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -575,7 +575,7 @@ func TestSessionNamesProduceSafeRuntimeIdentities(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -620,7 +620,7 @@ func TestSessionWaitsForBoundRWXThenCreatesExactlyOnePodAndService(t *testing.T) workspace.Status.Phase = clusterv1alpha1.InfrastructurePending pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimPending}, } session := testSession() @@ -755,7 +755,7 @@ func TestSessionOMPModeOmitsCredentialSecretReference(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -791,7 +791,7 @@ func TestSessionRejectsUnownedDeterministicResources(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -826,7 +826,7 @@ func TestSessionRecreatesPodWhenImmutableDesiredStateChanges(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -881,7 +881,7 @@ func TestSessionPodHashIncludesEveryOMPReference(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -927,7 +927,7 @@ func TestSessionRecreatesPodWhenOMPResourceVersionChanges(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -970,7 +970,7 @@ func TestSessionRuntimeReferenceRevocationStopsAuthority(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -1024,7 +1024,7 @@ func TestSessionFailsClosedWhenOMPConfigMapIsMissing(t *testing.T) { scheme := testScheme(t) workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) workspace.Status.PVCName = "workspace-a-data" - pvc := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}} + pvc := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}} session := testSession() objects := []client.Object{testHost(), rwxStorageClass(), workspace, pvc, session} if test.configMap != nil { @@ -1054,7 +1054,7 @@ func TestSessionRecreatesExternallyExposedOwnedService(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -1102,7 +1102,7 @@ func TestSessionRestoresRequiredPodSelectorLabelsBeforeAvailability(t *testing.T workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -1308,6 +1308,14 @@ func TestSessionDependencyRevocationCleansOwnedResourcesAndConvergesAfterRestart host.Spec.StorageClassName = otherClass.Name return c.Update(ctx, &host) }}, + {name: "classless Workspace PVC", conditionType: "WorkspaceReady", wantReason: "StorageClassMismatch", revoke: func(ctx context.Context, c client.Client) error { + var pvc corev1.PersistentVolumeClaim + if err := c.Get(ctx, types.NamespacedName{Namespace: "team", Name: "workspace-a-data"}, &pvc); err != nil { + return err + } + pvc.Spec.StorageClassName = nil + return c.Update(ctx, &pvc) + }}, } { t.Run(test.name, func(t *testing.T) { ctx := context.Background() @@ -1384,57 +1392,67 @@ func TestSessionDependencyRevocationCleansOwnedResourcesAndConvergesAfterRestart } func TestWorkspaceHostStorageClassDriftFailsClosedWithoutRecreatingPVC(t *testing.T) { - ctx := context.Background() - scheme := testScheme(t) - workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) - workspace.UID = "workspace-uid" - workspace.Status.ObservedGeneration = workspace.Generation - workspace.Status.PVCName = controllers.WorkspacePVCName(workspace) - workspace.Status.Phase = clusterv1alpha1.InfrastructureReady - workspace.Status.Conditions = []metav1.Condition{ - {Type: "StorageReady", Status: metav1.ConditionTrue, Reason: controllers.ReasonStorageReady, ObservedGeneration: workspace.Generation}, - {Type: "Ready", Status: metav1.ConditionTrue, Reason: "PVCBound", ObservedGeneration: workspace.Generation}, - } oldClass := "portable-rwx" - pvc := &corev1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: workspace.Status.PVCName, Namespace: workspace.Namespace, - Annotations: map[string]string{clusterv1alpha1.WorkspaceUIDAnnotation: string(workspace.UID)}, - OwnerReferences: []metav1.OwnerReference{{APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Workspace", Name: workspace.Name, UID: workspace.UID, Controller: ptr(true)}}, - }, - Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: &oldClass, AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, - Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, - } - host := testHost() - host.Spec.StorageClassName = "other-rwx" - otherClass := &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: "other-rwx", Annotations: map[string]string{clusterv1alpha1.RWXStorageClassAnnotation: string(corev1.ReadWriteMany)}}, Provisioner: "example.invalid/csi"} - c := fake.NewClientBuilder().WithScheme(scheme). - WithStatusSubresource(&clusterv1alpha1.T4Workspace{}, &corev1.PersistentVolumeClaim{}). - WithObjects(host, otherClass, workspace, pvc).Build() - r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} + for _, test := range []struct { + name string + claimClass *string + }{ + {name: "different class", claimClass: &oldClass}, + {name: "class omitted"}, + } { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + workspace.Status.ObservedGeneration = workspace.Generation + workspace.Status.PVCName = controllers.WorkspacePVCName(workspace) + workspace.Status.Phase = clusterv1alpha1.InfrastructureReady + workspace.Status.Conditions = []metav1.Condition{ + {Type: "StorageReady", Status: metav1.ConditionTrue, Reason: controllers.ReasonStorageReady, ObservedGeneration: workspace.Generation}, + {Type: "Ready", Status: metav1.ConditionTrue, Reason: "PVCBound", ObservedGeneration: workspace.Generation}, + } + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: workspace.Status.PVCName, Namespace: workspace.Namespace, + Annotations: map[string]string{clusterv1alpha1.WorkspaceUIDAnnotation: string(workspace.UID)}, + OwnerReferences: []metav1.OwnerReference{{APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Workspace", Name: workspace.Name, UID: workspace.UID, Controller: ptr(true)}}, + }, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: test.claimClass, AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } + host := testHost() + host.Spec.StorageClassName = "other-rwx" + otherClass := &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: "other-rwx", Annotations: map[string]string{clusterv1alpha1.RWXStorageClassAnnotation: string(corev1.ReadWriteMany)}}, Provisioner: "example.invalid/csi"} + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Workspace{}, &corev1.PersistentVolumeClaim{}). + WithObjects(host, otherClass, workspace, pvc).Build() + r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} - result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}) - if err != nil { - t.Fatal(err) - } - if result.RequeueAfter <= 0 || result.RequeueAfter > 30*time.Second { - t.Fatalf("storage drift requeue = %s, want bounded positive retry", result.RequeueAfter) - } - var got clusterv1alpha1.T4Workspace - if err := c.Get(ctx, client.ObjectKeyFromObject(workspace), &got); err != nil { - t.Fatal(err) - } - storageReady := findCondition(got.Status.Conditions, "StorageReady") - ready := findCondition(got.Status.Conditions, "Ready") - if got.Status.Phase != clusterv1alpha1.InfrastructureFailed || storageReady == nil || storageReady.Status != metav1.ConditionFalse || storageReady.Reason != "StorageClassMismatch" || ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != "StorageClassMismatch" { - t.Fatalf("storage drift remained Ready: status=%#v StorageReady=%#v Ready=%#v", got.Status, storageReady, ready) - } - var retained corev1.PersistentVolumeClaim - if err := c.Get(ctx, client.ObjectKeyFromObject(pvc), &retained); err != nil { - t.Fatalf("storage drift removed the data PVC: %v", err) - } - if retained.Spec.StorageClassName == nil || *retained.Spec.StorageClassName != oldClass { - t.Fatalf("storage drift recreated or mutated PVC class: %#v", retained.Spec.StorageClassName) + result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}) + if err != nil { + t.Fatal(err) + } + if result.RequeueAfter <= 0 || result.RequeueAfter > 30*time.Second { + t.Fatalf("storage drift requeue = %s, want bounded positive retry", result.RequeueAfter) + } + var got clusterv1alpha1.T4Workspace + if err := c.Get(ctx, client.ObjectKeyFromObject(workspace), &got); err != nil { + t.Fatal(err) + } + storageReady := findCondition(got.Status.Conditions, "StorageReady") + ready := findCondition(got.Status.Conditions, "Ready") + if got.Status.Phase != clusterv1alpha1.InfrastructureFailed || storageReady == nil || storageReady.Status != metav1.ConditionFalse || storageReady.Reason != "StorageClassMismatch" || ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != "StorageClassMismatch" { + t.Fatalf("storage drift remained Ready: status=%#v StorageReady=%#v Ready=%#v", got.Status, storageReady, ready) + } + var retained corev1.PersistentVolumeClaim + if err := c.Get(ctx, client.ObjectKeyFromObject(pvc), &retained); err != nil { + t.Fatalf("storage drift removed the data PVC: %v", err) + } + if !reflect.DeepEqual(retained.Spec.StorageClassName, test.claimClass) { + t.Fatalf("storage drift recreated or mutated PVC class: got %#v, want %#v", retained.Spec.StorageClassName, test.claimClass) + } + }) } } @@ -1512,6 +1530,31 @@ func TestHostDependencyRecoveryReplacesFalseConditions(t *testing.T) { t.Fatalf("recovered Session retained stale HostReady: %#v", hostReady) } }) + + t.Run("static runtime failure", func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + session := testSession() + session.Status.ObservedGeneration = session.Generation + session.Status.Phase = clusterv1alpha1.InfrastructureFailed + session.Status.Conditions = []metav1.Condition{{Type: "HostReady", Status: metav1.ConditionFalse, Reason: "HostNotFound", ObservedGeneration: session.Generation}} + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}). + WithObjects(testHost(), session).Build() + r := configuredSessionReconciler(c, scheme) + r.RuntimeImage = "registry.example/session:latest" + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + var failed clusterv1alpha1.T4Session + if err := c.Get(ctx, client.ObjectKeyFromObject(session), &failed); err != nil { + t.Fatal(err) + } + hostReady := findCondition(failed.Status.Conditions, "HostReady") + if hostReady == nil || hostReady.Status != metav1.ConditionUnknown || hostReady.Reason != "NotEvaluated" || hostReady.ObservedGeneration != failed.Generation { + t.Fatalf("static runtime failure retained stale HostReady: %#v", hostReady) + } + }) } func configuredSessionReconciler(c client.Client, scheme *runtime.Scheme) *controllers.SessionReconciler { diff --git a/packages/cluster-operator/controllers/session_controller.go b/packages/cluster-operator/controllers/session_controller.go index 26e516e1..9b4318b8 100644 --- a/packages/cluster-operator/controllers/session_controller.go +++ b/packages/cluster-operator/controllers/session_controller.go @@ -380,11 +380,11 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) } return ctrl.Result{}, err } - if pvc.Spec.StorageClassName != nil && *pvc.Spec.StorageClassName != host.Spec.StorageClassName { + if pvcStorageClassName(&pvc) != host.Spec.StorageClassName { if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", ReasonStorageClassMismatch, fmt.Sprintf("workspace PVC uses StorageClass %q instead of host-selected %q", *pvc.Spec.StorageClassName, host.Spec.StorageClassName)) + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", ReasonStorageClassMismatch, fmt.Sprintf("workspace PVC uses StorageClass %q instead of host-selected %q", pvcStorageClassName(&pvc), host.Spec.StorageClassName)) } if pvc.Status.Phase != corev1.ClaimBound || !pvcHasRWX(&pvc) { if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { @@ -751,6 +751,8 @@ func (r *SessionReconciler) updateSessionFailure(ctx context.Context, session *c } if hostReady { meta.SetStatusCondition(&session.Status.Conditions, condition("HostReady", metav1.ConditionTrue, "HostResolved", "referenced T4ClusterHost is available", session.Generation)) + } else if conditionType != "HostReady" { + meta.SetStatusCondition(&session.Status.Conditions, condition("HostReady", metav1.ConditionUnknown, "NotEvaluated", "host dependency was not evaluated because static runtime configuration is invalid", session.Generation)) } session.Status.ObservedGeneration = session.Generation session.Status.PodName = "" diff --git a/packages/cluster-operator/controllers/workspace_controller.go b/packages/cluster-operator/controllers/workspace_controller.go index 97d95819..da1b555b 100644 --- a/packages/cluster-operator/controllers/workspace_controller.go +++ b/packages/cluster-operator/controllers/workspace_controller.go @@ -105,8 +105,8 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, request ctrl.Reques return ctrl.Result{}, err } else if !workspaceOwnsPVC(&workspace, &pvc) { return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCOwnershipConflict", "deterministic workspace PVC does not belong to this workspace") - } else if pvc.Spec.StorageClassName != nil && *pvc.Spec.StorageClassName != storageClassName { - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", ReasonStorageClassMismatch, fmt.Sprintf("workspace PVC uses StorageClass %q instead of host-selected %q; data-bearing PVCs are never recreated automatically", *pvc.Spec.StorageClassName, storageClassName)) + } else if pvcStorageClassName(&pvc) != storageClassName { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", ReasonStorageClassMismatch, fmt.Sprintf("workspace PVC uses StorageClass %q instead of host-selected %q; data-bearing PVCs are never recreated automatically", pvcStorageClassName(&pvc), storageClassName)) } else if workspace.Spec.RetentionPolicy == clusterv1alpha1.RetentionPolicyRetain && metav1.IsControlledBy(&pvc, &workspace) { before := pvc.DeepCopy() pvc.OwnerReferences = removeWorkspaceOwnerReference(pvc.OwnerReferences, workspace.UID) From 30752e273db257440e150aa51df2b81b13739783 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 21:40:01 +0000 Subject: [PATCH 21/43] test(operator): reject stale dependency authority --- .../controllers/reconciler_test.go | 299 +++++++++++++++++- 1 file changed, 297 insertions(+), 2 deletions(-) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index 8342d571..3f51cba1 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -1144,7 +1144,7 @@ func TestSessionRestoresRequiredPodSelectorLabelsBeforeAvailability(t *testing.T func TestWorkspaceDeletionRefusesForeignDeterministicPVC(t *testing.T) { for _, policy := range []clusterv1alpha1.RetentionPolicy{clusterv1alpha1.RetentionPolicyRetain, clusterv1alpha1.RetentionPolicyDelete} { - for _, mismatch := range []string{"uid-annotation", "controller-owner"} { + for _, mismatch := range []string{"uid-annotation", "controller-owner", "foreign-non-controller-owner"} { t.Run(string(policy)+"/"+mismatch, func(t *testing.T) { scheme := testScheme(t) workspace := testWorkspace(policy) @@ -1156,10 +1156,14 @@ func TestWorkspaceDeletionRefusesForeignDeterministicPVC(t *testing.T) { }} if mismatch == "uid-annotation" { pvc.Annotations[clusterv1alpha1.WorkspaceUIDAnnotation] = "foreign-workspace-uid" - } else { + } else if mismatch == "controller-owner" { pvc.OwnerReferences = []metav1.OwnerReference{{ APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Workspace", Name: "foreign", UID: "foreign-workspace-uid", Controller: ptr(true), }} + } else { + pvc.OwnerReferences = []metav1.OwnerReference{{ + APIVersion: "example.test/v1", Kind: "Foreign", Name: "foreign", UID: "foreign-uid", + }} } expectedOwnerCount := len(pvc.OwnerReferences) c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Workspace{}).WithObjects(workspace, pvc).Build() @@ -1557,6 +1561,297 @@ func TestHostDependencyRecoveryReplacesFalseConditions(t *testing.T) { }) } +func TestWorkspaceFailureRevokesPublishedPVCAuthorityAndRefreshesConditions(t *testing.T) { + for _, test := range []struct { + name string + objects func(*clusterv1alpha1.T4Workspace) []client.Object + wantHostStatus metav1.ConditionStatus + wantStorageReason string + }{ + { + name: "missing Host", + objects: func(workspace *clusterv1alpha1.T4Workspace) []client.Object { + return []client.Object{workspace} + }, + wantHostStatus: metav1.ConditionFalse, + wantStorageReason: "NotEvaluated", + }, + { + name: "missing StorageClass", + objects: func(workspace *clusterv1alpha1.T4Workspace) []client.Object { + return []client.Object{testHost(), workspace} + }, + wantHostStatus: metav1.ConditionTrue, + wantStorageReason: controllers.ReasonStorageClassNotFound, + }, + { + name: "non-RWX StorageClass", + objects: func(workspace *clusterv1alpha1.T4Workspace) []client.Object { + class := rwxStorageClass() + class.Annotations = nil + return []client.Object{testHost(), class, workspace} + }, + wantHostStatus: metav1.ConditionTrue, + wantStorageReason: controllers.ReasonStorageClassNotRWX, + }, + { + name: "PVC class drift", + objects: func(workspace *clusterv1alpha1.T4Workspace) []client.Object { + pvc := ownedWorkspacePVC(workspace) + pvc.Spec.StorageClassName = ptr("old-rwx") + return []client.Object{testHost(), rwxStorageClass(), workspace, pvc} + }, + wantHostStatus: metav1.ConditionTrue, + wantStorageReason: controllers.ReasonStorageClassMismatch, + }, + { + name: "PVC ownership conflict", + objects: func(workspace *clusterv1alpha1.T4Workspace) []client.Object { + pvc := ownedWorkspacePVC(workspace) + pvc.OwnerReferences = []metav1.OwnerReference{{APIVersion: "example.test/v1", Kind: "Foreign", Name: "foreign", UID: "foreign-uid", Controller: ptr(true)}} + return []client.Object{testHost(), rwxStorageClass(), workspace, pvc} + }, + wantHostStatus: metav1.ConditionTrue, + wantStorageReason: "PVCOwnershipConflict", + }, + } { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + workspace.Generation = 7 + workspace.Status.ObservedGeneration = 6 + workspace.Status.PVCName = controllers.WorkspacePVCName(workspace) + workspace.Status.PVCPhase = corev1.ClaimBound + workspace.Status.Capacity = apiresource.MustParse("10Gi") + workspace.Status.Phase = clusterv1alpha1.InfrastructureReady + workspace.Status.Conditions = []metav1.Condition{ + {Type: "HostReady", Status: metav1.ConditionTrue, Reason: "Stale", ObservedGeneration: 6}, + {Type: "StorageReady", Status: metav1.ConditionTrue, Reason: "Stale", ObservedGeneration: 6}, + {Type: "Ready", Status: metav1.ConditionTrue, Reason: "Stale", ObservedGeneration: 6}, + } + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Workspace{}, &corev1.PersistentVolumeClaim{}).WithObjects(test.objects(workspace)...).Build() + r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + var failed clusterv1alpha1.T4Workspace + if err := c.Get(ctx, client.ObjectKeyFromObject(workspace), &failed); err != nil { + t.Fatal(err) + } + if failed.Status.PVCName != "" || failed.Status.PVCPhase != "" || !failed.Status.Capacity.IsZero() { + t.Fatalf("failed Workspace retained PVC authority: %#v", failed.Status) + } + hostReady := findCondition(failed.Status.Conditions, "HostReady") + storageReady := findCondition(failed.Status.Conditions, "StorageReady") + ready := findCondition(failed.Status.Conditions, "Ready") + if hostReady == nil || hostReady.Status != test.wantHostStatus || hostReady.ObservedGeneration != failed.Generation || + storageReady == nil || storageReady.Status == metav1.ConditionTrue || storageReady.Reason != test.wantStorageReason || storageReady.ObservedGeneration != failed.Generation || + ready == nil || ready.Status != metav1.ConditionFalse || ready.ObservedGeneration != failed.Generation { + t.Fatalf("failure conditions are stale: HostReady=%#v StorageReady=%#v Ready=%#v", hostReady, storageReady, ready) + } + }) + } +} + +func TestSessionRejectsWorkspacePVCWithoutExactIdentityAndOwnership(t *testing.T) { + for _, test := range []struct { + name string + mutatePVC func(*clusterv1alpha1.T4Workspace, *corev1.PersistentVolumeClaim) + }{ + {name: "foreign deterministic PVC", mutatePVC: func(_ *clusterv1alpha1.T4Workspace, pvc *corev1.PersistentVolumeClaim) { + pvc.OwnerReferences = []metav1.OwnerReference{{APIVersion: "example.test/v1", Kind: "Foreign", Name: "foreign", UID: "foreign-uid", Controller: ptr(true)}} + }}, + {name: "tampered Workspace status name", mutatePVC: func(workspace *clusterv1alpha1.T4Workspace, pvc *corev1.PersistentVolumeClaim) { + workspace.Status.PVCName = "foreign-data" + pvc.Name = workspace.Status.PVCName + }}, + } { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + workspace.Status.PVCName = controllers.WorkspacePVCName(workspace) + pvc := ownedWorkspacePVC(workspace) + test.mutatePVC(workspace, pvc) + session := testSession() + session.UID = "session-uid" + pod, service := ownedSessionResources(session) + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(testHost(), workspace, pvc, session, pod, service).Build() + r := configuredSessionReconciler(c, scheme) + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + assertObjectCounts(t, c, 0, 0) + var failed clusterv1alpha1.T4Session + if err := c.Get(ctx, client.ObjectKeyFromObject(session), &failed); err != nil { + t.Fatal(err) + } + condition := findCondition(failed.Status.Conditions, "WorkspaceReady") + if failed.Status.PodName != "" || failed.Status.ServiceName != "" || condition == nil || condition.Status != metav1.ConditionFalse || condition.ObservedGeneration != failed.Generation { + t.Fatalf("untrusted Workspace PVC retained session authority: status=%#v WorkspaceReady=%#v", failed.Status, condition) + } + }) + } +} + +func TestSessionFailureAndPendingRefreshEveryCondition(t *testing.T) { + t.Run("failure", func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + session := testSession() + session.Generation = 8 + session.Status.ObservedGeneration = 7 + session.Status.Conditions = staleSessionConditions(7) + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Session{}).WithObjects(session).Build() + r := configuredSessionReconciler(c, scheme) + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + var failed clusterv1alpha1.T4Session + if err := c.Get(ctx, client.ObjectKeyFromObject(session), &failed); err != nil { + t.Fatal(err) + } + assertCurrentSessionConditions(t, &failed, map[string]metav1.ConditionStatus{ + "HostReady": metav1.ConditionFalse, "WorkspaceReady": metav1.ConditionUnknown, "RuntimeConfigured": metav1.ConditionUnknown, "Available": metav1.ConditionFalse, + }) + }) + + t.Run("pending", func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + workspace.Status.PVCName = controllers.WorkspacePVCName(workspace) + pvc := ownedWorkspacePVC(workspace) + session := testSession() + session.UID = "session-uid" + session.Generation = 8 + session.Status.ObservedGeneration = 7 + session.Status.Conditions = staleSessionConditions(7) + _, service := ownedSessionResources(session) + service.Spec.Type = corev1.ServiceTypeNodePort + service.Spec.Ports = []corev1.ServicePort{{Name: "host", Port: 8787, NodePort: 32080}} + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(testHost(), workspace, pvc, session, service).Build() + r := configuredSessionReconciler(c, scheme) + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + var pending clusterv1alpha1.T4Session + if err := c.Get(ctx, client.ObjectKeyFromObject(session), &pending); err != nil { + t.Fatal(err) + } + assertCurrentSessionConditions(t, &pending, map[string]metav1.ConditionStatus{ + "HostReady": metav1.ConditionTrue, "WorkspaceReady": metav1.ConditionTrue, "RuntimeConfigured": metav1.ConditionTrue, "Available": metav1.ConditionFalse, + }) + }) +} + +func TestSessionResourcesWithAnyForeignOwnerFailClosed(t *testing.T) { + for _, path := range []string{"normal", "dependency-cleanup"} { + for _, kind := range []string{"Pod", "Service"} { + t.Run(path+"/"+kind, func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + session := testSession() + session.UID = "session-uid" + pod, service := ownedSessionResources(session) + foreignOwner := metav1.OwnerReference{APIVersion: "example.test/v1", Kind: "Foreign", Name: "foreign", UID: "foreign-uid"} + if kind == "Pod" { + pod.OwnerReferences = append(pod.OwnerReferences, foreignOwner) + } else { + service.OwnerReferences = append(service.OwnerReferences, foreignOwner) + } + objects := []client.Object{session, pod, service} + if path == "normal" { + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + workspace.Status.PVCName = controllers.WorkspacePVCName(workspace) + objects = append(objects, testHost(), workspace, ownedWorkspacePVC(workspace)) + } + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(objects...).Build() + r := configuredSessionReconciler(c, scheme) + beforePod := pod.DeepCopy() + beforeService := service.DeepCopy() + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + if kind == "Pod" { + var got corev1.Pod + if err := c.Get(ctx, client.ObjectKeyFromObject(pod), &got); err != nil { + t.Fatalf("foreign-owned Pod was deleted: %v", err) + } + if !reflect.DeepEqual(got.ObjectMeta, beforePod.ObjectMeta) || !reflect.DeepEqual(got.Spec, beforePod.Spec) { + t.Fatalf("Pod with foreign OwnerReference was mutated: %#v", got) + } + } else { + var got corev1.Service + if err := c.Get(ctx, client.ObjectKeyFromObject(service), &got); err != nil { + t.Fatalf("foreign-owned Service was deleted: %v", err) + } + if !reflect.DeepEqual(got.ObjectMeta, beforeService.ObjectMeta) || !reflect.DeepEqual(got.Spec, beforeService.Spec) { + t.Fatalf("Service with foreign OwnerReference was mutated: %#v", got) + } + } + var failed clusterv1alpha1.T4Session + if err := c.Get(ctx, client.ObjectKeyFromObject(session), &failed); err != nil { + t.Fatal(err) + } + available := findCondition(failed.Status.Conditions, "Available") + if available == nil || available.Status != metav1.ConditionFalse || !strings.Contains(available.Reason, "OwnershipConflict") { + t.Fatalf("foreign owner did not produce stable ownership conflict: %#v", available) + } + }) + } + } +} + +func ownedWorkspacePVC(workspace *clusterv1alpha1.T4Workspace) *corev1.PersistentVolumeClaim { + return &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: controllers.WorkspacePVCName(workspace), Namespace: workspace.Namespace, + Annotations: map[string]string{clusterv1alpha1.WorkspaceUIDAnnotation: string(workspace.UID)}, + OwnerReferences: []metav1.OwnerReference{{APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Workspace", Name: workspace.Name, UID: workspace.UID, Controller: ptr(true)}}, + }, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, + } +} + +func ownedSessionResources(session *clusterv1alpha1.T4Session) (*corev1.Pod, *corev1.Service) { + owner := metav1.OwnerReference{APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Session", Name: session.Name, UID: session.UID, Controller: ptr(true)} + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionPodName(session), Namespace: session.Namespace, OwnerReferences: []metav1.OwnerReference{owner}}} + service := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionServiceName(session), Namespace: session.Namespace, OwnerReferences: []metav1.OwnerReference{owner}}} + return pod, service +} + +func staleSessionConditions(generation int64) []metav1.Condition { + return []metav1.Condition{ + {Type: "HostReady", Status: metav1.ConditionTrue, Reason: "Stale", ObservedGeneration: generation}, + {Type: "WorkspaceReady", Status: metav1.ConditionTrue, Reason: "Stale", ObservedGeneration: generation}, + {Type: "RuntimeConfigured", Status: metav1.ConditionTrue, Reason: "Stale", ObservedGeneration: generation}, + {Type: "Available", Status: metav1.ConditionTrue, Reason: "Stale", ObservedGeneration: generation}, + } +} + +func assertCurrentSessionConditions(t *testing.T, session *clusterv1alpha1.T4Session, want map[string]metav1.ConditionStatus) { + t.Helper() + for conditionType, status := range want { + condition := findCondition(session.Status.Conditions, conditionType) + if condition == nil || condition.Status != status || condition.ObservedGeneration != session.Generation { + t.Fatalf("%s = %#v, want %s at generation %d", conditionType, condition, status, session.Generation) + } + } +} + func configuredSessionReconciler(c client.Client, scheme *runtime.Scheme) *controllers.SessionReconciler { for _, object := range []client.Object{ &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "omp-runtime-config", Namespace: "team"}, Data: map[string]string{ From 95da911ad8f262814c515292d147d05360165b50 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 21:47:16 +0000 Subject: [PATCH 22/43] fix(operator): revoke untrusted dependency authority --- .../controllers/reconciler_test.go | 2 +- .../controllers/session_controller.go | 106 +++++++++++++----- .../controllers/workspace_controller.go | 22 +++- 3 files changed, 95 insertions(+), 35 deletions(-) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index 3f51cba1..564e4211 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -1829,7 +1829,7 @@ func ownedWorkspacePVC(workspace *clusterv1alpha1.T4Workspace) *corev1.Persisten func ownedSessionResources(session *clusterv1alpha1.T4Session) (*corev1.Pod, *corev1.Service) { owner := metav1.OwnerReference{APIVersion: clusterv1alpha1.GroupVersion.String(), Kind: "T4Session", Name: session.Name, UID: session.UID, Controller: ptr(true)} pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionPodName(session), Namespace: session.Namespace, OwnerReferences: []metav1.OwnerReference{owner}}} - service := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionServiceName(session), Namespace: session.Namespace, OwnerReferences: []metav1.OwnerReference{owner}}} + service := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionServiceName(session), Namespace: session.Namespace, OwnerReferences: []metav1.OwnerReference{owner}}, Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeClusterIP}} return pod, service } diff --git a/packages/cluster-operator/controllers/session_controller.go b/packages/cluster-operator/controllers/session_controller.go index 9b4318b8..df4c492c 100644 --- a/packages/cluster-operator/controllers/session_controller.go +++ b/packages/cluster-operator/controllers/session_controller.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "encoding/json" + "errors" "fmt" "net/url" "reflect" @@ -42,6 +43,8 @@ const ( sessionWorkspaceRefIndexField = "t4.session.spec.workspaceRef" ) +var errSessionResourceOwnershipConflict = errors.New("session resource ownership conflict") + var ( configMapKeyPattern = regexp.MustCompile(`^[-._A-Za-z0-9]+$`) runtimeImagePattern = regexp.MustCompile(`^(?:(?:[A-Za-z0-9](?:[A-Za-z0-9.-]*[A-Za-z0-9])?|\[[A-Fa-f0-9:]+\])(?::[0-9]+)?/)?[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*(?:/[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*)*@sha256:[a-f0-9]{64}$`) @@ -289,6 +292,10 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) var session clusterv1alpha1.T4Session found := false defer func() { + if errors.Is(err, errSessionResourceOwnershipConflict) { + result = ctrl.Result{RequeueAfter: 30 * time.Second} + err = nil + } observeReconcile(metricKindSession, request.NamespacedName, session.Status.Conditions, conditionObjectPresent(&session, found, err), err) }() if err := r.Get(ctx, request.NamespacedName, &session); err != nil { @@ -307,13 +314,13 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, false, "RuntimeConfigured", reason, message) + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, false, false, "RuntimeConfigured", reason, message) } if reason, message := r.OMPConfig.validationFailure(); reason != "" { if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, false, "RuntimeConfigured", reason, message) + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, false, false, "RuntimeConfigured", reason, message) } var host clusterv1alpha1.T4ClusterHost @@ -322,7 +329,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, false, "HostReady", "HostNotFound", "referenced T4ClusterHost does not exist") + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, false, false, "HostReady", "HostNotFound", "referenced T4ClusterHost does not exist") } return ctrl.Result{}, err } @@ -330,7 +337,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "RuntimeConfigured", "RuntimeProfileNotAllowed", "runtime profile is not allowed by the referenced T4ClusterHost") + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "RuntimeConfigured", "RuntimeProfileNotAllowed", "runtime profile is not allowed by the referenced T4ClusterHost") } var storageClass storagev1.StorageClass if err := r.Get(ctx, types.NamespacedName{Name: host.Spec.StorageClassName}, &storageClass); err != nil { @@ -338,7 +345,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", ReasonStorageClassNotFound, fmt.Sprintf("StorageClass %q selected by the referenced T4ClusterHost does not exist", host.Spec.StorageClassName)) + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", ReasonStorageClassNotFound, fmt.Sprintf("StorageClass %q selected by the referenced T4ClusterHost does not exist", host.Spec.StorageClassName)) } return ctrl.Result{}, err } @@ -346,7 +353,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", ReasonStorageClassNotRWX, fmt.Sprintf("StorageClass %q selected by the referenced T4ClusterHost is not administrator-declared ReadWriteMany", host.Spec.StorageClassName)) + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", ReasonStorageClassNotRWX, fmt.Sprintf("StorageClass %q selected by the referenced T4ClusterHost is not administrator-declared ReadWriteMany", host.Spec.StorageClassName)) } var workspace clusterv1alpha1.T4Workspace if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: session.Spec.WorkspaceRef}, &workspace); err != nil { @@ -354,7 +361,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", "WorkspaceNotFound", "referenced T4Workspace does not exist") + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", "WorkspaceNotFound", "referenced T4Workspace does not exist") } return ctrl.Result{}, err } @@ -362,13 +369,19 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", "HostMismatch", "session and workspace must reference the same T4ClusterHost") + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", "HostMismatch", "session and workspace must reference the same T4ClusterHost") } if workspace.Status.PVCName == "" { if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", "PVCNotDeclared", "workspace controller has not declared a PVC") + return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", "PVCNotDeclared", "workspace controller has not declared a PVC") + } + if workspace.UID != "" && workspace.Status.PVCName != WorkspacePVCName(&workspace) { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", "PVCIdentityMismatch", "workspace status does not reference its deterministic PVC") } var pvc corev1.PersistentVolumeClaim if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: workspace.Status.PVCName}, &pvc); err != nil { @@ -376,21 +389,27 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", "PVCNotFound", "workspace PVC does not exist") + return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", "PVCNotFound", "workspace PVC does not exist") } return ctrl.Result{}, err } + if workspace.UID != "" && !workspaceOwnsPVC(&workspace, &pvc) { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", "PVCOwnershipConflict", "workspace PVC identity or ownership is not authoritative") + } if pvcStorageClassName(&pvc) != host.Spec.StorageClassName { if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", ReasonStorageClassMismatch, fmt.Sprintf("workspace PVC uses StorageClass %q instead of host-selected %q", pvcStorageClassName(&pvc), host.Spec.StorageClassName)) + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", ReasonStorageClassMismatch, fmt.Sprintf("workspace PVC uses StorageClass %q instead of host-selected %q", pvcStorageClassName(&pvc), host.Spec.StorageClassName)) } if pvc.Status.Phase != corev1.ClaimBound || !pvcHasRWX(&pvc) { if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, true, "WorkspaceReady", "PVCNotBoundRWX", "workspace PVC must be Bound and ReadWriteMany before a session starts") + return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", "PVCNotBoundRWX", "workspace PVC must be Bound and ReadWriteMany before a session starts") } runtimeVersions, reason, message, err := r.loadOMPResourceVersions(ctx, session.Namespace) if err != nil { @@ -400,7 +419,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { return ctrl.Result{}, err } - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "RuntimeConfigured", reason, message) + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, true, "RuntimeConfigured", reason, message) } serviceName := SessionServiceName(&session) @@ -429,8 +448,8 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) } } else if err != nil { return ctrl.Result{}, err - } else if !metav1.IsControlledBy(&service, &session) { - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "Available", "ServiceOwnershipConflict", "deterministic session Service is not controlled by this session") + } else if !sessionExclusivelyOwnsResource(&service, &session) { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, true, "Available", "ServiceOwnershipConflict", "deterministic session Service has an unexpected owner") } else if !serviceExposureIsInternal(&service) { if err := r.Delete(ctx, &service); err != nil && !apierrors.IsNotFound(err) { return ctrl.Result{}, err @@ -463,8 +482,8 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) } } else if err != nil { return ctrl.Result{}, err - } else if !metav1.IsControlledBy(&pod, &session) { - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, "Available", "PodOwnershipConflict", "deterministic session Pod is not controlled by this session") + } else if !sessionExclusivelyOwnsResource(&pod, &session) { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, true, "Available", "PodOwnershipConflict", "deterministic session Pod has an unexpected owner") } else if !labelsContain(pod.Labels, desiredPod.Labels) { if pod.Labels == nil { pod.Labels = map[string]string{} @@ -654,6 +673,8 @@ func (r *SessionReconciler) updateSessionPending(ctx context.Context, session *c } session.Status.ObservedGeneration = session.Generation meta.SetStatusCondition(&session.Status.Conditions, condition("HostReady", metav1.ConditionTrue, "HostResolved", "referenced T4ClusterHost is available", session.Generation)) + meta.SetStatusCondition(&session.Status.Conditions, condition("WorkspaceReady", metav1.ConditionTrue, "PVCBoundRWX", "workspace PVC is Bound and ReadWriteMany", session.Generation)) + meta.SetStatusCondition(&session.Status.Conditions, condition("RuntimeConfigured", metav1.ConditionTrue, "OMPReferencesReady", "administrator-owned OMP runtime references are configured", session.Generation)) session.Status.PodName = podName session.Status.ServiceName = serviceName session.Status.Phase = clusterv1alpha1.InfrastructurePending @@ -693,7 +714,7 @@ func (r *SessionReconciler) reconcileDelete(ctx context.Context, session *cluste if err != nil { return ctrl.Result{}, err } - if !metav1.IsControlledBy(object, session) { + if !sessionExclusivelyOwnsResource(object, session) { before := session.Status if session.Status.Conditions != nil { before.Conditions = append([]metav1.Condition(nil), session.Status.Conditions...) @@ -734,8 +755,11 @@ func (r *SessionReconciler) deleteOwnedSessionResources(ctx context.Context, ses } continue } - if !metav1.IsControlledBy(object, session) { - continue + if !sessionExclusivelyOwnsResource(object, session) { + if err := r.updateSessionFailure(ctx, session, false, false, "Available", "ResourceOwnershipConflict", fmt.Sprintf("deterministic %T has an unexpected owner", object)); err != nil { + return err + } + return errSessionResourceOwnershipConflict } if err := r.Delete(ctx, object); err != nil && !apierrors.IsNotFound(err) { return err @@ -744,30 +768,56 @@ func (r *SessionReconciler) deleteOwnedSessionResources(ctx context.Context, ses return nil } -func (r *SessionReconciler) updateSessionFailure(ctx context.Context, session *clusterv1alpha1.T4Session, hostReady bool, conditionType, reason, message string) error { +func (r *SessionReconciler) updateSessionFailure(ctx context.Context, session *clusterv1alpha1.T4Session, hostReady, workspaceReady bool, conditionType, reason, message string) error { original := session.Status if session.Status.Conditions != nil { original.Conditions = append([]metav1.Condition(nil), session.Status.Conditions...) } - if hostReady { + if conditionType == "HostReady" { + meta.SetStatusCondition(&session.Status.Conditions, condition("HostReady", metav1.ConditionFalse, reason, message, session.Generation)) + } else if hostReady { meta.SetStatusCondition(&session.Status.Conditions, condition("HostReady", metav1.ConditionTrue, "HostResolved", "referenced T4ClusterHost is available", session.Generation)) - } else if conditionType != "HostReady" { - meta.SetStatusCondition(&session.Status.Conditions, condition("HostReady", metav1.ConditionUnknown, "NotEvaluated", "host dependency was not evaluated because static runtime configuration is invalid", session.Generation)) + } else { + meta.SetStatusCondition(&session.Status.Conditions, condition("HostReady", metav1.ConditionUnknown, "NotEvaluated", "host dependency was not evaluated", session.Generation)) + } + if conditionType == "WorkspaceReady" { + meta.SetStatusCondition(&session.Status.Conditions, condition("WorkspaceReady", metav1.ConditionFalse, reason, message, session.Generation)) + } else if workspaceReady { + meta.SetStatusCondition(&session.Status.Conditions, condition("WorkspaceReady", metav1.ConditionTrue, "PVCBoundRWX", "workspace PVC is Bound and ReadWriteMany", session.Generation)) + } else { + meta.SetStatusCondition(&session.Status.Conditions, condition("WorkspaceReady", metav1.ConditionUnknown, "NotEvaluated", "workspace dependency was not evaluated", session.Generation)) + } + if conditionType == "RuntimeConfigured" { + meta.SetStatusCondition(&session.Status.Conditions, condition("RuntimeConfigured", metav1.ConditionFalse, reason, message, session.Generation)) + } else if workspaceReady { + meta.SetStatusCondition(&session.Status.Conditions, condition("RuntimeConfigured", metav1.ConditionTrue, "OMPReferencesReady", "administrator-owned OMP runtime references are configured", session.Generation)) + } else { + meta.SetStatusCondition(&session.Status.Conditions, condition("RuntimeConfigured", metav1.ConditionUnknown, "NotEvaluated", "runtime configuration was not evaluated", session.Generation)) } session.Status.ObservedGeneration = session.Generation session.Status.PodName = "" session.Status.ServiceName = "" session.Status.Phase = clusterv1alpha1.InfrastructureFailed - meta.SetStatusCondition(&session.Status.Conditions, condition(conditionType, metav1.ConditionFalse, reason, message, session.Generation)) - if conditionType != "Available" { - meta.SetStatusCondition(&session.Status.Conditions, condition("Available", metav1.ConditionFalse, reason, message, session.Generation)) - } + meta.SetStatusCondition(&session.Status.Conditions, condition("Available", metav1.ConditionFalse, reason, message, session.Generation)) if reflect.DeepEqual(original, session.Status) { return nil } return r.Status().Update(ctx, session) } +func sessionExclusivelyOwnsResource(object metav1.Object, session *clusterv1alpha1.T4Session) bool { + controller := metav1.GetControllerOf(object) + if controller == nil || controller.APIVersion != clusterv1alpha1.GroupVersion.String() || controller.Kind != "T4Session" || controller.Name != session.Name || controller.UID != session.UID { + return false + } + for _, reference := range object.GetOwnerReferences() { + if reference.APIVersion != clusterv1alpha1.GroupVersion.String() || reference.Kind != "T4Session" || reference.Name != session.Name || reference.UID != session.UID { + return false + } + } + return true +} + func serviceExposureIsInternal(service *corev1.Service) bool { if service.Spec.Type != corev1.ServiceTypeClusterIP || service.Spec.ClusterIP == corev1.ClusterIPNone || service.Spec.ExternalName != "" || len(service.Spec.ExternalIPs) != 0 || service.Spec.LoadBalancerIP != "" || len(service.Spec.LoadBalancerSourceRanges) != 0 || diff --git a/packages/cluster-operator/controllers/workspace_controller.go b/packages/cluster-operator/controllers/workspace_controller.go index da1b555b..65e47608 100644 --- a/packages/cluster-operator/controllers/workspace_controller.go +++ b/packages/cluster-operator/controllers/workspace_controller.go @@ -10,6 +10,7 @@ import ( storagev1 "k8s.io/api/storage/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" meta "k8s.io/apimachinery/pkg/api/meta" + apiresource "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -161,6 +162,11 @@ func workspaceOwnsPVC(workspace *clusterv1alpha1.T4Workspace, pvc *corev1.Persis if pvc.Annotations[clusterv1alpha1.WorkspaceUIDAnnotation] != string(workspace.UID) { return false } + for _, reference := range pvc.OwnerReferences { + if reference.APIVersion != clusterv1alpha1.GroupVersion.String() || reference.Kind != "T4Workspace" || reference.Name != workspace.Name || reference.UID != workspace.UID { + return false + } + } controller := metav1.GetControllerOf(pvc) if workspace.Spec.RetentionPolicy == clusterv1alpha1.RetentionPolicyDelete { return controller != nil && controller.UID == workspace.UID @@ -270,15 +276,19 @@ func (r *WorkspaceReconciler) updateWorkspaceFailure(ctx context.Context, worksp if workspace.Status.Conditions != nil { original.Conditions = append([]metav1.Condition(nil), workspace.Status.Conditions...) } - if conditionType != "HostReady" { - meta.SetStatusCondition(&workspace.Status.Conditions, condition("HostReady", metav1.ConditionTrue, "HostResolved", "referenced T4ClusterHost is available", workspace.Generation)) - } workspace.Status.ObservedGeneration = workspace.Generation + workspace.Status.PVCName = "" + workspace.Status.PVCPhase = "" + workspace.Status.Capacity = apiresource.Quantity{} workspace.Status.Phase = clusterv1alpha1.InfrastructureFailed - meta.SetStatusCondition(&workspace.Status.Conditions, condition(conditionType, metav1.ConditionFalse, reason, message, workspace.Generation)) - if conditionType != "Ready" { - meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionFalse, reason, message, workspace.Generation)) + if conditionType == "HostReady" { + meta.SetStatusCondition(&workspace.Status.Conditions, condition("HostReady", metav1.ConditionFalse, reason, message, workspace.Generation)) + meta.SetStatusCondition(&workspace.Status.Conditions, condition("StorageReady", metav1.ConditionUnknown, "NotEvaluated", "storage dependency was not evaluated because the referenced host is unavailable", workspace.Generation)) + } else { + meta.SetStatusCondition(&workspace.Status.Conditions, condition("HostReady", metav1.ConditionTrue, "HostResolved", "referenced T4ClusterHost is available", workspace.Generation)) + meta.SetStatusCondition(&workspace.Status.Conditions, condition("StorageReady", metav1.ConditionFalse, reason, message, workspace.Generation)) } + meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionFalse, reason, message, workspace.Generation)) if reflect.DeepEqual(original, workspace.Status) { return nil } From d954667c18f6600baf813909adf656b44892f2fb Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 21:58:23 +0000 Subject: [PATCH 23/43] test(operator): revoke terminal storage authority --- .../controllers/reconciler_test.go | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index 564e4211..8d48f6da 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -1801,6 +1801,17 @@ func TestSessionResourcesWithAnyForeignOwnerFailClosed(t *testing.T) { t.Fatalf("Service with foreign OwnerReference was mutated: %#v", got) } } + if path == "dependency-cleanup" { + var sibling client.Object + if kind == "Pod" { + sibling = &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionServiceName(session), Namespace: session.Namespace}} + } else { + sibling = &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionPodName(session), Namespace: session.Namespace}} + } + if err := c.Get(ctx, client.ObjectKeyFromObject(sibling), sibling); !apierrors.IsNotFound(err) { + t.Fatalf("exclusively owned sibling remained after dependency cleanup: %v", err) + } + } var failed clusterv1alpha1.T4Session if err := c.Get(ctx, client.ObjectKeyFromObject(session), &failed); err != nil { t.Fatal(err) @@ -1814,6 +1825,45 @@ func TestSessionResourcesWithAnyForeignOwnerFailClosed(t *testing.T) { } } +func TestWorkspaceTerminalPVCFailuresRevokePublishedAuthority(t *testing.T) { + for _, test := range []struct { + name string + mutate func(*corev1.PersistentVolumeClaim) + reason string + }{ + {name: "Bound without RWX", reason: "PVCNotRWX", mutate: func(pvc *corev1.PersistentVolumeClaim) { pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} }}, + {name: "Lost", reason: "PVCLost", mutate: func(pvc *corev1.PersistentVolumeClaim) { pvc.Status.Phase = corev1.ClaimLost }}, + } { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + pvc := ownedWorkspacePVC(workspace) + test.mutate(pvc) + c := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Workspace{}, &corev1.PersistentVolumeClaim{}). + WithObjects(testHost(), rwxStorageClass(), workspace, pvc).Build() + r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + var failed clusterv1alpha1.T4Workspace + if err := c.Get(ctx, client.ObjectKeyFromObject(workspace), &failed); err != nil { + t.Fatal(err) + } + storageReady := findCondition(failed.Status.Conditions, "StorageReady") + ready := findCondition(failed.Status.Conditions, "Ready") + if failed.Status.PVCName != "" || failed.Status.PVCPhase != "" || !failed.Status.Capacity.IsZero() || + storageReady == nil || storageReady.Status != metav1.ConditionFalse || storageReady.Reason != test.reason || storageReady.ObservedGeneration != failed.Generation || + ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != test.reason || ready.ObservedGeneration != failed.Generation { + t.Fatalf("terminal PVC failure retained authority: status=%#v StorageReady=%#v Ready=%#v", failed.Status, storageReady, ready) + } + }) + } +} + + func ownedWorkspacePVC(workspace *clusterv1alpha1.T4Workspace) *corev1.PersistentVolumeClaim { return &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ From 187baf2649105f3bd29cf4df37468bbe894b8674 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 22:01:56 +0000 Subject: [PATCH 24/43] fix(operator): clear terminal storage authority --- .../controllers/session_controller.go | 13 +++++++++---- .../controllers/workspace_controller.go | 18 ++++++++---------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/packages/cluster-operator/controllers/session_controller.go b/packages/cluster-operator/controllers/session_controller.go index df4c492c..7a837e43 100644 --- a/packages/cluster-operator/controllers/session_controller.go +++ b/packages/cluster-operator/controllers/session_controller.go @@ -748,6 +748,7 @@ func (r *SessionReconciler) deleteOwnedSessionResources(ctx context.Context, ses &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: SessionPodName(session), Namespace: session.Namespace}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: SessionServiceName(session), Namespace: session.Namespace}}, } + ownershipConflict := false for _, object := range objects { if err := r.Get(ctx, client.ObjectKeyFromObject(object), object); err != nil { if err := client.IgnoreNotFound(err); err != nil { @@ -756,15 +757,19 @@ func (r *SessionReconciler) deleteOwnedSessionResources(ctx context.Context, ses continue } if !sessionExclusivelyOwnsResource(object, session) { - if err := r.updateSessionFailure(ctx, session, false, false, "Available", "ResourceOwnershipConflict", fmt.Sprintf("deterministic %T has an unexpected owner", object)); err != nil { - return err - } - return errSessionResourceOwnershipConflict + ownershipConflict = true + continue } if err := r.Delete(ctx, object); err != nil && !apierrors.IsNotFound(err) { return err } } + if ownershipConflict { + if err := r.updateSessionFailure(ctx, session, false, false, "Available", "ResourceOwnershipConflict", "one or more deterministic session resources have an unexpected owner"); err != nil { + return err + } + return errSessionResourceOwnershipConflict + } return nil } diff --git a/packages/cluster-operator/controllers/workspace_controller.go b/packages/cluster-operator/controllers/workspace_controller.go index 65e47608..d32920f8 100644 --- a/packages/cluster-operator/controllers/workspace_controller.go +++ b/packages/cluster-operator/controllers/workspace_controller.go @@ -118,6 +118,12 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, request ctrl.Reques return ctrl.Result{Requeue: true}, nil } } + if pvc.Status.Phase == corev1.ClaimBound && !pvcHasRWX(&pvc) { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCNotRWX", "bound workspace PVC does not request ReadWriteMany") + } + if pvc.Status.Phase == corev1.ClaimLost { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCLost", "workspace PVC lost its volume") + } original := workspace.Status original.Capacity = workspace.Status.Capacity.DeepCopy() @@ -133,16 +139,8 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, request ctrl.Reques meta.SetStatusCondition(&workspace.Status.Conditions, condition("StorageReady", metav1.ConditionTrue, ReasonStorageReady, "RWX StorageClass and workspace PVC are accepted", workspace.Generation)) switch pvc.Status.Phase { case corev1.ClaimBound: - if !pvcHasRWX(&pvc) { - workspace.Status.Phase = clusterv1alpha1.InfrastructureFailed - meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionFalse, "PVCNotRWX", "bound workspace PVC does not request ReadWriteMany", workspace.Generation)) - } else { - workspace.Status.Phase = clusterv1alpha1.InfrastructureReady - meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionTrue, "PVCBound", "workspace PVC is bound with ReadWriteMany access", workspace.Generation)) - } - case corev1.ClaimLost: - workspace.Status.Phase = clusterv1alpha1.InfrastructureFailed - meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionFalse, "PVCLost", "workspace PVC lost its volume", workspace.Generation)) + workspace.Status.Phase = clusterv1alpha1.InfrastructureReady + meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionTrue, "PVCBound", "workspace PVC is bound with ReadWriteMany access", workspace.Generation)) default: workspace.Status.Phase = clusterv1alpha1.InfrastructurePending meta.SetStatusCondition(&workspace.Status.Conditions, condition("Ready", metav1.ConditionFalse, "PVCBinding", "workspace PVC is waiting to bind", workspace.Generation)) From 75762dafcaf7725fad155ba4898c49c58e788872 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 22:10:25 +0000 Subject: [PATCH 25/43] test(operator): stop owned siblings on conflict --- .../controllers/reconciler_test.go | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index 8d48f6da..a1533f3f 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -1801,16 +1801,14 @@ func TestSessionResourcesWithAnyForeignOwnerFailClosed(t *testing.T) { t.Fatalf("Service with foreign OwnerReference was mutated: %#v", got) } } - if path == "dependency-cleanup" { - var sibling client.Object - if kind == "Pod" { - sibling = &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionServiceName(session), Namespace: session.Namespace}} - } else { - sibling = &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionPodName(session), Namespace: session.Namespace}} - } - if err := c.Get(ctx, client.ObjectKeyFromObject(sibling), sibling); !apierrors.IsNotFound(err) { - t.Fatalf("exclusively owned sibling remained after dependency cleanup: %v", err) - } + var sibling client.Object + if kind == "Pod" { + sibling = &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionServiceName(session), Namespace: session.Namespace}} + } else { + sibling = &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: controllers.SessionPodName(session), Namespace: session.Namespace}} + } + if err := c.Get(ctx, client.ObjectKeyFromObject(sibling), sibling); !apierrors.IsNotFound(err) { + t.Fatalf("exclusively owned sibling remained after ownership conflict: %v", err) } var failed clusterv1alpha1.T4Session if err := c.Get(ctx, client.ObjectKeyFromObject(session), &failed); err != nil { From 55539e5f10c21f3dcc14ea84340ee18fabff63a0 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 22:14:32 +0000 Subject: [PATCH 26/43] fix(operator): stop owned siblings on conflict --- .../cluster-operator/controllers/session_controller.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/cluster-operator/controllers/session_controller.go b/packages/cluster-operator/controllers/session_controller.go index 7a837e43..c7834cea 100644 --- a/packages/cluster-operator/controllers/session_controller.go +++ b/packages/cluster-operator/controllers/session_controller.go @@ -449,7 +449,10 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) } else if err != nil { return ctrl.Result{}, err } else if !sessionExclusivelyOwnsResource(&service, &session) { - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, true, "Available", "ServiceOwnershipConflict", "deterministic session Service has an unexpected owner") + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } else if !serviceExposureIsInternal(&service) { if err := r.Delete(ctx, &service); err != nil && !apierrors.IsNotFound(err) { return ctrl.Result{}, err @@ -483,7 +486,10 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) } else if err != nil { return ctrl.Result{}, err } else if !sessionExclusivelyOwnsResource(&pod, &session) { - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, true, "Available", "PodOwnershipConflict", "deterministic session Pod has an unexpected owner") + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } else if !labelsContain(pod.Labels, desiredPod.Labels) { if pod.Labels == nil { pod.Labels = map[string]string{} From 764f46094951d40d84f8389ad83070609fe4d5e7 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 22:26:47 +0000 Subject: [PATCH 27/43] test(operator): preserve dependency status on conflicts --- packages/cluster-operator/controllers/reconciler_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index a1533f3f..b92fd46b 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -1818,6 +1818,11 @@ func TestSessionResourcesWithAnyForeignOwnerFailClosed(t *testing.T) { if available == nil || available.Status != metav1.ConditionFalse || !strings.Contains(available.Reason, "OwnershipConflict") { t.Fatalf("foreign owner did not produce stable ownership conflict: %#v", available) } + if path == "normal" { + assertCurrentSessionConditions(t, &failed, map[string]metav1.ConditionStatus{ + "HostReady": metav1.ConditionTrue, "WorkspaceReady": metav1.ConditionTrue, "RuntimeConfigured": metav1.ConditionTrue, "Available": metav1.ConditionFalse, + }) + } }) } } From e24b6dcefc8ba92be906a4a26a47545b52c26fab Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 22:34:18 +0000 Subject: [PATCH 28/43] fix(operator): preserve ready dependencies on conflicts --- .../controllers/session_controller.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/cluster-operator/controllers/session_controller.go b/packages/cluster-operator/controllers/session_controller.go index c7834cea..1d52a20b 100644 --- a/packages/cluster-operator/controllers/session_controller.go +++ b/packages/cluster-operator/controllers/session_controller.go @@ -449,7 +449,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) } else if err != nil { return ctrl.Result{}, err } else if !sessionExclusivelyOwnsResource(&service, &session) { - if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + if err := r.deleteOwnedSessionResourcesAfterVerifiedDependencies(ctx, &session, "ServiceOwnershipConflict", "deterministic session Service has an unexpected owner"); err != nil { return ctrl.Result{}, err } return ctrl.Result{RequeueAfter: 30 * time.Second}, nil @@ -486,7 +486,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) } else if err != nil { return ctrl.Result{}, err } else if !sessionExclusivelyOwnsResource(&pod, &session) { - if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + if err := r.deleteOwnedSessionResourcesAfterVerifiedDependencies(ctx, &session, "PodOwnershipConflict", "deterministic session Pod has an unexpected owner"); err != nil { return ctrl.Result{}, err } return ctrl.Result{RequeueAfter: 30 * time.Second}, nil @@ -750,6 +750,14 @@ func (r *SessionReconciler) reconcileDelete(ctx context.Context, session *cluste } func (r *SessionReconciler) deleteOwnedSessionResources(ctx context.Context, session *clusterv1alpha1.T4Session) error { + return r.deleteOwnedSessionResourcesWithFailure(ctx, session, false, false, "ResourceOwnershipConflict", "one or more deterministic session resources have an unexpected owner") +} + +func (r *SessionReconciler) deleteOwnedSessionResourcesAfterVerifiedDependencies(ctx context.Context, session *clusterv1alpha1.T4Session, reason, message string) error { + return r.deleteOwnedSessionResourcesWithFailure(ctx, session, true, true, reason, message) +} + +func (r *SessionReconciler) deleteOwnedSessionResourcesWithFailure(ctx context.Context, session *clusterv1alpha1.T4Session, hostReady, workspaceReady bool, reason, message string) error { objects := []client.Object{ &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: SessionPodName(session), Namespace: session.Namespace}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: SessionServiceName(session), Namespace: session.Namespace}}, @@ -771,7 +779,7 @@ func (r *SessionReconciler) deleteOwnedSessionResources(ctx context.Context, ses } } if ownershipConflict { - if err := r.updateSessionFailure(ctx, session, false, false, "Available", "ResourceOwnershipConflict", "one or more deterministic session resources have an unexpected owner"); err != nil { + if err := r.updateSessionFailure(ctx, session, hostReady, workspaceReady, "Available", reason, message); err != nil { return err } return errSessionResourceOwnershipConflict From 96ccd96fb89c6c85076e02476c213c85bc4b89ff Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 22:41:51 +0000 Subject: [PATCH 29/43] test(operator): cover cleanup and create races --- .../controllers/reconciler_test.go | 119 +++++++++++++++++- 1 file changed, 118 insertions(+), 1 deletion(-) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index b92fd46b..2b0fe1a4 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -13,6 +13,7 @@ import ( apiresource "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" utilvalidation "k8s.io/apimachinery/pkg/util/validation" ctrl "sigs.k8s.io/controller-runtime" @@ -1212,6 +1213,8 @@ func TestSessionDeletionRefusesForeignDeterministicResources(t *testing.T) { } else { service.OwnerReferences = nil } + beforePod := pod.DeepCopy() + beforeService := service.DeepCopy() c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Session{}).WithObjects(session, pod, service).Build() if err := c.Delete(context.Background(), session); err != nil { t.Fatal(err) @@ -1220,7 +1223,28 @@ func TestSessionDeletionRefusesForeignDeterministicResources(t *testing.T) { if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { t.Fatal(err) } - assertObjectCounts(t, c, 1, 1) + if foreignKind == "Pod" { + assertObjectCounts(t, c, 1, 0) + } else { + assertObjectCounts(t, c, 0, 1) + } + if foreignKind == "Pod" { + var got corev1.Pod + if err := c.Get(context.Background(), client.ObjectKeyFromObject(pod), &got); err != nil { + t.Fatalf("foreign Pod was deleted: %v", err) + } + if !reflect.DeepEqual(got.ObjectMeta, beforePod.ObjectMeta) || !reflect.DeepEqual(got.Spec, beforePod.Spec) { + t.Fatalf("foreign Pod was mutated during finalizer cleanup: %#v", got) + } + } else { + var got corev1.Service + if err := c.Get(context.Background(), client.ObjectKeyFromObject(service), &got); err != nil { + t.Fatalf("foreign Service was deleted: %v", err) + } + if !reflect.DeepEqual(got.ObjectMeta, beforeService.ObjectMeta) || !reflect.DeepEqual(got.Spec, beforeService.Spec) { + t.Fatalf("foreign Service was mutated during finalizer cleanup: %#v", got) + } + } var waiting clusterv1alpha1.T4Session if err := c.Get(context.Background(), client.ObjectKeyFromObject(session), &waiting); err != nil { t.Fatalf("session finalizer was removed on cleanup conflict: %v", err) @@ -1828,6 +1852,70 @@ func TestSessionResourcesWithAnyForeignOwnerFailClosed(t *testing.T) { } } +func TestSessionCreateAlreadyExistsRefetchesForeignWinner(t *testing.T) { + for _, kind := range []string{"Pod", "Service"} { + t.Run(kind, func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + workspace.Status.PVCName = controllers.WorkspacePVCName(workspace) + session := testSession() + session.UID = "session-uid" + pod, service := ownedSessionResources(session) + objects := []client.Object{testHost(), workspace, ownedWorkspacePVC(workspace), session} + if kind == "Pod" { + objects = append(objects, service) + } else { + objects = append(objects, pod) + } + base := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(objects...).Build() + c := &createAlreadyExistsClient{Client: base, raceKind: kind} + r := configuredSessionReconciler(c, scheme) + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + if c.winner == nil { + t.Fatalf("%s create race was not exercised", kind) + } + if kind == "Pod" { + var got corev1.Pod + if err := base.Get(ctx, client.ObjectKeyFromObject(c.winner), &got); err != nil { + t.Fatalf("foreign Pod winner was deleted: %v", err) + } + want := c.winner.(*corev1.Pod) + if !reflect.DeepEqual(got.ObjectMeta, want.ObjectMeta) || !reflect.DeepEqual(got.Spec, want.Spec) { + t.Fatalf("foreign Pod winner was mutated: %#v", got) + } + assertObjectCounts(t, base, 1, 0) + } else { + var got corev1.Service + if err := base.Get(ctx, client.ObjectKeyFromObject(c.winner), &got); err != nil { + t.Fatalf("foreign Service winner was deleted: %v", err) + } + want := c.winner.(*corev1.Service) + if !reflect.DeepEqual(got.ObjectMeta, want.ObjectMeta) || !reflect.DeepEqual(got.Spec, want.Spec) { + t.Fatalf("foreign Service winner was mutated: %#v", got) + } + assertObjectCounts(t, base, 0, 1) + } + var failed clusterv1alpha1.T4Session + if err := base.Get(ctx, client.ObjectKeyFromObject(session), &failed); err != nil { + t.Fatal(err) + } + assertCurrentSessionConditions(t, &failed, map[string]metav1.ConditionStatus{ + "HostReady": metav1.ConditionTrue, "WorkspaceReady": metav1.ConditionTrue, "RuntimeConfigured": metav1.ConditionTrue, "Available": metav1.ConditionFalse, + }) + available := findCondition(failed.Status.Conditions, "Available") + if available.Reason != kind+"OwnershipConflict" { + t.Fatalf("Available reason = %q, want %sOwnershipConflict", available.Reason, kind) + } + }) + } +} + func TestWorkspaceTerminalPVCFailuresRevokePublishedAuthority(t *testing.T) { for _, test := range []struct { name string @@ -2034,4 +2122,33 @@ func hasReadOnlyMount(mounts []corev1.VolumeMount, name, path string) bool { return false } +type createAlreadyExistsClient struct { + client.Client + raceKind string + winner client.Object +} + +func (c *createAlreadyExistsClient) Create(ctx context.Context, object client.Object, options ...client.CreateOption) error { + var winner client.Object + switch object := object.(type) { + case *corev1.Pod: + if c.raceKind == "Pod" { + winner = object.DeepCopy() + } + case *corev1.Service: + if c.raceKind == "Service" { + winner = object.DeepCopy() + } + } + if winner == nil || c.winner != nil { + return c.Client.Create(ctx, object, options...) + } + winner.SetOwnerReferences(nil) + if err := c.Client.Create(ctx, winner, options...); err != nil { + return err + } + c.winner = winner.DeepCopyObject().(client.Object) + return apierrors.NewAlreadyExists(schema.GroupResource{Resource: strings.ToLower(c.raceKind) + "s"}, object.GetName()) +} + func ptr[T any](value T) *T { return &value } From 2a8b7ae3e99c9499fddf035cc0f3a8ae6b0ff20d Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 22:44:52 +0000 Subject: [PATCH 30/43] fix(operator): close child ownership races --- .../controllers/session_controller.go | 65 ++++++++++++++----- 1 file changed, 47 insertions(+), 18 deletions(-) diff --git a/packages/cluster-operator/controllers/session_controller.go b/packages/cluster-operator/controllers/session_controller.go index 1d52a20b..95f076d0 100644 --- a/packages/cluster-operator/controllers/session_controller.go +++ b/packages/cluster-operator/controllers/session_controller.go @@ -441,14 +441,25 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) return ctrl.Result{}, err } var service corev1.Service - if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: serviceName}, &service); apierrors.IsNotFound(err) { + serviceKey := types.NamespacedName{Namespace: session.Namespace, Name: serviceName} + if err := r.Get(ctx, serviceKey, &service); apierrors.IsNotFound(err) { service = desiredService - if err := r.Create(ctx, &service); err != nil && !apierrors.IsAlreadyExists(err) { - return ctrl.Result{}, err + if err := r.Create(ctx, &service); err != nil { + if !apierrors.IsAlreadyExists(err) { + return ctrl.Result{}, err + } + reader := r.APIReader + if reader == nil { + reader = r.Client + } + if err := reader.Get(ctx, serviceKey, &service); err != nil { + return ctrl.Result{}, err + } } } else if err != nil { return ctrl.Result{}, err - } else if !sessionExclusivelyOwnsResource(&service, &session) { + } + if !sessionExclusivelyOwnsResource(&service, &session) { if err := r.deleteOwnedSessionResourcesAfterVerifiedDependencies(ctx, &session, "ServiceOwnershipConflict", "deterministic session Service has an unexpected owner"); err != nil { return ctrl.Result{}, err } @@ -478,14 +489,25 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) return ctrl.Result{}, err } var pod corev1.Pod - if err := r.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: podName}, &pod); apierrors.IsNotFound(err) { + podKey := types.NamespacedName{Namespace: session.Namespace, Name: podName} + if err := r.Get(ctx, podKey, &pod); apierrors.IsNotFound(err) { pod = desiredPod - if err := r.Create(ctx, &pod); err != nil && !apierrors.IsAlreadyExists(err) { - return ctrl.Result{}, err + if err := r.Create(ctx, &pod); err != nil { + if !apierrors.IsAlreadyExists(err) { + return ctrl.Result{}, err + } + reader := r.APIReader + if reader == nil { + reader = r.Client + } + if err := reader.Get(ctx, podKey, &pod); err != nil { + return ctrl.Result{}, err + } } } else if err != nil { return ctrl.Result{}, err - } else if !sessionExclusivelyOwnsResource(&pod, &session) { + } + if !sessionExclusivelyOwnsResource(&pod, &session) { if err := r.deleteOwnedSessionResourcesAfterVerifiedDependencies(ctx, &session, "PodOwnershipConflict", "deterministic session Pod has an unexpected owner"); err != nil { return ctrl.Result{}, err } @@ -712,6 +734,7 @@ func (r *SessionReconciler) reconcileDelete(ctx context.Context, session *cluste &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: SessionServiceName(session), Namespace: session.Namespace}}, } existing := make([]client.Object, 0, len(objects)) + var ownershipConflict client.Object for _, object := range objects { err := r.Get(ctx, client.ObjectKeyFromObject(object), object) if apierrors.IsNotFound(err) { @@ -721,17 +744,10 @@ func (r *SessionReconciler) reconcileDelete(ctx context.Context, session *cluste return ctrl.Result{}, err } if !sessionExclusivelyOwnsResource(object, session) { - before := session.Status - if session.Status.Conditions != nil { - before.Conditions = append([]metav1.Condition(nil), session.Status.Conditions...) - } - meta.SetStatusCondition(&session.Status.Conditions, condition("Available", metav1.ConditionFalse, "CleanupOwnershipConflict", fmt.Sprintf("deterministic %T is not controlled by this session", object), session.Generation)) - if !reflect.DeepEqual(before, session.Status) { - if err := r.Status().Update(ctx, session); err != nil { - return ctrl.Result{}, err - } + if ownershipConflict == nil { + ownershipConflict = object } - return ctrl.Result{RequeueAfter: 30 * time.Second}, nil + continue } existing = append(existing, object) } @@ -742,6 +758,19 @@ func (r *SessionReconciler) reconcileDelete(ctx context.Context, session *cluste } } } + if ownershipConflict != nil { + before := session.Status + if session.Status.Conditions != nil { + before.Conditions = append([]metav1.Condition(nil), session.Status.Conditions...) + } + meta.SetStatusCondition(&session.Status.Conditions, condition("Available", metav1.ConditionFalse, "CleanupOwnershipConflict", fmt.Sprintf("deterministic %T is not controlled by this session", ownershipConflict), session.Generation)) + if !reflect.DeepEqual(before, session.Status) { + if err := r.Status().Update(ctx, session); err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil + } if len(existing) > 0 { return ctrl.Result{RequeueAfter: time.Second}, nil } From e366c52605a253348ad0d41784adb868b0272aa8 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 22:51:48 +0000 Subject: [PATCH 31/43] test(operator): model stale cache after create race --- .../controllers/reconciler_test.go | 34 ++++++++++++++++--- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index 2b0fe1a4..79e7bb29 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -1213,9 +1213,15 @@ func TestSessionDeletionRefusesForeignDeterministicResources(t *testing.T) { } else { service.OwnerReferences = nil } - beforePod := pod.DeepCopy() - beforeService := service.DeepCopy() c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Session{}).WithObjects(session, pod, service).Build() + beforePod := &corev1.Pod{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(pod), beforePod); err != nil { + t.Fatal(err) + } + beforeService := &corev1.Service{} + if err := c.Get(context.Background(), client.ObjectKeyFromObject(service), beforeService); err != nil { + t.Fatal(err) + } if err := c.Delete(context.Background(), session); err != nil { t.Fatal(err) } @@ -1872,8 +1878,9 @@ func TestSessionCreateAlreadyExistsRefetchesForeignWinner(t *testing.T) { base := fake.NewClientBuilder().WithScheme(scheme). WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). WithObjects(objects...).Build() - c := &createAlreadyExistsClient{Client: base, raceKind: kind} + c := &createAlreadyExistsClient{Client: base, raceKind: kind, hideWinnerFromCache: true} r := configuredSessionReconciler(c, scheme) + r.APIReader = base if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { t.Fatal(err) } @@ -2124,8 +2131,25 @@ func hasReadOnlyMount(mounts []corev1.VolumeMount, name, path string) bool { type createAlreadyExistsClient struct { client.Client - raceKind string - winner client.Object + raceKind string + winner client.Object + hideWinnerFromCache bool +} + +func (c *createAlreadyExistsClient) Get(ctx context.Context, key client.ObjectKey, object client.Object, options ...client.GetOption) error { + if c.hideWinnerFromCache && c.winner != nil && key == client.ObjectKeyFromObject(c.winner) { + switch object.(type) { + case *corev1.Pod: + if c.raceKind == "Pod" { + return apierrors.NewNotFound(schema.GroupResource{Resource: "pods"}, key.Name) + } + case *corev1.Service: + if c.raceKind == "Service" { + return apierrors.NewNotFound(schema.GroupResource{Resource: "services"}, key.Name) + } + } + } + return c.Client.Get(ctx, key, object, options...) } func (c *createAlreadyExistsClient) Create(ctx context.Context, object client.Object, options ...client.CreateOption) error { From 4916c10a68036fc31f276a20203b5c4ed79c2266 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 22:54:45 +0000 Subject: [PATCH 32/43] test(operator): cover workspace PVC create race --- .../controllers/reconciler_test.go | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index 79e7bb29..3195e777 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -72,6 +72,40 @@ func TestWorkspaceReconcileIsIdempotentAcrossDuplicateEvents(t *testing.T) { } } +func TestWorkspaceCreateAlreadyExistsRefetchesForeignPVC(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + base := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Workspace{}, &corev1.PersistentVolumeClaim{}). + WithObjects(testHost(), rwxStorageClass(), workspace).Build() + c := &createAlreadyExistsClient{Client: base, raceKind: "PVC"} + r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + if c.winner == nil { + t.Fatal("PVC create race was not exercised") + } + var pvc corev1.PersistentVolumeClaim + if err := base.Get(ctx, client.ObjectKeyFromObject(c.winner), &pvc); err != nil { + t.Fatalf("foreign PVC winner was deleted: %v", err) + } + want := c.winner.(*corev1.PersistentVolumeClaim) + if !reflect.DeepEqual(pvc.ObjectMeta, want.ObjectMeta) || !reflect.DeepEqual(pvc.Spec, want.Spec) { + t.Fatalf("foreign PVC winner was mutated: %#v", pvc) + } + var failed clusterv1alpha1.T4Workspace + if err := base.Get(ctx, client.ObjectKeyFromObject(workspace), &failed); err != nil { + t.Fatal(err) + } + storageReady := findCondition(failed.Status.Conditions, "StorageReady") + if failed.Status.PVCName != "" || storageReady == nil || storageReady.Status != metav1.ConditionFalse || storageReady.Reason != "PVCOwnershipConflict" || storageReady.ObservedGeneration != failed.Generation { + t.Fatalf("foreign PVC winner was published as authoritative: status=%#v StorageReady=%#v", failed.Status, storageReady) + } +} + func TestRetainWorkspaceCreatesPVCWithoutGarbageCollectableOwner(t *testing.T) { scheme := testScheme(t) workspace := testWorkspace(clusterv1alpha1.RetentionPolicyRetain) @@ -2163,6 +2197,10 @@ func (c *createAlreadyExistsClient) Create(ctx context.Context, object client.Ob if c.raceKind == "Service" { winner = object.DeepCopy() } + case *corev1.PersistentVolumeClaim: + if c.raceKind == "PVC" { + winner = object.DeepCopy() + } } if winner == nil || c.winner != nil { return c.Client.Create(ctx, object, options...) From db95dd6c4bdc0b2cf64c0d9886181b2339e86e7b Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 22:57:58 +0000 Subject: [PATCH 33/43] fix(operator): validate authoritative create winners --- packages/cluster-operator/cmd/manager/main.go | 2 +- .../controllers/reconciler_test.go | 4 ++-- .../controllers/session_controller.go | 12 ++++++---- .../controllers/workspace_controller.go | 22 ++++++++++++++----- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/packages/cluster-operator/cmd/manager/main.go b/packages/cluster-operator/cmd/manager/main.go index d216db5a..f33af52c 100644 --- a/packages/cluster-operator/cmd/manager/main.go +++ b/packages/cluster-operator/cmd/manager/main.go @@ -70,7 +70,7 @@ func main() { ctrl.Log.Error(err, "unable to register T4ClusterHost controller") os.Exit(1) } - if err := (&controllers.WorkspaceReconciler{Client: manager.GetClient(), Scheme: manager.GetScheme()}).SetupWithManager(manager); err != nil { + if err := (&controllers.WorkspaceReconciler{Client: manager.GetClient(), APIReader: manager.GetAPIReader(), Scheme: manager.GetScheme()}).SetupWithManager(manager); err != nil { ctrl.Log.Error(err, "unable to register T4Workspace controller") os.Exit(1) } diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index 3195e777..e4f4f42d 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -80,8 +80,8 @@ func TestWorkspaceCreateAlreadyExistsRefetchesForeignPVC(t *testing.T) { base := fake.NewClientBuilder().WithScheme(scheme). WithStatusSubresource(&clusterv1alpha1.T4Workspace{}, &corev1.PersistentVolumeClaim{}). WithObjects(testHost(), rwxStorageClass(), workspace).Build() - c := &createAlreadyExistsClient{Client: base, raceKind: "PVC"} - r := &controllers.WorkspaceReconciler{Client: c, Scheme: scheme} + c := &createAlreadyExistsClient{Client: base, raceKind: "PVC", hideWinnerFromCache: true} + r := &controllers.WorkspaceReconciler{Client: c, APIReader: base, Scheme: scheme} if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { t.Fatal(err) } diff --git a/packages/cluster-operator/controllers/session_controller.go b/packages/cluster-operator/controllers/session_controller.go index 95f076d0..3f1bd28d 100644 --- a/packages/cluster-operator/controllers/session_controller.go +++ b/packages/cluster-operator/controllers/session_controller.go @@ -779,21 +779,25 @@ func (r *SessionReconciler) reconcileDelete(ctx context.Context, session *cluste } func (r *SessionReconciler) deleteOwnedSessionResources(ctx context.Context, session *clusterv1alpha1.T4Session) error { - return r.deleteOwnedSessionResourcesWithFailure(ctx, session, false, false, "ResourceOwnershipConflict", "one or more deterministic session resources have an unexpected owner") + return r.deleteOwnedSessionResourcesWithFailure(ctx, r.Client, session, false, false, "ResourceOwnershipConflict", "one or more deterministic session resources have an unexpected owner") } func (r *SessionReconciler) deleteOwnedSessionResourcesAfterVerifiedDependencies(ctx context.Context, session *clusterv1alpha1.T4Session, reason, message string) error { - return r.deleteOwnedSessionResourcesWithFailure(ctx, session, true, true, reason, message) + reader := r.APIReader + if reader == nil { + reader = r.Client + } + return r.deleteOwnedSessionResourcesWithFailure(ctx, reader, session, true, true, reason, message) } -func (r *SessionReconciler) deleteOwnedSessionResourcesWithFailure(ctx context.Context, session *clusterv1alpha1.T4Session, hostReady, workspaceReady bool, reason, message string) error { +func (r *SessionReconciler) deleteOwnedSessionResourcesWithFailure(ctx context.Context, reader client.Reader, session *clusterv1alpha1.T4Session, hostReady, workspaceReady bool, reason, message string) error { objects := []client.Object{ &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: SessionPodName(session), Namespace: session.Namespace}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: SessionServiceName(session), Namespace: session.Namespace}}, } ownershipConflict := false for _, object := range objects { - if err := r.Get(ctx, client.ObjectKeyFromObject(object), object); err != nil { + if err := reader.Get(ctx, client.ObjectKeyFromObject(object), object); err != nil { if err := client.IgnoreNotFound(err); err != nil { return err } diff --git a/packages/cluster-operator/controllers/workspace_controller.go b/packages/cluster-operator/controllers/workspace_controller.go index d32920f8..d6c2a54d 100644 --- a/packages/cluster-operator/controllers/workspace_controller.go +++ b/packages/cluster-operator/controllers/workspace_controller.go @@ -24,7 +24,8 @@ import ( type WorkspaceReconciler struct { client.Client - Scheme *runtime.Scheme + APIReader client.Reader + Scheme *runtime.Scheme } const ( @@ -74,8 +75,9 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, request ctrl.Reques } pvcName := WorkspacePVCName(&workspace) + pvcKey := types.NamespacedName{Namespace: workspace.Namespace, Name: pvcName} var pvc corev1.PersistentVolumeClaim - err = r.Get(ctx, types.NamespacedName{Namespace: workspace.Namespace, Name: pvcName}, &pvc) + err = r.Get(ctx, pvcKey, &pvc) if apierrors.IsNotFound(err) { volumeMode := corev1.PersistentVolumeFilesystem pvc = corev1.PersistentVolumeClaim{ @@ -99,12 +101,22 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, request ctrl.Reques return ctrl.Result{}, err } } - if err := r.Create(ctx, &pvc); err != nil && !apierrors.IsAlreadyExists(err) { - return ctrl.Result{}, err + if err := r.Create(ctx, &pvc); err != nil { + if !apierrors.IsAlreadyExists(err) { + return ctrl.Result{}, err + } + reader := r.APIReader + if reader == nil { + reader = r.Client + } + if err := reader.Get(ctx, pvcKey, &pvc); err != nil { + return ctrl.Result{}, err + } } } else if err != nil { return ctrl.Result{}, err - } else if !workspaceOwnsPVC(&workspace, &pvc) { + } + if !workspaceOwnsPVC(&workspace, &pvc) { return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCOwnershipConflict", "deterministic workspace PVC does not belong to this workspace") } else if pvcStorageClassName(&pvc) != storageClassName { return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", ReasonStorageClassMismatch, fmt.Sprintf("workspace PVC uses StorageClass %q instead of host-selected %q; data-bearing PVCs are never recreated automatically", pvcStorageClassName(&pvc), storageClassName)) From ee5d00903272e00b584ce2bac16fd33e7cf80352 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 23:10:02 +0000 Subject: [PATCH 34/43] test(operator): cover authoritative cleanup policy --- .../controllers/reconciler_test.go | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index e4f4f42d..6e45a9a0 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -106,6 +106,67 @@ func TestWorkspaceCreateAlreadyExistsRefetchesForeignPVC(t *testing.T) { } } +func TestWorkspacePendingPVCPolicyFailsBeforeAuthority(t *testing.T) { + for _, test := range []struct { + name string + mutate func(*corev1.PersistentVolumeClaim) + reason string + }{ + {name: "wrong class", reason: controllers.ReasonStorageClassMismatch, mutate: func(pvc *corev1.PersistentVolumeClaim) { pvc.Spec.StorageClassName = ptr("other-rwx") }}, + {name: "wrong access", reason: "PVCNotRWX", mutate: func(pvc *corev1.PersistentVolumeClaim) { pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} }}, + } { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + pvc := ownedWorkspacePVC(workspace) + pvc.Status.Phase = corev1.ClaimPending + test.mutate(pvc) + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Workspace{}, &corev1.PersistentVolumeClaim{}).WithObjects(testHost(), rwxStorageClass(), workspace, pvc).Build() + r := &controllers.WorkspaceReconciler{Client: c, APIReader: c, Scheme: scheme} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + var failed clusterv1alpha1.T4Workspace + if err := c.Get(ctx, client.ObjectKeyFromObject(workspace), &failed); err != nil { + t.Fatal(err) + } + condition := findCondition(failed.Status.Conditions, "StorageReady") + if failed.Status.PVCName != "" || condition == nil || condition.Status != metav1.ConditionFalse || condition.Reason != test.reason { + t.Fatalf("incompatible Pending PVC published authority: status=%#v condition=%#v", failed.Status, condition) + } + }) + } +} + +func TestWorkspaceDeletionUsesAuthoritativePVCReader(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + workspace.Finalizers = []string{clusterv1alpha1.WorkspaceFinalizer} + pvc := ownedWorkspacePVC(workspace) + pvc.OwnerReferences = nil + base := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Workspace{}).WithObjects(workspace, pvc).Build() + if err := base.Delete(ctx, workspace); err != nil { + t.Fatal(err) + } + c := &createAlreadyExistsClient{Client: base, raceKind: "PVC", winner: pvc, hideWinnerFromCache: true} + r := &controllers.WorkspaceReconciler{Client: c, APIReader: base, Scheme: scheme} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + var waiting clusterv1alpha1.T4Workspace + if err := base.Get(ctx, client.ObjectKeyFromObject(workspace), &waiting); err != nil { + t.Fatalf("workspace finalizer ignored authoritative PVC conflict: %v", err) + } + condition := findCondition(waiting.Status.Conditions, "Ready") + if condition == nil || condition.Reason != "CleanupOwnershipConflict" || !contains(waiting.Finalizers, clusterv1alpha1.WorkspaceFinalizer) { + t.Fatalf("authoritative PVC conflict not retained: %#v", waiting) + } +} + func TestRetainWorkspaceCreatesPVCWithoutGarbageCollectableOwner(t *testing.T) { scheme := testScheme(t) workspace := testWorkspace(clusterv1alpha1.RetentionPolicyRetain) @@ -1297,6 +1358,35 @@ func TestSessionDeletionRefusesForeignDeterministicResources(t *testing.T) { } } +func TestSessionDeletionUsesAuthoritativeChildReader(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + session := testSession() + session.UID = "session-uid" + session.Finalizers = []string{clusterv1alpha1.SessionFinalizer} + pod, service := ownedSessionResources(session) + pod.OwnerReferences = nil + base := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Session{}).WithObjects(session, pod, service).Build() + if err := base.Delete(ctx, session); err != nil { + t.Fatal(err) + } + c := &createAlreadyExistsClient{Client: base, raceKind: "Pod", winner: pod, hideWinnerFromCache: true} + r := configuredSessionReconciler(c, scheme) + r.APIReader = base + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + assertObjectCounts(t, base, 1, 0) + var waiting clusterv1alpha1.T4Session + if err := base.Get(ctx, client.ObjectKeyFromObject(session), &waiting); err != nil { + t.Fatalf("session finalizer ignored authoritative Pod conflict: %v", err) + } + condition := findCondition(waiting.Status.Conditions, "Available") + if condition == nil || condition.Reason != "CleanupOwnershipConflict" || !contains(waiting.Finalizers, clusterv1alpha1.SessionFinalizer) { + t.Fatalf("authoritative child conflict not retained: %#v", waiting) + } +} + func TestSessionDeletionCleansResourcesBeforeFinalizer(t *testing.T) { scheme := testScheme(t) session := testSession() @@ -2181,6 +2271,10 @@ func (c *createAlreadyExistsClient) Get(ctx context.Context, key client.ObjectKe if c.raceKind == "Service" { return apierrors.NewNotFound(schema.GroupResource{Resource: "services"}, key.Name) } + case *corev1.PersistentVolumeClaim: + if c.raceKind == "PVC" { + return apierrors.NewNotFound(schema.GroupResource{Resource: "persistentvolumeclaims"}, key.Name) + } } } return c.Client.Get(ctx, key, object, options...) From d9ef139dd752f3f7ceb79707b6be45e87f589b58 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 23:12:47 +0000 Subject: [PATCH 35/43] fix(operator): precondition authoritative cleanup --- .../controllers/session_controller.go | 43 +++++++++++++++---- .../controllers/workspace_controller.go | 13 +++--- 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/packages/cluster-operator/controllers/session_controller.go b/packages/cluster-operator/controllers/session_controller.go index 3f1bd28d..7523bbbb 100644 --- a/packages/cluster-operator/controllers/session_controller.go +++ b/packages/cluster-operator/controllers/session_controller.go @@ -465,7 +465,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) } return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } else if !serviceExposureIsInternal(&service) { - if err := r.Delete(ctx, &service); err != nil && !apierrors.IsNotFound(err) { + if err := deleteWithPreconditions(ctx, r.Client, &service); err != nil && !apierrors.IsNotFound(err) { return ctrl.Result{}, err } if err := r.updateSessionPending(ctx, &session, podName, serviceName, "ServiceExposureChanged", "session Service is being recreated with ClusterIP-only exposure"); err != nil { @@ -527,7 +527,7 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) } return ctrl.Result{RequeueAfter: time.Second}, nil } else if pod.Annotations[clusterv1alpha1.SessionPodSpecHashAnnotation] != desiredPod.Annotations[clusterv1alpha1.SessionPodSpecHashAnnotation] { - if err := r.Delete(ctx, &pod); err != nil && !apierrors.IsNotFound(err) { + if err := deleteWithPreconditions(ctx, r.Client, &pod); err != nil && !apierrors.IsNotFound(err) { return ctrl.Result{}, err } if err := r.updateSessionPending(ctx, &session, podName, serviceName, "PodSpecChanged", "session Pod is being recreated to apply immutable desired state"); err != nil { @@ -735,8 +735,12 @@ func (r *SessionReconciler) reconcileDelete(ctx context.Context, session *cluste } existing := make([]client.Object, 0, len(objects)) var ownershipConflict client.Object + reader := r.APIReader + if reader == nil { + reader = r.Client + } for _, object := range objects { - err := r.Get(ctx, client.ObjectKeyFromObject(object), object) + err := reader.Get(ctx, client.ObjectKeyFromObject(object), object) if apierrors.IsNotFound(err) { continue } @@ -753,7 +757,7 @@ func (r *SessionReconciler) reconcileDelete(ctx context.Context, session *cluste } for _, object := range existing { if object.GetDeletionTimestamp().IsZero() { - if err := r.Delete(ctx, object); err != nil && !apierrors.IsNotFound(err) { + if err := deleteWithPreconditions(ctx, r.Client, object); err != nil && !apierrors.IsNotFound(err) { return ctrl.Result{}, err } } @@ -779,7 +783,7 @@ func (r *SessionReconciler) reconcileDelete(ctx context.Context, session *cluste } func (r *SessionReconciler) deleteOwnedSessionResources(ctx context.Context, session *clusterv1alpha1.T4Session) error { - return r.deleteOwnedSessionResourcesWithFailure(ctx, r.Client, session, false, false, "ResourceOwnershipConflict", "one or more deterministic session resources have an unexpected owner") + return r.deleteOwnedSessionResourcesWithFailure(ctx, r.Client, session, true, false, false, "ResourceOwnershipConflict", "one or more deterministic session resources have an unexpected owner") } func (r *SessionReconciler) deleteOwnedSessionResourcesAfterVerifiedDependencies(ctx context.Context, session *clusterv1alpha1.T4Session, reason, message string) error { @@ -787,14 +791,15 @@ func (r *SessionReconciler) deleteOwnedSessionResourcesAfterVerifiedDependencies if reader == nil { reader = r.Client } - return r.deleteOwnedSessionResourcesWithFailure(ctx, reader, session, true, true, reason, message) + return r.deleteOwnedSessionResourcesWithFailure(ctx, reader, session, false, true, true, reason, message) } -func (r *SessionReconciler) deleteOwnedSessionResourcesWithFailure(ctx context.Context, reader client.Reader, session *clusterv1alpha1.T4Session, hostReady, workspaceReady bool, reason, message string) error { +func (r *SessionReconciler) deleteOwnedSessionResourcesWithFailure(ctx context.Context, reader client.Reader, session *clusterv1alpha1.T4Session, deleteWithoutConflict, hostReady, workspaceReady bool, reason, message string) error { objects := []client.Object{ &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: SessionPodName(session), Namespace: session.Namespace}}, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: SessionServiceName(session), Namespace: session.Namespace}}, } + owned := make([]client.Object, 0, len(objects)) ownershipConflict := false for _, object := range objects { if err := reader.Get(ctx, client.ObjectKeyFromObject(object), object); err != nil { @@ -807,8 +812,13 @@ func (r *SessionReconciler) deleteOwnedSessionResourcesWithFailure(ctx context.C ownershipConflict = true continue } - if err := r.Delete(ctx, object); err != nil && !apierrors.IsNotFound(err) { - return err + owned = append(owned, object) + } + if ownershipConflict || deleteWithoutConflict { + for _, object := range owned { + if err := deleteWithPreconditions(ctx, r.Client, object); err != nil && !apierrors.IsNotFound(err) { + return err + } } } if ownershipConflict { @@ -820,6 +830,21 @@ func (r *SessionReconciler) deleteOwnedSessionResourcesWithFailure(ctx context.C return nil } +func deleteWithPreconditions(ctx context.Context, writer client.Client, object client.Object) error { + preconditions := metav1.Preconditions{} + if uid := object.GetUID(); uid != "" { + preconditions.UID = &uid + } + if resourceVersion := object.GetResourceVersion(); resourceVersion != "" { + preconditions.ResourceVersion = &resourceVersion + } + options := &client.DeleteOptions{} + if preconditions.UID != nil || preconditions.ResourceVersion != nil { + options.Preconditions = &preconditions + } + return writer.Delete(ctx, object, options) +} + func (r *SessionReconciler) updateSessionFailure(ctx context.Context, session *clusterv1alpha1.T4Session, hostReady, workspaceReady bool, conditionType, reason, message string) error { original := session.Status if session.Status.Conditions != nil { diff --git a/packages/cluster-operator/controllers/workspace_controller.go b/packages/cluster-operator/controllers/workspace_controller.go index d6c2a54d..441e785a 100644 --- a/packages/cluster-operator/controllers/workspace_controller.go +++ b/packages/cluster-operator/controllers/workspace_controller.go @@ -118,6 +118,8 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, request ctrl.Reques } if !workspaceOwnsPVC(&workspace, &pvc) { return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCOwnershipConflict", "deterministic workspace PVC does not belong to this workspace") + } else if !pvcHasRWX(&pvc) { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCNotRWX", "workspace PVC does not request ReadWriteMany") } else if pvcStorageClassName(&pvc) != storageClassName { return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", ReasonStorageClassMismatch, fmt.Sprintf("workspace PVC uses StorageClass %q instead of host-selected %q; data-bearing PVCs are never recreated automatically", pvcStorageClassName(&pvc), storageClassName)) } else if workspace.Spec.RetentionPolicy == clusterv1alpha1.RetentionPolicyRetain && metav1.IsControlledBy(&pvc, &workspace) { @@ -130,9 +132,6 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, request ctrl.Reques return ctrl.Result{Requeue: true}, nil } } - if pvc.Status.Phase == corev1.ClaimBound && !pvcHasRWX(&pvc) { - return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCNotRWX", "bound workspace PVC does not request ReadWriteMany") - } if pvc.Status.Phase == corev1.ClaimLost { return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCLost", "workspace PVC lost its volume") } @@ -235,7 +234,11 @@ func (r *WorkspaceReconciler) reconcileDelete(ctx context.Context, workspace *cl } pvcKey := types.NamespacedName{Namespace: workspace.Namespace, Name: WorkspacePVCName(workspace)} var pvc corev1.PersistentVolumeClaim - err := r.Get(ctx, pvcKey, &pvc) + reader := r.APIReader + if reader == nil { + reader = r.Client + } + err := reader.Get(ctx, pvcKey, &pvc) if err == nil && !workspaceOwnsPVC(workspace, &pvc) { before := workspace.Status if workspace.Status.Conditions != nil { @@ -267,7 +270,7 @@ func (r *WorkspaceReconciler) reconcileDelete(ctx context.Context, workspace *cl } } else { if err == nil { - if err := r.Delete(ctx, &pvc); err != nil && !apierrors.IsNotFound(err) { + if err := deleteWithPreconditions(ctx, r.Client, &pvc); err != nil && !apierrors.IsNotFound(err) { return ctrl.Result{}, err } return ctrl.Result{RequeueAfter: time.Second}, nil From 32d4a0454496024e5f00e7ab6f6ae082db93dde4 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 23:28:51 +0000 Subject: [PATCH 36/43] test(operator): cover split-reader dependency cleanup --- .../controllers/reconciler_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index 6e45a9a0..5f244bf5 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -1387,6 +1387,22 @@ func TestSessionDeletionUsesAuthoritativeChildReader(t *testing.T) { } } +func TestSessionDependencyCleanupUsesAuthoritativeChildReader(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + session := testSession() + session.UID = "session-uid" + pod, service := ownedSessionResources(session) + base := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.Pod{}).WithObjects(session, pod, service).Build() + c := &createAlreadyExistsClient{Client: base, raceKind: "Pod", winner: pod, hideWinnerFromCache: true} + r := configuredSessionReconciler(c, scheme) + r.APIReader = base + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + assertObjectCounts(t, base, 0, 0) +} + func TestSessionDeletionCleansResourcesBeforeFinalizer(t *testing.T) { scheme := testScheme(t) session := testSession() From a10e70be3060db423a3c823fa6c083bd0903a724 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 23:30:43 +0000 Subject: [PATCH 37/43] fix(operator): authoritatively clean revoked sessions --- packages/cluster-operator/controllers/session_controller.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/cluster-operator/controllers/session_controller.go b/packages/cluster-operator/controllers/session_controller.go index 7523bbbb..21b1254e 100644 --- a/packages/cluster-operator/controllers/session_controller.go +++ b/packages/cluster-operator/controllers/session_controller.go @@ -783,7 +783,11 @@ func (r *SessionReconciler) reconcileDelete(ctx context.Context, session *cluste } func (r *SessionReconciler) deleteOwnedSessionResources(ctx context.Context, session *clusterv1alpha1.T4Session) error { - return r.deleteOwnedSessionResourcesWithFailure(ctx, r.Client, session, true, false, false, "ResourceOwnershipConflict", "one or more deterministic session resources have an unexpected owner") + reader := r.APIReader + if reader == nil { + reader = r.Client + } + return r.deleteOwnedSessionResourcesWithFailure(ctx, reader, session, true, false, false, "ResourceOwnershipConflict", "one or more deterministic session resources have an unexpected owner") } func (r *SessionReconciler) deleteOwnedSessionResourcesAfterVerifiedDependencies(ctx context.Context, session *clusterv1alpha1.T4Session, reason, message string) error { From 3d09b4ac29d04cdf2415a72c16af1bc94154dc3e Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 23:41:18 +0000 Subject: [PATCH 38/43] test(operator): expose stale workspace session cache --- .../controllers/reconciler_test.go | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index 5f244bf5..cb4959bf 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -167,6 +167,41 @@ func TestWorkspaceDeletionUsesAuthoritativePVCReader(t *testing.T) { } } +func TestWorkspaceDeletionUsesAuthoritativeSessionReader(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + workspace.Finalizers = []string{clusterv1alpha1.WorkspaceFinalizer} + pvc := ownedWorkspacePVC(workspace) + session := testSession() + session.Spec.WorkspaceRef = workspace.Name + cacheClient := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&clusterv1alpha1.T4Workspace{}).WithObjects(workspace, pvc).Build() + apiReader := fake.NewClientBuilder().WithScheme(scheme).WithObjects(pvc, session).Build() + if err := cacheClient.Delete(ctx, workspace); err != nil { + t.Fatal(err) + } + r := &controllers.WorkspaceReconciler{Client: cacheClient, APIReader: apiReader, Scheme: scheme} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + var waiting clusterv1alpha1.T4Workspace + if err := cacheClient.Get(ctx, client.ObjectKeyFromObject(workspace), &waiting); err != nil { + t.Fatalf("workspace finalizer ignored authoritative session: %v", err) + } + ready := findCondition(waiting.Status.Conditions, "Ready") + if ready == nil || ready.Status != metav1.ConditionFalse || ready.Reason != "SessionsRemain" || ready.ObservedGeneration != waiting.Generation { + t.Fatalf("Ready = %#v, want current-generation False/SessionsRemain", ready) + } + if !contains(waiting.Finalizers, clusterv1alpha1.WorkspaceFinalizer) { + t.Fatal("workspace finalizer was removed while authoritative session remains") + } + var remainingPVC corev1.PersistentVolumeClaim + if err := cacheClient.Get(ctx, client.ObjectKeyFromObject(pvc), &remainingPVC); err != nil { + t.Fatalf("workspace PVC was deleted while authoritative session remains: %v", err) + } +} + func TestRetainWorkspaceCreatesPVCWithoutGarbageCollectableOwner(t *testing.T) { scheme := testScheme(t) workspace := testWorkspace(clusterv1alpha1.RetentionPolicyRetain) From d428c0a0d17d548e9666e5635652a32be5976756 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Tue, 21 Jul 2026 23:48:00 +0000 Subject: [PATCH 39/43] fix(operator): list workspace sessions authoritatively --- .../cluster-operator/controllers/workspace_controller.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/cluster-operator/controllers/workspace_controller.go b/packages/cluster-operator/controllers/workspace_controller.go index 441e785a..f3b0fd90 100644 --- a/packages/cluster-operator/controllers/workspace_controller.go +++ b/packages/cluster-operator/controllers/workspace_controller.go @@ -210,7 +210,11 @@ func (r *WorkspaceReconciler) reconcileDelete(ctx context.Context, workspace *cl } } var sessions clusterv1alpha1.T4SessionList - if err := r.List(ctx, &sessions, client.InNamespace(workspace.Namespace)); err != nil { + sessionReader := r.APIReader + if sessionReader == nil { + sessionReader = r.Client + } + if err := sessionReader.List(ctx, &sessions, client.InNamespace(workspace.Namespace)); err != nil { return ctrl.Result{}, err } remainingSessions := 0 From 4a7cdf66bbdcfbf325d1bc2439a8727478976027 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 03:45:50 +0000 Subject: [PATCH 40/43] test(operator): reject stale PVC authority --- .../controllers/reconciler_test.go | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index cb4959bf..f5e00c84 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -106,6 +106,42 @@ func TestWorkspaceCreateAlreadyExistsRefetchesForeignPVC(t *testing.T) { } } +func TestWorkspaceReadinessRejectsAuthoritativeForeignPVCReplacement(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + cachedPVC := ownedWorkspacePVC(workspace) + cachedPVC.UID = "cached-pvc-uid" + authoritativePVC := cachedPVC.DeepCopy() + authoritativePVC.UID = "replacement-pvc-uid" + authoritativePVC.Annotations = nil + authoritativePVC.OwnerReferences = nil + cacheClient := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Workspace{}, &corev1.PersistentVolumeClaim{}). + WithObjects(testHost(), rwxStorageClass(), workspace, cachedPVC).Build() + r := &controllers.WorkspaceReconciler{ + Client: cacheClient, + APIReader: &pvcOverrideReader{Reader: cacheClient, pvc: authoritativePVC}, + Scheme: scheme, + } + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(workspace)}); err != nil { + t.Fatal(err) + } + var got clusterv1alpha1.T4Workspace + if err := cacheClient.Get(ctx, client.ObjectKeyFromObject(workspace), &got); err != nil { + t.Fatal(err) + } + storageReady := findCondition(got.Status.Conditions, "StorageReady") + ready := findCondition(got.Status.Conditions, "Ready") + if got.Status.PVCName != "" || got.Status.PVCPhase != "" || !got.Status.Capacity.IsZero() || + storageReady == nil || storageReady.Status != metav1.ConditionFalse || storageReady.Reason != "PVCOwnershipConflict" || storageReady.ObservedGeneration != got.Generation || + ready == nil || ready.Status != metav1.ConditionFalse || ready.ObservedGeneration != got.Generation { + t.Fatalf("stale cached PVC published workspace authority: status=%#v StorageReady=%#v Ready=%#v", got.Status, storageReady, ready) + } +} + + func TestWorkspacePendingPVCPolicyFailsBeforeAuthority(t *testing.T) { for _, test := range []struct { name string @@ -347,6 +383,59 @@ func TestWorkspaceDeletionWaitsForSessionResources(t *testing.T) { } } +func TestSessionPodCreateRevalidatesAuthoritativePVC(t *testing.T) { + tests := []struct { + name string + mutate func(*corev1.PersistentVolumeClaim) + }{ + {name: "replacement UID", mutate: func(pvc *corev1.PersistentVolumeClaim) { pvc.UID = "replacement-pvc-uid" }}, + {name: "foreign owner", mutate: func(pvc *corev1.PersistentVolumeClaim) { + pvc.OwnerReferences = []metav1.OwnerReference{{APIVersion: "v1", Kind: "Secret", Name: "foreign", UID: "foreign-uid", Controller: ptr(true)}} + }}, + {name: "storage class drift", mutate: func(pvc *corev1.PersistentVolumeClaim) { pvc.Spec.StorageClassName = ptr("other-rwx") }}, + {name: "access mode drift", mutate: func(pvc *corev1.PersistentVolumeClaim) { pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} }}, + {name: "unbound replacement", mutate: func(pvc *corev1.PersistentVolumeClaim) { pvc.Status.Phase = corev1.ClaimPending }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + workspace.Status.PVCName = controllers.WorkspacePVCName(workspace) + cachedPVC := ownedWorkspacePVC(workspace) + cachedPVC.UID = "cached-pvc-uid" + authoritativePVC := cachedPVC.DeepCopy() + test.mutate(authoritativePVC) + session := testSession() + session.UID = "session-uid" + cacheClient := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(testHost(), workspace, cachedPVC, session).Build() + r := configuredSessionReconciler(cacheClient, scheme) + r.APIReader = &pvcOverrideReader{Reader: cacheClient, pvc: authoritativePVC} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + var pods corev1.PodList + if err := cacheClient.List(ctx, &pods, client.InNamespace(session.Namespace)); err != nil { + t.Fatal(err) + } + if len(pods.Items) != 0 { + t.Fatalf("created %d Pods after authoritative PVC %s", len(pods.Items), test.name) + } + var got clusterv1alpha1.T4Session + if err := cacheClient.Get(ctx, client.ObjectKeyFromObject(session), &got); err != nil { + t.Fatal(err) + } + workspaceReady := findCondition(got.Status.Conditions, "WorkspaceReady") + if got.Status.PodName != "" || workspaceReady == nil || workspaceReady.Status != metav1.ConditionFalse || workspaceReady.Reason != "PVCAuthorityChanged" || workspaceReady.ObservedGeneration != got.Generation { + t.Fatalf("authoritative PVC %s published session authority: status=%#v WorkspaceReady=%#v", test.name, got.Status, workspaceReady) + } + }) + } +} + func TestSessionFailsClosedWhenAnyOMPReferenceIsMissing(t *testing.T) { for _, test := range []struct { name string @@ -2304,6 +2393,19 @@ func hasReadOnlyMount(mounts []corev1.VolumeMount, name, path string) bool { return false } +type pvcOverrideReader struct { + client.Reader + pvc *corev1.PersistentVolumeClaim +} + +func (r *pvcOverrideReader) Get(ctx context.Context, key client.ObjectKey, object client.Object, options ...client.GetOption) error { + if pvc, ok := object.(*corev1.PersistentVolumeClaim); ok && key == client.ObjectKeyFromObject(r.pvc) { + r.pvc.DeepCopyInto(pvc) + return nil + } + return r.Reader.Get(ctx, key, object, options...) +} + type createAlreadyExistsClient struct { client.Client raceKind string From b98470dd2c696d3b5f68624015e757530039497c Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 03:54:16 +0000 Subject: [PATCH 41/43] fix(operator): revalidate PVC authority --- .../controllers/session_controller.go | 51 +++++++++++++++++++ .../controllers/workspace_controller.go | 22 ++++++++ 2 files changed, 73 insertions(+) diff --git a/packages/cluster-operator/controllers/session_controller.go b/packages/cluster-operator/controllers/session_controller.go index 21b1254e..ed45cb6c 100644 --- a/packages/cluster-operator/controllers/session_controller.go +++ b/packages/cluster-operator/controllers/session_controller.go @@ -491,6 +491,16 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) var pod corev1.Pod podKey := types.NamespacedName{Namespace: session.Namespace, Name: podName} if err := r.Get(ctx, podKey, &pod); apierrors.IsNotFound(err) { + reason, message, err := r.authoritativePVCValidation(ctx, &workspace, &pvc, host.Spec.StorageClassName) + if err != nil { + return ctrl.Result{}, err + } + if reason != "" { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", reason, message) + } pod = desiredPod if err := r.Create(ctx, &pod); err != nil { if !apierrors.IsAlreadyExists(err) { @@ -536,6 +546,17 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) return ctrl.Result{RequeueAfter: time.Second}, nil } + reason, message, err = r.authoritativePVCValidation(ctx, &workspace, &pvc, host.Spec.StorageClassName) + if err != nil { + return ctrl.Result{}, err + } + if reason != "" { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", reason, message) + } + original := session.Status if session.Status.Conditions != nil { original.Conditions = append([]metav1.Condition(nil), session.Status.Conditions...) @@ -567,6 +588,36 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } +func (r *SessionReconciler) authoritativePVCValidation(ctx context.Context, workspace *clusterv1alpha1.T4Workspace, cachedPVC *corev1.PersistentVolumeClaim, storageClassName string) (string, string, error) { + reader := r.APIReader + if reader == nil { + reader = r.Client + } + var authoritativePVC corev1.PersistentVolumeClaim + if err := reader.Get(ctx, client.ObjectKeyFromObject(cachedPVC), &authoritativePVC); err != nil { + if apierrors.IsNotFound(err) { + return "PVCAuthorityChanged", "workspace PVC does not exist in authoritative API state", nil + } + return "", "", err + } + if authoritativePVC.UID != cachedPVC.UID { + return "PVCAuthorityChanged", "authoritative workspace PVC UID differs from the validated cached PVC", nil + } + if workspace.UID != "" && !workspaceOwnsPVC(workspace, &authoritativePVC) { + return "PVCAuthorityChanged", "authoritative workspace PVC owner reference does not belong to the workspace", nil + } + if pvcStorageClassName(&authoritativePVC) != storageClassName { + return "PVCAuthorityChanged", fmt.Sprintf("authoritative workspace PVC uses StorageClass %q instead of host-selected %q", pvcStorageClassName(&authoritativePVC), storageClassName), nil + } + if !pvcHasRWX(&authoritativePVC) { + return "PVCAuthorityChanged", "authoritative workspace PVC does not request ReadWriteMany", nil + } + if authoritativePVC.Status.Phase != corev1.ClaimBound { + return "PVCAuthorityChanged", "authoritative workspace PVC is not Bound", nil + } + return "", "", nil +} + func (r *SessionReconciler) desiredPod(session *clusterv1alpha1.T4Session, pvcName, podName string, labels map[string]string, runtimeVersions ompResourceVersions) (corev1.Pod, error) { falseValue := false trueValue := true diff --git a/packages/cluster-operator/controllers/workspace_controller.go b/packages/cluster-operator/controllers/workspace_controller.go index f3b0fd90..e1b616dc 100644 --- a/packages/cluster-operator/controllers/workspace_controller.go +++ b/packages/cluster-operator/controllers/workspace_controller.go @@ -132,6 +132,28 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, request ctrl.Reques return ctrl.Result{Requeue: true}, nil } } + reader := r.APIReader + if reader == nil { + reader = r.Client + } + var authoritativePVC corev1.PersistentVolumeClaim + if err := reader.Get(ctx, pvcKey, &authoritativePVC); err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{RequeueAfter: 5 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCNotFound", "workspace PVC does not exist in authoritative API state") + } + return ctrl.Result{}, err + } + if authoritativePVC.UID != pvc.UID || !workspaceOwnsPVC(&workspace, &authoritativePVC) { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCOwnershipConflict", "authoritative workspace PVC identity or ownership does not belong to this workspace") + } + if !pvcHasRWX(&authoritativePVC) { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCNotRWX", "authoritative workspace PVC does not request ReadWriteMany") + } + if pvcStorageClassName(&authoritativePVC) != storageClassName { + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", ReasonStorageClassMismatch, fmt.Sprintf("authoritative workspace PVC uses StorageClass %q instead of host-selected %q; data-bearing PVCs are never recreated automatically", pvcStorageClassName(&authoritativePVC), storageClassName)) + } + pvc = authoritativePVC + if pvc.Status.Phase == corev1.ClaimLost { return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateWorkspaceFailure(ctx, &workspace, "StorageReady", "PVCLost", "workspace PVC lost its volume") } From ece837c095a464fe9e3034901fce8affb56e7132 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 04:31:41 +0000 Subject: [PATCH 42/43] test(operator): gate existing pods on PVC authority --- .../controllers/reconciler_test.go | 56 ++++++++++++++++--- 1 file changed, 48 insertions(+), 8 deletions(-) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index f5e00c84..40cc9b02 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -436,6 +436,54 @@ func TestSessionPodCreateRevalidatesAuthoritativePVC(t *testing.T) { } } +func TestSessionExistingPodRepairRejectsAuthoritativeForeignPVCReplacement(t *testing.T) { + ctx := context.Background() + scheme := testScheme(t) + workspace := testWorkspace(clusterv1alpha1.RetentionPolicyDelete) + workspace.UID = "workspace-uid" + workspace.Status.PVCName = controllers.WorkspacePVCName(workspace) + cachedPVC := ownedWorkspacePVC(workspace) + cachedPVC.UID = "cached-pvc-uid" + session := testSession() + session.UID = "session-uid" + cacheClient := fake.NewClientBuilder().WithScheme(scheme). + WithStatusSubresource(&clusterv1alpha1.T4Session{}, &corev1.PersistentVolumeClaim{}, &corev1.Pod{}). + WithObjects(testHost(), workspace, cachedPVC, session).Build() + r := configuredSessionReconciler(cacheClient, scheme) + reconcileMany(t, 2, func() error { + _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}) + return err + }) + var pod corev1.Pod + if err := cacheClient.Get(ctx, types.NamespacedName{Namespace: session.Namespace, Name: controllers.SessionPodName(session)}, &pod); err != nil { + t.Fatal(err) + } + pod.Labels = nil + if err := cacheClient.Update(ctx, &pod); err != nil { + t.Fatal(err) + } + authoritativePVC := cachedPVC.DeepCopy() + authoritativePVC.UID = "replacement-pvc-uid" + authoritativePVC.Annotations = nil + authoritativePVC.OwnerReferences = nil + r.APIReader = &pvcOverrideReader{Reader: cacheClient, pvc: authoritativePVC} + if _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(session)}); err != nil { + t.Fatal(err) + } + if err := cacheClient.Get(ctx, client.ObjectKeyFromObject(&pod), &pod); !apierrors.IsNotFound(err) { + t.Fatalf("existing Pod survived authoritative PVC replacement: %v", err) + } + var got clusterv1alpha1.T4Session + if err := cacheClient.Get(ctx, client.ObjectKeyFromObject(session), &got); err != nil { + t.Fatal(err) + } + workspaceReady := findCondition(got.Status.Conditions, "WorkspaceReady") + if got.Status.PodName != "" || workspaceReady == nil || workspaceReady.Status != metav1.ConditionFalse || workspaceReady.Reason != "PVCAuthorityChanged" || workspaceReady.ObservedGeneration != got.Generation { + t.Fatalf("existing Pod repair retained stale PVC authority: status=%#v WorkspaceReady=%#v", got.Status, workspaceReady) + } +} + + func TestSessionFailsClosedWhenAnyOMPReferenceIsMissing(t *testing.T) { for _, test := range []struct { name string @@ -1586,14 +1634,6 @@ func TestSessionDependencyRevocationCleansOwnedResourcesAndConvergesAfterRestart {name: "missing OMP ConfigMap", conditionType: "RuntimeConfigured", wantReason: "OMPConfigMapNotFound", revoke: func(ctx context.Context, c client.Client) error { return c.Delete(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "omp-runtime-config", Namespace: "team"}}) }}, - {name: "invalid OMP credential Secret", conditionType: "RuntimeConfigured", wantReason: "OMPCredentialSecretInvalid", revoke: func(ctx context.Context, c client.Client) error { - var secret corev1.Secret - if err := c.Get(ctx, types.NamespacedName{Namespace: "team", Name: "omp-runtime-credential"}, &secret); err != nil { - return err - } - delete(secret.Data, "MODEL_API_KEY") - return c.Update(ctx, &secret) - }}, {name: "mismatched Host storage class", conditionType: "WorkspaceReady", wantReason: "StorageClassMismatch", revoke: func(ctx context.Context, c client.Client) error { otherClass := &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: "other-rwx", Annotations: map[string]string{clusterv1alpha1.RWXStorageClassAnnotation: string(corev1.ReadWriteMany)}}, Provisioner: "example.invalid/csi"} if err := c.Create(ctx, otherClass); err != nil { From 20e98781f966ed9b17f1b990ba521231a44e7764 Mon Sep 17 00:00:00 2001 From: usr-bin-roygbiv Date: Wed, 22 Jul 2026 04:39:43 +0000 Subject: [PATCH 43/43] fix(operator): gate child repairs on PVC authority --- .../cluster-operator/controllers/reconciler_test.go | 8 ++++---- .../controllers/session_controller.go | 11 +++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/packages/cluster-operator/controllers/reconciler_test.go b/packages/cluster-operator/controllers/reconciler_test.go index 40cc9b02..2c9db16f 100644 --- a/packages/cluster-operator/controllers/reconciler_test.go +++ b/packages/cluster-operator/controllers/reconciler_test.go @@ -501,7 +501,7 @@ func TestSessionFailsClosedWhenAnyOMPReferenceIsMissing(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -563,7 +563,7 @@ func TestSessionCredentialModeUpgradeStopsExistingAuthorityAndClearsRoute(t *tes workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -625,7 +625,7 @@ func TestSessionRejectsCredentialBearingModelsConfiguration(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() @@ -676,7 +676,7 @@ func TestSessionRejectsCredentialBearingSettingsConfiguration(t *testing.T) { workspace.Status.PVCName = "workspace-a-data" pvc := &corev1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{Name: workspace.Status.PVCName, Namespace: "team"}, - Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, + Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: ptr("portable-rwx"), AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}}, Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimBound}, } session := testSession() diff --git a/packages/cluster-operator/controllers/session_controller.go b/packages/cluster-operator/controllers/session_controller.go index ed45cb6c..345dbeb0 100644 --- a/packages/cluster-operator/controllers/session_controller.go +++ b/packages/cluster-operator/controllers/session_controller.go @@ -421,6 +421,17 @@ func (r *SessionReconciler) Reconcile(ctx context.Context, request ctrl.Request) } return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, true, "RuntimeConfigured", reason, message) } + reason, message, err = r.authoritativePVCValidation(ctx, &workspace, &pvc, host.Spec.StorageClassName) + if err != nil { + return ctrl.Result{}, err + } + if reason != "" { + if err := r.deleteOwnedSessionResources(ctx, &session); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 30 * time.Second}, r.updateSessionFailure(ctx, &session, true, false, "WorkspaceReady", reason, message) + } + serviceName := SessionServiceName(&session) podName := SessionPodName(&session)