From f08ee99595ff2d7bf638c991e0f3e03fa5ffd1f3 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Fri, 21 Aug 2026 14:19:20 -0700 Subject: [PATCH 01/56] refactor: extract run-ID and label primitives to neutral packages Signed-off-by: Alex Yuskauskas --- pkg/k8s/labels/labels.go | 39 ++++++++++++++++++++++++++++ pkg/runid/runid.go | 46 ++++++++++++++++++++++++++++++++++ pkg/runid/runid_test.go | 43 +++++++++++++++++++++++++++++++ pkg/validator/labels/labels.go | 20 +++++++++------ pkg/validator/v1/job_plan.go | 16 ++---------- 5 files changed, 143 insertions(+), 21 deletions(-) create mode 100644 pkg/k8s/labels/labels.go create mode 100644 pkg/runid/runid.go create mode 100644 pkg/runid/runid_test.go diff --git a/pkg/k8s/labels/labels.go b/pkg/k8s/labels/labels.go new file mode 100644 index 000000000..05005b523 --- /dev/null +++ b/pkg/k8s/labels/labels.go @@ -0,0 +1,39 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package labels provides shared Kubernetes label constants used by both +// the validator (pkg/validator) and the snapshot agent (pkg/k8s/agent), so +// neither has to import the other to agree on label keys and values. +package labels + +import "github.com/NVIDIA/aicr/pkg/header" + +// Standard Kubernetes label keys. +const ( + Name = "app.kubernetes.io/name" + Component = "app.kubernetes.io/component" + ManagedBy = "app.kubernetes.io/managed-by" +) + +// RunID scopes every resource to the run that created it. +const RunID = header.Domain + "/run-id" + +// Common label values. +const ( + // ValueAICR is the shared app name. + ValueAICR = "aicr" + + // ValueSnapshotAgent identifies snapshot-agent-owned resources. + ValueSnapshotAgent = "snapshot-agent" +) diff --git a/pkg/runid/runid.go b/pkg/runid/runid.go new file mode 100644 index 000000000..175404183 --- /dev/null +++ b/pkg/runid/runid.go @@ -0,0 +1,46 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package runid generates unique run identifiers shared by the validator +// and the snapshot agent so both subsystems use one format and one +// generator. +package runid + +import ( + "crypto/rand" + "encoding/hex" + "fmt" + "time" +) + +// Generate creates a unique run identifier. +// Format: {timestamp}-{random-hex} (e.g., "20260514-123045-abc123def456"). +// Callers use this to generate runIDs before creating ConfigMaps and +// rendering Jobs. +// +// Panics if the system's random number generator fails. Entropy failures are +// exceptional and we prefer to fail fast rather than generate predictable IDs +// that could collide across concurrent runs. +func Generate() string { + timestamp := time.Now().Format("20060102-150405") + randomBytes := make([]byte, 8) + n, err := rand.Read(randomBytes) + if err != nil { + panic(fmt.Sprintf("failed to generate random bytes for runID: %v", err)) + } + if n != len(randomBytes) { + panic(fmt.Sprintf("failed to generate runID: read %d bytes, expected %d", n, len(randomBytes))) + } + return fmt.Sprintf("%s-%s", timestamp, hex.EncodeToString(randomBytes)) +} diff --git a/pkg/runid/runid_test.go b/pkg/runid/runid_test.go new file mode 100644 index 000000000..5ada35fe9 --- /dev/null +++ b/pkg/runid/runid_test.go @@ -0,0 +1,43 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package runid + +import ( + "regexp" + "testing" +) + +var runIDPattern = regexp.MustCompile(`^\d{8}-\d{6}-[0-9a-f]{16}$`) + +func TestGenerateFormat(t *testing.T) { + id := Generate() + if !runIDPattern.MatchString(id) { + t.Errorf("Generate() = %q, want match %s", id, runIDPattern) + } + if len(id) != 32 { + t.Errorf("len(Generate()) = %d, want 32", len(id)) + } +} + +func TestGenerateUnique(t *testing.T) { + seen := make(map[string]struct{}, 100) + for range 100 { + id := Generate() + if _, dup := seen[id]; dup { + t.Fatalf("Generate() returned duplicate %q", id) + } + seen[id] = struct{}{} + } +} diff --git a/pkg/validator/labels/labels.go b/pkg/validator/labels/labels.go index 9c7c9988f..6f99b6fa1 100644 --- a/pkg/validator/labels/labels.go +++ b/pkg/validator/labels/labels.go @@ -15,19 +15,25 @@ // Package labels provides shared label constants for validation resources. package labels -import "github.com/NVIDIA/aicr/pkg/header" +import ( + "github.com/NVIDIA/aicr/pkg/header" + k8slabels "github.com/NVIDIA/aicr/pkg/k8s/labels" +) -// Standard Kubernetes label keys. +// Standard Kubernetes label keys. These are aliases of pkg/k8s/labels, which +// is the single source of truth shared with the snapshot agent +// (pkg/k8s/agent) — validator code should keep referring to them as +// labels.Name etc. rather than importing pkg/k8s/labels directly. const ( - Name = "app.kubernetes.io/name" - Component = "app.kubernetes.io/component" - ManagedBy = "app.kubernetes.io/managed-by" + Name = k8slabels.Name + Component = k8slabels.Component + ManagedBy = k8slabels.ManagedBy + RunID = k8slabels.RunID ) // AICR-specific label keys, keyed on the canonical AICR API domain. const ( JobType = header.Domain + "/job-type" - RunID = header.Domain + "/run-id" Validator = header.Domain + "/validator" Phase = header.Domain + "/phase" ReportType = header.Domain + "/report-type" @@ -35,7 +41,7 @@ const ( // Common label values. const ( - ValueAICR = "aicr" + ValueAICR = k8slabels.ValueAICR ValueValidation = "validation" ValueValidator = "aicr-validator" ) diff --git a/pkg/validator/v1/job_plan.go b/pkg/validator/v1/job_plan.go index d25fc5aba..c3b2312bc 100644 --- a/pkg/validator/v1/job_plan.go +++ b/pkg/validator/v1/job_plan.go @@ -15,14 +15,11 @@ package v1 import ( - "crypto/rand" - "encoding/hex" - "fmt" "strings" - "time" "github.com/NVIDIA/aicr/pkg/defaults" "github.com/NVIDIA/aicr/pkg/recipe" + "github.com/NVIDIA/aicr/pkg/runid" "github.com/NVIDIA/aicr/pkg/validator/labels" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -105,16 +102,7 @@ type JobPlan struct { // exceptional and we prefer to fail fast rather than generate predictable IDs // that could collide across concurrent runs. func GenerateRunID() string { - timestamp := time.Now().Format("20060102-150405") - randomBytes := make([]byte, 8) - n, err := rand.Read(randomBytes) - if err != nil { - panic(fmt.Sprintf("failed to generate random bytes for runID: %v", err)) - } - if n != len(randomBytes) { - panic(fmt.Sprintf("failed to generate runID: read %d bytes, expected %d", n, len(randomBytes))) - } - return fmt.Sprintf("%s-%s", timestamp, hex.EncodeToString(randomBytes)) + return runid.Generate() } // ImagePullPolicy determines the pull policy for a container image. From 29574fa274fad96547987c64cc7a7438f9cf0afa Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Fri, 21 Aug 2026 14:29:37 -0700 Subject: [PATCH 02/56] feat(agent): add run-scoped name helpers Signed-off-by: Alex Yuskauskas --- pkg/defaults/k8s.go | 10 +++ pkg/k8s/agent/deployer.go | 4 +- pkg/k8s/agent/deployer_test.go | 18 +++--- pkg/k8s/agent/names.go | 98 ++++++++++++++++++++++++++++ pkg/k8s/agent/names_test.go | 115 +++++++++++++++++++++++++++++++++ pkg/k8s/agent/rbac.go | 10 +-- pkg/k8s/agent/types.go | 36 +++++++---- 7 files changed, 261 insertions(+), 30 deletions(-) create mode 100644 pkg/k8s/agent/names.go create mode 100644 pkg/k8s/agent/names_test.go diff --git a/pkg/defaults/k8s.go b/pkg/defaults/k8s.go index 64b1b0185..8a6f2d559 100644 --- a/pkg/defaults/k8s.go +++ b/pkg/defaults/k8s.go @@ -40,3 +40,13 @@ const ( // (discovery + the first batch of GETs) is not immediately throttled. ValidatorClientBurst = 100 ) + +// MaxK8sNameLength is the maximum length of a Kubernetes object name built +// from a run-scoped prefix. 63 characters is the general DNS label ceiling +// (RFC 1123) that Kubernetes enforces on object names, but the binding +// constraint here is narrower: Jobs propagate their name into the +// batch.kubernetes.io/job-name label on every Pod they create, and label +// values share the same 63-character ceiling. A run-scoped Job name that +// fits the object-name limit but not the label limit would fail Pod +// creation, so name helpers must budget against this constant. +const MaxK8sNameLength = 63 diff --git a/pkg/k8s/agent/deployer.go b/pkg/k8s/agent/deployer.go index 184f8b0d2..36db0a6de 100644 --- a/pkg/k8s/agent/deployer.go +++ b/pkg/k8s/agent/deployer.go @@ -116,8 +116,8 @@ func (d *Deployer) Cleanup(ctx context.Context, opts CleanupOptions) error { {fmt.Sprintf("ServiceAccount %q", d.config.ServiceAccountName), d.deleteServiceAccount}, {fmt.Sprintf("Role %q", d.config.ServiceAccountName), d.deleteRole}, {fmt.Sprintf("RoleBinding %q", d.config.ServiceAccountName), d.deleteRoleBinding}, - {fmt.Sprintf("ClusterRole %q", clusterRoleName), d.deleteClusterRole}, - {fmt.Sprintf("ClusterRoleBinding %q", clusterRoleName), d.deleteClusterRoleBinding}, + {fmt.Sprintf("ClusterRole %q", d.clusterRoleName()), d.deleteClusterRole}, + {fmt.Sprintf("ClusterRoleBinding %q", d.clusterRoleName()), d.deleteClusterRoleBinding}, } // sync.WaitGroup (not errgroup) is intentional here: cleanup must diff --git a/pkg/k8s/agent/deployer_test.go b/pkg/k8s/agent/deployer_test.go index 9e2effdf9..3a6a94404 100644 --- a/pkg/k8s/agent/deployer_test.go +++ b/pkg/k8s/agent/deployer_test.go @@ -150,7 +150,7 @@ func TestDeployer_EnsureRBAC(t *testing.T) { } cr, err := clientset.RbacV1().ClusterRoles(). - Get(ctx, "aicr-node-reader", metav1.GetOptions{}) + Get(ctx, deployer.clusterRoleName(), metav1.GetOptions{}) if err != nil { t.Fatalf("ClusterRole not found: %v", err) } @@ -227,7 +227,7 @@ func TestDeployer_EnsureRBAC(t *testing.T) { t.Fatalf("failed to create ClusterRole: %v", err) } cr, err := discoverClientset.RbacV1().ClusterRoles(). - Get(ctx, "aicr-node-reader", metav1.GetOptions{}) + Get(ctx, d.clusterRoleName(), metav1.GetOptions{}) if err != nil { t.Fatalf("ClusterRole not found: %v", err) } @@ -283,7 +283,7 @@ func TestDeployer_EnsureRBAC(t *testing.T) { } crb, err := clientset.RbacV1().ClusterRoleBindings(). - Get(ctx, "aicr-node-reader", metav1.GetOptions{}) + Get(ctx, deployer.clusterRoleName(), metav1.GetOptions{}) if err != nil { t.Fatalf("ClusterRoleBinding not found: %v", err) } @@ -294,8 +294,8 @@ func TestDeployer_EnsureRBAC(t *testing.T) { } // Verify roleRef - if crb.RoleRef.Name != "aicr-node-reader" { - t.Errorf("expected roleRef name 'aicr-node-reader', got %q", crb.RoleRef.Name) + if crb.RoleRef.Name != deployer.clusterRoleName() { + t.Errorf("expected roleRef name %q, got %q", deployer.clusterRoleName(), crb.RoleRef.Name) } }) } @@ -562,14 +562,14 @@ func TestDeployer_Deploy(t *testing.T) { // Verify ClusterRole _, err = clientset.RbacV1().ClusterRoles(). - Get(ctx, "aicr-node-reader", metav1.GetOptions{}) + Get(ctx, deployer.clusterRoleName(), metav1.GetOptions{}) if err != nil { t.Errorf("ClusterRole not created: %v", err) } // Verify ClusterRoleBinding _, err = clientset.RbacV1().ClusterRoleBindings(). - Get(ctx, "aicr-node-reader", metav1.GetOptions{}) + Get(ctx, deployer.clusterRoleName(), metav1.GetOptions{}) if err != nil { t.Errorf("ClusterRoleBinding not created: %v", err) } @@ -702,13 +702,13 @@ func TestDeployer_Cleanup_AttemptsAllDeletions(t *testing.T) { } _, err = clientset.RbacV1().ClusterRoles(). - Get(ctx, clusterRoleName, metav1.GetOptions{}) + Get(ctx, deployer.clusterRoleName(), metav1.GetOptions{}) if err == nil { t.Error("ClusterRole should be deleted") } _, err = clientset.RbacV1().ClusterRoleBindings(). - Get(ctx, clusterRoleName, metav1.GetOptions{}) + Get(ctx, deployer.clusterRoleName(), metav1.GetOptions{}) if err == nil { t.Error("ClusterRoleBinding should be deleted") } diff --git a/pkg/k8s/agent/names.go b/pkg/k8s/agent/names.go new file mode 100644 index 000000000..763f1b554 --- /dev/null +++ b/pkg/k8s/agent/names.go @@ -0,0 +1,98 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package agent + +import ( + "strings" + + "github.com/NVIDIA/aicr/pkg/defaults" +) + +// defaultNameBase is the prefix used for generated resource names when the +// caller does not set Config.NameBase. +const defaultNameBase = "aicr" + +// staticClusterRoleName is the un-scoped ClusterRole/ClusterRoleBinding +// name used only as the prefix input to nameWithRunID. +const staticClusterRoleName = "aicr-node-reader" + +// staticStagingConfigMapName is the un-scoped staging ConfigMap name used +// only as the prefix input to nameWithRunID. +const staticStagingConfigMapName = "aicr-snapshot" + +// nameWithRunID joins prefix and runID, truncating prefix so the result fits +// within the Kubernetes name ceiling. An empty prefix yields the bare runID. +func nameWithRunID(prefix, runID string) string { + if prefix == "" { + return runID + } + budget := defaults.MaxK8sNameLength - len(runID) - 1 + if budget < 0 { + budget = 0 + } + if len(prefix) > budget { + prefix = prefix[:budget] + } + prefix = strings.TrimRight(prefix, "-") + if prefix == "" { + return runID + } + return prefix + "-" + runID +} + +// base returns the configured name base, defaulting to "aicr" when unset. +func (d *Deployer) base() string { + if d.config.NameBase != "" { + return d.config.NameBase + } + return defaultNameBase +} + +// jobName returns the run-scoped name for the agent Job. +func (d *Deployer) jobName() string { + prefix := d.config.JobName + if prefix == "" { + prefix = d.base() + } + return nameWithRunID(prefix, d.config.RunID) +} + +// saName returns the run-scoped name for the agent ServiceAccount. +func (d *Deployer) saName() string { + prefix := d.config.ServiceAccountName + if prefix == "" { + prefix = d.base() + } + return nameWithRunID(prefix, d.config.RunID) +} + +// roleName returns the run-scoped name for the agent Role and RoleBinding. +// It shares the ServiceAccount's name, matching the existing convention +// where the Role/RoleBinding are named after the ServiceAccount they bind. +func (d *Deployer) roleName() string { + return d.saName() +} + +// clusterRoleName returns the run-scoped name for the agent ClusterRole and +// ClusterRoleBinding. +func (d *Deployer) clusterRoleName() string { + return nameWithRunID(staticClusterRoleName, d.config.RunID) +} + +// stagingConfigMapName returns the run-scoped name for the staging +// ConfigMap the agent writes its snapshot result to. +func (d *Deployer) stagingConfigMapName() string { + return nameWithRunID(staticStagingConfigMapName, d.config.RunID) +} diff --git a/pkg/k8s/agent/names_test.go b/pkg/k8s/agent/names_test.go new file mode 100644 index 000000000..047e9ce28 --- /dev/null +++ b/pkg/k8s/agent/names_test.go @@ -0,0 +1,115 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package agent + +import ( + "strings" + "testing" +) + +func TestNameWithRunID(t *testing.T) { + const runID = "20260821-142233-9f3a1c0b7e2d4a55" // 32 chars + + tests := []struct { + name string + prefix string + want string + }{ + {"short prefix", "aicr", "aicr-" + runID}, + {"exactly at budget", strings.Repeat("a", 30), strings.Repeat("a", 30) + "-" + runID}, + {"over budget truncates", strings.Repeat("b", 40), strings.Repeat("b", 30) + "-" + runID}, + {"trailing dash trimmed", strings.Repeat("c", 29) + "-", strings.Repeat("c", 29) + "-" + runID}, + {"empty prefix", "", runID}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := nameWithRunID(tt.prefix, runID) + if got != tt.want { + t.Errorf("nameWithRunID(%q, runID) = %q, want %q", tt.prefix, got, tt.want) + } + if len(got) > 63 { + t.Errorf("len = %d, exceeds 63-char ceiling", len(got)) + } + }) + } +} + +func TestDeployerNameAccessors(t *testing.T) { + const runID = "20260821-142233-9f3a1c0b7e2d4a55" // 32 chars + + tests := []struct { + name string + config Config + get func(*Deployer) string + want string + }{ + { + name: "jobName uses configured JobName", + config: Config{JobName: "my-job", RunID: runID}, + get: (*Deployer).jobName, + want: "my-job-" + runID, + }, + { + name: "jobName falls back to NameBase", + config: Config{NameBase: "custom-base", RunID: runID}, + get: (*Deployer).jobName, + want: "custom-base-" + runID, + }, + { + name: "jobName falls back to default base", + config: Config{RunID: runID}, + get: (*Deployer).jobName, + want: "aicr-" + runID, + }, + { + name: "saName uses configured ServiceAccountName", + config: Config{ServiceAccountName: "my-sa", RunID: runID}, + get: (*Deployer).saName, + want: "my-sa-" + runID, + }, + { + name: "saName falls back to default base", + config: Config{RunID: runID}, + get: (*Deployer).saName, + want: "aicr-" + runID, + }, + { + name: "roleName matches saName", + config: Config{ServiceAccountName: "my-sa", RunID: runID}, + get: (*Deployer).roleName, + want: "my-sa-" + runID, + }, + { + name: "clusterRoleName is run-scoped", + config: Config{RunID: runID}, + get: (*Deployer).clusterRoleName, + want: "aicr-node-reader-" + runID, + }, + { + name: "stagingConfigMapName is run-scoped", + config: Config{RunID: runID}, + get: (*Deployer).stagingConfigMapName, + want: "aicr-snapshot-" + runID, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := &Deployer{config: tt.config} + if got := tt.get(d); got != tt.want { + t.Errorf("got %q, want %q", got, tt.want) + } + }) + } +} diff --git a/pkg/k8s/agent/rbac.go b/pkg/k8s/agent/rbac.go index fc9c7951d..1e6780938 100644 --- a/pkg/k8s/agent/rbac.go +++ b/pkg/k8s/agent/rbac.go @@ -211,7 +211,7 @@ func (d *Deployer) ensureClusterRole(ctx context.Context) error { cr := &rbacv1.ClusterRole{ ObjectMeta: metav1.ObjectMeta{ - Name: clusterRoleName, + Name: d.clusterRoleName(), }, Rules: rules, } @@ -234,7 +234,7 @@ func (d *Deployer) ensureClusterRole(ctx context.Context) error { func (d *Deployer) ensureClusterRoleBinding(ctx context.Context) error { crb := &rbacv1.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ - Name: clusterRoleName, + Name: d.clusterRoleName(), }, Subjects: []rbacv1.Subject{ { @@ -246,7 +246,7 @@ func (d *Deployer) ensureClusterRoleBinding(ctx context.Context) error { RoleRef: rbacv1.RoleRef{ APIGroup: rbacAPIGroup, Kind: "ClusterRole", - Name: clusterRoleName, + Name: d.clusterRoleName(), }, } @@ -292,7 +292,7 @@ func (d *Deployer) deleteRoleBinding(ctx context.Context) error { // If the ClusterRole doesn't exist, this is a no-op (idempotent). func (d *Deployer) deleteClusterRole(ctx context.Context) error { err := d.clientset.RbacV1().ClusterRoles(). - Delete(ctx, clusterRoleName, metav1.DeleteOptions{}) + Delete(ctx, d.clusterRoleName(), metav1.DeleteOptions{}) return k8s.IgnoreNotFound(err) } @@ -300,7 +300,7 @@ func (d *Deployer) deleteClusterRole(ctx context.Context) error { // If the ClusterRoleBinding doesn't exist, this is a no-op (idempotent). func (d *Deployer) deleteClusterRoleBinding(ctx context.Context) error { err := d.clientset.RbacV1().ClusterRoleBindings(). - Delete(ctx, clusterRoleName, metav1.DeleteOptions{}) + Delete(ctx, d.clusterRoleName(), metav1.DeleteOptions{}) return k8s.IgnoreNotFound(err) } diff --git a/pkg/k8s/agent/types.go b/pkg/k8s/agent/types.go index af16dedcd..f0f8312cf 100644 --- a/pkg/k8s/agent/types.go +++ b/pkg/k8s/agent/types.go @@ -19,9 +19,6 @@ import ( "k8s.io/client-go/kubernetes" ) -// clusterRoleName is the name used for the ClusterRole and ClusterRoleBinding. -const clusterRoleName = "aicr-node-reader" - // Standard Kubernetes recommended labels applied to all agent-managed // resources. Centralized here so selectors and resource templates stay in sync. const ( @@ -36,17 +33,28 @@ type Config struct { Namespace string ServiceAccountName string JobName string - Image string - ImagePullSecrets []string - NodeSelector map[string]string - Tolerations []corev1.Toleration - Output string - Debug bool - Privileged bool // If true, run with privileged security context (required for GPU/SystemD collectors) - RequireGPU bool // If true, request nvidia.com/gpu resource (required for CDI environments) - RuntimeClassName string // If set, use this runtimeClassName on the pod and inject NVIDIA_VISIBLE_DEVICES=all (alternative to RequireGPU) - MaxNodesPerEntry int // Max node names per topology entry (0 = unlimited) - OS string // Recipe OS criteria value. When set to oskind.Talos, systemd hostPath mounts are skipped and the in-pod agent uses the Talos service backend. + + // RunID scopes every resource this Deployer creates to a single run, + // so concurrent snapshot-agent runs never collide on a shared resource + // name. Callers generate it with runid.Generate() before deploying. + RunID string + + // NameBase prefixes generated resource names only — it has no effect + // when ServiceAccountName or JobName is already set. Defaults to + // "aicr" when empty. + NameBase string + + Image string + ImagePullSecrets []string + NodeSelector map[string]string + Tolerations []corev1.Toleration + Output string + Debug bool + Privileged bool // If true, run with privileged security context (required for GPU/SystemD collectors) + RequireGPU bool // If true, request nvidia.com/gpu resource (required for CDI environments) + RuntimeClassName string // If set, use this runtimeClassName on the pod and inject NVIDIA_VISIBLE_DEVICES=all (alternative to RequireGPU) + MaxNodesPerEntry int // Max node names per topology entry (0 = unlimited) + OS string // Recipe OS criteria value. When set to oskind.Talos, systemd hostPath mounts are skipped and the in-pod agent uses the Talos service backend. // ClusterConfigPath, when set, forwards to the in-pod network // collector via AICR_CLUSTER_CONFIG_PATH so it ingests an existing From c72afe27ef809317d49cc7b2991bfe7eaa0b0127 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Fri, 21 Aug 2026 14:33:40 -0700 Subject: [PATCH 03/56] fix(agent): empty runID must not yield a trailing-dash name Signed-off-by: Alex Yuskauskas --- pkg/k8s/agent/names.go | 8 ++++++++ pkg/k8s/agent/names_test.go | 23 ++++++++++++++++------- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/pkg/k8s/agent/names.go b/pkg/k8s/agent/names.go index 763f1b554..4959c1a41 100644 --- a/pkg/k8s/agent/names.go +++ b/pkg/k8s/agent/names.go @@ -34,10 +34,18 @@ const staticStagingConfigMapName = "aicr-snapshot" // nameWithRunID joins prefix and runID, truncating prefix so the result fits // within the Kubernetes name ceiling. An empty prefix yields the bare runID. +// An empty runID yields the bare (trimmed) prefix rather than appending a +// trailing "-": a trailing separator would leave a Kubernetes object name +// that fails validation (names must end in an alphanumeric character), and +// falling back to the unscoped prefix also keeps deploys working between +// this task and the task that wires Config.RunID through every caller. func nameWithRunID(prefix, runID string) string { if prefix == "" { return runID } + if runID == "" { + return prefix + } budget := defaults.MaxK8sNameLength - len(runID) - 1 if budget < 0 { budget = 0 diff --git a/pkg/k8s/agent/names_test.go b/pkg/k8s/agent/names_test.go index 047e9ce28..570a6194a 100644 --- a/pkg/k8s/agent/names_test.go +++ b/pkg/k8s/agent/names_test.go @@ -25,23 +25,32 @@ func TestNameWithRunID(t *testing.T) { tests := []struct { name string prefix string + runID string want string }{ - {"short prefix", "aicr", "aicr-" + runID}, - {"exactly at budget", strings.Repeat("a", 30), strings.Repeat("a", 30) + "-" + runID}, - {"over budget truncates", strings.Repeat("b", 40), strings.Repeat("b", 30) + "-" + runID}, - {"trailing dash trimmed", strings.Repeat("c", 29) + "-", strings.Repeat("c", 29) + "-" + runID}, - {"empty prefix", "", runID}, + {"short prefix", "aicr", runID, "aicr-" + runID}, + {"exactly at budget", strings.Repeat("a", 30), runID, strings.Repeat("a", 30) + "-" + runID}, + {"over budget truncates", strings.Repeat("b", 40), runID, strings.Repeat("b", 30) + "-" + runID}, + {"trailing dash trimmed", strings.Repeat("c", 29) + "-", runID, strings.Repeat("c", 29) + "-" + runID}, + {"empty prefix", "", runID, runID}, + // A zero-value Config.RunID (before a caller wires it in) must fall + // back to the bare prefix, never a prefix with a trailing "-" — that + // would be an invalid Kubernetes object name. + {"empty runID falls back to bare prefix", "aicr", "", "aicr"}, + {"empty prefix and empty runID", "", "", ""}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := nameWithRunID(tt.prefix, runID) + got := nameWithRunID(tt.prefix, tt.runID) if got != tt.want { - t.Errorf("nameWithRunID(%q, runID) = %q, want %q", tt.prefix, got, tt.want) + t.Errorf("nameWithRunID(%q, %q) = %q, want %q", tt.prefix, tt.runID, got, tt.want) } if len(got) > 63 { t.Errorf("len = %d, exceeds 63-char ceiling", len(got)) } + if strings.HasSuffix(got, "-") { + t.Errorf("nameWithRunID(%q, %q) = %q, ends in a trailing separator (invalid Kubernetes name)", tt.prefix, tt.runID, got) + } }) } } From 675a8fe3f5ed65bf8825da891a2caf67bb318566 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Fri, 21 Aug 2026 14:52:35 -0700 Subject: [PATCH 04/56] feat(agent): create run-scoped resources and drop create-or-update Every ensure* function in rbac.go and job.go now issues a single Create against the run-scoped name (Task 2's jobName/saName/roleName/ clusterRoleName methods) instead of create-or-update / delete-and- recreate. Because names now carry the run ID, concurrent runs can no longer collide on a shared object, so the guard logic that handled collisions is removed rather than preserved: - rbac.go: the five ensure* functions drop their apierrors.IsAlreadyExists + Update branches; AlreadyExists now returns an error naming the duplicate-RunID scenario. ensureServiceAccount additionally Gets the bare (unscoped) prefix name first and slog.Warns if it already exists, since --service-account-name callers relying on IgnoreAlreadyExists adoption of a pre-created ServiceAccount now silently get a fresh run-owned one instead. The five delete* helpers (still called by Cleanup) now delete by the same run-scoped name the ensure* functions create, so cleanup keeps finding what it created. ensureNamespace is unchanged (namespace is ensured, not run-owned). - job.go: ensureJob no longer deletes an existing Job before creating; buildJob names the Job via jobName() and applies objectLabels() to both Job.ObjectMeta and Job.Spec.Template.ObjectMeta (Job labels do not propagate to its Pods). buildPodSpec now sets ServiceAccountName from saName() to match the ServiceAccount ensureServiceAccount actually creates. deleteJob deletes by jobName(). waitForJobDeletion is deleted along with its "k8s.io/apimachinery/pkg/watch" import. - types.go: adds (*Deployer) objectLabels(), the standard label set (labels.Name/ManagedBy/Component/RunID) attached to every object this Deployer creates. Deleted tests (assert removed create-or-update / delete-and-recreate behavior, not a regression): - TestDeployer_EnsureRBAC_Idempotent: asserted that calling ensureServiceAccount twice was a idempotent no-op; a second call now correctly errors (AlreadyExists under a run-scoped name means a bug, not a legitimate re-run). - TestDeployer_EnsureJob's "recreate Job deletes old one" subtest: asserted ensureJob deletes-and-recreates; it now errors on the second call for the same reason. - job_watch_test.go's four TestWaitForJobDeletion_* tests: exercised the deleted waitForJobDeletion function. Added TestDeployUsesRunScopedNamesAndLabels (deployer_test.go) verifying Deploy() creates the Job and ClusterRole under run-scoped names and that the Job's pod template (not just the Job object) carries all four labels. Signed-off-by: Alex Yuskauskas --- pkg/k8s/agent/deployer_test.go | 100 +++++++++++++------------- pkg/k8s/agent/job.go | 103 ++++----------------------- pkg/k8s/agent/job_watch_test.go | 121 -------------------------------- pkg/k8s/agent/rbac.go | 92 ++++++++++++++---------- pkg/k8s/agent/types.go | 15 ++++ 5 files changed, 130 insertions(+), 301 deletions(-) diff --git a/pkg/k8s/agent/deployer_test.go b/pkg/k8s/agent/deployer_test.go index 3a6a94404..be177a890 100644 --- a/pkg/k8s/agent/deployer_test.go +++ b/pkg/k8s/agent/deployer_test.go @@ -26,6 +26,7 @@ import ( "time" aicrerrors "github.com/NVIDIA/aicr/pkg/errors" + "github.com/NVIDIA/aicr/pkg/k8s/labels" "github.com/NVIDIA/aicr/pkg/k8s/pod" authv1 "k8s.io/api/authorization/v1" corev1 "k8s.io/api/core/v1" @@ -300,38 +301,6 @@ func TestDeployer_EnsureRBAC(t *testing.T) { }) } -func TestDeployer_EnsureRBAC_Idempotent(t *testing.T) { - clientset := fake.NewClientset() - config := Config{ - Namespace: "test-namespace", - ServiceAccountName: testName, - JobName: testName, - Image: "ghcr.io/nvidia/aicr-validator:latest", - Output: "cm://test-namespace/aicr-snapshot", - } - deployer := NewDeployer(clientset, config) - ctx := context.Background() - - // Create resources twice - second call should be idempotent - if err := deployer.ensureServiceAccount(ctx); err != nil { - t.Fatalf("first create failed: %v", err) - } - - if err := deployer.ensureServiceAccount(ctx); err != nil { - t.Fatalf("second create failed (not idempotent): %v", err) - } - - // Verify only one ServiceAccount exists - saList, err := clientset.CoreV1().ServiceAccounts(config.Namespace). - List(ctx, metav1.ListOptions{}) - if err != nil { - t.Fatalf("failed to list ServiceAccounts: %v", err) - } - if len(saList.Items) != 1 { - t.Errorf("expected 1 ServiceAccount, got %d", len(saList.Items)) - } -} - func TestDeployer_EnsureJob(t *testing.T) { clientset := fake.NewClientset() config := Config{ @@ -408,26 +377,6 @@ func TestDeployer_EnsureJob(t *testing.T) { t.Errorf("expected 3 volumes, got %d", len(job.Spec.Template.Spec.Volumes)) } }) - - t.Run("recreate Job deletes old one", func(t *testing.T) { - // Create Job first time - if err := deployer.ensureJob(ctx); err != nil { - t.Fatalf("first create failed: %v", err) - } - - // Create Job second time - should delete and recreate - if err := deployer.ensureJob(ctx); err != nil { - t.Fatalf("second create failed: %v", err) - } - - // Verify Job still exists (fake client doesn't support watch/wait, - // but we can verify the Job exists) - _, err := clientset.BatchV1().Jobs(config.Namespace). - Get(ctx, config.JobName, metav1.GetOptions{}) - if err != nil { - t.Errorf("Job should exist after recreate: %v", err) - } - }) } func TestDeployer_EnsureJob_Unprivileged(t *testing.T) { @@ -582,6 +531,53 @@ func TestDeployer_Deploy(t *testing.T) { } } +// TestDeployUsesRunScopedNamesAndLabels verifies that Deploy() creates every +// object under a run-scoped name (prefix-runID) and that the Job's pod +// template carries the full label set, not just the Job object itself — +// Job labels do not propagate to the Pods a Job creates. +// +// Deviation from the plan's literal test: the brief's snippet omits the +// SelfSubjectAccessReview-allow reactor that every other Deploy()-calling +// test in this file installs. The fake clientset denies all permission +// checks by default (Status.Allowed defaults to false), so Deploy() fails +// at the Step-0 CheckPermissions gate before creating anything — a false +// RED unrelated to run-scoped naming. Added the same reactor used by +// TestDeployer_Deploy et al. so the test exercises the behavior it names. +func TestDeployUsesRunScopedNamesAndLabels(t *testing.T) { + ctx := context.Background() + client := fake.NewSimpleClientset() + client.PrependReactor("create", "selfsubjectaccessreviews", func(action k8stesting.Action) (bool, runtime.Object, error) { + return true, &authv1.SelfSubjectAccessReview{ + Status: authv1.SubjectAccessReviewStatus{ + Allowed: true, + Reason: "test permissions allowed", + }, + }, nil + }) + d := NewDeployer(client, Config{ + Namespace: "test-ns", + Image: "aicr:test", + RunID: "20260821-142233-9f3a1c0b7e2d4a55", + }) + if err := d.Deploy(ctx); err != nil { + t.Fatalf("Deploy() error = %v", err) + } + + wantSuffix := "-20260821-142233-9f3a1c0b7e2d4a55" + job, err := client.BatchV1().Jobs("test-ns").Get(ctx, "aicr"+wantSuffix, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Job not found under run-scoped name: %v", err) + } + for _, key := range []string{labels.Name, labels.ManagedBy, labels.Component, labels.RunID} { + if _, ok := job.Spec.Template.Labels[key]; !ok { + t.Errorf("pod template missing label %q", key) + } + } + if _, err := client.RbacV1().ClusterRoles().Get(ctx, "aicr-node-reader"+wantSuffix, metav1.GetOptions{}); err != nil { + t.Errorf("ClusterRole not found under run-scoped name: %v", err) + } +} + func TestDeployer_Cleanup(t *testing.T) { clientset := fake.NewClientset() diff --git a/pkg/k8s/agent/job.go b/pkg/k8s/agent/job.go index 5a2c06e7f..f6d99ef46 100644 --- a/pkg/k8s/agent/job.go +++ b/pkg/k8s/agent/job.go @@ -28,37 +28,17 @@ import ( "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/watch" "k8s.io/utils/ptr" ) -// ensureJob deletes any existing Job and creates a fresh one. +// ensureJob creates the run-scoped agent Job. func (d *Deployer) ensureJob(ctx context.Context) error { - // Delete existing Job if present - propagationPolicy := metav1.DeletePropagationForeground - err := d.clientset.BatchV1().Jobs(d.config.Namespace).Delete( - ctx, - d.config.JobName, - metav1.DeleteOptions{ - PropagationPolicy: &propagationPolicy, - }, - ) - if err != nil && !errors.IsNotFound(err) { - return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to delete existing Job", err) - } - - // Wait for Job to be fully deleted - jobExisted := err == nil // Job existed and was deleted - if jobExisted { - if waitErr := d.waitForJobDeletion(ctx); waitErr != nil { - return aicrerrors.Wrap(aicrerrors.ErrCodeTimeout, "timeout waiting for Job deletion", waitErr) - } - } - - // Create fresh Job job := d.buildJob() - _, err = d.clientset.BatchV1().Jobs(d.config.Namespace). + _, err := d.clientset.BatchV1().Jobs(d.config.Namespace). Create(ctx, job, metav1.CreateOptions{}) + if errors.IsAlreadyExists(err) { + return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "Job already exists under run-scoped name (duplicate RunID?)", err) + } if err != nil { return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to create Job", err) } @@ -79,11 +59,9 @@ func (d *Deployer) buildJob() *batchv1.Job { return &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ - Name: d.config.JobName, + Name: d.jobName(), Namespace: d.config.Namespace, - Labels: map[string]string{ - labelAppName: appName, - }, + Labels: d.objectLabels(), }, Spec: batchv1.JobSpec{ Completions: ptr.To(int32(1)), @@ -94,9 +72,10 @@ func (d *Deployer) buildJob() *batchv1.Job { ActiveDeadlineSeconds: ptr.To(int64(defaults.AgentJobActiveDeadline.Seconds())), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - labelAppName: appName, - }, + // Job metadata.labels do NOT propagate to the Pods a Job + // creates — the pod template needs its own copy so + // label-selector pod lookups (findPodName et al.) work. + Labels: d.objectLabels(), }, Spec: podSpec, }, @@ -109,7 +88,7 @@ func (d *Deployer) buildJob() *batchv1.Job { // When Privileged=false: PSS-compliant restricted pod, only K8s collector works. func (d *Deployer) buildPodSpec(args []string) corev1.PodSpec { spec := corev1.PodSpec{ - ServiceAccountName: d.config.ServiceAccountName, + ServiceAccountName: d.saName(), RestartPolicy: corev1.RestartPolicyNever, NodeSelector: d.config.NodeSelector, Tolerations: d.config.Tolerations, @@ -353,7 +332,7 @@ func (d *Deployer) deleteJob(ctx context.Context) error { propagationPolicy := metav1.DeletePropagationForeground err := d.clientset.BatchV1().Jobs(d.config.Namespace).Delete( ctx, - d.config.JobName, + d.jobName(), metav1.DeleteOptions{ PropagationPolicy: &propagationPolicy, }, @@ -361,62 +340,6 @@ func (d *Deployer) deleteJob(ctx context.Context) error { return k8s.IgnoreNotFound(err) } -// waitForJobDeletion waits for the Job to be fully deleted using the watch API. -// Returns nil when the Job is observed deleted (Get returns NotFound, or a -// watch.Deleted event is received). Returns ErrCodeTimeout if the cleanup -// deadline elapses before deletion is observed. -func (d *Deployer) waitForJobDeletion(ctx context.Context) error { - timeoutCtx, cancel := context.WithTimeout(ctx, defaults.K8sCleanupTimeout) - defer cancel() - - // Fast path: already deleted. Note: IgnoreNotFound(nil) returns nil, - // so check NotFound explicitly — otherwise a successful Get (Job still - // exists) would incorrectly short-circuit as "deleted". - current, err := d.clientset.BatchV1().Jobs(d.config.Namespace). - Get(timeoutCtx, d.config.JobName, metav1.GetOptions{}) - if errors.IsNotFound(err) { - return nil - } - if err != nil { - return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to get Job", err) - } - - watcher, err := d.clientset.BatchV1().Jobs(d.config.Namespace).Watch(timeoutCtx, metav1.ListOptions{ - FieldSelector: "metadata.name=" + d.config.JobName, - ResourceVersion: current.ResourceVersion, - }) - if err != nil { - return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to watch Job", err) - } - defer watcher.Stop() - - for { - select { - case <-timeoutCtx.Done(): - return aicrerrors.Wrap(aicrerrors.ErrCodeTimeout, "Job deletion wait timeout", timeoutCtx.Err()) - case event, ok := <-watcher.ResultChan(): - if !ok { - // Channel closed; verify with a Get to handle missed events. - // Use explicit NotFound check (IgnoreNotFound(nil) returns nil - // and would falsely report success when the Job still exists). - _, getErr := d.clientset.BatchV1().Jobs(d.config.Namespace). - Get(timeoutCtx, d.config.JobName, metav1.GetOptions{}) - if errors.IsNotFound(getErr) { - return nil - } - if getErr != nil { - return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "Job watch channel closed", getErr) - } - return aicrerrors.Wrap(aicrerrors.ErrCodeUnavailable, - "Job watch channel closed before deletion observed", nil) - } - if event.Type == watch.Deleted { - return nil - } - } - } -} - // mustParseQuantity parses a resource quantity or panics. func mustParseQuantity(s string) resource.Quantity { q := resource.MustParse(s) diff --git a/pkg/k8s/agent/job_watch_test.go b/pkg/k8s/agent/job_watch_test.go index 01e96822e..d28ea2f92 100644 --- a/pkg/k8s/agent/job_watch_test.go +++ b/pkg/k8s/agent/job_watch_test.go @@ -21,7 +21,6 @@ import ( "time" aicrerrors "github.com/NVIDIA/aicr/pkg/errors" - batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/watch" @@ -34,126 +33,6 @@ import ( // event fails the test instead of stalling the suite. const jobWatchTimeout = 5 * time.Second -// TestWaitForJobDeletion_AlreadyDeleted exercises the fast-path Get returning -// NotFound (no Job exists in the clientset). -func TestWaitForJobDeletion_AlreadyDeleted(t *testing.T) { - t.Parallel() - - clientset := fake.NewClientset() // no jobs - d := NewDeployer(clientset, Config{ - Namespace: "test-ns", - JobName: "test-job", - }) - - if err := d.waitForJobDeletion(context.Background()); err != nil { - t.Fatalf("expected nil for already-deleted Job, got %v", err) - } -} - -// TestWaitForJobDeletion_DeletedEvent exercises the watch.Deleted path: the -// Job is present at Get time, then the fake watcher emits a Deleted event. -func TestWaitForJobDeletion_DeletedEvent(t *testing.T) { - t.Parallel() - - job := &batchv1.Job{ - ObjectMeta: metav1.ObjectMeta{Name: "test-job", Namespace: "test-ns", ResourceVersion: "1"}, - } - clientset := fake.NewClientset(job) - - w := watch.NewFake() - clientset.PrependWatchReactor("jobs", k8stesting.DefaultWatchReactor(w, nil)) - - go func() { - // Unbuffered FakeWatcher channel: Delete blocks until consumed. - w.Delete(job) - }() - - d := NewDeployer(clientset, Config{ - Namespace: "test-ns", - JobName: "test-job", - }) - - if err := d.waitForJobDeletion(context.Background()); err != nil { - t.Fatalf("expected nil after Deleted event, got %v", err) - } -} - -// TestWaitForJobDeletion_ChannelCloseWithNotFound exercises the fallback path -// where the watch channel closes without a Deleted event but a follow-up Get -// shows the Job has been deleted. -func TestWaitForJobDeletion_ChannelCloseWithNotFound(t *testing.T) { - t.Parallel() - - job := &batchv1.Job{ - ObjectMeta: metav1.ObjectMeta{Name: "test-job", Namespace: "test-ns", ResourceVersion: "1"}, - } - clientset := fake.NewClientset(job) - - w := watch.NewFake() - clientset.PrependWatchReactor("jobs", k8stesting.DefaultWatchReactor(w, nil)) - - go func() { - // Delete the Job from the fake clientset, then close the watch - // channel without firing a Deleted event so the close-fallback Get - // returns NotFound. - _ = clientset.BatchV1().Jobs("test-ns").Delete( - context.Background(), "test-job", metav1.DeleteOptions{}, - ) - w.Stop() - }() - - d := NewDeployer(clientset, Config{ - Namespace: "test-ns", - JobName: "test-job", - }) - - if err := d.waitForJobDeletion(context.Background()); err != nil { - t.Fatalf("expected nil when channel closes after deletion, got %v", err) - } -} - -// TestWaitForJobDeletion_ContextCanceled exercises the timeout branch when -// the parent context is canceled while the watcher remains idle. -func TestWaitForJobDeletion_ContextCanceled(t *testing.T) { - t.Parallel() - - job := &batchv1.Job{ - ObjectMeta: metav1.ObjectMeta{Name: "test-job", Namespace: "test-ns", ResourceVersion: "1"}, - } - clientset := fake.NewClientset(job) - - // Empty fake watcher that never emits events; force the select to - // block on ctx.Done() rather than racing against a default Added event. - w := watch.NewFake() - clientset.PrependWatchReactor("jobs", k8stesting.DefaultWatchReactor(w, nil)) - - d := NewDeployer(clientset, Config{ - Namespace: "test-ns", - JobName: "test-job", - }) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - // Cancel after a short delay so Get + Watch setup completes first. - go func() { - // Wait until the watcher has at least one consumer, signaled by - // the watcher's no-op behavior. Cancel to force the timeout branch. - cancel() - }() - - err := d.waitForJobDeletion(ctx) - if err == nil { - t.Fatal("expected error for canceled context") - } - var sErr *aicrerrors.StructuredError - if !stderrors.As(err, &sErr) { - t.Fatalf("expected *StructuredError, got %T: %v", err, err) - } - if sErr.Code != aicrerrors.ErrCodeTimeout { - t.Errorf("expected ErrCodeTimeout, got %v", sErr.Code) - } -} - // TestFindOrWatchPodName_WatchAddedEvent exercises the watch path: List // returns no matching pods, then a fake watcher emits an Added event. func TestFindOrWatchPodName_WatchAddedEvent(t *testing.T) { diff --git a/pkg/k8s/agent/rbac.go b/pkg/k8s/agent/rbac.go index 1e6780938..7a8ba5357 100644 --- a/pkg/k8s/agent/rbac.go +++ b/pkg/k8s/agent/rbac.go @@ -17,6 +17,7 @@ package agent import ( "context" "fmt" + "log/slog" "github.com/NVIDIA/aicr/pkg/errors" "github.com/NVIDIA/aicr/pkg/k8s" @@ -79,26 +80,54 @@ func (d *Deployer) ensureNamespace(ctx context.Context) error { return nil } -// ensureServiceAccount creates the ServiceAccount for the agent. -// If the ServiceAccount already exists, this is a no-op (idempotent). +// ensureServiceAccount creates the run-scoped ServiceAccount for the agent. +// +// Before creating, it checks whether a ServiceAccount already exists under +// the bare (unscoped) prefix name. Previously, a caller passing +// --service-account-name to target a ServiceAccount they created out of +// band (e.g. with cloud IAM annotations for IRSA/Workload Identity) got it +// silently adopted via IgnoreAlreadyExists. Now every run gets its own +// run-scoped ServiceAccount, so that adoption no longer happens; warn +// loudly instead of leaving the caller to discover it the hard way. A +// NotFound Get is the normal path and stays silent. func (d *Deployer) ensureServiceAccount(ctx context.Context) error { + name := d.saName() + bareName := d.config.ServiceAccountName + if bareName == "" { + bareName = d.base() + } + if _, err := d.clientset.CoreV1().ServiceAccounts(d.config.Namespace).Get(ctx, bareName, metav1.GetOptions{}); err == nil { + slog.Warn("ServiceAccount already exists under the unscoped name; aicr is creating a run-scoped ServiceAccount instead of adopting it", + "existing", bareName, "creating", name) + } else if !apierrors.IsNotFound(err) { + return errors.Wrap(errors.ErrCodeInternal, "failed to check for pre-existing ServiceAccount", err) + } + sa := &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ - Name: d.config.ServiceAccountName, + Name: name, Namespace: d.config.Namespace, + Labels: d.objectLabels(), }, } _, err := d.clientset.CoreV1().ServiceAccounts(d.config.Namespace).Create(ctx, sa, metav1.CreateOptions{}) - return k8s.IgnoreAlreadyExists(err) + if apierrors.IsAlreadyExists(err) { + return errors.Wrap(errors.ErrCodeInternal, "ServiceAccount already exists under run-scoped name (duplicate RunID?)", err) + } + if err != nil { + return errors.Wrap(errors.ErrCodeInternal, "failed to create ServiceAccount", err) + } + return nil } -// ensureRole creates or updates the Role for ConfigMap access. +// ensureRole creates the run-scoped Role for ConfigMap access. func (d *Deployer) ensureRole(ctx context.Context) error { role := &rbacv1.Role{ ObjectMeta: metav1.ObjectMeta{ - Name: d.config.ServiceAccountName, + Name: d.roleName(), Namespace: d.config.Namespace, + Labels: d.objectLabels(), }, Rules: []rbacv1.PolicyRule{ { @@ -116,11 +145,7 @@ func (d *Deployer) ensureRole(ctx context.Context) error { _, err := d.clientset.RbacV1().Roles(d.config.Namespace).Create(ctx, role, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { - _, err = d.clientset.RbacV1().Roles(d.config.Namespace).Update(ctx, role, metav1.UpdateOptions{}) - if err != nil { - return errors.Wrap(errors.ErrCodeInternal, "failed to update Role", err) - } - return nil + return errors.Wrap(errors.ErrCodeInternal, "Role already exists under run-scoped name (duplicate RunID?)", err) } if err != nil { return errors.Wrap(errors.ErrCodeInternal, "failed to create Role", err) @@ -128,34 +153,31 @@ func (d *Deployer) ensureRole(ctx context.Context) error { return nil } -// ensureRoleBinding creates or updates the RoleBinding to bind the Role to the ServiceAccount. +// ensureRoleBinding creates the run-scoped RoleBinding binding the Role to the ServiceAccount. func (d *Deployer) ensureRoleBinding(ctx context.Context) error { rb := &rbacv1.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ - Name: d.config.ServiceAccountName, + Name: d.roleName(), Namespace: d.config.Namespace, + Labels: d.objectLabels(), }, Subjects: []rbacv1.Subject{ { Kind: "ServiceAccount", - Name: d.config.ServiceAccountName, + Name: d.saName(), Namespace: d.config.Namespace, }, }, RoleRef: rbacv1.RoleRef{ APIGroup: rbacAPIGroup, Kind: "Role", - Name: d.config.ServiceAccountName, + Name: d.roleName(), }, } _, err := d.clientset.RbacV1().RoleBindings(d.config.Namespace).Create(ctx, rb, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { - _, err = d.clientset.RbacV1().RoleBindings(d.config.Namespace).Update(ctx, rb, metav1.UpdateOptions{}) - if err != nil { - return errors.Wrap(errors.ErrCodeInternal, "failed to update RoleBinding", err) - } - return nil + return errors.Wrap(errors.ErrCodeInternal, "RoleBinding already exists under run-scoped name (duplicate RunID?)", err) } if err != nil { return errors.Wrap(errors.ErrCodeInternal, "failed to create RoleBinding", err) @@ -163,7 +185,7 @@ func (d *Deployer) ensureRoleBinding(ctx context.Context) error { return nil } -// ensureClusterRole creates or updates the ClusterRole for node and cluster-wide resource access. +// ensureClusterRole creates the run-scoped ClusterRole for node and cluster-wide resource access. func (d *Deployer) ensureClusterRole(ctx context.Context) error { rules := []rbacv1.PolicyRule{ { @@ -211,18 +233,15 @@ func (d *Deployer) ensureClusterRole(ctx context.Context) error { cr := &rbacv1.ClusterRole{ ObjectMeta: metav1.ObjectMeta{ - Name: d.clusterRoleName(), + Name: d.clusterRoleName(), + Labels: d.objectLabels(), }, Rules: rules, } _, err := d.clientset.RbacV1().ClusterRoles().Create(ctx, cr, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { - _, err = d.clientset.RbacV1().ClusterRoles().Update(ctx, cr, metav1.UpdateOptions{}) - if err != nil { - return errors.Wrap(errors.ErrCodeInternal, "failed to update ClusterRole", err) - } - return nil + return errors.Wrap(errors.ErrCodeInternal, "ClusterRole already exists under run-scoped name (duplicate RunID?)", err) } if err != nil { return errors.Wrap(errors.ErrCodeInternal, "failed to create ClusterRole", err) @@ -230,16 +249,17 @@ func (d *Deployer) ensureClusterRole(ctx context.Context) error { return nil } -// ensureClusterRoleBinding creates or updates the ClusterRoleBinding to bind the ClusterRole to the ServiceAccount. +// ensureClusterRoleBinding creates the run-scoped ClusterRoleBinding binding the ClusterRole to the ServiceAccount. func (d *Deployer) ensureClusterRoleBinding(ctx context.Context) error { crb := &rbacv1.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ - Name: d.clusterRoleName(), + Name: d.clusterRoleName(), + Labels: d.objectLabels(), }, Subjects: []rbacv1.Subject{ { Kind: "ServiceAccount", - Name: d.config.ServiceAccountName, + Name: d.saName(), Namespace: d.config.Namespace, }, }, @@ -252,11 +272,7 @@ func (d *Deployer) ensureClusterRoleBinding(ctx context.Context) error { _, err := d.clientset.RbacV1().ClusterRoleBindings().Create(ctx, crb, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { - _, err = d.clientset.RbacV1().ClusterRoleBindings().Update(ctx, crb, metav1.UpdateOptions{}) - if err != nil { - return errors.Wrap(errors.ErrCodeInternal, "failed to update ClusterRoleBinding", err) - } - return nil + return errors.Wrap(errors.ErrCodeInternal, "ClusterRoleBinding already exists under run-scoped name (duplicate RunID?)", err) } if err != nil { return errors.Wrap(errors.ErrCodeInternal, "failed to create ClusterRoleBinding", err) @@ -268,7 +284,7 @@ func (d *Deployer) ensureClusterRoleBinding(ctx context.Context) error { // If the ServiceAccount doesn't exist, this is a no-op (idempotent). func (d *Deployer) deleteServiceAccount(ctx context.Context) error { err := d.clientset.CoreV1().ServiceAccounts(d.config.Namespace). - Delete(ctx, d.config.ServiceAccountName, metav1.DeleteOptions{}) + Delete(ctx, d.saName(), metav1.DeleteOptions{}) return k8s.IgnoreNotFound(err) } @@ -276,7 +292,7 @@ func (d *Deployer) deleteServiceAccount(ctx context.Context) error { // If the Role doesn't exist, this is a no-op (idempotent). func (d *Deployer) deleteRole(ctx context.Context) error { err := d.clientset.RbacV1().Roles(d.config.Namespace). - Delete(ctx, d.config.ServiceAccountName, metav1.DeleteOptions{}) + Delete(ctx, d.roleName(), metav1.DeleteOptions{}) return k8s.IgnoreNotFound(err) } @@ -284,7 +300,7 @@ func (d *Deployer) deleteRole(ctx context.Context) error { // If the RoleBinding doesn't exist, this is a no-op (idempotent). func (d *Deployer) deleteRoleBinding(ctx context.Context) error { err := d.clientset.RbacV1().RoleBindings(d.config.Namespace). - Delete(ctx, d.config.ServiceAccountName, metav1.DeleteOptions{}) + Delete(ctx, d.roleName(), metav1.DeleteOptions{}) return k8s.IgnoreNotFound(err) } diff --git a/pkg/k8s/agent/types.go b/pkg/k8s/agent/types.go index f0f8312cf..673b8590b 100644 --- a/pkg/k8s/agent/types.go +++ b/pkg/k8s/agent/types.go @@ -15,6 +15,7 @@ package agent import ( + "github.com/NVIDIA/aicr/pkg/k8s/labels" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" ) @@ -102,6 +103,20 @@ func NewDeployer(clientset kubernetes.Interface, config Config) *Deployer { } } +// objectLabels returns the standard label set applied to every run-owned +// object this Deployer creates: the ServiceAccount, Role, RoleBinding, +// ClusterRole, ClusterRoleBinding, Job, and the Job's pod template. Each +// call returns a fresh map so callers attaching it to two objects (e.g. a +// Job and its pod template) never alias the same underlying map. +func (d *Deployer) objectLabels() map[string]string { + return map[string]string{ + labels.Name: labels.ValueAICR, + labels.ManagedBy: labels.ValueAICR, + labels.Component: labels.ValueSnapshotAgent, + labels.RunID: d.config.RunID, + } +} + // CleanupOptions controls what resources to remove during cleanup. type CleanupOptions struct { Enabled bool // If true, removes Job and all RBAC resources From dc2b9f10d7c41247f68ebf4deefc1f2c34cb88e4 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Fri, 21 Aug 2026 15:18:10 -0700 Subject: [PATCH 05/56] feat(agent): scope cleanup to created objects with UID preconditions Signed-off-by: Alex Yuskauskas --- pkg/k8s/agent/deployer.go | 69 +++++++-- pkg/k8s/agent/deployer_test.go | 253 +++++++++++++++++++++++++++++++++ pkg/k8s/agent/job.go | 17 ++- pkg/k8s/agent/permissions.go | 1 + pkg/k8s/agent/rbac.go | 71 +++++---- pkg/k8s/agent/types.go | 80 +++++++++++ pkg/k8s/agent/wait.go | 24 ++++ 7 files changed, 466 insertions(+), 49 deletions(-) diff --git a/pkg/k8s/agent/deployer.go b/pkg/k8s/agent/deployer.go index 36db0a6de..bd41b2104 100644 --- a/pkg/k8s/agent/deployer.go +++ b/pkg/k8s/agent/deployer.go @@ -94,30 +94,39 @@ func (d *Deployer) GetSnapshot(ctx context.Context) ([]byte, error) { return d.getSnapshotFromConfigMap(ctx) } -// Cleanup removes the agent Job and RBAC resources. -// If opts.Enabled is false, no cleanup is performed (resources are kept for debugging). -// All resources are attempted for deletion even if some fail, and a combined error is returned. -// Deletions are fanned out concurrently so a slow apiserver does not serialize the wall clock. +// Cleanup removes exactly the objects this Deployer created: the Job, the +// RBAC resources, and — when this Deployer owns the output ConfigMap — the +// staging ConfigMap. If opts.Enabled is false, no cleanup is performed +// (resources are kept for debugging). All resources are attempted for +// deletion even if some fail, and a combined error is returned. Deletions +// are fanned out concurrently so a slow apiserver does not serialize the +// wall clock. func (d *Deployer) Cleanup(ctx context.Context, opts CleanupOptions) error { if !opts.Enabled { return nil } + // Build the task list from what this Deployer actually created, not + // from configured names — a run must never delete an object it did + // not create (e.g. a same-named object left behind by an unrelated + // run or user). Each delete is additionally pinned to the recorded + // UID via metav1.Preconditions. + created := d.createdSnapshot() + type result struct { label string err error } - tasks := []struct { + tasks := make([]struct { label string op func(context.Context) error - }{ - {fmt.Sprintf("Job %q", d.config.JobName), d.deleteJob}, - {fmt.Sprintf("ServiceAccount %q", d.config.ServiceAccountName), d.deleteServiceAccount}, - {fmt.Sprintf("Role %q", d.config.ServiceAccountName), d.deleteRole}, - {fmt.Sprintf("RoleBinding %q", d.config.ServiceAccountName), d.deleteRoleBinding}, - {fmt.Sprintf("ClusterRole %q", d.clusterRoleName()), d.deleteClusterRole}, - {fmt.Sprintf("ClusterRoleBinding %q", d.clusterRoleName()), d.deleteClusterRoleBinding}, + }, len(created)) + for i, obj := range created { + tasks[i].label = fmt.Sprintf("%s %q", obj.kind, obj.name) + tasks[i].op = func(ctx context.Context) error { + return d.deleteCreatedObject(ctx, obj) + } } // sync.WaitGroup (not errgroup) is intentional here: cleanup must @@ -158,6 +167,42 @@ func (d *Deployer) Cleanup(ctx context.Context, opts CleanupOptions) error { return nil } +// deleteCreatedObject deletes a single created-set entry by dispatching to +// the resource-specific delete call for obj.kind, passing obj.name and +// obj.uid through so every delete is UID-pinned. +func (d *Deployer) deleteCreatedObject(ctx context.Context, obj createdObject) error { + switch obj.kind { + case kindJob: + return d.deleteJob(ctx, obj.name, obj.uid) + case kindServiceAccount: + return d.deleteServiceAccount(ctx, obj.name, obj.uid) + case kindRole: + return d.deleteRole(ctx, obj.name, obj.uid) + case kindRoleBinding: + return d.deleteRoleBinding(ctx, obj.name, obj.uid) + case kindClusterRole: + return d.deleteClusterRole(ctx, obj.name, obj.uid) + case kindClusterRoleBinding: + return d.deleteClusterRoleBinding(ctx, obj.name, obj.uid) + case kindConfigMap: + return d.deleteStagingConfigMap(ctx, obj.name, obj.uid) + default: + return aicrerrors.New(aicrerrors.ErrCodeInternal, fmt.Sprintf("cleanup: unknown created-object kind %q", obj.kind)) + } +} + +// ignoreNotFoundOrConflict returns nil when err is "not found" (already +// deleted) or "conflict" (the UID precondition did not match — some other +// object now holds this name; it has already been replaced and is not +// ours to delete). Both are success from Cleanup's perspective: the object +// this run created is gone. +func ignoreNotFoundOrConflict(err error) error { + if k8serrors.IsNotFound(err) || k8serrors.IsConflict(err) { + return nil + } + return err +} + // validateRuntimeClass checks that the specified RuntimeClass exists in the cluster. func (d *Deployer) validateRuntimeClass(ctx context.Context) error { ctx, cancel := context.WithTimeout(ctx, defaults.RuntimeClassCheckTimeout) diff --git a/pkg/k8s/agent/deployer_test.go b/pkg/k8s/agent/deployer_test.go index be177a890..04d4508bb 100644 --- a/pkg/k8s/agent/deployer_test.go +++ b/pkg/k8s/agent/deployer_test.go @@ -18,9 +18,11 @@ import ( "bytes" "context" "errors" + "fmt" "net" "slices" "strings" + "sync" "syscall" "testing" "time" @@ -32,8 +34,11 @@ import ( corev1 "k8s.io/api/core/v1" nodev1 "k8s.io/api/node/v1" rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/fake" k8stesting "k8s.io/client-go/testing" ) @@ -731,6 +736,254 @@ func TestDeployer_Cleanup_ReportsAllErrors(t *testing.T) { } } +// TestCleanupDeletesOnlyWhatItCreated verifies Cleanup builds its delete +// list from the created-set (Deploy's own objects), not from configured +// names — a foreign object that happens to share no name with this run +// must survive even though Cleanup is enabled. +// +// Deviation from the plan's literal test: as with +// TestDeployUsesRunScopedNamesAndLabels above, the brief's snippet omits +// the SelfSubjectAccessReview-allow reactor. Without it the fake clientset +// denies all permission checks by default, so Deploy() fails at the Step-0 +// CheckPermissions gate before creating anything — a false RED unrelated to +// created-set cleanup scoping. Added the same reactor used elsewhere in +// this file. +func TestCleanupDeletesOnlyWhatItCreated(t *testing.T) { + ctx := context.Background() + client := fake.NewSimpleClientset() + client.PrependReactor("create", "selfsubjectaccessreviews", func(action k8stesting.Action) (bool, runtime.Object, error) { + return true, &authv1.SelfSubjectAccessReview{ + Status: authv1.SubjectAccessReviewStatus{ + Allowed: true, + Reason: "test permissions allowed", + }, + }, nil + }) + + // A foreign object sharing no name with this run must survive. + foreign := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "aicr-other", Namespace: "test-ns"}} + if _, err := client.CoreV1().ServiceAccounts("test-ns").Create(ctx, foreign, metav1.CreateOptions{}); err != nil { + t.Fatalf("seed: %v", err) + } + + d := NewDeployer(client, Config{Namespace: "test-ns", Image: "aicr:test", RunID: "20260821-142233-9f3a1c0b7e2d4a55"}) + if err := d.Deploy(ctx); err != nil { + t.Fatalf("Deploy() error = %v", err) + } + if err := d.Cleanup(ctx, CleanupOptions{Enabled: true}); err != nil { + t.Fatalf("Cleanup() error = %v", err) + } + + if _, err := client.CoreV1().ServiceAccounts("test-ns").Get(ctx, "aicr-other", metav1.GetOptions{}); err != nil { + t.Errorf("Cleanup deleted a ServiceAccount it did not create: %v", err) + } + if _, err := client.CoreV1().ServiceAccounts("test-ns").Get(ctx, "aicr-20260821-142233-9f3a1c0b7e2d4a55", metav1.GetOptions{}); !apierrors.IsNotFound(err) { + t.Errorf("Cleanup did not delete its own ServiceAccount, err = %v", err) + } +} + +// TestCleanupPassesUIDPrecondition verifies every delete Cleanup issues +// carries Preconditions.UID set to the UID recorded at create time. The +// fake clientset's ObjectTracker neither assigns UIDs on Create nor +// enforces Preconditions on Delete (it ignores DeleteOptions entirely), so +// this records a known UID directly via recordCreated and spies on the +// outgoing delete action rather than relying on tracker behavior. +func TestCleanupPassesUIDPrecondition(t *testing.T) { + ctx := context.Background() + client := fake.NewSimpleClientset() + + const wantUID = types.UID("sa-uid-123") + var sawUID types.UID + var sawPreconditions bool + client.PrependReactor("delete", "serviceaccounts", func(action k8stesting.Action) (bool, runtime.Object, error) { + da, ok := action.(k8stesting.DeleteActionImpl) + if ok && da.DeleteOptions.Preconditions != nil && da.DeleteOptions.Preconditions.UID != nil { + sawPreconditions = true + sawUID = *da.DeleteOptions.Preconditions.UID + } + return false, nil, nil // not handled: fall through to the default tracker delete + }) + + d := NewDeployer(client, Config{Namespace: "test-ns"}) + d.recordCreated(kindServiceAccount, "aicr-sa", wantUID) + + if err := d.Cleanup(ctx, CleanupOptions{Enabled: true}); err != nil { + t.Fatalf("Cleanup() error = %v", err) + } + + if !sawPreconditions { + t.Fatal("ServiceAccount delete did not carry Preconditions.UID") + } + if sawUID != wantUID { + t.Errorf("Preconditions.UID = %q, want %q", sawUID, wantUID) + } +} + +// TestCleanupTreatsConflictAsSuccess verifies a Conflict response (the UID +// precondition did not match — the name now belongs to a different object) +// is treated as success, same as NotFound, rather than surfaced as a +// Cleanup failure. +func TestCleanupTreatsConflictAsSuccess(t *testing.T) { + ctx := context.Background() + client := fake.NewSimpleClientset() + client.PrependReactor("create", "selfsubjectaccessreviews", func(action k8stesting.Action) (bool, runtime.Object, error) { + return true, &authv1.SelfSubjectAccessReview{ + Status: authv1.SubjectAccessReviewStatus{Allowed: true, Reason: "test permissions allowed"}, + }, nil + }) + client.PrependReactor("delete", "serviceaccounts", func(action k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewConflict(schema.GroupResource{Resource: "serviceaccounts"}, "aicr", errors.New("uid mismatch")) + }) + + d := NewDeployer(client, Config{Namespace: "test-ns", Image: "aicr:test", RunID: "20260821-142233-9f3a1c0b7e2d4a55"}) + if err := d.Deploy(ctx); err != nil { + t.Fatalf("Deploy() error = %v", err) + } + if err := d.Cleanup(ctx, CleanupOptions{Enabled: true}); err != nil { + t.Fatalf("Cleanup() should treat a Conflict delete response as success, got: %v", err) + } +} + +// TestRecordCreatedAndJobUID verifies jobUID() returns the zero UID before +// any Job is recorded, and the recorded Job's UID afterward — even when +// other kinds have been recorded too. +func TestRecordCreatedAndJobUID(t *testing.T) { + d := NewDeployer(fake.NewSimpleClientset(), Config{Namespace: "test-ns"}) + + if got := d.jobUID(); got != "" { + t.Fatalf("jobUID() before any Job recorded = %q, want zero UID", got) + } + + d.recordCreated(kindServiceAccount, "aicr-sa", types.UID("sa-uid")) + if got := d.jobUID(); got != "" { + t.Fatalf("jobUID() after recording a non-Job kind = %q, want zero UID", got) + } + + d.recordCreated(kindJob, "aicr-job", types.UID("job-uid")) + if got := d.jobUID(); got != "job-uid" { + t.Fatalf("jobUID() = %q, want %q", got, "job-uid") + } +} + +// TestCreatedSnapshotIsDefensiveCopy verifies createdSnapshot returns a copy +// that mutation cannot use to corrupt the Deployer's internal created-set. +func TestCreatedSnapshotIsDefensiveCopy(t *testing.T) { + d := NewDeployer(fake.NewSimpleClientset(), Config{Namespace: "test-ns"}) + d.recordCreated(kindServiceAccount, "aicr-sa", types.UID("sa-uid")) + + snap := d.createdSnapshot() + if len(snap) != 1 { + t.Fatalf("createdSnapshot() length = %d, want 1", len(snap)) + } + snap[0].name = "mutated" + + again := d.createdSnapshot() + if again[0].name != "aicr-sa" { + t.Fatalf("createdSnapshot() mutation leaked into Deployer state: got %q, want %q", again[0].name, "aicr-sa") + } +} + +// TestRecordCreatedConcurrentSafe exercises recordCreated from many +// goroutines at once so `go test -race` can catch a data race on the +// created-set if the locking is ever removed or narrowed incorrectly. +func TestRecordCreatedConcurrentSafe(t *testing.T) { + d := NewDeployer(fake.NewSimpleClientset(), Config{Namespace: "test-ns"}) + + const n = 50 + var wg sync.WaitGroup + for i := 0; i < n; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + d.recordCreated(kindConfigMap, fmt.Sprintf("cm-%d", i), types.UID(fmt.Sprintf("uid-%d", i))) + }(i) + } + wg.Wait() + + if got := len(d.createdSnapshot()); got != n { + t.Fatalf("createdSnapshot() length = %d, want %d", got, n) + } +} + +// TestGetSnapshotFromConfigMap_RecordsUID_WhenOwned verifies +// getSnapshotFromConfigMap enters the staging ConfigMap into the +// created-set (for a UID-pinned Cleanup delete) only when +// Config.OwnsOutputConfigMap is true — a caller-supplied `cm://` output is +// the caller's artifact and must never be deleted by this Deployer. +func TestGetSnapshotFromConfigMap_RecordsUID_WhenOwned(t *testing.T) { + tests := []struct { + name string + ownsOutput bool + wantRecord bool + }{ + {name: "owned output is recorded", ownsOutput: true, wantRecord: true}, + {name: "caller-supplied output is not recorded", ownsOutput: false, wantRecord: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "aicr-snapshot", + Namespace: "test-namespace", + UID: types.UID("cm-uid"), + }, + Data: map[string]string{"snapshot.yaml": "data"}, + } + clientset := fake.NewClientset(cm) + d := NewDeployer(clientset, Config{ + Namespace: "test-namespace", + Output: "cm://test-namespace/aicr-snapshot", + OwnsOutputConfigMap: tt.ownsOutput, + }) + + if _, err := d.getSnapshotFromConfigMap(context.Background()); err != nil { + t.Fatalf("getSnapshotFromConfigMap() error = %v", err) + } + + snap := d.createdSnapshot() + gotRecorded := len(snap) == 1 && snap[0].kind == kindConfigMap && snap[0].uid == types.UID("cm-uid") + if gotRecorded != tt.wantRecord { + t.Errorf("recorded = %v (snapshot = %+v), want %v", gotRecorded, snap, tt.wantRecord) + } + }) + } +} + +// TestCleanupDeletesStagingConfigMapWhenOwned verifies Cleanup deletes the +// staging ConfigMap once getSnapshotFromConfigMap has recorded it (i.e. +// Config.OwnsOutputConfigMap was true), exercising deleteStagingConfigMap's +// dispatch from Cleanup end to end. +func TestCleanupDeletesStagingConfigMapWhenOwned(t *testing.T) { + ctx := context.Background() + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "aicr-snapshot", + Namespace: "test-namespace", + UID: types.UID("cm-uid"), + }, + Data: map[string]string{"snapshot.yaml": "data"}, + } + clientset := fake.NewClientset(cm) + d := NewDeployer(clientset, Config{ + Namespace: "test-namespace", + Output: "cm://test-namespace/aicr-snapshot", + OwnsOutputConfigMap: true, + }) + + if _, err := d.getSnapshotFromConfigMap(ctx); err != nil { + t.Fatalf("getSnapshotFromConfigMap() error = %v", err) + } + + if err := d.Cleanup(ctx, CleanupOptions{Enabled: true}); err != nil { + t.Fatalf("Cleanup() error = %v", err) + } + + if _, err := clientset.CoreV1().ConfigMaps("test-namespace").Get(ctx, "aicr-snapshot", metav1.GetOptions{}); !apierrors.IsNotFound(err) { + t.Errorf("Cleanup did not delete the owned staging ConfigMap, err = %v", err) + } +} + func TestParseConfigMapName(t *testing.T) { tests := []struct { name string diff --git a/pkg/k8s/agent/job.go b/pkg/k8s/agent/job.go index f6d99ef46..d1f2fa59b 100644 --- a/pkg/k8s/agent/job.go +++ b/pkg/k8s/agent/job.go @@ -21,20 +21,20 @@ import ( "github.com/NVIDIA/aicr/pkg/defaults" aicrerrors "github.com/NVIDIA/aicr/pkg/errors" - "github.com/NVIDIA/aicr/pkg/k8s" "github.com/NVIDIA/aicr/pkg/recipe/oskind" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" ) // ensureJob creates the run-scoped agent Job. func (d *Deployer) ensureJob(ctx context.Context) error { job := d.buildJob() - _, err := d.clientset.BatchV1().Jobs(d.config.Namespace). + created, err := d.clientset.BatchV1().Jobs(d.config.Namespace). Create(ctx, job, metav1.CreateOptions{}) if errors.IsAlreadyExists(err) { return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "Job already exists under run-scoped name (duplicate RunID?)", err) @@ -42,6 +42,7 @@ func (d *Deployer) ensureJob(ctx context.Context) error { if err != nil { return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to create Job", err) } + d.recordCreated(kindJob, created.Name, created.UID) return nil } @@ -327,17 +328,21 @@ func (d *Deployer) buildEnvVars() []corev1.EnvVar { return envVars } -// deleteJob deletes the Job. -func (d *Deployer) deleteJob(ctx context.Context) error { +// deleteJob deletes the Job, pinning the delete to uid so a same-named Job +// belonging to a different run is never collected. If the Job is already +// gone, or uid no longer matches (already replaced, not ours), this is a +// no-op (idempotent). +func (d *Deployer) deleteJob(ctx context.Context, name string, uid types.UID) error { propagationPolicy := metav1.DeletePropagationForeground err := d.clientset.BatchV1().Jobs(d.config.Namespace).Delete( ctx, - d.jobName(), + name, metav1.DeleteOptions{ PropagationPolicy: &propagationPolicy, + Preconditions: &metav1.Preconditions{UID: &uid}, }, ) - return k8s.IgnoreNotFound(err) + return ignoreNotFoundOrConflict(err) } // mustParseQuantity parses a resource quantity or panics. diff --git a/pkg/k8s/agent/permissions.go b/pkg/k8s/agent/permissions.go index a13d1c746..7dd955b87 100644 --- a/pkg/k8s/agent/permissions.go +++ b/pkg/k8s/agent/permissions.go @@ -61,6 +61,7 @@ func (d *Deployer) CheckPermissions(ctx context.Context) ([]permissionCheck, err // Cleanup permissions {"jobs", "delete", d.config.Namespace}, + {resourceCM, "delete", d.config.Namespace}, } // SelfSubjectAccessReview is a read-only query; running the N required diff --git a/pkg/k8s/agent/rbac.go b/pkg/k8s/agent/rbac.go index 7a8ba5357..e9f9e3cec 100644 --- a/pkg/k8s/agent/rbac.go +++ b/pkg/k8s/agent/rbac.go @@ -20,7 +20,6 @@ import ( "log/slog" "github.com/NVIDIA/aicr/pkg/errors" - "github.com/NVIDIA/aicr/pkg/k8s" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -111,13 +110,14 @@ func (d *Deployer) ensureServiceAccount(ctx context.Context) error { }, } - _, err := d.clientset.CoreV1().ServiceAccounts(d.config.Namespace).Create(ctx, sa, metav1.CreateOptions{}) + created, err := d.clientset.CoreV1().ServiceAccounts(d.config.Namespace).Create(ctx, sa, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { return errors.Wrap(errors.ErrCodeInternal, "ServiceAccount already exists under run-scoped name (duplicate RunID?)", err) } if err != nil { return errors.Wrap(errors.ErrCodeInternal, "failed to create ServiceAccount", err) } + d.recordCreated(kindServiceAccount, created.Name, created.UID) return nil } @@ -143,13 +143,14 @@ func (d *Deployer) ensureRole(ctx context.Context) error { }, } - _, err := d.clientset.RbacV1().Roles(d.config.Namespace).Create(ctx, role, metav1.CreateOptions{}) + created, err := d.clientset.RbacV1().Roles(d.config.Namespace).Create(ctx, role, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { return errors.Wrap(errors.ErrCodeInternal, "Role already exists under run-scoped name (duplicate RunID?)", err) } if err != nil { return errors.Wrap(errors.ErrCodeInternal, "failed to create Role", err) } + d.recordCreated(kindRole, created.Name, created.UID) return nil } @@ -175,13 +176,14 @@ func (d *Deployer) ensureRoleBinding(ctx context.Context) error { }, } - _, err := d.clientset.RbacV1().RoleBindings(d.config.Namespace).Create(ctx, rb, metav1.CreateOptions{}) + created, err := d.clientset.RbacV1().RoleBindings(d.config.Namespace).Create(ctx, rb, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { return errors.Wrap(errors.ErrCodeInternal, "RoleBinding already exists under run-scoped name (duplicate RunID?)", err) } if err != nil { return errors.Wrap(errors.ErrCodeInternal, "failed to create RoleBinding", err) } + d.recordCreated(kindRoleBinding, created.Name, created.UID) return nil } @@ -239,13 +241,14 @@ func (d *Deployer) ensureClusterRole(ctx context.Context) error { Rules: rules, } - _, err := d.clientset.RbacV1().ClusterRoles().Create(ctx, cr, metav1.CreateOptions{}) + created, err := d.clientset.RbacV1().ClusterRoles().Create(ctx, cr, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { return errors.Wrap(errors.ErrCodeInternal, "ClusterRole already exists under run-scoped name (duplicate RunID?)", err) } if err != nil { return errors.Wrap(errors.ErrCodeInternal, "failed to create ClusterRole", err) } + d.recordCreated(kindClusterRole, created.Name, created.UID) return nil } @@ -270,54 +273,60 @@ func (d *Deployer) ensureClusterRoleBinding(ctx context.Context) error { }, } - _, err := d.clientset.RbacV1().ClusterRoleBindings().Create(ctx, crb, metav1.CreateOptions{}) + created, err := d.clientset.RbacV1().ClusterRoleBindings().Create(ctx, crb, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { return errors.Wrap(errors.ErrCodeInternal, "ClusterRoleBinding already exists under run-scoped name (duplicate RunID?)", err) } if err != nil { return errors.Wrap(errors.ErrCodeInternal, "failed to create ClusterRoleBinding", err) } + d.recordCreated(kindClusterRoleBinding, created.Name, created.UID) return nil } -// deleteServiceAccount deletes the ServiceAccount. -// If the ServiceAccount doesn't exist, this is a no-op (idempotent). -func (d *Deployer) deleteServiceAccount(ctx context.Context) error { +// deleteServiceAccount deletes the ServiceAccount, pinning the delete to uid +// so a same-named ServiceAccount belonging to a different run is never +// collected. If the ServiceAccount is already gone, or uid no longer +// matches (already replaced, not ours), this is a no-op (idempotent). +func (d *Deployer) deleteServiceAccount(ctx context.Context, name string, uid types.UID) error { err := d.clientset.CoreV1().ServiceAccounts(d.config.Namespace). - Delete(ctx, d.saName(), metav1.DeleteOptions{}) - return k8s.IgnoreNotFound(err) + Delete(ctx, name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}) + return ignoreNotFoundOrConflict(err) } -// deleteRole deletes the Role. -// If the Role doesn't exist, this is a no-op (idempotent). -func (d *Deployer) deleteRole(ctx context.Context) error { +// deleteRole deletes the Role, pinning the delete to uid. If the Role is +// already gone, or uid no longer matches, this is a no-op (idempotent). +func (d *Deployer) deleteRole(ctx context.Context, name string, uid types.UID) error { err := d.clientset.RbacV1().Roles(d.config.Namespace). - Delete(ctx, d.roleName(), metav1.DeleteOptions{}) - return k8s.IgnoreNotFound(err) + Delete(ctx, name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}) + return ignoreNotFoundOrConflict(err) } -// deleteRoleBinding deletes the RoleBinding. -// If the RoleBinding doesn't exist, this is a no-op (idempotent). -func (d *Deployer) deleteRoleBinding(ctx context.Context) error { +// deleteRoleBinding deletes the RoleBinding, pinning the delete to uid. If +// the RoleBinding is already gone, or uid no longer matches, this is a +// no-op (idempotent). +func (d *Deployer) deleteRoleBinding(ctx context.Context, name string, uid types.UID) error { err := d.clientset.RbacV1().RoleBindings(d.config.Namespace). - Delete(ctx, d.roleName(), metav1.DeleteOptions{}) - return k8s.IgnoreNotFound(err) + Delete(ctx, name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}) + return ignoreNotFoundOrConflict(err) } -// deleteClusterRole deletes the ClusterRole. -// If the ClusterRole doesn't exist, this is a no-op (idempotent). -func (d *Deployer) deleteClusterRole(ctx context.Context) error { +// deleteClusterRole deletes the ClusterRole, pinning the delete to uid. If +// the ClusterRole is already gone, or uid no longer matches, this is a +// no-op (idempotent). +func (d *Deployer) deleteClusterRole(ctx context.Context, name string, uid types.UID) error { err := d.clientset.RbacV1().ClusterRoles(). - Delete(ctx, d.clusterRoleName(), metav1.DeleteOptions{}) - return k8s.IgnoreNotFound(err) + Delete(ctx, name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}) + return ignoreNotFoundOrConflict(err) } -// deleteClusterRoleBinding deletes the ClusterRoleBinding. -// If the ClusterRoleBinding doesn't exist, this is a no-op (idempotent). -func (d *Deployer) deleteClusterRoleBinding(ctx context.Context) error { +// deleteClusterRoleBinding deletes the ClusterRoleBinding, pinning the +// delete to uid. If the ClusterRoleBinding is already gone, or uid no +// longer matches, this is a no-op (idempotent). +func (d *Deployer) deleteClusterRoleBinding(ctx context.Context, name string, uid types.UID) error { err := d.clientset.RbacV1().ClusterRoleBindings(). - Delete(ctx, d.clusterRoleName(), metav1.DeleteOptions{}) - return k8s.IgnoreNotFound(err) + Delete(ctx, name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}) + return ignoreNotFoundOrConflict(err) } // discoverNetworkClusterRules returns the cluster-scoped policy rules diff --git a/pkg/k8s/agent/types.go b/pkg/k8s/agent/types.go index 673b8590b..7d5799725 100644 --- a/pkg/k8s/agent/types.go +++ b/pkg/k8s/agent/types.go @@ -15,8 +15,11 @@ package agent import ( + "sync" + "github.com/NVIDIA/aicr/pkg/k8s/labels" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" ) @@ -29,6 +32,30 @@ const ( agentLabelSelector = labelAppName + "=" + appName ) +// Kind labels recorded in Deployer.created for each run-owned object type. +// Cleanup dispatches on these to call the matching resource-specific delete. +const ( + kindServiceAccount = "ServiceAccount" + kindRole = "Role" + kindRoleBinding = "RoleBinding" + kindClusterRole = "ClusterRole" + kindClusterRoleBinding = "ClusterRoleBinding" + kindJob = "Job" + kindConfigMap = "ConfigMap" +) + +// createdObject records one object this Deployer created (or, for the +// staging ConfigMap it does not itself create, observed itself owning) so +// Cleanup can delete exactly this set instead of deriving a delete list from +// configured names. name is the run-scoped name the object was created +// under; uid pins the eventual delete via metav1.Preconditions so a +// same-named object belonging to a different run is never collected. +type createdObject struct { + kind string + name string + uid types.UID +} + // Config holds the configuration for deploying the agent. type Config struct { Namespace string @@ -87,12 +114,29 @@ type Config struct { // caller did not already supply that key — so a caller can request // e.g. nvidia.com/gpu=4 alongside RequireGPU and keep their value. Limits corev1.ResourceList + + // OwnsOutputConfigMap is true when Output names the staging ConfigMap + // this Deployer's own Job writes (the default run-scoped + // `cm:///` URI), rather than a ConfigMap + // the caller supplied out of band via a hand-written `cm://` Output + // URI. GetSnapshot enters the ConfigMap into the created-set for + // Cleanup only when this is true — a caller-supplied ConfigMap is + // the caller's artifact and must never be deleted by this Deployer. + OwnsOutputConfigMap bool } // Deployer manages the deployment and lifecycle of the agent Job. type Deployer struct { clientset kubernetes.Interface config Config + + // mu guards created. Deploy's ensure* steps run sequentially today, + // but GetSnapshot (which records the staging ConfigMap) can be + // invoked from a different goroutine than Deploy, and Cleanup reads + // the created-set while a caller could still be recording into it, + // so every access is mutex-guarded. + mu sync.Mutex + created []createdObject } // NewDeployer creates a new agent Deployer with the given configuration. @@ -117,6 +161,42 @@ func (d *Deployer) objectLabels() map[string]string { } } +// recordCreated appends a run-owned object to the created-set. Cleanup +// builds its UID-pinned delete list from exactly this set, so every ensure* +// call that successfully creates an object — and GetSnapshot, for the +// staging ConfigMap it observes but does not itself create — must call this +// on success. Safe for concurrent use. +func (d *Deployer) recordCreated(kind, name string, uid types.UID) { + d.mu.Lock() + defer d.mu.Unlock() + d.created = append(d.created, createdObject{kind: kind, name: name, uid: uid}) +} + +// createdSnapshot returns a defensive copy of the created-set taken under +// lock. Callers must not read d.created directly. +func (d *Deployer) createdSnapshot() []createdObject { + d.mu.Lock() + defer d.mu.Unlock() + out := make([]createdObject, len(d.created)) + copy(out, d.created) + return out +} + +// jobUID returns the UID of the Job this Deployer created, or the zero UID +// if Deploy has not (yet) reached the Job-create step — including when +// Deploy failed before getting there. Task 5 uses this to authorize pod +// selection against exactly this run's Job. +func (d *Deployer) jobUID() types.UID { + d.mu.Lock() + defer d.mu.Unlock() + for _, c := range d.created { + if c.kind == kindJob { + return c.uid + } + } + return "" +} + // CleanupOptions controls what resources to remove during cleanup. type CleanupOptions struct { Enabled bool // If true, removes Job and all RBAC resources diff --git a/pkg/k8s/agent/wait.go b/pkg/k8s/agent/wait.go index f0ecae931..19051d253 100644 --- a/pkg/k8s/agent/wait.go +++ b/pkg/k8s/agent/wait.go @@ -24,6 +24,7 @@ import ( "github.com/NVIDIA/aicr/pkg/k8s/pod" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ) // waitForJobCompletion waits for the Job to complete successfully or fail. @@ -45,6 +46,16 @@ func (d *Deployer) getSnapshotFromConfigMap(ctx context.Context) ([]byte, error) return nil, errors.Wrap(errors.ErrCodeNotFound, fmt.Sprintf("failed to get ConfigMap %s/%s", namespace, name), err) } + // The staging ConfigMap is written by the in-pod agent, not this + // controller, so there is no Create response to record a UID from — + // this Get is the only point where we observe it. Record it into the + // created-set (for a UID-pinned Cleanup delete) only when this + // Deployer owns the output: a caller-supplied `cm://` Output URI + // names the caller's own artifact, which must never be deleted here. + if d.config.OwnsOutputConfigMap { + d.recordCreated(kindConfigMap, name, cm.UID) + } + // Extract snapshot data snapshot, ok := cm.Data["snapshot.yaml"] if !ok { @@ -54,6 +65,19 @@ func (d *Deployer) getSnapshotFromConfigMap(ctx context.Context) ([]byte, error) return []byte(snapshot), nil } +// deleteStagingConfigMap deletes the staging ConfigMap in d.config.Namespace +// by name, pinning the delete to uid so a same-named ConfigMap belonging to +// a different run is never collected. If the ConfigMap is already gone, or +// uid no longer matches (already replaced, not ours), this is a no-op +// (idempotent). Only reached via Cleanup's created-set dispatch, which only +// contains an entry for this ConfigMap when Config.OwnsOutputConfigMap was +// true at record time (see getSnapshotFromConfigMap). +func (d *Deployer) deleteStagingConfigMap(ctx context.Context, name string, uid types.UID) error { + err := d.clientset.CoreV1().ConfigMaps(d.config.Namespace). + Delete(ctx, name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}) + return ignoreNotFoundOrConflict(err) +} + // StreamLogs streams logs from the Job's Pod to the provided writer. // It will follow the logs until the context is canceled. // Returns when the context is canceled or an error occurs. From 9062f45b0f8e2f93379614ddc7bfc4d8d388bece Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Fri, 21 Aug 2026 15:39:43 -0700 Subject: [PATCH 06/56] feat(agent): authorize pod selection via controlling ownerReference Signed-off-by: Alex Yuskauskas --- pkg/k8s/agent/deployer_test.go | 6 +- pkg/k8s/agent/types.go | 7 +- pkg/k8s/agent/wait.go | 65 +++++++++++--- pkg/k8s/agent/wait_test.go | 150 ++++++++++++++++++++++++++++++++- 4 files changed, 209 insertions(+), 19 deletions(-) diff --git a/pkg/k8s/agent/deployer_test.go b/pkg/k8s/agent/deployer_test.go index 04d4508bb..edec895ba 100644 --- a/pkg/k8s/agent/deployer_test.go +++ b/pkg/k8s/agent/deployer_test.go @@ -1149,7 +1149,8 @@ func TestDeployer_WaitForPodReady(t *testing.T) { Name: "aicr-xyz", Namespace: "test-namespace", Labels: map[string]string{ - "app.kubernetes.io/name": "aicr", + labels.Name: labels.ValueAICR, + labels.RunID: "", }, }, Status: corev1.PodStatus{ @@ -1203,7 +1204,8 @@ func TestDeployer_WaitForPodReady_PodFailed(t *testing.T) { Name: "aicr-xyz", Namespace: "test-namespace", Labels: map[string]string{ - "app.kubernetes.io/name": "aicr", + labels.Name: labels.ValueAICR, + labels.RunID: "", }, }, Status: corev1.PodStatus{ diff --git a/pkg/k8s/agent/types.go b/pkg/k8s/agent/types.go index 7d5799725..1cb6a7a35 100644 --- a/pkg/k8s/agent/types.go +++ b/pkg/k8s/agent/types.go @@ -26,10 +26,9 @@ import ( // Standard Kubernetes recommended labels applied to all agent-managed // resources. Centralized here so selectors and resource templates stay in sync. const ( - labelAppName = "app.kubernetes.io/name" - labelAppManagedBy = "app.kubernetes.io/managed-by" - appName = "aicr" - agentLabelSelector = labelAppName + "=" + appName + labelAppName = "app.kubernetes.io/name" + labelAppManagedBy = "app.kubernetes.io/managed-by" + appName = "aicr" ) // Kind labels recorded in Deployer.created for each run-owned object type. diff --git a/pkg/k8s/agent/wait.go b/pkg/k8s/agent/wait.go index 19051d253..5d5b169f7 100644 --- a/pkg/k8s/agent/wait.go +++ b/pkg/k8s/agent/wait.go @@ -21,6 +21,7 @@ import ( "time" "github.com/NVIDIA/aicr/pkg/errors" + "github.com/NVIDIA/aicr/pkg/k8s/labels" "github.com/NVIDIA/aicr/pkg/k8s/pod" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -29,7 +30,7 @@ import ( // waitForJobCompletion waits for the Job to complete successfully or fail. func (d *Deployer) waitForJobCompletion(ctx context.Context, timeout time.Duration) error { - return pod.WaitForJobCompletion(ctx, d.clientset, d.config.Namespace, d.config.JobName, timeout) + return pod.WaitForJobCompletion(ctx, d.clientset, d.config.Namespace, d.jobName(), timeout) } // getSnapshotFromConfigMap retrieves the snapshot data from ConfigMap. @@ -132,28 +133,60 @@ func (d *Deployer) WaitForPodReady(ctx context.Context, timeout time.Duration) e return pod.WaitForPodReady(ctx, d.clientset, d.config.Namespace, podName, remainingTimeout) } +// podLabelSelector returns the List/Watch label selector for this run's +// agent pod: app name plus RunID. This only narrows the candidate set — +// pod labels, including a forged aicr.run/run-id, are writable by anything +// that can update pods in the namespace. ownedByJob (applied by pickLivePod +// and the watch loop in findOrWatchPodName) is what authorizes selection. +func (d *Deployer) podLabelSelector() string { + return fmt.Sprintf("%s=%s,%s=%s", labels.Name, labels.ValueAICR, labels.RunID, d.config.RunID) +} + +// ownedByJob reports whether pod is controlled by the Job with jobUID. +// Pod labels are writable by anything that can update pods in the +// namespace, so the controlling ownerReference — not +// batch.kubernetes.io/controller-uid — is what authorizes selection. +// jobUID must be non-zero; callers fall back to label-only narrowing (see +// pickLivePod) instead of passing the zero UID here. +func ownedByJob(pod *corev1.Pod, jobUID types.UID) bool { + for i := range pod.OwnerReferences { + ref := &pod.OwnerReferences[i] + if ref.Kind == "Job" && ref.UID == jobUID && ref.Controller != nil && *ref.Controller { + return true + } + } + return false +} + // findPodName finds the pod name by label selector for this Job. // One-shot: returns ErrCodeNotFound if no pod is currently labeled. // Skips pods that are being deleted or have already failed so an // orphaned pod from a prior run is not selected. func (d *Deployer) findPodName(ctx context.Context) (string, error) { pods, err := d.clientset.CoreV1().Pods(d.config.Namespace).List(ctx, metav1.ListOptions{ - LabelSelector: agentLabelSelector, + LabelSelector: d.podLabelSelector(), }) if err != nil { return "", errors.Wrap(errors.ErrCodeInternal, "failed to list Pods", err) } - name := pickLivePod(pods.Items) + name := pickLivePod(pods.Items, d.jobUID()) if name == "" { - return "", errors.New(errors.ErrCodeNotFound, fmt.Sprintf("no Pods found for Job %s", d.config.JobName)) + return "", errors.New(errors.ErrCodeNotFound, fmt.Sprintf("no Pods found for Job %s", d.jobName())) } return name, nil } // pickLivePod returns the name of the youngest pod that is neither being -// deleted nor in a Failed phase. Returns "" if no usable pod exists. -func pickLivePod(pods []corev1.Pod) string { +// deleted nor in a Failed phase. When jobUID is non-zero, a pod must also be +// owned by that Job (see ownedByJob) — the label selector used to build the +// candidate list only narrows it; the controlling ownerReference is what +// authorizes selection. When jobUID is the zero UID (the Job hasn't been +// recorded yet — WaitForPodReady's watch can start before Deploy returns), +// ownership is not checked here; callers re-check on every call since +// d.jobUID() is queried live, not cached. Returns "" if no usable pod +// exists. +func pickLivePod(pods []corev1.Pod, jobUID types.UID) string { var best *corev1.Pod for i := range pods { p := &pods[i] @@ -163,6 +196,9 @@ func pickLivePod(pods []corev1.Pod) string { if p.Status.Phase == corev1.PodFailed { continue } + if jobUID != "" && !ownedByJob(p, jobUID) { + continue + } if best == nil || p.CreationTimestamp.After(best.CreationTimestamp.Time) { best = p } @@ -177,18 +213,19 @@ func pickLivePod(pods []corev1.Pod) string { // (List), return immediately; otherwise watch for an Added event until ctx is // canceled. func (d *Deployer) findOrWatchPodName(ctx context.Context) (string, error) { + selector := d.podLabelSelector() pods, err := d.clientset.CoreV1().Pods(d.config.Namespace).List(ctx, metav1.ListOptions{ - LabelSelector: agentLabelSelector, + LabelSelector: selector, }) if err != nil { return "", errors.Wrap(errors.ErrCodeInternal, "failed to list Pods", err) } - if name := pickLivePod(pods.Items); name != "" { + if name := pickLivePod(pods.Items, d.jobUID()); name != "" { return name, nil } watcher, err := d.clientset.CoreV1().Pods(d.config.Namespace).Watch(ctx, metav1.ListOptions{ - LabelSelector: agentLabelSelector, + LabelSelector: selector, ResourceVersion: pods.ResourceVersion, }) if err != nil { @@ -206,12 +243,12 @@ func (d *Deployer) findOrWatchPodName(ctx context.Context) (string, error) { // close watch channels without the pod actually failing to // appear. Re-List before declaring failure. pods, listErr := d.clientset.CoreV1().Pods(d.config.Namespace).List(ctx, metav1.ListOptions{ - LabelSelector: agentLabelSelector, + LabelSelector: selector, }) if listErr != nil { return "", errors.Wrap(errors.ErrCodeUnavailable, "Pod watch channel closed and re-List failed", listErr) } - if name := pickLivePod(pods.Items); name != "" { + if name := pickLivePod(pods.Items, d.jobUID()); name != "" { return name, nil } return "", errors.New(errors.ErrCodeUnavailable, "Pod watch channel closed before pod observed") @@ -223,6 +260,12 @@ func (d *Deployer) findOrWatchPodName(ctx context.Context) (string, error) { if p.DeletionTimestamp != nil || p.Status.Phase == corev1.PodFailed { continue } + // jobUID is re-queried per event (not cached at loop entry) so a + // Job UID recorded by Deploy after this watch started is + // honored on the very next event. + if jobUID := d.jobUID(); jobUID != "" && !ownedByJob(p, jobUID) { + continue + } return p.Name, nil } } diff --git a/pkg/k8s/agent/wait_test.go b/pkg/k8s/agent/wait_test.go index 2e29d17b3..4da0e09ad 100644 --- a/pkg/k8s/agent/wait_test.go +++ b/pkg/k8s/agent/wait_test.go @@ -21,10 +21,12 @@ import ( "testing" "time" + "github.com/NVIDIA/aicr/pkg/k8s/labels" "github.com/NVIDIA/aicr/pkg/k8s/pod" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/fake" ) @@ -265,7 +267,7 @@ func TestDeployer_WaitForPodReady_Extended(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "test-pod", Namespace: ns, - Labels: map[string]string{"app.kubernetes.io/name": "aicr"}, + Labels: map[string]string{labels.Name: labels.ValueAICR, labels.RunID: ""}, }, Status: corev1.PodStatus{ Phase: corev1.PodRunning, @@ -295,7 +297,7 @@ func TestDeployer_WaitForPodReady_Extended(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "test-pod", Namespace: ns, - Labels: map[string]string{"app.kubernetes.io/name": "aicr"}, + Labels: map[string]string{labels.Name: labels.ValueAICR, labels.RunID: ""}, }, Status: corev1.PodStatus{ Phase: corev1.PodFailed, @@ -328,3 +330,147 @@ func TestDeployer_WaitForPodReady_Extended(t *testing.T) { } }) } + +// podWithOwner returns a Pod whose sole OwnerReference is of the given kind, +// uid, and controller flag. Used to exercise ownedByJob's ownership checks +// in isolation from label-based pod selection. +func podWithOwner(kind string, uid types.UID, controller bool) corev1.Pod { + return corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + OwnerReferences: []metav1.OwnerReference{ + { + Kind: kind, + UID: uid, + Controller: &controller, + }, + }, + }, + } +} + +// podWithForgedLabel returns a Pod with no OwnerReferences but a +// batch.kubernetes.io/controller-uid label set to uid — the label any +// client that can update pods in the namespace could set directly, unlike +// the controller-managed OwnerReferences. ownedByJob must not be fooled by +// it. +func podWithForgedLabel(uid types.UID) corev1.Pod { + return corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "batch.kubernetes.io/controller-uid": string(uid), + }, + }, + } +} + +func TestOwnedByJob(t *testing.T) { + const want = types.UID("job-uid-1") + tests := []struct { + name string + pod corev1.Pod + ok bool + }{ + {"controller job matching uid", podWithOwner("Job", want, true), true}, + {"controller job wrong uid", podWithOwner("Job", types.UID("other"), true), false}, + {"non-controller ref", podWithOwner("Job", want, false), false}, + {"wrong kind", podWithOwner("ReplicaSet", want, true), false}, + {"no owner refs", corev1.Pod{}, false}, + {"forged label only", podWithForgedLabel(want), false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ownedByJob(&tt.pod, want); got != tt.ok { + t.Errorf("ownedByJob() = %v, want %v", got, tt.ok) + } + }) + } +} + +// TestPickLivePod exercises the jobUID-gated ownership filtering pickLivePod +// applies on top of the existing DeletionTimestamp/Failed-phase filtering. +func TestPickLivePod(t *testing.T) { + const jobUID = types.UID("job-uid-1") + + older := metav1.NewTime(time.Unix(1000, 0)) + younger := metav1.NewTime(time.Unix(2000, 0)) + + ownedOlder := podWithOwner("Job", jobUID, true) + ownedOlder.Name = "owned-older" + ownedOlder.CreationTimestamp = older + + ownedYounger := podWithOwner("Job", jobUID, true) + ownedYounger.Name = "owned-younger" + ownedYounger.CreationTimestamp = younger + + unowned := podWithForgedLabel(jobUID) // forged label, no real ownerRef + unowned.Name = "unowned-forged" + unowned.CreationTimestamp = younger // younger than both owned pods + + deleting := podWithOwner("Job", jobUID, true) + deleting.Name = "deleting" + deleting.CreationTimestamp = younger + now := metav1.Now() + deleting.DeletionTimestamp = &now + + failed := podWithOwner("Job", jobUID, true) + failed.Name = "failed" + failed.CreationTimestamp = younger + failed.Status.Phase = corev1.PodFailed + + tests := []struct { + name string + pods []corev1.Pod + jobUID types.UID + wantPod string + }{ + { + name: "zero jobUID falls back to youngest live pod regardless of ownership", + pods: []corev1.Pod{ownedOlder, unowned}, + jobUID: "", + wantPod: "unowned-forged", + }, + { + name: "known jobUID rejects unowned pod even if younger", + pods: []corev1.Pod{ownedOlder, unowned}, + jobUID: jobUID, + wantPod: "owned-older", + }, + { + name: "known jobUID picks youngest among owned pods", + pods: []corev1.Pod{ownedOlder, ownedYounger}, + jobUID: jobUID, + wantPod: "owned-younger", + }, + { + name: "known jobUID with only unowned pods returns none", + pods: []corev1.Pod{unowned}, + jobUID: jobUID, + wantPod: "", + }, + { + name: "deleting owned pod is skipped", + pods: []corev1.Pod{deleting, ownedOlder}, + jobUID: jobUID, + wantPod: "owned-older", + }, + { + name: "failed owned pod is skipped", + pods: []corev1.Pod{failed, ownedOlder}, + jobUID: jobUID, + wantPod: "owned-older", + }, + { + name: "no pods", + pods: nil, + jobUID: jobUID, + wantPod: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := pickLivePod(tt.pods, tt.jobUID); got != tt.wantPod { + t.Errorf("pickLivePod() = %q, want %q", got, tt.wantPod) + } + }) + } +} From 04252801c02f3dd6946d2a1b5322c9a9da28a59a Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Fri, 21 Aug 2026 15:50:13 -0700 Subject: [PATCH 07/56] fix(agent): fail closed in ownedByJob on a zero Job UID Signed-off-by: Alex Yuskauskas --- pkg/k8s/agent/wait.go | 13 ++++++++++--- pkg/k8s/agent/wait_test.go | 35 ++++++++++++++++++++--------------- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/pkg/k8s/agent/wait.go b/pkg/k8s/agent/wait.go index 5d5b169f7..1f3be4b70 100644 --- a/pkg/k8s/agent/wait.go +++ b/pkg/k8s/agent/wait.go @@ -146,12 +146,19 @@ func (d *Deployer) podLabelSelector() string { // Pod labels are writable by anything that can update pods in the // namespace, so the controlling ownerReference — not // batch.kubernetes.io/controller-uid — is what authorizes selection. -// jobUID must be non-zero; callers fall back to label-only narrowing (see -// pickLivePod) instead of passing the zero UID here. +// Callers fall back to label-only narrowing (see pickLivePod) instead of +// calling this with the zero UID; the guard below is defense-in-depth so +// the predicate itself fails closed if ever called standalone. func ownedByJob(pod *corev1.Pod, jobUID types.UID) bool { + if jobUID == "" { + // A zero Job UID means ownership is not yet establishable — fail + // closed rather than risk matching a pod whose own ownerRef UID + // happens to also be empty. + return false + } for i := range pod.OwnerReferences { ref := &pod.OwnerReferences[i] - if ref.Kind == "Job" && ref.UID == jobUID && ref.Controller != nil && *ref.Controller { + if ref.Kind == kindJob && ref.UID == jobUID && ref.Controller != nil && *ref.Controller { return true } } diff --git a/pkg/k8s/agent/wait_test.go b/pkg/k8s/agent/wait_test.go index 4da0e09ad..986841b1c 100644 --- a/pkg/k8s/agent/wait_test.go +++ b/pkg/k8s/agent/wait_test.go @@ -357,7 +357,7 @@ func podWithForgedLabel(uid types.UID) corev1.Pod { return corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - "batch.kubernetes.io/controller-uid": string(uid), + batchv1.ControllerUidLabel: string(uid), }, }, } @@ -366,20 +366,25 @@ func podWithForgedLabel(uid types.UID) corev1.Pod { func TestOwnedByJob(t *testing.T) { const want = types.UID("job-uid-1") tests := []struct { - name string - pod corev1.Pod - ok bool + name string + pod corev1.Pod + jobUID types.UID + ok bool }{ - {"controller job matching uid", podWithOwner("Job", want, true), true}, - {"controller job wrong uid", podWithOwner("Job", types.UID("other"), true), false}, - {"non-controller ref", podWithOwner("Job", want, false), false}, - {"wrong kind", podWithOwner("ReplicaSet", want, true), false}, - {"no owner refs", corev1.Pod{}, false}, - {"forged label only", podWithForgedLabel(want), false}, + {"controller job matching uid", podWithOwner(kindJob, want, true), want, true}, + {"controller job wrong uid", podWithOwner(kindJob, types.UID("other"), true), want, false}, + {"non-controller ref", podWithOwner(kindJob, want, false), want, false}, + {"wrong kind", podWithOwner("ReplicaSet", want, true), want, false}, + {"no owner refs", corev1.Pod{}, want, false}, + {"forged label only", podWithForgedLabel(want), want, false}, + // jobUID == "" must fail closed even when the pod's own ownerRef UID + // also happens to be "" — ownership is never establishable from a + // zero Job UID, regardless of what the pod carries. + {"zero jobUID never matches, even a zero-UID owner ref", podWithOwner(kindJob, "", true), "", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := ownedByJob(&tt.pod, want); got != tt.ok { + if got := ownedByJob(&tt.pod, tt.jobUID); got != tt.ok { t.Errorf("ownedByJob() = %v, want %v", got, tt.ok) } }) @@ -394,11 +399,11 @@ func TestPickLivePod(t *testing.T) { older := metav1.NewTime(time.Unix(1000, 0)) younger := metav1.NewTime(time.Unix(2000, 0)) - ownedOlder := podWithOwner("Job", jobUID, true) + ownedOlder := podWithOwner(kindJob, jobUID, true) ownedOlder.Name = "owned-older" ownedOlder.CreationTimestamp = older - ownedYounger := podWithOwner("Job", jobUID, true) + ownedYounger := podWithOwner(kindJob, jobUID, true) ownedYounger.Name = "owned-younger" ownedYounger.CreationTimestamp = younger @@ -406,13 +411,13 @@ func TestPickLivePod(t *testing.T) { unowned.Name = "unowned-forged" unowned.CreationTimestamp = younger // younger than both owned pods - deleting := podWithOwner("Job", jobUID, true) + deleting := podWithOwner(kindJob, jobUID, true) deleting.Name = "deleting" deleting.CreationTimestamp = younger now := metav1.Now() deleting.DeletionTimestamp = &now - failed := podWithOwner("Job", jobUID, true) + failed := podWithOwner(kindJob, jobUID, true) failed.Name = "failed" failed.CreationTimestamp = younger failed.Status.Phase = corev1.PodFailed From 813df45f914dd8a578875d075169e7fd10536bd5 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Fri, 21 Aug 2026 16:13:41 -0700 Subject: [PATCH 08/56] feat(snapshotter): scope staging ConfigMap to the run and unify validate's run ID Signed-off-by: Alex Yuskauskas --- pkg/cli/validate.go | 40 +++++++-- pkg/snapshotter/agent.go | 113 ++++++++++++++++++-------- pkg/snapshotter/agent_test.go | 147 ++++++++++++++++++++++++++++++---- 3 files changed, 244 insertions(+), 56 deletions(-) diff --git a/pkg/cli/validate.go b/pkg/cli/validate.go index 859830060..8e07529cd 100644 --- a/pkg/cli/validate.go +++ b/pkg/cli/validate.go @@ -207,6 +207,13 @@ type validationConfig struct { // Input phases []validator.Phase + // runID is generated once, up front, for the whole `aicr validate` + // invocation (see validateCmd's Action) and reused here instead of a + // second v1.GenerateRunID() call. Sharing one ID keeps names, labels, + // logs, and cleanup hints correlated to a single command instead of + // fragmenting across an agent-side ID and a validator-side one. + runID string + // Kubeconfig path; propagated to ConfigMap reads/writes so a single // validate invocation can target a non-default cluster end-to-end. kubeconfig string @@ -257,12 +264,13 @@ func runValidation( slog.Info("running validation", "phases", cfg.phases) - // Generate the run ID CLI-side rather than letting the validator - // auto-generate it internally: the no-cleanup debug log below needs to - // surface the same ID the validator stamps on its Jobs/RBAC so an - // operator can locate the kept resources. Passing it via - // WithValidationRunID keeps that value in our hands. - runID := v1.GenerateRunID() + // Generated once CLI-side, before either snapshot source ran (see + // validateCmd's Action), rather than letting the validator auto-generate + // it internally: the no-cleanup debug log below needs to surface the + // same ID the validator stamps on its Jobs/RBAC so an operator can + // locate the kept resources. Passing it via WithValidationRunID keeps + // that value in our hands. + runID := cfg.runID // Translate the resolved CLI values into facade ValidateOptions. These // mirror the validator.With* options the direct invocation used to set; @@ -805,6 +813,25 @@ constraint (e.g. K8s version) is not met — --fail-on-error scopes to phase che return err } + // Generate the run ID once, up front, for this whole invocation — + // before either snapshot source below runs. The validator Jobs + // deployed by runValidation need it (passed through + // validationConfig.runID); generating it here rather than inside + // runValidation means a single `aicr validate` no longer risks + // splitting its correlation ID across two independent + // v1.GenerateRunID() calls. + // + // KNOWN GAP: the live-capture branch below + // (deployAgentForValidation -> Client.CollectSnapshot) cannot yet + // forward this ID to the snapshot-capture agent's Job/RBAC — + // pkg/client/v1.AgentConfig has no RunID field to receive it + // (pkg/snapshotter.AgentConfig does; mirroring it onto the facade + // is a separate, later change per ADR-020's implementation plan). + // Until that facade field exists and is wired here, a + // live-capture run's agent Job still gets its own independently + // generated RunID from DeployAndCollect's default. + runID := v1.GenerateRunID() + var snap *aicr.Snapshot // --no-cluster means "do not touch the cluster". The agent-deploy @@ -861,6 +888,7 @@ constraint (e.g. K8s version) is not met — --fail-on-error scopes to phase che return runValidation(ctx, client, rec, snap, validationConfig{ phases: phases, + runID: runID, kubeconfig: kubeconfig, output: cmd.String("output"), outFormat: serializer.FormatJSON, diff --git a/pkg/snapshotter/agent.go b/pkg/snapshotter/agent.go index 356169d03..c00ac0f25 100644 --- a/pkg/snapshotter/agent.go +++ b/pkg/snapshotter/agent.go @@ -36,6 +36,7 @@ import ( k8sclient "github.com/NVIDIA/aicr/pkg/k8s/client" "github.com/NVIDIA/aicr/pkg/k8s/pod" "github.com/NVIDIA/aicr/pkg/measurement" + "github.com/NVIDIA/aicr/pkg/runid" "github.com/NVIDIA/aicr/pkg/serializer" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -148,38 +149,59 @@ type AgentConfig struct { // that key in Limits — e.g. --require-gpu --limits nvidia.com/gpu=4 // keeps 4, not 1. Limits corev1.ResourceList + + // RunID scopes every resource this deployment creates (Job, RBAC, and + // the internal staging ConfigMap when Output does not name one) to a + // single run, so concurrent snapshot-agent runs never collide on a + // shared resource name. DeployAndCollect defaults it with + // runid.Generate() when empty — callers normally leave it unset; + // setting it explicitly is for correlating this run with an external + // identifier (e.g. sharing one ID with a downstream validator run). + RunID string } // buildAgentConfig projects snapshotter configuration onto the deployer's // Job configuration. Keep scheduling defaults at this projection boundary so // every snapshot-agent caller gets the same nil-versus-empty behavior. -func buildAgentConfig(config *AgentConfig, agentOutput string) agent.Config { +// +// ownsOutput is agentConfigMapTarget's second return value, forwarded +// verbatim: it is true only when agentOutput is the internal staging +// ConfigMap this run owns (the caller did not name a cm:// destination), so +// Cleanup may delete it. A caller-supplied cm:// Output is never owned. +func buildAgentConfig(config *AgentConfig, agentOutput string, ownsOutput bool) agent.Config { return agent.Config{ - Namespace: config.Namespace, - ServiceAccountName: config.ServiceAccountName, - JobName: config.JobName, - Image: config.Image, - ImagePullSecrets: config.ImagePullSecrets, - NodeSelector: config.NodeSelector, - Tolerations: effectiveAgentTolerations(config.Tolerations), - Output: agentOutput, - Debug: config.Debug, - Privileged: config.Privileged, - RequireGPU: config.RequireGPU, - RuntimeClassName: config.RuntimeClassName, - MaxNodesPerEntry: config.MaxNodesPerEntry, - OS: config.OS, - ClusterConfigPath: config.ClusterConfigPath, - DiscoverNetwork: config.DiscoverNetwork, - Requests: config.Requests, - Limits: config.Limits, + Namespace: config.Namespace, + ServiceAccountName: config.ServiceAccountName, + JobName: config.JobName, + RunID: config.RunID, + Image: config.Image, + ImagePullSecrets: config.ImagePullSecrets, + NodeSelector: config.NodeSelector, + Tolerations: effectiveAgentTolerations(config.Tolerations), + Output: agentOutput, + Debug: config.Debug, + Privileged: config.Privileged, + RequireGPU: config.RequireGPU, + RuntimeClassName: config.RuntimeClassName, + MaxNodesPerEntry: config.MaxNodesPerEntry, + OS: config.OS, + ClusterConfigPath: config.ClusterConfigPath, + DiscoverNetwork: config.DiscoverNetwork, + Requests: config.Requests, + Limits: config.Limits, + OwnsOutputConfigMap: ownsOutput, } } // deployAndWaitForResult handles the common deploy-wait-retrieve lifecycle for an agent Job. // It creates the deployer, deploys RBAC and the Job, streams logs, waits for completion, // and retrieves the snapshot data from the result ConfigMap. -func deployAndWaitForResult(ctx context.Context, clientset k8sclient.Interface, config *AgentConfig, agentOutput string, deliverViaConfigMap bool) ([]byte, error) { +// +// ownsOutput is agentConfigMapTarget's second return value: true when +// agentOutput is the internal staging ConfigMap this run owns, false when it +// is a caller-supplied cm:// destination. See buildAgentConfig and +// rewriteMergedSnapshotConfigMap for how each consumes it. +func deployAndWaitForResult(ctx context.Context, clientset k8sclient.Interface, config *AgentConfig, agentOutput string, ownsOutput bool) ([]byte, error) { // The pool projection is pure file processing on the caller's host — // project it BEFORE deploying so a bad file fails in milliseconds, // not after a Job round-trip, and merge it into the returned snapshot @@ -198,7 +220,7 @@ func deployAndWaitForResult(ctx context.Context, clientset k8sclient.Interface, // name the injected selector (TOCTOU: node may be cordoned after detection). autoInjectedGPUSelector := maybeInjectGPUNodeSelector(ctx, clientset, config) - agentConfig := buildAgentConfig(config, agentOutput) + agentConfig := buildAgentConfig(config, agentOutput, ownsOutput) deployer := agent.NewDeployer(clientset, agentConfig) @@ -325,8 +347,10 @@ func deployAndWaitForResult(ctx context.Context, clientset k8sclient.Interface, // merged reading, so a transient Apply failure on the internal // hygiene rewrite must not discard an already-captured snapshot; // warn loudly instead (the orphaned ConfigMap stays pre-merge). + // ownsOutput is the run-owns-the-staging-ConfigMap flag, so its + // logical inverse is "the ConfigMap is the user's delivery vehicle". if err := rewriteMergedSnapshotConfigMap(ctx, agentOutput, config.Kubeconfig, - snapshotData, deliverViaConfigMap); err != nil { + snapshotData, !ownsOutput); err != nil { return nil, err } } @@ -482,10 +506,27 @@ func DeployAndCollect(ctx context.Context, config *AgentConfig) (*Snapshot, []by "Namespace is required: it is where the agent Job, its RBAC, and the result ConfigMap are created") } + // Default RunID before any cluster access — and before + // agentConfigMapTarget below, which folds it into the internal staging + // ConfigMap's name — so every resource this run creates shares one + // scope. An empty RunID reaching pkg/k8s/agent would silently fall back + // to unscoped, collision-prone names (nameWithRunID's deploy-between- + // tasks safety net), so a whitespace-only value that slips past this + // simple emptiness check must fail closed rather than reach that + // fallback. + if config.RunID == "" { + config.RunID = runid.Generate() + } + if strings.TrimSpace(config.RunID) == "" { + return nil, nil, errors.New(errors.ErrCodeInvalidRequest, + "RunID must not be all-whitespace; leave it unset to auto-generate one") + } + slog.Info("snapshot agent run", slog.String("runID", config.RunID)) + // Resolve (and validate) the Job's ConfigMap target before any cluster // access: a malformed cm:// Output must not cost the caller a deployed // Job and a cluster-admin binding. - agentOutput, deliverViaConfigMap, err := agentConfigMapTarget(config) + agentOutput, ownsOutput, err := agentConfigMapTarget(config) if err != nil { return nil, nil, err } @@ -497,7 +538,7 @@ func DeployAndCollect(ctx context.Context, config *AgentConfig) (*Snapshot, []by return nil, nil, err } - snapshotData, err := deployAndWaitForResult(ctx, clientset, config, agentOutput, deliverViaConfigMap) + snapshotData, err := deployAndWaitForResult(ctx, clientset, config, agentOutput, ownsOutput) if err != nil { return nil, nil, err } @@ -513,21 +554,27 @@ func DeployAndCollect(ctx context.Context, config *AgentConfig) (*Snapshot, []by } // agentConfigMapTarget resolves where the agent Job stages its result and -// whether that ConfigMap is the user's delivery vehicle. +// whether this run owns that ConfigMap. // // The Job always writes to a ConfigMap. When config.Output is a cm:// URI the -// user asked for that exact ConfigMap, so the Job targets it directly and -// deliverViaConfigMap is true — which makes a failed AKS-pool-merge rewrite -// fatal rather than a warning, because the bytes the user will read live -// there. Any other Output (file, stdout, template, or unset) stages to an -// internal ConfigMap in config.Namespace that the caller never sees. +// user asked for that exact ConfigMap, so the Job targets it directly, +// ownsOutput is false, and a failed AKS-pool-merge rewrite is fatal rather +// than a warning — the bytes the user will read live there, and this run +// must never delete an artifact it does not own. Any other Output (file, +// stdout, template, or unset) stages to an internal, run-scoped ConfigMap in +// config.Namespace that the caller never names: ownsOutput is true, so +// Cleanup may delete it. +// +// The returned uri's namespace equals config.Namespace in the owned case — +// callers (deleteStagingConfigMap in pkg/k8s/agent) rely on that invariant +// rather than re-parsing the URI for a delete namespace. // // A cm:// Output is fully parsed here, not merely prefix-matched. The // namespace/name only has to be well-formed for the in-pod writer much later, // so a typo like "cm://aicr-snapshot" (no namespace) would otherwise surface // as a Job failure — after RBAC and the Job exist, and with Cleanup false // (the zero value) they stay behind. Returns ErrCodeInvalidRequest instead. -func agentConfigMapTarget(config *AgentConfig) (uri string, deliverViaConfigMap bool, err error) { +func agentConfigMapTarget(config *AgentConfig) (uri string, ownsOutput bool, err error) { if strings.HasPrefix(config.Output, serializer.ConfigMapURIScheme) { if _, _, parseErr := pod.ParseConfigMapURI(config.Output); parseErr != nil { // Wrap with the same code rather than PropagateOrWrap: the inner @@ -537,9 +584,9 @@ func agentConfigMapTarget(config *AgentConfig) (uri string, deliverViaConfigMap fmt.Sprintf("invalid ConfigMap output URI %q (expected cm://namespace/name)", config.Output), parseErr) } - return config.Output, true, nil + return config.Output, false, nil } - return fmt.Sprintf("%s%s/aicr-snapshot", serializer.ConfigMapURIScheme, config.Namespace), false, nil + return fmt.Sprintf("%s%s/aicr-snapshot-%s", serializer.ConfigMapURIScheme, config.Namespace, config.RunID), true, nil } // SnapshotDelivery describes where DeliverSnapshot writes captured bytes. diff --git a/pkg/snapshotter/agent_test.go b/pkg/snapshotter/agent_test.go index 9db0d0979..edfa11609 100644 --- a/pkg/snapshotter/agent_test.go +++ b/pkg/snapshotter/agent_test.go @@ -22,6 +22,7 @@ import ( "os" "path/filepath" "reflect" + "regexp" "strings" "testing" @@ -88,7 +89,7 @@ func TestBuildAgentConfigTolerations(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := buildAgentConfig(&AgentConfig{Tolerations: tt.input}, "snapshot.yaml").Tolerations + got := buildAgentConfig(&AgentConfig{Tolerations: tt.input}, "snapshot.yaml", false).Tolerations if !reflect.DeepEqual(got, tt.want) { t.Errorf("buildAgentConfig().Tolerations = %#v, want %#v", got, tt.want) } @@ -96,6 +97,34 @@ func TestBuildAgentConfigTolerations(t *testing.T) { } } +// TestBuildAgentConfigPropagatesRunIDAndOwnership confirms buildAgentConfig +// forwards AgentConfig.RunID and its ownsOutput parameter onto +// agent.Config.RunID / agent.Config.OwnsOutputConfigMap — the projection +// deployAndWaitForResult relies on so the deployer scopes every resource +// name to this run and Cleanup knows whether it may delete the staging +// ConfigMap. +func TestBuildAgentConfigPropagatesRunIDAndOwnership(t *testing.T) { + tests := []struct { + name string + ownsOutput bool + }{ + {name: "owned staging ConfigMap", ownsOutput: true}, + {name: "user-supplied ConfigMap", ownsOutput: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := buildAgentConfig(&AgentConfig{RunID: "20260821-142233-9f3a1c0b7e2d4a55"}, + "cm://ns/name", tt.ownsOutput) + if got.RunID != "20260821-142233-9f3a1c0b7e2d4a55" { + t.Errorf("agent.Config.RunID = %q, want the AgentConfig.RunID value", got.RunID) + } + if got.OwnsOutputConfigMap != tt.ownsOutput { + t.Errorf("agent.Config.OwnsOutputConfigMap = %v, want %v", got.OwnsOutputConfigMap, tt.ownsOutput) + } + }) + } +} + func TestAgentConfig_Defaults(t *testing.T) { // Test that AgentConfig can be instantiated with zero values cfg := AgentConfig{} @@ -505,11 +534,15 @@ func TestParseTolerationsOperator(t *testing.T) { // TestAgentOutputURILogic exercises agentConfigMapTarget — the rule that // decides where the agent Job stages its result: -// 1. A file path leaves the Job on the default ConfigMap in its namespace. -// 2. A cm:// URI makes that ConfigMap the Job's target AND the delivery -// vehicle, so a rewrite failure must be fatal rather than a warning. +// 1. A file path leaves the Job on the run-scoped internal ConfigMap in its +// namespace, which this run owns. +// 2. A cm:// URI makes that ConfigMap the Job's target directly; this run +// does NOT own it (the caller supplied it), so a rewrite failure must be +// fatal rather than a warning. // 3. Stdout (empty or "-") behaves like a file path. func TestAgentOutputURILogic(t *testing.T) { + const testRunID = "20260821-142233-9f3a1c0b7e2d4a55" + tests := []struct { name string agentNamespace string @@ -518,24 +551,24 @@ func TestAgentOutputURILogic(t *testing.T) { wantUsesUserOutput bool // whether agentOutput should equal userOutput }{ { - name: "file output uses default ConfigMap with agent namespace", + name: "file output uses run-scoped internal ConfigMap", agentNamespace: "default", userOutput: "snapshot.yaml", - wantAgentOutputHas: "cm://default/aicr-snapshot", + wantAgentOutputHas: "cm://default/aicr-snapshot-" + testRunID, wantUsesUserOutput: false, }, { - name: "stdout uses default ConfigMap with agent namespace", + name: "stdout uses run-scoped internal ConfigMap", agentNamespace: "default", userOutput: "", - wantAgentOutputHas: "cm://default/aicr-snapshot", + wantAgentOutputHas: "cm://default/aicr-snapshot-" + testRunID, wantUsesUserOutput: false, }, { - name: "dash stdout uses default ConfigMap with agent namespace", + name: "dash stdout uses run-scoped internal ConfigMap", agentNamespace: "default", userOutput: "-", - wantAgentOutputHas: "cm://default/aicr-snapshot", + wantAgentOutputHas: "cm://default/aicr-snapshot-" + testRunID, wantUsesUserOutput: false, }, { @@ -546,19 +579,20 @@ func TestAgentOutputURILogic(t *testing.T) { wantUsesUserOutput: true, }, { - name: "custom namespace uses that namespace for default ConfigMap", + name: "custom namespace uses that namespace for the run-scoped ConfigMap", agentNamespace: "custom-namespace", userOutput: "output.yaml", - wantAgentOutputHas: "cm://custom-namespace/aicr-snapshot", + wantAgentOutputHas: "cm://custom-namespace/aicr-snapshot-" + testRunID, wantUsesUserOutput: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - agentOutput, deliverViaConfigMap, err := agentConfigMapTarget(&AgentConfig{ + agentOutput, ownsOutput, err := agentConfigMapTarget(&AgentConfig{ Namespace: tt.agentNamespace, Output: tt.userOutput, + RunID: testRunID, }) if err != nil { t.Fatalf("agentConfigMapTarget: %v", err) @@ -573,15 +607,52 @@ func TestAgentOutputURILogic(t *testing.T) { t.Errorf("agentOutput = %q, want %q", agentOutput, tt.wantAgentOutputHas) } } - if deliverViaConfigMap != tt.wantUsesUserOutput { - t.Errorf("deliverViaConfigMap = %v, want %v — the flag must track whether the "+ - "ConfigMap is the user's delivery vehicle, since it decides whether an "+ - "AKS-pool-merge rewrite failure is fatal", deliverViaConfigMap, tt.wantUsesUserOutput) + // ownsOutput is the logical inverse of "the user supplied this + // output": the run owns (and Cleanup may delete) the staging + // ConfigMap only when it did NOT come from the user. + wantOwnsOutput := !tt.wantUsesUserOutput + if ownsOutput != wantOwnsOutput { + t.Errorf("ownsOutput = %v, want %v — it must track whether this run owns the "+ + "ConfigMap, since it decides whether Cleanup may delete it and whether an "+ + "AKS-pool-merge rewrite failure is fatal", ownsOutput, wantOwnsOutput) } }) } } +// TestAgentConfigMapTargetIsRunScoped and TestAgentConfigMapTargetLeavesUserURIAlone +// pin the ownsOutput inversion: the run owns (and Cleanup may delete) the +// staging ConfigMap only when the user did NOT name a cm:// destination. +// Getting this backwards deletes the user's own output ConfigMap. +func TestAgentConfigMapTargetIsRunScoped(t *testing.T) { + cfg := &AgentConfig{Namespace: "gpu-operator", RunID: "20260821-142233-9f3a1c0b7e2d4a55"} + uri, ownsOutput, err := agentConfigMapTarget(cfg) + if err != nil { + t.Fatalf("agentConfigMapTarget() error = %v", err) + } + want := "cm://gpu-operator/aicr-snapshot-20260821-142233-9f3a1c0b7e2d4a55" + if uri != want { + t.Errorf("uri = %q, want %q", uri, want) + } + if !ownsOutput { + t.Error("ownsOutput = false, want true for the internal staging ConfigMap") + } +} + +func TestAgentConfigMapTargetLeavesUserURIAlone(t *testing.T) { + cfg := &AgentConfig{Namespace: "gpu-operator", Output: "cm://gpu-operator/aicr-snapshot", RunID: "20260821-142233-9f3a1c0b7e2d4a55"} + uri, ownsOutput, err := agentConfigMapTarget(cfg) + if err != nil { + t.Fatalf("agentConfigMapTarget() error = %v", err) + } + if uri != cfg.Output { + t.Errorf("uri = %q, want the user's URI %q unchanged", uri, cfg.Output) + } + if ownsOutput { + t.Error("ownsOutput = true; a user-supplied cm:// output is delivered, not run-owned") + } +} + func TestAgentConfigWithTemplatePath(t *testing.T) { // Test that AgentConfig can hold TemplatePath cfg := AgentConfig{ @@ -1114,6 +1185,20 @@ func TestDeployAndCollectRejectsBeforeClusterAccess(t *testing.T) { }, wantMsg: "Namespace is required", }, + { + // Ruling 5 mitigation: nameWithRunID (pkg/k8s/agent) silently + // falls back to unscoped, collision-prone names when RunID is + // empty. A caller-supplied all-whitespace RunID bypasses the + // simple "== \"\"" default check, so it must fail closed here + // rather than reach that fallback deep in agent.Deploy. + name: "whitespace-only RunID", + config: &AgentConfig{ + Namespace: "default", + Kubeconfig: badKubeconfig, + RunID: " ", + }, + wantMsg: "RunID must not be all-whitespace", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -1131,3 +1216,31 @@ func TestDeployAndCollectRejectsBeforeClusterAccess(t *testing.T) { }) } } + +// TestDeployAndCollectDefaultsRunID confirms DeployAndCollect fills an empty +// RunID with a freshly generated one before it is folded into the internal +// staging ConfigMap's name — observable here because the malformed-Output +// rejection below fires AFTER that defaulting step, and DeployAndCollect +// mutates the caller's *AgentConfig in place. +func TestDeployAndCollectDefaultsRunID(t *testing.T) { + runIDPattern := regexp.MustCompile(`^\d{8}-\d{6}-[0-9a-f]{16}$`) + + cfg := &AgentConfig{ + Namespace: "default", + Kubeconfig: filepath.Join(t.TempDir(), "does-not-exist.kubeconfig"), + Output: "cm://aicr-snapshot", // malformed: no namespace — rejected after RunID defaulting + } + if cfg.RunID != "" { + t.Fatalf("test precondition: cfg.RunID = %q, want empty", cfg.RunID) + } + + _, _, err := DeployAndCollect(t.Context(), cfg) + if err == nil { + t.Fatal("DeployAndCollect() = nil error, want rejection from the malformed Output") + } + + if !runIDPattern.MatchString(cfg.RunID) { + t.Errorf("cfg.RunID = %q after DeployAndCollect, want it filled with a generated ID matching %s", + cfg.RunID, runIDPattern) + } +} From 1e641f4c07ec85e717b2b59c096e6ec8880d98c0 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Fri, 21 Aug 2026 16:33:40 -0700 Subject: [PATCH 09/56] feat(cli): treat job and service-account names as prefixes Signed-off-by: Alex Yuskauskas --- pkg/cli/snapshot.go | 7 ++-- pkg/cli/snapshot_config_test.go | 1 + pkg/cli/validate.go | 56 +++++++++++++++++++--------- pkg/cli/validate_test.go | 65 +++++++++++++++++++++++++++++++++ pkg/client/v1/aicr.go | 21 +++++++++-- pkg/client/v1/translate.go | 2 + pkg/client/v1/types.go | 18 +++++++++ pkg/config/resolve.go | 20 ++++++++-- pkg/snapshotter/agent.go | 8 ++++ pkg/snapshotter/agent_test.go | 20 ++++++---- 10 files changed, 182 insertions(+), 36 deletions(-) diff --git a/pkg/cli/snapshot.go b/pkg/cli/snapshot.go index b5978d997..e0e92d6c0 100644 --- a/pkg/cli/snapshot.go +++ b/pkg/cli/snapshot.go @@ -153,6 +153,7 @@ func (o *snapshotCmdOptions) toAgentConfig() *aicr.AgentConfig { DiscoverNetwork: o.discoverNetwork, Requests: o.requests, Limits: o.limits, + NameBase: name, } } @@ -344,14 +345,12 @@ func snapshotCmdFlags() []cli.Flag { }, &cli.StringFlag{ Name: "job-name", - Usage: "Override default Job name", - Value: name, + Usage: "Job name prefix (default: \"aicr\"); the run ID is always appended", Category: catAgentDeployment, }, &cli.StringFlag{ Name: "service-account-name", - Usage: "Override default ServiceAccount name", - Value: name, + Usage: "ServiceAccount name prefix (default: \"aicr\"); the run ID is always appended", Category: catAgentDeployment, }, &cli.StringSliceFlag{ diff --git a/pkg/cli/snapshot_config_test.go b/pkg/cli/snapshot_config_test.go index 4d0e2820b..256a87179 100644 --- a/pkg/cli/snapshot_config_test.go +++ b/pkg/cli/snapshot_config_test.go @@ -590,6 +590,7 @@ func TestSnapshotCmdOptions_ToAgentConfig(t *testing.T) { {"ClusterConfigPath", ac.ClusterConfigPath, "/l8k/cluster-config.yaml"}, {"AKSGPUPoolsPath", ac.AKSGPUPoolsPath, "/aks/pools.json"}, {"DiscoverNetwork", ac.DiscoverNetwork, true}, + {"NameBase", ac.NameBase, name}, } for _, w := range wants { if !reflect.DeepEqual(w.got, w.want) { diff --git a/pkg/cli/validate.go b/pkg/cli/validate.go index 8e07529cd..8d7b28a0e 100644 --- a/pkg/cli/validate.go +++ b/pkg/cli/validate.go @@ -39,6 +39,14 @@ import ( v1 "github.com/NVIDIA/aicr/pkg/validator/v1" ) +// validateNameBase is the prefix applied to the live-capture snapshot +// agent's Job/ServiceAccount/RBAC names when --job-name / --service-account-name +// are left unset, distinguishing `aicr validate`'s agent resources from +// `aicr snapshot`'s (which use the package-level "aicr" base). RunID is +// always appended, so the deployed names are run-scoped regardless of +// whether this base is used. +const validateNameBase = "aicr-validate" + // validateAgentConfig holds parsed agent configuration for validate command. type validateAgentConfig struct { kubeconfig string @@ -54,6 +62,14 @@ type validateAgentConfig struct { debug bool requireGPU bool aksGPUPoolsPath string + + // runID correlates this run's live-capture snapshot agent with the + // validator Jobs runValidation deploys for the same `aicr validate` + // invocation — both are given the SAME id (generated once, up front, + // by the caller of parseValidateAgentConfig) so ADR-020 run isolation + // scopes every resource this command creates to a single run rather + // than splitting it across two independently generated ids. + runID string } // parseValidateAgentConfig builds the snapshot-capture agent's deployment @@ -61,10 +77,17 @@ type validateAgentConfig struct { // namespace, cleanup) are resolved once by the caller and passed in; this // keeps any CLI-overrides-config slog.Info from firing twice when both // the agent and the downstream validator job want the same value. +// +// runID is the single id generated once for the whole `aicr validate` +// invocation; passing it in here (rather than leaving AgentConfig.RunID +// empty and letting the snapshot agent default its own) is what keeps the +// live-capture agent and the validator Jobs it feeds correlated under one +// RunID. func parseValidateAgentConfig( cmd *cli.Command, resolved *config.ValidateResolved, shared validateSharedResolved, + runID string, ) *validateAgentConfig { return &validateAgentConfig{ @@ -81,6 +104,7 @@ func parseValidateAgentConfig( debug: cmd.Bool("debug"), requireGPU: boolFlagOrConfig(cmd, "require-gpu", resolved.RequireGPU), aksGPUPoolsPath: cmd.String("aks-gpu-pools"), + runID: runID, } } @@ -156,7 +180,11 @@ func resolveValidateTolerations(cmd *cli.Command, resolved *config.ValidateResol // toAgentConfig projects the validate command's resolved flags onto the // facade AgentConfig that Client.CollectSnapshot consumes. Privileged is // unconditional here: the validation snapshot needs the GPU and SystemD -// collectors, which do not work in restricted mode. +// collectors, which do not work in restricted mode. RunID forwards the +// single id parseValidateAgentConfig's caller generated for this whole +// `aicr validate` invocation, so the live-capture agent's Job/RBAC share it +// with the validator Jobs runValidation deploys — one RunID per command, +// per ADR-020, instead of the agent falling back to its own default. func (c *validateAgentConfig) toAgentConfig() *aicr.AgentConfig { return &aicr.AgentConfig{ Kubeconfig: c.kubeconfig, @@ -173,6 +201,8 @@ func (c *validateAgentConfig) toAgentConfig() *aicr.AgentConfig { Privileged: true, RequireGPU: c.requireGPU, AKSGPUPoolsPath: c.aksGPUPoolsPath, + RunID: c.runID, + NameBase: validateNameBase, } } @@ -463,14 +493,12 @@ func validateCmdFlags() []cli.Flag { }, &cli.StringFlag{ Name: "job-name", - Usage: "Override default Job name", - Value: "aicr-validate", + Usage: "Job name prefix (default: \"aicr-validate\"); the run ID is always appended", Category: catAgentDeployment, }, &cli.StringFlag{ Name: "service-account-name", - Usage: "Override default ServiceAccount name", - Value: name, + Usage: "ServiceAccount name prefix (default: \"aicr-validate\"); the run ID is always appended", Category: catAgentDeployment, }, &cli.StringSliceFlag{ @@ -819,17 +847,11 @@ constraint (e.g. K8s version) is not met — --fail-on-error scopes to phase che // validationConfig.runID); generating it here rather than inside // runValidation means a single `aicr validate` no longer risks // splitting its correlation ID across two independent - // v1.GenerateRunID() calls. - // - // KNOWN GAP: the live-capture branch below - // (deployAgentForValidation -> Client.CollectSnapshot) cannot yet - // forward this ID to the snapshot-capture agent's Job/RBAC — - // pkg/client/v1.AgentConfig has no RunID field to receive it - // (pkg/snapshotter.AgentConfig does; mirroring it onto the facade - // is a separate, later change per ADR-020's implementation plan). - // Until that facade field exists and is wired here, a - // live-capture run's agent Job still gets its own independently - // generated RunID from DeployAndCollect's default. + // v1.GenerateRunID() calls. The same runID is also threaded into + // parseValidateAgentConfig below, so the live-capture branch's + // snapshot agent shares this run's id with the validator Jobs + // rather than defaulting its own inside DeployAndCollect — + // ADR-020 requires one RunID per `aicr validate` invocation. runID := v1.GenerateRunID() var snap *aicr.Snapshot @@ -853,7 +875,7 @@ constraint (e.g. K8s version) is not met — --fail-on-error scopes to phase che } else { slog.Info("deploying agent to capture snapshot") - agentCfg := parseValidateAgentConfig(cmd, resolved, shared) + agentCfg := parseValidateAgentConfig(cmd, resolved, shared, runID) var deployErr error snap, deployErr = deployAgentForValidation(ctx, client, agentCfg) diff --git a/pkg/cli/validate_test.go b/pkg/cli/validate_test.go index cd5db95f3..6c4a780c5 100644 --- a/pkg/cli/validate_test.go +++ b/pkg/cli/validate_test.go @@ -470,6 +470,71 @@ func TestDeployAgentForValidation_ExplicitKubeconfigFailsFast(t *testing.T) { } } +// TestValidateAgentConfig_ToAgentConfig_ForwardsRunID pins half of the +// ADR-020 Ruling 7 contract: validateAgentConfig.toAgentConfig must forward +// runID onto the facade AgentConfig.RunID field unchanged, and must supply +// validateNameBase ("aicr-validate") rather than leaving JobName/ +// ServiceAccountName to carry the naming prefix. If toAgentConfig ever drops +// or misassigns runID, this fails even though parseValidateAgentConfig (the +// other half, tested below) still wires the caller's id in correctly. +func TestValidateAgentConfig_ToAgentConfig_ForwardsRunID(t *testing.T) { + cfg := &validateAgentConfig{ + namespace: "aicr-validation-test", + runID: "20260821-142233-9f3a1c0b7e2d4a55", + } + + ac := cfg.toAgentConfig() + + if ac.RunID != cfg.runID { + t.Errorf("AgentConfig.RunID = %q, want %q (validateAgentConfig.runID)", ac.RunID, cfg.runID) + } + if ac.NameBase != validateNameBase { + t.Errorf("AgentConfig.NameBase = %q, want %q", ac.NameBase, validateNameBase) + } +} + +// TestParseValidateAgentConfig_SingleRunIDPerInvocation pins the other half +// of the ADR-020 Ruling 7 contract: `aicr validate` generates exactly one +// RunID per invocation (see the Action's `runID := v1.GenerateRunID()`) and +// must hand that SAME id to parseValidateAgentConfig, which is what +// eventually reaches the live-capture snapshot agent's Job/RBAC via +// toAgentConfig above. Combined with that test, this closes the loop: if a +// future change reintroduces a second, independently generated id for the +// snapshot agent (e.g. reverting to a per-call v1.GenerateRunID() inside +// parseValidateAgentConfig, or simply forgetting to thread the caller's id +// through), one of these two tests fails — exactly the two-run-IDs-per- +// command split ADR-020 forbids. +func TestParseValidateAgentConfig_SingleRunIDPerInvocation(t *testing.T) { + const wantRunID = "20260821-142233-9f3a1c0b7e2d4a55" + + var captured *validateAgentConfig + cmd := validateCmd() + cmd.Action = func(ctx context.Context, c *cli.Command) error { + cfg, err := loadCmdConfig(ctx, c) + if err != nil { + return err + } + resolved, err := cfg.Validation().Resolve() + if err != nil { + return err + } + shared := validateSharedResolved{namespace: "aicr-validation-test"} + // wantRunID stands in for the Action's single + // `runID := v1.GenerateRunID()` call — the production code path + // passes that same local variable to both parseValidateAgentConfig + // (here) and validationConfig.runID (runValidation). + captured = parseValidateAgentConfig(c, resolved, shared, wantRunID) + return nil + } + if err := cmd.Run(t.Context(), []string{"validate", "--no-cluster"}); err != nil { + t.Fatalf("validate run: %v", err) + } + + if captured.runID != wantRunID { + t.Errorf("validateAgentConfig.runID = %q, want %q", captured.runID, wantRunID) + } +} + // TestClassifyIgnoredAKSGPUPools pins the provenance matrix of the // ignored-projection note: explicit CLI presence (either flag form) always // warns; a purely ambient env source is demoted to debug; nothing logs diff --git a/pkg/client/v1/aicr.go b/pkg/client/v1/aicr.go index 4fa6c3b12..391f77965 100644 --- a/pkg/client/v1/aicr.go +++ b/pkg/client/v1/aicr.go @@ -1725,8 +1725,12 @@ func resolveHelmComponentValues( // without breaking signatures. CollectSnapshot is therefore safe even // on a Client whose recipe source is unrelated to the target cluster. // -// cfg.Kubeconfig is the path (or empty for in-cluster). cfg.Namespace, -// cfg.Image, cfg.ServiceAccountName must be set; other fields fall +// cfg.Kubeconfig is the path (or empty for in-cluster). cfg.Namespace and +// cfg.Image must be set. cfg.JobName and cfg.ServiceAccountName are +// optional naming prefixes, not required names — leaving them empty is +// fine: cfg.NameBase (default "aicr") supplies the prefix instead, and +// cfg.RunID is appended to whichever prefix applies, so every object this +// call deploys is named uniquely to this run either way. Other fields fall // back to package defaults documented on snapshotter.AgentConfig. // // # Output and delivery @@ -1784,8 +1788,17 @@ func resolveHelmComponentValues( // carry the appropriate pkg/errors codes (ErrCodeInternal for // deployment failures, ErrCodeTimeout for context expiry, etc.). // -// Concurrent CollectSnapshot calls are safe; each call constructs an -// independent run. +// Concurrent CollectSnapshot calls are safe: each gets its own RunID +// (cfg.RunID when set, otherwise generated) and, from it, its own Job, +// RBAC, and — when cfg.Output does not name one — its own staging +// ConfigMap; independence means each call's objects, permissions, +// captured result, and Cleanup are scoped to that RunID alone, so +// concurrent runs never collide on a shared resource name or delete +// something another run created. The one effect that is still shared by +// design: two calls that set cfg.Output to the SAME explicit +// cm://namespace/name URI write to that one caller-named ConfigMap and +// overwrite each other — Output identifies a caller-owned destination, +// not a run-scoped one, so RunID does not disambiguate it. func (c *Client) CollectSnapshot(ctx context.Context, cfg *AgentConfig) (*Snapshot, error) { if c == nil { return nil, errors.New(errors.ErrCodeInvalidRequest, "aicr client not initialized") diff --git a/pkg/client/v1/translate.go b/pkg/client/v1/translate.go index 8bbbfc905..a7377b592 100644 --- a/pkg/client/v1/translate.go +++ b/pkg/client/v1/translate.go @@ -67,6 +67,8 @@ func toInternalAgentConfig(cfg *AgentConfig) *snapshotter.AgentConfig { Requests: cfg.Requests, Limits: cfg.Limits, AKSGPUPoolsPath: cfg.AKSGPUPoolsPath, + RunID: cfg.RunID, + NameBase: cfg.NameBase, } } diff --git a/pkg/client/v1/types.go b/pkg/client/v1/types.go index 87693ed84..1b0cbfb52 100644 --- a/pkg/client/v1/types.go +++ b/pkg/client/v1/types.go @@ -175,6 +175,24 @@ type AgentConfig struct { // Required for AKS profile-qualified resolution from a collected // snapshot; empty disables the projection. AKSGPUPoolsPath string + + // RunID scopes every resource this deployment creates (Job, RBAC, and + // the internal staging ConfigMap when Output does not name one) to a + // single run, so concurrent snapshot-agent runs never collide on a + // shared resource name. Empty lets CollectSnapshot's underlying + // deployment generate one; SDK and CLI callers normally leave it + // unset. Set it explicitly to correlate this run with an external + // identifier — `aicr validate` does this to give its live-capture + // snapshot agent and its validator Jobs the same RunID. + RunID string + + // NameBase prefixes generated Job/ServiceAccount/RBAC names when + // JobName and ServiceAccountName are left empty; it has no effect + // once either of those is set. Defaults to "aicr" when also empty. + // JobName and ServiceAccountName themselves are optional prefixes, + // not required names — RunID is appended to whichever prefix + // applies, so the deployed object names are always run-scoped. + NameBase string } // Criteria is the facade-owned, semver-stable shape of a recipe-resolution diff --git a/pkg/config/resolve.go b/pkg/config/resolve.go index f0231da19..187271f4a 100644 --- a/pkg/config/resolve.go +++ b/pkg/config/resolve.go @@ -337,10 +337,16 @@ type ValidateResolved struct { // config did not set the field. ImagePullSecrets []string - // JobName is spec.validate.agent.jobName. + // JobName is spec.validate.agent.jobName — an optional Job name + // prefix, not a required name. Empty (config unset and no + // --job-name) lets the CLI's own default prefix ("aicr-validate") + // apply instead; either way the run ID is appended, so the deployed + // Job name is always run-scoped. JobName string - // ServiceAccountName is spec.validate.agent.serviceAccountName. + // ServiceAccountName is spec.validate.agent.serviceAccountName — an + // optional ServiceAccount name prefix, not a required name. Same + // empty-value behavior as JobName. ServiceAccountName string // NodeSelector is spec.validate.agent.nodeSelector. Nil if unset; @@ -641,10 +647,16 @@ type SnapshotResolved struct { // config did not set the field. ImagePullSecrets []string - // JobName is spec.snapshot.agent.jobName. + // JobName is spec.snapshot.agent.jobName — an optional Job name + // prefix, not a required name. Empty (config unset and no + // --job-name) lets the CLI's own default prefix ("aicr") apply + // instead; either way the run ID is appended, so the deployed Job + // name is always run-scoped. JobName string - // ServiceAccountName is spec.snapshot.agent.serviceAccountName. + // ServiceAccountName is spec.snapshot.agent.serviceAccountName — an + // optional ServiceAccount name prefix, not a required name. Same + // empty-value behavior as JobName. ServiceAccountName string // NodeSelector is spec.snapshot.agent.nodeSelector. Nil if unset; diff --git a/pkg/snapshotter/agent.go b/pkg/snapshotter/agent.go index c00ac0f25..1d577f3de 100644 --- a/pkg/snapshotter/agent.go +++ b/pkg/snapshotter/agent.go @@ -158,6 +158,13 @@ type AgentConfig struct { // setting it explicitly is for correlating this run with an external // identifier (e.g. sharing one ID with a downstream validator run). RunID string + + // NameBase prefixes generated resource names (Job, ServiceAccount, + // Role/RoleBinding) when JobName / ServiceAccountName are left empty. + // It has no effect once either of those is set. Forwarded verbatim + // to pkg/k8s/agent.Config.NameBase, which defaults to "aicr" when + // also empty. + NameBase string } // buildAgentConfig projects snapshotter configuration onto the deployer's @@ -174,6 +181,7 @@ func buildAgentConfig(config *AgentConfig, agentOutput string, ownsOutput bool) ServiceAccountName: config.ServiceAccountName, JobName: config.JobName, RunID: config.RunID, + NameBase: config.NameBase, Image: config.Image, ImagePullSecrets: config.ImagePullSecrets, NodeSelector: config.NodeSelector, diff --git a/pkg/snapshotter/agent_test.go b/pkg/snapshotter/agent_test.go index edfa11609..f6f295fac 100644 --- a/pkg/snapshotter/agent_test.go +++ b/pkg/snapshotter/agent_test.go @@ -98,11 +98,12 @@ func TestBuildAgentConfigTolerations(t *testing.T) { } // TestBuildAgentConfigPropagatesRunIDAndOwnership confirms buildAgentConfig -// forwards AgentConfig.RunID and its ownsOutput parameter onto -// agent.Config.RunID / agent.Config.OwnsOutputConfigMap — the projection -// deployAndWaitForResult relies on so the deployer scopes every resource -// name to this run and Cleanup knows whether it may delete the staging -// ConfigMap. +// forwards AgentConfig.RunID, AgentConfig.NameBase, and its ownsOutput +// parameter onto agent.Config.RunID / agent.Config.NameBase / +// agent.Config.OwnsOutputConfigMap — the projection deployAndWaitForResult +// relies on so the deployer scopes every resource name to this run, applies +// the caller's naming prefix, and Cleanup knows whether it may delete the +// staging ConfigMap. func TestBuildAgentConfigPropagatesRunIDAndOwnership(t *testing.T) { tests := []struct { name string @@ -113,11 +114,16 @@ func TestBuildAgentConfigPropagatesRunIDAndOwnership(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := buildAgentConfig(&AgentConfig{RunID: "20260821-142233-9f3a1c0b7e2d4a55"}, - "cm://ns/name", tt.ownsOutput) + got := buildAgentConfig(&AgentConfig{ + RunID: "20260821-142233-9f3a1c0b7e2d4a55", + NameBase: "aicr-validate", + }, "cm://ns/name", tt.ownsOutput) if got.RunID != "20260821-142233-9f3a1c0b7e2d4a55" { t.Errorf("agent.Config.RunID = %q, want the AgentConfig.RunID value", got.RunID) } + if got.NameBase != "aicr-validate" { + t.Errorf("agent.Config.NameBase = %q, want the AgentConfig.NameBase value", got.NameBase) + } if got.OwnsOutputConfigMap != tt.ownsOutput { t.Errorf("agent.Config.OwnsOutputConfigMap = %v, want %v", got.OwnsOutputConfigMap, tt.ownsOutput) } From d7d3c2b0026ff5b6c3f993b91f4d693c85719449 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Fri, 21 Aug 2026 16:47:27 -0700 Subject: [PATCH 10/56] fix(cli): stop Ruling-7 tests from overclaiming single-generation coverage Signed-off-by: Alex Yuskauskas --- pkg/cli/validate.go | 17 +++++++++++++ pkg/cli/validate_test.go | 55 +++++++++++++++++++++++----------------- 2 files changed, 49 insertions(+), 23 deletions(-) diff --git a/pkg/cli/validate.go b/pkg/cli/validate.go index 8d7b28a0e..1ccab3c2d 100644 --- a/pkg/cli/validate.go +++ b/pkg/cli/validate.go @@ -852,6 +852,23 @@ constraint (e.g. K8s version) is not met — --fail-on-error scopes to phase che // snapshot agent shares this run's id with the validator Jobs // rather than defaulting its own inside DeployAndCollect — // ADR-020 requires one RunID per `aicr validate` invocation. + // + // WARNING, not enforced by an automated test: this line must + // stay the ONLY v1.GenerateRunID() call in this Action. Adding a + // second call — e.g. generating a fresh id inline for the + // parseValidateAgentConfig call below, or for the + // validationConfig{runID: ...} literal further down — silently + // splits a single `aicr validate` invocation back into two + // uncorrelated runs. The command's two consumer sites + // (live-capture agent vs. validator Jobs) sit on mutually + // exclusive code paths (--no-cluster requires --snapshot, which + // skips the live-capture branch entirely), so no unit test can + // observe both consumers receiving the SAME value in one + // invocation without a live cluster; see + // TestValidateAgentConfig_ToAgentConfig_ForwardsRunID and + // TestParseValidateAgentConfig_ForwardsCallerRunID in + // validate_test.go for what IS covered (passthrough at each + // function boundary only) and what is not. runID := v1.GenerateRunID() var snap *aicr.Snapshot diff --git a/pkg/cli/validate_test.go b/pkg/cli/validate_test.go index 6c4a780c5..a78ff169e 100644 --- a/pkg/cli/validate_test.go +++ b/pkg/cli/validate_test.go @@ -470,13 +470,20 @@ func TestDeployAgentForValidation_ExplicitKubeconfigFailsFast(t *testing.T) { } } -// TestValidateAgentConfig_ToAgentConfig_ForwardsRunID pins half of the -// ADR-020 Ruling 7 contract: validateAgentConfig.toAgentConfig must forward -// runID onto the facade AgentConfig.RunID field unchanged, and must supply -// validateNameBase ("aicr-validate") rather than leaving JobName/ -// ServiceAccountName to carry the naming prefix. If toAgentConfig ever drops -// or misassigns runID, this fails even though parseValidateAgentConfig (the -// other half, tested below) still wires the caller's id in correctly. +// TestValidateAgentConfig_ToAgentConfig_ForwardsRunID covers ONLY the +// toAgentConfig projection boundary: given a validateAgentConfig whose +// runID field is already set, toAgentConfig must copy it onto the facade +// AgentConfig.RunID unchanged (and set NameBase to validateNameBase rather +// than leaving JobName/ServiceAccountName to carry the naming prefix). +// +// What this does NOT cover: it never runs the validateCmd Action, so it +// says nothing about whether `aicr validate` generates exactly one RunID +// per invocation and hands that SAME value to both the live-capture +// snapshot agent and the validator Jobs — that single-generation invariant +// (ADR-020 Ruling 7) is not exercised by an automated test at all; see the +// WARNING comment on the Action's `runID := v1.GenerateRunID()` call in +// validate.go for why, and TestParseValidateAgentConfig_ForwardsCallerRunID +// below for the adjacent (also boundary-only) coverage on the parsing side. func TestValidateAgentConfig_ToAgentConfig_ForwardsRunID(t *testing.T) { cfg := &validateAgentConfig{ namespace: "aicr-validation-test", @@ -493,18 +500,24 @@ func TestValidateAgentConfig_ToAgentConfig_ForwardsRunID(t *testing.T) { } } -// TestParseValidateAgentConfig_SingleRunIDPerInvocation pins the other half -// of the ADR-020 Ruling 7 contract: `aicr validate` generates exactly one -// RunID per invocation (see the Action's `runID := v1.GenerateRunID()`) and -// must hand that SAME id to parseValidateAgentConfig, which is what -// eventually reaches the live-capture snapshot agent's Job/RBAC via -// toAgentConfig above. Combined with that test, this closes the loop: if a -// future change reintroduces a second, independently generated id for the -// snapshot agent (e.g. reverting to a per-call v1.GenerateRunID() inside -// parseValidateAgentConfig, or simply forgetting to thread the caller's id -// through), one of these two tests fails — exactly the two-run-IDs-per- -// command split ADR-020 forbids. -func TestParseValidateAgentConfig_SingleRunIDPerInvocation(t *testing.T) { +// TestParseValidateAgentConfig_ForwardsCallerRunID covers ONLY +// parseValidateAgentConfig's own mapping: the runID parameter it is given +// lands unchanged on the returned validateAgentConfig.runID field. +// +// This test replaces cmd.Action outright (to isolate that mapping from +// recipe/snapshot I/O without touching a cluster), so NONE of the +// production Action code at validate.go's `runID := v1.GenerateRunID()` +// call through the parseValidateAgentConfig call site actually runs here. +// It cannot detect a regression in how the real Action generates or +// threads runID — in particular it says nothing about ADR-020 Ruling 7's +// single-generation invariant (one v1.GenerateRunID() call feeding BOTH +// the live-capture agent and the validator Jobs). See the WARNING comment +// on that call site in validate.go: no automated test in this package +// enforces single-generation, because the Action's two consumer sites sit +// on mutually exclusive code paths (--no-cluster requires --snapshot, +// which skips the live-capture branch entirely) and cannot both be +// exercised in one invocation without a live cluster. +func TestParseValidateAgentConfig_ForwardsCallerRunID(t *testing.T) { const wantRunID = "20260821-142233-9f3a1c0b7e2d4a55" var captured *validateAgentConfig @@ -519,10 +532,6 @@ func TestParseValidateAgentConfig_SingleRunIDPerInvocation(t *testing.T) { return err } shared := validateSharedResolved{namespace: "aicr-validation-test"} - // wantRunID stands in for the Action's single - // `runID := v1.GenerateRunID()` call — the production code path - // passes that same local variable to both parseValidateAgentConfig - // (here) and validationConfig.runID (runValidation). captured = parseValidateAgentConfig(c, resolved, shared, wantRunID) return nil } From 46a14694247c53e8b41f62e27deff7172db3545a Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Fri, 21 Aug 2026 16:59:41 -0700 Subject: [PATCH 11/56] docs: update selectors and names for run-scoped agent resources Signed-off-by: Alex Yuskauskas --- .../debug-snapshot-job.sh | 6 +- docs/integrator/automation.md | 3 +- docs/integrator/go-library.md | 24 ++++---- docs/user/agent-deployment.md | 55 ++++++++++--------- docs/user/cli-reference.md | 10 ++-- tests/e2e/run.sh | 6 +- tools/cleanup | 17 ++++-- 7 files changed, 67 insertions(+), 54 deletions(-) diff --git a/.github/actions/gpu-snapshot-validate/debug-snapshot-job.sh b/.github/actions/gpu-snapshot-validate/debug-snapshot-job.sh index aa94bcd7e..0549f50dc 100644 --- a/.github/actions/gpu-snapshot-validate/debug-snapshot-job.sh +++ b/.github/actions/gpu-snapshot-validate/debug-snapshot-job.sh @@ -20,11 +20,11 @@ kubectl_kind() { } echo "=== Snapshot Job ===" -kubectl_kind -n default get job aicr -o yaml || true +kubectl_kind -n default get job -l app.kubernetes.io/name=aicr -o yaml || true echo "=== Snapshot Pods ===" kubectl_kind -n default get pods -l app.kubernetes.io/name=aicr -o wide || true echo "=== Snapshot Job describe ===" -kubectl_kind -n default describe job aicr || true +kubectl_kind -n default describe job -l app.kubernetes.io/name=aicr || true echo "=== Snapshot Pod describe ===" kubectl_kind -n default describe pods -l app.kubernetes.io/name=aicr || true echo "=== Snapshot current logs ===" @@ -32,4 +32,4 @@ kubectl_kind -n default logs -l app.kubernetes.io/name=aicr --all-containers --t echo "=== Snapshot previous logs ===" kubectl_kind -n default logs -l app.kubernetes.io/name=aicr --all-containers --previous --tail=200 || true echo "=== Snapshot ConfigMap ===" -kubectl_kind -n default get configmap aicr-snapshot -o yaml || true +kubectl_kind -n default get configmap -l app.kubernetes.io/name=aicr -o yaml || true diff --git a/docs/integrator/automation.md b/docs/integrator/automation.md index 9c46cd411..17c0da66d 100644 --- a/docs/integrator/automation.md +++ b/docs/integrator/automation.md @@ -497,7 +497,8 @@ metadata: spec: podSelector: matchLabels: - job-name: aicr + app.kubernetes.io/name: aicr + app.kubernetes.io/component: snapshot-agent policyTypes: - Egress egress: diff --git a/docs/integrator/go-library.md b/docs/integrator/go-library.md index 97059175a..4341a33b9 100644 --- a/docs/integrator/go-library.md +++ b/docs/integrator/go-library.md @@ -248,18 +248,18 @@ snapCtx, cancelSnap := context.WithTimeout(context.Background(), 10*time.Minute) defer cancelSnap() snap, err := client.CollectSnapshot(snapCtx, &aicr.AgentConfig{ Kubeconfig: "/path/to/target-kubeconfig", - // Namespace, Image, JobName, and ServiceAccountName are all required on - // the SDK path. Only Namespace is validated; the rest are copied straight - // into the Job and RBAC objects, so an empty value becomes an empty - // metadata.name or container image that the API server rejects. The CLI - // defaults them from its own flags, which the facade does not share. - Namespace: "aicr-snapshot", - Image: "ghcr.io/nvidia/aicr:v0.19.0", - JobName: "aicr-snapshot", - ServiceAccountName: "aicr-agent", - Timeout: 5 * time.Minute, - Cleanup: true, - AKSGPUPoolsPath: "/path/to/aks-gpu-pools.json", // AKS only + // Namespace is required and validated (it becomes the RBAC/Job namespace + // and the internal staging ConfigMap's namespace). Image is not + // validated — an empty value becomes an empty container image that the + // API server rejects. JobName and ServiceAccountName are optional name + // prefixes; leaving them unset defaults both to "aicr" with a generated + // run ID appended, so every run gets its own uniquely named Job and + // ServiceAccount without the caller having to manage that. + Namespace: "aicr-snapshot", + Image: "ghcr.io/nvidia/aicr:v0.19.0", + Timeout: 5 * time.Minute, + Cleanup: true, + AKSGPUPoolsPath: "/path/to/aks-gpu-pools.json", // AKS only }) if err != nil { log.Fatalf("collect snapshot: %v", err) diff --git a/docs/user/agent-deployment.md b/docs/user/agent-deployment.md index f59bada92..b4b139e52 100644 --- a/docs/user/agent-deployment.md +++ b/docs/user/agent-deployment.md @@ -129,12 +129,12 @@ aicr snapshot \ - `--namespace`: Deployment namespace (default: `default`) - `--image`: Container image (default: matches the CLI version, e.g. `ghcr.io/nvidia/aicr:v0.19.0`; dev and snapshot builds use `:latest`) - `--image-pull-secret`: Secret name for pulling the agent image from a private registry (repeatable) -- `--job-name`: Job name (default: `aicr`) -- `--service-account-name`: ServiceAccount name (default: `aicr`) +- `--job-name`: Job name prefix (default: `aicr`); the run ID is always appended (`-`) +- `--service-account-name`: ServiceAccount name prefix (default: `aicr`); the run ID is always appended (`-`) - `--node-selector`: Node selector (format: `key=value`, repeatable) - `--toleration`: Toleration (format: `key=value:effect`, repeatable). **Default: all taints are tolerated** (uses `operator: Exists` without key). Only specify this flag if you want to restrict which taints the Job can tolerate. - `--timeout`: Wait timeout (default: `5m`) -- `--no-cleanup`: Skip removal of Job and RBAC resources on completion. **Warning:** leaves the `aicr-node-reader` ClusterRole and ClusterRoleBinding active. By default these grant only read access to nodes, pods, ClusterPolicy CRDs, Slinky Controller/NodeSet/LoginSet/RestApi/Accounting CRs, and official MariaDB CRs (not cluster-admin); however, when combined with `--discover-network` the retained ClusterRole also carries the cluster-scoped **mutating** discovery rules (CRD/namespace/DaemonSet create-delete, `pods/exec`, `nodes/patch`, `NicClusterPolicy` patch — see [Security Considerations](#security-considerations)), so it is **not** read-only in that case. +- `--no-cleanup`: Skip removal of Job and RBAC resources on completion. **Warning:** leaves the run-scoped `aicr-node-reader-` ClusterRole and ClusterRoleBinding active. By default these grant only read access to nodes, pods, ClusterPolicy CRDs, Slinky Controller/NodeSet/LoginSet/RestApi/Accounting CRs, and official MariaDB CRs (not cluster-admin); however, when combined with `--discover-network` the retained ClusterRole also carries the cluster-scoped **mutating** discovery rules (CRD/namespace/DaemonSet create-delete, `pods/exec`, `nodes/patch`, `NicClusterPolicy` patch — see [Security Considerations](#security-considerations)), so it is **not** read-only in that case. - `--privileged`: Run agent in privileged mode (default: enabled; required for GPU/SystemD collectors). Set to `false` for PSS-restricted namespaces. - `--require-gpu`: Fail the snapshot if no GPU is found. In agent mode also requests an `nvidia.com/gpu` resource for the pod (required in CDI environments). - `--runtime-class`: Set `runtimeClassName` on the agent pod for `nvidia-smi` access without consuming a GPU. Use with `--node-selector` to target GPU nodes. @@ -150,13 +150,13 @@ If something goes wrong, check Job logs: ```shell # Get Job status -kubectl get jobs -n gpu-operator +kubectl get jobs -n gpu-operator -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent # View logs -kubectl logs -n gpu-operator job/aicr +kubectl logs -n gpu-operator -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent # Describe Job for events -kubectl describe job aicr -n gpu-operator +kubectl describe job -n gpu-operator -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent ``` ## Customization @@ -326,22 +326,23 @@ aicr diff --baseline baseline.yaml --target current.yaml --fail-on-drift \ ### Job Fails to Start -Check RBAC permissions: +Check RBAC permissions. The ServiceAccount name is run-scoped (`aicr-`), so look it up first: ```shell -kubectl auth can-i get nodes --as=system:serviceaccount:gpu-operator:aicr -kubectl auth can-i get pods --as=system:serviceaccount:gpu-operator:aicr +SA=$(kubectl get sa -n gpu-operator -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent -o jsonpath='{.items[0].metadata.name}') +kubectl auth can-i get nodes --as=system:serviceaccount:gpu-operator:$SA +kubectl auth can-i get pods --as=system:serviceaccount:gpu-operator:$SA kubectl auth can-i list controllers.slinky.slurm.net --all-namespaces \ - --as=system:serviceaccount:gpu-operator:aicr + --as=system:serviceaccount:gpu-operator:$SA kubectl auth can-i list nodesets.slinky.slurm.net --all-namespaces \ - --as=system:serviceaccount:gpu-operator:aicr + --as=system:serviceaccount:gpu-operator:$SA kubectl auth can-i list loginsets.slinky.slurm.net --all-namespaces \ - --as=system:serviceaccount:gpu-operator:aicr + --as=system:serviceaccount:gpu-operator:$SA kubectl auth can-i list restapis.slinky.slurm.net --all-namespaces \ - --as=system:serviceaccount:gpu-operator:aicr + --as=system:serviceaccount:gpu-operator:$SA kubectl auth can-i list accountings.slinky.slurm.net --all-namespaces \ - --as=system:serviceaccount:gpu-operator:aicr + --as=system:serviceaccount:gpu-operator:$SA kubectl auth can-i list mariadbs.k8s.mariadb.com --all-namespaces \ - --as=system:serviceaccount:gpu-operator:aicr + --as=system:serviceaccount:gpu-operator:$SA ``` ### Job Pending @@ -349,7 +350,7 @@ kubectl auth can-i list mariadbs.k8s.mariadb.com --all-namespaces \ Check node selectors and tolerations: ```shell # View pod events -kubectl describe pod -n gpu-operator -l job-name=aicr +kubectl describe pod -n gpu-operator -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent # Check node labels kubectl get nodes --show-labels @@ -369,25 +370,25 @@ kubectl get configmap aicr-snapshot -n gpu-operator kubectl get configmap aicr-snapshot -n gpu-operator -o yaml # View pod logs for errors -kubectl logs -n gpu-operator -l job-name=aicr +kubectl logs -n gpu-operator -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent ``` ### Permission Denied Ensure RBAC is correctly deployed: ```shell -# Verify ClusterRole -kubectl get clusterrole aicr-node-reader +# Verify ClusterRole (run-scoped: "aicr-node-reader-") +kubectl get clusterrole -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent # Verify ClusterRoleBinding -kubectl get clusterrolebinding aicr-node-reader +kubectl get clusterrolebinding -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent -# Verify Role and RoleBinding -kubectl get role aicr -n gpu-operator -kubectl get rolebinding aicr -n gpu-operator +# Verify Role and RoleBinding (run-scoped: "aicr-") +kubectl get role -n gpu-operator -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent +kubectl get rolebinding -n gpu-operator -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent -# Verify ServiceAccount -kubectl get serviceaccount aicr -n gpu-operator +# Verify ServiceAccount (run-scoped: "aicr-") +kubectl get serviceaccount -n gpu-operator -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent ``` ## Security Considerations @@ -395,8 +396,8 @@ kubectl get serviceaccount aicr -n gpu-operator ### RBAC Permissions The agent requires these permissions (created automatically by the CLI): -- **ClusterRole** (`aicr-node-reader`): Read access to nodes and pods; `get`/`list` access to ClusterPolicy CRDs (`nvidia.com`); cluster-wide `list` access to Slinky Controller, NodeSet, LoginSet, RestApi, and Accounting CRs (`slinky.slurm.net`); and cluster-wide `list` access to official MariaDB CRs (`k8s.mariadb.com`) -- **Role** (`aicr`): Create/update ConfigMaps and list pods in the deployment namespace +- **ClusterRole** (`aicr-node-reader-`, run-scoped): Read access to nodes and pods; `get`/`list` access to ClusterPolicy CRDs (`nvidia.com`); cluster-wide `list` access to Slinky Controller, NodeSet, LoginSet, RestApi, and Accounting CRs (`slinky.slurm.net`); and cluster-wide `list` access to official MariaDB CRs (`k8s.mariadb.com`) +- **Role** (`aicr-`, run-scoped): Create/update ConfigMaps and list pods in the deployment namespace The baseline ClusterRole above is read-only (`get`/`list` only). Slinky detection projects only allowlisted identity, association, and boolean fields; diff --git a/docs/user/cli-reference.md b/docs/user/cli-reference.md index 73e6c5c1e..8b025f525 100644 --- a/docs/user/cli-reference.md +++ b/docs/user/cli-reference.md @@ -84,12 +84,12 @@ aicr snapshot [flags] | `--kubeconfig` | `-k` | string | ~/.kube/config | Path to kubeconfig file (overrides KUBECONFIG env). Also used when `--output` is a ConfigMap URI so reads and writes target the same cluster. | | `--namespace` | `-n` | string | default | Kubernetes namespace for agent deployment | | `--image` | | string | matches CLI version | Container image for agent Job. Release builds default to `ghcr.io/nvidia/aicr:v`; dev and `-next` snapshot builds default to `ghcr.io/nvidia/aicr:latest`. | -| `--job-name` | | string | aicr | Name for the agent Job | -| `--service-account-name` | | string | aicr | ServiceAccount name for agent Job | +| `--job-name` | | string | aicr | Prefix for the agent Job name; the run ID is always appended (`-`) | +| `--service-account-name` | | string | aicr | Prefix for the agent Job's ServiceAccount name; the run ID is always appended (`-`) | | `--node-selector` | | string[] | auto | Node selector for agent scheduling (key=value, repeatable). When omitted (and neither `--require-gpu` nor `--runtime-class` is set), the agent auto-targets GPU nodes labeled `nvidia.com/gpu.present=true` if the cluster has any — see [Agent Deployment](agent-deployment.md). Pass an explicit selector to override. | | `--toleration` | | string[] | all taints | Tolerations for agent scheduling (key=value:effect, repeatable). **Default: all taints tolerated** (uses `operator: Exists`). Only specify to restrict which taints are tolerated. | | `--timeout` | | duration | 5m | Timeout for agent Job completion | -| `--no-cleanup` | | bool | false | Skip removal of Job and RBAC resources on completion. **Warning:** leaves the agent's `aicr-node-reader` ClusterRoleBinding active. By default this grants only read-only access; with `--discover-network` the retained ClusterRole also carries the mutating rules live network discovery needs (CRD/namespace/daemonset create, pod exec, node patch, NicClusterPolicy). | +| `--no-cleanup` | | bool | false | Skip removal of Job and RBAC resources on completion. **Warning:** leaves the agent's run-scoped `aicr-node-reader-` ClusterRoleBinding active. By default this grants only read-only access; with `--discover-network` the retained ClusterRole also carries the mutating rules live network discovery needs (CRD/namespace/daemonset create, pod exec, node patch, NicClusterPolicy). | | `--privileged` | | bool | true | Run agent in privileged mode (required for GPU/SystemD collectors). Set to false for PSS-restricted namespaces. | | `--image-pull-secret` | | string[] | | Image pull secrets for private registries (repeatable) | | `--require-gpu` | | bool | false | Require GPU resources on the agent pod (mutually exclusive with `--runtime-class`) | @@ -1031,8 +1031,8 @@ aicr validate [flags] | `--namespace` | `-n` | string | aicr-validation | Kubernetes namespace for validation Job deployment | | `--image` | | string | ghcr.io/nvidia/aicr:latest | Container image for validation Job | | `--image-pull-secret` | | string[] | | Image pull secrets for private registries (repeatable) | -| `--job-name` | | string | aicr-validate | Name for the validation Job | -| `--service-account-name` | | string | aicr | ServiceAccount name for validation Job | +| `--job-name` | | string | aicr-validate | Prefix for the validation Job name; the run ID is always appended (`-`) | +| `--service-account-name` | | string | aicr-validate | Prefix for the validation Job's ServiceAccount name; the run ID is always appended (`-`) | | `--node-selector` | | string[] | | Override GPU node selection for the live snapshot agent (when `--snapshot` is omitted) and inner validation workloads. Replaces platform-specific selectors (e.g., `cloud.google.com/gke-accelerator`, `node.kubernetes.io/instance-type`) on inner workloads like NCCL benchmark pods. Use when GPU nodes have non-standard labels. Does not affect the validator orchestrator Job. (format: key=value, repeatable) | | `--toleration` | | string[] | | Override tolerations for the live snapshot agent (when `--snapshot` is omitted) and inner validation workloads. When omitted, the snapshot agent tolerates all taints. Does not affect the validator orchestrator Job. (format: key=value:effect, repeatable) | | `--timeout` | | duration | 5m | Timeout for validation Job completion | diff --git a/tests/e2e/run.sh b/tests/e2e/run.sh index 6ef031851..d098721d2 100755 --- a/tests/e2e/run.sh +++ b/tests/e2e/run.sh @@ -1879,8 +1879,10 @@ cleanup_e2e() { msg "Cleaning up e2e resources" msg "==========================================" - # Clean up snapshot resources - kubectl delete job aicr-e2e-snapshot -n "$SNAPSHOT_NAMESPACE" --ignore-not-found=true > /dev/null 2>&1 || true + # Clean up snapshot resources. The Job name is run-scoped ("aicr-"), + # so select by the label every run-owned agent resource carries instead of + # a fixed name. + kubectl delete job -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent -n "$SNAPSHOT_NAMESPACE" --ignore-not-found=true > /dev/null 2>&1 || true kubectl delete cm "$SNAPSHOT_CM" -n "$SNAPSHOT_NAMESPACE" --ignore-not-found=true > /dev/null 2>&1 || true msg "Cleanup complete" diff --git a/tools/cleanup b/tools/cleanup index 44cfdd278..a1b33039b 100755 --- a/tools/cleanup +++ b/tools/cleanup @@ -391,10 +391,19 @@ if ! $DRY_RUN; then fi kc delete ns aicr-validation --ignore-not-found --wait=false # Legacy on-cluster agent leftovers from the older deployment pattern. -kc -n gpu-operator delete job aicr --ignore-not-found -kc -n gpu-operator delete sa aicr --ignore-not-found -kc -n gpu-operator delete role aicr --ignore-not-found -kc -n gpu-operator delete rolebinding aicr --ignore-not-found +# Run-scoped agent resources (Job/SA/Role/RoleBinding) are named +# "aicr-", so a fixed-name delete never matches current runs; select +# by the label every one of them carries instead. +kc -n gpu-operator delete job -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent --ignore-not-found +kc -n gpu-operator delete sa -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent --ignore-not-found +kc -n gpu-operator delete role -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent --ignore-not-found +kc -n gpu-operator delete rolebinding -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent --ignore-not-found +# One-time removal of the pre-change unlabeled cluster RBAC: before this +# labeling was introduced, the agent's ClusterRole/ClusterRoleBinding were +# named literally "aicr-node-reader" with no labels, so no selector above can +# find them. +kc delete clusterrole aicr-node-reader --ignore-not-found +kc delete clusterrolebinding aicr-node-reader --ignore-not-found # Phase 3: Component CRDs. # Helm does NOT remove CRDs on uninstall — they must be deleted manually or a From 82e1a6da62bb07073e4af4e324d5ff57ac4b3fef Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Sat, 22 Aug 2026 18:21:56 -0700 Subject: [PATCH 12/56] test(agent): prove concurrent runs are isolated Signed-off-by: Alex Yuskauskas --- pkg/k8s/agent/concurrency_test.go | 401 ++++++++++++++++++++++++++++++ 1 file changed, 401 insertions(+) create mode 100644 pkg/k8s/agent/concurrency_test.go diff --git a/pkg/k8s/agent/concurrency_test.go b/pkg/k8s/agent/concurrency_test.go new file mode 100644 index 000000000..58c45b2d4 --- /dev/null +++ b/pkg/k8s/agent/concurrency_test.go @@ -0,0 +1,401 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package agent + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + authv1 "k8s.io/api/authorization/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" +) + +// concurrencyTestNamespace hosts both runs in TestConcurrentRuns. Both runs +// share one namespace on purpose: name-based isolation within a shared +// namespace is exactly the property issue #2120 reported broken (fixed +// object names meant two runs clobbered each other's Job, RBAC, and +// snapshot ConfigMap). +const concurrencyTestNamespace = "concurrent-runs-ns" + +// Two run IDs in runid.Generate()'s YYYYMMDD-HHMMSS-<16 hex> format, equal +// in every field except the random suffix — isolation must hold between +// runs started in the same instant, not just runs separated in time. +const ( + concurrencyRunIDA = "20260821-090000-aaaaaaaaaaaaaaaa" + concurrencyRunIDB = "20260821-090000-bbbbbbbbbbbbbbbb" +) + +// concurrencyPodTimeEarly and concurrencyPodTimeLate seed distinct, ordered +// CreationTimestamps for TestConcurrentRuns's pod-selection assertion — see +// the comment above seedImposterPod's call site for why the ordering +// matters. +var ( + concurrencyPodTimeEarly = metav1.NewTime(time.Unix(1000, 0)) + concurrencyPodTimeLate = metav1.NewTime(time.Unix(2000, 0)) +) + +// TestConcurrentRuns is the concurrency proof required by issue #2120 +// acceptance criterion #5: two overlapping snapshot-agent runs, deployed +// from separate goroutines against one shared fake clientset and one shared +// namespace, must never collide on object names, must never leak one run's +// RBAC permissions (specifically DiscoverNetwork's extra cluster rules) +// into the other, must each select only their own Pod, must each read back +// only their own snapshot bytes, and one run's Cleanup must never touch the +// other's objects. Run under -race. +// +// Fake-clientset limitations this test works around, not around-asserts: +// +// - No Job controller runs against the fake clientset, so Pods never +// appear on their own. This test creates them explicitly (seedRunPod), +// carrying the run's RunID label and a controlling ownerReference to +// that run's Job, mirroring what a real kube-controller-manager +// produces. +// - The fake ObjectTracker does not assign a UID on Create. Without one, +// jobUID() would stay the zero UID after Deploy(), and pickLivePod +// would silently skip the ownedByJob authorization check entirely +// (see wait.go: "jobUID != "" && !ownedByJob(...)" only runs the check +// when a UID is present) — degrading pod selection to label-only +// matching and defeating the point of this test. A "create","jobs" +// reactor below assigns each Job a real, name-derived UID so +// ownedByJob is exercised for real. +// - List DOES filter by LabelSelector against the fake clientset: +// k8s.io/client-go/gentype's alsoFakeLister.List applies the selector +// client-side after the ObjectTracker's own (selector-blind) List. +// findPodName below is List-based, so it exercises real selection +// logic — this test does not need to fall back to asserting a raw +// selector string for that path. Watch does NOT filter (the fake +// Watch reactor ignores ListOptions.LabelSelector entirely); this +// test does not exercise the watch-based pod-discovery path +// (findOrWatchPodName), so that gap does not apply here. +func TestConcurrentRuns(t *testing.T) { + ctx := context.Background() + clientset := fake.NewClientset() + + // Deploy's Step 0 is a permissions gate; allow everything so the test + // exercises isolation, not authorization plumbing already covered by + // permissions_test.go. + clientset.PrependReactor("create", "selfsubjectaccessreviews", func(action k8stesting.Action) (bool, runtime.Object, error) { + return true, &authv1.SelfSubjectAccessReview{ + Status: authv1.SubjectAccessReviewStatus{Allowed: true, Reason: "test permissions allowed"}, + }, nil + }) + + // See the fake-clientset limitations note above: give every created Job + // a deterministic, name-derived UID since the ObjectTracker assigns + // none on its own. + clientset.PrependReactor("create", "jobs", func(action k8stesting.Action) (bool, runtime.Object, error) { + ca, ok := action.(k8stesting.CreateActionImpl) + if !ok { + return false, nil, nil + } + job, ok := ca.GetObject().(*batchv1.Job) + if !ok { + return false, nil, nil + } + job.UID = types.UID(job.Name + "-uid") + return false, nil, nil // not handled: fall through to the default tracker create + }) + + dA := NewDeployer(clientset, Config{ + Namespace: concurrencyTestNamespace, + Image: "aicr:test", + RunID: concurrencyRunIDA, + Output: fmt.Sprintf("cm://%s/%s", concurrencyTestNamespace, nameWithRunID(staticStagingConfigMapName, concurrencyRunIDA)), + OwnsOutputConfigMap: true, + DiscoverNetwork: false, + }) + dB := NewDeployer(clientset, Config{ + Namespace: concurrencyTestNamespace, + Image: "aicr:test", + RunID: concurrencyRunIDB, + Output: fmt.Sprintf("cm://%s/%s", concurrencyTestNamespace, nameWithRunID(staticStagingConfigMapName, concurrencyRunIDB)), + OwnsOutputConfigMap: true, + DiscoverNetwork: true, + }) + + // Deploy both runs concurrently from separate goroutines — the exact + // overlap issue #2120 reported as unsafe. + var wg sync.WaitGroup + deployErrs := make([]error, 2) + wg.Add(2) + go func() { defer wg.Done(); deployErrs[0] = dA.Deploy(ctx) }() + go func() { defer wg.Done(); deployErrs[1] = dB.Deploy(ctx) }() + wg.Wait() + if deployErrs[0] != nil { + t.Fatalf("run A Deploy() error = %v", deployErrs[0]) + } + if deployErrs[1] != nil { + t.Fatalf("run B Deploy() error = %v", deployErrs[1]) + } + + // The staging ConfigMap is written by the in-pod agent, not Deploy(); + // seed it directly for both runs so the seventh object kind exists and + // GetSnapshot has something run-specific to read back. + seedStagingConfigMap(t, ctx, clientset, dA, "cm-uid-a", "snapshot-bytes-for-run-A") + seedStagingConfigMap(t, ctx, clientset, dB, "cm-uid-b", "snapshot-bytes-for-run-B") + + // --- Assertion 1: all seven object kinds exist twice, under distinct names. + t.Run("all seven kinds exist twice under distinct names", func(t *testing.T) { + assertSevenKindsExist(t, ctx, clientset, dA) + assertSevenKindsExist(t, ctx, clientset, dB) + + namesA := []string{dA.saName(), dA.roleName(), dA.clusterRoleName(), dA.jobName(), dA.stagingConfigMapName()} + namesB := []string{dB.saName(), dB.roleName(), dB.clusterRoleName(), dB.jobName(), dB.stagingConfigMapName()} + for i := range namesA { + if namesA[i] == namesB[i] { + t.Errorf("run A and run B share object name %q; runs are not name-isolated", namesA[i]) + } + } + }) + + // --- Assertion 2: DiscoverNetwork's extra ClusterRole rules do not leak across runs. + t.Run("ClusterRole rules are isolated per run's DiscoverNetwork setting", func(t *testing.T) { + crA, err := clientset.RbacV1().ClusterRoles().Get(ctx, dA.clusterRoleName(), metav1.GetOptions{}) + if err != nil { + t.Fatalf("run A ClusterRole not found: %v", err) + } + crB, err := clientset.RbacV1().ClusterRoles().Get(ctx, dB.clusterRoleName(), metav1.GetOptions{}) + if err != nil { + t.Fatalf("run B ClusterRole not found: %v", err) + } + + if clusterRoleHasRule(crA.Rules, "pods/exec", "create") { + t.Error("non-discovery run A's ClusterRole grants pods/exec create; expected none") + } + if clusterRoleHasRule(crA.Rules, "nodes", "patch") { + t.Error("non-discovery run A's ClusterRole grants nodes patch; expected none") + } + if !clusterRoleHasRule(crB.Rules, "pods/exec", "create") { + t.Error("discovery run B's ClusterRole is missing pods/exec create") + } + if !clusterRoleHasRule(crB.Rules, "nodes", "patch") { + t.Error("discovery run B's ClusterRole is missing nodes patch") + } + }) + + // Seed Pods: the fake clientset runs no Job controller, so Pods never + // appear on their own. podA/podB get an explicit, earlier + // CreationTimestamp than the imposter below: pickLivePod prefers the + // youngest candidate, so an imposter that merely passed the label + // filter (without also failing ownedByJob) would beat podA on + // recency — making the ownership check load-bearing for this + // assertion rather than incidentally masked by Pod name sort order. + podA := seedRunPodAt(t, ctx, clientset, dA, "agent-pod-a", concurrencyPodTimeEarly) + podB := seedRunPodAt(t, ctx, clientset, dB, "agent-pod-b", concurrencyPodTimeEarly) + // Imposter: carries run A's RunID label (so it passes the label + // selector dA.findPodName uses — List DOES honor selectors against the + // fake clientset, see the top-of-function note), is strictly younger + // than podA, but its controlling ownerReference points at run B's Job. + // Proves pickLivePod's ownedByJob check, not the (forgeable) RunID + // label or Pod recency, is what authorizes selection. + seedImposterPod(t, ctx, clientset, dA, dB, "agent-pod-imposter", concurrencyPodTimeLate) + + // --- Assertion 3: each run's pod selection returns only its own pod. + t.Run("pod selection returns only the run's own pod", func(t *testing.T) { + gotA, err := dA.findPodName(ctx) + if err != nil { + t.Fatalf("run A findPodName() error = %v", err) + } + if gotA != podA.Name { + t.Errorf("run A findPodName() = %q, want %q (imposter or run B pod leaked through)", gotA, podA.Name) + } + + gotB, err := dB.findPodName(ctx) + if err != nil { + t.Fatalf("run B findPodName() error = %v", err) + } + if gotB != podB.Name { + t.Errorf("run B findPodName() = %q, want %q", gotB, podB.Name) + } + }) + + // --- Assertion 4: GetSnapshot returns each run's own staging ConfigMap bytes. + t.Run("GetSnapshot returns each run's own bytes", func(t *testing.T) { + gotA, err := dA.GetSnapshot(ctx) + if err != nil { + t.Fatalf("run A GetSnapshot() error = %v", err) + } + if string(gotA) != "snapshot-bytes-for-run-A" { + t.Errorf("run A GetSnapshot() = %q, want %q", gotA, "snapshot-bytes-for-run-A") + } + + gotB, err := dB.GetSnapshot(ctx) + if err != nil { + t.Fatalf("run B GetSnapshot() error = %v", err) + } + if string(gotB) != "snapshot-bytes-for-run-B" { + t.Errorf("run B GetSnapshot() = %q, want %q", gotB, "snapshot-bytes-for-run-B") + } + }) + + // --- Assertion 5: run A's Cleanup must not touch any of run B's objects. + t.Run("run A Cleanup leaves every run B object intact", func(t *testing.T) { + if err := dA.Cleanup(ctx, CleanupOptions{Enabled: true}); err != nil { + t.Fatalf("run A Cleanup() error = %v", err) + } + + assertSevenKindsExist(t, ctx, clientset, dB) + + // Confirm run A's own Job is actually gone, so this isn't a + // Cleanup that vacuously "succeeds" without deleting anything. + if _, err := clientset.BatchV1().Jobs(concurrencyTestNamespace).Get(ctx, dA.jobName(), metav1.GetOptions{}); !apierrors.IsNotFound(err) { + t.Errorf("run A's Job should be deleted by its own Cleanup, err = %v", err) + } + }) +} + +// assertSevenKindsExist verifies each of the seven run-owned object kinds a +// Deployer creates (ServiceAccount, Role, RoleBinding, ClusterRole, +// ClusterRoleBinding, Job, and the staging ConfigMap — see kindServiceAccount +// et al. in types.go) exists under d's run-scoped names. +func assertSevenKindsExist(t *testing.T, ctx context.Context, clientset kubernetes.Interface, d *Deployer) { + t.Helper() + ns := d.config.Namespace + + if _, err := clientset.CoreV1().ServiceAccounts(ns).Get(ctx, d.saName(), metav1.GetOptions{}); err != nil { + t.Errorf("ServiceAccount %q not found: %v", d.saName(), err) + } + if _, err := clientset.RbacV1().Roles(ns).Get(ctx, d.roleName(), metav1.GetOptions{}); err != nil { + t.Errorf("Role %q not found: %v", d.roleName(), err) + } + if _, err := clientset.RbacV1().RoleBindings(ns).Get(ctx, d.roleName(), metav1.GetOptions{}); err != nil { + t.Errorf("RoleBinding %q not found: %v", d.roleName(), err) + } + if _, err := clientset.RbacV1().ClusterRoles().Get(ctx, d.clusterRoleName(), metav1.GetOptions{}); err != nil { + t.Errorf("ClusterRole %q not found: %v", d.clusterRoleName(), err) + } + if _, err := clientset.RbacV1().ClusterRoleBindings().Get(ctx, d.clusterRoleName(), metav1.GetOptions{}); err != nil { + t.Errorf("ClusterRoleBinding %q not found: %v", d.clusterRoleName(), err) + } + if _, err := clientset.BatchV1().Jobs(ns).Get(ctx, d.jobName(), metav1.GetOptions{}); err != nil { + t.Errorf("Job %q not found: %v", d.jobName(), err) + } + if _, err := clientset.CoreV1().ConfigMaps(ns).Get(ctx, d.stagingConfigMapName(), metav1.GetOptions{}); err != nil { + t.Errorf("staging ConfigMap %q not found: %v", d.stagingConfigMapName(), err) + } +} + +// seedStagingConfigMap creates d's staging ConfigMap directly, standing in +// for the in-pod agent write that Deploy() itself never performs against +// the fake clientset. +func seedStagingConfigMap(t *testing.T, ctx context.Context, clientset kubernetes.Interface, d *Deployer, uid, snapshotYAML string) { + t.Helper() + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: d.stagingConfigMapName(), + Namespace: d.config.Namespace, + UID: types.UID(uid), + Labels: d.objectLabels(), + }, + Data: map[string]string{"snapshot.yaml": snapshotYAML}, + } + if _, err := clientset.CoreV1().ConfigMaps(d.config.Namespace).Create(ctx, cm, metav1.CreateOptions{}); err != nil { + t.Fatalf("seed staging ConfigMap %q: %v", d.stagingConfigMapName(), err) + } +} + +// seedRunPodAt creates the agent Pod for d's Job, since the fake clientset +// runs no Job controller to create it automatically. The Pod carries d's +// full label set (RunID included), createdAt as its CreationTimestamp, and +// a controlling ownerReference to d's Job, mirroring what a real Job +// controller produces. +func seedRunPodAt(t *testing.T, ctx context.Context, clientset kubernetes.Interface, d *Deployer, podName string, createdAt metav1.Time) *corev1.Pod { + t.Helper() + controller := true + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: d.config.Namespace, + Labels: d.objectLabels(), + CreationTimestamp: createdAt, + OwnerReferences: []metav1.OwnerReference{ + { + Kind: kindJob, + Name: d.jobName(), + UID: d.jobUID(), + Controller: &controller, + }, + }, + }, + } + created, err := clientset.CoreV1().Pods(d.config.Namespace).Create(ctx, pod, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("seed pod %q: %v", podName, err) + } + return created +} + +// seedImposterPod creates a Pod labeled as labelOwner's own (so it passes +// labelOwner's List label selector) with createdAt as its +// CreationTimestamp, but controlled by jobOwner's Job — proving pod +// selection is authorized by the controlling ownerReference, not the RunID +// label alone (writable by anything that can update Pods in the namespace) +// or Pod recency. +func seedImposterPod(t *testing.T, ctx context.Context, clientset kubernetes.Interface, labelOwner, jobOwner *Deployer, podName string, createdAt metav1.Time) *corev1.Pod { + t.Helper() + controller := true + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: labelOwner.config.Namespace, + Labels: labelOwner.objectLabels(), + CreationTimestamp: createdAt, + OwnerReferences: []metav1.OwnerReference{ + { + Kind: kindJob, + Name: jobOwner.jobName(), + UID: jobOwner.jobUID(), + Controller: &controller, + }, + }, + }, + } + created, err := clientset.CoreV1().Pods(labelOwner.config.Namespace).Create(ctx, pod, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("seed imposter pod %q: %v", podName, err) + } + return created +} + +// clusterRoleHasRule reports whether rules contains a rule permitting verb +// on resource, mirroring the resource/verb matching TestDeployer_EnsureRBAC's +// discovery subtest uses elsewhere in this package. +func clusterRoleHasRule(rules []rbacv1.PolicyRule, resource, verb string) bool { + for _, r := range rules { + for _, res := range r.Resources { + if res != resource { + continue + } + for _, v := range r.Verbs { + if v == verb { + return true + } + } + } + } + return false +} From 5cc3d9f82bb5683368e9cb2e9a8929492f51dc08 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Sat, 22 Aug 2026 19:04:28 -0700 Subject: [PATCH 13/56] test(agent): add cleanup-ownership discriminator to concurrency proof Assertion 5 (run A's Cleanup must not touch run B's objects) passed equally against a hypothetical name-derived Cleanup, since distinct run IDs already guarantee distinct names (assertion 1). Add a same-run discriminator: run A's staging ConfigMap sits at the exact name run A's own naming formula computes, but OwnsOutputConfigMap is now false for run A so it is never recorded into run A's created-set. Only created-set-scoped cleanup leaves it standing. Signed-off-by: Alex Yuskauskas --- pkg/k8s/agent/concurrency_test.go | 44 ++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/pkg/k8s/agent/concurrency_test.go b/pkg/k8s/agent/concurrency_test.go index 58c45b2d4..50e19acd5 100644 --- a/pkg/k8s/agent/concurrency_test.go +++ b/pkg/k8s/agent/concurrency_test.go @@ -65,7 +65,9 @@ var ( // RBAC permissions (specifically DiscoverNetwork's extra cluster rules) // into the other, must each select only their own Pod, must each read back // only their own snapshot bytes, and one run's Cleanup must never touch the -// other's objects. Run under -race. +// other's objects — nor an object that merely sits at a name that run's own +// naming formula would produce but that it never actually created. Run +// under -race. // // Fake-clientset limitations this test works around, not around-asserts: // @@ -121,11 +123,20 @@ func TestConcurrentRuns(t *testing.T) { }) dA := NewDeployer(clientset, Config{ - Namespace: concurrencyTestNamespace, - Image: "aicr:test", - RunID: concurrencyRunIDA, - Output: fmt.Sprintf("cm://%s/%s", concurrencyTestNamespace, nameWithRunID(staticStagingConfigMapName, concurrencyRunIDA)), - OwnsOutputConfigMap: true, + Namespace: concurrencyTestNamespace, + Image: "aicr:test", + RunID: concurrencyRunIDA, + Output: fmt.Sprintf("cm://%s/%s", concurrencyTestNamespace, nameWithRunID(staticStagingConfigMapName, concurrencyRunIDA)), + // OwnsOutputConfigMap is false for run A on purpose (unlike run B + // below): it models an object — the staging ConfigMap seeded for + // run A below — that sits at exactly the name run A's own naming + // formula computes, but that run A never created or recorded into + // its created-set. Assertion 5's discriminator subtest needs + // exactly this shape: name-scoping alone (assertion 1) cannot + // explain that object surviving run A's Cleanup, only created-set + // scoping can. Run B keeps OwnsOutputConfigMap true so the + // "owned and recorded" path stays covered too (assertion 4). + OwnsOutputConfigMap: false, DiscoverNetwork: false, }) dB := NewDeployer(clientset, Config{ @@ -266,6 +277,27 @@ func TestConcurrentRuns(t *testing.T) { t.Errorf("run A's Job should be deleted by its own Cleanup, err = %v", err) } }) + + // --- Assertion 5 (cleanup ownership discriminator): run A's Cleanup + // must not touch an object it never created, even when that object + // sits at exactly the name run A's own naming formula computes. + // + // This is deliberately a separate subtest from the one above: the run-B + // check above would pass equally against a hypothetical name-derived + // Cleanup, because run A and run B always compute different names + // (assertion 1 already proves that) — so it cannot by itself prove + // Cleanup is scoped by the created-set rather than by recomputed names. + // This subtest supplies the missing case: dA's staging ConfigMap name + // collides with dA's own formula, but Config.OwnsOutputConfigMap was + // false for run A, so getSnapshotFromConfigMap (assertion 4) never + // recorded it into run A's created-set. See the RED/GREEN evidence in + // task-9-report.md for confirmation this genuinely discriminates + // against a name-derived Cleanup. + t.Run("run A Cleanup leaves its own unrecorded staging ConfigMap intact", func(t *testing.T) { + if _, err := clientset.CoreV1().ConfigMaps(concurrencyTestNamespace).Get(ctx, dA.stagingConfigMapName(), metav1.GetOptions{}); err != nil { + t.Errorf("run A's own staging ConfigMap %q should survive run A's Cleanup (never recorded, so not owned), err = %v", dA.stagingConfigMapName(), err) + } + }) } // assertSevenKindsExist verifies each of the seven run-owned object kinds a From e0325983d86bd9832ec56694bbc386fb16e142c1 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Sat, 22 Aug 2026 20:16:46 -0700 Subject: [PATCH 14/56] fix(agent): isolate the staging ConfigMap from the validator's and sweep it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `aicr validate` hands one run ID to both the snapshot agent and the validator Jobs in one namespace, and both built `aicr-snapshot-` (pkg/k8s/agent/names.go vs pkg/validator/validator.go). Same namespace, same name, two owners: with --no-cleanup and live capture, the agent's staging ConfigMap survives, EnsureDataConfigMaps then adopts it on AlreadyExists and overwrites its Data and labels, silently replacing the artifact --no-cleanup promised to preserve. The validator's own cleanup deletes that name with no UID precondition. Prefix the agent's staging ConfigMap `aicr-agent-snapshot-` so the two namespaces of generated names are disjoint by construction, and export StagingConfigMapName so pkg/snapshotter's agentConfigMapTarget builds the Job's cm:// URI from the same place Cleanup deletes it, rather than from a second copy of the format string. Also close the staging ConfigMap's leak: it entered the created-set only on a successful GetSnapshot, so a run that failed after the in-pod agent wrote it (Job timeout, wait error) left it behind — one object per failed run under per-run naming. Cleanup now Gets it by its run-scoped name when the run owns the output and nothing was recorded, and deletes it pinned to the UID that Get returned, so the sweep stays ownership-scoped. Alongside: - Rewrite the package godoc, which still described create-or-update RBAC, a delete-and-recreate Job, and waitForJobDeletion (deleted in ace3817c), and whose usage example set no RunID — teaching the unscoped-name pattern this branch exists to remove. Same for the two stale comments in deployer.go and the example in pkg/k8s/doc.go. - Log the run-scoped Job name via a new Deployer.JobName(): the snapshotter logged agentConfig.JobName, which is now only the user's prefix and is empty by default, so those lines printed job="". - Drop stale plan-task references in names.go, types.go, and snapshotter/agent.go, and make nameWithRunID's empty-runID branch trim a trailing dash as its godoc already claimed. - Give TestDeployer_Cleanup and TestDeployer_Cleanup_AttemptsAllDeletions a real RunID; they asserted bare configured names and passed only because Config.RunID was empty, so they no longer exercised production naming. - Point the "Job Completes but No Output" troubleshooting steps at the label selector: that ConfigMap is run-ID-suffixed and cleanup deletes it. Signed-off-by: Alex Yuskauskas --- docs/user/agent-deployment.md | 9 +- pkg/k8s/agent/deployer.go | 36 +++++++- pkg/k8s/agent/deployer_test.go | 163 +++++++++++++++++++++++++++++++-- pkg/k8s/agent/doc.go | 95 +++++++++++++------ pkg/k8s/agent/names.go | 38 ++++++-- pkg/k8s/agent/names_test.go | 76 ++++++++++++++- pkg/k8s/agent/types.go | 19 +++- pkg/k8s/agent/wait.go | 28 ++++++ pkg/k8s/doc.go | 9 +- pkg/snapshotter/agent.go | 25 +++-- pkg/snapshotter/agent_test.go | 21 ++++- 11 files changed, 447 insertions(+), 72 deletions(-) diff --git a/docs/user/agent-deployment.md b/docs/user/agent-deployment.md index b4b139e52..3da553b34 100644 --- a/docs/user/agent-deployment.md +++ b/docs/user/agent-deployment.md @@ -363,11 +363,14 @@ kubectl get nodes -o custom-columns=NAME:.metadata.name,TAINTS:.spec.taints Check ConfigMap and container logs: ```shell -# Check if ConfigMap was created -kubectl get configmap aicr-snapshot -n gpu-operator +# Check if the staging ConfigMap was created. Without an explicit +# "-o cm:///", the agent stages its result in a run-scoped +# ConfigMap named "aicr-agent-snapshot-" that cleanup deletes when +# the run ends — pass --no-cleanup to keep it around for inspection. +kubectl get configmap -n gpu-operator -l app.kubernetes.io/name=aicr # View ConfigMap contents -kubectl get configmap aicr-snapshot -n gpu-operator -o yaml +kubectl get configmap -n gpu-operator -l app.kubernetes.io/name=aicr -o yaml # View pod logs for errors kubectl logs -n gpu-operator -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent diff --git a/pkg/k8s/agent/deployer.go b/pkg/k8s/agent/deployer.go index bd41b2104..f1c109b79 100644 --- a/pkg/k8s/agent/deployer.go +++ b/pkg/k8s/agent/deployer.go @@ -53,7 +53,9 @@ func (d *Deployer) Deploy(ctx context.Context) error { return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to ensure namespace", err) } - // Step 2: Ensure RBAC resources (idempotent - reuses if already exists) + // Step 2: Create this run's RBAC. Every name carries the run ID, so + // nothing here can already exist; an AlreadyExists is reported as an + // error rather than adopted or overwritten. if err := d.ensureServiceAccount(ctx); err != nil { return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to create ServiceAccount", err) } @@ -74,7 +76,7 @@ func (d *Deployer) Deploy(ctx context.Context) error { return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to create ClusterRoleBinding", err) } - // Step 2: Ensure Job (delete existing + recreate) + // Step 3: Create this run's Job under its run-scoped name. if err := d.ensureJob(ctx); err != nil { return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to create Job", err) } @@ -82,6 +84,15 @@ func (d *Deployer) Deploy(ctx context.Context) error { return nil } +// JobName returns the run-scoped name of the Job this Deployer deploys — +// Config.JobName (or the name base) suffixed with Config.RunID. Callers that +// surface the Job to an operator (log lines, kubectl hints) must use this +// rather than Config.JobName, which is only the optional prefix and is empty +// by default. +func (d *Deployer) JobName() string { + return d.jobName() +} + // WaitForCompletion waits for the agent Job to complete successfully. // Returns error if the Job fails or times out. func (d *Deployer) WaitForCompletion(ctx context.Context, timeout time.Duration) error { @@ -118,10 +129,12 @@ func (d *Deployer) Cleanup(ctx context.Context, opts CleanupOptions) error { err error } - tasks := make([]struct { + type task struct { label string op func(context.Context) error - }, len(created)) + } + + tasks := make([]task, len(created)) for i, obj := range created { tasks[i].label = fmt.Sprintf("%s %q", obj.kind, obj.name) tasks[i].op = func(ctx context.Context) error { @@ -129,6 +142,21 @@ func (d *Deployer) Cleanup(ctx context.Context, opts CleanupOptions) error { } } + // The staging ConfigMap is written by the in-pod agent, so it only + // enters the created-set when getSnapshotFromConfigMap got far enough + // to observe its UID. A run that fails after the agent wrote it (Job + // timeout, wait error, canceled context) would otherwise leak it — and + // with run-scoped naming that is one leaked object per failed run, not + // one shared object. Sweep it here: the name is run-unique, and the + // delete is still UID-pinned against the UID observed by the Get. + if d.config.OwnsOutputConfigMap && !d.hasCreated(kindConfigMap) { + name := d.stagingConfigMapName() + tasks = append(tasks, task{ + label: fmt.Sprintf("%s %q", kindConfigMap, name), + op: d.deleteUnrecordedStagingConfigMap, + }) + } + // sync.WaitGroup (not errgroup) is intentional here: cleanup must // attempt every delete even if earlier ones fail, AND surface every // failure in the combined error message below. errgroup.WithContext diff --git a/pkg/k8s/agent/deployer_test.go b/pkg/k8s/agent/deployer_test.go index edec895ba..30c3de340 100644 --- a/pkg/k8s/agent/deployer_test.go +++ b/pkg/k8s/agent/deployer_test.go @@ -45,6 +45,12 @@ import ( const testName = "aicr" +// testRunID is a fixed, well-formed run ID (the shape runid.Generate emits: +// UTC timestamp + 16 hex bytes). Tests that exercise production naming must +// set Config.RunID — leaving it empty falls back to unscoped names no +// production caller ever builds. +const testRunID = "20260821-142233-9f3a1c0b7e2d4a55" + func TestDeployer_EnsureRBAC(t *testing.T) { clientset := fake.NewClientset() config := Config{ @@ -596,15 +602,21 @@ func TestDeployer_Cleanup(t *testing.T) { }, nil }) + // JobName / ServiceAccountName are prefixes: the objects Cleanup must + // find are named "-", so RunID is set here to exercise + // the same naming production uses. config := Config{ Namespace: "test-namespace", ServiceAccountName: testName, JobName: testName, + RunID: testRunID, Image: "ghcr.io/nvidia/aicr-validator:latest", Output: "cm://test-namespace/aicr-snapshot", } deployer := NewDeployer(clientset, config) ctx := context.Background() + scopedJob := testName + "-" + testRunID + scopedSA := testName + "-" + testRunID // Deploy first if err := deployer.Deploy(ctx); err != nil { @@ -618,14 +630,14 @@ func TestDeployer_Cleanup(t *testing.T) { // Job should still exist (cleanup disabled) _, err := clientset.BatchV1().Jobs(config.Namespace). - Get(ctx, config.JobName, metav1.GetOptions{}) + Get(ctx, scopedJob, metav1.GetOptions{}) if err != nil { t.Errorf("Job should still exist when cleanup disabled: %v", err) } // ServiceAccount should still exist _, err = clientset.CoreV1().ServiceAccounts(config.Namespace). - Get(ctx, testName, metav1.GetOptions{}) + Get(ctx, scopedSA, metav1.GetOptions{}) if err != nil { t.Errorf("ServiceAccount should still exist: %v", err) } @@ -637,7 +649,7 @@ func TestDeployer_Cleanup(t *testing.T) { // Job should be deleted _, err = clientset.BatchV1().Jobs(config.Namespace). - Get(ctx, config.JobName, metav1.GetOptions{}) + Get(ctx, scopedJob, metav1.GetOptions{}) if err == nil { t.Errorf("Job should be deleted") } @@ -656,15 +668,19 @@ func TestDeployer_Cleanup_AttemptsAllDeletions(t *testing.T) { }, nil }) + // RunID set for the same reason as TestDeployer_Cleanup: without it the + // test asserts on bare names production never creates. config := Config{ Namespace: "test-namespace", ServiceAccountName: testName, JobName: testName, + RunID: testRunID, Image: "ghcr.io/nvidia/aicr-validator:latest", Output: "cm://test-namespace/aicr-snapshot", } deployer := NewDeployer(clientset, config) ctx := context.Background() + scopedName := testName + "-" + testRunID // Deploy first if err := deployer.Deploy(ctx); err != nil { @@ -673,7 +689,7 @@ func TestDeployer_Cleanup_AttemptsAllDeletions(t *testing.T) { // Manually delete the Job to simulate it already being cleaned up // This tests that cleanup continues to delete other resources - if err := clientset.BatchV1().Jobs(config.Namespace).Delete(ctx, config.JobName, metav1.DeleteOptions{}); err != nil { + if err := clientset.BatchV1().Jobs(config.Namespace).Delete(ctx, scopedName, metav1.DeleteOptions{}); err != nil { t.Fatalf("Failed to pre-delete Job: %v", err) } @@ -685,19 +701,19 @@ func TestDeployer_Cleanup_AttemptsAllDeletions(t *testing.T) { // Verify all RBAC resources were deleted _, err := clientset.CoreV1().ServiceAccounts(config.Namespace). - Get(ctx, testName, metav1.GetOptions{}) + Get(ctx, scopedName, metav1.GetOptions{}) if err == nil { t.Error("ServiceAccount should be deleted") } _, err = clientset.RbacV1().Roles(config.Namespace). - Get(ctx, testName, metav1.GetOptions{}) + Get(ctx, scopedName, metav1.GetOptions{}) if err == nil { t.Error("Role should be deleted") } _, err = clientset.RbacV1().RoleBindings(config.Namespace). - Get(ctx, testName, metav1.GetOptions{}) + Get(ctx, scopedName, metav1.GetOptions{}) if err == nil { t.Error("RoleBinding should be deleted") } @@ -984,6 +1000,139 @@ func TestCleanupDeletesStagingConfigMapWhenOwned(t *testing.T) { } } +// TestCleanupSweepsUnrecordedStagingConfigMap covers the leak path: the +// in-pod agent wrote the staging ConfigMap, but the run failed (Job timeout, +// wait error, canceled context) before getSnapshotFromConfigMap could observe +// its UID, so nothing was recorded. With run-scoped naming that would leak one +// ConfigMap per failed run, so Cleanup Gets it by its run-scoped name and +// deletes it pinned to the UID that Get returned. +func TestCleanupSweepsUnrecordedStagingConfigMap(t *testing.T) { + ctx := context.Background() + name := StagingConfigMapName(testRunID) + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "test-namespace", + UID: types.UID("staging-uid"), + }, + Data: map[string]string{"snapshot.yaml": "data"}, + } + clientset := fake.NewClientset(cm) + + var sawUIDPrecondition bool + clientset.PrependReactor("delete", "configmaps", func(action k8stesting.Action) (bool, runtime.Object, error) { + del, ok := action.(k8stesting.DeleteActionImpl) + if !ok || del.DeleteOptions.Preconditions == nil || del.DeleteOptions.Preconditions.UID == nil { + return false, nil, nil + } + if *del.DeleteOptions.Preconditions.UID == types.UID("staging-uid") { + sawUIDPrecondition = true + } + return false, nil, nil + }) + + d := NewDeployer(clientset, Config{ + Namespace: "test-namespace", + RunID: testRunID, + Output: "cm://test-namespace/" + name, + OwnsOutputConfigMap: true, + }) + + // Deliberately no getSnapshotFromConfigMap call: this is the failed run. + if d.hasCreated(kindConfigMap) { + t.Fatal("precondition: created-set must not hold the staging ConfigMap") + } + + if err := d.Cleanup(ctx, CleanupOptions{Enabled: true}); err != nil { + t.Fatalf("Cleanup() error = %v", err) + } + + if _, err := clientset.CoreV1().ConfigMaps("test-namespace").Get(ctx, name, metav1.GetOptions{}); !apierrors.IsNotFound(err) { + t.Errorf("Cleanup leaked the staging ConfigMap, Get err = %v", err) + } + if !sawUIDPrecondition { + t.Error("staging ConfigMap delete was not pinned to the observed UID") + } +} + +// TestCleanupSkipsStagingConfigMapSweepWhenNotOwned asserts the sweep stays +// ownership-scoped: a caller-supplied cm:// Output is the caller's artifact +// (OwnsOutputConfigMap false) and must survive Cleanup even when it happens to +// carry this run's staging name. +func TestCleanupSkipsStagingConfigMapSweepWhenNotOwned(t *testing.T) { + ctx := context.Background() + name := StagingConfigMapName(testRunID) + clientset := fake.NewClientset(&corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "test-namespace", + UID: types.UID("callers-uid"), + }, + Data: map[string]string{"snapshot.yaml": "data"}, + }) + + d := NewDeployer(clientset, Config{ + Namespace: "test-namespace", + RunID: testRunID, + Output: "cm://test-namespace/" + name, + OwnsOutputConfigMap: false, + }) + + if err := d.Cleanup(ctx, CleanupOptions{Enabled: true}); err != nil { + t.Fatalf("Cleanup() error = %v", err) + } + + if _, err := clientset.CoreV1().ConfigMaps("test-namespace").Get(ctx, name, metav1.GetOptions{}); err != nil { + t.Errorf("Cleanup deleted a ConfigMap this run does not own: %v", err) + } +} + +// TestCleanupSweepNoOpWhenStagingConfigMapAbsent covers the common failure +// shape — the Job never got far enough to write anything — where the sweep's +// Get is a NotFound and Cleanup must still report success. +func TestCleanupSweepNoOpWhenStagingConfigMapAbsent(t *testing.T) { + ctx := context.Background() + clientset := fake.NewClientset() + + d := NewDeployer(clientset, Config{ + Namespace: "test-namespace", + RunID: testRunID, + Output: "cm://test-namespace/" + StagingConfigMapName(testRunID), + OwnsOutputConfigMap: true, + }) + + if err := d.Cleanup(ctx, CleanupOptions{Enabled: true}); err != nil { + t.Fatalf("Cleanup() error = %v, want nil when the staging ConfigMap was never written", err) + } +} + +// TestCleanupSweepSurfacesUnexpectedGetError fails closed: an apiserver error +// other than NotFound while looking for the staging ConfigMap means cleanup +// cannot prove the object is gone, so it must be reported rather than +// silently swallowed. +func TestCleanupSweepSurfacesUnexpectedGetError(t *testing.T) { + ctx := context.Background() + clientset := fake.NewClientset() + clientset.PrependReactor("get", "configmaps", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewInternalError(errors.New("apiserver exploded")) + }) + + d := NewDeployer(clientset, Config{ + Namespace: "test-namespace", + RunID: testRunID, + Output: "cm://test-namespace/" + StagingConfigMapName(testRunID), + OwnsOutputConfigMap: true, + }) + + err := d.Cleanup(ctx, CleanupOptions{Enabled: true}) + if err == nil { + t.Fatal("Cleanup() error = nil, want the unexpected Get error surfaced") + } + if !strings.Contains(err.Error(), StagingConfigMapName(testRunID)) { + t.Errorf("error %q does not name the staging ConfigMap", err) + } +} + func TestParseConfigMapName(t *testing.T) { tests := []struct { name string diff --git a/pkg/k8s/agent/doc.go b/pkg/k8s/agent/doc.go index 5c6e15b97..5c4c36bdc 100644 --- a/pkg/k8s/agent/doc.go +++ b/pkg/k8s/agent/doc.go @@ -19,20 +19,57 @@ The agent package deploys a Kubernetes Job that runs aicr snapshot on GPU nodes and writes output to ConfigMap storage. It handles RBAC setup, Job lifecycle management, and snapshot retrieval. -# Deployment Strategy - -RBAC resources (ServiceAccount, Role, RoleBinding, ClusterRole, ClusterRoleBinding) -are created idempotently - if they exist, they are reused. Mutable resources -(Role, RoleBinding, ClusterRole, ClusterRoleBinding) use create-or-update -semantics so stale rules from a previous run cannot persist. - -The agent Namespace is created with an "app.kubernetes.io/managed-by=aicr" -label; if the namespace pre-existed without that label, ensureNamespace -patches the label rather than silently dropping intent. - -The Job is deleted and recreated for each snapshot to ensure clean state. -Job and Pod lifecycle waits use the Kubernetes watch API (not polling) -for efficiency. +# Run Scoping + +Every deployment belongs to a single run identified by Config.RunID (generate +one with runid.Generate). The run ID is suffixed onto every object this package +creates — Job, ServiceAccount, Role, RoleBinding, ClusterRole, +ClusterRoleBinding, and the staging ConfigMap — so concurrent runs never share +an object. Config.JobName and Config.ServiceAccountName are prefixes, not exact +names; when empty they fall back to Config.NameBase (default "aicr"). See +ADR-020 (docs/design/020-snapshot-agent-run-isolation.md). + +Because a run-scoped name cannot already belong to another run, creates are +plain creates: there is no delete-and-recreate of the Job, and no +create-or-update of the RBAC objects. An AlreadyExists implies a duplicate +RunID and is returned as an error rather than adopted or overwritten. The one +courtesy check is in ensureServiceAccount, which warns when a ServiceAccount +already exists under the bare (unscoped) prefix so a caller relying on the old +adoption behavior is not left guessing; it never blocks the deploy. + +Two objects are deliberately NOT run-scoped: + + - The Namespace is ensured, never deleted: it is created if absent and + labeled "app.kubernetes.io/managed-by=aicr", patching the label onto a + pre-existing namespace rather than silently dropping intent. + - A caller-supplied "cm://namespace/name" Output is the caller's delivered + artifact. It is written on purpose and never deleted + (Config.OwnsOutputConfigMap is false for it). + +Every created object carries app.kubernetes.io/name=aicr, +app.kubernetes.io/managed-by=aicr, +app.kubernetes.io/component=snapshot-agent, and aicr.run/run-id=, on the +Job's pod template as well as the Job itself. Select agent pods across runs +with the component label; the Job name changes every run. + +Job and Pod lifecycle waits use the Kubernetes watch API (not polling) for +efficiency. Pod selection narrows by label and then authorizes the candidate +against the controlling ownerReference carrying the recorded Job UID, since pod +labels are writable by anything that can update pods in the namespace. + +# Cleanup + +The Deployer records (kind, name, UID) for each object it successfully creates. +Cleanup deletes exactly that set, passing the recorded UID as a +metav1.Preconditions so a same-named object belonging to another run is never +collected; a UID mismatch (Conflict) and a NotFound are both treated as +success. Cleanup also runs on the Deploy failure path, which is why it is +scoped to what was created rather than to configured names. + +The staging ConfigMap is written by the in-pod agent, so its UID is observed +when GetSnapshot reads it. When the run owns that ConfigMap and failed before +it could be observed, Cleanup Gets it by its run-scoped name and deletes it +pinned to the UID that Get returned. # Usage Example @@ -44,6 +81,7 @@ for efficiency. "github.com/NVIDIA/aicr/pkg/k8s/agent" "github.com/NVIDIA/aicr/pkg/k8s/client" + "github.com/NVIDIA/aicr/pkg/runid" ) func main() { @@ -55,11 +93,19 @@ for efficiency. panic(err) } + // One run ID scopes every object this deployment creates. + runID := runid.Generate() + // Configure deployer config := agent.Config{ Namespace: "gpu-operator", + RunID: runID, Image: "ghcr.io/nvidia/aicr-validator:latest", - Output: "cm://gpu-operator/aicr-snapshot", + Output: "cm://gpu-operator/" + agent.StagingConfigMapName(runID), + // Output is owned by this run, so Cleanup may delete it. Point + // Output at a ConfigMap of your own and leave this false: an + // artifact you named is never deleted here. + OwnsOutputConfigMap: true, NodeSelector: map[string]string{ "nodeGroup": "customer-gpu", }, @@ -68,12 +114,17 @@ for efficiency. // Create deployer deployer := agent.NewDeployer(clientset, config) + // Always clean up this run's objects, including on the failure path. + defer func() { + _ = deployer.Cleanup(context.Background(), agent.CleanupOptions{Enabled: true}) + }() + // Deploy RBAC and Job if err := deployer.Deploy(ctx); err != nil { panic(err) } - // Wait for completion + // Wait for completion (deployer.JobName() is the run-scoped name) if err := deployer.WaitForCompletion(ctx, 5*time.Minute); err != nil { panic(err) } @@ -87,17 +138,6 @@ for efficiency. // Use snapshot... } -# Reconciliation - -The deployer ensures idempotent operation: - - Namespace: Created with managed-by label, or patched if pre-existing - - Immutable RBAC (ServiceAccount): Created if missing, reused if exists - - Mutable RBAC (Role/RoleBinding/ClusterRole/ClusterRoleBinding): - create-or-update semantics so stale rules cannot persist - - Job: Deleted and recreated for clean state each run; deletion is - observed via watch (watch.Deleted event), not polling - - ConfigMap: Created or updated with latest snapshot - # Testing The package is designed for testability with Kubernetes fake clients: @@ -111,6 +151,7 @@ The package is designed for testability with Kubernetes fake clients: clientset := fake.NewSimpleClientset() deployer := agent.NewDeployer(clientset, agent.Config{ Namespace: "test", + RunID: "20260821-142233-9f3a1c0b7e2d4a55", Image: "test:latest", }) // Test deployment logic... diff --git a/pkg/k8s/agent/names.go b/pkg/k8s/agent/names.go index 4959c1a41..d0e0dfcf9 100644 --- a/pkg/k8s/agent/names.go +++ b/pkg/k8s/agent/names.go @@ -30,21 +30,34 @@ const staticClusterRoleName = "aicr-node-reader" // staticStagingConfigMapName is the un-scoped staging ConfigMap name used // only as the prefix input to nameWithRunID. -const staticStagingConfigMapName = "aicr-snapshot" +// +// It is deliberately NOT "aicr-snapshot": pkg/validator names its own +// snapshot data ConfigMap "aicr-snapshot-" (see EnsureDataConfigMaps +// and cleanupDataConfigMaps in pkg/validator/validator.go, and the volume in +// pkg/validator/v1/job_plan_internal.go). `aicr validate` hands ONE run ID to +// both the snapshot agent and the validator Jobs and points both at the same +// namespace, so a shared prefix would put two owners on one object: the +// validator would adopt and overwrite the agent's staging ConfigMap, and its +// (UID-unpinned) cleanup would delete it. Distinct prefixes keep the two +// namespaces of generated names disjoint by construction. +const staticStagingConfigMapName = "aicr-agent-snapshot" // nameWithRunID joins prefix and runID, truncating prefix so the result fits // within the Kubernetes name ceiling. An empty prefix yields the bare runID. -// An empty runID yields the bare (trimmed) prefix rather than appending a -// trailing "-": a trailing separator would leave a Kubernetes object name -// that fails validation (names must end in an alphanumeric character), and -// falling back to the unscoped prefix also keeps deploys working between -// this task and the task that wires Config.RunID through every caller. +// An empty runID yields the prefix with any trailing "-" trimmed rather than +// appending one: a trailing separator would leave a Kubernetes object name +// that fails validation (names must end in an alphanumeric character). +// +// Every in-tree caller supplies a RunID — pkg/snapshotter defaults and +// rejects an all-whitespace one before building an agent Config — so the +// empty-runID fallback is reachable only by an SDK caller constructing a +// Config directly, which then gets unscoped, collision-prone names. func nameWithRunID(prefix, runID string) string { if prefix == "" { return runID } if runID == "" { - return prefix + return strings.TrimRight(prefix, "-") } budget := defaults.MaxK8sNameLength - len(runID) - 1 if budget < 0 { @@ -99,8 +112,17 @@ func (d *Deployer) clusterRoleName() string { return nameWithRunID(staticClusterRoleName, d.config.RunID) } +// StagingConfigMapName returns the run-scoped name of the internal staging +// ConfigMap the agent Job writes its snapshot result to for the given run ID. +// It is exported so the one caller that builds the Job's `cm://` output URI +// (pkg/snapshotter's agentConfigMapTarget) derives that name from the same +// place Cleanup deletes it, instead of repeating the format string. +func StagingConfigMapName(runID string) string { + return nameWithRunID(staticStagingConfigMapName, runID) +} + // stagingConfigMapName returns the run-scoped name for the staging // ConfigMap the agent writes its snapshot result to. func (d *Deployer) stagingConfigMapName() string { - return nameWithRunID(staticStagingConfigMapName, d.config.RunID) + return StagingConfigMapName(d.config.RunID) } diff --git a/pkg/k8s/agent/names_test.go b/pkg/k8s/agent/names_test.go index 570a6194a..b7430ccbc 100644 --- a/pkg/k8s/agent/names_test.go +++ b/pkg/k8s/agent/names_test.go @@ -33,10 +33,12 @@ func TestNameWithRunID(t *testing.T) { {"over budget truncates", strings.Repeat("b", 40), runID, strings.Repeat("b", 30) + "-" + runID}, {"trailing dash trimmed", strings.Repeat("c", 29) + "-", runID, strings.Repeat("c", 29) + "-" + runID}, {"empty prefix", "", runID, runID}, - // A zero-value Config.RunID (before a caller wires it in) must fall - // back to the bare prefix, never a prefix with a trailing "-" — that - // would be an invalid Kubernetes object name. + // A zero-value Config.RunID (only reachable from an SDK caller + // constructing a Config directly) must fall back to the bare prefix, + // never a prefix with a trailing "-" — that would be an invalid + // Kubernetes object name. {"empty runID falls back to bare prefix", "aicr", "", "aicr"}, + {"empty runID trims the prefix's trailing dash", "aicr-", "", "aicr"}, {"empty prefix and empty runID", "", "", ""}, } for _, tt := range tests { @@ -110,7 +112,7 @@ func TestDeployerNameAccessors(t *testing.T) { name: "stagingConfigMapName is run-scoped", config: Config{RunID: runID}, get: (*Deployer).stagingConfigMapName, - want: "aicr-snapshot-" + runID, + want: "aicr-agent-snapshot-" + runID, }, } for _, tt := range tests { @@ -122,3 +124,69 @@ func TestDeployerNameAccessors(t *testing.T) { }) } } + +// TestStagingConfigMapNameDoesNotCollideWithValidator pins the reason the +// agent's staging ConfigMap is prefixed "aicr-agent-snapshot" and not +// "aicr-snapshot": pkg/validator builds its own snapshot data ConfigMap as +// "aicr-snapshot-" (EnsureDataConfigMaps and cleanupDataConfigMaps in +// pkg/validator/validator.go, plus the Job volume in +// pkg/validator/v1/job_plan_internal.go). `aicr validate` hands ONE run ID to +// both subsystems and points both at the same namespace, so equal names would +// mean two owners on one object: the validator adopts and overwrites the +// agent's staging data (silently replacing the artifact --no-cleanup was +// asked to preserve), and its UID-unpinned cleanup deletes it. +// +// The validator name is spelled out here rather than imported because it is +// built inline there; if that ever changes, this test is the tripwire. +func TestStagingConfigMapNameDoesNotCollideWithValidator(t *testing.T) { + const runID = "20260821-142233-9f3a1c0b7e2d4a55" + + validatorSnapshotCM := "aicr-snapshot-" + runID + agentStagingCM := StagingConfigMapName(runID) + + if agentStagingCM == validatorSnapshotCM { + t.Fatalf("agent staging ConfigMap name %q collides with the validator's snapshot data ConfigMap name for the same run ID", agentStagingCM) + } + // A shared prefix is the collision hazard in the other direction: the + // validator's name must not be a prefix of the agent's (or vice versa) + // once a run ID is appended by either side. + if strings.HasPrefix(agentStagingCM, "aicr-snapshot-") { + t.Errorf("agent staging ConfigMap name %q reuses the validator's %q prefix", agentStagingCM, "aicr-snapshot-") + } + if want := "aicr-agent-snapshot-" + runID; agentStagingCM != want { + t.Errorf("StagingConfigMapName(%q) = %q, want %q", runID, agentStagingCM, want) + } +} + +// TestJobNameIsRunScoped covers the exported accessor callers use when they +// surface the Job to an operator (pkg/snapshotter logs it while waiting for +// completion). Config.JobName is only the prefix and is empty by default, so +// logging that field prints an empty name. +func TestJobNameIsRunScoped(t *testing.T) { + const runID = "20260821-142233-9f3a1c0b7e2d4a55" + + d := NewDeployer(nil, Config{RunID: runID}) + if got, want := d.JobName(), "aicr-"+runID; got != want { + t.Errorf("JobName() with no configured prefix = %q, want %q", got, want) + } + + withPrefix := NewDeployer(nil, Config{JobName: "my-job", RunID: runID}) + if got, want := withPrefix.JobName(), "my-job-"+runID; got != want { + t.Errorf("JobName() with a configured prefix = %q, want %q", got, want) + } + if withPrefix.JobName() == withPrefix.config.JobName { + t.Error("JobName() returned the bare prefix; it must be run-scoped") + } +} + +// TestStagingConfigMapNameMatchesDeployerMethod guards a single source of +// truth for the staging ConfigMap's name: the exported helper pkg/snapshotter +// uses to build the Job's cm:// output URI and the name Cleanup deletes must +// be the same string for the same run. +func TestStagingConfigMapNameMatchesDeployerMethod(t *testing.T) { + const runID = "20260821-142233-9f3a1c0b7e2d4a55" + d := &Deployer{config: Config{RunID: runID}} + if got, want := d.stagingConfigMapName(), StagingConfigMapName(runID); got != want { + t.Errorf("stagingConfigMapName() = %q, StagingConfigMapName(%q) = %q; they must agree", got, runID, want) + } +} diff --git a/pkg/k8s/agent/types.go b/pkg/k8s/agent/types.go index 1cb6a7a35..1b8956dbb 100644 --- a/pkg/k8s/agent/types.go +++ b/pkg/k8s/agent/types.go @@ -183,8 +183,8 @@ func (d *Deployer) createdSnapshot() []createdObject { // jobUID returns the UID of the Job this Deployer created, or the zero UID // if Deploy has not (yet) reached the Job-create step — including when -// Deploy failed before getting there. Task 5 uses this to authorize pod -// selection against exactly this run's Job. +// Deploy failed before getting there. Pod selection (see ownedByJob in +// wait.go) authorizes candidates against exactly this UID. func (d *Deployer) jobUID() types.UID { d.mu.Lock() defer d.mu.Unlock() @@ -196,6 +196,21 @@ func (d *Deployer) jobUID() types.UID { return "" } +// hasCreated reports whether the created-set already holds an object of +// kind. Cleanup uses it to decide whether the staging ConfigMap still needs +// a name-based sweep (the run failed before getSnapshotFromConfigMap could +// observe its UID) or was already recorded. Safe for concurrent use. +func (d *Deployer) hasCreated(kind string) bool { + d.mu.Lock() + defer d.mu.Unlock() + for _, c := range d.created { + if c.kind == kind { + return true + } + } + return false +} + // CleanupOptions controls what resources to remove during cleanup. type CleanupOptions struct { Enabled bool // If true, removes Job and all RBAC resources diff --git a/pkg/k8s/agent/wait.go b/pkg/k8s/agent/wait.go index 1f3be4b70..324dfcdef 100644 --- a/pkg/k8s/agent/wait.go +++ b/pkg/k8s/agent/wait.go @@ -24,6 +24,7 @@ import ( "github.com/NVIDIA/aicr/pkg/k8s/labels" "github.com/NVIDIA/aicr/pkg/k8s/pod" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ) @@ -79,6 +80,33 @@ func (d *Deployer) deleteStagingConfigMap(ctx context.Context, name string, uid return ignoreNotFoundOrConflict(err) } +// deleteUnrecordedStagingConfigMap deletes this run's staging ConfigMap when +// the in-pod agent wrote it but the controller never observed it — the run +// failed between the agent's write and getSnapshotFromConfigMap, so there is +// no recorded UID to pin the delete to. It Gets the ConfigMap first and pins +// the delete to the UID it observes there, so the sweep is still +// ownership-scoped rather than name-only. +// +// Only called from Cleanup, and only when Config.OwnsOutputConfigMap is true. +// That flag means Output is the run-scoped staging URI pkg/snapshotter builds +// from StagingConfigMapName, so d.stagingConfigMapName() names exactly that +// object in d.config.Namespace. A caller that sets the flag while pointing +// Output elsewhere simply finds nothing here (NotFound is a no-op) — this +// deletes nothing it does not own. +func (d *Deployer) deleteUnrecordedStagingConfigMap(ctx context.Context) error { + name := d.stagingConfigMapName() + cm, err := d.clientset.CoreV1().ConfigMaps(d.config.Namespace).Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + // The Job never got far enough to write it: nothing to clean up. + return nil + } + if err != nil { + return errors.Wrap(errors.ErrCodeInternal, + fmt.Sprintf("failed to get staging ConfigMap %s/%s", d.config.Namespace, name), err) + } + return d.deleteStagingConfigMap(ctx, cm.Name, cm.UID) +} + // StreamLogs streams logs from the Job's Pod to the provided writer. // It will follow the logs until the context is canceled. // Returns when the context is canceled or an error occurs. diff --git a/pkg/k8s/doc.go b/pkg/k8s/doc.go index c321620b0..9aa8fa906 100644 --- a/pkg/k8s/doc.go +++ b/pkg/k8s/doc.go @@ -60,11 +60,16 @@ // // For agent deployment, import the agent sub-package: // -// import "github.com/NVIDIA/aicr/pkg/k8s/agent" +// import ( +// "github.com/NVIDIA/aicr/pkg/k8s/agent" +// "github.com/NVIDIA/aicr/pkg/runid" +// ) // -// // Deploy snapshot agent +// // Deploy snapshot agent. RunID scopes every object the deployment +// // creates, so concurrent runs never collide; see pkg/k8s/agent. // config := agent.Config{ // Namespace: "gpu-operator", +// RunID: runid.Generate(), // Image: "ghcr.io/nvidia/aicr-validator:latest", // } // deployer := agent.NewDeployer(clientset, config) diff --git a/pkg/snapshotter/agent.go b/pkg/snapshotter/agent.go index 1d577f3de..2237c5887 100644 --- a/pkg/snapshotter/agent.go +++ b/pkg/snapshotter/agent.go @@ -259,8 +259,11 @@ func deployAndWaitForResult(ctx context.Context, clientset k8sclient.Interface, timeout = defaults.K8sJobCompletionTimeout } + // Log the run-scoped Job name, not agentConfig.JobName: that field is + // only the user's optional prefix and is empty by default, so logging it + // prints job="". slog.Info("waiting for Job completion", - slog.String("job", agentConfig.JobName), + slog.String("job", deployer.JobName()), slog.Duration("timeout", timeout)) // Stream logs in background while waiting for Job completion. @@ -286,7 +289,7 @@ func deployAndWaitForResult(ctx context.Context, clientset k8sclient.Interface, if logCtx.Err() == nil { slog.Warn("agent log streaming skipped: pod did not become ready", slog.String("namespace", agentConfig.Namespace), - slog.String("job", agentConfig.JobName), + slog.String("job", deployer.JobName()), "error", podErr) } return @@ -518,10 +521,9 @@ func DeployAndCollect(ctx context.Context, config *AgentConfig) (*Snapshot, []by // agentConfigMapTarget below, which folds it into the internal staging // ConfigMap's name — so every resource this run creates shares one // scope. An empty RunID reaching pkg/k8s/agent would silently fall back - // to unscoped, collision-prone names (nameWithRunID's deploy-between- - // tasks safety net), so a whitespace-only value that slips past this - // simple emptiness check must fail closed rather than reach that - // fallback. + // to unscoped, collision-prone names (nameWithRunID's empty-runID + // branch), so a whitespace-only value that slips past this simple + // emptiness check must fail closed rather than reach that fallback. if config.RunID == "" { config.RunID = runid.Generate() } @@ -573,9 +575,12 @@ func DeployAndCollect(ctx context.Context, config *AgentConfig) (*Snapshot, []by // config.Namespace that the caller never names: ownsOutput is true, so // Cleanup may delete it. // -// The returned uri's namespace equals config.Namespace in the owned case — -// callers (deleteStagingConfigMap in pkg/k8s/agent) rely on that invariant -// rather than re-parsing the URI for a delete namespace. +// In the owned case the returned uri is exactly +// cm:///. Cleanup +// in pkg/k8s/agent relies on both halves of that invariant rather than +// re-parsing the URI: deleteStagingConfigMap deletes in Config.Namespace, and +// deleteUnrecordedStagingConfigMap (the sweep for a run that failed before +// the ConfigMap's UID was observed) looks it up by that same generated name. // // A cm:// Output is fully parsed here, not merely prefix-matched. The // namespace/name only has to be well-formed for the in-pod writer much later, @@ -594,7 +599,7 @@ func agentConfigMapTarget(config *AgentConfig) (uri string, ownsOutput bool, err } return config.Output, false, nil } - return fmt.Sprintf("%s%s/aicr-snapshot-%s", serializer.ConfigMapURIScheme, config.Namespace, config.RunID), true, nil + return serializer.ConfigMapURIScheme + config.Namespace + "/" + agent.StagingConfigMapName(config.RunID), true, nil } // SnapshotDelivery describes where DeliverSnapshot writes captured bytes. diff --git a/pkg/snapshotter/agent_test.go b/pkg/snapshotter/agent_test.go index f6f295fac..2010d1eca 100644 --- a/pkg/snapshotter/agent_test.go +++ b/pkg/snapshotter/agent_test.go @@ -27,6 +27,7 @@ import ( "testing" "github.com/NVIDIA/aicr/pkg/errors" + "github.com/NVIDIA/aicr/pkg/k8s/agent" "github.com/NVIDIA/aicr/pkg/serializer" corev1 "k8s.io/api/core/v1" ) @@ -560,21 +561,21 @@ func TestAgentOutputURILogic(t *testing.T) { name: "file output uses run-scoped internal ConfigMap", agentNamespace: "default", userOutput: "snapshot.yaml", - wantAgentOutputHas: "cm://default/aicr-snapshot-" + testRunID, + wantAgentOutputHas: "cm://default/aicr-agent-snapshot-" + testRunID, wantUsesUserOutput: false, }, { name: "stdout uses run-scoped internal ConfigMap", agentNamespace: "default", userOutput: "", - wantAgentOutputHas: "cm://default/aicr-snapshot-" + testRunID, + wantAgentOutputHas: "cm://default/aicr-agent-snapshot-" + testRunID, wantUsesUserOutput: false, }, { name: "dash stdout uses run-scoped internal ConfigMap", agentNamespace: "default", userOutput: "-", - wantAgentOutputHas: "cm://default/aicr-snapshot-" + testRunID, + wantAgentOutputHas: "cm://default/aicr-agent-snapshot-" + testRunID, wantUsesUserOutput: false, }, { @@ -588,7 +589,7 @@ func TestAgentOutputURILogic(t *testing.T) { name: "custom namespace uses that namespace for the run-scoped ConfigMap", agentNamespace: "custom-namespace", userOutput: "output.yaml", - wantAgentOutputHas: "cm://custom-namespace/aicr-snapshot-" + testRunID, + wantAgentOutputHas: "cm://custom-namespace/aicr-agent-snapshot-" + testRunID, wantUsesUserOutput: false, }, } @@ -636,10 +637,20 @@ func TestAgentConfigMapTargetIsRunScoped(t *testing.T) { if err != nil { t.Fatalf("agentConfigMapTarget() error = %v", err) } - want := "cm://gpu-operator/aicr-snapshot-20260821-142233-9f3a1c0b7e2d4a55" + // The staging ConfigMap is deliberately prefixed "aicr-agent-snapshot", + // NOT "aicr-snapshot": pkg/validator names its own snapshot data + // ConfigMap "aicr-snapshot-", and `aicr validate` gives both + // subsystems the same run ID in the same namespace. See + // TestStagingConfigMapNameDoesNotCollideWithValidator in pkg/k8s/agent. + want := "cm://gpu-operator/aicr-agent-snapshot-20260821-142233-9f3a1c0b7e2d4a55" if uri != want { t.Errorf("uri = %q, want %q", uri, want) } + // The URI must be built from the one exported helper the agent package + // also deletes by, not from a second copy of the format string. + if wantHelper := "cm://gpu-operator/" + agent.StagingConfigMapName(cfg.RunID); uri != wantHelper { + t.Errorf("uri = %q, want %q (agent.StagingConfigMapName)", uri, wantHelper) + } if !ownsOutput { t.Error("ownsOutput = false, want true for the internal staging ConfigMap") } From 21b986ec92e5d4358c66b4a5e05bcaad8062f06e Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Sat, 22 Aug 2026 20:17:00 -0700 Subject: [PATCH 15/56] fix(agent): do not fail Deploy on a forbidden adoption-drift Get MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ensureServiceAccount Gets the bare (unscoped) ServiceAccount name so it can warn that aicr is no longer adopting an out-of-band ServiceAccount, and returned ErrCodeInternal on any non-NotFound error. But `serviceaccounts get` is not in CheckPermissions' requiredChecks, so an identity scoped to exactly the pre-flight verb set passed the pre-flight and then failed Deploy with "failed to check for pre-existing ServiceAccount" — a permission problem surfaced as an internal error. That Get is a diagnostic courtesy and must not gate deployment: Forbidden now downgrades to a debug line and the deploy proceeds. Every other unexpected error still fails closed. Tests cover all four branches of the Get (pre-existing, absent, forbidden, unexpected error) plus the end-to-end shape: a full Deploy succeeds while every ServiceAccount read is forbidden. Signed-off-by: Alex Yuskauskas --- pkg/k8s/agent/rbac.go | 18 +++- pkg/k8s/agent/rbac_test.go | 203 +++++++++++++++++++++++++++++++++++++ 2 files changed, 219 insertions(+), 2 deletions(-) create mode 100644 pkg/k8s/agent/rbac_test.go diff --git a/pkg/k8s/agent/rbac.go b/pkg/k8s/agent/rbac.go index e9f9e3cec..257cf00d6 100644 --- a/pkg/k8s/agent/rbac.go +++ b/pkg/k8s/agent/rbac.go @@ -89,16 +89,30 @@ func (d *Deployer) ensureNamespace(ctx context.Context) error { // run-scoped ServiceAccount, so that adoption no longer happens; warn // loudly instead of leaving the caller to discover it the hard way. A // NotFound Get is the normal path and stays silent. +// +// That Get is a diagnostic courtesy and must not gate the deployment: +// `serviceaccounts get` is deliberately absent from CheckPermissions' +// requiredChecks (permissions.go), so an identity scoped to exactly the +// pre-flight verb set would otherwise pass the pre-flight and then fail +// Deploy with an ErrCodeInternal — a permission problem reported as an +// internal error. Forbidden therefore downgrades to a debug line and the +// deployment proceeds; every other unexpected error still fails closed. func (d *Deployer) ensureServiceAccount(ctx context.Context) error { name := d.saName() bareName := d.config.ServiceAccountName if bareName == "" { bareName = d.base() } - if _, err := d.clientset.CoreV1().ServiceAccounts(d.config.Namespace).Get(ctx, bareName, metav1.GetOptions{}); err == nil { + switch _, err := d.clientset.CoreV1().ServiceAccounts(d.config.Namespace).Get(ctx, bareName, metav1.GetOptions{}); { + case err == nil: slog.Warn("ServiceAccount already exists under the unscoped name; aicr is creating a run-scoped ServiceAccount instead of adopting it", "existing", bareName, "creating", name) - } else if !apierrors.IsNotFound(err) { + case apierrors.IsNotFound(err): + // Normal path: nothing to warn about. + case apierrors.IsForbidden(err): + slog.Debug("skipping adoption-drift check: not permitted to read ServiceAccounts in this namespace", + "name", bareName, "namespace", d.config.Namespace, "error", err) + default: return errors.Wrap(errors.ErrCodeInternal, "failed to check for pre-existing ServiceAccount", err) } diff --git a/pkg/k8s/agent/rbac_test.go b/pkg/k8s/agent/rbac_test.go new file mode 100644 index 000000000..fd03e2c60 --- /dev/null +++ b/pkg/k8s/agent/rbac_test.go @@ -0,0 +1,203 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package agent + +import ( + "bytes" + "context" + stderrors "errors" + "log/slog" + "strings" + "testing" + + aicrerrors "github.com/NVIDIA/aicr/pkg/errors" + authv1 "k8s.io/api/authorization/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" +) + +// captureLogs redirects the default slog logger into a buffer at debug level +// for the duration of the test and returns the buffer. +func captureLogs(t *testing.T) *bytes.Buffer { + t.Helper() + var buf bytes.Buffer + original := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))) + t.Cleanup(func() { slog.SetDefault(original) }) + return &buf +} + +// TestEnsureServiceAccount_AdoptionDriftCheck covers every branch of the +// bare-name Get that warns when a ServiceAccount already exists under the +// unscoped prefix (ADR-020's adoption-drift warning). +// +// The Forbidden branch is the load-bearing one: `serviceaccounts get` is NOT +// in CheckPermissions' requiredChecks, so an identity holding exactly the +// pre-flight verb set must still be able to deploy. That Get is a diagnostic +// courtesy and must never gate the deployment. +func TestEnsureServiceAccount_AdoptionDriftCheck(t *testing.T) { + saGR := schema.GroupResource{Group: "", Resource: "serviceaccounts"} + saGVR := corev1.SchemeGroupVersion.WithResource("serviceaccounts") + + tests := []struct { + name string + // getErr, when non-nil, is returned by every ServiceAccount Get. + getErr error + // existingBareSA pre-creates a ServiceAccount under the unscoped + // prefix name. + existingBareSA bool + wantErr bool + wantCreated bool + wantLogSubstr string + notWantLog string + }{ + { + name: "pre-existing unscoped ServiceAccount warns but still creates the run-scoped one", + existingBareSA: true, + wantCreated: true, + wantLogSubstr: "already exists under the unscoped name", + }, + { + name: "no pre-existing ServiceAccount is the silent normal path", + wantCreated: true, + notWantLog: "already exists under the unscoped name", + }, + { + name: "forbidden Get does not block the deployment", + getErr: apierrors.NewForbidden(saGR, testName, stderrors.New("no get permission")), + wantCreated: true, + wantLogSubstr: "skipping adoption-drift check", + notWantLog: "already exists under the unscoped name", + }, + { + name: "unexpected Get error fails closed", + getErr: apierrors.NewInternalError(stderrors.New("apiserver exploded")), + wantErr: true, + wantCreated: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + buf := captureLogs(t) + + clientset := fake.NewClientset() + if tt.existingBareSA { + if _, err := clientset.CoreV1().ServiceAccounts("test-ns").Create(ctx, &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: testName, + Namespace: "test-ns", + Annotations: map[string]string{"eks.amazonaws.com/role-arn": "arn:aws:iam::123456789012:role/example"}, + }, + }, metav1.CreateOptions{}); err != nil { + t.Fatalf("seeding unscoped ServiceAccount: %v", err) + } + } + if tt.getErr != nil { + clientset.PrependReactor("get", "serviceaccounts", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, tt.getErr + }) + } + + d := NewDeployer(clientset, Config{Namespace: "test-ns", RunID: testRunID}) + err := d.ensureServiceAccount(ctx) + + if (err != nil) != tt.wantErr { + t.Fatalf("ensureServiceAccount() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr && !stderrors.Is(err, aicrerrors.New(aicrerrors.ErrCodeInternal, "")) { + t.Errorf("error = %v, want ErrCodeInternal", err) + } + + // The run-scoped ServiceAccount is what the Job actually binds + // to; a suppressed diagnostic must not suppress its creation. + // Read through the tracker, not the clientset: the reactor above + // stands in for an identity that cannot read ServiceAccounts at + // all, and that must not also blind the assertions. + scoped := testName + "-" + testRunID + _, getErr := clientset.Tracker().Get(saGVR, "test-ns", scoped) + if gotCreated := getErr == nil; gotCreated != tt.wantCreated { + t.Errorf("run-scoped ServiceAccount %q created = %v (err %v), want %v", scoped, gotCreated, getErr, tt.wantCreated) + } + if tt.wantCreated && !d.hasCreated(kindServiceAccount) { + t.Error("created-set has no ServiceAccount entry; Cleanup would not delete it") + } + + if tt.existingBareSA { + // Adoption is exactly what this release stopped doing: the + // out-of-band ServiceAccount must be left untouched. + obj, bareErr := clientset.Tracker().Get(saGVR, "test-ns", testName) + if bareErr != nil { + t.Fatalf("unscoped ServiceAccount disappeared: %v", bareErr) + } + bare, ok := obj.(*corev1.ServiceAccount) + if !ok { + t.Fatalf("tracker returned %T, want *corev1.ServiceAccount", obj) + } + if bare.Annotations["eks.amazonaws.com/role-arn"] == "" { + t.Error("unscoped ServiceAccount annotations were modified") + } + } + + if tt.wantLogSubstr != "" && !strings.Contains(buf.String(), tt.wantLogSubstr) { + t.Errorf("log = %q, want it to contain %q", buf.String(), tt.wantLogSubstr) + } + if tt.notWantLog != "" && strings.Contains(buf.String(), tt.notWantLog) { + t.Errorf("log = %q, want it NOT to contain %q", buf.String(), tt.notWantLog) + } + }) + } +} + +// TestDeploy_SucceedsWhenServiceAccountGetForbidden is the end-to-end shape of +// the same bug: an identity authorized for exactly CheckPermissions' +// requiredChecks (which do not include `serviceaccounts get`) passes the +// pre-flight, so Deploy must not then fail on the adoption-drift Get. +func TestDeploy_SucceedsWhenServiceAccountGetForbidden(t *testing.T) { + ctx := context.Background() + captureLogs(t) + + clientset := fake.NewClientset() + clientset.PrependReactor("create", "selfsubjectaccessreviews", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, &authv1.SelfSubjectAccessReview{ + Status: authv1.SubjectAccessReviewStatus{Allowed: true, Reason: "pre-flight verb set granted"}, + }, nil + }) + clientset.PrependReactor("get", "serviceaccounts", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewForbidden( + schema.GroupResource{Group: "", Resource: "serviceaccounts"}, testName, + stderrors.New(`User "snapshot-runner" cannot get resource "serviceaccounts"`)) + }) + + d := NewDeployer(clientset, Config{ + Namespace: "test-ns", + Image: "aicr:test", + RunID: testRunID, + }) + + if err := d.Deploy(ctx); err != nil { + t.Fatalf("Deploy() error = %v, want nil (the adoption-drift Get must not gate deployment)", err) + } + + if _, err := clientset.BatchV1().Jobs("test-ns").Get(ctx, "aicr-"+testRunID, metav1.GetOptions{}); err != nil { + t.Errorf("Job not created: %v", err) + } +} From 6dd0e6cfae34ebf17e7c03b51fac01335c72f280 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Sat, 22 Aug 2026 20:17:09 -0700 Subject: [PATCH 16/56] docs(cli): correct why the single-run-ID invariant is untested MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The WARNING on `runID := v1.GenerateRunID()` and the matching test comment claimed the Action's two consumer sites "sit on mutually exclusive code paths". They do not: with neither --snapshot nor --no-cluster, the live-capture branch and runValidation both consume the same id in one invocation — which is precisely the case the invariant protects. What blocks a unit test is live-cluster I/O with no injectable seam in the Action. This comment is the only protection for an untested invariant, so a wrong mental model in it is worse than no comment. Signed-off-by: Alex Yuskauskas --- pkg/cli/validate.go | 14 ++++++++------ pkg/cli/validate_test.go | 9 +++++---- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/pkg/cli/validate.go b/pkg/cli/validate.go index 1ccab3c2d..4dfafa843 100644 --- a/pkg/cli/validate.go +++ b/pkg/cli/validate.go @@ -859,12 +859,14 @@ constraint (e.g. K8s version) is not met — --fail-on-error scopes to phase che // parseValidateAgentConfig call below, or for the // validationConfig{runID: ...} literal further down — silently // splits a single `aicr validate` invocation back into two - // uncorrelated runs. The command's two consumer sites - // (live-capture agent vs. validator Jobs) sit on mutually - // exclusive code paths (--no-cluster requires --snapshot, which - // skips the live-capture branch entirely), so no unit test can - // observe both consumers receiving the SAME value in one - // invocation without a live cluster; see + // uncorrelated runs. The two consumer sites are NOT mutually + // exclusive: with neither --snapshot nor --no-cluster, the + // live-capture branch below and runValidation both consume this + // id in the same invocation — which is exactly the case the + // invariant protects. What blocks a unit test is that reaching + // both requires live-cluster I/O (deployAgentForValidation + // deploys a Job; runValidation deploys validator Jobs) with no + // injectable seam here to fake it. See // TestValidateAgentConfig_ToAgentConfig_ForwardsRunID and // TestParseValidateAgentConfig_ForwardsCallerRunID in // validate_test.go for what IS covered (passthrough at each diff --git a/pkg/cli/validate_test.go b/pkg/cli/validate_test.go index a78ff169e..22402bd1c 100644 --- a/pkg/cli/validate_test.go +++ b/pkg/cli/validate_test.go @@ -513,10 +513,11 @@ func TestValidateAgentConfig_ToAgentConfig_ForwardsRunID(t *testing.T) { // single-generation invariant (one v1.GenerateRunID() call feeding BOTH // the live-capture agent and the validator Jobs). See the WARNING comment // on that call site in validate.go: no automated test in this package -// enforces single-generation, because the Action's two consumer sites sit -// on mutually exclusive code paths (--no-cluster requires --snapshot, -// which skips the live-capture branch entirely) and cannot both be -// exercised in one invocation without a live cluster. +// enforces single-generation. The two consumer sites are NOT mutually +// exclusive — with neither --snapshot nor --no-cluster, the live-capture +// branch and runValidation both consume the id in one invocation. What +// blocks the test is that exercising both requires live-cluster I/O (both +// branches deploy Jobs) with no injectable seam in the Action to fake it. func TestParseValidateAgentConfig_ForwardsCallerRunID(t *testing.T) { const wantRunID = "20260821-142233-9f3a1c0b7e2d4a55" From 44989898ee59c2fbaa98c33403b5c916eb968307 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Mon, 24 Aug 2026 10:41:58 -0700 Subject: [PATCH 17/56] test(e2e): add cluster-backed snapshot run-isolation checks Signed-off-by: Alex Yuskauskas --- tests/e2e/run.sh | 203 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 203 insertions(+) diff --git a/tests/e2e/run.sh b/tests/e2e/run.sh index d098721d2..849b42279 100755 --- a/tests/e2e/run.sh +++ b/tests/e2e/run.sh @@ -510,6 +510,208 @@ test_snapshot() { fi } +# ============================================================================= +# Snapshot Run Isolation Tests (ADR-020, issue #2120) +# ============================================================================= + +# Verifies that concurrent snapshot runs own and delete only their own +# Kubernetes resources. These checks are cluster-backed on purpose: the +# unit tests in pkg/k8s/agent run against a fake clientset, which runs no +# Job controller (so pod ownerReferences must be hand-seeded there) and does +# not enforce metav1.Preconditions on delete. Only a real apiserver exercises +# both. +# +# Scope note on UID preconditions: the delete-with-stale-UID race itself is +# not reachable from outside the CLI process — the window between an object's +# creation and the deferred Cleanup is the Job wait, and swapping a live +# object's UID mid-run revokes the running agent's own credentials. That +# mechanism is covered by TestCleanupPassesUIDPrecondition and +# TestCleanupTreatsConflictAsSuccess in pkg/k8s/agent/deployer_test.go. What +# is verified here is the ownership contract those preconditions enforce and +# that IS externally observable: cleanup deletes only objects this run +# created, never one that merely matches its labels or name shape. +test_snapshot_run_isolation() { + msg "==========================================" + msg "Testing snapshot run isolation" + msg "==========================================" + + if [ "$FAKE_GPU_ENABLED" != "true" ]; then + skip "snapshot/isolation" "Fake GPU not enabled" + return 0 + fi + + local ns="$SNAPSHOT_NAMESPACE" + local decoy="aicr-node-reader-e2edecoy" + local rc=0 + + # --- Decoy: labelled like a run-owned object, but created by nobody's run --- + # A cleanup that swept by label or by name shape would collect this. A + # cleanup scoped to its own created-set must leave it alone. + msg "--- Test: cleanup is scoped to created objects, not to labels ---" + kubectl delete clusterrole "$decoy" --ignore-not-found=true > /dev/null 2>&1 || true + kubectl create clusterrole "$decoy" --verb=get --resource=nodes > /dev/null 2>&1 || true + kubectl label clusterrole "$decoy" \ + app.kubernetes.io/name=aicr \ + app.kubernetes.io/managed-by=aicr \ + app.kubernetes.io/component=snapshot-agent \ + aicr.run/run-id=e2edecoy --overwrite > /dev/null 2>&1 || true + local decoy_uid_before + decoy_uid_before=$(kubectl get clusterrole "$decoy" -o jsonpath='{.metadata.uid}' 2>/dev/null || echo "") + + # --- Retained run: --no-cleanup, so its objects must outlive later runs --- + msg "--- Test: a retained run's resources survive concurrent runs ---" + local retained_log="${OUTPUT_DIR}/isolation-retained.log" + "${AICR_BIN}" snapshot \ + --image "${AICR_IMAGE}" \ + --namespace "${ns}" \ + --no-cleanup \ + --output "${OUTPUT_DIR}/isolation-retained.yaml" \ + --timeout 180s \ + --privileged \ + --node-selector kubernetes.io/os=linux > "$retained_log" 2>&1 || rc=$? + + if [ "$rc" -ne 0 ]; then + cat "$retained_log" + fail "snapshot/isolation/retained-run" "retained snapshot run failed" + return 1 + fi + + local retained_id + retained_id=$(kubectl get jobs -n "$ns" -l app.kubernetes.io/name=aicr \ + -o jsonpath='{.items[0].metadata.labels.aicr\.run/run-id}' 2>/dev/null || echo "") + if [ -z "$retained_id" ]; then + fail "snapshot/isolation/run-id-label" "no aicr.run/run-id label on the retained Job" + return 1 + fi + detail "retained run ID: ${retained_id}" + pass "snapshot/isolation/run-id-label" + + # Every run-owned object must carry the run ID in its name. + local missing="" + kubectl get job -n "$ns" "aicr-${retained_id}" > /dev/null 2>&1 || missing="${missing} job" + kubectl get sa -n "$ns" "aicr-${retained_id}" > /dev/null 2>&1 || missing="${missing} sa" + kubectl get role -n "$ns" "aicr-${retained_id}" > /dev/null 2>&1 || missing="${missing} role" + kubectl get rolebinding -n "$ns" "aicr-${retained_id}" > /dev/null 2>&1 || missing="${missing} rolebinding" + kubectl get cm -n "$ns" "aicr-agent-snapshot-${retained_id}" > /dev/null 2>&1 || missing="${missing} staging-cm" + kubectl get clusterrole "aicr-node-reader-${retained_id}" > /dev/null 2>&1 || missing="${missing} clusterrole" + kubectl get clusterrolebinding "aicr-node-reader-${retained_id}" > /dev/null 2>&1 || missing="${missing} clusterrolebinding" + if [ -n "$missing" ]; then + fail "snapshot/isolation/run-scoped-names" "not found under run-scoped names:${missing}" + return 1 + fi + pass "snapshot/isolation/run-scoped-names" + + # The agent's staging ConfigMap must not reuse the validator's name shape. + # aicr validate hands one run ID to both subsystems in one namespace, so a + # shared aicr-snapshot- prefix would give two owners one object. + if kubectl get cm -n "$ns" "aicr-snapshot-${retained_id}" > /dev/null 2>&1; then + fail "snapshot/isolation/staging-cm-name" "found aicr-snapshot-${retained_id}; collides with the validator's data ConfigMap" + return 1 + fi + pass "snapshot/isolation/staging-cm-name" + + # The pod must be authorized by its controlling ownerReference, not by a + # label — pod labels are writable by anything that can update pods. + msg "--- Test: pod is owned by its Job (controlling ownerReference) ---" + local pod job_uid owner_uid + pod=$(kubectl get pods -n "$ns" -l "aicr.run/run-id=${retained_id}" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "") + job_uid=$(kubectl get job -n "$ns" "aicr-${retained_id}" -o jsonpath='{.metadata.uid}' 2>/dev/null || echo "") + owner_uid=$(kubectl get pod -n "$ns" "$pod" \ + -o jsonpath='{.metadata.ownerReferences[?(@.controller==true)].uid}' 2>/dev/null || echo "") + if [ -n "$pod" ] && [ -n "$job_uid" ] && [ "$job_uid" = "$owner_uid" ]; then + detail "pod ${pod} controlled by Job UID ${job_uid}" + pass "snapshot/isolation/pod-owned-by-job" + else + fail "snapshot/isolation/pod-owned-by-job" "pod=${pod} jobUID=${job_uid} ownerUID=${owner_uid}" + return 1 + fi + + # --- Two concurrent runs, with the retained run's objects still present --- + msg "--- Test: two concurrent runs are independent ---" + local log_a="${OUTPUT_DIR}/isolation-a.log" + local log_b="${OUTPUT_DIR}/isolation-b.log" + local rc_a=0 rc_b=0 pid_a pid_b + + "${AICR_BIN}" snapshot --image "${AICR_IMAGE}" --namespace "${ns}" \ + --output "${OUTPUT_DIR}/isolation-a.yaml" --timeout 180s --privileged \ + --node-selector kubernetes.io/os=linux > "$log_a" 2>&1 & + pid_a=$! + "${AICR_BIN}" snapshot --image "${AICR_IMAGE}" --namespace "${ns}" \ + --output "${OUTPUT_DIR}/isolation-b.yaml" --timeout 180s --privileged \ + --node-selector kubernetes.io/os=linux > "$log_b" 2>&1 & + pid_b=$! + + wait "$pid_a" || rc_a=$? + wait "$pid_b" || rc_b=$? + + if [ "$rc_a" -ne 0 ] || [ "$rc_b" -ne 0 ]; then + echo "--- concurrent run A ---"; tail -30 "$log_a" + echo "--- concurrent run B ---"; tail -30 "$log_b" + fail "snapshot/isolation/concurrent-runs" "run A rc=${rc_a}, run B rc=${rc_b}" + return 1 + fi + + local id_a id_b + id_a=$(grep -o 'runID=[0-9a-f-]*' "$log_a" | head -1 | cut -d= -f2 || echo "") + id_b=$(grep -o 'runID=[0-9a-f-]*' "$log_b" | head -1 | cut -d= -f2 || echo "") + if [ -z "$id_a" ] || [ -z "$id_b" ] || [ "$id_a" = "$id_b" ]; then + fail "snapshot/isolation/concurrent-runs" "expected two distinct run IDs, got '${id_a}' and '${id_b}'" + return 1 + fi + detail "run A: ${id_a}" + detail "run B: ${id_b}" + if [ ! -s "${OUTPUT_DIR}/isolation-a.yaml" ] || [ ! -s "${OUTPUT_DIR}/isolation-b.yaml" ]; then + fail "snapshot/isolation/concurrent-runs" "a concurrent run produced an empty snapshot" + return 1 + fi + pass "snapshot/isolation/concurrent-runs" + + # The retained run's objects must be untouched by both concurrent runs. + # Before ADR-020 this is exactly what broke: a second run's ensureJob + # deleted the same-named Job and its cleanup deleted the shared RBAC. + local destroyed="" + kubectl get job -n "$ns" "aicr-${retained_id}" > /dev/null 2>&1 || destroyed="${destroyed} job" + kubectl get sa -n "$ns" "aicr-${retained_id}" > /dev/null 2>&1 || destroyed="${destroyed} sa" + kubectl get role -n "$ns" "aicr-${retained_id}" > /dev/null 2>&1 || destroyed="${destroyed} role" + kubectl get rolebinding -n "$ns" "aicr-${retained_id}" > /dev/null 2>&1 || destroyed="${destroyed} rolebinding" + kubectl get clusterrole "aicr-node-reader-${retained_id}" > /dev/null 2>&1 || destroyed="${destroyed} clusterrole" + kubectl get clusterrolebinding "aicr-node-reader-${retained_id}" > /dev/null 2>&1 || destroyed="${destroyed} clusterrolebinding" + if [ -n "$destroyed" ]; then + fail "snapshot/isolation/retained-run-survives" "concurrent runs destroyed:${destroyed}" + return 1 + fi + pass "snapshot/isolation/retained-run-survives" + + # Each concurrent run must have removed its own resources. + local leftover_jobs + leftover_jobs=$(kubectl get jobs -n "$ns" -l app.kubernetes.io/name=aicr -o name 2>/dev/null | wc -l | tr -d ' ') + if [ "$leftover_jobs" != "1" ]; then + kubectl get jobs -n "$ns" -l app.kubernetes.io/name=aicr -o name || true + fail "snapshot/isolation/self-cleanup" "expected only the retained Job to remain, found ${leftover_jobs}" + return 1 + fi + pass "snapshot/isolation/self-cleanup" + + # The decoy must have survived every run above. + local decoy_uid_after + decoy_uid_after=$(kubectl get clusterrole "$decoy" -o jsonpath='{.metadata.uid}' 2>/dev/null || echo "") + if [ -n "$decoy_uid_before" ] && [ "$decoy_uid_before" = "$decoy_uid_after" ]; then + pass "snapshot/isolation/cleanup-scoped-to-created" + else + fail "snapshot/isolation/cleanup-scoped-to-created" \ + "aicr-labelled ClusterRole it never created was deleted or replaced (before=${decoy_uid_before} after=${decoy_uid_after})" + return 1 + fi + + # Housekeeping: remove the decoy and the retained run's resources. + kubectl delete clusterrole "$decoy" --ignore-not-found=true > /dev/null 2>&1 || true + kubectl delete job "aicr-${retained_id}" -n "$ns" --ignore-not-found=true > /dev/null 2>&1 || true + kubectl delete sa,role,rolebinding "aicr-${retained_id}" -n "$ns" --ignore-not-found=true > /dev/null 2>&1 || true + kubectl delete cm "aicr-agent-snapshot-${retained_id}" -n "$ns" --ignore-not-found=true > /dev/null 2>&1 || true + kubectl delete clusterrole,clusterrolebinding "aicr-node-reader-${retained_id}" --ignore-not-found=true > /dev/null 2>&1 || true +} + # ============================================================================= # Recipe from Snapshot Tests (from e2e.md) # ============================================================================= @@ -1947,6 +2149,7 @@ main() { # Setup fake GPU environment and run snapshot tests if setup_fake_gpu; then test_snapshot + test_snapshot_run_isolation test_recipe_from_snapshot test_validate test_validate_deployment_checks From f494122cd753c302264e7708a9f2ca717d27a7f0 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Mon, 24 Aug 2026 11:04:54 -0700 Subject: [PATCH 18/56] test(e2e): drop RBAC fixture superseded by run-scoped agent RBAC Signed-off-by: Alex Yuskauskas --- tests/e2e/run.sh | 41 +++++++++-------------------------------- 1 file changed, 9 insertions(+), 32 deletions(-) diff --git a/tests/e2e/run.sh b/tests/e2e/run.sh index 849b42279..f8c07ddad 100755 --- a/tests/e2e/run.sh +++ b/tests/e2e/run.sh @@ -356,38 +356,15 @@ setup_fake_gpu() { # Create namespace for snapshot tests (if it doesn't exist) kubectl create namespace "$SNAPSHOT_NAMESPACE" --dry-run=client -o yaml | kubectl apply -f - - # Create RBAC for snapshot agent - msg "Creating RBAC for snapshot agent" - kubectl apply -f - << EOF -apiVersion: v1 -kind: ServiceAccount -metadata: - name: aicr - namespace: ${SNAPSHOT_NAMESPACE} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: aicr-e2e-reader -rules: -- apiGroups: [""] - resources: ["nodes", "pods", "configmaps"] - verbs: ["get", "list", "watch", "create", "update", "patch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: aicr-e2e-reader -subjects: -- kind: ServiceAccount - name: aicr - namespace: ${SNAPSHOT_NAMESPACE} -roleRef: - kind: ClusterRole - name: aicr-e2e-reader - apiGroup: rbac.authorization.k8s.io -EOF - pass "setup/rbac" + # No RBAC is pre-provisioned here. Per ADR-020 the agent creates its own + # run-scoped ServiceAccount, Role/RoleBinding and ClusterRole/ClusterRoleBinding + # ("aicr-" / "aicr-node-reader-") and deletes exactly those at + # the end of the run. The fixture this replaced pre-created a ServiceAccount + # named "aicr" bound to an "aicr-e2e-reader" ClusterRole granting + # nodes/pods/configmaps — the same access the agent now grants itself. Nothing + # referenced that ServiceAccount once the agent stopped using a fixed name, and + # its presence tripped the agent's adoption-drift warning on every run because + # it collided with the default name prefix. return 0 } From 49e8f15e157c37cfa327ad1ee2bb2fa08f000ed7 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Mon, 24 Aug 2026 11:23:35 -0700 Subject: [PATCH 19/56] docs: select the agent ServiceAccount by run ID when debugging Signed-off-by: Alex Yuskauskas --- docs/user/agent-deployment.md | 14 ++++++++++++-- tools/cleanup | 6 ------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/docs/user/agent-deployment.md b/docs/user/agent-deployment.md index 3da553b34..ad20b6838 100644 --- a/docs/user/agent-deployment.md +++ b/docs/user/agent-deployment.md @@ -326,9 +326,19 @@ aicr diff --baseline baseline.yaml --target current.yaml --fail-on-drift \ ### Job Fails to Start -Check RBAC permissions. The ServiceAccount name is run-scoped (`aicr-`), so look it up first: +Check RBAC permissions. The ServiceAccount name is run-scoped (`aicr-`), so look it up first. + +Select it by run ID, not by position: with concurrent snapshot runs the namespace +holds one ServiceAccount per run, and `.items[0]` would pick an arbitrary one — so +the checks below could report on a healthy run while you are debugging a failed one. +The CLI logs the run ID when it starts (`snapshot agent run: runID=...`), and every +resource the run created carries it as the `aicr.run/run-id` label: + ```shell -SA=$(kubectl get sa -n gpu-operator -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent -o jsonpath='{.items[0].metadata.name}') +# From the failing Job (or use the runID the CLI logged at start). +RUN_ID=$(kubectl get job -n gpu-operator -o jsonpath='{.metadata.labels.aicr\.run/run-id}') + +SA=$(kubectl get sa -n gpu-operator -l app.kubernetes.io/name=aicr,aicr.run/run-id=$RUN_ID -o jsonpath='{.items[0].metadata.name}') kubectl auth can-i get nodes --as=system:serviceaccount:gpu-operator:$SA kubectl auth can-i get pods --as=system:serviceaccount:gpu-operator:$SA kubectl auth can-i list controllers.slinky.slurm.net --all-namespaces \ diff --git a/tools/cleanup b/tools/cleanup index a1b33039b..034aaf874 100755 --- a/tools/cleanup +++ b/tools/cleanup @@ -398,12 +398,6 @@ kc -n gpu-operator delete job -l app.kubernetes.io/name=aicr,app.kubernetes.io/c kc -n gpu-operator delete sa -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent --ignore-not-found kc -n gpu-operator delete role -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent --ignore-not-found kc -n gpu-operator delete rolebinding -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent --ignore-not-found -# One-time removal of the pre-change unlabeled cluster RBAC: before this -# labeling was introduced, the agent's ClusterRole/ClusterRoleBinding were -# named literally "aicr-node-reader" with no labels, so no selector above can -# find them. -kc delete clusterrole aicr-node-reader --ignore-not-found -kc delete clusterrolebinding aicr-node-reader --ignore-not-found # Phase 3: Component CRDs. # Helm does NOT remove CRDs on uninstall — they must be deleted manually or a From f7ab9b8882647b85ccd4d6800e5b981d381b9bde Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Mon, 24 Aug 2026 11:45:42 -0700 Subject: [PATCH 20/56] fix(agent): gate the configmaps delete pre-flight on output ownership CheckPermissions fails closed, so an unconditional `configmaps: delete` entry made Deploy return ErrCodeUnauthorized at Step 0 for a caller who supplied their own cm:// output URI -- a run that never deletes a ConfigMap, because Cleanup's staging sweep and getSnapshotFromConfigMap's created-set record are both gated on Config.OwnsOutputConfigMap. Gate the check the same way ensureClusterRole gates its DiscoverNetwork rules. Signed-off-by: Alex Yuskauskas --- pkg/k8s/agent/consts.go | 1 + pkg/k8s/agent/permissions.go | 17 ++++++- pkg/k8s/agent/permissions_test.go | 76 +++++++++++++++++++++++++++++++ pkg/k8s/agent/rbac.go | 2 +- 4 files changed, 93 insertions(+), 3 deletions(-) diff --git a/pkg/k8s/agent/consts.go b/pkg/k8s/agent/consts.go index 137954a13..fe8ceb100 100644 --- a/pkg/k8s/agent/consts.go +++ b/pkg/k8s/agent/consts.go @@ -19,6 +19,7 @@ const ( verbCreate = "create" verbList = "list" verbGet = "get" + verbDelete = "delete" resourceCM = "configmaps" slinkyAPIGroup = "slinky.slurm.net" diff --git a/pkg/k8s/agent/permissions.go b/pkg/k8s/agent/permissions.go index 7dd955b87..4bde39fc0 100644 --- a/pkg/k8s/agent/permissions.go +++ b/pkg/k8s/agent/permissions.go @@ -60,8 +60,21 @@ func (d *Deployer) CheckPermissions(ctx context.Context) ([]permissionCheck, err {"clusterrolebindings", verbCreate, ""}, // Cleanup permissions - {"jobs", "delete", d.config.Namespace}, - {resourceCM, "delete", d.config.Namespace}, + {"jobs", verbDelete, d.config.Namespace}, + } + + // `configmaps: delete` is required only when this Deployer owns the + // output ConfigMap. Cleanup's staging-ConfigMap sweep and + // getSnapshotFromConfigMap's created-set record are both gated on + // Config.OwnsOutputConfigMap, so a caller who supplied their own + // `cm://` output URI never has a ConfigMap deleted on their behalf. + // CheckPermissions fails closed — a denied check makes Deploy return + // ErrCodeUnauthorized at Step 0 — so demanding an unconditional + // delete grant would block deployment for identities that are + // perfectly capable of the run they actually asked for. Gate it the + // same way ensureClusterRole gates its DiscoverNetwork rules. + if d.config.OwnsOutputConfigMap { + requiredChecks = append(requiredChecks, permCheck{resourceCM, verbDelete, d.config.Namespace}) } // SelfSubjectAccessReview is a read-only query; running the N required diff --git a/pkg/k8s/agent/permissions_test.go b/pkg/k8s/agent/permissions_test.go index f45ab1705..34597bd2a 100644 --- a/pkg/k8s/agent/permissions_test.go +++ b/pkg/k8s/agent/permissions_test.go @@ -154,3 +154,79 @@ func TestCheckPermission(t *testing.T) { }) } } + +// TestCheckPermissions_ConfigMapDeleteGatedOnOwnership pins the gate on the +// `configmaps: delete` verb. CheckPermissions fails closed, so an +// unconditional entry would make Deploy return ErrCodeUnauthorized at Step 0 +// for a caller who supplied their own `cm://` output URI — a run that never +// deletes a ConfigMap at all, because both Cleanup's staging sweep and +// getSnapshotFromConfigMap's created-set record are gated on +// Config.OwnsOutputConfigMap. +func TestCheckPermissions_ConfigMapDeleteGatedOnOwnership(t *testing.T) { + tests := []struct { + name string + ownsOutput bool + wantCMDeleteCheck bool + wantErr bool + }{ + { + name: "owns output ConfigMap requires configmaps delete", + ownsOutput: true, + wantCMDeleteCheck: true, + // The identity below is denied exactly `configmaps: delete`, + // so a required check makes the whole pre-flight fail. + wantErr: true, + }, + { + name: "caller-supplied output ConfigMap does not require configmaps delete", + ownsOutput: false, + wantCMDeleteCheck: false, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clientset := fake.NewClientset() + + // Deny only `configmaps: delete`; allow everything else. This + // models the least-privilege identity the gate exists for. + clientset.PrependReactor("create", "selfsubjectaccessreviews", func(action k8stesting.Action) (bool, runtime.Object, error) { + create, ok := action.(k8stesting.CreateAction) + if !ok { + t.Fatalf("action %T is not a CreateAction", action) + } + review, ok := create.GetObject().(*authv1.SelfSubjectAccessReview) + if !ok { + t.Fatalf("object %T is not a SelfSubjectAccessReview", create.GetObject()) + } + attrs := review.Spec.ResourceAttributes + allowed := attrs.Resource != resourceCM || attrs.Verb != verbDelete + return true, &authv1.SelfSubjectAccessReview{ + Status: authv1.SubjectAccessReviewStatus{Allowed: allowed, Reason: "test reason"}, + }, nil + }) + + deployer := NewDeployer(clientset, Config{ + Namespace: "gpu-operator", + RunID: testRunID, + OwnsOutputConfigMap: tt.ownsOutput, + }) + + checks, err := deployer.CheckPermissions(context.Background()) + if (err != nil) != tt.wantErr { + t.Fatalf("CheckPermissions() error = %v, wantErr %v", err, tt.wantErr) + } + + gotCMDelete := false + for _, c := range checks { + if c.Resource == resourceCM && c.Verb == verbDelete { + gotCMDelete = true + } + } + if gotCMDelete != tt.wantCMDeleteCheck { + t.Errorf("configmaps delete check present = %v, want %v", gotCMDelete, tt.wantCMDeleteCheck) + } + }) + } +} diff --git a/pkg/k8s/agent/rbac.go b/pkg/k8s/agent/rbac.go index 257cf00d6..be98d59a2 100644 --- a/pkg/k8s/agent/rbac.go +++ b/pkg/k8s/agent/rbac.go @@ -363,7 +363,7 @@ func (d *Deployer) deleteClusterRoleBinding(ctx context.Context, name string, ui // - nicclusterpolicies: l8k patches the user's NicClusterPolicy // (NicConfigurationOperator section) via server-side apply. func discoverNetworkClusterRules() []rbacv1.PolicyRule { - const verbUpdate, verbPatch, verbWatch, verbDelete = "update", "patch", "watch", "delete" + const verbUpdate, verbPatch, verbWatch = "update", "patch", "watch" return []rbacv1.PolicyRule{ { APIGroups: []string{"apiextensions.k8s.io"}, From c6e76cd9fc763b377e5af882e6b24b68655dd4f1 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Mon, 24 Aug 2026 11:45:50 -0700 Subject: [PATCH 21/56] fix(agent): reject an invalid RunID before Deploy creates anything RunID is folded into every run-owned object name, so a value that is not a DNS-1123 label -- empty, "build/42", "-build", uppercase, or long enough to push a generated name past 63 characters -- previously surfaced as an opaque apiserver "Invalid value: metadata.name" from partway through the ensure* chain, with some objects already created. Deploy now validates it up front and returns ErrCodeInvalidRequest naming the field and the value. pkg/snapshotter's existing guard stays: it fires earlier, with a message that points at the knob a CLI caller controls. Signed-off-by: Alex Yuskauskas --- pkg/k8s/agent/deployer.go | 10 +++++ pkg/k8s/agent/names.go | 44 +++++++++++++++++++-- pkg/k8s/agent/names_test.go | 79 +++++++++++++++++++++++++++++++++++++ pkg/k8s/agent/types.go | 6 +++ pkg/snapshotter/agent.go | 9 +++-- 5 files changed, 140 insertions(+), 8 deletions(-) diff --git a/pkg/k8s/agent/deployer.go b/pkg/k8s/agent/deployer.go index f1c109b79..b217aa808 100644 --- a/pkg/k8s/agent/deployer.go +++ b/pkg/k8s/agent/deployer.go @@ -31,6 +31,16 @@ import ( // Deploy deploys the agent with all required resources (RBAC + Job). // This is the main entry point that orchestrates the deployment. func (d *Deployer) Deploy(ctx context.Context) error { + // Pre-flight, ahead of any cluster call: reject a run ID that cannot + // be folded into a valid object name. Every object created below is + // named "-", so validating here is what keeps an + // invalid value from surfacing as an apiserver + // "Invalid value: metadata.name" partway through the ensure* chain, + // with some objects already created. + if err := d.validateRunID(); err != nil { + return err + } + // Step 0: Check permissions before attempting deployment _, err := d.CheckPermissions(ctx) if err != nil { diff --git a/pkg/k8s/agent/names.go b/pkg/k8s/agent/names.go index d0e0dfcf9..aa7df4ce4 100644 --- a/pkg/k8s/agent/names.go +++ b/pkg/k8s/agent/names.go @@ -15,9 +15,12 @@ package agent import ( + "fmt" "strings" "github.com/NVIDIA/aicr/pkg/defaults" + "github.com/NVIDIA/aicr/pkg/errors" + "k8s.io/apimachinery/pkg/util/validation" ) // defaultNameBase is the prefix used for generated resource names when the @@ -48,10 +51,11 @@ const staticStagingConfigMapName = "aicr-agent-snapshot" // appending one: a trailing separator would leave a Kubernetes object name // that fails validation (names must end in an alphanumeric character). // -// Every in-tree caller supplies a RunID — pkg/snapshotter defaults and -// rejects an all-whitespace one before building an agent Config — so the -// empty-runID fallback is reachable only by an SDK caller constructing a -// Config directly, which then gets unscoped, collision-prone names. +// The empty-runID fallback is not reachable through Deploy, which rejects +// an empty or malformed RunID up front (see validateRunID). It survives for +// the accessors a caller can reach without deploying — JobName, Cleanup on a +// Deployer that never ran — where returning a name that fails Kubernetes +// validation would be strictly worse than returning the bare prefix. func nameWithRunID(prefix, runID string) string { if prefix == "" { return runID @@ -73,6 +77,38 @@ func nameWithRunID(prefix, runID string) string { return prefix + "-" + runID } +// validateRunID rejects a Config.RunID that cannot be folded into a valid +// Kubernetes object name. Deploy calls it before any object is created: +// every run-owned name is "-", so a bad run ID would +// otherwise surface as an opaque apiserver "Invalid value: metadata.name" +// from deep inside the ensure* chain — after some objects already exist. +// +// The constraint is exactly DNS-1123 label: lowercase alphanumerics and +// "-", starting and ending alphanumeric, at most 63 characters. The length +// bound is what keeps the generated name inside defaults.MaxK8sNameLength: +// nameWithRunID truncates the prefix to fit, so an over-long run ID is the +// one input it cannot compensate for (its budget floors at zero and the +// bare run ID is returned). +// +// pkg/snapshotter defaults and whitespace-checks RunID before building an +// agent Config, but that guard does not cover callers who construct a +// pkg/k8s/agent Config directly, which is the public SDK surface. +func (d *Deployer) validateRunID() error { + runID := d.config.RunID + if runID == "" { + return errors.NewWithContext(errors.ErrCodeInvalidRequest, + "Config.RunID is required: every object this Deployer creates is named \"-\"; generate one with runid.Generate()", + map[string]any{"field": "Config.RunID", "value": runID}) + } + if problems := validation.IsDNS1123Label(runID); len(problems) > 0 { + return errors.NewWithContext(errors.ErrCodeInvalidRequest, + fmt.Sprintf("Config.RunID %q is not a valid Kubernetes name segment: %s", + runID, strings.Join(problems, "; ")), + map[string]any{"field": "Config.RunID", "value": runID}) + } + return nil +} + // base returns the configured name base, defaulting to "aicr" when unset. func (d *Deployer) base() string { if d.config.NameBase != "" { diff --git a/pkg/k8s/agent/names_test.go b/pkg/k8s/agent/names_test.go index b7430ccbc..579a1b557 100644 --- a/pkg/k8s/agent/names_test.go +++ b/pkg/k8s/agent/names_test.go @@ -15,8 +15,13 @@ package agent import ( + "context" + stderrors "errors" "strings" "testing" + + "github.com/NVIDIA/aicr/pkg/errors" + "k8s.io/client-go/kubernetes/fake" ) func TestNameWithRunID(t *testing.T) { @@ -190,3 +195,77 @@ func TestStagingConfigMapNameMatchesDeployerMethod(t *testing.T) { t.Errorf("stagingConfigMapName() = %q, StagingConfigMapName(%q) = %q; they must agree", got, runID, want) } } + +// TestValidateRunID covers the run IDs that are non-empty but still cannot +// be folded into a Kubernetes object name. Without this gate they reach the +// apiserver as an opaque "Invalid value: metadata.name" from partway through +// Deploy's ensure* chain, after some objects already exist. +func TestValidateRunID(t *testing.T) { + tests := []struct { + name string + runID string + wantErr bool + }{ + {"well-formed generated run ID", "20260821-142233-9f3a1c0b7e2d4a55", false}, + {"single character", "a", false}, + {"exactly at the DNS-1123 label ceiling", strings.Repeat("a", 63), false}, + {"empty", "", true}, + {"whitespace", " ", true}, + {"embedded slash", "build/42", true}, + {"leading dash", "-build", true}, + {"trailing dash", "build-", true}, + {"uppercase", "Build42", true}, + {"one over the DNS-1123 label ceiling", strings.Repeat("a", 64), true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := NewDeployer(fake.NewClientset(), Config{Namespace: "test-ns", RunID: tt.runID}) + err := d.validateRunID() + if (err != nil) != tt.wantErr { + t.Fatalf("validateRunID() error = %v, wantErr %v", err, tt.wantErr) + } + if err == nil { + return + } + if !stderrors.Is(err, errors.New(errors.ErrCodeInvalidRequest, "")) { + t.Errorf("error code = %v, want %v", err, errors.ErrCodeInvalidRequest) + } + // The message must name the field and echo the offending + // value so an operator can see what to change. + if !strings.Contains(err.Error(), "Config.RunID") { + t.Errorf("error %q does not name the offending field", err.Error()) + } + if tt.runID != "" && !strings.Contains(err.Error(), tt.runID) { + t.Errorf("error %q does not echo the offending value %q", err.Error(), tt.runID) + } + }) + } +} + +// TestDeployRejectsInvalidRunIDBeforeCreatingAnything is the end-to-end half +// of the gate: Deploy must fail before it reaches the ensure* chain, so a +// rejected run leaves no partially-created RBAC behind. +func TestDeployRejectsInvalidRunIDBeforeCreatingAnything(t *testing.T) { + ctx := context.Background() + for _, runID := range []string{"", " ", "build/42", "-build", "build-", "Build42", strings.Repeat("a", 64)} { + t.Run(runID, func(t *testing.T) { + clientset := fake.NewClientset() + d := NewDeployer(clientset, Config{Namespace: "test-ns", Image: "aicr:test", RunID: runID}) + + err := d.Deploy(ctx) + if err == nil { + t.Fatalf("Deploy() with RunID %q should fail", runID) + } + if !stderrors.Is(err, errors.New(errors.ErrCodeInvalidRequest, "")) { + t.Errorf("Deploy() error = %v, want code %v", err, errors.ErrCodeInvalidRequest) + } + + // Nothing may have been created — not even the Namespace, + // which Deploy ensures before any run-owned object. + if actions := clientset.Actions(); len(actions) != 0 { + t.Errorf("Deploy() issued %d API call(s) before rejecting an invalid RunID: %v", len(actions), actions) + } + }) + } +} diff --git a/pkg/k8s/agent/types.go b/pkg/k8s/agent/types.go index 1b8956dbb..39d6dea71 100644 --- a/pkg/k8s/agent/types.go +++ b/pkg/k8s/agent/types.go @@ -64,6 +64,12 @@ type Config struct { // RunID scopes every resource this Deployer creates to a single run, // so concurrent snapshot-agent runs never collide on a shared resource // name. Callers generate it with runid.Generate() before deploying. + // + // Required, and validated by Deploy before any object is created: it + // is folded into every run-owned name, so it must be a DNS-1123 label + // (lowercase alphanumerics and "-", starting and ending alphanumeric, + // at most 63 characters). Anything else fails with + // errors.ErrCodeInvalidRequest. RunID string // NameBase prefixes generated resource names only — it has no effect diff --git a/pkg/snapshotter/agent.go b/pkg/snapshotter/agent.go index 2237c5887..f156bbe52 100644 --- a/pkg/snapshotter/agent.go +++ b/pkg/snapshotter/agent.go @@ -520,10 +520,11 @@ func DeployAndCollect(ctx context.Context, config *AgentConfig) (*Snapshot, []by // Default RunID before any cluster access — and before // agentConfigMapTarget below, which folds it into the internal staging // ConfigMap's name — so every resource this run creates shares one - // scope. An empty RunID reaching pkg/k8s/agent would silently fall back - // to unscoped, collision-prone names (nameWithRunID's empty-runID - // branch), so a whitespace-only value that slips past this simple - // emptiness check must fail closed rather than reach that fallback. + // scope. pkg/k8s/agent's Deploy independently rejects an empty or + // malformed RunID, but only as an ErrCodeInvalidRequest naming a Config + // field a CLI user never set — so catch the whitespace-only value that + // slips past this simple emptiness check here, where the message can + // point at the knob the caller actually controls. if config.RunID == "" { config.RunID = runid.Generate() } From 7d7e8c878fcc7d836c7c7ad50a9eff0277bdd359 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Mon, 24 Aug 2026 11:45:53 -0700 Subject: [PATCH 22/56] test(agent): run-scope the names asserted by the deployer tests TestDeployer_EnsureRBAC, _EnsureJob, _EnsureJob_Unprivileged and _Deploy left RunID empty and asserted bare names no production caller can produce. Give each a realistic RunID and assert through the deployer's own name accessors. TestDeployer_Deploy_NetworkError and TestDeployer_Deploy_RuntimeClassNotFound also needed one, since Deploy now rejects an empty RunID before reaching the behavior they cover. Signed-off-by: Alex Yuskauskas --- pkg/k8s/agent/deployer_test.go | 47 +++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/pkg/k8s/agent/deployer_test.go b/pkg/k8s/agent/deployer_test.go index 30c3de340..d7f90eed0 100644 --- a/pkg/k8s/agent/deployer_test.go +++ b/pkg/k8s/agent/deployer_test.go @@ -53,10 +53,15 @@ const testRunID = "20260821-142233-9f3a1c0b7e2d4a55" func TestDeployer_EnsureRBAC(t *testing.T) { clientset := fake.NewClientset() + // ServiceAccountName and JobName are prefixes, not names: every + // run-owned object lands at "-". Assertions below go + // through the deployer's own name accessors so they track the names a + // production caller actually gets. config := Config{ Namespace: "test-namespace", ServiceAccountName: testName, JobName: testName, + RunID: testRunID, Image: "ghcr.io/nvidia/aicr-validator:latest", Output: "cm://test-namespace/aicr-snapshot", } @@ -93,12 +98,12 @@ func TestDeployer_EnsureRBAC(t *testing.T) { } sa, err := clientset.CoreV1().ServiceAccounts(config.Namespace). - Get(ctx, testName, metav1.GetOptions{}) + Get(ctx, deployer.saName(), metav1.GetOptions{}) if err != nil { t.Fatalf("ServiceAccount not found: %v", err) } - if sa.Name != testName { - t.Errorf("expected SA name %q, got %q", testName, sa.Name) + if sa.Name != deployer.saName() { + t.Errorf("expected SA name %q, got %q", deployer.saName(), sa.Name) } }) @@ -109,7 +114,7 @@ func TestDeployer_EnsureRBAC(t *testing.T) { } role, err := clientset.RbacV1().Roles(config.Namespace). - Get(ctx, testName, metav1.GetOptions{}) + Get(ctx, deployer.roleName(), metav1.GetOptions{}) if err != nil { t.Fatalf("Role not found: %v", err) } @@ -136,7 +141,7 @@ func TestDeployer_EnsureRBAC(t *testing.T) { } rb, err := clientset.RbacV1().RoleBindings(config.Namespace). - Get(ctx, testName, metav1.GetOptions{}) + Get(ctx, deployer.roleName(), metav1.GetOptions{}) if err != nil { t.Fatalf("RoleBinding not found: %v", err) } @@ -145,13 +150,13 @@ func TestDeployer_EnsureRBAC(t *testing.T) { if len(rb.Subjects) != 1 { t.Errorf("expected 1 subject, got %d", len(rb.Subjects)) } - if rb.Subjects[0].Name != testName { - t.Errorf("expected subject name 'aicr', got %q", rb.Subjects[0].Name) + if rb.Subjects[0].Name != deployer.saName() { + t.Errorf("expected subject name %q, got %q", deployer.saName(), rb.Subjects[0].Name) } // Verify roleRef - if rb.RoleRef.Name != testName { - t.Errorf("expected roleRef name 'aicr', got %q", rb.RoleRef.Name) + if rb.RoleRef.Name != deployer.roleName() { + t.Errorf("expected roleRef name %q, got %q", deployer.roleName(), rb.RoleRef.Name) } }) @@ -318,6 +323,7 @@ func TestDeployer_EnsureJob(t *testing.T) { Namespace: "test-namespace", ServiceAccountName: testName, JobName: testName, + RunID: testRunID, Image: "ghcr.io/nvidia/aicr-validator:latest", Output: "cm://test-namespace/aicr-snapshot", Privileged: true, // Test privileged mode (default for agent deployment) @@ -342,15 +348,16 @@ func TestDeployer_EnsureJob(t *testing.T) { } job, err := clientset.BatchV1().Jobs(config.Namespace). - Get(ctx, config.JobName, metav1.GetOptions{}) + Get(ctx, deployer.jobName(), metav1.GetOptions{}) if err != nil { t.Fatalf("Job not found: %v", err) } - // Verify Job spec - if job.Spec.Template.Spec.ServiceAccountName != config.ServiceAccountName { + // Verify Job spec. The pod's ServiceAccountName is the run-scoped + // SA name, not the configured prefix. + if job.Spec.Template.Spec.ServiceAccountName != deployer.saName() { t.Errorf("expected ServiceAccountName %q, got %q", - config.ServiceAccountName, job.Spec.Template.Spec.ServiceAccountName) + deployer.saName(), job.Spec.Template.Spec.ServiceAccountName) } // Verify host settings @@ -396,6 +403,7 @@ func TestDeployer_EnsureJob_Unprivileged(t *testing.T) { Namespace: "test-namespace", ServiceAccountName: testName, JobName: testName, + RunID: testRunID, Image: "ghcr.io/nvidia/aicr-validator:latest", Output: "cm://test-namespace/aicr-snapshot", Privileged: false, // Test unprivileged mode for PSS-restricted namespaces @@ -408,7 +416,7 @@ func TestDeployer_EnsureJob_Unprivileged(t *testing.T) { } job, err := clientset.BatchV1().Jobs(config.Namespace). - Get(ctx, config.JobName, metav1.GetOptions{}) + Get(ctx, deployer.jobName(), metav1.GetOptions{}) if err != nil { t.Fatalf("Job not found: %v", err) } @@ -481,6 +489,7 @@ func TestDeployer_Deploy(t *testing.T) { Namespace: "test-namespace", ServiceAccountName: testName, JobName: testName, + RunID: testRunID, Image: "ghcr.io/nvidia/aicr-validator:latest", Output: "cm://test-namespace/aicr-snapshot", } @@ -501,21 +510,21 @@ func TestDeployer_Deploy(t *testing.T) { // Verify ServiceAccount _, err = clientset.CoreV1().ServiceAccounts(config.Namespace). - Get(ctx, testName, metav1.GetOptions{}) + Get(ctx, deployer.saName(), metav1.GetOptions{}) if err != nil { t.Errorf("ServiceAccount not created: %v", err) } // Verify Role _, err = clientset.RbacV1().Roles(config.Namespace). - Get(ctx, testName, metav1.GetOptions{}) + Get(ctx, deployer.roleName(), metav1.GetOptions{}) if err != nil { t.Errorf("Role not created: %v", err) } // Verify RoleBinding _, err = clientset.RbacV1().RoleBindings(config.Namespace). - Get(ctx, testName, metav1.GetOptions{}) + Get(ctx, deployer.roleName(), metav1.GetOptions{}) if err != nil { t.Errorf("RoleBinding not created: %v", err) } @@ -536,7 +545,7 @@ func TestDeployer_Deploy(t *testing.T) { // Verify Job _, err = clientset.BatchV1().Jobs(config.Namespace). - Get(ctx, config.JobName, metav1.GetOptions{}) + Get(ctx, deployer.jobName(), metav1.GetOptions{}) if err != nil { t.Errorf("Job not created: %v", err) } @@ -1417,6 +1426,7 @@ func TestDeployer_Deploy_NetworkError(t *testing.T) { Namespace: "test-namespace", ServiceAccountName: testName, JobName: testName, + RunID: testRunID, Image: "ghcr.io/nvidia/aicr-validator:latest", Output: "cm://test-namespace/aicr-snapshot", } @@ -1517,6 +1527,7 @@ func TestDeployer_Deploy_RuntimeClassNotFound(t *testing.T) { Namespace: "test-namespace", ServiceAccountName: testName, JobName: testName, + RunID: testRunID, Image: "ghcr.io/nvidia/aicr-validator:latest", Output: "cm://test-namespace/aicr-snapshot", RuntimeClassName: "nvidia", From abd8c04d54bf1e7abe9a91e6f7d31b2a8355cfda Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Mon, 24 Aug 2026 11:46:02 -0700 Subject: [PATCH 23/56] test(agent): cover a nil Controller ownerReference in ownedByJob podWithOwner always takes the address of a bool, so ownedByJob's `ref.Controller != nil` guard -- the one standing between it and a nil dereference on an ownerReference written without the field -- was never exercised. Add a helper that omits Controller and a table case expecting false. Signed-off-by: Alex Yuskauskas --- pkg/k8s/agent/wait_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pkg/k8s/agent/wait_test.go b/pkg/k8s/agent/wait_test.go index 986841b1c..05bebff33 100644 --- a/pkg/k8s/agent/wait_test.go +++ b/pkg/k8s/agent/wait_test.go @@ -348,6 +348,21 @@ func podWithOwner(kind string, uid types.UID, controller bool) corev1.Pod { } } +// podWithNilControllerOwner returns a Pod whose sole OwnerReference omits +// the Controller field. Controller is a *bool, so an ownerReference written +// by a client that never set it leaves nil there — the exact case +// ownedByJob's `ref.Controller != nil` guard exists to survive, and one +// podWithOwner cannot produce because it always takes the address of a bool. +func podWithNilControllerOwner(kind string, uid types.UID) corev1.Pod { + return corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + OwnerReferences: []metav1.OwnerReference{ + {Kind: kind, UID: uid}, + }, + }, + } +} + // podWithForgedLabel returns a Pod with no OwnerReferences but a // batch.kubernetes.io/controller-uid label set to uid — the label any // client that can update pods in the namespace could set directly, unlike @@ -374,6 +389,9 @@ func TestOwnedByJob(t *testing.T) { {"controller job matching uid", podWithOwner(kindJob, want, true), want, true}, {"controller job wrong uid", podWithOwner(kindJob, types.UID("other"), true), want, false}, {"non-controller ref", podWithOwner(kindJob, want, false), want, false}, + // Controller is a *bool: an ownerReference written without it + // must be rejected, not dereferenced. + {"nil Controller on an otherwise matching ref", podWithNilControllerOwner(kindJob, want), want, false}, {"wrong kind", podWithOwner("ReplicaSet", want, true), want, false}, {"no owner refs", corev1.Pod{}, want, false}, {"forged label only", podWithForgedLabel(want), want, false}, From d83360ea138d041b46b4ffdfb80958be5954d474 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Mon, 24 Aug 2026 11:46:02 -0700 Subject: [PATCH 24/56] test(agent): state the cleanup discriminator instead of citing a report The comment pointed at task-9-report.md, a development artifact that is gitignored and not in the repository. Replace it with a direct statement of the property: a Cleanup recomputing its delete list from d.stagingConfigMapName() would delete this ConfigMap; one driven by the created-set leaves it standing. Signed-off-by: Alex Yuskauskas --- pkg/k8s/agent/concurrency_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/k8s/agent/concurrency_test.go b/pkg/k8s/agent/concurrency_test.go index 50e19acd5..823be6214 100644 --- a/pkg/k8s/agent/concurrency_test.go +++ b/pkg/k8s/agent/concurrency_test.go @@ -290,9 +290,10 @@ func TestConcurrentRuns(t *testing.T) { // This subtest supplies the missing case: dA's staging ConfigMap name // collides with dA's own formula, but Config.OwnsOutputConfigMap was // false for run A, so getSnapshotFromConfigMap (assertion 4) never - // recorded it into run A's created-set. See the RED/GREEN evidence in - // task-9-report.md for confirmation this genuinely discriminates - // against a name-derived Cleanup. + // recorded it into run A's created-set. So the property under test is + // precisely this: a Cleanup that recomputed its delete list from + // d.stagingConfigMapName() would delete this ConfigMap, while one + // driven by the created-set leaves it standing. t.Run("run A Cleanup leaves its own unrecorded staging ConfigMap intact", func(t *testing.T) { if _, err := clientset.CoreV1().ConfigMaps(concurrencyTestNamespace).Get(ctx, dA.stagingConfigMapName(), metav1.GetOptions{}); err != nil { t.Errorf("run A's own staging ConfigMap %q should survive run A's Cleanup (never recorded, so not owned), err = %v", dA.stagingConfigMapName(), err) From b57e4a02015d0202956d93721fe835ccf72e73a4 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Mon, 24 Aug 2026 11:46:13 -0700 Subject: [PATCH 25/56] test(e2e): scope agent Job cleanup to this script's own runs cleanup_e2e deleted every Job matching the snapshot agent's labels, which a concurrent run started by someone else against the same cluster also carries -- terminating it. Record the run ID each aicr snapshot invocation prints and delete only those Jobs. test_snapshot_run_isolation took its retained run ID from .items[0] of a label query, so an aborted or concurrent run could supply it -- and every assertion below, including the housekeeping delete, is keyed on that value. Take it from the run's own output instead, and narrow the leftover-Job count to app.kubernetes.io/component=snapshot-agent. Housekeeping (the decoy ClusterRole and the retained run's objects) moves to a wrapper that runs it whichever way the body exits; every earlier `return 1` used to skip it and leak a labelled ClusterRole and a full set of run-scoped objects into later tests and later CI runs. The body's exit status is preserved so a failed assertion still fails the suite. Signed-off-by: Alex Yuskauskas --- tests/e2e/run.sh | 120 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 104 insertions(+), 16 deletions(-) diff --git a/tests/e2e/run.sh b/tests/e2e/run.sh index f8c07ddad..4b15754c9 100755 --- a/tests/e2e/run.sh +++ b/tests/e2e/run.sh @@ -56,6 +56,29 @@ CREATED_FAKE_GPU_OPERATOR_DEPLOYMENT=false CREATED_FAKE_CLUSTER_POLICY=false CREATED_FAKE_CLUSTER_POLICY_CRD=false +# Run IDs of the snapshot-agent runs this script launched, space separated. +# cleanup_e2e deletes only these Jobs. Every agent run carries the same +# app.kubernetes.io/{name,component} labels, so a label-only sweep would also +# delete -- and thereby terminate -- a concurrent snapshot run someone else +# started against the same cluster. +E2E_AGENT_RUN_IDS="" + +# extract_run_id prints the agent run ID an `aicr snapshot` invocation logged +# ("runID=" on its "snapshot agent run" line), read from the log file +# given as $1. Prints nothing when the run failed before generating one. +extract_run_id() { + grep -o 'runID=[0-9a-f-]*' "$1" | head -1 | cut -d= -f2 || true +} + +# remember_agent_run_id records a run ID in E2E_AGENT_RUN_IDS so cleanup_e2e +# can sweep that run's Job. An empty argument is ignored: a run that never +# reported an ID created no Job for cleanup to find. +remember_agent_run_id() { + if [ -n "$1" ]; then + E2E_AGENT_RUN_IDS="${E2E_AGENT_RUN_IDS} $1" + fi +} + # Test counters TOTAL_TESTS=0 PASSED_TESTS=0 @@ -393,6 +416,7 @@ test_snapshot() { detail "Output: cm://${SNAPSHOT_NAMESPACE}/${SNAPSHOT_CM}" echo -e "${DIM} \$ aicr snapshot --image ${AICR_IMAGE} --namespace ${SNAPSHOT_NAMESPACE} -o cm://${SNAPSHOT_NAMESPACE}/${SNAPSHOT_CM}${NC}" + local snapshot_log="${OUTPUT_DIR}/snapshot-agent.log" local snapshot_output snapshot_output=$("${AICR_BIN}" snapshot \ --image "${AICR_IMAGE}" \ @@ -401,6 +425,8 @@ test_snapshot() { --timeout 120s \ --privileged \ --node-selector kubernetes.io/os=linux 2>&1) || true + printf '%s\n' "$snapshot_output" > "$snapshot_log" + remember_agent_run_id "$(extract_run_id "$snapshot_log")" if kubectl get cm "$SNAPSHOT_CM" -n "$SNAPSHOT_NAMESPACE" > /dev/null 2>&1; then pass "snapshot/agent" @@ -491,6 +517,30 @@ test_snapshot() { # Snapshot Run Isolation Tests (ADR-020, issue #2120) # ============================================================================= +# Objects snapshot_run_isolation_body creates that outlive its assertions. +# Published as globals so cleanup_snapshot_run_isolation can remove them on +# every exit path, including an early `return 1` from a failed assertion. +ISOLATION_DECOY="" +ISOLATION_RETAINED_ID="" + +# cleanup_snapshot_run_isolation removes the decoy ClusterRole and the +# retained (--no-cleanup) run's objects. Safe to call when the body bailed +# before creating either: both globals are empty then. +cleanup_snapshot_run_isolation() { + if [ -n "$ISOLATION_DECOY" ]; then + kubectl delete clusterrole "$ISOLATION_DECOY" --ignore-not-found=true > /dev/null 2>&1 || true + fi + if [ -n "$ISOLATION_RETAINED_ID" ]; then + local ns="$SNAPSHOT_NAMESPACE" + kubectl delete job "aicr-${ISOLATION_RETAINED_ID}" -n "$ns" --ignore-not-found=true > /dev/null 2>&1 || true + kubectl delete sa,role,rolebinding "aicr-${ISOLATION_RETAINED_ID}" -n "$ns" --ignore-not-found=true > /dev/null 2>&1 || true + kubectl delete cm "aicr-agent-snapshot-${ISOLATION_RETAINED_ID}" -n "$ns" --ignore-not-found=true > /dev/null 2>&1 || true + kubectl delete clusterrole,clusterrolebinding "aicr-node-reader-${ISOLATION_RETAINED_ID}" --ignore-not-found=true > /dev/null 2>&1 || true + fi + ISOLATION_DECOY="" + ISOLATION_RETAINED_ID="" +} + # Verifies that concurrent snapshot runs own and delete only their own # Kubernetes resources. These checks are cluster-backed on purpose: the # unit tests in pkg/k8s/agent run against a fake clientset, which runs no @@ -508,6 +558,18 @@ test_snapshot() { # that IS externally observable: cleanup deletes only objects this run # created, never one that merely matches its labels or name shape. test_snapshot_run_isolation() { + local rc=0 + snapshot_run_isolation_body || rc=$? + # Housekeeping must run whether the body passed or bailed early. Every + # `return 1` below used to skip it, leaving an aicr-labelled ClusterRole + # and a full set of run-scoped objects behind for later tests -- and later + # CI runs on the same cluster -- to trip over. The body's status is + # preserved so a failed assertion still fails the suite. + cleanup_snapshot_run_isolation + return "$rc" +} + +snapshot_run_isolation_body() { msg "==========================================" msg "Testing snapshot run isolation" msg "==========================================" @@ -532,6 +594,7 @@ test_snapshot_run_isolation() { app.kubernetes.io/managed-by=aicr \ app.kubernetes.io/component=snapshot-agent \ aicr.run/run-id=e2edecoy --overwrite > /dev/null 2>&1 || true + ISOLATION_DECOY="$decoy" local decoy_uid_before decoy_uid_before=$(kubectl get clusterrole "$decoy" -o jsonpath='{.metadata.uid}' 2>/dev/null || echo "") @@ -553,11 +616,29 @@ test_snapshot_run_isolation() { return 1 fi + # Take the run ID from THIS run's own output. A label query returning + # `.items[0]` would happily hand back an aborted run's Job, or a concurrent + # snapshot someone else started -- and everything below, including the + # housekeeping delete, is keyed on this value. local retained_id - retained_id=$(kubectl get jobs -n "$ns" -l app.kubernetes.io/name=aicr \ - -o jsonpath='{.items[0].metadata.labels.aicr\.run/run-id}' 2>/dev/null || echo "") + retained_id=$(extract_run_id "$retained_log") if [ -z "$retained_id" ]; then - fail "snapshot/isolation/run-id-label" "no aicr.run/run-id label on the retained Job" + cat "$retained_log" + fail "snapshot/isolation/run-id-label" "retained run printed no runID" + return 1 + fi + ISOLATION_RETAINED_ID="$retained_id" + remember_agent_run_id "$retained_id" + + # That Job must carry the run ID as a label, and the stable + # component label consumers select on across runs. + local retained_job_labels + retained_job_labels=$(kubectl get job -n "$ns" "aicr-${retained_id}" \ + -o jsonpath='{.metadata.labels.aicr\.run/run-id}/{.metadata.labels.app\.kubernetes\.io/component}' \ + 2>/dev/null || echo "") + if [ "$retained_job_labels" != "${retained_id}/snapshot-agent" ]; then + fail "snapshot/isolation/run-id-label" \ + "Job aicr-${retained_id} labels run-id/component = '${retained_job_labels}', want '${retained_id}/snapshot-agent'" return 1 fi detail "retained run ID: ${retained_id}" @@ -630,12 +711,14 @@ test_snapshot_run_isolation() { fi local id_a id_b - id_a=$(grep -o 'runID=[0-9a-f-]*' "$log_a" | head -1 | cut -d= -f2 || echo "") - id_b=$(grep -o 'runID=[0-9a-f-]*' "$log_b" | head -1 | cut -d= -f2 || echo "") + id_a=$(extract_run_id "$log_a") + id_b=$(extract_run_id "$log_b") if [ -z "$id_a" ] || [ -z "$id_b" ] || [ "$id_a" = "$id_b" ]; then fail "snapshot/isolation/concurrent-runs" "expected two distinct run IDs, got '${id_a}' and '${id_b}'" return 1 fi + remember_agent_run_id "$id_a" + remember_agent_run_id "$id_b" detail "run A: ${id_a}" detail "run B: ${id_b}" if [ ! -s "${OUTPUT_DIR}/isolation-a.yaml" ] || [ ! -s "${OUTPUT_DIR}/isolation-b.yaml" ]; then @@ -662,9 +745,10 @@ test_snapshot_run_isolation() { # Each concurrent run must have removed its own resources. local leftover_jobs - leftover_jobs=$(kubectl get jobs -n "$ns" -l app.kubernetes.io/name=aicr -o name 2>/dev/null | wc -l | tr -d ' ') + local agent_job_selector="app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent" + leftover_jobs=$(kubectl get jobs -n "$ns" -l "$agent_job_selector" -o name 2>/dev/null | wc -l | tr -d ' ') if [ "$leftover_jobs" != "1" ]; then - kubectl get jobs -n "$ns" -l app.kubernetes.io/name=aicr -o name || true + kubectl get jobs -n "$ns" -l "$agent_job_selector" -o name || true fail "snapshot/isolation/self-cleanup" "expected only the retained Job to remain, found ${leftover_jobs}" return 1 fi @@ -681,12 +765,9 @@ test_snapshot_run_isolation() { return 1 fi - # Housekeeping: remove the decoy and the retained run's resources. - kubectl delete clusterrole "$decoy" --ignore-not-found=true > /dev/null 2>&1 || true - kubectl delete job "aicr-${retained_id}" -n "$ns" --ignore-not-found=true > /dev/null 2>&1 || true - kubectl delete sa,role,rolebinding "aicr-${retained_id}" -n "$ns" --ignore-not-found=true > /dev/null 2>&1 || true - kubectl delete cm "aicr-agent-snapshot-${retained_id}" -n "$ns" --ignore-not-found=true > /dev/null 2>&1 || true - kubectl delete clusterrole,clusterrolebinding "aicr-node-reader-${retained_id}" --ignore-not-found=true > /dev/null 2>&1 || true + # Housekeeping (the decoy and the retained run's objects) is the wrapper's + # job -- see cleanup_snapshot_run_isolation -- so it also happens on the + # `return 1` paths above. } # ============================================================================= @@ -2059,9 +2140,16 @@ cleanup_e2e() { msg "==========================================" # Clean up snapshot resources. The Job name is run-scoped ("aicr-"), - # so select by the label every run-owned agent resource carries instead of - # a fixed name. - kubectl delete job -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent -n "$SNAPSHOT_NAMESPACE" --ignore-not-found=true > /dev/null 2>&1 || true + # so select by label -- but scoped to the run IDs THIS script launched. + # A sweep of every Job matching the agent's labels would also delete, and + # so terminate, a concurrent snapshot run started against the same cluster + # by someone else. + local run_id + for run_id in $E2E_AGENT_RUN_IDS; do + kubectl delete job \ + -l "app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent,aicr.run/run-id=${run_id}" \ + -n "$SNAPSHOT_NAMESPACE" --ignore-not-found=true > /dev/null 2>&1 || true + done kubectl delete cm "$SNAPSHOT_CM" -n "$SNAPSHOT_NAMESPACE" --ignore-not-found=true > /dev/null 2>&1 || true msg "Cleanup complete" From 7070e4b347155c97f4c0bc5ac4f3cede2de07968 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Mon, 24 Aug 2026 11:46:13 -0700 Subject: [PATCH 26/56] docs: add the blank line markdownlint MD031 wants before code fences Five opening fences in agent-deployment.md sat directly against the preceding prose line, three of them in Troubleshooting. Fixed all five, not only the reported three. Signed-off-by: Alex Yuskauskas --- docs/user/agent-deployment.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/user/agent-deployment.md b/docs/user/agent-deployment.md index ad20b6838..db96136ff 100644 --- a/docs/user/agent-deployment.md +++ b/docs/user/agent-deployment.md @@ -30,6 +30,7 @@ The agent is a Kubernetes Job that captures system configuration and writes outp ### ConfigMap storage Agent uses ConfigMap URI scheme (`cm://namespace/name`) to write snapshots: + ```bash aicr snapshot --namespace gpu-operator --output cm://gpu-operator/aicr-snapshot ``` @@ -40,6 +41,7 @@ namespace **must match `--namespace`** — otherwise the Job's ServiceAccount ha no permission to create the ConfigMap and the snapshot write fails. This creates: + ```yaml apiVersion: v1 kind: ConfigMap @@ -358,6 +360,7 @@ kubectl auth can-i list mariadbs.k8s.mariadb.com --all-namespaces \ ### Job Pending Check node selectors and tolerations: + ```shell # View pod events kubectl describe pod -n gpu-operator -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent @@ -372,6 +375,7 @@ kubectl get nodes -o custom-columns=NAME:.metadata.name,TAINTS:.spec.taints ### Job Completes but No Output Check ConfigMap and container logs: + ```shell # Check if the staging ConfigMap was created. Without an explicit # "-o cm:///", the agent stages its result in a run-scoped @@ -389,6 +393,7 @@ kubectl logs -n gpu-operator -l app.kubernetes.io/name=aicr,app.kubernetes.io/co ### Permission Denied Ensure RBAC is correctly deployed: + ```shell # Verify ClusterRole (run-scoped: "aicr-node-reader-") kubectl get clusterrole -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent From e9b57c9a3b621a28672d61d2e6edf085d9988db7 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Mon, 24 Aug 2026 13:00:51 -0700 Subject: [PATCH 27/56] fix(agent): record create intent so a lost response cannot orphan an object MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit recordCreated only ran after a successful Create. If the apiserver committed the Create but the response never arrived (client timeout, apiserver rollout, LB 502/504, connection reset, ctx cancel in the response window), the ensure* call returned a non-AlreadyExists error, Deploy aborted, and the deferred Cleanup never learned the object existed. With run-scoped names no later run reclaims it either, so the orphan was permanent. Each ensure* now records (kind, name) with the zero UID immediately before its Create and upserts the observed UID on success. AlreadyExists — the one response that proves the object is not ours — discards the entry again, so a duplicate RunID can never hand this run a delete of another run's object. The delete path omits metav1.Preconditions entirely for a zero-UID entry rather than passing an empty-string UID, which the apiserver would reject with a Conflict that ignoreNotFoundOrConflict swallows as success. Also in Cleanup: derive the staging-ConfigMap presence test from the created-set snapshot already taken instead of re-entering the lock via hasCreated. Two acquisitions let a concurrent recordCreated land between them, producing a snapshot that misses the ConfigMap while the second read reports it present — skipping both delete paths and leaking one run-scoped object. TestCleanupPassesUIDPrecondition documented itself as covering every delete Cleanup issues but reactored only on ("delete","serviceaccounts"), leaving five of seven dispatch arms unguarded. It now spies over "delete","*" and asserts one object of each kind carried its own recorded UID. Corrects the Config.NameBase godoc, which claimed NameBase "has no effect once either of those is set". The fallback is per name: jobName() and saName() each consult it independently. Refs: ADR-020 Signed-off-by: Alex Yuskauskas --- pkg/k8s/agent/deployer.go | 27 ++++- pkg/k8s/agent/deployer_test.go | 214 ++++++++++++++++++++++++++++++--- pkg/k8s/agent/job.go | 6 +- pkg/k8s/agent/rbac.go | 34 ++++-- pkg/k8s/agent/types.go | 90 ++++++++++++-- pkg/k8s/agent/wait.go | 2 +- 6 files changed, 333 insertions(+), 40 deletions(-) diff --git a/pkg/k8s/agent/deployer.go b/pkg/k8s/agent/deployer.go index b217aa808..7afda73ee 100644 --- a/pkg/k8s/agent/deployer.go +++ b/pkg/k8s/agent/deployer.go @@ -26,6 +26,7 @@ import ( aicrerrors "github.com/NVIDIA/aicr/pkg/errors" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ) // Deploy deploys the agent with all required resources (RBAC + Job). @@ -159,7 +160,13 @@ func (d *Deployer) Cleanup(ctx context.Context, opts CleanupOptions) error { // with run-scoped naming that is one leaked object per failed run, not // one shared object. Sweep it here: the name is run-unique, and the // delete is still UID-pinned against the UID observed by the Get. - if d.config.OwnsOutputConfigMap && !d.hasCreated(kindConfigMap) { + // + // The presence test reads the `created` snapshot taken above rather than + // re-entering the mutex (see containsKind): two separate lock + // acquisitions would let a concurrent recordCreated land between them, + // producing a snapshot without the ConfigMap and a second read reporting + // it present — skipping both delete paths and leaking the object. + if d.config.OwnsOutputConfigMap && !containsKind(created, kindConfigMap) { name := d.stagingConfigMapName() tasks = append(tasks, task{ label: fmt.Sprintf("%s %q", kindConfigMap, name), @@ -229,6 +236,24 @@ func (d *Deployer) deleteCreatedObject(ctx context.Context, obj createdObject) e } } +// uidPreconditions returns the DeleteOptions precondition pinning a delete +// to uid, or nil when uid is the zero UID. +// +// A zero UID means recordIntent entered the object before its Create and no +// Create response ever confirmed a UID (see recordIntent). Omitting the +// precondition entirely is required — metav1.Preconditions{UID: &""} is NOT +// equivalent to no precondition: the apiserver compares it against the live +// object's UID and rejects every delete with a Conflict, which +// ignoreNotFoundOrConflict then swallows as success, leaking the object this +// entry exists to reclaim. A bare-name delete is safe here precisely because +// the name carries this run's ID and no other run can produce it. +func uidPreconditions(uid types.UID) *metav1.Preconditions { + if uid == "" { + return nil + } + return &metav1.Preconditions{UID: &uid} +} + // ignoreNotFoundOrConflict returns nil when err is "not found" (already // deleted) or "conflict" (the UID precondition did not match — some other // object now holds this name; it has already been replaced and is not diff --git a/pkg/k8s/agent/deployer_test.go b/pkg/k8s/agent/deployer_test.go index d7f90eed0..defa138ff 100644 --- a/pkg/k8s/agent/deployer_test.go +++ b/pkg/k8s/agent/deployer_test.go @@ -807,40 +807,216 @@ func TestCleanupDeletesOnlyWhatItCreated(t *testing.T) { } } +// observedDelete is one delete action captured by spyOnDeletes. +type observedDelete struct { + resource string + name string + uid *types.UID // nil when the delete carried no UID precondition +} + +// spyOnDeletes installs a reactor over EVERY resource that records each +// outgoing delete's resource, name, and Preconditions.UID, then falls through +// to the default tracker delete. Reactors run under the fake Clientset's own +// lock, but Cleanup fans its deletes out concurrently, so the slice is +// mutex-guarded regardless; the returned accessor takes the same lock. +func spyOnDeletes(client *fake.Clientset) func() []observedDelete { + var mu sync.Mutex + var observed []observedDelete + client.PrependReactor("delete", "*", func(action k8stesting.Action) (bool, runtime.Object, error) { + da, ok := action.(k8stesting.DeleteActionImpl) + if !ok { + return false, nil, nil + } + rec := observedDelete{resource: da.GetResource().Resource, name: da.GetName()} + if da.DeleteOptions.Preconditions != nil && da.DeleteOptions.Preconditions.UID != nil { + uid := *da.DeleteOptions.Preconditions.UID + rec.uid = &uid + } + mu.Lock() + observed = append(observed, rec) + mu.Unlock() + return false, nil, nil // not handled: fall through to the default tracker delete + }) + return func() []observedDelete { + mu.Lock() + defer mu.Unlock() + out := make([]observedDelete, len(observed)) + copy(out, observed) + return out + } +} + // TestCleanupPassesUIDPrecondition verifies every delete Cleanup issues -// carries Preconditions.UID set to the UID recorded at create time. The -// fake clientset's ObjectTracker neither assigns UIDs on Create nor +// carries Preconditions.UID set to the UID recorded at create time — for all +// seven kinds deleteCreatedObject dispatches on, not just one. A reactor +// scoped to a single resource would leave the other six dispatch arms free to +// drop the precondition unnoticed, so the spy here is installed over "*". +// +// The fake clientset's ObjectTracker neither assigns UIDs on Create nor // enforces Preconditions on Delete (it ignores DeleteOptions entirely), so -// this records a known UID directly via recordCreated and spies on the -// outgoing delete action rather than relying on tracker behavior. +// this records known UIDs directly via recordCreated and inspects the +// outgoing delete actions rather than relying on tracker behavior. func TestCleanupPassesUIDPrecondition(t *testing.T) { ctx := context.Background() client := fake.NewSimpleClientset() + deletes := spyOnDeletes(client) + + // One object per kind, each with a distinct UID so a dispatch arm that + // passed some OTHER entry's UID would be caught as well as one that + // passed none. + created := []struct { + kind string + name string + resource string + uid types.UID + }{ + {kindServiceAccount, "aicr-sa", "serviceaccounts", "sa-uid-123"}, + {kindRole, "aicr-role", "roles", "role-uid-123"}, + {kindRoleBinding, "aicr-rb", "rolebindings", "rb-uid-123"}, + {kindClusterRole, "aicr-cr", "clusterroles", "cr-uid-123"}, + {kindClusterRoleBinding, "aicr-crb", "clusterrolebindings", "crb-uid-123"}, + {kindJob, "aicr-job", "jobs", "job-uid-123"}, + {kindConfigMap, "aicr-agent-snapshot", "configmaps", "cm-uid-123"}, + } - const wantUID = types.UID("sa-uid-123") - var sawUID types.UID - var sawPreconditions bool - client.PrependReactor("delete", "serviceaccounts", func(action k8stesting.Action) (bool, runtime.Object, error) { - da, ok := action.(k8stesting.DeleteActionImpl) - if ok && da.DeleteOptions.Preconditions != nil && da.DeleteOptions.Preconditions.UID != nil { - sawPreconditions = true - sawUID = *da.DeleteOptions.Preconditions.UID + d := NewDeployer(client, Config{Namespace: "test-ns"}) + for _, c := range created { + d.recordCreated(c.kind, c.name, c.uid) + } + + if err := d.Cleanup(ctx, CleanupOptions{Enabled: true}); err != nil { + t.Fatalf("Cleanup() error = %v", err) + } + + observed := deletes() + if len(observed) != len(created) { + t.Fatalf("Cleanup issued %d deletes, want %d: %+v", len(observed), len(created), observed) + } + for _, c := range created { + t.Run(c.kind, func(t *testing.T) { + idx := slices.IndexFunc(observed, func(o observedDelete) bool { + return o.resource == c.resource && o.name == c.name + }) + if idx < 0 { + t.Fatalf("Cleanup issued no delete for %s %q (resource %q); observed: %+v", + c.kind, c.name, c.resource, observed) + } + got := observed[idx] + if got.uid == nil { + t.Fatalf("%s delete did not carry Preconditions.UID", c.kind) + } + if *got.uid != c.uid { + t.Errorf("%s delete Preconditions.UID = %q, want %q", c.kind, *got.uid, c.uid) + } + }) + } +} + +// TestCleanupDeletesUnconfirmedCreateByBareName covers the lost-Create-response +// path: recordIntent enters an object BEFORE its Create, so an entry can reach +// Cleanup with the zero UID. Such a delete must omit Preconditions entirely. +// +// Passing &"" instead would be worse than useless: the apiserver would compare +// the empty UID against the live object's real one, reject every attempt with +// a Conflict, and ignoreNotFoundOrConflict would swallow that as success — +// leaking the very object this entry exists to reclaim. A bare-name delete is +// safe here because the name carries this run's ID. +func TestCleanupDeletesUnconfirmedCreateByBareName(t *testing.T) { + ctx := context.Background() + client := fake.NewSimpleClientset() + deletes := spyOnDeletes(client) + + d := NewDeployer(client, Config{Namespace: "test-ns"}) + d.recordIntent(kindServiceAccount, "aicr-20260821-142233-9f3a1c0b7e2d4a55") + + if err := d.Cleanup(ctx, CleanupOptions{Enabled: true}); err != nil { + t.Fatalf("Cleanup() error = %v", err) + } + + observed := deletes() + if len(observed) != 1 { + t.Fatalf("Cleanup issued %d deletes, want 1: %+v", len(observed), observed) + } + if observed[0].name != "aicr-20260821-142233-9f3a1c0b7e2d4a55" { + t.Errorf("delete name = %q, want the recorded run-scoped name", observed[0].name) + } + if observed[0].uid != nil { + t.Errorf("delete carried Preconditions.UID = %q; a zero-UID entry must delete by bare name", + *observed[0].uid) + } +} + +// TestEnsureRecordsIntentBeforeCreate is the reason recordIntent exists: an +// apiserver that commits a Create but never delivers the response (client +// timeout, apiserver rollout, LB 502/504, connection reset) must not leave an +// object nothing will ever delete. The run-scoped name means no later run +// reclaims it, so the orphan would be permanent. +// +// The reactor below reproduces exactly that: the object is written into the +// tracker and THEN an error is returned, so ensureServiceAccount fails while +// the ServiceAccount exists. Cleanup must still delete it. +func TestEnsureRecordsIntentBeforeCreate(t *testing.T) { + ctx := context.Background() + const ns = "test-ns" + client := fake.NewSimpleClientset() + + d := NewDeployer(client, Config{Namespace: ns, RunID: testRunID}) + saName := d.saName() + + client.PrependReactor("create", "serviceaccounts", func(action k8stesting.Action) (bool, runtime.Object, error) { + ca, ok := action.(k8stesting.CreateActionImpl) + if !ok { + return false, nil, nil } - return false, nil, nil // not handled: fall through to the default tracker delete + // Commit the object the way a real apiserver would... + if err := client.Tracker().Create(ca.GetResource(), ca.GetObject(), ns); err != nil { + return true, nil, err + } + // ...then lose the response on the way back to the client. + return true, nil, syscall.ECONNRESET }) - d := NewDeployer(client, Config{Namespace: "test-ns"}) - d.recordCreated(kindServiceAccount, "aicr-sa", wantUID) + if err := d.ensureServiceAccount(ctx); err == nil { + t.Fatal("ensureServiceAccount() = nil error, want the simulated lost response") + } + + if _, err := client.CoreV1().ServiceAccounts(ns).Get(ctx, saName, metav1.GetOptions{}); err != nil { + t.Fatalf("test precondition: the ServiceAccount must exist in the cluster despite the "+ + "failed call, Get err = %v", err) + } if err := d.Cleanup(ctx, CleanupOptions{Enabled: true}); err != nil { t.Fatalf("Cleanup() error = %v", err) } - if !sawPreconditions { - t.Fatal("ServiceAccount delete did not carry Preconditions.UID") + if _, err := client.CoreV1().ServiceAccounts(ns).Get(ctx, saName, metav1.GetOptions{}); !apierrors.IsNotFound(err) { + t.Errorf("Cleanup leaked the ServiceAccount created by the lost-response Create; Get err = %v", err) } - if sawUID != wantUID { - t.Errorf("Preconditions.UID = %q, want %q", sawUID, wantUID) +} + +// TestEnsureDiscardsIntentOnAlreadyExists is recordIntent's counterweight: an +// AlreadyExists response is the one outcome that proves the object at that +// name is NOT ours (a duplicate RunID, or a 16-byte random collision). Keeping +// the intent entry would hand this run a bare-name delete of another run's +// object — strictly worse than the leak recordIntent prevents. +func TestEnsureDiscardsIntentOnAlreadyExists(t *testing.T) { + ctx := context.Background() + const ns = "test-ns" + client := fake.NewSimpleClientset() + + d := NewDeployer(client, Config{Namespace: ns, RunID: testRunID}) + client.PrependReactor("create", "serviceaccounts", func(action k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewAlreadyExists( + schema.GroupResource{Resource: "serviceaccounts"}, d.saName()) + }) + + if err := d.ensureServiceAccount(ctx); err == nil { + t.Fatal("ensureServiceAccount() = nil error, want AlreadyExists to be reported") + } + + if got := d.createdSnapshot(); len(got) != 0 { + t.Errorf("created-set = %+v, want empty — an AlreadyExists object was not created by "+ + "this run and must not enter its delete list", got) } } diff --git a/pkg/k8s/agent/job.go b/pkg/k8s/agent/job.go index d1f2fa59b..fb5dedd70 100644 --- a/pkg/k8s/agent/job.go +++ b/pkg/k8s/agent/job.go @@ -34,9 +34,13 @@ import ( // ensureJob creates the run-scoped agent Job. func (d *Deployer) ensureJob(ctx context.Context) error { job := d.buildJob() + // Record the intent before the Create so a committed create whose + // response is lost still enters Cleanup's delete list (see recordIntent). + d.recordIntent(kindJob, job.Name) created, err := d.clientset.BatchV1().Jobs(d.config.Namespace). Create(ctx, job, metav1.CreateOptions{}) if errors.IsAlreadyExists(err) { + d.discardIntent(kindJob, job.Name) return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "Job already exists under run-scoped name (duplicate RunID?)", err) } if err != nil { @@ -339,7 +343,7 @@ func (d *Deployer) deleteJob(ctx context.Context, name string, uid types.UID) er name, metav1.DeleteOptions{ PropagationPolicy: &propagationPolicy, - Preconditions: &metav1.Preconditions{UID: &uid}, + Preconditions: uidPreconditions(uid), }, ) return ignoreNotFoundOrConflict(err) diff --git a/pkg/k8s/agent/rbac.go b/pkg/k8s/agent/rbac.go index be98d59a2..0f391e21b 100644 --- a/pkg/k8s/agent/rbac.go +++ b/pkg/k8s/agent/rbac.go @@ -124,8 +124,12 @@ func (d *Deployer) ensureServiceAccount(ctx context.Context) error { }, } + // Record the intent before the Create so a committed create whose + // response is lost still enters Cleanup's delete list (see recordIntent). + d.recordIntent(kindServiceAccount, name) created, err := d.clientset.CoreV1().ServiceAccounts(d.config.Namespace).Create(ctx, sa, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { + d.discardIntent(kindServiceAccount, name) return errors.Wrap(errors.ErrCodeInternal, "ServiceAccount already exists under run-scoped name (duplicate RunID?)", err) } if err != nil { @@ -137,9 +141,10 @@ func (d *Deployer) ensureServiceAccount(ctx context.Context) error { // ensureRole creates the run-scoped Role for ConfigMap access. func (d *Deployer) ensureRole(ctx context.Context) error { + name := d.roleName() role := &rbacv1.Role{ ObjectMeta: metav1.ObjectMeta{ - Name: d.roleName(), + Name: name, Namespace: d.config.Namespace, Labels: d.objectLabels(), }, @@ -157,8 +162,10 @@ func (d *Deployer) ensureRole(ctx context.Context) error { }, } + d.recordIntent(kindRole, name) created, err := d.clientset.RbacV1().Roles(d.config.Namespace).Create(ctx, role, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { + d.discardIntent(kindRole, name) return errors.Wrap(errors.ErrCodeInternal, "Role already exists under run-scoped name (duplicate RunID?)", err) } if err != nil { @@ -170,9 +177,10 @@ func (d *Deployer) ensureRole(ctx context.Context) error { // ensureRoleBinding creates the run-scoped RoleBinding binding the Role to the ServiceAccount. func (d *Deployer) ensureRoleBinding(ctx context.Context) error { + name := d.roleName() rb := &rbacv1.RoleBinding{ ObjectMeta: metav1.ObjectMeta{ - Name: d.roleName(), + Name: name, Namespace: d.config.Namespace, Labels: d.objectLabels(), }, @@ -190,8 +198,10 @@ func (d *Deployer) ensureRoleBinding(ctx context.Context) error { }, } + d.recordIntent(kindRoleBinding, name) created, err := d.clientset.RbacV1().RoleBindings(d.config.Namespace).Create(ctx, rb, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { + d.discardIntent(kindRoleBinding, name) return errors.Wrap(errors.ErrCodeInternal, "RoleBinding already exists under run-scoped name (duplicate RunID?)", err) } if err != nil { @@ -247,16 +257,19 @@ func (d *Deployer) ensureClusterRole(ctx context.Context) error { rules = append(rules, discoverNetworkClusterRules()...) } + name := d.clusterRoleName() cr := &rbacv1.ClusterRole{ ObjectMeta: metav1.ObjectMeta{ - Name: d.clusterRoleName(), + Name: name, Labels: d.objectLabels(), }, Rules: rules, } + d.recordIntent(kindClusterRole, name) created, err := d.clientset.RbacV1().ClusterRoles().Create(ctx, cr, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { + d.discardIntent(kindClusterRole, name) return errors.Wrap(errors.ErrCodeInternal, "ClusterRole already exists under run-scoped name (duplicate RunID?)", err) } if err != nil { @@ -268,9 +281,10 @@ func (d *Deployer) ensureClusterRole(ctx context.Context) error { // ensureClusterRoleBinding creates the run-scoped ClusterRoleBinding binding the ClusterRole to the ServiceAccount. func (d *Deployer) ensureClusterRoleBinding(ctx context.Context) error { + name := d.clusterRoleName() crb := &rbacv1.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ - Name: d.clusterRoleName(), + Name: name, Labels: d.objectLabels(), }, Subjects: []rbacv1.Subject{ @@ -287,8 +301,10 @@ func (d *Deployer) ensureClusterRoleBinding(ctx context.Context) error { }, } + d.recordIntent(kindClusterRoleBinding, name) created, err := d.clientset.RbacV1().ClusterRoleBindings().Create(ctx, crb, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { + d.discardIntent(kindClusterRoleBinding, name) return errors.Wrap(errors.ErrCodeInternal, "ClusterRoleBinding already exists under run-scoped name (duplicate RunID?)", err) } if err != nil { @@ -304,7 +320,7 @@ func (d *Deployer) ensureClusterRoleBinding(ctx context.Context) error { // matches (already replaced, not ours), this is a no-op (idempotent). func (d *Deployer) deleteServiceAccount(ctx context.Context, name string, uid types.UID) error { err := d.clientset.CoreV1().ServiceAccounts(d.config.Namespace). - Delete(ctx, name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}) + Delete(ctx, name, metav1.DeleteOptions{Preconditions: uidPreconditions(uid)}) return ignoreNotFoundOrConflict(err) } @@ -312,7 +328,7 @@ func (d *Deployer) deleteServiceAccount(ctx context.Context, name string, uid ty // already gone, or uid no longer matches, this is a no-op (idempotent). func (d *Deployer) deleteRole(ctx context.Context, name string, uid types.UID) error { err := d.clientset.RbacV1().Roles(d.config.Namespace). - Delete(ctx, name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}) + Delete(ctx, name, metav1.DeleteOptions{Preconditions: uidPreconditions(uid)}) return ignoreNotFoundOrConflict(err) } @@ -321,7 +337,7 @@ func (d *Deployer) deleteRole(ctx context.Context, name string, uid types.UID) e // no-op (idempotent). func (d *Deployer) deleteRoleBinding(ctx context.Context, name string, uid types.UID) error { err := d.clientset.RbacV1().RoleBindings(d.config.Namespace). - Delete(ctx, name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}) + Delete(ctx, name, metav1.DeleteOptions{Preconditions: uidPreconditions(uid)}) return ignoreNotFoundOrConflict(err) } @@ -330,7 +346,7 @@ func (d *Deployer) deleteRoleBinding(ctx context.Context, name string, uid types // no-op (idempotent). func (d *Deployer) deleteClusterRole(ctx context.Context, name string, uid types.UID) error { err := d.clientset.RbacV1().ClusterRoles(). - Delete(ctx, name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}) + Delete(ctx, name, metav1.DeleteOptions{Preconditions: uidPreconditions(uid)}) return ignoreNotFoundOrConflict(err) } @@ -339,7 +355,7 @@ func (d *Deployer) deleteClusterRole(ctx context.Context, name string, uid types // longer matches, this is a no-op (idempotent). func (d *Deployer) deleteClusterRoleBinding(ctx context.Context, name string, uid types.UID) error { err := d.clientset.RbacV1().ClusterRoleBindings(). - Delete(ctx, name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}) + Delete(ctx, name, metav1.DeleteOptions{Preconditions: uidPreconditions(uid)}) return ignoreNotFoundOrConflict(err) } diff --git a/pkg/k8s/agent/types.go b/pkg/k8s/agent/types.go index 39d6dea71..e42338a13 100644 --- a/pkg/k8s/agent/types.go +++ b/pkg/k8s/agent/types.go @@ -49,6 +49,10 @@ const ( // configured names. name is the run-scoped name the object was created // under; uid pins the eventual delete via metav1.Preconditions so a // same-named object belonging to a different run is never collected. +// +// uid is the zero UID for an entry recorded by recordIntent but never +// confirmed by a Create response. Such an entry is deleted by bare name (no +// Preconditions) — see uidPreconditions. type createdObject struct { kind string name string @@ -72,8 +76,11 @@ type Config struct { // errors.ErrCodeInvalidRequest. RunID string - // NameBase prefixes generated resource names only — it has no effect - // when ServiceAccountName or JobName is already set. Defaults to + // NameBase prefixes generated resource names. It applies per name, + // not all-or-nothing: jobName() falls back to it when JobName is + // empty, and saName() (which also names the Role and RoleBinding) + // falls back to it when ServiceAccountName is empty — so setting only + // one of the two leaves NameBase governing the other. Defaults to // "aicr" when empty. NameBase string @@ -166,17 +173,79 @@ func (d *Deployer) objectLabels() map[string]string { } } -// recordCreated appends a run-owned object to the created-set. Cleanup +// recordIntent enters a run-owned object into the created-set with the zero +// UID, immediately BEFORE its Create call. It closes the window in which a +// committed Create whose response never arrives (client timeout, apiserver +// rollout, LB 502/504, connection reset, context cancellation in the +// response window) leaves an object nothing will ever delete: the ensure* +// call returns a non-AlreadyExists error, Deploy aborts, and the deferred +// Cleanup — enabled by default — would otherwise never learn the object +// exists. Because the name is run-unique, no later run reclaims it either, +// so the orphan is permanent. +// +// Cleanup deletes a zero-UID entry by bare name (uidPreconditions returns +// nil), which is safe precisely because the name carries this run's ID. An +// AlreadyExists response is the one case that proves the object is NOT ours, +// so every ensure* discards the intent on that branch (see discardIntent). +// Safe for concurrent use. +func (d *Deployer) recordIntent(kind, name string) { + d.mu.Lock() + defer d.mu.Unlock() + d.created = append(d.created, createdObject{kind: kind, name: name}) +} + +// discardIntent drops the zero-UID entry recordIntent added for (kind, +// name). Called only when a Create returns AlreadyExists: the object at that +// name exists but this run did not create it, so it must not enter this +// run's delete list. Safe for concurrent use. +func (d *Deployer) discardIntent(kind, name string) { + d.mu.Lock() + defer d.mu.Unlock() + for i := range d.created { + if d.created[i].kind == kind && d.created[i].name == name && d.created[i].uid == "" { + d.created = append(d.created[:i], d.created[i+1:]...) + return + } + } +} + +// recordCreated confirms a run-owned object in the created-set. Cleanup // builds its UID-pinned delete list from exactly this set, so every ensure* // call that successfully creates an object — and GetSnapshot, for the // staging ConfigMap it observes but does not itself create — must call this -// on success. Safe for concurrent use. +// on success. +// +// It upserts: when recordIntent already entered (kind, name) with the zero +// UID, the observed UID is written onto that entry rather than appended as a +// second one, so the set holds one entry per object and jobUID() sees the +// real Job UID. Safe for concurrent use. func (d *Deployer) recordCreated(kind, name string, uid types.UID) { d.mu.Lock() defer d.mu.Unlock() + for i := range d.created { + if d.created[i].kind == kind && d.created[i].name == name && d.created[i].uid == "" { + d.created[i].uid = uid + return + } + } d.created = append(d.created, createdObject{kind: kind, name: name, uid: uid}) } +// containsKind reports whether objs holds an entry of kind. Cleanup uses it +// against the single created-set snapshot it already took, rather than +// re-entering the mutex: reading the set twice would let a recordCreated +// landing between the two reads produce a snapshot that misses the staging +// ConfigMap while the second read reports it present, skipping both the +// created-set delete and the name-based sweep and leaking the object. +func containsKind(objs []createdObject, kind string) bool { + for _, o := range objs { + if o.kind == kind { + return true + } + } + return false +} + // createdSnapshot returns a defensive copy of the created-set taken under // lock. Callers must not read d.created directly. func (d *Deployer) createdSnapshot() []createdObject { @@ -189,8 +258,10 @@ func (d *Deployer) createdSnapshot() []createdObject { // jobUID returns the UID of the Job this Deployer created, or the zero UID // if Deploy has not (yet) reached the Job-create step — including when -// Deploy failed before getting there. Pod selection (see ownedByJob in -// wait.go) authorizes candidates against exactly this UID. +// Deploy failed before getting there, and while the Job's Create is in +// flight (recordIntent has entered the name but no UID is known yet). Pod +// selection (see ownedByJob in wait.go) authorizes candidates against +// exactly this UID. func (d *Deployer) jobUID() types.UID { d.mu.Lock() defer d.mu.Unlock() @@ -203,9 +274,10 @@ func (d *Deployer) jobUID() types.UID { } // hasCreated reports whether the created-set already holds an object of -// kind. Cleanup uses it to decide whether the staging ConfigMap still needs -// a name-based sweep (the run failed before getSnapshotFromConfigMap could -// observe its UID) or was already recorded. Safe for concurrent use. +// kind. Cleanup does NOT use it — it derives the same answer from the single +// snapshot it already took, via containsKind, so the decision cannot straddle +// two lock acquisitions. This remains as the locked accessor for callers +// (tests) that hold no snapshot. Safe for concurrent use. func (d *Deployer) hasCreated(kind string) bool { d.mu.Lock() defer d.mu.Unlock() diff --git a/pkg/k8s/agent/wait.go b/pkg/k8s/agent/wait.go index 324dfcdef..53e581718 100644 --- a/pkg/k8s/agent/wait.go +++ b/pkg/k8s/agent/wait.go @@ -76,7 +76,7 @@ func (d *Deployer) getSnapshotFromConfigMap(ctx context.Context) ([]byte, error) // true at record time (see getSnapshotFromConfigMap). func (d *Deployer) deleteStagingConfigMap(ctx context.Context, name string, uid types.UID) error { err := d.clientset.CoreV1().ConfigMaps(d.config.Namespace). - Delete(ctx, name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}) + Delete(ctx, name, metav1.DeleteOptions{Preconditions: uidPreconditions(uid)}) return ignoreNotFoundOrConflict(err) } From d221e2637a865c49cb06f004622f4a4b99c74c80 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Mon, 24 Aug 2026 13:01:11 -0700 Subject: [PATCH 28/56] test(agent): cover ownership on the watch-based pod-discovery path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit findOrWatchPodName is the primary production discovery path — pkg/snapshotter calls WaitForPodReady immediately after Deploy, before any pod exists, so the fast List misses and the watch loop is what selects the pod. Its ownership authorization had no coverage: every test reaching it left jobUID() empty, which makes both pickLivePod calls and the per-event guard no-ops. Mutation evidence: replacing both pickLivePod(pods.Items, d.jobUID()) calls in findOrWatchPodName with pickLivePod(pods.Items, "") and deleting the per-event ownedByJob guard left the whole package green. With the new test, four of its five subtests fail. The new cases mirror TestConcurrentRuns's imposter — a pod carrying this run's labels but a controlling ownerReference to a different Job UID, emitted before the real pod — plus an ownerReference-less pod, and three cases over the watch-channel-closed re-List branch, which had zero coverage: re-List finds the owned pod, re-List finds only the foreign pod (must fail closed), and re-List itself errors. Also rewords two comments in concurrency_test.go that overclaimed. The staging-ConfigMap subtest said it proved Cleanup is "scoped by the created-set rather than by recomputed names"; mutation testing showed it passes unchanged for any name-derived Cleanup that keeps the OwnsOutputConfigMap gate. What it pins is the ownership gate. The reworded comment says so and cross-references the tests that do discriminate created-set scoping. No assertion was weakened. Refs: ADR-020 Signed-off-by: Alex Yuskauskas --- pkg/k8s/agent/concurrency_test.go | 67 +++++---- pkg/k8s/agent/wait_test.go | 226 ++++++++++++++++++++++++++++++ 2 files changed, 266 insertions(+), 27 deletions(-) diff --git a/pkg/k8s/agent/concurrency_test.go b/pkg/k8s/agent/concurrency_test.go index 823be6214..f9fcf6bd0 100644 --- a/pkg/k8s/agent/concurrency_test.go +++ b/pkg/k8s/agent/concurrency_test.go @@ -65,9 +65,9 @@ var ( // RBAC permissions (specifically DiscoverNetwork's extra cluster rules) // into the other, must each select only their own Pod, must each read back // only their own snapshot bytes, and one run's Cleanup must never touch the -// other's objects — nor an object that merely sits at a name that run's own -// naming formula would produce but that it never actually created. Run -// under -race. +// other's objects — nor a ConfigMap that merely sits at a name that run's own +// naming formula would produce but that the run does not own. Run under +// -race. // // Fake-clientset limitations this test works around, not around-asserts: // @@ -92,7 +92,10 @@ var ( // selector string for that path. Watch does NOT filter (the fake // Watch reactor ignores ListOptions.LabelSelector entirely); this // test does not exercise the watch-based pod-discovery path -// (findOrWatchPodName), so that gap does not apply here. +// (findOrWatchPodName), so that gap does not apply here. That path — +// the one production actually takes, since WaitForPodReady runs right +// after Deploy — has its own ownership coverage in +// TestFindOrWatchPodNameAuthorizesByJobOwnership (wait_test.go). func TestConcurrentRuns(t *testing.T) { ctx := context.Background() clientset := fake.NewClientset() @@ -130,12 +133,11 @@ func TestConcurrentRuns(t *testing.T) { // OwnsOutputConfigMap is false for run A on purpose (unlike run B // below): it models an object — the staging ConfigMap seeded for // run A below — that sits at exactly the name run A's own naming - // formula computes, but that run A never created or recorded into - // its created-set. Assertion 5's discriminator subtest needs - // exactly this shape: name-scoping alone (assertion 1) cannot - // explain that object surviving run A's Cleanup, only created-set - // scoping can. Run B keeps OwnsOutputConfigMap true so the - // "owned and recorded" path stays covered too (assertion 4). + // formula computes, but that run A does not own. Assertion 5's + // ownership subtest needs exactly this shape, since name-scoping + // alone (assertion 1) cannot explain that object surviving run A's + // Cleanup. Run B keeps OwnsOutputConfigMap true so the "owned and + // recorded" path stays covered too (assertion 4). OwnsOutputConfigMap: false, DiscoverNetwork: false, }) @@ -278,25 +280,30 @@ func TestConcurrentRuns(t *testing.T) { } }) - // --- Assertion 5 (cleanup ownership discriminator): run A's Cleanup - // must not touch an object it never created, even when that object - // sits at exactly the name run A's own naming formula computes. + // --- Assertion 5 (staging-ConfigMap ownership gate): run A's Cleanup + // must not touch a staging-named ConfigMap this run does not own, even + // when that name is exactly what run A's own naming formula computes. // - // This is deliberately a separate subtest from the one above: the run-B - // check above would pass equally against a hypothetical name-derived - // Cleanup, because run A and run B always compute different names - // (assertion 1 already proves that) — so it cannot by itself prove - // Cleanup is scoped by the created-set rather than by recomputed names. - // This subtest supplies the missing case: dA's staging ConfigMap name - // collides with dA's own formula, but Config.OwnsOutputConfigMap was - // false for run A, so getSnapshotFromConfigMap (assertion 4) never - // recorded it into run A's created-set. So the property under test is - // precisely this: a Cleanup that recomputed its delete list from - // d.stagingConfigMapName() would delete this ConfigMap, while one - // driven by the created-set leaves it standing. - t.Run("run A Cleanup leaves its own unrecorded staging ConfigMap intact", func(t *testing.T) { + // This is deliberately a separate subtest from the one above, which + // compares run A against run B and so is satisfied by name-scoping + // alone (assertion 1 already proves the two runs compute different + // names). Here the names collide, so only Config.OwnsOutputConfigMap — + // false for run A, which is why getSnapshotFromConfigMap in assertion 4 + // did not record the ConfigMap and why the name-based sweep does not + // fire either — can explain the object surviving. + // + // Scope note: this pins the OWNERSHIP GATE, not created-set scoping as + // such. A hypothetical Cleanup that recomputed its delete list from + // d.stagingConfigMapName() but kept the same OwnsOutputConfigMap gate + // would pass this subtest unchanged. The tests that actually + // discriminate created-set scoping are TestCleanupPassesUIDPrecondition + // (every delete carries the UID from its Create response — a value no + // name-derived delete list can supply) and + // TestCleanupDeletesUnconfirmedCreateByBareName, both in + // deployer_test.go. + t.Run("run A Cleanup leaves its own unowned staging ConfigMap intact", func(t *testing.T) { if _, err := clientset.CoreV1().ConfigMaps(concurrencyTestNamespace).Get(ctx, dA.stagingConfigMapName(), metav1.GetOptions{}); err != nil { - t.Errorf("run A's own staging ConfigMap %q should survive run A's Cleanup (never recorded, so not owned), err = %v", dA.stagingConfigMapName(), err) + t.Errorf("the ConfigMap at run A's staging name %q should survive run A's Cleanup (OwnsOutputConfigMap is false, so it is not run A's to delete), err = %v", dA.stagingConfigMapName(), err) } }) } @@ -335,6 +342,12 @@ func assertSevenKindsExist(t *testing.T, ctx context.Context, clientset kubernet // seedStagingConfigMap creates d's staging ConfigMap directly, standing in // for the in-pod agent write that Deploy() itself never performs against // the fake clientset. +// +// The labels below are d's own set for convenience only. The real in-pod +// writer (pkg/serializer's ConfigMap writer) stamps a different, smaller set — +// no managed-by and no run-ID label — because it also produces the user's +// delivered cm:// artifact. Nothing here selects on these labels: run scoping +// for this object comes from its name. func seedStagingConfigMap(t *testing.T, ctx context.Context, clientset kubernetes.Interface, d *Deployer, uid, snapshotYAML string) { t.Helper() cm := &corev1.ConfigMap{ diff --git a/pkg/k8s/agent/wait_test.go b/pkg/k8s/agent/wait_test.go index 05bebff33..19baab173 100644 --- a/pkg/k8s/agent/wait_test.go +++ b/pkg/k8s/agent/wait_test.go @@ -17,17 +17,23 @@ package agent import ( "bytes" "context" + stderrors "errors" "io" + "sync" "testing" "time" + aicrerrors "github.com/NVIDIA/aicr/pkg/errors" "github.com/NVIDIA/aicr/pkg/k8s/labels" "github.com/NVIDIA/aicr/pkg/k8s/pod" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" ) func TestParseConfigMapName_Extended(t *testing.T) { @@ -497,3 +503,223 @@ func TestPickLivePod(t *testing.T) { }) } } + +// Two Job UIDs for the watch-path ownership tests: A is the run under test, +// B stands in for any concurrent run whose pod could reach this run's watch. +const ( + watchJobUIDA = types.UID("job-uid-a") + watchJobUIDB = types.UID("job-uid-b") +) + +// runLabeledPod returns a Pod carrying d's full label set — so it passes +// d.podLabelSelector() — controlled by the Job with ownerUID. Passing a UID +// other than d's own recorded Job UID produces the imposter: a pod that +// anything able to update pods in the namespace could label as this run's, +// but that this run's Job does not control. +func runLabeledPod(d *Deployer, name string, ownerUID types.UID) *corev1.Pod { + controller := true + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: d.config.Namespace, + Labels: d.objectLabels(), + OwnerReferences: []metav1.OwnerReference{ + { + Kind: kindJob, + Name: "some-other-run-job", + UID: ownerUID, + Controller: &controller, + }, + }, + }, + } +} + +// watchDeployer returns a Deployer for the watch-path tests with run A's Job +// UID already recorded, so jobUID() is non-zero and the ownership checks in +// findOrWatchPodName are actually exercised rather than skipped. +func watchDeployer(client *fake.Clientset, namespace string) *Deployer { + d := NewDeployer(client, Config{Namespace: namespace, RunID: testRunID}) + d.recordCreated(kindJob, d.jobName(), watchJobUIDA) + return d +} + +// TestFindOrWatchPodNameAuthorizesByJobOwnership covers the watch-based +// discovery path, which is the PRIMARY production path: pkg/snapshotter calls +// WaitForPodReady immediately after Deploy, before any pod exists, so the fast +// List misses and the watch loop is what actually selects the pod. +// +// Every other test in this package that reaches findOrWatchPodName leaves +// jobUID() empty, which makes both pickLivePod calls AND the per-event guard +// no-ops — so the ownership authorization on this path was previously +// untested. Each subtest here records a real Job UID first. +// +// Fake-clientset limitation this works around: the fake Watch reactor ignores +// ListOptions.LabelSelector entirely, so every emitted event reaches the loop. +// That is fine — and in fact necessary — here, because the property under test +// is that the controlling ownerReference (not the forgeable RunID label) is +// what authorizes selection. The List-side selector filtering IS honored by +// the fake, so the re-List subtests exercise it for real. +func TestFindOrWatchPodNameAuthorizesByJobOwnership(t *testing.T) { + const ns = "watch-ns" + + t.Run("watch event for another run's Job is skipped", func(t *testing.T) { + client := fake.NewClientset() + w := watch.NewRaceFreeFake() + client.PrependWatchReactor("pods", k8stesting.DefaultWatchReactor(w, nil)) + + d := watchDeployer(client, ns) + + // The imposter arrives FIRST: a watch loop that returned the first + // event passing the label filter would take it and never see the + // real pod. + w.Add(runLabeledPod(d, "imposter-pod", watchJobUIDB)) + w.Add(runLabeledPod(d, "agent-pod-a", watchJobUIDA)) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + got, err := d.findOrWatchPodName(ctx) + if err != nil { + t.Fatalf("findOrWatchPodName() error = %v", err) + } + if got != "agent-pod-a" { + t.Errorf("findOrWatchPodName() = %q, want %q — a pod controlled by another run's "+ + "Job was selected on the watch path", got, "agent-pod-a") + } + }) + + t.Run("watch event with no ownerReference is skipped", func(t *testing.T) { + client := fake.NewClientset() + w := watch.NewRaceFreeFake() + client.PrependWatchReactor("pods", k8stesting.DefaultWatchReactor(w, nil)) + + d := watchDeployer(client, ns) + + // Labels alone, no controlling ownerReference at all — the shape a + // caller with pods/update in the namespace can produce directly. + orphan := runLabeledPod(d, "orphan-pod", watchJobUIDA) + orphan.OwnerReferences = nil + w.Add(orphan) + w.Add(runLabeledPod(d, "agent-pod-a", watchJobUIDA)) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + got, err := d.findOrWatchPodName(ctx) + if err != nil { + t.Fatalf("findOrWatchPodName() error = %v", err) + } + if got != "agent-pod-a" { + t.Errorf("findOrWatchPodName() = %q, want %q", got, "agent-pod-a") + } + }) + + t.Run("closed watch channel re-Lists and authorizes the re-Listed pod", func(t *testing.T) { + client := fake.NewClientset() + d := watchDeployer(client, ns) + + // The fast-path List must miss so the watch is reached at all; the + // re-List after the channel closes then returns both pods. + installStagedPodLister(client, func(call int) []corev1.Pod { + if call == 1 { + return nil + } + return []corev1.Pod{ + *runLabeledPod(d, "imposter-pod", watchJobUIDB), + *runLabeledPod(d, "agent-pod-a", watchJobUIDA), + } + }) + + w := watch.NewRaceFreeFake() + w.Stop() // apiserver hiccup / LB drop: channel closed, no event + client.PrependWatchReactor("pods", k8stesting.DefaultWatchReactor(w, nil)) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + got, err := d.findOrWatchPodName(ctx) + if err != nil { + t.Fatalf("findOrWatchPodName() error = %v — a closed watch channel must re-List "+ + "before declaring failure", err) + } + if got != "agent-pod-a" { + t.Errorf("findOrWatchPodName() = %q, want %q — the re-List branch must apply the "+ + "same ownership check", got, "agent-pod-a") + } + }) + + t.Run("closed watch channel with only a foreign pod fails closed", func(t *testing.T) { + client := fake.NewClientset() + d := watchDeployer(client, ns) + + installStagedPodLister(client, func(call int) []corev1.Pod { + if call == 1 { + return nil + } + return []corev1.Pod{*runLabeledPod(d, "imposter-pod", watchJobUIDB)} + }) + + w := watch.NewRaceFreeFake() + w.Stop() + client.PrependWatchReactor("pods", k8stesting.DefaultWatchReactor(w, nil)) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + got, err := d.findOrWatchPodName(ctx) + if err == nil { + t.Fatalf("findOrWatchPodName() = %q, nil error; a pod owned by another run's Job "+ + "must not satisfy the re-List branch", got) + } + if !stderrors.Is(err, aicrerrors.New(aicrerrors.ErrCodeUnavailable, "")) { + t.Errorf("error = %v, want code ErrCodeUnavailable", err) + } + }) + + t.Run("closed watch channel surfaces a failed re-List", func(t *testing.T) { + client := fake.NewClientset() + d := watchDeployer(client, ns) + + var mu sync.Mutex + var calls int + client.PrependReactor("list", "pods", func(k8stesting.Action) (bool, runtime.Object, error) { + mu.Lock() + defer mu.Unlock() + calls++ + if calls == 1 { + return true, &corev1.PodList{}, nil + } + return true, nil, stderrors.New("apiserver unreachable") + }) + + w := watch.NewRaceFreeFake() + w.Stop() + client.PrependWatchReactor("pods", k8stesting.DefaultWatchReactor(w, nil)) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if _, err := d.findOrWatchPodName(ctx); err == nil { + t.Fatal("findOrWatchPodName() = nil error, want the re-List failure surfaced") + } else if !stderrors.Is(err, aicrerrors.New(aicrerrors.ErrCodeUnavailable, "")) { + t.Errorf("error = %v, want code ErrCodeUnavailable", err) + } + }) +} + +// installStagedPodLister makes successive Pod List calls return different +// results: items(1) answers the fast-path List in findOrWatchPodName, items(2) +// answers the re-List after the watch channel closes. The fake still applies +// ListOptions.LabelSelector to whatever this returns, so the label narrowing +// is exercised for real. +func installStagedPodLister(client *fake.Clientset, items func(call int) []corev1.Pod) { + var mu sync.Mutex + var calls int + client.PrependReactor("list", "pods", func(k8stesting.Action) (bool, runtime.Object, error) { + mu.Lock() + defer mu.Unlock() + calls++ + return true, &corev1.PodList{Items: items(calls)}, nil + }) +} From 19bc99aaa42e07d2dd786e9bd6e1052aa2a1c15b Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Mon, 24 Aug 2026 13:01:11 -0700 Subject: [PATCH 29/56] fix(snapshotter): stop DeployAndCollect mutating the caller's AgentConfig MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit config.RunID = runid.Generate() wrote the generated ID back into the caller-owned *AgentConfig. A caller reusing one config pointer for a second run silently became a caller who pinned a duplicate RunID — the one state ADR-020 declares unsupported — and run 2 hard-failed ErrCodeInternal on the first still-existing run-scoped object. The resolved ID now stays in a local and is passed into agentConfigMapTarget, deployAndWaitForResult, and buildAgentConfig instead of being read back off the config. In-tree callers go through the pkg/client/v1 facade, which builds a fresh internal config per call, so nothing in this repo was affected — but pkg/snapshotter is public and the mutation was undocumented. TestDeployAndCollectDefaultsRunID asserted the mutation as the behavior under test. It is replaced by TestDeployAndCollectGeneratesRunIDWithoutMutatingConfig, which observes the generated ID through the "snapshot agent run" log line and asserts cfg.RunID is left empty. Refs: ADR-020 Signed-off-by: Alex Yuskauskas --- pkg/snapshotter/agent.go | 72 +++++++++++++++++++++++---------- pkg/snapshotter/agent_test.go | 76 ++++++++++++++++++++++++----------- 2 files changed, 102 insertions(+), 46 deletions(-) diff --git a/pkg/snapshotter/agent.go b/pkg/snapshotter/agent.go index f156bbe52..7c3d4ad0d 100644 --- a/pkg/snapshotter/agent.go +++ b/pkg/snapshotter/agent.go @@ -153,17 +153,27 @@ type AgentConfig struct { // RunID scopes every resource this deployment creates (Job, RBAC, and // the internal staging ConfigMap when Output does not name one) to a // single run, so concurrent snapshot-agent runs never collide on a - // shared resource name. DeployAndCollect defaults it with - // runid.Generate() when empty — callers normally leave it unset; - // setting it explicitly is for correlating this run with an external - // identifier (e.g. sharing one ID with a downstream validator run). + // shared resource name. DeployAndCollect generates one with + // runid.Generate() when this is empty — callers normally leave it + // unset; setting it explicitly is for correlating this run with an + // external identifier (e.g. sharing one ID with a downstream + // validator run). + // + // DeployAndCollect never writes the generated value back here: the + // AgentConfig belongs to the caller, and a caller reusing one config + // pointer across two runs would otherwise silently become a caller + // pinning a duplicate RunID — the one state ADR-020 declares + // unsupported, which fails the second run with ErrCodeInternal on the + // first still-existing run-scoped object. RunID string // NameBase prefixes generated resource names (Job, ServiceAccount, - // Role/RoleBinding) when JobName / ServiceAccountName are left empty. - // It has no effect once either of those is set. Forwarded verbatim - // to pkg/k8s/agent.Config.NameBase, which defaults to "aicr" when - // also empty. + // Role/RoleBinding). It applies per name: JobName falls back to it + // when JobName is empty, and ServiceAccountName (which also names the + // Role and RoleBinding) falls back to it when ServiceAccountName is + // empty — so setting only one of the two leaves NameBase governing the + // other. Forwarded verbatim to pkg/k8s/agent.Config.NameBase, which + // defaults to "aicr" when also empty. NameBase string } @@ -175,12 +185,18 @@ type AgentConfig struct { // verbatim: it is true only when agentOutput is the internal staging // ConfigMap this run owns (the caller did not name a cm:// destination), so // Cleanup may delete it. A caller-supplied cm:// Output is never owned. -func buildAgentConfig(config *AgentConfig, agentOutput string, ownsOutput bool) agent.Config { +// +// runID is the resolved run ID for this invocation — config.RunID when the +// caller pinned one, otherwise the value DeployAndCollect generated. It is +// passed in rather than read from config because DeployAndCollect must not +// write the generated ID back into the caller's AgentConfig (see +// AgentConfig.RunID). +func buildAgentConfig(config *AgentConfig, runID, agentOutput string, ownsOutput bool) agent.Config { return agent.Config{ Namespace: config.Namespace, ServiceAccountName: config.ServiceAccountName, JobName: config.JobName, - RunID: config.RunID, + RunID: runID, NameBase: config.NameBase, Image: config.Image, ImagePullSecrets: config.ImagePullSecrets, @@ -209,7 +225,9 @@ func buildAgentConfig(config *AgentConfig, agentOutput string, ownsOutput bool) // agentOutput is the internal staging ConfigMap this run owns, false when it // is a caller-supplied cm:// destination. See buildAgentConfig and // rewriteMergedSnapshotConfigMap for how each consumes it. -func deployAndWaitForResult(ctx context.Context, clientset k8sclient.Interface, config *AgentConfig, agentOutput string, ownsOutput bool) ([]byte, error) { +// +// runID is the resolved run ID for this invocation; see buildAgentConfig. +func deployAndWaitForResult(ctx context.Context, clientset k8sclient.Interface, config *AgentConfig, runID, agentOutput string, ownsOutput bool) ([]byte, error) { // The pool projection is pure file processing on the caller's host — // project it BEFORE deploying so a bad file fails in milliseconds, // not after a Job round-trip, and merge it into the returned snapshot @@ -228,7 +246,7 @@ func deployAndWaitForResult(ctx context.Context, clientset k8sclient.Interface, // name the injected selector (TOCTOU: node may be cordoned after detection). autoInjectedGPUSelector := maybeInjectGPUNodeSelector(ctx, clientset, config) - agentConfig := buildAgentConfig(config, agentOutput, ownsOutput) + agentConfig := buildAgentConfig(config, runID, agentOutput, ownsOutput) deployer := agent.NewDeployer(clientset, agentConfig) @@ -525,19 +543,24 @@ func DeployAndCollect(ctx context.Context, config *AgentConfig) (*Snapshot, []by // field a CLI user never set — so catch the whitespace-only value that // slips past this simple emptiness check here, where the message can // point at the knob the caller actually controls. - if config.RunID == "" { - config.RunID = runid.Generate() - } - if strings.TrimSpace(config.RunID) == "" { + // + // The resolved value stays in a local: writing it back into the + // caller-owned *AgentConfig would turn a caller who reuses one config + // pointer for a second run into a caller pinning a duplicate RunID. + runID := config.RunID + if runID == "" { + runID = runid.Generate() + } + if strings.TrimSpace(runID) == "" { return nil, nil, errors.New(errors.ErrCodeInvalidRequest, "RunID must not be all-whitespace; leave it unset to auto-generate one") } - slog.Info("snapshot agent run", slog.String("runID", config.RunID)) + slog.Info("snapshot agent run", slog.String("runID", runID)) // Resolve (and validate) the Job's ConfigMap target before any cluster // access: a malformed cm:// Output must not cost the caller a deployed // Job and a cluster-admin binding. - agentOutput, ownsOutput, err := agentConfigMapTarget(config) + agentOutput, ownsOutput, err := agentConfigMapTarget(config, runID) if err != nil { return nil, nil, err } @@ -549,7 +572,7 @@ func DeployAndCollect(ctx context.Context, config *AgentConfig) (*Snapshot, []by return nil, nil, err } - snapshotData, err := deployAndWaitForResult(ctx, clientset, config, agentOutput, ownsOutput) + snapshotData, err := deployAndWaitForResult(ctx, clientset, config, runID, agentOutput, ownsOutput) if err != nil { return nil, nil, err } @@ -576,8 +599,13 @@ func DeployAndCollect(ctx context.Context, config *AgentConfig) (*Snapshot, []by // config.Namespace that the caller never names: ownsOutput is true, so // Cleanup may delete it. // +// runID is the resolved run ID for this invocation (config.RunID when the +// caller pinned one, otherwise the value DeployAndCollect generated); it is a +// parameter rather than a config field read because DeployAndCollect never +// writes the generated ID back into the caller's AgentConfig. +// // In the owned case the returned uri is exactly -// cm:///. Cleanup +// cm:///. Cleanup // in pkg/k8s/agent relies on both halves of that invariant rather than // re-parsing the URI: deleteStagingConfigMap deletes in Config.Namespace, and // deleteUnrecordedStagingConfigMap (the sweep for a run that failed before @@ -588,7 +616,7 @@ func DeployAndCollect(ctx context.Context, config *AgentConfig) (*Snapshot, []by // so a typo like "cm://aicr-snapshot" (no namespace) would otherwise surface // as a Job failure — after RBAC and the Job exist, and with Cleanup false // (the zero value) they stay behind. Returns ErrCodeInvalidRequest instead. -func agentConfigMapTarget(config *AgentConfig) (uri string, ownsOutput bool, err error) { +func agentConfigMapTarget(config *AgentConfig, runID string) (uri string, ownsOutput bool, err error) { if strings.HasPrefix(config.Output, serializer.ConfigMapURIScheme) { if _, _, parseErr := pod.ParseConfigMapURI(config.Output); parseErr != nil { // Wrap with the same code rather than PropagateOrWrap: the inner @@ -600,7 +628,7 @@ func agentConfigMapTarget(config *AgentConfig) (uri string, ownsOutput bool, err } return config.Output, false, nil } - return serializer.ConfigMapURIScheme + config.Namespace + "/" + agent.StagingConfigMapName(config.RunID), true, nil + return serializer.ConfigMapURIScheme + config.Namespace + "/" + agent.StagingConfigMapName(runID), true, nil } // SnapshotDelivery describes where DeliverSnapshot writes captured bytes. diff --git a/pkg/snapshotter/agent_test.go b/pkg/snapshotter/agent_test.go index 2010d1eca..9ac04e756 100644 --- a/pkg/snapshotter/agent_test.go +++ b/pkg/snapshotter/agent_test.go @@ -19,6 +19,7 @@ import ( "encoding/json" stderrors "errors" "io" + "log/slog" "os" "path/filepath" "reflect" @@ -90,7 +91,7 @@ func TestBuildAgentConfigTolerations(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := buildAgentConfig(&AgentConfig{Tolerations: tt.input}, "snapshot.yaml", false).Tolerations + got := buildAgentConfig(&AgentConfig{Tolerations: tt.input}, "20260821-142233-9f3a1c0b7e2d4a55", "snapshot.yaml", false).Tolerations if !reflect.DeepEqual(got, tt.want) { t.Errorf("buildAgentConfig().Tolerations = %#v, want %#v", got, tt.want) } @@ -99,12 +100,18 @@ func TestBuildAgentConfigTolerations(t *testing.T) { } // TestBuildAgentConfigPropagatesRunIDAndOwnership confirms buildAgentConfig -// forwards AgentConfig.RunID, AgentConfig.NameBase, and its ownsOutput +// forwards its runID argument, AgentConfig.NameBase, and its ownsOutput // parameter onto agent.Config.RunID / agent.Config.NameBase / // agent.Config.OwnsOutputConfigMap — the projection deployAndWaitForResult // relies on so the deployer scopes every resource name to this run, applies // the caller's naming prefix, and Cleanup knows whether it may delete the // staging ConfigMap. +// +// runID is a parameter, not a read of AgentConfig.RunID: DeployAndCollect +// resolves it into a local and never writes a generated ID back into the +// caller's config. The AgentConfig below deliberately leaves RunID empty so a +// projection that regressed to reading config.RunID would produce "" and fail +// here. func TestBuildAgentConfigPropagatesRunIDAndOwnership(t *testing.T) { tests := []struct { name string @@ -116,11 +123,10 @@ func TestBuildAgentConfigPropagatesRunIDAndOwnership(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got := buildAgentConfig(&AgentConfig{ - RunID: "20260821-142233-9f3a1c0b7e2d4a55", NameBase: "aicr-validate", - }, "cm://ns/name", tt.ownsOutput) + }, "20260821-142233-9f3a1c0b7e2d4a55", "cm://ns/name", tt.ownsOutput) if got.RunID != "20260821-142233-9f3a1c0b7e2d4a55" { - t.Errorf("agent.Config.RunID = %q, want the AgentConfig.RunID value", got.RunID) + t.Errorf("agent.Config.RunID = %q, want the runID argument", got.RunID) } if got.NameBase != "aicr-validate" { t.Errorf("agent.Config.NameBase = %q, want the AgentConfig.NameBase value", got.NameBase) @@ -599,8 +605,7 @@ func TestAgentOutputURILogic(t *testing.T) { agentOutput, ownsOutput, err := agentConfigMapTarget(&AgentConfig{ Namespace: tt.agentNamespace, Output: tt.userOutput, - RunID: testRunID, - }) + }, testRunID) if err != nil { t.Fatalf("agentConfigMapTarget: %v", err) } @@ -632,8 +637,9 @@ func TestAgentOutputURILogic(t *testing.T) { // staging ConfigMap only when the user did NOT name a cm:// destination. // Getting this backwards deletes the user's own output ConfigMap. func TestAgentConfigMapTargetIsRunScoped(t *testing.T) { - cfg := &AgentConfig{Namespace: "gpu-operator", RunID: "20260821-142233-9f3a1c0b7e2d4a55"} - uri, ownsOutput, err := agentConfigMapTarget(cfg) + const runID = "20260821-142233-9f3a1c0b7e2d4a55" + cfg := &AgentConfig{Namespace: "gpu-operator"} + uri, ownsOutput, err := agentConfigMapTarget(cfg, runID) if err != nil { t.Fatalf("agentConfigMapTarget() error = %v", err) } @@ -648,7 +654,7 @@ func TestAgentConfigMapTargetIsRunScoped(t *testing.T) { } // The URI must be built from the one exported helper the agent package // also deletes by, not from a second copy of the format string. - if wantHelper := "cm://gpu-operator/" + agent.StagingConfigMapName(cfg.RunID); uri != wantHelper { + if wantHelper := "cm://gpu-operator/" + agent.StagingConfigMapName(runID); uri != wantHelper { t.Errorf("uri = %q, want %q (agent.StagingConfigMapName)", uri, wantHelper) } if !ownsOutput { @@ -657,8 +663,8 @@ func TestAgentConfigMapTargetIsRunScoped(t *testing.T) { } func TestAgentConfigMapTargetLeavesUserURIAlone(t *testing.T) { - cfg := &AgentConfig{Namespace: "gpu-operator", Output: "cm://gpu-operator/aicr-snapshot", RunID: "20260821-142233-9f3a1c0b7e2d4a55"} - uri, ownsOutput, err := agentConfigMapTarget(cfg) + cfg := &AgentConfig{Namespace: "gpu-operator", Output: "cm://gpu-operator/aicr-snapshot"} + uri, ownsOutput, err := agentConfigMapTarget(cfg, "20260821-142233-9f3a1c0b7e2d4a55") if err != nil { t.Fatalf("agentConfigMapTarget() error = %v", err) } @@ -1139,7 +1145,7 @@ func TestAgentConfigMapTargetRejectsMalformedURI(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, _, err := agentConfigMapTarget(&AgentConfig{Namespace: "default", Output: tt.output}) + _, _, err := agentConfigMapTarget(&AgentConfig{Namespace: "default", Output: tt.output}, "20260821-142233-9f3a1c0b7e2d4a55") if err == nil { t.Fatalf("agentConfigMapTarget(%q) = nil error, want rejection before any cluster access", tt.output) } @@ -1234,18 +1240,34 @@ func TestDeployAndCollectRejectsBeforeClusterAccess(t *testing.T) { } } -// TestDeployAndCollectDefaultsRunID confirms DeployAndCollect fills an empty -// RunID with a freshly generated one before it is folded into the internal -// staging ConfigMap's name — observable here because the malformed-Output -// rejection below fires AFTER that defaulting step, and DeployAndCollect -// mutates the caller's *AgentConfig in place. -func TestDeployAndCollectDefaultsRunID(t *testing.T) { - runIDPattern := regexp.MustCompile(`^\d{8}-\d{6}-[0-9a-f]{16}$`) +// TestDeployAndCollectGeneratesRunIDWithoutMutatingConfig confirms +// DeployAndCollect resolves an empty RunID to a freshly generated one before +// it is folded into the internal staging ConfigMap's name, AND that it leaves +// the caller's *AgentConfig untouched while doing so. +// +// The non-mutation half is the load-bearing one. Writing the generated ID +// back into the caller's config would turn a caller who reuses one config +// pointer for a second run into a caller pinning a duplicate RunID — the one +// state ADR-020 declares unsupported — and run 2 would hard-fail +// ErrCodeInternal on the first still-existing run-scoped object. In-tree +// callers reach this through the pkg/client/v1 facade, which builds a fresh +// internal config per call, but pkg/snapshotter is public. +// +// The generated value is observed through the "snapshot agent run" log line, +// which is emitted just before the malformed-Output rejection below fires; +// there is no other seam that does not require cluster access. +func TestDeployAndCollectGeneratesRunIDWithoutMutatingConfig(t *testing.T) { + runIDPattern := regexp.MustCompile(`runID=(\d{8}-\d{6}-[0-9a-f]{16})`) + + var logs bytes.Buffer + previousLogger := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug}))) + t.Cleanup(func() { slog.SetDefault(previousLogger) }) cfg := &AgentConfig{ Namespace: "default", Kubeconfig: filepath.Join(t.TempDir(), "does-not-exist.kubeconfig"), - Output: "cm://aicr-snapshot", // malformed: no namespace — rejected after RunID defaulting + Output: "cm://aicr-snapshot", // malformed: no namespace — rejected after RunID resolution } if cfg.RunID != "" { t.Fatalf("test precondition: cfg.RunID = %q, want empty", cfg.RunID) @@ -1256,8 +1278,14 @@ func TestDeployAndCollectDefaultsRunID(t *testing.T) { t.Fatal("DeployAndCollect() = nil error, want rejection from the malformed Output") } - if !runIDPattern.MatchString(cfg.RunID) { - t.Errorf("cfg.RunID = %q after DeployAndCollect, want it filled with a generated ID matching %s", - cfg.RunID, runIDPattern) + if !runIDPattern.MatchString(logs.String()) { + t.Errorf("no generated run ID matching %s in log output; DeployAndCollect must "+ + "resolve an empty RunID with runid.Generate(). Logs:\n%s", runIDPattern, logs.String()) + } + + if cfg.RunID != "" { + t.Errorf("cfg.RunID = %q after DeployAndCollect, want it left empty — the generated ID "+ + "must stay in a local so a caller reusing this config for a second run does not "+ + "silently pin a duplicate RunID", cfg.RunID) } } From 85d94502fc33e4b9bc48fbce7efd4bd7c97d4c96 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Mon, 24 Aug 2026 13:01:30 -0700 Subject: [PATCH 30/56] docs: correct agent label coverage, name prefixes, and validate flag targets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four documentation claims did not match the code. Label coverage. pkg/k8s/agent/doc.go and docs/user/agent-deployment.md both asserted that every created object — explicitly counting the staging ConfigMap — carries managed-by=aicr, component=snapshot-agent and aicr.run/run-id. The staging ConfigMap carries none of those: it is written by serializer.ConfigMapWriter, which stamps only app.kubernetes.io/name, app.kubernetes.io/component (the header kind) and app.kubernetes.io/version. An earlier review decided not to add the labels at the serializer, because that writer also produces the user's delivered cm:// artifact and stamping the sweep key on a never-delete object is a hazard — so the documentation is corrected and the serializer is left alone. The agent-deployment troubleshooting recipe now addresses the staging ConfigMap by its run-scoped name. Shipped config examples. ADR-020 decision 8 requires examples pinning jobName/serviceAccountName to be updated; four blocks still pinned them (two in cli-config.md, two in cli-reference.md), and one contradicted a neighbouring row this branch already updated — pinning serviceAccountName: aicr for validate while the reference gives the default as aicr-validate. The pins are dropped and both agent.* schema rows now state the fields are optional prefixes with the run ID appended. NameBase godoc. The pkg/client/v1 copy repeated the claim that NameBase "has no effect once either of those is set". The fallback is per name. Validate flag targets. The aicr validate rows for --job-name / --service-account-name attributed them to "the validation Job" and its ServiceAccount. Both feed only the optional live snapshot-capture agent: the validator Jobs are named aicr-- and their ServiceAccount aicr-validator-, neither influenced by these flags, and both flags are inert when --snapshot is supplied. Refs: ADR-020 Signed-off-by: Alex Yuskauskas --- docs/user/agent-deployment.md | 20 ++++++++++++++++---- docs/user/cli-config.md | 12 ++++++------ docs/user/cli-reference.md | 13 +++++++------ pkg/client/v1/types.go | 10 +++++++--- pkg/k8s/agent/doc.go | 33 +++++++++++++++++++++++++-------- 5 files changed, 61 insertions(+), 27 deletions(-) diff --git a/docs/user/agent-deployment.md b/docs/user/agent-deployment.md index db96136ff..cfca9c47b 100644 --- a/docs/user/agent-deployment.md +++ b/docs/user/agent-deployment.md @@ -333,8 +333,12 @@ Check RBAC permissions. The ServiceAccount name is run-scoped (`aicr-`), Select it by run ID, not by position: with concurrent snapshot runs the namespace holds one ServiceAccount per run, and `.items[0]` would pick an arbitrary one — so the checks below could report on a healthy run while you are debugging a failed one. -The CLI logs the run ID when it starts (`snapshot agent run: runID=...`), and every -resource the run created carries it as the `aicr.run/run-id` label: +The CLI logs the run ID when it starts (`snapshot agent run: runID=...`), and the +Job, its pods, and its RBAC resources all carry it as the `aicr.run/run-id` +label. (The staging ConfigMap is the exception — it is written from inside the +pod and carries only `app.kubernetes.io/name`, `app.kubernetes.io/component` and +`app.kubernetes.io/version`; find it by its run-scoped name instead, see +[Job Completes but No Output](#job-completes-but-no-output).) ```shell # From the failing Job (or use the runID the CLI logged at start). @@ -381,10 +385,18 @@ Check ConfigMap and container logs: # "-o cm:///", the agent stages its result in a run-scoped # ConfigMap named "aicr-agent-snapshot-" that cleanup deletes when # the run ends — pass --no-cleanup to keep it around for inspection. -kubectl get configmap -n gpu-operator -l app.kubernetes.io/name=aicr +# +# This object is written by the in-pod agent, not by the CLI, so it does NOT +# carry the aicr.run/run-id label the Job and RBAC resources do. Address it by +# name: with concurrent runs, the label selector below matches every run's +# staging ConfigMap plus any delivered cm:// artifact in the namespace. +kubectl get configmap -n gpu-operator aicr-agent-snapshot-$RUN_ID # View ConfigMap contents -kubectl get configmap -n gpu-operator -l app.kubernetes.io/name=aicr -o yaml +kubectl get configmap -n gpu-operator aicr-agent-snapshot-$RUN_ID -o yaml + +# Or list every aicr-written ConfigMap in the namespace (all runs) +kubectl get configmap -n gpu-operator -l app.kubernetes.io/name=aicr # View pod logs for errors kubectl logs -n gpu-operator -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent diff --git a/docs/user/cli-config.md b/docs/user/cli-config.md index 85465ef1a..9f22abf21 100644 --- a/docs/user/cli-config.md +++ b/docs/user/cli-config.md @@ -98,8 +98,8 @@ spec: namespace: aicr-validation image: "" # default: ghcr.io/nvidia/aicr:latest imagePullSecrets: [] - jobName: aicr - serviceAccountName: aicr + # jobName / serviceAccountName are optional PREFIXES, not names — the + # run ID is always appended. Omit them to take the defaults. nodeSelector: nodeGroup: gpu-worker tolerations: @@ -181,8 +181,8 @@ spec: namespace: aicr-validation image: "" imagePullSecrets: [] - jobName: aicr - serviceAccountName: aicr + # Optional prefixes; omitted here so the defaults apply + # (jobName: aicr-validate, serviceAccountName: aicr). nodeSelector: nodeGroup: gpu-worker tolerations: @@ -229,7 +229,7 @@ produced from the live cluster. | `output.path` | string | Output file path (same as `-o`) | | `output.format` | string | `yaml` \| `json` \| `table` | | `output.template` | string | Optional Go template path | -| `agent.*` | object | In-cluster capture Job pod: `namespace`, `image`, `imagePullSecrets`, `jobName`, `serviceAccountName`, `nodeSelector`, `tolerations`, `requireGpu`, `runtimeClassName` (mutually exclusive with `requireGpu`), `os`, `requests`, `limits`. Mirrors `spec.validate.agent` so one file pins matching placement for both | +| `agent.*` | object | In-cluster capture Job pod: `namespace`, `image`, `imagePullSecrets`, `jobName`, `serviceAccountName`, `nodeSelector`, `tolerations`, `requireGpu`, `runtimeClassName` (mutually exclusive with `requireGpu`), `os`, `requests`, `limits`. `jobName` and `serviceAccountName` are optional **prefixes**, not exact names — the run ID is appended (`-`), so omit them unless you need a custom prefix. Mirrors `spec.validate.agent` so one file pins matching placement for both | | `execution.timeout` | duration string | e.g. `5m` | | `execution.noCleanup` | bool | Keep the capture Job after completion | | `execution.privileged` | bool (tri-state) | Set `false` for PSS-restricted namespaces | @@ -289,7 +289,7 @@ Inputs to `aicr validate`. | Field | Type | Notes | |-------|------|-------| | `input.recipe` / `.snapshot` | string | Recipe + snapshot to validate | -| `agent.*` | object | In-cluster validation Job pod; same fields and nil-vs-empty semantics as `spec.snapshot.agent` (minus `runtimeClassName`/`os`/`requests`/`limits`) | +| `agent.*` | object | The **live snapshot-capture** Job pod `aicr validate` deploys when `input.snapshot` is empty; same fields and nil-vs-empty semantics as `spec.snapshot.agent` (minus `runtimeClassName`/`os`/`requests`/`limits`). `jobName` and `serviceAccountName` are optional prefixes with the run ID appended (defaults: both `aicr-validate`); they do not name the validator Jobs | | `execution.phases` | []string | e.g. `[deployment, conformance, performance]` | | `execution.failOnError` | bool (tri-state) | Absent = CLI default (`true`); explicit `false` opts out | | `execution.failFast` | bool (tri-state) | Stop after the first failed phase | diff --git a/docs/user/cli-reference.md b/docs/user/cli-reference.md index 8b025f525..df29d77b3 100644 --- a/docs/user/cli-reference.md +++ b/docs/user/cli-reference.md @@ -208,8 +208,8 @@ spec: namespace: aicr-validation image: "" # default: ghcr.io/nvidia/aicr:latest imagePullSecrets: [] - jobName: aicr - serviceAccountName: aicr + # jobName / serviceAccountName are optional PREFIXES, not names — the + # run ID is always appended. Omit them to take the defaults. nodeSelector: nodeGroup: gpu-worker tolerations: @@ -1031,8 +1031,8 @@ aicr validate [flags] | `--namespace` | `-n` | string | aicr-validation | Kubernetes namespace for validation Job deployment | | `--image` | | string | ghcr.io/nvidia/aicr:latest | Container image for validation Job | | `--image-pull-secret` | | string[] | | Image pull secrets for private registries (repeatable) | -| `--job-name` | | string | aicr-validate | Prefix for the validation Job name; the run ID is always appended (`-`) | -| `--service-account-name` | | string | aicr-validate | Prefix for the validation Job's ServiceAccount name; the run ID is always appended (`-`) | +| `--job-name` | | string | aicr-validate | Prefix for the **live snapshot-capture agent's** Job name; the run ID is always appended (`-`). Inert when `--snapshot` is supplied — no agent is deployed. Does not name the validator Jobs (`aicr--`) | +| `--service-account-name` | | string | aicr-validate | Prefix for the **live snapshot-capture agent's** ServiceAccount, Role, and RoleBinding; the run ID is always appended (`-`). Inert when `--snapshot` is supplied. Does not name the validator Jobs' ServiceAccount (`aicr-validator-`) | | `--node-selector` | | string[] | | Override GPU node selection for the live snapshot agent (when `--snapshot` is omitted) and inner validation workloads. Replaces platform-specific selectors (e.g., `cloud.google.com/gke-accelerator`, `node.kubernetes.io/instance-type`) on inner workloads like NCCL benchmark pods. Use when GPU nodes have non-standard labels. Does not affect the validator orchestrator Job. (format: key=value, repeatable) | | `--toleration` | | string[] | | Override tolerations for the live snapshot agent (when `--snapshot` is omitted) and inner validation workloads. When omitted, the snapshot agent tolerates all taints. Does not affect the validator orchestrator Job. (format: key=value:effect, repeatable) | | `--timeout` | | duration | 5m | Timeout for validation Job completion | @@ -1248,8 +1248,9 @@ spec: namespace: aicr-validation image: ghcr.io/nvidia/aicr:v0.19.0 imagePullSecrets: [registry-secret] - jobName: aicr-validate - serviceAccountName: aicr + # Optional prefixes for the live-capture agent, not names — the run ID + # is always appended. Omitted here so the defaults apply + # (both aicr-validate). nodeSelector: my-org/gpu-pool: "true" tolerations: # [] clears the live snapshot agent's tolerate-all default diff --git a/pkg/client/v1/types.go b/pkg/client/v1/types.go index 1b0cbfb52..55c469fd0 100644 --- a/pkg/client/v1/types.go +++ b/pkg/client/v1/types.go @@ -186,9 +186,13 @@ type AgentConfig struct { // snapshot agent and its validator Jobs the same RunID. RunID string - // NameBase prefixes generated Job/ServiceAccount/RBAC names when - // JobName and ServiceAccountName are left empty; it has no effect - // once either of those is set. Defaults to "aicr" when also empty. + // NameBase prefixes generated Job/ServiceAccount/RBAC names. The + // fallback is per name, not all-or-nothing: the Job uses JobName when + // set and NameBase otherwise, while the ServiceAccount, Role and + // RoleBinding use ServiceAccountName when set and NameBase otherwise. + // Setting only one of the two therefore leaves NameBase governing the + // other. Defaults to "aicr" when also empty. + // // JobName and ServiceAccountName themselves are optional prefixes, // not required names — RunID is appended to whichever prefix // applies, so the deployed object names are always run-scoped. diff --git a/pkg/k8s/agent/doc.go b/pkg/k8s/agent/doc.go index 5c4c36bdc..c66fbe3d5 100644 --- a/pkg/k8s/agent/doc.go +++ b/pkg/k8s/agent/doc.go @@ -46,12 +46,22 @@ Two objects are deliberately NOT run-scoped: artifact. It is written on purpose and never deleted (Config.OwnsOutputConfigMap is false for it). -Every created object carries app.kubernetes.io/name=aicr, -app.kubernetes.io/managed-by=aicr, +Every object this package itself creates — the Job, ServiceAccount, Role, +RoleBinding, ClusterRole, and ClusterRoleBinding — carries +app.kubernetes.io/name=aicr, app.kubernetes.io/managed-by=aicr, app.kubernetes.io/component=snapshot-agent, and aicr.run/run-id=, on the Job's pod template as well as the Job itself. Select agent pods across runs with the component label; the Job name changes every run. +The staging ConfigMap is the exception: it is written from inside the pod by +pkg/serializer's ConfigMap writer, which stamps app.kubernetes.io/name=aicr, +app.kubernetes.io/component= and app.kubernetes.io/version — not +managed-by and not the run-ID label. That writer also produces the user's +delivered cm:// artifact, so it deliberately does not stamp the run-ID sweep +key onto an object this package must never delete. Run scoping for the staging +ConfigMap comes from its name (see StagingConfigMapName), which is what both +Cleanup paths key on. + Job and Pod lifecycle waits use the Kubernetes watch API (not polling) for efficiency. Pod selection narrows by label and then authorizes the candidate against the controlling ownerReference carrying the recorded Job UID, since pod @@ -59,12 +69,19 @@ labels are writable by anything that can update pods in the namespace. # Cleanup -The Deployer records (kind, name, UID) for each object it successfully creates. -Cleanup deletes exactly that set, passing the recorded UID as a -metav1.Preconditions so a same-named object belonging to another run is never -collected; a UID mismatch (Conflict) and a NotFound are both treated as -success. Cleanup also runs on the Deploy failure path, which is why it is -scoped to what was created rather than to configured names. +The Deployer records (kind, name) immediately before each Create and writes the +returned UID onto that entry on success. Cleanup deletes exactly that set, +passing the recorded UID as a metav1.Preconditions so a same-named object +belonging to another run is never collected; a UID mismatch (Conflict) and a +NotFound are both treated as success. Cleanup also runs on the Deploy failure +path, which is why it is scoped to what was created rather than to configured +names. + +Recording before the Create is what keeps a lost Create response from orphaning +an object forever: if the apiserver commits the create but the response never +arrives, the entry is already in the set and Cleanup deletes it by its +(run-unique) name with no UID precondition. The one response that proves the +object is not ours — AlreadyExists — discards the entry again. The staging ConfigMap is written by the in-pod agent, so its UID is observed when GetSnapshot reads it. When the run owns that ConfigMap and failed before From 7828018b8b407e3268c46f8eaf594a7dc5cbbb48 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Mon, 24 Aug 2026 13:01:31 -0700 Subject: [PATCH 31/56] fix(cleanup): restore the name-based sweep for pre-ADR-020 agent leftovers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An earlier commit on this branch replaced four name-based deletes (job, sa, role, rolebinding named "aicr" in gpu-operator) with a label selector on app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent. Pre-branch objects never carried those labels: origin/main's ensureServiceAccount, ensureRole and ensureRoleBinding set no Labels at all, and its Job carried only app.kubernetes.io/name. So the block, still commented "Legacy on-cluster agent leftovers from the older deployment pattern", swept only current-run objects and no legacy one. Both sweeps are now present. They cannot collide: a run-scoped name is always "aicr-", never the bare "aicr". The comment describes what the code actually does. The aicr-node-reader ClusterRole/ClusterRoleBinding lines are deliberately untouched — whether to add a name-based delete for the legacy pair is a separate decision. Refs: ADR-020 Signed-off-by: Alex Yuskauskas --- tools/cleanup | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/tools/cleanup b/tools/cleanup index 034aaf874..e3ae584e7 100755 --- a/tools/cleanup +++ b/tools/cleanup @@ -390,14 +390,28 @@ if ! $DRY_RUN; then | xargs -r kubectl delete --ignore-not-found || true fi kc delete ns aicr-validation --ignore-not-found --wait=false -# Legacy on-cluster agent leftovers from the older deployment pattern. -# Run-scoped agent resources (Job/SA/Role/RoleBinding) are named -# "aicr-", so a fixed-name delete never matches current runs; select -# by the label every one of them carries instead. +# On-cluster snapshot-agent leftovers, current and legacy. Both sweeps are +# needed and neither subsumes the other: +# +# - Current (ADR-020) runs name their Job/SA/Role/RoleBinding +# "aicr-", so no fixed-name delete matches them; they are found by +# the label set every one of them carries. +# - Pre-ADR-020 runs used the fixed name "aicr" and carried none of those +# labels — the Job had only app.kubernetes.io/name, and the SA, Role and +# RoleBinding had no labels at all — so the label selector above misses +# them entirely. The name-based deletes below are the only thing that +# collects them. +# +# The two cannot collide: a run-scoped name is always "aicr-", never +# the bare "aicr". kc -n gpu-operator delete job -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent --ignore-not-found kc -n gpu-operator delete sa -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent --ignore-not-found kc -n gpu-operator delete role -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent --ignore-not-found kc -n gpu-operator delete rolebinding -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent --ignore-not-found +kc -n gpu-operator delete job aicr --ignore-not-found +kc -n gpu-operator delete sa aicr --ignore-not-found +kc -n gpu-operator delete role aicr --ignore-not-found +kc -n gpu-operator delete rolebinding aicr --ignore-not-found # Phase 3: Component CRDs. # Helm does NOT remove CRDs on uninstall — they must be deleted manually or a From 9620e98446f658482c995907404b768f2354a22f Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Mon, 24 Aug 2026 13:07:04 -0700 Subject: [PATCH 32/56] fix(cleanup): remove the pre-ADR-020 unlabeled cluster RBAC pair Signed-off-by: Alex Yuskauskas --- docs/user/agent-deployment.md | 2 +- tools/cleanup | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/user/agent-deployment.md b/docs/user/agent-deployment.md index cfca9c47b..3494c67de 100644 --- a/docs/user/agent-deployment.md +++ b/docs/user/agent-deployment.md @@ -50,7 +50,7 @@ metadata: namespace: gpu-operator labels: app.kubernetes.io/name: aicr - app.kubernetes.io/component: snapshot + app.kubernetes.io/component: Snapshot app.kubernetes.io/version: data: snapshot.yaml: | # Complete snapshot YAML diff --git a/tools/cleanup b/tools/cleanup index e3ae584e7..1cf26a414 100755 --- a/tools/cleanup +++ b/tools/cleanup @@ -412,6 +412,17 @@ kc -n gpu-operator delete job aicr --ignore-not-found kc -n gpu-operator delete sa aicr --ignore-not-found kc -n gpu-operator delete role aicr --ignore-not-found kc -n gpu-operator delete rolebinding aicr --ignore-not-found +# The pre-ADR-020 cluster-scoped pair, for the same reason: it was named +# literally "aicr-node-reader" and carried no labels, so every selector above +# misses it and the name-grep fallback ('/aicr-validator-|/aicr$') does not +# match it either. Nothing else in the tree deletes it now that run-owned +# names are "aicr-node-reader-" — a later run no longer reclaims the +# bare name — so without this it is a permanent orphan. That matters because a +# leftover from `--discover-network` carries mutating grants (nodes: patch, +# pods/exec: create, CRD and namespace create/delete). The exact-name delete +# cannot touch a live run, whose name always carries a run-ID suffix. +kc delete clusterrole aicr-node-reader --ignore-not-found +kc delete clusterrolebinding aicr-node-reader --ignore-not-found # Phase 3: Component CRDs. # Helm does NOT remove CRDs on uninstall — they must be deleted manually or a From e8f859b40572dd4c98b28c88ca749594b767c52d Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Mon, 24 Aug 2026 15:35:17 -0700 Subject: [PATCH 33/56] test(agent): drop the always-constant namespace param from watchDeployer Signed-off-by: Alex Yuskauskas --- pkg/k8s/agent/wait_test.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pkg/k8s/agent/wait_test.go b/pkg/k8s/agent/wait_test.go index 19baab173..af72d75e2 100644 --- a/pkg/k8s/agent/wait_test.go +++ b/pkg/k8s/agent/wait_test.go @@ -535,11 +535,14 @@ func runLabeledPod(d *Deployer, name string, ownerUID types.UID) *corev1.Pod { } } +// watchNamespace is the namespace every watch-path subtest deploys into. +const watchNamespace = "watch-ns" + // watchDeployer returns a Deployer for the watch-path tests with run A's Job // UID already recorded, so jobUID() is non-zero and the ownership checks in // findOrWatchPodName are actually exercised rather than skipped. -func watchDeployer(client *fake.Clientset, namespace string) *Deployer { - d := NewDeployer(client, Config{Namespace: namespace, RunID: testRunID}) +func watchDeployer(client *fake.Clientset) *Deployer { + d := NewDeployer(client, Config{Namespace: watchNamespace, RunID: testRunID}) d.recordCreated(kindJob, d.jobName(), watchJobUIDA) return d } @@ -561,14 +564,13 @@ func watchDeployer(client *fake.Clientset, namespace string) *Deployer { // what authorizes selection. The List-side selector filtering IS honored by // the fake, so the re-List subtests exercise it for real. func TestFindOrWatchPodNameAuthorizesByJobOwnership(t *testing.T) { - const ns = "watch-ns" t.Run("watch event for another run's Job is skipped", func(t *testing.T) { client := fake.NewClientset() w := watch.NewRaceFreeFake() client.PrependWatchReactor("pods", k8stesting.DefaultWatchReactor(w, nil)) - d := watchDeployer(client, ns) + d := watchDeployer(client) // The imposter arrives FIRST: a watch loop that returned the first // event passing the label filter would take it and never see the @@ -594,7 +596,7 @@ func TestFindOrWatchPodNameAuthorizesByJobOwnership(t *testing.T) { w := watch.NewRaceFreeFake() client.PrependWatchReactor("pods", k8stesting.DefaultWatchReactor(w, nil)) - d := watchDeployer(client, ns) + d := watchDeployer(client) // Labels alone, no controlling ownerReference at all — the shape a // caller with pods/update in the namespace can produce directly. @@ -617,7 +619,7 @@ func TestFindOrWatchPodNameAuthorizesByJobOwnership(t *testing.T) { t.Run("closed watch channel re-Lists and authorizes the re-Listed pod", func(t *testing.T) { client := fake.NewClientset() - d := watchDeployer(client, ns) + d := watchDeployer(client) // The fast-path List must miss so the watch is reached at all; the // re-List after the channel closes then returns both pods. @@ -651,7 +653,7 @@ func TestFindOrWatchPodNameAuthorizesByJobOwnership(t *testing.T) { t.Run("closed watch channel with only a foreign pod fails closed", func(t *testing.T) { client := fake.NewClientset() - d := watchDeployer(client, ns) + d := watchDeployer(client) installStagedPodLister(client, func(call int) []corev1.Pod { if call == 1 { @@ -679,7 +681,7 @@ func TestFindOrWatchPodNameAuthorizesByJobOwnership(t *testing.T) { t.Run("closed watch channel surfaces a failed re-List", func(t *testing.T) { client := fake.NewClientset() - d := watchDeployer(client, ns) + d := watchDeployer(client) var mu sync.Mutex var calls int From d68edf5f44af9cb6c5b736e39683e4f670fe58c7 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Mon, 24 Aug 2026 15:40:38 -0700 Subject: [PATCH 34/56] test(agent): drop the leading blank line in the watch-path test Signed-off-by: Alex Yuskauskas --- pkg/k8s/agent/wait_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/k8s/agent/wait_test.go b/pkg/k8s/agent/wait_test.go index af72d75e2..8b66fefa3 100644 --- a/pkg/k8s/agent/wait_test.go +++ b/pkg/k8s/agent/wait_test.go @@ -564,7 +564,6 @@ func watchDeployer(client *fake.Clientset) *Deployer { // what authorizes selection. The List-side selector filtering IS honored by // the fake, so the re-List subtests exercise it for real. func TestFindOrWatchPodNameAuthorizesByJobOwnership(t *testing.T) { - t.Run("watch event for another run's Job is skipped", func(t *testing.T) { client := fake.NewClientset() w := watch.NewRaceFreeFake() From bffa50cc5c533c60591cf309751ec5ce86c6b607 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Mon, 24 Aug 2026 16:24:36 -0700 Subject: [PATCH 35/56] test(agent): derive name-limit boundaries from defaults.MaxK8sNameLength The name-helper tests hard-coded 63 and the 30-character prefix budget it implies for a 32-character run ID. Both are consequences of defaults.MaxK8sNameLength, so raising or lowering that constant would leave the assertions passing against boundaries the code no longer uses. Derive the budget the same way nameWithRunID does (MaxK8sNameLength minus the run ID minus the separator) and express the DNS-1123 label ceiling cases in terms of the constant too. Signed-off-by: Alex Yuskauskas --- pkg/k8s/agent/names_test.go | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/pkg/k8s/agent/names_test.go b/pkg/k8s/agent/names_test.go index 579a1b557..412a3d846 100644 --- a/pkg/k8s/agent/names_test.go +++ b/pkg/k8s/agent/names_test.go @@ -20,12 +20,19 @@ import ( "strings" "testing" + "github.com/NVIDIA/aicr/pkg/defaults" "github.com/NVIDIA/aicr/pkg/errors" "k8s.io/client-go/kubernetes/fake" ) func TestNameWithRunID(t *testing.T) { - const runID = "20260821-142233-9f3a1c0b7e2d4a55" // 32 chars + const runID = "20260821-142233-9f3a1c0b7e2d4a55" + + // nameWithRunID budgets the prefix against defaults.MaxK8sNameLength, + // reserving len(runID) plus the "-" separator. Derive the budget the same + // way rather than hard-coding it, so raising or lowering the constant + // moves these boundaries with it instead of silently invalidating them. + budget := defaults.MaxK8sNameLength - len(runID) - 1 tests := []struct { name string @@ -34,9 +41,9 @@ func TestNameWithRunID(t *testing.T) { want string }{ {"short prefix", "aicr", runID, "aicr-" + runID}, - {"exactly at budget", strings.Repeat("a", 30), runID, strings.Repeat("a", 30) + "-" + runID}, - {"over budget truncates", strings.Repeat("b", 40), runID, strings.Repeat("b", 30) + "-" + runID}, - {"trailing dash trimmed", strings.Repeat("c", 29) + "-", runID, strings.Repeat("c", 29) + "-" + runID}, + {"exactly at budget", strings.Repeat("a", budget), runID, strings.Repeat("a", budget) + "-" + runID}, + {"over budget truncates", strings.Repeat("b", budget+10), runID, strings.Repeat("b", budget) + "-" + runID}, + {"trailing dash trimmed", strings.Repeat("c", budget-1) + "-", runID, strings.Repeat("c", budget-1) + "-" + runID}, {"empty prefix", "", runID, runID}, // A zero-value Config.RunID (only reachable from an SDK caller // constructing a Config directly) must fall back to the bare prefix, @@ -52,8 +59,8 @@ func TestNameWithRunID(t *testing.T) { if got != tt.want { t.Errorf("nameWithRunID(%q, %q) = %q, want %q", tt.prefix, tt.runID, got, tt.want) } - if len(got) > 63 { - t.Errorf("len = %d, exceeds 63-char ceiling", len(got)) + if len(got) > defaults.MaxK8sNameLength { + t.Errorf("len = %d, exceeds the %d-char ceiling", len(got), defaults.MaxK8sNameLength) } if strings.HasSuffix(got, "-") { t.Errorf("nameWithRunID(%q, %q) = %q, ends in a trailing separator (invalid Kubernetes name)", tt.prefix, tt.runID, got) @@ -208,14 +215,14 @@ func TestValidateRunID(t *testing.T) { }{ {"well-formed generated run ID", "20260821-142233-9f3a1c0b7e2d4a55", false}, {"single character", "a", false}, - {"exactly at the DNS-1123 label ceiling", strings.Repeat("a", 63), false}, + {"exactly at the DNS-1123 label ceiling", strings.Repeat("a", defaults.MaxK8sNameLength), false}, {"empty", "", true}, {"whitespace", " ", true}, {"embedded slash", "build/42", true}, {"leading dash", "-build", true}, {"trailing dash", "build-", true}, {"uppercase", "Build42", true}, - {"one over the DNS-1123 label ceiling", strings.Repeat("a", 64), true}, + {"one over the DNS-1123 label ceiling", strings.Repeat("a", defaults.MaxK8sNameLength+1), true}, } for _, tt := range tests { @@ -248,7 +255,7 @@ func TestValidateRunID(t *testing.T) { // rejected run leaves no partially-created RBAC behind. func TestDeployRejectsInvalidRunIDBeforeCreatingAnything(t *testing.T) { ctx := context.Background() - for _, runID := range []string{"", " ", "build/42", "-build", "build-", "Build42", strings.Repeat("a", 64)} { + for _, runID := range []string{"", " ", "build/42", "-build", "build-", "Build42", strings.Repeat("a", defaults.MaxK8sNameLength+1)} { t.Run(runID, func(t *testing.T) { clientset := fake.NewClientset() d := NewDeployer(clientset, Config{Namespace: "test-ns", Image: "aicr:test", RunID: runID}) From d1fb564febcd59b92bc43877933876e9c3a5de95 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Mon, 24 Aug 2026 16:25:02 -0700 Subject: [PATCH 36/56] fix(agent): reject an invalid resolved object name before Deploy writes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deploy validated Config.RunID but not the names it is folded into. NameBase, JobName and ServiceAccountName are caller-supplied too, so an underscore- bearing prefix yields "agent_-" — a metadata.name the apiserver rejects. That surfaced as an opaque "Invalid value: metadata.name" from partway through the ensure* chain, after some run-owned objects already existed. Validate the resolved jobName() and saName() in the same pre-flight, ahead of CheckPermissions and any write, and fail with ErrCodeInvalidRequest naming the Config field at fault, the prefix it held, and the name that prefix produced. saName() also names the Role and RoleBinding, so one check covers three objects; the ClusterRole and staging ConfigMap prefixes are package constants and carry no caller input beyond the run ID. Signed-off-by: Alex Yuskauskas --- pkg/k8s/agent/deployer.go | 14 +-- pkg/k8s/agent/names.go | 79 ++++++++++++++++ pkg/k8s/agent/names_test.go | 184 ++++++++++++++++++++++++++++++++++++ 3 files changed, 270 insertions(+), 7 deletions(-) diff --git a/pkg/k8s/agent/deployer.go b/pkg/k8s/agent/deployer.go index 7afda73ee..b0c043a1f 100644 --- a/pkg/k8s/agent/deployer.go +++ b/pkg/k8s/agent/deployer.go @@ -32,13 +32,13 @@ import ( // Deploy deploys the agent with all required resources (RBAC + Job). // This is the main entry point that orchestrates the deployment. func (d *Deployer) Deploy(ctx context.Context) error { - // Pre-flight, ahead of any cluster call: reject a run ID that cannot - // be folded into a valid object name. Every object created below is - // named "-", so validating here is what keeps an - // invalid value from surfacing as an apiserver - // "Invalid value: metadata.name" partway through the ensure* chain, - // with some objects already created. - if err := d.validateRunID(); err != nil { + // Pre-flight, ahead of any cluster call: reject a run ID — or a + // caller-supplied name prefix — that cannot be folded into a valid + // object name. Every object created below is named "-", + // so validating here is what keeps an invalid value from surfacing as + // an apiserver "Invalid value: metadata.name" partway through the + // ensure* chain, with some objects already created. + if err := d.validateNames(); err != nil { return err } diff --git a/pkg/k8s/agent/names.go b/pkg/k8s/agent/names.go index aa7df4ce4..1958a0520 100644 --- a/pkg/k8s/agent/names.go +++ b/pkg/k8s/agent/names.go @@ -109,6 +109,85 @@ func (d *Deployer) validateRunID() error { return nil } +// resolvedName is one caller-influenced object name this Deployer would +// create, carried together with the Config field that supplied its prefix. +// Validation reports the field rather than only the derived string, because +// the field is what a caller can actually change. +type resolvedName struct { + field string // Config field the prefix came from + prefix string // the prefix value that field held + value string // the resolved object name, "-" + // objects names what value is the name of, so a rejection states the + // blast radius: saName() also names the Role and the RoleBinding. + objects string +} + +// resolvedNames returns every object name this Deployer builds from a +// caller-supplied prefix, paired with its source field. +// +// The ClusterRole/ClusterRoleBinding and staging ConfigMap names are +// deliberately absent: their prefixes are package constants +// (staticClusterRoleName, staticStagingConfigMapName), so the only +// caller-supplied input they carry is the run ID, which validateRunID +// already covers. +func (d *Deployer) resolvedNames() []resolvedName { + jobField, jobPrefix := "Config.NameBase", d.base() + if d.config.JobName != "" { + jobField, jobPrefix = "Config.JobName", d.config.JobName + } + saField, saPrefix := "Config.NameBase", d.base() + if d.config.ServiceAccountName != "" { + saField, saPrefix = "Config.ServiceAccountName", d.config.ServiceAccountName + } + return []resolvedName{ + {field: jobField, prefix: jobPrefix, value: d.jobName(), objects: "Job"}, + {field: saField, prefix: saPrefix, value: d.saName(), objects: "ServiceAccount, Role and RoleBinding"}, + } +} + +// validateResolvedNames rejects a generated object name Kubernetes would +// refuse. validateRunID covers one half of every run-owned name; this covers +// the other. NameBase, JobName and ServiceAccountName are caller-supplied +// too, and a prefix such as "agent_" yields "agent_-" — which the +// apiserver rejects with an opaque "Invalid value: metadata.name" from +// partway through Deploy's ensure* chain, after some objects already exist. +// +// The constraint is DNS-1123 subdomain, which is what the apiserver enforces +// on Job and ServiceAccount names, plus the narrower defaults.MaxK8sNameLength +// ceiling this package budgets against (a Job name also becomes the +// batch.kubernetes.io/job-name label value on every Pod the Job creates, and +// label values share that ceiling). nameWithRunID truncates the prefix to +// that budget, so the length branch is reachable only for a run ID that is +// itself over-long — which validateRunID rejects first when both run under +// validateNames, but not when this method is called on its own. +func (d *Deployer) validateResolvedNames() error { + for _, n := range d.resolvedNames() { + problems := validation.IsDNS1123Subdomain(n.value) + if len(problems) == 0 && len(n.value) > defaults.MaxK8sNameLength { + problems = []string{fmt.Sprintf("must be no more than %d characters", defaults.MaxK8sNameLength)} + } + if len(problems) == 0 { + continue + } + return errors.NewWithContext(errors.ErrCodeInvalidRequest, + fmt.Sprintf("%s %q yields the %s name %q, which is not a valid Kubernetes object name: %s", + n.field, n.prefix, n.objects, n.value, strings.Join(problems, "; ")), + map[string]any{"field": n.field, "value": n.prefix, "resolvedName": n.value}) + } + return nil +} + +// validateNames is Deploy's naming pre-flight: it rejects both halves of a +// run-owned name — the run ID and the resolved object names built from the +// caller's prefixes — before any cluster call is made, so an invalid value +// can never leave a partially-created deployment behind. +func (d *Deployer) validateNames() error { + if err := d.validateRunID(); err != nil { + return err + } + return d.validateResolvedNames() +} + // base returns the configured name base, defaulting to "aicr" when unset. func (d *Deployer) base() string { if d.config.NameBase != "" { diff --git a/pkg/k8s/agent/names_test.go b/pkg/k8s/agent/names_test.go index 412a3d846..979827143 100644 --- a/pkg/k8s/agent/names_test.go +++ b/pkg/k8s/agent/names_test.go @@ -276,3 +276,187 @@ func TestDeployRejectsInvalidRunIDBeforeCreatingAnything(t *testing.T) { }) } } + +// TestValidateResolvedNames covers the other half of the naming gate: +// Config.RunID may be well-formed while the caller-supplied prefix it is +// appended to is not. Every case here calls validateResolvedNames directly +// rather than validateNames, so the over-long run ID case can reach the +// length branch that validateRunID would otherwise short-circuit. +func TestValidateResolvedNames(t *testing.T) { + const runID = "20260821-142233-9f3a1c0b7e2d4a55" + + // A prefix that leaves no room at all: nameWithRunID's budget floors at + // zero, so the resolved name is the bare (over-long) run ID. + overLongRunID := strings.Repeat("a", defaults.MaxK8sNameLength+17) + + tests := []struct { + name string + // config carries only the naming fields; RunID defaults to runID + // unless the case overrides it. + config Config + // wantErr is the substring the message must name — the Config field + // at fault. Empty means the names must be accepted. + wantField string + // wantValue is echoed in the message alongside the field so an + // operator sees what to change. Only checked when wantField is set. + wantValue string + }{ + { + name: "default prefixes are valid", + config: Config{RunID: runID}, + }, + { + name: "explicit prefixes are valid", + config: Config{JobName: "my-job", ServiceAccountName: "my-sa", RunID: runID}, + }, + { + name: "a dot is a legal DNS-1123 subdomain character", + config: Config{JobName: "aicr.agent", RunID: runID}, + }, + { + // nameWithRunID trims the separator rather than doubling it, + // so this resolves to a valid name and must be accepted. + name: "trailing dash on the prefix is trimmed, not rejected", + config: Config{JobName: "agent-", RunID: runID}, + }, + { + // The budget truncation keeps the resolved name inside + // defaults.MaxK8sNameLength, so an over-long prefix is not an + // error — it is silently shortened. + name: "over-length prefix truncates to a valid name", + config: Config{JobName: strings.Repeat("a", defaults.MaxK8sNameLength*3), RunID: runID}, + }, + { + // Truncation plus trailing-dash trimming can empty the prefix + // entirely; the result is the bare run ID, which is valid. + name: "prefix that empties after truncation degrades to the bare run ID", + config: Config{JobName: "----", RunID: runID}, + }, + { + name: "underscore in JobName", + config: Config{JobName: "agent_", RunID: runID}, + wantField: "Config.JobName", + wantValue: "agent_", + }, + { + name: "underscore in ServiceAccountName", + config: Config{ServiceAccountName: "agent_sa", RunID: runID}, + wantField: "Config.ServiceAccountName", + wantValue: "agent_sa", + }, + { + name: "underscore in NameBase governs both names", + config: Config{NameBase: "agent_base", RunID: runID}, + wantField: "Config.NameBase", + wantValue: "agent_base", + }, + { + name: "uppercase in JobName", + config: Config{JobName: "Agent", RunID: runID}, + wantField: "Config.JobName", + wantValue: "Agent", + }, + { + name: "leading dash in JobName", + config: Config{JobName: "-agent", RunID: runID}, + wantField: "Config.JobName", + wantValue: "-agent", + }, + { + name: "slash in ServiceAccountName", + config: Config{ServiceAccountName: "team/agent", RunID: runID}, + wantField: "Config.ServiceAccountName", + wantValue: "team/agent", + }, + { + // Reachable only by calling validateResolvedNames directly: + // validateNames rejects this run ID first. It exists so the + // defaults.MaxK8sNameLength branch is covered rather than + // trusted. + name: "over-long run ID leaves a name past the length ceiling", + config: Config{RunID: overLongRunID}, + wantField: "Config.NameBase", + wantValue: defaultNameBase, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := NewDeployer(fake.NewClientset(), tt.config) + err := d.validateResolvedNames() + if tt.wantField == "" { + if err != nil { + t.Fatalf("validateResolvedNames() error = %v, want nil", err) + } + return + } + if err == nil { + t.Fatalf("validateResolvedNames() = nil, want an error naming %s", tt.wantField) + } + if !stderrors.Is(err, errors.New(errors.ErrCodeInvalidRequest, "")) { + t.Errorf("error = %v, want code %v", err, errors.ErrCodeInvalidRequest) + } + if !strings.Contains(err.Error(), tt.wantField) { + t.Errorf("error %q does not name the offending field %q", err.Error(), tt.wantField) + } + if !strings.Contains(err.Error(), tt.wantValue) { + t.Errorf("error %q does not echo the offending value %q", err.Error(), tt.wantValue) + } + }) + } +} + +// TestValidateNamesChecksRunIDFirst pins the order inside the pre-flight: a +// caller who gets both halves wrong should hear about the run ID, which is +// the value they are least likely to have set deliberately. +func TestValidateNamesChecksRunIDFirst(t *testing.T) { + d := NewDeployer(fake.NewClientset(), Config{JobName: "agent_", RunID: "Bad/ID"}) + err := d.validateNames() + if err == nil { + t.Fatal("validateNames() = nil, want an error") + } + if !strings.Contains(err.Error(), "Config.RunID") { + t.Errorf("error %q does not name Config.RunID", err.Error()) + } +} + +// TestDeployRejectsInvalidResolvedNameBeforeCreatingAnything is the +// end-to-end half of the resolved-name gate, mirroring the run-ID test +// above: an invalid prefix must be rejected before CheckPermissions and +// before any write, so no partially-created RBAC is left behind. +func TestDeployRejectsInvalidResolvedNameBeforeCreatingAnything(t *testing.T) { + const runID = "20260821-142233-9f3a1c0b7e2d4a55" + ctx := context.Background() + + tests := []struct { + name string + config Config + }{ + {"underscore JobName", Config{JobName: "agent_", RunID: runID}}, + {"uppercase ServiceAccountName", Config{ServiceAccountName: "AgentSA", RunID: runID}}, + {"underscore NameBase", Config{NameBase: "agent_base", RunID: runID}}, + {"leading dash JobName", Config{JobName: "-agent", RunID: runID}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clientset := fake.NewClientset() + cfg := tt.config + cfg.Namespace = "test-ns" + cfg.Image = "aicr:test" + + err := NewDeployer(clientset, cfg).Deploy(ctx) + if err == nil { + t.Fatalf("Deploy() with config %+v should fail", cfg) + } + if !stderrors.Is(err, errors.New(errors.ErrCodeInvalidRequest, "")) { + t.Errorf("Deploy() error = %v, want code %v", err, errors.ErrCodeInvalidRequest) + } + // Not even the Step-0 SelfSubjectAccessReview may have been + // issued: the gate runs ahead of CheckPermissions. + if actions := clientset.Actions(); len(actions) != 0 { + t.Errorf("Deploy() issued %d API call(s) before rejecting an invalid name: %v", len(actions), actions) + } + }) + } +} From 5e3a43ea263cd2bebdd1a555c2045685d8341aa2 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Mon, 24 Aug 2026 16:25:10 -0700 Subject: [PATCH 37/56] test(agent): report reactor failures without t.Fatalf on a worker goroutine CheckPermissions fans its checks out over an errgroup, so a fake-clientset reactor installed by the test runs on worker goroutines, not the test goroutine. t.Fatalf there calls runtime.Goexit and terminates only that worker: the test does not stop as intended, and the errgroup is left waiting on a goroutine that never returns a value. Report with t.Errorf and hand the same failure back as the reactor's error so the call under test terminates on its own. Signed-off-by: Alex Yuskauskas --- pkg/k8s/agent/permissions_test.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pkg/k8s/agent/permissions_test.go b/pkg/k8s/agent/permissions_test.go index 34597bd2a..1ee8896ff 100644 --- a/pkg/k8s/agent/permissions_test.go +++ b/pkg/k8s/agent/permissions_test.go @@ -16,6 +16,7 @@ package agent import ( "context" + "fmt" "strings" "testing" @@ -191,14 +192,25 @@ func TestCheckPermissions_ConfigMapDeleteGatedOnOwnership(t *testing.T) { // Deny only `configmaps: delete`; allow everything else. This // models the least-privilege identity the gate exists for. + // + // CheckPermissions fans the checks out over an errgroup, so this + // reactor runs on worker goroutines, not the test goroutine. + // t.Fatalf there would Goexit only the worker and leave the + // errgroup waiting on a goroutine that never returns a value; + // report with t.Errorf and hand the failure back as the + // reactor's error so the call under test terminates. clientset.PrependReactor("create", "selfsubjectaccessreviews", func(action k8stesting.Action) (bool, runtime.Object, error) { create, ok := action.(k8stesting.CreateAction) if !ok { - t.Fatalf("action %T is not a CreateAction", action) + reactorErr := fmt.Errorf("action %T is not a CreateAction", action) + t.Error(reactorErr) + return true, nil, reactorErr } review, ok := create.GetObject().(*authv1.SelfSubjectAccessReview) if !ok { - t.Fatalf("object %T is not a SelfSubjectAccessReview", create.GetObject()) + reactorErr := fmt.Errorf("object %T is not a SelfSubjectAccessReview", create.GetObject()) + t.Error(reactorErr) + return true, nil, reactorErr } attrs := review.Spec.ResourceAttributes allowed := attrs.Resource != resourceCM || attrs.Verb != verbDelete From 0b4b46eb6780a9d11790ae3e6e198b02b5e59579 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Mon, 24 Aug 2026 16:25:10 -0700 Subject: [PATCH 38/56] test(agent): use fake.NewClientset over the deprecated constructor The rest of the package already builds fakes with fake.NewClientset; the tests added on this branch reintroduced fake.NewSimpleClientset. Switch all of them, plus the testing example in the package doc comment, for consistency. Signed-off-by: Alex Yuskauskas --- pkg/k8s/agent/deployer_test.go | 20 ++++++++++---------- pkg/k8s/agent/doc.go | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/k8s/agent/deployer_test.go b/pkg/k8s/agent/deployer_test.go index defa138ff..9a9278442 100644 --- a/pkg/k8s/agent/deployer_test.go +++ b/pkg/k8s/agent/deployer_test.go @@ -565,7 +565,7 @@ func TestDeployer_Deploy(t *testing.T) { // TestDeployer_Deploy et al. so the test exercises the behavior it names. func TestDeployUsesRunScopedNamesAndLabels(t *testing.T) { ctx := context.Background() - client := fake.NewSimpleClientset() + client := fake.NewClientset() client.PrependReactor("create", "selfsubjectaccessreviews", func(action k8stesting.Action) (bool, runtime.Object, error) { return true, &authv1.SelfSubjectAccessReview{ Status: authv1.SubjectAccessReviewStatus{ @@ -775,7 +775,7 @@ func TestDeployer_Cleanup_ReportsAllErrors(t *testing.T) { // this file. func TestCleanupDeletesOnlyWhatItCreated(t *testing.T) { ctx := context.Background() - client := fake.NewSimpleClientset() + client := fake.NewClientset() client.PrependReactor("create", "selfsubjectaccessreviews", func(action k8stesting.Action) (bool, runtime.Object, error) { return true, &authv1.SelfSubjectAccessReview{ Status: authv1.SubjectAccessReviewStatus{ @@ -858,7 +858,7 @@ func spyOnDeletes(client *fake.Clientset) func() []observedDelete { // outgoing delete actions rather than relying on tracker behavior. func TestCleanupPassesUIDPrecondition(t *testing.T) { ctx := context.Background() - client := fake.NewSimpleClientset() + client := fake.NewClientset() deletes := spyOnDeletes(client) // One object per kind, each with a distinct UID so a dispatch arm that @@ -923,7 +923,7 @@ func TestCleanupPassesUIDPrecondition(t *testing.T) { // safe here because the name carries this run's ID. func TestCleanupDeletesUnconfirmedCreateByBareName(t *testing.T) { ctx := context.Background() - client := fake.NewSimpleClientset() + client := fake.NewClientset() deletes := spyOnDeletes(client) d := NewDeployer(client, Config{Namespace: "test-ns"}) @@ -958,7 +958,7 @@ func TestCleanupDeletesUnconfirmedCreateByBareName(t *testing.T) { func TestEnsureRecordsIntentBeforeCreate(t *testing.T) { ctx := context.Background() const ns = "test-ns" - client := fake.NewSimpleClientset() + client := fake.NewClientset() d := NewDeployer(client, Config{Namespace: ns, RunID: testRunID}) saName := d.saName() @@ -1002,7 +1002,7 @@ func TestEnsureRecordsIntentBeforeCreate(t *testing.T) { func TestEnsureDiscardsIntentOnAlreadyExists(t *testing.T) { ctx := context.Background() const ns = "test-ns" - client := fake.NewSimpleClientset() + client := fake.NewClientset() d := NewDeployer(client, Config{Namespace: ns, RunID: testRunID}) client.PrependReactor("create", "serviceaccounts", func(action k8stesting.Action) (bool, runtime.Object, error) { @@ -1026,7 +1026,7 @@ func TestEnsureDiscardsIntentOnAlreadyExists(t *testing.T) { // Cleanup failure. func TestCleanupTreatsConflictAsSuccess(t *testing.T) { ctx := context.Background() - client := fake.NewSimpleClientset() + client := fake.NewClientset() client.PrependReactor("create", "selfsubjectaccessreviews", func(action k8stesting.Action) (bool, runtime.Object, error) { return true, &authv1.SelfSubjectAccessReview{ Status: authv1.SubjectAccessReviewStatus{Allowed: true, Reason: "test permissions allowed"}, @@ -1049,7 +1049,7 @@ func TestCleanupTreatsConflictAsSuccess(t *testing.T) { // any Job is recorded, and the recorded Job's UID afterward — even when // other kinds have been recorded too. func TestRecordCreatedAndJobUID(t *testing.T) { - d := NewDeployer(fake.NewSimpleClientset(), Config{Namespace: "test-ns"}) + d := NewDeployer(fake.NewClientset(), Config{Namespace: "test-ns"}) if got := d.jobUID(); got != "" { t.Fatalf("jobUID() before any Job recorded = %q, want zero UID", got) @@ -1069,7 +1069,7 @@ func TestRecordCreatedAndJobUID(t *testing.T) { // TestCreatedSnapshotIsDefensiveCopy verifies createdSnapshot returns a copy // that mutation cannot use to corrupt the Deployer's internal created-set. func TestCreatedSnapshotIsDefensiveCopy(t *testing.T) { - d := NewDeployer(fake.NewSimpleClientset(), Config{Namespace: "test-ns"}) + d := NewDeployer(fake.NewClientset(), Config{Namespace: "test-ns"}) d.recordCreated(kindServiceAccount, "aicr-sa", types.UID("sa-uid")) snap := d.createdSnapshot() @@ -1088,7 +1088,7 @@ func TestCreatedSnapshotIsDefensiveCopy(t *testing.T) { // goroutines at once so `go test -race` can catch a data race on the // created-set if the locking is ever removed or narrowed incorrectly. func TestRecordCreatedConcurrentSafe(t *testing.T) { - d := NewDeployer(fake.NewSimpleClientset(), Config{Namespace: "test-ns"}) + d := NewDeployer(fake.NewClientset(), Config{Namespace: "test-ns"}) const n = 50 var wg sync.WaitGroup diff --git a/pkg/k8s/agent/doc.go b/pkg/k8s/agent/doc.go index c66fbe3d5..2395423a8 100644 --- a/pkg/k8s/agent/doc.go +++ b/pkg/k8s/agent/doc.go @@ -165,7 +165,7 @@ The package is designed for testability with Kubernetes fake clients: ) func TestDeployer(t *testing.T) { - clientset := fake.NewSimpleClientset() + clientset := fake.NewClientset() deployer := agent.NewDeployer(clientset, agent.Config{ Namespace: "test", RunID: "20260821-142233-9f3a1c0b7e2d4a55", From 7135aee9ba4db59428b5cd6c696f0bc483a5b3c0 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Mon, 24 Aug 2026 16:25:19 -0700 Subject: [PATCH 39/56] docs: make the run-ID lookup self-contained and fix agent default drift Three corrections to the run-scoped agent docs: - agent-deployment.md's "Job Completes but No Output" section used $RUN_ID but only "Job Fails to Start" defined it, so a reader who opened that section alone ran `kubectl get configmap ... aicr-agent-snapshot-` and got a not-found. Repeat the lookup there. - cli-config.md said validate's serviceAccountName default is "aicr". It is "aicr-validate" (validateNameBase in pkg/cli/validate.go, applied as Config.NameBase for both names), which is what the same file's spec.validate table and cli-reference.md already say. - cli-reference.md's --no-cleanup warning named only the ClusterRoleBinding, then referred to "the retained ClusterRole" without introducing it. The ClusterRole is the object that carries the mutating --discover-network rules, so name both. Signed-off-by: Alex Yuskauskas --- docs/user/agent-deployment.md | 5 +++++ docs/user/cli-config.md | 2 +- docs/user/cli-reference.md | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/user/agent-deployment.md b/docs/user/agent-deployment.md index 3494c67de..f5d380b24 100644 --- a/docs/user/agent-deployment.md +++ b/docs/user/agent-deployment.md @@ -381,6 +381,11 @@ kubectl get nodes -o custom-columns=NAME:.metadata.name,TAINTS:.spec.taints Check ConfigMap and container logs: ```shell +# Every name below is run-scoped, so start from the run ID. The CLI logs it +# when it starts ("snapshot agent run: runID=..."); this reads it back off the +# Job instead. Substitute your own Job name (or use the logged run ID). +RUN_ID=$(kubectl get job -n gpu-operator -o jsonpath='{.metadata.labels.aicr\.run/run-id}') + # Check if the staging ConfigMap was created. Without an explicit # "-o cm:///", the agent stages its result in a run-scoped # ConfigMap named "aicr-agent-snapshot-" that cleanup deletes when diff --git a/docs/user/cli-config.md b/docs/user/cli-config.md index 9f22abf21..42d1499b9 100644 --- a/docs/user/cli-config.md +++ b/docs/user/cli-config.md @@ -182,7 +182,7 @@ spec: image: "" imagePullSecrets: [] # Optional prefixes; omitted here so the defaults apply - # (jobName: aicr-validate, serviceAccountName: aicr). + # (jobName and serviceAccountName both default to aicr-validate). nodeSelector: nodeGroup: gpu-worker tolerations: diff --git a/docs/user/cli-reference.md b/docs/user/cli-reference.md index df29d77b3..5e1d09a7e 100644 --- a/docs/user/cli-reference.md +++ b/docs/user/cli-reference.md @@ -89,7 +89,7 @@ aicr snapshot [flags] | `--node-selector` | | string[] | auto | Node selector for agent scheduling (key=value, repeatable). When omitted (and neither `--require-gpu` nor `--runtime-class` is set), the agent auto-targets GPU nodes labeled `nvidia.com/gpu.present=true` if the cluster has any — see [Agent Deployment](agent-deployment.md). Pass an explicit selector to override. | | `--toleration` | | string[] | all taints | Tolerations for agent scheduling (key=value:effect, repeatable). **Default: all taints tolerated** (uses `operator: Exists`). Only specify to restrict which taints are tolerated. | | `--timeout` | | duration | 5m | Timeout for agent Job completion | -| `--no-cleanup` | | bool | false | Skip removal of Job and RBAC resources on completion. **Warning:** leaves the agent's run-scoped `aicr-node-reader-` ClusterRoleBinding active. By default this grants only read-only access; with `--discover-network` the retained ClusterRole also carries the mutating rules live network discovery needs (CRD/namespace/daemonset create, pod exec, node patch, NicClusterPolicy). | +| `--no-cleanup` | | bool | false | Skip removal of Job and RBAC resources on completion. **Warning:** leaves both the agent's run-scoped `aicr-node-reader-` ClusterRole and the identically named ClusterRoleBinding active. By default the ClusterRole grants only read-only access; with `--discover-network` it also carries the mutating rules live network discovery needs (CRD/namespace/daemonset create, pod exec, node patch, NicClusterPolicy). Delete both when you are done — removing only the binding leaves the grant definition behind. | | `--privileged` | | bool | true | Run agent in privileged mode (required for GPU/SystemD collectors). Set to false for PSS-restricted namespaces. | | `--image-pull-secret` | | string[] | | Image pull secrets for private registries (repeatable) | | `--require-gpu` | | bool | false | Require GPU resources on the agent pod (mutually exclusive with `--runtime-class`) | From 1d51d6047c72b75e2fe87abdf84b2010bc41f60f Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Mon, 24 Aug 2026 16:25:28 -0700 Subject: [PATCH 40/56] docs(cleanup): warn that the agent label sweep terminates a live run tools/cleanup deletes every Job, SA, Role and RoleBinding in gpu-operator carrying the snapshot-agent label set. Every run carries it, so running the tool during a snapshot deletes that run's Job and revokes its ServiceAccount. That is the right contract for a manual teardown tool -- it must reclaim the namespace regardless of which run left an object behind, including runs whose IDs it never saw -- so the behavior is unchanged. Document the hazard above the sweep and in the header block an operator reads first. Signed-off-by: Alex Yuskauskas --- tools/cleanup | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tools/cleanup b/tools/cleanup index 1cf26a414..4fe75abaa 100755 --- a/tools/cleanup +++ b/tools/cleanup @@ -26,6 +26,10 @@ # are touched. Cluster-scoped resources outside the AICR component domains are # left alone. # +# Not narrow, by design: the snapshot-agent sweep in phase 2 is label-based and +# matches every run, including one in flight. See the HAZARD note there before +# running this against a cluster someone else is using. +# # On shared / bring-up clusters a registry component may be deliberately owned # out-of-band (installed by the platform team, excluded from the AICR recipe). # Destroying it is data loss AICR does not own, so `--exclude-ns` and @@ -404,6 +408,17 @@ kc delete ns aicr-validation --ignore-not-found --wait=false # # The two cannot collide: a run-scoped name is always "aicr-", never # the bare "aicr". +# +# HAZARD -- this sweep is not run-scoped, and deliberately so. EVERY agent run +# carries this exact label set, so running `tools/cleanup` while a snapshot is +# in flight deletes that run's Job and revokes its ServiceAccount, Role and +# RoleBinding; the run fails partway through. That is the intended contract for +# a manual teardown tool: it reclaims the namespace regardless of which run +# left an object behind, including runs whose IDs this invocation never saw. +# Contrast tests/e2e/run.sh, which records the run IDs it launched and sweeps +# only those: an unattended suite shares its cluster and cannot make this +# choice. Do not run this tool against a cluster with an active +# `aicr snapshot` or `aicr validate`. kc -n gpu-operator delete job -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent --ignore-not-found kc -n gpu-operator delete sa -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent --ignore-not-found kc -n gpu-operator delete role -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent --ignore-not-found From d3e06737f36f6ebbaa9eed9d7860910ff4596de6 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Mon, 24 Aug 2026 16:25:28 -0700 Subject: [PATCH 41/56] test(e2e): poll until the leftover agent Job count settles The self-cleanup assertion counted Jobs once and required exactly 1. Kubernetes deletion is asynchronous, so a Job whose delete the concurrent runs already issued and acked stays listable while its pods terminate and its finalizers clear -- the assertion then failed with "found 2" for a cleanup that worked correctly. Poll until the count settles, failing only after AGENT_JOB_SETTLE_TIMEOUT (60s, overridable). Only the timing becomes tolerant; the final assertion is still exactly 1. Signed-off-by: Alex Yuskauskas --- tests/e2e/run.sh | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/tests/e2e/run.sh b/tests/e2e/run.sh index 4b15754c9..46806ac45 100755 --- a/tests/e2e/run.sh +++ b/tests/e2e/run.sh @@ -56,6 +56,11 @@ CREATED_FAKE_GPU_OPERATOR_DEPLOYMENT=false CREATED_FAKE_CLUSTER_POLICY=false CREATED_FAKE_CLUSTER_POLICY_CRD=false +# Seconds to wait for the concurrent runs' self-deleted Jobs to disappear from +# the API before the self-cleanup assertion gives up. Deletion is asynchronous, +# so the count settles shortly after the CLI returns rather than at that instant. +AGENT_JOB_SETTLE_TIMEOUT="${AGENT_JOB_SETTLE_TIMEOUT:-60}" + # Run IDs of the snapshot-agent runs this script launched, space separated. # cleanup_e2e deletes only these Jobs. Every agent run carries the same # app.kubernetes.io/{name,component} labels, so a label-only sweep would also @@ -743,15 +748,30 @@ snapshot_run_isolation_body() { fi pass "snapshot/isolation/retained-run-survives" - # Each concurrent run must have removed its own resources. - local leftover_jobs + # Each concurrent run must have removed its own resources: exactly the + # retained Job survives. + # + # Poll rather than sample once. Kubernetes deletion is asynchronous -- a Job + # whose delete the CLI already issued and acked stays listable while its + # pods terminate and its finalizers clear, so a single read can legitimately + # still see it and report "found 2" for a cleanup that worked. Only the + # timing is tolerant; the assertion below is still exact. + local leftover_jobs="" local agent_job_selector="app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent" - leftover_jobs=$(kubectl get jobs -n "$ns" -l "$agent_job_selector" -o name 2>/dev/null | wc -l | tr -d ' ') - if [ "$leftover_jobs" != "1" ]; then - kubectl get jobs -n "$ns" -l "$agent_job_selector" -o name || true - fail "snapshot/isolation/self-cleanup" "expected only the retained Job to remain, found ${leftover_jobs}" - return 1 - fi + local deadline=$((SECONDS + AGENT_JOB_SETTLE_TIMEOUT)) + while :; do + leftover_jobs=$(kubectl get jobs -n "$ns" -l "$agent_job_selector" -o name 2>/dev/null | wc -l | tr -d ' ') + if [ "$leftover_jobs" = "1" ]; then + break + fi + if [ "$SECONDS" -ge "$deadline" ]; then + kubectl get jobs -n "$ns" -l "$agent_job_selector" -o name || true + fail "snapshot/isolation/self-cleanup" \ + "expected only the retained Job to remain after ${AGENT_JOB_SETTLE_TIMEOUT}s, found ${leftover_jobs}" + return 1 + fi + sleep 2 + done pass "snapshot/isolation/self-cleanup" # The decoy must have survived every run above. From 3ee9bc86a279ef77418e89081027320b903ed399 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Mon, 24 Aug 2026 16:33:24 -0700 Subject: [PATCH 42/56] refactor(agent): name the shared error-context keys Signed-off-by: Alex Yuskauskas --- pkg/k8s/agent/names.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/pkg/k8s/agent/names.go b/pkg/k8s/agent/names.go index 1958a0520..a9d92e92d 100644 --- a/pkg/k8s/agent/names.go +++ b/pkg/k8s/agent/names.go @@ -93,18 +93,26 @@ func nameWithRunID(prefix, runID string) string { // pkg/snapshotter defaults and whitespace-checks RunID before building an // agent Config, but that guard does not cover callers who construct a // pkg/k8s/agent Config directly, which is the public SDK surface. +// Error-context keys shared by the name-validation failures below, so a +// caller parsing a structured error keys off one spelling. +const ( + ctxKeyField = "field" + ctxKeyValue = "value" + ctxKeyResolvedName = "resolvedName" +) + func (d *Deployer) validateRunID() error { runID := d.config.RunID if runID == "" { return errors.NewWithContext(errors.ErrCodeInvalidRequest, "Config.RunID is required: every object this Deployer creates is named \"-\"; generate one with runid.Generate()", - map[string]any{"field": "Config.RunID", "value": runID}) + map[string]any{ctxKeyField: "Config.RunID", ctxKeyValue: runID}) } if problems := validation.IsDNS1123Label(runID); len(problems) > 0 { return errors.NewWithContext(errors.ErrCodeInvalidRequest, fmt.Sprintf("Config.RunID %q is not a valid Kubernetes name segment: %s", runID, strings.Join(problems, "; ")), - map[string]any{"field": "Config.RunID", "value": runID}) + map[string]any{ctxKeyField: "Config.RunID", ctxKeyValue: runID}) } return nil } @@ -172,7 +180,7 @@ func (d *Deployer) validateResolvedNames() error { return errors.NewWithContext(errors.ErrCodeInvalidRequest, fmt.Sprintf("%s %q yields the %s name %q, which is not a valid Kubernetes object name: %s", n.field, n.prefix, n.objects, n.value, strings.Join(problems, "; ")), - map[string]any{"field": n.field, "value": n.prefix, "resolvedName": n.value}) + map[string]any{ctxKeyField: n.field, ctxKeyValue: n.prefix, ctxKeyResolvedName: n.value}) } return nil } From f92aaba17b5e4f2ca0d535ebc8d02d2a777434c0 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Tue, 25 Aug 2026 10:53:37 -0700 Subject: [PATCH 43/56] fix(agent): establish cleanup ownership at create time, not by name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cleanup inferred ownership from a name plus a freshly-read UID, which proves only that the object did not change between the Get and the Delete. Two paths could therefore collect an object the run never created. A failed run reusing another run's RunID swept the staging ConfigMap the first run was still using: it resolved the same name and deleted it with the UID it had just read. Gate that sweep on this run holding an apiserver-confirmed Job — the staging ConfigMap is written only by the in-pod agent that Job runs, so no confirmed Job means no ConfigMap of ours — and re-check that the object carries app.kubernetes.io/name=aicr, the only label pkg/serializer stamps on it, before deleting. An entry recorded by recordIntent but never confirmed by a Create response deleted by bare name, collecting a replacement created under that name in the meantime. Record ownership at create time instead (createdObject.confirmed), and for an unconfirmed entry re-establish it from the live object's own aicr.run/run-id label set, deleting pinned to the UID that read observed. When the labels do not match, keep the object and warn with its identity rather than delete blind; a non-NotFound Get fails the cleanup rather than being swallowed. The lost-response case recordIntent exists for is still reclaimed. Signed-off-by: Alex Yuskauskas --- docs/user/agent-deployment.md | 28 +++ pkg/k8s/agent/concurrency_test.go | 6 +- pkg/k8s/agent/deployer.go | 143 +++++++++++-- pkg/k8s/agent/deployer_test.go | 327 +++++++++++++++++++++++++++--- pkg/k8s/agent/types.go | 94 ++++++--- pkg/k8s/agent/wait.go | 29 ++- 6 files changed, 556 insertions(+), 71 deletions(-) diff --git a/docs/user/agent-deployment.md b/docs/user/agent-deployment.md index f5d380b24..5e0f77029 100644 --- a/docs/user/agent-deployment.md +++ b/docs/user/agent-deployment.md @@ -426,6 +426,34 @@ kubectl get rolebinding -n gpu-operator -l app.kubernetes.io/name=aicr,app.kuber kubectl get serviceaccount -n gpu-operator -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent ``` +### Cleanup Left a Resource Behind + +Cleanup removes only the objects the run itself created, and pins every delete +to the UID the apiserver returned when it created them. If a create's response +is lost in flight, the run knows the name it used but never learned the +object's identity — so cleanup re-reads the object and deletes it only when it +still carries that run's `aicr.run/run-id` label. When something else has taken +the name over, cleanup keeps the object and says so: + +```text +WARN cleanup left behind an object it cannot prove this run created; if it is a +stale orphan of this run, remove it by hand kind=Role name=aicr- +runID= objectRunID= +``` + +Inspect the named object and delete it yourself if it is a stale leftover: + +```shell +kubectl get role -n gpu-operator aicr- -o yaml +``` + +The staging ConfigMap is warned about the same way. It is written by the in-pod +agent rather than by the CLI, so it carries no `aicr.run/run-id` label to check +(see [Job Completes but No Output](#job-completes-but-no-output)); cleanup +sweeps it only when this run's Job was created successfully — the only way this +run could have produced one — and only when the object looks like an aicr +artifact (`app.kubernetes.io/name=aicr`). + ## Security Considerations ### RBAC Permissions diff --git a/pkg/k8s/agent/concurrency_test.go b/pkg/k8s/agent/concurrency_test.go index f9fcf6bd0..eae7e639b 100644 --- a/pkg/k8s/agent/concurrency_test.go +++ b/pkg/k8s/agent/concurrency_test.go @@ -299,8 +299,10 @@ func TestConcurrentRuns(t *testing.T) { // discriminate created-set scoping are TestCleanupPassesUIDPrecondition // (every delete carries the UID from its Create response — a value no // name-derived delete list can supply) and - // TestCleanupDeletesUnconfirmedCreateByBareName, both in - // deployer_test.go. + // TestCleanupResolvesUnconfirmedEntryBeforeDeleting, both in + // deployer_test.go. The duplicate-RunID case, where name scoping cannot + // help because both runs compute the same names, is + // TestCleanupDuplicateRunIDKeepsFirstRunsStagingConfigMap. t.Run("run A Cleanup leaves its own unowned staging ConfigMap intact", func(t *testing.T) { if _, err := clientset.CoreV1().ConfigMaps(concurrencyTestNamespace).Get(ctx, dA.stagingConfigMapName(), metav1.GetOptions{}); err != nil { t.Errorf("the ConfigMap at run A's staging name %q should survive run A's Cleanup (OwnsOutputConfigMap is false, so it is not run A's to delete), err = %v", dA.stagingConfigMapName(), err) diff --git a/pkg/k8s/agent/deployer.go b/pkg/k8s/agent/deployer.go index b0c043a1f..9952f3ee6 100644 --- a/pkg/k8s/agent/deployer.go +++ b/pkg/k8s/agent/deployer.go @@ -24,6 +24,7 @@ import ( "github.com/NVIDIA/aicr/pkg/defaults" aicrerrors "github.com/NVIDIA/aicr/pkg/errors" + "github.com/NVIDIA/aicr/pkg/k8s/labels" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -158,15 +159,15 @@ func (d *Deployer) Cleanup(ctx context.Context, opts CleanupOptions) error { // to observe its UID. A run that fails after the agent wrote it (Job // timeout, wait error, canceled context) would otherwise leak it — and // with run-scoped naming that is one leaked object per failed run, not - // one shared object. Sweep it here: the name is run-unique, and the - // delete is still UID-pinned against the UID observed by the Get. + // one shared object. Sweep it here. // - // The presence test reads the `created` snapshot taken above rather than - // re-entering the mutex (see containsKind): two separate lock - // acquisitions would let a concurrent recordCreated land between them, - // producing a snapshot without the ConfigMap and a second read reporting - // it present — skipping both delete paths and leaking the object. - if d.config.OwnsOutputConfigMap && !containsKind(created, kindConfigMap) { + // The name alone does not license that delete: Config.RunID is caller- + // settable, so a second run reusing a RunID resolves the same staging + // name as the first. needsStagingConfigMapSweep therefore requires this + // run to hold a CONFIRMED Job entry — the only way this run could have + // produced a staging ConfigMap at all — and deleteUnrecordedStagingConfigMap + // re-checks the object it finds before deleting it. + if d.config.OwnsOutputConfigMap && needsStagingConfigMapSweep(created) { name := d.stagingConfigMapName() tasks = append(tasks, task{ label: fmt.Sprintf("%s %q", kindConfigMap, name), @@ -215,7 +216,20 @@ func (d *Deployer) Cleanup(ctx context.Context, opts CleanupOptions) error { // deleteCreatedObject deletes a single created-set entry by dispatching to // the resource-specific delete call for obj.kind, passing obj.name and // obj.uid through so every delete is UID-pinned. +// +// An unconfirmed entry carries no UID (no Create response ever named the +// object), so its ownership is re-established first — resolveIntentUID either +// returns the live object's UID after proving the object carries this run's +// labels, or reports that there is nothing this run may delete. func (d *Deployer) deleteCreatedObject(ctx context.Context, obj createdObject) error { + if !obj.confirmed { + uid, ours, err := d.resolveIntentUID(ctx, obj) + if err != nil || !ours { + return err + } + obj.uid = uid + } + switch obj.kind { case kindJob: return d.deleteJob(ctx, obj.name, obj.uid) @@ -236,17 +250,114 @@ func (d *Deployer) deleteCreatedObject(ctx context.Context, obj createdObject) e } } +// resolveIntentUID re-establishes ownership of an unconfirmed created-set +// entry — one recordIntent added whose Create response never arrived — and +// returns the UID to pin its delete to. ours is false when this run may not +// delete the object, in which case the caller must issue no delete at all. +// +// The entry's run-scoped name is not evidence: it says what this run WOULD +// have created, not what is standing there now. Get the live object and +// require it to carry the label set objectLabels() stamps on everything +// Deploy creates. That set is written at creation time by this run, so: +// +// - Labels match: the Create did commit and this is our object. Delete it, +// pinned to the UID this Get observed. (If it is replaced again between +// the Get and the Delete, the precondition turns that into a Conflict, +// which ignoreNotFoundOrConflict treats as "already gone".) +// - Labels do not match: whatever holds the name was not created by this +// run — an operator's object, another subsystem's, or a replacement made +// after this run's object was deleted. Deleting it would collect an +// object this run never owned, so fail closed and warn instead. +// - NotFound: nothing to reclaim. +// +// Residual, and deliberately not papered over: a second run that reuses this +// run's RunID stamps an identical label set, so the two are indistinguishable +// here. The ADR treats a duplicate RunID as an unsupported caller error — +// every ensure* fails closed on the AlreadyExists it normally produces (see +// discardIntent); this path is reachable only when that response was also +// lost. +func (d *Deployer) resolveIntentUID(ctx context.Context, obj createdObject) (uid types.UID, ours bool, err error) { + live, err := d.getCreatedObject(ctx, obj.kind, obj.name) + if k8serrors.IsNotFound(err) { + return "", false, nil + } + if err != nil { + return "", false, aicrerrors.Wrap(aicrerrors.ErrCodeInternal, + fmt.Sprintf("failed to read %s %q to confirm this run created it before deleting it", obj.kind, obj.name), err) + } + + // A real apiserver always assigns a UID, so an empty one here means the + // delete could not be pinned. Refuse rather than fall back to a + // bare-name delete: that is exactly the blind delete this path exists + // to prevent. + if !d.createdByThisRun(live.GetLabels()) || live.GetUID() == "" { + slog.Warn("cleanup left behind an object it cannot prove this run created; if it is a stale orphan of this run, remove it by hand", + slog.String("kind", obj.kind), + slog.String("name", obj.name), + slog.String("namespace", live.GetNamespace()), + slog.String("uid", string(live.GetUID())), + slog.String("runID", d.config.RunID), + slog.String("objectRunID", live.GetLabels()[labels.RunID])) + return "", false, nil + } + return live.GetUID(), true, nil +} + +// getCreatedObject reads the live object a created-set entry names. The +// apiserver error is returned unwrapped so callers can classify it with +// k8serrors.IsNotFound before wrapping. +func (d *Deployer) getCreatedObject(ctx context.Context, kind, name string) (metav1.Object, error) { + ns := d.config.Namespace + switch kind { + case kindJob: + return d.clientset.BatchV1().Jobs(ns).Get(ctx, name, metav1.GetOptions{}) + case kindServiceAccount: + return d.clientset.CoreV1().ServiceAccounts(ns).Get(ctx, name, metav1.GetOptions{}) + case kindRole: + return d.clientset.RbacV1().Roles(ns).Get(ctx, name, metav1.GetOptions{}) + case kindRoleBinding: + return d.clientset.RbacV1().RoleBindings(ns).Get(ctx, name, metav1.GetOptions{}) + case kindClusterRole: + return d.clientset.RbacV1().ClusterRoles().Get(ctx, name, metav1.GetOptions{}) + case kindClusterRoleBinding: + return d.clientset.RbacV1().ClusterRoleBindings().Get(ctx, name, metav1.GetOptions{}) + case kindConfigMap: + return d.clientset.CoreV1().ConfigMaps(ns).Get(ctx, name, metav1.GetOptions{}) + default: + return nil, aicrerrors.New(aicrerrors.ErrCodeInternal, fmt.Sprintf("cleanup: cannot read unknown created-object kind %q", kind)) + } +} + +// createdByThisRun reports whether objLabels is the label set objectLabels() +// stamps on every object this Deployer creates. All four keys are required: +// aicr.run/run-id alone would also match a validator-owned object carrying +// the same ID (`aicr validate` hands one run ID to both subsystems), and an +// empty Config.RunID must never match a label-less object. +func (d *Deployer) createdByThisRun(objLabels map[string]string) bool { + if d.config.RunID == "" { + return false + } + return objLabels[labels.RunID] == d.config.RunID && + objLabels[labels.Name] == labels.ValueAICR && + objLabels[labels.ManagedBy] == labels.ValueAICR && + objLabels[labels.Component] == labels.ValueSnapshotAgent +} + // uidPreconditions returns the DeleteOptions precondition pinning a delete // to uid, or nil when uid is the zero UID. // -// A zero UID means recordIntent entered the object before its Create and no -// Create response ever confirmed a UID (see recordIntent). Omitting the -// precondition entirely is required — metav1.Preconditions{UID: &""} is NOT -// equivalent to no precondition: the apiserver compares it against the live -// object's UID and rejects every delete with a Conflict, which -// ignoreNotFoundOrConflict then swallows as success, leaking the object this -// entry exists to reclaim. A bare-name delete is safe here precisely because -// the name carries this run's ID and no other run can produce it. +// Only a confirmed entry can reach here with the zero UID: a Create response +// named the object — establishing this run's ownership — but carried no UID. +// A real apiserver always assigns one, so that shape belongs to fake +// clientsets in tests; the delete then falls back to the run-scoped name the +// Create response confirmed. Unconfirmed entries never reach here without a +// UID (see resolveIntentUID). +// +// Omitting the precondition entirely is required in that case — +// metav1.Preconditions{UID: &""} is NOT equivalent to no precondition: the +// apiserver compares it against the live object's UID and rejects every +// delete with a Conflict, which ignoreNotFoundOrConflict then swallows as +// success, leaking the object this entry exists to reclaim. func uidPreconditions(uid types.UID) *metav1.Preconditions { if uid == "" { return nil diff --git a/pkg/k8s/agent/deployer_test.go b/pkg/k8s/agent/deployer_test.go index 9a9278442..f351137d1 100644 --- a/pkg/k8s/agent/deployer_test.go +++ b/pkg/k8s/agent/deployer_test.go @@ -28,6 +28,7 @@ import ( "time" aicrerrors "github.com/NVIDIA/aicr/pkg/errors" + "github.com/NVIDIA/aicr/pkg/header" "github.com/NVIDIA/aicr/pkg/k8s/labels" "github.com/NVIDIA/aicr/pkg/k8s/pod" authv1 "k8s.io/api/authorization/v1" @@ -912,37 +913,152 @@ func TestCleanupPassesUIDPrecondition(t *testing.T) { } } -// TestCleanupDeletesUnconfirmedCreateByBareName covers the lost-Create-response -// path: recordIntent enters an object BEFORE its Create, so an entry can reach -// Cleanup with the zero UID. Such a delete must omit Preconditions entirely. +// TestCleanupResolvesUnconfirmedEntryBeforeDeleting covers the +// lost-Create-response path: recordIntent enters an object BEFORE its Create, +// so an entry can reach Cleanup with no UID and no Create response ever having +// named it. The run-scoped name alone is not ownership evidence — it says what +// this run WOULD have created, not what is standing there now — so Cleanup +// must recover the UID from the live object and prove that object carries this +// run's labels before deleting anything. // +// The "replaced under the same name" case is the one this replaces a bare-name +// delete for: an object at that name with a different UID and someone else's +// labels must survive, and the operator must be told it was left behind. +// +// Deleting by bare name with no Preconditions would collect the replacement. // Passing &"" instead would be worse than useless: the apiserver would compare // the empty UID against the live object's real one, reject every attempt with -// a Conflict, and ignoreNotFoundOrConflict would swallow that as success — -// leaking the very object this entry exists to reclaim. A bare-name delete is -// safe here because the name carries this run's ID. -func TestCleanupDeletesUnconfirmedCreateByBareName(t *testing.T) { +// a Conflict, and ignoreNotFoundOrConflict would swallow that as success. +func TestCleanupResolvesUnconfirmedEntryBeforeDeleting(t *testing.T) { + const ns = "test-ns" + d := NewDeployer(fake.NewClientset(), Config{Namespace: ns, RunID: testRunID}) + saName := d.saName() + ourLabels := d.objectLabels() + + // A replacement created after this run's object was deleted: same name, + // different identity. Carries a foreign run's labels, which is what a + // second aicr run standing an object up at this name would stamp. + foreignLabels := map[string]string{ + labels.Name: labels.ValueAICR, + labels.ManagedBy: labels.ValueAICR, + labels.Component: labels.ValueSnapshotAgent, + labels.RunID: "20260822-090000-0011223344556677", + } + + tests := []struct { + name string + seed *corev1.ServiceAccount // nil: nothing at the name + wantDelete bool + wantUID types.UID // expected Preconditions.UID when wantDelete + wantWarn bool + }{ + { + name: "this run's object is deleted pinned to the recovered UID", + seed: &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{ + Name: saName, Namespace: ns, UID: types.UID("ours-uid"), Labels: ourLabels, + }}, + wantDelete: true, + wantUID: types.UID("ours-uid"), + }, + { + name: "nothing at the name issues no delete", + wantDelete: false, + }, + { + name: "a replacement under the same name survives", + seed: &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{ + Name: saName, Namespace: ns, UID: types.UID("replacement-uid"), Labels: foreignLabels, + }}, + wantDelete: false, + wantWarn: true, + }, + { + name: "an unlabeled object under the same name survives", + seed: &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{ + Name: saName, Namespace: ns, UID: types.UID("operators-uid"), + }}, + wantDelete: false, + wantWarn: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + logs := captureLogs(t) + client := fake.NewClientset() + if tt.seed != nil { + if _, err := client.CoreV1().ServiceAccounts(ns).Create(ctx, tt.seed, metav1.CreateOptions{}); err != nil { + t.Fatalf("seed: %v", err) + } + } + deletes := spyOnDeletes(client) + + run := NewDeployer(client, Config{Namespace: ns, RunID: testRunID}) + run.recordIntent(kindServiceAccount, saName) + + if err := run.Cleanup(ctx, CleanupOptions{Enabled: true}); err != nil { + t.Fatalf("Cleanup() error = %v", err) + } + + observed := deletes() + if !tt.wantDelete { + if len(observed) != 0 { + t.Fatalf("Cleanup issued %d deletes, want none: %+v", len(observed), observed) + } + if tt.seed != nil { + live, err := client.CoreV1().ServiceAccounts(ns).Get(ctx, saName, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Cleanup deleted an object this run cannot prove it created: %v", err) + } + if live.UID != tt.seed.UID { + t.Errorf("surviving object UID = %q, want %q", live.UID, tt.seed.UID) + } + } + } else { + if len(observed) != 1 { + t.Fatalf("Cleanup issued %d deletes, want 1: %+v", len(observed), observed) + } + if observed[0].uid == nil || *observed[0].uid != tt.wantUID { + t.Errorf("delete Preconditions.UID = %v, want %q", observed[0].uid, tt.wantUID) + } + } + + gotWarn := strings.Contains(logs.String(), "cannot prove this run created") + if gotWarn != tt.wantWarn { + t.Errorf("warned about an ambiguous orphan = %v, want %v; logs: %s", gotWarn, tt.wantWarn, logs.String()) + } + if tt.wantWarn && !strings.Contains(logs.String(), saName) { + t.Errorf("warning does not name the object left behind; logs: %s", logs.String()) + } + }) + } +} + +// TestCleanupSurfacesIntentResolutionGetError fails closed on an apiserver +// error other than NotFound while resolving an unconfirmed entry: cleanup can +// neither prove the object is ours nor prove it is gone, so it must report the +// failure rather than delete blind or silently skip. +func TestCleanupSurfacesIntentResolutionGetError(t *testing.T) { ctx := context.Background() client := fake.NewClientset() + client.PrependReactor("get", "serviceaccounts", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewInternalError(errors.New("apiserver exploded")) + }) deletes := spyOnDeletes(client) - d := NewDeployer(client, Config{Namespace: "test-ns"}) - d.recordIntent(kindServiceAccount, "aicr-20260821-142233-9f3a1c0b7e2d4a55") - - if err := d.Cleanup(ctx, CleanupOptions{Enabled: true}); err != nil { - t.Fatalf("Cleanup() error = %v", err) - } + d := NewDeployer(client, Config{Namespace: "test-ns", RunID: testRunID}) + d.recordIntent(kindServiceAccount, d.saName()) - observed := deletes() - if len(observed) != 1 { - t.Fatalf("Cleanup issued %d deletes, want 1: %+v", len(observed), observed) + err := d.Cleanup(ctx, CleanupOptions{Enabled: true}) + if err == nil { + t.Fatal("Cleanup() error = nil, want the unexpected Get error surfaced") } - if observed[0].name != "aicr-20260821-142233-9f3a1c0b7e2d4a55" { - t.Errorf("delete name = %q, want the recorded run-scoped name", observed[0].name) + if !strings.Contains(err.Error(), d.saName()) { + t.Errorf("error %q does not name the unresolved object", err) } - if observed[0].uid != nil { - t.Errorf("delete carried Preconditions.UID = %q; a zero-UID entry must delete by bare name", - *observed[0].uid) + if observed := deletes(); len(observed) != 0 { + t.Errorf("Cleanup issued %d deletes despite an unresolvable entry: %+v", len(observed), observed) } } @@ -953,23 +1069,33 @@ func TestCleanupDeletesUnconfirmedCreateByBareName(t *testing.T) { // reclaims it, so the orphan would be permanent. // // The reactor below reproduces exactly that: the object is written into the -// tracker and THEN an error is returned, so ensureServiceAccount fails while -// the ServiceAccount exists. Cleanup must still delete it. +// tracker — with a UID, as a real apiserver assigns and the fake ObjectTracker +// does not — and THEN an error is returned, so ensureServiceAccount fails while +// the ServiceAccount exists. Cleanup must still delete it, and (since the +// Create response never named it) must delete it pinned to the UID it +// recovers from the live object, not by bare name. func TestEnsureRecordsIntentBeforeCreate(t *testing.T) { ctx := context.Background() const ns = "test-ns" client := fake.NewClientset() + deletes := spyOnDeletes(client) d := NewDeployer(client, Config{Namespace: ns, RunID: testRunID}) saName := d.saName() + committedUID := types.UID(saName + "-uid") client.PrependReactor("create", "serviceaccounts", func(action k8stesting.Action) (bool, runtime.Object, error) { ca, ok := action.(k8stesting.CreateActionImpl) if !ok { return false, nil, nil } - // Commit the object the way a real apiserver would... - if err := client.Tracker().Create(ca.GetResource(), ca.GetObject(), ns); err != nil { + sa, ok := ca.GetObject().(*corev1.ServiceAccount) + if !ok { + return false, nil, nil + } + // Commit the object the way a real apiserver would, UID and all... + sa.UID = committedUID + if err := client.Tracker().Create(ca.GetResource(), sa, ns); err != nil { return true, nil, err } // ...then lose the response on the way back to the client. @@ -992,6 +1118,15 @@ func TestEnsureRecordsIntentBeforeCreate(t *testing.T) { if _, err := client.CoreV1().ServiceAccounts(ns).Get(ctx, saName, metav1.GetOptions{}); !apierrors.IsNotFound(err) { t.Errorf("Cleanup leaked the ServiceAccount created by the lost-response Create; Get err = %v", err) } + + observed := deletes() + if len(observed) != 1 { + t.Fatalf("Cleanup issued %d deletes, want 1: %+v", len(observed), observed) + } + if observed[0].uid == nil || *observed[0].uid != committedUID { + t.Errorf("delete Preconditions.UID = %v, want the UID recovered from the live object (%q)", + observed[0].uid, committedUID) + } } // TestEnsureDiscardsIntentOnAlreadyExists is recordIntent's counterweight: an @@ -1191,6 +1326,13 @@ func TestCleanupDeletesStagingConfigMapWhenOwned(t *testing.T) { // its UID, so nothing was recorded. With run-scoped naming that would leak one // ConfigMap per failed run, so Cleanup Gets it by its run-scoped name and // deletes it pinned to the UID that Get returned. +// +// The sweep is licensed by this run holding a CONFIRMED Job — the only thing +// that can produce a staging ConfigMap is the in-pod agent that Job runs — so +// the Job is recorded here as Deploy would have recorded it. The seeded +// ConfigMap carries the label set pkg/serializer's ConfigMapWriter actually +// stamps from inside the pod: app.kubernetes.io/name plus component and +// version, and NO aicr.run/run-id. func TestCleanupSweepsUnrecordedStagingConfigMap(t *testing.T) { ctx := context.Background() name := StagingConfigMapName(testRunID) @@ -1199,6 +1341,7 @@ func TestCleanupSweepsUnrecordedStagingConfigMap(t *testing.T) { Name: name, Namespace: "test-namespace", UID: types.UID("staging-uid"), + Labels: stagingConfigMapLabels(), }, Data: map[string]string{"snapshot.yaml": "data"}, } @@ -1224,6 +1367,7 @@ func TestCleanupSweepsUnrecordedStagingConfigMap(t *testing.T) { }) // Deliberately no getSnapshotFromConfigMap call: this is the failed run. + d.recordCreated(kindJob, d.jobName(), types.UID("job-uid")) if d.hasCreated(kindConfigMap) { t.Fatal("precondition: created-set must not hold the staging ConfigMap") } @@ -1240,6 +1384,137 @@ func TestCleanupSweepsUnrecordedStagingConfigMap(t *testing.T) { } } +// stagingConfigMapLabels returns the label set pkg/serializer's +// ConfigMapWriter stamps on the staging ConfigMap it writes from inside the +// agent pod (Serialize in pkg/serializer/configmap.go). Deliberately NOT +// objectLabels(): that object is written by the in-pod agent rather than by +// this controller, so it carries neither aicr.run/run-id nor managed-by — +// which is why the sweep's ownership evidence is the confirmed Job, and why +// the check on the object itself can only be app.kubernetes.io/name. +func stagingConfigMapLabels() map[string]string { + return map[string]string{ + labels.Name: labels.ValueAICR, + labels.Component: header.KindSnapshot.String(), + } +} + +// TestCleanupDuplicateRunIDKeepsFirstRunsStagingConfigMap is the +// duplicate-RunID failure case. Config.RunID is public SDK surface and +// deliberately settable (pinned e2e/chainsaw runs), so a second run can +// resolve the first run's exact staging name. +// +// Run B reuses run A's RunID and fails on its very first AlreadyExists, before +// recording anything. Its deferred Cleanup still runs — Cleanup is registered +// before Deploy — and must not sweep the staging ConfigMap run A is still +// using: run B never created a Job, so nothing it did could have produced a +// ConfigMap at that name. +func TestCleanupDuplicateRunIDKeepsFirstRunsStagingConfigMap(t *testing.T) { + ctx := context.Background() + const ns = "test-namespace" + stagingName := StagingConfigMapName(testRunID) + + client := fake.NewClientset() + client.PrependReactor("create", "selfsubjectaccessreviews", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, &authv1.SelfSubjectAccessReview{ + Status: authv1.SubjectAccessReviewStatus{Allowed: true, Reason: "test permissions allowed"}, + }, nil + }) + + runA := NewDeployer(client, Config{ + Namespace: ns, + Image: "aicr:test", + RunID: testRunID, + Output: "cm://" + ns + "/" + stagingName, + OwnsOutputConfigMap: true, + }) + if err := runA.Deploy(ctx); err != nil { + t.Fatalf("run A Deploy() error = %v", err) + } + // Run A's in-pod agent has staged its result; Deploy() itself never + // writes this object, so seed it the way the agent would. + if _, err := client.CoreV1().ConfigMaps(ns).Create(ctx, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: stagingName, + Namespace: ns, + UID: types.UID("run-a-staging-uid"), + Labels: stagingConfigMapLabels(), + }, + Data: map[string]string{"snapshot.yaml": "run A's snapshot"}, + }, metav1.CreateOptions{}); err != nil { + t.Fatalf("seed run A staging ConfigMap: %v", err) + } + + deletes := spyOnDeletes(client) + + // Run B pins the SAME run ID and therefore collides on run A's + // ServiceAccount, the first object Deploy creates. + runB := NewDeployer(client, Config{ + Namespace: ns, + Image: "aicr:test", + RunID: testRunID, + Output: "cm://" + ns + "/" + stagingName, + OwnsOutputConfigMap: true, + }) + if err := runB.Deploy(ctx); err == nil { + t.Fatal("run B Deploy() = nil error, want AlreadyExists on the duplicate RunID") + } + if err := runB.Cleanup(ctx, CleanupOptions{Enabled: true}); err != nil { + t.Fatalf("run B Cleanup() error = %v", err) + } + + cm, err := client.CoreV1().ConfigMaps(ns).Get(ctx, stagingName, metav1.GetOptions{}) + if err != nil { + t.Fatalf("run B's failed Cleanup deleted run A's staging ConfigMap: %v", err) + } + if cm.UID != types.UID("run-a-staging-uid") { + t.Errorf("staging ConfigMap UID = %q, want run A's %q", cm.UID, "run-a-staging-uid") + } + if observed := deletes(); len(observed) != 0 { + t.Errorf("run B's Cleanup issued %d deletes despite creating nothing: %+v", len(observed), observed) + } +} + +// TestCleanupSweepKeepsForeignConfigMapAtStagingName is the sweep's own +// fail-closed check, downstream of the confirmed-Job gate: a ConfigMap parked +// at this run's staging name that does not carry app.kubernetes.io/name=aicr +// was not written by pkg/serializer's in-pod writer, so this run did not +// produce it. It must survive, and the operator must hear about it. +func TestCleanupSweepKeepsForeignConfigMapAtStagingName(t *testing.T) { + ctx := context.Background() + const ns = "test-namespace" + name := StagingConfigMapName(testRunID) + logs := captureLogs(t) + + client := fake.NewClientset(&corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + UID: types.UID("someone-elses-uid"), + Labels: map[string]string{"app.kubernetes.io/name": "not-aicr"}, + }, + Data: map[string]string{"unrelated": "data"}, + }) + + d := NewDeployer(client, Config{ + Namespace: ns, + RunID: testRunID, + Output: "cm://" + ns + "/" + name, + OwnsOutputConfigMap: true, + }) + d.recordCreated(kindJob, d.jobName(), types.UID("job-uid")) + + if err := d.Cleanup(ctx, CleanupOptions{Enabled: true}); err != nil { + t.Fatalf("Cleanup() error = %v", err) + } + + if _, err := client.CoreV1().ConfigMaps(ns).Get(ctx, name, metav1.GetOptions{}); err != nil { + t.Errorf("Cleanup deleted a ConfigMap this run did not write: %v", err) + } + if !strings.Contains(logs.String(), name) { + t.Errorf("no warning naming the ConfigMap left behind; logs: %s", logs.String()) + } +} + // TestCleanupSkipsStagingConfigMapSweepWhenNotOwned asserts the sweep stays // ownership-scoped: a caller-supplied cm:// Output is the caller's artifact // (OwnsOutputConfigMap false) and must survive Cleanup even when it happens to @@ -1285,6 +1560,7 @@ func TestCleanupSweepNoOpWhenStagingConfigMapAbsent(t *testing.T) { Output: "cm://test-namespace/" + StagingConfigMapName(testRunID), OwnsOutputConfigMap: true, }) + d.recordCreated(kindJob, d.jobName(), types.UID("job-uid")) if err := d.Cleanup(ctx, CleanupOptions{Enabled: true}); err != nil { t.Fatalf("Cleanup() error = %v, want nil when the staging ConfigMap was never written", err) @@ -1308,6 +1584,7 @@ func TestCleanupSweepSurfacesUnexpectedGetError(t *testing.T) { Output: "cm://test-namespace/" + StagingConfigMapName(testRunID), OwnsOutputConfigMap: true, }) + d.recordCreated(kindJob, d.jobName(), types.UID("job-uid")) err := d.Cleanup(ctx, CleanupOptions{Enabled: true}) if err == nil { diff --git a/pkg/k8s/agent/types.go b/pkg/k8s/agent/types.go index e42338a13..cb4a4993c 100644 --- a/pkg/k8s/agent/types.go +++ b/pkg/k8s/agent/types.go @@ -50,13 +50,21 @@ const ( // under; uid pins the eventual delete via metav1.Preconditions so a // same-named object belonging to a different run is never collected. // -// uid is the zero UID for an entry recorded by recordIntent but never -// confirmed by a Create response. Such an entry is deleted by bare name (no -// Preconditions) — see uidPreconditions. +// confirmed records whether ownership was established at creation time — a +// Create response came back naming this object. It is the discriminator +// Cleanup keys off, NOT uid == "": ownership is a property of how the entry +// was obtained, and a run must never infer it from a name plus a freshly-read +// UID (a Get followed by a UID-pinned delete proves only that the object did +// not change between the two calls). +// +// An entry stays unconfirmed when recordIntent added it and no Create +// response ever arrived. Cleanup must then re-establish ownership from the +// live object's own labels before deleting it — see resolveIntentUID. type createdObject struct { - kind string - name string - uid types.UID + kind string + name string + uid types.UID + confirmed bool } // Config holds the configuration for deploying the agent. @@ -183,10 +191,13 @@ func (d *Deployer) objectLabels() map[string]string { // exists. Because the name is run-unique, no later run reclaims it either, // so the orphan is permanent. // -// Cleanup deletes a zero-UID entry by bare name (uidPreconditions returns -// nil), which is safe precisely because the name carries this run's ID. An -// AlreadyExists response is the one case that proves the object is NOT ours, -// so every ensure* discards the intent on that branch (see discardIntent). +// The entry is unconfirmed: no Create response ever named the object, so the +// run-scoped name is the only thing tying it to this run, and a name proves +// nothing about who created what currently sits at it. Cleanup therefore +// re-establishes ownership from the live object's labels before deleting it +// (resolveIntentUID) rather than deleting by bare name. An AlreadyExists +// response is the one case that proves the object is NOT ours, so every +// ensure* discards the intent on that branch (see discardIntent). // Safe for concurrent use. func (d *Deployer) recordIntent(kind, name string) { d.mu.Lock() @@ -194,7 +205,7 @@ func (d *Deployer) recordIntent(kind, name string) { d.created = append(d.created, createdObject{kind: kind, name: name}) } -// discardIntent drops the zero-UID entry recordIntent added for (kind, +// discardIntent drops the unconfirmed entry recordIntent added for (kind, // name). Called only when a Create returns AlreadyExists: the object at that // name exists but this run did not create it, so it must not enter this // run's delete list. Safe for concurrent use. @@ -202,7 +213,7 @@ func (d *Deployer) discardIntent(kind, name string) { d.mu.Lock() defer d.mu.Unlock() for i := range d.created { - if d.created[i].kind == kind && d.created[i].name == name && d.created[i].uid == "" { + if d.created[i].kind == kind && d.created[i].name == name && !d.created[i].confirmed { d.created = append(d.created[:i], d.created[i+1:]...) return } @@ -215,35 +226,66 @@ func (d *Deployer) discardIntent(kind, name string) { // staging ConfigMap it observes but does not itself create — must call this // on success. // -// It upserts: when recordIntent already entered (kind, name) with the zero -// UID, the observed UID is written onto that entry rather than appended as a -// second one, so the set holds one entry per object and jobUID() sees the -// real Job UID. Safe for concurrent use. +// It marks the entry confirmed — this is the point at which this run's +// ownership of the object is established, and the only place that flag is +// set. +// +// It upserts: when recordIntent already entered (kind, name) unconfirmed, the +// observed UID is written onto that entry rather than appended as a second +// one, so the set holds one entry per object and jobUID() sees the real Job +// UID. Safe for concurrent use. func (d *Deployer) recordCreated(kind, name string, uid types.UID) { d.mu.Lock() defer d.mu.Unlock() for i := range d.created { - if d.created[i].kind == kind && d.created[i].name == name && d.created[i].uid == "" { + if d.created[i].kind == kind && d.created[i].name == name && !d.created[i].confirmed { d.created[i].uid = uid + d.created[i].confirmed = true return } } - d.created = append(d.created, createdObject{kind: kind, name: name, uid: uid}) + d.created = append(d.created, createdObject{kind: kind, name: name, uid: uid, confirmed: true}) } -// containsKind reports whether objs holds an entry of kind. Cleanup uses it -// against the single created-set snapshot it already took, rather than -// re-entering the mutex: reading the set twice would let a recordCreated +// needsStagingConfigMapSweep reports whether Cleanup must look for a staging +// ConfigMap that is not in the created-set, given that set. Both halves of +// the answer are read from the single snapshot Cleanup already took, rather +// than re-entering the mutex: reading the set twice would let a recordCreated // landing between the two reads produce a snapshot that misses the staging // ConfigMap while the second read reports it present, skipping both the -// created-set delete and the name-based sweep and leaking the object. -func containsKind(objs []createdObject, kind string) bool { +// created-set delete and the sweep and leaking the object. +// +// Two conditions must hold. +// +// No ConfigMap entry: an entry means getSnapshotFromConfigMap already +// observed the object's UID, so Cleanup deletes it from the created-set and +// the sweep would be a redundant second delete of the same name. +// +// A CONFIRMED Job entry: the staging ConfigMap is written only by this run's +// in-pod agent, so this run cannot have produced one unless the apiserver +// confirmed the Job that runs that agent. Anything sitting at the staging +// name when no confirmed Job exists belongs to someone else — notably a +// caller that reused another run's RunID (Config.RunID is public SDK surface, +// deliberately settable for pinned e2e/chainsaw runs) and failed on its first +// AlreadyExists before recording anything. Sweeping there would delete the +// first run's live staging ConfigMap. +// +// An unconfirmed (recordIntent-only) Job entry deliberately does not qualify. +// It cannot be told apart from a duplicate-RunID collision whose AlreadyExists +// never came back, and the window it forfeits is empty in practice: Deploy +// aborts the moment that Create fails and Cleanup runs seconds later, far +// short of the time the agent needs to collect a snapshot and write it. +func needsStagingConfigMapSweep(objs []createdObject) bool { + var jobConfirmed bool for _, o := range objs { - if o.kind == kind { - return true + switch o.kind { + case kindConfigMap: + return false + case kindJob: + jobConfirmed = jobConfirmed || o.confirmed } } - return false + return jobConfirmed } // createdSnapshot returns a defensive copy of the created-set taken under diff --git a/pkg/k8s/agent/wait.go b/pkg/k8s/agent/wait.go index 53e581718..701e09556 100644 --- a/pkg/k8s/agent/wait.go +++ b/pkg/k8s/agent/wait.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "io" + "log/slog" "time" "github.com/NVIDIA/aicr/pkg/errors" @@ -84,8 +85,24 @@ func (d *Deployer) deleteStagingConfigMap(ctx context.Context, name string, uid // the in-pod agent wrote it but the controller never observed it — the run // failed between the agent's write and getSnapshotFromConfigMap, so there is // no recorded UID to pin the delete to. It Gets the ConfigMap first and pins -// the delete to the UID it observes there, so the sweep is still -// ownership-scoped rather than name-only. +// the delete to the UID it observes there. +// +// Ownership is NOT inferred from the name. Config.RunID is caller-settable +// (public SDK surface), so two runs can resolve the same staging name, and a +// Get plus a UID-pinned delete would only prove the object did not change +// between the two calls. Cleanup gates this call on a confirmed Job entry +// (needsStagingConfigMapSweep) — this run cannot have produced a staging +// ConfigMap without the Job whose in-pod agent writes it — and this function +// re-checks the object it finds. +// +// That re-check is app.kubernetes.io/name only. The staging ConfigMap is +// written by pkg/serializer's ConfigMapWriter from inside the pod, which +// stamps name/component/version and — unlike objectLabels() — no +// aicr.run/run-id and no managed-by, so createdByThisRun() does not apply to +// it. The check still rules out an unrelated ConfigMap parked at this name, and +// component is deliberately not required: its value is the snapshot Kind +// written by the agent image, which may be a different aicr version than the +// controller. // // Only called from Cleanup, and only when Config.OwnsOutputConfigMap is true. // That flag means Output is the run-scoped staging URI pkg/snapshotter builds @@ -104,6 +121,14 @@ func (d *Deployer) deleteUnrecordedStagingConfigMap(ctx context.Context) error { return errors.Wrap(errors.ErrCodeInternal, fmt.Sprintf("failed to get staging ConfigMap %s/%s", d.config.Namespace, name), err) } + if cm.Labels[labels.Name] != labels.ValueAICR { + slog.Warn("cleanup left behind the ConfigMap at this run's staging name: it does not look like an aicr snapshot artifact, so this run did not write it", + slog.String("namespace", d.config.Namespace), + slog.String("name", name), + slog.String("uid", string(cm.UID)), + slog.String("runID", d.config.RunID)) + return nil + } return d.deleteStagingConfigMap(ctx, cm.Name, cm.UID) } From 4598fbc0b478435debc73915ba7ca2cb782d5025 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Tue, 25 Aug 2026 11:33:15 -0700 Subject: [PATCH 44/56] feat(agent): run as an existing ServiceAccount and manage none of its RBAC Config.ServiceAccountName becomes exact-if-exists. When a ServiceAccount of exactly that name already exists in the namespace, the agent pod runs as it verbatim and the run creates no ServiceAccount, Role, RoleBinding, ClusterRole or ClusterRoleBinding -- and deletes none at cleanup. aicr adds and removes no permissions on an identity it did not create. Otherwise the value stays a prefix and behavior is unchanged. This is what a pre-created ServiceAccount carrying IRSA (eks.amazonaws.com/role-arn) or GKE Workload Identity (iam.gke.io/gcp-service-account) annotations needs: both providers pin trust to the ServiceAccount name -- IRSA conditions on system:serviceaccount::, GKE names PROJECT.svc.id.goog[/] and accepts no wildcard -- so a run-scoped name can never be trusted by either, and copying the annotations onto one would not help. Run isolation turned the flag into a prefix, which silently dropped those credentials on upgrade; the adoption-drift warning that noted it is replaced by an slog.Info naming the ServiceAccount in use and stating that aicr manages no RBAC for the run. An unset ServiceAccountName is never probed, so a stray ServiceAccount at the default base cannot capture a run. A Forbidden Get still falls back to prefix mode rather than failing, because `serviceaccounts get` is absent from CheckPermissions' pre-flight set. Cleanup needs no new branch: nothing is created, so nothing enters the created-set it builds its delete list from. Also extracts the Role and ClusterRole rule sets into namespacedRules and clusterRules so a second consumer cannot drift from them, and names the namespace / name / runID log attributes shared across this package. Signed-off-by: Alex Yuskauskas --- pkg/k8s/agent/consts.go | 11 ++ pkg/k8s/agent/deployer.go | 52 ++++-- pkg/k8s/agent/doc.go | 39 +++- pkg/k8s/agent/job.go | 2 +- pkg/k8s/agent/names.go | 16 ++ pkg/k8s/agent/permissions.go | 10 + pkg/k8s/agent/rbac.go | 220 +++++++++++++--------- pkg/k8s/agent/rbac_test.go | 351 +++++++++++++++++++++++++++-------- pkg/k8s/agent/types.go | 77 +++++++- pkg/k8s/agent/wait.go | 6 +- 10 files changed, 582 insertions(+), 202 deletions(-) diff --git a/pkg/k8s/agent/consts.go b/pkg/k8s/agent/consts.go index fe8ceb100..3706f340f 100644 --- a/pkg/k8s/agent/consts.go +++ b/pkg/k8s/agent/consts.go @@ -37,3 +37,14 @@ const ( // / ClusterRole / ClusterRoleBinding resources. rbacAPIGroup = "rbac.authorization.k8s.io" ) + +// Attribute keys shared by this package's structured log lines and error +// contexts. Named for the same reason the ctxKey* constants in names.go are: +// one spelling reaches every consumer that parses them, and a literal +// repeated across files does not drift. +const ( + attrNamespace = "namespace" + attrName = "name" + attrRunID = "runID" + attrServiceAccount = "serviceAccount" +) diff --git a/pkg/k8s/agent/deployer.go b/pkg/k8s/agent/deployer.go index 9952f3ee6..2d125fccf 100644 --- a/pkg/k8s/agent/deployer.go +++ b/pkg/k8s/agent/deployer.go @@ -65,27 +65,45 @@ func (d *Deployer) Deploy(ctx context.Context) error { return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to ensure namespace", err) } + // Step 1.5: Decide whether Config.ServiceAccountName names a + // ServiceAccount that already exists (use it verbatim, manage none of + // its permissions) or is a prefix for one this run creates and owns. + // It runs after ensureNamespace so the Get is issued against a + // namespace that exists, and before Step 2 because it decides whether + // Step 2 happens at all. + if err := d.resolveServiceAccount(ctx); err != nil { + return err + } + // Step 2: Create this run's RBAC. Every name carries the run ID, so // nothing here can already exist; an AlreadyExists is reported as an // error rather than adopted or overwritten. - if err := d.ensureServiceAccount(ctx); err != nil { - return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to create ServiceAccount", err) - } + // + // Skipped entirely in exact-ServiceAccount mode: aicr will not add or + // remove permissions on a ServiceAccount it did not create. Nothing is + // created, so nothing enters the created-set, so Cleanup deletes none + // of these kinds — the operator's grants outlive the run. Provision + // them once with ProvisionServiceAccountRoles. + if d.managesRBAC() { + if err := d.ensureServiceAccount(ctx); err != nil { + return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to create ServiceAccount", err) + } - if err := d.ensureRole(ctx); err != nil { - return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to create Role", err) - } + if err := d.ensureRole(ctx); err != nil { + return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to create Role", err) + } - if err := d.ensureRoleBinding(ctx); err != nil { - return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to create RoleBinding", err) - } + if err := d.ensureRoleBinding(ctx); err != nil { + return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to create RoleBinding", err) + } - if err := d.ensureClusterRole(ctx); err != nil { - return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to create ClusterRole", err) - } + if err := d.ensureClusterRole(ctx); err != nil { + return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to create ClusterRole", err) + } - if err := d.ensureClusterRoleBinding(ctx); err != nil { - return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to create ClusterRoleBinding", err) + if err := d.ensureClusterRoleBinding(ctx); err != nil { + return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to create ClusterRoleBinding", err) + } } // Step 3: Create this run's Job under its run-scoped name. @@ -293,10 +311,10 @@ func (d *Deployer) resolveIntentUID(ctx context.Context, obj createdObject) (uid if !d.createdByThisRun(live.GetLabels()) || live.GetUID() == "" { slog.Warn("cleanup left behind an object it cannot prove this run created; if it is a stale orphan of this run, remove it by hand", slog.String("kind", obj.kind), - slog.String("name", obj.name), - slog.String("namespace", live.GetNamespace()), + slog.String(attrName, obj.name), + slog.String(attrNamespace, live.GetNamespace()), slog.String("uid", string(live.GetUID())), - slog.String("runID", d.config.RunID), + slog.String(attrRunID, d.config.RunID), slog.String("objectRunID", live.GetLabels()[labels.RunID])) return "", false, nil } diff --git a/pkg/k8s/agent/doc.go b/pkg/k8s/agent/doc.go index 2395423a8..06cbb6195 100644 --- a/pkg/k8s/agent/doc.go +++ b/pkg/k8s/agent/doc.go @@ -25,17 +25,42 @@ Every deployment belongs to a single run identified by Config.RunID (generate one with runid.Generate). The run ID is suffixed onto every object this package creates — Job, ServiceAccount, Role, RoleBinding, ClusterRole, ClusterRoleBinding, and the staging ConfigMap — so concurrent runs never share -an object. Config.JobName and Config.ServiceAccountName are prefixes, not exact -names; when empty they fall back to Config.NameBase (default "aicr"). See -ADR-020 (docs/design/020-snapshot-agent-run-isolation.md). +an object. Config.JobName is a prefix, not an exact name; +Config.ServiceAccountName is a prefix only when no ServiceAccount of that +exact name exists (see Existing ServiceAccounts below). When empty they fall +back to Config.NameBase (default "aicr"). See ADR-020 +(docs/design/020-snapshot-agent-run-isolation.md). Because a run-scoped name cannot already belong to another run, creates are plain creates: there is no delete-and-recreate of the Job, and no create-or-update of the RBAC objects. An AlreadyExists implies a duplicate -RunID and is returned as an error rather than adopted or overwritten. The one -courtesy check is in ensureServiceAccount, which warns when a ServiceAccount -already exists under the bare (unscoped) prefix so a caller relying on the old -adoption behavior is not left guessing; it never blocks the deploy. +RunID and is returned as an error rather than adopted or overwritten. + +# Existing ServiceAccounts + +Config.ServiceAccountName is exact-if-exists. When a ServiceAccount of exactly +that name already exists in the namespace, the agent pod runs as it verbatim +and the run creates NO ServiceAccount, Role, RoleBinding, ClusterRole or +ClusterRoleBinding — aicr adds and removes no permissions on an identity it +did not create. Nothing of those kinds enters the created-set, so Cleanup has +nothing of those kinds to delete and the operator's grants outlive the run. + +This exists because IRSA (eks.amazonaws.com/role-arn) and GKE Workload +Identity (iam.gke.io/gcp-service-account) both pin trust to the ServiceAccount +NAME — IRSA's trust policy conditions on system:serviceaccount:/, +GKE's IAM binding names PROJECT.svc.id.goog[/] and accepts no +wildcard — so a per-run name can never be trusted by either, and copying the +annotations onto a run-scoped ServiceAccount would not help. + +Grant the agent's permissions to such a ServiceAccount once with +ProvisionServiceAccountRoles (CLI: aicr snapshot +--add-roles-to-service-account). What it creates is permanent: no run-ID +label, never in a created-set, never deleted by run cleanup. + +The trade-off is deliberate and opt-in: an adopted ServiceAccount waives +per-run permission isolation. Concurrent runs sharing it share its grants, and +a DiscoverNetwork provisioning leaves cluster-scoped mutating permissions in +place permanently rather than for one run's lifetime. Two objects are deliberately NOT run-scoped: diff --git a/pkg/k8s/agent/job.go b/pkg/k8s/agent/job.go index fb5dedd70..a1c76d1dc 100644 --- a/pkg/k8s/agent/job.go +++ b/pkg/k8s/agent/job.go @@ -93,7 +93,7 @@ func (d *Deployer) buildJob() *batchv1.Job { // When Privileged=false: PSS-compliant restricted pod, only K8s collector works. func (d *Deployer) buildPodSpec(args []string) corev1.PodSpec { spec := corev1.PodSpec{ - ServiceAccountName: d.saName(), + ServiceAccountName: d.podServiceAccountName(), RestartPolicy: corev1.RestartPolicyNever, NodeSelector: d.config.NodeSelector, Tolerations: d.config.Tolerations, diff --git a/pkg/k8s/agent/names.go b/pkg/k8s/agent/names.go index a9d92e92d..8759b9d3f 100644 --- a/pkg/k8s/agent/names.go +++ b/pkg/k8s/agent/names.go @@ -213,6 +213,22 @@ func (d *Deployer) jobName() string { return nameWithRunID(prefix, d.config.RunID) } +// podServiceAccountName returns the ServiceAccount the agent pod actually +// runs as: the operator's already-existing ServiceAccount when Deploy +// resolved Config.ServiceAccountName to an exact match, otherwise this run's +// own run-scoped one. +// +// It is deliberately separate from saName(), which stays the run-scoped +// name unconditionally. saName() also names the Role and RoleBinding, and +// those exist only in prefix mode — folding the exact name into it would +// make roleName() silently return an operator-owned ServiceAccount's name. +func (d *Deployer) podServiceAccountName() string { + if name := d.existingServiceAccount(); name != "" { + return name + } + return d.saName() +} + // saName returns the run-scoped name for the agent ServiceAccount. func (d *Deployer) saName() string { prefix := d.config.ServiceAccountName diff --git a/pkg/k8s/agent/permissions.go b/pkg/k8s/agent/permissions.go index 4bde39fc0..538ed6ed2 100644 --- a/pkg/k8s/agent/permissions.go +++ b/pkg/k8s/agent/permissions.go @@ -77,6 +77,16 @@ func (d *Deployer) CheckPermissions(ctx context.Context) ([]permissionCheck, err requiredChecks = append(requiredChecks, permCheck{resourceCM, verbDelete, d.config.Namespace}) } + // The RBAC create verbs above stay unconditional even though + // exact-ServiceAccount mode creates none of those objects + // (resolveServiceAccount). Narrowing them would mean resolving the + // ServiceAccount before this pre-flight, and this pre-flight is + // deliberately Deploy's first cluster call — the one thing that runs + // before any write. The cost of leaving them is that an operator using + // an existing ServiceAccount still needs the same grants they needed + // before, which is no regression; the cost of moving them would be a + // weaker fail-before-mutate guarantee. + // SelfSubjectAccessReview is a read-only query; running the N required // checks in parallel cuts wall-clock from N×RTT to one RTT against the // apiserver. Each iteration's index is preserved so the response slice diff --git a/pkg/k8s/agent/rbac.go b/pkg/k8s/agent/rbac.go index 0f391e21b..ff1847301 100644 --- a/pkg/k8s/agent/rbac.go +++ b/pkg/k8s/agent/rbac.go @@ -79,43 +79,65 @@ func (d *Deployer) ensureNamespace(ctx context.Context) error { return nil } -// ensureServiceAccount creates the run-scoped ServiceAccount for the agent. +// resolveServiceAccount decides, once per Deploy, whether this run creates +// and owns its own run-scoped ServiceAccount or runs as an +// already-existing one the operator named exactly. // -// Before creating, it checks whether a ServiceAccount already exists under -// the bare (unscoped) prefix name. Previously, a caller passing -// --service-account-name to target a ServiceAccount they created out of -// band (e.g. with cloud IAM annotations for IRSA/Workload Identity) got it -// silently adopted via IgnoreAlreadyExists. Now every run gets its own -// run-scoped ServiceAccount, so that adoption no longer happens; warn -// loudly instead of leaving the caller to discover it the hard way. A -// NotFound Get is the normal path and stays silent. +// Config.ServiceAccountName is exact-if-exists. When it is set and a +// ServiceAccount of exactly that name already exists in the namespace, the +// agent pod runs as that ServiceAccount verbatim and this run creates NO +// ServiceAccount, Role, RoleBinding, ClusterRole or ClusterRoleBinding — +// aicr manages no permissions for an identity it did not create. That is +// what keeps a pre-created ServiceAccount carrying IRSA +// (eks.amazonaws.com/role-arn) or GKE Workload Identity +// (iam.gke.io/gcp-service-account) annotations usable: both providers pin +// trust to the ServiceAccount NAME, so a per-run name can never be trusted +// by either and copying the annotations onto one would not help. // -// That Get is a diagnostic courtesy and must not gate the deployment: -// `serviceaccounts get` is deliberately absent from CheckPermissions' -// requiredChecks (permissions.go), so an identity scoped to exactly the -// pre-flight verb set would otherwise pass the pre-flight and then fail -// Deploy with an ErrCodeInternal — a permission problem reported as an -// internal error. Forbidden therefore downgrades to a debug line and the -// deployment proceeds; every other unexpected error still fails closed. -func (d *Deployer) ensureServiceAccount(ctx context.Context) error { - name := d.saName() - bareName := d.config.ServiceAccountName - if bareName == "" { - bareName = d.base() +// When the name does not exist, the value stays a prefix and the run +// creates - plus its RBAC, exactly as before. An unset +// Config.ServiceAccountName is never probed: the fallback base ("aicr") is +// aicr's own default, not something the operator asked for, so a stray +// ServiceAccount sitting at that name must not silently capture the run. +// +// The Get must not gate the deployment: `serviceaccounts get` is +// deliberately absent from CheckPermissions' requiredChecks +// (permissions.go), so an identity scoped to exactly the pre-flight verb +// set would otherwise pass the pre-flight and then fail Deploy with an +// ErrCodeInternal — a permission problem reported as an internal error. +// Forbidden therefore downgrades to a debug line and the run proceeds in +// prefix mode, which is the mode that identity has the permissions for. +// Every other unexpected error still fails closed. +func (d *Deployer) resolveServiceAccount(ctx context.Context) error { + name := d.config.ServiceAccountName + if name == "" { + return nil } - switch _, err := d.clientset.CoreV1().ServiceAccounts(d.config.Namespace).Get(ctx, bareName, metav1.GetOptions{}); { + + switch _, err := d.clientset.CoreV1().ServiceAccounts(d.config.Namespace).Get(ctx, name, metav1.GetOptions{}); { case err == nil: - slog.Warn("ServiceAccount already exists under the unscoped name; aicr is creating a run-scoped ServiceAccount instead of adopting it", - "existing", bareName, "creating", name) + d.setExistingServiceAccount(name) + slog.Info("using the existing ServiceAccount named by --service-account-name; aicr manages no RBAC for this run", + attrServiceAccount, name, + attrNamespace, d.config.Namespace, + attrRunID, d.config.RunID, + "note", "no ServiceAccount, Role, RoleBinding, ClusterRole or ClusterRoleBinding is created or deleted; grant the agent's permissions once with 'aicr snapshot --add-roles-to-service-account "+name+"'") case apierrors.IsNotFound(err): - // Normal path: nothing to warn about. + // Normal path: the value is a prefix and this run creates its own + // run-scoped ServiceAccount below. case apierrors.IsForbidden(err): - slog.Debug("skipping adoption-drift check: not permitted to read ServiceAccounts in this namespace", - "name", bareName, "namespace", d.config.Namespace, "error", err) + slog.Debug("cannot read ServiceAccounts in this namespace; treating --service-account-name as a prefix", + attrName, name, attrNamespace, d.config.Namespace, "error", err) default: - return errors.Wrap(errors.ErrCodeInternal, "failed to check for pre-existing ServiceAccount", err) + return errors.Wrap(errors.ErrCodeInternal, "failed to check for an existing ServiceAccount", err) } + return nil +} +// ensureServiceAccount creates the run-scoped ServiceAccount for the agent. +// Deploy calls it only in prefix mode; see resolveServiceAccount. +func (d *Deployer) ensureServiceAccount(ctx context.Context) error { + name := d.saName() sa := &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -139,6 +161,81 @@ func (d *Deployer) ensureServiceAccount(ctx context.Context) error { return nil } +// namespacedRules returns the namespace-scoped policy rules the agent needs: +// writing its snapshot result into a staging ConfigMap, and reading pods. +// +// It is the single definition consumed by both the run-scoped Role +// ensureRole creates and the permanent Role ProvisionServiceAccountRoles +// grants to an operator-supplied ServiceAccount, so the two can never drift. +func namespacedRules() []rbacv1.PolicyRule { + return []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{resourceCM}, + Verbs: []string{verbCreate, verbGet, "update", "patch"}, + }, + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{verbGet, verbList}, + }, + } +} + +// clusterRules returns the cluster-scoped policy rules the agent needs. The +// baseline set is read-only; discoverNetwork appends the mutating rules live +// l8k network discovery requires (see discoverNetworkClusterRules). +// +// It is the single definition consumed by both the run-scoped ClusterRole +// ensureClusterRole creates and the permanent ClusterRole +// ProvisionServiceAccountRoles grants. +func clusterRules(discoverNetwork bool) []rbacv1.PolicyRule { + rules := []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"nodes"}, + Verbs: []string{verbGet, verbList}, + }, + { + APIGroups: []string{""}, + Resources: []string{"pods"}, + Verbs: []string{verbGet, verbList}, + }, + { + APIGroups: []string{"nvidia.com"}, + Resources: []string{"clusterpolicies"}, + Verbs: []string{verbGet, verbList}, + }, + { + APIGroups: []string{slinkyAPIGroup}, + Resources: []string{ + slinkyControllerResource, + slinkyNodeSetResource, + slinkyLoginSetResource, + slinkyRestAPIResource, + slinkyAccountingResource, + }, + Verbs: []string{verbList}, + }, + { + APIGroups: []string{mariaDBAPIGroup}, + Resources: []string{mariaDBResource}, + Verbs: []string{verbList}, + }, + } + + // Live l8k network discovery stands up a nic-configuration-daemon + // DaemonSet in its own namespace, exec's into the daemon pods, + // writes nvidia.kubernetes-launch-kit.{machine,gpu} labels onto + // nodes, and patches mellanox.com NicClusterPolicy via server-side + // apply. Grant the extra cluster-scoped rules only when discovery was + // opted into so non-network snapshots stay minimal-priv. + if discoverNetwork { + rules = append(rules, discoverNetworkClusterRules()...) + } + return rules +} + // ensureRole creates the run-scoped Role for ConfigMap access. func (d *Deployer) ensureRole(ctx context.Context) error { name := d.roleName() @@ -148,18 +245,7 @@ func (d *Deployer) ensureRole(ctx context.Context) error { Namespace: d.config.Namespace, Labels: d.objectLabels(), }, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{resourceCM}, - Verbs: []string{verbCreate, verbGet, "update", "patch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"get", verbList}, - }, - }, + Rules: namespacedRules(), } d.recordIntent(kindRole, name) @@ -186,14 +272,14 @@ func (d *Deployer) ensureRoleBinding(ctx context.Context) error { }, Subjects: []rbacv1.Subject{ { - Kind: "ServiceAccount", + Kind: kindServiceAccount, Name: d.saName(), Namespace: d.config.Namespace, }, }, RoleRef: rbacv1.RoleRef{ APIGroup: rbacAPIGroup, - Kind: "Role", + Kind: kindRole, Name: d.roleName(), }, } @@ -213,57 +299,13 @@ func (d *Deployer) ensureRoleBinding(ctx context.Context) error { // ensureClusterRole creates the run-scoped ClusterRole for node and cluster-wide resource access. func (d *Deployer) ensureClusterRole(ctx context.Context) error { - rules := []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{"nodes"}, - Verbs: []string{verbGet, verbList}, - }, - { - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{verbGet, verbList}, - }, - { - APIGroups: []string{"nvidia.com"}, - Resources: []string{"clusterpolicies"}, - Verbs: []string{verbGet, verbList}, - }, - { - APIGroups: []string{slinkyAPIGroup}, - Resources: []string{ - slinkyControllerResource, - slinkyNodeSetResource, - slinkyLoginSetResource, - slinkyRestAPIResource, - slinkyAccountingResource, - }, - Verbs: []string{verbList}, - }, - { - APIGroups: []string{mariaDBAPIGroup}, - Resources: []string{mariaDBResource}, - Verbs: []string{verbList}, - }, - } - - // Live l8k network discovery stands up a nic-configuration-daemon - // DaemonSet in its own namespace, exec's into the daemon pods, - // writes nvidia.kubernetes-launch-kit.{machine,gpu} labels onto - // nodes, and patches mellanox.com NicClusterPolicy via server-side - // apply. Grant the extra cluster-scoped rules only when the snapshot - // opted into discovery so non-network snapshots stay minimal-priv. - if d.config.DiscoverNetwork { - rules = append(rules, discoverNetworkClusterRules()...) - } - name := d.clusterRoleName() cr := &rbacv1.ClusterRole{ ObjectMeta: metav1.ObjectMeta{ Name: name, Labels: d.objectLabels(), }, - Rules: rules, + Rules: clusterRules(d.config.DiscoverNetwork), } d.recordIntent(kindClusterRole, name) @@ -289,14 +331,14 @@ func (d *Deployer) ensureClusterRoleBinding(ctx context.Context) error { }, Subjects: []rbacv1.Subject{ { - Kind: "ServiceAccount", + Kind: kindServiceAccount, Name: d.saName(), Namespace: d.config.Namespace, }, }, RoleRef: rbacv1.RoleRef{ APIGroup: rbacAPIGroup, - Kind: "ClusterRole", + Kind: kindClusterRole, Name: d.clusterRoleName(), }, } diff --git a/pkg/k8s/agent/rbac_test.go b/pkg/k8s/agent/rbac_test.go index fd03e2c60..a541aca5c 100644 --- a/pkg/k8s/agent/rbac_test.go +++ b/pkg/k8s/agent/rbac_test.go @@ -33,6 +33,10 @@ import ( k8stesting "k8s.io/client-go/testing" ) +// testNamespace is the namespace every ServiceAccount-resolution test +// deploys into. +const testNamespace = "test-ns" + // captureLogs redirects the default slog logger into a buffer at debug level // for the duration of the test and returns the buffer. func captureLogs(t *testing.T) *bytes.Buffer { @@ -44,53 +48,64 @@ func captureLogs(t *testing.T) *bytes.Buffer { return &buf } -// TestEnsureServiceAccount_AdoptionDriftCheck covers every branch of the -// bare-name Get that warns when a ServiceAccount already exists under the -// unscoped prefix (ADR-020's adoption-drift warning). +// TestResolveServiceAccount covers every branch of the exact-if-exists +// resolution of Config.ServiceAccountName: whether the run adopts an +// operator-supplied ServiceAccount verbatim (and manages none of its +// permissions) or treats the value as a prefix and creates its own. // // The Forbidden branch is the load-bearing one: `serviceaccounts get` is NOT // in CheckPermissions' requiredChecks, so an identity holding exactly the -// pre-flight verb set must still be able to deploy. That Get is a diagnostic -// courtesy and must never gate the deployment. -func TestEnsureServiceAccount_AdoptionDriftCheck(t *testing.T) { +// pre-flight verb set must still be able to deploy. It falls back to prefix +// mode, which is the mode that identity has the permissions for. +func TestResolveServiceAccount(t *testing.T) { saGR := schema.GroupResource{Group: "", Resource: "serviceaccounts"} - saGVR := corev1.SchemeGroupVersion.WithResource("serviceaccounts") tests := []struct { name string + // configured is Config.ServiceAccountName. + configured string + // seeded, when non-empty, pre-creates a ServiceAccount of that + // name in the namespace. + seeded string // getErr, when non-nil, is returned by every ServiceAccount Get. getErr error - // existingBareSA pre-creates a ServiceAccount under the unscoped - // prefix name. - existingBareSA bool - wantErr bool - wantCreated bool - wantLogSubstr string - notWantLog string + // wantExisting is the ServiceAccount the run should adopt + // verbatim; "" means prefix mode. + wantExisting string + wantErr bool + wantLogSubstr string + notWantLog string }{ { - name: "pre-existing unscoped ServiceAccount warns but still creates the run-scoped one", - existingBareSA: true, - wantCreated: true, - wantLogSubstr: "already exists under the unscoped name", + name: "exact match adopts the operator's ServiceAccount", + configured: "irsa-snapshotter", + seeded: "irsa-snapshotter", + wantExisting: "irsa-snapshotter", + wantLogSubstr: "aicr manages no RBAC for this run", + }, + { + name: "no match keeps prefix mode", + configured: "irsa-snapshotter", + notWantLog: "aicr manages no RBAC for this run", }, { - name: "no pre-existing ServiceAccount is the silent normal path", - wantCreated: true, - notWantLog: "already exists under the unscoped name", + name: "unset name is never probed, even when the base name exists", + seeded: testName, + notWantLog: "aicr manages no RBAC for this run", }, { - name: "forbidden Get does not block the deployment", - getErr: apierrors.NewForbidden(saGR, testName, stderrors.New("no get permission")), - wantCreated: true, - wantLogSubstr: "skipping adoption-drift check", - notWantLog: "already exists under the unscoped name", + name: "forbidden Get falls back to prefix mode", + configured: "irsa-snapshotter", + seeded: "irsa-snapshotter", + getErr: apierrors.NewForbidden(saGR, "irsa-snapshotter", stderrors.New("no get permission")), + wantLogSubstr: "treating --service-account-name as a prefix", + notWantLog: "aicr manages no RBAC for this run", }, { - name: "unexpected Get error fails closed", - getErr: apierrors.NewInternalError(stderrors.New("apiserver exploded")), - wantErr: true, - wantCreated: false, + name: "unexpected Get error fails closed", + configured: "irsa-snapshotter", + getErr: apierrors.NewInternalError(stderrors.New("apiserver exploded")), + wantErr: true, }, } @@ -100,15 +115,11 @@ func TestEnsureServiceAccount_AdoptionDriftCheck(t *testing.T) { buf := captureLogs(t) clientset := fake.NewClientset() - if tt.existingBareSA { - if _, err := clientset.CoreV1().ServiceAccounts("test-ns").Create(ctx, &corev1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{ - Name: testName, - Namespace: "test-ns", - Annotations: map[string]string{"eks.amazonaws.com/role-arn": "arn:aws:iam::123456789012:role/example"}, - }, + if tt.seeded != "" { + if _, err := clientset.CoreV1().ServiceAccounts(testNamespace).Create(ctx, &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{Name: tt.seeded, Namespace: testNamespace}, }, metav1.CreateOptions{}); err != nil { - t.Fatalf("seeding unscoped ServiceAccount: %v", err) + t.Fatalf("seeding ServiceAccount: %v", err) } } if tt.getErr != nil { @@ -117,44 +128,34 @@ func TestEnsureServiceAccount_AdoptionDriftCheck(t *testing.T) { }) } - d := NewDeployer(clientset, Config{Namespace: "test-ns", RunID: testRunID}) - err := d.ensureServiceAccount(ctx) + d := NewDeployer(clientset, Config{ + Namespace: testNamespace, + ServiceAccountName: tt.configured, + RunID: testRunID, + }) + err := d.resolveServiceAccount(ctx) if (err != nil) != tt.wantErr { - t.Fatalf("ensureServiceAccount() error = %v, wantErr %v", err, tt.wantErr) + t.Fatalf("resolveServiceAccount() error = %v, wantErr %v", err, tt.wantErr) } if tt.wantErr && !stderrors.Is(err, aicrerrors.New(aicrerrors.ErrCodeInternal, "")) { t.Errorf("error = %v, want ErrCodeInternal", err) } - - // The run-scoped ServiceAccount is what the Job actually binds - // to; a suppressed diagnostic must not suppress its creation. - // Read through the tracker, not the clientset: the reactor above - // stands in for an identity that cannot read ServiceAccounts at - // all, and that must not also blind the assertions. - scoped := testName + "-" + testRunID - _, getErr := clientset.Tracker().Get(saGVR, "test-ns", scoped) - if gotCreated := getErr == nil; gotCreated != tt.wantCreated { - t.Errorf("run-scoped ServiceAccount %q created = %v (err %v), want %v", scoped, gotCreated, getErr, tt.wantCreated) + if got := d.existingServiceAccount(); got != tt.wantExisting { + t.Errorf("existingServiceAccount() = %q, want %q", got, tt.wantExisting) } - if tt.wantCreated && !d.hasCreated(kindServiceAccount) { - t.Error("created-set has no ServiceAccount entry; Cleanup would not delete it") + // managesRBAC is what gates Deploy's whole RBAC block, so + // assert it rather than inferring it from the field. + if got, want := d.managesRBAC(), tt.wantExisting == ""; got != want { + t.Errorf("managesRBAC() = %v, want %v", got, want) } - - if tt.existingBareSA { - // Adoption is exactly what this release stopped doing: the - // out-of-band ServiceAccount must be left untouched. - obj, bareErr := clientset.Tracker().Get(saGVR, "test-ns", testName) - if bareErr != nil { - t.Fatalf("unscoped ServiceAccount disappeared: %v", bareErr) - } - bare, ok := obj.(*corev1.ServiceAccount) - if !ok { - t.Fatalf("tracker returned %T, want *corev1.ServiceAccount", obj) - } - if bare.Annotations["eks.amazonaws.com/role-arn"] == "" { - t.Error("unscoped ServiceAccount annotations were modified") - } + // The pod must run as whichever ServiceAccount was resolved. + wantPodSA := tt.wantExisting + if wantPodSA == "" { + wantPodSA = d.saName() + } + if got := d.podServiceAccountName(); got != wantPodSA { + t.Errorf("podServiceAccountName() = %q, want %q", got, wantPodSA) } if tt.wantLogSubstr != "" && !strings.Contains(buf.String(), tt.wantLogSubstr) { @@ -167,20 +168,201 @@ func TestEnsureServiceAccount_AdoptionDriftCheck(t *testing.T) { } } -// TestDeploy_SucceedsWhenServiceAccountGetForbidden is the end-to-end shape of -// the same bug: an identity authorized for exactly CheckPermissions' -// requiredChecks (which do not include `serviceaccounts get`) passes the -// pre-flight, so Deploy must not then fail on the adoption-drift Get. -func TestDeploy_SucceedsWhenServiceAccountGetForbidden(t *testing.T) { +// TestDeploy_ExistingServiceAccountCreatesAndDeletesNoRBAC is the end-to-end +// contract of exact-ServiceAccount mode: aicr will not add or remove +// permissions on a ServiceAccount it did not create. +// +// It asserts the whole chain rather than just the flag — no RBAC object of +// any kind is created, the created-set holds none of those kinds, and +// therefore Cleanup (which builds its delete list from exactly that set) +// leaves the operator's ServiceAccount and everything around it alone. The +// Cleanup assertion is deliberately not "cleanup skips these kinds": there is +// no such branch in Cleanup, and the point is that none is needed. +func TestDeploy_ExistingServiceAccountCreatesAndDeletesNoRBAC(t *testing.T) { ctx := context.Background() captureLogs(t) + const saName = "irsa-snapshotter" + const roleARN = "arn:aws:iam::123456789012:role/aicr-snapshot" + clientset := fake.NewClientset() + allowAllPermissionChecks(clientset) + if _, err := clientset.CoreV1().ServiceAccounts(testNamespace).Create(ctx, &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: saName, + Namespace: testNamespace, + Annotations: map[string]string{"eks.amazonaws.com/role-arn": roleARN}, + }, + }, metav1.CreateOptions{}); err != nil { + t.Fatalf("seeding ServiceAccount: %v", err) + } + + d := NewDeployer(clientset, Config{ + Namespace: testNamespace, + ServiceAccountName: saName, + Image: "aicr:test", + RunID: testRunID, + DiscoverNetwork: true, + }) + if err := d.Deploy(ctx); err != nil { + t.Fatalf("Deploy() error = %v", err) + } + + // The Job — the one object this run still owns — must run as the + // operator's ServiceAccount verbatim, not as "-". + job, err := clientset.BatchV1().Jobs(testNamespace).Get(ctx, d.jobName(), metav1.GetOptions{}) + if err != nil { + t.Fatalf("Job not created: %v", err) + } + if got := job.Spec.Template.Spec.ServiceAccountName; got != saName { + t.Errorf("pod ServiceAccountName = %q, want %q", got, saName) + } + + // Nothing run-scoped exists for any RBAC kind. + if sas, listErr := clientset.CoreV1().ServiceAccounts(testNamespace).List(ctx, metav1.ListOptions{}); listErr != nil { + t.Fatalf("listing ServiceAccounts: %v", listErr) + } else if len(sas.Items) != 1 || sas.Items[0].Name != saName { + t.Errorf("ServiceAccounts = %v, want only the pre-existing %q", saNames(sas.Items), saName) + } + assertNoRBACObjects(ctx, t, clientset) + + // The created-set is what Cleanup deletes from, so its contents are + // the actual guarantee. + for _, kind := range []string{kindServiceAccount, kindRole, kindRoleBinding, kindClusterRole, kindClusterRoleBinding} { + if d.hasCreated(kind) { + t.Errorf("created-set holds a %s entry; Cleanup would delete an object this run did not create", kind) + } + } + + if err := d.Cleanup(ctx, CleanupOptions{Enabled: true}); err != nil { + t.Fatalf("Cleanup() error = %v", err) + } + + sa, err := clientset.CoreV1().ServiceAccounts(testNamespace).Get(ctx, saName, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Cleanup deleted the operator's ServiceAccount: %v", err) + } + if sa.Annotations["eks.amazonaws.com/role-arn"] != roleARN { + t.Errorf("IRSA annotation = %q, want %q", sa.Annotations["eks.amazonaws.com/role-arn"], roleARN) + } + assertNoRBACObjects(ctx, t, clientset) +} + +// TestDeploy_AbsentServiceAccountKeepsPrefixBehavior pins the other half of +// exact-if-exists: a --service-account-name naming nothing that exists is +// still a prefix, and the run creates and owns the full run-scoped RBAC set +// exactly as it did before. +func TestDeploy_AbsentServiceAccountKeepsPrefixBehavior(t *testing.T) { + ctx := context.Background() + captureLogs(t) + + clientset := fake.NewClientset() + allowAllPermissionChecks(clientset) + + d := NewDeployer(clientset, Config{ + Namespace: testNamespace, + ServiceAccountName: "irsa-snapshotter", + Image: "aicr:test", + RunID: testRunID, + }) + if err := d.Deploy(ctx); err != nil { + t.Fatalf("Deploy() error = %v", err) + } + + scoped := "irsa-snapshotter-" + testRunID + if d.saName() != scoped { + t.Fatalf("saName() = %q, want %q", d.saName(), scoped) + } + if _, err := clientset.CoreV1().ServiceAccounts(testNamespace).Get(ctx, scoped, metav1.GetOptions{}); err != nil { + t.Errorf("run-scoped ServiceAccount %q not created: %v", scoped, err) + } + if _, err := clientset.RbacV1().Roles(testNamespace).Get(ctx, scoped, metav1.GetOptions{}); err != nil { + t.Errorf("run-scoped Role %q not created: %v", scoped, err) + } + if _, err := clientset.RbacV1().RoleBindings(testNamespace).Get(ctx, scoped, metav1.GetOptions{}); err != nil { + t.Errorf("run-scoped RoleBinding %q not created: %v", scoped, err) + } + if _, err := clientset.RbacV1().ClusterRoles().Get(ctx, d.clusterRoleName(), metav1.GetOptions{}); err != nil { + t.Errorf("run-scoped ClusterRole %q not created: %v", d.clusterRoleName(), err) + } + if _, err := clientset.RbacV1().ClusterRoleBindings().Get(ctx, d.clusterRoleName(), metav1.GetOptions{}); err != nil { + t.Errorf("run-scoped ClusterRoleBinding %q not created: %v", d.clusterRoleName(), err) + } + + job, err := clientset.BatchV1().Jobs(testNamespace).Get(ctx, d.jobName(), metav1.GetOptions{}) + if err != nil { + t.Fatalf("Job not created: %v", err) + } + if got := job.Spec.Template.Spec.ServiceAccountName; got != scoped { + t.Errorf("pod ServiceAccountName = %q, want %q", got, scoped) + } +} + +// saNames projects a ServiceAccount list to its names for error messages. +func saNames(items []corev1.ServiceAccount) []string { + out := make([]string, 0, len(items)) + for i := range items { + out = append(out, items[i].Name) + } + return out +} + +// assertNoRBACObjects fails when any Role, RoleBinding, ClusterRole or +// ClusterRoleBinding exists in the cluster. +func assertNoRBACObjects(ctx context.Context, t *testing.T, clientset *fake.Clientset) { + t.Helper() + roles, err := clientset.RbacV1().Roles(testNamespace).List(ctx, metav1.ListOptions{}) + if err != nil { + t.Fatalf("listing Roles: %v", err) + } + if len(roles.Items) != 0 { + t.Errorf("Roles = %d, want 0", len(roles.Items)) + } + rbs, err := clientset.RbacV1().RoleBindings(testNamespace).List(ctx, metav1.ListOptions{}) + if err != nil { + t.Fatalf("listing RoleBindings: %v", err) + } + if len(rbs.Items) != 0 { + t.Errorf("RoleBindings = %d, want 0", len(rbs.Items)) + } + crs, err := clientset.RbacV1().ClusterRoles().List(ctx, metav1.ListOptions{}) + if err != nil { + t.Fatalf("listing ClusterRoles: %v", err) + } + if len(crs.Items) != 0 { + t.Errorf("ClusterRoles = %d, want 0", len(crs.Items)) + } + crbs, err := clientset.RbacV1().ClusterRoleBindings().List(ctx, metav1.ListOptions{}) + if err != nil { + t.Fatalf("listing ClusterRoleBindings: %v", err) + } + if len(crbs.Items) != 0 { + t.Errorf("ClusterRoleBindings = %d, want 0", len(crbs.Items)) + } +} + +// allowAllPermissionChecks makes every SelfSubjectAccessReview succeed so a +// test exercises Deploy past its Step 0 pre-flight. +func allowAllPermissionChecks(clientset *fake.Clientset) { clientset.PrependReactor("create", "selfsubjectaccessreviews", func(k8stesting.Action) (bool, runtime.Object, error) { return true, &authv1.SelfSubjectAccessReview{ Status: authv1.SubjectAccessReviewStatus{Allowed: true, Reason: "pre-flight verb set granted"}, }, nil }) +} + +// TestDeploy_SucceedsWhenServiceAccountGetForbidden is the end-to-end shape of +// the same bug: an identity authorized for exactly CheckPermissions' +// requiredChecks (which do not include `serviceaccounts get`) passes the +// pre-flight, so Deploy must not then fail on the exact-if-exists Get. +// ServiceAccountName is set because that Get is issued only when it is — +// leaving it empty would make the test pass without reaching the branch. +func TestDeploy_SucceedsWhenServiceAccountGetForbidden(t *testing.T) { + ctx := context.Background() + captureLogs(t) + + clientset := fake.NewClientset() + allowAllPermissionChecks(clientset) clientset.PrependReactor("get", "serviceaccounts", func(k8stesting.Action) (bool, runtime.Object, error) { return true, nil, apierrors.NewForbidden( schema.GroupResource{Group: "", Resource: "serviceaccounts"}, testName, @@ -188,16 +370,29 @@ func TestDeploy_SucceedsWhenServiceAccountGetForbidden(t *testing.T) { }) d := NewDeployer(clientset, Config{ - Namespace: "test-ns", - Image: "aicr:test", - RunID: testRunID, + Namespace: testNamespace, + ServiceAccountName: testName, + Image: "aicr:test", + RunID: testRunID, }) if err := d.Deploy(ctx); err != nil { - t.Fatalf("Deploy() error = %v, want nil (the adoption-drift Get must not gate deployment)", err) + t.Fatalf("Deploy() error = %v, want nil (the exact-if-exists Get must not gate deployment)", err) } - if _, err := clientset.BatchV1().Jobs("test-ns").Get(ctx, "aicr-"+testRunID, metav1.GetOptions{}); err != nil { + if _, err := clientset.BatchV1().Jobs(testNamespace).Get(ctx, "aicr-"+testRunID, metav1.GetOptions{}); err != nil { t.Errorf("Job not created: %v", err) } + // Forbidden must fall back to prefix mode, which still creates the + // run-scoped RBAC set. + if !d.managesRBAC() { + t.Error("managesRBAC() = false; a Forbidden Get must not be read as an adopted ServiceAccount") + } + // Read through the tracker, not the clientset: the reactor above stands + // in for an identity that cannot read ServiceAccounts at all, and that + // must not also blind the assertion. + saGVR := corev1.SchemeGroupVersion.WithResource("serviceaccounts") + if _, err := clientset.Tracker().Get(saGVR, testNamespace, "aicr-"+testRunID); err != nil { + t.Errorf("run-scoped ServiceAccount not created: %v", err) + } } diff --git a/pkg/k8s/agent/types.go b/pkg/k8s/agent/types.go index cb4a4993c..90b09f1f7 100644 --- a/pkg/k8s/agent/types.go +++ b/pkg/k8s/agent/types.go @@ -69,9 +69,35 @@ type createdObject struct { // Config holds the configuration for deploying the agent. type Config struct { - Namespace string + Namespace string + + // ServiceAccountName is exact-if-exists, and therefore carries two + // meanings resolved once per Deploy (see resolveServiceAccount): + // + // - A ServiceAccount of exactly this name already exists in + // Namespace: the agent pod runs as it verbatim and this run + // creates NO ServiceAccount, Role, RoleBinding, ClusterRole or + // ClusterRoleBinding. aicr adds and removes no permissions on an + // identity it did not create, and cleanup deletes none of them. + // This is what keeps a ServiceAccount carrying IRSA or GKE + // Workload Identity annotations usable: both providers pin trust + // to the ServiceAccount NAME, which a run-scoped name can never + // satisfy. Grant the agent's permissions to such a + // ServiceAccount once with ProvisionServiceAccountRoles. + // - Otherwise: a prefix. The run creates "-" and + // the full run-scoped RBAC set, and deletes them at cleanup. + // + // Empty falls back to NameBase and is never probed for existence — + // the fallback base is aicr's own default, not a name the caller + // asked for, so a stray ServiceAccount sitting at it must not + // silently capture the run. + // + // Exact mode waives per-run permission isolation: concurrent runs + // sharing the ServiceAccount share its grants, and grants provisioned + // for DiscoverNetwork persist beyond any one run. ServiceAccountName string - JobName string + + JobName string // RunID scopes every resource this Deployer creates to a single run, // so concurrent snapshot-agent runs never collide on a shared resource @@ -150,13 +176,22 @@ type Deployer struct { clientset kubernetes.Interface config Config - // mu guards created. Deploy's ensure* steps run sequentially today, - // but GetSnapshot (which records the staging ConfigMap) can be - // invoked from a different goroutine than Deploy, and Cleanup reads - // the created-set while a caller could still be recording into it, - // so every access is mutex-guarded. + // mu guards created and existingSA. Deploy's ensure* steps run + // sequentially today, but GetSnapshot (which records the staging + // ConfigMap) can be invoked from a different goroutine than Deploy, + // and Cleanup reads the created-set while a caller could still be + // recording into it, so every access is mutex-guarded. mu sync.Mutex created []createdObject + + // existingSA is the exact, operator-named ServiceAccount this run + // runs as instead of creating its own, or "" in prefix mode. It is + // resolved once by resolveServiceAccount at the top of Deploy and + // read afterwards by the Job builder. It shares mu with created + // rather than carrying its own: the Deployer is reachable from the + // caller's log-streaming and cleanup goroutines, so a field Deploy + // writes must not be read unsynchronized from any of them. + existingSA string } // NewDeployer creates a new agent Deployer with the given configuration. @@ -181,6 +216,34 @@ func (d *Deployer) objectLabels() map[string]string { } } +// setExistingServiceAccount records that this run uses the operator's +// already-existing ServiceAccount verbatim rather than creating its own. +// Called once, by resolveServiceAccount. Safe for concurrent use. +func (d *Deployer) setExistingServiceAccount(name string) { + d.mu.Lock() + defer d.mu.Unlock() + d.existingSA = name +} + +// existingServiceAccount returns the operator-supplied ServiceAccount this +// run adopted verbatim, or "" when the run creates and owns its own +// run-scoped one. Safe for concurrent use. +func (d *Deployer) existingServiceAccount() string { + d.mu.Lock() + defer d.mu.Unlock() + return d.existingSA +} + +// managesRBAC reports whether this run creates (and therefore later deletes) +// the ServiceAccount, Role, RoleBinding, ClusterRole and ClusterRoleBinding. +// False in exact-ServiceAccount mode: aicr adds and removes no permissions +// on an identity it did not create, so none of those objects is created, +// none enters the created-set, and Cleanup consequently has nothing of those +// kinds to delete. +func (d *Deployer) managesRBAC() bool { + return d.existingServiceAccount() == "" +} + // recordIntent enters a run-owned object into the created-set with the zero // UID, immediately BEFORE its Create call. It closes the window in which a // committed Create whose response never arrives (client timeout, apiserver diff --git a/pkg/k8s/agent/wait.go b/pkg/k8s/agent/wait.go index 701e09556..ebb497082 100644 --- a/pkg/k8s/agent/wait.go +++ b/pkg/k8s/agent/wait.go @@ -123,10 +123,10 @@ func (d *Deployer) deleteUnrecordedStagingConfigMap(ctx context.Context) error { } if cm.Labels[labels.Name] != labels.ValueAICR { slog.Warn("cleanup left behind the ConfigMap at this run's staging name: it does not look like an aicr snapshot artifact, so this run did not write it", - slog.String("namespace", d.config.Namespace), - slog.String("name", name), + slog.String(attrNamespace, d.config.Namespace), + slog.String(attrName, name), slog.String("uid", string(cm.UID)), - slog.String("runID", d.config.RunID)) + slog.String(attrRunID, d.config.RunID)) return nil } return d.deleteStagingConfigMap(ctx, cm.Name, cm.UID) From ae75239ad0c54b1b3dd553c7cbae3e081d9aed16 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Tue, 25 Aug 2026 11:33:30 -0700 Subject: [PATCH 45/56] feat(agent): add --add-roles-to-service-account to grant the agent's RBAC A provision-and-exit invocation, run once by an admin: it grants the snapshot agent's permissions to an already-existing ServiceAccount and returns without deploying a Job or capturing a snapshot. It is the counterpart to exact-if-exists --service-account-name, where a run that adopts an existing ServiceAccount creates and deletes no RBAC of its own. What it creates is permanent and outside every run's lifecycle: no run-ID label, never in a created-set, never deleted by run cleanup. Teardown is the operator's job, and tools/cleanup's managed-by sweep now excludes it so a developer teardown cannot revoke an admin's grant. Naming is deterministic -- aicr-agent--rbac in the namespace, and aicr-agent---rbac cluster-scoped -- so re-running is idempotent and refreshes the rules in place after an upgrade. The "-rbac" suffix is what keeps these out of the run-scoped name space: every run-scoped name ends in a run ID whose last segment is 16 lowercase-hex characters, and "r" is not a hex digit, so the two sets are disjoint by construction. The cluster-scoped name joins two "-"-bearing segments and is therefore not injective, so a second provisioning that would retarget another ServiceAccount's binding fails with ErrCodeConflict instead of silently revoking its grants. Combined with --discover-network it also provisions the mutating discovery rules; without it, only the read-only base set. A missing ServiceAccount fails with ErrCodeNotFound -- aicr grants permissions to an identity the operator controls and never creates one. The CLI stays an adapter: naming, existence check, rule set and create-or-update live in pkg/k8s/agent, reached through snapshotter.ProvisionAgentRoles. Signed-off-by: Alex Yuskauskas --- pkg/cli/consts.go | 5 + pkg/cli/snapshot.go | 80 +++++- pkg/cli/snapshot_test.go | 129 ++++++++++ pkg/defaults/timeouts.go | 8 + pkg/k8s/agent/provision.go | 349 ++++++++++++++++++++++++++ pkg/k8s/agent/provision_test.go | 398 ++++++++++++++++++++++++++++++ pkg/k8s/labels/labels.go | 9 + pkg/snapshotter/provision.go | 123 +++++++++ pkg/snapshotter/provision_test.go | 66 +++++ tools/cleanup | 11 +- 10 files changed, 1174 insertions(+), 4 deletions(-) create mode 100644 pkg/k8s/agent/provision.go create mode 100644 pkg/k8s/agent/provision_test.go create mode 100644 pkg/snapshotter/provision.go create mode 100644 pkg/snapshotter/provision_test.go diff --git a/pkg/cli/consts.go b/pkg/cli/consts.go index bd7713910..22685df37 100644 --- a/pkg/cli/consts.go +++ b/pkg/cli/consts.go @@ -39,6 +39,11 @@ const ( flagSlurmAccountingMode = "slurm-accounting-mode" flagRuntimeInventory = "runtime-inventory" flagNoHealth = "no-health" + + // flagAddRolesToSA switches `aicr snapshot` into a provision-and-exit + // invocation that grants the agent's permissions to an existing + // ServiceAccount. No snapshot is taken. + flagAddRolesToSA = "add-roles-to-service-account" ) // criteriaAny is the wildcard value for any criteria dimension. diff --git a/pkg/cli/snapshot.go b/pkg/cli/snapshot.go index e0e92d6c0..04ecafe39 100644 --- a/pkg/cli/snapshot.go +++ b/pkg/cli/snapshot.go @@ -16,6 +16,8 @@ package cli import ( "context" + "fmt" + "io" "log/slog" "os" "strings" @@ -176,7 +178,7 @@ func (o *snapshotCmdOptions) toSnapshotDelivery() snapshotter.SnapshotDelivery { // over config values. Returns a fully-typed snapshotCmdOptions that callers // can pass to the snapshotter without further parsing. func parseSnapshotCmdOptions(cmd *cli.Command, cfg *config.AICRConfig) (*snapshotCmdOptions, error) { - if err := validateSingleValueFlags(cmd, "namespace", "image", "job-name", "service-account-name", "timeout", "template", "max-nodes-per-entry", "runtime-class", "output", "format", "config", "os", "requests", "limits", "cluster-config", "aks-gpu-pools"); err != nil { + if err := validateSingleValueFlags(cmd, "namespace", "image", "job-name", "service-account-name", flagAddRolesToSA, "timeout", "template", "max-nodes-per-entry", "runtime-class", "output", "format", "config", "os", "requests", "limits", "cluster-config", "aks-gpu-pools"); err != nil { return nil, err } @@ -269,6 +271,66 @@ func parseSnapshotCmdOptions(cmd *cli.Command, cfg *config.AICRConfig) (*snapsho }, nil } +// runAddRolesToServiceAccount handles the provision-and-exit invocation +// `aicr snapshot --add-roles-to-service-account `: it grants the agent's +// permissions to an already-existing ServiceAccount and returns without +// deploying a Job or capturing anything. +// +// It is a thin adapter — every decision (name derivation, existence check, +// rule set, idempotent create-or-update) lives in pkg/snapshotter and +// pkg/k8s/agent. What belongs here is only presenting the outcome, including +// the two properties an operator must not have to discover on their own: the +// objects are permanent, and adopting one ServiceAccount across runs waives +// per-run permission isolation. +func runAddRolesToServiceAccount(ctx context.Context, cmd *cli.Command, opts *snapshotCmdOptions, saName string) error { + res, err := snapshotter.ProvisionAgentRoles(ctx, &snapshotter.AgentRolesConfig{ + Kubeconfig: opts.kubeconfig, + Namespace: opts.namespace, + ServiceAccountName: saName, + DiscoverNetwork: opts.discoverNetwork, + }) + if err != nil { + return err + } + + writeProvisionReport(cmd.Root().Writer, res) + return nil +} + +// writeProvisionReport renders the outcome of a provisioning run. It is split +// out from runAddRolesToServiceAccount so the two properties an operator must +// not have to discover on their own — the objects are permanent, and adopting +// one ServiceAccount across runs waives per-run permission isolation — are +// assertable without a cluster. +func writeProvisionReport(w io.Writer, res *snapshotter.AgentRolesResult) { + fmt.Fprintf(w, `Granted the snapshot agent's permissions to ServiceAccount %[1]q in namespace %[2]q. + +Created or updated: + role/%[3]s (namespace %[2]s) + rolebinding/%[4]s (namespace %[2]s) + clusterrole/%[5]s + clusterrolebinding/%[6]s + +These objects are permanent: no aicr run creates, updates, or deletes them. +Re-run this command after an aicr upgrade to refresh the rules; delete the four +objects by hand when the ServiceAccount no longer needs them. + +Capture a snapshot as this ServiceAccount with: + aicr snapshot --namespace %[2]s --service-account-name %[1]s + +Trade-off: runs that share this ServiceAccount share its permissions, so per-run +permission isolation is waived for them. +`, res.ServiceAccountName, res.Namespace, res.Role, res.RoleBinding, res.ClusterRole, res.ClusterRoleBinding) + + if res.DiscoverNetwork { + fmt.Fprint(w, ` +WARNING: --discover-network also granted cluster-scoped MUTATING rules +(nodes: patch, pods/exec: create, CRD/namespace/DaemonSet create-delete). +This ServiceAccount now carries them permanently, not for one run. +`) + } +} + // snapshotTemplateOptions holds parsed template options for the snapshot command. type snapshotTemplateOptions struct { templatePath string @@ -350,7 +412,12 @@ func snapshotCmdFlags() []cli.Flag { }, &cli.StringFlag{ Name: "service-account-name", - Usage: "ServiceAccount name prefix (default: \"aicr\"); the run ID is always appended", + Usage: "ServiceAccount to run the agent as. Exact-if-exists: when a ServiceAccount of exactly this name already exists in --namespace it is used verbatim and aicr creates and deletes no RBAC for the run (grant it permissions once with --add-roles-to-service-account). Otherwise it is a name prefix (default: \"aicr\") and the run ID is appended.", + Category: catAgentDeployment, + }, + &cli.StringFlag{ + Name: flagAddRolesToSA, + Usage: "Grant the agent's permissions to the named EXISTING ServiceAccount in --namespace, then exit without taking a snapshot. Creates permanent, non-run-scoped Role/RoleBinding and ClusterRole/ClusterRoleBinding that no run cleanup removes; idempotent. Add --discover-network to also grant the mutating live-discovery rules.", Category: catAgentDeployment, }, &cli.StringSliceFlag{ @@ -526,6 +593,15 @@ See examples/templates/snapshot-template.md.tmpl for a sample template. return err } + // Provision-and-exit: --add-roles-to-service-account grants the + // agent's permissions to an existing ServiceAccount and takes no + // snapshot. Checked ahead of every collection path (including the + // in-pod one below) so the flag can never be combined with a + // capture. + if saName := cmd.String(flagAddRolesToSA); saName != "" { + return runAddRolesToServiceAccount(ctx, cmd, opts, saName) + } + agentCfg := opts.toAgentConfig() // When running inside an agent Job, collect locally instead of diff --git a/pkg/cli/snapshot_test.go b/pkg/cli/snapshot_test.go index 0efeab428..0d8b281f9 100644 --- a/pkg/cli/snapshot_test.go +++ b/pkg/cli/snapshot_test.go @@ -15,6 +15,7 @@ package cli import ( + "bytes" "context" "os" "path/filepath" @@ -450,3 +451,131 @@ func TestOutputDestinationParsing(t *testing.T) { }) } } + +// TestSnapshotCmd_AddRolesFlagWiring covers the CLI surface of the +// provision-and-exit invocation. The provisioning itself is cluster work and +// is covered in pkg/k8s/agent; what this asserts is the wiring an operator +// touches: the flag exists under the agent-deployment category, and it is +// single-valued so a repeated flag is rejected rather than silently taking +// the last value and provisioning the wrong ServiceAccount. +func TestSnapshotCmd_AddRolesFlagWiring(t *testing.T) { + t.Run("flag is registered under agent deployment", func(t *testing.T) { + var found cli.Flag + for _, f := range snapshotCmd().Flags { + for _, n := range f.Names() { + if n == flagAddRolesToSA { + found = f + } + } + } + if found == nil { + t.Fatalf("snapshot command must define --%s", flagAddRolesToSA) + } + sf, ok := found.(*cli.StringFlag) + if !ok { + t.Fatalf("--%s is %T, want *cli.StringFlag", flagAddRolesToSA, found) + } + if sf.Category != catAgentDeployment { + t.Errorf("--%s category = %q, want %q", flagAddRolesToSA, sf.Category, catAgentDeployment) + } + if !strings.Contains(sf.Usage, "without taking a snapshot") { + t.Errorf("--%s usage must say it takes no snapshot; got %q", flagAddRolesToSA, sf.Usage) + } + }) + + t.Run("repeated flag is rejected", func(t *testing.T) { + err := runSnapshotCmdExpectErr(t, []string{ + "--" + flagAddRolesToSA, "sa-one", + "--" + flagAddRolesToSA, "sa-two", + }) + if err == nil { + t.Fatalf("repeated --%s accepted; the last value would silently win", flagAddRolesToSA) + } + if !strings.Contains(err.Error(), flagAddRolesToSA) { + t.Errorf("error = %q, want it to name --%s", err.Error(), flagAddRolesToSA) + } + }) +} + +// TestSnapshotCmd_ServiceAccountNameUsageStatesExactIfExists pins the one +// thing an operator has to learn from `--help`: --service-account-name is no +// longer only a prefix. A usage string that still says "prefix" alone would +// send an IRSA user straight into the silent credential loss this change +// exists to close. +func TestSnapshotCmd_ServiceAccountNameUsageStatesExactIfExists(t *testing.T) { + for _, f := range snapshotCmd().Flags { + sf, ok := f.(*cli.StringFlag) + if !ok || sf.Name != "service-account-name" { + continue + } + if !strings.Contains(sf.Usage, "xact-if-exists") { + t.Errorf("--service-account-name usage does not state the exact-if-exists rule; got %q", sf.Usage) + } + return + } + t.Fatal("snapshot command must define --service-account-name") +} + +// TestWriteProvisionReport asserts the two properties the provisioning output +// must state outright, because an operator who does not read them cannot +// discover either from the cluster: the objects are permanent (nothing in aicr +// will ever remove them), and adopting one ServiceAccount across runs waives +// per-run permission isolation. The --discover-network warning is separate +// because that grant is the one that is also mutating. +func TestWriteProvisionReport(t *testing.T) { + res := &snapshotter.AgentRolesResult{ + Namespace: "gpu-operator", + ServiceAccountName: "irsa-snapshotter", + Role: "aicr-agent-irsa-snapshotter-rbac", + RoleBinding: "aicr-agent-irsa-snapshotter-rbac", + ClusterRole: "aicr-agent-gpu-operator-irsa-snapshotter-rbac", + ClusterRoleBinding: "aicr-agent-gpu-operator-irsa-snapshotter-rbac", + } + + tests := []struct { + name string + discoverNetwork bool + wantSubstrings []string + notWant string + }{ + { + name: "read-only grant", + wantSubstrings: []string{ + `ServiceAccount "irsa-snapshotter" in namespace "gpu-operator"`, + "role/aicr-agent-irsa-snapshotter-rbac", + "clusterrolebinding/aicr-agent-gpu-operator-irsa-snapshotter-rbac", + "These objects are permanent", + "permission isolation is waived", + "aicr snapshot --namespace gpu-operator --service-account-name irsa-snapshotter", + }, + notWant: "MUTATING", + }, + { + name: "discovery grant warns about the permanent mutating rules", + discoverNetwork: true, + wantSubstrings: []string{ + "These objects are permanent", + "MUTATING", + "carries them permanently, not for one run", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := *res + r.DiscoverNetwork = tt.discoverNetwork + var buf bytes.Buffer + writeProvisionReport(&buf, &r) + + for _, want := range tt.wantSubstrings { + if !strings.Contains(buf.String(), want) { + t.Errorf("report does not contain %q; got:\n%s", want, buf.String()) + } + } + if tt.notWant != "" && strings.Contains(buf.String(), tt.notWant) { + t.Errorf("report contains %q but should not; got:\n%s", tt.notWant, buf.String()) + } + }) + } +} diff --git a/pkg/defaults/timeouts.go b/pkg/defaults/timeouts.go index 9dd457a28..335e16fc9 100644 --- a/pkg/defaults/timeouts.go +++ b/pkg/defaults/timeouts.go @@ -239,6 +239,14 @@ const ( // K8sCleanupTimeout is the timeout for cleanup operations. K8sCleanupTimeout = 30 * time.Second + // AgentRBACProvisionTimeout bounds + // `aicr snapshot --add-roles-to-service-account`, which reads the + // target ServiceAccount and then creates-or-updates four RBAC objects. + // That is a handful of small writes with no waiting on cluster state, + // so the budget only has to absorb apiserver latency and a retry or + // two — not a Job round trip. + AgentRBACProvisionTimeout = 60 * time.Second + // DiscoveryRefreshCooldown rate-limits how often the shared cluster // fetcher (pkg/chainsaw) invalidates its cached discovery data after a // no-match. The refresh exists so a CRD installed by the component being diff --git a/pkg/k8s/agent/provision.go b/pkg/k8s/agent/provision.go new file mode 100644 index 000000000..a083d3abe --- /dev/null +++ b/pkg/k8s/agent/provision.go @@ -0,0 +1,349 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package agent + +import ( + "context" + "fmt" + "strings" + + "github.com/NVIDIA/aicr/pkg/errors" + "github.com/NVIDIA/aicr/pkg/k8s/labels" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/client-go/kubernetes" +) + +// Naming of the permanent RBAC objects ProvisionServiceAccountRoles creates. +// +// The names are deterministic — the same (namespace, ServiceAccount) pair +// always resolves to the same four names, which is what makes re-running the +// provisioning idempotent — and they cannot collide with a run-scoped name. +// +// The non-collision is structural, not probabilistic. Every run-scoped name +// is "-" and every runID ends in 16 lowercase-hex characters +// (runid.Generate emits "--<16 hex>"). These names end in +// the literal "-rbac", and "r" is not a hex digit, so no run-scoped name can +// ever equal one of them regardless of what prefix a caller supplies. +const ( + provisionedNamePrefix = "aicr-agent-" + provisionedNameSuffix = "-rbac" +) + +// ProvisionOptions selects the ServiceAccount that +// ProvisionServiceAccountRoles grants the snapshot agent's permissions to. +type ProvisionOptions struct { + // Namespace holds the ServiceAccount and is where the Role and + // RoleBinding are created. Required. + Namespace string + + // ServiceAccountName is the EXACT name of an already-existing + // ServiceAccount. Required; provisioning fails with ErrCodeNotFound + // when no such ServiceAccount exists, because the whole point is to + // grant permissions to an identity the operator created and controls + // (typically one carrying IRSA or GKE Workload Identity annotations). + ServiceAccountName string + + // DiscoverNetwork also grants the cluster-scoped MUTATING rules that + // `aicr snapshot --discover-network` needs — nodes: patch, + // pods/exec: create, and CRD, namespace, DaemonSet and namespaced-RBAC + // create/delete (see discoverNetworkClusterRules). + // + // Unlike a run-scoped grant, this one is permanent: the ServiceAccount + // carries those permissions until the operator removes them, not for + // one run's lifetime. + DiscoverNetwork bool +} + +// ProvisionResult names what ProvisionServiceAccountRoles created or +// updated, so a caller can report it without rebuilding the names. +type ProvisionResult struct { + Namespace string + ServiceAccountName string + Role string + RoleBinding string + ClusterRole string + ClusterRoleBinding string + + // DiscoverNetwork echoes ProvisionOptions.DiscoverNetwork: it is the + // difference between a read-only grant and one carrying cluster-scoped + // mutating rules, so a caller reporting the result must be able to say + // which was provisioned. + DiscoverNetwork bool +} + +// ProvisionServiceAccountRoles grants the snapshot agent's permissions to an +// already-existing, operator-supplied ServiceAccount by creating a Role, +// RoleBinding, ClusterRole and ClusterRoleBinding for it. +// +// These four objects are PERMANENT and deliberately outside every run's +// lifecycle: they carry no run-ID label, they never enter any Deployer's +// created-set, and no run's Cleanup deletes them. Removing them is the +// operator's job. That is the counterpart to Config.ServiceAccountName's +// exact-if-exists behavior, where a run that adopts an existing +// ServiceAccount creates and deletes no RBAC of its own. +// +// The call is idempotent: each object is created, or updated in place when +// it already exists, so re-running it after an aicr upgrade refreshes the +// rules rather than failing or leaving a stale rule set behind. +// +// Trade-off the caller must surface to the operator: an adopted +// ServiceAccount waives per-run permission isolation. Concurrent runs using +// it share its grants, and a DiscoverNetwork provisioning leaves mutating +// cluster permissions in place permanently rather than for one run. +func ProvisionServiceAccountRoles(ctx context.Context, clientset kubernetes.Interface, opts ProvisionOptions) (*ProvisionResult, error) { + if strings.TrimSpace(opts.Namespace) == "" { + return nil, errors.New(errors.ErrCodeInvalidRequest, + "namespace is required: it is where the ServiceAccount, Role and RoleBinding live") + } + if strings.TrimSpace(opts.ServiceAccountName) == "" { + return nil, errors.New(errors.ErrCodeInvalidRequest, + "ServiceAccount name is required: provisioning grants permissions to an existing ServiceAccount, it does not create one") + } + + // Both halves of each composed name are valid on their own, but their + // concatenation can exceed the length ceiling. Reject that here, before + // anything is created, rather than as an opaque apiserver "Invalid + // value: metadata.name" partway through. + roleName := provisionedRoleName(opts.ServiceAccountName) + clusterRoleName := provisionedClusterRoleName(opts.Namespace, opts.ServiceAccountName) + for _, name := range []string{roleName, clusterRoleName} { + problems := validation.IsDNS1123Subdomain(name) + if len(problems) == 0 { + continue + } + return nil, errors.NewWithContext(errors.ErrCodeInvalidRequest, + fmt.Sprintf("ServiceAccount %q yields the RBAC object name %q, which is not a valid Kubernetes object name: %s", + opts.ServiceAccountName, name, strings.Join(problems, "; ")), + map[string]any{ctxKeyValue: opts.ServiceAccountName, ctxKeyResolvedName: name}) + } + + // Fail closed before creating anything when the ServiceAccount is + // absent. Provisioning permissions for an identity that does not exist + // would leave four dangling objects and a binding to nothing, and the + // most likely cause is a typo the operator needs to see. + if _, err := clientset.CoreV1().ServiceAccounts(opts.Namespace). + Get(ctx, opts.ServiceAccountName, metav1.GetOptions{}); err != nil { + if apierrors.IsNotFound(err) { + return nil, errors.NewWithContext(errors.ErrCodeNotFound, + fmt.Sprintf("ServiceAccount %q not found in namespace %q; create it first (aicr grants permissions to an existing ServiceAccount, it never creates one)", + opts.ServiceAccountName, opts.Namespace), + map[string]any{attrServiceAccount: opts.ServiceAccountName, attrNamespace: opts.Namespace}) + } + return nil, errors.Wrap(errors.ErrCodeInternal, "failed to read the target ServiceAccount", err) + } + + subjects := []rbacv1.Subject{{ + Kind: kindServiceAccount, + Name: opts.ServiceAccountName, + Namespace: opts.Namespace, + }} + + if err := provisionRole(ctx, clientset, opts.Namespace, roleName); err != nil { + return nil, err + } + if err := provisionRoleBinding(ctx, clientset, opts.Namespace, roleName, subjects); err != nil { + return nil, err + } + if err := provisionClusterRole(ctx, clientset, clusterRoleName, opts.DiscoverNetwork); err != nil { + return nil, err + } + if err := provisionClusterRoleBinding(ctx, clientset, clusterRoleName, subjects); err != nil { + return nil, err + } + + return &ProvisionResult{ + Namespace: opts.Namespace, + ServiceAccountName: opts.ServiceAccountName, + Role: roleName, + RoleBinding: roleName, + ClusterRole: clusterRoleName, + ClusterRoleBinding: clusterRoleName, + DiscoverNetwork: opts.DiscoverNetwork, + }, nil +} + +// provisionedRoleName returns the permanent Role and RoleBinding name for a +// ServiceAccount. It is injective within a namespace — the name is a pure +// function of the ServiceAccount name, and a ServiceAccount name is unique +// in its namespace — so two ServiceAccounts can never share these objects. +func provisionedRoleName(serviceAccount string) string { + return provisionedNamePrefix + serviceAccount + provisionedNameSuffix +} + +// provisionedClusterRoleName returns the permanent ClusterRole and +// ClusterRoleBinding name for a ServiceAccount. The namespace is part of the +// name because these objects are cluster-scoped and the same ServiceAccount +// name can exist in several namespaces. +// +// Joining two "-"-bearing segments is not injective ("a-b"/"c" and +// "a"/"b-c" compose the same string), so provisionClusterRoleBinding +// additionally refuses to retarget a binding that already names a different +// subject rather than silently revoking the first ServiceAccount's grants. +func provisionedClusterRoleName(namespace, serviceAccount string) string { + return provisionedNamePrefix + namespace + "-" + serviceAccount + provisionedNameSuffix +} + +// provisionedLabels is the label set stamped on every permanent object. +// It deliberately omits labels.RunID: these objects belong to no run, so +// Deployer.createdByThisRun can never match one and no run's Cleanup can +// reclaim it. The component value is what distinguishes them from the +// run-scoped snapshot-agent objects in selectors and sweeps. +func provisionedLabels() map[string]string { + return map[string]string{ + labels.Name: labels.ValueAICR, + labels.ManagedBy: labels.ValueAICR, + labels.Component: labels.ValueAgentRBAC, + } +} + +// provisionRole creates the permanent Role, or refreshes its rules in place +// when it already exists so an aicr upgrade that changes the rule set takes +// effect on a re-run instead of leaving stale rules behind. +func provisionRole(ctx context.Context, clientset kubernetes.Interface, namespace, name string) error { + role := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: provisionedLabels(), + }, + Rules: namespacedRules(), + } + _, err := clientset.RbacV1().Roles(namespace).Create(ctx, role, metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(err) { + if _, err = clientset.RbacV1().Roles(namespace).Update(ctx, role, metav1.UpdateOptions{}); err != nil { + return errors.Wrap(errors.ErrCodeInternal, "failed to update the permanent Role", err) + } + return nil + } + if err != nil { + return errors.Wrap(errors.ErrCodeInternal, "failed to create the permanent Role", err) + } + return nil +} + +// provisionRoleBinding creates or refreshes the permanent RoleBinding. +// It needs no subject-collision guard: the name is a pure function of the +// ServiceAccount name within one namespace, so it can only ever refer to +// the ServiceAccount it is being written for. +func provisionRoleBinding(ctx context.Context, clientset kubernetes.Interface, namespace, name string, subjects []rbacv1.Subject) error { + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: provisionedLabels(), + }, + Subjects: subjects, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacAPIGroup, + Kind: kindRole, + Name: name, + }, + } + _, err := clientset.RbacV1().RoleBindings(namespace).Create(ctx, rb, metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(err) { + if _, err = clientset.RbacV1().RoleBindings(namespace).Update(ctx, rb, metav1.UpdateOptions{}); err != nil { + return errors.Wrap(errors.ErrCodeInternal, "failed to update the permanent RoleBinding", err) + } + return nil + } + if err != nil { + return errors.Wrap(errors.ErrCodeInternal, "failed to create the permanent RoleBinding", err) + } + return nil +} + +// provisionClusterRole creates or refreshes the permanent ClusterRole. +func provisionClusterRole(ctx context.Context, clientset kubernetes.Interface, name string, discoverNetwork bool) error { + cr := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: provisionedLabels(), + }, + Rules: clusterRules(discoverNetwork), + } + _, err := clientset.RbacV1().ClusterRoles().Create(ctx, cr, metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(err) { + if _, err = clientset.RbacV1().ClusterRoles().Update(ctx, cr, metav1.UpdateOptions{}); err != nil { + return errors.Wrap(errors.ErrCodeInternal, "failed to update the permanent ClusterRole", err) + } + return nil + } + if err != nil { + return errors.Wrap(errors.ErrCodeInternal, "failed to create the permanent ClusterRole", err) + } + return nil +} + +// provisionClusterRoleBinding creates or refreshes the permanent +// ClusterRoleBinding. +// +// Before updating an existing one it checks the subject: because the +// cluster-scoped name joins namespace and ServiceAccount with "-", two +// distinct pairs can compose the same name (see +// provisionedClusterRoleName). Overwriting a binding that names a different +// ServiceAccount would silently revoke that ServiceAccount's cluster grants, +// so refuse with ErrCodeConflict — the request is well formed, the cluster +// state is what makes it unserviceable. +func provisionClusterRoleBinding(ctx context.Context, clientset kubernetes.Interface, name string, subjects []rbacv1.Subject) error { + crb := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: provisionedLabels(), + }, + Subjects: subjects, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacAPIGroup, + Kind: kindClusterRole, + Name: name, + }, + } + _, err := clientset.RbacV1().ClusterRoleBindings().Create(ctx, crb, metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(err) { + existing, getErr := clientset.RbacV1().ClusterRoleBindings().Get(ctx, name, metav1.GetOptions{}) + if getErr != nil { + return errors.Wrap(errors.ErrCodeInternal, "failed to read the existing ClusterRoleBinding", getErr) + } + if other := conflictingSubject(existing.Subjects, subjects[0]); other != "" { + return errors.NewWithContext(errors.ErrCodeConflict, + fmt.Sprintf("ClusterRoleBinding %q already grants these permissions to %s; updating it would revoke them. Rename one of the two ServiceAccounts or namespaces so the generated names differ", + name, other), + map[string]any{ctxKeyResolvedName: name, "existingSubject": other}) + } + if _, err = clientset.RbacV1().ClusterRoleBindings().Update(ctx, crb, metav1.UpdateOptions{}); err != nil { + return errors.Wrap(errors.ErrCodeInternal, "failed to update the permanent ClusterRoleBinding", err) + } + return nil + } + if err != nil { + return errors.Wrap(errors.ErrCodeInternal, "failed to create the permanent ClusterRoleBinding", err) + } + return nil +} + +// conflictingSubject returns a description of the first subject in existing +// that is not want, or "" when every subject is want (the idempotent +// re-provisioning case) or existing is empty. +func conflictingSubject(existing []rbacv1.Subject, want rbacv1.Subject) string { + for _, s := range existing { + if s.Kind == want.Kind && s.Name == want.Name && s.Namespace == want.Namespace { + continue + } + return fmt.Sprintf("%s %q in namespace %q", s.Kind, s.Name, s.Namespace) + } + return "" +} diff --git a/pkg/k8s/agent/provision_test.go b/pkg/k8s/agent/provision_test.go new file mode 100644 index 000000000..602873b80 --- /dev/null +++ b/pkg/k8s/agent/provision_test.go @@ -0,0 +1,398 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package agent + +import ( + "context" + stderrors "errors" + "reflect" + "strings" + "testing" + + aicrerrors "github.com/NVIDIA/aicr/pkg/errors" + "github.com/NVIDIA/aicr/pkg/k8s/labels" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +const provisionSA = "irsa-snapshotter" + +// seedServiceAccount creates the target ServiceAccount provisioning requires. +func seedServiceAccount(ctx context.Context, t *testing.T, clientset *fake.Clientset, namespace, name string) { + t.Helper() + if _, err := clientset.CoreV1().ServiceAccounts(namespace).Create(ctx, &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + }, metav1.CreateOptions{}); err != nil { + t.Fatalf("seeding ServiceAccount: %v", err) + } +} + +// TestProvisionServiceAccountRoles_Names pins the naming scheme, because its +// only real requirement is structural: a provisioned name must never be +// mistakable for a run-scoped one. Every run-scoped name ends in a run ID +// whose final segment is 16 lowercase-hex characters, and these end in +// "-rbac" — "r" is not a hex digit, so the two name spaces are disjoint by +// construction rather than by luck. +func TestProvisionServiceAccountRoles_Names(t *testing.T) { + ctx := context.Background() + clientset := fake.NewClientset() + seedServiceAccount(ctx, t, clientset, testNamespace, provisionSA) + + res, err := ProvisionServiceAccountRoles(ctx, clientset, ProvisionOptions{ + Namespace: testNamespace, + ServiceAccountName: provisionSA, + }) + if err != nil { + t.Fatalf("ProvisionServiceAccountRoles() error = %v", err) + } + + wantRole := "aicr-agent-" + provisionSA + "-rbac" + wantClusterRole := "aicr-agent-" + testNamespace + "-" + provisionSA + "-rbac" + if res.Role != wantRole || res.RoleBinding != wantRole { + t.Errorf("Role/RoleBinding = %q/%q, want %q", res.Role, res.RoleBinding, wantRole) + } + if res.ClusterRole != wantClusterRole || res.ClusterRoleBinding != wantClusterRole { + t.Errorf("ClusterRole/ClusterRoleBinding = %q/%q, want %q", res.ClusterRole, res.ClusterRoleBinding, wantClusterRole) + } + + // A run-scoped name is "-". Nothing provisioned may be + // one, whatever prefix a caller supplies. + for _, name := range []string{res.Role, res.RoleBinding, res.ClusterRole, res.ClusterRoleBinding} { + if strings.HasSuffix(name, "-"+testRunID) { + t.Errorf("provisioned name %q collides with the run-scoped name space", name) + } + if !strings.HasSuffix(name, provisionedNameSuffix) { + t.Errorf("provisioned name %q does not carry the %q suffix that keeps it out of the run-scoped name space", name, provisionedNameSuffix) + } + } +} + +// TestProvisionServiceAccountRoles_ObjectsArePermanentAndUnscoped asserts the +// property the whole feature rests on: nothing provisioned carries a run ID, +// so no run's cleanup can ever reclaim it. Deployer.createdByThisRun requires +// the run-ID label, and these objects deliberately have none. +func TestProvisionServiceAccountRoles_ObjectsArePermanentAndUnscoped(t *testing.T) { + ctx := context.Background() + clientset := fake.NewClientset() + seedServiceAccount(ctx, t, clientset, testNamespace, provisionSA) + + res, err := ProvisionServiceAccountRoles(ctx, clientset, ProvisionOptions{ + Namespace: testNamespace, + ServiceAccountName: provisionSA, + }) + if err != nil { + t.Fatalf("ProvisionServiceAccountRoles() error = %v", err) + } + + role, err := clientset.RbacV1().Roles(testNamespace).Get(ctx, res.Role, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Role not created: %v", err) + } + cr, err := clientset.RbacV1().ClusterRoles().Get(ctx, res.ClusterRole, metav1.GetOptions{}) + if err != nil { + t.Fatalf("ClusterRole not created: %v", err) + } + + for name, got := range map[string]map[string]string{res.Role: role.Labels, res.ClusterRole: cr.Labels} { + if _, ok := got[labels.RunID]; ok { + t.Errorf("%s carries the %s label; a run's cleanup could reclaim it", name, labels.RunID) + } + if got[labels.Component] != labels.ValueAgentRBAC { + t.Errorf("%s component label = %q, want %q", name, got[labels.Component], labels.ValueAgentRBAC) + } + if got[labels.Component] == labels.ValueSnapshotAgent { + t.Errorf("%s is labeled as a run-scoped snapshot-agent object", name) + } + } + + // A Deployer must not be able to claim these as its own, whatever run + // ID it holds. + d := NewDeployer(clientset, Config{Namespace: testNamespace, RunID: testRunID}) + if d.createdByThisRun(role.Labels) || d.createdByThisRun(cr.Labels) { + t.Error("createdByThisRun matched a provisioned object; run cleanup would delete a permanent grant") + } + + // The bindings must point at the operator's ServiceAccount. + rb, err := clientset.RbacV1().RoleBindings(testNamespace).Get(ctx, res.RoleBinding, metav1.GetOptions{}) + if err != nil { + t.Fatalf("RoleBinding not created: %v", err) + } + crb, err := clientset.RbacV1().ClusterRoleBindings().Get(ctx, res.ClusterRoleBinding, metav1.GetOptions{}) + if err != nil { + t.Fatalf("ClusterRoleBinding not created: %v", err) + } + want := []rbacv1.Subject{{Kind: kindServiceAccount, Name: provisionSA, Namespace: testNamespace}} + if !reflect.DeepEqual(rb.Subjects, want) { + t.Errorf("RoleBinding subjects = %v, want %v", rb.Subjects, want) + } + if !reflect.DeepEqual(crb.Subjects, want) { + t.Errorf("ClusterRoleBinding subjects = %v, want %v", crb.Subjects, want) + } +} + +// TestProvisionServiceAccountRoles_DiscoverNetworkRules covers both rule sets: +// the read-only baseline, and the baseline plus the cluster-scoped mutating +// rules live discovery needs. The distinction matters because a provisioned +// grant is permanent — a --discover-network provisioning leaves nodes:patch +// and pods/exec:create on the ServiceAccount indefinitely. +func TestProvisionServiceAccountRoles_DiscoverNetworkRules(t *testing.T) { + tests := []struct { + name string + discoverNetwork bool + wantMutating bool + }{ + {name: "read-only baseline", discoverNetwork: false, wantMutating: false}, + {name: "discovery adds the mutating rules", discoverNetwork: true, wantMutating: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + clientset := fake.NewClientset() + seedServiceAccount(ctx, t, clientset, testNamespace, provisionSA) + + res, err := ProvisionServiceAccountRoles(ctx, clientset, ProvisionOptions{ + Namespace: testNamespace, + ServiceAccountName: provisionSA, + DiscoverNetwork: tt.discoverNetwork, + }) + if err != nil { + t.Fatalf("ProvisionServiceAccountRoles() error = %v", err) + } + if res.DiscoverNetwork != tt.discoverNetwork { + t.Errorf("result DiscoverNetwork = %v, want %v", res.DiscoverNetwork, tt.discoverNetwork) + } + + cr, err := clientset.RbacV1().ClusterRoles().Get(ctx, res.ClusterRole, metav1.GetOptions{}) + if err != nil { + t.Fatalf("ClusterRole not created: %v", err) + } + if got := hasRule(cr.Rules, "", "nodes", "patch"); got != tt.wantMutating { + t.Errorf("nodes:patch granted = %v, want %v", got, tt.wantMutating) + } + if got := hasRule(cr.Rules, "", "pods/exec", verbCreate); got != tt.wantMutating { + t.Errorf("pods/exec:create granted = %v, want %v", got, tt.wantMutating) + } + // The baseline read-only rules are present either way. + if !hasRule(cr.Rules, "", "nodes", verbList) { + t.Error("baseline nodes:list rule missing") + } + + // The provisioned ClusterRole must carry exactly what a + // run-scoped one would, so an adopted ServiceAccount is not + // quietly less capable than a run-owned one. + if !reflect.DeepEqual(cr.Rules, clusterRules(tt.discoverNetwork)) { + t.Error("provisioned ClusterRole rules differ from the run-scoped agent's") + } + role, err := clientset.RbacV1().Roles(testNamespace).Get(ctx, res.Role, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Role not created: %v", err) + } + if !reflect.DeepEqual(role.Rules, namespacedRules()) { + t.Error("provisioned Role rules differ from the run-scoped agent's") + } + }) + } +} + +// TestProvisionServiceAccountRoles_Idempotent re-runs provisioning over a +// stale rule set and asserts the rules are refreshed in place rather than the +// call failing or the stale grant surviving. This is the aicr-upgrade path: +// an operator re-runs the command and expects the current rules. +func TestProvisionServiceAccountRoles_Idempotent(t *testing.T) { + ctx := context.Background() + clientset := fake.NewClientset() + seedServiceAccount(ctx, t, clientset, testNamespace, provisionSA) + + opts := ProvisionOptions{Namespace: testNamespace, ServiceAccountName: provisionSA, DiscoverNetwork: true} + res, err := ProvisionServiceAccountRoles(ctx, clientset, opts) + if err != nil { + t.Fatalf("first ProvisionServiceAccountRoles() error = %v", err) + } + + // Simulate a stale grant left by an older aicr: strip the rules from + // both roles. A merely-idempotent implementation that skipped existing + // objects would leave them stripped. + stale, err := clientset.RbacV1().ClusterRoles().Get(ctx, res.ClusterRole, metav1.GetOptions{}) + if err != nil { + t.Fatalf("reading ClusterRole: %v", err) + } + stale.Rules = nil + if _, err = clientset.RbacV1().ClusterRoles().Update(ctx, stale, metav1.UpdateOptions{}); err != nil { + t.Fatalf("staling ClusterRole: %v", err) + } + + res2, err := ProvisionServiceAccountRoles(ctx, clientset, opts) + if err != nil { + t.Fatalf("second ProvisionServiceAccountRoles() error = %v", err) + } + if !reflect.DeepEqual(res, res2) { + t.Errorf("second result = %+v, want %+v (names must be deterministic)", res2, res) + } + + refreshed, err := clientset.RbacV1().ClusterRoles().Get(ctx, res.ClusterRole, metav1.GetOptions{}) + if err != nil { + t.Fatalf("reading ClusterRole: %v", err) + } + if !reflect.DeepEqual(refreshed.Rules, clusterRules(true)) { + t.Error("re-provisioning did not refresh the stale ClusterRole rules") + } + + // Exactly one of each object, not a duplicate per run. + crs, err := clientset.RbacV1().ClusterRoles().List(ctx, metav1.ListOptions{}) + if err != nil { + t.Fatalf("listing ClusterRoles: %v", err) + } + if len(crs.Items) != 1 { + t.Errorf("ClusterRoles = %d, want 1", len(crs.Items)) + } + roles, err := clientset.RbacV1().Roles(testNamespace).List(ctx, metav1.ListOptions{}) + if err != nil { + t.Fatalf("listing Roles: %v", err) + } + if len(roles.Items) != 1 { + t.Errorf("Roles = %d, want 1", len(roles.Items)) + } +} + +// TestProvisionServiceAccountRoles_Rejections covers every input the call +// refuses before writing anything — most importantly a ServiceAccount that +// does not exist, which must be ErrCodeNotFound rather than four dangling +// objects bound to nothing. +func TestProvisionServiceAccountRoles_Rejections(t *testing.T) { + tests := []struct { + name string + seed string + opts ProvisionOptions + wantCode aicrerrors.ErrorCode + wantInMsg string + }{ + { + name: "missing ServiceAccount", + opts: ProvisionOptions{Namespace: testNamespace, ServiceAccountName: provisionSA}, + wantCode: aicrerrors.ErrCodeNotFound, + wantInMsg: "not found in namespace", + }, + { + name: "empty namespace", + opts: ProvisionOptions{ServiceAccountName: provisionSA}, + wantCode: aicrerrors.ErrCodeInvalidRequest, + }, + { + name: "empty ServiceAccount name", + opts: ProvisionOptions{Namespace: testNamespace}, + wantCode: aicrerrors.ErrCodeInvalidRequest, + }, + { + name: "whitespace ServiceAccount name", + opts: ProvisionOptions{Namespace: testNamespace, ServiceAccountName: " "}, + wantCode: aicrerrors.ErrCodeInvalidRequest, + }, + { + name: "name too long to compose", + seed: strings.Repeat("a", 250), + opts: ProvisionOptions{Namespace: testNamespace, ServiceAccountName: strings.Repeat("a", 250)}, + wantCode: aicrerrors.ErrCodeInvalidRequest, + wantInMsg: "not a valid Kubernetes object name", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + clientset := fake.NewClientset() + if tt.seed != "" { + seedServiceAccount(ctx, t, clientset, testNamespace, tt.seed) + } + + _, err := ProvisionServiceAccountRoles(ctx, clientset, tt.opts) + if err == nil { + t.Fatal("ProvisionServiceAccountRoles() error = nil, want an error") + } + if !stderrors.Is(err, aicrerrors.New(tt.wantCode, "")) { + t.Errorf("error = %v, want code %s", err, tt.wantCode) + } + if tt.wantInMsg != "" && !strings.Contains(err.Error(), tt.wantInMsg) { + t.Errorf("error = %q, want it to contain %q", err.Error(), tt.wantInMsg) + } + + // Nothing may be written on a rejected call. + crs, listErr := clientset.RbacV1().ClusterRoles().List(ctx, metav1.ListOptions{}) + if listErr != nil { + t.Fatalf("listing ClusterRoles: %v", listErr) + } + if len(crs.Items) != 0 { + t.Errorf("ClusterRoles = %d, want 0 (a rejected call must write nothing)", len(crs.Items)) + } + }) + } +} + +// TestProvisionServiceAccountRoles_RefusesToRetargetAnotherSubject covers the +// one way the cluster-scoped name can be ambiguous: it joins namespace and +// ServiceAccount with "-", so ("a-b", "c") and ("a", "b-c") compose the same +// name. Silently updating the binding would revoke the first ServiceAccount's +// cluster grants, so the second provisioning must fail closed. +func TestProvisionServiceAccountRoles_RefusesToRetargetAnotherSubject(t *testing.T) { + ctx := context.Background() + clientset := fake.NewClientset() + seedServiceAccount(ctx, t, clientset, "a-b", "c") + seedServiceAccount(ctx, t, clientset, "a", "b-c") + + first, err := ProvisionServiceAccountRoles(ctx, clientset, ProvisionOptions{Namespace: "a-b", ServiceAccountName: "c"}) + if err != nil { + t.Fatalf("first ProvisionServiceAccountRoles() error = %v", err) + } + + _, err = ProvisionServiceAccountRoles(ctx, clientset, ProvisionOptions{Namespace: "a", ServiceAccountName: "b-c"}) + if err == nil { + t.Fatal("second ProvisionServiceAccountRoles() error = nil, want a conflict") + } + if !stderrors.Is(err, aicrerrors.New(aicrerrors.ErrCodeConflict, "")) { + t.Errorf("error = %v, want ErrCodeConflict", err) + } + + // The first ServiceAccount keeps its grant. + crb, err := clientset.RbacV1().ClusterRoleBindings().Get(ctx, first.ClusterRoleBinding, metav1.GetOptions{}) + if err != nil { + t.Fatalf("reading ClusterRoleBinding: %v", err) + } + want := []rbacv1.Subject{{Kind: kindServiceAccount, Name: "c", Namespace: "a-b"}} + if !reflect.DeepEqual(crb.Subjects, want) { + t.Errorf("ClusterRoleBinding subjects = %v, want %v (the first grant must survive)", crb.Subjects, want) + } +} + +// hasRule reports whether rules grant verb on resource in apiGroup. +func hasRule(rules []rbacv1.PolicyRule, apiGroup, resource, verb string) bool { + for _, r := range rules { + if !contains(r.APIGroups, apiGroup) || !contains(r.Resources, resource) || !contains(r.Verbs, verb) { + continue + } + return true + } + return false +} + +func contains(haystack []string, needle string) bool { + for _, s := range haystack { + if s == needle { + return true + } + } + return false +} diff --git a/pkg/k8s/labels/labels.go b/pkg/k8s/labels/labels.go index 05005b523..8102e34bf 100644 --- a/pkg/k8s/labels/labels.go +++ b/pkg/k8s/labels/labels.go @@ -36,4 +36,13 @@ const ( // ValueSnapshotAgent identifies snapshot-agent-owned resources. ValueSnapshotAgent = "snapshot-agent" + + // ValueAgentRBAC identifies the permanent, NON-run-scoped Role, + // RoleBinding, ClusterRole and ClusterRoleBinding that + // `aicr snapshot --add-roles-to-service-account` provisions onto an + // operator-supplied ServiceAccount. Objects carrying this value are + // deliberately outside every run's lifecycle: they carry no RunID + // label, never enter a run's created-set, and are never deleted by + // run cleanup. Teardown is the operator's job. + ValueAgentRBAC = "agent-rbac" ) diff --git a/pkg/snapshotter/provision.go b/pkg/snapshotter/provision.go new file mode 100644 index 000000000..beb4c22c3 --- /dev/null +++ b/pkg/snapshotter/provision.go @@ -0,0 +1,123 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package snapshotter + +import ( + "context" + "strings" + + "github.com/NVIDIA/aicr/pkg/defaults" + "github.com/NVIDIA/aicr/pkg/errors" + "github.com/NVIDIA/aicr/pkg/k8s/agent" +) + +// AgentRolesConfig selects the ServiceAccount that ProvisionAgentRoles +// grants the snapshot agent's permissions to, and the cluster it lives in. +type AgentRolesConfig struct { + // Kubeconfig is an optional path override; empty uses default + // discovery (KUBECONFIG, then ~/.kube/config, then in-cluster). + Kubeconfig string + + // Namespace holds the ServiceAccount and receives the Role and + // RoleBinding. Required. + Namespace string + + // ServiceAccountName is the EXACT name of an already-existing + // ServiceAccount. Required. + ServiceAccountName string + + // DiscoverNetwork also grants the cluster-scoped MUTATING rules that + // `aicr snapshot --discover-network` needs. Permanently, not for one + // run's lifetime. + DiscoverNetwork bool +} + +// AgentRolesResult names what ProvisionAgentRoles created or updated, so a +// caller can report it without rebuilding the names. +// +// It is snapshotter-owned rather than pkg/k8s/agent's own ProvisionResult +// so callers presenting the outcome — the CLI among them — need no +// dependency on the Kubernetes-facing package. +type AgentRolesResult struct { + Namespace string + ServiceAccountName string + Role string + RoleBinding string + ClusterRole string + ClusterRoleBinding string + + // DiscoverNetwork echoes AgentRolesConfig.DiscoverNetwork: it is the + // difference between a read-only grant and one carrying cluster-scoped + // mutating rules, so anything reporting this result can say which was + // provisioned. + DiscoverNetwork bool +} + +// ProvisionAgentRoles grants the snapshot agent's permissions to an +// existing, operator-supplied ServiceAccount so that ServiceAccount can be +// named exactly via AgentConfig.ServiceAccountName +// (`--service-account-name`) and keep its own identity — the IRSA or GKE +// Workload Identity annotations a run-scoped ServiceAccount cannot carry, +// because both providers pin trust to the ServiceAccount name. +// +// It provisions and returns; it deploys no Job and collects no snapshot. +// The objects it creates are PERMANENT: they carry no run-ID label, never +// enter a run's created-set, and no run's cleanup deletes them. Removing +// them is the operator's job. +// +// Idempotent — re-run it after an aicr upgrade to refresh the rules in +// place. Returns ErrCodeNotFound when the named ServiceAccount does not +// exist. +func ProvisionAgentRoles(ctx context.Context, config *AgentRolesConfig) (*AgentRolesResult, error) { + if config == nil { + return nil, errors.New(errors.ErrCodeInvalidRequest, "agent roles config is required") + } + // Reject what can be rejected without contacting the cluster, so a bad + // value is never masked by a kubeconfig error. + if strings.TrimSpace(config.Namespace) == "" { + return nil, errors.New(errors.ErrCodeInvalidRequest, + "Namespace is required: it is where the ServiceAccount, Role and RoleBinding live") + } + if strings.TrimSpace(config.ServiceAccountName) == "" { + return nil, errors.New(errors.ErrCodeInvalidRequest, + "ServiceAccountName is required: provisioning grants permissions to an existing ServiceAccount, it does not create one") + } + + clientset, err := getKubeClient(config.Kubeconfig) + if err != nil { + return nil, err + } + + ctx, cancel := context.WithTimeout(ctx, defaults.AgentRBACProvisionTimeout) + defer cancel() + + res, err := agent.ProvisionServiceAccountRoles(ctx, clientset, agent.ProvisionOptions{ + Namespace: config.Namespace, + ServiceAccountName: config.ServiceAccountName, + DiscoverNetwork: config.DiscoverNetwork, + }) + if err != nil { + return nil, err + } + return &AgentRolesResult{ + Namespace: res.Namespace, + ServiceAccountName: res.ServiceAccountName, + Role: res.Role, + RoleBinding: res.RoleBinding, + ClusterRole: res.ClusterRole, + ClusterRoleBinding: res.ClusterRoleBinding, + DiscoverNetwork: res.DiscoverNetwork, + }, nil +} diff --git a/pkg/snapshotter/provision_test.go b/pkg/snapshotter/provision_test.go new file mode 100644 index 000000000..1982b13db --- /dev/null +++ b/pkg/snapshotter/provision_test.go @@ -0,0 +1,66 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package snapshotter + +import ( + "context" + stderrors "errors" + "testing" + + "github.com/NVIDIA/aicr/pkg/errors" +) + +// TestProvisionAgentRoles_RejectsBeforeClusterAccess covers the fail-before- +// connect contract: every input ProvisionAgentRoles can reject without a +// cluster must be rejected before the Kubernetes client is built, so a bad +// value is reported as itself rather than masked by a kubeconfig error on a +// machine with no cluster configured. +// +// The cluster-side behavior (existence check, naming, idempotent +// create-or-update) is covered against a fake clientset in pkg/k8s/agent. +func TestProvisionAgentRoles_RejectsBeforeClusterAccess(t *testing.T) { + tests := []struct { + name string + config *AgentRolesConfig + }{ + {name: "nil config", config: nil}, + {name: "empty namespace", config: &AgentRolesConfig{ServiceAccountName: "irsa-snapshotter"}}, + {name: "whitespace namespace", config: &AgentRolesConfig{Namespace: " ", ServiceAccountName: "irsa-snapshotter"}}, + {name: "empty ServiceAccount name", config: &AgentRolesConfig{Namespace: "gpu-operator"}}, + {name: "whitespace ServiceAccount name", config: &AgentRolesConfig{Namespace: "gpu-operator", ServiceAccountName: " "}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // A kubeconfig path that cannot resolve: if the rejection ever + // moved after client construction, this test would start + // failing on the wrong error instead of passing silently. + if tt.config != nil { + tt.config.Kubeconfig = "/nonexistent/kubeconfig-that-must-not-be-read" + } + + res, err := ProvisionAgentRoles(context.Background(), tt.config) + if err == nil { + t.Fatal("ProvisionAgentRoles() error = nil, want ErrCodeInvalidRequest") + } + if res != nil { + t.Errorf("result = %+v, want nil", res) + } + if !stderrors.Is(err, errors.New(errors.ErrCodeInvalidRequest, "")) { + t.Errorf("error = %v, want ErrCodeInvalidRequest", err) + } + }) + } +} diff --git a/tools/cleanup b/tools/cleanup index 4fe75abaa..eae80dcca 100755 --- a/tools/cleanup +++ b/tools/cleanup @@ -380,8 +380,15 @@ fi # runID suffix; we delete by label selector to catch all runs. Legacy resources # from the older job pattern lived in gpu-operator under the name "aicr". msg "Phase 2: Removing AICR validator artifacts..." -kc delete clusterrolebinding -l app.kubernetes.io/managed-by=aicr --ignore-not-found -kc delete clusterrole -l app.kubernetes.io/managed-by=aicr --ignore-not-found +# The component!=agent-rbac term spares the PERMANENT cluster RBAC that +# `aicr snapshot --add-roles-to-service-account` grants to an operator-supplied +# ServiceAccount. Those objects belong to no run — they carry no run-ID label +# and no run's cleanup deletes them — and re-creating them needs the admin who +# provisioned them, so a teardown tool must not sweep them away with the +# per-run leftovers it exists to reclaim. Delete them by hand when the +# ServiceAccount is retired. +kc delete clusterrolebinding -l 'app.kubernetes.io/managed-by=aicr,app.kubernetes.io/component!=agent-rbac' --ignore-not-found +kc delete clusterrole -l 'app.kubernetes.io/managed-by=aicr,app.kubernetes.io/component!=agent-rbac' --ignore-not-found kc delete clusterrolebinding -l app.kubernetes.io/name=aicr-validator --ignore-not-found kc delete clusterrole -l app.kubernetes.io/name=aicr-validator --ignore-not-found # Fall back to name-based deletion for any without the expected labels. From db4397435db19e38d6e3aef19230cc5c3bc7e5d9 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Tue, 25 Aug 2026 11:33:40 -0700 Subject: [PATCH 46/56] docs: document exact-if-exists ServiceAccounts and the isolation waiver Covers both flags and the trade-off an operator must be able to learn before choosing it, rather than after. docs/user/cli-reference.md states the exact-if-exists rule for --service-account-name on both `aicr snapshot` and `aicr validate`, adds --add-roles-to-service-account, and replaces the stale `--service-account-name aicr` example with the provision-then-capture pair. docs/user/agent-deployment.md gains a migration section: why a pre-created ServiceAccount stopped being used (IRSA and GKE Workload Identity both pin trust to the ServiceAccount name), the supported flow, what provisioning creates and how to remove it, and the waiver -- concurrent runs sharing one ServiceAccount share its grants, and a --discover-network provisioning leaves mutating cluster permissions in place permanently rather than for one run's lifetime. docs/integrator/go-library.md and the AgentConfig godoc in pkg/client/v1, pkg/snapshotter and pkg/config carry the same dual semantics for ServiceAccountName; the facade/internal mirror the doc relies on is enforced by TestAgentConfigMirrorsInternal, now named there. Signed-off-by: Alex Yuskauskas --- docs/integrator/go-library.md | 65 +++++++++++++++++++-- docs/user/agent-deployment.md | 106 +++++++++++++++++++++++++++++++++- docs/user/cli-reference.md | 19 +++++- pkg/cli/validate.go | 2 +- pkg/client/v1/aicr.go | 17 ++++-- pkg/client/v1/types.go | 68 +++++++++++++++------- pkg/config/resolve.go | 18 ++++-- pkg/snapshotter/agent.go | 26 ++++++++- 8 files changed, 276 insertions(+), 45 deletions(-) diff --git a/docs/integrator/go-library.md b/docs/integrator/go-library.md index 4341a33b9..5e006cef8 100644 --- a/docs/integrator/go-library.md +++ b/docs/integrator/go-library.md @@ -221,8 +221,8 @@ reported as no drift. // CollectSnapshot deploys a snapshotter Job to the target cluster and // returns the resulting Snapshot. cfg is a facade-owned struct that // mirrors pkg/snapshotter.AgentConfig field for field; the mirror is -// enforced by a test, so a field added upstream cannot silently stay at -// its zero value here. +// enforced by TestAgentConfigMirrorsInternal, so a field added upstream +// cannot silently stay at its zero value here. // // The returned Snapshot carries the parsed form plus Snapshot.Raw — the // exact bytes the agent emitted. Persist Raw rather than re-serializing @@ -251,10 +251,17 @@ snap, err := client.CollectSnapshot(snapCtx, &aicr.AgentConfig{ // Namespace is required and validated (it becomes the RBAC/Job namespace // and the internal staging ConfigMap's namespace). Image is not // validated — an empty value becomes an empty container image that the - // API server rejects. JobName and ServiceAccountName are optional name - // prefixes; leaving them unset defaults both to "aicr" with a generated - // run ID appended, so every run gets its own uniquely named Job and - // ServiceAccount without the caller having to manage that. + // API server rejects. JobName is an optional name prefix; leaving it + // unset defaults to "aicr" with a generated run ID appended, so every + // run gets its own uniquely named Job without the caller managing that. + // + // ServiceAccountName carries two meanings and is EXACT-IF-EXISTS. When + // a ServiceAccount of exactly that name already exists in Namespace it + // is used verbatim and the run creates NO ServiceAccount, Role, + // RoleBinding, ClusterRole or ClusterRoleBinding — and deletes none at + // cleanup. Otherwise it is a prefix and the run creates and owns the + // full run-scoped RBAC set. Leaving it unset keeps the run-scoped + // default and never probes for an existing ServiceAccount. Namespace: "aicr-snapshot", Image: "ghcr.io/nvidia/aicr:v0.19.0", Timeout: 5 * time.Minute, @@ -338,6 +345,52 @@ Valid phase values are `PhaseDeployment`, `PhaseConformance`, and `ErrCodeInvalidRequest` before any cluster work, so a typo cannot silently degrade to an empty run. +#### Running the agent as an existing ServiceAccount + +`AgentConfig.ServiceAccountName` is **exact-if-exists**, so it carries two +meanings depending on the cluster: + +- A ServiceAccount of exactly that name already exists in `Namespace`: the + agent pod runs as it verbatim, and `CollectSnapshot` creates **no** + ServiceAccount, Role, RoleBinding, ClusterRole, or ClusterRoleBinding, and + deletes none at cleanup. +- Otherwise it is a name prefix and the run creates and owns the full + run-scoped RBAC set, named `-`. + +Leaving the field empty keeps the run-scoped default and never probes for an +existing ServiceAccount, so a stray ServiceAccount cannot capture a run. + +Use the first form when the ServiceAccount must carry EKS IRSA or GKE Workload +Identity annotations: both providers pin trust to the ServiceAccount *name*, so +a run-scoped name can never be trusted by either. Grant it the agent's +permissions once — the objects it creates are permanent and no run cleanup +removes them: + +```go +// Admin step, run once. Provisions and returns; it deploys no Job. +// Returns ErrCodeNotFound when the ServiceAccount does not exist. +res, err := snapshotter.ProvisionAgentRoles(ctx, &snapshotter.AgentRolesConfig{ + Kubeconfig: "/path/to/target-kubeconfig", + Namespace: "gpu-operator", + ServiceAccountName: "irsa-snapshotter", + // DiscoverNetwork also grants the cluster-scoped MUTATING rules live + // network discovery needs — permanently, not for one run's lifetime. + DiscoverNetwork: false, +}) +if err != nil { + log.Fatalf("provision agent roles: %v", err) +} +log.Printf("granted via %s/%s and %s/%s", + res.Role, res.RoleBinding, res.ClusterRole, res.ClusterRoleBinding) +``` + +Adopting one ServiceAccount across runs waives per-run permission isolation: +concurrent runs sharing it hold the same grants, and a `DiscoverNetwork` +provisioning leaves mutating cluster permissions in place until an operator +removes them. See +[Agent Deployment](../user/agent-deployment.md#using-an-existing-serviceaccount-irsa-and-workload-identity) +for the full migration path and teardown commands. + ### Loading an existing recipe When a recipe has already been resolved and persisted (for example a diff --git a/docs/user/agent-deployment.md b/docs/user/agent-deployment.md index 5e0f77029..47cee070f 100644 --- a/docs/user/agent-deployment.md +++ b/docs/user/agent-deployment.md @@ -132,7 +132,8 @@ aicr snapshot \ - `--image`: Container image (default: matches the CLI version, e.g. `ghcr.io/nvidia/aicr:v0.19.0`; dev and snapshot builds use `:latest`) - `--image-pull-secret`: Secret name for pulling the agent image from a private registry (repeatable) - `--job-name`: Job name prefix (default: `aicr`); the run ID is always appended (`-`) -- `--service-account-name`: ServiceAccount name prefix (default: `aicr`); the run ID is always appended (`-`) +- `--service-account-name`: ServiceAccount the agent pod runs as. **Exact-if-exists** — an existing ServiceAccount of exactly this name in `--namespace` is used verbatim and the run creates no RBAC; otherwise it is a name prefix (default: `aicr`) and the run ID is appended (`-`). See [Using an existing ServiceAccount](#using-an-existing-serviceaccount-irsa-and-workload-identity) +- `--add-roles-to-service-account`: Grant the agent's permissions to the named **existing** ServiceAccount and exit **without taking a snapshot**. Idempotent; what it creates is permanent. See [Using an existing ServiceAccount](#using-an-existing-serviceaccount-irsa-and-workload-identity) - `--node-selector`: Node selector (format: `key=value`, repeatable) - `--toleration`: Toleration (format: `key=value:effect`, repeatable). **Default: all taints are tolerated** (uses `operator: Exists` without key). Only specify this flag if you want to restrict which taints the Job can tolerate. - `--timeout`: Wait timeout (default: `5m`) @@ -204,6 +205,109 @@ aicr snapshot --image ghcr.io/nvidia/aicr:v0.19.0 - [GitHub Releases](https://github.com/NVIDIA/aicr/releases) - Container registry: [ghcr.io/nvidia/aicr](https://github.com/NVIDIA/aicr/pkgs/container/aicr) +## Using an existing ServiceAccount (IRSA and Workload Identity) + +By default the agent creates its own ServiceAccount for each run and deletes it +at cleanup, so no two runs share an identity. That does not work when the +ServiceAccount must carry cloud IAM credentials: **EKS IRSA** +(`eks.amazonaws.com/role-arn`) and **GKE Workload Identity** +(`iam.gke.io/gcp-service-account`) both pin trust to the ServiceAccount *name*. +IRSA's role trust policy conditions on +`system:serviceaccount::`, and a GKE IAM binding names the KSA +as `PROJECT.svc.id.goog[/]` and accepts no wildcard. A +per-run name can never be trusted by either, and copying the annotations onto a +run-scoped ServiceAccount does not help. + +`--service-account-name` therefore behaves as **exact-if-exists**: + +| Does a ServiceAccount of exactly that name exist in `--namespace`? | Behavior | +|---|---| +| Yes | Used verbatim. aicr creates **no** ServiceAccount, Role, RoleBinding, ClusterRole, or ClusterRoleBinding for the run, binds nothing to it, and deletes nothing at cleanup. | +| No | The value is a name prefix. The run creates `-` plus the full run-scoped RBAC set, and deletes them at cleanup — the pre-existing behavior. | + +An unset `--service-account-name` is never probed for existence, so a stray +ServiceAccount named `aicr` cannot silently capture a run. + +### Migrating a pre-created ServiceAccount + +**What changed.** Before run isolation, passing `--service-account-name` at a +ServiceAccount you had created out of band got that ServiceAccount adopted, and +aicr attached its Role and RoleBinding to it. Run isolation made every +run-owned object run-scoped, which turned the flag into a prefix — so a +pre-created ServiceAccount stopped being used at all, and an agent pod that +had been running with IRSA or Workload Identity credentials silently started +running without them. Exact-if-exists restores the pre-created ServiceAccount +as the one the pod runs as; what does **not** come back is aicr managing its +permissions. + +**Supported flow.** Run the provisioning command once, as an admin, then take +snapshots normally: + +```shell +# 1. Your ServiceAccount, created and annotated by you (or by eksctl / Terraform). +kubectl create serviceaccount irsa-snapshotter -n gpu-operator +kubectl annotate serviceaccount irsa-snapshotter -n gpu-operator \ + eks.amazonaws.com/role-arn=arn:aws:iam::123456789012:role/aicr-snapshot + +# 2. Grant it the agent's permissions. Provisions and exits - no snapshot is taken. +aicr snapshot --namespace gpu-operator --add-roles-to-service-account irsa-snapshotter + +# 3. Capture snapshots as that ServiceAccount, as often as you like. +aicr snapshot \ + --namespace gpu-operator \ + --service-account-name irsa-snapshotter \ + --output cm://gpu-operator/aicr-snapshot +``` + +Step 2 fails with `NOT_FOUND` when the ServiceAccount does not exist: aicr +grants permissions to an identity you control, and never creates one. + +### What provisioning creates + +| Object | Name | Scope | +|---|---|---| +| `Role`, `RoleBinding` | `aicr-agent--rbac` | `--namespace` | +| `ClusterRole`, `ClusterRoleBinding` | `aicr-agent---rbac` | Cluster | + +The rules are the same ones a run-scoped grant carries, so an adopted +ServiceAccount is never less capable than a run-owned one. The names end in +`-rbac`, which no run-scoped name can (a run-scoped name always ends in a run +ID whose last segment is hexadecimal), so the two name spaces cannot collide. + +**These objects are permanent.** They carry no `aicr.run/run-id` label, no run +enters them into its cleanup list, and no `aicr snapshot` or `aicr validate` +invocation deletes them. Removing them is your job: + +```shell +kubectl delete role,rolebinding aicr-agent-irsa-snapshotter-rbac -n gpu-operator +kubectl delete clusterrole,clusterrolebinding aicr-agent-gpu-operator-irsa-snapshotter-rbac +``` + +The command is idempotent — re-run it after an aicr upgrade and the rules are +refreshed in place rather than duplicated or left stale. + +### Trade-off: per-run permission isolation is waived + +Using an existing ServiceAccount is an opt-in exchange, and it is worth +understanding before you choose it: + +- **Concurrent runs share one identity.** Two `aicr snapshot` invocations using + the same ServiceAccount hold exactly the same grants. Run isolation's + guarantee that one run's permissions cannot reach another run's does not + apply to them. +- **`--discover-network` grants become permanent.** With a run-owned + ServiceAccount, the cluster-scoped **mutating** discovery rules + (`nodes: patch`, `pods/exec: create`, CRD / namespace / DaemonSet + create-delete — see [Security Considerations](#security-considerations)) + exist for one run and are revoked at cleanup. Provisioned with + `aicr snapshot --add-roles-to-service-account --discover-network`, they + sit on that ServiceAccount until you remove them. + +Provision without `--discover-network` unless you need live network discovery; +that grant is read-only. If you need discovery only occasionally, prefer a +run-owned ServiceAccount for those runs, or provision a separate +ServiceAccount used only for discovery. + ## Post-Deployment ### Retrieve Snapshot diff --git a/docs/user/cli-reference.md b/docs/user/cli-reference.md index 5e1d09a7e..ae5a2bdb0 100644 --- a/docs/user/cli-reference.md +++ b/docs/user/cli-reference.md @@ -85,7 +85,8 @@ aicr snapshot [flags] | `--namespace` | `-n` | string | default | Kubernetes namespace for agent deployment | | `--image` | | string | matches CLI version | Container image for agent Job. Release builds default to `ghcr.io/nvidia/aicr:v`; dev and `-next` snapshot builds default to `ghcr.io/nvidia/aicr:latest`. | | `--job-name` | | string | aicr | Prefix for the agent Job name; the run ID is always appended (`-`) | -| `--service-account-name` | | string | aicr | Prefix for the agent Job's ServiceAccount name; the run ID is always appended (`-`) | +| `--service-account-name` | | string | aicr | ServiceAccount the agent pod runs as. **Exact-if-exists:** when a ServiceAccount of exactly this name already exists in `--namespace`, it is used verbatim and the run creates **no** ServiceAccount, Role, RoleBinding, ClusterRole, or ClusterRoleBinding — and deletes none at cleanup. Otherwise the value is a name prefix and the run ID is appended (`-`). See [Using an existing ServiceAccount](agent-deployment.md#using-an-existing-serviceaccount-irsa-and-workload-identity) | +| `--add-roles-to-service-account` | | string | | Grant the agent's permissions to the named **existing** ServiceAccount in `--namespace`, then exit **without taking a snapshot**. Creates permanent, non-run-scoped `Role`/`RoleBinding` (`aicr-agent--rbac`) and `ClusterRole`/`ClusterRoleBinding` (`aicr-agent---rbac`) that no run cleanup ever removes. Idempotent — re-run it after an aicr upgrade to refresh the rules. Fails with `NOT_FOUND` when the ServiceAccount does not exist. Combine with `--discover-network` to also grant the mutating live-discovery rules, permanently | | `--node-selector` | | string[] | auto | Node selector for agent scheduling (key=value, repeatable). When omitted (and neither `--require-gpu` nor `--runtime-class` is set), the agent auto-targets GPU nodes labeled `nvidia.com/gpu.present=true` if the cluster has any — see [Agent Deployment](agent-deployment.md). Pass an explicit selector to override. | | `--toleration` | | string[] | all taints | Tolerations for agent scheduling (key=value:effect, repeatable). **Default: all taints tolerated** (uses `operator: Exists`). Only specify to restrict which taints are tolerated. | | `--timeout` | | duration | 5m | Timeout for agent Job completion | @@ -169,13 +170,25 @@ aicr snapshot \ --namespace gpu-operator \ --image ghcr.io/nvidia/aicr:v0.19.0 \ --job-name snapshot-gpu-nodes \ - --service-account-name aicr \ --node-selector accelerator=nvidia-h100 \ --toleration nvidia.com/gpu:NoSchedule \ --timeout 10m \ --output cm://gpu-operator/aicr-snapshot \ --no-cleanup +# Grant the agent's permissions to an existing ServiceAccount, then exit. +# Run once, by an admin. Takes no snapshot; what it creates is permanent. +aicr snapshot \ + --namespace gpu-operator \ + --add-roles-to-service-account irsa-snapshotter + +# Capture as that ServiceAccount. Because it already exists, the name is used +# verbatim and this run creates and deletes no RBAC of its own. +aicr snapshot \ + --namespace gpu-operator \ + --service-account-name irsa-snapshotter \ + --output cm://gpu-operator/aicr-snapshot + # Custom template formatting aicr snapshot --template examples/templates/snapshot-template.md.tmpl @@ -1032,7 +1045,7 @@ aicr validate [flags] | `--image` | | string | ghcr.io/nvidia/aicr:latest | Container image for validation Job | | `--image-pull-secret` | | string[] | | Image pull secrets for private registries (repeatable) | | `--job-name` | | string | aicr-validate | Prefix for the **live snapshot-capture agent's** Job name; the run ID is always appended (`-`). Inert when `--snapshot` is supplied — no agent is deployed. Does not name the validator Jobs (`aicr--`) | -| `--service-account-name` | | string | aicr-validate | Prefix for the **live snapshot-capture agent's** ServiceAccount, Role, and RoleBinding; the run ID is always appended (`-`). Inert when `--snapshot` is supplied. Does not name the validator Jobs' ServiceAccount (`aicr-validator-`) | +| `--service-account-name` | | string | aicr-validate | ServiceAccount the **live snapshot-capture agent** runs as. **Exact-if-exists:** an existing ServiceAccount of exactly this name in `--namespace` is used verbatim and the agent creates no RBAC for the run; otherwise the value is a prefix for the agent's ServiceAccount, Role, and RoleBinding and the run ID is appended (`-`). Inert when `--snapshot` is supplied. Does not name the validator Jobs' ServiceAccount (`aicr-validator-`), whose RBAC is always run-scoped. Provision an existing ServiceAccount with `aicr snapshot --namespace --add-roles-to-service-account `, matching this command's `--namespace` | | `--node-selector` | | string[] | | Override GPU node selection for the live snapshot agent (when `--snapshot` is omitted) and inner validation workloads. Replaces platform-specific selectors (e.g., `cloud.google.com/gke-accelerator`, `node.kubernetes.io/instance-type`) on inner workloads like NCCL benchmark pods. Use when GPU nodes have non-standard labels. Does not affect the validator orchestrator Job. (format: key=value, repeatable) | | `--toleration` | | string[] | | Override tolerations for the live snapshot agent (when `--snapshot` is omitted) and inner validation workloads. When omitted, the snapshot agent tolerates all taints. Does not affect the validator orchestrator Job. (format: key=value:effect, repeatable) | | `--timeout` | | duration | 5m | Timeout for validation Job completion | diff --git a/pkg/cli/validate.go b/pkg/cli/validate.go index 4dfafa843..79296660e 100644 --- a/pkg/cli/validate.go +++ b/pkg/cli/validate.go @@ -498,7 +498,7 @@ func validateCmdFlags() []cli.Flag { }, &cli.StringFlag{ Name: "service-account-name", - Usage: "ServiceAccount name prefix (default: \"aicr-validate\"); the run ID is always appended", + Usage: "ServiceAccount the live snapshot-capture agent runs as. Exact-if-exists: when a ServiceAccount of exactly this name already exists in --namespace it is used verbatim and the agent creates and deletes no RBAC for the run (grant it permissions once with 'aicr snapshot --add-roles-to-service-account'). Otherwise it is a name prefix (default: \"aicr-validate\") and the run ID is appended.", Category: catAgentDeployment, }, &cli.StringSliceFlag{ diff --git a/pkg/client/v1/aicr.go b/pkg/client/v1/aicr.go index 391f77965..4aacfe65f 100644 --- a/pkg/client/v1/aicr.go +++ b/pkg/client/v1/aicr.go @@ -1726,12 +1726,17 @@ func resolveHelmComponentValues( // on a Client whose recipe source is unrelated to the target cluster. // // cfg.Kubeconfig is the path (or empty for in-cluster). cfg.Namespace and -// cfg.Image must be set. cfg.JobName and cfg.ServiceAccountName are -// optional naming prefixes, not required names — leaving them empty is -// fine: cfg.NameBase (default "aicr") supplies the prefix instead, and -// cfg.RunID is appended to whichever prefix applies, so every object this -// call deploys is named uniquely to this run either way. Other fields fall -// back to package defaults documented on snapshotter.AgentConfig. +// cfg.Image must be set. cfg.JobName is an optional naming prefix, not a +// required name — leaving it empty is fine: cfg.NameBase (default "aicr") +// supplies the prefix instead, and cfg.RunID is appended to whichever +// prefix applies, so the Job this call deploys is named uniquely to this +// run either way. +// +// cfg.ServiceAccountName is exact-if-exists: it names an existing +// ServiceAccount when one of exactly that name is already in cfg.Namespace +// — in which case this call creates and deletes no RBAC at all — and is a +// prefix otherwise. See its own documentation on AgentConfig. Other fields +// fall back to package defaults documented on snapshotter.AgentConfig. // // # Output and delivery // diff --git a/pkg/client/v1/types.go b/pkg/client/v1/types.go index 55c469fd0..97f7545de 100644 --- a/pkg/client/v1/types.go +++ b/pkg/client/v1/types.go @@ -122,25 +122,49 @@ func (s *Snapshot) Unwrap() *snapshotter.Snapshot { // this type — so an unplumbed field is a test failure rather than a silent // zero value. type AgentConfig struct { - Kubeconfig string - Namespace string - Image string - ImagePullSecrets []string - JobName string + Kubeconfig string + Namespace string + Image string + ImagePullSecrets []string + JobName string + + // ServiceAccountName selects the ServiceAccount the agent pod runs + // as. It is EXACT-IF-EXISTS, so it carries two meanings: + // + // - A ServiceAccount of exactly this name already exists in + // Namespace: it is used verbatim, and the run creates NO + // ServiceAccount, Role, RoleBinding, ClusterRole or + // ClusterRoleBinding — and deletes none at cleanup. This is how a + // ServiceAccount carrying IRSA (eks.amazonaws.com/role-arn) or + // GKE Workload Identity (iam.gke.io/gcp-service-account) + // annotations stays usable: both providers pin trust to the + // ServiceAccount NAME, which a run-scoped name can never satisfy. + // Grant it the agent's permissions once with + // snapshotter.ProvisionAgentRoles (CLI: + // `aicr snapshot --add-roles-to-service-account`). + // - Otherwise: a name prefix. The run creates "-" + // and the full run-scoped RBAC set, and deletes them at cleanup. + // + // Empty falls back to NameBase and is never probed for existence. + // + // Using an existing ServiceAccount waives per-run permission + // isolation: concurrent runs sharing it share its grants, and grants + // provisioned for DiscoverNetwork persist beyond any one run. ServiceAccountName string - NodeSelector map[string]string - Tolerations []corev1.Toleration - Timeout time.Duration - Cleanup bool - Debug bool - Privileged bool - RequireGPU bool - RuntimeClassName string - TemplatePath string - MaxNodesPerEntry int - OS string - Requests corev1.ResourceList - Limits corev1.ResourceList + + NodeSelector map[string]string + Tolerations []corev1.Toleration + Timeout time.Duration + Cleanup bool + Debug bool + Privileged bool + RequireGPU bool + RuntimeClassName string + TemplatePath string + MaxNodesPerEntry int + OS string + Requests corev1.ResourceList + Limits corev1.ResourceList // Output selects where the agent Job stages its result. A cm://namespace/name // URI makes that ConfigMap the delivery vehicle — the Job writes there and @@ -193,9 +217,11 @@ type AgentConfig struct { // Setting only one of the two therefore leaves NameBase governing the // other. Defaults to "aicr" when also empty. // - // JobName and ServiceAccountName themselves are optional prefixes, - // not required names — RunID is appended to whichever prefix - // applies, so the deployed object names are always run-scoped. + // JobName is likewise an optional prefix, not a required name — + // RunID is appended to whichever prefix applies, so the deployed Job + // name is always run-scoped. ServiceAccountName is a prefix only + // when no ServiceAccount of that exact name exists; see its own + // documentation above. NameBase string } diff --git a/pkg/config/resolve.go b/pkg/config/resolve.go index 187271f4a..fb4299fc4 100644 --- a/pkg/config/resolve.go +++ b/pkg/config/resolve.go @@ -344,9 +344,12 @@ type ValidateResolved struct { // Job name is always run-scoped. JobName string - // ServiceAccountName is spec.validate.agent.serviceAccountName — an - // optional ServiceAccount name prefix, not a required name. Same - // empty-value behavior as JobName. + // ServiceAccountName is spec.validate.agent.serviceAccountName. It is + // exact-if-exists: when a ServiceAccount of exactly this name already + // exists in the namespace, the live snapshot-capture agent runs as it + // verbatim and creates no RBAC for the run; otherwise it is an + // optional name prefix with the same empty-value behavior as JobName. + // See pkg/snapshotter.AgentConfig.ServiceAccountName. ServiceAccountName string // NodeSelector is spec.validate.agent.nodeSelector. Nil if unset; @@ -654,9 +657,12 @@ type SnapshotResolved struct { // name is always run-scoped. JobName string - // ServiceAccountName is spec.snapshot.agent.serviceAccountName — an - // optional ServiceAccount name prefix, not a required name. Same - // empty-value behavior as JobName. + // ServiceAccountName is spec.snapshot.agent.serviceAccountName. It is + // exact-if-exists: when a ServiceAccount of exactly this name already + // exists in the namespace, the agent runs as it verbatim and creates + // no RBAC for the run; otherwise it is an optional name prefix with + // the same empty-value behavior as JobName. See + // pkg/snapshotter.AgentConfig.ServiceAccountName. ServiceAccountName string // NodeSelector is spec.snapshot.agent.nodeSelector. Nil if unset; diff --git a/pkg/snapshotter/agent.go b/pkg/snapshotter/agent.go index 7c3d4ad0d..22882442b 100644 --- a/pkg/snapshotter/agent.go +++ b/pkg/snapshotter/agent.go @@ -65,7 +65,31 @@ type AgentConfig struct { // JobName for the agent Job JobName string - // ServiceAccountName for the agent + // ServiceAccountName selects the ServiceAccount the agent pod runs + // as. It is EXACT-IF-EXISTS, so it carries two meanings resolved once + // per deployment: + // + // - A ServiceAccount of exactly this name already exists in + // Namespace: it is used verbatim, and the run creates NO + // ServiceAccount, Role, RoleBinding, ClusterRole or + // ClusterRoleBinding — and deletes none at cleanup. aicr adds and + // removes no permissions on an identity it did not create. This + // is how a ServiceAccount carrying IRSA + // (eks.amazonaws.com/role-arn) or GKE Workload Identity + // (iam.gke.io/gcp-service-account) annotations stays usable: both + // providers pin trust to the ServiceAccount NAME, which a + // run-scoped name can never satisfy. Grant it the agent's + // permissions once with ProvisionAgentRoles. + // - Otherwise: a name prefix. The run creates "-" + // and the full run-scoped RBAC set, and deletes them at cleanup. + // + // Empty falls back to NameBase and is never probed for existence, so + // a stray ServiceAccount sitting at the default base cannot silently + // capture the run. + // + // Using an existing ServiceAccount waives per-run permission + // isolation: concurrent runs sharing it share its grants, and grants + // provisioned for DiscoverNetwork persist beyond any one run. ServiceAccountName string // NodeSelector for targeting specific nodes From 811d05a8c0c511eeca601d42aabb1156fbdef7e8 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Tue, 25 Aug 2026 13:33:05 -0700 Subject: [PATCH 47/56] test(agent): stop shadowing err in the existing-ServiceAccount test Signed-off-by: Alex Yuskauskas --- pkg/k8s/agent/rbac_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/k8s/agent/rbac_test.go b/pkg/k8s/agent/rbac_test.go index a541aca5c..ea71ac436 100644 --- a/pkg/k8s/agent/rbac_test.go +++ b/pkg/k8s/agent/rbac_test.go @@ -234,8 +234,8 @@ func TestDeploy_ExistingServiceAccountCreatesAndDeletesNoRBAC(t *testing.T) { } } - if err := d.Cleanup(ctx, CleanupOptions{Enabled: true}); err != nil { - t.Fatalf("Cleanup() error = %v", err) + if cleanupErr := d.Cleanup(ctx, CleanupOptions{Enabled: true}); cleanupErr != nil { + t.Fatalf("Cleanup() error = %v", cleanupErr) } sa, err := clientset.CoreV1().ServiceAccounts(testNamespace).Get(ctx, saName, metav1.GetOptions{}) From dec685d8126da0a5e9e4435f3006587b471e25e9 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Tue, 25 Aug 2026 14:35:28 -0700 Subject: [PATCH 48/56] feat(agent): write --add-roles-to-service-account RBAC as manifests The flag no longer submits anything to the cluster. It renders the Role, RoleBinding, ClusterRole and ClusterRoleBinding that grant the snapshot agent's permissions to an operator-supplied ServiceAccount into a new snapshot-rbac-/ directory, prints the apply and delete commands, and exits. The grant it hands out is not small -- under --discover-network it carries nodes: patch, pods/exec: create, and cluster-wide CRD creation, and those outlive every run. An operator consenting to that should be able to read exactly what they are granting first, which a command that provisions on their behalf does not allow. So the decision moves to them: generate, review, kubectl apply -f /, and kubectl delete -f / when the ServiceAccount is done with it. That delete is now the supported teardown, which is why the directory is worth keeping. One object per file, numerically prefixed so kubectl's lexical directory walk applies each Role ahead of its binding and so a reader can take the files in one at a time. Each opens with a YAML comment header naming what the object grants and why the agent needs each rule; the --discover-network ClusterRole gets a warning block mapping every mutating rule to the concrete discovery step it exists for, so "nodes: patch" is never unexplained. Nothing on the path touches a cluster: no clientset, no ServiceAccount Get, no permission pre-flight. It works with no kubeconfig and no privileges. That drops the old ErrCodeNotFound on a missing ServiceAccount -- nothing is consulted that could know -- so a mistyped name now yields manifests the operator inspects before applying, and the rendered RoleBinding tells them how to check. The cluster-scoped name is not injective, and the subject collision the old path detected with a Get is likewise now a warning in the ClusterRoleBinding header rather than an error. The rules come from namespacedRules and clusterRules unchanged -- the same definitions ensureRole and ensureClusterRole build from -- so a rendered manifest cannot drift from what a run-owned grant carries. An existing output directory fails with ErrCodeConflict rather than overwriting a set someone is midway through reading, and a failed write removes the partial directory so the retry is not blocked by it. The flag name reads like it mutates the cluster, so its usage string now opens with "WRITES MANIFESTS AND APPLIES NOTHING" and names both kubectl commands. Signed-off-by: Alex Yuskauskas --- docs/user/agent-deployment.md | 97 +++-- docs/user/cli-reference.md | 12 +- pkg/cli/consts.go | 7 +- pkg/cli/snapshot.go | 125 +++--- pkg/cli/snapshot_test.go | 127 ++++-- pkg/cli/validate.go | 2 +- pkg/client/v1/types.go | 7 +- pkg/defaults/k8s.go | 28 ++ pkg/defaults/timeouts.go | 8 - pkg/k8s/agent/deployer.go | 5 +- pkg/k8s/agent/doc.go | 14 +- pkg/k8s/agent/provision.go | 629 +++++++++++++++++++----------- pkg/k8s/agent/provision_test.go | 584 +++++++++++++-------------- pkg/k8s/agent/rbac.go | 10 +- pkg/k8s/agent/types.go | 5 +- pkg/k8s/labels/labels.go | 15 +- pkg/snapshotter/agent.go | 4 +- pkg/snapshotter/provision.go | 200 +++++++--- pkg/snapshotter/provision_test.go | 305 +++++++++++++-- tools/cleanup | 15 +- 20 files changed, 1459 insertions(+), 740 deletions(-) diff --git a/docs/user/agent-deployment.md b/docs/user/agent-deployment.md index 47cee070f..3b30dda09 100644 --- a/docs/user/agent-deployment.md +++ b/docs/user/agent-deployment.md @@ -133,7 +133,7 @@ aicr snapshot \ - `--image-pull-secret`: Secret name for pulling the agent image from a private registry (repeatable) - `--job-name`: Job name prefix (default: `aicr`); the run ID is always appended (`-`) - `--service-account-name`: ServiceAccount the agent pod runs as. **Exact-if-exists** — an existing ServiceAccount of exactly this name in `--namespace` is used verbatim and the run creates no RBAC; otherwise it is a name prefix (default: `aicr`) and the run ID is appended (`-`). See [Using an existing ServiceAccount](#using-an-existing-serviceaccount-irsa-and-workload-identity) -- `--add-roles-to-service-account`: Grant the agent's permissions to the named **existing** ServiceAccount and exit **without taking a snapshot**. Idempotent; what it creates is permanent. See [Using an existing ServiceAccount](#using-an-existing-serviceaccount-irsa-and-workload-identity) +- `--add-roles-to-service-account`: **Writes manifests and applies nothing.** Renders the RBAC that grants the agent's permissions to the named ServiceAccount into `./snapshot-rbac-/` and exits **without taking a snapshot**. No cluster is contacted. You review the files, then apply and later delete them yourself. See [Using an existing ServiceAccount](#using-an-existing-serviceaccount-irsa-and-workload-identity) - `--node-selector`: Node selector (format: `key=value`, repeatable) - `--toleration`: Toleration (format: `key=value:effect`, repeatable). **Default: all taints are tolerated** (uses `operator: Exists` without key). Only specify this flag if you want to restrict which taints the Job can tolerate. - `--timeout`: Wait timeout (default: `5m`) @@ -240,8 +240,8 @@ running without them. Exact-if-exists restores the pre-created ServiceAccount as the one the pod runs as; what does **not** come back is aicr managing its permissions. -**Supported flow.** Run the provisioning command once, as an admin, then take -snapshots normally: +**Supported flow.** Generate the RBAC manifests, read them, apply them, then +take snapshots normally: ```shell # 1. Your ServiceAccount, created and annotated by you (or by eksctl / Terraform). @@ -249,42 +249,84 @@ kubectl create serviceaccount irsa-snapshotter -n gpu-operator kubectl annotate serviceaccount irsa-snapshotter -n gpu-operator \ eks.amazonaws.com/role-arn=arn:aws:iam::123456789012:role/aicr-snapshot -# 2. Grant it the agent's permissions. Provisions and exits - no snapshot is taken. +# 2. Write the RBAC manifests. Applies NOTHING, contacts no cluster, takes no +# snapshot. Prints the directory it wrote and the commands below. aicr snapshot --namespace gpu-operator --add-roles-to-service-account irsa-snapshotter -# 3. Capture snapshots as that ServiceAccount, as often as you like. +# 3. Read what you are about to grant. Each file explains its rules. +less snapshot-rbac-/*.yaml + +# 4. Grant it, once you are satisfied. +kubectl apply -f snapshot-rbac-/ + +# 5. Capture snapshots as that ServiceAccount, as often as you like. aicr snapshot \ --namespace gpu-operator \ --service-account-name irsa-snapshotter \ --output cm://gpu-operator/aicr-snapshot + +# 6. Revoke the grant when the ServiceAccount no longer needs it. +kubectl delete -f snapshot-rbac-/ ``` -Step 2 fails with `NOT_FOUND` when the ServiceAccount does not exist: aicr -grants permissions to an identity you control, and never creates one. +Step 2 does **not** check that the ServiceAccount exists — it contacts no +cluster at all, so it cannot. A mistyped name yields manifests naming a subject +that does not resolve; Kubernetes accepts such a binding and it simply grants +nothing. The generated `02-rolebinding.yaml` tells you how to verify the name, +and `aicr` never creates a ServiceAccount for you. -### What provisioning creates +### What the manifests contain + +`aicr snapshot --add-roles-to-service-account ` writes a new directory in +your current working directory, one object per file: + +```text +snapshot-rbac-/ +├── 01-role.yaml Role/aicr-agent--rbac (namespaced) +├── 02-rolebinding.yaml RoleBinding/aicr-agent--rbac (namespaced) +├── 03-clusterrole.yaml ClusterRole/aicr-agent---rbac (cluster) +└── 04-clusterrolebinding.yaml ClusterRoleBinding/aicr-agent---rbac (cluster) +``` -| Object | Name | Scope | -|---|---|---| -| `Role`, `RoleBinding` | `aicr-agent--rbac` | `--namespace` | -| `ClusterRole`, `ClusterRoleBinding` | `aicr-agent---rbac` | Cluster | +Every file opens with a YAML comment header naming what the object grants and +why the agent needs each rule, so you can decide rule by rule whether to apply +it. The numeric prefixes exist because `kubectl apply -f /` visits a +directory in lexical order — they keep each Role ahead of the binding that +references it. You can also apply or read the files individually. -The rules are the same ones a run-scoped grant carries, so an adopted +The rules are the same ones a run-scoped grant carries, so a shared ServiceAccount is never less capable than a run-owned one. The names end in `-rbac`, which no run-scoped name can (a run-scoped name always ends in a run ID whose last segment is hexadecimal), so the two name spaces cannot collide. -**These objects are permanent.** They carry no `aicr.run/run-id` label, no run -enters them into its cleanup list, and no `aicr snapshot` or `aicr validate` -invocation deletes them. Removing them is your job: +**Nothing is applied for you, and nothing is removed for you.** The objects you +apply carry no `aicr.run/run-id` label, no run enters them into its cleanup +list, and no `aicr snapshot` or `aicr validate` invocation deletes them. +Teardown is one command, which is why keeping the directory is worth it: + +```shell +kubectl delete -f snapshot-rbac-/ +``` + +If you no longer have the directory, delete the objects by name instead: ```shell kubectl delete role,rolebinding aicr-agent-irsa-snapshotter-rbac -n gpu-operator kubectl delete clusterrole,clusterrolebinding aicr-agent-gpu-operator-irsa-snapshotter-rbac ``` -The command is idempotent — re-run it after an aicr upgrade and the rules are -refreshed in place rather than duplicated or left stale. +The directory name carries a fresh run ID on every invocation, so generating +twice never overwrites a set you are still reviewing; a directory that already +exists fails the command with `CONFLICT`. After an aicr upgrade, re-generate +and `kubectl apply -f` the new directory to refresh the rules in place. + +**Check for a name collision before applying the cluster-scoped pair.** Their +name joins the namespace and the ServiceAccount name with `-`, which is not +injective: namespace `a-b` with ServiceAccount `c` and namespace `a` with +ServiceAccount `b-c` both compose `aicr-agent-a-b-c-rbac`. Because nothing +reads your cluster, applying over an existing binding of that name would +retarget it and revoke the other ServiceAccount's grants. The generated +`04-clusterrolebinding.yaml` says so and gives you the `kubectl get` to run. ### Trade-off: per-run permission isolation is waived @@ -299,14 +341,17 @@ understanding before you choose it: ServiceAccount, the cluster-scoped **mutating** discovery rules (`nodes: patch`, `pods/exec: create`, CRD / namespace / DaemonSet create-delete — see [Security Considerations](#security-considerations)) - exist for one run and are revoked at cleanup. Provisioned with - `aicr snapshot --add-roles-to-service-account --discover-network`, they - sit on that ServiceAccount until you remove them. - -Provision without `--discover-network` unless you need live network discovery; -that grant is read-only. If you need discovery only occasionally, prefer a -run-owned ServiceAccount for those runs, or provision a separate -ServiceAccount used only for discovery. + exist for one run and are revoked at cleanup. Rendered with + `aicr snapshot --add-roles-to-service-account --discover-network` and + applied, they sit on that ServiceAccount until you remove them. + +Generate without `--discover-network` unless you need live network discovery; +that grant is read-only. When you do use it, `03-clusterrole.yaml` carries a +warning header naming every mutating rule and the discovery step it exists for +— read it before applying, which is precisely what writing manifests instead of +applying them is for. If you need discovery only occasionally, prefer a +run-owned ServiceAccount for those runs, or keep a separate ServiceAccount used +only for discovery. ## Post-Deployment diff --git a/docs/user/cli-reference.md b/docs/user/cli-reference.md index ae5a2bdb0..da0af2dac 100644 --- a/docs/user/cli-reference.md +++ b/docs/user/cli-reference.md @@ -86,7 +86,7 @@ aicr snapshot [flags] | `--image` | | string | matches CLI version | Container image for agent Job. Release builds default to `ghcr.io/nvidia/aicr:v`; dev and `-next` snapshot builds default to `ghcr.io/nvidia/aicr:latest`. | | `--job-name` | | string | aicr | Prefix for the agent Job name; the run ID is always appended (`-`) | | `--service-account-name` | | string | aicr | ServiceAccount the agent pod runs as. **Exact-if-exists:** when a ServiceAccount of exactly this name already exists in `--namespace`, it is used verbatim and the run creates **no** ServiceAccount, Role, RoleBinding, ClusterRole, or ClusterRoleBinding — and deletes none at cleanup. Otherwise the value is a name prefix and the run ID is appended (`-`). See [Using an existing ServiceAccount](agent-deployment.md#using-an-existing-serviceaccount-irsa-and-workload-identity) | -| `--add-roles-to-service-account` | | string | | Grant the agent's permissions to the named **existing** ServiceAccount in `--namespace`, then exit **without taking a snapshot**. Creates permanent, non-run-scoped `Role`/`RoleBinding` (`aicr-agent--rbac`) and `ClusterRole`/`ClusterRoleBinding` (`aicr-agent---rbac`) that no run cleanup ever removes. Idempotent — re-run it after an aicr upgrade to refresh the rules. Fails with `NOT_FOUND` when the ServiceAccount does not exist. Combine with `--discover-network` to also grant the mutating live-discovery rules, permanently | +| `--add-roles-to-service-account` | | string | | **Writes manifests and applies nothing.** Renders the `Role`/`RoleBinding` (`aicr-agent--rbac`) and `ClusterRole`/`ClusterRoleBinding` (`aicr-agent---rbac`) that grant the agent's permissions to the named ServiceAccount into `./snapshot-rbac-/`, one object per file with a comment header explaining what it grants, then exits **without taking a snapshot**. **No cluster is contacted** — no kubeconfig or privileges needed, and the ServiceAccount is not checked for existence. Review the files, then apply with `kubectl apply -f /` and revoke with `kubectl delete -f /` yourself; no run cleanup ever touches them. Fails with `CONFLICT` if the directory already exists. Combine with `--discover-network` to also render the mutating live-discovery rules | | `--node-selector` | | string[] | auto | Node selector for agent scheduling (key=value, repeatable). When omitted (and neither `--require-gpu` nor `--runtime-class` is set), the agent auto-targets GPU nodes labeled `nvidia.com/gpu.present=true` if the cluster has any — see [Agent Deployment](agent-deployment.md). Pass an explicit selector to override. | | `--toleration` | | string[] | all taints | Tolerations for agent scheduling (key=value:effect, repeatable). **Default: all taints tolerated** (uses `operator: Exists`). Only specify to restrict which taints are tolerated. | | `--timeout` | | duration | 5m | Timeout for agent Job completion | @@ -176,12 +176,16 @@ aicr snapshot \ --output cm://gpu-operator/aicr-snapshot \ --no-cleanup -# Grant the agent's permissions to an existing ServiceAccount, then exit. -# Run once, by an admin. Takes no snapshot; what it creates is permanent. +# Write the RBAC manifests that grant the agent's permissions to an existing +# ServiceAccount, then exit. Applies nothing and contacts no cluster; takes no +# snapshot. Writes ./snapshot-rbac-/. aicr snapshot \ --namespace gpu-operator \ --add-roles-to-service-account irsa-snapshotter +# Review what each file grants, then apply them yourself. +kubectl apply -f snapshot-rbac-/ + # Capture as that ServiceAccount. Because it already exists, the name is used # verbatim and this run creates and deletes no RBAC of its own. aicr snapshot \ @@ -1045,7 +1049,7 @@ aicr validate [flags] | `--image` | | string | ghcr.io/nvidia/aicr:latest | Container image for validation Job | | `--image-pull-secret` | | string[] | | Image pull secrets for private registries (repeatable) | | `--job-name` | | string | aicr-validate | Prefix for the **live snapshot-capture agent's** Job name; the run ID is always appended (`-`). Inert when `--snapshot` is supplied — no agent is deployed. Does not name the validator Jobs (`aicr--`) | -| `--service-account-name` | | string | aicr-validate | ServiceAccount the **live snapshot-capture agent** runs as. **Exact-if-exists:** an existing ServiceAccount of exactly this name in `--namespace` is used verbatim and the agent creates no RBAC for the run; otherwise the value is a prefix for the agent's ServiceAccount, Role, and RoleBinding and the run ID is appended (`-`). Inert when `--snapshot` is supplied. Does not name the validator Jobs' ServiceAccount (`aicr-validator-`), whose RBAC is always run-scoped. Provision an existing ServiceAccount with `aicr snapshot --namespace --add-roles-to-service-account `, matching this command's `--namespace` | +| `--service-account-name` | | string | aicr-validate | ServiceAccount the **live snapshot-capture agent** runs as. **Exact-if-exists:** an existing ServiceAccount of exactly this name in `--namespace` is used verbatim and the agent creates no RBAC for the run; otherwise the value is a prefix for the agent's ServiceAccount, Role, and RoleBinding and the run ID is appended (`-`). Inert when `--snapshot` is supplied. Does not name the validator Jobs' ServiceAccount (`aicr-validator-`), whose RBAC is always run-scoped. Generate that ServiceAccount's RBAC manifests with `aicr snapshot --namespace --add-roles-to-service-account ` (matching this command's `--namespace`) and apply them yourself — that command applies nothing | | `--node-selector` | | string[] | | Override GPU node selection for the live snapshot agent (when `--snapshot` is omitted) and inner validation workloads. Replaces platform-specific selectors (e.g., `cloud.google.com/gke-accelerator`, `node.kubernetes.io/instance-type`) on inner workloads like NCCL benchmark pods. Use when GPU nodes have non-standard labels. Does not affect the validator orchestrator Job. (format: key=value, repeatable) | | `--toleration` | | string[] | | Override tolerations for the live snapshot agent (when `--snapshot` is omitted) and inner validation workloads. When omitted, the snapshot agent tolerates all taints. Does not affect the validator orchestrator Job. (format: key=value:effect, repeatable) | | `--timeout` | | duration | 5m | Timeout for validation Job completion | diff --git a/pkg/cli/consts.go b/pkg/cli/consts.go index 22685df37..7db2a5ad7 100644 --- a/pkg/cli/consts.go +++ b/pkg/cli/consts.go @@ -40,9 +40,10 @@ const ( flagRuntimeInventory = "runtime-inventory" flagNoHealth = "no-health" - // flagAddRolesToSA switches `aicr snapshot` into a provision-and-exit - // invocation that grants the agent's permissions to an existing - // ServiceAccount. No snapshot is taken. + // flagAddRolesToSA switches `aicr snapshot` into a generate-and-exit + // invocation that WRITES the RBAC manifests granting the agent's + // permissions to a named ServiceAccount and applies none of them. No + // cluster is contacted and no snapshot is taken. flagAddRolesToSA = "add-roles-to-service-account" ) diff --git a/pkg/cli/snapshot.go b/pkg/cli/snapshot.go index 04ecafe39..df8a659db 100644 --- a/pkg/cli/snapshot.go +++ b/pkg/cli/snapshot.go @@ -20,6 +20,7 @@ import ( "io" "log/slog" "os" + "path/filepath" "strings" "time" @@ -271,20 +272,23 @@ func parseSnapshotCmdOptions(cmd *cli.Command, cfg *config.AICRConfig) (*snapsho }, nil } -// runAddRolesToServiceAccount handles the provision-and-exit invocation -// `aicr snapshot --add-roles-to-service-account `: it grants the agent's -// permissions to an already-existing ServiceAccount and returns without -// deploying a Job or capturing anything. +// runWriteRoleManifests handles the generate-and-exit invocation +// `aicr snapshot --add-roles-to-service-account `: it writes the RBAC +// manifests that would grant the agent's permissions to that ServiceAccount +// and returns, applying nothing, contacting no cluster, and capturing +// nothing. // -// It is a thin adapter — every decision (name derivation, existence check, -// rule set, idempotent create-or-update) lives in pkg/snapshotter and +// It takes no context because there is no I/O to bound beyond writing four +// local files. +// +// It is a thin adapter — name derivation, the rule sets, the explanatory +// headers, and the directory layout all live in pkg/snapshotter and // pkg/k8s/agent. What belongs here is only presenting the outcome, including -// the two properties an operator must not have to discover on their own: the -// objects are permanent, and adopting one ServiceAccount across runs waives -// per-run permission isolation. -func runAddRolesToServiceAccount(ctx context.Context, cmd *cli.Command, opts *snapshotCmdOptions, saName string) error { - res, err := snapshotter.ProvisionAgentRoles(ctx, &snapshotter.AgentRolesConfig{ - Kubeconfig: opts.kubeconfig, +// the two things an operator must not have to discover on their own: nothing +// is live yet, and the exact commands that make it live and take it away +// again. +func runWriteRoleManifests(cmd *cli.Command, opts *snapshotCmdOptions, saName string) error { + res, err := snapshotter.WriteAgentRoleManifests(&snapshotter.AgentRolesConfig{ Namespace: opts.namespace, ServiceAccountName: saName, DiscoverNetwork: opts.discoverNetwork, @@ -293,42 +297,73 @@ func runAddRolesToServiceAccount(ctx context.Context, cmd *cli.Command, opts *sn return err } - writeProvisionReport(cmd.Root().Writer, res) + writeManifestReport(cmd.Root().Writer, res) return nil } -// writeProvisionReport renders the outcome of a provisioning run. It is split -// out from runAddRolesToServiceAccount so the two properties an operator must -// not have to discover on their own — the objects are permanent, and adopting -// one ServiceAccount across runs waives per-run permission isolation — are -// assertable without a cluster. -func writeProvisionReport(w io.Writer, res *snapshotter.AgentRolesResult) { - fmt.Fprintf(w, `Granted the snapshot agent's permissions to ServiceAccount %[1]q in namespace %[2]q. +// writeManifestReport renders the outcome of a manifest-generating run. It is +// split out from runWriteRoleManifests so the properties an operator must not +// have to discover on their own — nothing was applied, how to apply it, how to +// remove it again, and that a shared ServiceAccount waives per-run permission +// isolation — are assertable without a cluster or a filesystem. +func writeManifestReport(w io.Writer, res *snapshotter.AgentRolesResult) { + fmt.Fprintf(w, `Wrote the snapshot agent's RBAC manifests for ServiceAccount %[1]q in namespace +%[2]q to: + + %[3]s/ -Created or updated: - role/%[3]s (namespace %[2]s) - rolebinding/%[4]s (namespace %[2]s) - clusterrole/%[5]s - clusterrolebinding/%[6]s +NOTHING WAS APPLIED. No cluster was contacted, and %[1]s has no new permissions +yet. -These objects are permanent: no aicr run creates, updates, or deletes them. -Re-run this command after an aicr upgrade to refresh the rules; delete the four -objects by hand when the ServiceAccount no longer needs them. +`, res.ServiceAccountName, res.Namespace, res.Dir) + + for _, obj := range res.Objects { + fmt.Fprintf(w, " %-28s %s/%s\n", filepath.Base(obj.Path), strings.ToLower(obj.Kind), obj.Name) + } + + fmt.Fprintf(w, ` +Read them — each file explains what it grants and why the agent needs it — then +apply them yourself: + kubectl apply -f %[1]s/ Capture a snapshot as this ServiceAccount with: - aicr snapshot --namespace %[2]s --service-account-name %[1]s + aicr snapshot --namespace %[2]s --service-account-name %[3]s + +Remove the grant when the ServiceAccount no longer needs it: + kubectl delete -f %[1]s/ + +That delete is the only teardown: no aicr run creates, refreshes, or deletes +these objects. Keep the directory for as long as you want the easy teardown. + +The ServiceAccount is not verified to exist — aicr contacted no cluster. Check +it before you rely on the grant: + kubectl get serviceaccount %[3]s -n %[2]s Trade-off: runs that share this ServiceAccount share its permissions, so per-run permission isolation is waived for them. -`, res.ServiceAccountName, res.Namespace, res.Role, res.RoleBinding, res.ClusterRole, res.ClusterRoleBinding) - - if res.DiscoverNetwork { - fmt.Fprint(w, ` -WARNING: --discover-network also granted cluster-scoped MUTATING rules -(nodes: patch, pods/exec: create, CRD/namespace/DaemonSet create-delete). -This ServiceAccount now carries them permanently, not for one run. -`) +`, res.Dir, res.Namespace, res.ServiceAccountName) + + if !res.DiscoverNetwork { + return + } + fmt.Fprintf(w, ` +WARNING: --discover-network means the ClusterRole in this directory also carries +cluster-scoped MUTATING rules (nodes: patch, pods/exec: create, CRD/namespace/ +DaemonSet create-delete). Applying it grants them permanently, not for one run. +Each rule and the discovery step it exists for is explained in %s. +`, clusterRoleManifestName(res)) +} + +// clusterRoleManifestName returns the file name of the rendered ClusterRole, +// looked up by kind rather than by position so the discovery warning keeps +// pointing at the right file if the manifest order ever changes. +func clusterRoleManifestName(res *snapshotter.AgentRolesResult) string { + for _, obj := range res.Objects { + if obj.Kind == "ClusterRole" { + return filepath.Base(obj.Path) + } } + return "the ClusterRole manifest" } // snapshotTemplateOptions holds parsed template options for the snapshot command. @@ -412,12 +447,12 @@ func snapshotCmdFlags() []cli.Flag { }, &cli.StringFlag{ Name: "service-account-name", - Usage: "ServiceAccount to run the agent as. Exact-if-exists: when a ServiceAccount of exactly this name already exists in --namespace it is used verbatim and aicr creates and deletes no RBAC for the run (grant it permissions once with --add-roles-to-service-account). Otherwise it is a name prefix (default: \"aicr\") and the run ID is appended.", + Usage: "ServiceAccount to run the agent as. Exact-if-exists: when a ServiceAccount of exactly this name already exists in --namespace it is used verbatim and aicr creates and deletes no RBAC for the run (generate its RBAC manifests with --add-roles-to-service-account, then apply them yourself). Otherwise it is a name prefix (default: \"aicr\") and the run ID is appended.", Category: catAgentDeployment, }, &cli.StringFlag{ Name: flagAddRolesToSA, - Usage: "Grant the agent's permissions to the named EXISTING ServiceAccount in --namespace, then exit without taking a snapshot. Creates permanent, non-run-scoped Role/RoleBinding and ClusterRole/ClusterRoleBinding that no run cleanup removes; idempotent. Add --discover-network to also grant the mutating live-discovery rules.", + Usage: "WRITES MANIFESTS AND APPLIES NOTHING. Renders the Role, RoleBinding, ClusterRole and ClusterRoleBinding that grant the agent's permissions to the named ServiceAccount into ./snapshot-rbac-/, then exits without taking a snapshot. No cluster is contacted, so no kubeconfig or privileges are needed. Review the files, then grant with 'kubectl apply -f /' and revoke with 'kubectl delete -f /' yourself. Add --discover-network to include the mutating live-discovery rules.", Category: catAgentDeployment, }, &cli.StringSliceFlag{ @@ -593,13 +628,13 @@ See examples/templates/snapshot-template.md.tmpl for a sample template. return err } - // Provision-and-exit: --add-roles-to-service-account grants the - // agent's permissions to an existing ServiceAccount and takes no - // snapshot. Checked ahead of every collection path (including the - // in-pod one below) so the flag can never be combined with a - // capture. + // Generate-and-exit: --add-roles-to-service-account writes the + // RBAC manifests for an existing ServiceAccount, applies nothing, + // and takes no snapshot. Checked ahead of every collection path + // (including the in-pod one below) so the flag can never be + // combined with a capture. if saName := cmd.String(flagAddRolesToSA); saName != "" { - return runAddRolesToServiceAccount(ctx, cmd, opts, saName) + return runWriteRoleManifests(cmd, opts, saName) } agentCfg := opts.toAgentConfig() diff --git a/pkg/cli/snapshot_test.go b/pkg/cli/snapshot_test.go index 0d8b281f9..e69714577 100644 --- a/pkg/cli/snapshot_test.go +++ b/pkg/cli/snapshot_test.go @@ -453,11 +453,13 @@ func TestOutputDestinationParsing(t *testing.T) { } // TestSnapshotCmd_AddRolesFlagWiring covers the CLI surface of the -// provision-and-exit invocation. The provisioning itself is cluster work and -// is covered in pkg/k8s/agent; what this asserts is the wiring an operator -// touches: the flag exists under the agent-deployment category, and it is -// single-valued so a repeated flag is rejected rather than silently taking -// the last value and provisioning the wrong ServiceAccount. +// generate-and-exit invocation. Rendering the manifests is covered in +// pkg/k8s/agent and pkg/snapshotter; what this asserts is the wiring an +// operator touches: the flag exists under the agent-deployment category, its +// usage says outright that it applies nothing (the flag NAME reads like it +// mutates the cluster, so nobody may run it and assume the grant is live), +// and it is single-valued so a repeated flag is rejected rather than silently +// taking the last value and writing manifests for the wrong ServiceAccount. func TestSnapshotCmd_AddRolesFlagWiring(t *testing.T) { t.Run("flag is registered under agent deployment", func(t *testing.T) { var found cli.Flag @@ -481,6 +483,12 @@ func TestSnapshotCmd_AddRolesFlagWiring(t *testing.T) { if !strings.Contains(sf.Usage, "without taking a snapshot") { t.Errorf("--%s usage must say it takes no snapshot; got %q", flagAddRolesToSA, sf.Usage) } + if !strings.Contains(sf.Usage, "APPLIES NOTHING") { + t.Errorf("--%s usage must state plainly that it applies nothing; got %q", flagAddRolesToSA, sf.Usage) + } + if !strings.Contains(sf.Usage, "kubectl apply") || !strings.Contains(sf.Usage, "kubectl delete") { + t.Errorf("--%s usage must name the apply and delete commands; got %q", flagAddRolesToSA, sf.Usage) + } }) t.Run("repeated flag is rejected", func(t *testing.T) { @@ -516,20 +524,90 @@ func TestSnapshotCmd_ServiceAccountNameUsageStatesExactIfExists(t *testing.T) { t.Fatal("snapshot command must define --service-account-name") } -// TestWriteProvisionReport asserts the two properties the provisioning output -// must state outright, because an operator who does not read them cannot -// discover either from the cluster: the objects are permanent (nothing in aicr -// will ever remove them), and adopting one ServiceAccount across runs waives -// per-run permission isolation. The --discover-network warning is separate -// because that grant is the one that is also mutating. -func TestWriteProvisionReport(t *testing.T) { +// TestSnapshotCmd_AddRolesWritesManifestsWithoutACluster runs the real command +// action end to end. It is the test that pins the whole point of the flag: with +// KUBECONFIG pointed at a file no client can be built from and the in-cluster +// environment cleared, the invocation still succeeds and writes a reviewable +// directory. Any clientset construction, ServiceAccount lookup, or permission +// pre-flight on this path would fail here rather than pass quietly. +func TestSnapshotCmd_AddRolesWritesManifestsWithoutACluster(t *testing.T) { + unreachable := filepath.Join(t.TempDir(), "not-a-kubeconfig") + if seedErr := os.WriteFile(unreachable, []byte("this is not a kubeconfig\n"), 0o600); seedErr != nil { + t.Fatalf("seeding the kubeconfig: %v", seedErr) + } + t.Setenv("KUBECONFIG", unreachable) + t.Setenv("HOME", t.TempDir()) + t.Setenv("KUBERNETES_SERVICE_HOST", "") + t.Setenv("KUBERNETES_SERVICE_PORT", "") + t.Chdir(t.TempDir()) + + var buf bytes.Buffer + cmd := snapshotCmd() + cmd.Writer = &buf + if err := cmd.Run(context.Background(), []string{ + "snapshot", + "--namespace", "gpu-operator", + "--" + flagAddRolesToSA, "irsa-snapshotter", + "--discover-network", + }); err != nil { + t.Fatalf("snapshot --%s error = %v; the path must need no cluster at all", flagAddRolesToSA, err) + } + + entries, readErr := os.ReadDir(".") + if readErr != nil { + t.Fatalf("reading the working directory: %v", readErr) + } + var dir string + for _, e := range entries { + if e.IsDir() && strings.HasPrefix(e.Name(), "snapshot-rbac-") { + dir = e.Name() + } + } + if dir == "" { + t.Fatalf("no snapshot-rbac- directory written; got %v", entries) + } + + for _, name := range []string{"01-role.yaml", "02-rolebinding.yaml", "03-clusterrole.yaml", "04-clusterrolebinding.yaml"} { + body, err := os.ReadFile(filepath.Join(dir, name)) + if err != nil { + t.Errorf("reading %s: %v", name, err) + continue + } + if !strings.HasPrefix(string(body), "# ") { + t.Errorf("%s does not open with a YAML comment header", name) + } + } + + // The report goes to the command writer, not stdout, and must name the + // directory and both halves of the operator's workflow. + out := buf.String() + for _, want := range []string{"NOTHING WAS APPLIED", dir, "kubectl apply -f", "kubectl delete -f", "MUTATING"} { + if !strings.Contains(out, want) { + t.Errorf("command output does not contain %q; got:\n%s", want, out) + } + } +} + +// TestWriteManifestReport asserts the properties the output must state +// outright, because an operator who does not read them cannot discover any of +// them from the cluster: nothing was applied, where the manifests are, the +// command that makes the grant live, the command that removes it again, and +// that sharing one ServiceAccount waives per-run permission isolation. The +// --discover-network warning is separate because that grant is the one that +// is also mutating. +func TestWriteManifestReport(t *testing.T) { + dir := "snapshot-rbac-20260821-142233-9f3a1c0b7e2d4a55" res := &snapshotter.AgentRolesResult{ + Dir: dir, + RunID: "20260821-142233-9f3a1c0b7e2d4a55", Namespace: "gpu-operator", ServiceAccountName: "irsa-snapshotter", - Role: "aicr-agent-irsa-snapshotter-rbac", - RoleBinding: "aicr-agent-irsa-snapshotter-rbac", - ClusterRole: "aicr-agent-gpu-operator-irsa-snapshotter-rbac", - ClusterRoleBinding: "aicr-agent-gpu-operator-irsa-snapshotter-rbac", + Objects: []snapshotter.AgentRoleObject{ + {Kind: "Role", Name: "aicr-agent-irsa-snapshotter-rbac", Path: dir + "/01-role.yaml"}, + {Kind: "RoleBinding", Name: "aicr-agent-irsa-snapshotter-rbac", Path: dir + "/02-rolebinding.yaml"}, + {Kind: "ClusterRole", Name: "aicr-agent-gpu-operator-irsa-snapshotter-rbac", Path: dir + "/03-clusterrole.yaml"}, + {Kind: "ClusterRoleBinding", Name: "aicr-agent-gpu-operator-irsa-snapshotter-rbac", Path: dir + "/04-clusterrolebinding.yaml"}, + }, } tests := []struct { @@ -541,22 +619,27 @@ func TestWriteProvisionReport(t *testing.T) { { name: "read-only grant", wantSubstrings: []string{ - `ServiceAccount "irsa-snapshotter" in namespace "gpu-operator"`, + `ServiceAccount "irsa-snapshotter" in namespace`, + "NOTHING WAS APPLIED", + "kubectl apply -f " + dir + "/", + "kubectl delete -f " + dir + "/", + "01-role.yaml", "role/aicr-agent-irsa-snapshotter-rbac", "clusterrolebinding/aicr-agent-gpu-operator-irsa-snapshotter-rbac", - "These objects are permanent", + "not verified to exist", "permission isolation is waived", "aicr snapshot --namespace gpu-operator --service-account-name irsa-snapshotter", }, notWant: "MUTATING", }, { - name: "discovery grant warns about the permanent mutating rules", + name: "discovery grant warns about the mutating rules", discoverNetwork: true, wantSubstrings: []string{ - "These objects are permanent", + "NOTHING WAS APPLIED", "MUTATING", - "carries them permanently, not for one run", + "grants them permanently, not for one run", + "03-clusterrole.yaml", }, }, } @@ -566,7 +649,7 @@ func TestWriteProvisionReport(t *testing.T) { r := *res r.DiscoverNetwork = tt.discoverNetwork var buf bytes.Buffer - writeProvisionReport(&buf, &r) + writeManifestReport(&buf, &r) for _, want := range tt.wantSubstrings { if !strings.Contains(buf.String(), want) { diff --git a/pkg/cli/validate.go b/pkg/cli/validate.go index 79296660e..395e2a92f 100644 --- a/pkg/cli/validate.go +++ b/pkg/cli/validate.go @@ -498,7 +498,7 @@ func validateCmdFlags() []cli.Flag { }, &cli.StringFlag{ Name: "service-account-name", - Usage: "ServiceAccount the live snapshot-capture agent runs as. Exact-if-exists: when a ServiceAccount of exactly this name already exists in --namespace it is used verbatim and the agent creates and deletes no RBAC for the run (grant it permissions once with 'aicr snapshot --add-roles-to-service-account'). Otherwise it is a name prefix (default: \"aicr-validate\") and the run ID is appended.", + Usage: "ServiceAccount the live snapshot-capture agent runs as. Exact-if-exists: when a ServiceAccount of exactly this name already exists in --namespace it is used verbatim and the agent creates and deletes no RBAC for the run (generate its RBAC manifests with 'aicr snapshot --add-roles-to-service-account', then apply them yourself). Otherwise it is a name prefix (default: \"aicr-validate\") and the run ID is appended.", Category: catAgentDeployment, }, &cli.StringSliceFlag{ diff --git a/pkg/client/v1/types.go b/pkg/client/v1/types.go index 97f7545de..2e5d68dee 100644 --- a/pkg/client/v1/types.go +++ b/pkg/client/v1/types.go @@ -139,9 +139,10 @@ type AgentConfig struct { // GKE Workload Identity (iam.gke.io/gcp-service-account) // annotations stays usable: both providers pin trust to the // ServiceAccount NAME, which a run-scoped name can never satisfy. - // Grant it the agent's permissions once with - // snapshotter.ProvisionAgentRoles (CLI: - // `aicr snapshot --add-roles-to-service-account`). + // Generate the RBAC that grants it the agent's permissions with + // snapshotter.WriteAgentRoleManifests (CLI: + // `aicr snapshot --add-roles-to-service-account`), which writes + // manifests and applies nothing, then apply them yourself. // - Otherwise: a name prefix. The run creates "-" // and the full run-scoped RBAC set, and deletes them at cleanup. // diff --git a/pkg/defaults/k8s.go b/pkg/defaults/k8s.go index 8a6f2d559..2c81de63d 100644 --- a/pkg/defaults/k8s.go +++ b/pkg/defaults/k8s.go @@ -14,6 +14,8 @@ package defaults +import "os" + // Kubernetes REST client rate limits for validator containers. // // client-go's default client-side rate limiter (QPS 5, Burst 10) is tuned for @@ -50,3 +52,29 @@ const ( // fits the object-name limit but not the label limit would fail Pod // creation, so name helpers must budget against this constant. const MaxK8sNameLength = 63 + +// Layout of the RBAC manifest directory that +// `aicr snapshot --add-roles-to-service-account` writes. +// +// That invocation applies nothing and contacts no cluster: it renders the +// Role, RoleBinding, ClusterRole and ClusterRoleBinding that grant the +// snapshot agent's permissions to an operator-supplied ServiceAccount, and +// leaves applying them — and later deleting them — to the operator. The +// directory is what makes both halves a single `kubectl -f` argument. +const ( + // AgentRBACManifestDirPrefix is prepended to the run ID to form the + // output directory name (`snapshot-rbac-`). The run ID makes + // the name unique per invocation, so a second run never overwrites a + // set of manifests an operator is still reviewing. + AgentRBACManifestDirPrefix = "snapshot-rbac-" + + // AgentRBACManifestDirMode is the permission of that directory. + // Owner-only: the manifests describe a permission grant, and there is + // no reason for another local user to read or replace them between + // generation and `kubectl apply`. + AgentRBACManifestDirMode os.FileMode = 0o700 + + // AgentRBACManifestFileMode is the permission of each manifest file, + // owner read/write only for the same reason as the directory. + AgentRBACManifestFileMode os.FileMode = 0o600 +) diff --git a/pkg/defaults/timeouts.go b/pkg/defaults/timeouts.go index 335e16fc9..9dd457a28 100644 --- a/pkg/defaults/timeouts.go +++ b/pkg/defaults/timeouts.go @@ -239,14 +239,6 @@ const ( // K8sCleanupTimeout is the timeout for cleanup operations. K8sCleanupTimeout = 30 * time.Second - // AgentRBACProvisionTimeout bounds - // `aicr snapshot --add-roles-to-service-account`, which reads the - // target ServiceAccount and then creates-or-updates four RBAC objects. - // That is a handful of small writes with no waiting on cluster state, - // so the budget only has to absorb apiserver latency and a retry or - // two — not a Job round trip. - AgentRBACProvisionTimeout = 60 * time.Second - // DiscoveryRefreshCooldown rate-limits how often the shared cluster // fetcher (pkg/chainsaw) invalidates its cached discovery data after a // no-match. The refresh exists so a CRD installed by the component being diff --git a/pkg/k8s/agent/deployer.go b/pkg/k8s/agent/deployer.go index 2d125fccf..a63db3c4c 100644 --- a/pkg/k8s/agent/deployer.go +++ b/pkg/k8s/agent/deployer.go @@ -82,8 +82,9 @@ func (d *Deployer) Deploy(ctx context.Context) error { // Skipped entirely in exact-ServiceAccount mode: aicr will not add or // remove permissions on a ServiceAccount it did not create. Nothing is // created, so nothing enters the created-set, so Cleanup deletes none - // of these kinds — the operator's grants outlive the run. Provision - // them once with ProvisionServiceAccountRoles. + // of these kinds — the operator's grants outlive the run. Generate its + // RBAC manifests with BuildServiceAccountRoleManifests and apply them + // out of band. if d.managesRBAC() { if err := d.ensureServiceAccount(ctx); err != nil { return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to create ServiceAccount", err) diff --git a/pkg/k8s/agent/doc.go b/pkg/k8s/agent/doc.go index 06cbb6195..c075b1124 100644 --- a/pkg/k8s/agent/doc.go +++ b/pkg/k8s/agent/doc.go @@ -52,14 +52,18 @@ GKE's IAM binding names PROJECT.svc.id.goog[/] and accepts no wildcard — so a per-run name can never be trusted by either, and copying the annotations onto a run-scoped ServiceAccount would not help. -Grant the agent's permissions to such a ServiceAccount once with -ProvisionServiceAccountRoles (CLI: aicr snapshot ---add-roles-to-service-account). What it creates is permanent: no run-ID -label, never in a created-set, never deleted by run cleanup. +Render the RBAC that grants such a ServiceAccount the agent's permissions +with BuildServiceAccountRoleManifests (CLI: aicr snapshot +--add-roles-to-service-account). That path APPLIES NOTHING and contacts no +cluster: it writes manifests the operator reviews and applies themselves, so +the decision to grant cluster-scoped -- and, under DiscoverNetwork, mutating +-- permissions is an informed one. What they then apply sits outside every +run: no run-ID label, never in a created-set, never deleted by run cleanup, +and removed with kubectl delete -f. The trade-off is deliberate and opt-in: an adopted ServiceAccount waives per-run permission isolation. Concurrent runs sharing it share its grants, and -a DiscoverNetwork provisioning leaves cluster-scoped mutating permissions in +a DiscoverNetwork grant leaves cluster-scoped mutating permissions in place permanently rather than for one run's lifetime. Two objects are deliberately NOT run-scoped: diff --git a/pkg/k8s/agent/provision.go b/pkg/k8s/agent/provision.go index a083d3abe..6646b6154 100644 --- a/pkg/k8s/agent/provision.go +++ b/pkg/k8s/agent/provision.go @@ -15,24 +15,22 @@ package agent import ( - "context" "fmt" "strings" "github.com/NVIDIA/aicr/pkg/errors" "github.com/NVIDIA/aicr/pkg/k8s/labels" rbacv1 "k8s.io/api/rbac/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation" - "k8s.io/client-go/kubernetes" + "sigs.k8s.io/yaml" ) -// Naming of the permanent RBAC objects ProvisionServiceAccountRoles creates. +// Naming of the RBAC objects BuildServiceAccountRoleManifests renders. // // The names are deterministic — the same (namespace, ServiceAccount) pair -// always resolves to the same four names, which is what makes re-running the -// provisioning idempotent — and they cannot collide with a run-scoped name. +// always resolves to the same four names — and they cannot collide with a +// run-scoped name. // // The non-collision is structural, not probabilistic. Every run-scoped name // is "-" and every runID ends in 16 lowercase-hex characters @@ -44,81 +42,112 @@ const ( provisionedNameSuffix = "-rbac" ) -// ProvisionOptions selects the ServiceAccount that -// ProvisionServiceAccountRoles grants the snapshot agent's permissions to. -type ProvisionOptions struct { - // Namespace holds the ServiceAccount and is where the Role and - // RoleBinding are created. Required. +// File names of the rendered manifests, one object per file. +// +// The numeric prefixes are not decoration. `kubectl apply -f /` visits a +// directory in lexical order, so they put each Role ahead of the RoleBinding +// that references it, and they keep the reading order an operator sees when +// they list the directory the same as the order the objects take effect in. +const ( + roleFileName = "01-role.yaml" + roleBindingFileName = "02-rolebinding.yaml" + clusterRoleFileName = "03-clusterrole.yaml" + clusterRoleBindingFileName = "04-clusterrolebinding.yaml" +) + +// ManifestOptions selects the ServiceAccount that +// BuildServiceAccountRoleManifests renders RBAC for. +type ManifestOptions struct { + // Namespace is the namespace of the ServiceAccount, and the namespace + // the rendered Role and RoleBinding declare. Required. Namespace string - // ServiceAccountName is the EXACT name of an already-existing - // ServiceAccount. Required; provisioning fails with ErrCodeNotFound - // when no such ServiceAccount exists, because the whole point is to - // grant permissions to an identity the operator created and controls - // (typically one carrying IRSA or GKE Workload Identity annotations). + // ServiceAccountName is the name of the ServiceAccount the rendered + // RoleBinding and ClusterRoleBinding name as their subject. Required. + // + // It is NOT verified to exist: rendering contacts no cluster, by + // design (see BuildServiceAccountRoleManifests). A name that does not + // resolve produces manifests that grant nothing, which the operator + // sees when they review the files or when the ServiceAccount they + // meant to name still cannot snapshot. ServiceAccountName string - // DiscoverNetwork also grants the cluster-scoped MUTATING rules that + // DiscoverNetwork also renders the cluster-scoped MUTATING rules that // `aicr snapshot --discover-network` needs — nodes: patch, // pods/exec: create, and CRD, namespace, DaemonSet and namespaced-RBAC // create/delete (see discoverNetworkClusterRules). // - // Unlike a run-scoped grant, this one is permanent: the ServiceAccount - // carries those permissions until the operator removes them, not for - // one run's lifetime. + // The rendered ClusterRole carries an explicit warning header + // enumerating each mutating rule and the discovery step it exists for, + // because deciding whether to grant them is the whole reason these + // manifests are written out instead of applied. DiscoverNetwork bool } -// ProvisionResult names what ProvisionServiceAccountRoles created or -// updated, so a caller can report it without rebuilding the names. -type ProvisionResult struct { - Namespace string - ServiceAccountName string - Role string - RoleBinding string - ClusterRole string - ClusterRoleBinding string +// Manifest is one rendered RBAC object: the bytes to write, the file name to +// write them under, and the object's kind and name so a caller can report +// what it wrote without re-deriving either. +type Manifest struct { + // FileName is the name of the file within the output directory. It is + // a bare file name, never a path. + FileName string - // DiscoverNetwork echoes ProvisionOptions.DiscoverNetwork: it is the - // difference between a read-only grant and one carrying cluster-scoped - // mutating rules, so a caller reporting the result must be able to say - // which was provisioned. - DiscoverNetwork bool + // Kind is the Kubernetes kind ("Role", "RoleBinding", "ClusterRole", + // "ClusterRoleBinding"). + Kind string + + // Name is the object's metadata.name. + Name string + + // Content is the complete file body: a YAML comment header explaining + // what the object grants and why the agent needs it, followed by the + // object itself. + Content []byte } -// ProvisionServiceAccountRoles grants the snapshot agent's permissions to an -// already-existing, operator-supplied ServiceAccount by creating a Role, -// RoleBinding, ClusterRole and ClusterRoleBinding for it. +// BuildServiceAccountRoleManifests renders the Role, RoleBinding, +// ClusterRole and ClusterRoleBinding that grant the snapshot agent's +// permissions to an operator-supplied ServiceAccount. +// +// It APPLIES NOTHING and contacts no cluster. There is no clientset, no +// ServiceAccount lookup and no permission pre-flight on this path, so it +// works with no kubeconfig and no cluster privileges at all. Applying the +// manifests, and deleting them when the grant is no longer wanted, is the +// operator's decision and the operator's command. +// +// That is deliberate. The rules being granted include, under +// DiscoverNetwork, cluster-scoped mutating permissions (nodes: patch, +// pods/exec: create, CRD create) that outlive any single run. An operator +// consenting to that should be able to read exactly what they are granting +// first, which a command that provisions on their behalf does not allow. // -// These four objects are PERMANENT and deliberately outside every run's -// lifecycle: they carry no run-ID label, they never enter any Deployer's -// created-set, and no run's Cleanup deletes them. Removing them is the -// operator's job. That is the counterpart to Config.ServiceAccountName's -// exact-if-exists behavior, where a run that adopts an existing -// ServiceAccount creates and deletes no RBAC of its own. +// Every rule set comes from namespacedRules and clusterRules — the same +// definitions the run-scoped ensureRole and ensureClusterRole build from — +// so a rendered manifest can never drift from what a run-owned grant +// carries. // -// The call is idempotent: each object is created, or updated in place when -// it already exists, so re-running it after an aicr upgrade refreshes the -// rules rather than failing or leaving a stale rule set behind. +// The objects are NOT run-scoped: they carry no run-ID label, never enter a +// Deployer's created-set, and no run's Cleanup deletes them. Teardown is +// `kubectl delete -f /`. // -// Trade-off the caller must surface to the operator: an adopted -// ServiceAccount waives per-run permission isolation. Concurrent runs using -// it share its grants, and a DiscoverNetwork provisioning leaves mutating -// cluster permissions in place permanently rather than for one run. -func ProvisionServiceAccountRoles(ctx context.Context, clientset kubernetes.Interface, opts ProvisionOptions) (*ProvisionResult, error) { +// Trade-off the caller must surface to the operator: a shared ServiceAccount +// waives per-run permission isolation. Concurrent runs using it share its +// grants, and a DiscoverNetwork grant leaves mutating cluster permissions in +// place until the operator removes them. +func BuildServiceAccountRoleManifests(opts ManifestOptions) ([]Manifest, error) { if strings.TrimSpace(opts.Namespace) == "" { return nil, errors.New(errors.ErrCodeInvalidRequest, - "namespace is required: it is where the ServiceAccount, Role and RoleBinding live") + "namespace is required: it is the namespace the rendered Role and RoleBinding declare") } if strings.TrimSpace(opts.ServiceAccountName) == "" { return nil, errors.New(errors.ErrCodeInvalidRequest, - "ServiceAccount name is required: provisioning grants permissions to an existing ServiceAccount, it does not create one") + "ServiceAccount name is required: the rendered bindings need a subject to name") } // Both halves of each composed name are valid on their own, but their - // concatenation can exceed the length ceiling. Reject that here, before - // anything is created, rather than as an opaque apiserver "Invalid - // value: metadata.name" partway through. + // concatenation can exceed the length ceiling. Reject that here, while + // rendering, rather than leaving the operator to discover it as an + // opaque apiserver "Invalid value: metadata.name" at apply time. roleName := provisionedRoleName(opts.ServiceAccountName) clusterRoleName := provisionedClusterRoleName(opts.Namespace, opts.ServiceAccountName) for _, name := range []string{roleName, clusterRoleName} { @@ -132,52 +161,321 @@ func ProvisionServiceAccountRoles(ctx context.Context, clientset kubernetes.Inte map[string]any{ctxKeyValue: opts.ServiceAccountName, ctxKeyResolvedName: name}) } - // Fail closed before creating anything when the ServiceAccount is - // absent. Provisioning permissions for an identity that does not exist - // would leave four dangling objects and a binding to nothing, and the - // most likely cause is a typo the operator needs to see. - if _, err := clientset.CoreV1().ServiceAccounts(opts.Namespace). - Get(ctx, opts.ServiceAccountName, metav1.GetOptions{}); err != nil { - if apierrors.IsNotFound(err) { - return nil, errors.NewWithContext(errors.ErrCodeNotFound, - fmt.Sprintf("ServiceAccount %q not found in namespace %q; create it first (aicr grants permissions to an existing ServiceAccount, it never creates one)", - opts.ServiceAccountName, opts.Namespace), - map[string]any{attrServiceAccount: opts.ServiceAccountName, attrNamespace: opts.Namespace}) - } - return nil, errors.Wrap(errors.ErrCodeInternal, "failed to read the target ServiceAccount", err) - } - subjects := []rbacv1.Subject{{ Kind: kindServiceAccount, Name: opts.ServiceAccountName, Namespace: opts.Namespace, }} + // Each object gets its own ObjectMeta rather than sharing one value: + // ObjectMeta copies by value but its Labels map does not, and four + // objects aliasing one map is a mutation hazard for anything that later + // adjusts labels per object. + namespacedMeta := func() metav1.ObjectMeta { + return metav1.ObjectMeta{Name: roleName, Namespace: opts.Namespace, Labels: provisionedLabels()} + } + clusterMeta := func() metav1.ObjectMeta { + return metav1.ObjectMeta{Name: clusterRoleName, Labels: provisionedLabels()} + } - if err := provisionRole(ctx, clientset, opts.Namespace, roleName); err != nil { - return nil, err + objects := []struct { + fileName string + kind string + name string + header string + object any + }{ + { + fileName: roleFileName, + kind: kindRole, + name: roleName, + header: roleHeader(roleName, opts.Namespace, opts.ServiceAccountName), + object: &rbacv1.Role{ + TypeMeta: rbacTypeMeta(kindRole), + ObjectMeta: namespacedMeta(), + Rules: namespacedRules(), + }, + }, + { + fileName: roleBindingFileName, + kind: kindRoleBinding, + name: roleName, + header: roleBindingHeader(roleName, opts.Namespace, opts.ServiceAccountName), + object: &rbacv1.RoleBinding{ + TypeMeta: rbacTypeMeta(kindRoleBinding), + ObjectMeta: namespacedMeta(), + Subjects: subjects, + RoleRef: rbacv1.RoleRef{APIGroup: rbacAPIGroup, Kind: kindRole, Name: roleName}, + }, + }, + { + fileName: clusterRoleFileName, + kind: kindClusterRole, + name: clusterRoleName, + header: clusterRoleHeader(clusterRoleName, opts.ServiceAccountName, opts.DiscoverNetwork), + object: &rbacv1.ClusterRole{ + TypeMeta: rbacTypeMeta(kindClusterRole), + ObjectMeta: clusterMeta(), + Rules: clusterRules(opts.DiscoverNetwork), + }, + }, + { + fileName: clusterRoleBindingFileName, + kind: kindClusterRoleBinding, + name: clusterRoleName, + header: clusterRoleBindingHeader(clusterRoleName, opts.Namespace, opts.ServiceAccountName), + object: &rbacv1.ClusterRoleBinding{ + TypeMeta: rbacTypeMeta(kindClusterRoleBinding), + ObjectMeta: clusterMeta(), + Subjects: subjects, + RoleRef: rbacv1.RoleRef{APIGroup: rbacAPIGroup, Kind: kindClusterRole, Name: clusterRoleName}, + }, + }, } - if err := provisionRoleBinding(ctx, clientset, opts.Namespace, roleName, subjects); err != nil { - return nil, err + + manifests := make([]Manifest, 0, len(objects)) + for _, o := range objects { + content, err := renderManifest(o.header, o.object, opts.ServiceAccountName) + if err != nil { + return nil, err + } + manifests = append(manifests, Manifest{ + FileName: o.fileName, + Kind: o.kind, + Name: o.name, + Content: content, + }) } - if err := provisionClusterRole(ctx, clientset, clusterRoleName, opts.DiscoverNetwork); err != nil { - return nil, err + return manifests, nil +} + +// rbacTypeMeta returns the apiVersion/kind pair a standalone manifest needs. +// The typed objects client-go hands back leave TypeMeta empty — the wire +// format carries it out of band — so a file written from one is unusable +// with `kubectl apply` until it is set explicitly. +func rbacTypeMeta(kind string) metav1.TypeMeta { + return metav1.TypeMeta{APIVersion: rbacv1.SchemeGroupVersion.String(), Kind: kind} +} + +// renderManifest assembles one file: the explanatory header, the marshaled +// object, and the shared trailer that states nothing was applied and how to +// apply and revoke. +// +// Marshaling goes through sigs.k8s.io/yaml, which encodes the typed struct +// via encoding/json rather than walking a Go map, so the field order is the +// struct's and two runs over the same inputs produce identical bytes. +func renderManifest(header string, obj any, serviceAccount string) ([]byte, error) { + body, err := yaml.Marshal(obj) + if err != nil { + return nil, errors.Wrap(errors.ErrCodeInternal, "failed to render the RBAC manifest", err) } - if err := provisionClusterRoleBinding(ctx, clientset, clusterRoleName, subjects); err != nil { - return nil, err + var buf strings.Builder + buf.WriteString(header) + buf.WriteString(manifestTrailer(serviceAccount)) + buf.WriteString("---\n") + buf.Write(body) + return []byte(buf.String()), nil +} + +// manifestTrailer is the block every rendered file ends its header with. It +// repeats on all four because an operator may open, review, and act on one +// file alone, and the single fact none of them can afford to omit is that +// nothing has been applied yet. +func manifestTrailer(serviceAccount string) string { + return fmt.Sprintf(`# +# Generated by: aicr snapshot --add-roles-to-service-account %s +# +# NOTHING HAS BEEN APPLIED TO YOUR CLUSTER. aicr wrote these files and +# contacted no cluster to do it. Review them, then grant and revoke yourself: +# +# kubectl apply -f / # grant the permissions +# kubectl delete -f / # revoke them again +# +# No aicr run creates, refreshes, or deletes these objects. They last until +# you delete them. +# +`, serviceAccount) +} + +// roleHeader explains the namespaced grant: two rules, both confined to one +// namespace, and what the agent does with each. +func roleHeader(name, namespace, serviceAccount string) string { + return fmt.Sprintf(`# aicr snapshot agent -- namespaced permissions +# +# Role/%s +# in namespace %s +# +# Bound to ServiceAccount %q by %s. +# +# Grants only what the agent Job needs inside this one namespace: +# +# configmaps: create, get, update, patch +# The agent runs as a Kubernetes Job and cannot hand its result back to +# the CLI directly. It stages the snapshot it collected in a ConfigMap +# in this namespace and the CLI reads that ConfigMap. Confined to this +# namespace: it grants nothing in any other. +# +# pods: get, list +# The agent reads its own pod to learn which node it was scheduled onto, +# and lists pods when collecting workload state. +# +# Read-mostly and namespace-local: nothing here is cluster-scoped, and +# nothing here can modify a node, a CRD, or any workload. +`, name, namespace, serviceAccount, roleBindingFileName) +} + +// roleBindingHeader explains that this file is what makes the Role take +// effect, and states the consequence of the unverified ServiceAccount name: +// Kubernetes accepts a binding to a subject that does not exist and it +// simply grants nothing, so a typo fails silently. +func roleBindingHeader(name, namespace, serviceAccount string) string { + return fmt.Sprintf(`# aicr snapshot agent -- binds the namespaced permissions +# +# RoleBinding/%[1]s +# in namespace %[2]s +# Role/%[1]s -> ServiceAccount %[2]s/%[3]s +# +# Applying this is what actually gives ServiceAccount %[3]q the rules +# in %[4]s. Until it is applied, that Role grants nothing to anyone. +# +# CHECK THE NAME FIRST. aicr rendered these manifests without contacting a +# cluster, so it did not verify that this ServiceAccount exists. Kubernetes +# accepts a binding whose subject does not exist and simply grants nothing, +# so a typo here fails silently rather than loudly: +# +# kubectl get serviceaccount %[3]s -n %[2]s +# +# The ServiceAccount must be one you created and control -- typically one +# carrying IRSA (eks.amazonaws.com/role-arn) or GKE Workload Identity +# (iam.gke.io/gcp-service-account) annotations. aicr never creates it. +`, name, namespace, serviceAccount, roleFileName) +} + +// clusterRoleHeader explains the cluster-scoped grant. Without +// discoverNetwork it is entirely read-only and says so; with it, the +// mutating rules get a rule-by-rule account of the discovery step each one +// exists for, since that is the grant an operator most needs to read before +// consenting to it. +func clusterRoleHeader(name, serviceAccount string, discoverNetwork bool) string { + header := fmt.Sprintf(`# aicr snapshot agent -- cluster-scoped permissions +# +# ClusterRole/%s +# (cluster-scoped -- these rules apply in EVERY namespace) +# +# Bound to ServiceAccount %q by %s. +# +# The baseline rule set below is READ-ONLY -- every verb is get, list or +# watch, and nothing in it creates, patches, or deletes anything: +# +# nodes: get, list +# Node inventory: labels, taints, allocatable capacity, kubelet and +# container-runtime versions, OS image. +# +# pods: get, list +# Cluster-wide workload inventory, used to detect which GPU and +# networking components are already deployed. +# +# nvidia.com clusterpolicies: get, list +# GPU Operator ClusterPolicy: the driver, toolkit, and device-plugin +# configuration currently in effect. +# +# slinky.slurm.net controllers, nodesets, loginsets, restapis, +# accountings: list +# Slurm-on-Kubernetes topology, when Slinky is installed. +# +# k8s.mariadb.com mariadbs: list +# The MariaDB instance backing Slurm accounting, when present. +`, name, serviceAccount, clusterRoleBindingFileName) + if !discoverNetwork { + return header } + return header + discoverNetworkHeaderSection() +} + +// discoverNetworkHeaderSection is the warning block appended to the +// ClusterRole manifest when DiscoverNetwork is set. Each rule is named +// alongside the concrete discovery step that needs it, so an operator +// reading "nodes: patch" can tell from the file why it is there. +func discoverNetworkHeaderSection() string { + return `# +# ========================================================================== +# WARNING -- --discover-network was requested, so this ClusterRole ALSO +# carries MUTATING, cluster-scoped rules. Read them before you apply it. +# ========================================================================== +# +# Live network discovery (k8s-launch-kit) does not read topology from the +# API. It stands up a probe DaemonSet, execs into it, and writes what it +# learns back onto your cluster. Every mutating rule below maps to one +# concrete step of that flow: +# +# nodes: patch +# Writes nvidia.kubernetes-launch-kit.machine and .gpu labels onto every +# node discovery matches. This modifies YOUR nodes, and the labels +# remain after the run finishes. +# +# pods/exec: create +# Execs into each probe pod to read NIC VPD and link metadata via the +# in-pod CLI. Note that pods/exec: create at cluster scope permits +# running commands in ANY pod in the cluster, not only the probe pods. +# +# namespaces: get, create, delete +# apps daemonsets: get, list, watch, create, delete +# serviceaccounts, configmaps: get, create, delete +# rbac.authorization.k8s.io roles, rolebindings: get, create, delete +# Discovery creates the nvidia-k8s-launch-kit namespace, deploys the +# nic-configuration-daemon DaemonSet and its supporting RBAC into it, +# and deletes the namespace when it is done. +# +# apiextensions.k8s.io customresourcedefinitions: +# get, list, create, update, patch +# Installs the nic-configuration-operator CRDs (NicDevice, +# NicClusterPolicy) when they are absent. This is a cluster-wide schema +# change that outlives the run. +# +# configuration.net.nvidia.com nicdevices: get, list +# Reads the NicDevice CRs the probe daemon publishes. +# +# mellanox.com nicclusterpolicies: get, patch +# Server-side-applies the NicConfigurationOperator section of YOUR +# existing NicClusterPolicy. +# +# These permissions last as long as the objects do -- they are NOT scoped to +# one run. A run that lets aicr create its own ServiceAccount instead gets +# the same rules for the lifetime of that one run and has them revoked at +# cleanup. If you need discovery only occasionally, prefer that, or keep a +# separate ServiceAccount used only for discovery runs. +` +} - return &ProvisionResult{ - Namespace: opts.Namespace, - ServiceAccountName: opts.ServiceAccountName, - Role: roleName, - RoleBinding: roleName, - ClusterRole: clusterRoleName, - ClusterRoleBinding: clusterRoleName, - DiscoverNetwork: opts.DiscoverNetwork, - }, nil +// clusterRoleBindingHeader explains the cluster-scoped binding and warns +// about the one hazard aicr can no longer detect for the operator: the +// generated name is not injective, so an apply can silently retarget an +// existing binding. Reading the file is the check that replaces the cluster +// read the old provisioning path performed. +func clusterRoleBindingHeader(name, namespace, serviceAccount string) string { + return fmt.Sprintf(`# aicr snapshot agent -- binds the cluster-scoped permissions +# +# ClusterRoleBinding/%[1]s +# (cluster-scoped) +# ClusterRole/%[1]s -> ServiceAccount %[2]s/%[3]s +# +# Applying this is what gives ServiceAccount %[3]q the rules in +# %[4]s, across every namespace in the cluster. +# +# CHECK FOR A NAME COLLISION FIRST. This name joins the namespace and the +# ServiceAccount name with "-", which is not injective: namespace "a-b" with +# ServiceAccount "c" and namespace "a" with ServiceAccount "b-c" both compose +# "aicr-agent-a-b-c-rbac". aicr contacted no cluster and could not check. +# Applying over an existing binding of this name would retarget it and revoke +# the other ServiceAccount's cluster permissions: +# +# kubectl get clusterrolebinding %[1]s -o yaml +# +# If it already exists and names a different subject, rename one of the two +# namespaces or ServiceAccounts so the generated names differ. +`, name, namespace, serviceAccount, clusterRoleFileName) } -// provisionedRoleName returns the permanent Role and RoleBinding name for a +// provisionedRoleName returns the Role and RoleBinding name for a // ServiceAccount. It is injective within a namespace — the name is a pure // function of the ServiceAccount name, and a ServiceAccount name is unique // in its namespace — so two ServiceAccounts can never share these objects. @@ -185,20 +483,20 @@ func provisionedRoleName(serviceAccount string) string { return provisionedNamePrefix + serviceAccount + provisionedNameSuffix } -// provisionedClusterRoleName returns the permanent ClusterRole and -// ClusterRoleBinding name for a ServiceAccount. The namespace is part of the -// name because these objects are cluster-scoped and the same ServiceAccount -// name can exist in several namespaces. +// provisionedClusterRoleName returns the ClusterRole and ClusterRoleBinding +// name for a ServiceAccount. The namespace is part of the name because these +// objects are cluster-scoped and the same ServiceAccount name can exist in +// several namespaces. // // Joining two "-"-bearing segments is not injective ("a-b"/"c" and -// "a"/"b-c" compose the same string), so provisionClusterRoleBinding -// additionally refuses to retarget a binding that already names a different -// subject rather than silently revoking the first ServiceAccount's grants. +// "a"/"b-c" compose the same string). Nothing here can detect that — no +// cluster is read — so the rendered ClusterRoleBinding warns about it in its +// header and tells the operator how to check before applying. func provisionedClusterRoleName(namespace, serviceAccount string) string { return provisionedNamePrefix + namespace + "-" + serviceAccount + provisionedNameSuffix } -// provisionedLabels is the label set stamped on every permanent object. +// provisionedLabels is the label set stamped on every rendered object. // It deliberately omits labels.RunID: these objects belong to no run, so // Deployer.createdByThisRun can never match one and no run's Cleanup can // reclaim it. The component value is what distinguishes them from the @@ -210,140 +508,3 @@ func provisionedLabels() map[string]string { labels.Component: labels.ValueAgentRBAC, } } - -// provisionRole creates the permanent Role, or refreshes its rules in place -// when it already exists so an aicr upgrade that changes the rule set takes -// effect on a re-run instead of leaving stale rules behind. -func provisionRole(ctx context.Context, clientset kubernetes.Interface, namespace, name string) error { - role := &rbacv1.Role{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - Labels: provisionedLabels(), - }, - Rules: namespacedRules(), - } - _, err := clientset.RbacV1().Roles(namespace).Create(ctx, role, metav1.CreateOptions{}) - if apierrors.IsAlreadyExists(err) { - if _, err = clientset.RbacV1().Roles(namespace).Update(ctx, role, metav1.UpdateOptions{}); err != nil { - return errors.Wrap(errors.ErrCodeInternal, "failed to update the permanent Role", err) - } - return nil - } - if err != nil { - return errors.Wrap(errors.ErrCodeInternal, "failed to create the permanent Role", err) - } - return nil -} - -// provisionRoleBinding creates or refreshes the permanent RoleBinding. -// It needs no subject-collision guard: the name is a pure function of the -// ServiceAccount name within one namespace, so it can only ever refer to -// the ServiceAccount it is being written for. -func provisionRoleBinding(ctx context.Context, clientset kubernetes.Interface, namespace, name string, subjects []rbacv1.Subject) error { - rb := &rbacv1.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - Labels: provisionedLabels(), - }, - Subjects: subjects, - RoleRef: rbacv1.RoleRef{ - APIGroup: rbacAPIGroup, - Kind: kindRole, - Name: name, - }, - } - _, err := clientset.RbacV1().RoleBindings(namespace).Create(ctx, rb, metav1.CreateOptions{}) - if apierrors.IsAlreadyExists(err) { - if _, err = clientset.RbacV1().RoleBindings(namespace).Update(ctx, rb, metav1.UpdateOptions{}); err != nil { - return errors.Wrap(errors.ErrCodeInternal, "failed to update the permanent RoleBinding", err) - } - return nil - } - if err != nil { - return errors.Wrap(errors.ErrCodeInternal, "failed to create the permanent RoleBinding", err) - } - return nil -} - -// provisionClusterRole creates or refreshes the permanent ClusterRole. -func provisionClusterRole(ctx context.Context, clientset kubernetes.Interface, name string, discoverNetwork bool) error { - cr := &rbacv1.ClusterRole{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Labels: provisionedLabels(), - }, - Rules: clusterRules(discoverNetwork), - } - _, err := clientset.RbacV1().ClusterRoles().Create(ctx, cr, metav1.CreateOptions{}) - if apierrors.IsAlreadyExists(err) { - if _, err = clientset.RbacV1().ClusterRoles().Update(ctx, cr, metav1.UpdateOptions{}); err != nil { - return errors.Wrap(errors.ErrCodeInternal, "failed to update the permanent ClusterRole", err) - } - return nil - } - if err != nil { - return errors.Wrap(errors.ErrCodeInternal, "failed to create the permanent ClusterRole", err) - } - return nil -} - -// provisionClusterRoleBinding creates or refreshes the permanent -// ClusterRoleBinding. -// -// Before updating an existing one it checks the subject: because the -// cluster-scoped name joins namespace and ServiceAccount with "-", two -// distinct pairs can compose the same name (see -// provisionedClusterRoleName). Overwriting a binding that names a different -// ServiceAccount would silently revoke that ServiceAccount's cluster grants, -// so refuse with ErrCodeConflict — the request is well formed, the cluster -// state is what makes it unserviceable. -func provisionClusterRoleBinding(ctx context.Context, clientset kubernetes.Interface, name string, subjects []rbacv1.Subject) error { - crb := &rbacv1.ClusterRoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Labels: provisionedLabels(), - }, - Subjects: subjects, - RoleRef: rbacv1.RoleRef{ - APIGroup: rbacAPIGroup, - Kind: kindClusterRole, - Name: name, - }, - } - _, err := clientset.RbacV1().ClusterRoleBindings().Create(ctx, crb, metav1.CreateOptions{}) - if apierrors.IsAlreadyExists(err) { - existing, getErr := clientset.RbacV1().ClusterRoleBindings().Get(ctx, name, metav1.GetOptions{}) - if getErr != nil { - return errors.Wrap(errors.ErrCodeInternal, "failed to read the existing ClusterRoleBinding", getErr) - } - if other := conflictingSubject(existing.Subjects, subjects[0]); other != "" { - return errors.NewWithContext(errors.ErrCodeConflict, - fmt.Sprintf("ClusterRoleBinding %q already grants these permissions to %s; updating it would revoke them. Rename one of the two ServiceAccounts or namespaces so the generated names differ", - name, other), - map[string]any{ctxKeyResolvedName: name, "existingSubject": other}) - } - if _, err = clientset.RbacV1().ClusterRoleBindings().Update(ctx, crb, metav1.UpdateOptions{}); err != nil { - return errors.Wrap(errors.ErrCodeInternal, "failed to update the permanent ClusterRoleBinding", err) - } - return nil - } - if err != nil { - return errors.Wrap(errors.ErrCodeInternal, "failed to create the permanent ClusterRoleBinding", err) - } - return nil -} - -// conflictingSubject returns a description of the first subject in existing -// that is not want, or "" when every subject is want (the idempotent -// re-provisioning case) or existing is empty. -func conflictingSubject(existing []rbacv1.Subject, want rbacv1.Subject) string { - for _, s := range existing { - if s.Kind == want.Kind && s.Name == want.Name && s.Namespace == want.Namespace { - continue - } - return fmt.Sprintf("%s %q in namespace %q", s.Kind, s.Name, s.Namespace) - } - return "" -} diff --git a/pkg/k8s/agent/provision_test.go b/pkg/k8s/agent/provision_test.go index 602873b80..b9359b845 100644 --- a/pkg/k8s/agent/provision_test.go +++ b/pkg/k8s/agent/provision_test.go @@ -15,7 +15,6 @@ package agent import ( - "context" stderrors "errors" "reflect" "strings" @@ -23,376 +22,395 @@ import ( aicrerrors "github.com/NVIDIA/aicr/pkg/errors" "github.com/NVIDIA/aicr/pkg/k8s/labels" - corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes/fake" + "sigs.k8s.io/yaml" ) const provisionSA = "irsa-snapshotter" -// seedServiceAccount creates the target ServiceAccount provisioning requires. -func seedServiceAccount(ctx context.Context, t *testing.T, clientset *fake.Clientset, namespace, name string) { +// buildManifests renders the four manifests for the standard test inputs and +// fails the test on any error. +func buildManifests(t *testing.T, discoverNetwork bool) []Manifest { t.Helper() - if _, err := clientset.CoreV1().ServiceAccounts(namespace).Create(ctx, &corev1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, - }, metav1.CreateOptions{}); err != nil { - t.Fatalf("seeding ServiceAccount: %v", err) - } -} - -// TestProvisionServiceAccountRoles_Names pins the naming scheme, because its -// only real requirement is structural: a provisioned name must never be -// mistakable for a run-scoped one. Every run-scoped name ends in a run ID -// whose final segment is 16 lowercase-hex characters, and these end in -// "-rbac" — "r" is not a hex digit, so the two name spaces are disjoint by -// construction rather than by luck. -func TestProvisionServiceAccountRoles_Names(t *testing.T) { - ctx := context.Background() - clientset := fake.NewClientset() - seedServiceAccount(ctx, t, clientset, testNamespace, provisionSA) - - res, err := ProvisionServiceAccountRoles(ctx, clientset, ProvisionOptions{ + manifests, err := BuildServiceAccountRoleManifests(ManifestOptions{ Namespace: testNamespace, ServiceAccountName: provisionSA, + DiscoverNetwork: discoverNetwork, }) if err != nil { - t.Fatalf("ProvisionServiceAccountRoles() error = %v", err) - } - - wantRole := "aicr-agent-" + provisionSA + "-rbac" - wantClusterRole := "aicr-agent-" + testNamespace + "-" + provisionSA + "-rbac" - if res.Role != wantRole || res.RoleBinding != wantRole { - t.Errorf("Role/RoleBinding = %q/%q, want %q", res.Role, res.RoleBinding, wantRole) - } - if res.ClusterRole != wantClusterRole || res.ClusterRoleBinding != wantClusterRole { - t.Errorf("ClusterRole/ClusterRoleBinding = %q/%q, want %q", res.ClusterRole, res.ClusterRoleBinding, wantClusterRole) + t.Fatalf("BuildServiceAccountRoleManifests() error = %v", err) } + return manifests +} - // A run-scoped name is "-". Nothing provisioned may be - // one, whatever prefix a caller supplies. - for _, name := range []string{res.Role, res.RoleBinding, res.ClusterRole, res.ClusterRoleBinding} { - if strings.HasSuffix(name, "-"+testRunID) { - t.Errorf("provisioned name %q collides with the run-scoped name space", name) - } - if !strings.HasSuffix(name, provisionedNameSuffix) { - t.Errorf("provisioned name %q does not carry the %q suffix that keeps it out of the run-scoped name space", name, provisionedNameSuffix) - } +// manifestByFile indexes rendered manifests by file name. +func manifestByFile(manifests []Manifest) map[string]Manifest { + byFile := make(map[string]Manifest, len(manifests)) + for _, m := range manifests { + byFile[m.FileName] = m } + return byFile } -// TestProvisionServiceAccountRoles_ObjectsArePermanentAndUnscoped asserts the -// property the whole feature rests on: nothing provisioned carries a run ID, -// so no run's cleanup can ever reclaim it. Deployer.createdByThisRun requires -// the run-ID label, and these objects deliberately have none. -func TestProvisionServiceAccountRoles_ObjectsArePermanentAndUnscoped(t *testing.T) { - ctx := context.Background() - clientset := fake.NewClientset() - seedServiceAccount(ctx, t, clientset, testNamespace, provisionSA) +// TestBuildServiceAccountRoleManifests_FilesAndNames pins the file layout and +// the naming scheme together, because both are contracts an operator's +// commands depend on: `kubectl apply -f /` relies on one parseable +// object per file in an order that puts a Role ahead of its RoleBinding, and +// a rendered name must never be mistakable for a run-scoped one. Every +// run-scoped name ends in a run ID whose final segment is 16 lowercase-hex +// characters, and these end in "-rbac" — "r" is not a hex digit, so the two +// name spaces are disjoint by construction rather than by luck. +func TestBuildServiceAccountRoleManifests_FilesAndNames(t *testing.T) { + manifests := buildManifests(t, false) - res, err := ProvisionServiceAccountRoles(ctx, clientset, ProvisionOptions{ - Namespace: testNamespace, - ServiceAccountName: provisionSA, - }) - if err != nil { - t.Fatalf("ProvisionServiceAccountRoles() error = %v", err) - } - - role, err := clientset.RbacV1().Roles(testNamespace).Get(ctx, res.Role, metav1.GetOptions{}) - if err != nil { - t.Fatalf("Role not created: %v", err) - } - cr, err := clientset.RbacV1().ClusterRoles().Get(ctx, res.ClusterRole, metav1.GetOptions{}) - if err != nil { - t.Fatalf("ClusterRole not created: %v", err) - } + wantRole := "aicr-agent-" + provisionSA + "-rbac" + wantClusterRole := "aicr-agent-" + testNamespace + "-" + provisionSA + "-rbac" - for name, got := range map[string]map[string]string{res.Role: role.Labels, res.ClusterRole: cr.Labels} { - if _, ok := got[labels.RunID]; ok { - t.Errorf("%s carries the %s label; a run's cleanup could reclaim it", name, labels.RunID) + want := []struct { + file string + kind string + name string + }{ + {roleFileName, kindRole, wantRole}, + {roleBindingFileName, kindRoleBinding, wantRole}, + {clusterRoleFileName, kindClusterRole, wantClusterRole}, + {clusterRoleBindingFileName, kindClusterRoleBinding, wantClusterRole}, + } + if len(manifests) != len(want) { + t.Fatalf("manifests = %d, want %d (one file per object)", len(manifests), len(want)) + } + for i, w := range want { + got := manifests[i] + if got.FileName != w.file { + t.Errorf("manifest[%d].FileName = %q, want %q (lexical order must apply a Role before its binding)", i, got.FileName, w.file) + } + if got.Kind != w.kind { + t.Errorf("manifest[%d].Kind = %q, want %q", i, got.Kind, w.kind) } - if got[labels.Component] != labels.ValueAgentRBAC { - t.Errorf("%s component label = %q, want %q", name, got[labels.Component], labels.ValueAgentRBAC) + if got.Name != w.name { + t.Errorf("manifest[%d].Name = %q, want %q", i, got.Name, w.name) } - if got[labels.Component] == labels.ValueSnapshotAgent { - t.Errorf("%s is labeled as a run-scoped snapshot-agent object", name) + if !strings.HasSuffix(got.Name, "-rbac") { + t.Errorf("manifest[%d].Name = %q, want a name ending in -rbac so it cannot collide with a run-scoped name", i, got.Name) } } +} - // A Deployer must not be able to claim these as its own, whatever run - // ID it holds. - d := NewDeployer(clientset, Config{Namespace: testNamespace, RunID: testRunID}) - if d.createdByThisRun(role.Labels) || d.createdByThisRun(cr.Labels) { - t.Error("createdByThisRun matched a provisioned object; run cleanup would delete a permanent grant") - } +// TestBuildServiceAccountRoleManifests_ParseableYAML asserts each file +// round-trips through a YAML decoder into the typed object it claims to be, +// with apiVersion and kind set. A typed object straight out of client-go +// leaves TypeMeta empty, which `kubectl apply` rejects, so this is the check +// that the files are usable at all. +func TestBuildServiceAccountRoleManifests_ParseableYAML(t *testing.T) { + byFile := manifestByFile(buildManifests(t, false)) + wantAPIVersion := rbacv1.SchemeGroupVersion.String() - // The bindings must point at the operator's ServiceAccount. - rb, err := clientset.RbacV1().RoleBindings(testNamespace).Get(ctx, res.RoleBinding, metav1.GetOptions{}) - if err != nil { - t.Fatalf("RoleBinding not created: %v", err) - } - crb, err := clientset.RbacV1().ClusterRoleBindings().Get(ctx, res.ClusterRoleBinding, metav1.GetOptions{}) - if err != nil { - t.Fatalf("ClusterRoleBinding not created: %v", err) - } - want := []rbacv1.Subject{{Kind: kindServiceAccount, Name: provisionSA, Namespace: testNamespace}} - if !reflect.DeepEqual(rb.Subjects, want) { - t.Errorf("RoleBinding subjects = %v, want %v", rb.Subjects, want) + tests := []struct { + name string + file string + wantKind string + into func() any + check func(t *testing.T, obj any) + }{ + { + name: "role carries the namespaced rules", + file: roleFileName, + wantKind: kindRole, + into: func() any { return &rbacv1.Role{} }, + check: func(t *testing.T, obj any) { + t.Helper() + role, ok := obj.(*rbacv1.Role) + if !ok { + t.Fatalf("decoded %T, want *rbacv1.Role", obj) + } + if role.Namespace != testNamespace { + t.Errorf("Role namespace = %q, want %q", role.Namespace, testNamespace) + } + if !reflect.DeepEqual(role.Rules, namespacedRules()) { + t.Errorf("Role rules drifted from namespacedRules(); got %+v", role.Rules) + } + }, + }, + { + name: "rolebinding names the ServiceAccount subject", + file: roleBindingFileName, + wantKind: kindRoleBinding, + into: func() any { return &rbacv1.RoleBinding{} }, + check: func(t *testing.T, obj any) { + t.Helper() + rb, ok := obj.(*rbacv1.RoleBinding) + if !ok { + t.Fatalf("decoded %T, want *rbacv1.RoleBinding", obj) + } + wantSubjects := []rbacv1.Subject{{Kind: kindServiceAccount, Name: provisionSA, Namespace: testNamespace}} + if !reflect.DeepEqual(rb.Subjects, wantSubjects) { + t.Errorf("RoleBinding subjects = %+v, want %+v", rb.Subjects, wantSubjects) + } + if rb.RoleRef.Kind != kindRole { + t.Errorf("RoleBinding roleRef kind = %q, want %q", rb.RoleRef.Kind, kindRole) + } + }, + }, + { + name: "clusterrole carries the cluster rules and no namespace", + file: clusterRoleFileName, + wantKind: kindClusterRole, + into: func() any { return &rbacv1.ClusterRole{} }, + check: func(t *testing.T, obj any) { + t.Helper() + cr, ok := obj.(*rbacv1.ClusterRole) + if !ok { + t.Fatalf("decoded %T, want *rbacv1.ClusterRole", obj) + } + if cr.Namespace != "" { + t.Errorf("ClusterRole namespace = %q, want empty (it is cluster-scoped)", cr.Namespace) + } + if !reflect.DeepEqual(cr.Rules, clusterRules(false)) { + t.Errorf("ClusterRole rules drifted from clusterRules(false); got %+v", cr.Rules) + } + }, + }, + { + name: "clusterrolebinding binds the clusterrole", + file: clusterRoleBindingFileName, + wantKind: kindClusterRoleBinding, + into: func() any { return &rbacv1.ClusterRoleBinding{} }, + check: func(t *testing.T, obj any) { + t.Helper() + crb, ok := obj.(*rbacv1.ClusterRoleBinding) + if !ok { + t.Fatalf("decoded %T, want *rbacv1.ClusterRoleBinding", obj) + } + if crb.RoleRef.Kind != kindClusterRole { + t.Errorf("ClusterRoleBinding roleRef kind = %q, want %q", crb.RoleRef.Kind, kindClusterRole) + } + if got := crb.Labels[labels.Component]; got != labels.ValueAgentRBAC { + t.Errorf("component label = %q, want %q", got, labels.ValueAgentRBAC) + } + if _, ok := crb.Labels[labels.RunID]; ok { + t.Error("rendered object carries a run-ID label; these objects belong to no run") + } + }, + }, } - if !reflect.DeepEqual(crb.Subjects, want) { - t.Errorf("ClusterRoleBinding subjects = %v, want %v", crb.Subjects, want) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m, ok := byFile[tt.file] + if !ok { + t.Fatalf("no manifest rendered for %s", tt.file) + } + obj := tt.into() + if err := yaml.Unmarshal(m.Content, obj); err != nil { + t.Fatalf("unmarshalling %s: %v\n%s", tt.file, err, m.Content) + } + var apiVersion, kind string + switch o := obj.(type) { + case *rbacv1.Role: + apiVersion, kind = o.APIVersion, o.Kind + case *rbacv1.RoleBinding: + apiVersion, kind = o.APIVersion, o.Kind + case *rbacv1.ClusterRole: + apiVersion, kind = o.APIVersion, o.Kind + case *rbacv1.ClusterRoleBinding: + apiVersion, kind = o.APIVersion, o.Kind + } + if apiVersion != wantAPIVersion { + t.Errorf("%s apiVersion = %q, want %q", tt.file, apiVersion, wantAPIVersion) + } + if kind != tt.wantKind { + t.Errorf("%s kind = %q, want %q", tt.file, kind, tt.wantKind) + } + tt.check(t, obj) + }) } } -// TestProvisionServiceAccountRoles_DiscoverNetworkRules covers both rule sets: -// the read-only baseline, and the baseline plus the cluster-scoped mutating -// rules live discovery needs. The distinction matters because a provisioned -// grant is permanent — a --discover-network provisioning leaves nodes:patch -// and pods/exec:create on the ServiceAccount indefinitely. -func TestProvisionServiceAccountRoles_DiscoverNetworkRules(t *testing.T) { +// TestBuildServiceAccountRoleManifests_DiscoverNetwork covers both rule sets. +// The default grant must stay read-only, and the --discover-network grant +// must carry the mutating rules AND explain them in the header — the header +// is what an operator reads to decide whether to apply the file at all, so a +// silent "nodes: patch" would defeat the point of writing manifests out +// instead of applying them. +func TestBuildServiceAccountRoleManifests_DiscoverNetwork(t *testing.T) { tests := []struct { name string discoverNetwork bool + wantRuleCount int wantMutating bool }{ - {name: "read-only baseline", discoverNetwork: false, wantMutating: false}, - {name: "discovery adds the mutating rules", discoverNetwork: true, wantMutating: true}, + {name: "default grant is read-only", discoverNetwork: false, wantRuleCount: len(clusterRules(false))}, + {name: "discovery grant adds the mutating rules", discoverNetwork: true, wantRuleCount: len(clusterRules(true)), wantMutating: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ctx := context.Background() - clientset := fake.NewClientset() - seedServiceAccount(ctx, t, clientset, testNamespace, provisionSA) - - res, err := ProvisionServiceAccountRoles(ctx, clientset, ProvisionOptions{ - Namespace: testNamespace, - ServiceAccountName: provisionSA, - DiscoverNetwork: tt.discoverNetwork, - }) - if err != nil { - t.Fatalf("ProvisionServiceAccountRoles() error = %v", err) - } - if res.DiscoverNetwork != tt.discoverNetwork { - t.Errorf("result DiscoverNetwork = %v, want %v", res.DiscoverNetwork, tt.discoverNetwork) + m, ok := manifestByFile(buildManifests(t, tt.discoverNetwork))[clusterRoleFileName] + if !ok { + t.Fatalf("no ClusterRole manifest rendered") } - - cr, err := clientset.RbacV1().ClusterRoles().Get(ctx, res.ClusterRole, metav1.GetOptions{}) - if err != nil { - t.Fatalf("ClusterRole not created: %v", err) - } - if got := hasRule(cr.Rules, "", "nodes", "patch"); got != tt.wantMutating { - t.Errorf("nodes:patch granted = %v, want %v", got, tt.wantMutating) + cr := &rbacv1.ClusterRole{} + if err := yaml.Unmarshal(m.Content, cr); err != nil { + t.Fatalf("unmarshalling ClusterRole: %v", err) } - if got := hasRule(cr.Rules, "", "pods/exec", verbCreate); got != tt.wantMutating { - t.Errorf("pods/exec:create granted = %v, want %v", got, tt.wantMutating) + if len(cr.Rules) != tt.wantRuleCount { + t.Errorf("ClusterRole rules = %d, want %d", len(cr.Rules), tt.wantRuleCount) } - // The baseline read-only rules are present either way. - if !hasRule(cr.Rules, "", "nodes", verbList) { - t.Error("baseline nodes:list rule missing") + for _, want := range []struct{ resource, verb string }{ + {"nodes", "patch"}, + {"pods/exec", verbCreate}, + {"customresourcedefinitions", verbCreate}, + } { + if got := hasRule(cr.Rules, want.resource, want.verb); got != tt.wantMutating { + t.Errorf("%s: %s present = %v, want %v", want.resource, want.verb, got, tt.wantMutating) + } } - // The provisioned ClusterRole must carry exactly what a - // run-scoped one would, so an adopted ServiceAccount is not - // quietly less capable than a run-owned one. - if !reflect.DeepEqual(cr.Rules, clusterRules(tt.discoverNetwork)) { - t.Error("provisioned ClusterRole rules differ from the run-scoped agent's") + header := string(m.Content) + for _, want := range []string{"nodes: patch", "pods/exec: create", "MUTATING"} { + if strings.Contains(header, want) != tt.wantMutating { + t.Errorf("header mentions %q = %v, want %v; header:\n%s", want, !tt.wantMutating, tt.wantMutating, header) + } } - role, err := clientset.RbacV1().Roles(testNamespace).Get(ctx, res.Role, metav1.GetOptions{}) - if err != nil { - t.Fatalf("Role not created: %v", err) - } - if !reflect.DeepEqual(role.Rules, namespacedRules()) { - t.Error("provisioned Role rules differ from the run-scoped agent's") + if !tt.wantMutating && !strings.Contains(header, "READ-ONLY") { + t.Errorf("read-only grant does not say so in the header:\n%s", header) } }) } } -// TestProvisionServiceAccountRoles_Idempotent re-runs provisioning over a -// stale rule set and asserts the rules are refreshed in place rather than the -// call failing or the stale grant surviving. This is the aicr-upgrade path: -// an operator re-runs the command and expects the current rules. -func TestProvisionServiceAccountRoles_Idempotent(t *testing.T) { - ctx := context.Background() - clientset := fake.NewClientset() - seedServiceAccount(ctx, t, clientset, testNamespace, provisionSA) - - opts := ProvisionOptions{Namespace: testNamespace, ServiceAccountName: provisionSA, DiscoverNetwork: true} - res, err := ProvisionServiceAccountRoles(ctx, clientset, opts) - if err != nil { - t.Fatalf("first ProvisionServiceAccountRoles() error = %v", err) - } - - // Simulate a stale grant left by an older aicr: strip the rules from - // both roles. A merely-idempotent implementation that skipped existing - // objects would leave them stripped. - stale, err := clientset.RbacV1().ClusterRoles().Get(ctx, res.ClusterRole, metav1.GetOptions{}) - if err != nil { - t.Fatalf("reading ClusterRole: %v", err) - } - stale.Rules = nil - if _, err = clientset.RbacV1().ClusterRoles().Update(ctx, stale, metav1.UpdateOptions{}); err != nil { - t.Fatalf("staling ClusterRole: %v", err) - } - - res2, err := ProvisionServiceAccountRoles(ctx, clientset, opts) - if err != nil { - t.Fatalf("second ProvisionServiceAccountRoles() error = %v", err) - } - if !reflect.DeepEqual(res, res2) { - t.Errorf("second result = %+v, want %+v (names must be deterministic)", res2, res) +// hasRule reports whether any rule grants verb on resource. +func hasRule(rules []rbacv1.PolicyRule, resource, verb string) bool { + for _, r := range rules { + for _, res := range r.Resources { + if res != resource { + continue + } + for _, v := range r.Verbs { + if v == verb { + return true + } + } + } } + return false +} - refreshed, err := clientset.RbacV1().ClusterRoles().Get(ctx, res.ClusterRole, metav1.GetOptions{}) - if err != nil { - t.Fatalf("reading ClusterRole: %v", err) - } - if !reflect.DeepEqual(refreshed.Rules, clusterRules(true)) { - t.Error("re-provisioning did not refresh the stale ClusterRole rules") +// TestBuildServiceAccountRoleManifests_HeadersExplainTheGrant asserts every +// file states, in its own header, that nothing was applied and how to apply +// and revoke. An operator may open exactly one of these files, and the fact +// they must not miss is that the grant is not live yet. +func TestBuildServiceAccountRoleManifests_HeadersExplainTheGrant(t *testing.T) { + wantEverywhere := []string{ + "NOTHING HAS BEEN APPLIED", + "kubectl apply -f", + "kubectl delete -f", + provisionSA, + } + for _, m := range buildManifests(t, false) { + body := string(m.Content) + if !strings.HasPrefix(body, "# ") { + t.Errorf("%s does not open with a YAML comment header; got %.40q", m.FileName, body) + } + for _, want := range wantEverywhere { + if !strings.Contains(body, want) { + t.Errorf("%s header does not contain %q", m.FileName, want) + } + } } +} - // Exactly one of each object, not a duplicate per run. - crs, err := clientset.RbacV1().ClusterRoles().List(ctx, metav1.ListOptions{}) - if err != nil { - t.Fatalf("listing ClusterRoles: %v", err) - } - if len(crs.Items) != 1 { - t.Errorf("ClusterRoles = %d, want 1", len(crs.Items)) - } - roles, err := clientset.RbacV1().Roles(testNamespace).List(ctx, metav1.ListOptions{}) - if err != nil { - t.Fatalf("listing Roles: %v", err) - } - if len(roles.Items) != 1 { - t.Errorf("Roles = %d, want 1", len(roles.Items)) +// TestBuildServiceAccountRoleManifests_Deterministic asserts two renders of +// the same inputs are byte-identical. The manifests are reviewed by hand and +// diffed against a previous grant, so incidental churn between runs would +// make a real change hard to spot. +func TestBuildServiceAccountRoleManifests_Deterministic(t *testing.T) { + first := buildManifests(t, true) + second := buildManifests(t, true) + if len(first) != len(second) { + t.Fatalf("manifest counts differ: %d vs %d", len(first), len(second)) + } + for i := range first { + if string(first[i].Content) != string(second[i].Content) { + t.Errorf("%s differs between renders:\n--- first ---\n%s\n--- second ---\n%s", + first[i].FileName, first[i].Content, second[i].Content) + } } } -// TestProvisionServiceAccountRoles_Rejections covers every input the call -// refuses before writing anything — most importantly a ServiceAccount that -// does not exist, which must be ErrCodeNotFound rather than four dangling -// objects bound to nothing. -func TestProvisionServiceAccountRoles_Rejections(t *testing.T) { +// TestBuildServiceAccountRoleManifests_Rejections covers every input the call +// refuses before rendering anything. A ServiceAccount that does not exist is +// deliberately NOT among them: rendering contacts no cluster, so a mistyped +// name yields manifests the operator inspects before applying. +func TestBuildServiceAccountRoleManifests_Rejections(t *testing.T) { tests := []struct { name string - seed string - opts ProvisionOptions - wantCode aicrerrors.ErrorCode + opts ManifestOptions wantInMsg string }{ { - name: "missing ServiceAccount", - opts: ProvisionOptions{Namespace: testNamespace, ServiceAccountName: provisionSA}, - wantCode: aicrerrors.ErrCodeNotFound, - wantInMsg: "not found in namespace", + name: "empty namespace", + opts: ManifestOptions{ServiceAccountName: provisionSA}, }, { - name: "empty namespace", - opts: ProvisionOptions{ServiceAccountName: provisionSA}, - wantCode: aicrerrors.ErrCodeInvalidRequest, + name: "whitespace namespace", + opts: ManifestOptions{Namespace: " ", ServiceAccountName: provisionSA}, }, { - name: "empty ServiceAccount name", - opts: ProvisionOptions{Namespace: testNamespace}, - wantCode: aicrerrors.ErrCodeInvalidRequest, + name: "empty ServiceAccount name", + opts: ManifestOptions{Namespace: testNamespace}, }, { - name: "whitespace ServiceAccount name", - opts: ProvisionOptions{Namespace: testNamespace, ServiceAccountName: " "}, - wantCode: aicrerrors.ErrCodeInvalidRequest, + name: "whitespace ServiceAccount name", + opts: ManifestOptions{Namespace: testNamespace, ServiceAccountName: " "}, }, { name: "name too long to compose", - seed: strings.Repeat("a", 250), - opts: ProvisionOptions{Namespace: testNamespace, ServiceAccountName: strings.Repeat("a", 250)}, - wantCode: aicrerrors.ErrCodeInvalidRequest, + opts: ManifestOptions{Namespace: testNamespace, ServiceAccountName: strings.Repeat("a", 250)}, wantInMsg: "not a valid Kubernetes object name", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ctx := context.Background() - clientset := fake.NewClientset() - if tt.seed != "" { - seedServiceAccount(ctx, t, clientset, testNamespace, tt.seed) - } - - _, err := ProvisionServiceAccountRoles(ctx, clientset, tt.opts) + manifests, err := BuildServiceAccountRoleManifests(tt.opts) if err == nil { - t.Fatal("ProvisionServiceAccountRoles() error = nil, want an error") + t.Fatal("BuildServiceAccountRoleManifests() error = nil, want ErrCodeInvalidRequest") } - if !stderrors.Is(err, aicrerrors.New(tt.wantCode, "")) { - t.Errorf("error = %v, want code %s", err, tt.wantCode) + if !stderrors.Is(err, aicrerrors.New(aicrerrors.ErrCodeInvalidRequest, "")) { + t.Errorf("error = %v, want code %s", err, aicrerrors.ErrCodeInvalidRequest) } if tt.wantInMsg != "" && !strings.Contains(err.Error(), tt.wantInMsg) { t.Errorf("error = %q, want it to contain %q", err.Error(), tt.wantInMsg) } - - // Nothing may be written on a rejected call. - crs, listErr := clientset.RbacV1().ClusterRoles().List(ctx, metav1.ListOptions{}) - if listErr != nil { - t.Fatalf("listing ClusterRoles: %v", listErr) - } - if len(crs.Items) != 0 { - t.Errorf("ClusterRoles = %d, want 0 (a rejected call must write nothing)", len(crs.Items)) + if manifests != nil { + t.Errorf("manifests = %v, want nil on a rejected call", manifests) } }) } } -// TestProvisionServiceAccountRoles_RefusesToRetargetAnotherSubject covers the -// one way the cluster-scoped name can be ambiguous: it joins namespace and -// ServiceAccount with "-", so ("a-b", "c") and ("a", "b-c") compose the same -// name. Silently updating the binding would revoke the first ServiceAccount's -// cluster grants, so the second provisioning must fail closed. -func TestProvisionServiceAccountRoles_RefusesToRetargetAnotherSubject(t *testing.T) { - ctx := context.Background() - clientset := fake.NewClientset() - seedServiceAccount(ctx, t, clientset, "a-b", "c") - seedServiceAccount(ctx, t, clientset, "a", "b-c") - - first, err := ProvisionServiceAccountRoles(ctx, clientset, ProvisionOptions{Namespace: "a-b", ServiceAccountName: "c"}) - if err != nil { - t.Fatalf("first ProvisionServiceAccountRoles() error = %v", err) - } - - _, err = ProvisionServiceAccountRoles(ctx, clientset, ProvisionOptions{Namespace: "a", ServiceAccountName: "b-c"}) - if err == nil { - t.Fatal("second ProvisionServiceAccountRoles() error = nil, want a conflict") - } - if !stderrors.Is(err, aicrerrors.New(aicrerrors.ErrCodeConflict, "")) { - t.Errorf("error = %v, want ErrCodeConflict", err) - } - - // The first ServiceAccount keeps its grant. - crb, err := clientset.RbacV1().ClusterRoleBindings().Get(ctx, first.ClusterRoleBinding, metav1.GetOptions{}) +// TestBuildServiceAccountRoleManifests_UnknownServiceAccountStillRenders pins +// the deliberate simplification: the old provisioning path failed with +// ErrCodeNotFound when the ServiceAccount was absent. Rendering has no +// cluster to ask, so it renders regardless and the RoleBinding header tells +// the operator to check the name themselves. +func TestBuildServiceAccountRoleManifests_UnknownServiceAccountStillRenders(t *testing.T) { + manifests, err := BuildServiceAccountRoleManifests(ManifestOptions{ + Namespace: testNamespace, + ServiceAccountName: "no-such-serviceaccount", + }) if err != nil { - t.Fatalf("reading ClusterRoleBinding: %v", err) + t.Fatalf("BuildServiceAccountRoleManifests() error = %v, want manifests for an unverified name", err) } - want := []rbacv1.Subject{{Kind: kindServiceAccount, Name: "c", Namespace: "a-b"}} - if !reflect.DeepEqual(crb.Subjects, want) { - t.Errorf("ClusterRoleBinding subjects = %v, want %v (the first grant must survive)", crb.Subjects, want) + if len(manifests) != 4 { + t.Fatalf("manifests = %d, want 4", len(manifests)) } -} - -// hasRule reports whether rules grant verb on resource in apiGroup. -func hasRule(rules []rbacv1.PolicyRule, apiGroup, resource, verb string) bool { - for _, r := range rules { - if !contains(r.APIGroups, apiGroup) || !contains(r.Resources, resource) || !contains(r.Verbs, verb) { - continue - } - return true + rb, ok := manifestByFile(manifests)[roleBindingFileName] + if !ok { + t.Fatal("no RoleBinding manifest rendered") } - return false -} - -func contains(haystack []string, needle string) bool { - for _, s := range haystack { - if s == needle { - return true - } + if !strings.Contains(string(rb.Content), "kubectl get serviceaccount no-such-serviceaccount") { + t.Errorf("RoleBinding header does not tell the operator how to verify the name:\n%s", rb.Content) } - return false } diff --git a/pkg/k8s/agent/rbac.go b/pkg/k8s/agent/rbac.go index ff1847301..8a8a7126d 100644 --- a/pkg/k8s/agent/rbac.go +++ b/pkg/k8s/agent/rbac.go @@ -121,7 +121,7 @@ func (d *Deployer) resolveServiceAccount(ctx context.Context) error { attrServiceAccount, name, attrNamespace, d.config.Namespace, attrRunID, d.config.RunID, - "note", "no ServiceAccount, Role, RoleBinding, ClusterRole or ClusterRoleBinding is created or deleted; grant the agent's permissions once with 'aicr snapshot --add-roles-to-service-account "+name+"'") + "note", "no ServiceAccount, Role, RoleBinding, ClusterRole or ClusterRoleBinding is created or deleted; generate this ServiceAccount's RBAC manifests with 'aicr snapshot --add-roles-to-service-account "+name+"' and apply them yourself") case apierrors.IsNotFound(err): // Normal path: the value is a prefix and this run creates its own // run-scoped ServiceAccount below. @@ -165,8 +165,8 @@ func (d *Deployer) ensureServiceAccount(ctx context.Context) error { // writing its snapshot result into a staging ConfigMap, and reading pods. // // It is the single definition consumed by both the run-scoped Role -// ensureRole creates and the permanent Role ProvisionServiceAccountRoles -// grants to an operator-supplied ServiceAccount, so the two can never drift. +// ensureRole creates and the Role BuildServiceAccountRoleManifests renders +// for an operator-supplied ServiceAccount, so the two can never drift. func namespacedRules() []rbacv1.PolicyRule { return []rbacv1.PolicyRule{ { @@ -187,8 +187,8 @@ func namespacedRules() []rbacv1.PolicyRule { // l8k network discovery requires (see discoverNetworkClusterRules). // // It is the single definition consumed by both the run-scoped ClusterRole -// ensureClusterRole creates and the permanent ClusterRole -// ProvisionServiceAccountRoles grants. +// ensureClusterRole creates and the ClusterRole +// BuildServiceAccountRoleManifests renders. func clusterRules(discoverNetwork bool) []rbacv1.PolicyRule { rules := []rbacv1.PolicyRule{ { diff --git a/pkg/k8s/agent/types.go b/pkg/k8s/agent/types.go index 90b09f1f7..e755bdcea 100644 --- a/pkg/k8s/agent/types.go +++ b/pkg/k8s/agent/types.go @@ -82,8 +82,9 @@ type Config struct { // This is what keeps a ServiceAccount carrying IRSA or GKE // Workload Identity annotations usable: both providers pin trust // to the ServiceAccount NAME, which a run-scoped name can never - // satisfy. Grant the agent's permissions to such a - // ServiceAccount once with ProvisionServiceAccountRoles. + // satisfy. Generate the RBAC that grants such a ServiceAccount + // the agent's permissions with + // BuildServiceAccountRoleManifests, then apply it out of band. // - Otherwise: a prefix. The run creates "-" and // the full run-scoped RBAC set, and deletes them at cleanup. // diff --git a/pkg/k8s/labels/labels.go b/pkg/k8s/labels/labels.go index 8102e34bf..2d0fca49a 100644 --- a/pkg/k8s/labels/labels.go +++ b/pkg/k8s/labels/labels.go @@ -37,12 +37,13 @@ const ( // ValueSnapshotAgent identifies snapshot-agent-owned resources. ValueSnapshotAgent = "snapshot-agent" - // ValueAgentRBAC identifies the permanent, NON-run-scoped Role, - // RoleBinding, ClusterRole and ClusterRoleBinding that - // `aicr snapshot --add-roles-to-service-account` provisions onto an - // operator-supplied ServiceAccount. Objects carrying this value are - // deliberately outside every run's lifecycle: they carry no RunID - // label, never enter a run's created-set, and are never deleted by - // run cleanup. Teardown is the operator's job. + // ValueAgentRBAC identifies the NON-run-scoped Role, RoleBinding, + // ClusterRole and ClusterRoleBinding that + // `aicr snapshot --add-roles-to-service-account` renders as manifests + // for an operator-supplied ServiceAccount. aicr applies none of them; + // the operator does. Objects carrying this value are deliberately + // outside every run's lifecycle: they carry no RunID label, never + // enter a run's created-set, and are never deleted by run cleanup. + // Teardown is the operator's `kubectl delete -f`. ValueAgentRBAC = "agent-rbac" ) diff --git a/pkg/snapshotter/agent.go b/pkg/snapshotter/agent.go index 22882442b..216d1fc99 100644 --- a/pkg/snapshotter/agent.go +++ b/pkg/snapshotter/agent.go @@ -78,8 +78,8 @@ type AgentConfig struct { // (eks.amazonaws.com/role-arn) or GKE Workload Identity // (iam.gke.io/gcp-service-account) annotations stays usable: both // providers pin trust to the ServiceAccount NAME, which a - // run-scoped name can never satisfy. Grant it the agent's - // permissions once with ProvisionAgentRoles. + // run-scoped name can never satisfy. Generate its RBAC manifests + // with WriteAgentRoleManifests, then apply them out of band. // - Otherwise: a name prefix. The run creates "-" // and the full run-scoped RBAC set, and deletes them at cleanup. // diff --git a/pkg/snapshotter/provision.go b/pkg/snapshotter/provision.go index beb4c22c3..235267c70 100644 --- a/pkg/snapshotter/provision.go +++ b/pkg/snapshotter/provision.go @@ -15,95 +15,130 @@ package snapshotter import ( - "context" + stderrors "errors" + "io/fs" + "log/slog" + "os" + "path/filepath" "strings" "github.com/NVIDIA/aicr/pkg/defaults" "github.com/NVIDIA/aicr/pkg/errors" "github.com/NVIDIA/aicr/pkg/k8s/agent" + "github.com/NVIDIA/aicr/pkg/runid" ) -// AgentRolesConfig selects the ServiceAccount that ProvisionAgentRoles -// grants the snapshot agent's permissions to, and the cluster it lives in. +// AgentRolesConfig selects the ServiceAccount that WriteAgentRoleManifests +// renders the snapshot agent's RBAC for. +// +// There is deliberately no Kubeconfig field: writing the manifests contacts +// no cluster, so there is no connection to configure. type AgentRolesConfig struct { - // Kubeconfig is an optional path override; empty uses default - // discovery (KUBECONFIG, then ~/.kube/config, then in-cluster). - Kubeconfig string - - // Namespace holds the ServiceAccount and receives the Role and - // RoleBinding. Required. + // Namespace is the namespace of the ServiceAccount, and the namespace + // the rendered Role and RoleBinding declare. Required. Namespace string - // ServiceAccountName is the EXACT name of an already-existing - // ServiceAccount. Required. + // ServiceAccountName is the name of the ServiceAccount the rendered + // bindings name as their subject. Required, and not verified to + // exist — see WriteAgentRoleManifests. ServiceAccountName string - // DiscoverNetwork also grants the cluster-scoped MUTATING rules that - // `aicr snapshot --discover-network` needs. Permanently, not for one - // run's lifetime. + // DiscoverNetwork also renders the cluster-scoped MUTATING rules that + // `aicr snapshot --discover-network` needs, with a header enumerating + // each one and the discovery step it exists for. DiscoverNetwork bool + + // RunID names the output directory (`snapshot-rbac-`). Empty + // generates one, which is the normal path; it is injectable so tests + // and automation can pin a directory name. + RunID string } -// AgentRolesResult names what ProvisionAgentRoles created or updated, so a -// caller can report it without rebuilding the names. +// AgentRoleObject identifies one written manifest so a caller can report +// what landed where without re-deriving either the name or the file. +type AgentRoleObject struct { + // Kind is the Kubernetes kind ("Role", "RoleBinding", "ClusterRole", + // "ClusterRoleBinding"). + Kind string + + // Name is the object's metadata.name. + Name string + + // Path is the manifest's path, including the output directory. + Path string +} + +// AgentRolesResult names the directory WriteAgentRoleManifests wrote and +// what it put there. // -// It is snapshotter-owned rather than pkg/k8s/agent's own ProvisionResult -// so callers presenting the outcome — the CLI among them — need no -// dependency on the Kubernetes-facing package. +// It is snapshotter-owned rather than pkg/k8s/agent's own Manifest type so +// callers presenting the outcome — the CLI among them — need no dependency +// on the Kubernetes-facing package. type AgentRolesResult struct { + // Dir is the output directory, relative to the working directory the + // call was made from. + Dir string + + // RunID is the run ID the directory name was built from. + RunID string + Namespace string ServiceAccountName string - Role string - RoleBinding string - ClusterRole string - ClusterRoleBinding string + + // Objects lists what was written, in the order the files apply. + Objects []AgentRoleObject // DiscoverNetwork echoes AgentRolesConfig.DiscoverNetwork: it is the // difference between a read-only grant and one carrying cluster-scoped // mutating rules, so anything reporting this result can say which was - // provisioned. + // written. DiscoverNetwork bool } -// ProvisionAgentRoles grants the snapshot agent's permissions to an -// existing, operator-supplied ServiceAccount so that ServiceAccount can be -// named exactly via AgentConfig.ServiceAccountName -// (`--service-account-name`) and keep its own identity — the IRSA or GKE -// Workload Identity annotations a run-scoped ServiceAccount cannot carry, -// because both providers pin trust to the ServiceAccount name. +// WriteAgentRoleManifests writes the RBAC manifests that grant the snapshot +// agent's permissions to an operator-supplied ServiceAccount into a new +// `snapshot-rbac-` directory in the current working directory. +// +// It APPLIES NOTHING and contacts no cluster. No clientset is built, the +// ServiceAccount is never looked up, and no permission pre-flight runs — so +// the call succeeds with no kubeconfig and no cluster privileges at all. The +// operator reviews the files and then applies them: +// +// kubectl apply -f snapshot-rbac-/ +// +// and removes the grant with the matching delete: // -// It provisions and returns; it deploys no Job and collects no snapshot. -// The objects it creates are PERMANENT: they carry no run-ID label, never -// enter a run's created-set, and no run's cleanup deletes them. Removing -// them is the operator's job. +// kubectl delete -f snapshot-rbac-/ // -// Idempotent — re-run it after an aicr upgrade to refresh the rules in -// place. Returns ErrCodeNotFound when the named ServiceAccount does not -// exist. -func ProvisionAgentRoles(ctx context.Context, config *AgentRolesConfig) (*AgentRolesResult, error) { +// The ServiceAccount named in ServiceAccountName is NOT verified to exist. +// That is a deliberate simplification of the earlier behavior, which failed +// with ErrCodeNotFound against the cluster: a mistyped name now yields +// manifests the operator inspects before applying, and the rendered +// RoleBinding tells them how to check. +// +// The directory must not already exist. Colliding with one returns +// ErrCodeConflict rather than overwriting, because the manifests an operator +// is midway through reviewing are exactly what must not change under them. +// +// The objects are outside every run's lifecycle: no run-ID label, never in a +// run's created-set, never deleted by run cleanup. Teardown is the +// operator's `kubectl delete`. +func WriteAgentRoleManifests(config *AgentRolesConfig) (*AgentRolesResult, error) { if config == nil { return nil, errors.New(errors.ErrCodeInvalidRequest, "agent roles config is required") } - // Reject what can be rejected without contacting the cluster, so a bad - // value is never masked by a kubeconfig error. + // Reject what can be rejected before touching the filesystem, so a bad + // value never leaves a half-written directory behind. if strings.TrimSpace(config.Namespace) == "" { return nil, errors.New(errors.ErrCodeInvalidRequest, - "Namespace is required: it is where the ServiceAccount, Role and RoleBinding live") + "Namespace is required: it is the namespace the rendered Role and RoleBinding declare") } if strings.TrimSpace(config.ServiceAccountName) == "" { return nil, errors.New(errors.ErrCodeInvalidRequest, - "ServiceAccountName is required: provisioning grants permissions to an existing ServiceAccount, it does not create one") + "ServiceAccountName is required: the rendered bindings need a subject to name") } - clientset, err := getKubeClient(config.Kubeconfig) - if err != nil { - return nil, err - } - - ctx, cancel := context.WithTimeout(ctx, defaults.AgentRBACProvisionTimeout) - defer cancel() - - res, err := agent.ProvisionServiceAccountRoles(ctx, clientset, agent.ProvisionOptions{ + manifests, err := agent.BuildServiceAccountRoleManifests(agent.ManifestOptions{ Namespace: config.Namespace, ServiceAccountName: config.ServiceAccountName, DiscoverNetwork: config.DiscoverNetwork, @@ -111,13 +146,62 @@ func ProvisionAgentRoles(ctx context.Context, config *AgentRolesConfig) (*AgentR if err != nil { return nil, err } + + runID := config.RunID + if runID == "" { + runID = runid.Generate() + } + dir := defaults.AgentRBACManifestDirPrefix + runID + + // Mkdir, not MkdirAll: an existing directory must fail rather than have + // its contents joined by a second run's files. + if mkErr := os.Mkdir(dir, defaults.AgentRBACManifestDirMode); mkErr != nil { + if stderrors.Is(mkErr, fs.ErrExist) { + return nil, errors.NewWithContext(errors.ErrCodeConflict, + "refusing to overwrite the existing directory "+dir+ + "; move or delete it, or pass a different run ID", + map[string]any{"directory": dir}) + } + return nil, errors.Wrap(errors.ErrCodeInternal, + "failed to create the RBAC manifest directory", mkErr) + } + + objects, writeErr := writeManifestFiles(dir, manifests) + if writeErr != nil { + // A partially written directory would block the retry with the + // ErrCodeConflict above, so remove what this call created. The + // write error is what the caller needs to see, not this one. + if rmErr := os.RemoveAll(dir); rmErr != nil { + slog.Warn("failed to remove the partially written RBAC manifest directory", + "directory", dir, "error", rmErr) + } + return nil, writeErr + } + return &AgentRolesResult{ - Namespace: res.Namespace, - ServiceAccountName: res.ServiceAccountName, - Role: res.Role, - RoleBinding: res.RoleBinding, - ClusterRole: res.ClusterRole, - ClusterRoleBinding: res.ClusterRoleBinding, - DiscoverNetwork: res.DiscoverNetwork, + Dir: dir, + RunID: runID, + Namespace: config.Namespace, + ServiceAccountName: config.ServiceAccountName, + Objects: objects, + DiscoverNetwork: config.DiscoverNetwork, }, nil } + +// writeManifestFiles writes each manifest into dir and returns what it +// wrote. os.WriteFile is used rather than an explicit Create/Write/Close: it +// reports the Close error, which for a writable handle is where a failed +// flush surfaces. +func writeManifestFiles(dir string, manifests []agent.Manifest) ([]AgentRoleObject, error) { + objects := make([]AgentRoleObject, 0, len(manifests)) + for _, m := range manifests { + path := filepath.Join(dir, m.FileName) + if err := os.WriteFile(path, m.Content, defaults.AgentRBACManifestFileMode); err != nil { + return nil, errors.WrapWithContext(errors.ErrCodeInternal, + "failed to write the RBAC manifest", err, + map[string]any{"path": path, "kind": m.Kind}) + } + objects = append(objects, AgentRoleObject{Kind: m.Kind, Name: m.Name, Path: path}) + } + return objects, nil +} diff --git a/pkg/snapshotter/provision_test.go b/pkg/snapshotter/provision_test.go index 1982b13db..f18c58b7a 100644 --- a/pkg/snapshotter/provision_test.go +++ b/pkg/snapshotter/provision_test.go @@ -15,51 +15,310 @@ package snapshotter import ( - "context" stderrors "errors" + "os" + "path/filepath" + "sort" + "strings" "testing" "github.com/NVIDIA/aicr/pkg/errors" + rbacv1 "k8s.io/api/rbac/v1" + "sigs.k8s.io/yaml" ) -// TestProvisionAgentRoles_RejectsBeforeClusterAccess covers the fail-before- -// connect contract: every input ProvisionAgentRoles can reject without a -// cluster must be rejected before the Kubernetes client is built, so a bad -// value is reported as itself rather than masked by a kubeconfig error on a -// machine with no cluster configured. +const ( + provisionNamespace = "gpu-operator" + provisionSAName = "irsa-snapshotter" + provisionRunID = "20260821-142233-9f3a1c0b7e2d4a55" +) + +// writeRoles renders into a scratch working directory and returns the result. +// It pins RunID so the directory name is deterministic, and t.Chdir keeps the +// output out of the repository — the call writes to the working directory by +// design. +func writeRoles(t *testing.T, config *AgentRolesConfig) *AgentRolesResult { + t.Helper() + t.Chdir(t.TempDir()) + res, err := WriteAgentRoleManifests(config) + if err != nil { + t.Fatalf("WriteAgentRoleManifests() error = %v", err) + } + return res +} + +// defaultRolesConfig is the standard input: a read-only grant with a pinned +// run ID. +func defaultRolesConfig() *AgentRolesConfig { + return &AgentRolesConfig{ + Namespace: provisionNamespace, + ServiceAccountName: provisionSAName, + RunID: provisionRunID, + } +} + +// TestWriteAgentRoleManifests_WritesReviewableDirectory covers the layout the +// documented workflow depends on: a `snapshot-rbac-` directory holding +// one parseable object per file, which is what makes both +// `kubectl apply -f /` and the `kubectl delete -f /` teardown work. +func TestWriteAgentRoleManifests_WritesReviewableDirectory(t *testing.T) { + res := writeRoles(t, defaultRolesConfig()) + + wantDir := "snapshot-rbac-" + provisionRunID + if res.Dir != wantDir { + t.Errorf("Dir = %q, want %q", res.Dir, wantDir) + } + if res.RunID != provisionRunID { + t.Errorf("RunID = %q, want %q", res.RunID, provisionRunID) + } + + entries, err := os.ReadDir(res.Dir) + if err != nil { + t.Fatalf("reading %s: %v", res.Dir, err) + } + got := make([]string, 0, len(entries)) + for _, e := range entries { + got = append(got, e.Name()) + } + sort.Strings(got) + want := []string{"01-role.yaml", "02-rolebinding.yaml", "03-clusterrole.yaml", "04-clusterrolebinding.yaml"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Errorf("directory contents = %v, want %v", got, want) + } + + if len(res.Objects) != len(want) { + t.Fatalf("Objects = %d, want %d", len(res.Objects), len(want)) + } + wantKinds := []string{"Role", "RoleBinding", "ClusterRole", "ClusterRoleBinding"} + for i, obj := range res.Objects { + if obj.Kind != wantKinds[i] { + t.Errorf("Objects[%d].Kind = %q, want %q", i, obj.Kind, wantKinds[i]) + } + if obj.Path != filepath.Join(wantDir, want[i]) { + t.Errorf("Objects[%d].Path = %q, want %q", i, obj.Path, filepath.Join(wantDir, want[i])) + } + body, readErr := os.ReadFile(obj.Path) + if readErr != nil { + t.Fatalf("reading %s: %v", obj.Path, readErr) + } + if !strings.HasPrefix(string(body), "# ") { + t.Errorf("%s does not open with a YAML comment header", obj.Path) + } + var parsed map[string]any + if unmarshalErr := yaml.Unmarshal(body, &parsed); unmarshalErr != nil { + t.Errorf("%s is not parseable YAML: %v", obj.Path, unmarshalErr) + } + if parsed["kind"] != obj.Kind { + t.Errorf("%s kind = %v, want %q", obj.Path, parsed["kind"], obj.Kind) + } + if parsed["metadata"] == nil { + t.Errorf("%s has no metadata; the comment header may have swallowed the object", obj.Path) + } + } +} + +// TestWriteAgentRoleManifests_DiscoverNetwork asserts the flag is what decides +// whether the written ClusterRole carries the mutating discovery rules, and +// that the plain form carries none of them. +func TestWriteAgentRoleManifests_DiscoverNetwork(t *testing.T) { + tests := []struct { + name string + discoverNetwork bool + wantMutating bool + }{ + {name: "plain grant stays read-only", discoverNetwork: false}, + {name: "discovery grant adds the mutating rules", discoverNetwork: true, wantMutating: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + config := defaultRolesConfig() + config.DiscoverNetwork = tt.discoverNetwork + res := writeRoles(t, config) + + if res.DiscoverNetwork != tt.discoverNetwork { + t.Errorf("DiscoverNetwork = %v, want %v", res.DiscoverNetwork, tt.discoverNetwork) + } + + body, err := os.ReadFile(filepath.Join(res.Dir, "03-clusterrole.yaml")) + if err != nil { + t.Fatalf("reading the ClusterRole: %v", err) + } + cr := &rbacv1.ClusterRole{} + if unmarshalErr := yaml.Unmarshal(body, cr); unmarshalErr != nil { + t.Fatalf("unmarshalling the ClusterRole: %v", unmarshalErr) + } + + mutating := map[string]string{"nodes": "patch", "pods/exec": "create", "customresourcedefinitions": "create"} + for resource, verb := range mutating { + if got := clusterRoleGrants(cr, resource, verb); got != tt.wantMutating { + t.Errorf("%s: %s = %v, want %v", resource, verb, got, tt.wantMutating) + } + } + + // The header must explain the mutating rules, not merely carry + // them: reading the file is how an operator consents to them. + if strings.Contains(string(body), "nodes: patch") != tt.wantMutating { + t.Errorf("header explains nodes: patch = %v, want %v", !tt.wantMutating, tt.wantMutating) + } + }) + } +} + +// clusterRoleGrants reports whether cr grants verb on resource. +func clusterRoleGrants(cr *rbacv1.ClusterRole, resource, verb string) bool { + for _, rule := range cr.Rules { + for _, res := range rule.Resources { + if res != resource { + continue + } + for _, v := range rule.Verbs { + if v == verb { + return true + } + } + } + } + return false +} + +// TestWriteAgentRoleManifests_ExistingDirectory asserts a collision fails with +// a structured conflict instead of overwriting. The manifests an operator is +// midway through reviewing are exactly what must not change under them. +func TestWriteAgentRoleManifests_ExistingDirectory(t *testing.T) { + t.Chdir(t.TempDir()) + dir := "snapshot-rbac-" + provisionRunID + if mkErr := os.Mkdir(dir, 0o700); mkErr != nil { + t.Fatalf("seeding the directory: %v", mkErr) + } + sentinel := filepath.Join(dir, "01-role.yaml") + if seedErr := os.WriteFile(sentinel, []byte("# operator is reading this\n"), 0o600); seedErr != nil { + t.Fatalf("seeding a file: %v", seedErr) + } + + res, err := WriteAgentRoleManifests(defaultRolesConfig()) + if err == nil { + t.Fatal("WriteAgentRoleManifests() error = nil, want ErrCodeConflict") + } + if res != nil { + t.Errorf("result = %+v, want nil", res) + } + if !stderrors.Is(err, errors.New(errors.ErrCodeConflict, "")) { + t.Errorf("error = %v, want code %s", err, errors.ErrCodeConflict) + } + if !strings.Contains(err.Error(), dir) { + t.Errorf("error = %q, want it to name the directory %q", err.Error(), dir) + } + + body, readErr := os.ReadFile(sentinel) + if readErr != nil { + t.Fatalf("reading the seeded file: %v", readErr) + } + if string(body) != "# operator is reading this\n" { + t.Errorf("the existing file was overwritten; got %q", body) + } +} + +// TestWriteAgentRoleManifests_GeneratesRunIDWhenUnset covers the normal path, +// where the run ID comes from the shared generator rather than the caller. +func TestWriteAgentRoleManifests_GeneratesRunIDWhenUnset(t *testing.T) { + config := defaultRolesConfig() + config.RunID = "" + res := writeRoles(t, config) + + if res.RunID == "" { + t.Fatal("RunID = \"\", want a generated run ID") + } + if !strings.HasPrefix(res.Dir, "snapshot-rbac-") { + t.Errorf("Dir = %q, want the snapshot-rbac- prefix", res.Dir) + } + if res.Dir != "snapshot-rbac-"+res.RunID { + t.Errorf("Dir = %q, want it built from RunID %q", res.Dir, res.RunID) + } + if _, err := os.Stat(res.Dir); err != nil { + t.Errorf("generated directory %q not created: %v", res.Dir, err) + } +} + +// TestWriteAgentRoleManifests_NoClusterAccess is the load-bearing test for +// this path: it must work with no kubeconfig and no cluster at all. KUBECONFIG +// is pointed at a file that cannot yield a client and the in-cluster +// environment is cleared, so any attempt to build a clientset or read the +// ServiceAccount would fail the call rather than silently pass. // -// The cluster-side behavior (existence check, naming, idempotent -// create-or-update) is covered against a fake clientset in pkg/k8s/agent. -func TestProvisionAgentRoles_RejectsBeforeClusterAccess(t *testing.T) { +// It also pins the deliberate simplification: a ServiceAccount that does not +// exist is no longer an ErrCodeNotFound. Nothing is consulted that could +// know, and the operator reviews the manifests before applying them. +func TestWriteAgentRoleManifests_NoClusterAccess(t *testing.T) { + unreachable := filepath.Join(t.TempDir(), "not-a-kubeconfig") + if seedErr := os.WriteFile(unreachable, []byte("this is not a kubeconfig\n"), 0o600); seedErr != nil { + t.Fatalf("seeding the kubeconfig: %v", seedErr) + } + t.Setenv("KUBECONFIG", unreachable) + t.Setenv("HOME", t.TempDir()) + t.Setenv("KUBERNETES_SERVICE_HOST", "") + t.Setenv("KUBERNETES_SERVICE_PORT", "") + + config := defaultRolesConfig() + config.ServiceAccountName = "no-such-serviceaccount" + res := writeRoles(t, config) + + if len(res.Objects) != 4 { + t.Fatalf("Objects = %d, want 4 written without any cluster access", len(res.Objects)) + } + body, err := os.ReadFile(res.Objects[1].Path) + if err != nil { + t.Fatalf("reading the RoleBinding: %v", err) + } + if !strings.Contains(string(body), "no-such-serviceaccount") { + t.Errorf("RoleBinding does not name the unverified ServiceAccount:\n%s", body) + } +} + +// TestWriteAgentRoleManifests_Rejections covers every input refused before +// anything reaches the filesystem, so a bad value never leaves a directory +// behind that would block the corrected retry. +func TestWriteAgentRoleManifests_Rejections(t *testing.T) { tests := []struct { name string config *AgentRolesConfig }{ {name: "nil config", config: nil}, - {name: "empty namespace", config: &AgentRolesConfig{ServiceAccountName: "irsa-snapshotter"}}, - {name: "whitespace namespace", config: &AgentRolesConfig{Namespace: " ", ServiceAccountName: "irsa-snapshotter"}}, - {name: "empty ServiceAccount name", config: &AgentRolesConfig{Namespace: "gpu-operator"}}, - {name: "whitespace ServiceAccount name", config: &AgentRolesConfig{Namespace: "gpu-operator", ServiceAccountName: " "}}, + {name: "empty namespace", config: &AgentRolesConfig{ServiceAccountName: provisionSAName}}, + {name: "whitespace namespace", config: &AgentRolesConfig{Namespace: " ", ServiceAccountName: provisionSAName}}, + {name: "empty ServiceAccount name", config: &AgentRolesConfig{Namespace: provisionNamespace}}, + {name: "whitespace ServiceAccount name", config: &AgentRolesConfig{Namespace: provisionNamespace, ServiceAccountName: " "}}, + { + name: "name too long to compose", + config: &AgentRolesConfig{ + Namespace: provisionNamespace, + ServiceAccountName: strings.Repeat("a", 250), + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - // A kubeconfig path that cannot resolve: if the rejection ever - // moved after client construction, this test would start - // failing on the wrong error instead of passing silently. - if tt.config != nil { - tt.config.Kubeconfig = "/nonexistent/kubeconfig-that-must-not-be-read" - } + dir := t.TempDir() + t.Chdir(dir) - res, err := ProvisionAgentRoles(context.Background(), tt.config) + res, err := WriteAgentRoleManifests(tt.config) if err == nil { - t.Fatal("ProvisionAgentRoles() error = nil, want ErrCodeInvalidRequest") + t.Fatal("WriteAgentRoleManifests() error = nil, want ErrCodeInvalidRequest") + } + if !stderrors.Is(err, errors.New(errors.ErrCodeInvalidRequest, "")) { + t.Errorf("error = %v, want code %s", err, errors.ErrCodeInvalidRequest) } if res != nil { t.Errorf("result = %+v, want nil", res) } - if !stderrors.Is(err, errors.New(errors.ErrCodeInvalidRequest, "")) { - t.Errorf("error = %v, want ErrCodeInvalidRequest", err) + + entries, readErr := os.ReadDir(dir) + if readErr != nil { + t.Fatalf("reading the working directory: %v", readErr) + } + if len(entries) != 0 { + t.Errorf("working directory has %d entries, want 0 (a rejected call must write nothing)", len(entries)) } }) } diff --git a/tools/cleanup b/tools/cleanup index eae80dcca..6ff0f987a 100755 --- a/tools/cleanup +++ b/tools/cleanup @@ -380,13 +380,14 @@ fi # runID suffix; we delete by label selector to catch all runs. Legacy resources # from the older job pattern lived in gpu-operator under the name "aicr". msg "Phase 2: Removing AICR validator artifacts..." -# The component!=agent-rbac term spares the PERMANENT cluster RBAC that -# `aicr snapshot --add-roles-to-service-account` grants to an operator-supplied -# ServiceAccount. Those objects belong to no run — they carry no run-ID label -# and no run's cleanup deletes them — and re-creating them needs the admin who -# provisioned them, so a teardown tool must not sweep them away with the -# per-run leftovers it exists to reclaim. Delete them by hand when the -# ServiceAccount is retired. +# The component!=agent-rbac term spares the cluster RBAC an operator applied +# from the manifests `aicr snapshot --add-roles-to-service-account` wrote for +# an operator-supplied ServiceAccount. Those objects belong to no run — they +# carry no run-ID label and no run's cleanup deletes them — and restoring them +# needs the admin who applied them, so a teardown tool must not sweep them away +# with the per-run leftovers it exists to reclaim. Revoke them deliberately +# with `kubectl delete -f snapshot-rbac-/` when the ServiceAccount is +# retired. kc delete clusterrolebinding -l 'app.kubernetes.io/managed-by=aicr,app.kubernetes.io/component!=agent-rbac' --ignore-not-found kc delete clusterrole -l 'app.kubernetes.io/managed-by=aicr,app.kubernetes.io/component!=agent-rbac' --ignore-not-found kc delete clusterrolebinding -l app.kubernetes.io/name=aicr-validator --ignore-not-found From 8609496045875fe0b3d3f94234727eb4531b3aa3 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Tue, 25 Aug 2026 16:16:10 -0700 Subject: [PATCH 49/56] feat(agent): verify caller and ServiceAccount permissions at the gate CheckPermissions becomes the authoritative pre-flight for the whole run: it verifies every permission the run will actually exercise, for the caller AND for the ServiceAccount the agent pod runs as, and fails before any write. Closes two holes reviewers found independently: A. resolveServiceAccount downgraded a Forbidden ServiceAccount Get to a debug line and continued in prefix mode, so an operator naming their IRSA or Workload Identity ServiceAccount silently ran under a generated one with none of its cloud annotations. `serviceaccounts: get` is now required, and the Forbidden branch fails closed with ErrCodeUnauthorized. B. The gate demanded `create` but never `delete` on the five RBAC kinds. The deferred Cleanup always runs, so a create-but-not-delete identity passed a green pre-flight and leaked a full run-scoped RBAC set, cluster-scoped objects included, once per run. The verb set is mode-aware. Prefix mode requires create AND delete on serviceaccounts, roles, rolebindings, clusterroles and clusterrolebindings; exact-ServiceAccount mode requires none of them, because aicr creates and deletes no RBAC there. Resolving the mode needs a read-only Get, so the gate runs check -> resolve -> mode-specific checks; every step before it closes is a read, and Deploy's Step 1.5 resolution is now redundant and removed. In exact mode the ServiceAccount's own rules are verified with SubjectAccessReview naming system:serviceaccount::, derived from namespacedRules/clusterRules so the gate cannot drift from what the agent needs. A caller that may not create a SubjectAccessReview is told so and the run continues; it is never silently skipped. Reviews now carry the correct API group (roles and clusterroles were being asked about in the core group), every failure is reported in one error naming verb, resource, scope and subject, and Deploy propagates that error as-is instead of burying it behind a generic message. Signed-off-by: Alex Yuskauskas --- docs/user/agent-deployment.md | 110 +++- docs/user/cli-reference.md | 2 +- pkg/defaults/k8s.go | 12 + pkg/k8s/agent/consts.go | 25 +- pkg/k8s/agent/deployer.go | 32 +- pkg/k8s/agent/permissions.go | 685 ++++++++++++++++++++---- pkg/k8s/agent/permissions_test.go | 862 +++++++++++++++++++++++++----- pkg/k8s/agent/rbac.go | 50 +- pkg/k8s/agent/rbac_test.go | 106 ++-- 9 files changed, 1579 insertions(+), 305 deletions(-) diff --git a/docs/user/agent-deployment.md b/docs/user/agent-deployment.md index 3b30dda09..ae6c836e8 100644 --- a/docs/user/agent-deployment.md +++ b/docs/user/agent-deployment.md @@ -66,7 +66,13 @@ data: - Kubernetes cluster with GPU nodes - aicr CLI installed - GPU Operator installed (or appropriate namespace configured via `--namespace`) -- Cluster admin permissions (for RBAC setup) +- Permission to create and delete the run's Job and RBAC in the target + namespace, plus the cluster-scoped `ClusterRole`/`ClusterRoleBinding` the + agent needs. Every run starts by verifying this and stops before touching + the cluster if anything is missing — see + [Pre-flight permission gate](#pre-flight-permission-gate). Pointing + `--service-account-name` at a ServiceAccount you provisioned yourself + requires **no** RBAC permissions at all ## Quick Start @@ -558,7 +564,13 @@ kubectl logs -n gpu-operator -l app.kubernetes.io/name=aicr,app.kubernetes.io/co ### Permission Denied -Ensure RBAC is correctly deployed: +A run that stops with `missing required permissions` never reached the cluster +— read the list it printed, which names every missing verb, its scope, and +whether you or the agent ServiceAccount lacked it. See +[Pre-flight permission gate](#pre-flight-permission-gate) for the full matrix +and for why exact-ServiceAccount mode needs fewer of them. + +If the run got past the gate, verify the RBAC it deployed: ```shell # Verify ClusterRole (run-scoped: "aicr-node-reader-") @@ -635,6 +647,100 @@ than read access: Use `--discover-network` only against clusters where this mutation and the broader RBAC grant are acceptable. +### Pre-flight permission gate + +Every run begins by verifying the permissions it will actually use and stops +before writing anything if any are missing. The gate is read-only: an access +review is an authorization query that persists nothing, and the one object it +reads — the ServiceAccount named by `--service-account-name` — is read with +`get`. A failed gate therefore leaves the cluster exactly as it found it. + +All failures are reported together, so one run tells you everything to fix. +Each line names the verb, the resource, the scope, and which of the two +identities lacked it: + +```text +missing required permissions: + - the caller (your kubeconfig identity) cannot "delete" clusterroles.rbac.authorization.k8s.io (cluster-scoped) + - agent ServiceAccount "system:serviceaccount:gpu-operator/irsa-snapshotter" cannot "list" nodes (cluster-scoped) +``` + +#### What the caller must be able to do + +Checked with `SelfSubjectAccessReview` against your kubeconfig identity. The +first group is required in both ServiceAccount modes: + +| Resource | Verbs | Scope | Used by | +|---|---|---|---| +| `serviceaccounts` | `get` | namespace | Deciding whether `--service-account-name` names an existing ServiceAccount or is a prefix | +| `jobs.batch` | `create`, `get`, `list`, `watch`, `delete` | namespace | Creating the agent Job, waiting on it, cleaning it up | +| `pods` | `get`, `list`, `watch` | namespace | Finding the agent pod and waiting for it to be ready | +| `pods/log` | `get` | namespace | Streaming the agent's output back to your terminal | +| `configmaps` | `get`, `list` | namespace | Reading the snapshot the agent staged | +| `configmaps` | `delete` | namespace | Only when aicr owns the output ConfigMap (i.e. you did not pass your own `cm://` URI) | + +`serviceaccounts: get` is not optional. Without it the run cannot tell whether +`--service-account-name` names an existing ServiceAccount or is a prefix, and +guessing "prefix" would run the agent under a generated ServiceAccount carrying +none of the named account's IRSA or Workload Identity annotations. The run +stops instead of guessing. + +The second group is required **only in prefix mode** — when the run creates +its own run-scoped ServiceAccount: + +| Resource | Verbs | Scope | +|---|---|---| +| `serviceaccounts` | `create`, `delete` | namespace | +| `roles.rbac.authorization.k8s.io` | `create`, `delete` | namespace | +| `rolebindings.rbac.authorization.k8s.io` | `create`, `delete` | namespace | +| `clusterroles.rbac.authorization.k8s.io` | `create`, `delete` | cluster | +| `clusterrolebindings.rbac.authorization.k8s.io` | `create`, `delete` | cluster | + +`delete` is required alongside `create` because cleanup always runs, including +on the failure path. An identity that can create but not delete would leave a +ServiceAccount, Role, RoleBinding, ClusterRole and ClusterRoleBinding behind on +every single run. + +**Exact-ServiceAccount mode requires fewer caller permissions.** When +`--service-account-name` names a ServiceAccount that already exists, the run +creates and deletes no RBAC at all, so none of the five kinds above is +demanded of you. What it requires instead is that the ServiceAccount was +actually provisioned — see below. + +#### What the agent ServiceAccount must be able to do + +The agent pod runs as a ServiceAccount, not as you, so its permissions are +checked separately with `SubjectAccessReview` naming +`system:serviceaccount::` as the subject. The questions are +derived from the same rule set the run-scoped `Role` and `ClusterRole` grant +(and that `--add-roles-to-service-account` renders), so the gate cannot fall +behind what the agent needs: namespaced `configmaps` and `pods` access, plus +cluster-scoped `nodes`, `pods`, `nvidia.com` ClusterPolicies, the Slinky CRs +and the MariaDB CRs — widened to the full mutating set when +`--discover-network` is passed. + +This check runs **in exact-ServiceAccount mode only**. In prefix mode the +ServiceAccount does not exist yet and the run is about to grant it exactly +those rules, so there is nothing to verify. In exact mode aicr grants nothing, +and the most common failure is that the manifests from +`--add-roles-to-service-account` were rendered but never applied. The gate +catches that up front and tells you how to fix it, rather than letting a Job +start and fail inside the pod minutes later. + +**When the ServiceAccount's permissions cannot be verified.** Creating a +`SubjectAccessReview` is itself a privilege. If you do not hold it, the run +does **not** silently skip the check — it says so and continues, because the +agent will still fail visibly in-pod if a rule is missing: + +```text +WARN could not verify the agent ServiceAccount's own permissions; continuing, +but a missing rule will surface as an in-pod failure minutes from now instead +of here serviceAccount=irsa-snapshotter namespace=gpu-operator +uncheckedRules=18 remedy="grant the caller 'create +subjectaccessreviews.authorization.k8s.io', or verify by hand with: kubectl +auth can-i --list --as system:serviceaccount:gpu-operator/irsa-snapshotter" +``` + ### Pod Security Context The agent requires elevated privileges to collect system configuration from the host: diff --git a/docs/user/cli-reference.md b/docs/user/cli-reference.md index da0af2dac..b9595bc7e 100644 --- a/docs/user/cli-reference.md +++ b/docs/user/cli-reference.md @@ -85,7 +85,7 @@ aicr snapshot [flags] | `--namespace` | `-n` | string | default | Kubernetes namespace for agent deployment | | `--image` | | string | matches CLI version | Container image for agent Job. Release builds default to `ghcr.io/nvidia/aicr:v`; dev and `-next` snapshot builds default to `ghcr.io/nvidia/aicr:latest`. | | `--job-name` | | string | aicr | Prefix for the agent Job name; the run ID is always appended (`-`) | -| `--service-account-name` | | string | aicr | ServiceAccount the agent pod runs as. **Exact-if-exists:** when a ServiceAccount of exactly this name already exists in `--namespace`, it is used verbatim and the run creates **no** ServiceAccount, Role, RoleBinding, ClusterRole, or ClusterRoleBinding — and deletes none at cleanup. Otherwise the value is a name prefix and the run ID is appended (`-`). See [Using an existing ServiceAccount](agent-deployment.md#using-an-existing-serviceaccount-irsa-and-workload-identity) | +| `--service-account-name` | | string | aicr | ServiceAccount the agent pod runs as. **Exact-if-exists:** when a ServiceAccount of exactly this name already exists in `--namespace`, it is used verbatim and the run creates **no** ServiceAccount, Role, RoleBinding, ClusterRole, or ClusterRoleBinding — and deletes none at cleanup. Otherwise the value is a name prefix and the run ID is appended (`-`). Exact-if-exists mode needs **fewer** caller permissions — the run's pre-flight gate stops demanding `create`/`delete` on the five RBAC kinds — but requires the ServiceAccount to already carry the agent's rules, which the gate verifies with a `SubjectAccessReview`. See [Using an existing ServiceAccount](agent-deployment.md#using-an-existing-serviceaccount-irsa-and-workload-identity) and [Pre-flight permission gate](agent-deployment.md#pre-flight-permission-gate) | | `--add-roles-to-service-account` | | string | | **Writes manifests and applies nothing.** Renders the `Role`/`RoleBinding` (`aicr-agent--rbac`) and `ClusterRole`/`ClusterRoleBinding` (`aicr-agent---rbac`) that grant the agent's permissions to the named ServiceAccount into `./snapshot-rbac-/`, one object per file with a comment header explaining what it grants, then exits **without taking a snapshot**. **No cluster is contacted** — no kubeconfig or privileges needed, and the ServiceAccount is not checked for existence. Review the files, then apply with `kubectl apply -f /` and revoke with `kubectl delete -f /` yourself; no run cleanup ever touches them. Fails with `CONFLICT` if the directory already exists. Combine with `--discover-network` to also render the mutating live-discovery rules | | `--node-selector` | | string[] | auto | Node selector for agent scheduling (key=value, repeatable). When omitted (and neither `--require-gpu` nor `--runtime-class` is set), the agent auto-targets GPU nodes labeled `nvidia.com/gpu.present=true` if the cluster has any — see [Agent Deployment](agent-deployment.md). Pass an explicit selector to override. | | `--toleration` | | string[] | all taints | Tolerations for agent scheduling (key=value:effect, repeatable). **Default: all taints tolerated** (uses `operator: Exists`). Only specify to restrict which taints are tolerated. | diff --git a/pkg/defaults/k8s.go b/pkg/defaults/k8s.go index 2c81de63d..cc4c4b95a 100644 --- a/pkg/defaults/k8s.go +++ b/pkg/defaults/k8s.go @@ -43,6 +43,18 @@ const ( ValidatorClientBurst = 100 ) +// K8sAccessReviewConcurrency bounds how many Self/SubjectAccessReview +// requests a pre-flight permission gate keeps in flight at once. +// +// An access review is read-only, so the checks fan out concurrently: N +// sequential reviews cost N round trips against the apiserver while one +// batch costs roughly one. The bound exists because the set is not small — +// the snapshot agent expands the agent ServiceAccount's own PolicyRules into +// one review per (apiGroup, resource, verb), which reaches several dozen on +// a --discover-network run — and opening that many connections at once buys +// nothing over a handful of waves while adding avoidable apiserver load. +const K8sAccessReviewConcurrency = 16 + // MaxK8sNameLength is the maximum length of a Kubernetes object name built // from a run-scoped prefix. 63 characters is the general DNS label ceiling // (RFC 1123) that Kubernetes enforces on object names, but the binding diff --git a/pkg/k8s/agent/consts.go b/pkg/k8s/agent/consts.go index 3706f340f..b8ac2956b 100644 --- a/pkg/k8s/agent/consts.go +++ b/pkg/k8s/agent/consts.go @@ -20,7 +20,30 @@ const ( verbList = "list" verbGet = "get" verbDelete = "delete" - resourceCM = "configmaps" + verbWatch = "watch" + verbUpdate = "update" + verbPatch = "patch" + + // Resource names, as they appear in an RBAC PolicyRule and in the + // ResourceAttributes of an access review. Named so the RBAC this + // package builds and the pre-flight gate that checks for it can never + // spell the same resource two different ways. + resourceCM = "configmaps" + resourceServiceAccounts = "serviceaccounts" + resourceRoles = "roles" + resourceRoleBindings = "rolebindings" + resourceClusterRoles = "clusterroles" + resourceClusterRoleBindings = "clusterrolebindings" + resourceJobs = "jobs" + resourcePods = "pods" + resourceNodes = "nodes" + + // subresourceLog is the pods subresource the CLI reads to stream the + // agent's output back to the operator's terminal. + subresourceLog = "log" + + // batchAPIGroup is the API group the agent Job lives in. + batchAPIGroup = "batch" slinkyAPIGroup = "slinky.slurm.net" slinkyControllerResource = "controllers" diff --git a/pkg/k8s/agent/deployer.go b/pkg/k8s/agent/deployer.go index a63db3c4c..8fe1f54ff 100644 --- a/pkg/k8s/agent/deployer.go +++ b/pkg/k8s/agent/deployer.go @@ -43,14 +43,22 @@ func (d *Deployer) Deploy(ctx context.Context) error { return err } - // Step 0: Check permissions before attempting deployment - _, err := d.CheckPermissions(ctx) - if err != nil { + // Step 0: the authoritative permission gate for the whole run. It + // verifies every permission this run will exercise — for the caller AND + // for the ServiceAccount the agent pod runs as — and resolves which + // ServiceAccount mode the run is in, which is what decides the verb set + // it demands. Everything it does is a read, so nothing below has been + // written yet when it fails. + if _, err := d.CheckPermissions(ctx); err != nil { if aicrerrors.IsNetworkError(err) { return aicrerrors.Wrap(aicrerrors.ErrCodeUnavailable, "cannot reach Kubernetes API server\n\nCheck your network connectivity:\n - Is your VPN connected?\n - Is the cluster endpoint correct in your kubeconfig?\n - Are firewall rules allowing egress to the API server?", err) } - return aicrerrors.Wrap(aicrerrors.ErrCodeUnauthorized, "insufficient permissions to deploy agent\n\nTo deploy the agent, you need cluster admin privileges.\nRun: aicr snapshot", err) + // Propagated as-is: CheckPermissions already returns + // ErrCodeUnauthorized carrying the complete list of what is + // missing, for which subject, and how to fix it. Re-wrapping would + // bury that behind a generic sentence. + return err } // Step 0.5: Validate RuntimeClass exists if configured @@ -65,15 +73,13 @@ func (d *Deployer) Deploy(ctx context.Context) error { return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to ensure namespace", err) } - // Step 1.5: Decide whether Config.ServiceAccountName names a - // ServiceAccount that already exists (use it verbatim, manage none of - // its permissions) or is a prefix for one this run creates and owns. - // It runs after ensureNamespace so the Get is issued against a - // namespace that exists, and before Step 2 because it decides whether - // Step 2 happens at all. - if err := d.resolveServiceAccount(ctx); err != nil { - return err - } + // ServiceAccount mode is already decided at this point: Step 0's gate + // resolves it (resolveServiceAccount) because the permissions it must + // demand differ between the two modes. Resolving there rather than here + // also means the decision is made before ensureNamespace, which is + // harmless — a ServiceAccount cannot exist in a namespace that does + // not, so the Get correctly reports NotFound and the run stays in + // prefix mode. // Step 2: Create this run's RBAC. Every name carries the run ID, so // nothing here can already exist; an AlreadyExists is reported as an diff --git a/pkg/k8s/agent/permissions.go b/pkg/k8s/agent/permissions.go index 538ed6ed2..0c068a01c 100644 --- a/pkg/k8s/agent/permissions.go +++ b/pkg/k8s/agent/permissions.go @@ -17,50 +17,224 @@ package agent import ( "context" "fmt" + "log/slog" "strings" - "sync" + "github.com/NVIDIA/aicr/pkg/defaults" "github.com/NVIDIA/aicr/pkg/errors" + "github.com/NVIDIA/aicr/pkg/k8s/pod" "golang.org/x/sync/errgroup" authv1 "k8s.io/api/authorization/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// permissionCheck represents a single permission check result. +// Labels used when rendering a permission failure for an operator. The +// caller label names the kubeconfig identity running aicr; the +// ServiceAccount label is filled in with the subject's +// "system:serviceaccount::" username. +const ( + callerSubjectLabel = "the caller (your kubeconfig identity)" + + // serviceAccountUserPrefix is the username the apiserver authenticates + // a ServiceAccount as, and therefore the SubjectAccessReview subject + // that answers for the agent pod rather than for the caller. + serviceAccountUserPrefix = "system:serviceaccount:" + + // Virtual groups every ServiceAccount is a member of. RBAC bindings + // routinely target them instead of an individual ServiceAccount, so a + // SubjectAccessReview that omits them under-reports the subject's + // real permissions and would fail a correctly-provisioned operator. + groupServiceAccounts = "system:serviceaccounts" + groupServiceAccountsPrefix = "system:serviceaccounts:" + groupAuthenticated = "system:authenticated" +) + +// Sizing hints for the two slices this file builds. Neither is a bound. +const ( + // rbacKindCount is the number of RBAC kinds a prefix-mode run creates + // and later deletes: ServiceAccount, Role, RoleBinding, ClusterRole, + // ClusterRoleBinding. + rbacKindCount = 5 + + // rulesPerPolicyHint is the average number of (group, resource, verb) + // triples one PolicyRule expands into across namespacedRules and + // clusterRules. + rulesPerPolicyHint = 4 +) + +// accessCheck is one authorization question: may `subject` perform `verb` on +// `group/resource[/subresource]` at `namespace` (empty means cluster scope)? +// +// subject is "" for the caller — answered with a SelfSubjectAccessReview — +// or a "system:serviceaccount::" username for the agent's +// ServiceAccount, answered with a SubjectAccessReview. The struct is +// deliberately all-comparable so dedupeChecks can key a map on it. +type accessCheck struct { + group string + resource string + subresource string + verb string + namespace string + subject string +} + +// permissionCheck is one answered accessCheck, returned to callers so they +// can render the full pre-flight rather than only its failures. type permissionCheck struct { - Resource string - Verb string - Namespace string - Allowed bool - Reason string + Group string + Resource string + Subresource string + Verb string + Namespace string + + // Subject is "" for the caller, or the ServiceAccount's + // "system:serviceaccount::" username. + Subject string + + Allowed bool + Reason string + + // Unverified marks a ServiceAccount check the apiserver refused to + // answer because the caller may not create a SubjectAccessReview. + // Allowed is false on such an entry, but it is NOT counted as a + // missing permission: nothing was learned either way. CheckPermissions + // says so out loud instead of silently dropping the check. + Unverified bool } -// CheckPermissions verifies if the current user has the required permissions -// to deploy the agent. Returns a list of permission checks and an error if any -// required permissions are missing. +// CheckPermissions is the authoritative pre-flight gate for an entire agent +// run. It verifies every permission the run will actually exercise, for both +// identities involved — the caller (the kubeconfig identity running aicr) +// and the ServiceAccount the agent pod runs as — and fails before anything +// is written to the cluster when any of them is missing. +// +// # Ordering, and why it is still fail-before-mutate +// +// The required verb set depends on which ServiceAccount mode this run is in +// (see resolveServiceAccount), and the mode cannot be known without reading +// ServiceAccounts. The gate therefore runs in two phases: +// +// 1. Check the caller permissions every run needs in either mode, +// `serviceaccounts: get` among them. +// 2. Resolve the ServiceAccount (a read-only Get), then check the +// mode-specific set. +// +// Every step up to the point the gate closes is a read: a +// Self/SubjectAccessReview is a non-persisted authorization query, and the +// resolution Get creates nothing. The fail-before-mutate guarantee is about +// not WRITING before validation, and no write is issued until Deploy's +// ensure* chain, which runs only after this returns nil. +// +// # Mode-specific verbs +// +// Prefix mode (aicr creates its own run-scoped ServiceAccount) needs create +// AND delete on all five RBAC kinds: the deferred Cleanup is registered +// before Deploy and always runs, so an identity that can create but not +// delete would pass a green pre-flight and then leak a full run-scoped RBAC +// set — cluster-scoped objects included — on every run. +// +// Exact-ServiceAccount mode creates and deletes no RBAC at all, so demanding +// those verbs would block operators who legitimately hold none. It instead +// verifies that the operator actually provisioned the ServiceAccount, which +// aicr does not do for them. +// +// # Reporting +// +// Every check is evaluated before any failure is reported, so an operator +// fixing permissions gets the complete list in one run. Each failure names +// the verb, the resource, the scope, and which subject lacked it. func (d *Deployer) CheckPermissions(ctx context.Context) ([]permissionCheck, error) { - type permCheck struct { - resource string - verb string - namespace string + // Phase 1: the caller-side set every run needs regardless of mode. + results, err := d.runAccessChecks(ctx, dedupeChecks(d.callerCommonChecks())) + if err != nil { + return nil, err + } + + // `serviceaccounts: get` gates the rest of the pre-flight rather than + // merely joining it. Without it the mode is unknowable, and the + // historical behavior — treat an unreadable ServiceAccount as a + // prefix — silently ran an operator who named their IRSA / Workload + // Identity ServiceAccount under a fresh, un-annotated one instead. + // Fail here with everything phase 1 found, and say why the rest was + // not evaluated. + if !callerMayReadServiceAccounts(results) { + return results, missingPermissionsError(results, unresolvableModeHint(d.config.Namespace)) } - // Required permissions for deployment - requiredChecks := []permCheck{ - // Namespace-scoped resources - {"serviceaccounts", verbCreate, d.config.Namespace}, - {"roles", verbCreate, d.config.Namespace}, - {"rolebindings", verbCreate, d.config.Namespace}, - {"jobs", verbCreate, d.config.Namespace}, - {resourceCM, verbGet, d.config.Namespace}, - {resourceCM, verbList, d.config.Namespace}, + // Read-only: decides which verb set phase 2 demands, and which + // ServiceAccount the agent pod will run as. + if err = d.resolveServiceAccount(ctx); err != nil { + return results, err + } + + modeResults, err := d.runAccessChecks(ctx, dedupeChecks(d.modeSpecificChecks())) + if err != nil { + return nil, err + } + results = append(results, modeResults...) - // Cluster-scoped resources - {"clusterroles", verbCreate, ""}, - {"clusterrolebindings", verbCreate, ""}, + // The meta-permission is handled out loud, never silently: a caller who + // cannot create a SubjectAccessReview learns that the ServiceAccount's + // own permissions went unverified and that the agent will surface any + // gap in-pod instead. + d.warnUnverified(results) - // Cleanup permissions - {"jobs", verbDelete, d.config.Namespace}, + if hasMissing(results) { + return results, missingPermissionsError(results, d.remediationHints(results)) + } + return results, nil +} + +// callerCommonChecks returns the permissions the caller needs in either +// ServiceAccount mode: everything Deploy, WaitForCompletion, GetSnapshot and +// Cleanup issue that is not RBAC creation or deletion. +// +// Deliberately absent: Namespace create/get/patch. ensureNamespace's three +// branches need a different verb depending on whether the namespace already +// exists and is already labeled, and the documented path — an operator's +// pre-existing, previously-labeled namespace such as `gpu-operator` — +// needs none of them. Demanding the cluster-scoped `namespaces: create` +// that only the fresh-namespace branch uses would fail the majority case. +// +// Also absent: `get` on Role, RoleBinding, ClusterRole and ClusterRoleBinding. +// Cleanup reads those only to re-establish ownership of an entry whose +// Create response was lost (resolveIntentUID), and that path already fails +// closed with a warning rather than a blind delete, so a missing `get` +// degrades to a logged orphan instead of a wrong deletion. `serviceaccounts: +// get` is required anyway, for the mode resolution above. +func (d *Deployer) callerCommonChecks() []accessCheck { + ns := d.config.Namespace + checks := []accessCheck{ + // Mode resolution (resolveServiceAccount) and Cleanup's + // ownership re-check (resolveIntentUID). + {resource: resourceServiceAccounts, verb: verbGet, namespace: ns}, + + // ensureJob, waitForJobCompletion (Get + Watch, List on watch + // resume) and Cleanup's UID-pinned delete. + {group: batchAPIGroup, resource: resourceJobs, verb: verbCreate, namespace: ns}, + {group: batchAPIGroup, resource: resourceJobs, verb: verbGet, namespace: ns}, + {group: batchAPIGroup, resource: resourceJobs, verb: verbList, namespace: ns}, + {group: batchAPIGroup, resource: resourceJobs, verb: verbWatch, namespace: ns}, + {group: batchAPIGroup, resource: resourceJobs, verb: verbDelete, namespace: ns}, + + // Pod discovery (findPodName / findOrWatchPodName), readiness + // waiting, and log streaming back to the operator's terminal. + {resource: resourcePods, verb: verbGet, namespace: ns}, + {resource: resourcePods, verb: verbList, namespace: ns}, + {resource: resourcePods, verb: verbWatch, namespace: ns}, + {resource: resourcePods, subresource: subresourceLog, verb: verbGet, namespace: ns}, + + // Reading the snapshot the agent staged. + {resource: resourceCM, verb: verbGet, namespace: ns}, + {resource: resourceCM, verb: verbList, namespace: ns}, + } + + // A caller-supplied `cm:///` Output is read + // where it points, not in the agent's namespace. + if outputNS, _, parseErr := pod.ParseConfigMapURI(d.config.Output); parseErr == nil && outputNS != "" { + checks = append(checks, accessCheck{resource: resourceCM, verb: verbGet, namespace: outputNS}) } // `configmaps: delete` is required only when this Deployer owns the @@ -68,104 +242,409 @@ func (d *Deployer) CheckPermissions(ctx context.Context) ([]permissionCheck, err // getSnapshotFromConfigMap's created-set record are both gated on // Config.OwnsOutputConfigMap, so a caller who supplied their own // `cm://` output URI never has a ConfigMap deleted on their behalf. - // CheckPermissions fails closed — a denied check makes Deploy return - // ErrCodeUnauthorized at Step 0 — so demanding an unconditional - // delete grant would block deployment for identities that are - // perfectly capable of the run they actually asked for. Gate it the - // same way ensureClusterRole gates its DiscoverNetwork rules. + // This gate fails closed, so demanding an unconditional delete grant + // would block deployment for identities perfectly capable of the run + // they actually asked for. if d.config.OwnsOutputConfigMap { - requiredChecks = append(requiredChecks, permCheck{resourceCM, verbDelete, d.config.Namespace}) - } - - // The RBAC create verbs above stay unconditional even though - // exact-ServiceAccount mode creates none of those objects - // (resolveServiceAccount). Narrowing them would mean resolving the - // ServiceAccount before this pre-flight, and this pre-flight is - // deliberately Deploy's first cluster call — the one thing that runs - // before any write. The cost of leaving them is that an operator using - // an existing ServiceAccount still needs the same grants they needed - // before, which is no regression; the cost of moving them would be a - // weaker fail-before-mutate guarantee. - - // SelfSubjectAccessReview is a read-only query; running the N required - // checks in parallel cuts wall-clock from N×RTT to one RTT against the - // apiserver. Each iteration's index is preserved so the response slice - // keeps a deterministic order regardless of completion timing. - results := make([]permissionCheck, len(requiredChecks)) + checks = append(checks, accessCheck{resource: resourceCM, verb: verbDelete, namespace: ns}) + } + return checks +} + +// modeSpecificChecks returns the half of the gate that depends on which +// ServiceAccount mode resolveServiceAccount settled on. +// +// Prefix mode: the caller creates and later deletes the full run-scoped RBAC +// set, so both verbs are required on all five kinds. +// +// Exact-ServiceAccount mode: aicr creates and deletes nothing, so none of +// those verbs is required of the caller. What IS required is that the +// operator already granted the agent's rules to the ServiceAccount they +// named — `aicr snapshot --add-roles-to-service-account` renders those +// manifests but applies nothing, so "you never applied them" is the +// failure this catches, at the gate rather than minutes later inside a pod. +func (d *Deployer) modeSpecificChecks() []accessCheck { + if !d.managesRBAC() { + return d.serviceAccountChecks(serviceAccountUsername(d.config.Namespace, d.existingServiceAccount())) + } + + ns := d.config.Namespace + verbs := []string{verbCreate, verbDelete} + checks := make([]accessCheck, 0, len(verbs)*rbacKindCount) + for _, verb := range verbs { + checks = append(checks, + accessCheck{resource: resourceServiceAccounts, verb: verb, namespace: ns}, + accessCheck{group: rbacAPIGroup, resource: resourceRoles, verb: verb, namespace: ns}, + accessCheck{group: rbacAPIGroup, resource: resourceRoleBindings, verb: verb, namespace: ns}, + accessCheck{group: rbacAPIGroup, resource: resourceClusterRoles, verb: verb}, + accessCheck{group: rbacAPIGroup, resource: resourceClusterRoleBindings, verb: verb}, + ) + } + return checks +} + +// serviceAccountChecks returns the questions asked of the agent's own +// ServiceAccount: exactly the rules ensureRole and ensureClusterRole grant +// in prefix mode, expanded to one check per (group, resource, verb). +// +// Deriving them from namespacedRules and clusterRules — the same two +// definitions the run-scoped Role and ClusterRole are built from, and the +// same ones BuildServiceAccountRoleManifests renders — is what keeps the +// gate from drifting away from what the agent actually needs. +// +// Issued only in exact-ServiceAccount mode. In prefix mode the +// ServiceAccount does not exist yet and its RBAC has not been created, so +// every answer would be a truthful "denied" about an identity that is about +// to be granted those very rules. +func (d *Deployer) serviceAccountChecks(subject string) []accessCheck { + checks := checksFromRules(namespacedRules(), d.config.Namespace, subject) + return append(checks, checksFromRules(clusterRules(d.config.DiscoverNetwork), "", subject)...) +} + +// checksFromRules expands PolicyRules into one accessCheck per +// (apiGroup, resource, verb) triple at the given scope. A "resource/sub" +// entry such as "pods/exec" is split so the review carries Subresource, +// which is how the apiserver evaluates it. +func checksFromRules(rules []rbacv1.PolicyRule, namespace, subject string) []accessCheck { + // Capacity is a hint, not a bound: most rules name one API group and a + // handful of verbs, so this lands close without a counting pass. + checks := make([]accessCheck, 0, len(rules)*rulesPerPolicyHint) + for _, rule := range rules { + for _, group := range rule.APIGroups { + for _, res := range rule.Resources { + resource, subresource, _ := strings.Cut(res, "/") + for _, verb := range rule.Verbs { + checks = append(checks, accessCheck{ + group: group, + resource: resource, + subresource: subresource, + verb: verb, + namespace: namespace, + subject: subject, + }) + } + } + } + } + return checks +} + +// dedupeChecks drops exact duplicates while preserving first-seen order, so +// the same question is never asked of the apiserver twice and never reported +// twice. Near-duplicates that differ only in scope (namespaced `pods: list` +// vs cluster-wide `pods: list`) are deliberately kept: an operator who +// applied the Role but not the ClusterRole fails exactly one of them. +func dedupeChecks(checks []accessCheck) []accessCheck { + seen := make(map[accessCheck]struct{}, len(checks)) + out := make([]accessCheck, 0, len(checks)) + for _, c := range checks { + if _, dup := seen[c]; dup { + continue + } + seen[c] = struct{}{} + out = append(out, c) + } + return out +} + +// runAccessChecks answers every check and returns the results in input +// order. An access review is a read-only query, so they fan out +// concurrently: N sequential reviews cost N round trips, one batch costs +// one. Concurrency is bounded because the expanded rule set can reach +// several dozen checks on a --discover-network run and there is no reason +// to open that many connections at once. +// +// A ServiceAccount check the apiserver declines to answer is recorded as +// Unverified rather than failing the run; see CheckPermissions. +func (d *Deployer) runAccessChecks(ctx context.Context, checks []accessCheck) ([]permissionCheck, error) { + results := make([]permissionCheck, len(checks)) g, gctx := errgroup.WithContext(ctx) - var mu sync.Mutex - var firstErr error - for i := range requiredChecks { - check := requiredChecks[i] + g.SetLimit(defaults.K8sAccessReviewConcurrency) + for i := range checks { + check := checks[i] g.Go(func() error { - allowed, reason, err := d.checkPermission(gctx, check.resource, check.verb, check.namespace) - if err != nil { + allowed, reason, reviewErr := d.reviewAccess(gctx, check) + if reviewErr != nil { + if check.subject != "" && isUnanswerable(reviewErr) { + results[i] = check.result(false, reviewErr.Error()) + results[i].Unverified = true + return nil + } code := errors.ErrCodeInternal - if errors.IsNetworkError(err) { + if errors.IsNetworkError(reviewErr) { code = errors.ErrCodeUnavailable } - mu.Lock() - if firstErr == nil { - firstErr = errors.Wrap(code, fmt.Sprintf("failed to check permission for %s %s", check.verb, check.resource), err) - } - mu.Unlock() - return err - } - results[i] = permissionCheck{ - Resource: check.resource, - Verb: check.verb, - Namespace: check.namespace, - Allowed: allowed, - Reason: reason, + return errors.Wrap(code, + fmt.Sprintf("failed to check whether %s may %q %s (%s)", + check.subjectLabel(), check.verb, check.resourceLabel(), check.scopeLabel()), + reviewErr) } + results[i] = check.result(allowed, reason) return nil }) } if err := g.Wait(); err != nil { - if firstErr != nil { - return nil, firstErr + return nil, err + } + return results, nil +} + +// reviewAccess asks the apiserver one authorization question. +// +// A caller check uses SelfSubjectAccessReview, which answers only for the +// identity in the kubeconfig. A ServiceAccount check must use +// SubjectAccessReview with that ServiceAccount as the subject — the agent +// pod runs as it, and a SelfSubjectAccessReview cannot answer for anyone but +// the caller. +// +// The apiserver error is returned unwrapped so the caller can classify it +// (Forbidden / not served / network) before deciding whether it is fatal. +func (d *Deployer) reviewAccess(ctx context.Context, check accessCheck) (bool, string, error) { + attrs := &authv1.ResourceAttributes{ + Verb: check.verb, + Group: check.group, + Resource: check.resource, + Subresource: check.subresource, + Namespace: check.namespace, + } + + if check.subject == "" { + review := &authv1.SelfSubjectAccessReview{ + Spec: authv1.SelfSubjectAccessReviewSpec{ResourceAttributes: attrs}, } - return nil, errors.Wrap(errors.ErrCodeInternal, "permission check failed", err) + result, err := d.clientset.AuthorizationV1().SelfSubjectAccessReviews().Create(ctx, review, metav1.CreateOptions{}) + if err != nil { + return false, "", err + } + return result.Status.Allowed, result.Status.Reason, nil } - checks := make([]permissionCheck, 0, len(results)) - checks = append(checks, results...) - var missingPermissions []string - for _, result := range results { - if !result.Allowed { - scope := "cluster-scoped" - if result.Namespace != "" { - scope = fmt.Sprintf("namespace %q", result.Namespace) - } - missingPermissions = append(missingPermissions, - fmt.Sprintf("%s %s (%s)", result.Verb, result.Resource, scope)) + review := &authv1.SubjectAccessReview{ + Spec: authv1.SubjectAccessReviewSpec{ + ResourceAttributes: attrs, + User: check.subject, + Groups: serviceAccountGroups(d.config.Namespace), + }, + } + result, err := d.clientset.AuthorizationV1().SubjectAccessReviews().Create(ctx, review, metav1.CreateOptions{}) + if err != nil { + return false, "", err + } + return result.Status.Allowed, result.Status.Reason, nil +} + +// isUnanswerable reports whether err means the apiserver would not answer a +// SubjectAccessReview at all, as opposed to answering "denied". +// +// Creating a SubjectAccessReview is itself a privilege, and some clusters do +// not serve the endpoint. Neither says anything about the ServiceAccount's +// permissions, so neither may be read as a pass OR as a failure — the run +// continues with the gap reported (see warnUnverified). +func isUnanswerable(err error) bool { + return apierrors.IsForbidden(err) || + apierrors.IsUnauthorized(err) || + apierrors.IsNotFound(err) || + apierrors.IsMethodNotSupported(err) +} + +// serviceAccountUsername returns the username the apiserver authenticates a +// ServiceAccount as, which is the SubjectAccessReview subject that answers +// for the agent pod. +func serviceAccountUsername(namespace, name string) string { + return serviceAccountUserPrefix + namespace + ":" + name +} + +// serviceAccountGroups returns the virtual groups every ServiceAccount in +// namespace belongs to, so a SubjectAccessReview reflects grants made to +// those groups rather than only to the individual subject. +func serviceAccountGroups(namespace string) []string { + return []string{groupServiceAccounts, groupServiceAccountsPrefix + namespace, groupAuthenticated} +} + +// result converts an answered check into the reportable form. +func (c accessCheck) result(allowed bool, reason string) permissionCheck { + return permissionCheck{ + Group: c.group, + Resource: c.resource, + Subresource: c.subresource, + Verb: c.verb, + Namespace: c.namespace, + Subject: c.subject, + Allowed: allowed, + Reason: reason, + } +} + +// subjectLabel names whose permissions this check answers for. +func (c accessCheck) subjectLabel() string { + return subjectLabel(c.subject) +} + +// resourceLabel renders "[/][.]", the spelling +// an operator would use in a PolicyRule. +func (c accessCheck) resourceLabel() string { + return resourceLabel(c.resource, c.subresource, c.group) +} + +// scopeLabel renders the scope the check was evaluated at. +func (c accessCheck) scopeLabel() string { + return scopeLabel(c.namespace) +} + +func subjectLabel(subject string) string { + if subject == "" { + return callerSubjectLabel + } + return fmt.Sprintf("agent ServiceAccount %q", subject) +} + +func resourceLabel(resource, subresource, group string) string { + name := resource + if subresource != "" { + name += "/" + subresource + } + if group != "" { + name += "." + group + } + return name +} + +func scopeLabel(namespace string) string { + if namespace == "" { + return "cluster-scoped" + } + return fmt.Sprintf("namespace %q", namespace) +} + +// callerMayReadServiceAccounts reports whether the caller's +// `serviceaccounts: get` check came back allowed. It is the one check that +// gates the rest of the pre-flight rather than merely joining it: without it +// the ServiceAccount mode is unknowable. +func callerMayReadServiceAccounts(results []permissionCheck) bool { + for _, r := range results { + if r.Subject == "" && r.Resource == resourceServiceAccounts && r.Verb == verbGet && r.Allowed { + return true + } + } + return false +} + +// hasMissing reports whether any check came back denied. An Unverified entry +// does not count: nothing was learned about it either way. +func hasMissing(results []permissionCheck) bool { + for _, r := range results { + if !r.Allowed && !r.Unverified { + return true } } + return false +} - if len(missingPermissions) > 0 { - return checks, errors.New(errors.ErrCodeUnauthorized, fmt.Sprintf("missing required permissions:\n - %s", - strings.Join(missingPermissions, "\n - "))) +// missingPermissionsError renders every denied check as one actionable line +// — subject, verb, resource, scope, and the authorizer's reason when it gave +// one — followed by hint. Reporting all of them together is the point: an +// operator fixing permissions should need one run, not one run per verb. +func missingPermissionsError(results []permissionCheck, hint string) error { + var missing []string + for _, r := range results { + if r.Allowed || r.Unverified { + continue + } + line := fmt.Sprintf("%s cannot %q %s (%s)", + subjectLabel(r.Subject), r.Verb, + resourceLabel(r.Resource, r.Subresource, r.Group), scopeLabel(r.Namespace)) + if r.Reason != "" { + line += ": " + r.Reason + } + missing = append(missing, line) } + msg := fmt.Sprintf("missing required permissions:\n - %s", strings.Join(missing, "\n - ")) + if hint != "" { + msg += "\n\n" + hint + } + return errors.New(errors.ErrCodeUnauthorized, msg) +} - return checks, nil +// unresolvableModeHint explains why the pre-flight stopped early when the +// caller cannot read ServiceAccounts, and why that is not something aicr +// works around. +func unresolvableModeHint(namespace string) string { + return fmt.Sprintf( + "Reading ServiceAccounts in namespace %q is required before anything else: it is what\n"+ + "decides whether --service-account-name names an existing ServiceAccount to run as\n"+ + "verbatim, or is a prefix for one this run creates. Without it, a ServiceAccount you\n"+ + "named explicitly would be silently replaced by a generated one carrying none of its\n"+ + "cloud credentials (IRSA / Workload Identity), so the run stops instead.\n"+ + "The remaining, mode-specific permissions were not evaluated; fix the above and re-run.", + namespace) } -// checkPermission checks if the current user can perform the specified action. -func (d *Deployer) checkPermission(ctx context.Context, resource, verb, namespace string) (bool, string, error) { - review := &authv1.SelfSubjectAccessReview{ - Spec: authv1.SelfSubjectAccessReviewSpec{ - ResourceAttributes: &authv1.ResourceAttributes{ - Verb: verb, - Resource: resource, - Namespace: namespace, - }, - }, +// remediationHints returns the operator-facing next step for whichever kind +// of failure occurred, so the message can be acted on without reading the +// source. +func (d *Deployer) remediationHints(results []permissionCheck) string { + var callerMissing, subjectMissing bool + for _, r := range results { + if r.Allowed || r.Unverified { + continue + } + if r.Subject == "" { + callerMissing = true + } else { + subjectMissing = true + } } - result, err := d.clientset.AuthorizationV1().SelfSubjectAccessReviews().Create(ctx, review, metav1.CreateOptions{}) - if err != nil { - return false, "", errors.Wrap(errors.ErrCodeInternal, "failed to create SelfSubjectAccessReview", err) + var hints []string + if subjectMissing { + hints = append(hints, fmt.Sprintf( + "The agent pod runs as ServiceAccount %q, and aicr manages no permissions for a\n"+ + "ServiceAccount it did not create. Generate the RBAC that grants it what the agent\n"+ + "needs, review it, and apply it yourself:\n"+ + " aicr snapshot --add-roles-to-service-account %s --namespace %s\n"+ + " kubectl apply -f snapshot-rbac-/", + d.existingServiceAccount(), d.existingServiceAccount(), d.config.Namespace)) } + if callerMissing && d.managesRBAC() { + hints = append(hints, ""+ + "This run creates and deletes its own run-scoped RBAC, which is why both create and\n"+ + "delete are required: cleanup always runs, and an identity that can create but not\n"+ + "delete leaks a ServiceAccount, Role, RoleBinding, ClusterRole and ClusterRoleBinding\n"+ + "on every run. To need none of those verbs, pre-create a ServiceAccount and pass its\n"+ + "exact name with --service-account-name.") + } + return strings.Join(hints, "\n\n") +} - return result.Status.Allowed, result.Status.Reason, nil +// warnUnverified reports, once, that the agent ServiceAccount's own +// permissions could not be checked. +// +// Creating a SubjectAccessReview is itself a privilege. A caller who lacks +// it learns nothing about the ServiceAccount, and silently dropping the +// check would be the same defect class the `serviceaccounts: get` gate above +// closes — a missing answer read as a passing one. The run continues, +// because the agent still fails visibly inside the pod, but the operator is +// told the gate did not cover it and why. +func (d *Deployer) warnUnverified(results []permissionCheck) { + var count int + var reason string + for _, r := range results { + if !r.Unverified { + continue + } + count++ + if reason == "" { + reason = r.Reason + } + } + if count == 0 { + return + } + slog.Warn("could not verify the agent ServiceAccount's own permissions; continuing, but a missing rule will surface as an in-pod failure minutes from now instead of here", + slog.String(attrServiceAccount, d.existingServiceAccount()), + slog.String(attrNamespace, d.config.Namespace), + slog.String(attrRunID, d.config.RunID), + slog.Int("uncheckedRules", count), + slog.String("cause", reason), + slog.String("remedy", "grant the caller 'create subjectaccessreviews.authorization.k8s.io', or verify by hand with: kubectl auth can-i --list --as "+serviceAccountUsername(d.config.Namespace, d.existingServiceAccount()))) } diff --git a/pkg/k8s/agent/permissions_test.go b/pkg/k8s/agent/permissions_test.go index 1ee8896ff..a36d5c296 100644 --- a/pkg/k8s/agent/permissions_test.go +++ b/pkg/k8s/agent/permissions_test.go @@ -16,141 +16,587 @@ package agent import ( "context" + stderrors "errors" "fmt" "strings" + "sync" "testing" + aicrerrors "github.com/NVIDIA/aicr/pkg/errors" authv1 "k8s.io/api/authorization/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes/fake" k8stesting "k8s.io/client-go/testing" ) -func TestCheckPermissions(t *testing.T) { - tests := []struct { - name string - allowed bool - wantErr bool - errContains string +// exactSAName is the operator-provisioned ServiceAccount the +// exact-ServiceAccount-mode cases name verbatim. +const exactSAName = "irsa-snapshotter" + +// The two access-review resources the gate posts to. They are "created" +// over the REST API but persist nothing — the create IS the authorization +// query — which is why the write guard below exempts them. +const ( + selfReviewResource = "selfsubjectaccessreviews" + subjectReviewResource = "subjectaccessreviews" +) + +// askedAccess is one authorization question a reactor observed, flattened +// out of whichever review kind carried it. subject is "" when the question +// came from a SelfSubjectAccessReview (i.e. it was asked about the caller). +type askedAccess struct { + subject string + group string + resource string + subresource string + verb string + namespace string +} + +// String renders a question the way an assertion failure should read. +func (a askedAccess) String() string { + return fmt.Sprintf("%s %s (%s) subject=%q", a.verb, + resourceLabel(a.resource, a.subresource, a.group), scopeLabel(a.namespace), a.subject) +} + +// reviewRecorder collects every question the gate asked. CheckPermissions +// fans its checks out over an errgroup, so the reactors run on worker +// goroutines and every field must be mutex-guarded. +type reviewRecorder struct { + mu sync.Mutex + asked []askedAccess +} + +func (r *reviewRecorder) record(a askedAccess) { + r.mu.Lock() + defer r.mu.Unlock() + r.asked = append(r.asked, a) +} + +// questions returns a copy of what was asked, taken under lock. +func (r *reviewRecorder) questions() []askedAccess { + r.mu.Lock() + defer r.mu.Unlock() + out := make([]askedAccess, len(r.asked)) + copy(out, r.asked) + return out +} + +// asked reports whether a question matching pred was put to the apiserver. +func (r *reviewRecorder) asked1(pred func(askedAccess) bool) bool { + for _, q := range r.questions() { + if pred(q) { + return true + } + } + return false +} + +// installReviewReactors answers every Self/SubjectAccessReview with allow(q) +// and records the question. A nil allow permits everything. +// +// Failures are reported with t.Errorf and handed back as the reactor's error +// rather than with t.Fatalf: the reactor runs on an errgroup worker, and +// Goexit there would leave the group waiting on a goroutine that never +// returns. +func installReviewReactors(t *testing.T, cs *fake.Clientset, allow func(askedAccess) bool) *reviewRecorder { + t.Helper() + rec := &reviewRecorder{} + if allow == nil { + allow = func(askedAccess) bool { return true } + } + + answer := func(action k8stesting.Action) (askedAccess, bool, error) { + create, ok := action.(k8stesting.CreateAction) + if !ok { + err := fmt.Errorf("action %T is not a CreateAction", action) + t.Error(err) + return askedAccess{}, false, err + } + var attrs *authv1.ResourceAttributes + var subject string + switch obj := create.GetObject().(type) { + case *authv1.SelfSubjectAccessReview: + attrs = obj.Spec.ResourceAttributes + case *authv1.SubjectAccessReview: + attrs, subject = obj.Spec.ResourceAttributes, obj.Spec.User + if subject == "" { + err := stderrors.New("SubjectAccessReview carries no User; it would answer for nobody") + t.Error(err) + return askedAccess{}, false, err + } + default: + err := fmt.Errorf("object %T is not an access review", create.GetObject()) + t.Error(err) + return askedAccess{}, false, err + } + if attrs == nil { + err := stderrors.New("access review carries no ResourceAttributes") + t.Error(err) + return askedAccess{}, false, err + } + q := askedAccess{ + subject: subject, + group: attrs.Group, + resource: attrs.Resource, + subresource: attrs.Subresource, + verb: attrs.Verb, + namespace: attrs.Namespace, + } + rec.record(q) + return q, allow(q), nil + } + + cs.PrependReactor(verbCreate, selfReviewResource, func(action k8stesting.Action) (bool, runtime.Object, error) { + q, allowed, err := answer(action) + if err != nil { + return true, nil, err + } + return true, &authv1.SelfSubjectAccessReview{ + Status: authv1.SubjectAccessReviewStatus{Allowed: allowed, Reason: "caller policy for " + q.String()}, + }, nil + }) + cs.PrependReactor(verbCreate, subjectReviewResource, func(action k8stesting.Action) (bool, runtime.Object, error) { + q, allowed, err := answer(action) + if err != nil { + return true, nil, err + } + return true, &authv1.SubjectAccessReview{ + Status: authv1.SubjectAccessReviewStatus{Allowed: allowed, Reason: "ServiceAccount policy for " + q.String()}, + }, nil + }) + return rec +} + +// seedServiceAccount pre-creates the operator-provisioned ServiceAccount +// that puts a Deployer into exact-ServiceAccount mode. +func seedServiceAccount(t *testing.T, cs *fake.Clientset, namespace, name string) { + t.Helper() + if _, err := cs.CoreV1().ServiceAccounts(namespace).Create(context.Background(), &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + }, metav1.CreateOptions{}); err != nil { + t.Fatalf("seeding ServiceAccount %s/%s: %v", namespace, name, err) + } +} + +// hasCheck reports whether results holds an answered check matching pred. +func hasCheck(results []permissionCheck, pred func(permissionCheck) bool) bool { + for _, r := range results { + if pred(r) { + return true + } + } + return false +} + +// isCallerRBACCheck matches a caller-side check on one of the five RBAC +// kinds the run creates and deletes in prefix mode. +func isCallerRBACCheck(p permissionCheck) bool { + if p.Subject != "" { + return false + } + switch p.Resource { + case resourceRoles, resourceRoleBindings, resourceClusterRoles, resourceClusterRoleBindings: + return true + case resourceServiceAccounts: + return p.Verb == verbCreate || p.Verb == verbDelete + default: + return false + } +} + +// TestCheckPermissions_PrefixModeRequiresRBACCreateAndDelete pins hole B: the +// deferred Cleanup is registered before Deploy and always runs, issuing a +// UID-pinned delete for every RBAC object the run created. An identity with +// create-but-not-delete used to pass a green pre-flight, deploy, and then +// leak a full run-scoped RBAC set — cluster-scoped objects included — once +// per run. +func TestCheckPermissions_PrefixModeRequiresRBACCreateAndDelete(t *testing.T) { + // Every (resource, verb, cluster-scoped?) triple prefix mode must + // demand of the caller. Each is denied on its own to prove it is + // individually load-bearing rather than incidentally covered. + required := []struct { + resource string + verb string + cluster bool }{ - { - name: "all permissions allowed", - allowed: true, - wantErr: false, - }, - { - name: "permissions denied", - allowed: false, - wantErr: true, - errContains: "missing required permissions", - }, + {resourceServiceAccounts, verbCreate, false}, + {resourceServiceAccounts, verbDelete, false}, + {resourceRoles, verbCreate, false}, + {resourceRoles, verbDelete, false}, + {resourceRoleBindings, verbCreate, false}, + {resourceRoleBindings, verbDelete, false}, + {resourceClusterRoles, verbCreate, true}, + {resourceClusterRoles, verbDelete, true}, + {resourceClusterRoleBindings, verbCreate, true}, + {resourceClusterRoleBindings, verbDelete, true}, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { + for _, req := range required { + t.Run(req.verb+" "+req.resource, func(t *testing.T) { clientset := fake.NewClientset() - - // Mock SelfSubjectAccessReview responses - clientset.PrependReactor("create", "selfsubjectaccessreviews", func(action k8stesting.Action) (bool, runtime.Object, error) { - return true, &authv1.SelfSubjectAccessReview{ - Status: authv1.SubjectAccessReviewStatus{ - Allowed: tt.allowed, - Reason: "test reason", - }, - }, nil + rec := installReviewReactors(t, clientset, func(q askedAccess) bool { + return q.resource != req.resource || q.verb != req.verb }) - deployer := NewDeployer(clientset, Config{ - Namespace: "gpu-operator", - ServiceAccountName: "aicr", - JobName: "aicr", + d := NewDeployer(clientset, Config{ + Namespace: testNamespace, + ServiceAccountName: exactSAName, // nothing seeded: prefix mode + RunID: testRunID, }) - - ctx := context.Background() - checks, err := deployer.CheckPermissions(ctx) - - if (err != nil) != tt.wantErr { - t.Errorf("CheckPermissions() error = %v, wantErr %v", err, tt.wantErr) - return + results, err := d.CheckPermissions(context.Background()) + if err == nil { + t.Fatalf("CheckPermissions() error = nil; %s %s must be required in prefix mode", req.verb, req.resource) } - - if tt.wantErr && err != nil && tt.errContains != "" { - if !strings.Contains(err.Error(), tt.errContains) { - t.Errorf("CheckPermissions() error = %v, should contain %q", err, tt.errContains) - } + if !stderrors.Is(err, aicrerrors.New(aicrerrors.ErrCodeUnauthorized, "")) { + t.Errorf("error code = %v, want ErrCodeUnauthorized", err) } - if !tt.wantErr && len(checks) == 0 { - t.Error("CheckPermissions() returned no checks") + wantScope := scopeLabel(testNamespace) + if req.cluster { + wantScope = scopeLabel("") } - - // Verify all checks match expected result - for _, check := range checks { - if check.Allowed != tt.allowed { - t.Errorf("Check %s %s: got allowed=%v, want %v", check.Verb, check.Resource, check.Allowed, tt.allowed) - } + wantLine := fmt.Sprintf("%s cannot %q %s (%s)", callerSubjectLabel, req.verb, + resourceLabel(req.resource, "", rbacGroupFor(req.resource)), wantScope) + if !strings.Contains(err.Error(), wantLine) { + t.Errorf("error = %v\nwant a line containing %q", err, wantLine) + } + if !hasCheck(results, func(p permissionCheck) bool { + return p.Resource == req.resource && p.Verb == req.verb && !p.Allowed + }) { + t.Errorf("results do not carry the denied %s %s check", req.verb, req.resource) + } + // The gate must have resolved the mode first, which means it + // asked whether it may read ServiceAccounts. + if !rec.asked1(func(q askedAccess) bool { + return q.subject == "" && q.resource == resourceServiceAccounts && q.verb == verbGet + }) { + t.Error("gate never asked for `serviceaccounts: get`, so it cannot have resolved the mode") } }) } } -func TestCheckPermission(t *testing.T) { +// rbacGroupFor returns the API group a resource lives in, for building the +// exact message line the gate is expected to render. +func rbacGroupFor(resource string) string { + if resource == resourceServiceAccounts { + return "" + } + return rbacAPIGroup +} + +// TestCheckPermissions_ExactModeSkipsCallerRBACVerbs is the other half of the +// mode split. In exact-ServiceAccount mode aicr creates and deletes no RBAC +// at all, so demanding create/delete on the five kinds would fail operators +// who legitimately hold none of those grants — the very operators the exact +// mode exists for. +func TestCheckPermissions_ExactModeSkipsCallerRBACVerbs(t *testing.T) { + clientset := fake.NewClientset() + seedServiceAccount(t, clientset, testNamespace, exactSAName) + + // Deny every caller-side RBAC verb outright. A correct gate never asks. + rec := installReviewReactors(t, clientset, func(q askedAccess) bool { + if q.subject != "" { + return true + } + switch q.resource { + case resourceRoles, resourceRoleBindings, resourceClusterRoles, resourceClusterRoleBindings: + return false + case resourceServiceAccounts: + return q.verb == verbGet + default: + return true + } + }) + + d := NewDeployer(clientset, Config{ + Namespace: testNamespace, + ServiceAccountName: exactSAName, + RunID: testRunID, + }) + results, err := d.CheckPermissions(context.Background()) + if err != nil { + t.Fatalf("CheckPermissions() error = %v, want nil (exact mode needs no RBAC create/delete)", err) + } + if d.managesRBAC() { + t.Fatal("managesRBAC() = true; the seeded ServiceAccount should have put the run in exact mode") + } + if hasCheck(results, isCallerRBACCheck) { + t.Error("gate demanded a caller RBAC verb in exact-ServiceAccount mode") + } + if rec.asked1(func(q askedAccess) bool { + return q.subject == "" && q.resource == resourceClusterRoles + }) { + t.Error("gate asked the apiserver about clusterroles in exact-ServiceAccount mode") + } + + // It must instead have asked the ServiceAccount's own questions, and + // asked them of the ServiceAccount rather than of the caller. + subject := serviceAccountUsername(testNamespace, exactSAName) + if !rec.asked1(func(q askedAccess) bool { + return q.subject == subject && q.resource == resourceNodes && q.verb == verbList + }) { + t.Errorf("gate never asked whether %s may list nodes", subject) + } +} + +// TestCheckPermissions_ServiceAccountGetIsRequired pins hole A at the gate. +// Without `serviceaccounts: get` the ServiceAccount mode is unknowable, and +// the old behavior — treat an unreadable ServiceAccount as a name prefix — +// silently ran an operator's explicitly-named IRSA / Workload Identity +// account as a generated one carrying none of its cloud annotations. +func TestCheckPermissions_ServiceAccountGetIsRequired(t *testing.T) { + clientset := fake.NewClientset() + seedServiceAccount(t, clientset, testNamespace, exactSAName) + rec := installReviewReactors(t, clientset, func(q askedAccess) bool { + return q.resource != resourceServiceAccounts || q.verb != verbGet + }) + + d := NewDeployer(clientset, Config{ + Namespace: testNamespace, + ServiceAccountName: exactSAName, + RunID: testRunID, + }) + results, err := d.CheckPermissions(context.Background()) + if err == nil { + t.Fatal("CheckPermissions() error = nil; a caller that cannot read ServiceAccounts must fail the gate") + } + if !stderrors.Is(err, aicrerrors.New(aicrerrors.ErrCodeUnauthorized, "")) { + t.Errorf("error code = %v, want ErrCodeUnauthorized", err) + } + for _, want := range []string{ + fmt.Sprintf("%s cannot %q %s", callerSubjectLabel, verbGet, resourceServiceAccounts), + "mode-specific permissions were not evaluated", + } { + if !strings.Contains(err.Error(), want) { + t.Errorf("error = %v\nwant it to contain %q", err, want) + } + } + + // It must NOT have silently downgraded to prefix mode and carried on: + // no mode-specific question may have been asked, and the run must not + // have adopted the seeded ServiceAccount either. + if hasCheck(results, isCallerRBACCheck) { + t.Error("gate evaluated prefix-mode RBAC verbs despite being unable to resolve the mode") + } + if rec.asked1(func(q askedAccess) bool { return q.subject != "" }) { + t.Error("gate issued a SubjectAccessReview despite being unable to resolve the mode") + } + if d.existingServiceAccount() != "" { + t.Errorf("existingServiceAccount() = %q, want \"\" (nothing may be resolved)", d.existingServiceAccount()) + } +} + +// TestCheckPermissions_ServiceAccountSubjectFailureNamesTheSubject covers +// requirement 3: the agent pod runs as the ServiceAccount, so the gate must +// verify that identity's own permissions with a SubjectAccessReview, not the +// caller's with a SelfSubjectAccessReview. In exact mode aicr grants nothing +// and the operator was supposed to have applied +// `--add-roles-to-service-account`; "you never applied those manifests" is +// far better caught here than in a pod minutes later. +func TestCheckPermissions_ServiceAccountSubjectFailureNamesTheSubject(t *testing.T) { + clientset := fake.NewClientset() + seedServiceAccount(t, clientset, testNamespace, exactSAName) + installReviewReactors(t, clientset, func(q askedAccess) bool { + // The caller is fully privileged; the ServiceAccount is missing + // exactly the cluster-scoped node read the agent cannot work without. + return q.subject == "" || q.resource != resourceNodes + }) + + d := NewDeployer(clientset, Config{ + Namespace: testNamespace, + ServiceAccountName: exactSAName, + RunID: testRunID, + }) + results, err := d.CheckPermissions(context.Background()) + if err == nil { + t.Fatal("CheckPermissions() error = nil; a ServiceAccount missing `nodes: list` must fail the gate") + } + + subject := serviceAccountUsername(testNamespace, exactSAName) + for _, want := range []string{ + fmt.Sprintf("agent ServiceAccount %q cannot %q %s (%s)", subject, verbList, resourceNodes, scopeLabel("")), + "--add-roles-to-service-account " + exactSAName, + } { + if !strings.Contains(err.Error(), want) { + t.Errorf("error = %v\nwant it to contain %q", err, want) + } + } + // A caller-scoped answer must never be substituted for the subject's. + if !hasCheck(results, func(p permissionCheck) bool { + return p.Subject == subject && p.Resource == resourceNodes && !p.Allowed && !p.Unverified + }) { + t.Error("results carry no denied ServiceAccount-subject check for nodes") + } +} + +// TestCheckPermissions_SubjectAccessReviewForbiddenReportsAndContinues covers +// the meta-permission honestly. Creating a SubjectAccessReview is itself a +// privilege; a caller without it learns nothing about the ServiceAccount. +// Silently skipping would be the same defect class as hole A — a missing +// answer read as a passing one — so the gate records the checks as +// unverified, warns, and lets the run proceed to fail visibly in-pod. +func TestCheckPermissions_SubjectAccessReviewForbiddenReportsAndContinues(t *testing.T) { + clientset := fake.NewClientset() + seedServiceAccount(t, clientset, testNamespace, exactSAName) + installReviewReactors(t, clientset, nil) + // Prepended after installReviewReactors, so it wins for this resource. + clientset.PrependReactor(verbCreate, subjectReviewResource, func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewForbidden( + schema.GroupResource{Group: "authorization.k8s.io", Resource: subjectReviewResource}, "", + stderrors.New(`User "snapshot-runner" cannot create resource "subjectaccessreviews"`)) + }) + + logs := captureLogs(t) + d := NewDeployer(clientset, Config{ + Namespace: testNamespace, + ServiceAccountName: exactSAName, + RunID: testRunID, + }) + results, err := d.CheckPermissions(context.Background()) + if err != nil { + t.Fatalf("CheckPermissions() error = %v, want nil (an unanswerable check is not a failed one)", err) + } + + var unverified int + for _, r := range results { + if r.Unverified { + unverified++ + if r.Subject == "" { + t.Error("a caller check was marked unverified; only ServiceAccount checks may be") + } + } + } + if unverified == 0 { + t.Fatal("no check was marked Unverified; the gate silently dropped the ServiceAccount's permissions") + } + + for _, want := range []string{ + "could not verify the agent ServiceAccount's own permissions", + "cannot create resource \\\"subjectaccessreviews\\\"", + "kubectl auth can-i --list --as " + serviceAccountUsername(testNamespace, exactSAName), + } { + if !strings.Contains(logs.String(), want) { + t.Errorf("warning log = %s\nwant it to contain %q", logs.String(), want) + } + } +} + +// TestCheckPermissions_ReportsEveryMissingPermissionAtOnce pins requirement +// 4: an operator fixing permissions should get the complete list in one run +// rather than discovering them one denial at a time. +func TestCheckPermissions_ReportsEveryMissingPermissionAtOnce(t *testing.T) { + clientset := fake.NewClientset() + denied := map[string]string{ + resourceClusterRoles: verbDelete, + resourceClusterRoleBindings: verbDelete, + resourceJobs: verbCreate, + resourcePods: verbWatch, + } + installReviewReactors(t, clientset, func(q askedAccess) bool { + return denied[q.resource] != q.verb + }) + + d := NewDeployer(clientset, Config{ + Namespace: testNamespace, + RunID: testRunID, + }) + _, err := d.CheckPermissions(context.Background()) + if err == nil { + t.Fatal("CheckPermissions() error = nil, want the four denied permissions reported") + } + for resource, verb := range denied { + line := fmt.Sprintf("cannot %q %s", verb, resource) + if !strings.Contains(err.Error(), line) { + t.Errorf("error = %v\nwant it to report %q too", err, line) + } + } + // Prefix mode is what this Deployer is in, so the remediation must point + // at the exact-ServiceAccount escape hatch rather than at manifests. + if !strings.Contains(err.Error(), "--service-account-name") { + t.Errorf("error = %v\nwant prefix-mode remediation naming --service-account-name", err) + } +} + +// TestCheckPermissions_IssuesNoWriteBeforeTheGateCloses is the structural +// guarantee behind resolving the ServiceAccount inside the gate: resolving +// is a read, and fail-before-mutate is about not WRITING before validation. +// A reactor rejects every create/update/patch/delete of a real object, so any +// regression that mutates during the pre-flight fails here rather than in +// production. +func TestCheckPermissions_IssuesNoWriteBeforeTheGateCloses(t *testing.T) { tests := []struct { - name string - resource string - verb string - namespace string - allowed bool - reason string + name string + exact bool + allow func(askedAccess) bool + wantErr bool }{ + {name: "prefix mode, all granted"}, + {name: "exact mode, all granted", exact: true}, { - name: "allowed permission", - resource: "jobs", - verb: "create", - namespace: "gpu-operator", - allowed: true, - reason: "user has permission", + name: "prefix mode, denied", + allow: func(q askedAccess) bool { return q.resource != resourceClusterRoles }, + wantErr: true, }, { - name: "denied permission", - resource: "jobs", - verb: "create", - namespace: "gpu-operator", - allowed: false, - reason: "user lacks permission", + name: "exact mode, ServiceAccount denied", + exact: true, + allow: func(q askedAccess) bool { return q.subject == "" }, + wantErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { clientset := fake.NewClientset() + if tt.exact { + // Seeded through the tracker before the guard is armed. + seedServiceAccount(t, clientset, testNamespace, exactSAName) + } + installReviewReactors(t, clientset, tt.allow) - clientset.PrependReactor("create", "selfsubjectaccessreviews", func(action k8stesting.Action) (bool, runtime.Object, error) { - return true, &authv1.SelfSubjectAccessReview{ - Status: authv1.SubjectAccessReviewStatus{ - Allowed: tt.allowed, - Reason: tt.reason, - }, - }, nil + var mu sync.Mutex + var writes []string + clientset.PrependReactor("*", "*", func(action k8stesting.Action) (bool, runtime.Object, error) { + switch action.GetVerb() { + case verbCreate, verbUpdate, verbPatch, verbDelete, "deletecollection": + default: + return false, nil, nil + } + // Access reviews are "created" over the REST API but + // persist nothing — they are the authorization query + // itself. Everything else is a real mutation. + if res := action.GetResource().Resource; res == selfReviewResource || res == subjectReviewResource { + return false, nil, nil + } + write := action.GetVerb() + " " + action.GetResource().Resource + mu.Lock() + writes = append(writes, write) + mu.Unlock() + return true, nil, fmt.Errorf("pre-flight issued a write: %s", write) }) - deployer := NewDeployer(clientset, Config{ - Namespace: tt.namespace, + d := NewDeployer(clientset, Config{ + Namespace: testNamespace, + ServiceAccountName: exactSAName, + RunID: testRunID, + OwnsOutputConfigMap: true, + Output: "cm://" + testNamespace + "/" + StagingConfigMapName(testRunID), }) - - ctx := context.Background() - allowed, reason, err := deployer.checkPermission(ctx, tt.resource, tt.verb, tt.namespace) - - if err != nil { - t.Fatalf("checkPermission() error = %v", err) - } - - if allowed != tt.allowed { - t.Errorf("checkPermission() allowed = %v, want %v", allowed, tt.allowed) + _, err := d.CheckPermissions(context.Background()) + if (err != nil) != tt.wantErr { + t.Fatalf("CheckPermissions() error = %v, wantErr %v", err, tt.wantErr) } - if reason != tt.reason { - t.Errorf("checkPermission() reason = %q, want %q", reason, tt.reason) + mu.Lock() + defer mu.Unlock() + if len(writes) > 0 { + t.Errorf("pre-flight mutated the cluster before the gate closed: %v", writes) } }) } @@ -189,38 +635,14 @@ func TestCheckPermissions_ConfigMapDeleteGatedOnOwnership(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { clientset := fake.NewClientset() - // Deny only `configmaps: delete`; allow everything else. This // models the least-privilege identity the gate exists for. - // - // CheckPermissions fans the checks out over an errgroup, so this - // reactor runs on worker goroutines, not the test goroutine. - // t.Fatalf there would Goexit only the worker and leave the - // errgroup waiting on a goroutine that never returns a value; - // report with t.Errorf and hand the failure back as the - // reactor's error so the call under test terminates. - clientset.PrependReactor("create", "selfsubjectaccessreviews", func(action k8stesting.Action) (bool, runtime.Object, error) { - create, ok := action.(k8stesting.CreateAction) - if !ok { - reactorErr := fmt.Errorf("action %T is not a CreateAction", action) - t.Error(reactorErr) - return true, nil, reactorErr - } - review, ok := create.GetObject().(*authv1.SelfSubjectAccessReview) - if !ok { - reactorErr := fmt.Errorf("object %T is not a SelfSubjectAccessReview", create.GetObject()) - t.Error(reactorErr) - return true, nil, reactorErr - } - attrs := review.Spec.ResourceAttributes - allowed := attrs.Resource != resourceCM || attrs.Verb != verbDelete - return true, &authv1.SelfSubjectAccessReview{ - Status: authv1.SubjectAccessReviewStatus{Allowed: allowed, Reason: "test reason"}, - }, nil + installReviewReactors(t, clientset, func(q askedAccess) bool { + return q.resource != resourceCM || q.verb != verbDelete }) deployer := NewDeployer(clientset, Config{ - Namespace: "gpu-operator", + Namespace: testNamespace, RunID: testRunID, OwnsOutputConfigMap: tt.ownsOutput, }) @@ -230,15 +652,205 @@ func TestCheckPermissions_ConfigMapDeleteGatedOnOwnership(t *testing.T) { t.Fatalf("CheckPermissions() error = %v, wantErr %v", err, tt.wantErr) } - gotCMDelete := false - for _, c := range checks { - if c.Resource == resourceCM && c.Verb == verbDelete { - gotCMDelete = true - } - } + gotCMDelete := hasCheck(checks, func(p permissionCheck) bool { + return p.Resource == resourceCM && p.Verb == verbDelete + }) if gotCMDelete != tt.wantCMDeleteCheck { t.Errorf("configmaps delete check present = %v, want %v", gotCMDelete, tt.wantCMDeleteCheck) } }) } } + +// TestCheckPermissions_AllGrantedPassesBothModes is the happy path, and also +// asserts the gate covers the run's non-RBAC work — the Job it creates and +// waits on, the pods it discovers and streams logs from, and the ConfigMap +// it reads the snapshot back out of. +func TestCheckPermissions_AllGrantedPassesBothModes(t *testing.T) { + tests := []struct { + name string + exact bool + }{ + {name: "prefix mode"}, + {name: "exact-ServiceAccount mode", exact: true}, + } + + want := []askedAccess{ + {group: batchAPIGroup, resource: resourceJobs, verb: verbCreate, namespace: testNamespace}, + {group: batchAPIGroup, resource: resourceJobs, verb: verbWatch, namespace: testNamespace}, + {group: batchAPIGroup, resource: resourceJobs, verb: verbDelete, namespace: testNamespace}, + {resource: resourcePods, verb: verbList, namespace: testNamespace}, + {resource: resourcePods, subresource: subresourceLog, verb: verbGet, namespace: testNamespace}, + {resource: resourceCM, verb: verbGet, namespace: testNamespace}, + {resource: resourceServiceAccounts, verb: verbGet, namespace: testNamespace}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clientset := fake.NewClientset() + if tt.exact { + seedServiceAccount(t, clientset, testNamespace, exactSAName) + } + rec := installReviewReactors(t, clientset, nil) + + d := NewDeployer(clientset, Config{ + Namespace: testNamespace, + ServiceAccountName: exactSAName, + RunID: testRunID, + }) + results, err := d.CheckPermissions(context.Background()) + if err != nil { + t.Fatalf("CheckPermissions() error = %v, want nil", err) + } + if len(results) == 0 { + t.Fatal("CheckPermissions() returned no checks") + } + for _, r := range results { + if !r.Allowed { + t.Errorf("check %s %s (%s) denied under an allow-all policy", r.Verb, r.Resource, r.Namespace) + } + } + for _, q := range want { + if !rec.asked1(func(got askedAccess) bool { return got == q }) { + t.Errorf("gate never asked: %s", q) + } + } + }) + } +} + +// TestReviewAccess covers the two review kinds one question at a time: a +// caller check must go out as a SelfSubjectAccessReview and a ServiceAccount +// check as a SubjectAccessReview naming that subject, because the former +// cannot answer for anyone but the caller. +func TestReviewAccess(t *testing.T) { + subject := serviceAccountUsername(testNamespace, exactSAName) + + tests := []struct { + name string + check accessCheck + allowed bool + }{ + { + name: "caller check allowed", + check: accessCheck{group: batchAPIGroup, resource: resourceJobs, verb: verbCreate, namespace: testNamespace}, + allowed: true, + }, + { + name: "caller check denied", + check: accessCheck{group: batchAPIGroup, resource: resourceJobs, verb: verbCreate, namespace: testNamespace}, + }, + { + name: "ServiceAccount subresource check allowed", + check: accessCheck{resource: resourcePods, subresource: "exec", verb: verbCreate, subject: subject}, + allowed: true, + }, + { + name: "ServiceAccount cluster-scoped check denied", + check: accessCheck{resource: resourceNodes, verb: verbList, subject: subject}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clientset := fake.NewClientset() + rec := installReviewReactors(t, clientset, func(askedAccess) bool { return tt.allowed }) + + d := NewDeployer(clientset, Config{Namespace: testNamespace, RunID: testRunID}) + allowed, reason, err := d.reviewAccess(context.Background(), tt.check) + if err != nil { + t.Fatalf("reviewAccess() error = %v", err) + } + if allowed != tt.allowed { + t.Errorf("reviewAccess() allowed = %v, want %v", allowed, tt.allowed) + } + if reason == "" { + t.Error("reviewAccess() reason is empty; the authorizer's explanation must be carried through") + } + + asked := rec.questions() + if len(asked) != 1 { + t.Fatalf("questions asked = %d, want exactly 1", len(asked)) + } + want := askedAccess{ + subject: tt.check.subject, + group: tt.check.group, + resource: tt.check.resource, + subresource: tt.check.subresource, + verb: tt.check.verb, + namespace: tt.check.namespace, + } + if asked[0] != want { + t.Errorf("asked %s, want %s", asked[0], want) + } + }) + } +} + +// TestDedupeChecks pins that the gate never asks the apiserver the same +// question twice, while keeping questions that differ only in scope: an +// operator who applied the Role but not the ClusterRole must fail exactly +// the cluster-scoped one. +func TestDedupeChecks(t *testing.T) { + nsPods := accessCheck{resource: resourcePods, verb: verbList, namespace: testNamespace} + clusterPods := accessCheck{resource: resourcePods, verb: verbList} + + got := dedupeChecks([]accessCheck{nsPods, clusterPods, nsPods, clusterPods, nsPods}) + want := []accessCheck{nsPods, clusterPods} + if len(got) != len(want) { + t.Fatalf("dedupeChecks() returned %d checks, want %d: %+v", len(got), len(want), got) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("check %d = %+v, want %+v (first-seen order must be preserved)", i, got[i], want[i]) + } + } +} + +// TestServiceAccountChecksTrackTheGrantedRules pins the anti-drift property: +// the questions asked of the agent ServiceAccount are derived from the same +// namespacedRules / clusterRules definitions the run-scoped Role and +// ClusterRole are built from, so the gate cannot fall behind what the agent +// needs. --discover-network widens the rule set, and the gate must widen +// with it. +func TestServiceAccountChecksTrackTheGrantedRules(t *testing.T) { + subject := serviceAccountUsername(testNamespace, exactSAName) + + for _, discover := range []bool{false, true} { + name := "baseline" + if discover { + name = "discover-network" + } + t.Run(name, func(t *testing.T) { + d := NewDeployer(fake.NewClientset(), Config{ + Namespace: testNamespace, + RunID: testRunID, + DiscoverNetwork: discover, + }) + got := make(map[accessCheck]struct{}) + for _, c := range d.serviceAccountChecks(subject) { + got[c] = struct{}{} + } + + var wantAll []accessCheck + wantAll = append(wantAll, checksFromRules(namespacedRules(), testNamespace, subject)...) + wantAll = append(wantAll, checksFromRules(clusterRules(discover), "", subject)...) + for _, c := range wantAll { + if _, ok := got[c]; !ok { + t.Errorf("granted rule not checked: %+v", c) + } + if c.subject != subject { + t.Errorf("check %+v is not asked of the ServiceAccount", c) + } + } + + // pods/exec is the rule most easily lost to naive splitting: + // the apiserver evaluates it as resource "pods", subresource + // "exec", never as a resource literally named "pods/exec". + execCheck := accessCheck{resource: resourcePods, subresource: "exec", verb: verbCreate, subject: subject} + if _, ok := got[execCheck]; ok != discover { + t.Errorf("pods/exec check present = %v, want %v", ok, discover) + } + }) + } +} diff --git a/pkg/k8s/agent/rbac.go b/pkg/k8s/agent/rbac.go index 8a8a7126d..ce3c69e12 100644 --- a/pkg/k8s/agent/rbac.go +++ b/pkg/k8s/agent/rbac.go @@ -100,14 +100,22 @@ func (d *Deployer) ensureNamespace(ctx context.Context) error { // aicr's own default, not something the operator asked for, so a stray // ServiceAccount sitting at that name must not silently capture the run. // -// The Get must not gate the deployment: `serviceaccounts get` is -// deliberately absent from CheckPermissions' requiredChecks -// (permissions.go), so an identity scoped to exactly the pre-flight verb -// set would otherwise pass the pre-flight and then fail Deploy with an -// ErrCodeInternal — a permission problem reported as an internal error. -// Forbidden therefore downgrades to a debug line and the run proceeds in -// prefix mode, which is the mode that identity has the permissions for. -// Every other unexpected error still fails closed. +// It is called from CheckPermissions, not from Deploy directly: the verb set +// the pre-flight demands depends on which of the two modes this run is in, +// so the gate has to resolve before it can finish. The Get is read-only, so +// resolving inside the gate does not weaken fail-before-mutate — no write is +// issued until Deploy's ensure* chain, which runs only once the gate passes. +// +// Every error fails closed, Forbidden included. `serviceaccounts: get` is a +// REQUIRED check in CheckPermissions, evaluated before this runs, so a +// caller that reaches here has been told by the apiserver's own authorizer +// that it may read ServiceAccounts; a Forbidden anyway means the answer and +// the behavior disagree, which is not a state to guess through. The +// previous downgrade — log at debug, continue in prefix mode — is the exact +// credential-loss seam this package exists to close: an operator who passed +// --service-account-name pointing at their IRSA or Workload Identity +// ServiceAccount would silently run under a fresh "-" account +// carrying none of its cloud annotations, with no visible signal. func (d *Deployer) resolveServiceAccount(ctx context.Context) error { name := d.config.ServiceAccountName if name == "" { @@ -126,8 +134,13 @@ func (d *Deployer) resolveServiceAccount(ctx context.Context) error { // Normal path: the value is a prefix and this run creates its own // run-scoped ServiceAccount below. case apierrors.IsForbidden(err): - slog.Debug("cannot read ServiceAccounts in this namespace; treating --service-account-name as a prefix", - attrName, name, attrNamespace, d.config.Namespace, "error", err) + return errors.WrapWithContext(errors.ErrCodeUnauthorized, + fmt.Sprintf("cannot read ServiceAccount %q in namespace %q, so it is impossible to tell whether "+ + "--service-account-name names an existing ServiceAccount to run as verbatim or is a prefix "+ + "for one this run creates; refusing to guess, because guessing \"prefix\" would run the agent "+ + "under a generated ServiceAccount carrying none of that account's cloud credentials. "+ + "Grant 'get serviceaccounts' in this namespace and re-run", name, d.config.Namespace), + err, map[string]any{attrName: name, attrNamespace: d.config.Namespace}) default: return errors.Wrap(errors.ErrCodeInternal, "failed to check for an existing ServiceAccount", err) } @@ -172,11 +185,11 @@ func namespacedRules() []rbacv1.PolicyRule { { APIGroups: []string{""}, Resources: []string{resourceCM}, - Verbs: []string{verbCreate, verbGet, "update", "patch"}, + Verbs: []string{verbCreate, verbGet, verbUpdate, verbPatch}, }, { APIGroups: []string{""}, - Resources: []string{"pods"}, + Resources: []string{resourcePods}, Verbs: []string{verbGet, verbList}, }, } @@ -193,12 +206,12 @@ func clusterRules(discoverNetwork bool) []rbacv1.PolicyRule { rules := []rbacv1.PolicyRule{ { APIGroups: []string{""}, - Resources: []string{"nodes"}, + Resources: []string{resourceNodes}, Verbs: []string{verbGet, verbList}, }, { APIGroups: []string{""}, - Resources: []string{"pods"}, + Resources: []string{resourcePods}, Verbs: []string{verbGet, verbList}, }, { @@ -421,7 +434,6 @@ func (d *Deployer) deleteClusterRoleBinding(ctx context.Context, name string, ui // - nicclusterpolicies: l8k patches the user's NicClusterPolicy // (NicConfigurationOperator section) via server-side apply. func discoverNetworkClusterRules() []rbacv1.PolicyRule { - const verbUpdate, verbPatch, verbWatch = "update", "patch", "watch" return []rbacv1.PolicyRule{ { APIGroups: []string{"apiextensions.k8s.io"}, @@ -440,22 +452,22 @@ func discoverNetworkClusterRules() []rbacv1.PolicyRule { }, { APIGroups: []string{""}, - Resources: []string{"serviceaccounts", "configmaps"}, + Resources: []string{resourceServiceAccounts, resourceCM}, Verbs: []string{verbGet, verbCreate, verbDelete}, }, { APIGroups: []string{rbacAPIGroup}, - Resources: []string{"roles", "rolebindings"}, + Resources: []string{resourceRoles, resourceRoleBindings}, Verbs: []string{verbGet, verbCreate, verbDelete}, }, { APIGroups: []string{""}, - Resources: []string{"pods/exec"}, + Resources: []string{resourcePods + "/exec"}, Verbs: []string{verbCreate}, }, { APIGroups: []string{""}, - Resources: []string{"nodes"}, + Resources: []string{resourceNodes}, Verbs: []string{verbPatch}, }, { diff --git a/pkg/k8s/agent/rbac_test.go b/pkg/k8s/agent/rbac_test.go index ea71ac436..d6140fbf6 100644 --- a/pkg/k8s/agent/rbac_test.go +++ b/pkg/k8s/agent/rbac_test.go @@ -53,10 +53,12 @@ func captureLogs(t *testing.T) *bytes.Buffer { // operator-supplied ServiceAccount verbatim (and manages none of its // permissions) or treats the value as a prefix and creates its own. // -// The Forbidden branch is the load-bearing one: `serviceaccounts get` is NOT -// in CheckPermissions' requiredChecks, so an identity holding exactly the -// pre-flight verb set must still be able to deploy. It falls back to prefix -// mode, which is the mode that identity has the permissions for. +// The Forbidden branch is the load-bearing one, and it now fails closed. +// `serviceaccounts get` is a REQUIRED check in CheckPermissions, evaluated +// before this runs, so a Forbidden here means the authorizer's answer and +// the apiserver's behavior disagree. The old downgrade — continue in prefix +// mode — silently ran an operator who named their IRSA / Workload Identity +// ServiceAccount under a generated one carrying none of its annotations. func TestResolveServiceAccount(t *testing.T) { saGR := schema.GroupResource{Group: "", Resource: "serviceaccounts"} @@ -69,6 +71,10 @@ func TestResolveServiceAccount(t *testing.T) { seeded string // getErr, when non-nil, is returned by every ServiceAccount Get. getErr error + // wantErrCode is the structured code the failure must carry. A + // permission problem must surface as ErrCodeUnauthorized, not as + // an opaque internal error. + wantErrCode aicrerrors.ErrorCode // wantExisting is the ServiceAccount the run should adopt // verbatim; "" means prefix mode. wantExisting string @@ -94,18 +100,20 @@ func TestResolveServiceAccount(t *testing.T) { notWantLog: "aicr manages no RBAC for this run", }, { - name: "forbidden Get falls back to prefix mode", - configured: "irsa-snapshotter", - seeded: "irsa-snapshotter", - getErr: apierrors.NewForbidden(saGR, "irsa-snapshotter", stderrors.New("no get permission")), - wantLogSubstr: "treating --service-account-name as a prefix", - notWantLog: "aicr manages no RBAC for this run", + name: "forbidden Get fails closed instead of downgrading to a prefix", + configured: "irsa-snapshotter", + seeded: "irsa-snapshotter", + getErr: apierrors.NewForbidden(saGR, "irsa-snapshotter", stderrors.New("no get permission")), + wantErr: true, + wantErrCode: aicrerrors.ErrCodeUnauthorized, + notWantLog: "aicr manages no RBAC for this run", }, { - name: "unexpected Get error fails closed", - configured: "irsa-snapshotter", - getErr: apierrors.NewInternalError(stderrors.New("apiserver exploded")), - wantErr: true, + name: "unexpected Get error fails closed", + configured: "irsa-snapshotter", + getErr: apierrors.NewInternalError(stderrors.New("apiserver exploded")), + wantErr: true, + wantErrCode: aicrerrors.ErrCodeInternal, }, } @@ -138,8 +146,8 @@ func TestResolveServiceAccount(t *testing.T) { if (err != nil) != tt.wantErr { t.Fatalf("resolveServiceAccount() error = %v, wantErr %v", err, tt.wantErr) } - if tt.wantErr && !stderrors.Is(err, aicrerrors.New(aicrerrors.ErrCodeInternal, "")) { - t.Errorf("error = %v, want ErrCodeInternal", err) + if tt.wantErr && !stderrors.Is(err, aicrerrors.New(tt.wantErrCode, "")) { + t.Errorf("error = %v, want code %s", err, tt.wantErrCode) } if got := d.existingServiceAccount(); got != tt.wantExisting { t.Errorf("existingServiceAccount() = %q, want %q", got, tt.wantExisting) @@ -341,23 +349,35 @@ func assertNoRBACObjects(ctx context.Context, t *testing.T, clientset *fake.Clie } } -// allowAllPermissionChecks makes every SelfSubjectAccessReview succeed so a -// test exercises Deploy past its Step 0 pre-flight. +// allowAllPermissionChecks makes every access review succeed so a test +// exercises Deploy past its Step 0 pre-flight. Both kinds are answered: +// SelfSubjectAccessReview covers the caller's verbs, and SubjectAccessReview +// covers the agent ServiceAccount's own rules, which the gate checks in +// exact-ServiceAccount mode. func allowAllPermissionChecks(clientset *fake.Clientset) { clientset.PrependReactor("create", "selfsubjectaccessreviews", func(k8stesting.Action) (bool, runtime.Object, error) { return true, &authv1.SelfSubjectAccessReview{ Status: authv1.SubjectAccessReviewStatus{Allowed: true, Reason: "pre-flight verb set granted"}, }, nil }) + clientset.PrependReactor("create", "subjectaccessreviews", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, &authv1.SubjectAccessReview{ + Status: authv1.SubjectAccessReviewStatus{Allowed: true, Reason: "ServiceAccount rules granted"}, + }, nil + }) } -// TestDeploy_SucceedsWhenServiceAccountGetForbidden is the end-to-end shape of -// the same bug: an identity authorized for exactly CheckPermissions' -// requiredChecks (which do not include `serviceaccounts get`) passes the -// pre-flight, so Deploy must not then fail on the exact-if-exists Get. -// ServiceAccountName is set because that Get is issued only when it is — -// leaving it empty would make the test pass without reaching the branch. -func TestDeploy_SucceedsWhenServiceAccountGetForbidden(t *testing.T) { +// TestDeploy_FailsWhenServiceAccountGetForbidden is the end-to-end shape of +// the closed hole: an explicitly-named ServiceAccount that cannot be read +// must stop the run, not be silently reinterpreted as a name prefix. +// +// The SelfSubjectAccessReview reactor says `serviceaccounts: get` is +// allowed while the Get itself returns Forbidden — the authorizer and the +// apiserver disagreeing. That is precisely the state the run must not guess +// through, because guessing "prefix" deploys the agent under a generated +// ServiceAccount carrying none of the named account's cloud credentials. +// ServiceAccountName is set because the Get is issued only when it is. +func TestDeploy_FailsWhenServiceAccountGetForbidden(t *testing.T) { ctx := context.Background() captureLogs(t) @@ -365,7 +385,7 @@ func TestDeploy_SucceedsWhenServiceAccountGetForbidden(t *testing.T) { allowAllPermissionChecks(clientset) clientset.PrependReactor("get", "serviceaccounts", func(k8stesting.Action) (bool, runtime.Object, error) { return true, nil, apierrors.NewForbidden( - schema.GroupResource{Group: "", Resource: "serviceaccounts"}, testName, + schema.GroupResource{Group: "", Resource: resourceServiceAccounts}, testName, stderrors.New(`User "snapshot-runner" cannot get resource "serviceaccounts"`)) }) @@ -376,23 +396,27 @@ func TestDeploy_SucceedsWhenServiceAccountGetForbidden(t *testing.T) { RunID: testRunID, }) - if err := d.Deploy(ctx); err != nil { - t.Fatalf("Deploy() error = %v, want nil (the exact-if-exists Get must not gate deployment)", err) + err := d.Deploy(ctx) + if err == nil { + t.Fatal("Deploy() error = nil; an unreadable, explicitly-named ServiceAccount must fail the run") } - - if _, err := clientset.BatchV1().Jobs(testNamespace).Get(ctx, "aicr-"+testRunID, metav1.GetOptions{}); err != nil { - t.Errorf("Job not created: %v", err) + if !stderrors.Is(err, aicrerrors.New(aicrerrors.ErrCodeUnauthorized, "")) { + t.Errorf("Deploy() error code = %v, want ErrCodeUnauthorized", err) } - // Forbidden must fall back to prefix mode, which still creates the - // run-scoped RBAC set. - if !d.managesRBAC() { - t.Error("managesRBAC() = false; a Forbidden Get must not be read as an adopted ServiceAccount") + if !strings.Contains(err.Error(), "refusing to guess") { + t.Errorf("Deploy() error = %v, want it to say the run refuses to guess the mode", err) } - // Read through the tracker, not the clientset: the reactor above stands - // in for an identity that cannot read ServiceAccounts at all, and that - // must not also blind the assertion. - saGVR := corev1.SchemeGroupVersion.WithResource("serviceaccounts") - if _, err := clientset.Tracker().Get(saGVR, testNamespace, "aicr-"+testRunID); err != nil { - t.Errorf("run-scoped ServiceAccount not created: %v", err) + + // Nothing may have been written: the gate fails before Deploy's ensure* + // chain runs. Read through the tracker, not the clientset — the reactor + // above stands in for an identity that cannot read ServiceAccounts at + // all, and that must not also blind the assertion. + if _, jobErr := clientset.BatchV1().Jobs(testNamespace).Get(ctx, "aicr-"+testRunID, metav1.GetOptions{}); !apierrors.IsNotFound(jobErr) { + t.Errorf("Job Get error = %v, want NotFound (no Job may be created)", jobErr) } + saGVR := corev1.SchemeGroupVersion.WithResource(resourceServiceAccounts) + if _, saErr := clientset.Tracker().Get(saGVR, testNamespace, "aicr-"+testRunID); !apierrors.IsNotFound(saErr) { + t.Errorf("run-scoped ServiceAccount Get error = %v, want NotFound", saErr) + } + assertNoRBACObjects(ctx, t, clientset) } From 85403e8a1599bac1a068533bec9aae8f38dcc51c Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Tue, 25 Aug 2026 16:25:28 -0700 Subject: [PATCH 50/56] docs(agent): document cleanup's label+UID re-verification of lost creates The package godoc still described the pre-hardening cleanup algorithm: a lost-Create entry deleted by its run-unique name with no UID precondition. resolveIntentUID has not done that since the created-set was hardened -- it Gets the live object, requires the full createdByThisRun label set AND a non-empty UID, and fails closed (no delete, warn) otherwise. Rewrite the paragraph to match, so nobody 'restores consistency' by reintroducing the bare-name delete. Signed-off-by: Alex Yuskauskas --- pkg/k8s/agent/doc.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/pkg/k8s/agent/doc.go b/pkg/k8s/agent/doc.go index c075b1124..f9a9f8a38 100644 --- a/pkg/k8s/agent/doc.go +++ b/pkg/k8s/agent/doc.go @@ -106,11 +106,18 @@ NotFound are both treated as success. Cleanup also runs on the Deploy failure path, which is why it is scoped to what was created rather than to configured names. -Recording before the Create is what keeps a lost Create response from orphaning -an object forever: if the apiserver commits the create but the response never -arrives, the entry is already in the set and Cleanup deletes it by its -(run-unique) name with no UID precondition. The one response that proves the -object is not ours — AlreadyExists — discards the entry again. +Recording before the Create is what keeps a lost Create response from +orphaning an object forever: if the apiserver commits the create but the +response never arrives, the entry is already in the set. That entry carries no +UID, and its (run-unique) name is not evidence of ownership — it says what +this run WOULD have created, not what is standing there now — so Cleanup never +deletes it by bare name. It Gets the live object and re-verifies it: the +delete is issued only when that object carries the full label set this run +stamps at creation time AND a non-empty UID, and it is pinned to the UID that +Get observed. A label mismatch or a missing UID fails closed — no delete at +all, and a warning names the object left behind for an operator to judge — +while a NotFound means there is nothing to reclaim. The one response that +proves the object is not ours — AlreadyExists — discards the entry again. The staging ConfigMap is written by the in-pod agent, so its UID is observed when GetSnapshot reads it. When the run owns that ConfigMap and failed before From a9e0d0577f903d86466b820c2c266414cfb1b8c5 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Tue, 25 Aug 2026 16:25:35 -0700 Subject: [PATCH 51/56] test(agent): cover cleanup's empty-RunID and missing-UID refuse paths Two clauses that keep cleanup from deleting an object this run cannot prove it created had no test driving them: - createdByThisRun's RunID == "" guard: with no run ID, an empty label value must not match a label-less object. - resolveIntentUID's live.GetUID() == "" clause: every seeded object carried a UID, so the refuse path was only ever reached through the label-mismatch clause. Add a row for each to the existing table, plus an optional per-row RunID override (nil keeps testRunID). Signed-off-by: Alex Yuskauskas --- pkg/k8s/agent/deployer_test.go | 37 ++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/pkg/k8s/agent/deployer_test.go b/pkg/k8s/agent/deployer_test.go index f351137d1..345fbc793 100644 --- a/pkg/k8s/agent/deployer_test.go +++ b/pkg/k8s/agent/deployer_test.go @@ -42,6 +42,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/fake" k8stesting "k8s.io/client-go/testing" + "k8s.io/utils/ptr" ) const testName = "aicr" @@ -919,7 +920,8 @@ func TestCleanupPassesUIDPrecondition(t *testing.T) { // named it. The run-scoped name alone is not ownership evidence — it says what // this run WOULD have created, not what is standing there now — so Cleanup // must recover the UID from the live object and prove that object carries this -// run's labels before deleting anything. +// run's labels before deleting anything. Ownership takes both halves: a label +// mismatch and a missing UID each refuse the delete on their own. // // The "replaced under the same name" case is the one this replaces a bare-name // delete for: an object at that name with a different UID and someone else's @@ -948,6 +950,7 @@ func TestCleanupResolvesUnconfirmedEntryBeforeDeleting(t *testing.T) { tests := []struct { name string seed *corev1.ServiceAccount // nil: nothing at the name + runID *string // nil: testRunID wantDelete bool wantUID types.UID // expected Preconditions.UID when wantDelete wantWarn bool @@ -980,6 +983,32 @@ func TestCleanupResolvesUnconfirmedEntryBeforeDeleting(t *testing.T) { wantDelete: false, wantWarn: true, }, + { + // Labels alone would clear this object — they are this run's + // own — but a real apiserver always assigns a UID, so a + // missing one means the delete cannot be pinned. Refusing is + // the point: the fallback would be the bare-name delete this + // path exists to prevent. + name: "this run's own labels without a UID still survive", + seed: &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{ + Name: saName, Namespace: ns, Labels: ourLabels, + }}, + wantDelete: false, + wantWarn: true, + }, + { + // An empty Config.RunID is not a wildcard: the run-ID + // comparison alone would let "" == "" pass a label-less + // object off as this run's, so createdByThisRun matches + // nothing at all when this run has no ID to match on. + name: "an empty RunID proves ownership of nothing", + seed: &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{ + Name: saName, Namespace: ns, UID: types.UID("operators-uid"), + }}, + runID: ptr.To(""), + wantDelete: false, + wantWarn: true, + }, } for _, tt := range tests { @@ -994,7 +1023,11 @@ func TestCleanupResolvesUnconfirmedEntryBeforeDeleting(t *testing.T) { } deletes := spyOnDeletes(client) - run := NewDeployer(client, Config{Namespace: ns, RunID: testRunID}) + runID := testRunID + if tt.runID != nil { + runID = *tt.runID + } + run := NewDeployer(client, Config{Namespace: ns, RunID: runID}) run.recordIntent(kindServiceAccount, saName) if err := run.Cleanup(ctx, CleanupOptions{Enabled: true}); err != nil { From c328b21d77972a329f909d8b078220290fff0137 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Tue, 25 Aug 2026 16:28:00 -0700 Subject: [PATCH 52/56] test(agent): make the empty-RunID guard row fail without the guard The row seeded a label-less object, which createdByThisRun rejects on the labels.Name comparison whether or not the empty-RunID guard exists, so deleting the guard left the test green. Seed every aicr label except the run ID instead. objLabels[labels.RunID] is then "", so against an empty Config.RunID all four comparisons pass and the guard is the only thing refusing the match. Verified by removing the guard: the row fails. Signed-off-by: Alex Yuskauskas --- pkg/k8s/agent/deployer_test.go | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/pkg/k8s/agent/deployer_test.go b/pkg/k8s/agent/deployer_test.go index 345fbc793..f7279ab0b 100644 --- a/pkg/k8s/agent/deployer_test.go +++ b/pkg/k8s/agent/deployer_test.go @@ -937,6 +937,18 @@ func TestCleanupResolvesUnconfirmedEntryBeforeDeleting(t *testing.T) { saName := d.saName() ourLabels := d.objectLabels() + // Every aicr label except the run ID. This is the shape that makes the + // empty-RunID guard load-bearing: objLabels[labels.RunID] is "" here, so + // against an empty Config.RunID the run-ID comparison SUCCEEDS ("" == "") + // and the other three match too. Without the guard, createdByThisRun + // would claim this object. A label-less seed cannot show that — it is + // rejected on labels.Name regardless. + runIDLessLabels := map[string]string{ + labels.Name: labels.ValueAICR, + labels.ManagedBy: labels.ValueAICR, + labels.Component: labels.ValueSnapshotAgent, + } + // A replacement created after this run's object was deleted: same name, // different identity. Carries a foreign run's labels, which is what a // second aicr run standing an object up at this name would stamp. @@ -997,13 +1009,16 @@ func TestCleanupResolvesUnconfirmedEntryBeforeDeleting(t *testing.T) { wantWarn: true, }, { - // An empty Config.RunID is not a wildcard: the run-ID - // comparison alone would let "" == "" pass a label-less - // object off as this run's, so createdByThisRun matches - // nothing at all when this run has no ID to match on. + // An empty Config.RunID is not a wildcard. The seed carries + // every aicr label but the run ID, so all four comparisons + // would pass against an empty RunID -- "" == "" included. + // createdByThisRun must still match nothing, because a run + // with no ID has no ownership to prove. Deleting the guard + // fails this row; a label-less seed would not. name: "an empty RunID proves ownership of nothing", seed: &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{ Name: saName, Namespace: ns, UID: types.UID("operators-uid"), + Labels: runIDLessLabels, }}, runID: ptr.To(""), wantDelete: false, From af60ee6acfdb2e306619d9ae77919927832564b2 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Wed, 26 Aug 2026 15:20:23 -0700 Subject: [PATCH 53/56] test(agent): satisfy whitespace, unparam and prealloc in the gate tests Seven golangci-lint findings, all in permissions_test.go: - whitespace/multi-if (x5): hoist each inline closure to a named variable so the 'if' condition is single-line. The package's other tests avoid multi-line if conditions entirely, so this matches them rather than padding the body with a blank line. - unparam: seedServiceAccount's namespace parameter only ever received testNamespace; drop it. - prealloc: size wantAll from the two slices it concatenates. Signed-off-by: Alex Yuskauskas --- pkg/k8s/agent/permissions_test.go | 53 +++++++++++++++++-------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/pkg/k8s/agent/permissions_test.go b/pkg/k8s/agent/permissions_test.go index a36d5c296..ea57df0be 100644 --- a/pkg/k8s/agent/permissions_test.go +++ b/pkg/k8s/agent/permissions_test.go @@ -174,12 +174,12 @@ func installReviewReactors(t *testing.T, cs *fake.Clientset, allow func(askedAcc // seedServiceAccount pre-creates the operator-provisioned ServiceAccount // that puts a Deployer into exact-ServiceAccount mode. -func seedServiceAccount(t *testing.T, cs *fake.Clientset, namespace, name string) { +func seedServiceAccount(t *testing.T, cs *fake.Clientset, name string) { t.Helper() - if _, err := cs.CoreV1().ServiceAccounts(namespace).Create(context.Background(), &corev1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + if _, err := cs.CoreV1().ServiceAccounts(testNamespace).Create(context.Background(), &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: testNamespace}, }, metav1.CreateOptions{}); err != nil { - t.Fatalf("seeding ServiceAccount %s/%s: %v", namespace, name, err) + t.Fatalf("seeding ServiceAccount %s/%s: %v", testNamespace, name, err) } } @@ -265,16 +265,18 @@ func TestCheckPermissions_PrefixModeRequiresRBACCreateAndDelete(t *testing.T) { if !strings.Contains(err.Error(), wantLine) { t.Errorf("error = %v\nwant a line containing %q", err, wantLine) } - if !hasCheck(results, func(p permissionCheck) bool { + carriesDenied := func(p permissionCheck) bool { return p.Resource == req.resource && p.Verb == req.verb && !p.Allowed - }) { + } + if !hasCheck(results, carriesDenied) { t.Errorf("results do not carry the denied %s %s check", req.verb, req.resource) } // The gate must have resolved the mode first, which means it // asked whether it may read ServiceAccounts. - if !rec.asked1(func(q askedAccess) bool { + askedSAGet := func(q askedAccess) bool { return q.subject == "" && q.resource == resourceServiceAccounts && q.verb == verbGet - }) { + } + if !rec.asked1(askedSAGet) { t.Error("gate never asked for `serviceaccounts: get`, so it cannot have resolved the mode") } }) @@ -297,7 +299,7 @@ func rbacGroupFor(resource string) string { // mode exists for. func TestCheckPermissions_ExactModeSkipsCallerRBACVerbs(t *testing.T) { clientset := fake.NewClientset() - seedServiceAccount(t, clientset, testNamespace, exactSAName) + seedServiceAccount(t, clientset, exactSAName) // Deny every caller-side RBAC verb outright. A correct gate never asks. rec := installReviewReactors(t, clientset, func(q askedAccess) bool { @@ -329,18 +331,20 @@ func TestCheckPermissions_ExactModeSkipsCallerRBACVerbs(t *testing.T) { if hasCheck(results, isCallerRBACCheck) { t.Error("gate demanded a caller RBAC verb in exact-ServiceAccount mode") } - if rec.asked1(func(q askedAccess) bool { + askedCallerRBAC := func(q askedAccess) bool { return q.subject == "" && q.resource == resourceClusterRoles - }) { + } + if rec.asked1(askedCallerRBAC) { t.Error("gate asked the apiserver about clusterroles in exact-ServiceAccount mode") } // It must instead have asked the ServiceAccount's own questions, and // asked them of the ServiceAccount rather than of the caller. subject := serviceAccountUsername(testNamespace, exactSAName) - if !rec.asked1(func(q askedAccess) bool { + askedClusterRoles := func(q askedAccess) bool { return q.subject == subject && q.resource == resourceNodes && q.verb == verbList - }) { + } + if !rec.asked1(askedClusterRoles) { t.Errorf("gate never asked whether %s may list nodes", subject) } } @@ -352,7 +356,7 @@ func TestCheckPermissions_ExactModeSkipsCallerRBACVerbs(t *testing.T) { // account as a generated one carrying none of its cloud annotations. func TestCheckPermissions_ServiceAccountGetIsRequired(t *testing.T) { clientset := fake.NewClientset() - seedServiceAccount(t, clientset, testNamespace, exactSAName) + seedServiceAccount(t, clientset, exactSAName) rec := installReviewReactors(t, clientset, func(q askedAccess) bool { return q.resource != resourceServiceAccounts || q.verb != verbGet }) @@ -401,7 +405,7 @@ func TestCheckPermissions_ServiceAccountGetIsRequired(t *testing.T) { // far better caught here than in a pod minutes later. func TestCheckPermissions_ServiceAccountSubjectFailureNamesTheSubject(t *testing.T) { clientset := fake.NewClientset() - seedServiceAccount(t, clientset, testNamespace, exactSAName) + seedServiceAccount(t, clientset, exactSAName) installReviewReactors(t, clientset, func(q askedAccess) bool { // The caller is fully privileged; the ServiceAccount is missing // exactly the cluster-scoped node read the agent cannot work without. @@ -428,9 +432,10 @@ func TestCheckPermissions_ServiceAccountSubjectFailureNamesTheSubject(t *testing } } // A caller-scoped answer must never be substituted for the subject's. - if !hasCheck(results, func(p permissionCheck) bool { + askedSubjectNodes := func(p permissionCheck) bool { return p.Subject == subject && p.Resource == resourceNodes && !p.Allowed && !p.Unverified - }) { + } + if !hasCheck(results, askedSubjectNodes) { t.Error("results carry no denied ServiceAccount-subject check for nodes") } } @@ -443,7 +448,7 @@ func TestCheckPermissions_ServiceAccountSubjectFailureNamesTheSubject(t *testing // unverified, warns, and lets the run proceed to fail visibly in-pod. func TestCheckPermissions_SubjectAccessReviewForbiddenReportsAndContinues(t *testing.T) { clientset := fake.NewClientset() - seedServiceAccount(t, clientset, testNamespace, exactSAName) + seedServiceAccount(t, clientset, exactSAName) installReviewReactors(t, clientset, nil) // Prepended after installReviewReactors, so it wins for this resource. clientset.PrependReactor(verbCreate, subjectReviewResource, func(k8stesting.Action) (bool, runtime.Object, error) { @@ -556,7 +561,7 @@ func TestCheckPermissions_IssuesNoWriteBeforeTheGateCloses(t *testing.T) { clientset := fake.NewClientset() if tt.exact { // Seeded through the tracker before the guard is armed. - seedServiceAccount(t, clientset, testNamespace, exactSAName) + seedServiceAccount(t, clientset, exactSAName) } installReviewReactors(t, clientset, tt.allow) @@ -689,7 +694,7 @@ func TestCheckPermissions_AllGrantedPassesBothModes(t *testing.T) { t.Run(tt.name, func(t *testing.T) { clientset := fake.NewClientset() if tt.exact { - seedServiceAccount(t, clientset, testNamespace, exactSAName) + seedServiceAccount(t, clientset, exactSAName) } rec := installReviewReactors(t, clientset, nil) @@ -832,9 +837,11 @@ func TestServiceAccountChecksTrackTheGrantedRules(t *testing.T) { got[c] = struct{}{} } - var wantAll []accessCheck - wantAll = append(wantAll, checksFromRules(namespacedRules(), testNamespace, subject)...) - wantAll = append(wantAll, checksFromRules(clusterRules(discover), "", subject)...) + nsChecks := checksFromRules(namespacedRules(), testNamespace, subject) + clusterChecks := checksFromRules(clusterRules(discover), "", subject) + wantAll := make([]accessCheck, 0, len(nsChecks)+len(clusterChecks)) + wantAll = append(wantAll, nsChecks...) + wantAll = append(wantAll, clusterChecks...) for _, c := range wantAll { if _, ok := got[c]; !ok { t.Errorf("granted rule not checked: %+v", c) From 084e872fea39cd99fb57e01f1ba8c07c475c2ca2 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Wed, 26 Aug 2026 15:26:58 -0700 Subject: [PATCH 54/56] test(agent): drop seedServiceAccount's always-constant name param unparam cascades: removing the namespace parameter left name as the only argument, and every call site passed exactSAName. Fix both by naming the constants inside the helper, which is what the callers meant anyway. Signed-off-by: Alex Yuskauskas --- pkg/k8s/agent/permissions_test.go | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/pkg/k8s/agent/permissions_test.go b/pkg/k8s/agent/permissions_test.go index ea57df0be..87969b583 100644 --- a/pkg/k8s/agent/permissions_test.go +++ b/pkg/k8s/agent/permissions_test.go @@ -173,13 +173,16 @@ func installReviewReactors(t *testing.T, cs *fake.Clientset, allow func(askedAcc } // seedServiceAccount pre-creates the operator-provisioned ServiceAccount -// that puts a Deployer into exact-ServiceAccount mode. -func seedServiceAccount(t *testing.T, cs *fake.Clientset, name string) { +// that puts a Deployer into exact-ServiceAccount mode. Its namespace and +// name are fixed rather than parameters: every caller wants the one +// ServiceAccount that testNamespace/exactSAName names, and passing the same +// two constants at each call site is what unparam flags. +func seedServiceAccount(t *testing.T, cs *fake.Clientset) { t.Helper() if _, err := cs.CoreV1().ServiceAccounts(testNamespace).Create(context.Background(), &corev1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: testNamespace}, + ObjectMeta: metav1.ObjectMeta{Name: exactSAName, Namespace: testNamespace}, }, metav1.CreateOptions{}); err != nil { - t.Fatalf("seeding ServiceAccount %s/%s: %v", testNamespace, name, err) + t.Fatalf("seeding ServiceAccount %s/%s: %v", testNamespace, exactSAName, err) } } @@ -299,7 +302,7 @@ func rbacGroupFor(resource string) string { // mode exists for. func TestCheckPermissions_ExactModeSkipsCallerRBACVerbs(t *testing.T) { clientset := fake.NewClientset() - seedServiceAccount(t, clientset, exactSAName) + seedServiceAccount(t, clientset) // Deny every caller-side RBAC verb outright. A correct gate never asks. rec := installReviewReactors(t, clientset, func(q askedAccess) bool { @@ -356,7 +359,7 @@ func TestCheckPermissions_ExactModeSkipsCallerRBACVerbs(t *testing.T) { // account as a generated one carrying none of its cloud annotations. func TestCheckPermissions_ServiceAccountGetIsRequired(t *testing.T) { clientset := fake.NewClientset() - seedServiceAccount(t, clientset, exactSAName) + seedServiceAccount(t, clientset) rec := installReviewReactors(t, clientset, func(q askedAccess) bool { return q.resource != resourceServiceAccounts || q.verb != verbGet }) @@ -405,7 +408,7 @@ func TestCheckPermissions_ServiceAccountGetIsRequired(t *testing.T) { // far better caught here than in a pod minutes later. func TestCheckPermissions_ServiceAccountSubjectFailureNamesTheSubject(t *testing.T) { clientset := fake.NewClientset() - seedServiceAccount(t, clientset, exactSAName) + seedServiceAccount(t, clientset) installReviewReactors(t, clientset, func(q askedAccess) bool { // The caller is fully privileged; the ServiceAccount is missing // exactly the cluster-scoped node read the agent cannot work without. @@ -448,7 +451,7 @@ func TestCheckPermissions_ServiceAccountSubjectFailureNamesTheSubject(t *testing // unverified, warns, and lets the run proceed to fail visibly in-pod. func TestCheckPermissions_SubjectAccessReviewForbiddenReportsAndContinues(t *testing.T) { clientset := fake.NewClientset() - seedServiceAccount(t, clientset, exactSAName) + seedServiceAccount(t, clientset) installReviewReactors(t, clientset, nil) // Prepended after installReviewReactors, so it wins for this resource. clientset.PrependReactor(verbCreate, subjectReviewResource, func(k8stesting.Action) (bool, runtime.Object, error) { @@ -561,7 +564,7 @@ func TestCheckPermissions_IssuesNoWriteBeforeTheGateCloses(t *testing.T) { clientset := fake.NewClientset() if tt.exact { // Seeded through the tracker before the guard is armed. - seedServiceAccount(t, clientset, exactSAName) + seedServiceAccount(t, clientset) } installReviewReactors(t, clientset, tt.allow) @@ -694,7 +697,7 @@ func TestCheckPermissions_AllGrantedPassesBothModes(t *testing.T) { t.Run(tt.name, func(t *testing.T) { clientset := fake.NewClientset() if tt.exact { - seedServiceAccount(t, clientset, exactSAName) + seedServiceAccount(t, clientset) } rec := installReviewReactors(t, clientset, nil) From 7bec43ef45c2f3143cb39697d2028eaf0d8ca05f Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Wed, 26 Aug 2026 15:44:39 -0700 Subject: [PATCH 55/56] fix(agent): make the provisioned ClusterRole name injective The cluster-scoped pair joined namespace and ServiceAccount with "-", which is not injective: namespace "a-b" with ServiceAccount "c" and namespace "a" with ServiceAccount "b-c" compose the same name. Applying the second render over the first would retarget the live ClusterRoleBinding and revoke the first ServiceAccount's cluster permissions, and the generator reads no cluster so it could not detect the collision -- only warn about it. Join on "." instead. A namespace is a DNS-1123 label and cannot contain a dot, while a ClusterRole name is a DNS-1123 subdomain and can, so the first dot always separates the two segments whatever either holds. The rendered header and the operator-facing docs drop the collision warning they no longer need. Signed-off-by: Alex Yuskauskas --- docs/user/agent-deployment.md | 13 +++++------ pkg/cli/snapshot_test.go | 6 ++--- pkg/k8s/agent/provision.go | 31 +++++++++++++------------ pkg/k8s/agent/provision_test.go | 40 ++++++++++++++++++++++++++++++++- 4 files changed, 62 insertions(+), 28 deletions(-) diff --git a/docs/user/agent-deployment.md b/docs/user/agent-deployment.md index ae6c836e8..d182aed11 100644 --- a/docs/user/agent-deployment.md +++ b/docs/user/agent-deployment.md @@ -318,7 +318,7 @@ If you no longer have the directory, delete the objects by name instead: ```shell kubectl delete role,rolebinding aicr-agent-irsa-snapshotter-rbac -n gpu-operator -kubectl delete clusterrole,clusterrolebinding aicr-agent-gpu-operator-irsa-snapshotter-rbac +kubectl delete clusterrole,clusterrolebinding aicr-agent-gpu-operator.irsa-snapshotter-rbac ``` The directory name carries a fresh run ID on every invocation, so generating @@ -326,13 +326,10 @@ twice never overwrites a set you are still reviewing; a directory that already exists fails the command with `CONFLICT`. After an aicr upgrade, re-generate and `kubectl apply -f` the new directory to refresh the rules in place. -**Check for a name collision before applying the cluster-scoped pair.** Their -name joins the namespace and the ServiceAccount name with `-`, which is not -injective: namespace `a-b` with ServiceAccount `c` and namespace `a` with -ServiceAccount `b-c` both compose `aicr-agent-a-b-c-rbac`. Because nothing -reads your cluster, applying over an existing binding of that name would -retarget it and revoke the other ServiceAccount's grants. The generated -`04-clusterrolebinding.yaml` says so and gives you the `kubectl get` to run. +The cluster-scoped pair is named `aicr-agent-.-rbac`. +The two segments join on `.`, which a namespace can never contain, so no other +namespace and ServiceAccount combination can produce the same name — applying +one grant cannot retarget another's. ### Trade-off: per-run permission isolation is waived diff --git a/pkg/cli/snapshot_test.go b/pkg/cli/snapshot_test.go index e69714577..674970c78 100644 --- a/pkg/cli/snapshot_test.go +++ b/pkg/cli/snapshot_test.go @@ -605,8 +605,8 @@ func TestWriteManifestReport(t *testing.T) { Objects: []snapshotter.AgentRoleObject{ {Kind: "Role", Name: "aicr-agent-irsa-snapshotter-rbac", Path: dir + "/01-role.yaml"}, {Kind: "RoleBinding", Name: "aicr-agent-irsa-snapshotter-rbac", Path: dir + "/02-rolebinding.yaml"}, - {Kind: "ClusterRole", Name: "aicr-agent-gpu-operator-irsa-snapshotter-rbac", Path: dir + "/03-clusterrole.yaml"}, - {Kind: "ClusterRoleBinding", Name: "aicr-agent-gpu-operator-irsa-snapshotter-rbac", Path: dir + "/04-clusterrolebinding.yaml"}, + {Kind: "ClusterRole", Name: "aicr-agent-gpu-operator.irsa-snapshotter-rbac", Path: dir + "/03-clusterrole.yaml"}, + {Kind: "ClusterRoleBinding", Name: "aicr-agent-gpu-operator.irsa-snapshotter-rbac", Path: dir + "/04-clusterrolebinding.yaml"}, }, } @@ -625,7 +625,7 @@ func TestWriteManifestReport(t *testing.T) { "kubectl delete -f " + dir + "/", "01-role.yaml", "role/aicr-agent-irsa-snapshotter-rbac", - "clusterrolebinding/aicr-agent-gpu-operator-irsa-snapshotter-rbac", + "clusterrolebinding/aicr-agent-gpu-operator.irsa-snapshotter-rbac", "not verified to exist", "permission isolation is waived", "aicr snapshot --namespace gpu-operator --service-account-name irsa-snapshotter", diff --git a/pkg/k8s/agent/provision.go b/pkg/k8s/agent/provision.go index 6646b6154..491ecf27a 100644 --- a/pkg/k8s/agent/provision.go +++ b/pkg/k8s/agent/provision.go @@ -461,17 +461,9 @@ func clusterRoleBindingHeader(name, namespace, serviceAccount string) string { # Applying this is what gives ServiceAccount %[3]q the rules in # %[4]s, across every namespace in the cluster. # -# CHECK FOR A NAME COLLISION FIRST. This name joins the namespace and the -# ServiceAccount name with "-", which is not injective: namespace "a-b" with -# ServiceAccount "c" and namespace "a" with ServiceAccount "b-c" both compose -# "aicr-agent-a-b-c-rbac". aicr contacted no cluster and could not check. -# Applying over an existing binding of this name would retarget it and revoke -# the other ServiceAccount's cluster permissions: -# -# kubectl get clusterrolebinding %[1]s -o yaml -# -# If it already exists and names a different subject, rename one of the two -# namespaces or ServiceAccounts so the generated names differ. +# The name joins the namespace and the ServiceAccount on ".", which no +# namespace may contain, so no other (namespace, ServiceAccount) pair can +# compose this name. `, name, namespace, serviceAccount, clusterRoleFileName) } @@ -488,12 +480,19 @@ func provisionedRoleName(serviceAccount string) string { // objects are cluster-scoped and the same ServiceAccount name can exist in // several namespaces. // -// Joining two "-"-bearing segments is not injective ("a-b"/"c" and -// "a"/"b-c" compose the same string). Nothing here can detect that — no -// cluster is read — so the rendered ClusterRoleBinding warns about it in its -// header and tells the operator how to check before applying. +// The two segments join on "." rather than "-" so the composition is +// injective. A "-" join is not: namespace "a-b" with ServiceAccount "c" and +// namespace "a" with ServiceAccount "b-c" compose the same string, and +// applying the second render over the first would retarget the existing +// ClusterRoleBinding and revoke the other ServiceAccount's cluster +// permissions. Nothing here could detect that, because no cluster is read. +// +// "." is safe and sufficient: a namespace is a DNS-1123 *label*, which cannot +// contain a dot, while a ClusterRole name is a DNS-1123 *subdomain*, which +// can. The first dot therefore always separates namespace from ServiceAccount, +// whatever either contains. func provisionedClusterRoleName(namespace, serviceAccount string) string { - return provisionedNamePrefix + namespace + "-" + serviceAccount + provisionedNameSuffix + return provisionedNamePrefix + namespace + "." + serviceAccount + provisionedNameSuffix } // provisionedLabels is the label set stamped on every rendered object. diff --git a/pkg/k8s/agent/provision_test.go b/pkg/k8s/agent/provision_test.go index b9359b845..4ed8a99ba 100644 --- a/pkg/k8s/agent/provision_test.go +++ b/pkg/k8s/agent/provision_test.go @@ -16,6 +16,7 @@ package agent import ( stderrors "errors" + "k8s.io/apimachinery/pkg/util/validation" "reflect" "strings" "testing" @@ -52,6 +53,43 @@ func manifestByFile(manifests []Manifest) map[string]Manifest { return byFile } +// TestProvisionedClusterRoleNameIsInjective pins the property the "." join +// exists for. These two (namespace, ServiceAccount) pairs compose the same +// string under a "-" join, and the collision is not academic: the second +// render applied over the first retargets the live ClusterRoleBinding and +// revokes the first ServiceAccount's cluster permissions. Nothing in the +// generator can detect it, because it reads no cluster. +// +// A namespace is a DNS-1123 label and cannot contain ".", so the first dot +// always separates the two segments. Switching the join back to "-" fails +// this test. +func TestProvisionedClusterRoleNameIsInjective(t *testing.T) { + tests := []struct { + name string + namespace string + sa string + }{ + {"hyphen in the namespace", "a-b", "c"}, + {"hyphen in the ServiceAccount", "a", "b-c"}, + {"dot in the ServiceAccount is still unambiguous", "a", "b.c"}, + } + + seen := make(map[string]string, len(tests)) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := provisionedClusterRoleName(tt.namespace, tt.sa) + if errs := validation.IsDNS1123Subdomain(got); len(errs) > 0 { + t.Errorf("%q is not a valid ClusterRole name: %v", got, errs) + } + if prior, ok := seen[got]; ok { + t.Errorf("%s/%s collides with %s on name %q", + tt.namespace, tt.sa, prior, got) + } + seen[got] = tt.namespace + "/" + tt.sa + }) + } +} + // TestBuildServiceAccountRoleManifests_FilesAndNames pins the file layout and // the naming scheme together, because both are contracts an operator's // commands depend on: `kubectl apply -f /` relies on one parseable @@ -64,7 +102,7 @@ func TestBuildServiceAccountRoleManifests_FilesAndNames(t *testing.T) { manifests := buildManifests(t, false) wantRole := "aicr-agent-" + provisionSA + "-rbac" - wantClusterRole := "aicr-agent-" + testNamespace + "-" + provisionSA + "-rbac" + wantClusterRole := "aicr-agent-" + testNamespace + "." + provisionSA + "-rbac" want := []struct { file string From 583c79827cc2636245fbe5102635e6d09dc14306 Mon Sep 17 00:00:00 2001 From: Alex Yuskauskas Date: Wed, 26 Aug 2026 15:52:56 -0700 Subject: [PATCH 56/56] test(agent): group the validation import with the other k8s imports goimports keeps stdlib in the first group and third-party after it. The import was hand-placed into the stdlib group, which gofmt does not correct. Signed-off-by: Alex Yuskauskas --- pkg/k8s/agent/provision_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/k8s/agent/provision_test.go b/pkg/k8s/agent/provision_test.go index 4ed8a99ba..9e02d4de2 100644 --- a/pkg/k8s/agent/provision_test.go +++ b/pkg/k8s/agent/provision_test.go @@ -16,7 +16,6 @@ package agent import ( stderrors "errors" - "k8s.io/apimachinery/pkg/util/validation" "reflect" "strings" "testing" @@ -24,6 +23,7 @@ import ( aicrerrors "github.com/NVIDIA/aicr/pkg/errors" "github.com/NVIDIA/aicr/pkg/k8s/labels" rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/util/validation" "sigs.k8s.io/yaml" )