feat(agent): isolate concurrent snapshot runs with per-run resources

.github/actions/gpu-snapshot-validate/debug-snapshot-job.sh

@@ -20,16 +20,16 @@ kubectl_kind() {
 }
 
 echo "=== Snapshot Job ==="
-kubectl_kind -n default get job aicr -o yaml || true
+kubectl_kind -n default get job -l app.kubernetes.io/name=aicr -o yaml || true
 echo "=== Snapshot Pods ==="
 kubectl_kind -n default get pods -l app.kubernetes.io/name=aicr -o wide || true
 echo "=== Snapshot Job describe ==="
-kubectl_kind -n default describe job aicr || true
+kubectl_kind -n default describe job -l app.kubernetes.io/name=aicr || true
 echo "=== Snapshot Pod describe ==="
 kubectl_kind -n default describe pods -l app.kubernetes.io/name=aicr || true
 echo "=== Snapshot current logs ==="
 kubectl_kind -n default logs -l app.kubernetes.io/name=aicr --all-containers --tail=200 || true
 echo "=== Snapshot previous logs ==="
 kubectl_kind -n default logs -l app.kubernetes.io/name=aicr --all-containers --previous --tail=200 || true
 echo "=== Snapshot ConfigMap ==="
-kubectl_kind -n default get configmap aicr-snapshot -o yaml || true
+kubectl_kind -n default get configmap -l app.kubernetes.io/name=aicr -o yaml || true

docs/integrator/automation.md

@@ -497,7 +497,8 @@ metadata:
 spec:
   podSelector:
     matchLabels:
-      job-name: aicr
+      app.kubernetes.io/name: aicr
+      app.kubernetes.io/component: snapshot-agent
   policyTypes:
     - Egress
   egress:

docs/integrator/go-library.md

@@ -221,8 +221,8 @@ reported as no drift.
 // CollectSnapshot deploys a snapshotter Job to the target cluster and
 // returns the resulting Snapshot. cfg is a facade-owned struct that
 // mirrors pkg/snapshotter.AgentConfig field for field; the mirror is
-// enforced by a test, so a field added upstream cannot silently stay at
-// its zero value here.
+// enforced by TestAgentConfigMirrorsInternal, so a field added upstream
+// cannot silently stay at its zero value here.
 //
 // The returned Snapshot carries the parsed form plus Snapshot.Raw — the
 // exact bytes the agent emitted. Persist Raw rather than re-serializing
@@ -248,18 +248,25 @@ snapCtx, cancelSnap := context.WithTimeout(context.Background(), 10*time.Minute)
 defer cancelSnap()
 snap, err := client.CollectSnapshot(snapCtx, &aicr.AgentConfig{
 	Kubeconfig: "/path/to/target-kubeconfig",
-	// Namespace, Image, JobName, and ServiceAccountName are all required on
-	// the SDK path. Only Namespace is validated; the rest are copied straight
-	// into the Job and RBAC objects, so an empty value becomes an empty
-	// metadata.name or container image that the API server rejects. The CLI
-	// defaults them from its own flags, which the facade does not share.
-	Namespace:          "aicr-snapshot",
-	Image:              "ghcr.io/nvidia/aicr:v0.19.0",
-	JobName:            "aicr-snapshot",
-	ServiceAccountName: "aicr-agent",
-	Timeout:            5 * time.Minute,
-	Cleanup:            true,
-	AKSGPUPoolsPath:    "/path/to/aks-gpu-pools.json", // AKS only
+	// Namespace is required and validated (it becomes the RBAC/Job namespace
+	// and the internal staging ConfigMap's namespace). Image is not
+	// validated — an empty value becomes an empty container image that the
+	// API server rejects. JobName is an optional name prefix; leaving it
+	// unset defaults to "aicr" with a generated run ID appended, so every
+	// run gets its own uniquely named Job without the caller managing that.
+	//
+	// ServiceAccountName carries two meanings and is EXACT-IF-EXISTS. When
+	// a ServiceAccount of exactly that name already exists in Namespace it
+	// is used verbatim and the run creates NO ServiceAccount, Role,
+	// RoleBinding, ClusterRole or ClusterRoleBinding — and deletes none at
+	// cleanup. Otherwise it is a prefix and the run creates and owns the
+	// full run-scoped RBAC set. Leaving it unset keeps the run-scoped
+	// default and never probes for an existing ServiceAccount.
+	Namespace:       "aicr-snapshot",
+	Image:           "ghcr.io/nvidia/aicr:v0.19.0",
+	Timeout:         5 * time.Minute,
+	Cleanup:         true,
+	AKSGPUPoolsPath: "/path/to/aks-gpu-pools.json", // AKS only
 })
 if err != nil {
 	log.Fatalf("collect snapshot: %v", err)
@@ -338,6 +345,52 @@ Valid phase values are `PhaseDeployment`, `PhaseConformance`, and
 `ErrCodeInvalidRequest` before any cluster work, so a typo cannot
 silently degrade to an empty run.
 
+#### Running the agent as an existing ServiceAccount
+
+`AgentConfig.ServiceAccountName` is **exact-if-exists**, so it carries two
+meanings depending on the cluster:
+
+- A ServiceAccount of exactly that name already exists in `Namespace`: the
+  agent pod runs as it verbatim, and `CollectSnapshot` creates **no**
+  ServiceAccount, Role, RoleBinding, ClusterRole, or ClusterRoleBinding, and
+  deletes none at cleanup.
+- Otherwise it is a name prefix and the run creates and owns the full
+  run-scoped RBAC set, named `<prefix>-<run-id>`.
+
+Leaving the field empty keeps the run-scoped default and never probes for an
+existing ServiceAccount, so a stray ServiceAccount cannot capture a run.
+
+Use the first form when the ServiceAccount must carry EKS IRSA or GKE Workload
+Identity annotations: both providers pin trust to the ServiceAccount *name*, so
+a run-scoped name can never be trusted by either. Grant it the agent's
+permissions once — the objects it creates are permanent and no run cleanup
+removes them:
+
+```go
+// Admin step, run once. Provisions and returns; it deploys no Job.
+// Returns ErrCodeNotFound when the ServiceAccount does not exist.
+res, err := snapshotter.ProvisionAgentRoles(ctx, &snapshotter.AgentRolesConfig{
+	Kubeconfig:         "/path/to/target-kubeconfig",
+	Namespace:          "gpu-operator",
+	ServiceAccountName: "irsa-snapshotter",
+	// DiscoverNetwork also grants the cluster-scoped MUTATING rules live
+	// network discovery needs — permanently, not for one run's lifetime.
+	DiscoverNetwork: false,
+})
+if err != nil {
+	log.Fatalf("provision agent roles: %v", err)
+}
+log.Printf("granted via %s/%s and %s/%s",
+	res.Role, res.RoleBinding, res.ClusterRole, res.ClusterRoleBinding)
+```
+
+Adopting one ServiceAccount across runs waives per-run permission isolation:
+concurrent runs sharing it hold the same grants, and a `DiscoverNetwork`
+provisioning leaves mutating cluster permissions in place until an operator
+removes them. See
+[Agent Deployment](../user/agent-deployment.md#using-an-existing-serviceaccount-irsa-and-workload-identity)
+for the full migration path and teardown commands.
+
 ### Loading an existing recipe
 
 When a recipe has already been resolved and persisted (for example a