diff --git a/.github/actions/gpu-snapshot-validate/debug-snapshot-job.sh b/.github/actions/gpu-snapshot-validate/debug-snapshot-job.sh index aa94bcd7e..0549f50dc 100644 --- a/.github/actions/gpu-snapshot-validate/debug-snapshot-job.sh +++ b/.github/actions/gpu-snapshot-validate/debug-snapshot-job.sh @@ -20,11 +20,11 @@ kubectl_kind() { } echo "=== Snapshot Job ===" -kubectl_kind -n default get job aicr -o yaml || true +kubectl_kind -n default get job -l app.kubernetes.io/name=aicr -o yaml || true echo "=== Snapshot Pods ===" kubectl_kind -n default get pods -l app.kubernetes.io/name=aicr -o wide || true echo "=== Snapshot Job describe ===" -kubectl_kind -n default describe job aicr || true +kubectl_kind -n default describe job -l app.kubernetes.io/name=aicr || true echo "=== Snapshot Pod describe ===" kubectl_kind -n default describe pods -l app.kubernetes.io/name=aicr || true echo "=== Snapshot current logs ===" @@ -32,4 +32,4 @@ kubectl_kind -n default logs -l app.kubernetes.io/name=aicr --all-containers --t echo "=== Snapshot previous logs ===" kubectl_kind -n default logs -l app.kubernetes.io/name=aicr --all-containers --previous --tail=200 || true echo "=== Snapshot ConfigMap ===" -kubectl_kind -n default get configmap aicr-snapshot -o yaml || true +kubectl_kind -n default get configmap -l app.kubernetes.io/name=aicr -o yaml || true diff --git a/docs/integrator/automation.md b/docs/integrator/automation.md index 9c46cd411..17c0da66d 100644 --- a/docs/integrator/automation.md +++ b/docs/integrator/automation.md @@ -497,7 +497,8 @@ metadata: spec: podSelector: matchLabels: - job-name: aicr + app.kubernetes.io/name: aicr + app.kubernetes.io/component: snapshot-agent policyTypes: - Egress egress: diff --git a/docs/integrator/go-library.md b/docs/integrator/go-library.md index 97059175a..5e006cef8 100644 --- a/docs/integrator/go-library.md +++ b/docs/integrator/go-library.md @@ -221,8 +221,8 @@ reported as no drift. // CollectSnapshot deploys a snapshotter Job to the target cluster and // returns the resulting Snapshot. cfg is a facade-owned struct that // mirrors pkg/snapshotter.AgentConfig field for field; the mirror is -// enforced by a test, so a field added upstream cannot silently stay at -// its zero value here. +// enforced by TestAgentConfigMirrorsInternal, so a field added upstream +// cannot silently stay at its zero value here. // // The returned Snapshot carries the parsed form plus Snapshot.Raw — the // exact bytes the agent emitted. Persist Raw rather than re-serializing @@ -248,18 +248,25 @@ snapCtx, cancelSnap := context.WithTimeout(context.Background(), 10*time.Minute) defer cancelSnap() snap, err := client.CollectSnapshot(snapCtx, &aicr.AgentConfig{ Kubeconfig: "/path/to/target-kubeconfig", - // Namespace, Image, JobName, and ServiceAccountName are all required on - // the SDK path. Only Namespace is validated; the rest are copied straight - // into the Job and RBAC objects, so an empty value becomes an empty - // metadata.name or container image that the API server rejects. The CLI - // defaults them from its own flags, which the facade does not share. - Namespace: "aicr-snapshot", - Image: "ghcr.io/nvidia/aicr:v0.19.0", - JobName: "aicr-snapshot", - ServiceAccountName: "aicr-agent", - Timeout: 5 * time.Minute, - Cleanup: true, - AKSGPUPoolsPath: "/path/to/aks-gpu-pools.json", // AKS only + // Namespace is required and validated (it becomes the RBAC/Job namespace + // and the internal staging ConfigMap's namespace). Image is not + // validated — an empty value becomes an empty container image that the + // API server rejects. JobName is an optional name prefix; leaving it + // unset defaults to "aicr" with a generated run ID appended, so every + // run gets its own uniquely named Job without the caller managing that. + // + // ServiceAccountName carries two meanings and is EXACT-IF-EXISTS. When + // a ServiceAccount of exactly that name already exists in Namespace it + // is used verbatim and the run creates NO ServiceAccount, Role, + // RoleBinding, ClusterRole or ClusterRoleBinding — and deletes none at + // cleanup. Otherwise it is a prefix and the run creates and owns the + // full run-scoped RBAC set. Leaving it unset keeps the run-scoped + // default and never probes for an existing ServiceAccount. + Namespace: "aicr-snapshot", + Image: "ghcr.io/nvidia/aicr:v0.19.0", + Timeout: 5 * time.Minute, + Cleanup: true, + AKSGPUPoolsPath: "/path/to/aks-gpu-pools.json", // AKS only }) if err != nil { log.Fatalf("collect snapshot: %v", err) @@ -338,6 +345,52 @@ Valid phase values are `PhaseDeployment`, `PhaseConformance`, and `ErrCodeInvalidRequest` before any cluster work, so a typo cannot silently degrade to an empty run. +#### Running the agent as an existing ServiceAccount + +`AgentConfig.ServiceAccountName` is **exact-if-exists**, so it carries two +meanings depending on the cluster: + +- A ServiceAccount of exactly that name already exists in `Namespace`: the + agent pod runs as it verbatim, and `CollectSnapshot` creates **no** + ServiceAccount, Role, RoleBinding, ClusterRole, or ClusterRoleBinding, and + deletes none at cleanup. +- Otherwise it is a name prefix and the run creates and owns the full + run-scoped RBAC set, named `-`. + +Leaving the field empty keeps the run-scoped default and never probes for an +existing ServiceAccount, so a stray ServiceAccount cannot capture a run. + +Use the first form when the ServiceAccount must carry EKS IRSA or GKE Workload +Identity annotations: both providers pin trust to the ServiceAccount *name*, so +a run-scoped name can never be trusted by either. Grant it the agent's +permissions once — the objects it creates are permanent and no run cleanup +removes them: + +```go +// Admin step, run once. Provisions and returns; it deploys no Job. +// Returns ErrCodeNotFound when the ServiceAccount does not exist. +res, err := snapshotter.ProvisionAgentRoles(ctx, &snapshotter.AgentRolesConfig{ + Kubeconfig: "/path/to/target-kubeconfig", + Namespace: "gpu-operator", + ServiceAccountName: "irsa-snapshotter", + // DiscoverNetwork also grants the cluster-scoped MUTATING rules live + // network discovery needs — permanently, not for one run's lifetime. + DiscoverNetwork: false, +}) +if err != nil { + log.Fatalf("provision agent roles: %v", err) +} +log.Printf("granted via %s/%s and %s/%s", + res.Role, res.RoleBinding, res.ClusterRole, res.ClusterRoleBinding) +``` + +Adopting one ServiceAccount across runs waives per-run permission isolation: +concurrent runs sharing it hold the same grants, and a `DiscoverNetwork` +provisioning leaves mutating cluster permissions in place until an operator +removes them. See +[Agent Deployment](../user/agent-deployment.md#using-an-existing-serviceaccount-irsa-and-workload-identity) +for the full migration path and teardown commands. + ### Loading an existing recipe When a recipe has already been resolved and persisted (for example a diff --git a/docs/user/agent-deployment.md b/docs/user/agent-deployment.md index f59bada92..d182aed11 100644 --- a/docs/user/agent-deployment.md +++ b/docs/user/agent-deployment.md @@ -30,6 +30,7 @@ The agent is a Kubernetes Job that captures system configuration and writes outp ### ConfigMap storage Agent uses ConfigMap URI scheme (`cm://namespace/name`) to write snapshots: + ```bash aicr snapshot --namespace gpu-operator --output cm://gpu-operator/aicr-snapshot ``` @@ -40,6 +41,7 @@ namespace **must match `--namespace`** — otherwise the Job's ServiceAccount ha no permission to create the ConfigMap and the snapshot write fails. This creates: + ```yaml apiVersion: v1 kind: ConfigMap @@ -48,7 +50,7 @@ metadata: namespace: gpu-operator labels: app.kubernetes.io/name: aicr - app.kubernetes.io/component: snapshot + app.kubernetes.io/component: Snapshot app.kubernetes.io/version: data: snapshot.yaml: | # Complete snapshot YAML @@ -64,7 +66,13 @@ data: - Kubernetes cluster with GPU nodes - aicr CLI installed - GPU Operator installed (or appropriate namespace configured via `--namespace`) -- Cluster admin permissions (for RBAC setup) +- Permission to create and delete the run's Job and RBAC in the target + namespace, plus the cluster-scoped `ClusterRole`/`ClusterRoleBinding` the + agent needs. Every run starts by verifying this and stops before touching + the cluster if anything is missing — see + [Pre-flight permission gate](#pre-flight-permission-gate). Pointing + `--service-account-name` at a ServiceAccount you provisioned yourself + requires **no** RBAC permissions at all ## Quick Start @@ -129,12 +137,13 @@ aicr snapshot \ - `--namespace`: Deployment namespace (default: `default`) - `--image`: Container image (default: matches the CLI version, e.g. `ghcr.io/nvidia/aicr:v0.19.0`; dev and snapshot builds use `:latest`) - `--image-pull-secret`: Secret name for pulling the agent image from a private registry (repeatable) -- `--job-name`: Job name (default: `aicr`) -- `--service-account-name`: ServiceAccount name (default: `aicr`) +- `--job-name`: Job name prefix (default: `aicr`); the run ID is always appended (`-`) +- `--service-account-name`: ServiceAccount the agent pod runs as. **Exact-if-exists** — an existing ServiceAccount of exactly this name in `--namespace` is used verbatim and the run creates no RBAC; otherwise it is a name prefix (default: `aicr`) and the run ID is appended (`-`). See [Using an existing ServiceAccount](#using-an-existing-serviceaccount-irsa-and-workload-identity) +- `--add-roles-to-service-account`: **Writes manifests and applies nothing.** Renders the RBAC that grants the agent's permissions to the named ServiceAccount into `./snapshot-rbac-/` and exits **without taking a snapshot**. No cluster is contacted. You review the files, then apply and later delete them yourself. See [Using an existing ServiceAccount](#using-an-existing-serviceaccount-irsa-and-workload-identity) - `--node-selector`: Node selector (format: `key=value`, repeatable) - `--toleration`: Toleration (format: `key=value:effect`, repeatable). **Default: all taints are tolerated** (uses `operator: Exists` without key). Only specify this flag if you want to restrict which taints the Job can tolerate. - `--timeout`: Wait timeout (default: `5m`) -- `--no-cleanup`: Skip removal of Job and RBAC resources on completion. **Warning:** leaves the `aicr-node-reader` ClusterRole and ClusterRoleBinding active. By default these grant only read access to nodes, pods, ClusterPolicy CRDs, Slinky Controller/NodeSet/LoginSet/RestApi/Accounting CRs, and official MariaDB CRs (not cluster-admin); however, when combined with `--discover-network` the retained ClusterRole also carries the cluster-scoped **mutating** discovery rules (CRD/namespace/DaemonSet create-delete, `pods/exec`, `nodes/patch`, `NicClusterPolicy` patch — see [Security Considerations](#security-considerations)), so it is **not** read-only in that case. +- `--no-cleanup`: Skip removal of Job and RBAC resources on completion. **Warning:** leaves the run-scoped `aicr-node-reader-` ClusterRole and ClusterRoleBinding active. By default these grant only read access to nodes, pods, ClusterPolicy CRDs, Slinky Controller/NodeSet/LoginSet/RestApi/Accounting CRs, and official MariaDB CRs (not cluster-admin); however, when combined with `--discover-network` the retained ClusterRole also carries the cluster-scoped **mutating** discovery rules (CRD/namespace/DaemonSet create-delete, `pods/exec`, `nodes/patch`, `NicClusterPolicy` patch — see [Security Considerations](#security-considerations)), so it is **not** read-only in that case. - `--privileged`: Run agent in privileged mode (default: enabled; required for GPU/SystemD collectors). Set to `false` for PSS-restricted namespaces. - `--require-gpu`: Fail the snapshot if no GPU is found. In agent mode also requests an `nvidia.com/gpu` resource for the pod (required in CDI environments). - `--runtime-class`: Set `runtimeClassName` on the agent pod for `nvidia-smi` access without consuming a GPU. Use with `--node-selector` to target GPU nodes. @@ -150,13 +159,13 @@ If something goes wrong, check Job logs: ```shell # Get Job status -kubectl get jobs -n gpu-operator +kubectl get jobs -n gpu-operator -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent # View logs -kubectl logs -n gpu-operator job/aicr +kubectl logs -n gpu-operator -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent # Describe Job for events -kubectl describe job aicr -n gpu-operator +kubectl describe job -n gpu-operator -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent ``` ## Customization @@ -202,6 +211,151 @@ aicr snapshot --image ghcr.io/nvidia/aicr:v0.19.0 - [GitHub Releases](https://github.com/NVIDIA/aicr/releases) - Container registry: [ghcr.io/nvidia/aicr](https://github.com/NVIDIA/aicr/pkgs/container/aicr) +## Using an existing ServiceAccount (IRSA and Workload Identity) + +By default the agent creates its own ServiceAccount for each run and deletes it +at cleanup, so no two runs share an identity. That does not work when the +ServiceAccount must carry cloud IAM credentials: **EKS IRSA** +(`eks.amazonaws.com/role-arn`) and **GKE Workload Identity** +(`iam.gke.io/gcp-service-account`) both pin trust to the ServiceAccount *name*. +IRSA's role trust policy conditions on +`system:serviceaccount::`, and a GKE IAM binding names the KSA +as `PROJECT.svc.id.goog[/]` and accepts no wildcard. A +per-run name can never be trusted by either, and copying the annotations onto a +run-scoped ServiceAccount does not help. + +`--service-account-name` therefore behaves as **exact-if-exists**: + +| Does a ServiceAccount of exactly that name exist in `--namespace`? | Behavior | +|---|---| +| Yes | Used verbatim. aicr creates **no** ServiceAccount, Role, RoleBinding, ClusterRole, or ClusterRoleBinding for the run, binds nothing to it, and deletes nothing at cleanup. | +| No | The value is a name prefix. The run creates `-` plus the full run-scoped RBAC set, and deletes them at cleanup — the pre-existing behavior. | + +An unset `--service-account-name` is never probed for existence, so a stray +ServiceAccount named `aicr` cannot silently capture a run. + +### Migrating a pre-created ServiceAccount + +**What changed.** Before run isolation, passing `--service-account-name` at a +ServiceAccount you had created out of band got that ServiceAccount adopted, and +aicr attached its Role and RoleBinding to it. Run isolation made every +run-owned object run-scoped, which turned the flag into a prefix — so a +pre-created ServiceAccount stopped being used at all, and an agent pod that +had been running with IRSA or Workload Identity credentials silently started +running without them. Exact-if-exists restores the pre-created ServiceAccount +as the one the pod runs as; what does **not** come back is aicr managing its +permissions. + +**Supported flow.** Generate the RBAC manifests, read them, apply them, then +take snapshots normally: + +```shell +# 1. Your ServiceAccount, created and annotated by you (or by eksctl / Terraform). +kubectl create serviceaccount irsa-snapshotter -n gpu-operator +kubectl annotate serviceaccount irsa-snapshotter -n gpu-operator \ + eks.amazonaws.com/role-arn=arn:aws:iam::123456789012:role/aicr-snapshot + +# 2. Write the RBAC manifests. Applies NOTHING, contacts no cluster, takes no +# snapshot. Prints the directory it wrote and the commands below. +aicr snapshot --namespace gpu-operator --add-roles-to-service-account irsa-snapshotter + +# 3. Read what you are about to grant. Each file explains its rules. +less snapshot-rbac-/*.yaml + +# 4. Grant it, once you are satisfied. +kubectl apply -f snapshot-rbac-/ + +# 5. Capture snapshots as that ServiceAccount, as often as you like. +aicr snapshot \ + --namespace gpu-operator \ + --service-account-name irsa-snapshotter \ + --output cm://gpu-operator/aicr-snapshot + +# 6. Revoke the grant when the ServiceAccount no longer needs it. +kubectl delete -f snapshot-rbac-/ +``` + +Step 2 does **not** check that the ServiceAccount exists — it contacts no +cluster at all, so it cannot. A mistyped name yields manifests naming a subject +that does not resolve; Kubernetes accepts such a binding and it simply grants +nothing. The generated `02-rolebinding.yaml` tells you how to verify the name, +and `aicr` never creates a ServiceAccount for you. + +### What the manifests contain + +`aicr snapshot --add-roles-to-service-account ` writes a new directory in +your current working directory, one object per file: + +```text +snapshot-rbac-/ +├── 01-role.yaml Role/aicr-agent--rbac (namespaced) +├── 02-rolebinding.yaml RoleBinding/aicr-agent--rbac (namespaced) +├── 03-clusterrole.yaml ClusterRole/aicr-agent---rbac (cluster) +└── 04-clusterrolebinding.yaml ClusterRoleBinding/aicr-agent---rbac (cluster) +``` + +Every file opens with a YAML comment header naming what the object grants and +why the agent needs each rule, so you can decide rule by rule whether to apply +it. The numeric prefixes exist because `kubectl apply -f /` visits a +directory in lexical order — they keep each Role ahead of the binding that +references it. You can also apply or read the files individually. + +The rules are the same ones a run-scoped grant carries, so a shared +ServiceAccount is never less capable than a run-owned one. The names end in +`-rbac`, which no run-scoped name can (a run-scoped name always ends in a run +ID whose last segment is hexadecimal), so the two name spaces cannot collide. + +**Nothing is applied for you, and nothing is removed for you.** The objects you +apply carry no `aicr.run/run-id` label, no run enters them into its cleanup +list, and no `aicr snapshot` or `aicr validate` invocation deletes them. +Teardown is one command, which is why keeping the directory is worth it: + +```shell +kubectl delete -f snapshot-rbac-/ +``` + +If you no longer have the directory, delete the objects by name instead: + +```shell +kubectl delete role,rolebinding aicr-agent-irsa-snapshotter-rbac -n gpu-operator +kubectl delete clusterrole,clusterrolebinding aicr-agent-gpu-operator.irsa-snapshotter-rbac +``` + +The directory name carries a fresh run ID on every invocation, so generating +twice never overwrites a set you are still reviewing; a directory that already +exists fails the command with `CONFLICT`. After an aicr upgrade, re-generate +and `kubectl apply -f` the new directory to refresh the rules in place. + +The cluster-scoped pair is named `aicr-agent-.-rbac`. +The two segments join on `.`, which a namespace can never contain, so no other +namespace and ServiceAccount combination can produce the same name — applying +one grant cannot retarget another's. + +### Trade-off: per-run permission isolation is waived + +Using an existing ServiceAccount is an opt-in exchange, and it is worth +understanding before you choose it: + +- **Concurrent runs share one identity.** Two `aicr snapshot` invocations using + the same ServiceAccount hold exactly the same grants. Run isolation's + guarantee that one run's permissions cannot reach another run's does not + apply to them. +- **`--discover-network` grants become permanent.** With a run-owned + ServiceAccount, the cluster-scoped **mutating** discovery rules + (`nodes: patch`, `pods/exec: create`, CRD / namespace / DaemonSet + create-delete — see [Security Considerations](#security-considerations)) + exist for one run and are revoked at cleanup. Rendered with + `aicr snapshot --add-roles-to-service-account --discover-network` and + applied, they sit on that ServiceAccount until you remove them. + +Generate without `--discover-network` unless you need live network discovery; +that grant is read-only. When you do use it, `03-clusterrole.yaml` carries a +warning header naming every mutating rule and the discovery step it exists for +— read it before applying, which is precisely what writing manifests instead of +applying them is for. If you need discovery only occasionally, prefer a +run-owned ServiceAccount for those runs, or keep a separate ServiceAccount used +only for discovery. + ## Post-Deployment ### Retrieve Snapshot @@ -326,30 +480,46 @@ aicr diff --baseline baseline.yaml --target current.yaml --fail-on-drift \ ### Job Fails to Start -Check RBAC permissions: +Check RBAC permissions. The ServiceAccount name is run-scoped (`aicr-`), so look it up first. + +Select it by run ID, not by position: with concurrent snapshot runs the namespace +holds one ServiceAccount per run, and `.items[0]` would pick an arbitrary one — so +the checks below could report on a healthy run while you are debugging a failed one. +The CLI logs the run ID when it starts (`snapshot agent run: runID=...`), and the +Job, its pods, and its RBAC resources all carry it as the `aicr.run/run-id` +label. (The staging ConfigMap is the exception — it is written from inside the +pod and carries only `app.kubernetes.io/name`, `app.kubernetes.io/component` and +`app.kubernetes.io/version`; find it by its run-scoped name instead, see +[Job Completes but No Output](#job-completes-but-no-output).) + ```shell -kubectl auth can-i get nodes --as=system:serviceaccount:gpu-operator:aicr -kubectl auth can-i get pods --as=system:serviceaccount:gpu-operator:aicr +# From the failing Job (or use the runID the CLI logged at start). +RUN_ID=$(kubectl get job -n gpu-operator -o jsonpath='{.metadata.labels.aicr\.run/run-id}') + +SA=$(kubectl get sa -n gpu-operator -l app.kubernetes.io/name=aicr,aicr.run/run-id=$RUN_ID -o jsonpath='{.items[0].metadata.name}') +kubectl auth can-i get nodes --as=system:serviceaccount:gpu-operator:$SA +kubectl auth can-i get pods --as=system:serviceaccount:gpu-operator:$SA kubectl auth can-i list controllers.slinky.slurm.net --all-namespaces \ - --as=system:serviceaccount:gpu-operator:aicr + --as=system:serviceaccount:gpu-operator:$SA kubectl auth can-i list nodesets.slinky.slurm.net --all-namespaces \ - --as=system:serviceaccount:gpu-operator:aicr + --as=system:serviceaccount:gpu-operator:$SA kubectl auth can-i list loginsets.slinky.slurm.net --all-namespaces \ - --as=system:serviceaccount:gpu-operator:aicr + --as=system:serviceaccount:gpu-operator:$SA kubectl auth can-i list restapis.slinky.slurm.net --all-namespaces \ - --as=system:serviceaccount:gpu-operator:aicr + --as=system:serviceaccount:gpu-operator:$SA kubectl auth can-i list accountings.slinky.slurm.net --all-namespaces \ - --as=system:serviceaccount:gpu-operator:aicr + --as=system:serviceaccount:gpu-operator:$SA kubectl auth can-i list mariadbs.k8s.mariadb.com --all-namespaces \ - --as=system:serviceaccount:gpu-operator:aicr + --as=system:serviceaccount:gpu-operator:$SA ``` ### Job Pending Check node selectors and tolerations: + ```shell # View pod events -kubectl describe pod -n gpu-operator -l job-name=aicr +kubectl describe pod -n gpu-operator -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent # Check node labels kubectl get nodes --show-labels @@ -361,42 +531,94 @@ kubectl get nodes -o custom-columns=NAME:.metadata.name,TAINTS:.spec.taints ### Job Completes but No Output Check ConfigMap and container logs: + ```shell -# Check if ConfigMap was created -kubectl get configmap aicr-snapshot -n gpu-operator +# Every name below is run-scoped, so start from the run ID. The CLI logs it +# when it starts ("snapshot agent run: runID=..."); this reads it back off the +# Job instead. Substitute your own Job name (or use the logged run ID). +RUN_ID=$(kubectl get job -n gpu-operator -o jsonpath='{.metadata.labels.aicr\.run/run-id}') + +# Check if the staging ConfigMap was created. Without an explicit +# "-o cm:///", the agent stages its result in a run-scoped +# ConfigMap named "aicr-agent-snapshot-" that cleanup deletes when +# the run ends — pass --no-cleanup to keep it around for inspection. +# +# This object is written by the in-pod agent, not by the CLI, so it does NOT +# carry the aicr.run/run-id label the Job and RBAC resources do. Address it by +# name: with concurrent runs, the label selector below matches every run's +# staging ConfigMap plus any delivered cm:// artifact in the namespace. +kubectl get configmap -n gpu-operator aicr-agent-snapshot-$RUN_ID # View ConfigMap contents -kubectl get configmap aicr-snapshot -n gpu-operator -o yaml +kubectl get configmap -n gpu-operator aicr-agent-snapshot-$RUN_ID -o yaml + +# Or list every aicr-written ConfigMap in the namespace (all runs) +kubectl get configmap -n gpu-operator -l app.kubernetes.io/name=aicr # View pod logs for errors -kubectl logs -n gpu-operator -l job-name=aicr +kubectl logs -n gpu-operator -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent ``` ### Permission Denied -Ensure RBAC is correctly deployed: +A run that stops with `missing required permissions` never reached the cluster +— read the list it printed, which names every missing verb, its scope, and +whether you or the agent ServiceAccount lacked it. See +[Pre-flight permission gate](#pre-flight-permission-gate) for the full matrix +and for why exact-ServiceAccount mode needs fewer of them. + +If the run got past the gate, verify the RBAC it deployed: + ```shell -# Verify ClusterRole -kubectl get clusterrole aicr-node-reader +# Verify ClusterRole (run-scoped: "aicr-node-reader-") +kubectl get clusterrole -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent # Verify ClusterRoleBinding -kubectl get clusterrolebinding aicr-node-reader +kubectl get clusterrolebinding -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent + +# Verify Role and RoleBinding (run-scoped: "aicr-") +kubectl get role -n gpu-operator -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent +kubectl get rolebinding -n gpu-operator -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent -# Verify Role and RoleBinding -kubectl get role aicr -n gpu-operator -kubectl get rolebinding aicr -n gpu-operator +# Verify ServiceAccount (run-scoped: "aicr-") +kubectl get serviceaccount -n gpu-operator -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent +``` + +### Cleanup Left a Resource Behind + +Cleanup removes only the objects the run itself created, and pins every delete +to the UID the apiserver returned when it created them. If a create's response +is lost in flight, the run knows the name it used but never learned the +object's identity — so cleanup re-reads the object and deletes it only when it +still carries that run's `aicr.run/run-id` label. When something else has taken +the name over, cleanup keeps the object and says so: + +```text +WARN cleanup left behind an object it cannot prove this run created; if it is a +stale orphan of this run, remove it by hand kind=Role name=aicr- +runID= objectRunID= +``` + +Inspect the named object and delete it yourself if it is a stale leftover: -# Verify ServiceAccount -kubectl get serviceaccount aicr -n gpu-operator +```shell +kubectl get role -n gpu-operator aicr- -o yaml ``` +The staging ConfigMap is warned about the same way. It is written by the in-pod +agent rather than by the CLI, so it carries no `aicr.run/run-id` label to check +(see [Job Completes but No Output](#job-completes-but-no-output)); cleanup +sweeps it only when this run's Job was created successfully — the only way this +run could have produced one — and only when the object looks like an aicr +artifact (`app.kubernetes.io/name=aicr`). + ## Security Considerations ### RBAC Permissions The agent requires these permissions (created automatically by the CLI): -- **ClusterRole** (`aicr-node-reader`): Read access to nodes and pods; `get`/`list` access to ClusterPolicy CRDs (`nvidia.com`); cluster-wide `list` access to Slinky Controller, NodeSet, LoginSet, RestApi, and Accounting CRs (`slinky.slurm.net`); and cluster-wide `list` access to official MariaDB CRs (`k8s.mariadb.com`) -- **Role** (`aicr`): Create/update ConfigMaps and list pods in the deployment namespace +- **ClusterRole** (`aicr-node-reader-`, run-scoped): Read access to nodes and pods; `get`/`list` access to ClusterPolicy CRDs (`nvidia.com`); cluster-wide `list` access to Slinky Controller, NodeSet, LoginSet, RestApi, and Accounting CRs (`slinky.slurm.net`); and cluster-wide `list` access to official MariaDB CRs (`k8s.mariadb.com`) +- **Role** (`aicr-`, run-scoped): Create/update ConfigMaps and list pods in the deployment namespace The baseline ClusterRole above is read-only (`get`/`list` only). Slinky detection projects only allowlisted identity, association, and boolean fields; @@ -422,6 +644,100 @@ than read access: Use `--discover-network` only against clusters where this mutation and the broader RBAC grant are acceptable. +### Pre-flight permission gate + +Every run begins by verifying the permissions it will actually use and stops +before writing anything if any are missing. The gate is read-only: an access +review is an authorization query that persists nothing, and the one object it +reads — the ServiceAccount named by `--service-account-name` — is read with +`get`. A failed gate therefore leaves the cluster exactly as it found it. + +All failures are reported together, so one run tells you everything to fix. +Each line names the verb, the resource, the scope, and which of the two +identities lacked it: + +```text +missing required permissions: + - the caller (your kubeconfig identity) cannot "delete" clusterroles.rbac.authorization.k8s.io (cluster-scoped) + - agent ServiceAccount "system:serviceaccount:gpu-operator/irsa-snapshotter" cannot "list" nodes (cluster-scoped) +``` + +#### What the caller must be able to do + +Checked with `SelfSubjectAccessReview` against your kubeconfig identity. The +first group is required in both ServiceAccount modes: + +| Resource | Verbs | Scope | Used by | +|---|---|---|---| +| `serviceaccounts` | `get` | namespace | Deciding whether `--service-account-name` names an existing ServiceAccount or is a prefix | +| `jobs.batch` | `create`, `get`, `list`, `watch`, `delete` | namespace | Creating the agent Job, waiting on it, cleaning it up | +| `pods` | `get`, `list`, `watch` | namespace | Finding the agent pod and waiting for it to be ready | +| `pods/log` | `get` | namespace | Streaming the agent's output back to your terminal | +| `configmaps` | `get`, `list` | namespace | Reading the snapshot the agent staged | +| `configmaps` | `delete` | namespace | Only when aicr owns the output ConfigMap (i.e. you did not pass your own `cm://` URI) | + +`serviceaccounts: get` is not optional. Without it the run cannot tell whether +`--service-account-name` names an existing ServiceAccount or is a prefix, and +guessing "prefix" would run the agent under a generated ServiceAccount carrying +none of the named account's IRSA or Workload Identity annotations. The run +stops instead of guessing. + +The second group is required **only in prefix mode** — when the run creates +its own run-scoped ServiceAccount: + +| Resource | Verbs | Scope | +|---|---|---| +| `serviceaccounts` | `create`, `delete` | namespace | +| `roles.rbac.authorization.k8s.io` | `create`, `delete` | namespace | +| `rolebindings.rbac.authorization.k8s.io` | `create`, `delete` | namespace | +| `clusterroles.rbac.authorization.k8s.io` | `create`, `delete` | cluster | +| `clusterrolebindings.rbac.authorization.k8s.io` | `create`, `delete` | cluster | + +`delete` is required alongside `create` because cleanup always runs, including +on the failure path. An identity that can create but not delete would leave a +ServiceAccount, Role, RoleBinding, ClusterRole and ClusterRoleBinding behind on +every single run. + +**Exact-ServiceAccount mode requires fewer caller permissions.** When +`--service-account-name` names a ServiceAccount that already exists, the run +creates and deletes no RBAC at all, so none of the five kinds above is +demanded of you. What it requires instead is that the ServiceAccount was +actually provisioned — see below. + +#### What the agent ServiceAccount must be able to do + +The agent pod runs as a ServiceAccount, not as you, so its permissions are +checked separately with `SubjectAccessReview` naming +`system:serviceaccount::` as the subject. The questions are +derived from the same rule set the run-scoped `Role` and `ClusterRole` grant +(and that `--add-roles-to-service-account` renders), so the gate cannot fall +behind what the agent needs: namespaced `configmaps` and `pods` access, plus +cluster-scoped `nodes`, `pods`, `nvidia.com` ClusterPolicies, the Slinky CRs +and the MariaDB CRs — widened to the full mutating set when +`--discover-network` is passed. + +This check runs **in exact-ServiceAccount mode only**. In prefix mode the +ServiceAccount does not exist yet and the run is about to grant it exactly +those rules, so there is nothing to verify. In exact mode aicr grants nothing, +and the most common failure is that the manifests from +`--add-roles-to-service-account` were rendered but never applied. The gate +catches that up front and tells you how to fix it, rather than letting a Job +start and fail inside the pod minutes later. + +**When the ServiceAccount's permissions cannot be verified.** Creating a +`SubjectAccessReview` is itself a privilege. If you do not hold it, the run +does **not** silently skip the check — it says so and continues, because the +agent will still fail visibly in-pod if a rule is missing: + +```text +WARN could not verify the agent ServiceAccount's own permissions; continuing, +but a missing rule will surface as an in-pod failure minutes from now instead +of here serviceAccount=irsa-snapshotter namespace=gpu-operator +uncheckedRules=18 remedy="grant the caller 'create +subjectaccessreviews.authorization.k8s.io', or verify by hand with: kubectl +auth can-i --list --as system:serviceaccount:gpu-operator/irsa-snapshotter" +``` + ### Pod Security Context The agent requires elevated privileges to collect system configuration from the host: diff --git a/docs/user/cli-config.md b/docs/user/cli-config.md index 85465ef1a..42d1499b9 100644 --- a/docs/user/cli-config.md +++ b/docs/user/cli-config.md @@ -98,8 +98,8 @@ spec: namespace: aicr-validation image: "" # default: ghcr.io/nvidia/aicr:latest imagePullSecrets: [] - jobName: aicr - serviceAccountName: aicr + # jobName / serviceAccountName are optional PREFIXES, not names — the + # run ID is always appended. Omit them to take the defaults. nodeSelector: nodeGroup: gpu-worker tolerations: @@ -181,8 +181,8 @@ spec: namespace: aicr-validation image: "" imagePullSecrets: [] - jobName: aicr - serviceAccountName: aicr + # Optional prefixes; omitted here so the defaults apply + # (jobName and serviceAccountName both default to aicr-validate). nodeSelector: nodeGroup: gpu-worker tolerations: @@ -229,7 +229,7 @@ produced from the live cluster. | `output.path` | string | Output file path (same as `-o`) | | `output.format` | string | `yaml` \| `json` \| `table` | | `output.template` | string | Optional Go template path | -| `agent.*` | object | In-cluster capture Job pod: `namespace`, `image`, `imagePullSecrets`, `jobName`, `serviceAccountName`, `nodeSelector`, `tolerations`, `requireGpu`, `runtimeClassName` (mutually exclusive with `requireGpu`), `os`, `requests`, `limits`. Mirrors `spec.validate.agent` so one file pins matching placement for both | +| `agent.*` | object | In-cluster capture Job pod: `namespace`, `image`, `imagePullSecrets`, `jobName`, `serviceAccountName`, `nodeSelector`, `tolerations`, `requireGpu`, `runtimeClassName` (mutually exclusive with `requireGpu`), `os`, `requests`, `limits`. `jobName` and `serviceAccountName` are optional **prefixes**, not exact names — the run ID is appended (`-`), so omit them unless you need a custom prefix. Mirrors `spec.validate.agent` so one file pins matching placement for both | | `execution.timeout` | duration string | e.g. `5m` | | `execution.noCleanup` | bool | Keep the capture Job after completion | | `execution.privileged` | bool (tri-state) | Set `false` for PSS-restricted namespaces | @@ -289,7 +289,7 @@ Inputs to `aicr validate`. | Field | Type | Notes | |-------|------|-------| | `input.recipe` / `.snapshot` | string | Recipe + snapshot to validate | -| `agent.*` | object | In-cluster validation Job pod; same fields and nil-vs-empty semantics as `spec.snapshot.agent` (minus `runtimeClassName`/`os`/`requests`/`limits`) | +| `agent.*` | object | The **live snapshot-capture** Job pod `aicr validate` deploys when `input.snapshot` is empty; same fields and nil-vs-empty semantics as `spec.snapshot.agent` (minus `runtimeClassName`/`os`/`requests`/`limits`). `jobName` and `serviceAccountName` are optional prefixes with the run ID appended (defaults: both `aicr-validate`); they do not name the validator Jobs | | `execution.phases` | []string | e.g. `[deployment, conformance, performance]` | | `execution.failOnError` | bool (tri-state) | Absent = CLI default (`true`); explicit `false` opts out | | `execution.failFast` | bool (tri-state) | Stop after the first failed phase | diff --git a/docs/user/cli-reference.md b/docs/user/cli-reference.md index 73e6c5c1e..b9595bc7e 100644 --- a/docs/user/cli-reference.md +++ b/docs/user/cli-reference.md @@ -84,12 +84,13 @@ aicr snapshot [flags] | `--kubeconfig` | `-k` | string | ~/.kube/config | Path to kubeconfig file (overrides KUBECONFIG env). Also used when `--output` is a ConfigMap URI so reads and writes target the same cluster. | | `--namespace` | `-n` | string | default | Kubernetes namespace for agent deployment | | `--image` | | string | matches CLI version | Container image for agent Job. Release builds default to `ghcr.io/nvidia/aicr:v`; dev and `-next` snapshot builds default to `ghcr.io/nvidia/aicr:latest`. | -| `--job-name` | | string | aicr | Name for the agent Job | -| `--service-account-name` | | string | aicr | ServiceAccount name for agent Job | +| `--job-name` | | string | aicr | Prefix for the agent Job name; the run ID is always appended (`-`) | +| `--service-account-name` | | string | aicr | ServiceAccount the agent pod runs as. **Exact-if-exists:** when a ServiceAccount of exactly this name already exists in `--namespace`, it is used verbatim and the run creates **no** ServiceAccount, Role, RoleBinding, ClusterRole, or ClusterRoleBinding — and deletes none at cleanup. Otherwise the value is a name prefix and the run ID is appended (`-`). Exact-if-exists mode needs **fewer** caller permissions — the run's pre-flight gate stops demanding `create`/`delete` on the five RBAC kinds — but requires the ServiceAccount to already carry the agent's rules, which the gate verifies with a `SubjectAccessReview`. See [Using an existing ServiceAccount](agent-deployment.md#using-an-existing-serviceaccount-irsa-and-workload-identity) and [Pre-flight permission gate](agent-deployment.md#pre-flight-permission-gate) | +| `--add-roles-to-service-account` | | string | | **Writes manifests and applies nothing.** Renders the `Role`/`RoleBinding` (`aicr-agent--rbac`) and `ClusterRole`/`ClusterRoleBinding` (`aicr-agent---rbac`) that grant the agent's permissions to the named ServiceAccount into `./snapshot-rbac-/`, one object per file with a comment header explaining what it grants, then exits **without taking a snapshot**. **No cluster is contacted** — no kubeconfig or privileges needed, and the ServiceAccount is not checked for existence. Review the files, then apply with `kubectl apply -f /` and revoke with `kubectl delete -f /` yourself; no run cleanup ever touches them. Fails with `CONFLICT` if the directory already exists. Combine with `--discover-network` to also render the mutating live-discovery rules | | `--node-selector` | | string[] | auto | Node selector for agent scheduling (key=value, repeatable). When omitted (and neither `--require-gpu` nor `--runtime-class` is set), the agent auto-targets GPU nodes labeled `nvidia.com/gpu.present=true` if the cluster has any — see [Agent Deployment](agent-deployment.md). Pass an explicit selector to override. | | `--toleration` | | string[] | all taints | Tolerations for agent scheduling (key=value:effect, repeatable). **Default: all taints tolerated** (uses `operator: Exists`). Only specify to restrict which taints are tolerated. | | `--timeout` | | duration | 5m | Timeout for agent Job completion | -| `--no-cleanup` | | bool | false | Skip removal of Job and RBAC resources on completion. **Warning:** leaves the agent's `aicr-node-reader` ClusterRoleBinding active. By default this grants only read-only access; with `--discover-network` the retained ClusterRole also carries the mutating rules live network discovery needs (CRD/namespace/daemonset create, pod exec, node patch, NicClusterPolicy). | +| `--no-cleanup` | | bool | false | Skip removal of Job and RBAC resources on completion. **Warning:** leaves both the agent's run-scoped `aicr-node-reader-` ClusterRole and the identically named ClusterRoleBinding active. By default the ClusterRole grants only read-only access; with `--discover-network` it also carries the mutating rules live network discovery needs (CRD/namespace/daemonset create, pod exec, node patch, NicClusterPolicy). Delete both when you are done — removing only the binding leaves the grant definition behind. | | `--privileged` | | bool | true | Run agent in privileged mode (required for GPU/SystemD collectors). Set to false for PSS-restricted namespaces. | | `--image-pull-secret` | | string[] | | Image pull secrets for private registries (repeatable) | | `--require-gpu` | | bool | false | Require GPU resources on the agent pod (mutually exclusive with `--runtime-class`) | @@ -169,13 +170,29 @@ aicr snapshot \ --namespace gpu-operator \ --image ghcr.io/nvidia/aicr:v0.19.0 \ --job-name snapshot-gpu-nodes \ - --service-account-name aicr \ --node-selector accelerator=nvidia-h100 \ --toleration nvidia.com/gpu:NoSchedule \ --timeout 10m \ --output cm://gpu-operator/aicr-snapshot \ --no-cleanup +# Write the RBAC manifests that grant the agent's permissions to an existing +# ServiceAccount, then exit. Applies nothing and contacts no cluster; takes no +# snapshot. Writes ./snapshot-rbac-/. +aicr snapshot \ + --namespace gpu-operator \ + --add-roles-to-service-account irsa-snapshotter + +# Review what each file grants, then apply them yourself. +kubectl apply -f snapshot-rbac-/ + +# Capture as that ServiceAccount. Because it already exists, the name is used +# verbatim and this run creates and deletes no RBAC of its own. +aicr snapshot \ + --namespace gpu-operator \ + --service-account-name irsa-snapshotter \ + --output cm://gpu-operator/aicr-snapshot + # Custom template formatting aicr snapshot --template examples/templates/snapshot-template.md.tmpl @@ -208,8 +225,8 @@ spec: namespace: aicr-validation image: "" # default: ghcr.io/nvidia/aicr:latest imagePullSecrets: [] - jobName: aicr - serviceAccountName: aicr + # jobName / serviceAccountName are optional PREFIXES, not names — the + # run ID is always appended. Omit them to take the defaults. nodeSelector: nodeGroup: gpu-worker tolerations: @@ -1031,8 +1048,8 @@ aicr validate [flags] | `--namespace` | `-n` | string | aicr-validation | Kubernetes namespace for validation Job deployment | | `--image` | | string | ghcr.io/nvidia/aicr:latest | Container image for validation Job | | `--image-pull-secret` | | string[] | | Image pull secrets for private registries (repeatable) | -| `--job-name` | | string | aicr-validate | Name for the validation Job | -| `--service-account-name` | | string | aicr | ServiceAccount name for validation Job | +| `--job-name` | | string | aicr-validate | Prefix for the **live snapshot-capture agent's** Job name; the run ID is always appended (`-`). Inert when `--snapshot` is supplied — no agent is deployed. Does not name the validator Jobs (`aicr--`) | +| `--service-account-name` | | string | aicr-validate | ServiceAccount the **live snapshot-capture agent** runs as. **Exact-if-exists:** an existing ServiceAccount of exactly this name in `--namespace` is used verbatim and the agent creates no RBAC for the run; otherwise the value is a prefix for the agent's ServiceAccount, Role, and RoleBinding and the run ID is appended (`-`). Inert when `--snapshot` is supplied. Does not name the validator Jobs' ServiceAccount (`aicr-validator-`), whose RBAC is always run-scoped. Generate that ServiceAccount's RBAC manifests with `aicr snapshot --namespace --add-roles-to-service-account ` (matching this command's `--namespace`) and apply them yourself — that command applies nothing | | `--node-selector` | | string[] | | Override GPU node selection for the live snapshot agent (when `--snapshot` is omitted) and inner validation workloads. Replaces platform-specific selectors (e.g., `cloud.google.com/gke-accelerator`, `node.kubernetes.io/instance-type`) on inner workloads like NCCL benchmark pods. Use when GPU nodes have non-standard labels. Does not affect the validator orchestrator Job. (format: key=value, repeatable) | | `--toleration` | | string[] | | Override tolerations for the live snapshot agent (when `--snapshot` is omitted) and inner validation workloads. When omitted, the snapshot agent tolerates all taints. Does not affect the validator orchestrator Job. (format: key=value:effect, repeatable) | | `--timeout` | | duration | 5m | Timeout for validation Job completion | @@ -1248,8 +1265,9 @@ spec: namespace: aicr-validation image: ghcr.io/nvidia/aicr:v0.19.0 imagePullSecrets: [registry-secret] - jobName: aicr-validate - serviceAccountName: aicr + # Optional prefixes for the live-capture agent, not names — the run ID + # is always appended. Omitted here so the defaults apply + # (both aicr-validate). nodeSelector: my-org/gpu-pool: "true" tolerations: # [] clears the live snapshot agent's tolerate-all default diff --git a/pkg/cli/consts.go b/pkg/cli/consts.go index bd7713910..7db2a5ad7 100644 --- a/pkg/cli/consts.go +++ b/pkg/cli/consts.go @@ -39,6 +39,12 @@ const ( flagSlurmAccountingMode = "slurm-accounting-mode" flagRuntimeInventory = "runtime-inventory" flagNoHealth = "no-health" + + // flagAddRolesToSA switches `aicr snapshot` into a generate-and-exit + // invocation that WRITES the RBAC manifests granting the agent's + // permissions to a named ServiceAccount and applies none of them. No + // cluster is contacted and no snapshot is taken. + flagAddRolesToSA = "add-roles-to-service-account" ) // criteriaAny is the wildcard value for any criteria dimension. diff --git a/pkg/cli/snapshot.go b/pkg/cli/snapshot.go index b5978d997..df8a659db 100644 --- a/pkg/cli/snapshot.go +++ b/pkg/cli/snapshot.go @@ -16,8 +16,11 @@ package cli import ( "context" + "fmt" + "io" "log/slog" "os" + "path/filepath" "strings" "time" @@ -153,6 +156,7 @@ func (o *snapshotCmdOptions) toAgentConfig() *aicr.AgentConfig { DiscoverNetwork: o.discoverNetwork, Requests: o.requests, Limits: o.limits, + NameBase: name, } } @@ -175,7 +179,7 @@ func (o *snapshotCmdOptions) toSnapshotDelivery() snapshotter.SnapshotDelivery { // over config values. Returns a fully-typed snapshotCmdOptions that callers // can pass to the snapshotter without further parsing. func parseSnapshotCmdOptions(cmd *cli.Command, cfg *config.AICRConfig) (*snapshotCmdOptions, error) { - if err := validateSingleValueFlags(cmd, "namespace", "image", "job-name", "service-account-name", "timeout", "template", "max-nodes-per-entry", "runtime-class", "output", "format", "config", "os", "requests", "limits", "cluster-config", "aks-gpu-pools"); err != nil { + if err := validateSingleValueFlags(cmd, "namespace", "image", "job-name", "service-account-name", flagAddRolesToSA, "timeout", "template", "max-nodes-per-entry", "runtime-class", "output", "format", "config", "os", "requests", "limits", "cluster-config", "aks-gpu-pools"); err != nil { return nil, err } @@ -268,6 +272,100 @@ func parseSnapshotCmdOptions(cmd *cli.Command, cfg *config.AICRConfig) (*snapsho }, nil } +// runWriteRoleManifests handles the generate-and-exit invocation +// `aicr snapshot --add-roles-to-service-account `: it writes the RBAC +// manifests that would grant the agent's permissions to that ServiceAccount +// and returns, applying nothing, contacting no cluster, and capturing +// nothing. +// +// It takes no context because there is no I/O to bound beyond writing four +// local files. +// +// It is a thin adapter — name derivation, the rule sets, the explanatory +// headers, and the directory layout all live in pkg/snapshotter and +// pkg/k8s/agent. What belongs here is only presenting the outcome, including +// the two things an operator must not have to discover on their own: nothing +// is live yet, and the exact commands that make it live and take it away +// again. +func runWriteRoleManifests(cmd *cli.Command, opts *snapshotCmdOptions, saName string) error { + res, err := snapshotter.WriteAgentRoleManifests(&snapshotter.AgentRolesConfig{ + Namespace: opts.namespace, + ServiceAccountName: saName, + DiscoverNetwork: opts.discoverNetwork, + }) + if err != nil { + return err + } + + writeManifestReport(cmd.Root().Writer, res) + return nil +} + +// writeManifestReport renders the outcome of a manifest-generating run. It is +// split out from runWriteRoleManifests so the properties an operator must not +// have to discover on their own — nothing was applied, how to apply it, how to +// remove it again, and that a shared ServiceAccount waives per-run permission +// isolation — are assertable without a cluster or a filesystem. +func writeManifestReport(w io.Writer, res *snapshotter.AgentRolesResult) { + fmt.Fprintf(w, `Wrote the snapshot agent's RBAC manifests for ServiceAccount %[1]q in namespace +%[2]q to: + + %[3]s/ + +NOTHING WAS APPLIED. No cluster was contacted, and %[1]s has no new permissions +yet. + +`, res.ServiceAccountName, res.Namespace, res.Dir) + + for _, obj := range res.Objects { + fmt.Fprintf(w, " %-28s %s/%s\n", filepath.Base(obj.Path), strings.ToLower(obj.Kind), obj.Name) + } + + fmt.Fprintf(w, ` +Read them — each file explains what it grants and why the agent needs it — then +apply them yourself: + kubectl apply -f %[1]s/ + +Capture a snapshot as this ServiceAccount with: + aicr snapshot --namespace %[2]s --service-account-name %[3]s + +Remove the grant when the ServiceAccount no longer needs it: + kubectl delete -f %[1]s/ + +That delete is the only teardown: no aicr run creates, refreshes, or deletes +these objects. Keep the directory for as long as you want the easy teardown. + +The ServiceAccount is not verified to exist — aicr contacted no cluster. Check +it before you rely on the grant: + kubectl get serviceaccount %[3]s -n %[2]s + +Trade-off: runs that share this ServiceAccount share its permissions, so per-run +permission isolation is waived for them. +`, res.Dir, res.Namespace, res.ServiceAccountName) + + if !res.DiscoverNetwork { + return + } + fmt.Fprintf(w, ` +WARNING: --discover-network means the ClusterRole in this directory also carries +cluster-scoped MUTATING rules (nodes: patch, pods/exec: create, CRD/namespace/ +DaemonSet create-delete). Applying it grants them permanently, not for one run. +Each rule and the discovery step it exists for is explained in %s. +`, clusterRoleManifestName(res)) +} + +// clusterRoleManifestName returns the file name of the rendered ClusterRole, +// looked up by kind rather than by position so the discovery warning keeps +// pointing at the right file if the manifest order ever changes. +func clusterRoleManifestName(res *snapshotter.AgentRolesResult) string { + for _, obj := range res.Objects { + if obj.Kind == "ClusterRole" { + return filepath.Base(obj.Path) + } + } + return "the ClusterRole manifest" +} + // snapshotTemplateOptions holds parsed template options for the snapshot command. type snapshotTemplateOptions struct { templatePath string @@ -344,14 +442,17 @@ func snapshotCmdFlags() []cli.Flag { }, &cli.StringFlag{ Name: "job-name", - Usage: "Override default Job name", - Value: name, + Usage: "Job name prefix (default: \"aicr\"); the run ID is always appended", Category: catAgentDeployment, }, &cli.StringFlag{ Name: "service-account-name", - Usage: "Override default ServiceAccount name", - Value: name, + Usage: "ServiceAccount to run the agent as. Exact-if-exists: when a ServiceAccount of exactly this name already exists in --namespace it is used verbatim and aicr creates and deletes no RBAC for the run (generate its RBAC manifests with --add-roles-to-service-account, then apply them yourself). Otherwise it is a name prefix (default: \"aicr\") and the run ID is appended.", + Category: catAgentDeployment, + }, + &cli.StringFlag{ + Name: flagAddRolesToSA, + Usage: "WRITES MANIFESTS AND APPLIES NOTHING. Renders the Role, RoleBinding, ClusterRole and ClusterRoleBinding that grant the agent's permissions to the named ServiceAccount into ./snapshot-rbac-/, then exits without taking a snapshot. No cluster is contacted, so no kubeconfig or privileges are needed. Review the files, then grant with 'kubectl apply -f /' and revoke with 'kubectl delete -f /' yourself. Add --discover-network to include the mutating live-discovery rules.", Category: catAgentDeployment, }, &cli.StringSliceFlag{ @@ -527,6 +628,15 @@ See examples/templates/snapshot-template.md.tmpl for a sample template. return err } + // Generate-and-exit: --add-roles-to-service-account writes the + // RBAC manifests for an existing ServiceAccount, applies nothing, + // and takes no snapshot. Checked ahead of every collection path + // (including the in-pod one below) so the flag can never be + // combined with a capture. + if saName := cmd.String(flagAddRolesToSA); saName != "" { + return runWriteRoleManifests(cmd, opts, saName) + } + agentCfg := opts.toAgentConfig() // When running inside an agent Job, collect locally instead of diff --git a/pkg/cli/snapshot_config_test.go b/pkg/cli/snapshot_config_test.go index 4d0e2820b..256a87179 100644 --- a/pkg/cli/snapshot_config_test.go +++ b/pkg/cli/snapshot_config_test.go @@ -590,6 +590,7 @@ func TestSnapshotCmdOptions_ToAgentConfig(t *testing.T) { {"ClusterConfigPath", ac.ClusterConfigPath, "/l8k/cluster-config.yaml"}, {"AKSGPUPoolsPath", ac.AKSGPUPoolsPath, "/aks/pools.json"}, {"DiscoverNetwork", ac.DiscoverNetwork, true}, + {"NameBase", ac.NameBase, name}, } for _, w := range wants { if !reflect.DeepEqual(w.got, w.want) { diff --git a/pkg/cli/snapshot_test.go b/pkg/cli/snapshot_test.go index 0efeab428..674970c78 100644 --- a/pkg/cli/snapshot_test.go +++ b/pkg/cli/snapshot_test.go @@ -15,6 +15,7 @@ package cli import ( + "bytes" "context" "os" "path/filepath" @@ -450,3 +451,214 @@ func TestOutputDestinationParsing(t *testing.T) { }) } } + +// TestSnapshotCmd_AddRolesFlagWiring covers the CLI surface of the +// generate-and-exit invocation. Rendering the manifests is covered in +// pkg/k8s/agent and pkg/snapshotter; what this asserts is the wiring an +// operator touches: the flag exists under the agent-deployment category, its +// usage says outright that it applies nothing (the flag NAME reads like it +// mutates the cluster, so nobody may run it and assume the grant is live), +// and it is single-valued so a repeated flag is rejected rather than silently +// taking the last value and writing manifests for the wrong ServiceAccount. +func TestSnapshotCmd_AddRolesFlagWiring(t *testing.T) { + t.Run("flag is registered under agent deployment", func(t *testing.T) { + var found cli.Flag + for _, f := range snapshotCmd().Flags { + for _, n := range f.Names() { + if n == flagAddRolesToSA { + found = f + } + } + } + if found == nil { + t.Fatalf("snapshot command must define --%s", flagAddRolesToSA) + } + sf, ok := found.(*cli.StringFlag) + if !ok { + t.Fatalf("--%s is %T, want *cli.StringFlag", flagAddRolesToSA, found) + } + if sf.Category != catAgentDeployment { + t.Errorf("--%s category = %q, want %q", flagAddRolesToSA, sf.Category, catAgentDeployment) + } + if !strings.Contains(sf.Usage, "without taking a snapshot") { + t.Errorf("--%s usage must say it takes no snapshot; got %q", flagAddRolesToSA, sf.Usage) + } + if !strings.Contains(sf.Usage, "APPLIES NOTHING") { + t.Errorf("--%s usage must state plainly that it applies nothing; got %q", flagAddRolesToSA, sf.Usage) + } + if !strings.Contains(sf.Usage, "kubectl apply") || !strings.Contains(sf.Usage, "kubectl delete") { + t.Errorf("--%s usage must name the apply and delete commands; got %q", flagAddRolesToSA, sf.Usage) + } + }) + + t.Run("repeated flag is rejected", func(t *testing.T) { + err := runSnapshotCmdExpectErr(t, []string{ + "--" + flagAddRolesToSA, "sa-one", + "--" + flagAddRolesToSA, "sa-two", + }) + if err == nil { + t.Fatalf("repeated --%s accepted; the last value would silently win", flagAddRolesToSA) + } + if !strings.Contains(err.Error(), flagAddRolesToSA) { + t.Errorf("error = %q, want it to name --%s", err.Error(), flagAddRolesToSA) + } + }) +} + +// TestSnapshotCmd_ServiceAccountNameUsageStatesExactIfExists pins the one +// thing an operator has to learn from `--help`: --service-account-name is no +// longer only a prefix. A usage string that still says "prefix" alone would +// send an IRSA user straight into the silent credential loss this change +// exists to close. +func TestSnapshotCmd_ServiceAccountNameUsageStatesExactIfExists(t *testing.T) { + for _, f := range snapshotCmd().Flags { + sf, ok := f.(*cli.StringFlag) + if !ok || sf.Name != "service-account-name" { + continue + } + if !strings.Contains(sf.Usage, "xact-if-exists") { + t.Errorf("--service-account-name usage does not state the exact-if-exists rule; got %q", sf.Usage) + } + return + } + t.Fatal("snapshot command must define --service-account-name") +} + +// TestSnapshotCmd_AddRolesWritesManifestsWithoutACluster runs the real command +// action end to end. It is the test that pins the whole point of the flag: with +// KUBECONFIG pointed at a file no client can be built from and the in-cluster +// environment cleared, the invocation still succeeds and writes a reviewable +// directory. Any clientset construction, ServiceAccount lookup, or permission +// pre-flight on this path would fail here rather than pass quietly. +func TestSnapshotCmd_AddRolesWritesManifestsWithoutACluster(t *testing.T) { + unreachable := filepath.Join(t.TempDir(), "not-a-kubeconfig") + if seedErr := os.WriteFile(unreachable, []byte("this is not a kubeconfig\n"), 0o600); seedErr != nil { + t.Fatalf("seeding the kubeconfig: %v", seedErr) + } + t.Setenv("KUBECONFIG", unreachable) + t.Setenv("HOME", t.TempDir()) + t.Setenv("KUBERNETES_SERVICE_HOST", "") + t.Setenv("KUBERNETES_SERVICE_PORT", "") + t.Chdir(t.TempDir()) + + var buf bytes.Buffer + cmd := snapshotCmd() + cmd.Writer = &buf + if err := cmd.Run(context.Background(), []string{ + "snapshot", + "--namespace", "gpu-operator", + "--" + flagAddRolesToSA, "irsa-snapshotter", + "--discover-network", + }); err != nil { + t.Fatalf("snapshot --%s error = %v; the path must need no cluster at all", flagAddRolesToSA, err) + } + + entries, readErr := os.ReadDir(".") + if readErr != nil { + t.Fatalf("reading the working directory: %v", readErr) + } + var dir string + for _, e := range entries { + if e.IsDir() && strings.HasPrefix(e.Name(), "snapshot-rbac-") { + dir = e.Name() + } + } + if dir == "" { + t.Fatalf("no snapshot-rbac- directory written; got %v", entries) + } + + for _, name := range []string{"01-role.yaml", "02-rolebinding.yaml", "03-clusterrole.yaml", "04-clusterrolebinding.yaml"} { + body, err := os.ReadFile(filepath.Join(dir, name)) + if err != nil { + t.Errorf("reading %s: %v", name, err) + continue + } + if !strings.HasPrefix(string(body), "# ") { + t.Errorf("%s does not open with a YAML comment header", name) + } + } + + // The report goes to the command writer, not stdout, and must name the + // directory and both halves of the operator's workflow. + out := buf.String() + for _, want := range []string{"NOTHING WAS APPLIED", dir, "kubectl apply -f", "kubectl delete -f", "MUTATING"} { + if !strings.Contains(out, want) { + t.Errorf("command output does not contain %q; got:\n%s", want, out) + } + } +} + +// TestWriteManifestReport asserts the properties the output must state +// outright, because an operator who does not read them cannot discover any of +// them from the cluster: nothing was applied, where the manifests are, the +// command that makes the grant live, the command that removes it again, and +// that sharing one ServiceAccount waives per-run permission isolation. The +// --discover-network warning is separate because that grant is the one that +// is also mutating. +func TestWriteManifestReport(t *testing.T) { + dir := "snapshot-rbac-20260821-142233-9f3a1c0b7e2d4a55" + res := &snapshotter.AgentRolesResult{ + Dir: dir, + RunID: "20260821-142233-9f3a1c0b7e2d4a55", + Namespace: "gpu-operator", + ServiceAccountName: "irsa-snapshotter", + Objects: []snapshotter.AgentRoleObject{ + {Kind: "Role", Name: "aicr-agent-irsa-snapshotter-rbac", Path: dir + "/01-role.yaml"}, + {Kind: "RoleBinding", Name: "aicr-agent-irsa-snapshotter-rbac", Path: dir + "/02-rolebinding.yaml"}, + {Kind: "ClusterRole", Name: "aicr-agent-gpu-operator.irsa-snapshotter-rbac", Path: dir + "/03-clusterrole.yaml"}, + {Kind: "ClusterRoleBinding", Name: "aicr-agent-gpu-operator.irsa-snapshotter-rbac", Path: dir + "/04-clusterrolebinding.yaml"}, + }, + } + + tests := []struct { + name string + discoverNetwork bool + wantSubstrings []string + notWant string + }{ + { + name: "read-only grant", + wantSubstrings: []string{ + `ServiceAccount "irsa-snapshotter" in namespace`, + "NOTHING WAS APPLIED", + "kubectl apply -f " + dir + "/", + "kubectl delete -f " + dir + "/", + "01-role.yaml", + "role/aicr-agent-irsa-snapshotter-rbac", + "clusterrolebinding/aicr-agent-gpu-operator.irsa-snapshotter-rbac", + "not verified to exist", + "permission isolation is waived", + "aicr snapshot --namespace gpu-operator --service-account-name irsa-snapshotter", + }, + notWant: "MUTATING", + }, + { + name: "discovery grant warns about the mutating rules", + discoverNetwork: true, + wantSubstrings: []string{ + "NOTHING WAS APPLIED", + "MUTATING", + "grants them permanently, not for one run", + "03-clusterrole.yaml", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := *res + r.DiscoverNetwork = tt.discoverNetwork + var buf bytes.Buffer + writeManifestReport(&buf, &r) + + for _, want := range tt.wantSubstrings { + if !strings.Contains(buf.String(), want) { + t.Errorf("report does not contain %q; got:\n%s", want, buf.String()) + } + } + if tt.notWant != "" && strings.Contains(buf.String(), tt.notWant) { + t.Errorf("report contains %q but should not; got:\n%s", tt.notWant, buf.String()) + } + }) + } +} diff --git a/pkg/cli/validate.go b/pkg/cli/validate.go index 859830060..395e2a92f 100644 --- a/pkg/cli/validate.go +++ b/pkg/cli/validate.go @@ -39,6 +39,14 @@ import ( v1 "github.com/NVIDIA/aicr/pkg/validator/v1" ) +// validateNameBase is the prefix applied to the live-capture snapshot +// agent's Job/ServiceAccount/RBAC names when --job-name / --service-account-name +// are left unset, distinguishing `aicr validate`'s agent resources from +// `aicr snapshot`'s (which use the package-level "aicr" base). RunID is +// always appended, so the deployed names are run-scoped regardless of +// whether this base is used. +const validateNameBase = "aicr-validate" + // validateAgentConfig holds parsed agent configuration for validate command. type validateAgentConfig struct { kubeconfig string @@ -54,6 +62,14 @@ type validateAgentConfig struct { debug bool requireGPU bool aksGPUPoolsPath string + + // runID correlates this run's live-capture snapshot agent with the + // validator Jobs runValidation deploys for the same `aicr validate` + // invocation — both are given the SAME id (generated once, up front, + // by the caller of parseValidateAgentConfig) so ADR-020 run isolation + // scopes every resource this command creates to a single run rather + // than splitting it across two independently generated ids. + runID string } // parseValidateAgentConfig builds the snapshot-capture agent's deployment @@ -61,10 +77,17 @@ type validateAgentConfig struct { // namespace, cleanup) are resolved once by the caller and passed in; this // keeps any CLI-overrides-config slog.Info from firing twice when both // the agent and the downstream validator job want the same value. +// +// runID is the single id generated once for the whole `aicr validate` +// invocation; passing it in here (rather than leaving AgentConfig.RunID +// empty and letting the snapshot agent default its own) is what keeps the +// live-capture agent and the validator Jobs it feeds correlated under one +// RunID. func parseValidateAgentConfig( cmd *cli.Command, resolved *config.ValidateResolved, shared validateSharedResolved, + runID string, ) *validateAgentConfig { return &validateAgentConfig{ @@ -81,6 +104,7 @@ func parseValidateAgentConfig( debug: cmd.Bool("debug"), requireGPU: boolFlagOrConfig(cmd, "require-gpu", resolved.RequireGPU), aksGPUPoolsPath: cmd.String("aks-gpu-pools"), + runID: runID, } } @@ -156,7 +180,11 @@ func resolveValidateTolerations(cmd *cli.Command, resolved *config.ValidateResol // toAgentConfig projects the validate command's resolved flags onto the // facade AgentConfig that Client.CollectSnapshot consumes. Privileged is // unconditional here: the validation snapshot needs the GPU and SystemD -// collectors, which do not work in restricted mode. +// collectors, which do not work in restricted mode. RunID forwards the +// single id parseValidateAgentConfig's caller generated for this whole +// `aicr validate` invocation, so the live-capture agent's Job/RBAC share it +// with the validator Jobs runValidation deploys — one RunID per command, +// per ADR-020, instead of the agent falling back to its own default. func (c *validateAgentConfig) toAgentConfig() *aicr.AgentConfig { return &aicr.AgentConfig{ Kubeconfig: c.kubeconfig, @@ -173,6 +201,8 @@ func (c *validateAgentConfig) toAgentConfig() *aicr.AgentConfig { Privileged: true, RequireGPU: c.requireGPU, AKSGPUPoolsPath: c.aksGPUPoolsPath, + RunID: c.runID, + NameBase: validateNameBase, } } @@ -207,6 +237,13 @@ type validationConfig struct { // Input phases []validator.Phase + // runID is generated once, up front, for the whole `aicr validate` + // invocation (see validateCmd's Action) and reused here instead of a + // second v1.GenerateRunID() call. Sharing one ID keeps names, labels, + // logs, and cleanup hints correlated to a single command instead of + // fragmenting across an agent-side ID and a validator-side one. + runID string + // Kubeconfig path; propagated to ConfigMap reads/writes so a single // validate invocation can target a non-default cluster end-to-end. kubeconfig string @@ -257,12 +294,13 @@ func runValidation( slog.Info("running validation", "phases", cfg.phases) - // Generate the run ID CLI-side rather than letting the validator - // auto-generate it internally: the no-cleanup debug log below needs to - // surface the same ID the validator stamps on its Jobs/RBAC so an - // operator can locate the kept resources. Passing it via - // WithValidationRunID keeps that value in our hands. - runID := v1.GenerateRunID() + // Generated once CLI-side, before either snapshot source ran (see + // validateCmd's Action), rather than letting the validator auto-generate + // it internally: the no-cleanup debug log below needs to surface the + // same ID the validator stamps on its Jobs/RBAC so an operator can + // locate the kept resources. Passing it via WithValidationRunID keeps + // that value in our hands. + runID := cfg.runID // Translate the resolved CLI values into facade ValidateOptions. These // mirror the validator.With* options the direct invocation used to set; @@ -455,14 +493,12 @@ func validateCmdFlags() []cli.Flag { }, &cli.StringFlag{ Name: "job-name", - Usage: "Override default Job name", - Value: "aicr-validate", + Usage: "Job name prefix (default: \"aicr-validate\"); the run ID is always appended", Category: catAgentDeployment, }, &cli.StringFlag{ Name: "service-account-name", - Usage: "Override default ServiceAccount name", - Value: name, + Usage: "ServiceAccount the live snapshot-capture agent runs as. Exact-if-exists: when a ServiceAccount of exactly this name already exists in --namespace it is used verbatim and the agent creates and deletes no RBAC for the run (generate its RBAC manifests with 'aicr snapshot --add-roles-to-service-account', then apply them yourself). Otherwise it is a name prefix (default: \"aicr-validate\") and the run ID is appended.", Category: catAgentDeployment, }, &cli.StringSliceFlag{ @@ -805,6 +841,38 @@ constraint (e.g. K8s version) is not met — --fail-on-error scopes to phase che return err } + // Generate the run ID once, up front, for this whole invocation — + // before either snapshot source below runs. The validator Jobs + // deployed by runValidation need it (passed through + // validationConfig.runID); generating it here rather than inside + // runValidation means a single `aicr validate` no longer risks + // splitting its correlation ID across two independent + // v1.GenerateRunID() calls. The same runID is also threaded into + // parseValidateAgentConfig below, so the live-capture branch's + // snapshot agent shares this run's id with the validator Jobs + // rather than defaulting its own inside DeployAndCollect — + // ADR-020 requires one RunID per `aicr validate` invocation. + // + // WARNING, not enforced by an automated test: this line must + // stay the ONLY v1.GenerateRunID() call in this Action. Adding a + // second call — e.g. generating a fresh id inline for the + // parseValidateAgentConfig call below, or for the + // validationConfig{runID: ...} literal further down — silently + // splits a single `aicr validate` invocation back into two + // uncorrelated runs. The two consumer sites are NOT mutually + // exclusive: with neither --snapshot nor --no-cluster, the + // live-capture branch below and runValidation both consume this + // id in the same invocation — which is exactly the case the + // invariant protects. What blocks a unit test is that reaching + // both requires live-cluster I/O (deployAgentForValidation + // deploys a Job; runValidation deploys validator Jobs) with no + // injectable seam here to fake it. See + // TestValidateAgentConfig_ToAgentConfig_ForwardsRunID and + // TestParseValidateAgentConfig_ForwardsCallerRunID in + // validate_test.go for what IS covered (passthrough at each + // function boundary only) and what is not. + runID := v1.GenerateRunID() + var snap *aicr.Snapshot // --no-cluster means "do not touch the cluster". The agent-deploy @@ -826,7 +894,7 @@ constraint (e.g. K8s version) is not met — --fail-on-error scopes to phase che } else { slog.Info("deploying agent to capture snapshot") - agentCfg := parseValidateAgentConfig(cmd, resolved, shared) + agentCfg := parseValidateAgentConfig(cmd, resolved, shared, runID) var deployErr error snap, deployErr = deployAgentForValidation(ctx, client, agentCfg) @@ -861,6 +929,7 @@ constraint (e.g. K8s version) is not met — --fail-on-error scopes to phase che return runValidation(ctx, client, rec, snap, validationConfig{ phases: phases, + runID: runID, kubeconfig: kubeconfig, output: cmd.String("output"), outFormat: serializer.FormatJSON, diff --git a/pkg/cli/validate_test.go b/pkg/cli/validate_test.go index cd5db95f3..22402bd1c 100644 --- a/pkg/cli/validate_test.go +++ b/pkg/cli/validate_test.go @@ -470,6 +470,81 @@ func TestDeployAgentForValidation_ExplicitKubeconfigFailsFast(t *testing.T) { } } +// TestValidateAgentConfig_ToAgentConfig_ForwardsRunID covers ONLY the +// toAgentConfig projection boundary: given a validateAgentConfig whose +// runID field is already set, toAgentConfig must copy it onto the facade +// AgentConfig.RunID unchanged (and set NameBase to validateNameBase rather +// than leaving JobName/ServiceAccountName to carry the naming prefix). +// +// What this does NOT cover: it never runs the validateCmd Action, so it +// says nothing about whether `aicr validate` generates exactly one RunID +// per invocation and hands that SAME value to both the live-capture +// snapshot agent and the validator Jobs — that single-generation invariant +// (ADR-020 Ruling 7) is not exercised by an automated test at all; see the +// WARNING comment on the Action's `runID := v1.GenerateRunID()` call in +// validate.go for why, and TestParseValidateAgentConfig_ForwardsCallerRunID +// below for the adjacent (also boundary-only) coverage on the parsing side. +func TestValidateAgentConfig_ToAgentConfig_ForwardsRunID(t *testing.T) { + cfg := &validateAgentConfig{ + namespace: "aicr-validation-test", + runID: "20260821-142233-9f3a1c0b7e2d4a55", + } + + ac := cfg.toAgentConfig() + + if ac.RunID != cfg.runID { + t.Errorf("AgentConfig.RunID = %q, want %q (validateAgentConfig.runID)", ac.RunID, cfg.runID) + } + if ac.NameBase != validateNameBase { + t.Errorf("AgentConfig.NameBase = %q, want %q", ac.NameBase, validateNameBase) + } +} + +// TestParseValidateAgentConfig_ForwardsCallerRunID covers ONLY +// parseValidateAgentConfig's own mapping: the runID parameter it is given +// lands unchanged on the returned validateAgentConfig.runID field. +// +// This test replaces cmd.Action outright (to isolate that mapping from +// recipe/snapshot I/O without touching a cluster), so NONE of the +// production Action code at validate.go's `runID := v1.GenerateRunID()` +// call through the parseValidateAgentConfig call site actually runs here. +// It cannot detect a regression in how the real Action generates or +// threads runID — in particular it says nothing about ADR-020 Ruling 7's +// single-generation invariant (one v1.GenerateRunID() call feeding BOTH +// the live-capture agent and the validator Jobs). See the WARNING comment +// on that call site in validate.go: no automated test in this package +// enforces single-generation. The two consumer sites are NOT mutually +// exclusive — with neither --snapshot nor --no-cluster, the live-capture +// branch and runValidation both consume the id in one invocation. What +// blocks the test is that exercising both requires live-cluster I/O (both +// branches deploy Jobs) with no injectable seam in the Action to fake it. +func TestParseValidateAgentConfig_ForwardsCallerRunID(t *testing.T) { + const wantRunID = "20260821-142233-9f3a1c0b7e2d4a55" + + var captured *validateAgentConfig + cmd := validateCmd() + cmd.Action = func(ctx context.Context, c *cli.Command) error { + cfg, err := loadCmdConfig(ctx, c) + if err != nil { + return err + } + resolved, err := cfg.Validation().Resolve() + if err != nil { + return err + } + shared := validateSharedResolved{namespace: "aicr-validation-test"} + captured = parseValidateAgentConfig(c, resolved, shared, wantRunID) + return nil + } + if err := cmd.Run(t.Context(), []string{"validate", "--no-cluster"}); err != nil { + t.Fatalf("validate run: %v", err) + } + + if captured.runID != wantRunID { + t.Errorf("validateAgentConfig.runID = %q, want %q", captured.runID, wantRunID) + } +} + // TestClassifyIgnoredAKSGPUPools pins the provenance matrix of the // ignored-projection note: explicit CLI presence (either flag form) always // warns; a purely ambient env source is demoted to debug; nothing logs diff --git a/pkg/client/v1/aicr.go b/pkg/client/v1/aicr.go index 4fa6c3b12..4aacfe65f 100644 --- a/pkg/client/v1/aicr.go +++ b/pkg/client/v1/aicr.go @@ -1725,9 +1725,18 @@ func resolveHelmComponentValues( // without breaking signatures. CollectSnapshot is therefore safe even // on a Client whose recipe source is unrelated to the target cluster. // -// cfg.Kubeconfig is the path (or empty for in-cluster). cfg.Namespace, -// cfg.Image, cfg.ServiceAccountName must be set; other fields fall -// back to package defaults documented on snapshotter.AgentConfig. +// cfg.Kubeconfig is the path (or empty for in-cluster). cfg.Namespace and +// cfg.Image must be set. cfg.JobName is an optional naming prefix, not a +// required name — leaving it empty is fine: cfg.NameBase (default "aicr") +// supplies the prefix instead, and cfg.RunID is appended to whichever +// prefix applies, so the Job this call deploys is named uniquely to this +// run either way. +// +// cfg.ServiceAccountName is exact-if-exists: it names an existing +// ServiceAccount when one of exactly that name is already in cfg.Namespace +// — in which case this call creates and deletes no RBAC at all — and is a +// prefix otherwise. See its own documentation on AgentConfig. Other fields +// fall back to package defaults documented on snapshotter.AgentConfig. // // # Output and delivery // @@ -1784,8 +1793,17 @@ func resolveHelmComponentValues( // carry the appropriate pkg/errors codes (ErrCodeInternal for // deployment failures, ErrCodeTimeout for context expiry, etc.). // -// Concurrent CollectSnapshot calls are safe; each call constructs an -// independent run. +// Concurrent CollectSnapshot calls are safe: each gets its own RunID +// (cfg.RunID when set, otherwise generated) and, from it, its own Job, +// RBAC, and — when cfg.Output does not name one — its own staging +// ConfigMap; independence means each call's objects, permissions, +// captured result, and Cleanup are scoped to that RunID alone, so +// concurrent runs never collide on a shared resource name or delete +// something another run created. The one effect that is still shared by +// design: two calls that set cfg.Output to the SAME explicit +// cm://namespace/name URI write to that one caller-named ConfigMap and +// overwrite each other — Output identifies a caller-owned destination, +// not a run-scoped one, so RunID does not disambiguate it. func (c *Client) CollectSnapshot(ctx context.Context, cfg *AgentConfig) (*Snapshot, error) { if c == nil { return nil, errors.New(errors.ErrCodeInvalidRequest, "aicr client not initialized") diff --git a/pkg/client/v1/translate.go b/pkg/client/v1/translate.go index 8bbbfc905..a7377b592 100644 --- a/pkg/client/v1/translate.go +++ b/pkg/client/v1/translate.go @@ -67,6 +67,8 @@ func toInternalAgentConfig(cfg *AgentConfig) *snapshotter.AgentConfig { Requests: cfg.Requests, Limits: cfg.Limits, AKSGPUPoolsPath: cfg.AKSGPUPoolsPath, + RunID: cfg.RunID, + NameBase: cfg.NameBase, } } diff --git a/pkg/client/v1/types.go b/pkg/client/v1/types.go index 87693ed84..2e5d68dee 100644 --- a/pkg/client/v1/types.go +++ b/pkg/client/v1/types.go @@ -122,25 +122,50 @@ func (s *Snapshot) Unwrap() *snapshotter.Snapshot { // this type — so an unplumbed field is a test failure rather than a silent // zero value. type AgentConfig struct { - Kubeconfig string - Namespace string - Image string - ImagePullSecrets []string - JobName string + Kubeconfig string + Namespace string + Image string + ImagePullSecrets []string + JobName string + + // ServiceAccountName selects the ServiceAccount the agent pod runs + // as. It is EXACT-IF-EXISTS, so it carries two meanings: + // + // - A ServiceAccount of exactly this name already exists in + // Namespace: it is used verbatim, and the run creates NO + // ServiceAccount, Role, RoleBinding, ClusterRole or + // ClusterRoleBinding — and deletes none at cleanup. This is how a + // ServiceAccount carrying IRSA (eks.amazonaws.com/role-arn) or + // GKE Workload Identity (iam.gke.io/gcp-service-account) + // annotations stays usable: both providers pin trust to the + // ServiceAccount NAME, which a run-scoped name can never satisfy. + // Generate the RBAC that grants it the agent's permissions with + // snapshotter.WriteAgentRoleManifests (CLI: + // `aicr snapshot --add-roles-to-service-account`), which writes + // manifests and applies nothing, then apply them yourself. + // - Otherwise: a name prefix. The run creates "-" + // and the full run-scoped RBAC set, and deletes them at cleanup. + // + // Empty falls back to NameBase and is never probed for existence. + // + // Using an existing ServiceAccount waives per-run permission + // isolation: concurrent runs sharing it share its grants, and grants + // provisioned for DiscoverNetwork persist beyond any one run. ServiceAccountName string - NodeSelector map[string]string - Tolerations []corev1.Toleration - Timeout time.Duration - Cleanup bool - Debug bool - Privileged bool - RequireGPU bool - RuntimeClassName string - TemplatePath string - MaxNodesPerEntry int - OS string - Requests corev1.ResourceList - Limits corev1.ResourceList + + NodeSelector map[string]string + Tolerations []corev1.Toleration + Timeout time.Duration + Cleanup bool + Debug bool + Privileged bool + RequireGPU bool + RuntimeClassName string + TemplatePath string + MaxNodesPerEntry int + OS string + Requests corev1.ResourceList + Limits corev1.ResourceList // Output selects where the agent Job stages its result. A cm://namespace/name // URI makes that ConfigMap the delivery vehicle — the Job writes there and @@ -175,6 +200,30 @@ type AgentConfig struct { // Required for AKS profile-qualified resolution from a collected // snapshot; empty disables the projection. AKSGPUPoolsPath string + + // RunID scopes every resource this deployment creates (Job, RBAC, and + // the internal staging ConfigMap when Output does not name one) to a + // single run, so concurrent snapshot-agent runs never collide on a + // shared resource name. Empty lets CollectSnapshot's underlying + // deployment generate one; SDK and CLI callers normally leave it + // unset. Set it explicitly to correlate this run with an external + // identifier — `aicr validate` does this to give its live-capture + // snapshot agent and its validator Jobs the same RunID. + RunID string + + // NameBase prefixes generated Job/ServiceAccount/RBAC names. The + // fallback is per name, not all-or-nothing: the Job uses JobName when + // set and NameBase otherwise, while the ServiceAccount, Role and + // RoleBinding use ServiceAccountName when set and NameBase otherwise. + // Setting only one of the two therefore leaves NameBase governing the + // other. Defaults to "aicr" when also empty. + // + // JobName is likewise an optional prefix, not a required name — + // RunID is appended to whichever prefix applies, so the deployed Job + // name is always run-scoped. ServiceAccountName is a prefix only + // when no ServiceAccount of that exact name exists; see its own + // documentation above. + NameBase string } // Criteria is the facade-owned, semver-stable shape of a recipe-resolution diff --git a/pkg/config/resolve.go b/pkg/config/resolve.go index f0231da19..fb4299fc4 100644 --- a/pkg/config/resolve.go +++ b/pkg/config/resolve.go @@ -337,10 +337,19 @@ type ValidateResolved struct { // config did not set the field. ImagePullSecrets []string - // JobName is spec.validate.agent.jobName. + // JobName is spec.validate.agent.jobName — an optional Job name + // prefix, not a required name. Empty (config unset and no + // --job-name) lets the CLI's own default prefix ("aicr-validate") + // apply instead; either way the run ID is appended, so the deployed + // Job name is always run-scoped. JobName string - // ServiceAccountName is spec.validate.agent.serviceAccountName. + // ServiceAccountName is spec.validate.agent.serviceAccountName. It is + // exact-if-exists: when a ServiceAccount of exactly this name already + // exists in the namespace, the live snapshot-capture agent runs as it + // verbatim and creates no RBAC for the run; otherwise it is an + // optional name prefix with the same empty-value behavior as JobName. + // See pkg/snapshotter.AgentConfig.ServiceAccountName. ServiceAccountName string // NodeSelector is spec.validate.agent.nodeSelector. Nil if unset; @@ -641,10 +650,19 @@ type SnapshotResolved struct { // config did not set the field. ImagePullSecrets []string - // JobName is spec.snapshot.agent.jobName. + // JobName is spec.snapshot.agent.jobName — an optional Job name + // prefix, not a required name. Empty (config unset and no + // --job-name) lets the CLI's own default prefix ("aicr") apply + // instead; either way the run ID is appended, so the deployed Job + // name is always run-scoped. JobName string - // ServiceAccountName is spec.snapshot.agent.serviceAccountName. + // ServiceAccountName is spec.snapshot.agent.serviceAccountName. It is + // exact-if-exists: when a ServiceAccount of exactly this name already + // exists in the namespace, the agent runs as it verbatim and creates + // no RBAC for the run; otherwise it is an optional name prefix with + // the same empty-value behavior as JobName. See + // pkg/snapshotter.AgentConfig.ServiceAccountName. ServiceAccountName string // NodeSelector is spec.snapshot.agent.nodeSelector. Nil if unset; diff --git a/pkg/defaults/k8s.go b/pkg/defaults/k8s.go index 64b1b0185..cc4c4b95a 100644 --- a/pkg/defaults/k8s.go +++ b/pkg/defaults/k8s.go @@ -14,6 +14,8 @@ package defaults +import "os" + // Kubernetes REST client rate limits for validator containers. // // client-go's default client-side rate limiter (QPS 5, Burst 10) is tuned for @@ -40,3 +42,51 @@ const ( // (discovery + the first batch of GETs) is not immediately throttled. ValidatorClientBurst = 100 ) + +// K8sAccessReviewConcurrency bounds how many Self/SubjectAccessReview +// requests a pre-flight permission gate keeps in flight at once. +// +// An access review is read-only, so the checks fan out concurrently: N +// sequential reviews cost N round trips against the apiserver while one +// batch costs roughly one. The bound exists because the set is not small — +// the snapshot agent expands the agent ServiceAccount's own PolicyRules into +// one review per (apiGroup, resource, verb), which reaches several dozen on +// a --discover-network run — and opening that many connections at once buys +// nothing over a handful of waves while adding avoidable apiserver load. +const K8sAccessReviewConcurrency = 16 + +// MaxK8sNameLength is the maximum length of a Kubernetes object name built +// from a run-scoped prefix. 63 characters is the general DNS label ceiling +// (RFC 1123) that Kubernetes enforces on object names, but the binding +// constraint here is narrower: Jobs propagate their name into the +// batch.kubernetes.io/job-name label on every Pod they create, and label +// values share the same 63-character ceiling. A run-scoped Job name that +// fits the object-name limit but not the label limit would fail Pod +// creation, so name helpers must budget against this constant. +const MaxK8sNameLength = 63 + +// Layout of the RBAC manifest directory that +// `aicr snapshot --add-roles-to-service-account` writes. +// +// That invocation applies nothing and contacts no cluster: it renders the +// Role, RoleBinding, ClusterRole and ClusterRoleBinding that grant the +// snapshot agent's permissions to an operator-supplied ServiceAccount, and +// leaves applying them — and later deleting them — to the operator. The +// directory is what makes both halves a single `kubectl -f` argument. +const ( + // AgentRBACManifestDirPrefix is prepended to the run ID to form the + // output directory name (`snapshot-rbac-`). The run ID makes + // the name unique per invocation, so a second run never overwrites a + // set of manifests an operator is still reviewing. + AgentRBACManifestDirPrefix = "snapshot-rbac-" + + // AgentRBACManifestDirMode is the permission of that directory. + // Owner-only: the manifests describe a permission grant, and there is + // no reason for another local user to read or replace them between + // generation and `kubectl apply`. + AgentRBACManifestDirMode os.FileMode = 0o700 + + // AgentRBACManifestFileMode is the permission of each manifest file, + // owner read/write only for the same reason as the directory. + AgentRBACManifestFileMode os.FileMode = 0o600 +) diff --git a/pkg/k8s/agent/concurrency_test.go b/pkg/k8s/agent/concurrency_test.go new file mode 100644 index 000000000..eae7e639b --- /dev/null +++ b/pkg/k8s/agent/concurrency_test.go @@ -0,0 +1,449 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package agent + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + authv1 "k8s.io/api/authorization/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" +) + +// concurrencyTestNamespace hosts both runs in TestConcurrentRuns. Both runs +// share one namespace on purpose: name-based isolation within a shared +// namespace is exactly the property issue #2120 reported broken (fixed +// object names meant two runs clobbered each other's Job, RBAC, and +// snapshot ConfigMap). +const concurrencyTestNamespace = "concurrent-runs-ns" + +// Two run IDs in runid.Generate()'s YYYYMMDD-HHMMSS-<16 hex> format, equal +// in every field except the random suffix — isolation must hold between +// runs started in the same instant, not just runs separated in time. +const ( + concurrencyRunIDA = "20260821-090000-aaaaaaaaaaaaaaaa" + concurrencyRunIDB = "20260821-090000-bbbbbbbbbbbbbbbb" +) + +// concurrencyPodTimeEarly and concurrencyPodTimeLate seed distinct, ordered +// CreationTimestamps for TestConcurrentRuns's pod-selection assertion — see +// the comment above seedImposterPod's call site for why the ordering +// matters. +var ( + concurrencyPodTimeEarly = metav1.NewTime(time.Unix(1000, 0)) + concurrencyPodTimeLate = metav1.NewTime(time.Unix(2000, 0)) +) + +// TestConcurrentRuns is the concurrency proof required by issue #2120 +// acceptance criterion #5: two overlapping snapshot-agent runs, deployed +// from separate goroutines against one shared fake clientset and one shared +// namespace, must never collide on object names, must never leak one run's +// RBAC permissions (specifically DiscoverNetwork's extra cluster rules) +// into the other, must each select only their own Pod, must each read back +// only their own snapshot bytes, and one run's Cleanup must never touch the +// other's objects — nor a ConfigMap that merely sits at a name that run's own +// naming formula would produce but that the run does not own. Run under +// -race. +// +// Fake-clientset limitations this test works around, not around-asserts: +// +// - No Job controller runs against the fake clientset, so Pods never +// appear on their own. This test creates them explicitly (seedRunPod), +// carrying the run's RunID label and a controlling ownerReference to +// that run's Job, mirroring what a real kube-controller-manager +// produces. +// - The fake ObjectTracker does not assign a UID on Create. Without one, +// jobUID() would stay the zero UID after Deploy(), and pickLivePod +// would silently skip the ownedByJob authorization check entirely +// (see wait.go: "jobUID != "" && !ownedByJob(...)" only runs the check +// when a UID is present) — degrading pod selection to label-only +// matching and defeating the point of this test. A "create","jobs" +// reactor below assigns each Job a real, name-derived UID so +// ownedByJob is exercised for real. +// - List DOES filter by LabelSelector against the fake clientset: +// k8s.io/client-go/gentype's alsoFakeLister.List applies the selector +// client-side after the ObjectTracker's own (selector-blind) List. +// findPodName below is List-based, so it exercises real selection +// logic — this test does not need to fall back to asserting a raw +// selector string for that path. Watch does NOT filter (the fake +// Watch reactor ignores ListOptions.LabelSelector entirely); this +// test does not exercise the watch-based pod-discovery path +// (findOrWatchPodName), so that gap does not apply here. That path — +// the one production actually takes, since WaitForPodReady runs right +// after Deploy — has its own ownership coverage in +// TestFindOrWatchPodNameAuthorizesByJobOwnership (wait_test.go). +func TestConcurrentRuns(t *testing.T) { + ctx := context.Background() + clientset := fake.NewClientset() + + // Deploy's Step 0 is a permissions gate; allow everything so the test + // exercises isolation, not authorization plumbing already covered by + // permissions_test.go. + clientset.PrependReactor("create", "selfsubjectaccessreviews", func(action k8stesting.Action) (bool, runtime.Object, error) { + return true, &authv1.SelfSubjectAccessReview{ + Status: authv1.SubjectAccessReviewStatus{Allowed: true, Reason: "test permissions allowed"}, + }, nil + }) + + // See the fake-clientset limitations note above: give every created Job + // a deterministic, name-derived UID since the ObjectTracker assigns + // none on its own. + clientset.PrependReactor("create", "jobs", func(action k8stesting.Action) (bool, runtime.Object, error) { + ca, ok := action.(k8stesting.CreateActionImpl) + if !ok { + return false, nil, nil + } + job, ok := ca.GetObject().(*batchv1.Job) + if !ok { + return false, nil, nil + } + job.UID = types.UID(job.Name + "-uid") + return false, nil, nil // not handled: fall through to the default tracker create + }) + + dA := NewDeployer(clientset, Config{ + Namespace: concurrencyTestNamespace, + Image: "aicr:test", + RunID: concurrencyRunIDA, + Output: fmt.Sprintf("cm://%s/%s", concurrencyTestNamespace, nameWithRunID(staticStagingConfigMapName, concurrencyRunIDA)), + // OwnsOutputConfigMap is false for run A on purpose (unlike run B + // below): it models an object — the staging ConfigMap seeded for + // run A below — that sits at exactly the name run A's own naming + // formula computes, but that run A does not own. Assertion 5's + // ownership subtest needs exactly this shape, since name-scoping + // alone (assertion 1) cannot explain that object surviving run A's + // Cleanup. Run B keeps OwnsOutputConfigMap true so the "owned and + // recorded" path stays covered too (assertion 4). + OwnsOutputConfigMap: false, + DiscoverNetwork: false, + }) + dB := NewDeployer(clientset, Config{ + Namespace: concurrencyTestNamespace, + Image: "aicr:test", + RunID: concurrencyRunIDB, + Output: fmt.Sprintf("cm://%s/%s", concurrencyTestNamespace, nameWithRunID(staticStagingConfigMapName, concurrencyRunIDB)), + OwnsOutputConfigMap: true, + DiscoverNetwork: true, + }) + + // Deploy both runs concurrently from separate goroutines — the exact + // overlap issue #2120 reported as unsafe. + var wg sync.WaitGroup + deployErrs := make([]error, 2) + wg.Add(2) + go func() { defer wg.Done(); deployErrs[0] = dA.Deploy(ctx) }() + go func() { defer wg.Done(); deployErrs[1] = dB.Deploy(ctx) }() + wg.Wait() + if deployErrs[0] != nil { + t.Fatalf("run A Deploy() error = %v", deployErrs[0]) + } + if deployErrs[1] != nil { + t.Fatalf("run B Deploy() error = %v", deployErrs[1]) + } + + // The staging ConfigMap is written by the in-pod agent, not Deploy(); + // seed it directly for both runs so the seventh object kind exists and + // GetSnapshot has something run-specific to read back. + seedStagingConfigMap(t, ctx, clientset, dA, "cm-uid-a", "snapshot-bytes-for-run-A") + seedStagingConfigMap(t, ctx, clientset, dB, "cm-uid-b", "snapshot-bytes-for-run-B") + + // --- Assertion 1: all seven object kinds exist twice, under distinct names. + t.Run("all seven kinds exist twice under distinct names", func(t *testing.T) { + assertSevenKindsExist(t, ctx, clientset, dA) + assertSevenKindsExist(t, ctx, clientset, dB) + + namesA := []string{dA.saName(), dA.roleName(), dA.clusterRoleName(), dA.jobName(), dA.stagingConfigMapName()} + namesB := []string{dB.saName(), dB.roleName(), dB.clusterRoleName(), dB.jobName(), dB.stagingConfigMapName()} + for i := range namesA { + if namesA[i] == namesB[i] { + t.Errorf("run A and run B share object name %q; runs are not name-isolated", namesA[i]) + } + } + }) + + // --- Assertion 2: DiscoverNetwork's extra ClusterRole rules do not leak across runs. + t.Run("ClusterRole rules are isolated per run's DiscoverNetwork setting", func(t *testing.T) { + crA, err := clientset.RbacV1().ClusterRoles().Get(ctx, dA.clusterRoleName(), metav1.GetOptions{}) + if err != nil { + t.Fatalf("run A ClusterRole not found: %v", err) + } + crB, err := clientset.RbacV1().ClusterRoles().Get(ctx, dB.clusterRoleName(), metav1.GetOptions{}) + if err != nil { + t.Fatalf("run B ClusterRole not found: %v", err) + } + + if clusterRoleHasRule(crA.Rules, "pods/exec", "create") { + t.Error("non-discovery run A's ClusterRole grants pods/exec create; expected none") + } + if clusterRoleHasRule(crA.Rules, "nodes", "patch") { + t.Error("non-discovery run A's ClusterRole grants nodes patch; expected none") + } + if !clusterRoleHasRule(crB.Rules, "pods/exec", "create") { + t.Error("discovery run B's ClusterRole is missing pods/exec create") + } + if !clusterRoleHasRule(crB.Rules, "nodes", "patch") { + t.Error("discovery run B's ClusterRole is missing nodes patch") + } + }) + + // Seed Pods: the fake clientset runs no Job controller, so Pods never + // appear on their own. podA/podB get an explicit, earlier + // CreationTimestamp than the imposter below: pickLivePod prefers the + // youngest candidate, so an imposter that merely passed the label + // filter (without also failing ownedByJob) would beat podA on + // recency — making the ownership check load-bearing for this + // assertion rather than incidentally masked by Pod name sort order. + podA := seedRunPodAt(t, ctx, clientset, dA, "agent-pod-a", concurrencyPodTimeEarly) + podB := seedRunPodAt(t, ctx, clientset, dB, "agent-pod-b", concurrencyPodTimeEarly) + // Imposter: carries run A's RunID label (so it passes the label + // selector dA.findPodName uses — List DOES honor selectors against the + // fake clientset, see the top-of-function note), is strictly younger + // than podA, but its controlling ownerReference points at run B's Job. + // Proves pickLivePod's ownedByJob check, not the (forgeable) RunID + // label or Pod recency, is what authorizes selection. + seedImposterPod(t, ctx, clientset, dA, dB, "agent-pod-imposter", concurrencyPodTimeLate) + + // --- Assertion 3: each run's pod selection returns only its own pod. + t.Run("pod selection returns only the run's own pod", func(t *testing.T) { + gotA, err := dA.findPodName(ctx) + if err != nil { + t.Fatalf("run A findPodName() error = %v", err) + } + if gotA != podA.Name { + t.Errorf("run A findPodName() = %q, want %q (imposter or run B pod leaked through)", gotA, podA.Name) + } + + gotB, err := dB.findPodName(ctx) + if err != nil { + t.Fatalf("run B findPodName() error = %v", err) + } + if gotB != podB.Name { + t.Errorf("run B findPodName() = %q, want %q", gotB, podB.Name) + } + }) + + // --- Assertion 4: GetSnapshot returns each run's own staging ConfigMap bytes. + t.Run("GetSnapshot returns each run's own bytes", func(t *testing.T) { + gotA, err := dA.GetSnapshot(ctx) + if err != nil { + t.Fatalf("run A GetSnapshot() error = %v", err) + } + if string(gotA) != "snapshot-bytes-for-run-A" { + t.Errorf("run A GetSnapshot() = %q, want %q", gotA, "snapshot-bytes-for-run-A") + } + + gotB, err := dB.GetSnapshot(ctx) + if err != nil { + t.Fatalf("run B GetSnapshot() error = %v", err) + } + if string(gotB) != "snapshot-bytes-for-run-B" { + t.Errorf("run B GetSnapshot() = %q, want %q", gotB, "snapshot-bytes-for-run-B") + } + }) + + // --- Assertion 5: run A's Cleanup must not touch any of run B's objects. + t.Run("run A Cleanup leaves every run B object intact", func(t *testing.T) { + if err := dA.Cleanup(ctx, CleanupOptions{Enabled: true}); err != nil { + t.Fatalf("run A Cleanup() error = %v", err) + } + + assertSevenKindsExist(t, ctx, clientset, dB) + + // Confirm run A's own Job is actually gone, so this isn't a + // Cleanup that vacuously "succeeds" without deleting anything. + if _, err := clientset.BatchV1().Jobs(concurrencyTestNamespace).Get(ctx, dA.jobName(), metav1.GetOptions{}); !apierrors.IsNotFound(err) { + t.Errorf("run A's Job should be deleted by its own Cleanup, err = %v", err) + } + }) + + // --- Assertion 5 (staging-ConfigMap ownership gate): run A's Cleanup + // must not touch a staging-named ConfigMap this run does not own, even + // when that name is exactly what run A's own naming formula computes. + // + // This is deliberately a separate subtest from the one above, which + // compares run A against run B and so is satisfied by name-scoping + // alone (assertion 1 already proves the two runs compute different + // names). Here the names collide, so only Config.OwnsOutputConfigMap — + // false for run A, which is why getSnapshotFromConfigMap in assertion 4 + // did not record the ConfigMap and why the name-based sweep does not + // fire either — can explain the object surviving. + // + // Scope note: this pins the OWNERSHIP GATE, not created-set scoping as + // such. A hypothetical Cleanup that recomputed its delete list from + // d.stagingConfigMapName() but kept the same OwnsOutputConfigMap gate + // would pass this subtest unchanged. The tests that actually + // discriminate created-set scoping are TestCleanupPassesUIDPrecondition + // (every delete carries the UID from its Create response — a value no + // name-derived delete list can supply) and + // TestCleanupResolvesUnconfirmedEntryBeforeDeleting, both in + // deployer_test.go. The duplicate-RunID case, where name scoping cannot + // help because both runs compute the same names, is + // TestCleanupDuplicateRunIDKeepsFirstRunsStagingConfigMap. + t.Run("run A Cleanup leaves its own unowned staging ConfigMap intact", func(t *testing.T) { + if _, err := clientset.CoreV1().ConfigMaps(concurrencyTestNamespace).Get(ctx, dA.stagingConfigMapName(), metav1.GetOptions{}); err != nil { + t.Errorf("the ConfigMap at run A's staging name %q should survive run A's Cleanup (OwnsOutputConfigMap is false, so it is not run A's to delete), err = %v", dA.stagingConfigMapName(), err) + } + }) +} + +// assertSevenKindsExist verifies each of the seven run-owned object kinds a +// Deployer creates (ServiceAccount, Role, RoleBinding, ClusterRole, +// ClusterRoleBinding, Job, and the staging ConfigMap — see kindServiceAccount +// et al. in types.go) exists under d's run-scoped names. +func assertSevenKindsExist(t *testing.T, ctx context.Context, clientset kubernetes.Interface, d *Deployer) { + t.Helper() + ns := d.config.Namespace + + if _, err := clientset.CoreV1().ServiceAccounts(ns).Get(ctx, d.saName(), metav1.GetOptions{}); err != nil { + t.Errorf("ServiceAccount %q not found: %v", d.saName(), err) + } + if _, err := clientset.RbacV1().Roles(ns).Get(ctx, d.roleName(), metav1.GetOptions{}); err != nil { + t.Errorf("Role %q not found: %v", d.roleName(), err) + } + if _, err := clientset.RbacV1().RoleBindings(ns).Get(ctx, d.roleName(), metav1.GetOptions{}); err != nil { + t.Errorf("RoleBinding %q not found: %v", d.roleName(), err) + } + if _, err := clientset.RbacV1().ClusterRoles().Get(ctx, d.clusterRoleName(), metav1.GetOptions{}); err != nil { + t.Errorf("ClusterRole %q not found: %v", d.clusterRoleName(), err) + } + if _, err := clientset.RbacV1().ClusterRoleBindings().Get(ctx, d.clusterRoleName(), metav1.GetOptions{}); err != nil { + t.Errorf("ClusterRoleBinding %q not found: %v", d.clusterRoleName(), err) + } + if _, err := clientset.BatchV1().Jobs(ns).Get(ctx, d.jobName(), metav1.GetOptions{}); err != nil { + t.Errorf("Job %q not found: %v", d.jobName(), err) + } + if _, err := clientset.CoreV1().ConfigMaps(ns).Get(ctx, d.stagingConfigMapName(), metav1.GetOptions{}); err != nil { + t.Errorf("staging ConfigMap %q not found: %v", d.stagingConfigMapName(), err) + } +} + +// seedStagingConfigMap creates d's staging ConfigMap directly, standing in +// for the in-pod agent write that Deploy() itself never performs against +// the fake clientset. +// +// The labels below are d's own set for convenience only. The real in-pod +// writer (pkg/serializer's ConfigMap writer) stamps a different, smaller set — +// no managed-by and no run-ID label — because it also produces the user's +// delivered cm:// artifact. Nothing here selects on these labels: run scoping +// for this object comes from its name. +func seedStagingConfigMap(t *testing.T, ctx context.Context, clientset kubernetes.Interface, d *Deployer, uid, snapshotYAML string) { + t.Helper() + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: d.stagingConfigMapName(), + Namespace: d.config.Namespace, + UID: types.UID(uid), + Labels: d.objectLabels(), + }, + Data: map[string]string{"snapshot.yaml": snapshotYAML}, + } + if _, err := clientset.CoreV1().ConfigMaps(d.config.Namespace).Create(ctx, cm, metav1.CreateOptions{}); err != nil { + t.Fatalf("seed staging ConfigMap %q: %v", d.stagingConfigMapName(), err) + } +} + +// seedRunPodAt creates the agent Pod for d's Job, since the fake clientset +// runs no Job controller to create it automatically. The Pod carries d's +// full label set (RunID included), createdAt as its CreationTimestamp, and +// a controlling ownerReference to d's Job, mirroring what a real Job +// controller produces. +func seedRunPodAt(t *testing.T, ctx context.Context, clientset kubernetes.Interface, d *Deployer, podName string, createdAt metav1.Time) *corev1.Pod { + t.Helper() + controller := true + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: d.config.Namespace, + Labels: d.objectLabels(), + CreationTimestamp: createdAt, + OwnerReferences: []metav1.OwnerReference{ + { + Kind: kindJob, + Name: d.jobName(), + UID: d.jobUID(), + Controller: &controller, + }, + }, + }, + } + created, err := clientset.CoreV1().Pods(d.config.Namespace).Create(ctx, pod, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("seed pod %q: %v", podName, err) + } + return created +} + +// seedImposterPod creates a Pod labeled as labelOwner's own (so it passes +// labelOwner's List label selector) with createdAt as its +// CreationTimestamp, but controlled by jobOwner's Job — proving pod +// selection is authorized by the controlling ownerReference, not the RunID +// label alone (writable by anything that can update Pods in the namespace) +// or Pod recency. +func seedImposterPod(t *testing.T, ctx context.Context, clientset kubernetes.Interface, labelOwner, jobOwner *Deployer, podName string, createdAt metav1.Time) *corev1.Pod { + t.Helper() + controller := true + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: labelOwner.config.Namespace, + Labels: labelOwner.objectLabels(), + CreationTimestamp: createdAt, + OwnerReferences: []metav1.OwnerReference{ + { + Kind: kindJob, + Name: jobOwner.jobName(), + UID: jobOwner.jobUID(), + Controller: &controller, + }, + }, + }, + } + created, err := clientset.CoreV1().Pods(labelOwner.config.Namespace).Create(ctx, pod, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("seed imposter pod %q: %v", podName, err) + } + return created +} + +// clusterRoleHasRule reports whether rules contains a rule permitting verb +// on resource, mirroring the resource/verb matching TestDeployer_EnsureRBAC's +// discovery subtest uses elsewhere in this package. +func clusterRoleHasRule(rules []rbacv1.PolicyRule, resource, verb string) bool { + for _, r := range rules { + for _, res := range r.Resources { + if res != resource { + continue + } + for _, v := range r.Verbs { + if v == verb { + return true + } + } + } + } + return false +} diff --git a/pkg/k8s/agent/consts.go b/pkg/k8s/agent/consts.go index 137954a13..b8ac2956b 100644 --- a/pkg/k8s/agent/consts.go +++ b/pkg/k8s/agent/consts.go @@ -19,7 +19,31 @@ const ( verbCreate = "create" verbList = "list" verbGet = "get" - resourceCM = "configmaps" + verbDelete = "delete" + verbWatch = "watch" + verbUpdate = "update" + verbPatch = "patch" + + // Resource names, as they appear in an RBAC PolicyRule and in the + // ResourceAttributes of an access review. Named so the RBAC this + // package builds and the pre-flight gate that checks for it can never + // spell the same resource two different ways. + resourceCM = "configmaps" + resourceServiceAccounts = "serviceaccounts" + resourceRoles = "roles" + resourceRoleBindings = "rolebindings" + resourceClusterRoles = "clusterroles" + resourceClusterRoleBindings = "clusterrolebindings" + resourceJobs = "jobs" + resourcePods = "pods" + resourceNodes = "nodes" + + // subresourceLog is the pods subresource the CLI reads to stream the + // agent's output back to the operator's terminal. + subresourceLog = "log" + + // batchAPIGroup is the API group the agent Job lives in. + batchAPIGroup = "batch" slinkyAPIGroup = "slinky.slurm.net" slinkyControllerResource = "controllers" @@ -36,3 +60,14 @@ const ( // / ClusterRole / ClusterRoleBinding resources. rbacAPIGroup = "rbac.authorization.k8s.io" ) + +// Attribute keys shared by this package's structured log lines and error +// contexts. Named for the same reason the ctxKey* constants in names.go are: +// one spelling reaches every consumer that parses them, and a literal +// repeated across files does not drift. +const ( + attrNamespace = "namespace" + attrName = "name" + attrRunID = "runID" + attrServiceAccount = "serviceAccount" +) diff --git a/pkg/k8s/agent/deployer.go b/pkg/k8s/agent/deployer.go index 184f8b0d2..8fe1f54ff 100644 --- a/pkg/k8s/agent/deployer.go +++ b/pkg/k8s/agent/deployer.go @@ -24,21 +24,41 @@ import ( "github.com/NVIDIA/aicr/pkg/defaults" aicrerrors "github.com/NVIDIA/aicr/pkg/errors" + "github.com/NVIDIA/aicr/pkg/k8s/labels" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ) // Deploy deploys the agent with all required resources (RBAC + Job). // This is the main entry point that orchestrates the deployment. func (d *Deployer) Deploy(ctx context.Context) error { - // Step 0: Check permissions before attempting deployment - _, err := d.CheckPermissions(ctx) - if err != nil { + // Pre-flight, ahead of any cluster call: reject a run ID — or a + // caller-supplied name prefix — that cannot be folded into a valid + // object name. Every object created below is named "-", + // so validating here is what keeps an invalid value from surfacing as + // an apiserver "Invalid value: metadata.name" partway through the + // ensure* chain, with some objects already created. + if err := d.validateNames(); err != nil { + return err + } + + // Step 0: the authoritative permission gate for the whole run. It + // verifies every permission this run will exercise — for the caller AND + // for the ServiceAccount the agent pod runs as — and resolves which + // ServiceAccount mode the run is in, which is what decides the verb set + // it demands. Everything it does is a read, so nothing below has been + // written yet when it fails. + if _, err := d.CheckPermissions(ctx); err != nil { if aicrerrors.IsNetworkError(err) { return aicrerrors.Wrap(aicrerrors.ErrCodeUnavailable, "cannot reach Kubernetes API server\n\nCheck your network connectivity:\n - Is your VPN connected?\n - Is the cluster endpoint correct in your kubeconfig?\n - Are firewall rules allowing egress to the API server?", err) } - return aicrerrors.Wrap(aicrerrors.ErrCodeUnauthorized, "insufficient permissions to deploy agent\n\nTo deploy the agent, you need cluster admin privileges.\nRun: aicr snapshot", err) + // Propagated as-is: CheckPermissions already returns + // ErrCodeUnauthorized carrying the complete list of what is + // missing, for which subject, and how to fix it. Re-wrapping would + // bury that behind a generic sentence. + return err } // Step 0.5: Validate RuntimeClass exists if configured @@ -53,28 +73,47 @@ func (d *Deployer) Deploy(ctx context.Context) error { return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to ensure namespace", err) } - // Step 2: Ensure RBAC resources (idempotent - reuses if already exists) - if err := d.ensureServiceAccount(ctx); err != nil { - return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to create ServiceAccount", err) - } + // ServiceAccount mode is already decided at this point: Step 0's gate + // resolves it (resolveServiceAccount) because the permissions it must + // demand differ between the two modes. Resolving there rather than here + // also means the decision is made before ensureNamespace, which is + // harmless — a ServiceAccount cannot exist in a namespace that does + // not, so the Get correctly reports NotFound and the run stays in + // prefix mode. - if err := d.ensureRole(ctx); err != nil { - return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to create Role", err) - } + // Step 2: Create this run's RBAC. Every name carries the run ID, so + // nothing here can already exist; an AlreadyExists is reported as an + // error rather than adopted or overwritten. + // + // Skipped entirely in exact-ServiceAccount mode: aicr will not add or + // remove permissions on a ServiceAccount it did not create. Nothing is + // created, so nothing enters the created-set, so Cleanup deletes none + // of these kinds — the operator's grants outlive the run. Generate its + // RBAC manifests with BuildServiceAccountRoleManifests and apply them + // out of band. + if d.managesRBAC() { + if err := d.ensureServiceAccount(ctx); err != nil { + return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to create ServiceAccount", err) + } - if err := d.ensureRoleBinding(ctx); err != nil { - return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to create RoleBinding", err) - } + if err := d.ensureRole(ctx); err != nil { + return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to create Role", err) + } - if err := d.ensureClusterRole(ctx); err != nil { - return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to create ClusterRole", err) - } + if err := d.ensureRoleBinding(ctx); err != nil { + return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to create RoleBinding", err) + } - if err := d.ensureClusterRoleBinding(ctx); err != nil { - return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to create ClusterRoleBinding", err) + if err := d.ensureClusterRole(ctx); err != nil { + return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to create ClusterRole", err) + } + + if err := d.ensureClusterRoleBinding(ctx); err != nil { + return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to create ClusterRoleBinding", err) + } } - // Step 2: Ensure Job (delete existing + recreate) + // Step 3: Create this run's Job under its run-scoped name. if err := d.ensureJob(ctx); err != nil { return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to create Job", err) } @@ -82,6 +121,15 @@ func (d *Deployer) Deploy(ctx context.Context) error { return nil } +// JobName returns the run-scoped name of the Job this Deployer deploys — +// Config.JobName (or the name base) suffixed with Config.RunID. Callers that +// surface the Job to an operator (log lines, kubectl hints) must use this +// rather than Config.JobName, which is only the optional prefix and is empty +// by default. +func (d *Deployer) JobName() string { + return d.jobName() +} + // WaitForCompletion waits for the agent Job to complete successfully. // Returns error if the Job fails or times out. func (d *Deployer) WaitForCompletion(ctx context.Context, timeout time.Duration) error { @@ -94,30 +142,62 @@ func (d *Deployer) GetSnapshot(ctx context.Context) ([]byte, error) { return d.getSnapshotFromConfigMap(ctx) } -// Cleanup removes the agent Job and RBAC resources. -// If opts.Enabled is false, no cleanup is performed (resources are kept for debugging). -// All resources are attempted for deletion even if some fail, and a combined error is returned. -// Deletions are fanned out concurrently so a slow apiserver does not serialize the wall clock. +// Cleanup removes exactly the objects this Deployer created: the Job, the +// RBAC resources, and — when this Deployer owns the output ConfigMap — the +// staging ConfigMap. If opts.Enabled is false, no cleanup is performed +// (resources are kept for debugging). All resources are attempted for +// deletion even if some fail, and a combined error is returned. Deletions +// are fanned out concurrently so a slow apiserver does not serialize the +// wall clock. func (d *Deployer) Cleanup(ctx context.Context, opts CleanupOptions) error { if !opts.Enabled { return nil } + // Build the task list from what this Deployer actually created, not + // from configured names — a run must never delete an object it did + // not create (e.g. a same-named object left behind by an unrelated + // run or user). Each delete is additionally pinned to the recorded + // UID via metav1.Preconditions. + created := d.createdSnapshot() + type result struct { label string err error } - tasks := []struct { + type task struct { label string op func(context.Context) error - }{ - {fmt.Sprintf("Job %q", d.config.JobName), d.deleteJob}, - {fmt.Sprintf("ServiceAccount %q", d.config.ServiceAccountName), d.deleteServiceAccount}, - {fmt.Sprintf("Role %q", d.config.ServiceAccountName), d.deleteRole}, - {fmt.Sprintf("RoleBinding %q", d.config.ServiceAccountName), d.deleteRoleBinding}, - {fmt.Sprintf("ClusterRole %q", clusterRoleName), d.deleteClusterRole}, - {fmt.Sprintf("ClusterRoleBinding %q", clusterRoleName), d.deleteClusterRoleBinding}, + } + + tasks := make([]task, len(created)) + for i, obj := range created { + tasks[i].label = fmt.Sprintf("%s %q", obj.kind, obj.name) + tasks[i].op = func(ctx context.Context) error { + return d.deleteCreatedObject(ctx, obj) + } + } + + // The staging ConfigMap is written by the in-pod agent, so it only + // enters the created-set when getSnapshotFromConfigMap got far enough + // to observe its UID. A run that fails after the agent wrote it (Job + // timeout, wait error, canceled context) would otherwise leak it — and + // with run-scoped naming that is one leaked object per failed run, not + // one shared object. Sweep it here. + // + // The name alone does not license that delete: Config.RunID is caller- + // settable, so a second run reusing a RunID resolves the same staging + // name as the first. needsStagingConfigMapSweep therefore requires this + // run to hold a CONFIRMED Job entry — the only way this run could have + // produced a staging ConfigMap at all — and deleteUnrecordedStagingConfigMap + // re-checks the object it finds before deleting it. + if d.config.OwnsOutputConfigMap && needsStagingConfigMapSweep(created) { + name := d.stagingConfigMapName() + tasks = append(tasks, task{ + label: fmt.Sprintf("%s %q", kindConfigMap, name), + op: d.deleteUnrecordedStagingConfigMap, + }) } // sync.WaitGroup (not errgroup) is intentional here: cleanup must @@ -158,6 +238,170 @@ func (d *Deployer) Cleanup(ctx context.Context, opts CleanupOptions) error { return nil } +// deleteCreatedObject deletes a single created-set entry by dispatching to +// the resource-specific delete call for obj.kind, passing obj.name and +// obj.uid through so every delete is UID-pinned. +// +// An unconfirmed entry carries no UID (no Create response ever named the +// object), so its ownership is re-established first — resolveIntentUID either +// returns the live object's UID after proving the object carries this run's +// labels, or reports that there is nothing this run may delete. +func (d *Deployer) deleteCreatedObject(ctx context.Context, obj createdObject) error { + if !obj.confirmed { + uid, ours, err := d.resolveIntentUID(ctx, obj) + if err != nil || !ours { + return err + } + obj.uid = uid + } + + switch obj.kind { + case kindJob: + return d.deleteJob(ctx, obj.name, obj.uid) + case kindServiceAccount: + return d.deleteServiceAccount(ctx, obj.name, obj.uid) + case kindRole: + return d.deleteRole(ctx, obj.name, obj.uid) + case kindRoleBinding: + return d.deleteRoleBinding(ctx, obj.name, obj.uid) + case kindClusterRole: + return d.deleteClusterRole(ctx, obj.name, obj.uid) + case kindClusterRoleBinding: + return d.deleteClusterRoleBinding(ctx, obj.name, obj.uid) + case kindConfigMap: + return d.deleteStagingConfigMap(ctx, obj.name, obj.uid) + default: + return aicrerrors.New(aicrerrors.ErrCodeInternal, fmt.Sprintf("cleanup: unknown created-object kind %q", obj.kind)) + } +} + +// resolveIntentUID re-establishes ownership of an unconfirmed created-set +// entry — one recordIntent added whose Create response never arrived — and +// returns the UID to pin its delete to. ours is false when this run may not +// delete the object, in which case the caller must issue no delete at all. +// +// The entry's run-scoped name is not evidence: it says what this run WOULD +// have created, not what is standing there now. Get the live object and +// require it to carry the label set objectLabels() stamps on everything +// Deploy creates. That set is written at creation time by this run, so: +// +// - Labels match: the Create did commit and this is our object. Delete it, +// pinned to the UID this Get observed. (If it is replaced again between +// the Get and the Delete, the precondition turns that into a Conflict, +// which ignoreNotFoundOrConflict treats as "already gone".) +// - Labels do not match: whatever holds the name was not created by this +// run — an operator's object, another subsystem's, or a replacement made +// after this run's object was deleted. Deleting it would collect an +// object this run never owned, so fail closed and warn instead. +// - NotFound: nothing to reclaim. +// +// Residual, and deliberately not papered over: a second run that reuses this +// run's RunID stamps an identical label set, so the two are indistinguishable +// here. The ADR treats a duplicate RunID as an unsupported caller error — +// every ensure* fails closed on the AlreadyExists it normally produces (see +// discardIntent); this path is reachable only when that response was also +// lost. +func (d *Deployer) resolveIntentUID(ctx context.Context, obj createdObject) (uid types.UID, ours bool, err error) { + live, err := d.getCreatedObject(ctx, obj.kind, obj.name) + if k8serrors.IsNotFound(err) { + return "", false, nil + } + if err != nil { + return "", false, aicrerrors.Wrap(aicrerrors.ErrCodeInternal, + fmt.Sprintf("failed to read %s %q to confirm this run created it before deleting it", obj.kind, obj.name), err) + } + + // A real apiserver always assigns a UID, so an empty one here means the + // delete could not be pinned. Refuse rather than fall back to a + // bare-name delete: that is exactly the blind delete this path exists + // to prevent. + if !d.createdByThisRun(live.GetLabels()) || live.GetUID() == "" { + slog.Warn("cleanup left behind an object it cannot prove this run created; if it is a stale orphan of this run, remove it by hand", + slog.String("kind", obj.kind), + slog.String(attrName, obj.name), + slog.String(attrNamespace, live.GetNamespace()), + slog.String("uid", string(live.GetUID())), + slog.String(attrRunID, d.config.RunID), + slog.String("objectRunID", live.GetLabels()[labels.RunID])) + return "", false, nil + } + return live.GetUID(), true, nil +} + +// getCreatedObject reads the live object a created-set entry names. The +// apiserver error is returned unwrapped so callers can classify it with +// k8serrors.IsNotFound before wrapping. +func (d *Deployer) getCreatedObject(ctx context.Context, kind, name string) (metav1.Object, error) { + ns := d.config.Namespace + switch kind { + case kindJob: + return d.clientset.BatchV1().Jobs(ns).Get(ctx, name, metav1.GetOptions{}) + case kindServiceAccount: + return d.clientset.CoreV1().ServiceAccounts(ns).Get(ctx, name, metav1.GetOptions{}) + case kindRole: + return d.clientset.RbacV1().Roles(ns).Get(ctx, name, metav1.GetOptions{}) + case kindRoleBinding: + return d.clientset.RbacV1().RoleBindings(ns).Get(ctx, name, metav1.GetOptions{}) + case kindClusterRole: + return d.clientset.RbacV1().ClusterRoles().Get(ctx, name, metav1.GetOptions{}) + case kindClusterRoleBinding: + return d.clientset.RbacV1().ClusterRoleBindings().Get(ctx, name, metav1.GetOptions{}) + case kindConfigMap: + return d.clientset.CoreV1().ConfigMaps(ns).Get(ctx, name, metav1.GetOptions{}) + default: + return nil, aicrerrors.New(aicrerrors.ErrCodeInternal, fmt.Sprintf("cleanup: cannot read unknown created-object kind %q", kind)) + } +} + +// createdByThisRun reports whether objLabels is the label set objectLabels() +// stamps on every object this Deployer creates. All four keys are required: +// aicr.run/run-id alone would also match a validator-owned object carrying +// the same ID (`aicr validate` hands one run ID to both subsystems), and an +// empty Config.RunID must never match a label-less object. +func (d *Deployer) createdByThisRun(objLabels map[string]string) bool { + if d.config.RunID == "" { + return false + } + return objLabels[labels.RunID] == d.config.RunID && + objLabels[labels.Name] == labels.ValueAICR && + objLabels[labels.ManagedBy] == labels.ValueAICR && + objLabels[labels.Component] == labels.ValueSnapshotAgent +} + +// uidPreconditions returns the DeleteOptions precondition pinning a delete +// to uid, or nil when uid is the zero UID. +// +// Only a confirmed entry can reach here with the zero UID: a Create response +// named the object — establishing this run's ownership — but carried no UID. +// A real apiserver always assigns one, so that shape belongs to fake +// clientsets in tests; the delete then falls back to the run-scoped name the +// Create response confirmed. Unconfirmed entries never reach here without a +// UID (see resolveIntentUID). +// +// Omitting the precondition entirely is required in that case — +// metav1.Preconditions{UID: &""} is NOT equivalent to no precondition: the +// apiserver compares it against the live object's UID and rejects every +// delete with a Conflict, which ignoreNotFoundOrConflict then swallows as +// success, leaking the object this entry exists to reclaim. +func uidPreconditions(uid types.UID) *metav1.Preconditions { + if uid == "" { + return nil + } + return &metav1.Preconditions{UID: &uid} +} + +// ignoreNotFoundOrConflict returns nil when err is "not found" (already +// deleted) or "conflict" (the UID precondition did not match — some other +// object now holds this name; it has already been replaced and is not +// ours to delete). Both are success from Cleanup's perspective: the object +// this run created is gone. +func ignoreNotFoundOrConflict(err error) error { + if k8serrors.IsNotFound(err) || k8serrors.IsConflict(err) { + return nil + } + return err +} + // validateRuntimeClass checks that the specified RuntimeClass exists in the cluster. func (d *Deployer) validateRuntimeClass(ctx context.Context) error { ctx, cancel := context.WithTimeout(ctx, defaults.RuntimeClassCheckTimeout) diff --git a/pkg/k8s/agent/deployer_test.go b/pkg/k8s/agent/deployer_test.go index 9e2effdf9..f7279ab0b 100644 --- a/pkg/k8s/agent/deployer_test.go +++ b/pkg/k8s/agent/deployer_test.go @@ -18,33 +18,52 @@ import ( "bytes" "context" "errors" + "fmt" "net" "slices" "strings" + "sync" "syscall" "testing" "time" aicrerrors "github.com/NVIDIA/aicr/pkg/errors" + "github.com/NVIDIA/aicr/pkg/header" + "github.com/NVIDIA/aicr/pkg/k8s/labels" "github.com/NVIDIA/aicr/pkg/k8s/pod" authv1 "k8s.io/api/authorization/v1" corev1 "k8s.io/api/core/v1" nodev1 "k8s.io/api/node/v1" rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/fake" k8stesting "k8s.io/client-go/testing" + "k8s.io/utils/ptr" ) const testName = "aicr" +// testRunID is a fixed, well-formed run ID (the shape runid.Generate emits: +// UTC timestamp + 16 hex bytes). Tests that exercise production naming must +// set Config.RunID — leaving it empty falls back to unscoped names no +// production caller ever builds. +const testRunID = "20260821-142233-9f3a1c0b7e2d4a55" + func TestDeployer_EnsureRBAC(t *testing.T) { clientset := fake.NewClientset() + // ServiceAccountName and JobName are prefixes, not names: every + // run-owned object lands at "-". Assertions below go + // through the deployer's own name accessors so they track the names a + // production caller actually gets. config := Config{ Namespace: "test-namespace", ServiceAccountName: testName, JobName: testName, + RunID: testRunID, Image: "ghcr.io/nvidia/aicr-validator:latest", Output: "cm://test-namespace/aicr-snapshot", } @@ -81,12 +100,12 @@ func TestDeployer_EnsureRBAC(t *testing.T) { } sa, err := clientset.CoreV1().ServiceAccounts(config.Namespace). - Get(ctx, testName, metav1.GetOptions{}) + Get(ctx, deployer.saName(), metav1.GetOptions{}) if err != nil { t.Fatalf("ServiceAccount not found: %v", err) } - if sa.Name != testName { - t.Errorf("expected SA name %q, got %q", testName, sa.Name) + if sa.Name != deployer.saName() { + t.Errorf("expected SA name %q, got %q", deployer.saName(), sa.Name) } }) @@ -97,7 +116,7 @@ func TestDeployer_EnsureRBAC(t *testing.T) { } role, err := clientset.RbacV1().Roles(config.Namespace). - Get(ctx, testName, metav1.GetOptions{}) + Get(ctx, deployer.roleName(), metav1.GetOptions{}) if err != nil { t.Fatalf("Role not found: %v", err) } @@ -124,7 +143,7 @@ func TestDeployer_EnsureRBAC(t *testing.T) { } rb, err := clientset.RbacV1().RoleBindings(config.Namespace). - Get(ctx, testName, metav1.GetOptions{}) + Get(ctx, deployer.roleName(), metav1.GetOptions{}) if err != nil { t.Fatalf("RoleBinding not found: %v", err) } @@ -133,13 +152,13 @@ func TestDeployer_EnsureRBAC(t *testing.T) { if len(rb.Subjects) != 1 { t.Errorf("expected 1 subject, got %d", len(rb.Subjects)) } - if rb.Subjects[0].Name != testName { - t.Errorf("expected subject name 'aicr', got %q", rb.Subjects[0].Name) + if rb.Subjects[0].Name != deployer.saName() { + t.Errorf("expected subject name %q, got %q", deployer.saName(), rb.Subjects[0].Name) } // Verify roleRef - if rb.RoleRef.Name != testName { - t.Errorf("expected roleRef name 'aicr', got %q", rb.RoleRef.Name) + if rb.RoleRef.Name != deployer.roleName() { + t.Errorf("expected roleRef name %q, got %q", deployer.roleName(), rb.RoleRef.Name) } }) @@ -150,7 +169,7 @@ func TestDeployer_EnsureRBAC(t *testing.T) { } cr, err := clientset.RbacV1().ClusterRoles(). - Get(ctx, "aicr-node-reader", metav1.GetOptions{}) + Get(ctx, deployer.clusterRoleName(), metav1.GetOptions{}) if err != nil { t.Fatalf("ClusterRole not found: %v", err) } @@ -227,7 +246,7 @@ func TestDeployer_EnsureRBAC(t *testing.T) { t.Fatalf("failed to create ClusterRole: %v", err) } cr, err := discoverClientset.RbacV1().ClusterRoles(). - Get(ctx, "aicr-node-reader", metav1.GetOptions{}) + Get(ctx, d.clusterRoleName(), metav1.GetOptions{}) if err != nil { t.Fatalf("ClusterRole not found: %v", err) } @@ -283,7 +302,7 @@ func TestDeployer_EnsureRBAC(t *testing.T) { } crb, err := clientset.RbacV1().ClusterRoleBindings(). - Get(ctx, "aicr-node-reader", metav1.GetOptions{}) + Get(ctx, deployer.clusterRoleName(), metav1.GetOptions{}) if err != nil { t.Fatalf("ClusterRoleBinding not found: %v", err) } @@ -294,50 +313,19 @@ func TestDeployer_EnsureRBAC(t *testing.T) { } // Verify roleRef - if crb.RoleRef.Name != "aicr-node-reader" { - t.Errorf("expected roleRef name 'aicr-node-reader', got %q", crb.RoleRef.Name) + if crb.RoleRef.Name != deployer.clusterRoleName() { + t.Errorf("expected roleRef name %q, got %q", deployer.clusterRoleName(), crb.RoleRef.Name) } }) } -func TestDeployer_EnsureRBAC_Idempotent(t *testing.T) { - clientset := fake.NewClientset() - config := Config{ - Namespace: "test-namespace", - ServiceAccountName: testName, - JobName: testName, - Image: "ghcr.io/nvidia/aicr-validator:latest", - Output: "cm://test-namespace/aicr-snapshot", - } - deployer := NewDeployer(clientset, config) - ctx := context.Background() - - // Create resources twice - second call should be idempotent - if err := deployer.ensureServiceAccount(ctx); err != nil { - t.Fatalf("first create failed: %v", err) - } - - if err := deployer.ensureServiceAccount(ctx); err != nil { - t.Fatalf("second create failed (not idempotent): %v", err) - } - - // Verify only one ServiceAccount exists - saList, err := clientset.CoreV1().ServiceAccounts(config.Namespace). - List(ctx, metav1.ListOptions{}) - if err != nil { - t.Fatalf("failed to list ServiceAccounts: %v", err) - } - if len(saList.Items) != 1 { - t.Errorf("expected 1 ServiceAccount, got %d", len(saList.Items)) - } -} - func TestDeployer_EnsureJob(t *testing.T) { clientset := fake.NewClientset() config := Config{ Namespace: "test-namespace", ServiceAccountName: testName, JobName: testName, + RunID: testRunID, Image: "ghcr.io/nvidia/aicr-validator:latest", Output: "cm://test-namespace/aicr-snapshot", Privileged: true, // Test privileged mode (default for agent deployment) @@ -362,15 +350,16 @@ func TestDeployer_EnsureJob(t *testing.T) { } job, err := clientset.BatchV1().Jobs(config.Namespace). - Get(ctx, config.JobName, metav1.GetOptions{}) + Get(ctx, deployer.jobName(), metav1.GetOptions{}) if err != nil { t.Fatalf("Job not found: %v", err) } - // Verify Job spec - if job.Spec.Template.Spec.ServiceAccountName != config.ServiceAccountName { + // Verify Job spec. The pod's ServiceAccountName is the run-scoped + // SA name, not the configured prefix. + if job.Spec.Template.Spec.ServiceAccountName != deployer.saName() { t.Errorf("expected ServiceAccountName %q, got %q", - config.ServiceAccountName, job.Spec.Template.Spec.ServiceAccountName) + deployer.saName(), job.Spec.Template.Spec.ServiceAccountName) } // Verify host settings @@ -408,26 +397,6 @@ func TestDeployer_EnsureJob(t *testing.T) { t.Errorf("expected 3 volumes, got %d", len(job.Spec.Template.Spec.Volumes)) } }) - - t.Run("recreate Job deletes old one", func(t *testing.T) { - // Create Job first time - if err := deployer.ensureJob(ctx); err != nil { - t.Fatalf("first create failed: %v", err) - } - - // Create Job second time - should delete and recreate - if err := deployer.ensureJob(ctx); err != nil { - t.Fatalf("second create failed: %v", err) - } - - // Verify Job still exists (fake client doesn't support watch/wait, - // but we can verify the Job exists) - _, err := clientset.BatchV1().Jobs(config.Namespace). - Get(ctx, config.JobName, metav1.GetOptions{}) - if err != nil { - t.Errorf("Job should exist after recreate: %v", err) - } - }) } func TestDeployer_EnsureJob_Unprivileged(t *testing.T) { @@ -436,6 +405,7 @@ func TestDeployer_EnsureJob_Unprivileged(t *testing.T) { Namespace: "test-namespace", ServiceAccountName: testName, JobName: testName, + RunID: testRunID, Image: "ghcr.io/nvidia/aicr-validator:latest", Output: "cm://test-namespace/aicr-snapshot", Privileged: false, // Test unprivileged mode for PSS-restricted namespaces @@ -448,7 +418,7 @@ func TestDeployer_EnsureJob_Unprivileged(t *testing.T) { } job, err := clientset.BatchV1().Jobs(config.Namespace). - Get(ctx, config.JobName, metav1.GetOptions{}) + Get(ctx, deployer.jobName(), metav1.GetOptions{}) if err != nil { t.Fatalf("Job not found: %v", err) } @@ -521,6 +491,7 @@ func TestDeployer_Deploy(t *testing.T) { Namespace: "test-namespace", ServiceAccountName: testName, JobName: testName, + RunID: testRunID, Image: "ghcr.io/nvidia/aicr-validator:latest", Output: "cm://test-namespace/aicr-snapshot", } @@ -541,47 +512,94 @@ func TestDeployer_Deploy(t *testing.T) { // Verify ServiceAccount _, err = clientset.CoreV1().ServiceAccounts(config.Namespace). - Get(ctx, testName, metav1.GetOptions{}) + Get(ctx, deployer.saName(), metav1.GetOptions{}) if err != nil { t.Errorf("ServiceAccount not created: %v", err) } // Verify Role _, err = clientset.RbacV1().Roles(config.Namespace). - Get(ctx, testName, metav1.GetOptions{}) + Get(ctx, deployer.roleName(), metav1.GetOptions{}) if err != nil { t.Errorf("Role not created: %v", err) } // Verify RoleBinding _, err = clientset.RbacV1().RoleBindings(config.Namespace). - Get(ctx, testName, metav1.GetOptions{}) + Get(ctx, deployer.roleName(), metav1.GetOptions{}) if err != nil { t.Errorf("RoleBinding not created: %v", err) } // Verify ClusterRole _, err = clientset.RbacV1().ClusterRoles(). - Get(ctx, "aicr-node-reader", metav1.GetOptions{}) + Get(ctx, deployer.clusterRoleName(), metav1.GetOptions{}) if err != nil { t.Errorf("ClusterRole not created: %v", err) } // Verify ClusterRoleBinding _, err = clientset.RbacV1().ClusterRoleBindings(). - Get(ctx, "aicr-node-reader", metav1.GetOptions{}) + Get(ctx, deployer.clusterRoleName(), metav1.GetOptions{}) if err != nil { t.Errorf("ClusterRoleBinding not created: %v", err) } // Verify Job _, err = clientset.BatchV1().Jobs(config.Namespace). - Get(ctx, config.JobName, metav1.GetOptions{}) + Get(ctx, deployer.jobName(), metav1.GetOptions{}) if err != nil { t.Errorf("Job not created: %v", err) } } +// TestDeployUsesRunScopedNamesAndLabels verifies that Deploy() creates every +// object under a run-scoped name (prefix-runID) and that the Job's pod +// template carries the full label set, not just the Job object itself — +// Job labels do not propagate to the Pods a Job creates. +// +// Deviation from the plan's literal test: the brief's snippet omits the +// SelfSubjectAccessReview-allow reactor that every other Deploy()-calling +// test in this file installs. The fake clientset denies all permission +// checks by default (Status.Allowed defaults to false), so Deploy() fails +// at the Step-0 CheckPermissions gate before creating anything — a false +// RED unrelated to run-scoped naming. Added the same reactor used by +// TestDeployer_Deploy et al. so the test exercises the behavior it names. +func TestDeployUsesRunScopedNamesAndLabels(t *testing.T) { + ctx := context.Background() + client := fake.NewClientset() + client.PrependReactor("create", "selfsubjectaccessreviews", func(action k8stesting.Action) (bool, runtime.Object, error) { + return true, &authv1.SelfSubjectAccessReview{ + Status: authv1.SubjectAccessReviewStatus{ + Allowed: true, + Reason: "test permissions allowed", + }, + }, nil + }) + d := NewDeployer(client, Config{ + Namespace: "test-ns", + Image: "aicr:test", + RunID: "20260821-142233-9f3a1c0b7e2d4a55", + }) + if err := d.Deploy(ctx); err != nil { + t.Fatalf("Deploy() error = %v", err) + } + + wantSuffix := "-20260821-142233-9f3a1c0b7e2d4a55" + job, err := client.BatchV1().Jobs("test-ns").Get(ctx, "aicr"+wantSuffix, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Job not found under run-scoped name: %v", err) + } + for _, key := range []string{labels.Name, labels.ManagedBy, labels.Component, labels.RunID} { + if _, ok := job.Spec.Template.Labels[key]; !ok { + t.Errorf("pod template missing label %q", key) + } + } + if _, err := client.RbacV1().ClusterRoles().Get(ctx, "aicr-node-reader"+wantSuffix, metav1.GetOptions{}); err != nil { + t.Errorf("ClusterRole not found under run-scoped name: %v", err) + } +} + func TestDeployer_Cleanup(t *testing.T) { clientset := fake.NewClientset() @@ -595,15 +613,21 @@ func TestDeployer_Cleanup(t *testing.T) { }, nil }) + // JobName / ServiceAccountName are prefixes: the objects Cleanup must + // find are named "-", so RunID is set here to exercise + // the same naming production uses. config := Config{ Namespace: "test-namespace", ServiceAccountName: testName, JobName: testName, + RunID: testRunID, Image: "ghcr.io/nvidia/aicr-validator:latest", Output: "cm://test-namespace/aicr-snapshot", } deployer := NewDeployer(clientset, config) ctx := context.Background() + scopedJob := testName + "-" + testRunID + scopedSA := testName + "-" + testRunID // Deploy first if err := deployer.Deploy(ctx); err != nil { @@ -617,14 +641,14 @@ func TestDeployer_Cleanup(t *testing.T) { // Job should still exist (cleanup disabled) _, err := clientset.BatchV1().Jobs(config.Namespace). - Get(ctx, config.JobName, metav1.GetOptions{}) + Get(ctx, scopedJob, metav1.GetOptions{}) if err != nil { t.Errorf("Job should still exist when cleanup disabled: %v", err) } // ServiceAccount should still exist _, err = clientset.CoreV1().ServiceAccounts(config.Namespace). - Get(ctx, testName, metav1.GetOptions{}) + Get(ctx, scopedSA, metav1.GetOptions{}) if err != nil { t.Errorf("ServiceAccount should still exist: %v", err) } @@ -636,7 +660,7 @@ func TestDeployer_Cleanup(t *testing.T) { // Job should be deleted _, err = clientset.BatchV1().Jobs(config.Namespace). - Get(ctx, config.JobName, metav1.GetOptions{}) + Get(ctx, scopedJob, metav1.GetOptions{}) if err == nil { t.Errorf("Job should be deleted") } @@ -655,15 +679,19 @@ func TestDeployer_Cleanup_AttemptsAllDeletions(t *testing.T) { }, nil }) + // RunID set for the same reason as TestDeployer_Cleanup: without it the + // test asserts on bare names production never creates. config := Config{ Namespace: "test-namespace", ServiceAccountName: testName, JobName: testName, + RunID: testRunID, Image: "ghcr.io/nvidia/aicr-validator:latest", Output: "cm://test-namespace/aicr-snapshot", } deployer := NewDeployer(clientset, config) ctx := context.Background() + scopedName := testName + "-" + testRunID // Deploy first if err := deployer.Deploy(ctx); err != nil { @@ -672,7 +700,7 @@ func TestDeployer_Cleanup_AttemptsAllDeletions(t *testing.T) { // Manually delete the Job to simulate it already being cleaned up // This tests that cleanup continues to delete other resources - if err := clientset.BatchV1().Jobs(config.Namespace).Delete(ctx, config.JobName, metav1.DeleteOptions{}); err != nil { + if err := clientset.BatchV1().Jobs(config.Namespace).Delete(ctx, scopedName, metav1.DeleteOptions{}); err != nil { t.Fatalf("Failed to pre-delete Job: %v", err) } @@ -684,31 +712,31 @@ func TestDeployer_Cleanup_AttemptsAllDeletions(t *testing.T) { // Verify all RBAC resources were deleted _, err := clientset.CoreV1().ServiceAccounts(config.Namespace). - Get(ctx, testName, metav1.GetOptions{}) + Get(ctx, scopedName, metav1.GetOptions{}) if err == nil { t.Error("ServiceAccount should be deleted") } _, err = clientset.RbacV1().Roles(config.Namespace). - Get(ctx, testName, metav1.GetOptions{}) + Get(ctx, scopedName, metav1.GetOptions{}) if err == nil { t.Error("Role should be deleted") } _, err = clientset.RbacV1().RoleBindings(config.Namespace). - Get(ctx, testName, metav1.GetOptions{}) + Get(ctx, scopedName, metav1.GetOptions{}) if err == nil { t.Error("RoleBinding should be deleted") } _, err = clientset.RbacV1().ClusterRoles(). - Get(ctx, clusterRoleName, metav1.GetOptions{}) + Get(ctx, deployer.clusterRoleName(), metav1.GetOptions{}) if err == nil { t.Error("ClusterRole should be deleted") } _, err = clientset.RbacV1().ClusterRoleBindings(). - Get(ctx, clusterRoleName, metav1.GetOptions{}) + Get(ctx, deployer.clusterRoleName(), metav1.GetOptions{}) if err == nil { t.Error("ClusterRoleBinding should be deleted") } @@ -735,6 +763,886 @@ func TestDeployer_Cleanup_ReportsAllErrors(t *testing.T) { } } +// TestCleanupDeletesOnlyWhatItCreated verifies Cleanup builds its delete +// list from the created-set (Deploy's own objects), not from configured +// names — a foreign object that happens to share no name with this run +// must survive even though Cleanup is enabled. +// +// Deviation from the plan's literal test: as with +// TestDeployUsesRunScopedNamesAndLabels above, the brief's snippet omits +// the SelfSubjectAccessReview-allow reactor. Without it the fake clientset +// denies all permission checks by default, so Deploy() fails at the Step-0 +// CheckPermissions gate before creating anything — a false RED unrelated to +// created-set cleanup scoping. Added the same reactor used elsewhere in +// this file. +func TestCleanupDeletesOnlyWhatItCreated(t *testing.T) { + ctx := context.Background() + client := fake.NewClientset() + client.PrependReactor("create", "selfsubjectaccessreviews", func(action k8stesting.Action) (bool, runtime.Object, error) { + return true, &authv1.SelfSubjectAccessReview{ + Status: authv1.SubjectAccessReviewStatus{ + Allowed: true, + Reason: "test permissions allowed", + }, + }, nil + }) + + // A foreign object sharing no name with this run must survive. + foreign := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "aicr-other", Namespace: "test-ns"}} + if _, err := client.CoreV1().ServiceAccounts("test-ns").Create(ctx, foreign, metav1.CreateOptions{}); err != nil { + t.Fatalf("seed: %v", err) + } + + d := NewDeployer(client, Config{Namespace: "test-ns", Image: "aicr:test", RunID: "20260821-142233-9f3a1c0b7e2d4a55"}) + if err := d.Deploy(ctx); err != nil { + t.Fatalf("Deploy() error = %v", err) + } + if err := d.Cleanup(ctx, CleanupOptions{Enabled: true}); err != nil { + t.Fatalf("Cleanup() error = %v", err) + } + + if _, err := client.CoreV1().ServiceAccounts("test-ns").Get(ctx, "aicr-other", metav1.GetOptions{}); err != nil { + t.Errorf("Cleanup deleted a ServiceAccount it did not create: %v", err) + } + if _, err := client.CoreV1().ServiceAccounts("test-ns").Get(ctx, "aicr-20260821-142233-9f3a1c0b7e2d4a55", metav1.GetOptions{}); !apierrors.IsNotFound(err) { + t.Errorf("Cleanup did not delete its own ServiceAccount, err = %v", err) + } +} + +// observedDelete is one delete action captured by spyOnDeletes. +type observedDelete struct { + resource string + name string + uid *types.UID // nil when the delete carried no UID precondition +} + +// spyOnDeletes installs a reactor over EVERY resource that records each +// outgoing delete's resource, name, and Preconditions.UID, then falls through +// to the default tracker delete. Reactors run under the fake Clientset's own +// lock, but Cleanup fans its deletes out concurrently, so the slice is +// mutex-guarded regardless; the returned accessor takes the same lock. +func spyOnDeletes(client *fake.Clientset) func() []observedDelete { + var mu sync.Mutex + var observed []observedDelete + client.PrependReactor("delete", "*", func(action k8stesting.Action) (bool, runtime.Object, error) { + da, ok := action.(k8stesting.DeleteActionImpl) + if !ok { + return false, nil, nil + } + rec := observedDelete{resource: da.GetResource().Resource, name: da.GetName()} + if da.DeleteOptions.Preconditions != nil && da.DeleteOptions.Preconditions.UID != nil { + uid := *da.DeleteOptions.Preconditions.UID + rec.uid = &uid + } + mu.Lock() + observed = append(observed, rec) + mu.Unlock() + return false, nil, nil // not handled: fall through to the default tracker delete + }) + return func() []observedDelete { + mu.Lock() + defer mu.Unlock() + out := make([]observedDelete, len(observed)) + copy(out, observed) + return out + } +} + +// TestCleanupPassesUIDPrecondition verifies every delete Cleanup issues +// carries Preconditions.UID set to the UID recorded at create time — for all +// seven kinds deleteCreatedObject dispatches on, not just one. A reactor +// scoped to a single resource would leave the other six dispatch arms free to +// drop the precondition unnoticed, so the spy here is installed over "*". +// +// The fake clientset's ObjectTracker neither assigns UIDs on Create nor +// enforces Preconditions on Delete (it ignores DeleteOptions entirely), so +// this records known UIDs directly via recordCreated and inspects the +// outgoing delete actions rather than relying on tracker behavior. +func TestCleanupPassesUIDPrecondition(t *testing.T) { + ctx := context.Background() + client := fake.NewClientset() + deletes := spyOnDeletes(client) + + // One object per kind, each with a distinct UID so a dispatch arm that + // passed some OTHER entry's UID would be caught as well as one that + // passed none. + created := []struct { + kind string + name string + resource string + uid types.UID + }{ + {kindServiceAccount, "aicr-sa", "serviceaccounts", "sa-uid-123"}, + {kindRole, "aicr-role", "roles", "role-uid-123"}, + {kindRoleBinding, "aicr-rb", "rolebindings", "rb-uid-123"}, + {kindClusterRole, "aicr-cr", "clusterroles", "cr-uid-123"}, + {kindClusterRoleBinding, "aicr-crb", "clusterrolebindings", "crb-uid-123"}, + {kindJob, "aicr-job", "jobs", "job-uid-123"}, + {kindConfigMap, "aicr-agent-snapshot", "configmaps", "cm-uid-123"}, + } + + d := NewDeployer(client, Config{Namespace: "test-ns"}) + for _, c := range created { + d.recordCreated(c.kind, c.name, c.uid) + } + + if err := d.Cleanup(ctx, CleanupOptions{Enabled: true}); err != nil { + t.Fatalf("Cleanup() error = %v", err) + } + + observed := deletes() + if len(observed) != len(created) { + t.Fatalf("Cleanup issued %d deletes, want %d: %+v", len(observed), len(created), observed) + } + for _, c := range created { + t.Run(c.kind, func(t *testing.T) { + idx := slices.IndexFunc(observed, func(o observedDelete) bool { + return o.resource == c.resource && o.name == c.name + }) + if idx < 0 { + t.Fatalf("Cleanup issued no delete for %s %q (resource %q); observed: %+v", + c.kind, c.name, c.resource, observed) + } + got := observed[idx] + if got.uid == nil { + t.Fatalf("%s delete did not carry Preconditions.UID", c.kind) + } + if *got.uid != c.uid { + t.Errorf("%s delete Preconditions.UID = %q, want %q", c.kind, *got.uid, c.uid) + } + }) + } +} + +// TestCleanupResolvesUnconfirmedEntryBeforeDeleting covers the +// lost-Create-response path: recordIntent enters an object BEFORE its Create, +// so an entry can reach Cleanup with no UID and no Create response ever having +// named it. The run-scoped name alone is not ownership evidence — it says what +// this run WOULD have created, not what is standing there now — so Cleanup +// must recover the UID from the live object and prove that object carries this +// run's labels before deleting anything. Ownership takes both halves: a label +// mismatch and a missing UID each refuse the delete on their own. +// +// The "replaced under the same name" case is the one this replaces a bare-name +// delete for: an object at that name with a different UID and someone else's +// labels must survive, and the operator must be told it was left behind. +// +// Deleting by bare name with no Preconditions would collect the replacement. +// Passing &"" instead would be worse than useless: the apiserver would compare +// the empty UID against the live object's real one, reject every attempt with +// a Conflict, and ignoreNotFoundOrConflict would swallow that as success. +func TestCleanupResolvesUnconfirmedEntryBeforeDeleting(t *testing.T) { + const ns = "test-ns" + d := NewDeployer(fake.NewClientset(), Config{Namespace: ns, RunID: testRunID}) + saName := d.saName() + ourLabels := d.objectLabels() + + // Every aicr label except the run ID. This is the shape that makes the + // empty-RunID guard load-bearing: objLabels[labels.RunID] is "" here, so + // against an empty Config.RunID the run-ID comparison SUCCEEDS ("" == "") + // and the other three match too. Without the guard, createdByThisRun + // would claim this object. A label-less seed cannot show that — it is + // rejected on labels.Name regardless. + runIDLessLabels := map[string]string{ + labels.Name: labels.ValueAICR, + labels.ManagedBy: labels.ValueAICR, + labels.Component: labels.ValueSnapshotAgent, + } + + // A replacement created after this run's object was deleted: same name, + // different identity. Carries a foreign run's labels, which is what a + // second aicr run standing an object up at this name would stamp. + foreignLabels := map[string]string{ + labels.Name: labels.ValueAICR, + labels.ManagedBy: labels.ValueAICR, + labels.Component: labels.ValueSnapshotAgent, + labels.RunID: "20260822-090000-0011223344556677", + } + + tests := []struct { + name string + seed *corev1.ServiceAccount // nil: nothing at the name + runID *string // nil: testRunID + wantDelete bool + wantUID types.UID // expected Preconditions.UID when wantDelete + wantWarn bool + }{ + { + name: "this run's object is deleted pinned to the recovered UID", + seed: &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{ + Name: saName, Namespace: ns, UID: types.UID("ours-uid"), Labels: ourLabels, + }}, + wantDelete: true, + wantUID: types.UID("ours-uid"), + }, + { + name: "nothing at the name issues no delete", + wantDelete: false, + }, + { + name: "a replacement under the same name survives", + seed: &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{ + Name: saName, Namespace: ns, UID: types.UID("replacement-uid"), Labels: foreignLabels, + }}, + wantDelete: false, + wantWarn: true, + }, + { + name: "an unlabeled object under the same name survives", + seed: &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{ + Name: saName, Namespace: ns, UID: types.UID("operators-uid"), + }}, + wantDelete: false, + wantWarn: true, + }, + { + // Labels alone would clear this object — they are this run's + // own — but a real apiserver always assigns a UID, so a + // missing one means the delete cannot be pinned. Refusing is + // the point: the fallback would be the bare-name delete this + // path exists to prevent. + name: "this run's own labels without a UID still survive", + seed: &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{ + Name: saName, Namespace: ns, Labels: ourLabels, + }}, + wantDelete: false, + wantWarn: true, + }, + { + // An empty Config.RunID is not a wildcard. The seed carries + // every aicr label but the run ID, so all four comparisons + // would pass against an empty RunID -- "" == "" included. + // createdByThisRun must still match nothing, because a run + // with no ID has no ownership to prove. Deleting the guard + // fails this row; a label-less seed would not. + name: "an empty RunID proves ownership of nothing", + seed: &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{ + Name: saName, Namespace: ns, UID: types.UID("operators-uid"), + Labels: runIDLessLabels, + }}, + runID: ptr.To(""), + wantDelete: false, + wantWarn: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + logs := captureLogs(t) + client := fake.NewClientset() + if tt.seed != nil { + if _, err := client.CoreV1().ServiceAccounts(ns).Create(ctx, tt.seed, metav1.CreateOptions{}); err != nil { + t.Fatalf("seed: %v", err) + } + } + deletes := spyOnDeletes(client) + + runID := testRunID + if tt.runID != nil { + runID = *tt.runID + } + run := NewDeployer(client, Config{Namespace: ns, RunID: runID}) + run.recordIntent(kindServiceAccount, saName) + + if err := run.Cleanup(ctx, CleanupOptions{Enabled: true}); err != nil { + t.Fatalf("Cleanup() error = %v", err) + } + + observed := deletes() + if !tt.wantDelete { + if len(observed) != 0 { + t.Fatalf("Cleanup issued %d deletes, want none: %+v", len(observed), observed) + } + if tt.seed != nil { + live, err := client.CoreV1().ServiceAccounts(ns).Get(ctx, saName, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Cleanup deleted an object this run cannot prove it created: %v", err) + } + if live.UID != tt.seed.UID { + t.Errorf("surviving object UID = %q, want %q", live.UID, tt.seed.UID) + } + } + } else { + if len(observed) != 1 { + t.Fatalf("Cleanup issued %d deletes, want 1: %+v", len(observed), observed) + } + if observed[0].uid == nil || *observed[0].uid != tt.wantUID { + t.Errorf("delete Preconditions.UID = %v, want %q", observed[0].uid, tt.wantUID) + } + } + + gotWarn := strings.Contains(logs.String(), "cannot prove this run created") + if gotWarn != tt.wantWarn { + t.Errorf("warned about an ambiguous orphan = %v, want %v; logs: %s", gotWarn, tt.wantWarn, logs.String()) + } + if tt.wantWarn && !strings.Contains(logs.String(), saName) { + t.Errorf("warning does not name the object left behind; logs: %s", logs.String()) + } + }) + } +} + +// TestCleanupSurfacesIntentResolutionGetError fails closed on an apiserver +// error other than NotFound while resolving an unconfirmed entry: cleanup can +// neither prove the object is ours nor prove it is gone, so it must report the +// failure rather than delete blind or silently skip. +func TestCleanupSurfacesIntentResolutionGetError(t *testing.T) { + ctx := context.Background() + client := fake.NewClientset() + client.PrependReactor("get", "serviceaccounts", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewInternalError(errors.New("apiserver exploded")) + }) + deletes := spyOnDeletes(client) + + d := NewDeployer(client, Config{Namespace: "test-ns", RunID: testRunID}) + d.recordIntent(kindServiceAccount, d.saName()) + + err := d.Cleanup(ctx, CleanupOptions{Enabled: true}) + if err == nil { + t.Fatal("Cleanup() error = nil, want the unexpected Get error surfaced") + } + if !strings.Contains(err.Error(), d.saName()) { + t.Errorf("error %q does not name the unresolved object", err) + } + if observed := deletes(); len(observed) != 0 { + t.Errorf("Cleanup issued %d deletes despite an unresolvable entry: %+v", len(observed), observed) + } +} + +// TestEnsureRecordsIntentBeforeCreate is the reason recordIntent exists: an +// apiserver that commits a Create but never delivers the response (client +// timeout, apiserver rollout, LB 502/504, connection reset) must not leave an +// object nothing will ever delete. The run-scoped name means no later run +// reclaims it, so the orphan would be permanent. +// +// The reactor below reproduces exactly that: the object is written into the +// tracker — with a UID, as a real apiserver assigns and the fake ObjectTracker +// does not — and THEN an error is returned, so ensureServiceAccount fails while +// the ServiceAccount exists. Cleanup must still delete it, and (since the +// Create response never named it) must delete it pinned to the UID it +// recovers from the live object, not by bare name. +func TestEnsureRecordsIntentBeforeCreate(t *testing.T) { + ctx := context.Background() + const ns = "test-ns" + client := fake.NewClientset() + deletes := spyOnDeletes(client) + + d := NewDeployer(client, Config{Namespace: ns, RunID: testRunID}) + saName := d.saName() + committedUID := types.UID(saName + "-uid") + + client.PrependReactor("create", "serviceaccounts", func(action k8stesting.Action) (bool, runtime.Object, error) { + ca, ok := action.(k8stesting.CreateActionImpl) + if !ok { + return false, nil, nil + } + sa, ok := ca.GetObject().(*corev1.ServiceAccount) + if !ok { + return false, nil, nil + } + // Commit the object the way a real apiserver would, UID and all... + sa.UID = committedUID + if err := client.Tracker().Create(ca.GetResource(), sa, ns); err != nil { + return true, nil, err + } + // ...then lose the response on the way back to the client. + return true, nil, syscall.ECONNRESET + }) + + if err := d.ensureServiceAccount(ctx); err == nil { + t.Fatal("ensureServiceAccount() = nil error, want the simulated lost response") + } + + if _, err := client.CoreV1().ServiceAccounts(ns).Get(ctx, saName, metav1.GetOptions{}); err != nil { + t.Fatalf("test precondition: the ServiceAccount must exist in the cluster despite the "+ + "failed call, Get err = %v", err) + } + + if err := d.Cleanup(ctx, CleanupOptions{Enabled: true}); err != nil { + t.Fatalf("Cleanup() error = %v", err) + } + + if _, err := client.CoreV1().ServiceAccounts(ns).Get(ctx, saName, metav1.GetOptions{}); !apierrors.IsNotFound(err) { + t.Errorf("Cleanup leaked the ServiceAccount created by the lost-response Create; Get err = %v", err) + } + + observed := deletes() + if len(observed) != 1 { + t.Fatalf("Cleanup issued %d deletes, want 1: %+v", len(observed), observed) + } + if observed[0].uid == nil || *observed[0].uid != committedUID { + t.Errorf("delete Preconditions.UID = %v, want the UID recovered from the live object (%q)", + observed[0].uid, committedUID) + } +} + +// TestEnsureDiscardsIntentOnAlreadyExists is recordIntent's counterweight: an +// AlreadyExists response is the one outcome that proves the object at that +// name is NOT ours (a duplicate RunID, or a 16-byte random collision). Keeping +// the intent entry would hand this run a bare-name delete of another run's +// object — strictly worse than the leak recordIntent prevents. +func TestEnsureDiscardsIntentOnAlreadyExists(t *testing.T) { + ctx := context.Background() + const ns = "test-ns" + client := fake.NewClientset() + + d := NewDeployer(client, Config{Namespace: ns, RunID: testRunID}) + client.PrependReactor("create", "serviceaccounts", func(action k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewAlreadyExists( + schema.GroupResource{Resource: "serviceaccounts"}, d.saName()) + }) + + if err := d.ensureServiceAccount(ctx); err == nil { + t.Fatal("ensureServiceAccount() = nil error, want AlreadyExists to be reported") + } + + if got := d.createdSnapshot(); len(got) != 0 { + t.Errorf("created-set = %+v, want empty — an AlreadyExists object was not created by "+ + "this run and must not enter its delete list", got) + } +} + +// TestCleanupTreatsConflictAsSuccess verifies a Conflict response (the UID +// precondition did not match — the name now belongs to a different object) +// is treated as success, same as NotFound, rather than surfaced as a +// Cleanup failure. +func TestCleanupTreatsConflictAsSuccess(t *testing.T) { + ctx := context.Background() + client := fake.NewClientset() + client.PrependReactor("create", "selfsubjectaccessreviews", func(action k8stesting.Action) (bool, runtime.Object, error) { + return true, &authv1.SelfSubjectAccessReview{ + Status: authv1.SubjectAccessReviewStatus{Allowed: true, Reason: "test permissions allowed"}, + }, nil + }) + client.PrependReactor("delete", "serviceaccounts", func(action k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewConflict(schema.GroupResource{Resource: "serviceaccounts"}, "aicr", errors.New("uid mismatch")) + }) + + d := NewDeployer(client, Config{Namespace: "test-ns", Image: "aicr:test", RunID: "20260821-142233-9f3a1c0b7e2d4a55"}) + if err := d.Deploy(ctx); err != nil { + t.Fatalf("Deploy() error = %v", err) + } + if err := d.Cleanup(ctx, CleanupOptions{Enabled: true}); err != nil { + t.Fatalf("Cleanup() should treat a Conflict delete response as success, got: %v", err) + } +} + +// TestRecordCreatedAndJobUID verifies jobUID() returns the zero UID before +// any Job is recorded, and the recorded Job's UID afterward — even when +// other kinds have been recorded too. +func TestRecordCreatedAndJobUID(t *testing.T) { + d := NewDeployer(fake.NewClientset(), Config{Namespace: "test-ns"}) + + if got := d.jobUID(); got != "" { + t.Fatalf("jobUID() before any Job recorded = %q, want zero UID", got) + } + + d.recordCreated(kindServiceAccount, "aicr-sa", types.UID("sa-uid")) + if got := d.jobUID(); got != "" { + t.Fatalf("jobUID() after recording a non-Job kind = %q, want zero UID", got) + } + + d.recordCreated(kindJob, "aicr-job", types.UID("job-uid")) + if got := d.jobUID(); got != "job-uid" { + t.Fatalf("jobUID() = %q, want %q", got, "job-uid") + } +} + +// TestCreatedSnapshotIsDefensiveCopy verifies createdSnapshot returns a copy +// that mutation cannot use to corrupt the Deployer's internal created-set. +func TestCreatedSnapshotIsDefensiveCopy(t *testing.T) { + d := NewDeployer(fake.NewClientset(), Config{Namespace: "test-ns"}) + d.recordCreated(kindServiceAccount, "aicr-sa", types.UID("sa-uid")) + + snap := d.createdSnapshot() + if len(snap) != 1 { + t.Fatalf("createdSnapshot() length = %d, want 1", len(snap)) + } + snap[0].name = "mutated" + + again := d.createdSnapshot() + if again[0].name != "aicr-sa" { + t.Fatalf("createdSnapshot() mutation leaked into Deployer state: got %q, want %q", again[0].name, "aicr-sa") + } +} + +// TestRecordCreatedConcurrentSafe exercises recordCreated from many +// goroutines at once so `go test -race` can catch a data race on the +// created-set if the locking is ever removed or narrowed incorrectly. +func TestRecordCreatedConcurrentSafe(t *testing.T) { + d := NewDeployer(fake.NewClientset(), Config{Namespace: "test-ns"}) + + const n = 50 + var wg sync.WaitGroup + for i := 0; i < n; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + d.recordCreated(kindConfigMap, fmt.Sprintf("cm-%d", i), types.UID(fmt.Sprintf("uid-%d", i))) + }(i) + } + wg.Wait() + + if got := len(d.createdSnapshot()); got != n { + t.Fatalf("createdSnapshot() length = %d, want %d", got, n) + } +} + +// TestGetSnapshotFromConfigMap_RecordsUID_WhenOwned verifies +// getSnapshotFromConfigMap enters the staging ConfigMap into the +// created-set (for a UID-pinned Cleanup delete) only when +// Config.OwnsOutputConfigMap is true — a caller-supplied `cm://` output is +// the caller's artifact and must never be deleted by this Deployer. +func TestGetSnapshotFromConfigMap_RecordsUID_WhenOwned(t *testing.T) { + tests := []struct { + name string + ownsOutput bool + wantRecord bool + }{ + {name: "owned output is recorded", ownsOutput: true, wantRecord: true}, + {name: "caller-supplied output is not recorded", ownsOutput: false, wantRecord: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "aicr-snapshot", + Namespace: "test-namespace", + UID: types.UID("cm-uid"), + }, + Data: map[string]string{"snapshot.yaml": "data"}, + } + clientset := fake.NewClientset(cm) + d := NewDeployer(clientset, Config{ + Namespace: "test-namespace", + Output: "cm://test-namespace/aicr-snapshot", + OwnsOutputConfigMap: tt.ownsOutput, + }) + + if _, err := d.getSnapshotFromConfigMap(context.Background()); err != nil { + t.Fatalf("getSnapshotFromConfigMap() error = %v", err) + } + + snap := d.createdSnapshot() + gotRecorded := len(snap) == 1 && snap[0].kind == kindConfigMap && snap[0].uid == types.UID("cm-uid") + if gotRecorded != tt.wantRecord { + t.Errorf("recorded = %v (snapshot = %+v), want %v", gotRecorded, snap, tt.wantRecord) + } + }) + } +} + +// TestCleanupDeletesStagingConfigMapWhenOwned verifies Cleanup deletes the +// staging ConfigMap once getSnapshotFromConfigMap has recorded it (i.e. +// Config.OwnsOutputConfigMap was true), exercising deleteStagingConfigMap's +// dispatch from Cleanup end to end. +func TestCleanupDeletesStagingConfigMapWhenOwned(t *testing.T) { + ctx := context.Background() + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "aicr-snapshot", + Namespace: "test-namespace", + UID: types.UID("cm-uid"), + }, + Data: map[string]string{"snapshot.yaml": "data"}, + } + clientset := fake.NewClientset(cm) + d := NewDeployer(clientset, Config{ + Namespace: "test-namespace", + Output: "cm://test-namespace/aicr-snapshot", + OwnsOutputConfigMap: true, + }) + + if _, err := d.getSnapshotFromConfigMap(ctx); err != nil { + t.Fatalf("getSnapshotFromConfigMap() error = %v", err) + } + + if err := d.Cleanup(ctx, CleanupOptions{Enabled: true}); err != nil { + t.Fatalf("Cleanup() error = %v", err) + } + + if _, err := clientset.CoreV1().ConfigMaps("test-namespace").Get(ctx, "aicr-snapshot", metav1.GetOptions{}); !apierrors.IsNotFound(err) { + t.Errorf("Cleanup did not delete the owned staging ConfigMap, err = %v", err) + } +} + +// TestCleanupSweepsUnrecordedStagingConfigMap covers the leak path: the +// in-pod agent wrote the staging ConfigMap, but the run failed (Job timeout, +// wait error, canceled context) before getSnapshotFromConfigMap could observe +// its UID, so nothing was recorded. With run-scoped naming that would leak one +// ConfigMap per failed run, so Cleanup Gets it by its run-scoped name and +// deletes it pinned to the UID that Get returned. +// +// The sweep is licensed by this run holding a CONFIRMED Job — the only thing +// that can produce a staging ConfigMap is the in-pod agent that Job runs — so +// the Job is recorded here as Deploy would have recorded it. The seeded +// ConfigMap carries the label set pkg/serializer's ConfigMapWriter actually +// stamps from inside the pod: app.kubernetes.io/name plus component and +// version, and NO aicr.run/run-id. +func TestCleanupSweepsUnrecordedStagingConfigMap(t *testing.T) { + ctx := context.Background() + name := StagingConfigMapName(testRunID) + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "test-namespace", + UID: types.UID("staging-uid"), + Labels: stagingConfigMapLabels(), + }, + Data: map[string]string{"snapshot.yaml": "data"}, + } + clientset := fake.NewClientset(cm) + + var sawUIDPrecondition bool + clientset.PrependReactor("delete", "configmaps", func(action k8stesting.Action) (bool, runtime.Object, error) { + del, ok := action.(k8stesting.DeleteActionImpl) + if !ok || del.DeleteOptions.Preconditions == nil || del.DeleteOptions.Preconditions.UID == nil { + return false, nil, nil + } + if *del.DeleteOptions.Preconditions.UID == types.UID("staging-uid") { + sawUIDPrecondition = true + } + return false, nil, nil + }) + + d := NewDeployer(clientset, Config{ + Namespace: "test-namespace", + RunID: testRunID, + Output: "cm://test-namespace/" + name, + OwnsOutputConfigMap: true, + }) + + // Deliberately no getSnapshotFromConfigMap call: this is the failed run. + d.recordCreated(kindJob, d.jobName(), types.UID("job-uid")) + if d.hasCreated(kindConfigMap) { + t.Fatal("precondition: created-set must not hold the staging ConfigMap") + } + + if err := d.Cleanup(ctx, CleanupOptions{Enabled: true}); err != nil { + t.Fatalf("Cleanup() error = %v", err) + } + + if _, err := clientset.CoreV1().ConfigMaps("test-namespace").Get(ctx, name, metav1.GetOptions{}); !apierrors.IsNotFound(err) { + t.Errorf("Cleanup leaked the staging ConfigMap, Get err = %v", err) + } + if !sawUIDPrecondition { + t.Error("staging ConfigMap delete was not pinned to the observed UID") + } +} + +// stagingConfigMapLabels returns the label set pkg/serializer's +// ConfigMapWriter stamps on the staging ConfigMap it writes from inside the +// agent pod (Serialize in pkg/serializer/configmap.go). Deliberately NOT +// objectLabels(): that object is written by the in-pod agent rather than by +// this controller, so it carries neither aicr.run/run-id nor managed-by — +// which is why the sweep's ownership evidence is the confirmed Job, and why +// the check on the object itself can only be app.kubernetes.io/name. +func stagingConfigMapLabels() map[string]string { + return map[string]string{ + labels.Name: labels.ValueAICR, + labels.Component: header.KindSnapshot.String(), + } +} + +// TestCleanupDuplicateRunIDKeepsFirstRunsStagingConfigMap is the +// duplicate-RunID failure case. Config.RunID is public SDK surface and +// deliberately settable (pinned e2e/chainsaw runs), so a second run can +// resolve the first run's exact staging name. +// +// Run B reuses run A's RunID and fails on its very first AlreadyExists, before +// recording anything. Its deferred Cleanup still runs — Cleanup is registered +// before Deploy — and must not sweep the staging ConfigMap run A is still +// using: run B never created a Job, so nothing it did could have produced a +// ConfigMap at that name. +func TestCleanupDuplicateRunIDKeepsFirstRunsStagingConfigMap(t *testing.T) { + ctx := context.Background() + const ns = "test-namespace" + stagingName := StagingConfigMapName(testRunID) + + client := fake.NewClientset() + client.PrependReactor("create", "selfsubjectaccessreviews", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, &authv1.SelfSubjectAccessReview{ + Status: authv1.SubjectAccessReviewStatus{Allowed: true, Reason: "test permissions allowed"}, + }, nil + }) + + runA := NewDeployer(client, Config{ + Namespace: ns, + Image: "aicr:test", + RunID: testRunID, + Output: "cm://" + ns + "/" + stagingName, + OwnsOutputConfigMap: true, + }) + if err := runA.Deploy(ctx); err != nil { + t.Fatalf("run A Deploy() error = %v", err) + } + // Run A's in-pod agent has staged its result; Deploy() itself never + // writes this object, so seed it the way the agent would. + if _, err := client.CoreV1().ConfigMaps(ns).Create(ctx, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: stagingName, + Namespace: ns, + UID: types.UID("run-a-staging-uid"), + Labels: stagingConfigMapLabels(), + }, + Data: map[string]string{"snapshot.yaml": "run A's snapshot"}, + }, metav1.CreateOptions{}); err != nil { + t.Fatalf("seed run A staging ConfigMap: %v", err) + } + + deletes := spyOnDeletes(client) + + // Run B pins the SAME run ID and therefore collides on run A's + // ServiceAccount, the first object Deploy creates. + runB := NewDeployer(client, Config{ + Namespace: ns, + Image: "aicr:test", + RunID: testRunID, + Output: "cm://" + ns + "/" + stagingName, + OwnsOutputConfigMap: true, + }) + if err := runB.Deploy(ctx); err == nil { + t.Fatal("run B Deploy() = nil error, want AlreadyExists on the duplicate RunID") + } + if err := runB.Cleanup(ctx, CleanupOptions{Enabled: true}); err != nil { + t.Fatalf("run B Cleanup() error = %v", err) + } + + cm, err := client.CoreV1().ConfigMaps(ns).Get(ctx, stagingName, metav1.GetOptions{}) + if err != nil { + t.Fatalf("run B's failed Cleanup deleted run A's staging ConfigMap: %v", err) + } + if cm.UID != types.UID("run-a-staging-uid") { + t.Errorf("staging ConfigMap UID = %q, want run A's %q", cm.UID, "run-a-staging-uid") + } + if observed := deletes(); len(observed) != 0 { + t.Errorf("run B's Cleanup issued %d deletes despite creating nothing: %+v", len(observed), observed) + } +} + +// TestCleanupSweepKeepsForeignConfigMapAtStagingName is the sweep's own +// fail-closed check, downstream of the confirmed-Job gate: a ConfigMap parked +// at this run's staging name that does not carry app.kubernetes.io/name=aicr +// was not written by pkg/serializer's in-pod writer, so this run did not +// produce it. It must survive, and the operator must hear about it. +func TestCleanupSweepKeepsForeignConfigMapAtStagingName(t *testing.T) { + ctx := context.Background() + const ns = "test-namespace" + name := StagingConfigMapName(testRunID) + logs := captureLogs(t) + + client := fake.NewClientset(&corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + UID: types.UID("someone-elses-uid"), + Labels: map[string]string{"app.kubernetes.io/name": "not-aicr"}, + }, + Data: map[string]string{"unrelated": "data"}, + }) + + d := NewDeployer(client, Config{ + Namespace: ns, + RunID: testRunID, + Output: "cm://" + ns + "/" + name, + OwnsOutputConfigMap: true, + }) + d.recordCreated(kindJob, d.jobName(), types.UID("job-uid")) + + if err := d.Cleanup(ctx, CleanupOptions{Enabled: true}); err != nil { + t.Fatalf("Cleanup() error = %v", err) + } + + if _, err := client.CoreV1().ConfigMaps(ns).Get(ctx, name, metav1.GetOptions{}); err != nil { + t.Errorf("Cleanup deleted a ConfigMap this run did not write: %v", err) + } + if !strings.Contains(logs.String(), name) { + t.Errorf("no warning naming the ConfigMap left behind; logs: %s", logs.String()) + } +} + +// TestCleanupSkipsStagingConfigMapSweepWhenNotOwned asserts the sweep stays +// ownership-scoped: a caller-supplied cm:// Output is the caller's artifact +// (OwnsOutputConfigMap false) and must survive Cleanup even when it happens to +// carry this run's staging name. +func TestCleanupSkipsStagingConfigMapSweepWhenNotOwned(t *testing.T) { + ctx := context.Background() + name := StagingConfigMapName(testRunID) + clientset := fake.NewClientset(&corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "test-namespace", + UID: types.UID("callers-uid"), + }, + Data: map[string]string{"snapshot.yaml": "data"}, + }) + + d := NewDeployer(clientset, Config{ + Namespace: "test-namespace", + RunID: testRunID, + Output: "cm://test-namespace/" + name, + OwnsOutputConfigMap: false, + }) + + if err := d.Cleanup(ctx, CleanupOptions{Enabled: true}); err != nil { + t.Fatalf("Cleanup() error = %v", err) + } + + if _, err := clientset.CoreV1().ConfigMaps("test-namespace").Get(ctx, name, metav1.GetOptions{}); err != nil { + t.Errorf("Cleanup deleted a ConfigMap this run does not own: %v", err) + } +} + +// TestCleanupSweepNoOpWhenStagingConfigMapAbsent covers the common failure +// shape — the Job never got far enough to write anything — where the sweep's +// Get is a NotFound and Cleanup must still report success. +func TestCleanupSweepNoOpWhenStagingConfigMapAbsent(t *testing.T) { + ctx := context.Background() + clientset := fake.NewClientset() + + d := NewDeployer(clientset, Config{ + Namespace: "test-namespace", + RunID: testRunID, + Output: "cm://test-namespace/" + StagingConfigMapName(testRunID), + OwnsOutputConfigMap: true, + }) + d.recordCreated(kindJob, d.jobName(), types.UID("job-uid")) + + if err := d.Cleanup(ctx, CleanupOptions{Enabled: true}); err != nil { + t.Fatalf("Cleanup() error = %v, want nil when the staging ConfigMap was never written", err) + } +} + +// TestCleanupSweepSurfacesUnexpectedGetError fails closed: an apiserver error +// other than NotFound while looking for the staging ConfigMap means cleanup +// cannot prove the object is gone, so it must be reported rather than +// silently swallowed. +func TestCleanupSweepSurfacesUnexpectedGetError(t *testing.T) { + ctx := context.Background() + clientset := fake.NewClientset() + clientset.PrependReactor("get", "configmaps", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewInternalError(errors.New("apiserver exploded")) + }) + + d := NewDeployer(clientset, Config{ + Namespace: "test-namespace", + RunID: testRunID, + Output: "cm://test-namespace/" + StagingConfigMapName(testRunID), + OwnsOutputConfigMap: true, + }) + d.recordCreated(kindJob, d.jobName(), types.UID("job-uid")) + + err := d.Cleanup(ctx, CleanupOptions{Enabled: true}) + if err == nil { + t.Fatal("Cleanup() error = nil, want the unexpected Get error surfaced") + } + if !strings.Contains(err.Error(), StagingConfigMapName(testRunID)) { + t.Errorf("error %q does not name the staging ConfigMap", err) + } +} + func TestParseConfigMapName(t *testing.T) { tests := []struct { name string @@ -900,7 +1808,8 @@ func TestDeployer_WaitForPodReady(t *testing.T) { Name: "aicr-xyz", Namespace: "test-namespace", Labels: map[string]string{ - "app.kubernetes.io/name": "aicr", + labels.Name: labels.ValueAICR, + labels.RunID: "", }, }, Status: corev1.PodStatus{ @@ -954,7 +1863,8 @@ func TestDeployer_WaitForPodReady_PodFailed(t *testing.T) { Name: "aicr-xyz", Namespace: "test-namespace", Labels: map[string]string{ - "app.kubernetes.io/name": "aicr", + labels.Name: labels.ValueAICR, + labels.RunID: "", }, }, Status: corev1.PodStatus{ @@ -1017,6 +1927,7 @@ func TestDeployer_Deploy_NetworkError(t *testing.T) { Namespace: "test-namespace", ServiceAccountName: testName, JobName: testName, + RunID: testRunID, Image: "ghcr.io/nvidia/aicr-validator:latest", Output: "cm://test-namespace/aicr-snapshot", } @@ -1117,6 +2028,7 @@ func TestDeployer_Deploy_RuntimeClassNotFound(t *testing.T) { Namespace: "test-namespace", ServiceAccountName: testName, JobName: testName, + RunID: testRunID, Image: "ghcr.io/nvidia/aicr-validator:latest", Output: "cm://test-namespace/aicr-snapshot", RuntimeClassName: "nvidia", diff --git a/pkg/k8s/agent/doc.go b/pkg/k8s/agent/doc.go index 5c6e15b97..f9a9f8a38 100644 --- a/pkg/k8s/agent/doc.go +++ b/pkg/k8s/agent/doc.go @@ -19,20 +19,110 @@ The agent package deploys a Kubernetes Job that runs aicr snapshot on GPU nodes and writes output to ConfigMap storage. It handles RBAC setup, Job lifecycle management, and snapshot retrieval. -# Deployment Strategy - -RBAC resources (ServiceAccount, Role, RoleBinding, ClusterRole, ClusterRoleBinding) -are created idempotently - if they exist, they are reused. Mutable resources -(Role, RoleBinding, ClusterRole, ClusterRoleBinding) use create-or-update -semantics so stale rules from a previous run cannot persist. - -The agent Namespace is created with an "app.kubernetes.io/managed-by=aicr" -label; if the namespace pre-existed without that label, ensureNamespace -patches the label rather than silently dropping intent. - -The Job is deleted and recreated for each snapshot to ensure clean state. -Job and Pod lifecycle waits use the Kubernetes watch API (not polling) -for efficiency. +# Run Scoping + +Every deployment belongs to a single run identified by Config.RunID (generate +one with runid.Generate). The run ID is suffixed onto every object this package +creates — Job, ServiceAccount, Role, RoleBinding, ClusterRole, +ClusterRoleBinding, and the staging ConfigMap — so concurrent runs never share +an object. Config.JobName is a prefix, not an exact name; +Config.ServiceAccountName is a prefix only when no ServiceAccount of that +exact name exists (see Existing ServiceAccounts below). When empty they fall +back to Config.NameBase (default "aicr"). See ADR-020 +(docs/design/020-snapshot-agent-run-isolation.md). + +Because a run-scoped name cannot already belong to another run, creates are +plain creates: there is no delete-and-recreate of the Job, and no +create-or-update of the RBAC objects. An AlreadyExists implies a duplicate +RunID and is returned as an error rather than adopted or overwritten. + +# Existing ServiceAccounts + +Config.ServiceAccountName is exact-if-exists. When a ServiceAccount of exactly +that name already exists in the namespace, the agent pod runs as it verbatim +and the run creates NO ServiceAccount, Role, RoleBinding, ClusterRole or +ClusterRoleBinding — aicr adds and removes no permissions on an identity it +did not create. Nothing of those kinds enters the created-set, so Cleanup has +nothing of those kinds to delete and the operator's grants outlive the run. + +This exists because IRSA (eks.amazonaws.com/role-arn) and GKE Workload +Identity (iam.gke.io/gcp-service-account) both pin trust to the ServiceAccount +NAME — IRSA's trust policy conditions on system:serviceaccount:/, +GKE's IAM binding names PROJECT.svc.id.goog[/] and accepts no +wildcard — so a per-run name can never be trusted by either, and copying the +annotations onto a run-scoped ServiceAccount would not help. + +Render the RBAC that grants such a ServiceAccount the agent's permissions +with BuildServiceAccountRoleManifests (CLI: aicr snapshot +--add-roles-to-service-account). That path APPLIES NOTHING and contacts no +cluster: it writes manifests the operator reviews and applies themselves, so +the decision to grant cluster-scoped -- and, under DiscoverNetwork, mutating +-- permissions is an informed one. What they then apply sits outside every +run: no run-ID label, never in a created-set, never deleted by run cleanup, +and removed with kubectl delete -f. + +The trade-off is deliberate and opt-in: an adopted ServiceAccount waives +per-run permission isolation. Concurrent runs sharing it share its grants, and +a DiscoverNetwork grant leaves cluster-scoped mutating permissions in +place permanently rather than for one run's lifetime. + +Two objects are deliberately NOT run-scoped: + + - The Namespace is ensured, never deleted: it is created if absent and + labeled "app.kubernetes.io/managed-by=aicr", patching the label onto a + pre-existing namespace rather than silently dropping intent. + - A caller-supplied "cm://namespace/name" Output is the caller's delivered + artifact. It is written on purpose and never deleted + (Config.OwnsOutputConfigMap is false for it). + +Every object this package itself creates — the Job, ServiceAccount, Role, +RoleBinding, ClusterRole, and ClusterRoleBinding — carries +app.kubernetes.io/name=aicr, app.kubernetes.io/managed-by=aicr, +app.kubernetes.io/component=snapshot-agent, and aicr.run/run-id=, on the +Job's pod template as well as the Job itself. Select agent pods across runs +with the component label; the Job name changes every run. + +The staging ConfigMap is the exception: it is written from inside the pod by +pkg/serializer's ConfigMap writer, which stamps app.kubernetes.io/name=aicr, +app.kubernetes.io/component= and app.kubernetes.io/version — not +managed-by and not the run-ID label. That writer also produces the user's +delivered cm:// artifact, so it deliberately does not stamp the run-ID sweep +key onto an object this package must never delete. Run scoping for the staging +ConfigMap comes from its name (see StagingConfigMapName), which is what both +Cleanup paths key on. + +Job and Pod lifecycle waits use the Kubernetes watch API (not polling) for +efficiency. Pod selection narrows by label and then authorizes the candidate +against the controlling ownerReference carrying the recorded Job UID, since pod +labels are writable by anything that can update pods in the namespace. + +# Cleanup + +The Deployer records (kind, name) immediately before each Create and writes the +returned UID onto that entry on success. Cleanup deletes exactly that set, +passing the recorded UID as a metav1.Preconditions so a same-named object +belonging to another run is never collected; a UID mismatch (Conflict) and a +NotFound are both treated as success. Cleanup also runs on the Deploy failure +path, which is why it is scoped to what was created rather than to configured +names. + +Recording before the Create is what keeps a lost Create response from +orphaning an object forever: if the apiserver commits the create but the +response never arrives, the entry is already in the set. That entry carries no +UID, and its (run-unique) name is not evidence of ownership — it says what +this run WOULD have created, not what is standing there now — so Cleanup never +deletes it by bare name. It Gets the live object and re-verifies it: the +delete is issued only when that object carries the full label set this run +stamps at creation time AND a non-empty UID, and it is pinned to the UID that +Get observed. A label mismatch or a missing UID fails closed — no delete at +all, and a warning names the object left behind for an operator to judge — +while a NotFound means there is nothing to reclaim. The one response that +proves the object is not ours — AlreadyExists — discards the entry again. + +The staging ConfigMap is written by the in-pod agent, so its UID is observed +when GetSnapshot reads it. When the run owns that ConfigMap and failed before +it could be observed, Cleanup Gets it by its run-scoped name and deletes it +pinned to the UID that Get returned. # Usage Example @@ -44,6 +134,7 @@ for efficiency. "github.com/NVIDIA/aicr/pkg/k8s/agent" "github.com/NVIDIA/aicr/pkg/k8s/client" + "github.com/NVIDIA/aicr/pkg/runid" ) func main() { @@ -55,11 +146,19 @@ for efficiency. panic(err) } + // One run ID scopes every object this deployment creates. + runID := runid.Generate() + // Configure deployer config := agent.Config{ Namespace: "gpu-operator", + RunID: runID, Image: "ghcr.io/nvidia/aicr-validator:latest", - Output: "cm://gpu-operator/aicr-snapshot", + Output: "cm://gpu-operator/" + agent.StagingConfigMapName(runID), + // Output is owned by this run, so Cleanup may delete it. Point + // Output at a ConfigMap of your own and leave this false: an + // artifact you named is never deleted here. + OwnsOutputConfigMap: true, NodeSelector: map[string]string{ "nodeGroup": "customer-gpu", }, @@ -68,12 +167,17 @@ for efficiency. // Create deployer deployer := agent.NewDeployer(clientset, config) + // Always clean up this run's objects, including on the failure path. + defer func() { + _ = deployer.Cleanup(context.Background(), agent.CleanupOptions{Enabled: true}) + }() + // Deploy RBAC and Job if err := deployer.Deploy(ctx); err != nil { panic(err) } - // Wait for completion + // Wait for completion (deployer.JobName() is the run-scoped name) if err := deployer.WaitForCompletion(ctx, 5*time.Minute); err != nil { panic(err) } @@ -87,17 +191,6 @@ for efficiency. // Use snapshot... } -# Reconciliation - -The deployer ensures idempotent operation: - - Namespace: Created with managed-by label, or patched if pre-existing - - Immutable RBAC (ServiceAccount): Created if missing, reused if exists - - Mutable RBAC (Role/RoleBinding/ClusterRole/ClusterRoleBinding): - create-or-update semantics so stale rules cannot persist - - Job: Deleted and recreated for clean state each run; deletion is - observed via watch (watch.Deleted event), not polling - - ConfigMap: Created or updated with latest snapshot - # Testing The package is designed for testability with Kubernetes fake clients: @@ -108,9 +201,10 @@ The package is designed for testability with Kubernetes fake clients: ) func TestDeployer(t *testing.T) { - clientset := fake.NewSimpleClientset() + clientset := fake.NewClientset() deployer := agent.NewDeployer(clientset, agent.Config{ Namespace: "test", + RunID: "20260821-142233-9f3a1c0b7e2d4a55", Image: "test:latest", }) // Test deployment logic... diff --git a/pkg/k8s/agent/job.go b/pkg/k8s/agent/job.go index 5a2c06e7f..a1c76d1dc 100644 --- a/pkg/k8s/agent/job.go +++ b/pkg/k8s/agent/job.go @@ -21,47 +21,32 @@ import ( "github.com/NVIDIA/aicr/pkg/defaults" aicrerrors "github.com/NVIDIA/aicr/pkg/errors" - "github.com/NVIDIA/aicr/pkg/k8s" "github.com/NVIDIA/aicr/pkg/recipe/oskind" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/watch" + "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" ) -// ensureJob deletes any existing Job and creates a fresh one. +// ensureJob creates the run-scoped agent Job. func (d *Deployer) ensureJob(ctx context.Context) error { - // Delete existing Job if present - propagationPolicy := metav1.DeletePropagationForeground - err := d.clientset.BatchV1().Jobs(d.config.Namespace).Delete( - ctx, - d.config.JobName, - metav1.DeleteOptions{ - PropagationPolicy: &propagationPolicy, - }, - ) - if err != nil && !errors.IsNotFound(err) { - return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to delete existing Job", err) - } - - // Wait for Job to be fully deleted - jobExisted := err == nil // Job existed and was deleted - if jobExisted { - if waitErr := d.waitForJobDeletion(ctx); waitErr != nil { - return aicrerrors.Wrap(aicrerrors.ErrCodeTimeout, "timeout waiting for Job deletion", waitErr) - } - } - - // Create fresh Job job := d.buildJob() - _, err = d.clientset.BatchV1().Jobs(d.config.Namespace). + // Record the intent before the Create so a committed create whose + // response is lost still enters Cleanup's delete list (see recordIntent). + d.recordIntent(kindJob, job.Name) + created, err := d.clientset.BatchV1().Jobs(d.config.Namespace). Create(ctx, job, metav1.CreateOptions{}) + if errors.IsAlreadyExists(err) { + d.discardIntent(kindJob, job.Name) + return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "Job already exists under run-scoped name (duplicate RunID?)", err) + } if err != nil { return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to create Job", err) } + d.recordCreated(kindJob, created.Name, created.UID) return nil } @@ -79,11 +64,9 @@ func (d *Deployer) buildJob() *batchv1.Job { return &batchv1.Job{ ObjectMeta: metav1.ObjectMeta{ - Name: d.config.JobName, + Name: d.jobName(), Namespace: d.config.Namespace, - Labels: map[string]string{ - labelAppName: appName, - }, + Labels: d.objectLabels(), }, Spec: batchv1.JobSpec{ Completions: ptr.To(int32(1)), @@ -94,9 +77,10 @@ func (d *Deployer) buildJob() *batchv1.Job { ActiveDeadlineSeconds: ptr.To(int64(defaults.AgentJobActiveDeadline.Seconds())), Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - labelAppName: appName, - }, + // Job metadata.labels do NOT propagate to the Pods a Job + // creates — the pod template needs its own copy so + // label-selector pod lookups (findPodName et al.) work. + Labels: d.objectLabels(), }, Spec: podSpec, }, @@ -109,7 +93,7 @@ func (d *Deployer) buildJob() *batchv1.Job { // When Privileged=false: PSS-compliant restricted pod, only K8s collector works. func (d *Deployer) buildPodSpec(args []string) corev1.PodSpec { spec := corev1.PodSpec{ - ServiceAccountName: d.config.ServiceAccountName, + ServiceAccountName: d.podServiceAccountName(), RestartPolicy: corev1.RestartPolicyNever, NodeSelector: d.config.NodeSelector, Tolerations: d.config.Tolerations, @@ -348,73 +332,21 @@ func (d *Deployer) buildEnvVars() []corev1.EnvVar { return envVars } -// deleteJob deletes the Job. -func (d *Deployer) deleteJob(ctx context.Context) error { +// deleteJob deletes the Job, pinning the delete to uid so a same-named Job +// belonging to a different run is never collected. If the Job is already +// gone, or uid no longer matches (already replaced, not ours), this is a +// no-op (idempotent). +func (d *Deployer) deleteJob(ctx context.Context, name string, uid types.UID) error { propagationPolicy := metav1.DeletePropagationForeground err := d.clientset.BatchV1().Jobs(d.config.Namespace).Delete( ctx, - d.config.JobName, + name, metav1.DeleteOptions{ PropagationPolicy: &propagationPolicy, + Preconditions: uidPreconditions(uid), }, ) - return k8s.IgnoreNotFound(err) -} - -// waitForJobDeletion waits for the Job to be fully deleted using the watch API. -// Returns nil when the Job is observed deleted (Get returns NotFound, or a -// watch.Deleted event is received). Returns ErrCodeTimeout if the cleanup -// deadline elapses before deletion is observed. -func (d *Deployer) waitForJobDeletion(ctx context.Context) error { - timeoutCtx, cancel := context.WithTimeout(ctx, defaults.K8sCleanupTimeout) - defer cancel() - - // Fast path: already deleted. Note: IgnoreNotFound(nil) returns nil, - // so check NotFound explicitly — otherwise a successful Get (Job still - // exists) would incorrectly short-circuit as "deleted". - current, err := d.clientset.BatchV1().Jobs(d.config.Namespace). - Get(timeoutCtx, d.config.JobName, metav1.GetOptions{}) - if errors.IsNotFound(err) { - return nil - } - if err != nil { - return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to get Job", err) - } - - watcher, err := d.clientset.BatchV1().Jobs(d.config.Namespace).Watch(timeoutCtx, metav1.ListOptions{ - FieldSelector: "metadata.name=" + d.config.JobName, - ResourceVersion: current.ResourceVersion, - }) - if err != nil { - return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "failed to watch Job", err) - } - defer watcher.Stop() - - for { - select { - case <-timeoutCtx.Done(): - return aicrerrors.Wrap(aicrerrors.ErrCodeTimeout, "Job deletion wait timeout", timeoutCtx.Err()) - case event, ok := <-watcher.ResultChan(): - if !ok { - // Channel closed; verify with a Get to handle missed events. - // Use explicit NotFound check (IgnoreNotFound(nil) returns nil - // and would falsely report success when the Job still exists). - _, getErr := d.clientset.BatchV1().Jobs(d.config.Namespace). - Get(timeoutCtx, d.config.JobName, metav1.GetOptions{}) - if errors.IsNotFound(getErr) { - return nil - } - if getErr != nil { - return aicrerrors.Wrap(aicrerrors.ErrCodeInternal, "Job watch channel closed", getErr) - } - return aicrerrors.Wrap(aicrerrors.ErrCodeUnavailable, - "Job watch channel closed before deletion observed", nil) - } - if event.Type == watch.Deleted { - return nil - } - } - } + return ignoreNotFoundOrConflict(err) } // mustParseQuantity parses a resource quantity or panics. diff --git a/pkg/k8s/agent/job_watch_test.go b/pkg/k8s/agent/job_watch_test.go index 01e96822e..d28ea2f92 100644 --- a/pkg/k8s/agent/job_watch_test.go +++ b/pkg/k8s/agent/job_watch_test.go @@ -21,7 +21,6 @@ import ( "time" aicrerrors "github.com/NVIDIA/aicr/pkg/errors" - batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/watch" @@ -34,126 +33,6 @@ import ( // event fails the test instead of stalling the suite. const jobWatchTimeout = 5 * time.Second -// TestWaitForJobDeletion_AlreadyDeleted exercises the fast-path Get returning -// NotFound (no Job exists in the clientset). -func TestWaitForJobDeletion_AlreadyDeleted(t *testing.T) { - t.Parallel() - - clientset := fake.NewClientset() // no jobs - d := NewDeployer(clientset, Config{ - Namespace: "test-ns", - JobName: "test-job", - }) - - if err := d.waitForJobDeletion(context.Background()); err != nil { - t.Fatalf("expected nil for already-deleted Job, got %v", err) - } -} - -// TestWaitForJobDeletion_DeletedEvent exercises the watch.Deleted path: the -// Job is present at Get time, then the fake watcher emits a Deleted event. -func TestWaitForJobDeletion_DeletedEvent(t *testing.T) { - t.Parallel() - - job := &batchv1.Job{ - ObjectMeta: metav1.ObjectMeta{Name: "test-job", Namespace: "test-ns", ResourceVersion: "1"}, - } - clientset := fake.NewClientset(job) - - w := watch.NewFake() - clientset.PrependWatchReactor("jobs", k8stesting.DefaultWatchReactor(w, nil)) - - go func() { - // Unbuffered FakeWatcher channel: Delete blocks until consumed. - w.Delete(job) - }() - - d := NewDeployer(clientset, Config{ - Namespace: "test-ns", - JobName: "test-job", - }) - - if err := d.waitForJobDeletion(context.Background()); err != nil { - t.Fatalf("expected nil after Deleted event, got %v", err) - } -} - -// TestWaitForJobDeletion_ChannelCloseWithNotFound exercises the fallback path -// where the watch channel closes without a Deleted event but a follow-up Get -// shows the Job has been deleted. -func TestWaitForJobDeletion_ChannelCloseWithNotFound(t *testing.T) { - t.Parallel() - - job := &batchv1.Job{ - ObjectMeta: metav1.ObjectMeta{Name: "test-job", Namespace: "test-ns", ResourceVersion: "1"}, - } - clientset := fake.NewClientset(job) - - w := watch.NewFake() - clientset.PrependWatchReactor("jobs", k8stesting.DefaultWatchReactor(w, nil)) - - go func() { - // Delete the Job from the fake clientset, then close the watch - // channel without firing a Deleted event so the close-fallback Get - // returns NotFound. - _ = clientset.BatchV1().Jobs("test-ns").Delete( - context.Background(), "test-job", metav1.DeleteOptions{}, - ) - w.Stop() - }() - - d := NewDeployer(clientset, Config{ - Namespace: "test-ns", - JobName: "test-job", - }) - - if err := d.waitForJobDeletion(context.Background()); err != nil { - t.Fatalf("expected nil when channel closes after deletion, got %v", err) - } -} - -// TestWaitForJobDeletion_ContextCanceled exercises the timeout branch when -// the parent context is canceled while the watcher remains idle. -func TestWaitForJobDeletion_ContextCanceled(t *testing.T) { - t.Parallel() - - job := &batchv1.Job{ - ObjectMeta: metav1.ObjectMeta{Name: "test-job", Namespace: "test-ns", ResourceVersion: "1"}, - } - clientset := fake.NewClientset(job) - - // Empty fake watcher that never emits events; force the select to - // block on ctx.Done() rather than racing against a default Added event. - w := watch.NewFake() - clientset.PrependWatchReactor("jobs", k8stesting.DefaultWatchReactor(w, nil)) - - d := NewDeployer(clientset, Config{ - Namespace: "test-ns", - JobName: "test-job", - }) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - // Cancel after a short delay so Get + Watch setup completes first. - go func() { - // Wait until the watcher has at least one consumer, signaled by - // the watcher's no-op behavior. Cancel to force the timeout branch. - cancel() - }() - - err := d.waitForJobDeletion(ctx) - if err == nil { - t.Fatal("expected error for canceled context") - } - var sErr *aicrerrors.StructuredError - if !stderrors.As(err, &sErr) { - t.Fatalf("expected *StructuredError, got %T: %v", err, err) - } - if sErr.Code != aicrerrors.ErrCodeTimeout { - t.Errorf("expected ErrCodeTimeout, got %v", sErr.Code) - } -} - // TestFindOrWatchPodName_WatchAddedEvent exercises the watch path: List // returns no matching pods, then a fake watcher emits an Added event. func TestFindOrWatchPodName_WatchAddedEvent(t *testing.T) { diff --git a/pkg/k8s/agent/names.go b/pkg/k8s/agent/names.go new file mode 100644 index 000000000..8759b9d3f --- /dev/null +++ b/pkg/k8s/agent/names.go @@ -0,0 +1,267 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package agent + +import ( + "fmt" + "strings" + + "github.com/NVIDIA/aicr/pkg/defaults" + "github.com/NVIDIA/aicr/pkg/errors" + "k8s.io/apimachinery/pkg/util/validation" +) + +// defaultNameBase is the prefix used for generated resource names when the +// caller does not set Config.NameBase. +const defaultNameBase = "aicr" + +// staticClusterRoleName is the un-scoped ClusterRole/ClusterRoleBinding +// name used only as the prefix input to nameWithRunID. +const staticClusterRoleName = "aicr-node-reader" + +// staticStagingConfigMapName is the un-scoped staging ConfigMap name used +// only as the prefix input to nameWithRunID. +// +// It is deliberately NOT "aicr-snapshot": pkg/validator names its own +// snapshot data ConfigMap "aicr-snapshot-" (see EnsureDataConfigMaps +// and cleanupDataConfigMaps in pkg/validator/validator.go, and the volume in +// pkg/validator/v1/job_plan_internal.go). `aicr validate` hands ONE run ID to +// both the snapshot agent and the validator Jobs and points both at the same +// namespace, so a shared prefix would put two owners on one object: the +// validator would adopt and overwrite the agent's staging ConfigMap, and its +// (UID-unpinned) cleanup would delete it. Distinct prefixes keep the two +// namespaces of generated names disjoint by construction. +const staticStagingConfigMapName = "aicr-agent-snapshot" + +// nameWithRunID joins prefix and runID, truncating prefix so the result fits +// within the Kubernetes name ceiling. An empty prefix yields the bare runID. +// An empty runID yields the prefix with any trailing "-" trimmed rather than +// appending one: a trailing separator would leave a Kubernetes object name +// that fails validation (names must end in an alphanumeric character). +// +// The empty-runID fallback is not reachable through Deploy, which rejects +// an empty or malformed RunID up front (see validateRunID). It survives for +// the accessors a caller can reach without deploying — JobName, Cleanup on a +// Deployer that never ran — where returning a name that fails Kubernetes +// validation would be strictly worse than returning the bare prefix. +func nameWithRunID(prefix, runID string) string { + if prefix == "" { + return runID + } + if runID == "" { + return strings.TrimRight(prefix, "-") + } + budget := defaults.MaxK8sNameLength - len(runID) - 1 + if budget < 0 { + budget = 0 + } + if len(prefix) > budget { + prefix = prefix[:budget] + } + prefix = strings.TrimRight(prefix, "-") + if prefix == "" { + return runID + } + return prefix + "-" + runID +} + +// validateRunID rejects a Config.RunID that cannot be folded into a valid +// Kubernetes object name. Deploy calls it before any object is created: +// every run-owned name is "-", so a bad run ID would +// otherwise surface as an opaque apiserver "Invalid value: metadata.name" +// from deep inside the ensure* chain — after some objects already exist. +// +// The constraint is exactly DNS-1123 label: lowercase alphanumerics and +// "-", starting and ending alphanumeric, at most 63 characters. The length +// bound is what keeps the generated name inside defaults.MaxK8sNameLength: +// nameWithRunID truncates the prefix to fit, so an over-long run ID is the +// one input it cannot compensate for (its budget floors at zero and the +// bare run ID is returned). +// +// pkg/snapshotter defaults and whitespace-checks RunID before building an +// agent Config, but that guard does not cover callers who construct a +// pkg/k8s/agent Config directly, which is the public SDK surface. +// Error-context keys shared by the name-validation failures below, so a +// caller parsing a structured error keys off one spelling. +const ( + ctxKeyField = "field" + ctxKeyValue = "value" + ctxKeyResolvedName = "resolvedName" +) + +func (d *Deployer) validateRunID() error { + runID := d.config.RunID + if runID == "" { + return errors.NewWithContext(errors.ErrCodeInvalidRequest, + "Config.RunID is required: every object this Deployer creates is named \"-\"; generate one with runid.Generate()", + map[string]any{ctxKeyField: "Config.RunID", ctxKeyValue: runID}) + } + if problems := validation.IsDNS1123Label(runID); len(problems) > 0 { + return errors.NewWithContext(errors.ErrCodeInvalidRequest, + fmt.Sprintf("Config.RunID %q is not a valid Kubernetes name segment: %s", + runID, strings.Join(problems, "; ")), + map[string]any{ctxKeyField: "Config.RunID", ctxKeyValue: runID}) + } + return nil +} + +// resolvedName is one caller-influenced object name this Deployer would +// create, carried together with the Config field that supplied its prefix. +// Validation reports the field rather than only the derived string, because +// the field is what a caller can actually change. +type resolvedName struct { + field string // Config field the prefix came from + prefix string // the prefix value that field held + value string // the resolved object name, "-" + // objects names what value is the name of, so a rejection states the + // blast radius: saName() also names the Role and the RoleBinding. + objects string +} + +// resolvedNames returns every object name this Deployer builds from a +// caller-supplied prefix, paired with its source field. +// +// The ClusterRole/ClusterRoleBinding and staging ConfigMap names are +// deliberately absent: their prefixes are package constants +// (staticClusterRoleName, staticStagingConfigMapName), so the only +// caller-supplied input they carry is the run ID, which validateRunID +// already covers. +func (d *Deployer) resolvedNames() []resolvedName { + jobField, jobPrefix := "Config.NameBase", d.base() + if d.config.JobName != "" { + jobField, jobPrefix = "Config.JobName", d.config.JobName + } + saField, saPrefix := "Config.NameBase", d.base() + if d.config.ServiceAccountName != "" { + saField, saPrefix = "Config.ServiceAccountName", d.config.ServiceAccountName + } + return []resolvedName{ + {field: jobField, prefix: jobPrefix, value: d.jobName(), objects: "Job"}, + {field: saField, prefix: saPrefix, value: d.saName(), objects: "ServiceAccount, Role and RoleBinding"}, + } +} + +// validateResolvedNames rejects a generated object name Kubernetes would +// refuse. validateRunID covers one half of every run-owned name; this covers +// the other. NameBase, JobName and ServiceAccountName are caller-supplied +// too, and a prefix such as "agent_" yields "agent_-" — which the +// apiserver rejects with an opaque "Invalid value: metadata.name" from +// partway through Deploy's ensure* chain, after some objects already exist. +// +// The constraint is DNS-1123 subdomain, which is what the apiserver enforces +// on Job and ServiceAccount names, plus the narrower defaults.MaxK8sNameLength +// ceiling this package budgets against (a Job name also becomes the +// batch.kubernetes.io/job-name label value on every Pod the Job creates, and +// label values share that ceiling). nameWithRunID truncates the prefix to +// that budget, so the length branch is reachable only for a run ID that is +// itself over-long — which validateRunID rejects first when both run under +// validateNames, but not when this method is called on its own. +func (d *Deployer) validateResolvedNames() error { + for _, n := range d.resolvedNames() { + problems := validation.IsDNS1123Subdomain(n.value) + if len(problems) == 0 && len(n.value) > defaults.MaxK8sNameLength { + problems = []string{fmt.Sprintf("must be no more than %d characters", defaults.MaxK8sNameLength)} + } + if len(problems) == 0 { + continue + } + return errors.NewWithContext(errors.ErrCodeInvalidRequest, + fmt.Sprintf("%s %q yields the %s name %q, which is not a valid Kubernetes object name: %s", + n.field, n.prefix, n.objects, n.value, strings.Join(problems, "; ")), + map[string]any{ctxKeyField: n.field, ctxKeyValue: n.prefix, ctxKeyResolvedName: n.value}) + } + return nil +} + +// validateNames is Deploy's naming pre-flight: it rejects both halves of a +// run-owned name — the run ID and the resolved object names built from the +// caller's prefixes — before any cluster call is made, so an invalid value +// can never leave a partially-created deployment behind. +func (d *Deployer) validateNames() error { + if err := d.validateRunID(); err != nil { + return err + } + return d.validateResolvedNames() +} + +// base returns the configured name base, defaulting to "aicr" when unset. +func (d *Deployer) base() string { + if d.config.NameBase != "" { + return d.config.NameBase + } + return defaultNameBase +} + +// jobName returns the run-scoped name for the agent Job. +func (d *Deployer) jobName() string { + prefix := d.config.JobName + if prefix == "" { + prefix = d.base() + } + return nameWithRunID(prefix, d.config.RunID) +} + +// podServiceAccountName returns the ServiceAccount the agent pod actually +// runs as: the operator's already-existing ServiceAccount when Deploy +// resolved Config.ServiceAccountName to an exact match, otherwise this run's +// own run-scoped one. +// +// It is deliberately separate from saName(), which stays the run-scoped +// name unconditionally. saName() also names the Role and RoleBinding, and +// those exist only in prefix mode — folding the exact name into it would +// make roleName() silently return an operator-owned ServiceAccount's name. +func (d *Deployer) podServiceAccountName() string { + if name := d.existingServiceAccount(); name != "" { + return name + } + return d.saName() +} + +// saName returns the run-scoped name for the agent ServiceAccount. +func (d *Deployer) saName() string { + prefix := d.config.ServiceAccountName + if prefix == "" { + prefix = d.base() + } + return nameWithRunID(prefix, d.config.RunID) +} + +// roleName returns the run-scoped name for the agent Role and RoleBinding. +// It shares the ServiceAccount's name, matching the existing convention +// where the Role/RoleBinding are named after the ServiceAccount they bind. +func (d *Deployer) roleName() string { + return d.saName() +} + +// clusterRoleName returns the run-scoped name for the agent ClusterRole and +// ClusterRoleBinding. +func (d *Deployer) clusterRoleName() string { + return nameWithRunID(staticClusterRoleName, d.config.RunID) +} + +// StagingConfigMapName returns the run-scoped name of the internal staging +// ConfigMap the agent Job writes its snapshot result to for the given run ID. +// It is exported so the one caller that builds the Job's `cm://` output URI +// (pkg/snapshotter's agentConfigMapTarget) derives that name from the same +// place Cleanup deletes it, instead of repeating the format string. +func StagingConfigMapName(runID string) string { + return nameWithRunID(staticStagingConfigMapName, runID) +} + +// stagingConfigMapName returns the run-scoped name for the staging +// ConfigMap the agent writes its snapshot result to. +func (d *Deployer) stagingConfigMapName() string { + return StagingConfigMapName(d.config.RunID) +} diff --git a/pkg/k8s/agent/names_test.go b/pkg/k8s/agent/names_test.go new file mode 100644 index 000000000..979827143 --- /dev/null +++ b/pkg/k8s/agent/names_test.go @@ -0,0 +1,462 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package agent + +import ( + "context" + stderrors "errors" + "strings" + "testing" + + "github.com/NVIDIA/aicr/pkg/defaults" + "github.com/NVIDIA/aicr/pkg/errors" + "k8s.io/client-go/kubernetes/fake" +) + +func TestNameWithRunID(t *testing.T) { + const runID = "20260821-142233-9f3a1c0b7e2d4a55" + + // nameWithRunID budgets the prefix against defaults.MaxK8sNameLength, + // reserving len(runID) plus the "-" separator. Derive the budget the same + // way rather than hard-coding it, so raising or lowering the constant + // moves these boundaries with it instead of silently invalidating them. + budget := defaults.MaxK8sNameLength - len(runID) - 1 + + tests := []struct { + name string + prefix string + runID string + want string + }{ + {"short prefix", "aicr", runID, "aicr-" + runID}, + {"exactly at budget", strings.Repeat("a", budget), runID, strings.Repeat("a", budget) + "-" + runID}, + {"over budget truncates", strings.Repeat("b", budget+10), runID, strings.Repeat("b", budget) + "-" + runID}, + {"trailing dash trimmed", strings.Repeat("c", budget-1) + "-", runID, strings.Repeat("c", budget-1) + "-" + runID}, + {"empty prefix", "", runID, runID}, + // A zero-value Config.RunID (only reachable from an SDK caller + // constructing a Config directly) must fall back to the bare prefix, + // never a prefix with a trailing "-" — that would be an invalid + // Kubernetes object name. + {"empty runID falls back to bare prefix", "aicr", "", "aicr"}, + {"empty runID trims the prefix's trailing dash", "aicr-", "", "aicr"}, + {"empty prefix and empty runID", "", "", ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := nameWithRunID(tt.prefix, tt.runID) + if got != tt.want { + t.Errorf("nameWithRunID(%q, %q) = %q, want %q", tt.prefix, tt.runID, got, tt.want) + } + if len(got) > defaults.MaxK8sNameLength { + t.Errorf("len = %d, exceeds the %d-char ceiling", len(got), defaults.MaxK8sNameLength) + } + if strings.HasSuffix(got, "-") { + t.Errorf("nameWithRunID(%q, %q) = %q, ends in a trailing separator (invalid Kubernetes name)", tt.prefix, tt.runID, got) + } + }) + } +} + +func TestDeployerNameAccessors(t *testing.T) { + const runID = "20260821-142233-9f3a1c0b7e2d4a55" // 32 chars + + tests := []struct { + name string + config Config + get func(*Deployer) string + want string + }{ + { + name: "jobName uses configured JobName", + config: Config{JobName: "my-job", RunID: runID}, + get: (*Deployer).jobName, + want: "my-job-" + runID, + }, + { + name: "jobName falls back to NameBase", + config: Config{NameBase: "custom-base", RunID: runID}, + get: (*Deployer).jobName, + want: "custom-base-" + runID, + }, + { + name: "jobName falls back to default base", + config: Config{RunID: runID}, + get: (*Deployer).jobName, + want: "aicr-" + runID, + }, + { + name: "saName uses configured ServiceAccountName", + config: Config{ServiceAccountName: "my-sa", RunID: runID}, + get: (*Deployer).saName, + want: "my-sa-" + runID, + }, + { + name: "saName falls back to default base", + config: Config{RunID: runID}, + get: (*Deployer).saName, + want: "aicr-" + runID, + }, + { + name: "roleName matches saName", + config: Config{ServiceAccountName: "my-sa", RunID: runID}, + get: (*Deployer).roleName, + want: "my-sa-" + runID, + }, + { + name: "clusterRoleName is run-scoped", + config: Config{RunID: runID}, + get: (*Deployer).clusterRoleName, + want: "aicr-node-reader-" + runID, + }, + { + name: "stagingConfigMapName is run-scoped", + config: Config{RunID: runID}, + get: (*Deployer).stagingConfigMapName, + want: "aicr-agent-snapshot-" + runID, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := &Deployer{config: tt.config} + if got := tt.get(d); got != tt.want { + t.Errorf("got %q, want %q", got, tt.want) + } + }) + } +} + +// TestStagingConfigMapNameDoesNotCollideWithValidator pins the reason the +// agent's staging ConfigMap is prefixed "aicr-agent-snapshot" and not +// "aicr-snapshot": pkg/validator builds its own snapshot data ConfigMap as +// "aicr-snapshot-" (EnsureDataConfigMaps and cleanupDataConfigMaps in +// pkg/validator/validator.go, plus the Job volume in +// pkg/validator/v1/job_plan_internal.go). `aicr validate` hands ONE run ID to +// both subsystems and points both at the same namespace, so equal names would +// mean two owners on one object: the validator adopts and overwrites the +// agent's staging data (silently replacing the artifact --no-cleanup was +// asked to preserve), and its UID-unpinned cleanup deletes it. +// +// The validator name is spelled out here rather than imported because it is +// built inline there; if that ever changes, this test is the tripwire. +func TestStagingConfigMapNameDoesNotCollideWithValidator(t *testing.T) { + const runID = "20260821-142233-9f3a1c0b7e2d4a55" + + validatorSnapshotCM := "aicr-snapshot-" + runID + agentStagingCM := StagingConfigMapName(runID) + + if agentStagingCM == validatorSnapshotCM { + t.Fatalf("agent staging ConfigMap name %q collides with the validator's snapshot data ConfigMap name for the same run ID", agentStagingCM) + } + // A shared prefix is the collision hazard in the other direction: the + // validator's name must not be a prefix of the agent's (or vice versa) + // once a run ID is appended by either side. + if strings.HasPrefix(agentStagingCM, "aicr-snapshot-") { + t.Errorf("agent staging ConfigMap name %q reuses the validator's %q prefix", agentStagingCM, "aicr-snapshot-") + } + if want := "aicr-agent-snapshot-" + runID; agentStagingCM != want { + t.Errorf("StagingConfigMapName(%q) = %q, want %q", runID, agentStagingCM, want) + } +} + +// TestJobNameIsRunScoped covers the exported accessor callers use when they +// surface the Job to an operator (pkg/snapshotter logs it while waiting for +// completion). Config.JobName is only the prefix and is empty by default, so +// logging that field prints an empty name. +func TestJobNameIsRunScoped(t *testing.T) { + const runID = "20260821-142233-9f3a1c0b7e2d4a55" + + d := NewDeployer(nil, Config{RunID: runID}) + if got, want := d.JobName(), "aicr-"+runID; got != want { + t.Errorf("JobName() with no configured prefix = %q, want %q", got, want) + } + + withPrefix := NewDeployer(nil, Config{JobName: "my-job", RunID: runID}) + if got, want := withPrefix.JobName(), "my-job-"+runID; got != want { + t.Errorf("JobName() with a configured prefix = %q, want %q", got, want) + } + if withPrefix.JobName() == withPrefix.config.JobName { + t.Error("JobName() returned the bare prefix; it must be run-scoped") + } +} + +// TestStagingConfigMapNameMatchesDeployerMethod guards a single source of +// truth for the staging ConfigMap's name: the exported helper pkg/snapshotter +// uses to build the Job's cm:// output URI and the name Cleanup deletes must +// be the same string for the same run. +func TestStagingConfigMapNameMatchesDeployerMethod(t *testing.T) { + const runID = "20260821-142233-9f3a1c0b7e2d4a55" + d := &Deployer{config: Config{RunID: runID}} + if got, want := d.stagingConfigMapName(), StagingConfigMapName(runID); got != want { + t.Errorf("stagingConfigMapName() = %q, StagingConfigMapName(%q) = %q; they must agree", got, runID, want) + } +} + +// TestValidateRunID covers the run IDs that are non-empty but still cannot +// be folded into a Kubernetes object name. Without this gate they reach the +// apiserver as an opaque "Invalid value: metadata.name" from partway through +// Deploy's ensure* chain, after some objects already exist. +func TestValidateRunID(t *testing.T) { + tests := []struct { + name string + runID string + wantErr bool + }{ + {"well-formed generated run ID", "20260821-142233-9f3a1c0b7e2d4a55", false}, + {"single character", "a", false}, + {"exactly at the DNS-1123 label ceiling", strings.Repeat("a", defaults.MaxK8sNameLength), false}, + {"empty", "", true}, + {"whitespace", " ", true}, + {"embedded slash", "build/42", true}, + {"leading dash", "-build", true}, + {"trailing dash", "build-", true}, + {"uppercase", "Build42", true}, + {"one over the DNS-1123 label ceiling", strings.Repeat("a", defaults.MaxK8sNameLength+1), true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := NewDeployer(fake.NewClientset(), Config{Namespace: "test-ns", RunID: tt.runID}) + err := d.validateRunID() + if (err != nil) != tt.wantErr { + t.Fatalf("validateRunID() error = %v, wantErr %v", err, tt.wantErr) + } + if err == nil { + return + } + if !stderrors.Is(err, errors.New(errors.ErrCodeInvalidRequest, "")) { + t.Errorf("error code = %v, want %v", err, errors.ErrCodeInvalidRequest) + } + // The message must name the field and echo the offending + // value so an operator can see what to change. + if !strings.Contains(err.Error(), "Config.RunID") { + t.Errorf("error %q does not name the offending field", err.Error()) + } + if tt.runID != "" && !strings.Contains(err.Error(), tt.runID) { + t.Errorf("error %q does not echo the offending value %q", err.Error(), tt.runID) + } + }) + } +} + +// TestDeployRejectsInvalidRunIDBeforeCreatingAnything is the end-to-end half +// of the gate: Deploy must fail before it reaches the ensure* chain, so a +// rejected run leaves no partially-created RBAC behind. +func TestDeployRejectsInvalidRunIDBeforeCreatingAnything(t *testing.T) { + ctx := context.Background() + for _, runID := range []string{"", " ", "build/42", "-build", "build-", "Build42", strings.Repeat("a", defaults.MaxK8sNameLength+1)} { + t.Run(runID, func(t *testing.T) { + clientset := fake.NewClientset() + d := NewDeployer(clientset, Config{Namespace: "test-ns", Image: "aicr:test", RunID: runID}) + + err := d.Deploy(ctx) + if err == nil { + t.Fatalf("Deploy() with RunID %q should fail", runID) + } + if !stderrors.Is(err, errors.New(errors.ErrCodeInvalidRequest, "")) { + t.Errorf("Deploy() error = %v, want code %v", err, errors.ErrCodeInvalidRequest) + } + + // Nothing may have been created — not even the Namespace, + // which Deploy ensures before any run-owned object. + if actions := clientset.Actions(); len(actions) != 0 { + t.Errorf("Deploy() issued %d API call(s) before rejecting an invalid RunID: %v", len(actions), actions) + } + }) + } +} + +// TestValidateResolvedNames covers the other half of the naming gate: +// Config.RunID may be well-formed while the caller-supplied prefix it is +// appended to is not. Every case here calls validateResolvedNames directly +// rather than validateNames, so the over-long run ID case can reach the +// length branch that validateRunID would otherwise short-circuit. +func TestValidateResolvedNames(t *testing.T) { + const runID = "20260821-142233-9f3a1c0b7e2d4a55" + + // A prefix that leaves no room at all: nameWithRunID's budget floors at + // zero, so the resolved name is the bare (over-long) run ID. + overLongRunID := strings.Repeat("a", defaults.MaxK8sNameLength+17) + + tests := []struct { + name string + // config carries only the naming fields; RunID defaults to runID + // unless the case overrides it. + config Config + // wantErr is the substring the message must name — the Config field + // at fault. Empty means the names must be accepted. + wantField string + // wantValue is echoed in the message alongside the field so an + // operator sees what to change. Only checked when wantField is set. + wantValue string + }{ + { + name: "default prefixes are valid", + config: Config{RunID: runID}, + }, + { + name: "explicit prefixes are valid", + config: Config{JobName: "my-job", ServiceAccountName: "my-sa", RunID: runID}, + }, + { + name: "a dot is a legal DNS-1123 subdomain character", + config: Config{JobName: "aicr.agent", RunID: runID}, + }, + { + // nameWithRunID trims the separator rather than doubling it, + // so this resolves to a valid name and must be accepted. + name: "trailing dash on the prefix is trimmed, not rejected", + config: Config{JobName: "agent-", RunID: runID}, + }, + { + // The budget truncation keeps the resolved name inside + // defaults.MaxK8sNameLength, so an over-long prefix is not an + // error — it is silently shortened. + name: "over-length prefix truncates to a valid name", + config: Config{JobName: strings.Repeat("a", defaults.MaxK8sNameLength*3), RunID: runID}, + }, + { + // Truncation plus trailing-dash trimming can empty the prefix + // entirely; the result is the bare run ID, which is valid. + name: "prefix that empties after truncation degrades to the bare run ID", + config: Config{JobName: "----", RunID: runID}, + }, + { + name: "underscore in JobName", + config: Config{JobName: "agent_", RunID: runID}, + wantField: "Config.JobName", + wantValue: "agent_", + }, + { + name: "underscore in ServiceAccountName", + config: Config{ServiceAccountName: "agent_sa", RunID: runID}, + wantField: "Config.ServiceAccountName", + wantValue: "agent_sa", + }, + { + name: "underscore in NameBase governs both names", + config: Config{NameBase: "agent_base", RunID: runID}, + wantField: "Config.NameBase", + wantValue: "agent_base", + }, + { + name: "uppercase in JobName", + config: Config{JobName: "Agent", RunID: runID}, + wantField: "Config.JobName", + wantValue: "Agent", + }, + { + name: "leading dash in JobName", + config: Config{JobName: "-agent", RunID: runID}, + wantField: "Config.JobName", + wantValue: "-agent", + }, + { + name: "slash in ServiceAccountName", + config: Config{ServiceAccountName: "team/agent", RunID: runID}, + wantField: "Config.ServiceAccountName", + wantValue: "team/agent", + }, + { + // Reachable only by calling validateResolvedNames directly: + // validateNames rejects this run ID first. It exists so the + // defaults.MaxK8sNameLength branch is covered rather than + // trusted. + name: "over-long run ID leaves a name past the length ceiling", + config: Config{RunID: overLongRunID}, + wantField: "Config.NameBase", + wantValue: defaultNameBase, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := NewDeployer(fake.NewClientset(), tt.config) + err := d.validateResolvedNames() + if tt.wantField == "" { + if err != nil { + t.Fatalf("validateResolvedNames() error = %v, want nil", err) + } + return + } + if err == nil { + t.Fatalf("validateResolvedNames() = nil, want an error naming %s", tt.wantField) + } + if !stderrors.Is(err, errors.New(errors.ErrCodeInvalidRequest, "")) { + t.Errorf("error = %v, want code %v", err, errors.ErrCodeInvalidRequest) + } + if !strings.Contains(err.Error(), tt.wantField) { + t.Errorf("error %q does not name the offending field %q", err.Error(), tt.wantField) + } + if !strings.Contains(err.Error(), tt.wantValue) { + t.Errorf("error %q does not echo the offending value %q", err.Error(), tt.wantValue) + } + }) + } +} + +// TestValidateNamesChecksRunIDFirst pins the order inside the pre-flight: a +// caller who gets both halves wrong should hear about the run ID, which is +// the value they are least likely to have set deliberately. +func TestValidateNamesChecksRunIDFirst(t *testing.T) { + d := NewDeployer(fake.NewClientset(), Config{JobName: "agent_", RunID: "Bad/ID"}) + err := d.validateNames() + if err == nil { + t.Fatal("validateNames() = nil, want an error") + } + if !strings.Contains(err.Error(), "Config.RunID") { + t.Errorf("error %q does not name Config.RunID", err.Error()) + } +} + +// TestDeployRejectsInvalidResolvedNameBeforeCreatingAnything is the +// end-to-end half of the resolved-name gate, mirroring the run-ID test +// above: an invalid prefix must be rejected before CheckPermissions and +// before any write, so no partially-created RBAC is left behind. +func TestDeployRejectsInvalidResolvedNameBeforeCreatingAnything(t *testing.T) { + const runID = "20260821-142233-9f3a1c0b7e2d4a55" + ctx := context.Background() + + tests := []struct { + name string + config Config + }{ + {"underscore JobName", Config{JobName: "agent_", RunID: runID}}, + {"uppercase ServiceAccountName", Config{ServiceAccountName: "AgentSA", RunID: runID}}, + {"underscore NameBase", Config{NameBase: "agent_base", RunID: runID}}, + {"leading dash JobName", Config{JobName: "-agent", RunID: runID}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clientset := fake.NewClientset() + cfg := tt.config + cfg.Namespace = "test-ns" + cfg.Image = "aicr:test" + + err := NewDeployer(clientset, cfg).Deploy(ctx) + if err == nil { + t.Fatalf("Deploy() with config %+v should fail", cfg) + } + if !stderrors.Is(err, errors.New(errors.ErrCodeInvalidRequest, "")) { + t.Errorf("Deploy() error = %v, want code %v", err, errors.ErrCodeInvalidRequest) + } + // Not even the Step-0 SelfSubjectAccessReview may have been + // issued: the gate runs ahead of CheckPermissions. + if actions := clientset.Actions(); len(actions) != 0 { + t.Errorf("Deploy() issued %d API call(s) before rejecting an invalid name: %v", len(actions), actions) + } + }) + } +} diff --git a/pkg/k8s/agent/permissions.go b/pkg/k8s/agent/permissions.go index a13d1c746..0c068a01c 100644 --- a/pkg/k8s/agent/permissions.go +++ b/pkg/k8s/agent/permissions.go @@ -17,131 +17,634 @@ package agent import ( "context" "fmt" + "log/slog" "strings" - "sync" + "github.com/NVIDIA/aicr/pkg/defaults" "github.com/NVIDIA/aicr/pkg/errors" + "github.com/NVIDIA/aicr/pkg/k8s/pod" "golang.org/x/sync/errgroup" authv1 "k8s.io/api/authorization/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// permissionCheck represents a single permission check result. +// Labels used when rendering a permission failure for an operator. The +// caller label names the kubeconfig identity running aicr; the +// ServiceAccount label is filled in with the subject's +// "system:serviceaccount::" username. +const ( + callerSubjectLabel = "the caller (your kubeconfig identity)" + + // serviceAccountUserPrefix is the username the apiserver authenticates + // a ServiceAccount as, and therefore the SubjectAccessReview subject + // that answers for the agent pod rather than for the caller. + serviceAccountUserPrefix = "system:serviceaccount:" + + // Virtual groups every ServiceAccount is a member of. RBAC bindings + // routinely target them instead of an individual ServiceAccount, so a + // SubjectAccessReview that omits them under-reports the subject's + // real permissions and would fail a correctly-provisioned operator. + groupServiceAccounts = "system:serviceaccounts" + groupServiceAccountsPrefix = "system:serviceaccounts:" + groupAuthenticated = "system:authenticated" +) + +// Sizing hints for the two slices this file builds. Neither is a bound. +const ( + // rbacKindCount is the number of RBAC kinds a prefix-mode run creates + // and later deletes: ServiceAccount, Role, RoleBinding, ClusterRole, + // ClusterRoleBinding. + rbacKindCount = 5 + + // rulesPerPolicyHint is the average number of (group, resource, verb) + // triples one PolicyRule expands into across namespacedRules and + // clusterRules. + rulesPerPolicyHint = 4 +) + +// accessCheck is one authorization question: may `subject` perform `verb` on +// `group/resource[/subresource]` at `namespace` (empty means cluster scope)? +// +// subject is "" for the caller — answered with a SelfSubjectAccessReview — +// or a "system:serviceaccount::" username for the agent's +// ServiceAccount, answered with a SubjectAccessReview. The struct is +// deliberately all-comparable so dedupeChecks can key a map on it. +type accessCheck struct { + group string + resource string + subresource string + verb string + namespace string + subject string +} + +// permissionCheck is one answered accessCheck, returned to callers so they +// can render the full pre-flight rather than only its failures. type permissionCheck struct { - Resource string - Verb string - Namespace string - Allowed bool - Reason string + Group string + Resource string + Subresource string + Verb string + Namespace string + + // Subject is "" for the caller, or the ServiceAccount's + // "system:serviceaccount::" username. + Subject string + + Allowed bool + Reason string + + // Unverified marks a ServiceAccount check the apiserver refused to + // answer because the caller may not create a SubjectAccessReview. + // Allowed is false on such an entry, but it is NOT counted as a + // missing permission: nothing was learned either way. CheckPermissions + // says so out loud instead of silently dropping the check. + Unverified bool } -// CheckPermissions verifies if the current user has the required permissions -// to deploy the agent. Returns a list of permission checks and an error if any -// required permissions are missing. +// CheckPermissions is the authoritative pre-flight gate for an entire agent +// run. It verifies every permission the run will actually exercise, for both +// identities involved — the caller (the kubeconfig identity running aicr) +// and the ServiceAccount the agent pod runs as — and fails before anything +// is written to the cluster when any of them is missing. +// +// # Ordering, and why it is still fail-before-mutate +// +// The required verb set depends on which ServiceAccount mode this run is in +// (see resolveServiceAccount), and the mode cannot be known without reading +// ServiceAccounts. The gate therefore runs in two phases: +// +// 1. Check the caller permissions every run needs in either mode, +// `serviceaccounts: get` among them. +// 2. Resolve the ServiceAccount (a read-only Get), then check the +// mode-specific set. +// +// Every step up to the point the gate closes is a read: a +// Self/SubjectAccessReview is a non-persisted authorization query, and the +// resolution Get creates nothing. The fail-before-mutate guarantee is about +// not WRITING before validation, and no write is issued until Deploy's +// ensure* chain, which runs only after this returns nil. +// +// # Mode-specific verbs +// +// Prefix mode (aicr creates its own run-scoped ServiceAccount) needs create +// AND delete on all five RBAC kinds: the deferred Cleanup is registered +// before Deploy and always runs, so an identity that can create but not +// delete would pass a green pre-flight and then leak a full run-scoped RBAC +// set — cluster-scoped objects included — on every run. +// +// Exact-ServiceAccount mode creates and deletes no RBAC at all, so demanding +// those verbs would block operators who legitimately hold none. It instead +// verifies that the operator actually provisioned the ServiceAccount, which +// aicr does not do for them. +// +// # Reporting +// +// Every check is evaluated before any failure is reported, so an operator +// fixing permissions gets the complete list in one run. Each failure names +// the verb, the resource, the scope, and which subject lacked it. func (d *Deployer) CheckPermissions(ctx context.Context) ([]permissionCheck, error) { - type permCheck struct { - resource string - verb string - namespace string - } - - // Required permissions for deployment - requiredChecks := []permCheck{ - // Namespace-scoped resources - {"serviceaccounts", verbCreate, d.config.Namespace}, - {"roles", verbCreate, d.config.Namespace}, - {"rolebindings", verbCreate, d.config.Namespace}, - {"jobs", verbCreate, d.config.Namespace}, - {resourceCM, verbGet, d.config.Namespace}, - {resourceCM, verbList, d.config.Namespace}, - - // Cluster-scoped resources - {"clusterroles", verbCreate, ""}, - {"clusterrolebindings", verbCreate, ""}, - - // Cleanup permissions - {"jobs", "delete", d.config.Namespace}, - } - - // SelfSubjectAccessReview is a read-only query; running the N required - // checks in parallel cuts wall-clock from N×RTT to one RTT against the - // apiserver. Each iteration's index is preserved so the response slice - // keeps a deterministic order regardless of completion timing. - results := make([]permissionCheck, len(requiredChecks)) + // Phase 1: the caller-side set every run needs regardless of mode. + results, err := d.runAccessChecks(ctx, dedupeChecks(d.callerCommonChecks())) + if err != nil { + return nil, err + } + + // `serviceaccounts: get` gates the rest of the pre-flight rather than + // merely joining it. Without it the mode is unknowable, and the + // historical behavior — treat an unreadable ServiceAccount as a + // prefix — silently ran an operator who named their IRSA / Workload + // Identity ServiceAccount under a fresh, un-annotated one instead. + // Fail here with everything phase 1 found, and say why the rest was + // not evaluated. + if !callerMayReadServiceAccounts(results) { + return results, missingPermissionsError(results, unresolvableModeHint(d.config.Namespace)) + } + + // Read-only: decides which verb set phase 2 demands, and which + // ServiceAccount the agent pod will run as. + if err = d.resolveServiceAccount(ctx); err != nil { + return results, err + } + + modeResults, err := d.runAccessChecks(ctx, dedupeChecks(d.modeSpecificChecks())) + if err != nil { + return nil, err + } + results = append(results, modeResults...) + + // The meta-permission is handled out loud, never silently: a caller who + // cannot create a SubjectAccessReview learns that the ServiceAccount's + // own permissions went unverified and that the agent will surface any + // gap in-pod instead. + d.warnUnverified(results) + + if hasMissing(results) { + return results, missingPermissionsError(results, d.remediationHints(results)) + } + return results, nil +} + +// callerCommonChecks returns the permissions the caller needs in either +// ServiceAccount mode: everything Deploy, WaitForCompletion, GetSnapshot and +// Cleanup issue that is not RBAC creation or deletion. +// +// Deliberately absent: Namespace create/get/patch. ensureNamespace's three +// branches need a different verb depending on whether the namespace already +// exists and is already labeled, and the documented path — an operator's +// pre-existing, previously-labeled namespace such as `gpu-operator` — +// needs none of them. Demanding the cluster-scoped `namespaces: create` +// that only the fresh-namespace branch uses would fail the majority case. +// +// Also absent: `get` on Role, RoleBinding, ClusterRole and ClusterRoleBinding. +// Cleanup reads those only to re-establish ownership of an entry whose +// Create response was lost (resolveIntentUID), and that path already fails +// closed with a warning rather than a blind delete, so a missing `get` +// degrades to a logged orphan instead of a wrong deletion. `serviceaccounts: +// get` is required anyway, for the mode resolution above. +func (d *Deployer) callerCommonChecks() []accessCheck { + ns := d.config.Namespace + checks := []accessCheck{ + // Mode resolution (resolveServiceAccount) and Cleanup's + // ownership re-check (resolveIntentUID). + {resource: resourceServiceAccounts, verb: verbGet, namespace: ns}, + + // ensureJob, waitForJobCompletion (Get + Watch, List on watch + // resume) and Cleanup's UID-pinned delete. + {group: batchAPIGroup, resource: resourceJobs, verb: verbCreate, namespace: ns}, + {group: batchAPIGroup, resource: resourceJobs, verb: verbGet, namespace: ns}, + {group: batchAPIGroup, resource: resourceJobs, verb: verbList, namespace: ns}, + {group: batchAPIGroup, resource: resourceJobs, verb: verbWatch, namespace: ns}, + {group: batchAPIGroup, resource: resourceJobs, verb: verbDelete, namespace: ns}, + + // Pod discovery (findPodName / findOrWatchPodName), readiness + // waiting, and log streaming back to the operator's terminal. + {resource: resourcePods, verb: verbGet, namespace: ns}, + {resource: resourcePods, verb: verbList, namespace: ns}, + {resource: resourcePods, verb: verbWatch, namespace: ns}, + {resource: resourcePods, subresource: subresourceLog, verb: verbGet, namespace: ns}, + + // Reading the snapshot the agent staged. + {resource: resourceCM, verb: verbGet, namespace: ns}, + {resource: resourceCM, verb: verbList, namespace: ns}, + } + + // A caller-supplied `cm:///` Output is read + // where it points, not in the agent's namespace. + if outputNS, _, parseErr := pod.ParseConfigMapURI(d.config.Output); parseErr == nil && outputNS != "" { + checks = append(checks, accessCheck{resource: resourceCM, verb: verbGet, namespace: outputNS}) + } + + // `configmaps: delete` is required only when this Deployer owns the + // output ConfigMap. Cleanup's staging-ConfigMap sweep and + // getSnapshotFromConfigMap's created-set record are both gated on + // Config.OwnsOutputConfigMap, so a caller who supplied their own + // `cm://` output URI never has a ConfigMap deleted on their behalf. + // This gate fails closed, so demanding an unconditional delete grant + // would block deployment for identities perfectly capable of the run + // they actually asked for. + if d.config.OwnsOutputConfigMap { + checks = append(checks, accessCheck{resource: resourceCM, verb: verbDelete, namespace: ns}) + } + return checks +} + +// modeSpecificChecks returns the half of the gate that depends on which +// ServiceAccount mode resolveServiceAccount settled on. +// +// Prefix mode: the caller creates and later deletes the full run-scoped RBAC +// set, so both verbs are required on all five kinds. +// +// Exact-ServiceAccount mode: aicr creates and deletes nothing, so none of +// those verbs is required of the caller. What IS required is that the +// operator already granted the agent's rules to the ServiceAccount they +// named — `aicr snapshot --add-roles-to-service-account` renders those +// manifests but applies nothing, so "you never applied them" is the +// failure this catches, at the gate rather than minutes later inside a pod. +func (d *Deployer) modeSpecificChecks() []accessCheck { + if !d.managesRBAC() { + return d.serviceAccountChecks(serviceAccountUsername(d.config.Namespace, d.existingServiceAccount())) + } + + ns := d.config.Namespace + verbs := []string{verbCreate, verbDelete} + checks := make([]accessCheck, 0, len(verbs)*rbacKindCount) + for _, verb := range verbs { + checks = append(checks, + accessCheck{resource: resourceServiceAccounts, verb: verb, namespace: ns}, + accessCheck{group: rbacAPIGroup, resource: resourceRoles, verb: verb, namespace: ns}, + accessCheck{group: rbacAPIGroup, resource: resourceRoleBindings, verb: verb, namespace: ns}, + accessCheck{group: rbacAPIGroup, resource: resourceClusterRoles, verb: verb}, + accessCheck{group: rbacAPIGroup, resource: resourceClusterRoleBindings, verb: verb}, + ) + } + return checks +} + +// serviceAccountChecks returns the questions asked of the agent's own +// ServiceAccount: exactly the rules ensureRole and ensureClusterRole grant +// in prefix mode, expanded to one check per (group, resource, verb). +// +// Deriving them from namespacedRules and clusterRules — the same two +// definitions the run-scoped Role and ClusterRole are built from, and the +// same ones BuildServiceAccountRoleManifests renders — is what keeps the +// gate from drifting away from what the agent actually needs. +// +// Issued only in exact-ServiceAccount mode. In prefix mode the +// ServiceAccount does not exist yet and its RBAC has not been created, so +// every answer would be a truthful "denied" about an identity that is about +// to be granted those very rules. +func (d *Deployer) serviceAccountChecks(subject string) []accessCheck { + checks := checksFromRules(namespacedRules(), d.config.Namespace, subject) + return append(checks, checksFromRules(clusterRules(d.config.DiscoverNetwork), "", subject)...) +} + +// checksFromRules expands PolicyRules into one accessCheck per +// (apiGroup, resource, verb) triple at the given scope. A "resource/sub" +// entry such as "pods/exec" is split so the review carries Subresource, +// which is how the apiserver evaluates it. +func checksFromRules(rules []rbacv1.PolicyRule, namespace, subject string) []accessCheck { + // Capacity is a hint, not a bound: most rules name one API group and a + // handful of verbs, so this lands close without a counting pass. + checks := make([]accessCheck, 0, len(rules)*rulesPerPolicyHint) + for _, rule := range rules { + for _, group := range rule.APIGroups { + for _, res := range rule.Resources { + resource, subresource, _ := strings.Cut(res, "/") + for _, verb := range rule.Verbs { + checks = append(checks, accessCheck{ + group: group, + resource: resource, + subresource: subresource, + verb: verb, + namespace: namespace, + subject: subject, + }) + } + } + } + } + return checks +} + +// dedupeChecks drops exact duplicates while preserving first-seen order, so +// the same question is never asked of the apiserver twice and never reported +// twice. Near-duplicates that differ only in scope (namespaced `pods: list` +// vs cluster-wide `pods: list`) are deliberately kept: an operator who +// applied the Role but not the ClusterRole fails exactly one of them. +func dedupeChecks(checks []accessCheck) []accessCheck { + seen := make(map[accessCheck]struct{}, len(checks)) + out := make([]accessCheck, 0, len(checks)) + for _, c := range checks { + if _, dup := seen[c]; dup { + continue + } + seen[c] = struct{}{} + out = append(out, c) + } + return out +} + +// runAccessChecks answers every check and returns the results in input +// order. An access review is a read-only query, so they fan out +// concurrently: N sequential reviews cost N round trips, one batch costs +// one. Concurrency is bounded because the expanded rule set can reach +// several dozen checks on a --discover-network run and there is no reason +// to open that many connections at once. +// +// A ServiceAccount check the apiserver declines to answer is recorded as +// Unverified rather than failing the run; see CheckPermissions. +func (d *Deployer) runAccessChecks(ctx context.Context, checks []accessCheck) ([]permissionCheck, error) { + results := make([]permissionCheck, len(checks)) g, gctx := errgroup.WithContext(ctx) - var mu sync.Mutex - var firstErr error - for i := range requiredChecks { - check := requiredChecks[i] + g.SetLimit(defaults.K8sAccessReviewConcurrency) + for i := range checks { + check := checks[i] g.Go(func() error { - allowed, reason, err := d.checkPermission(gctx, check.resource, check.verb, check.namespace) - if err != nil { + allowed, reason, reviewErr := d.reviewAccess(gctx, check) + if reviewErr != nil { + if check.subject != "" && isUnanswerable(reviewErr) { + results[i] = check.result(false, reviewErr.Error()) + results[i].Unverified = true + return nil + } code := errors.ErrCodeInternal - if errors.IsNetworkError(err) { + if errors.IsNetworkError(reviewErr) { code = errors.ErrCodeUnavailable } - mu.Lock() - if firstErr == nil { - firstErr = errors.Wrap(code, fmt.Sprintf("failed to check permission for %s %s", check.verb, check.resource), err) - } - mu.Unlock() - return err - } - results[i] = permissionCheck{ - Resource: check.resource, - Verb: check.verb, - Namespace: check.namespace, - Allowed: allowed, - Reason: reason, + return errors.Wrap(code, + fmt.Sprintf("failed to check whether %s may %q %s (%s)", + check.subjectLabel(), check.verb, check.resourceLabel(), check.scopeLabel()), + reviewErr) } + results[i] = check.result(allowed, reason) return nil }) } if err := g.Wait(); err != nil { - if firstErr != nil { - return nil, firstErr + return nil, err + } + return results, nil +} + +// reviewAccess asks the apiserver one authorization question. +// +// A caller check uses SelfSubjectAccessReview, which answers only for the +// identity in the kubeconfig. A ServiceAccount check must use +// SubjectAccessReview with that ServiceAccount as the subject — the agent +// pod runs as it, and a SelfSubjectAccessReview cannot answer for anyone but +// the caller. +// +// The apiserver error is returned unwrapped so the caller can classify it +// (Forbidden / not served / network) before deciding whether it is fatal. +func (d *Deployer) reviewAccess(ctx context.Context, check accessCheck) (bool, string, error) { + attrs := &authv1.ResourceAttributes{ + Verb: check.verb, + Group: check.group, + Resource: check.resource, + Subresource: check.subresource, + Namespace: check.namespace, + } + + if check.subject == "" { + review := &authv1.SelfSubjectAccessReview{ + Spec: authv1.SelfSubjectAccessReviewSpec{ResourceAttributes: attrs}, + } + result, err := d.clientset.AuthorizationV1().SelfSubjectAccessReviews().Create(ctx, review, metav1.CreateOptions{}) + if err != nil { + return false, "", err } - return nil, errors.Wrap(errors.ErrCodeInternal, "permission check failed", err) + return result.Status.Allowed, result.Status.Reason, nil } - checks := make([]permissionCheck, 0, len(results)) - checks = append(checks, results...) - var missingPermissions []string - for _, result := range results { - if !result.Allowed { - scope := "cluster-scoped" - if result.Namespace != "" { - scope = fmt.Sprintf("namespace %q", result.Namespace) - } - missingPermissions = append(missingPermissions, - fmt.Sprintf("%s %s (%s)", result.Verb, result.Resource, scope)) + review := &authv1.SubjectAccessReview{ + Spec: authv1.SubjectAccessReviewSpec{ + ResourceAttributes: attrs, + User: check.subject, + Groups: serviceAccountGroups(d.config.Namespace), + }, + } + result, err := d.clientset.AuthorizationV1().SubjectAccessReviews().Create(ctx, review, metav1.CreateOptions{}) + if err != nil { + return false, "", err + } + return result.Status.Allowed, result.Status.Reason, nil +} + +// isUnanswerable reports whether err means the apiserver would not answer a +// SubjectAccessReview at all, as opposed to answering "denied". +// +// Creating a SubjectAccessReview is itself a privilege, and some clusters do +// not serve the endpoint. Neither says anything about the ServiceAccount's +// permissions, so neither may be read as a pass OR as a failure — the run +// continues with the gap reported (see warnUnverified). +func isUnanswerable(err error) bool { + return apierrors.IsForbidden(err) || + apierrors.IsUnauthorized(err) || + apierrors.IsNotFound(err) || + apierrors.IsMethodNotSupported(err) +} + +// serviceAccountUsername returns the username the apiserver authenticates a +// ServiceAccount as, which is the SubjectAccessReview subject that answers +// for the agent pod. +func serviceAccountUsername(namespace, name string) string { + return serviceAccountUserPrefix + namespace + ":" + name +} + +// serviceAccountGroups returns the virtual groups every ServiceAccount in +// namespace belongs to, so a SubjectAccessReview reflects grants made to +// those groups rather than only to the individual subject. +func serviceAccountGroups(namespace string) []string { + return []string{groupServiceAccounts, groupServiceAccountsPrefix + namespace, groupAuthenticated} +} + +// result converts an answered check into the reportable form. +func (c accessCheck) result(allowed bool, reason string) permissionCheck { + return permissionCheck{ + Group: c.group, + Resource: c.resource, + Subresource: c.subresource, + Verb: c.verb, + Namespace: c.namespace, + Subject: c.subject, + Allowed: allowed, + Reason: reason, + } +} + +// subjectLabel names whose permissions this check answers for. +func (c accessCheck) subjectLabel() string { + return subjectLabel(c.subject) +} + +// resourceLabel renders "[/][.]", the spelling +// an operator would use in a PolicyRule. +func (c accessCheck) resourceLabel() string { + return resourceLabel(c.resource, c.subresource, c.group) +} + +// scopeLabel renders the scope the check was evaluated at. +func (c accessCheck) scopeLabel() string { + return scopeLabel(c.namespace) +} + +func subjectLabel(subject string) string { + if subject == "" { + return callerSubjectLabel + } + return fmt.Sprintf("agent ServiceAccount %q", subject) +} + +func resourceLabel(resource, subresource, group string) string { + name := resource + if subresource != "" { + name += "/" + subresource + } + if group != "" { + name += "." + group + } + return name +} + +func scopeLabel(namespace string) string { + if namespace == "" { + return "cluster-scoped" + } + return fmt.Sprintf("namespace %q", namespace) +} + +// callerMayReadServiceAccounts reports whether the caller's +// `serviceaccounts: get` check came back allowed. It is the one check that +// gates the rest of the pre-flight rather than merely joining it: without it +// the ServiceAccount mode is unknowable. +func callerMayReadServiceAccounts(results []permissionCheck) bool { + for _, r := range results { + if r.Subject == "" && r.Resource == resourceServiceAccounts && r.Verb == verbGet && r.Allowed { + return true + } + } + return false +} + +// hasMissing reports whether any check came back denied. An Unverified entry +// does not count: nothing was learned about it either way. +func hasMissing(results []permissionCheck) bool { + for _, r := range results { + if !r.Allowed && !r.Unverified { + return true } } + return false +} - if len(missingPermissions) > 0 { - return checks, errors.New(errors.ErrCodeUnauthorized, fmt.Sprintf("missing required permissions:\n - %s", - strings.Join(missingPermissions, "\n - "))) +// missingPermissionsError renders every denied check as one actionable line +// — subject, verb, resource, scope, and the authorizer's reason when it gave +// one — followed by hint. Reporting all of them together is the point: an +// operator fixing permissions should need one run, not one run per verb. +func missingPermissionsError(results []permissionCheck, hint string) error { + var missing []string + for _, r := range results { + if r.Allowed || r.Unverified { + continue + } + line := fmt.Sprintf("%s cannot %q %s (%s)", + subjectLabel(r.Subject), r.Verb, + resourceLabel(r.Resource, r.Subresource, r.Group), scopeLabel(r.Namespace)) + if r.Reason != "" { + line += ": " + r.Reason + } + missing = append(missing, line) } + msg := fmt.Sprintf("missing required permissions:\n - %s", strings.Join(missing, "\n - ")) + if hint != "" { + msg += "\n\n" + hint + } + return errors.New(errors.ErrCodeUnauthorized, msg) +} - return checks, nil +// unresolvableModeHint explains why the pre-flight stopped early when the +// caller cannot read ServiceAccounts, and why that is not something aicr +// works around. +func unresolvableModeHint(namespace string) string { + return fmt.Sprintf( + "Reading ServiceAccounts in namespace %q is required before anything else: it is what\n"+ + "decides whether --service-account-name names an existing ServiceAccount to run as\n"+ + "verbatim, or is a prefix for one this run creates. Without it, a ServiceAccount you\n"+ + "named explicitly would be silently replaced by a generated one carrying none of its\n"+ + "cloud credentials (IRSA / Workload Identity), so the run stops instead.\n"+ + "The remaining, mode-specific permissions were not evaluated; fix the above and re-run.", + namespace) } -// checkPermission checks if the current user can perform the specified action. -func (d *Deployer) checkPermission(ctx context.Context, resource, verb, namespace string) (bool, string, error) { - review := &authv1.SelfSubjectAccessReview{ - Spec: authv1.SelfSubjectAccessReviewSpec{ - ResourceAttributes: &authv1.ResourceAttributes{ - Verb: verb, - Resource: resource, - Namespace: namespace, - }, - }, +// remediationHints returns the operator-facing next step for whichever kind +// of failure occurred, so the message can be acted on without reading the +// source. +func (d *Deployer) remediationHints(results []permissionCheck) string { + var callerMissing, subjectMissing bool + for _, r := range results { + if r.Allowed || r.Unverified { + continue + } + if r.Subject == "" { + callerMissing = true + } else { + subjectMissing = true + } } - result, err := d.clientset.AuthorizationV1().SelfSubjectAccessReviews().Create(ctx, review, metav1.CreateOptions{}) - if err != nil { - return false, "", errors.Wrap(errors.ErrCodeInternal, "failed to create SelfSubjectAccessReview", err) + var hints []string + if subjectMissing { + hints = append(hints, fmt.Sprintf( + "The agent pod runs as ServiceAccount %q, and aicr manages no permissions for a\n"+ + "ServiceAccount it did not create. Generate the RBAC that grants it what the agent\n"+ + "needs, review it, and apply it yourself:\n"+ + " aicr snapshot --add-roles-to-service-account %s --namespace %s\n"+ + " kubectl apply -f snapshot-rbac-/", + d.existingServiceAccount(), d.existingServiceAccount(), d.config.Namespace)) } + if callerMissing && d.managesRBAC() { + hints = append(hints, ""+ + "This run creates and deletes its own run-scoped RBAC, which is why both create and\n"+ + "delete are required: cleanup always runs, and an identity that can create but not\n"+ + "delete leaks a ServiceAccount, Role, RoleBinding, ClusterRole and ClusterRoleBinding\n"+ + "on every run. To need none of those verbs, pre-create a ServiceAccount and pass its\n"+ + "exact name with --service-account-name.") + } + return strings.Join(hints, "\n\n") +} - return result.Status.Allowed, result.Status.Reason, nil +// warnUnverified reports, once, that the agent ServiceAccount's own +// permissions could not be checked. +// +// Creating a SubjectAccessReview is itself a privilege. A caller who lacks +// it learns nothing about the ServiceAccount, and silently dropping the +// check would be the same defect class the `serviceaccounts: get` gate above +// closes — a missing answer read as a passing one. The run continues, +// because the agent still fails visibly inside the pod, but the operator is +// told the gate did not cover it and why. +func (d *Deployer) warnUnverified(results []permissionCheck) { + var count int + var reason string + for _, r := range results { + if !r.Unverified { + continue + } + count++ + if reason == "" { + reason = r.Reason + } + } + if count == 0 { + return + } + slog.Warn("could not verify the agent ServiceAccount's own permissions; continuing, but a missing rule will surface as an in-pod failure minutes from now instead of here", + slog.String(attrServiceAccount, d.existingServiceAccount()), + slog.String(attrNamespace, d.config.Namespace), + slog.String(attrRunID, d.config.RunID), + slog.Int("uncheckedRules", count), + slog.String("cause", reason), + slog.String("remedy", "grant the caller 'create subjectaccessreviews.authorization.k8s.io', or verify by hand with: kubectl auth can-i --list --as "+serviceAccountUsername(d.config.Namespace, d.existingServiceAccount()))) } diff --git a/pkg/k8s/agent/permissions_test.go b/pkg/k8s/agent/permissions_test.go index f45ab1705..87969b583 100644 --- a/pkg/k8s/agent/permissions_test.go +++ b/pkg/k8s/agent/permissions_test.go @@ -16,140 +16,850 @@ package agent import ( "context" + stderrors "errors" + "fmt" "strings" + "sync" "testing" + aicrerrors "github.com/NVIDIA/aicr/pkg/errors" authv1 "k8s.io/api/authorization/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes/fake" k8stesting "k8s.io/client-go/testing" ) -func TestCheckPermissions(t *testing.T) { +// exactSAName is the operator-provisioned ServiceAccount the +// exact-ServiceAccount-mode cases name verbatim. +const exactSAName = "irsa-snapshotter" + +// The two access-review resources the gate posts to. They are "created" +// over the REST API but persist nothing — the create IS the authorization +// query — which is why the write guard below exempts them. +const ( + selfReviewResource = "selfsubjectaccessreviews" + subjectReviewResource = "subjectaccessreviews" +) + +// askedAccess is one authorization question a reactor observed, flattened +// out of whichever review kind carried it. subject is "" when the question +// came from a SelfSubjectAccessReview (i.e. it was asked about the caller). +type askedAccess struct { + subject string + group string + resource string + subresource string + verb string + namespace string +} + +// String renders a question the way an assertion failure should read. +func (a askedAccess) String() string { + return fmt.Sprintf("%s %s (%s) subject=%q", a.verb, + resourceLabel(a.resource, a.subresource, a.group), scopeLabel(a.namespace), a.subject) +} + +// reviewRecorder collects every question the gate asked. CheckPermissions +// fans its checks out over an errgroup, so the reactors run on worker +// goroutines and every field must be mutex-guarded. +type reviewRecorder struct { + mu sync.Mutex + asked []askedAccess +} + +func (r *reviewRecorder) record(a askedAccess) { + r.mu.Lock() + defer r.mu.Unlock() + r.asked = append(r.asked, a) +} + +// questions returns a copy of what was asked, taken under lock. +func (r *reviewRecorder) questions() []askedAccess { + r.mu.Lock() + defer r.mu.Unlock() + out := make([]askedAccess, len(r.asked)) + copy(out, r.asked) + return out +} + +// asked reports whether a question matching pred was put to the apiserver. +func (r *reviewRecorder) asked1(pred func(askedAccess) bool) bool { + for _, q := range r.questions() { + if pred(q) { + return true + } + } + return false +} + +// installReviewReactors answers every Self/SubjectAccessReview with allow(q) +// and records the question. A nil allow permits everything. +// +// Failures are reported with t.Errorf and handed back as the reactor's error +// rather than with t.Fatalf: the reactor runs on an errgroup worker, and +// Goexit there would leave the group waiting on a goroutine that never +// returns. +func installReviewReactors(t *testing.T, cs *fake.Clientset, allow func(askedAccess) bool) *reviewRecorder { + t.Helper() + rec := &reviewRecorder{} + if allow == nil { + allow = func(askedAccess) bool { return true } + } + + answer := func(action k8stesting.Action) (askedAccess, bool, error) { + create, ok := action.(k8stesting.CreateAction) + if !ok { + err := fmt.Errorf("action %T is not a CreateAction", action) + t.Error(err) + return askedAccess{}, false, err + } + var attrs *authv1.ResourceAttributes + var subject string + switch obj := create.GetObject().(type) { + case *authv1.SelfSubjectAccessReview: + attrs = obj.Spec.ResourceAttributes + case *authv1.SubjectAccessReview: + attrs, subject = obj.Spec.ResourceAttributes, obj.Spec.User + if subject == "" { + err := stderrors.New("SubjectAccessReview carries no User; it would answer for nobody") + t.Error(err) + return askedAccess{}, false, err + } + default: + err := fmt.Errorf("object %T is not an access review", create.GetObject()) + t.Error(err) + return askedAccess{}, false, err + } + if attrs == nil { + err := stderrors.New("access review carries no ResourceAttributes") + t.Error(err) + return askedAccess{}, false, err + } + q := askedAccess{ + subject: subject, + group: attrs.Group, + resource: attrs.Resource, + subresource: attrs.Subresource, + verb: attrs.Verb, + namespace: attrs.Namespace, + } + rec.record(q) + return q, allow(q), nil + } + + cs.PrependReactor(verbCreate, selfReviewResource, func(action k8stesting.Action) (bool, runtime.Object, error) { + q, allowed, err := answer(action) + if err != nil { + return true, nil, err + } + return true, &authv1.SelfSubjectAccessReview{ + Status: authv1.SubjectAccessReviewStatus{Allowed: allowed, Reason: "caller policy for " + q.String()}, + }, nil + }) + cs.PrependReactor(verbCreate, subjectReviewResource, func(action k8stesting.Action) (bool, runtime.Object, error) { + q, allowed, err := answer(action) + if err != nil { + return true, nil, err + } + return true, &authv1.SubjectAccessReview{ + Status: authv1.SubjectAccessReviewStatus{Allowed: allowed, Reason: "ServiceAccount policy for " + q.String()}, + }, nil + }) + return rec +} + +// seedServiceAccount pre-creates the operator-provisioned ServiceAccount +// that puts a Deployer into exact-ServiceAccount mode. Its namespace and +// name are fixed rather than parameters: every caller wants the one +// ServiceAccount that testNamespace/exactSAName names, and passing the same +// two constants at each call site is what unparam flags. +func seedServiceAccount(t *testing.T, cs *fake.Clientset) { + t.Helper() + if _, err := cs.CoreV1().ServiceAccounts(testNamespace).Create(context.Background(), &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{Name: exactSAName, Namespace: testNamespace}, + }, metav1.CreateOptions{}); err != nil { + t.Fatalf("seeding ServiceAccount %s/%s: %v", testNamespace, exactSAName, err) + } +} + +// hasCheck reports whether results holds an answered check matching pred. +func hasCheck(results []permissionCheck, pred func(permissionCheck) bool) bool { + for _, r := range results { + if pred(r) { + return true + } + } + return false +} + +// isCallerRBACCheck matches a caller-side check on one of the five RBAC +// kinds the run creates and deletes in prefix mode. +func isCallerRBACCheck(p permissionCheck) bool { + if p.Subject != "" { + return false + } + switch p.Resource { + case resourceRoles, resourceRoleBindings, resourceClusterRoles, resourceClusterRoleBindings: + return true + case resourceServiceAccounts: + return p.Verb == verbCreate || p.Verb == verbDelete + default: + return false + } +} + +// TestCheckPermissions_PrefixModeRequiresRBACCreateAndDelete pins hole B: the +// deferred Cleanup is registered before Deploy and always runs, issuing a +// UID-pinned delete for every RBAC object the run created. An identity with +// create-but-not-delete used to pass a green pre-flight, deploy, and then +// leak a full run-scoped RBAC set — cluster-scoped objects included — once +// per run. +func TestCheckPermissions_PrefixModeRequiresRBACCreateAndDelete(t *testing.T) { + // Every (resource, verb, cluster-scoped?) triple prefix mode must + // demand of the caller. Each is denied on its own to prove it is + // individually load-bearing rather than incidentally covered. + required := []struct { + resource string + verb string + cluster bool + }{ + {resourceServiceAccounts, verbCreate, false}, + {resourceServiceAccounts, verbDelete, false}, + {resourceRoles, verbCreate, false}, + {resourceRoles, verbDelete, false}, + {resourceRoleBindings, verbCreate, false}, + {resourceRoleBindings, verbDelete, false}, + {resourceClusterRoles, verbCreate, true}, + {resourceClusterRoles, verbDelete, true}, + {resourceClusterRoleBindings, verbCreate, true}, + {resourceClusterRoleBindings, verbDelete, true}, + } + + for _, req := range required { + t.Run(req.verb+" "+req.resource, func(t *testing.T) { + clientset := fake.NewClientset() + rec := installReviewReactors(t, clientset, func(q askedAccess) bool { + return q.resource != req.resource || q.verb != req.verb + }) + + d := NewDeployer(clientset, Config{ + Namespace: testNamespace, + ServiceAccountName: exactSAName, // nothing seeded: prefix mode + RunID: testRunID, + }) + results, err := d.CheckPermissions(context.Background()) + if err == nil { + t.Fatalf("CheckPermissions() error = nil; %s %s must be required in prefix mode", req.verb, req.resource) + } + if !stderrors.Is(err, aicrerrors.New(aicrerrors.ErrCodeUnauthorized, "")) { + t.Errorf("error code = %v, want ErrCodeUnauthorized", err) + } + + wantScope := scopeLabel(testNamespace) + if req.cluster { + wantScope = scopeLabel("") + } + wantLine := fmt.Sprintf("%s cannot %q %s (%s)", callerSubjectLabel, req.verb, + resourceLabel(req.resource, "", rbacGroupFor(req.resource)), wantScope) + if !strings.Contains(err.Error(), wantLine) { + t.Errorf("error = %v\nwant a line containing %q", err, wantLine) + } + carriesDenied := func(p permissionCheck) bool { + return p.Resource == req.resource && p.Verb == req.verb && !p.Allowed + } + if !hasCheck(results, carriesDenied) { + t.Errorf("results do not carry the denied %s %s check", req.verb, req.resource) + } + // The gate must have resolved the mode first, which means it + // asked whether it may read ServiceAccounts. + askedSAGet := func(q askedAccess) bool { + return q.subject == "" && q.resource == resourceServiceAccounts && q.verb == verbGet + } + if !rec.asked1(askedSAGet) { + t.Error("gate never asked for `serviceaccounts: get`, so it cannot have resolved the mode") + } + }) + } +} + +// rbacGroupFor returns the API group a resource lives in, for building the +// exact message line the gate is expected to render. +func rbacGroupFor(resource string) string { + if resource == resourceServiceAccounts { + return "" + } + return rbacAPIGroup +} + +// TestCheckPermissions_ExactModeSkipsCallerRBACVerbs is the other half of the +// mode split. In exact-ServiceAccount mode aicr creates and deletes no RBAC +// at all, so demanding create/delete on the five kinds would fail operators +// who legitimately hold none of those grants — the very operators the exact +// mode exists for. +func TestCheckPermissions_ExactModeSkipsCallerRBACVerbs(t *testing.T) { + clientset := fake.NewClientset() + seedServiceAccount(t, clientset) + + // Deny every caller-side RBAC verb outright. A correct gate never asks. + rec := installReviewReactors(t, clientset, func(q askedAccess) bool { + if q.subject != "" { + return true + } + switch q.resource { + case resourceRoles, resourceRoleBindings, resourceClusterRoles, resourceClusterRoleBindings: + return false + case resourceServiceAccounts: + return q.verb == verbGet + default: + return true + } + }) + + d := NewDeployer(clientset, Config{ + Namespace: testNamespace, + ServiceAccountName: exactSAName, + RunID: testRunID, + }) + results, err := d.CheckPermissions(context.Background()) + if err != nil { + t.Fatalf("CheckPermissions() error = %v, want nil (exact mode needs no RBAC create/delete)", err) + } + if d.managesRBAC() { + t.Fatal("managesRBAC() = true; the seeded ServiceAccount should have put the run in exact mode") + } + if hasCheck(results, isCallerRBACCheck) { + t.Error("gate demanded a caller RBAC verb in exact-ServiceAccount mode") + } + askedCallerRBAC := func(q askedAccess) bool { + return q.subject == "" && q.resource == resourceClusterRoles + } + if rec.asked1(askedCallerRBAC) { + t.Error("gate asked the apiserver about clusterroles in exact-ServiceAccount mode") + } + + // It must instead have asked the ServiceAccount's own questions, and + // asked them of the ServiceAccount rather than of the caller. + subject := serviceAccountUsername(testNamespace, exactSAName) + askedClusterRoles := func(q askedAccess) bool { + return q.subject == subject && q.resource == resourceNodes && q.verb == verbList + } + if !rec.asked1(askedClusterRoles) { + t.Errorf("gate never asked whether %s may list nodes", subject) + } +} + +// TestCheckPermissions_ServiceAccountGetIsRequired pins hole A at the gate. +// Without `serviceaccounts: get` the ServiceAccount mode is unknowable, and +// the old behavior — treat an unreadable ServiceAccount as a name prefix — +// silently ran an operator's explicitly-named IRSA / Workload Identity +// account as a generated one carrying none of its cloud annotations. +func TestCheckPermissions_ServiceAccountGetIsRequired(t *testing.T) { + clientset := fake.NewClientset() + seedServiceAccount(t, clientset) + rec := installReviewReactors(t, clientset, func(q askedAccess) bool { + return q.resource != resourceServiceAccounts || q.verb != verbGet + }) + + d := NewDeployer(clientset, Config{ + Namespace: testNamespace, + ServiceAccountName: exactSAName, + RunID: testRunID, + }) + results, err := d.CheckPermissions(context.Background()) + if err == nil { + t.Fatal("CheckPermissions() error = nil; a caller that cannot read ServiceAccounts must fail the gate") + } + if !stderrors.Is(err, aicrerrors.New(aicrerrors.ErrCodeUnauthorized, "")) { + t.Errorf("error code = %v, want ErrCodeUnauthorized", err) + } + for _, want := range []string{ + fmt.Sprintf("%s cannot %q %s", callerSubjectLabel, verbGet, resourceServiceAccounts), + "mode-specific permissions were not evaluated", + } { + if !strings.Contains(err.Error(), want) { + t.Errorf("error = %v\nwant it to contain %q", err, want) + } + } + + // It must NOT have silently downgraded to prefix mode and carried on: + // no mode-specific question may have been asked, and the run must not + // have adopted the seeded ServiceAccount either. + if hasCheck(results, isCallerRBACCheck) { + t.Error("gate evaluated prefix-mode RBAC verbs despite being unable to resolve the mode") + } + if rec.asked1(func(q askedAccess) bool { return q.subject != "" }) { + t.Error("gate issued a SubjectAccessReview despite being unable to resolve the mode") + } + if d.existingServiceAccount() != "" { + t.Errorf("existingServiceAccount() = %q, want \"\" (nothing may be resolved)", d.existingServiceAccount()) + } +} + +// TestCheckPermissions_ServiceAccountSubjectFailureNamesTheSubject covers +// requirement 3: the agent pod runs as the ServiceAccount, so the gate must +// verify that identity's own permissions with a SubjectAccessReview, not the +// caller's with a SelfSubjectAccessReview. In exact mode aicr grants nothing +// and the operator was supposed to have applied +// `--add-roles-to-service-account`; "you never applied those manifests" is +// far better caught here than in a pod minutes later. +func TestCheckPermissions_ServiceAccountSubjectFailureNamesTheSubject(t *testing.T) { + clientset := fake.NewClientset() + seedServiceAccount(t, clientset) + installReviewReactors(t, clientset, func(q askedAccess) bool { + // The caller is fully privileged; the ServiceAccount is missing + // exactly the cluster-scoped node read the agent cannot work without. + return q.subject == "" || q.resource != resourceNodes + }) + + d := NewDeployer(clientset, Config{ + Namespace: testNamespace, + ServiceAccountName: exactSAName, + RunID: testRunID, + }) + results, err := d.CheckPermissions(context.Background()) + if err == nil { + t.Fatal("CheckPermissions() error = nil; a ServiceAccount missing `nodes: list` must fail the gate") + } + + subject := serviceAccountUsername(testNamespace, exactSAName) + for _, want := range []string{ + fmt.Sprintf("agent ServiceAccount %q cannot %q %s (%s)", subject, verbList, resourceNodes, scopeLabel("")), + "--add-roles-to-service-account " + exactSAName, + } { + if !strings.Contains(err.Error(), want) { + t.Errorf("error = %v\nwant it to contain %q", err, want) + } + } + // A caller-scoped answer must never be substituted for the subject's. + askedSubjectNodes := func(p permissionCheck) bool { + return p.Subject == subject && p.Resource == resourceNodes && !p.Allowed && !p.Unverified + } + if !hasCheck(results, askedSubjectNodes) { + t.Error("results carry no denied ServiceAccount-subject check for nodes") + } +} + +// TestCheckPermissions_SubjectAccessReviewForbiddenReportsAndContinues covers +// the meta-permission honestly. Creating a SubjectAccessReview is itself a +// privilege; a caller without it learns nothing about the ServiceAccount. +// Silently skipping would be the same defect class as hole A — a missing +// answer read as a passing one — so the gate records the checks as +// unverified, warns, and lets the run proceed to fail visibly in-pod. +func TestCheckPermissions_SubjectAccessReviewForbiddenReportsAndContinues(t *testing.T) { + clientset := fake.NewClientset() + seedServiceAccount(t, clientset) + installReviewReactors(t, clientset, nil) + // Prepended after installReviewReactors, so it wins for this resource. + clientset.PrependReactor(verbCreate, subjectReviewResource, func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewForbidden( + schema.GroupResource{Group: "authorization.k8s.io", Resource: subjectReviewResource}, "", + stderrors.New(`User "snapshot-runner" cannot create resource "subjectaccessreviews"`)) + }) + + logs := captureLogs(t) + d := NewDeployer(clientset, Config{ + Namespace: testNamespace, + ServiceAccountName: exactSAName, + RunID: testRunID, + }) + results, err := d.CheckPermissions(context.Background()) + if err != nil { + t.Fatalf("CheckPermissions() error = %v, want nil (an unanswerable check is not a failed one)", err) + } + + var unverified int + for _, r := range results { + if r.Unverified { + unverified++ + if r.Subject == "" { + t.Error("a caller check was marked unverified; only ServiceAccount checks may be") + } + } + } + if unverified == 0 { + t.Fatal("no check was marked Unverified; the gate silently dropped the ServiceAccount's permissions") + } + + for _, want := range []string{ + "could not verify the agent ServiceAccount's own permissions", + "cannot create resource \\\"subjectaccessreviews\\\"", + "kubectl auth can-i --list --as " + serviceAccountUsername(testNamespace, exactSAName), + } { + if !strings.Contains(logs.String(), want) { + t.Errorf("warning log = %s\nwant it to contain %q", logs.String(), want) + } + } +} + +// TestCheckPermissions_ReportsEveryMissingPermissionAtOnce pins requirement +// 4: an operator fixing permissions should get the complete list in one run +// rather than discovering them one denial at a time. +func TestCheckPermissions_ReportsEveryMissingPermissionAtOnce(t *testing.T) { + clientset := fake.NewClientset() + denied := map[string]string{ + resourceClusterRoles: verbDelete, + resourceClusterRoleBindings: verbDelete, + resourceJobs: verbCreate, + resourcePods: verbWatch, + } + installReviewReactors(t, clientset, func(q askedAccess) bool { + return denied[q.resource] != q.verb + }) + + d := NewDeployer(clientset, Config{ + Namespace: testNamespace, + RunID: testRunID, + }) + _, err := d.CheckPermissions(context.Background()) + if err == nil { + t.Fatal("CheckPermissions() error = nil, want the four denied permissions reported") + } + for resource, verb := range denied { + line := fmt.Sprintf("cannot %q %s", verb, resource) + if !strings.Contains(err.Error(), line) { + t.Errorf("error = %v\nwant it to report %q too", err, line) + } + } + // Prefix mode is what this Deployer is in, so the remediation must point + // at the exact-ServiceAccount escape hatch rather than at manifests. + if !strings.Contains(err.Error(), "--service-account-name") { + t.Errorf("error = %v\nwant prefix-mode remediation naming --service-account-name", err) + } +} + +// TestCheckPermissions_IssuesNoWriteBeforeTheGateCloses is the structural +// guarantee behind resolving the ServiceAccount inside the gate: resolving +// is a read, and fail-before-mutate is about not WRITING before validation. +// A reactor rejects every create/update/patch/delete of a real object, so any +// regression that mutates during the pre-flight fails here rather than in +// production. +func TestCheckPermissions_IssuesNoWriteBeforeTheGateCloses(t *testing.T) { tests := []struct { - name string - allowed bool - wantErr bool - errContains string + name string + exact bool + allow func(askedAccess) bool + wantErr bool }{ + {name: "prefix mode, all granted"}, + {name: "exact mode, all granted", exact: true}, { - name: "all permissions allowed", - allowed: true, - wantErr: false, + name: "prefix mode, denied", + allow: func(q askedAccess) bool { return q.resource != resourceClusterRoles }, + wantErr: true, }, { - name: "permissions denied", - allowed: false, - wantErr: true, - errContains: "missing required permissions", + name: "exact mode, ServiceAccount denied", + exact: true, + allow: func(q askedAccess) bool { return q.subject == "" }, + wantErr: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { clientset := fake.NewClientset() + if tt.exact { + // Seeded through the tracker before the guard is armed. + seedServiceAccount(t, clientset) + } + installReviewReactors(t, clientset, tt.allow) - // Mock SelfSubjectAccessReview responses - clientset.PrependReactor("create", "selfsubjectaccessreviews", func(action k8stesting.Action) (bool, runtime.Object, error) { - return true, &authv1.SelfSubjectAccessReview{ - Status: authv1.SubjectAccessReviewStatus{ - Allowed: tt.allowed, - Reason: "test reason", - }, - }, nil + var mu sync.Mutex + var writes []string + clientset.PrependReactor("*", "*", func(action k8stesting.Action) (bool, runtime.Object, error) { + switch action.GetVerb() { + case verbCreate, verbUpdate, verbPatch, verbDelete, "deletecollection": + default: + return false, nil, nil + } + // Access reviews are "created" over the REST API but + // persist nothing — they are the authorization query + // itself. Everything else is a real mutation. + if res := action.GetResource().Resource; res == selfReviewResource || res == subjectReviewResource { + return false, nil, nil + } + write := action.GetVerb() + " " + action.GetResource().Resource + mu.Lock() + writes = append(writes, write) + mu.Unlock() + return true, nil, fmt.Errorf("pre-flight issued a write: %s", write) }) - deployer := NewDeployer(clientset, Config{ - Namespace: "gpu-operator", - ServiceAccountName: "aicr", - JobName: "aicr", + d := NewDeployer(clientset, Config{ + Namespace: testNamespace, + ServiceAccountName: exactSAName, + RunID: testRunID, + OwnsOutputConfigMap: true, + Output: "cm://" + testNamespace + "/" + StagingConfigMapName(testRunID), + }) + _, err := d.CheckPermissions(context.Background()) + if (err != nil) != tt.wantErr { + t.Fatalf("CheckPermissions() error = %v, wantErr %v", err, tt.wantErr) + } + + mu.Lock() + defer mu.Unlock() + if len(writes) > 0 { + t.Errorf("pre-flight mutated the cluster before the gate closed: %v", writes) + } + }) + } +} + +// TestCheckPermissions_ConfigMapDeleteGatedOnOwnership pins the gate on the +// `configmaps: delete` verb. CheckPermissions fails closed, so an +// unconditional entry would make Deploy return ErrCodeUnauthorized at Step 0 +// for a caller who supplied their own `cm://` output URI — a run that never +// deletes a ConfigMap at all, because both Cleanup's staging sweep and +// getSnapshotFromConfigMap's created-set record are gated on +// Config.OwnsOutputConfigMap. +func TestCheckPermissions_ConfigMapDeleteGatedOnOwnership(t *testing.T) { + tests := []struct { + name string + ownsOutput bool + wantCMDeleteCheck bool + wantErr bool + }{ + { + name: "owns output ConfigMap requires configmaps delete", + ownsOutput: true, + wantCMDeleteCheck: true, + // The identity below is denied exactly `configmaps: delete`, + // so a required check makes the whole pre-flight fail. + wantErr: true, + }, + { + name: "caller-supplied output ConfigMap does not require configmaps delete", + ownsOutput: false, + wantCMDeleteCheck: false, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clientset := fake.NewClientset() + // Deny only `configmaps: delete`; allow everything else. This + // models the least-privilege identity the gate exists for. + installReviewReactors(t, clientset, func(q askedAccess) bool { + return q.resource != resourceCM || q.verb != verbDelete }) - ctx := context.Background() - checks, err := deployer.CheckPermissions(ctx) + deployer := NewDeployer(clientset, Config{ + Namespace: testNamespace, + RunID: testRunID, + OwnsOutputConfigMap: tt.ownsOutput, + }) + checks, err := deployer.CheckPermissions(context.Background()) if (err != nil) != tt.wantErr { - t.Errorf("CheckPermissions() error = %v, wantErr %v", err, tt.wantErr) - return + t.Fatalf("CheckPermissions() error = %v, wantErr %v", err, tt.wantErr) } - if tt.wantErr && err != nil && tt.errContains != "" { - if !strings.Contains(err.Error(), tt.errContains) { - t.Errorf("CheckPermissions() error = %v, should contain %q", err, tt.errContains) - } + gotCMDelete := hasCheck(checks, func(p permissionCheck) bool { + return p.Resource == resourceCM && p.Verb == verbDelete + }) + if gotCMDelete != tt.wantCMDeleteCheck { + t.Errorf("configmaps delete check present = %v, want %v", gotCMDelete, tt.wantCMDeleteCheck) } + }) + } +} - if !tt.wantErr && len(checks) == 0 { - t.Error("CheckPermissions() returned no checks") +// TestCheckPermissions_AllGrantedPassesBothModes is the happy path, and also +// asserts the gate covers the run's non-RBAC work — the Job it creates and +// waits on, the pods it discovers and streams logs from, and the ConfigMap +// it reads the snapshot back out of. +func TestCheckPermissions_AllGrantedPassesBothModes(t *testing.T) { + tests := []struct { + name string + exact bool + }{ + {name: "prefix mode"}, + {name: "exact-ServiceAccount mode", exact: true}, + } + + want := []askedAccess{ + {group: batchAPIGroup, resource: resourceJobs, verb: verbCreate, namespace: testNamespace}, + {group: batchAPIGroup, resource: resourceJobs, verb: verbWatch, namespace: testNamespace}, + {group: batchAPIGroup, resource: resourceJobs, verb: verbDelete, namespace: testNamespace}, + {resource: resourcePods, verb: verbList, namespace: testNamespace}, + {resource: resourcePods, subresource: subresourceLog, verb: verbGet, namespace: testNamespace}, + {resource: resourceCM, verb: verbGet, namespace: testNamespace}, + {resource: resourceServiceAccounts, verb: verbGet, namespace: testNamespace}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clientset := fake.NewClientset() + if tt.exact { + seedServiceAccount(t, clientset) } + rec := installReviewReactors(t, clientset, nil) - // Verify all checks match expected result - for _, check := range checks { - if check.Allowed != tt.allowed { - t.Errorf("Check %s %s: got allowed=%v, want %v", check.Verb, check.Resource, check.Allowed, tt.allowed) + d := NewDeployer(clientset, Config{ + Namespace: testNamespace, + ServiceAccountName: exactSAName, + RunID: testRunID, + }) + results, err := d.CheckPermissions(context.Background()) + if err != nil { + t.Fatalf("CheckPermissions() error = %v, want nil", err) + } + if len(results) == 0 { + t.Fatal("CheckPermissions() returned no checks") + } + for _, r := range results { + if !r.Allowed { + t.Errorf("check %s %s (%s) denied under an allow-all policy", r.Verb, r.Resource, r.Namespace) + } + } + for _, q := range want { + if !rec.asked1(func(got askedAccess) bool { return got == q }) { + t.Errorf("gate never asked: %s", q) } } }) } } -func TestCheckPermission(t *testing.T) { +// TestReviewAccess covers the two review kinds one question at a time: a +// caller check must go out as a SelfSubjectAccessReview and a ServiceAccount +// check as a SubjectAccessReview naming that subject, because the former +// cannot answer for anyone but the caller. +func TestReviewAccess(t *testing.T) { + subject := serviceAccountUsername(testNamespace, exactSAName) + tests := []struct { - name string - resource string - verb string - namespace string - allowed bool - reason string + name string + check accessCheck + allowed bool }{ { - name: "allowed permission", - resource: "jobs", - verb: "create", - namespace: "gpu-operator", - allowed: true, - reason: "user has permission", + name: "caller check allowed", + check: accessCheck{group: batchAPIGroup, resource: resourceJobs, verb: verbCreate, namespace: testNamespace}, + allowed: true, }, { - name: "denied permission", - resource: "jobs", - verb: "create", - namespace: "gpu-operator", - allowed: false, - reason: "user lacks permission", + name: "caller check denied", + check: accessCheck{group: batchAPIGroup, resource: resourceJobs, verb: verbCreate, namespace: testNamespace}, + }, + { + name: "ServiceAccount subresource check allowed", + check: accessCheck{resource: resourcePods, subresource: "exec", verb: verbCreate, subject: subject}, + allowed: true, + }, + { + name: "ServiceAccount cluster-scoped check denied", + check: accessCheck{resource: resourceNodes, verb: verbList, subject: subject}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { clientset := fake.NewClientset() + rec := installReviewReactors(t, clientset, func(askedAccess) bool { return tt.allowed }) - clientset.PrependReactor("create", "selfsubjectaccessreviews", func(action k8stesting.Action) (bool, runtime.Object, error) { - return true, &authv1.SelfSubjectAccessReview{ - Status: authv1.SubjectAccessReviewStatus{ - Allowed: tt.allowed, - Reason: tt.reason, - }, - }, nil - }) + d := NewDeployer(clientset, Config{Namespace: testNamespace, RunID: testRunID}) + allowed, reason, err := d.reviewAccess(context.Background(), tt.check) + if err != nil { + t.Fatalf("reviewAccess() error = %v", err) + } + if allowed != tt.allowed { + t.Errorf("reviewAccess() allowed = %v, want %v", allowed, tt.allowed) + } + if reason == "" { + t.Error("reviewAccess() reason is empty; the authorizer's explanation must be carried through") + } - deployer := NewDeployer(clientset, Config{ - Namespace: tt.namespace, - }) + asked := rec.questions() + if len(asked) != 1 { + t.Fatalf("questions asked = %d, want exactly 1", len(asked)) + } + want := askedAccess{ + subject: tt.check.subject, + group: tt.check.group, + resource: tt.check.resource, + subresource: tt.check.subresource, + verb: tt.check.verb, + namespace: tt.check.namespace, + } + if asked[0] != want { + t.Errorf("asked %s, want %s", asked[0], want) + } + }) + } +} - ctx := context.Background() - allowed, reason, err := deployer.checkPermission(ctx, tt.resource, tt.verb, tt.namespace) +// TestDedupeChecks pins that the gate never asks the apiserver the same +// question twice, while keeping questions that differ only in scope: an +// operator who applied the Role but not the ClusterRole must fail exactly +// the cluster-scoped one. +func TestDedupeChecks(t *testing.T) { + nsPods := accessCheck{resource: resourcePods, verb: verbList, namespace: testNamespace} + clusterPods := accessCheck{resource: resourcePods, verb: verbList} - if err != nil { - t.Fatalf("checkPermission() error = %v", err) + got := dedupeChecks([]accessCheck{nsPods, clusterPods, nsPods, clusterPods, nsPods}) + want := []accessCheck{nsPods, clusterPods} + if len(got) != len(want) { + t.Fatalf("dedupeChecks() returned %d checks, want %d: %+v", len(got), len(want), got) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("check %d = %+v, want %+v (first-seen order must be preserved)", i, got[i], want[i]) + } + } +} + +// TestServiceAccountChecksTrackTheGrantedRules pins the anti-drift property: +// the questions asked of the agent ServiceAccount are derived from the same +// namespacedRules / clusterRules definitions the run-scoped Role and +// ClusterRole are built from, so the gate cannot fall behind what the agent +// needs. --discover-network widens the rule set, and the gate must widen +// with it. +func TestServiceAccountChecksTrackTheGrantedRules(t *testing.T) { + subject := serviceAccountUsername(testNamespace, exactSAName) + + for _, discover := range []bool{false, true} { + name := "baseline" + if discover { + name = "discover-network" + } + t.Run(name, func(t *testing.T) { + d := NewDeployer(fake.NewClientset(), Config{ + Namespace: testNamespace, + RunID: testRunID, + DiscoverNetwork: discover, + }) + got := make(map[accessCheck]struct{}) + for _, c := range d.serviceAccountChecks(subject) { + got[c] = struct{}{} } - if allowed != tt.allowed { - t.Errorf("checkPermission() allowed = %v, want %v", allowed, tt.allowed) + nsChecks := checksFromRules(namespacedRules(), testNamespace, subject) + clusterChecks := checksFromRules(clusterRules(discover), "", subject) + wantAll := make([]accessCheck, 0, len(nsChecks)+len(clusterChecks)) + wantAll = append(wantAll, nsChecks...) + wantAll = append(wantAll, clusterChecks...) + for _, c := range wantAll { + if _, ok := got[c]; !ok { + t.Errorf("granted rule not checked: %+v", c) + } + if c.subject != subject { + t.Errorf("check %+v is not asked of the ServiceAccount", c) + } } - if reason != tt.reason { - t.Errorf("checkPermission() reason = %q, want %q", reason, tt.reason) + // pods/exec is the rule most easily lost to naive splitting: + // the apiserver evaluates it as resource "pods", subresource + // "exec", never as a resource literally named "pods/exec". + execCheck := accessCheck{resource: resourcePods, subresource: "exec", verb: verbCreate, subject: subject} + if _, ok := got[execCheck]; ok != discover { + t.Errorf("pods/exec check present = %v, want %v", ok, discover) } }) } diff --git a/pkg/k8s/agent/provision.go b/pkg/k8s/agent/provision.go new file mode 100644 index 000000000..491ecf27a --- /dev/null +++ b/pkg/k8s/agent/provision.go @@ -0,0 +1,509 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package agent + +import ( + "fmt" + "strings" + + "github.com/NVIDIA/aicr/pkg/errors" + "github.com/NVIDIA/aicr/pkg/k8s/labels" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation" + "sigs.k8s.io/yaml" +) + +// Naming of the RBAC objects BuildServiceAccountRoleManifests renders. +// +// The names are deterministic — the same (namespace, ServiceAccount) pair +// always resolves to the same four names — and they cannot collide with a +// run-scoped name. +// +// The non-collision is structural, not probabilistic. Every run-scoped name +// is "-" and every runID ends in 16 lowercase-hex characters +// (runid.Generate emits "--<16 hex>"). These names end in +// the literal "-rbac", and "r" is not a hex digit, so no run-scoped name can +// ever equal one of them regardless of what prefix a caller supplies. +const ( + provisionedNamePrefix = "aicr-agent-" + provisionedNameSuffix = "-rbac" +) + +// File names of the rendered manifests, one object per file. +// +// The numeric prefixes are not decoration. `kubectl apply -f /` visits a +// directory in lexical order, so they put each Role ahead of the RoleBinding +// that references it, and they keep the reading order an operator sees when +// they list the directory the same as the order the objects take effect in. +const ( + roleFileName = "01-role.yaml" + roleBindingFileName = "02-rolebinding.yaml" + clusterRoleFileName = "03-clusterrole.yaml" + clusterRoleBindingFileName = "04-clusterrolebinding.yaml" +) + +// ManifestOptions selects the ServiceAccount that +// BuildServiceAccountRoleManifests renders RBAC for. +type ManifestOptions struct { + // Namespace is the namespace of the ServiceAccount, and the namespace + // the rendered Role and RoleBinding declare. Required. + Namespace string + + // ServiceAccountName is the name of the ServiceAccount the rendered + // RoleBinding and ClusterRoleBinding name as their subject. Required. + // + // It is NOT verified to exist: rendering contacts no cluster, by + // design (see BuildServiceAccountRoleManifests). A name that does not + // resolve produces manifests that grant nothing, which the operator + // sees when they review the files or when the ServiceAccount they + // meant to name still cannot snapshot. + ServiceAccountName string + + // DiscoverNetwork also renders the cluster-scoped MUTATING rules that + // `aicr snapshot --discover-network` needs — nodes: patch, + // pods/exec: create, and CRD, namespace, DaemonSet and namespaced-RBAC + // create/delete (see discoverNetworkClusterRules). + // + // The rendered ClusterRole carries an explicit warning header + // enumerating each mutating rule and the discovery step it exists for, + // because deciding whether to grant them is the whole reason these + // manifests are written out instead of applied. + DiscoverNetwork bool +} + +// Manifest is one rendered RBAC object: the bytes to write, the file name to +// write them under, and the object's kind and name so a caller can report +// what it wrote without re-deriving either. +type Manifest struct { + // FileName is the name of the file within the output directory. It is + // a bare file name, never a path. + FileName string + + // Kind is the Kubernetes kind ("Role", "RoleBinding", "ClusterRole", + // "ClusterRoleBinding"). + Kind string + + // Name is the object's metadata.name. + Name string + + // Content is the complete file body: a YAML comment header explaining + // what the object grants and why the agent needs it, followed by the + // object itself. + Content []byte +} + +// BuildServiceAccountRoleManifests renders the Role, RoleBinding, +// ClusterRole and ClusterRoleBinding that grant the snapshot agent's +// permissions to an operator-supplied ServiceAccount. +// +// It APPLIES NOTHING and contacts no cluster. There is no clientset, no +// ServiceAccount lookup and no permission pre-flight on this path, so it +// works with no kubeconfig and no cluster privileges at all. Applying the +// manifests, and deleting them when the grant is no longer wanted, is the +// operator's decision and the operator's command. +// +// That is deliberate. The rules being granted include, under +// DiscoverNetwork, cluster-scoped mutating permissions (nodes: patch, +// pods/exec: create, CRD create) that outlive any single run. An operator +// consenting to that should be able to read exactly what they are granting +// first, which a command that provisions on their behalf does not allow. +// +// Every rule set comes from namespacedRules and clusterRules — the same +// definitions the run-scoped ensureRole and ensureClusterRole build from — +// so a rendered manifest can never drift from what a run-owned grant +// carries. +// +// The objects are NOT run-scoped: they carry no run-ID label, never enter a +// Deployer's created-set, and no run's Cleanup deletes them. Teardown is +// `kubectl delete -f /`. +// +// Trade-off the caller must surface to the operator: a shared ServiceAccount +// waives per-run permission isolation. Concurrent runs using it share its +// grants, and a DiscoverNetwork grant leaves mutating cluster permissions in +// place until the operator removes them. +func BuildServiceAccountRoleManifests(opts ManifestOptions) ([]Manifest, error) { + if strings.TrimSpace(opts.Namespace) == "" { + return nil, errors.New(errors.ErrCodeInvalidRequest, + "namespace is required: it is the namespace the rendered Role and RoleBinding declare") + } + if strings.TrimSpace(opts.ServiceAccountName) == "" { + return nil, errors.New(errors.ErrCodeInvalidRequest, + "ServiceAccount name is required: the rendered bindings need a subject to name") + } + + // Both halves of each composed name are valid on their own, but their + // concatenation can exceed the length ceiling. Reject that here, while + // rendering, rather than leaving the operator to discover it as an + // opaque apiserver "Invalid value: metadata.name" at apply time. + roleName := provisionedRoleName(opts.ServiceAccountName) + clusterRoleName := provisionedClusterRoleName(opts.Namespace, opts.ServiceAccountName) + for _, name := range []string{roleName, clusterRoleName} { + problems := validation.IsDNS1123Subdomain(name) + if len(problems) == 0 { + continue + } + return nil, errors.NewWithContext(errors.ErrCodeInvalidRequest, + fmt.Sprintf("ServiceAccount %q yields the RBAC object name %q, which is not a valid Kubernetes object name: %s", + opts.ServiceAccountName, name, strings.Join(problems, "; ")), + map[string]any{ctxKeyValue: opts.ServiceAccountName, ctxKeyResolvedName: name}) + } + + subjects := []rbacv1.Subject{{ + Kind: kindServiceAccount, + Name: opts.ServiceAccountName, + Namespace: opts.Namespace, + }} + // Each object gets its own ObjectMeta rather than sharing one value: + // ObjectMeta copies by value but its Labels map does not, and four + // objects aliasing one map is a mutation hazard for anything that later + // adjusts labels per object. + namespacedMeta := func() metav1.ObjectMeta { + return metav1.ObjectMeta{Name: roleName, Namespace: opts.Namespace, Labels: provisionedLabels()} + } + clusterMeta := func() metav1.ObjectMeta { + return metav1.ObjectMeta{Name: clusterRoleName, Labels: provisionedLabels()} + } + + objects := []struct { + fileName string + kind string + name string + header string + object any + }{ + { + fileName: roleFileName, + kind: kindRole, + name: roleName, + header: roleHeader(roleName, opts.Namespace, opts.ServiceAccountName), + object: &rbacv1.Role{ + TypeMeta: rbacTypeMeta(kindRole), + ObjectMeta: namespacedMeta(), + Rules: namespacedRules(), + }, + }, + { + fileName: roleBindingFileName, + kind: kindRoleBinding, + name: roleName, + header: roleBindingHeader(roleName, opts.Namespace, opts.ServiceAccountName), + object: &rbacv1.RoleBinding{ + TypeMeta: rbacTypeMeta(kindRoleBinding), + ObjectMeta: namespacedMeta(), + Subjects: subjects, + RoleRef: rbacv1.RoleRef{APIGroup: rbacAPIGroup, Kind: kindRole, Name: roleName}, + }, + }, + { + fileName: clusterRoleFileName, + kind: kindClusterRole, + name: clusterRoleName, + header: clusterRoleHeader(clusterRoleName, opts.ServiceAccountName, opts.DiscoverNetwork), + object: &rbacv1.ClusterRole{ + TypeMeta: rbacTypeMeta(kindClusterRole), + ObjectMeta: clusterMeta(), + Rules: clusterRules(opts.DiscoverNetwork), + }, + }, + { + fileName: clusterRoleBindingFileName, + kind: kindClusterRoleBinding, + name: clusterRoleName, + header: clusterRoleBindingHeader(clusterRoleName, opts.Namespace, opts.ServiceAccountName), + object: &rbacv1.ClusterRoleBinding{ + TypeMeta: rbacTypeMeta(kindClusterRoleBinding), + ObjectMeta: clusterMeta(), + Subjects: subjects, + RoleRef: rbacv1.RoleRef{APIGroup: rbacAPIGroup, Kind: kindClusterRole, Name: clusterRoleName}, + }, + }, + } + + manifests := make([]Manifest, 0, len(objects)) + for _, o := range objects { + content, err := renderManifest(o.header, o.object, opts.ServiceAccountName) + if err != nil { + return nil, err + } + manifests = append(manifests, Manifest{ + FileName: o.fileName, + Kind: o.kind, + Name: o.name, + Content: content, + }) + } + return manifests, nil +} + +// rbacTypeMeta returns the apiVersion/kind pair a standalone manifest needs. +// The typed objects client-go hands back leave TypeMeta empty — the wire +// format carries it out of band — so a file written from one is unusable +// with `kubectl apply` until it is set explicitly. +func rbacTypeMeta(kind string) metav1.TypeMeta { + return metav1.TypeMeta{APIVersion: rbacv1.SchemeGroupVersion.String(), Kind: kind} +} + +// renderManifest assembles one file: the explanatory header, the marshaled +// object, and the shared trailer that states nothing was applied and how to +// apply and revoke. +// +// Marshaling goes through sigs.k8s.io/yaml, which encodes the typed struct +// via encoding/json rather than walking a Go map, so the field order is the +// struct's and two runs over the same inputs produce identical bytes. +func renderManifest(header string, obj any, serviceAccount string) ([]byte, error) { + body, err := yaml.Marshal(obj) + if err != nil { + return nil, errors.Wrap(errors.ErrCodeInternal, "failed to render the RBAC manifest", err) + } + var buf strings.Builder + buf.WriteString(header) + buf.WriteString(manifestTrailer(serviceAccount)) + buf.WriteString("---\n") + buf.Write(body) + return []byte(buf.String()), nil +} + +// manifestTrailer is the block every rendered file ends its header with. It +// repeats on all four because an operator may open, review, and act on one +// file alone, and the single fact none of them can afford to omit is that +// nothing has been applied yet. +func manifestTrailer(serviceAccount string) string { + return fmt.Sprintf(`# +# Generated by: aicr snapshot --add-roles-to-service-account %s +# +# NOTHING HAS BEEN APPLIED TO YOUR CLUSTER. aicr wrote these files and +# contacted no cluster to do it. Review them, then grant and revoke yourself: +# +# kubectl apply -f / # grant the permissions +# kubectl delete -f / # revoke them again +# +# No aicr run creates, refreshes, or deletes these objects. They last until +# you delete them. +# +`, serviceAccount) +} + +// roleHeader explains the namespaced grant: two rules, both confined to one +// namespace, and what the agent does with each. +func roleHeader(name, namespace, serviceAccount string) string { + return fmt.Sprintf(`# aicr snapshot agent -- namespaced permissions +# +# Role/%s +# in namespace %s +# +# Bound to ServiceAccount %q by %s. +# +# Grants only what the agent Job needs inside this one namespace: +# +# configmaps: create, get, update, patch +# The agent runs as a Kubernetes Job and cannot hand its result back to +# the CLI directly. It stages the snapshot it collected in a ConfigMap +# in this namespace and the CLI reads that ConfigMap. Confined to this +# namespace: it grants nothing in any other. +# +# pods: get, list +# The agent reads its own pod to learn which node it was scheduled onto, +# and lists pods when collecting workload state. +# +# Read-mostly and namespace-local: nothing here is cluster-scoped, and +# nothing here can modify a node, a CRD, or any workload. +`, name, namespace, serviceAccount, roleBindingFileName) +} + +// roleBindingHeader explains that this file is what makes the Role take +// effect, and states the consequence of the unverified ServiceAccount name: +// Kubernetes accepts a binding to a subject that does not exist and it +// simply grants nothing, so a typo fails silently. +func roleBindingHeader(name, namespace, serviceAccount string) string { + return fmt.Sprintf(`# aicr snapshot agent -- binds the namespaced permissions +# +# RoleBinding/%[1]s +# in namespace %[2]s +# Role/%[1]s -> ServiceAccount %[2]s/%[3]s +# +# Applying this is what actually gives ServiceAccount %[3]q the rules +# in %[4]s. Until it is applied, that Role grants nothing to anyone. +# +# CHECK THE NAME FIRST. aicr rendered these manifests without contacting a +# cluster, so it did not verify that this ServiceAccount exists. Kubernetes +# accepts a binding whose subject does not exist and simply grants nothing, +# so a typo here fails silently rather than loudly: +# +# kubectl get serviceaccount %[3]s -n %[2]s +# +# The ServiceAccount must be one you created and control -- typically one +# carrying IRSA (eks.amazonaws.com/role-arn) or GKE Workload Identity +# (iam.gke.io/gcp-service-account) annotations. aicr never creates it. +`, name, namespace, serviceAccount, roleFileName) +} + +// clusterRoleHeader explains the cluster-scoped grant. Without +// discoverNetwork it is entirely read-only and says so; with it, the +// mutating rules get a rule-by-rule account of the discovery step each one +// exists for, since that is the grant an operator most needs to read before +// consenting to it. +func clusterRoleHeader(name, serviceAccount string, discoverNetwork bool) string { + header := fmt.Sprintf(`# aicr snapshot agent -- cluster-scoped permissions +# +# ClusterRole/%s +# (cluster-scoped -- these rules apply in EVERY namespace) +# +# Bound to ServiceAccount %q by %s. +# +# The baseline rule set below is READ-ONLY -- every verb is get, list or +# watch, and nothing in it creates, patches, or deletes anything: +# +# nodes: get, list +# Node inventory: labels, taints, allocatable capacity, kubelet and +# container-runtime versions, OS image. +# +# pods: get, list +# Cluster-wide workload inventory, used to detect which GPU and +# networking components are already deployed. +# +# nvidia.com clusterpolicies: get, list +# GPU Operator ClusterPolicy: the driver, toolkit, and device-plugin +# configuration currently in effect. +# +# slinky.slurm.net controllers, nodesets, loginsets, restapis, +# accountings: list +# Slurm-on-Kubernetes topology, when Slinky is installed. +# +# k8s.mariadb.com mariadbs: list +# The MariaDB instance backing Slurm accounting, when present. +`, name, serviceAccount, clusterRoleBindingFileName) + if !discoverNetwork { + return header + } + return header + discoverNetworkHeaderSection() +} + +// discoverNetworkHeaderSection is the warning block appended to the +// ClusterRole manifest when DiscoverNetwork is set. Each rule is named +// alongside the concrete discovery step that needs it, so an operator +// reading "nodes: patch" can tell from the file why it is there. +func discoverNetworkHeaderSection() string { + return `# +# ========================================================================== +# WARNING -- --discover-network was requested, so this ClusterRole ALSO +# carries MUTATING, cluster-scoped rules. Read them before you apply it. +# ========================================================================== +# +# Live network discovery (k8s-launch-kit) does not read topology from the +# API. It stands up a probe DaemonSet, execs into it, and writes what it +# learns back onto your cluster. Every mutating rule below maps to one +# concrete step of that flow: +# +# nodes: patch +# Writes nvidia.kubernetes-launch-kit.machine and .gpu labels onto every +# node discovery matches. This modifies YOUR nodes, and the labels +# remain after the run finishes. +# +# pods/exec: create +# Execs into each probe pod to read NIC VPD and link metadata via the +# in-pod CLI. Note that pods/exec: create at cluster scope permits +# running commands in ANY pod in the cluster, not only the probe pods. +# +# namespaces: get, create, delete +# apps daemonsets: get, list, watch, create, delete +# serviceaccounts, configmaps: get, create, delete +# rbac.authorization.k8s.io roles, rolebindings: get, create, delete +# Discovery creates the nvidia-k8s-launch-kit namespace, deploys the +# nic-configuration-daemon DaemonSet and its supporting RBAC into it, +# and deletes the namespace when it is done. +# +# apiextensions.k8s.io customresourcedefinitions: +# get, list, create, update, patch +# Installs the nic-configuration-operator CRDs (NicDevice, +# NicClusterPolicy) when they are absent. This is a cluster-wide schema +# change that outlives the run. +# +# configuration.net.nvidia.com nicdevices: get, list +# Reads the NicDevice CRs the probe daemon publishes. +# +# mellanox.com nicclusterpolicies: get, patch +# Server-side-applies the NicConfigurationOperator section of YOUR +# existing NicClusterPolicy. +# +# These permissions last as long as the objects do -- they are NOT scoped to +# one run. A run that lets aicr create its own ServiceAccount instead gets +# the same rules for the lifetime of that one run and has them revoked at +# cleanup. If you need discovery only occasionally, prefer that, or keep a +# separate ServiceAccount used only for discovery runs. +` +} + +// clusterRoleBindingHeader explains the cluster-scoped binding and warns +// about the one hazard aicr can no longer detect for the operator: the +// generated name is not injective, so an apply can silently retarget an +// existing binding. Reading the file is the check that replaces the cluster +// read the old provisioning path performed. +func clusterRoleBindingHeader(name, namespace, serviceAccount string) string { + return fmt.Sprintf(`# aicr snapshot agent -- binds the cluster-scoped permissions +# +# ClusterRoleBinding/%[1]s +# (cluster-scoped) +# ClusterRole/%[1]s -> ServiceAccount %[2]s/%[3]s +# +# Applying this is what gives ServiceAccount %[3]q the rules in +# %[4]s, across every namespace in the cluster. +# +# The name joins the namespace and the ServiceAccount on ".", which no +# namespace may contain, so no other (namespace, ServiceAccount) pair can +# compose this name. +`, name, namespace, serviceAccount, clusterRoleFileName) +} + +// provisionedRoleName returns the Role and RoleBinding name for a +// ServiceAccount. It is injective within a namespace — the name is a pure +// function of the ServiceAccount name, and a ServiceAccount name is unique +// in its namespace — so two ServiceAccounts can never share these objects. +func provisionedRoleName(serviceAccount string) string { + return provisionedNamePrefix + serviceAccount + provisionedNameSuffix +} + +// provisionedClusterRoleName returns the ClusterRole and ClusterRoleBinding +// name for a ServiceAccount. The namespace is part of the name because these +// objects are cluster-scoped and the same ServiceAccount name can exist in +// several namespaces. +// +// The two segments join on "." rather than "-" so the composition is +// injective. A "-" join is not: namespace "a-b" with ServiceAccount "c" and +// namespace "a" with ServiceAccount "b-c" compose the same string, and +// applying the second render over the first would retarget the existing +// ClusterRoleBinding and revoke the other ServiceAccount's cluster +// permissions. Nothing here could detect that, because no cluster is read. +// +// "." is safe and sufficient: a namespace is a DNS-1123 *label*, which cannot +// contain a dot, while a ClusterRole name is a DNS-1123 *subdomain*, which +// can. The first dot therefore always separates namespace from ServiceAccount, +// whatever either contains. +func provisionedClusterRoleName(namespace, serviceAccount string) string { + return provisionedNamePrefix + namespace + "." + serviceAccount + provisionedNameSuffix +} + +// provisionedLabels is the label set stamped on every rendered object. +// It deliberately omits labels.RunID: these objects belong to no run, so +// Deployer.createdByThisRun can never match one and no run's Cleanup can +// reclaim it. The component value is what distinguishes them from the +// run-scoped snapshot-agent objects in selectors and sweeps. +func provisionedLabels() map[string]string { + return map[string]string{ + labels.Name: labels.ValueAICR, + labels.ManagedBy: labels.ValueAICR, + labels.Component: labels.ValueAgentRBAC, + } +} diff --git a/pkg/k8s/agent/provision_test.go b/pkg/k8s/agent/provision_test.go new file mode 100644 index 000000000..9e02d4de2 --- /dev/null +++ b/pkg/k8s/agent/provision_test.go @@ -0,0 +1,454 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package agent + +import ( + stderrors "errors" + "reflect" + "strings" + "testing" + + aicrerrors "github.com/NVIDIA/aicr/pkg/errors" + "github.com/NVIDIA/aicr/pkg/k8s/labels" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/util/validation" + "sigs.k8s.io/yaml" +) + +const provisionSA = "irsa-snapshotter" + +// buildManifests renders the four manifests for the standard test inputs and +// fails the test on any error. +func buildManifests(t *testing.T, discoverNetwork bool) []Manifest { + t.Helper() + manifests, err := BuildServiceAccountRoleManifests(ManifestOptions{ + Namespace: testNamespace, + ServiceAccountName: provisionSA, + DiscoverNetwork: discoverNetwork, + }) + if err != nil { + t.Fatalf("BuildServiceAccountRoleManifests() error = %v", err) + } + return manifests +} + +// manifestByFile indexes rendered manifests by file name. +func manifestByFile(manifests []Manifest) map[string]Manifest { + byFile := make(map[string]Manifest, len(manifests)) + for _, m := range manifests { + byFile[m.FileName] = m + } + return byFile +} + +// TestProvisionedClusterRoleNameIsInjective pins the property the "." join +// exists for. These two (namespace, ServiceAccount) pairs compose the same +// string under a "-" join, and the collision is not academic: the second +// render applied over the first retargets the live ClusterRoleBinding and +// revokes the first ServiceAccount's cluster permissions. Nothing in the +// generator can detect it, because it reads no cluster. +// +// A namespace is a DNS-1123 label and cannot contain ".", so the first dot +// always separates the two segments. Switching the join back to "-" fails +// this test. +func TestProvisionedClusterRoleNameIsInjective(t *testing.T) { + tests := []struct { + name string + namespace string + sa string + }{ + {"hyphen in the namespace", "a-b", "c"}, + {"hyphen in the ServiceAccount", "a", "b-c"}, + {"dot in the ServiceAccount is still unambiguous", "a", "b.c"}, + } + + seen := make(map[string]string, len(tests)) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := provisionedClusterRoleName(tt.namespace, tt.sa) + if errs := validation.IsDNS1123Subdomain(got); len(errs) > 0 { + t.Errorf("%q is not a valid ClusterRole name: %v", got, errs) + } + if prior, ok := seen[got]; ok { + t.Errorf("%s/%s collides with %s on name %q", + tt.namespace, tt.sa, prior, got) + } + seen[got] = tt.namespace + "/" + tt.sa + }) + } +} + +// TestBuildServiceAccountRoleManifests_FilesAndNames pins the file layout and +// the naming scheme together, because both are contracts an operator's +// commands depend on: `kubectl apply -f /` relies on one parseable +// object per file in an order that puts a Role ahead of its RoleBinding, and +// a rendered name must never be mistakable for a run-scoped one. Every +// run-scoped name ends in a run ID whose final segment is 16 lowercase-hex +// characters, and these end in "-rbac" — "r" is not a hex digit, so the two +// name spaces are disjoint by construction rather than by luck. +func TestBuildServiceAccountRoleManifests_FilesAndNames(t *testing.T) { + manifests := buildManifests(t, false) + + wantRole := "aicr-agent-" + provisionSA + "-rbac" + wantClusterRole := "aicr-agent-" + testNamespace + "." + provisionSA + "-rbac" + + want := []struct { + file string + kind string + name string + }{ + {roleFileName, kindRole, wantRole}, + {roleBindingFileName, kindRoleBinding, wantRole}, + {clusterRoleFileName, kindClusterRole, wantClusterRole}, + {clusterRoleBindingFileName, kindClusterRoleBinding, wantClusterRole}, + } + if len(manifests) != len(want) { + t.Fatalf("manifests = %d, want %d (one file per object)", len(manifests), len(want)) + } + for i, w := range want { + got := manifests[i] + if got.FileName != w.file { + t.Errorf("manifest[%d].FileName = %q, want %q (lexical order must apply a Role before its binding)", i, got.FileName, w.file) + } + if got.Kind != w.kind { + t.Errorf("manifest[%d].Kind = %q, want %q", i, got.Kind, w.kind) + } + if got.Name != w.name { + t.Errorf("manifest[%d].Name = %q, want %q", i, got.Name, w.name) + } + if !strings.HasSuffix(got.Name, "-rbac") { + t.Errorf("manifest[%d].Name = %q, want a name ending in -rbac so it cannot collide with a run-scoped name", i, got.Name) + } + } +} + +// TestBuildServiceAccountRoleManifests_ParseableYAML asserts each file +// round-trips through a YAML decoder into the typed object it claims to be, +// with apiVersion and kind set. A typed object straight out of client-go +// leaves TypeMeta empty, which `kubectl apply` rejects, so this is the check +// that the files are usable at all. +func TestBuildServiceAccountRoleManifests_ParseableYAML(t *testing.T) { + byFile := manifestByFile(buildManifests(t, false)) + wantAPIVersion := rbacv1.SchemeGroupVersion.String() + + tests := []struct { + name string + file string + wantKind string + into func() any + check func(t *testing.T, obj any) + }{ + { + name: "role carries the namespaced rules", + file: roleFileName, + wantKind: kindRole, + into: func() any { return &rbacv1.Role{} }, + check: func(t *testing.T, obj any) { + t.Helper() + role, ok := obj.(*rbacv1.Role) + if !ok { + t.Fatalf("decoded %T, want *rbacv1.Role", obj) + } + if role.Namespace != testNamespace { + t.Errorf("Role namespace = %q, want %q", role.Namespace, testNamespace) + } + if !reflect.DeepEqual(role.Rules, namespacedRules()) { + t.Errorf("Role rules drifted from namespacedRules(); got %+v", role.Rules) + } + }, + }, + { + name: "rolebinding names the ServiceAccount subject", + file: roleBindingFileName, + wantKind: kindRoleBinding, + into: func() any { return &rbacv1.RoleBinding{} }, + check: func(t *testing.T, obj any) { + t.Helper() + rb, ok := obj.(*rbacv1.RoleBinding) + if !ok { + t.Fatalf("decoded %T, want *rbacv1.RoleBinding", obj) + } + wantSubjects := []rbacv1.Subject{{Kind: kindServiceAccount, Name: provisionSA, Namespace: testNamespace}} + if !reflect.DeepEqual(rb.Subjects, wantSubjects) { + t.Errorf("RoleBinding subjects = %+v, want %+v", rb.Subjects, wantSubjects) + } + if rb.RoleRef.Kind != kindRole { + t.Errorf("RoleBinding roleRef kind = %q, want %q", rb.RoleRef.Kind, kindRole) + } + }, + }, + { + name: "clusterrole carries the cluster rules and no namespace", + file: clusterRoleFileName, + wantKind: kindClusterRole, + into: func() any { return &rbacv1.ClusterRole{} }, + check: func(t *testing.T, obj any) { + t.Helper() + cr, ok := obj.(*rbacv1.ClusterRole) + if !ok { + t.Fatalf("decoded %T, want *rbacv1.ClusterRole", obj) + } + if cr.Namespace != "" { + t.Errorf("ClusterRole namespace = %q, want empty (it is cluster-scoped)", cr.Namespace) + } + if !reflect.DeepEqual(cr.Rules, clusterRules(false)) { + t.Errorf("ClusterRole rules drifted from clusterRules(false); got %+v", cr.Rules) + } + }, + }, + { + name: "clusterrolebinding binds the clusterrole", + file: clusterRoleBindingFileName, + wantKind: kindClusterRoleBinding, + into: func() any { return &rbacv1.ClusterRoleBinding{} }, + check: func(t *testing.T, obj any) { + t.Helper() + crb, ok := obj.(*rbacv1.ClusterRoleBinding) + if !ok { + t.Fatalf("decoded %T, want *rbacv1.ClusterRoleBinding", obj) + } + if crb.RoleRef.Kind != kindClusterRole { + t.Errorf("ClusterRoleBinding roleRef kind = %q, want %q", crb.RoleRef.Kind, kindClusterRole) + } + if got := crb.Labels[labels.Component]; got != labels.ValueAgentRBAC { + t.Errorf("component label = %q, want %q", got, labels.ValueAgentRBAC) + } + if _, ok := crb.Labels[labels.RunID]; ok { + t.Error("rendered object carries a run-ID label; these objects belong to no run") + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m, ok := byFile[tt.file] + if !ok { + t.Fatalf("no manifest rendered for %s", tt.file) + } + obj := tt.into() + if err := yaml.Unmarshal(m.Content, obj); err != nil { + t.Fatalf("unmarshalling %s: %v\n%s", tt.file, err, m.Content) + } + var apiVersion, kind string + switch o := obj.(type) { + case *rbacv1.Role: + apiVersion, kind = o.APIVersion, o.Kind + case *rbacv1.RoleBinding: + apiVersion, kind = o.APIVersion, o.Kind + case *rbacv1.ClusterRole: + apiVersion, kind = o.APIVersion, o.Kind + case *rbacv1.ClusterRoleBinding: + apiVersion, kind = o.APIVersion, o.Kind + } + if apiVersion != wantAPIVersion { + t.Errorf("%s apiVersion = %q, want %q", tt.file, apiVersion, wantAPIVersion) + } + if kind != tt.wantKind { + t.Errorf("%s kind = %q, want %q", tt.file, kind, tt.wantKind) + } + tt.check(t, obj) + }) + } +} + +// TestBuildServiceAccountRoleManifests_DiscoverNetwork covers both rule sets. +// The default grant must stay read-only, and the --discover-network grant +// must carry the mutating rules AND explain them in the header — the header +// is what an operator reads to decide whether to apply the file at all, so a +// silent "nodes: patch" would defeat the point of writing manifests out +// instead of applying them. +func TestBuildServiceAccountRoleManifests_DiscoverNetwork(t *testing.T) { + tests := []struct { + name string + discoverNetwork bool + wantRuleCount int + wantMutating bool + }{ + {name: "default grant is read-only", discoverNetwork: false, wantRuleCount: len(clusterRules(false))}, + {name: "discovery grant adds the mutating rules", discoverNetwork: true, wantRuleCount: len(clusterRules(true)), wantMutating: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m, ok := manifestByFile(buildManifests(t, tt.discoverNetwork))[clusterRoleFileName] + if !ok { + t.Fatalf("no ClusterRole manifest rendered") + } + cr := &rbacv1.ClusterRole{} + if err := yaml.Unmarshal(m.Content, cr); err != nil { + t.Fatalf("unmarshalling ClusterRole: %v", err) + } + if len(cr.Rules) != tt.wantRuleCount { + t.Errorf("ClusterRole rules = %d, want %d", len(cr.Rules), tt.wantRuleCount) + } + for _, want := range []struct{ resource, verb string }{ + {"nodes", "patch"}, + {"pods/exec", verbCreate}, + {"customresourcedefinitions", verbCreate}, + } { + if got := hasRule(cr.Rules, want.resource, want.verb); got != tt.wantMutating { + t.Errorf("%s: %s present = %v, want %v", want.resource, want.verb, got, tt.wantMutating) + } + } + + header := string(m.Content) + for _, want := range []string{"nodes: patch", "pods/exec: create", "MUTATING"} { + if strings.Contains(header, want) != tt.wantMutating { + t.Errorf("header mentions %q = %v, want %v; header:\n%s", want, !tt.wantMutating, tt.wantMutating, header) + } + } + if !tt.wantMutating && !strings.Contains(header, "READ-ONLY") { + t.Errorf("read-only grant does not say so in the header:\n%s", header) + } + }) + } +} + +// hasRule reports whether any rule grants verb on resource. +func hasRule(rules []rbacv1.PolicyRule, resource, verb string) bool { + for _, r := range rules { + for _, res := range r.Resources { + if res != resource { + continue + } + for _, v := range r.Verbs { + if v == verb { + return true + } + } + } + } + return false +} + +// TestBuildServiceAccountRoleManifests_HeadersExplainTheGrant asserts every +// file states, in its own header, that nothing was applied and how to apply +// and revoke. An operator may open exactly one of these files, and the fact +// they must not miss is that the grant is not live yet. +func TestBuildServiceAccountRoleManifests_HeadersExplainTheGrant(t *testing.T) { + wantEverywhere := []string{ + "NOTHING HAS BEEN APPLIED", + "kubectl apply -f", + "kubectl delete -f", + provisionSA, + } + for _, m := range buildManifests(t, false) { + body := string(m.Content) + if !strings.HasPrefix(body, "# ") { + t.Errorf("%s does not open with a YAML comment header; got %.40q", m.FileName, body) + } + for _, want := range wantEverywhere { + if !strings.Contains(body, want) { + t.Errorf("%s header does not contain %q", m.FileName, want) + } + } + } +} + +// TestBuildServiceAccountRoleManifests_Deterministic asserts two renders of +// the same inputs are byte-identical. The manifests are reviewed by hand and +// diffed against a previous grant, so incidental churn between runs would +// make a real change hard to spot. +func TestBuildServiceAccountRoleManifests_Deterministic(t *testing.T) { + first := buildManifests(t, true) + second := buildManifests(t, true) + if len(first) != len(second) { + t.Fatalf("manifest counts differ: %d vs %d", len(first), len(second)) + } + for i := range first { + if string(first[i].Content) != string(second[i].Content) { + t.Errorf("%s differs between renders:\n--- first ---\n%s\n--- second ---\n%s", + first[i].FileName, first[i].Content, second[i].Content) + } + } +} + +// TestBuildServiceAccountRoleManifests_Rejections covers every input the call +// refuses before rendering anything. A ServiceAccount that does not exist is +// deliberately NOT among them: rendering contacts no cluster, so a mistyped +// name yields manifests the operator inspects before applying. +func TestBuildServiceAccountRoleManifests_Rejections(t *testing.T) { + tests := []struct { + name string + opts ManifestOptions + wantInMsg string + }{ + { + name: "empty namespace", + opts: ManifestOptions{ServiceAccountName: provisionSA}, + }, + { + name: "whitespace namespace", + opts: ManifestOptions{Namespace: " ", ServiceAccountName: provisionSA}, + }, + { + name: "empty ServiceAccount name", + opts: ManifestOptions{Namespace: testNamespace}, + }, + { + name: "whitespace ServiceAccount name", + opts: ManifestOptions{Namespace: testNamespace, ServiceAccountName: " "}, + }, + { + name: "name too long to compose", + opts: ManifestOptions{Namespace: testNamespace, ServiceAccountName: strings.Repeat("a", 250)}, + wantInMsg: "not a valid Kubernetes object name", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + manifests, err := BuildServiceAccountRoleManifests(tt.opts) + if err == nil { + t.Fatal("BuildServiceAccountRoleManifests() error = nil, want ErrCodeInvalidRequest") + } + if !stderrors.Is(err, aicrerrors.New(aicrerrors.ErrCodeInvalidRequest, "")) { + t.Errorf("error = %v, want code %s", err, aicrerrors.ErrCodeInvalidRequest) + } + if tt.wantInMsg != "" && !strings.Contains(err.Error(), tt.wantInMsg) { + t.Errorf("error = %q, want it to contain %q", err.Error(), tt.wantInMsg) + } + if manifests != nil { + t.Errorf("manifests = %v, want nil on a rejected call", manifests) + } + }) + } +} + +// TestBuildServiceAccountRoleManifests_UnknownServiceAccountStillRenders pins +// the deliberate simplification: the old provisioning path failed with +// ErrCodeNotFound when the ServiceAccount was absent. Rendering has no +// cluster to ask, so it renders regardless and the RoleBinding header tells +// the operator to check the name themselves. +func TestBuildServiceAccountRoleManifests_UnknownServiceAccountStillRenders(t *testing.T) { + manifests, err := BuildServiceAccountRoleManifests(ManifestOptions{ + Namespace: testNamespace, + ServiceAccountName: "no-such-serviceaccount", + }) + if err != nil { + t.Fatalf("BuildServiceAccountRoleManifests() error = %v, want manifests for an unverified name", err) + } + if len(manifests) != 4 { + t.Fatalf("manifests = %d, want 4", len(manifests)) + } + rb, ok := manifestByFile(manifests)[roleBindingFileName] + if !ok { + t.Fatal("no RoleBinding manifest rendered") + } + if !strings.Contains(string(rb.Content), "kubectl get serviceaccount no-such-serviceaccount") { + t.Errorf("RoleBinding header does not tell the operator how to verify the name:\n%s", rb.Content) + } +} diff --git a/pkg/k8s/agent/rbac.go b/pkg/k8s/agent/rbac.go index fc9c7951d..ce3c69e12 100644 --- a/pkg/k8s/agent/rbac.go +++ b/pkg/k8s/agent/rbac.go @@ -17,9 +17,9 @@ package agent import ( "context" "fmt" + "log/slog" "github.com/NVIDIA/aicr/pkg/errors" - "github.com/NVIDIA/aicr/pkg/k8s" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -79,101 +79,139 @@ func (d *Deployer) ensureNamespace(ctx context.Context) error { return nil } -// ensureServiceAccount creates the ServiceAccount for the agent. -// If the ServiceAccount already exists, this is a no-op (idempotent). -func (d *Deployer) ensureServiceAccount(ctx context.Context) error { - sa := &corev1.ServiceAccount{ - ObjectMeta: metav1.ObjectMeta{ - Name: d.config.ServiceAccountName, - Namespace: d.config.Namespace, - }, +// resolveServiceAccount decides, once per Deploy, whether this run creates +// and owns its own run-scoped ServiceAccount or runs as an +// already-existing one the operator named exactly. +// +// Config.ServiceAccountName is exact-if-exists. When it is set and a +// ServiceAccount of exactly that name already exists in the namespace, the +// agent pod runs as that ServiceAccount verbatim and this run creates NO +// ServiceAccount, Role, RoleBinding, ClusterRole or ClusterRoleBinding — +// aicr manages no permissions for an identity it did not create. That is +// what keeps a pre-created ServiceAccount carrying IRSA +// (eks.amazonaws.com/role-arn) or GKE Workload Identity +// (iam.gke.io/gcp-service-account) annotations usable: both providers pin +// trust to the ServiceAccount NAME, so a per-run name can never be trusted +// by either and copying the annotations onto one would not help. +// +// When the name does not exist, the value stays a prefix and the run +// creates - plus its RBAC, exactly as before. An unset +// Config.ServiceAccountName is never probed: the fallback base ("aicr") is +// aicr's own default, not something the operator asked for, so a stray +// ServiceAccount sitting at that name must not silently capture the run. +// +// It is called from CheckPermissions, not from Deploy directly: the verb set +// the pre-flight demands depends on which of the two modes this run is in, +// so the gate has to resolve before it can finish. The Get is read-only, so +// resolving inside the gate does not weaken fail-before-mutate — no write is +// issued until Deploy's ensure* chain, which runs only once the gate passes. +// +// Every error fails closed, Forbidden included. `serviceaccounts: get` is a +// REQUIRED check in CheckPermissions, evaluated before this runs, so a +// caller that reaches here has been told by the apiserver's own authorizer +// that it may read ServiceAccounts; a Forbidden anyway means the answer and +// the behavior disagree, which is not a state to guess through. The +// previous downgrade — log at debug, continue in prefix mode — is the exact +// credential-loss seam this package exists to close: an operator who passed +// --service-account-name pointing at their IRSA or Workload Identity +// ServiceAccount would silently run under a fresh "-" account +// carrying none of its cloud annotations, with no visible signal. +func (d *Deployer) resolveServiceAccount(ctx context.Context) error { + name := d.config.ServiceAccountName + if name == "" { + return nil } - _, err := d.clientset.CoreV1().ServiceAccounts(d.config.Namespace).Create(ctx, sa, metav1.CreateOptions{}) - return k8s.IgnoreAlreadyExists(err) + switch _, err := d.clientset.CoreV1().ServiceAccounts(d.config.Namespace).Get(ctx, name, metav1.GetOptions{}); { + case err == nil: + d.setExistingServiceAccount(name) + slog.Info("using the existing ServiceAccount named by --service-account-name; aicr manages no RBAC for this run", + attrServiceAccount, name, + attrNamespace, d.config.Namespace, + attrRunID, d.config.RunID, + "note", "no ServiceAccount, Role, RoleBinding, ClusterRole or ClusterRoleBinding is created or deleted; generate this ServiceAccount's RBAC manifests with 'aicr snapshot --add-roles-to-service-account "+name+"' and apply them yourself") + case apierrors.IsNotFound(err): + // Normal path: the value is a prefix and this run creates its own + // run-scoped ServiceAccount below. + case apierrors.IsForbidden(err): + return errors.WrapWithContext(errors.ErrCodeUnauthorized, + fmt.Sprintf("cannot read ServiceAccount %q in namespace %q, so it is impossible to tell whether "+ + "--service-account-name names an existing ServiceAccount to run as verbatim or is a prefix "+ + "for one this run creates; refusing to guess, because guessing \"prefix\" would run the agent "+ + "under a generated ServiceAccount carrying none of that account's cloud credentials. "+ + "Grant 'get serviceaccounts' in this namespace and re-run", name, d.config.Namespace), + err, map[string]any{attrName: name, attrNamespace: d.config.Namespace}) + default: + return errors.Wrap(errors.ErrCodeInternal, "failed to check for an existing ServiceAccount", err) + } + return nil } -// ensureRole creates or updates the Role for ConfigMap access. -func (d *Deployer) ensureRole(ctx context.Context) error { - role := &rbacv1.Role{ +// ensureServiceAccount creates the run-scoped ServiceAccount for the agent. +// Deploy calls it only in prefix mode; see resolveServiceAccount. +func (d *Deployer) ensureServiceAccount(ctx context.Context) error { + name := d.saName() + sa := &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ - Name: d.config.ServiceAccountName, + Name: name, Namespace: d.config.Namespace, - }, - Rules: []rbacv1.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{resourceCM}, - Verbs: []string{verbCreate, verbGet, "update", "patch"}, - }, - { - APIGroups: []string{""}, - Resources: []string{"pods"}, - Verbs: []string{"get", verbList}, - }, + Labels: d.objectLabels(), }, } - _, err := d.clientset.RbacV1().Roles(d.config.Namespace).Create(ctx, role, metav1.CreateOptions{}) + // Record the intent before the Create so a committed create whose + // response is lost still enters Cleanup's delete list (see recordIntent). + d.recordIntent(kindServiceAccount, name) + created, err := d.clientset.CoreV1().ServiceAccounts(d.config.Namespace).Create(ctx, sa, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { - _, err = d.clientset.RbacV1().Roles(d.config.Namespace).Update(ctx, role, metav1.UpdateOptions{}) - if err != nil { - return errors.Wrap(errors.ErrCodeInternal, "failed to update Role", err) - } - return nil + d.discardIntent(kindServiceAccount, name) + return errors.Wrap(errors.ErrCodeInternal, "ServiceAccount already exists under run-scoped name (duplicate RunID?)", err) } if err != nil { - return errors.Wrap(errors.ErrCodeInternal, "failed to create Role", err) + return errors.Wrap(errors.ErrCodeInternal, "failed to create ServiceAccount", err) } + d.recordCreated(kindServiceAccount, created.Name, created.UID) return nil } -// ensureRoleBinding creates or updates the RoleBinding to bind the Role to the ServiceAccount. -func (d *Deployer) ensureRoleBinding(ctx context.Context) error { - rb := &rbacv1.RoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: d.config.ServiceAccountName, - Namespace: d.config.Namespace, - }, - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: d.config.ServiceAccountName, - Namespace: d.config.Namespace, - }, +// namespacedRules returns the namespace-scoped policy rules the agent needs: +// writing its snapshot result into a staging ConfigMap, and reading pods. +// +// It is the single definition consumed by both the run-scoped Role +// ensureRole creates and the Role BuildServiceAccountRoleManifests renders +// for an operator-supplied ServiceAccount, so the two can never drift. +func namespacedRules() []rbacv1.PolicyRule { + return []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{resourceCM}, + Verbs: []string{verbCreate, verbGet, verbUpdate, verbPatch}, }, - RoleRef: rbacv1.RoleRef{ - APIGroup: rbacAPIGroup, - Kind: "Role", - Name: d.config.ServiceAccountName, + { + APIGroups: []string{""}, + Resources: []string{resourcePods}, + Verbs: []string{verbGet, verbList}, }, } - - _, err := d.clientset.RbacV1().RoleBindings(d.config.Namespace).Create(ctx, rb, metav1.CreateOptions{}) - if apierrors.IsAlreadyExists(err) { - _, err = d.clientset.RbacV1().RoleBindings(d.config.Namespace).Update(ctx, rb, metav1.UpdateOptions{}) - if err != nil { - return errors.Wrap(errors.ErrCodeInternal, "failed to update RoleBinding", err) - } - return nil - } - if err != nil { - return errors.Wrap(errors.ErrCodeInternal, "failed to create RoleBinding", err) - } - return nil } -// ensureClusterRole creates or updates the ClusterRole for node and cluster-wide resource access. -func (d *Deployer) ensureClusterRole(ctx context.Context) error { +// clusterRules returns the cluster-scoped policy rules the agent needs. The +// baseline set is read-only; discoverNetwork appends the mutating rules live +// l8k network discovery requires (see discoverNetworkClusterRules). +// +// It is the single definition consumed by both the run-scoped ClusterRole +// ensureClusterRole creates and the ClusterRole +// BuildServiceAccountRoleManifests renders. +func clusterRules(discoverNetwork bool) []rbacv1.PolicyRule { rules := []rbacv1.PolicyRule{ { APIGroups: []string{""}, - Resources: []string{"nodes"}, + Resources: []string{resourceNodes}, Verbs: []string{verbGet, verbList}, }, { APIGroups: []string{""}, - Resources: []string{"pods"}, + Resources: []string{resourcePods}, Verbs: []string{verbGet, verbList}, }, { @@ -203,105 +241,177 @@ func (d *Deployer) ensureClusterRole(ctx context.Context) error { // DaemonSet in its own namespace, exec's into the daemon pods, // writes nvidia.kubernetes-launch-kit.{machine,gpu} labels onto // nodes, and patches mellanox.com NicClusterPolicy via server-side - // apply. Grant the extra cluster-scoped rules only when the snapshot - // opted into discovery so non-network snapshots stay minimal-priv. - if d.config.DiscoverNetwork { + // apply. Grant the extra cluster-scoped rules only when discovery was + // opted into so non-network snapshots stay minimal-priv. + if discoverNetwork { rules = append(rules, discoverNetworkClusterRules()...) } + return rules +} + +// ensureRole creates the run-scoped Role for ConfigMap access. +func (d *Deployer) ensureRole(ctx context.Context) error { + name := d.roleName() + role := &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: d.config.Namespace, + Labels: d.objectLabels(), + }, + Rules: namespacedRules(), + } + + d.recordIntent(kindRole, name) + created, err := d.clientset.RbacV1().Roles(d.config.Namespace).Create(ctx, role, metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(err) { + d.discardIntent(kindRole, name) + return errors.Wrap(errors.ErrCodeInternal, "Role already exists under run-scoped name (duplicate RunID?)", err) + } + if err != nil { + return errors.Wrap(errors.ErrCodeInternal, "failed to create Role", err) + } + d.recordCreated(kindRole, created.Name, created.UID) + return nil +} + +// ensureRoleBinding creates the run-scoped RoleBinding binding the Role to the ServiceAccount. +func (d *Deployer) ensureRoleBinding(ctx context.Context) error { + name := d.roleName() + rb := &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: d.config.Namespace, + Labels: d.objectLabels(), + }, + Subjects: []rbacv1.Subject{ + { + Kind: kindServiceAccount, + Name: d.saName(), + Namespace: d.config.Namespace, + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacAPIGroup, + Kind: kindRole, + Name: d.roleName(), + }, + } + d.recordIntent(kindRoleBinding, name) + created, err := d.clientset.RbacV1().RoleBindings(d.config.Namespace).Create(ctx, rb, metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(err) { + d.discardIntent(kindRoleBinding, name) + return errors.Wrap(errors.ErrCodeInternal, "RoleBinding already exists under run-scoped name (duplicate RunID?)", err) + } + if err != nil { + return errors.Wrap(errors.ErrCodeInternal, "failed to create RoleBinding", err) + } + d.recordCreated(kindRoleBinding, created.Name, created.UID) + return nil +} + +// ensureClusterRole creates the run-scoped ClusterRole for node and cluster-wide resource access. +func (d *Deployer) ensureClusterRole(ctx context.Context) error { + name := d.clusterRoleName() cr := &rbacv1.ClusterRole{ ObjectMeta: metav1.ObjectMeta{ - Name: clusterRoleName, + Name: name, + Labels: d.objectLabels(), }, - Rules: rules, + Rules: clusterRules(d.config.DiscoverNetwork), } - _, err := d.clientset.RbacV1().ClusterRoles().Create(ctx, cr, metav1.CreateOptions{}) + d.recordIntent(kindClusterRole, name) + created, err := d.clientset.RbacV1().ClusterRoles().Create(ctx, cr, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { - _, err = d.clientset.RbacV1().ClusterRoles().Update(ctx, cr, metav1.UpdateOptions{}) - if err != nil { - return errors.Wrap(errors.ErrCodeInternal, "failed to update ClusterRole", err) - } - return nil + d.discardIntent(kindClusterRole, name) + return errors.Wrap(errors.ErrCodeInternal, "ClusterRole already exists under run-scoped name (duplicate RunID?)", err) } if err != nil { return errors.Wrap(errors.ErrCodeInternal, "failed to create ClusterRole", err) } + d.recordCreated(kindClusterRole, created.Name, created.UID) return nil } -// ensureClusterRoleBinding creates or updates the ClusterRoleBinding to bind the ClusterRole to the ServiceAccount. +// ensureClusterRoleBinding creates the run-scoped ClusterRoleBinding binding the ClusterRole to the ServiceAccount. func (d *Deployer) ensureClusterRoleBinding(ctx context.Context) error { + name := d.clusterRoleName() crb := &rbacv1.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ - Name: clusterRoleName, + Name: name, + Labels: d.objectLabels(), }, Subjects: []rbacv1.Subject{ { - Kind: "ServiceAccount", - Name: d.config.ServiceAccountName, + Kind: kindServiceAccount, + Name: d.saName(), Namespace: d.config.Namespace, }, }, RoleRef: rbacv1.RoleRef{ APIGroup: rbacAPIGroup, - Kind: "ClusterRole", - Name: clusterRoleName, + Kind: kindClusterRole, + Name: d.clusterRoleName(), }, } - _, err := d.clientset.RbacV1().ClusterRoleBindings().Create(ctx, crb, metav1.CreateOptions{}) + d.recordIntent(kindClusterRoleBinding, name) + created, err := d.clientset.RbacV1().ClusterRoleBindings().Create(ctx, crb, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { - _, err = d.clientset.RbacV1().ClusterRoleBindings().Update(ctx, crb, metav1.UpdateOptions{}) - if err != nil { - return errors.Wrap(errors.ErrCodeInternal, "failed to update ClusterRoleBinding", err) - } - return nil + d.discardIntent(kindClusterRoleBinding, name) + return errors.Wrap(errors.ErrCodeInternal, "ClusterRoleBinding already exists under run-scoped name (duplicate RunID?)", err) } if err != nil { return errors.Wrap(errors.ErrCodeInternal, "failed to create ClusterRoleBinding", err) } + d.recordCreated(kindClusterRoleBinding, created.Name, created.UID) return nil } -// deleteServiceAccount deletes the ServiceAccount. -// If the ServiceAccount doesn't exist, this is a no-op (idempotent). -func (d *Deployer) deleteServiceAccount(ctx context.Context) error { +// deleteServiceAccount deletes the ServiceAccount, pinning the delete to uid +// so a same-named ServiceAccount belonging to a different run is never +// collected. If the ServiceAccount is already gone, or uid no longer +// matches (already replaced, not ours), this is a no-op (idempotent). +func (d *Deployer) deleteServiceAccount(ctx context.Context, name string, uid types.UID) error { err := d.clientset.CoreV1().ServiceAccounts(d.config.Namespace). - Delete(ctx, d.config.ServiceAccountName, metav1.DeleteOptions{}) - return k8s.IgnoreNotFound(err) + Delete(ctx, name, metav1.DeleteOptions{Preconditions: uidPreconditions(uid)}) + return ignoreNotFoundOrConflict(err) } -// deleteRole deletes the Role. -// If the Role doesn't exist, this is a no-op (idempotent). -func (d *Deployer) deleteRole(ctx context.Context) error { +// deleteRole deletes the Role, pinning the delete to uid. If the Role is +// already gone, or uid no longer matches, this is a no-op (idempotent). +func (d *Deployer) deleteRole(ctx context.Context, name string, uid types.UID) error { err := d.clientset.RbacV1().Roles(d.config.Namespace). - Delete(ctx, d.config.ServiceAccountName, metav1.DeleteOptions{}) - return k8s.IgnoreNotFound(err) + Delete(ctx, name, metav1.DeleteOptions{Preconditions: uidPreconditions(uid)}) + return ignoreNotFoundOrConflict(err) } -// deleteRoleBinding deletes the RoleBinding. -// If the RoleBinding doesn't exist, this is a no-op (idempotent). -func (d *Deployer) deleteRoleBinding(ctx context.Context) error { +// deleteRoleBinding deletes the RoleBinding, pinning the delete to uid. If +// the RoleBinding is already gone, or uid no longer matches, this is a +// no-op (idempotent). +func (d *Deployer) deleteRoleBinding(ctx context.Context, name string, uid types.UID) error { err := d.clientset.RbacV1().RoleBindings(d.config.Namespace). - Delete(ctx, d.config.ServiceAccountName, metav1.DeleteOptions{}) - return k8s.IgnoreNotFound(err) + Delete(ctx, name, metav1.DeleteOptions{Preconditions: uidPreconditions(uid)}) + return ignoreNotFoundOrConflict(err) } -// deleteClusterRole deletes the ClusterRole. -// If the ClusterRole doesn't exist, this is a no-op (idempotent). -func (d *Deployer) deleteClusterRole(ctx context.Context) error { +// deleteClusterRole deletes the ClusterRole, pinning the delete to uid. If +// the ClusterRole is already gone, or uid no longer matches, this is a +// no-op (idempotent). +func (d *Deployer) deleteClusterRole(ctx context.Context, name string, uid types.UID) error { err := d.clientset.RbacV1().ClusterRoles(). - Delete(ctx, clusterRoleName, metav1.DeleteOptions{}) - return k8s.IgnoreNotFound(err) + Delete(ctx, name, metav1.DeleteOptions{Preconditions: uidPreconditions(uid)}) + return ignoreNotFoundOrConflict(err) } -// deleteClusterRoleBinding deletes the ClusterRoleBinding. -// If the ClusterRoleBinding doesn't exist, this is a no-op (idempotent). -func (d *Deployer) deleteClusterRoleBinding(ctx context.Context) error { +// deleteClusterRoleBinding deletes the ClusterRoleBinding, pinning the +// delete to uid. If the ClusterRoleBinding is already gone, or uid no +// longer matches, this is a no-op (idempotent). +func (d *Deployer) deleteClusterRoleBinding(ctx context.Context, name string, uid types.UID) error { err := d.clientset.RbacV1().ClusterRoleBindings(). - Delete(ctx, clusterRoleName, metav1.DeleteOptions{}) - return k8s.IgnoreNotFound(err) + Delete(ctx, name, metav1.DeleteOptions{Preconditions: uidPreconditions(uid)}) + return ignoreNotFoundOrConflict(err) } // discoverNetworkClusterRules returns the cluster-scoped policy rules @@ -324,7 +434,6 @@ func (d *Deployer) deleteClusterRoleBinding(ctx context.Context) error { // - nicclusterpolicies: l8k patches the user's NicClusterPolicy // (NicConfigurationOperator section) via server-side apply. func discoverNetworkClusterRules() []rbacv1.PolicyRule { - const verbUpdate, verbPatch, verbWatch, verbDelete = "update", "patch", "watch", "delete" return []rbacv1.PolicyRule{ { APIGroups: []string{"apiextensions.k8s.io"}, @@ -343,22 +452,22 @@ func discoverNetworkClusterRules() []rbacv1.PolicyRule { }, { APIGroups: []string{""}, - Resources: []string{"serviceaccounts", "configmaps"}, + Resources: []string{resourceServiceAccounts, resourceCM}, Verbs: []string{verbGet, verbCreate, verbDelete}, }, { APIGroups: []string{rbacAPIGroup}, - Resources: []string{"roles", "rolebindings"}, + Resources: []string{resourceRoles, resourceRoleBindings}, Verbs: []string{verbGet, verbCreate, verbDelete}, }, { APIGroups: []string{""}, - Resources: []string{"pods/exec"}, + Resources: []string{resourcePods + "/exec"}, Verbs: []string{verbCreate}, }, { APIGroups: []string{""}, - Resources: []string{"nodes"}, + Resources: []string{resourceNodes}, Verbs: []string{verbPatch}, }, { diff --git a/pkg/k8s/agent/rbac_test.go b/pkg/k8s/agent/rbac_test.go new file mode 100644 index 000000000..d6140fbf6 --- /dev/null +++ b/pkg/k8s/agent/rbac_test.go @@ -0,0 +1,422 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package agent + +import ( + "bytes" + "context" + stderrors "errors" + "log/slog" + "strings" + "testing" + + aicrerrors "github.com/NVIDIA/aicr/pkg/errors" + authv1 "k8s.io/api/authorization/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" +) + +// testNamespace is the namespace every ServiceAccount-resolution test +// deploys into. +const testNamespace = "test-ns" + +// captureLogs redirects the default slog logger into a buffer at debug level +// for the duration of the test and returns the buffer. +func captureLogs(t *testing.T) *bytes.Buffer { + t.Helper() + var buf bytes.Buffer + original := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug}))) + t.Cleanup(func() { slog.SetDefault(original) }) + return &buf +} + +// TestResolveServiceAccount covers every branch of the exact-if-exists +// resolution of Config.ServiceAccountName: whether the run adopts an +// operator-supplied ServiceAccount verbatim (and manages none of its +// permissions) or treats the value as a prefix and creates its own. +// +// The Forbidden branch is the load-bearing one, and it now fails closed. +// `serviceaccounts get` is a REQUIRED check in CheckPermissions, evaluated +// before this runs, so a Forbidden here means the authorizer's answer and +// the apiserver's behavior disagree. The old downgrade — continue in prefix +// mode — silently ran an operator who named their IRSA / Workload Identity +// ServiceAccount under a generated one carrying none of its annotations. +func TestResolveServiceAccount(t *testing.T) { + saGR := schema.GroupResource{Group: "", Resource: "serviceaccounts"} + + tests := []struct { + name string + // configured is Config.ServiceAccountName. + configured string + // seeded, when non-empty, pre-creates a ServiceAccount of that + // name in the namespace. + seeded string + // getErr, when non-nil, is returned by every ServiceAccount Get. + getErr error + // wantErrCode is the structured code the failure must carry. A + // permission problem must surface as ErrCodeUnauthorized, not as + // an opaque internal error. + wantErrCode aicrerrors.ErrorCode + // wantExisting is the ServiceAccount the run should adopt + // verbatim; "" means prefix mode. + wantExisting string + wantErr bool + wantLogSubstr string + notWantLog string + }{ + { + name: "exact match adopts the operator's ServiceAccount", + configured: "irsa-snapshotter", + seeded: "irsa-snapshotter", + wantExisting: "irsa-snapshotter", + wantLogSubstr: "aicr manages no RBAC for this run", + }, + { + name: "no match keeps prefix mode", + configured: "irsa-snapshotter", + notWantLog: "aicr manages no RBAC for this run", + }, + { + name: "unset name is never probed, even when the base name exists", + seeded: testName, + notWantLog: "aicr manages no RBAC for this run", + }, + { + name: "forbidden Get fails closed instead of downgrading to a prefix", + configured: "irsa-snapshotter", + seeded: "irsa-snapshotter", + getErr: apierrors.NewForbidden(saGR, "irsa-snapshotter", stderrors.New("no get permission")), + wantErr: true, + wantErrCode: aicrerrors.ErrCodeUnauthorized, + notWantLog: "aicr manages no RBAC for this run", + }, + { + name: "unexpected Get error fails closed", + configured: "irsa-snapshotter", + getErr: apierrors.NewInternalError(stderrors.New("apiserver exploded")), + wantErr: true, + wantErrCode: aicrerrors.ErrCodeInternal, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + buf := captureLogs(t) + + clientset := fake.NewClientset() + if tt.seeded != "" { + if _, err := clientset.CoreV1().ServiceAccounts(testNamespace).Create(ctx, &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{Name: tt.seeded, Namespace: testNamespace}, + }, metav1.CreateOptions{}); err != nil { + t.Fatalf("seeding ServiceAccount: %v", err) + } + } + if tt.getErr != nil { + clientset.PrependReactor("get", "serviceaccounts", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, tt.getErr + }) + } + + d := NewDeployer(clientset, Config{ + Namespace: testNamespace, + ServiceAccountName: tt.configured, + RunID: testRunID, + }) + err := d.resolveServiceAccount(ctx) + + if (err != nil) != tt.wantErr { + t.Fatalf("resolveServiceAccount() error = %v, wantErr %v", err, tt.wantErr) + } + if tt.wantErr && !stderrors.Is(err, aicrerrors.New(tt.wantErrCode, "")) { + t.Errorf("error = %v, want code %s", err, tt.wantErrCode) + } + if got := d.existingServiceAccount(); got != tt.wantExisting { + t.Errorf("existingServiceAccount() = %q, want %q", got, tt.wantExisting) + } + // managesRBAC is what gates Deploy's whole RBAC block, so + // assert it rather than inferring it from the field. + if got, want := d.managesRBAC(), tt.wantExisting == ""; got != want { + t.Errorf("managesRBAC() = %v, want %v", got, want) + } + // The pod must run as whichever ServiceAccount was resolved. + wantPodSA := tt.wantExisting + if wantPodSA == "" { + wantPodSA = d.saName() + } + if got := d.podServiceAccountName(); got != wantPodSA { + t.Errorf("podServiceAccountName() = %q, want %q", got, wantPodSA) + } + + if tt.wantLogSubstr != "" && !strings.Contains(buf.String(), tt.wantLogSubstr) { + t.Errorf("log = %q, want it to contain %q", buf.String(), tt.wantLogSubstr) + } + if tt.notWantLog != "" && strings.Contains(buf.String(), tt.notWantLog) { + t.Errorf("log = %q, want it NOT to contain %q", buf.String(), tt.notWantLog) + } + }) + } +} + +// TestDeploy_ExistingServiceAccountCreatesAndDeletesNoRBAC is the end-to-end +// contract of exact-ServiceAccount mode: aicr will not add or remove +// permissions on a ServiceAccount it did not create. +// +// It asserts the whole chain rather than just the flag — no RBAC object of +// any kind is created, the created-set holds none of those kinds, and +// therefore Cleanup (which builds its delete list from exactly that set) +// leaves the operator's ServiceAccount and everything around it alone. The +// Cleanup assertion is deliberately not "cleanup skips these kinds": there is +// no such branch in Cleanup, and the point is that none is needed. +func TestDeploy_ExistingServiceAccountCreatesAndDeletesNoRBAC(t *testing.T) { + ctx := context.Background() + captureLogs(t) + + const saName = "irsa-snapshotter" + const roleARN = "arn:aws:iam::123456789012:role/aicr-snapshot" + + clientset := fake.NewClientset() + allowAllPermissionChecks(clientset) + if _, err := clientset.CoreV1().ServiceAccounts(testNamespace).Create(ctx, &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: saName, + Namespace: testNamespace, + Annotations: map[string]string{"eks.amazonaws.com/role-arn": roleARN}, + }, + }, metav1.CreateOptions{}); err != nil { + t.Fatalf("seeding ServiceAccount: %v", err) + } + + d := NewDeployer(clientset, Config{ + Namespace: testNamespace, + ServiceAccountName: saName, + Image: "aicr:test", + RunID: testRunID, + DiscoverNetwork: true, + }) + if err := d.Deploy(ctx); err != nil { + t.Fatalf("Deploy() error = %v", err) + } + + // The Job — the one object this run still owns — must run as the + // operator's ServiceAccount verbatim, not as "-". + job, err := clientset.BatchV1().Jobs(testNamespace).Get(ctx, d.jobName(), metav1.GetOptions{}) + if err != nil { + t.Fatalf("Job not created: %v", err) + } + if got := job.Spec.Template.Spec.ServiceAccountName; got != saName { + t.Errorf("pod ServiceAccountName = %q, want %q", got, saName) + } + + // Nothing run-scoped exists for any RBAC kind. + if sas, listErr := clientset.CoreV1().ServiceAccounts(testNamespace).List(ctx, metav1.ListOptions{}); listErr != nil { + t.Fatalf("listing ServiceAccounts: %v", listErr) + } else if len(sas.Items) != 1 || sas.Items[0].Name != saName { + t.Errorf("ServiceAccounts = %v, want only the pre-existing %q", saNames(sas.Items), saName) + } + assertNoRBACObjects(ctx, t, clientset) + + // The created-set is what Cleanup deletes from, so its contents are + // the actual guarantee. + for _, kind := range []string{kindServiceAccount, kindRole, kindRoleBinding, kindClusterRole, kindClusterRoleBinding} { + if d.hasCreated(kind) { + t.Errorf("created-set holds a %s entry; Cleanup would delete an object this run did not create", kind) + } + } + + if cleanupErr := d.Cleanup(ctx, CleanupOptions{Enabled: true}); cleanupErr != nil { + t.Fatalf("Cleanup() error = %v", cleanupErr) + } + + sa, err := clientset.CoreV1().ServiceAccounts(testNamespace).Get(ctx, saName, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Cleanup deleted the operator's ServiceAccount: %v", err) + } + if sa.Annotations["eks.amazonaws.com/role-arn"] != roleARN { + t.Errorf("IRSA annotation = %q, want %q", sa.Annotations["eks.amazonaws.com/role-arn"], roleARN) + } + assertNoRBACObjects(ctx, t, clientset) +} + +// TestDeploy_AbsentServiceAccountKeepsPrefixBehavior pins the other half of +// exact-if-exists: a --service-account-name naming nothing that exists is +// still a prefix, and the run creates and owns the full run-scoped RBAC set +// exactly as it did before. +func TestDeploy_AbsentServiceAccountKeepsPrefixBehavior(t *testing.T) { + ctx := context.Background() + captureLogs(t) + + clientset := fake.NewClientset() + allowAllPermissionChecks(clientset) + + d := NewDeployer(clientset, Config{ + Namespace: testNamespace, + ServiceAccountName: "irsa-snapshotter", + Image: "aicr:test", + RunID: testRunID, + }) + if err := d.Deploy(ctx); err != nil { + t.Fatalf("Deploy() error = %v", err) + } + + scoped := "irsa-snapshotter-" + testRunID + if d.saName() != scoped { + t.Fatalf("saName() = %q, want %q", d.saName(), scoped) + } + if _, err := clientset.CoreV1().ServiceAccounts(testNamespace).Get(ctx, scoped, metav1.GetOptions{}); err != nil { + t.Errorf("run-scoped ServiceAccount %q not created: %v", scoped, err) + } + if _, err := clientset.RbacV1().Roles(testNamespace).Get(ctx, scoped, metav1.GetOptions{}); err != nil { + t.Errorf("run-scoped Role %q not created: %v", scoped, err) + } + if _, err := clientset.RbacV1().RoleBindings(testNamespace).Get(ctx, scoped, metav1.GetOptions{}); err != nil { + t.Errorf("run-scoped RoleBinding %q not created: %v", scoped, err) + } + if _, err := clientset.RbacV1().ClusterRoles().Get(ctx, d.clusterRoleName(), metav1.GetOptions{}); err != nil { + t.Errorf("run-scoped ClusterRole %q not created: %v", d.clusterRoleName(), err) + } + if _, err := clientset.RbacV1().ClusterRoleBindings().Get(ctx, d.clusterRoleName(), metav1.GetOptions{}); err != nil { + t.Errorf("run-scoped ClusterRoleBinding %q not created: %v", d.clusterRoleName(), err) + } + + job, err := clientset.BatchV1().Jobs(testNamespace).Get(ctx, d.jobName(), metav1.GetOptions{}) + if err != nil { + t.Fatalf("Job not created: %v", err) + } + if got := job.Spec.Template.Spec.ServiceAccountName; got != scoped { + t.Errorf("pod ServiceAccountName = %q, want %q", got, scoped) + } +} + +// saNames projects a ServiceAccount list to its names for error messages. +func saNames(items []corev1.ServiceAccount) []string { + out := make([]string, 0, len(items)) + for i := range items { + out = append(out, items[i].Name) + } + return out +} + +// assertNoRBACObjects fails when any Role, RoleBinding, ClusterRole or +// ClusterRoleBinding exists in the cluster. +func assertNoRBACObjects(ctx context.Context, t *testing.T, clientset *fake.Clientset) { + t.Helper() + roles, err := clientset.RbacV1().Roles(testNamespace).List(ctx, metav1.ListOptions{}) + if err != nil { + t.Fatalf("listing Roles: %v", err) + } + if len(roles.Items) != 0 { + t.Errorf("Roles = %d, want 0", len(roles.Items)) + } + rbs, err := clientset.RbacV1().RoleBindings(testNamespace).List(ctx, metav1.ListOptions{}) + if err != nil { + t.Fatalf("listing RoleBindings: %v", err) + } + if len(rbs.Items) != 0 { + t.Errorf("RoleBindings = %d, want 0", len(rbs.Items)) + } + crs, err := clientset.RbacV1().ClusterRoles().List(ctx, metav1.ListOptions{}) + if err != nil { + t.Fatalf("listing ClusterRoles: %v", err) + } + if len(crs.Items) != 0 { + t.Errorf("ClusterRoles = %d, want 0", len(crs.Items)) + } + crbs, err := clientset.RbacV1().ClusterRoleBindings().List(ctx, metav1.ListOptions{}) + if err != nil { + t.Fatalf("listing ClusterRoleBindings: %v", err) + } + if len(crbs.Items) != 0 { + t.Errorf("ClusterRoleBindings = %d, want 0", len(crbs.Items)) + } +} + +// allowAllPermissionChecks makes every access review succeed so a test +// exercises Deploy past its Step 0 pre-flight. Both kinds are answered: +// SelfSubjectAccessReview covers the caller's verbs, and SubjectAccessReview +// covers the agent ServiceAccount's own rules, which the gate checks in +// exact-ServiceAccount mode. +func allowAllPermissionChecks(clientset *fake.Clientset) { + clientset.PrependReactor("create", "selfsubjectaccessreviews", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, &authv1.SelfSubjectAccessReview{ + Status: authv1.SubjectAccessReviewStatus{Allowed: true, Reason: "pre-flight verb set granted"}, + }, nil + }) + clientset.PrependReactor("create", "subjectaccessreviews", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, &authv1.SubjectAccessReview{ + Status: authv1.SubjectAccessReviewStatus{Allowed: true, Reason: "ServiceAccount rules granted"}, + }, nil + }) +} + +// TestDeploy_FailsWhenServiceAccountGetForbidden is the end-to-end shape of +// the closed hole: an explicitly-named ServiceAccount that cannot be read +// must stop the run, not be silently reinterpreted as a name prefix. +// +// The SelfSubjectAccessReview reactor says `serviceaccounts: get` is +// allowed while the Get itself returns Forbidden — the authorizer and the +// apiserver disagreeing. That is precisely the state the run must not guess +// through, because guessing "prefix" deploys the agent under a generated +// ServiceAccount carrying none of the named account's cloud credentials. +// ServiceAccountName is set because the Get is issued only when it is. +func TestDeploy_FailsWhenServiceAccountGetForbidden(t *testing.T) { + ctx := context.Background() + captureLogs(t) + + clientset := fake.NewClientset() + allowAllPermissionChecks(clientset) + clientset.PrependReactor("get", "serviceaccounts", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewForbidden( + schema.GroupResource{Group: "", Resource: resourceServiceAccounts}, testName, + stderrors.New(`User "snapshot-runner" cannot get resource "serviceaccounts"`)) + }) + + d := NewDeployer(clientset, Config{ + Namespace: testNamespace, + ServiceAccountName: testName, + Image: "aicr:test", + RunID: testRunID, + }) + + err := d.Deploy(ctx) + if err == nil { + t.Fatal("Deploy() error = nil; an unreadable, explicitly-named ServiceAccount must fail the run") + } + if !stderrors.Is(err, aicrerrors.New(aicrerrors.ErrCodeUnauthorized, "")) { + t.Errorf("Deploy() error code = %v, want ErrCodeUnauthorized", err) + } + if !strings.Contains(err.Error(), "refusing to guess") { + t.Errorf("Deploy() error = %v, want it to say the run refuses to guess the mode", err) + } + + // Nothing may have been written: the gate fails before Deploy's ensure* + // chain runs. Read through the tracker, not the clientset — the reactor + // above stands in for an identity that cannot read ServiceAccounts at + // all, and that must not also blind the assertion. + if _, jobErr := clientset.BatchV1().Jobs(testNamespace).Get(ctx, "aicr-"+testRunID, metav1.GetOptions{}); !apierrors.IsNotFound(jobErr) { + t.Errorf("Job Get error = %v, want NotFound (no Job may be created)", jobErr) + } + saGVR := corev1.SchemeGroupVersion.WithResource(resourceServiceAccounts) + if _, saErr := clientset.Tracker().Get(saGVR, testNamespace, "aicr-"+testRunID); !apierrors.IsNotFound(saErr) { + t.Errorf("run-scoped ServiceAccount Get error = %v, want NotFound", saErr) + } + assertNoRBACObjects(ctx, t, clientset) +} diff --git a/pkg/k8s/agent/types.go b/pkg/k8s/agent/types.go index af16dedcd..e755bdcea 100644 --- a/pkg/k8s/agent/types.go +++ b/pkg/k8s/agent/types.go @@ -15,38 +15,121 @@ package agent import ( + "sync" + + "github.com/NVIDIA/aicr/pkg/k8s/labels" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" ) -// clusterRoleName is the name used for the ClusterRole and ClusterRoleBinding. -const clusterRoleName = "aicr-node-reader" - // Standard Kubernetes recommended labels applied to all agent-managed // resources. Centralized here so selectors and resource templates stay in sync. const ( - labelAppName = "app.kubernetes.io/name" - labelAppManagedBy = "app.kubernetes.io/managed-by" - appName = "aicr" - agentLabelSelector = labelAppName + "=" + appName + labelAppName = "app.kubernetes.io/name" + labelAppManagedBy = "app.kubernetes.io/managed-by" + appName = "aicr" ) +// Kind labels recorded in Deployer.created for each run-owned object type. +// Cleanup dispatches on these to call the matching resource-specific delete. +const ( + kindServiceAccount = "ServiceAccount" + kindRole = "Role" + kindRoleBinding = "RoleBinding" + kindClusterRole = "ClusterRole" + kindClusterRoleBinding = "ClusterRoleBinding" + kindJob = "Job" + kindConfigMap = "ConfigMap" +) + +// createdObject records one object this Deployer created (or, for the +// staging ConfigMap it does not itself create, observed itself owning) so +// Cleanup can delete exactly this set instead of deriving a delete list from +// configured names. name is the run-scoped name the object was created +// under; uid pins the eventual delete via metav1.Preconditions so a +// same-named object belonging to a different run is never collected. +// +// confirmed records whether ownership was established at creation time — a +// Create response came back naming this object. It is the discriminator +// Cleanup keys off, NOT uid == "": ownership is a property of how the entry +// was obtained, and a run must never infer it from a name plus a freshly-read +// UID (a Get followed by a UID-pinned delete proves only that the object did +// not change between the two calls). +// +// An entry stays unconfirmed when recordIntent added it and no Create +// response ever arrived. Cleanup must then re-establish ownership from the +// live object's own labels before deleting it — see resolveIntentUID. +type createdObject struct { + kind string + name string + uid types.UID + confirmed bool +} + // Config holds the configuration for deploying the agent. type Config struct { - Namespace string + Namespace string + + // ServiceAccountName is exact-if-exists, and therefore carries two + // meanings resolved once per Deploy (see resolveServiceAccount): + // + // - A ServiceAccount of exactly this name already exists in + // Namespace: the agent pod runs as it verbatim and this run + // creates NO ServiceAccount, Role, RoleBinding, ClusterRole or + // ClusterRoleBinding. aicr adds and removes no permissions on an + // identity it did not create, and cleanup deletes none of them. + // This is what keeps a ServiceAccount carrying IRSA or GKE + // Workload Identity annotations usable: both providers pin trust + // to the ServiceAccount NAME, which a run-scoped name can never + // satisfy. Generate the RBAC that grants such a ServiceAccount + // the agent's permissions with + // BuildServiceAccountRoleManifests, then apply it out of band. + // - Otherwise: a prefix. The run creates "-" and + // the full run-scoped RBAC set, and deletes them at cleanup. + // + // Empty falls back to NameBase and is never probed for existence — + // the fallback base is aicr's own default, not a name the caller + // asked for, so a stray ServiceAccount sitting at it must not + // silently capture the run. + // + // Exact mode waives per-run permission isolation: concurrent runs + // sharing the ServiceAccount share its grants, and grants provisioned + // for DiscoverNetwork persist beyond any one run. ServiceAccountName string - JobName string - Image string - ImagePullSecrets []string - NodeSelector map[string]string - Tolerations []corev1.Toleration - Output string - Debug bool - Privileged bool // If true, run with privileged security context (required for GPU/SystemD collectors) - RequireGPU bool // If true, request nvidia.com/gpu resource (required for CDI environments) - RuntimeClassName string // If set, use this runtimeClassName on the pod and inject NVIDIA_VISIBLE_DEVICES=all (alternative to RequireGPU) - MaxNodesPerEntry int // Max node names per topology entry (0 = unlimited) - OS string // Recipe OS criteria value. When set to oskind.Talos, systemd hostPath mounts are skipped and the in-pod agent uses the Talos service backend. + + JobName string + + // RunID scopes every resource this Deployer creates to a single run, + // so concurrent snapshot-agent runs never collide on a shared resource + // name. Callers generate it with runid.Generate() before deploying. + // + // Required, and validated by Deploy before any object is created: it + // is folded into every run-owned name, so it must be a DNS-1123 label + // (lowercase alphanumerics and "-", starting and ending alphanumeric, + // at most 63 characters). Anything else fails with + // errors.ErrCodeInvalidRequest. + RunID string + + // NameBase prefixes generated resource names. It applies per name, + // not all-or-nothing: jobName() falls back to it when JobName is + // empty, and saName() (which also names the Role and RoleBinding) + // falls back to it when ServiceAccountName is empty — so setting only + // one of the two leaves NameBase governing the other. Defaults to + // "aicr" when empty. + NameBase string + + Image string + ImagePullSecrets []string + NodeSelector map[string]string + Tolerations []corev1.Toleration + Output string + Debug bool + Privileged bool // If true, run with privileged security context (required for GPU/SystemD collectors) + RequireGPU bool // If true, request nvidia.com/gpu resource (required for CDI environments) + RuntimeClassName string // If set, use this runtimeClassName on the pod and inject NVIDIA_VISIBLE_DEVICES=all (alternative to RequireGPU) + MaxNodesPerEntry int // Max node names per topology entry (0 = unlimited) + OS string // Recipe OS criteria value. When set to oskind.Talos, systemd hostPath mounts are skipped and the in-pod agent uses the Talos service backend. // ClusterConfigPath, when set, forwards to the in-pod network // collector via AICR_CLUSTER_CONFIG_PATH so it ingests an existing @@ -78,12 +161,38 @@ type Config struct { // caller did not already supply that key — so a caller can request // e.g. nvidia.com/gpu=4 alongside RequireGPU and keep their value. Limits corev1.ResourceList + + // OwnsOutputConfigMap is true when Output names the staging ConfigMap + // this Deployer's own Job writes (the default run-scoped + // `cm:///` URI), rather than a ConfigMap + // the caller supplied out of band via a hand-written `cm://` Output + // URI. GetSnapshot enters the ConfigMap into the created-set for + // Cleanup only when this is true — a caller-supplied ConfigMap is + // the caller's artifact and must never be deleted by this Deployer. + OwnsOutputConfigMap bool } // Deployer manages the deployment and lifecycle of the agent Job. type Deployer struct { clientset kubernetes.Interface config Config + + // mu guards created and existingSA. Deploy's ensure* steps run + // sequentially today, but GetSnapshot (which records the staging + // ConfigMap) can be invoked from a different goroutine than Deploy, + // and Cleanup reads the created-set while a caller could still be + // recording into it, so every access is mutex-guarded. + mu sync.Mutex + created []createdObject + + // existingSA is the exact, operator-named ServiceAccount this run + // runs as instead of creating its own, or "" in prefix mode. It is + // resolved once by resolveServiceAccount at the top of Deploy and + // read afterwards by the Job builder. It shares mu with created + // rather than carrying its own: the Deployer is reachable from the + // caller's log-streaming and cleanup goroutines, so a field Deploy + // writes must not be read unsynchronized from any of them. + existingSA string } // NewDeployer creates a new agent Deployer with the given configuration. @@ -94,6 +203,198 @@ func NewDeployer(clientset kubernetes.Interface, config Config) *Deployer { } } +// objectLabels returns the standard label set applied to every run-owned +// object this Deployer creates: the ServiceAccount, Role, RoleBinding, +// ClusterRole, ClusterRoleBinding, Job, and the Job's pod template. Each +// call returns a fresh map so callers attaching it to two objects (e.g. a +// Job and its pod template) never alias the same underlying map. +func (d *Deployer) objectLabels() map[string]string { + return map[string]string{ + labels.Name: labels.ValueAICR, + labels.ManagedBy: labels.ValueAICR, + labels.Component: labels.ValueSnapshotAgent, + labels.RunID: d.config.RunID, + } +} + +// setExistingServiceAccount records that this run uses the operator's +// already-existing ServiceAccount verbatim rather than creating its own. +// Called once, by resolveServiceAccount. Safe for concurrent use. +func (d *Deployer) setExistingServiceAccount(name string) { + d.mu.Lock() + defer d.mu.Unlock() + d.existingSA = name +} + +// existingServiceAccount returns the operator-supplied ServiceAccount this +// run adopted verbatim, or "" when the run creates and owns its own +// run-scoped one. Safe for concurrent use. +func (d *Deployer) existingServiceAccount() string { + d.mu.Lock() + defer d.mu.Unlock() + return d.existingSA +} + +// managesRBAC reports whether this run creates (and therefore later deletes) +// the ServiceAccount, Role, RoleBinding, ClusterRole and ClusterRoleBinding. +// False in exact-ServiceAccount mode: aicr adds and removes no permissions +// on an identity it did not create, so none of those objects is created, +// none enters the created-set, and Cleanup consequently has nothing of those +// kinds to delete. +func (d *Deployer) managesRBAC() bool { + return d.existingServiceAccount() == "" +} + +// recordIntent enters a run-owned object into the created-set with the zero +// UID, immediately BEFORE its Create call. It closes the window in which a +// committed Create whose response never arrives (client timeout, apiserver +// rollout, LB 502/504, connection reset, context cancellation in the +// response window) leaves an object nothing will ever delete: the ensure* +// call returns a non-AlreadyExists error, Deploy aborts, and the deferred +// Cleanup — enabled by default — would otherwise never learn the object +// exists. Because the name is run-unique, no later run reclaims it either, +// so the orphan is permanent. +// +// The entry is unconfirmed: no Create response ever named the object, so the +// run-scoped name is the only thing tying it to this run, and a name proves +// nothing about who created what currently sits at it. Cleanup therefore +// re-establishes ownership from the live object's labels before deleting it +// (resolveIntentUID) rather than deleting by bare name. An AlreadyExists +// response is the one case that proves the object is NOT ours, so every +// ensure* discards the intent on that branch (see discardIntent). +// Safe for concurrent use. +func (d *Deployer) recordIntent(kind, name string) { + d.mu.Lock() + defer d.mu.Unlock() + d.created = append(d.created, createdObject{kind: kind, name: name}) +} + +// discardIntent drops the unconfirmed entry recordIntent added for (kind, +// name). Called only when a Create returns AlreadyExists: the object at that +// name exists but this run did not create it, so it must not enter this +// run's delete list. Safe for concurrent use. +func (d *Deployer) discardIntent(kind, name string) { + d.mu.Lock() + defer d.mu.Unlock() + for i := range d.created { + if d.created[i].kind == kind && d.created[i].name == name && !d.created[i].confirmed { + d.created = append(d.created[:i], d.created[i+1:]...) + return + } + } +} + +// recordCreated confirms a run-owned object in the created-set. Cleanup +// builds its UID-pinned delete list from exactly this set, so every ensure* +// call that successfully creates an object — and GetSnapshot, for the +// staging ConfigMap it observes but does not itself create — must call this +// on success. +// +// It marks the entry confirmed — this is the point at which this run's +// ownership of the object is established, and the only place that flag is +// set. +// +// It upserts: when recordIntent already entered (kind, name) unconfirmed, the +// observed UID is written onto that entry rather than appended as a second +// one, so the set holds one entry per object and jobUID() sees the real Job +// UID. Safe for concurrent use. +func (d *Deployer) recordCreated(kind, name string, uid types.UID) { + d.mu.Lock() + defer d.mu.Unlock() + for i := range d.created { + if d.created[i].kind == kind && d.created[i].name == name && !d.created[i].confirmed { + d.created[i].uid = uid + d.created[i].confirmed = true + return + } + } + d.created = append(d.created, createdObject{kind: kind, name: name, uid: uid, confirmed: true}) +} + +// needsStagingConfigMapSweep reports whether Cleanup must look for a staging +// ConfigMap that is not in the created-set, given that set. Both halves of +// the answer are read from the single snapshot Cleanup already took, rather +// than re-entering the mutex: reading the set twice would let a recordCreated +// landing between the two reads produce a snapshot that misses the staging +// ConfigMap while the second read reports it present, skipping both the +// created-set delete and the sweep and leaking the object. +// +// Two conditions must hold. +// +// No ConfigMap entry: an entry means getSnapshotFromConfigMap already +// observed the object's UID, so Cleanup deletes it from the created-set and +// the sweep would be a redundant second delete of the same name. +// +// A CONFIRMED Job entry: the staging ConfigMap is written only by this run's +// in-pod agent, so this run cannot have produced one unless the apiserver +// confirmed the Job that runs that agent. Anything sitting at the staging +// name when no confirmed Job exists belongs to someone else — notably a +// caller that reused another run's RunID (Config.RunID is public SDK surface, +// deliberately settable for pinned e2e/chainsaw runs) and failed on its first +// AlreadyExists before recording anything. Sweeping there would delete the +// first run's live staging ConfigMap. +// +// An unconfirmed (recordIntent-only) Job entry deliberately does not qualify. +// It cannot be told apart from a duplicate-RunID collision whose AlreadyExists +// never came back, and the window it forfeits is empty in practice: Deploy +// aborts the moment that Create fails and Cleanup runs seconds later, far +// short of the time the agent needs to collect a snapshot and write it. +func needsStagingConfigMapSweep(objs []createdObject) bool { + var jobConfirmed bool + for _, o := range objs { + switch o.kind { + case kindConfigMap: + return false + case kindJob: + jobConfirmed = jobConfirmed || o.confirmed + } + } + return jobConfirmed +} + +// createdSnapshot returns a defensive copy of the created-set taken under +// lock. Callers must not read d.created directly. +func (d *Deployer) createdSnapshot() []createdObject { + d.mu.Lock() + defer d.mu.Unlock() + out := make([]createdObject, len(d.created)) + copy(out, d.created) + return out +} + +// jobUID returns the UID of the Job this Deployer created, or the zero UID +// if Deploy has not (yet) reached the Job-create step — including when +// Deploy failed before getting there, and while the Job's Create is in +// flight (recordIntent has entered the name but no UID is known yet). Pod +// selection (see ownedByJob in wait.go) authorizes candidates against +// exactly this UID. +func (d *Deployer) jobUID() types.UID { + d.mu.Lock() + defer d.mu.Unlock() + for _, c := range d.created { + if c.kind == kindJob { + return c.uid + } + } + return "" +} + +// hasCreated reports whether the created-set already holds an object of +// kind. Cleanup does NOT use it — it derives the same answer from the single +// snapshot it already took, via containsKind, so the decision cannot straddle +// two lock acquisitions. This remains as the locked accessor for callers +// (tests) that hold no snapshot. Safe for concurrent use. +func (d *Deployer) hasCreated(kind string) bool { + d.mu.Lock() + defer d.mu.Unlock() + for _, c := range d.created { + if c.kind == kind { + return true + } + } + return false +} + // CleanupOptions controls what resources to remove during cleanup. type CleanupOptions struct { Enabled bool // If true, removes Job and all RBAC resources diff --git a/pkg/k8s/agent/wait.go b/pkg/k8s/agent/wait.go index f0ecae931..ebb497082 100644 --- a/pkg/k8s/agent/wait.go +++ b/pkg/k8s/agent/wait.go @@ -18,17 +18,21 @@ import ( "context" "fmt" "io" + "log/slog" "time" "github.com/NVIDIA/aicr/pkg/errors" + "github.com/NVIDIA/aicr/pkg/k8s/labels" "github.com/NVIDIA/aicr/pkg/k8s/pod" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ) // waitForJobCompletion waits for the Job to complete successfully or fail. func (d *Deployer) waitForJobCompletion(ctx context.Context, timeout time.Duration) error { - return pod.WaitForJobCompletion(ctx, d.clientset, d.config.Namespace, d.config.JobName, timeout) + return pod.WaitForJobCompletion(ctx, d.clientset, d.config.Namespace, d.jobName(), timeout) } // getSnapshotFromConfigMap retrieves the snapshot data from ConfigMap. @@ -45,6 +49,16 @@ func (d *Deployer) getSnapshotFromConfigMap(ctx context.Context) ([]byte, error) return nil, errors.Wrap(errors.ErrCodeNotFound, fmt.Sprintf("failed to get ConfigMap %s/%s", namespace, name), err) } + // The staging ConfigMap is written by the in-pod agent, not this + // controller, so there is no Create response to record a UID from — + // this Get is the only point where we observe it. Record it into the + // created-set (for a UID-pinned Cleanup delete) only when this + // Deployer owns the output: a caller-supplied `cm://` Output URI + // names the caller's own artifact, which must never be deleted here. + if d.config.OwnsOutputConfigMap { + d.recordCreated(kindConfigMap, name, cm.UID) + } + // Extract snapshot data snapshot, ok := cm.Data["snapshot.yaml"] if !ok { @@ -54,6 +68,70 @@ func (d *Deployer) getSnapshotFromConfigMap(ctx context.Context) ([]byte, error) return []byte(snapshot), nil } +// deleteStagingConfigMap deletes the staging ConfigMap in d.config.Namespace +// by name, pinning the delete to uid so a same-named ConfigMap belonging to +// a different run is never collected. If the ConfigMap is already gone, or +// uid no longer matches (already replaced, not ours), this is a no-op +// (idempotent). Only reached via Cleanup's created-set dispatch, which only +// contains an entry for this ConfigMap when Config.OwnsOutputConfigMap was +// true at record time (see getSnapshotFromConfigMap). +func (d *Deployer) deleteStagingConfigMap(ctx context.Context, name string, uid types.UID) error { + err := d.clientset.CoreV1().ConfigMaps(d.config.Namespace). + Delete(ctx, name, metav1.DeleteOptions{Preconditions: uidPreconditions(uid)}) + return ignoreNotFoundOrConflict(err) +} + +// deleteUnrecordedStagingConfigMap deletes this run's staging ConfigMap when +// the in-pod agent wrote it but the controller never observed it — the run +// failed between the agent's write and getSnapshotFromConfigMap, so there is +// no recorded UID to pin the delete to. It Gets the ConfigMap first and pins +// the delete to the UID it observes there. +// +// Ownership is NOT inferred from the name. Config.RunID is caller-settable +// (public SDK surface), so two runs can resolve the same staging name, and a +// Get plus a UID-pinned delete would only prove the object did not change +// between the two calls. Cleanup gates this call on a confirmed Job entry +// (needsStagingConfigMapSweep) — this run cannot have produced a staging +// ConfigMap without the Job whose in-pod agent writes it — and this function +// re-checks the object it finds. +// +// That re-check is app.kubernetes.io/name only. The staging ConfigMap is +// written by pkg/serializer's ConfigMapWriter from inside the pod, which +// stamps name/component/version and — unlike objectLabels() — no +// aicr.run/run-id and no managed-by, so createdByThisRun() does not apply to +// it. The check still rules out an unrelated ConfigMap parked at this name, and +// component is deliberately not required: its value is the snapshot Kind +// written by the agent image, which may be a different aicr version than the +// controller. +// +// Only called from Cleanup, and only when Config.OwnsOutputConfigMap is true. +// That flag means Output is the run-scoped staging URI pkg/snapshotter builds +// from StagingConfigMapName, so d.stagingConfigMapName() names exactly that +// object in d.config.Namespace. A caller that sets the flag while pointing +// Output elsewhere simply finds nothing here (NotFound is a no-op) — this +// deletes nothing it does not own. +func (d *Deployer) deleteUnrecordedStagingConfigMap(ctx context.Context) error { + name := d.stagingConfigMapName() + cm, err := d.clientset.CoreV1().ConfigMaps(d.config.Namespace).Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + // The Job never got far enough to write it: nothing to clean up. + return nil + } + if err != nil { + return errors.Wrap(errors.ErrCodeInternal, + fmt.Sprintf("failed to get staging ConfigMap %s/%s", d.config.Namespace, name), err) + } + if cm.Labels[labels.Name] != labels.ValueAICR { + slog.Warn("cleanup left behind the ConfigMap at this run's staging name: it does not look like an aicr snapshot artifact, so this run did not write it", + slog.String(attrNamespace, d.config.Namespace), + slog.String(attrName, name), + slog.String("uid", string(cm.UID)), + slog.String(attrRunID, d.config.RunID)) + return nil + } + return d.deleteStagingConfigMap(ctx, cm.Name, cm.UID) +} + // StreamLogs streams logs from the Job's Pod to the provided writer. // It will follow the logs until the context is canceled. // Returns when the context is canceled or an error occurs. @@ -108,28 +186,67 @@ func (d *Deployer) WaitForPodReady(ctx context.Context, timeout time.Duration) e return pod.WaitForPodReady(ctx, d.clientset, d.config.Namespace, podName, remainingTimeout) } +// podLabelSelector returns the List/Watch label selector for this run's +// agent pod: app name plus RunID. This only narrows the candidate set — +// pod labels, including a forged aicr.run/run-id, are writable by anything +// that can update pods in the namespace. ownedByJob (applied by pickLivePod +// and the watch loop in findOrWatchPodName) is what authorizes selection. +func (d *Deployer) podLabelSelector() string { + return fmt.Sprintf("%s=%s,%s=%s", labels.Name, labels.ValueAICR, labels.RunID, d.config.RunID) +} + +// ownedByJob reports whether pod is controlled by the Job with jobUID. +// Pod labels are writable by anything that can update pods in the +// namespace, so the controlling ownerReference — not +// batch.kubernetes.io/controller-uid — is what authorizes selection. +// Callers fall back to label-only narrowing (see pickLivePod) instead of +// calling this with the zero UID; the guard below is defense-in-depth so +// the predicate itself fails closed if ever called standalone. +func ownedByJob(pod *corev1.Pod, jobUID types.UID) bool { + if jobUID == "" { + // A zero Job UID means ownership is not yet establishable — fail + // closed rather than risk matching a pod whose own ownerRef UID + // happens to also be empty. + return false + } + for i := range pod.OwnerReferences { + ref := &pod.OwnerReferences[i] + if ref.Kind == kindJob && ref.UID == jobUID && ref.Controller != nil && *ref.Controller { + return true + } + } + return false +} + // findPodName finds the pod name by label selector for this Job. // One-shot: returns ErrCodeNotFound if no pod is currently labeled. // Skips pods that are being deleted or have already failed so an // orphaned pod from a prior run is not selected. func (d *Deployer) findPodName(ctx context.Context) (string, error) { pods, err := d.clientset.CoreV1().Pods(d.config.Namespace).List(ctx, metav1.ListOptions{ - LabelSelector: agentLabelSelector, + LabelSelector: d.podLabelSelector(), }) if err != nil { return "", errors.Wrap(errors.ErrCodeInternal, "failed to list Pods", err) } - name := pickLivePod(pods.Items) + name := pickLivePod(pods.Items, d.jobUID()) if name == "" { - return "", errors.New(errors.ErrCodeNotFound, fmt.Sprintf("no Pods found for Job %s", d.config.JobName)) + return "", errors.New(errors.ErrCodeNotFound, fmt.Sprintf("no Pods found for Job %s", d.jobName())) } return name, nil } // pickLivePod returns the name of the youngest pod that is neither being -// deleted nor in a Failed phase. Returns "" if no usable pod exists. -func pickLivePod(pods []corev1.Pod) string { +// deleted nor in a Failed phase. When jobUID is non-zero, a pod must also be +// owned by that Job (see ownedByJob) — the label selector used to build the +// candidate list only narrows it; the controlling ownerReference is what +// authorizes selection. When jobUID is the zero UID (the Job hasn't been +// recorded yet — WaitForPodReady's watch can start before Deploy returns), +// ownership is not checked here; callers re-check on every call since +// d.jobUID() is queried live, not cached. Returns "" if no usable pod +// exists. +func pickLivePod(pods []corev1.Pod, jobUID types.UID) string { var best *corev1.Pod for i := range pods { p := &pods[i] @@ -139,6 +256,9 @@ func pickLivePod(pods []corev1.Pod) string { if p.Status.Phase == corev1.PodFailed { continue } + if jobUID != "" && !ownedByJob(p, jobUID) { + continue + } if best == nil || p.CreationTimestamp.After(best.CreationTimestamp.Time) { best = p } @@ -153,18 +273,19 @@ func pickLivePod(pods []corev1.Pod) string { // (List), return immediately; otherwise watch for an Added event until ctx is // canceled. func (d *Deployer) findOrWatchPodName(ctx context.Context) (string, error) { + selector := d.podLabelSelector() pods, err := d.clientset.CoreV1().Pods(d.config.Namespace).List(ctx, metav1.ListOptions{ - LabelSelector: agentLabelSelector, + LabelSelector: selector, }) if err != nil { return "", errors.Wrap(errors.ErrCodeInternal, "failed to list Pods", err) } - if name := pickLivePod(pods.Items); name != "" { + if name := pickLivePod(pods.Items, d.jobUID()); name != "" { return name, nil } watcher, err := d.clientset.CoreV1().Pods(d.config.Namespace).Watch(ctx, metav1.ListOptions{ - LabelSelector: agentLabelSelector, + LabelSelector: selector, ResourceVersion: pods.ResourceVersion, }) if err != nil { @@ -182,12 +303,12 @@ func (d *Deployer) findOrWatchPodName(ctx context.Context) (string, error) { // close watch channels without the pod actually failing to // appear. Re-List before declaring failure. pods, listErr := d.clientset.CoreV1().Pods(d.config.Namespace).List(ctx, metav1.ListOptions{ - LabelSelector: agentLabelSelector, + LabelSelector: selector, }) if listErr != nil { return "", errors.Wrap(errors.ErrCodeUnavailable, "Pod watch channel closed and re-List failed", listErr) } - if name := pickLivePod(pods.Items); name != "" { + if name := pickLivePod(pods.Items, d.jobUID()); name != "" { return name, nil } return "", errors.New(errors.ErrCodeUnavailable, "Pod watch channel closed before pod observed") @@ -199,6 +320,12 @@ func (d *Deployer) findOrWatchPodName(ctx context.Context) (string, error) { if p.DeletionTimestamp != nil || p.Status.Phase == corev1.PodFailed { continue } + // jobUID is re-queried per event (not cached at loop entry) so a + // Job UID recorded by Deploy after this watch started is + // honored on the very next event. + if jobUID := d.jobUID(); jobUID != "" && !ownedByJob(p, jobUID) { + continue + } return p.Name, nil } } diff --git a/pkg/k8s/agent/wait_test.go b/pkg/k8s/agent/wait_test.go index 2e29d17b3..8b66fefa3 100644 --- a/pkg/k8s/agent/wait_test.go +++ b/pkg/k8s/agent/wait_test.go @@ -17,15 +17,23 @@ package agent import ( "bytes" "context" + stderrors "errors" "io" + "sync" "testing" "time" + aicrerrors "github.com/NVIDIA/aicr/pkg/errors" + "github.com/NVIDIA/aicr/pkg/k8s/labels" "github.com/NVIDIA/aicr/pkg/k8s/pod" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" ) func TestParseConfigMapName_Extended(t *testing.T) { @@ -265,7 +273,7 @@ func TestDeployer_WaitForPodReady_Extended(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "test-pod", Namespace: ns, - Labels: map[string]string{"app.kubernetes.io/name": "aicr"}, + Labels: map[string]string{labels.Name: labels.ValueAICR, labels.RunID: ""}, }, Status: corev1.PodStatus{ Phase: corev1.PodRunning, @@ -295,7 +303,7 @@ func TestDeployer_WaitForPodReady_Extended(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "test-pod", Namespace: ns, - Labels: map[string]string{"app.kubernetes.io/name": "aicr"}, + Labels: map[string]string{labels.Name: labels.ValueAICR, labels.RunID: ""}, }, Status: corev1.PodStatus{ Phase: corev1.PodFailed, @@ -328,3 +336,391 @@ func TestDeployer_WaitForPodReady_Extended(t *testing.T) { } }) } + +// podWithOwner returns a Pod whose sole OwnerReference is of the given kind, +// uid, and controller flag. Used to exercise ownedByJob's ownership checks +// in isolation from label-based pod selection. +func podWithOwner(kind string, uid types.UID, controller bool) corev1.Pod { + return corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + OwnerReferences: []metav1.OwnerReference{ + { + Kind: kind, + UID: uid, + Controller: &controller, + }, + }, + }, + } +} + +// podWithNilControllerOwner returns a Pod whose sole OwnerReference omits +// the Controller field. Controller is a *bool, so an ownerReference written +// by a client that never set it leaves nil there — the exact case +// ownedByJob's `ref.Controller != nil` guard exists to survive, and one +// podWithOwner cannot produce because it always takes the address of a bool. +func podWithNilControllerOwner(kind string, uid types.UID) corev1.Pod { + return corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + OwnerReferences: []metav1.OwnerReference{ + {Kind: kind, UID: uid}, + }, + }, + } +} + +// podWithForgedLabel returns a Pod with no OwnerReferences but a +// batch.kubernetes.io/controller-uid label set to uid — the label any +// client that can update pods in the namespace could set directly, unlike +// the controller-managed OwnerReferences. ownedByJob must not be fooled by +// it. +func podWithForgedLabel(uid types.UID) corev1.Pod { + return corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + batchv1.ControllerUidLabel: string(uid), + }, + }, + } +} + +func TestOwnedByJob(t *testing.T) { + const want = types.UID("job-uid-1") + tests := []struct { + name string + pod corev1.Pod + jobUID types.UID + ok bool + }{ + {"controller job matching uid", podWithOwner(kindJob, want, true), want, true}, + {"controller job wrong uid", podWithOwner(kindJob, types.UID("other"), true), want, false}, + {"non-controller ref", podWithOwner(kindJob, want, false), want, false}, + // Controller is a *bool: an ownerReference written without it + // must be rejected, not dereferenced. + {"nil Controller on an otherwise matching ref", podWithNilControllerOwner(kindJob, want), want, false}, + {"wrong kind", podWithOwner("ReplicaSet", want, true), want, false}, + {"no owner refs", corev1.Pod{}, want, false}, + {"forged label only", podWithForgedLabel(want), want, false}, + // jobUID == "" must fail closed even when the pod's own ownerRef UID + // also happens to be "" — ownership is never establishable from a + // zero Job UID, regardless of what the pod carries. + {"zero jobUID never matches, even a zero-UID owner ref", podWithOwner(kindJob, "", true), "", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ownedByJob(&tt.pod, tt.jobUID); got != tt.ok { + t.Errorf("ownedByJob() = %v, want %v", got, tt.ok) + } + }) + } +} + +// TestPickLivePod exercises the jobUID-gated ownership filtering pickLivePod +// applies on top of the existing DeletionTimestamp/Failed-phase filtering. +func TestPickLivePod(t *testing.T) { + const jobUID = types.UID("job-uid-1") + + older := metav1.NewTime(time.Unix(1000, 0)) + younger := metav1.NewTime(time.Unix(2000, 0)) + + ownedOlder := podWithOwner(kindJob, jobUID, true) + ownedOlder.Name = "owned-older" + ownedOlder.CreationTimestamp = older + + ownedYounger := podWithOwner(kindJob, jobUID, true) + ownedYounger.Name = "owned-younger" + ownedYounger.CreationTimestamp = younger + + unowned := podWithForgedLabel(jobUID) // forged label, no real ownerRef + unowned.Name = "unowned-forged" + unowned.CreationTimestamp = younger // younger than both owned pods + + deleting := podWithOwner(kindJob, jobUID, true) + deleting.Name = "deleting" + deleting.CreationTimestamp = younger + now := metav1.Now() + deleting.DeletionTimestamp = &now + + failed := podWithOwner(kindJob, jobUID, true) + failed.Name = "failed" + failed.CreationTimestamp = younger + failed.Status.Phase = corev1.PodFailed + + tests := []struct { + name string + pods []corev1.Pod + jobUID types.UID + wantPod string + }{ + { + name: "zero jobUID falls back to youngest live pod regardless of ownership", + pods: []corev1.Pod{ownedOlder, unowned}, + jobUID: "", + wantPod: "unowned-forged", + }, + { + name: "known jobUID rejects unowned pod even if younger", + pods: []corev1.Pod{ownedOlder, unowned}, + jobUID: jobUID, + wantPod: "owned-older", + }, + { + name: "known jobUID picks youngest among owned pods", + pods: []corev1.Pod{ownedOlder, ownedYounger}, + jobUID: jobUID, + wantPod: "owned-younger", + }, + { + name: "known jobUID with only unowned pods returns none", + pods: []corev1.Pod{unowned}, + jobUID: jobUID, + wantPod: "", + }, + { + name: "deleting owned pod is skipped", + pods: []corev1.Pod{deleting, ownedOlder}, + jobUID: jobUID, + wantPod: "owned-older", + }, + { + name: "failed owned pod is skipped", + pods: []corev1.Pod{failed, ownedOlder}, + jobUID: jobUID, + wantPod: "owned-older", + }, + { + name: "no pods", + pods: nil, + jobUID: jobUID, + wantPod: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := pickLivePod(tt.pods, tt.jobUID); got != tt.wantPod { + t.Errorf("pickLivePod() = %q, want %q", got, tt.wantPod) + } + }) + } +} + +// Two Job UIDs for the watch-path ownership tests: A is the run under test, +// B stands in for any concurrent run whose pod could reach this run's watch. +const ( + watchJobUIDA = types.UID("job-uid-a") + watchJobUIDB = types.UID("job-uid-b") +) + +// runLabeledPod returns a Pod carrying d's full label set — so it passes +// d.podLabelSelector() — controlled by the Job with ownerUID. Passing a UID +// other than d's own recorded Job UID produces the imposter: a pod that +// anything able to update pods in the namespace could label as this run's, +// but that this run's Job does not control. +func runLabeledPod(d *Deployer, name string, ownerUID types.UID) *corev1.Pod { + controller := true + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: d.config.Namespace, + Labels: d.objectLabels(), + OwnerReferences: []metav1.OwnerReference{ + { + Kind: kindJob, + Name: "some-other-run-job", + UID: ownerUID, + Controller: &controller, + }, + }, + }, + } +} + +// watchNamespace is the namespace every watch-path subtest deploys into. +const watchNamespace = "watch-ns" + +// watchDeployer returns a Deployer for the watch-path tests with run A's Job +// UID already recorded, so jobUID() is non-zero and the ownership checks in +// findOrWatchPodName are actually exercised rather than skipped. +func watchDeployer(client *fake.Clientset) *Deployer { + d := NewDeployer(client, Config{Namespace: watchNamespace, RunID: testRunID}) + d.recordCreated(kindJob, d.jobName(), watchJobUIDA) + return d +} + +// TestFindOrWatchPodNameAuthorizesByJobOwnership covers the watch-based +// discovery path, which is the PRIMARY production path: pkg/snapshotter calls +// WaitForPodReady immediately after Deploy, before any pod exists, so the fast +// List misses and the watch loop is what actually selects the pod. +// +// Every other test in this package that reaches findOrWatchPodName leaves +// jobUID() empty, which makes both pickLivePod calls AND the per-event guard +// no-ops — so the ownership authorization on this path was previously +// untested. Each subtest here records a real Job UID first. +// +// Fake-clientset limitation this works around: the fake Watch reactor ignores +// ListOptions.LabelSelector entirely, so every emitted event reaches the loop. +// That is fine — and in fact necessary — here, because the property under test +// is that the controlling ownerReference (not the forgeable RunID label) is +// what authorizes selection. The List-side selector filtering IS honored by +// the fake, so the re-List subtests exercise it for real. +func TestFindOrWatchPodNameAuthorizesByJobOwnership(t *testing.T) { + t.Run("watch event for another run's Job is skipped", func(t *testing.T) { + client := fake.NewClientset() + w := watch.NewRaceFreeFake() + client.PrependWatchReactor("pods", k8stesting.DefaultWatchReactor(w, nil)) + + d := watchDeployer(client) + + // The imposter arrives FIRST: a watch loop that returned the first + // event passing the label filter would take it and never see the + // real pod. + w.Add(runLabeledPod(d, "imposter-pod", watchJobUIDB)) + w.Add(runLabeledPod(d, "agent-pod-a", watchJobUIDA)) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + got, err := d.findOrWatchPodName(ctx) + if err != nil { + t.Fatalf("findOrWatchPodName() error = %v", err) + } + if got != "agent-pod-a" { + t.Errorf("findOrWatchPodName() = %q, want %q — a pod controlled by another run's "+ + "Job was selected on the watch path", got, "agent-pod-a") + } + }) + + t.Run("watch event with no ownerReference is skipped", func(t *testing.T) { + client := fake.NewClientset() + w := watch.NewRaceFreeFake() + client.PrependWatchReactor("pods", k8stesting.DefaultWatchReactor(w, nil)) + + d := watchDeployer(client) + + // Labels alone, no controlling ownerReference at all — the shape a + // caller with pods/update in the namespace can produce directly. + orphan := runLabeledPod(d, "orphan-pod", watchJobUIDA) + orphan.OwnerReferences = nil + w.Add(orphan) + w.Add(runLabeledPod(d, "agent-pod-a", watchJobUIDA)) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + got, err := d.findOrWatchPodName(ctx) + if err != nil { + t.Fatalf("findOrWatchPodName() error = %v", err) + } + if got != "agent-pod-a" { + t.Errorf("findOrWatchPodName() = %q, want %q", got, "agent-pod-a") + } + }) + + t.Run("closed watch channel re-Lists and authorizes the re-Listed pod", func(t *testing.T) { + client := fake.NewClientset() + d := watchDeployer(client) + + // The fast-path List must miss so the watch is reached at all; the + // re-List after the channel closes then returns both pods. + installStagedPodLister(client, func(call int) []corev1.Pod { + if call == 1 { + return nil + } + return []corev1.Pod{ + *runLabeledPod(d, "imposter-pod", watchJobUIDB), + *runLabeledPod(d, "agent-pod-a", watchJobUIDA), + } + }) + + w := watch.NewRaceFreeFake() + w.Stop() // apiserver hiccup / LB drop: channel closed, no event + client.PrependWatchReactor("pods", k8stesting.DefaultWatchReactor(w, nil)) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + got, err := d.findOrWatchPodName(ctx) + if err != nil { + t.Fatalf("findOrWatchPodName() error = %v — a closed watch channel must re-List "+ + "before declaring failure", err) + } + if got != "agent-pod-a" { + t.Errorf("findOrWatchPodName() = %q, want %q — the re-List branch must apply the "+ + "same ownership check", got, "agent-pod-a") + } + }) + + t.Run("closed watch channel with only a foreign pod fails closed", func(t *testing.T) { + client := fake.NewClientset() + d := watchDeployer(client) + + installStagedPodLister(client, func(call int) []corev1.Pod { + if call == 1 { + return nil + } + return []corev1.Pod{*runLabeledPod(d, "imposter-pod", watchJobUIDB)} + }) + + w := watch.NewRaceFreeFake() + w.Stop() + client.PrependWatchReactor("pods", k8stesting.DefaultWatchReactor(w, nil)) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + got, err := d.findOrWatchPodName(ctx) + if err == nil { + t.Fatalf("findOrWatchPodName() = %q, nil error; a pod owned by another run's Job "+ + "must not satisfy the re-List branch", got) + } + if !stderrors.Is(err, aicrerrors.New(aicrerrors.ErrCodeUnavailable, "")) { + t.Errorf("error = %v, want code ErrCodeUnavailable", err) + } + }) + + t.Run("closed watch channel surfaces a failed re-List", func(t *testing.T) { + client := fake.NewClientset() + d := watchDeployer(client) + + var mu sync.Mutex + var calls int + client.PrependReactor("list", "pods", func(k8stesting.Action) (bool, runtime.Object, error) { + mu.Lock() + defer mu.Unlock() + calls++ + if calls == 1 { + return true, &corev1.PodList{}, nil + } + return true, nil, stderrors.New("apiserver unreachable") + }) + + w := watch.NewRaceFreeFake() + w.Stop() + client.PrependWatchReactor("pods", k8stesting.DefaultWatchReactor(w, nil)) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if _, err := d.findOrWatchPodName(ctx); err == nil { + t.Fatal("findOrWatchPodName() = nil error, want the re-List failure surfaced") + } else if !stderrors.Is(err, aicrerrors.New(aicrerrors.ErrCodeUnavailable, "")) { + t.Errorf("error = %v, want code ErrCodeUnavailable", err) + } + }) +} + +// installStagedPodLister makes successive Pod List calls return different +// results: items(1) answers the fast-path List in findOrWatchPodName, items(2) +// answers the re-List after the watch channel closes. The fake still applies +// ListOptions.LabelSelector to whatever this returns, so the label narrowing +// is exercised for real. +func installStagedPodLister(client *fake.Clientset, items func(call int) []corev1.Pod) { + var mu sync.Mutex + var calls int + client.PrependReactor("list", "pods", func(k8stesting.Action) (bool, runtime.Object, error) { + mu.Lock() + defer mu.Unlock() + calls++ + return true, &corev1.PodList{Items: items(calls)}, nil + }) +} diff --git a/pkg/k8s/doc.go b/pkg/k8s/doc.go index c321620b0..9aa8fa906 100644 --- a/pkg/k8s/doc.go +++ b/pkg/k8s/doc.go @@ -60,11 +60,16 @@ // // For agent deployment, import the agent sub-package: // -// import "github.com/NVIDIA/aicr/pkg/k8s/agent" +// import ( +// "github.com/NVIDIA/aicr/pkg/k8s/agent" +// "github.com/NVIDIA/aicr/pkg/runid" +// ) // -// // Deploy snapshot agent +// // Deploy snapshot agent. RunID scopes every object the deployment +// // creates, so concurrent runs never collide; see pkg/k8s/agent. // config := agent.Config{ // Namespace: "gpu-operator", +// RunID: runid.Generate(), // Image: "ghcr.io/nvidia/aicr-validator:latest", // } // deployer := agent.NewDeployer(clientset, config) diff --git a/pkg/k8s/labels/labels.go b/pkg/k8s/labels/labels.go new file mode 100644 index 000000000..2d0fca49a --- /dev/null +++ b/pkg/k8s/labels/labels.go @@ -0,0 +1,49 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package labels provides shared Kubernetes label constants used by both +// the validator (pkg/validator) and the snapshot agent (pkg/k8s/agent), so +// neither has to import the other to agree on label keys and values. +package labels + +import "github.com/NVIDIA/aicr/pkg/header" + +// Standard Kubernetes label keys. +const ( + Name = "app.kubernetes.io/name" + Component = "app.kubernetes.io/component" + ManagedBy = "app.kubernetes.io/managed-by" +) + +// RunID scopes every resource to the run that created it. +const RunID = header.Domain + "/run-id" + +// Common label values. +const ( + // ValueAICR is the shared app name. + ValueAICR = "aicr" + + // ValueSnapshotAgent identifies snapshot-agent-owned resources. + ValueSnapshotAgent = "snapshot-agent" + + // ValueAgentRBAC identifies the NON-run-scoped Role, RoleBinding, + // ClusterRole and ClusterRoleBinding that + // `aicr snapshot --add-roles-to-service-account` renders as manifests + // for an operator-supplied ServiceAccount. aicr applies none of them; + // the operator does. Objects carrying this value are deliberately + // outside every run's lifecycle: they carry no RunID label, never + // enter a run's created-set, and are never deleted by run cleanup. + // Teardown is the operator's `kubectl delete -f`. + ValueAgentRBAC = "agent-rbac" +) diff --git a/pkg/runid/runid.go b/pkg/runid/runid.go new file mode 100644 index 000000000..175404183 --- /dev/null +++ b/pkg/runid/runid.go @@ -0,0 +1,46 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package runid generates unique run identifiers shared by the validator +// and the snapshot agent so both subsystems use one format and one +// generator. +package runid + +import ( + "crypto/rand" + "encoding/hex" + "fmt" + "time" +) + +// Generate creates a unique run identifier. +// Format: {timestamp}-{random-hex} (e.g., "20260514-123045-abc123def456"). +// Callers use this to generate runIDs before creating ConfigMaps and +// rendering Jobs. +// +// Panics if the system's random number generator fails. Entropy failures are +// exceptional and we prefer to fail fast rather than generate predictable IDs +// that could collide across concurrent runs. +func Generate() string { + timestamp := time.Now().Format("20060102-150405") + randomBytes := make([]byte, 8) + n, err := rand.Read(randomBytes) + if err != nil { + panic(fmt.Sprintf("failed to generate random bytes for runID: %v", err)) + } + if n != len(randomBytes) { + panic(fmt.Sprintf("failed to generate runID: read %d bytes, expected %d", n, len(randomBytes))) + } + return fmt.Sprintf("%s-%s", timestamp, hex.EncodeToString(randomBytes)) +} diff --git a/pkg/runid/runid_test.go b/pkg/runid/runid_test.go new file mode 100644 index 000000000..5ada35fe9 --- /dev/null +++ b/pkg/runid/runid_test.go @@ -0,0 +1,43 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package runid + +import ( + "regexp" + "testing" +) + +var runIDPattern = regexp.MustCompile(`^\d{8}-\d{6}-[0-9a-f]{16}$`) + +func TestGenerateFormat(t *testing.T) { + id := Generate() + if !runIDPattern.MatchString(id) { + t.Errorf("Generate() = %q, want match %s", id, runIDPattern) + } + if len(id) != 32 { + t.Errorf("len(Generate()) = %d, want 32", len(id)) + } +} + +func TestGenerateUnique(t *testing.T) { + seen := make(map[string]struct{}, 100) + for range 100 { + id := Generate() + if _, dup := seen[id]; dup { + t.Fatalf("Generate() returned duplicate %q", id) + } + seen[id] = struct{}{} + } +} diff --git a/pkg/snapshotter/agent.go b/pkg/snapshotter/agent.go index 356169d03..216d1fc99 100644 --- a/pkg/snapshotter/agent.go +++ b/pkg/snapshotter/agent.go @@ -36,6 +36,7 @@ import ( k8sclient "github.com/NVIDIA/aicr/pkg/k8s/client" "github.com/NVIDIA/aicr/pkg/k8s/pod" "github.com/NVIDIA/aicr/pkg/measurement" + "github.com/NVIDIA/aicr/pkg/runid" "github.com/NVIDIA/aicr/pkg/serializer" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -64,7 +65,31 @@ type AgentConfig struct { // JobName for the agent Job JobName string - // ServiceAccountName for the agent + // ServiceAccountName selects the ServiceAccount the agent pod runs + // as. It is EXACT-IF-EXISTS, so it carries two meanings resolved once + // per deployment: + // + // - A ServiceAccount of exactly this name already exists in + // Namespace: it is used verbatim, and the run creates NO + // ServiceAccount, Role, RoleBinding, ClusterRole or + // ClusterRoleBinding — and deletes none at cleanup. aicr adds and + // removes no permissions on an identity it did not create. This + // is how a ServiceAccount carrying IRSA + // (eks.amazonaws.com/role-arn) or GKE Workload Identity + // (iam.gke.io/gcp-service-account) annotations stays usable: both + // providers pin trust to the ServiceAccount NAME, which a + // run-scoped name can never satisfy. Generate its RBAC manifests + // with WriteAgentRoleManifests, then apply them out of band. + // - Otherwise: a name prefix. The run creates "-" + // and the full run-scoped RBAC set, and deletes them at cleanup. + // + // Empty falls back to NameBase and is never probed for existence, so + // a stray ServiceAccount sitting at the default base cannot silently + // capture the run. + // + // Using an existing ServiceAccount waives per-run permission + // isolation: concurrent runs sharing it share its grants, and grants + // provisioned for DiscoverNetwork persist beyond any one run. ServiceAccountName string // NodeSelector for targeting specific nodes @@ -148,38 +173,85 @@ type AgentConfig struct { // that key in Limits — e.g. --require-gpu --limits nvidia.com/gpu=4 // keeps 4, not 1. Limits corev1.ResourceList + + // RunID scopes every resource this deployment creates (Job, RBAC, and + // the internal staging ConfigMap when Output does not name one) to a + // single run, so concurrent snapshot-agent runs never collide on a + // shared resource name. DeployAndCollect generates one with + // runid.Generate() when this is empty — callers normally leave it + // unset; setting it explicitly is for correlating this run with an + // external identifier (e.g. sharing one ID with a downstream + // validator run). + // + // DeployAndCollect never writes the generated value back here: the + // AgentConfig belongs to the caller, and a caller reusing one config + // pointer across two runs would otherwise silently become a caller + // pinning a duplicate RunID — the one state ADR-020 declares + // unsupported, which fails the second run with ErrCodeInternal on the + // first still-existing run-scoped object. + RunID string + + // NameBase prefixes generated resource names (Job, ServiceAccount, + // Role/RoleBinding). It applies per name: JobName falls back to it + // when JobName is empty, and ServiceAccountName (which also names the + // Role and RoleBinding) falls back to it when ServiceAccountName is + // empty — so setting only one of the two leaves NameBase governing the + // other. Forwarded verbatim to pkg/k8s/agent.Config.NameBase, which + // defaults to "aicr" when also empty. + NameBase string } // buildAgentConfig projects snapshotter configuration onto the deployer's // Job configuration. Keep scheduling defaults at this projection boundary so // every snapshot-agent caller gets the same nil-versus-empty behavior. -func buildAgentConfig(config *AgentConfig, agentOutput string) agent.Config { +// +// ownsOutput is agentConfigMapTarget's second return value, forwarded +// verbatim: it is true only when agentOutput is the internal staging +// ConfigMap this run owns (the caller did not name a cm:// destination), so +// Cleanup may delete it. A caller-supplied cm:// Output is never owned. +// +// runID is the resolved run ID for this invocation — config.RunID when the +// caller pinned one, otherwise the value DeployAndCollect generated. It is +// passed in rather than read from config because DeployAndCollect must not +// write the generated ID back into the caller's AgentConfig (see +// AgentConfig.RunID). +func buildAgentConfig(config *AgentConfig, runID, agentOutput string, ownsOutput bool) agent.Config { return agent.Config{ - Namespace: config.Namespace, - ServiceAccountName: config.ServiceAccountName, - JobName: config.JobName, - Image: config.Image, - ImagePullSecrets: config.ImagePullSecrets, - NodeSelector: config.NodeSelector, - Tolerations: effectiveAgentTolerations(config.Tolerations), - Output: agentOutput, - Debug: config.Debug, - Privileged: config.Privileged, - RequireGPU: config.RequireGPU, - RuntimeClassName: config.RuntimeClassName, - MaxNodesPerEntry: config.MaxNodesPerEntry, - OS: config.OS, - ClusterConfigPath: config.ClusterConfigPath, - DiscoverNetwork: config.DiscoverNetwork, - Requests: config.Requests, - Limits: config.Limits, + Namespace: config.Namespace, + ServiceAccountName: config.ServiceAccountName, + JobName: config.JobName, + RunID: runID, + NameBase: config.NameBase, + Image: config.Image, + ImagePullSecrets: config.ImagePullSecrets, + NodeSelector: config.NodeSelector, + Tolerations: effectiveAgentTolerations(config.Tolerations), + Output: agentOutput, + Debug: config.Debug, + Privileged: config.Privileged, + RequireGPU: config.RequireGPU, + RuntimeClassName: config.RuntimeClassName, + MaxNodesPerEntry: config.MaxNodesPerEntry, + OS: config.OS, + ClusterConfigPath: config.ClusterConfigPath, + DiscoverNetwork: config.DiscoverNetwork, + Requests: config.Requests, + Limits: config.Limits, + OwnsOutputConfigMap: ownsOutput, } } // deployAndWaitForResult handles the common deploy-wait-retrieve lifecycle for an agent Job. // It creates the deployer, deploys RBAC and the Job, streams logs, waits for completion, // and retrieves the snapshot data from the result ConfigMap. -func deployAndWaitForResult(ctx context.Context, clientset k8sclient.Interface, config *AgentConfig, agentOutput string, deliverViaConfigMap bool) ([]byte, error) { +// +// ownsOutput is agentConfigMapTarget's second return value: true when +// agentOutput is the internal staging ConfigMap this run owns, false when it +// is a caller-supplied cm:// destination. See buildAgentConfig and +// rewriteMergedSnapshotConfigMap for how each consumes it. +// +// runID is the resolved run ID for this invocation; see buildAgentConfig. +func deployAndWaitForResult(ctx context.Context, clientset k8sclient.Interface, config *AgentConfig, runID, agentOutput string, ownsOutput bool) ([]byte, error) { // The pool projection is pure file processing on the caller's host — // project it BEFORE deploying so a bad file fails in milliseconds, // not after a Job round-trip, and merge it into the returned snapshot @@ -198,7 +270,7 @@ func deployAndWaitForResult(ctx context.Context, clientset k8sclient.Interface, // name the injected selector (TOCTOU: node may be cordoned after detection). autoInjectedGPUSelector := maybeInjectGPUNodeSelector(ctx, clientset, config) - agentConfig := buildAgentConfig(config, agentOutput) + agentConfig := buildAgentConfig(config, runID, agentOutput, ownsOutput) deployer := agent.NewDeployer(clientset, agentConfig) @@ -229,8 +301,11 @@ func deployAndWaitForResult(ctx context.Context, clientset k8sclient.Interface, timeout = defaults.K8sJobCompletionTimeout } + // Log the run-scoped Job name, not agentConfig.JobName: that field is + // only the user's optional prefix and is empty by default, so logging it + // prints job="". slog.Info("waiting for Job completion", - slog.String("job", agentConfig.JobName), + slog.String("job", deployer.JobName()), slog.Duration("timeout", timeout)) // Stream logs in background while waiting for Job completion. @@ -256,7 +331,7 @@ func deployAndWaitForResult(ctx context.Context, clientset k8sclient.Interface, if logCtx.Err() == nil { slog.Warn("agent log streaming skipped: pod did not become ready", slog.String("namespace", agentConfig.Namespace), - slog.String("job", agentConfig.JobName), + slog.String("job", deployer.JobName()), "error", podErr) } return @@ -325,8 +400,10 @@ func deployAndWaitForResult(ctx context.Context, clientset k8sclient.Interface, // merged reading, so a transient Apply failure on the internal // hygiene rewrite must not discard an already-captured snapshot; // warn loudly instead (the orphaned ConfigMap stays pre-merge). + // ownsOutput is the run-owns-the-staging-ConfigMap flag, so its + // logical inverse is "the ConfigMap is the user's delivery vehicle". if err := rewriteMergedSnapshotConfigMap(ctx, agentOutput, config.Kubeconfig, - snapshotData, deliverViaConfigMap); err != nil { + snapshotData, !ownsOutput); err != nil { return nil, err } } @@ -482,10 +559,32 @@ func DeployAndCollect(ctx context.Context, config *AgentConfig) (*Snapshot, []by "Namespace is required: it is where the agent Job, its RBAC, and the result ConfigMap are created") } + // Default RunID before any cluster access — and before + // agentConfigMapTarget below, which folds it into the internal staging + // ConfigMap's name — so every resource this run creates shares one + // scope. pkg/k8s/agent's Deploy independently rejects an empty or + // malformed RunID, but only as an ErrCodeInvalidRequest naming a Config + // field a CLI user never set — so catch the whitespace-only value that + // slips past this simple emptiness check here, where the message can + // point at the knob the caller actually controls. + // + // The resolved value stays in a local: writing it back into the + // caller-owned *AgentConfig would turn a caller who reuses one config + // pointer for a second run into a caller pinning a duplicate RunID. + runID := config.RunID + if runID == "" { + runID = runid.Generate() + } + if strings.TrimSpace(runID) == "" { + return nil, nil, errors.New(errors.ErrCodeInvalidRequest, + "RunID must not be all-whitespace; leave it unset to auto-generate one") + } + slog.Info("snapshot agent run", slog.String("runID", runID)) + // Resolve (and validate) the Job's ConfigMap target before any cluster // access: a malformed cm:// Output must not cost the caller a deployed // Job and a cluster-admin binding. - agentOutput, deliverViaConfigMap, err := agentConfigMapTarget(config) + agentOutput, ownsOutput, err := agentConfigMapTarget(config, runID) if err != nil { return nil, nil, err } @@ -497,7 +596,7 @@ func DeployAndCollect(ctx context.Context, config *AgentConfig) (*Snapshot, []by return nil, nil, err } - snapshotData, err := deployAndWaitForResult(ctx, clientset, config, agentOutput, deliverViaConfigMap) + snapshotData, err := deployAndWaitForResult(ctx, clientset, config, runID, agentOutput, ownsOutput) if err != nil { return nil, nil, err } @@ -513,21 +612,35 @@ func DeployAndCollect(ctx context.Context, config *AgentConfig) (*Snapshot, []by } // agentConfigMapTarget resolves where the agent Job stages its result and -// whether that ConfigMap is the user's delivery vehicle. +// whether this run owns that ConfigMap. // // The Job always writes to a ConfigMap. When config.Output is a cm:// URI the -// user asked for that exact ConfigMap, so the Job targets it directly and -// deliverViaConfigMap is true — which makes a failed AKS-pool-merge rewrite -// fatal rather than a warning, because the bytes the user will read live -// there. Any other Output (file, stdout, template, or unset) stages to an -// internal ConfigMap in config.Namespace that the caller never sees. +// user asked for that exact ConfigMap, so the Job targets it directly, +// ownsOutput is false, and a failed AKS-pool-merge rewrite is fatal rather +// than a warning — the bytes the user will read live there, and this run +// must never delete an artifact it does not own. Any other Output (file, +// stdout, template, or unset) stages to an internal, run-scoped ConfigMap in +// config.Namespace that the caller never names: ownsOutput is true, so +// Cleanup may delete it. +// +// runID is the resolved run ID for this invocation (config.RunID when the +// caller pinned one, otherwise the value DeployAndCollect generated); it is a +// parameter rather than a config field read because DeployAndCollect never +// writes the generated ID back into the caller's AgentConfig. +// +// In the owned case the returned uri is exactly +// cm:///. Cleanup +// in pkg/k8s/agent relies on both halves of that invariant rather than +// re-parsing the URI: deleteStagingConfigMap deletes in Config.Namespace, and +// deleteUnrecordedStagingConfigMap (the sweep for a run that failed before +// the ConfigMap's UID was observed) looks it up by that same generated name. // // A cm:// Output is fully parsed here, not merely prefix-matched. The // namespace/name only has to be well-formed for the in-pod writer much later, // so a typo like "cm://aicr-snapshot" (no namespace) would otherwise surface // as a Job failure — after RBAC and the Job exist, and with Cleanup false // (the zero value) they stay behind. Returns ErrCodeInvalidRequest instead. -func agentConfigMapTarget(config *AgentConfig) (uri string, deliverViaConfigMap bool, err error) { +func agentConfigMapTarget(config *AgentConfig, runID string) (uri string, ownsOutput bool, err error) { if strings.HasPrefix(config.Output, serializer.ConfigMapURIScheme) { if _, _, parseErr := pod.ParseConfigMapURI(config.Output); parseErr != nil { // Wrap with the same code rather than PropagateOrWrap: the inner @@ -537,9 +650,9 @@ func agentConfigMapTarget(config *AgentConfig) (uri string, deliverViaConfigMap fmt.Sprintf("invalid ConfigMap output URI %q (expected cm://namespace/name)", config.Output), parseErr) } - return config.Output, true, nil + return config.Output, false, nil } - return fmt.Sprintf("%s%s/aicr-snapshot", serializer.ConfigMapURIScheme, config.Namespace), false, nil + return serializer.ConfigMapURIScheme + config.Namespace + "/" + agent.StagingConfigMapName(runID), true, nil } // SnapshotDelivery describes where DeliverSnapshot writes captured bytes. diff --git a/pkg/snapshotter/agent_test.go b/pkg/snapshotter/agent_test.go index 9db0d0979..9ac04e756 100644 --- a/pkg/snapshotter/agent_test.go +++ b/pkg/snapshotter/agent_test.go @@ -19,13 +19,16 @@ import ( "encoding/json" stderrors "errors" "io" + "log/slog" "os" "path/filepath" "reflect" + "regexp" "strings" "testing" "github.com/NVIDIA/aicr/pkg/errors" + "github.com/NVIDIA/aicr/pkg/k8s/agent" "github.com/NVIDIA/aicr/pkg/serializer" corev1 "k8s.io/api/core/v1" ) @@ -88,7 +91,7 @@ func TestBuildAgentConfigTolerations(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := buildAgentConfig(&AgentConfig{Tolerations: tt.input}, "snapshot.yaml").Tolerations + got := buildAgentConfig(&AgentConfig{Tolerations: tt.input}, "20260821-142233-9f3a1c0b7e2d4a55", "snapshot.yaml", false).Tolerations if !reflect.DeepEqual(got, tt.want) { t.Errorf("buildAgentConfig().Tolerations = %#v, want %#v", got, tt.want) } @@ -96,6 +99,45 @@ func TestBuildAgentConfigTolerations(t *testing.T) { } } +// TestBuildAgentConfigPropagatesRunIDAndOwnership confirms buildAgentConfig +// forwards its runID argument, AgentConfig.NameBase, and its ownsOutput +// parameter onto agent.Config.RunID / agent.Config.NameBase / +// agent.Config.OwnsOutputConfigMap — the projection deployAndWaitForResult +// relies on so the deployer scopes every resource name to this run, applies +// the caller's naming prefix, and Cleanup knows whether it may delete the +// staging ConfigMap. +// +// runID is a parameter, not a read of AgentConfig.RunID: DeployAndCollect +// resolves it into a local and never writes a generated ID back into the +// caller's config. The AgentConfig below deliberately leaves RunID empty so a +// projection that regressed to reading config.RunID would produce "" and fail +// here. +func TestBuildAgentConfigPropagatesRunIDAndOwnership(t *testing.T) { + tests := []struct { + name string + ownsOutput bool + }{ + {name: "owned staging ConfigMap", ownsOutput: true}, + {name: "user-supplied ConfigMap", ownsOutput: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := buildAgentConfig(&AgentConfig{ + NameBase: "aicr-validate", + }, "20260821-142233-9f3a1c0b7e2d4a55", "cm://ns/name", tt.ownsOutput) + if got.RunID != "20260821-142233-9f3a1c0b7e2d4a55" { + t.Errorf("agent.Config.RunID = %q, want the runID argument", got.RunID) + } + if got.NameBase != "aicr-validate" { + t.Errorf("agent.Config.NameBase = %q, want the AgentConfig.NameBase value", got.NameBase) + } + if got.OwnsOutputConfigMap != tt.ownsOutput { + t.Errorf("agent.Config.OwnsOutputConfigMap = %v, want %v", got.OwnsOutputConfigMap, tt.ownsOutput) + } + }) + } +} + func TestAgentConfig_Defaults(t *testing.T) { // Test that AgentConfig can be instantiated with zero values cfg := AgentConfig{} @@ -505,11 +547,15 @@ func TestParseTolerationsOperator(t *testing.T) { // TestAgentOutputURILogic exercises agentConfigMapTarget — the rule that // decides where the agent Job stages its result: -// 1. A file path leaves the Job on the default ConfigMap in its namespace. -// 2. A cm:// URI makes that ConfigMap the Job's target AND the delivery -// vehicle, so a rewrite failure must be fatal rather than a warning. +// 1. A file path leaves the Job on the run-scoped internal ConfigMap in its +// namespace, which this run owns. +// 2. A cm:// URI makes that ConfigMap the Job's target directly; this run +// does NOT own it (the caller supplied it), so a rewrite failure must be +// fatal rather than a warning. // 3. Stdout (empty or "-") behaves like a file path. func TestAgentOutputURILogic(t *testing.T) { + const testRunID = "20260821-142233-9f3a1c0b7e2d4a55" + tests := []struct { name string agentNamespace string @@ -518,24 +564,24 @@ func TestAgentOutputURILogic(t *testing.T) { wantUsesUserOutput bool // whether agentOutput should equal userOutput }{ { - name: "file output uses default ConfigMap with agent namespace", + name: "file output uses run-scoped internal ConfigMap", agentNamespace: "default", userOutput: "snapshot.yaml", - wantAgentOutputHas: "cm://default/aicr-snapshot", + wantAgentOutputHas: "cm://default/aicr-agent-snapshot-" + testRunID, wantUsesUserOutput: false, }, { - name: "stdout uses default ConfigMap with agent namespace", + name: "stdout uses run-scoped internal ConfigMap", agentNamespace: "default", userOutput: "", - wantAgentOutputHas: "cm://default/aicr-snapshot", + wantAgentOutputHas: "cm://default/aicr-agent-snapshot-" + testRunID, wantUsesUserOutput: false, }, { - name: "dash stdout uses default ConfigMap with agent namespace", + name: "dash stdout uses run-scoped internal ConfigMap", agentNamespace: "default", userOutput: "-", - wantAgentOutputHas: "cm://default/aicr-snapshot", + wantAgentOutputHas: "cm://default/aicr-agent-snapshot-" + testRunID, wantUsesUserOutput: false, }, { @@ -546,20 +592,20 @@ func TestAgentOutputURILogic(t *testing.T) { wantUsesUserOutput: true, }, { - name: "custom namespace uses that namespace for default ConfigMap", + name: "custom namespace uses that namespace for the run-scoped ConfigMap", agentNamespace: "custom-namespace", userOutput: "output.yaml", - wantAgentOutputHas: "cm://custom-namespace/aicr-snapshot", + wantAgentOutputHas: "cm://custom-namespace/aicr-agent-snapshot-" + testRunID, wantUsesUserOutput: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - agentOutput, deliverViaConfigMap, err := agentConfigMapTarget(&AgentConfig{ + agentOutput, ownsOutput, err := agentConfigMapTarget(&AgentConfig{ Namespace: tt.agentNamespace, Output: tt.userOutput, - }) + }, testRunID) if err != nil { t.Fatalf("agentConfigMapTarget: %v", err) } @@ -573,15 +619,63 @@ func TestAgentOutputURILogic(t *testing.T) { t.Errorf("agentOutput = %q, want %q", agentOutput, tt.wantAgentOutputHas) } } - if deliverViaConfigMap != tt.wantUsesUserOutput { - t.Errorf("deliverViaConfigMap = %v, want %v — the flag must track whether the "+ - "ConfigMap is the user's delivery vehicle, since it decides whether an "+ - "AKS-pool-merge rewrite failure is fatal", deliverViaConfigMap, tt.wantUsesUserOutput) + // ownsOutput is the logical inverse of "the user supplied this + // output": the run owns (and Cleanup may delete) the staging + // ConfigMap only when it did NOT come from the user. + wantOwnsOutput := !tt.wantUsesUserOutput + if ownsOutput != wantOwnsOutput { + t.Errorf("ownsOutput = %v, want %v — it must track whether this run owns the "+ + "ConfigMap, since it decides whether Cleanup may delete it and whether an "+ + "AKS-pool-merge rewrite failure is fatal", ownsOutput, wantOwnsOutput) } }) } } +// TestAgentConfigMapTargetIsRunScoped and TestAgentConfigMapTargetLeavesUserURIAlone +// pin the ownsOutput inversion: the run owns (and Cleanup may delete) the +// staging ConfigMap only when the user did NOT name a cm:// destination. +// Getting this backwards deletes the user's own output ConfigMap. +func TestAgentConfigMapTargetIsRunScoped(t *testing.T) { + const runID = "20260821-142233-9f3a1c0b7e2d4a55" + cfg := &AgentConfig{Namespace: "gpu-operator"} + uri, ownsOutput, err := agentConfigMapTarget(cfg, runID) + if err != nil { + t.Fatalf("agentConfigMapTarget() error = %v", err) + } + // The staging ConfigMap is deliberately prefixed "aicr-agent-snapshot", + // NOT "aicr-snapshot": pkg/validator names its own snapshot data + // ConfigMap "aicr-snapshot-", and `aicr validate` gives both + // subsystems the same run ID in the same namespace. See + // TestStagingConfigMapNameDoesNotCollideWithValidator in pkg/k8s/agent. + want := "cm://gpu-operator/aicr-agent-snapshot-20260821-142233-9f3a1c0b7e2d4a55" + if uri != want { + t.Errorf("uri = %q, want %q", uri, want) + } + // The URI must be built from the one exported helper the agent package + // also deletes by, not from a second copy of the format string. + if wantHelper := "cm://gpu-operator/" + agent.StagingConfigMapName(runID); uri != wantHelper { + t.Errorf("uri = %q, want %q (agent.StagingConfigMapName)", uri, wantHelper) + } + if !ownsOutput { + t.Error("ownsOutput = false, want true for the internal staging ConfigMap") + } +} + +func TestAgentConfigMapTargetLeavesUserURIAlone(t *testing.T) { + cfg := &AgentConfig{Namespace: "gpu-operator", Output: "cm://gpu-operator/aicr-snapshot"} + uri, ownsOutput, err := agentConfigMapTarget(cfg, "20260821-142233-9f3a1c0b7e2d4a55") + if err != nil { + t.Fatalf("agentConfigMapTarget() error = %v", err) + } + if uri != cfg.Output { + t.Errorf("uri = %q, want the user's URI %q unchanged", uri, cfg.Output) + } + if ownsOutput { + t.Error("ownsOutput = true; a user-supplied cm:// output is delivered, not run-owned") + } +} + func TestAgentConfigWithTemplatePath(t *testing.T) { // Test that AgentConfig can hold TemplatePath cfg := AgentConfig{ @@ -1051,7 +1145,7 @@ func TestAgentConfigMapTargetRejectsMalformedURI(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, _, err := agentConfigMapTarget(&AgentConfig{Namespace: "default", Output: tt.output}) + _, _, err := agentConfigMapTarget(&AgentConfig{Namespace: "default", Output: tt.output}, "20260821-142233-9f3a1c0b7e2d4a55") if err == nil { t.Fatalf("agentConfigMapTarget(%q) = nil error, want rejection before any cluster access", tt.output) } @@ -1114,6 +1208,20 @@ func TestDeployAndCollectRejectsBeforeClusterAccess(t *testing.T) { }, wantMsg: "Namespace is required", }, + { + // Ruling 5 mitigation: nameWithRunID (pkg/k8s/agent) silently + // falls back to unscoped, collision-prone names when RunID is + // empty. A caller-supplied all-whitespace RunID bypasses the + // simple "== \"\"" default check, so it must fail closed here + // rather than reach that fallback deep in agent.Deploy. + name: "whitespace-only RunID", + config: &AgentConfig{ + Namespace: "default", + Kubeconfig: badKubeconfig, + RunID: " ", + }, + wantMsg: "RunID must not be all-whitespace", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -1131,3 +1239,53 @@ func TestDeployAndCollectRejectsBeforeClusterAccess(t *testing.T) { }) } } + +// TestDeployAndCollectGeneratesRunIDWithoutMutatingConfig confirms +// DeployAndCollect resolves an empty RunID to a freshly generated one before +// it is folded into the internal staging ConfigMap's name, AND that it leaves +// the caller's *AgentConfig untouched while doing so. +// +// The non-mutation half is the load-bearing one. Writing the generated ID +// back into the caller's config would turn a caller who reuses one config +// pointer for a second run into a caller pinning a duplicate RunID — the one +// state ADR-020 declares unsupported — and run 2 would hard-fail +// ErrCodeInternal on the first still-existing run-scoped object. In-tree +// callers reach this through the pkg/client/v1 facade, which builds a fresh +// internal config per call, but pkg/snapshotter is public. +// +// The generated value is observed through the "snapshot agent run" log line, +// which is emitted just before the malformed-Output rejection below fires; +// there is no other seam that does not require cluster access. +func TestDeployAndCollectGeneratesRunIDWithoutMutatingConfig(t *testing.T) { + runIDPattern := regexp.MustCompile(`runID=(\d{8}-\d{6}-[0-9a-f]{16})`) + + var logs bytes.Buffer + previousLogger := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug}))) + t.Cleanup(func() { slog.SetDefault(previousLogger) }) + + cfg := &AgentConfig{ + Namespace: "default", + Kubeconfig: filepath.Join(t.TempDir(), "does-not-exist.kubeconfig"), + Output: "cm://aicr-snapshot", // malformed: no namespace — rejected after RunID resolution + } + if cfg.RunID != "" { + t.Fatalf("test precondition: cfg.RunID = %q, want empty", cfg.RunID) + } + + _, _, err := DeployAndCollect(t.Context(), cfg) + if err == nil { + t.Fatal("DeployAndCollect() = nil error, want rejection from the malformed Output") + } + + if !runIDPattern.MatchString(logs.String()) { + t.Errorf("no generated run ID matching %s in log output; DeployAndCollect must "+ + "resolve an empty RunID with runid.Generate(). Logs:\n%s", runIDPattern, logs.String()) + } + + if cfg.RunID != "" { + t.Errorf("cfg.RunID = %q after DeployAndCollect, want it left empty — the generated ID "+ + "must stay in a local so a caller reusing this config for a second run does not "+ + "silently pin a duplicate RunID", cfg.RunID) + } +} diff --git a/pkg/snapshotter/provision.go b/pkg/snapshotter/provision.go new file mode 100644 index 000000000..235267c70 --- /dev/null +++ b/pkg/snapshotter/provision.go @@ -0,0 +1,207 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package snapshotter + +import ( + stderrors "errors" + "io/fs" + "log/slog" + "os" + "path/filepath" + "strings" + + "github.com/NVIDIA/aicr/pkg/defaults" + "github.com/NVIDIA/aicr/pkg/errors" + "github.com/NVIDIA/aicr/pkg/k8s/agent" + "github.com/NVIDIA/aicr/pkg/runid" +) + +// AgentRolesConfig selects the ServiceAccount that WriteAgentRoleManifests +// renders the snapshot agent's RBAC for. +// +// There is deliberately no Kubeconfig field: writing the manifests contacts +// no cluster, so there is no connection to configure. +type AgentRolesConfig struct { + // Namespace is the namespace of the ServiceAccount, and the namespace + // the rendered Role and RoleBinding declare. Required. + Namespace string + + // ServiceAccountName is the name of the ServiceAccount the rendered + // bindings name as their subject. Required, and not verified to + // exist — see WriteAgentRoleManifests. + ServiceAccountName string + + // DiscoverNetwork also renders the cluster-scoped MUTATING rules that + // `aicr snapshot --discover-network` needs, with a header enumerating + // each one and the discovery step it exists for. + DiscoverNetwork bool + + // RunID names the output directory (`snapshot-rbac-`). Empty + // generates one, which is the normal path; it is injectable so tests + // and automation can pin a directory name. + RunID string +} + +// AgentRoleObject identifies one written manifest so a caller can report +// what landed where without re-deriving either the name or the file. +type AgentRoleObject struct { + // Kind is the Kubernetes kind ("Role", "RoleBinding", "ClusterRole", + // "ClusterRoleBinding"). + Kind string + + // Name is the object's metadata.name. + Name string + + // Path is the manifest's path, including the output directory. + Path string +} + +// AgentRolesResult names the directory WriteAgentRoleManifests wrote and +// what it put there. +// +// It is snapshotter-owned rather than pkg/k8s/agent's own Manifest type so +// callers presenting the outcome — the CLI among them — need no dependency +// on the Kubernetes-facing package. +type AgentRolesResult struct { + // Dir is the output directory, relative to the working directory the + // call was made from. + Dir string + + // RunID is the run ID the directory name was built from. + RunID string + + Namespace string + ServiceAccountName string + + // Objects lists what was written, in the order the files apply. + Objects []AgentRoleObject + + // DiscoverNetwork echoes AgentRolesConfig.DiscoverNetwork: it is the + // difference between a read-only grant and one carrying cluster-scoped + // mutating rules, so anything reporting this result can say which was + // written. + DiscoverNetwork bool +} + +// WriteAgentRoleManifests writes the RBAC manifests that grant the snapshot +// agent's permissions to an operator-supplied ServiceAccount into a new +// `snapshot-rbac-` directory in the current working directory. +// +// It APPLIES NOTHING and contacts no cluster. No clientset is built, the +// ServiceAccount is never looked up, and no permission pre-flight runs — so +// the call succeeds with no kubeconfig and no cluster privileges at all. The +// operator reviews the files and then applies them: +// +// kubectl apply -f snapshot-rbac-/ +// +// and removes the grant with the matching delete: +// +// kubectl delete -f snapshot-rbac-/ +// +// The ServiceAccount named in ServiceAccountName is NOT verified to exist. +// That is a deliberate simplification of the earlier behavior, which failed +// with ErrCodeNotFound against the cluster: a mistyped name now yields +// manifests the operator inspects before applying, and the rendered +// RoleBinding tells them how to check. +// +// The directory must not already exist. Colliding with one returns +// ErrCodeConflict rather than overwriting, because the manifests an operator +// is midway through reviewing are exactly what must not change under them. +// +// The objects are outside every run's lifecycle: no run-ID label, never in a +// run's created-set, never deleted by run cleanup. Teardown is the +// operator's `kubectl delete`. +func WriteAgentRoleManifests(config *AgentRolesConfig) (*AgentRolesResult, error) { + if config == nil { + return nil, errors.New(errors.ErrCodeInvalidRequest, "agent roles config is required") + } + // Reject what can be rejected before touching the filesystem, so a bad + // value never leaves a half-written directory behind. + if strings.TrimSpace(config.Namespace) == "" { + return nil, errors.New(errors.ErrCodeInvalidRequest, + "Namespace is required: it is the namespace the rendered Role and RoleBinding declare") + } + if strings.TrimSpace(config.ServiceAccountName) == "" { + return nil, errors.New(errors.ErrCodeInvalidRequest, + "ServiceAccountName is required: the rendered bindings need a subject to name") + } + + manifests, err := agent.BuildServiceAccountRoleManifests(agent.ManifestOptions{ + Namespace: config.Namespace, + ServiceAccountName: config.ServiceAccountName, + DiscoverNetwork: config.DiscoverNetwork, + }) + if err != nil { + return nil, err + } + + runID := config.RunID + if runID == "" { + runID = runid.Generate() + } + dir := defaults.AgentRBACManifestDirPrefix + runID + + // Mkdir, not MkdirAll: an existing directory must fail rather than have + // its contents joined by a second run's files. + if mkErr := os.Mkdir(dir, defaults.AgentRBACManifestDirMode); mkErr != nil { + if stderrors.Is(mkErr, fs.ErrExist) { + return nil, errors.NewWithContext(errors.ErrCodeConflict, + "refusing to overwrite the existing directory "+dir+ + "; move or delete it, or pass a different run ID", + map[string]any{"directory": dir}) + } + return nil, errors.Wrap(errors.ErrCodeInternal, + "failed to create the RBAC manifest directory", mkErr) + } + + objects, writeErr := writeManifestFiles(dir, manifests) + if writeErr != nil { + // A partially written directory would block the retry with the + // ErrCodeConflict above, so remove what this call created. The + // write error is what the caller needs to see, not this one. + if rmErr := os.RemoveAll(dir); rmErr != nil { + slog.Warn("failed to remove the partially written RBAC manifest directory", + "directory", dir, "error", rmErr) + } + return nil, writeErr + } + + return &AgentRolesResult{ + Dir: dir, + RunID: runID, + Namespace: config.Namespace, + ServiceAccountName: config.ServiceAccountName, + Objects: objects, + DiscoverNetwork: config.DiscoverNetwork, + }, nil +} + +// writeManifestFiles writes each manifest into dir and returns what it +// wrote. os.WriteFile is used rather than an explicit Create/Write/Close: it +// reports the Close error, which for a writable handle is where a failed +// flush surfaces. +func writeManifestFiles(dir string, manifests []agent.Manifest) ([]AgentRoleObject, error) { + objects := make([]AgentRoleObject, 0, len(manifests)) + for _, m := range manifests { + path := filepath.Join(dir, m.FileName) + if err := os.WriteFile(path, m.Content, defaults.AgentRBACManifestFileMode); err != nil { + return nil, errors.WrapWithContext(errors.ErrCodeInternal, + "failed to write the RBAC manifest", err, + map[string]any{"path": path, "kind": m.Kind}) + } + objects = append(objects, AgentRoleObject{Kind: m.Kind, Name: m.Name, Path: path}) + } + return objects, nil +} diff --git a/pkg/snapshotter/provision_test.go b/pkg/snapshotter/provision_test.go new file mode 100644 index 000000000..f18c58b7a --- /dev/null +++ b/pkg/snapshotter/provision_test.go @@ -0,0 +1,325 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package snapshotter + +import ( + stderrors "errors" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/NVIDIA/aicr/pkg/errors" + rbacv1 "k8s.io/api/rbac/v1" + "sigs.k8s.io/yaml" +) + +const ( + provisionNamespace = "gpu-operator" + provisionSAName = "irsa-snapshotter" + provisionRunID = "20260821-142233-9f3a1c0b7e2d4a55" +) + +// writeRoles renders into a scratch working directory and returns the result. +// It pins RunID so the directory name is deterministic, and t.Chdir keeps the +// output out of the repository — the call writes to the working directory by +// design. +func writeRoles(t *testing.T, config *AgentRolesConfig) *AgentRolesResult { + t.Helper() + t.Chdir(t.TempDir()) + res, err := WriteAgentRoleManifests(config) + if err != nil { + t.Fatalf("WriteAgentRoleManifests() error = %v", err) + } + return res +} + +// defaultRolesConfig is the standard input: a read-only grant with a pinned +// run ID. +func defaultRolesConfig() *AgentRolesConfig { + return &AgentRolesConfig{ + Namespace: provisionNamespace, + ServiceAccountName: provisionSAName, + RunID: provisionRunID, + } +} + +// TestWriteAgentRoleManifests_WritesReviewableDirectory covers the layout the +// documented workflow depends on: a `snapshot-rbac-` directory holding +// one parseable object per file, which is what makes both +// `kubectl apply -f /` and the `kubectl delete -f /` teardown work. +func TestWriteAgentRoleManifests_WritesReviewableDirectory(t *testing.T) { + res := writeRoles(t, defaultRolesConfig()) + + wantDir := "snapshot-rbac-" + provisionRunID + if res.Dir != wantDir { + t.Errorf("Dir = %q, want %q", res.Dir, wantDir) + } + if res.RunID != provisionRunID { + t.Errorf("RunID = %q, want %q", res.RunID, provisionRunID) + } + + entries, err := os.ReadDir(res.Dir) + if err != nil { + t.Fatalf("reading %s: %v", res.Dir, err) + } + got := make([]string, 0, len(entries)) + for _, e := range entries { + got = append(got, e.Name()) + } + sort.Strings(got) + want := []string{"01-role.yaml", "02-rolebinding.yaml", "03-clusterrole.yaml", "04-clusterrolebinding.yaml"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Errorf("directory contents = %v, want %v", got, want) + } + + if len(res.Objects) != len(want) { + t.Fatalf("Objects = %d, want %d", len(res.Objects), len(want)) + } + wantKinds := []string{"Role", "RoleBinding", "ClusterRole", "ClusterRoleBinding"} + for i, obj := range res.Objects { + if obj.Kind != wantKinds[i] { + t.Errorf("Objects[%d].Kind = %q, want %q", i, obj.Kind, wantKinds[i]) + } + if obj.Path != filepath.Join(wantDir, want[i]) { + t.Errorf("Objects[%d].Path = %q, want %q", i, obj.Path, filepath.Join(wantDir, want[i])) + } + body, readErr := os.ReadFile(obj.Path) + if readErr != nil { + t.Fatalf("reading %s: %v", obj.Path, readErr) + } + if !strings.HasPrefix(string(body), "# ") { + t.Errorf("%s does not open with a YAML comment header", obj.Path) + } + var parsed map[string]any + if unmarshalErr := yaml.Unmarshal(body, &parsed); unmarshalErr != nil { + t.Errorf("%s is not parseable YAML: %v", obj.Path, unmarshalErr) + } + if parsed["kind"] != obj.Kind { + t.Errorf("%s kind = %v, want %q", obj.Path, parsed["kind"], obj.Kind) + } + if parsed["metadata"] == nil { + t.Errorf("%s has no metadata; the comment header may have swallowed the object", obj.Path) + } + } +} + +// TestWriteAgentRoleManifests_DiscoverNetwork asserts the flag is what decides +// whether the written ClusterRole carries the mutating discovery rules, and +// that the plain form carries none of them. +func TestWriteAgentRoleManifests_DiscoverNetwork(t *testing.T) { + tests := []struct { + name string + discoverNetwork bool + wantMutating bool + }{ + {name: "plain grant stays read-only", discoverNetwork: false}, + {name: "discovery grant adds the mutating rules", discoverNetwork: true, wantMutating: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + config := defaultRolesConfig() + config.DiscoverNetwork = tt.discoverNetwork + res := writeRoles(t, config) + + if res.DiscoverNetwork != tt.discoverNetwork { + t.Errorf("DiscoverNetwork = %v, want %v", res.DiscoverNetwork, tt.discoverNetwork) + } + + body, err := os.ReadFile(filepath.Join(res.Dir, "03-clusterrole.yaml")) + if err != nil { + t.Fatalf("reading the ClusterRole: %v", err) + } + cr := &rbacv1.ClusterRole{} + if unmarshalErr := yaml.Unmarshal(body, cr); unmarshalErr != nil { + t.Fatalf("unmarshalling the ClusterRole: %v", unmarshalErr) + } + + mutating := map[string]string{"nodes": "patch", "pods/exec": "create", "customresourcedefinitions": "create"} + for resource, verb := range mutating { + if got := clusterRoleGrants(cr, resource, verb); got != tt.wantMutating { + t.Errorf("%s: %s = %v, want %v", resource, verb, got, tt.wantMutating) + } + } + + // The header must explain the mutating rules, not merely carry + // them: reading the file is how an operator consents to them. + if strings.Contains(string(body), "nodes: patch") != tt.wantMutating { + t.Errorf("header explains nodes: patch = %v, want %v", !tt.wantMutating, tt.wantMutating) + } + }) + } +} + +// clusterRoleGrants reports whether cr grants verb on resource. +func clusterRoleGrants(cr *rbacv1.ClusterRole, resource, verb string) bool { + for _, rule := range cr.Rules { + for _, res := range rule.Resources { + if res != resource { + continue + } + for _, v := range rule.Verbs { + if v == verb { + return true + } + } + } + } + return false +} + +// TestWriteAgentRoleManifests_ExistingDirectory asserts a collision fails with +// a structured conflict instead of overwriting. The manifests an operator is +// midway through reviewing are exactly what must not change under them. +func TestWriteAgentRoleManifests_ExistingDirectory(t *testing.T) { + t.Chdir(t.TempDir()) + dir := "snapshot-rbac-" + provisionRunID + if mkErr := os.Mkdir(dir, 0o700); mkErr != nil { + t.Fatalf("seeding the directory: %v", mkErr) + } + sentinel := filepath.Join(dir, "01-role.yaml") + if seedErr := os.WriteFile(sentinel, []byte("# operator is reading this\n"), 0o600); seedErr != nil { + t.Fatalf("seeding a file: %v", seedErr) + } + + res, err := WriteAgentRoleManifests(defaultRolesConfig()) + if err == nil { + t.Fatal("WriteAgentRoleManifests() error = nil, want ErrCodeConflict") + } + if res != nil { + t.Errorf("result = %+v, want nil", res) + } + if !stderrors.Is(err, errors.New(errors.ErrCodeConflict, "")) { + t.Errorf("error = %v, want code %s", err, errors.ErrCodeConflict) + } + if !strings.Contains(err.Error(), dir) { + t.Errorf("error = %q, want it to name the directory %q", err.Error(), dir) + } + + body, readErr := os.ReadFile(sentinel) + if readErr != nil { + t.Fatalf("reading the seeded file: %v", readErr) + } + if string(body) != "# operator is reading this\n" { + t.Errorf("the existing file was overwritten; got %q", body) + } +} + +// TestWriteAgentRoleManifests_GeneratesRunIDWhenUnset covers the normal path, +// where the run ID comes from the shared generator rather than the caller. +func TestWriteAgentRoleManifests_GeneratesRunIDWhenUnset(t *testing.T) { + config := defaultRolesConfig() + config.RunID = "" + res := writeRoles(t, config) + + if res.RunID == "" { + t.Fatal("RunID = \"\", want a generated run ID") + } + if !strings.HasPrefix(res.Dir, "snapshot-rbac-") { + t.Errorf("Dir = %q, want the snapshot-rbac- prefix", res.Dir) + } + if res.Dir != "snapshot-rbac-"+res.RunID { + t.Errorf("Dir = %q, want it built from RunID %q", res.Dir, res.RunID) + } + if _, err := os.Stat(res.Dir); err != nil { + t.Errorf("generated directory %q not created: %v", res.Dir, err) + } +} + +// TestWriteAgentRoleManifests_NoClusterAccess is the load-bearing test for +// this path: it must work with no kubeconfig and no cluster at all. KUBECONFIG +// is pointed at a file that cannot yield a client and the in-cluster +// environment is cleared, so any attempt to build a clientset or read the +// ServiceAccount would fail the call rather than silently pass. +// +// It also pins the deliberate simplification: a ServiceAccount that does not +// exist is no longer an ErrCodeNotFound. Nothing is consulted that could +// know, and the operator reviews the manifests before applying them. +func TestWriteAgentRoleManifests_NoClusterAccess(t *testing.T) { + unreachable := filepath.Join(t.TempDir(), "not-a-kubeconfig") + if seedErr := os.WriteFile(unreachable, []byte("this is not a kubeconfig\n"), 0o600); seedErr != nil { + t.Fatalf("seeding the kubeconfig: %v", seedErr) + } + t.Setenv("KUBECONFIG", unreachable) + t.Setenv("HOME", t.TempDir()) + t.Setenv("KUBERNETES_SERVICE_HOST", "") + t.Setenv("KUBERNETES_SERVICE_PORT", "") + + config := defaultRolesConfig() + config.ServiceAccountName = "no-such-serviceaccount" + res := writeRoles(t, config) + + if len(res.Objects) != 4 { + t.Fatalf("Objects = %d, want 4 written without any cluster access", len(res.Objects)) + } + body, err := os.ReadFile(res.Objects[1].Path) + if err != nil { + t.Fatalf("reading the RoleBinding: %v", err) + } + if !strings.Contains(string(body), "no-such-serviceaccount") { + t.Errorf("RoleBinding does not name the unverified ServiceAccount:\n%s", body) + } +} + +// TestWriteAgentRoleManifests_Rejections covers every input refused before +// anything reaches the filesystem, so a bad value never leaves a directory +// behind that would block the corrected retry. +func TestWriteAgentRoleManifests_Rejections(t *testing.T) { + tests := []struct { + name string + config *AgentRolesConfig + }{ + {name: "nil config", config: nil}, + {name: "empty namespace", config: &AgentRolesConfig{ServiceAccountName: provisionSAName}}, + {name: "whitespace namespace", config: &AgentRolesConfig{Namespace: " ", ServiceAccountName: provisionSAName}}, + {name: "empty ServiceAccount name", config: &AgentRolesConfig{Namespace: provisionNamespace}}, + {name: "whitespace ServiceAccount name", config: &AgentRolesConfig{Namespace: provisionNamespace, ServiceAccountName: " "}}, + { + name: "name too long to compose", + config: &AgentRolesConfig{ + Namespace: provisionNamespace, + ServiceAccountName: strings.Repeat("a", 250), + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + t.Chdir(dir) + + res, err := WriteAgentRoleManifests(tt.config) + if err == nil { + t.Fatal("WriteAgentRoleManifests() error = nil, want ErrCodeInvalidRequest") + } + if !stderrors.Is(err, errors.New(errors.ErrCodeInvalidRequest, "")) { + t.Errorf("error = %v, want code %s", err, errors.ErrCodeInvalidRequest) + } + if res != nil { + t.Errorf("result = %+v, want nil", res) + } + + entries, readErr := os.ReadDir(dir) + if readErr != nil { + t.Fatalf("reading the working directory: %v", readErr) + } + if len(entries) != 0 { + t.Errorf("working directory has %d entries, want 0 (a rejected call must write nothing)", len(entries)) + } + }) + } +} diff --git a/pkg/validator/labels/labels.go b/pkg/validator/labels/labels.go index 9c7c9988f..6f99b6fa1 100644 --- a/pkg/validator/labels/labels.go +++ b/pkg/validator/labels/labels.go @@ -15,19 +15,25 @@ // Package labels provides shared label constants for validation resources. package labels -import "github.com/NVIDIA/aicr/pkg/header" +import ( + "github.com/NVIDIA/aicr/pkg/header" + k8slabels "github.com/NVIDIA/aicr/pkg/k8s/labels" +) -// Standard Kubernetes label keys. +// Standard Kubernetes label keys. These are aliases of pkg/k8s/labels, which +// is the single source of truth shared with the snapshot agent +// (pkg/k8s/agent) — validator code should keep referring to them as +// labels.Name etc. rather than importing pkg/k8s/labels directly. const ( - Name = "app.kubernetes.io/name" - Component = "app.kubernetes.io/component" - ManagedBy = "app.kubernetes.io/managed-by" + Name = k8slabels.Name + Component = k8slabels.Component + ManagedBy = k8slabels.ManagedBy + RunID = k8slabels.RunID ) // AICR-specific label keys, keyed on the canonical AICR API domain. const ( JobType = header.Domain + "/job-type" - RunID = header.Domain + "/run-id" Validator = header.Domain + "/validator" Phase = header.Domain + "/phase" ReportType = header.Domain + "/report-type" @@ -35,7 +41,7 @@ const ( // Common label values. const ( - ValueAICR = "aicr" + ValueAICR = k8slabels.ValueAICR ValueValidation = "validation" ValueValidator = "aicr-validator" ) diff --git a/pkg/validator/v1/job_plan.go b/pkg/validator/v1/job_plan.go index d25fc5aba..c3b2312bc 100644 --- a/pkg/validator/v1/job_plan.go +++ b/pkg/validator/v1/job_plan.go @@ -15,14 +15,11 @@ package v1 import ( - "crypto/rand" - "encoding/hex" - "fmt" "strings" - "time" "github.com/NVIDIA/aicr/pkg/defaults" "github.com/NVIDIA/aicr/pkg/recipe" + "github.com/NVIDIA/aicr/pkg/runid" "github.com/NVIDIA/aicr/pkg/validator/labels" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -105,16 +102,7 @@ type JobPlan struct { // exceptional and we prefer to fail fast rather than generate predictable IDs // that could collide across concurrent runs. func GenerateRunID() string { - timestamp := time.Now().Format("20060102-150405") - randomBytes := make([]byte, 8) - n, err := rand.Read(randomBytes) - if err != nil { - panic(fmt.Sprintf("failed to generate random bytes for runID: %v", err)) - } - if n != len(randomBytes) { - panic(fmt.Sprintf("failed to generate runID: read %d bytes, expected %d", n, len(randomBytes))) - } - return fmt.Sprintf("%s-%s", timestamp, hex.EncodeToString(randomBytes)) + return runid.Generate() } // ImagePullPolicy determines the pull policy for a container image. diff --git a/tests/e2e/run.sh b/tests/e2e/run.sh index 6ef031851..46806ac45 100755 --- a/tests/e2e/run.sh +++ b/tests/e2e/run.sh @@ -56,6 +56,34 @@ CREATED_FAKE_GPU_OPERATOR_DEPLOYMENT=false CREATED_FAKE_CLUSTER_POLICY=false CREATED_FAKE_CLUSTER_POLICY_CRD=false +# Seconds to wait for the concurrent runs' self-deleted Jobs to disappear from +# the API before the self-cleanup assertion gives up. Deletion is asynchronous, +# so the count settles shortly after the CLI returns rather than at that instant. +AGENT_JOB_SETTLE_TIMEOUT="${AGENT_JOB_SETTLE_TIMEOUT:-60}" + +# Run IDs of the snapshot-agent runs this script launched, space separated. +# cleanup_e2e deletes only these Jobs. Every agent run carries the same +# app.kubernetes.io/{name,component} labels, so a label-only sweep would also +# delete -- and thereby terminate -- a concurrent snapshot run someone else +# started against the same cluster. +E2E_AGENT_RUN_IDS="" + +# extract_run_id prints the agent run ID an `aicr snapshot` invocation logged +# ("runID=" on its "snapshot agent run" line), read from the log file +# given as $1. Prints nothing when the run failed before generating one. +extract_run_id() { + grep -o 'runID=[0-9a-f-]*' "$1" | head -1 | cut -d= -f2 || true +} + +# remember_agent_run_id records a run ID in E2E_AGENT_RUN_IDS so cleanup_e2e +# can sweep that run's Job. An empty argument is ignored: a run that never +# reported an ID created no Job for cleanup to find. +remember_agent_run_id() { + if [ -n "$1" ]; then + E2E_AGENT_RUN_IDS="${E2E_AGENT_RUN_IDS} $1" + fi +} + # Test counters TOTAL_TESTS=0 PASSED_TESTS=0 @@ -356,38 +384,15 @@ setup_fake_gpu() { # Create namespace for snapshot tests (if it doesn't exist) kubectl create namespace "$SNAPSHOT_NAMESPACE" --dry-run=client -o yaml | kubectl apply -f - - # Create RBAC for snapshot agent - msg "Creating RBAC for snapshot agent" - kubectl apply -f - << EOF -apiVersion: v1 -kind: ServiceAccount -metadata: - name: aicr - namespace: ${SNAPSHOT_NAMESPACE} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: aicr-e2e-reader -rules: -- apiGroups: [""] - resources: ["nodes", "pods", "configmaps"] - verbs: ["get", "list", "watch", "create", "update", "patch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: aicr-e2e-reader -subjects: -- kind: ServiceAccount - name: aicr - namespace: ${SNAPSHOT_NAMESPACE} -roleRef: - kind: ClusterRole - name: aicr-e2e-reader - apiGroup: rbac.authorization.k8s.io -EOF - pass "setup/rbac" + # No RBAC is pre-provisioned here. Per ADR-020 the agent creates its own + # run-scoped ServiceAccount, Role/RoleBinding and ClusterRole/ClusterRoleBinding + # ("aicr-" / "aicr-node-reader-") and deletes exactly those at + # the end of the run. The fixture this replaced pre-created a ServiceAccount + # named "aicr" bound to an "aicr-e2e-reader" ClusterRole granting + # nodes/pods/configmaps — the same access the agent now grants itself. Nothing + # referenced that ServiceAccount once the agent stopped using a fixed name, and + # its presence tripped the agent's adoption-drift warning on every run because + # it collided with the default name prefix. return 0 } @@ -416,6 +421,7 @@ test_snapshot() { detail "Output: cm://${SNAPSHOT_NAMESPACE}/${SNAPSHOT_CM}" echo -e "${DIM} \$ aicr snapshot --image ${AICR_IMAGE} --namespace ${SNAPSHOT_NAMESPACE} -o cm://${SNAPSHOT_NAMESPACE}/${SNAPSHOT_CM}${NC}" + local snapshot_log="${OUTPUT_DIR}/snapshot-agent.log" local snapshot_output snapshot_output=$("${AICR_BIN}" snapshot \ --image "${AICR_IMAGE}" \ @@ -424,6 +430,8 @@ test_snapshot() { --timeout 120s \ --privileged \ --node-selector kubernetes.io/os=linux 2>&1) || true + printf '%s\n' "$snapshot_output" > "$snapshot_log" + remember_agent_run_id "$(extract_run_id "$snapshot_log")" if kubectl get cm "$SNAPSHOT_CM" -n "$SNAPSHOT_NAMESPACE" > /dev/null 2>&1; then pass "snapshot/agent" @@ -510,6 +518,278 @@ test_snapshot() { fi } +# ============================================================================= +# Snapshot Run Isolation Tests (ADR-020, issue #2120) +# ============================================================================= + +# Objects snapshot_run_isolation_body creates that outlive its assertions. +# Published as globals so cleanup_snapshot_run_isolation can remove them on +# every exit path, including an early `return 1` from a failed assertion. +ISOLATION_DECOY="" +ISOLATION_RETAINED_ID="" + +# cleanup_snapshot_run_isolation removes the decoy ClusterRole and the +# retained (--no-cleanup) run's objects. Safe to call when the body bailed +# before creating either: both globals are empty then. +cleanup_snapshot_run_isolation() { + if [ -n "$ISOLATION_DECOY" ]; then + kubectl delete clusterrole "$ISOLATION_DECOY" --ignore-not-found=true > /dev/null 2>&1 || true + fi + if [ -n "$ISOLATION_RETAINED_ID" ]; then + local ns="$SNAPSHOT_NAMESPACE" + kubectl delete job "aicr-${ISOLATION_RETAINED_ID}" -n "$ns" --ignore-not-found=true > /dev/null 2>&1 || true + kubectl delete sa,role,rolebinding "aicr-${ISOLATION_RETAINED_ID}" -n "$ns" --ignore-not-found=true > /dev/null 2>&1 || true + kubectl delete cm "aicr-agent-snapshot-${ISOLATION_RETAINED_ID}" -n "$ns" --ignore-not-found=true > /dev/null 2>&1 || true + kubectl delete clusterrole,clusterrolebinding "aicr-node-reader-${ISOLATION_RETAINED_ID}" --ignore-not-found=true > /dev/null 2>&1 || true + fi + ISOLATION_DECOY="" + ISOLATION_RETAINED_ID="" +} + +# Verifies that concurrent snapshot runs own and delete only their own +# Kubernetes resources. These checks are cluster-backed on purpose: the +# unit tests in pkg/k8s/agent run against a fake clientset, which runs no +# Job controller (so pod ownerReferences must be hand-seeded there) and does +# not enforce metav1.Preconditions on delete. Only a real apiserver exercises +# both. +# +# Scope note on UID preconditions: the delete-with-stale-UID race itself is +# not reachable from outside the CLI process — the window between an object's +# creation and the deferred Cleanup is the Job wait, and swapping a live +# object's UID mid-run revokes the running agent's own credentials. That +# mechanism is covered by TestCleanupPassesUIDPrecondition and +# TestCleanupTreatsConflictAsSuccess in pkg/k8s/agent/deployer_test.go. What +# is verified here is the ownership contract those preconditions enforce and +# that IS externally observable: cleanup deletes only objects this run +# created, never one that merely matches its labels or name shape. +test_snapshot_run_isolation() { + local rc=0 + snapshot_run_isolation_body || rc=$? + # Housekeeping must run whether the body passed or bailed early. Every + # `return 1` below used to skip it, leaving an aicr-labelled ClusterRole + # and a full set of run-scoped objects behind for later tests -- and later + # CI runs on the same cluster -- to trip over. The body's status is + # preserved so a failed assertion still fails the suite. + cleanup_snapshot_run_isolation + return "$rc" +} + +snapshot_run_isolation_body() { + msg "==========================================" + msg "Testing snapshot run isolation" + msg "==========================================" + + if [ "$FAKE_GPU_ENABLED" != "true" ]; then + skip "snapshot/isolation" "Fake GPU not enabled" + return 0 + fi + + local ns="$SNAPSHOT_NAMESPACE" + local decoy="aicr-node-reader-e2edecoy" + local rc=0 + + # --- Decoy: labelled like a run-owned object, but created by nobody's run --- + # A cleanup that swept by label or by name shape would collect this. A + # cleanup scoped to its own created-set must leave it alone. + msg "--- Test: cleanup is scoped to created objects, not to labels ---" + kubectl delete clusterrole "$decoy" --ignore-not-found=true > /dev/null 2>&1 || true + kubectl create clusterrole "$decoy" --verb=get --resource=nodes > /dev/null 2>&1 || true + kubectl label clusterrole "$decoy" \ + app.kubernetes.io/name=aicr \ + app.kubernetes.io/managed-by=aicr \ + app.kubernetes.io/component=snapshot-agent \ + aicr.run/run-id=e2edecoy --overwrite > /dev/null 2>&1 || true + ISOLATION_DECOY="$decoy" + local decoy_uid_before + decoy_uid_before=$(kubectl get clusterrole "$decoy" -o jsonpath='{.metadata.uid}' 2>/dev/null || echo "") + + # --- Retained run: --no-cleanup, so its objects must outlive later runs --- + msg "--- Test: a retained run's resources survive concurrent runs ---" + local retained_log="${OUTPUT_DIR}/isolation-retained.log" + "${AICR_BIN}" snapshot \ + --image "${AICR_IMAGE}" \ + --namespace "${ns}" \ + --no-cleanup \ + --output "${OUTPUT_DIR}/isolation-retained.yaml" \ + --timeout 180s \ + --privileged \ + --node-selector kubernetes.io/os=linux > "$retained_log" 2>&1 || rc=$? + + if [ "$rc" -ne 0 ]; then + cat "$retained_log" + fail "snapshot/isolation/retained-run" "retained snapshot run failed" + return 1 + fi + + # Take the run ID from THIS run's own output. A label query returning + # `.items[0]` would happily hand back an aborted run's Job, or a concurrent + # snapshot someone else started -- and everything below, including the + # housekeeping delete, is keyed on this value. + local retained_id + retained_id=$(extract_run_id "$retained_log") + if [ -z "$retained_id" ]; then + cat "$retained_log" + fail "snapshot/isolation/run-id-label" "retained run printed no runID" + return 1 + fi + ISOLATION_RETAINED_ID="$retained_id" + remember_agent_run_id "$retained_id" + + # That Job must carry the run ID as a label, and the stable + # component label consumers select on across runs. + local retained_job_labels + retained_job_labels=$(kubectl get job -n "$ns" "aicr-${retained_id}" \ + -o jsonpath='{.metadata.labels.aicr\.run/run-id}/{.metadata.labels.app\.kubernetes\.io/component}' \ + 2>/dev/null || echo "") + if [ "$retained_job_labels" != "${retained_id}/snapshot-agent" ]; then + fail "snapshot/isolation/run-id-label" \ + "Job aicr-${retained_id} labels run-id/component = '${retained_job_labels}', want '${retained_id}/snapshot-agent'" + return 1 + fi + detail "retained run ID: ${retained_id}" + pass "snapshot/isolation/run-id-label" + + # Every run-owned object must carry the run ID in its name. + local missing="" + kubectl get job -n "$ns" "aicr-${retained_id}" > /dev/null 2>&1 || missing="${missing} job" + kubectl get sa -n "$ns" "aicr-${retained_id}" > /dev/null 2>&1 || missing="${missing} sa" + kubectl get role -n "$ns" "aicr-${retained_id}" > /dev/null 2>&1 || missing="${missing} role" + kubectl get rolebinding -n "$ns" "aicr-${retained_id}" > /dev/null 2>&1 || missing="${missing} rolebinding" + kubectl get cm -n "$ns" "aicr-agent-snapshot-${retained_id}" > /dev/null 2>&1 || missing="${missing} staging-cm" + kubectl get clusterrole "aicr-node-reader-${retained_id}" > /dev/null 2>&1 || missing="${missing} clusterrole" + kubectl get clusterrolebinding "aicr-node-reader-${retained_id}" > /dev/null 2>&1 || missing="${missing} clusterrolebinding" + if [ -n "$missing" ]; then + fail "snapshot/isolation/run-scoped-names" "not found under run-scoped names:${missing}" + return 1 + fi + pass "snapshot/isolation/run-scoped-names" + + # The agent's staging ConfigMap must not reuse the validator's name shape. + # aicr validate hands one run ID to both subsystems in one namespace, so a + # shared aicr-snapshot- prefix would give two owners one object. + if kubectl get cm -n "$ns" "aicr-snapshot-${retained_id}" > /dev/null 2>&1; then + fail "snapshot/isolation/staging-cm-name" "found aicr-snapshot-${retained_id}; collides with the validator's data ConfigMap" + return 1 + fi + pass "snapshot/isolation/staging-cm-name" + + # The pod must be authorized by its controlling ownerReference, not by a + # label — pod labels are writable by anything that can update pods. + msg "--- Test: pod is owned by its Job (controlling ownerReference) ---" + local pod job_uid owner_uid + pod=$(kubectl get pods -n "$ns" -l "aicr.run/run-id=${retained_id}" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "") + job_uid=$(kubectl get job -n "$ns" "aicr-${retained_id}" -o jsonpath='{.metadata.uid}' 2>/dev/null || echo "") + owner_uid=$(kubectl get pod -n "$ns" "$pod" \ + -o jsonpath='{.metadata.ownerReferences[?(@.controller==true)].uid}' 2>/dev/null || echo "") + if [ -n "$pod" ] && [ -n "$job_uid" ] && [ "$job_uid" = "$owner_uid" ]; then + detail "pod ${pod} controlled by Job UID ${job_uid}" + pass "snapshot/isolation/pod-owned-by-job" + else + fail "snapshot/isolation/pod-owned-by-job" "pod=${pod} jobUID=${job_uid} ownerUID=${owner_uid}" + return 1 + fi + + # --- Two concurrent runs, with the retained run's objects still present --- + msg "--- Test: two concurrent runs are independent ---" + local log_a="${OUTPUT_DIR}/isolation-a.log" + local log_b="${OUTPUT_DIR}/isolation-b.log" + local rc_a=0 rc_b=0 pid_a pid_b + + "${AICR_BIN}" snapshot --image "${AICR_IMAGE}" --namespace "${ns}" \ + --output "${OUTPUT_DIR}/isolation-a.yaml" --timeout 180s --privileged \ + --node-selector kubernetes.io/os=linux > "$log_a" 2>&1 & + pid_a=$! + "${AICR_BIN}" snapshot --image "${AICR_IMAGE}" --namespace "${ns}" \ + --output "${OUTPUT_DIR}/isolation-b.yaml" --timeout 180s --privileged \ + --node-selector kubernetes.io/os=linux > "$log_b" 2>&1 & + pid_b=$! + + wait "$pid_a" || rc_a=$? + wait "$pid_b" || rc_b=$? + + if [ "$rc_a" -ne 0 ] || [ "$rc_b" -ne 0 ]; then + echo "--- concurrent run A ---"; tail -30 "$log_a" + echo "--- concurrent run B ---"; tail -30 "$log_b" + fail "snapshot/isolation/concurrent-runs" "run A rc=${rc_a}, run B rc=${rc_b}" + return 1 + fi + + local id_a id_b + id_a=$(extract_run_id "$log_a") + id_b=$(extract_run_id "$log_b") + if [ -z "$id_a" ] || [ -z "$id_b" ] || [ "$id_a" = "$id_b" ]; then + fail "snapshot/isolation/concurrent-runs" "expected two distinct run IDs, got '${id_a}' and '${id_b}'" + return 1 + fi + remember_agent_run_id "$id_a" + remember_agent_run_id "$id_b" + detail "run A: ${id_a}" + detail "run B: ${id_b}" + if [ ! -s "${OUTPUT_DIR}/isolation-a.yaml" ] || [ ! -s "${OUTPUT_DIR}/isolation-b.yaml" ]; then + fail "snapshot/isolation/concurrent-runs" "a concurrent run produced an empty snapshot" + return 1 + fi + pass "snapshot/isolation/concurrent-runs" + + # The retained run's objects must be untouched by both concurrent runs. + # Before ADR-020 this is exactly what broke: a second run's ensureJob + # deleted the same-named Job and its cleanup deleted the shared RBAC. + local destroyed="" + kubectl get job -n "$ns" "aicr-${retained_id}" > /dev/null 2>&1 || destroyed="${destroyed} job" + kubectl get sa -n "$ns" "aicr-${retained_id}" > /dev/null 2>&1 || destroyed="${destroyed} sa" + kubectl get role -n "$ns" "aicr-${retained_id}" > /dev/null 2>&1 || destroyed="${destroyed} role" + kubectl get rolebinding -n "$ns" "aicr-${retained_id}" > /dev/null 2>&1 || destroyed="${destroyed} rolebinding" + kubectl get clusterrole "aicr-node-reader-${retained_id}" > /dev/null 2>&1 || destroyed="${destroyed} clusterrole" + kubectl get clusterrolebinding "aicr-node-reader-${retained_id}" > /dev/null 2>&1 || destroyed="${destroyed} clusterrolebinding" + if [ -n "$destroyed" ]; then + fail "snapshot/isolation/retained-run-survives" "concurrent runs destroyed:${destroyed}" + return 1 + fi + pass "snapshot/isolation/retained-run-survives" + + # Each concurrent run must have removed its own resources: exactly the + # retained Job survives. + # + # Poll rather than sample once. Kubernetes deletion is asynchronous -- a Job + # whose delete the CLI already issued and acked stays listable while its + # pods terminate and its finalizers clear, so a single read can legitimately + # still see it and report "found 2" for a cleanup that worked. Only the + # timing is tolerant; the assertion below is still exact. + local leftover_jobs="" + local agent_job_selector="app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent" + local deadline=$((SECONDS + AGENT_JOB_SETTLE_TIMEOUT)) + while :; do + leftover_jobs=$(kubectl get jobs -n "$ns" -l "$agent_job_selector" -o name 2>/dev/null | wc -l | tr -d ' ') + if [ "$leftover_jobs" = "1" ]; then + break + fi + if [ "$SECONDS" -ge "$deadline" ]; then + kubectl get jobs -n "$ns" -l "$agent_job_selector" -o name || true + fail "snapshot/isolation/self-cleanup" \ + "expected only the retained Job to remain after ${AGENT_JOB_SETTLE_TIMEOUT}s, found ${leftover_jobs}" + return 1 + fi + sleep 2 + done + pass "snapshot/isolation/self-cleanup" + + # The decoy must have survived every run above. + local decoy_uid_after + decoy_uid_after=$(kubectl get clusterrole "$decoy" -o jsonpath='{.metadata.uid}' 2>/dev/null || echo "") + if [ -n "$decoy_uid_before" ] && [ "$decoy_uid_before" = "$decoy_uid_after" ]; then + pass "snapshot/isolation/cleanup-scoped-to-created" + else + fail "snapshot/isolation/cleanup-scoped-to-created" \ + "aicr-labelled ClusterRole it never created was deleted or replaced (before=${decoy_uid_before} after=${decoy_uid_after})" + return 1 + fi + + # Housekeeping (the decoy and the retained run's objects) is the wrapper's + # job -- see cleanup_snapshot_run_isolation -- so it also happens on the + # `return 1` paths above. +} + # ============================================================================= # Recipe from Snapshot Tests (from e2e.md) # ============================================================================= @@ -1879,8 +2159,17 @@ cleanup_e2e() { msg "Cleaning up e2e resources" msg "==========================================" - # Clean up snapshot resources - kubectl delete job aicr-e2e-snapshot -n "$SNAPSHOT_NAMESPACE" --ignore-not-found=true > /dev/null 2>&1 || true + # Clean up snapshot resources. The Job name is run-scoped ("aicr-"), + # so select by label -- but scoped to the run IDs THIS script launched. + # A sweep of every Job matching the agent's labels would also delete, and + # so terminate, a concurrent snapshot run started against the same cluster + # by someone else. + local run_id + for run_id in $E2E_AGENT_RUN_IDS; do + kubectl delete job \ + -l "app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent,aicr.run/run-id=${run_id}" \ + -n "$SNAPSHOT_NAMESPACE" --ignore-not-found=true > /dev/null 2>&1 || true + done kubectl delete cm "$SNAPSHOT_CM" -n "$SNAPSHOT_NAMESPACE" --ignore-not-found=true > /dev/null 2>&1 || true msg "Cleanup complete" @@ -1945,6 +2234,7 @@ main() { # Setup fake GPU environment and run snapshot tests if setup_fake_gpu; then test_snapshot + test_snapshot_run_isolation test_recipe_from_snapshot test_validate test_validate_deployment_checks diff --git a/tools/cleanup b/tools/cleanup index 44cfdd278..6ff0f987a 100755 --- a/tools/cleanup +++ b/tools/cleanup @@ -26,6 +26,10 @@ # are touched. Cluster-scoped resources outside the AICR component domains are # left alone. # +# Not narrow, by design: the snapshot-agent sweep in phase 2 is label-based and +# matches every run, including one in flight. See the HAZARD note there before +# running this against a cluster someone else is using. +# # On shared / bring-up clusters a registry component may be deliberately owned # out-of-band (installed by the platform team, excluded from the AICR recipe). # Destroying it is data loss AICR does not own, so `--exclude-ns` and @@ -376,8 +380,16 @@ fi # runID suffix; we delete by label selector to catch all runs. Legacy resources # from the older job pattern lived in gpu-operator under the name "aicr". msg "Phase 2: Removing AICR validator artifacts..." -kc delete clusterrolebinding -l app.kubernetes.io/managed-by=aicr --ignore-not-found -kc delete clusterrole -l app.kubernetes.io/managed-by=aicr --ignore-not-found +# The component!=agent-rbac term spares the cluster RBAC an operator applied +# from the manifests `aicr snapshot --add-roles-to-service-account` wrote for +# an operator-supplied ServiceAccount. Those objects belong to no run — they +# carry no run-ID label and no run's cleanup deletes them — and restoring them +# needs the admin who applied them, so a teardown tool must not sweep them away +# with the per-run leftovers it exists to reclaim. Revoke them deliberately +# with `kubectl delete -f snapshot-rbac-/` when the ServiceAccount is +# retired. +kc delete clusterrolebinding -l 'app.kubernetes.io/managed-by=aicr,app.kubernetes.io/component!=agent-rbac' --ignore-not-found +kc delete clusterrole -l 'app.kubernetes.io/managed-by=aicr,app.kubernetes.io/component!=agent-rbac' --ignore-not-found kc delete clusterrolebinding -l app.kubernetes.io/name=aicr-validator --ignore-not-found kc delete clusterrole -l app.kubernetes.io/name=aicr-validator --ignore-not-found # Fall back to name-based deletion for any without the expected labels. @@ -390,11 +402,50 @@ if ! $DRY_RUN; then | xargs -r kubectl delete --ignore-not-found || true fi kc delete ns aicr-validation --ignore-not-found --wait=false -# Legacy on-cluster agent leftovers from the older deployment pattern. +# On-cluster snapshot-agent leftovers, current and legacy. Both sweeps are +# needed and neither subsumes the other: +# +# - Current (ADR-020) runs name their Job/SA/Role/RoleBinding +# "aicr-", so no fixed-name delete matches them; they are found by +# the label set every one of them carries. +# - Pre-ADR-020 runs used the fixed name "aicr" and carried none of those +# labels — the Job had only app.kubernetes.io/name, and the SA, Role and +# RoleBinding had no labels at all — so the label selector above misses +# them entirely. The name-based deletes below are the only thing that +# collects them. +# +# The two cannot collide: a run-scoped name is always "aicr-", never +# the bare "aicr". +# +# HAZARD -- this sweep is not run-scoped, and deliberately so. EVERY agent run +# carries this exact label set, so running `tools/cleanup` while a snapshot is +# in flight deletes that run's Job and revokes its ServiceAccount, Role and +# RoleBinding; the run fails partway through. That is the intended contract for +# a manual teardown tool: it reclaims the namespace regardless of which run +# left an object behind, including runs whose IDs this invocation never saw. +# Contrast tests/e2e/run.sh, which records the run IDs it launched and sweeps +# only those: an unattended suite shares its cluster and cannot make this +# choice. Do not run this tool against a cluster with an active +# `aicr snapshot` or `aicr validate`. +kc -n gpu-operator delete job -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent --ignore-not-found +kc -n gpu-operator delete sa -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent --ignore-not-found +kc -n gpu-operator delete role -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent --ignore-not-found +kc -n gpu-operator delete rolebinding -l app.kubernetes.io/name=aicr,app.kubernetes.io/component=snapshot-agent --ignore-not-found kc -n gpu-operator delete job aicr --ignore-not-found kc -n gpu-operator delete sa aicr --ignore-not-found kc -n gpu-operator delete role aicr --ignore-not-found kc -n gpu-operator delete rolebinding aicr --ignore-not-found +# The pre-ADR-020 cluster-scoped pair, for the same reason: it was named +# literally "aicr-node-reader" and carried no labels, so every selector above +# misses it and the name-grep fallback ('/aicr-validator-|/aicr$') does not +# match it either. Nothing else in the tree deletes it now that run-owned +# names are "aicr-node-reader-" — a later run no longer reclaims the +# bare name — so without this it is a permanent orphan. That matters because a +# leftover from `--discover-network` carries mutating grants (nodes: patch, +# pods/exec: create, CRD and namespace create/delete). The exact-name delete +# cannot touch a live run, whose name always carries a run-ID suffix. +kc delete clusterrole aicr-node-reader --ignore-not-found +kc delete clusterrolebinding aicr-node-reader --ignore-not-found # Phase 3: Component CRDs. # Helm does NOT remove CRDs on uninstall — they must be deleted manually or a