diff --git a/deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml b/deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml index 220e0c033..bbbdb427c 100644 --- a/deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml +++ b/deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml @@ -20,8 +20,8 @@ # A transition value is a closed enum, not free text. A strategy name means # enabled; disabled means the workflow is unsupported. # A disabled transition means that the end-to-end workflow is not qualified. -# The strict loader validates this catalog. Runtime reconciliation does not -# consume it until the storage-selection change is released. +# NVCA reads this catalog when it creates a new model-cache storage selection. +# Only a registered, non-disabled transition can select durable caching. apiVersion: storage.nvcf.nvidia.com/v1alpha1 kind: StorageCapabilityCatalog drivers: diff --git a/docs/dev/sdd-storage-agnostic-cache-architecture.md b/docs/dev/sdd-storage-agnostic-cache-architecture.md index 8d8f6d806..313a99912 100644 --- a/docs/dev/sdd-storage-agnostic-cache-architecture.md +++ b/docs/dev/sdd-storage-agnostic-cache-architecture.md @@ -1,112 +1,106 @@ -# SDD: Storage-Agnostic Cache Architecture +# SDD: Storage-Agnostic Model Cache Architecture ## Summary -This document separates implemented foundation from target runtime behavior. +New model-cache requests select storage from two cluster objects: -Implemented foundation: +- `StorageClass/nvcf-sc` identifies the installed CSI provisioner. +- ConfigMap `nvcf-storage-capabilities` maps that exact provisioner to an NVCA transition. -- The NVCA Operator chart installs a public `v1alpha1` storage catalog and packages its JSON Schema. -- The catalog uses closed transition enums and records required reader-PV mount options. -- NVCA has a strict catalog loader and semantic validator. Tests call them; runtime reconciliation does not. -- Current public NVCA still uses legacy StorageClass-presence checks and feature flags for backend selection. -- Weka, OCI File Storage (FSS), and OCI Lustre model-cache transitions remain `disabled`. +NVCA records the selection on the `ICMSRequest`, creates an immutable `ModelCacheBinding`, and adds the exact request +UID to the binding before it creates cache storage. Runtime code then uses the recorded binding. It does not select a +different durable provider after a retry, restart, feature-gate change, or catalog change. -Target: +NVCA has two registered regular transition branches: `roxReadOnly` and `rwxReadOnly`. Only `roxReadOnly` is selected by the +catalog. The shipped Weka, OCI File Storage (FSS), and OCI Lustre entries remain `disabled` for regular and Helm +requests, so they do not enable NVCF cache traffic. Helm supports only `roxReadOnly`. -1. Deployment tooling renders exactly one selected provider as `StorageClass/nvcf-sc`. -2. The public NVCA catalog declares qualified access modes, reader-PV mount options, and regular/Helm cache - transitions. -3. NVCA resolves the live class and catalog entry, then persists a shared cache binding before storage side effects. -4. A provider transition remains disabled until its complete functional contract passes. +For `rwxReadOnly`, NVCA populates one RWX PVC and returns that same claim to workload Pods. Workload construction sets +the PVC volume source and every matching init-container and container mount read-only. The current branch accepts only +binding-safe writer Jobs with no environment input, image-pull Secret, or Secret-backed volume. Current translated +writer artifacts do not meet that restriction. This branch therefore supports controlled, credential-free NVCA +storage-path tests; it is not yet an end-to-end production model-cache path for Weka or OCI FSS. -Deployment tooling owns CSI-specific StorageClass parameters. NVCA owns model-cache transitions. A provider name, -provisioner, or access mode alone never enables a workflow. - -Editing the catalog ConfigMap does not enable a provider today. Runtime work is tracked in -[NVIDIA/nvcf#1326](https://github.com/NVIDIA/nvcf/issues/1326); this SDD defines the stable `nvcf-sc` target contract. +Automated tests exercise schema and selection validation, controller state transitions, fake-client actions, and +generated Pod objects. They do not mount a real CSI volume or prove backend write denial, multi-Pod access, data +identity, an NVCA process restart, provider failure behavior, or performance. ## Table of contents -- [Summary](#summary) - [Scope](#scope) -- [Terms](#terms) -- [Deployment configuration](#deployment-configuration) -- [Storage catalog](#storage-catalog) -- [Target runtime design](#target-runtime-design) -- [Model-cache contract](#model-cache-contract) -- [Failure, migration, and rollback](#failure-migration-and-rollback) -- [Provider qualification](#provider-qualification) -- [Test plan](#test-plan) -- [Security and observability](#security-and-observability) -- [Implementation order](#implementation-order) +- [Current support](#current-support) +- [Cluster configuration](#cluster-configuration) +- [Runtime architecture](#runtime-architecture) +- [Regular model cache](#regular-model-cache) +- [Helm model cache](#helm-model-cache) +- [Encryption](#encryption) +- [Failure and cleanup rules](#failure-and-cleanup-rules) +- [Compatibility and limitations](#compatibility-and-limitations) +- [Test contract](#test-contract) +- [Provider enablement](#provider-enablement) +- [Rollout and rollback](#rollout-and-rollback) - [Public source references](#public-source-references) ## Scope -This design covers regular and Helm/MiniService model cache across managed and self-managed deployments, including -NVMesh, Weka, OCI FSS, OCI Lustre, future CSI providers, functional qualification, migration, and rollback. +This design covers regular container model cache and Helm/MiniService model cache. It includes provider selection, +binding ownership, NVMesh regular and Helm execution, provider-neutral regular `rwxReadOnly` execution, Kubernetes +read-only intent, cleanup guards, and the contract for enabling another CSI provider. The `rwxReadOnly` execution +described here is limited to the credential-free writer contract above. -It excludes container cache; function, internal, and database storage; `nvcf-function-storage-sc`; CSI driver -installation/lifecycle; and performance qualification, which follows functional acceptance. +It excludes container cache, function storage, internal storage, database storage, CSI driver installation, and +performance qualification. -## Terms +## Current support -| Term | Meaning | -|---|---| -| Provider | Product or integration ID, such as `ociFss` or `weka` | -| Provisioner | Exact CSI string in `StorageClass.provisioner` | -| Access mode | Kubernetes PVC mode declared only after qualification for an exact provider configuration | -| Workflow | `regularModelCache` or `helmModelCache` | -| Transition | Named NVCA implementation that moves a cache from writer state to reusable reader state | -| Qualification record | Versioned evidence for the exact tested storage and cluster configuration | -| Sharing domain | Stable authorization and encryption scope, such as an NCA ID | -| Cache key | Tuple of workflow, sharing domain, and cache handle | -| Cache binding | Immutable provider and transition choice shared by all requests for one cache key | -| Request selection | Per-request mode and optional cache-binding reference persisted before side effects | -| Data identity | Provider-backed object containing one populated cache handle | -| Writer | Workload holding the binding Lease while it populates a data identity | -| Reader view | Provider-specific, namespace-local, read-only access to the same data identity | +| Provisioner | Catalog access modes | Required reader-PV mount options | Shipped regular transition | Shipped Helm transition | +|---|---|---|---|---| +| `nvmesh-csi.excelero.com` | RWO, ROX | `ro`, `norecovery`, `nouuid` | `roxReadOnly` | `roxReadOnly` | +| `csi.weka.io` | RWX, ROX | None | `disabled` | `disabled` | +| `fss.csi.oraclecloud.com` | RWX | None | `disabled` | `disabled` | +| `lustre.csi.oraclecloud.com` | None recorded | None | `disabled` | `disabled` | -`ReadOnlyMany` (ROX) is a PVC access mode. Mounting a `ReadWriteMany` (RWX) PVC with `readOnly: true` is not evidence -that the driver provisions or binds a ROX PVC. Neither condition alone proves backend-enforced write denial or -cross-namespace data identity. +RWO means `ReadWriteOnce`, ROX means `ReadOnlyMany`, and RWX means `ReadWriteMany`. -`ReadWriteOnce` (RWO) limits a volume to one node, not one Pod or writer. Kubernetes access modes describe attachment -and mount intent; NVCA must separately serialize writers and test reader write denial. +Catalog access modes record PVC modes exercised for an exact provider configuration. They do not prove an NVCF +model-cache transition. In particular, a read-only Pod mount of an RWX claim is not ROX evidence. -## Deployment configuration +`rwxReadOnly` is registered runtime code, not a shipped provider enablement. It is valid only for regular model cache +and requires `ReadWriteMany`. The schema and catalog validator reject it for Helm. It also rejects request-scoped +writer inputs until binding-scoped input identity and cleanup are implemented. The shipped Weka and FSS entries remain +`disabled` until that implementation gap is closed and their exact `nvcf-sc` configurations pass NVCA functional +qualification. -Managed input selects a provider through `spec.nvcfStorage.provider`, resolves -`spec.storage.drivers..storageClasses.nvcf-sc`, and renders `StorageClass/nvcf-sc`. +## Cluster configuration -The selected provider supplies its exact provisioner, parameters, mount options, binding mode, expansion setting, -and topology. Provider values must match the installed CSI driver. +### StorageClass contract -The class name is always `nvcf-sc`, its reclaim policy is always `Retain`, and exactly one installed provider is -primary even when multiple CSI drivers coexist. Rendering rejects an empty provisioner, invalid class fields, unknown -provider fields, and provider input for `reclaimPolicy`. +Deployment tooling owns provider-specific CSI parameters. NVCA requires: -When no provider is selected, managed output remains compatible with NVMesh. Self-managed deployments may use -different tooling, but NVCA sees the same live `StorageClass/nvcf-sc` contract. +- the class name is `nvcf-sc`; +- `provisioner` is non-empty; +- `reclaimPolicy` is `Retain`; +- exactly one provider supplies the class. -NVMesh supports volume expansion, so its `nvcf-sc` definition and regression tests must preserve -`allowVolumeExpansion: true`. Expansion is deployment behavior, not a field in the NVCA catalog. +The StorageClass digest covers provisioner, sorted parameters, reclaim policy, volume binding mode, mount options, and +allowed topologies. It excludes metadata and `allowVolumeExpansion`. Expansion remains a deployment setting. NVMesh +deployment configuration may keep `allowVolumeExpansion: true` without changing NVCA selection. -## Storage catalog +### Capability catalog -The chart installs ConfigMap `nvcf-storage-capabilities` in its release namespace. Data key -`storage-provider-capabilities.yaml` contains a `storage.nvcf.nvidia.com/v1alpha1` `StorageCapabilityCatalog`. Source -and release charts contain byte-identical catalog and schema files; rendering fails when the catalog file is absent. +The operator chart installs `nvcf-storage-capabilities`. The operator mirrors it into each NVCA agent namespace. Data +key `storage-provider-capabilities.yaml` contains the public catalog. -### Minimal shape +Minimal shape: ```yaml +apiVersion: storage.nvcf.nvidia.com/v1alpha1 +kind: StorageCapabilityCatalog drivers: : provider: accessModes: - - + - readerMountOptions: - transitions: @@ -114,50 +108,35 @@ drivers: helmModelCache: ``` -This is a shape illustration, not a valid provider entry. Actual entries must use exact provisioner strings, -Kubernetes access-mode names, and registered transition names. - -The exact provisioner is the lookup key. `provider` is a label, and `accessModes` cites externally qualified PVC -modes. Transition values are a closed enum, not free text. A strategy name means enabled; `disabled` means -unsupported. +The provisioner is the lookup key. `provider` is an identifier for logs and persisted state. Transition values are a +closed enum, not free text. A strategy name means enabled; `disabled` means unsupported. | Workflow field | Allowed values | |---|---| | `regularModelCache` | `disabled`, `roxReadOnly`, `rwxReadOnly` | | `helmModelCache` | `disabled`, `roxReadOnly` | -Transition code, not the flat access-mode list, defines state A to state B: - -| Transition | Writer to reader contract | Catalog restriction | +| Strategy | Runtime behavior | Current restriction | |---|---|---| -| `disabled` | No durable transition | Any provider | -| `roxReadOnly` | Populate an RWO writer, then publish ROX reader storage | Exact NVMesh provisioner and provider | -| `rwxReadOnly` | Populate one RWX claim, then serve it through read-only Pod mounts | Regular model cache only | - -Adding a transition requires dispatcher code, schema and semantic validation, a declared writer-to-reader mode pair, -and workflow tests. Access modes only show which required modes were qualified. +| `disabled` | Do not use durable storage for this workflow | None | +| `roxReadOnly` | Populate an RWO writer, then publish ROX reader storage with read-only Pod mounts | Exact NVMesh provisioner only | +| `rwxReadOnly` | Populate one RWX claim, then publish that same claim with read-only Pod mounts | Regular workflow only; no shipped provider is enabled yet | -`disabled` is the complete workflow-disabled state; a driver may still list qualified access modes. There is no -separate qualification field. Enabling a transition is a release decision allowed only after implementation and exact -workflow qualification; the schema cannot prove that evidence exists. +`readerMountOptions` is a required array. An empty array means only that the catalog records no reader-PV option +requirement; it is not provider qualification or enablement evidence. -`readerMountOptions` is a required array for options NVCA must apply when it creates or rewrites a reader PV. It is -not `StorageClass.mountOptions`, a Pod `volumeMount`, or the Pod-level read-only flag. `roxReadOnly` requires -`ro`, `norecovery`, and `nouuid`. `rwxReadOnly` uses the dynamically provisioned RWX claim and therefore -requires an empty array. An empty array is not provider qualification evidence. +Reader mount options apply only when a transition makes NVCA create or rewrite a read-only reader PV. They are not +`StorageClass.mountOptions`, arbitrary Pod `volumeMounts`, or the Pod-level read-only flag. The `rwxReadOnly` +transition returns the dynamically provisioned RWX claim and therefore uses no reader-PV mount options. +The catalog, request annotation, and binding are public Kubernetes data; mount options must never contain credentials +or other secrets. -The catalog intentionally does not contain a general CSI capability matrix. StorageClass rendering and provider -documentation own expansion, snapshots, clones, topology, and other CSI settings. +The parser rejects unknown fields, unknown access modes, duplicate modes or options, blank options, conflicting +required options, unknown transitions, and transition/provider mismatches. An NVMesh transition requires `ro`, +`norecovery`, and `nouuid`. Tests parse the shipped catalog and validate the shipped JSON Schema. -Access-mode rules: - -- Record only Kubernetes access modes exercised by functional evidence for the exact provider configuration. -- Do not infer ROX from a read-only Pod mount of an RWX claim. -- Do not infer RWX or ROX from CSI driver documentation alone. -- Do not infer cross-namespace sharing or backend write denial from an access mode. - -NVMesh uses transition `roxReadOnly` for both regular and Helm model cache. Samba is not an NVMesh transition. Weka, OCI -FSS, and OCI Lustre transitions remain `disabled` until their exact workflow qualification passes. +The catalog is not a general CSI capability matrix. It does not record expansion, snapshots, clones, performance, or +topology support. ### What operators set @@ -167,289 +146,416 @@ Use the exact installed CSI provisioner. Do not infer or enable a strategy from |---|---|---| | Shipped NVMesh configuration | `roxReadOnly` | `roxReadOnly` | | New or unqualified provider | `disabled` | `disabled` | -| Qualified shared-RWX regular workflow | `rwxReadOnly` | `disabled` | +| Qualified Weka, OCI FSS, VAST, or Lustre RWX regular workflow | `rwxReadOnly` | `disabled` | The last row is a target configuration, not current enablement. Set it only after the exact CSI and StorageClass -configuration passes the provider-qualification contract and its NVCA runtime transition is released. - -The target catalog does not register a generic `sharedfs` transition. Current NVCA retains a legacy, presence-selected -`sharedfs` route. Separate dynamic PVCs may expose different data, so class or driver presence cannot qualify it. - -### Validation contract +configuration passes the provider-qualification contract in this document and the listed NVCA limitations are closed. -CI validates structure, required fields, closed transition values, and workflow constraints with the packaged JSON -Schema; Helm only checks that the file exists. When called, the Go loader independently performs strict decoding and -semantic validation. It also rejects duplicate, blank, whitespace-padded, or conflicting reader mount options. The -test plan covers both validation layers. +### Selection outcomes -Catalog entries are configuration metadata, not credentials. The security requirements below govern their content. +For a cache request, NVCA evaluates `CachingSupport`, the Helm-only `HelmModelCaching` sub-gate, `nvcf-sc`, and the +catalog once, before it creates the `ICMSRequest`. -### Current runtime gap - -The loader has no runtime call site, and NVCA has no complete provider-neutral cache binding. Current requests persist -only a coarse backend value. Both gaps must be closed before the catalog can control runtime behavior. Until then, a -`disabled` catalog entry does not disable a legacy path selected by StorageClass presence. - -For Helm caching, current public NVCA first requires `CachingSupport` and `HelmModelCaching`. It then checks the legacy -`nvcf-sc-30` marker, the `nvcf-miniservice-sc` sharedfs sentinel, and a gated Samba fallback with a usable backing -class; otherwise it selects the per-Pod ephemeral cache. These are compatibility paths, not target provider selection. -The presence-only sharedfs branch cannot prove a cross-namespace backend identity. The target reads only `nvcf-sc`, -finds the exact provisioner entry, and selects its recorded transition. - -## Target runtime design - -### Selection algorithm - -For each request, NVCA must: +| Condition | Regular model cache | Helm model cache | +|---|---|---| +| Caching gate is off | `none` | `none` | +| `nvcf-sc` is absent | `none` | `ephemeral` | +| Selected workflow transition is `disabled` | `none` | `ephemeral` | +| Transition is `roxReadOnly` | `durable` | `durable` | +| Transition is `rwxReadOnly` | `durable` | Invalid catalog; request creation fails | +| Catalog is invalid or provisioner is unknown | Request creation fails | Request creation fails | +| `nvcf-sc` is not `Retain` | Request creation fails | Request creation fails | + +NVCA never falls through to a second durable provider. + +## Runtime architecture + +```text +ICMS cache request + -> persisted selection annotation + -> persisted NVCA request finalizer + -> Active ModelCacheBinding with exact request UID reference + -> binding UID labels on NVCA-created cache resources + -> no request-scoped metadata or ICMSRequest owner references on shared resources + -> exact bound-PV identity validation for rwxReadOnly + -> recorded provider transition + -> read-only workload volume and mounts +``` -1. Evaluate the applicable feature gates and workflow. Persist `none` or `ephemeral` on the request when selected; do - not create a durable binding. -2. For durable caching, derive `(workflow, sharingDomain, cacheHandle)` and read its binding from the model-cache - control namespace. -3. If it is `Active`, use it. If it is `Retiring`, retry or record a supported request fallback; never rebind it. -4. If no binding exists, read `StorageClass/nvcf-sc`, require `Retain`, and strictly load the catalog. -5. Find the exact provisioner entry and require a non-disabled transition with its required access modes and reader - mount options. -6. Create the binding by deterministic name and optimistic concurrency. A losing creator reads the winning binding. -7. Conditionally add the request namespace, name, and UID while the binding is `Active`. Then persist its name, UID, - and a request finalizer before any storage side effect. -8. Re-read the binding, confirm the reference and request finalizer are present, and execute only its transition. +### Persisted request selection -Unknown provisioners, invalid catalogs, non-`Retain` classes, and disabled transitions cannot select durable storage. -The only generic fallbacks are `none` and, where the workflow supports it, `ephemeral`; NVCA never switches to a -second durable provider. +Annotation `nvca.nvcf.nvidia.io/model-cache-storage-selection` records: -### Release qualification +- workflow and mode; +- StorageClass name, UID, and configuration digest; +- catalog payload digest; +- provider, provisioner, transition, required access modes, and required reader-PV mount options; +- the encryption decision, which may be `true` only for `roxReadOnly`; +- binding name and API-assigned UID after the binding is committed. -Provisioner matching selects code; it does not prove that a deployment was qualified. Before a transition is enabled, -dated evidence must identify the CSI provisioner and images, backend version/configuration, `nvcf-sc` digest, -Kubernetes version, eligible node/OS/architecture matrix, catalog/schema version, and evidence reference. +The annotation is strict JSON. Runtime validation rejects unknown fields, partial durable state, workflow changes, an +incomplete binding reference, and unsupported transitions. -Some backend fields are not discoverable through Kubernetes or CSI. The release or deployment qualification process, -not NVCA, owns this record and verifies those fields. NVCA snapshots only live Kubernetes data it can read: the -StorageClass name, UID, provisioner, configuration digest, and catalog digest. That snapshot detects drift for a cache -binding; it does not prove the backend product or version. +Immediately before first binding creation, NVCA revalidates the live StorageClass and exact catalog payload. A changed +UID, provisioner, `Retain` policy, StorageClass digest, catalog digest, provider, or transition fails before a storage +side effect. Transient API errors requeue. -Setting a transition to a non-disabled value is therefore an operator or release assertion that the qualification -record applies to the cluster. NVCA validates the catalog and live StorageClass, but cannot verify non-observable -fields. +After binding creation, the binding is authoritative. NVCA does not reselect from a later catalog revision. If a +binding-owned unencrypted writer PVC is missing and must be dynamically provisioned again, the live `nvcf-sc` must +still match the binding's StorageClass snapshot. Existing bound objects and static reader objects do not require that +live lookup. -### Cache binding and realized state +### ModelCacheBinding -Recomputing per request could mix providers for one cache key. The binding is one durable Kubernetes object in the -model-cache control namespace; every namespace-local request for that key references it. +`ModelCacheBinding` is a namespaced `v2beta1` API object in `nvca-modelcache-init`. -| Group | Required binding data | +| Area | Recorded data | +|---|---| +| Identity | Version, workflow, sharing-domain digest, cache-handle digest | +| Decision | Provider, provisioner, transition, required access modes, required reader-PV mount options, catalog digest, encryption decision | +| StorageClass | Name, UID, `Retain`, configuration digest | +| Resource intent | Writer namespace, deterministic PVC and Job names, optional Lease, encrypted class and Secret names | +| Lifecycle | `Active` or `Retiring`, exact request namespace/name/UID references, finalizer | + +The API server rejects `spec` mutation. It also rejects a transition from `Retiring` back to `Active` and a change to a +recorded provider data identity. + +Regular failure cleanup changes an `Active` binding to `Retiring` only when the exact request is its sole reference. +`Retiring` blocks new references and normal runtime. That same regular request may resume interrupted cleanup. +General zero-reference retirement, successful-cache data garbage collection, binding deletion, `status.realized`, and +conditions are not implemented. A zero-reference binding stays `Active`. Regular periodic cleanup skips binding-owned +PVCs. Helm idle cleanup skips binding-owned primary PVs and encrypted StorageClasses. No controller retires or deletes +a successful `rwxReadOnly` binding, populated PVC, retained completed Job, or backing data. + +Binding creation order: + +1. Persist the general NVCA finalizer on the `ICMSRequest` and stop that reconcile. +2. Build the deterministic binding intent. +3. If no binding exists, revalidate live selection inputs and create it. +4. Initialize it as `Active` and add the exact request UID reference. +5. Persist the binding name and UID in the request selection and stop that reconcile. +6. Re-read the binding, require exact immutable intent, `Active`, finalizer, UID, and request reference. +7. Start provider side effects. + +Before this request persists a binding reference, NVCA may reuse an exact Active binding and add the request reference, +or recover an exact newly created binding whose status is completely empty. After the request records the binding name +and UID, normal runtime retries require the exact UID, finalizer, immutable spec, Active phase, and request reference. +Partially initialized status, Retiring runtime use, new joins to Retiring bindings, and immutable collisions fail +closed. The exact sole regular request may resume cleanup against its Retiring binding. A stale same-name request +reference is replaced only after the old request UID is absent. If request deletion starts while the reference is being +committed, NVCA removes the newly added reference. + +The current binding name hashes only the cache handle because regular and Helm cache resource names are handle-scoped. +The same handle in another workflow or sharing domain collides and fails closed. A future resource-naming migration is +required before those domains can use independent bindings for the same handle. + +## Regular model cache + +The regular writer and workload Pods run in the Pod instance namespace. + +### `roxReadOnly` + +1. Create `rw-pvc-` with RWO and `writer-job-`. +2. Wait for population and volume detachment. +3. Set the retained PV to ROX and bind `ro-pvc-` to that PV. +4. Apply the binding's required reader-PV mount options before publishing the reader PVC. +5. Mount the reader PVC with `PersistentVolumeClaimVolumeSource.readOnly: true`. +6. Set every matching init-container and container `volumeMount.readOnly: true`. + +### `rwxReadOnly` + +1. Remove request-scoped metadata and all owner references from the shared PVC. Strip all preexisting labels, + annotations, and owner references from the Job and Pod template, then add only the binding identity and PVC-UID + witness. Disable automatic service-account token mounting. Reject any writer environment input, image-pull Secret, + or Secret-backed volume. +2. Create one `rw-pvc-` with RWX and `writer-job-`. The Job must not use + `ttlSecondsAfterFinished`; it must reference and mount that exact PVC writable. +3. After the API assigns the PVC UID, record that UID in the immutable Job Pod-template annotation. Reject a Job whose + recorded UID differs from the current PVC UID. +4. Wait for the PVC to bind and the writer Job to complete. +5. Require a non-terminating, Bound PVC and PV. Validate the PV by StorageClass, `Retain`, exact RWX mode, volume mode, + persisted CSI provisioner, non-empty CSI volume handle, and claim reference namespace, name, and PVC UID. +6. Revalidate the Active binding, bound PV, PVC-UID Job witness, and completed Job within each populated-marker update + attempt. After the marker update, repeat those checks before returning the claim. Retain the completed Job as the + publication fence. +7. On later reconciles, require both the populated PVC label and the exact completed Job. A missing, terminating, or + non-completed fence Job; a non-Active binding; or a storage-identity mismatch fails closed. +8. Return the same RWX PVC name. Set the workload PVC source and every matching init-container and container mount + read-only. + +This transition creates no reader PVC, does not modify the PV during publication, does not wait for detach, and performs +no clone or copy. The claim remains RWX. Read-only publication is Kubernetes Pod mount intent, not a conversion to ROX. +The API server admits only one object at the deterministic Job name, and replicas adopt only that exact object. +Retaining the completed Job prevents a replica with a stale pre-publication read from recreating a writer. The retained +Job is accepted only when it contains no environment input or Secret reference. + +Existing same-name PVCs and Jobs require the exact binding UID and immutable intent. Other same-name objects are not +adopted. Before creation, NVCA removes request-scoped labels, annotations, and all owner references from the shared +writer PVC. It strips all preexisting labels, annotations, and owner references from the Job and Pod template before +adding the binding label and PVC-UID witness. Existing shared objects that retain request ownership or lack the intended +binding metadata fail closed. This prevents deletion of one request from garbage-collecting shared cache objects. +Automated two-reference reuse tests use credential-free synthetic Jobs. Production reuse requires binding-scoped +writer input identity, Secret lifecycle, and failure recovery. + +For `roxReadOnly`, the binding UID labels the writer PVC, Job and Pod template, retained PV, and reader PVC. For +`rwxReadOnly`, it labels the writer PVC, Job, and Pod template. NVCA does not label the dynamically provisioned RWX PV; +ownership is proved through the exact PVC claim reference and persisted storage identity. The regular path has no +Kubernetes Lease. Its mutex serializes setup only within one NVCA process. + +Before any destructive failure cleanup, NVCA atomically changes the binding from `Active` to `Retiring` if the exact +request is its sole reference. A concurrent reference prevents retirement and cleanup. The same request can resume +interrupted cleanup while the binding remains `Retiring`. Cleanup inventories only resources recorded for the selected +transition, requires the binding UID on the Job and PVC, and validates the exact PV/PVC identity before each PV change. +Job and PVC deletes use UID and resourceVersion preconditions. For a bound claim, cleanup changes the exact PV from +`Retain` to `Delete` before deleting the PVC. A retry accepts an already-applied `Delete` policy only during cleanup, +revalidates every other identity field, and continues. Legacy periodic cleanup skips binding-owned PVCs. + +## Helm model cache + +Only `roxReadOnly` is executable for Helm. The schema, catalog validator, and persisted-selection validator reject +`rwxReadOnly`. The writer and readers use different namespaces, so they cannot reference one namespaced PVC. A +provider-neutral, no-copy namespace-local reader mapping and its lifecycle and cleanup logic are not implemented. + +The Helm writer runs in `nvca-modelcache-init`. + +1. A Lease named from the cache handle elects one writer request. +2. The writer populates `rw-pvc-` through `writer-job-`. +3. NVCA retains the primary PV as the cache data identity. +4. Each StorageRequest creates its namespace-local ROX PVC and secondary PV for the same NVMesh data identity, using + the binding's required reader-PV mount options. +5. The webhook sets the PVC volume source and every mount that references the model-cache volume read-only. + +The binding UID labels the StorageRequest, writer PVC, Job and Pod template, pull Secrets, Lease, primary PV, secondary +PV, and reader PVC. The secondary PV and reader PVC also carry +`nvca.nvcf.nvidia.io/model-cache-request-uid`, which rejects a stale same-name reader from an earlier ICMSRequest +generation. Shared writer objects do not carry a request UID label because multiple requests can share them. The +StorageRequest records the source UID in annotation `nvca.nvcf.nvidia.io/icms-request-uid`. Existing writer objects and +Leases with another or missing binding UID fail closed. An existing StorageRequest must match the exact persisted +selection and source ICMSRequest UID. Same-binding PVC, Job, Lease, and pull Secret adoption also requires immutable +intent to match before any object is created. + +Annotated cleanup validates the Active binding and exact per-request reader inventory before deletion. While the exact +binding reference is present, only a Lease holder containing the exact request UID may delete shared writer artifacts. +Before each shared-writer deletion, cleanup revalidates the Lease UID, resourceVersion, binding UID, and holder. After +that reference is released, the StorageRequest identity is a tombstone only if the exact ICMSRequest is deleting or +absent. The tombstone authorizes deletion of that request's reader PV/PVC only; shared writer cleanup is skipped. A live +unreferenced request, a same-name request with another UID, or a request-read error stops cleanup. Legacy idle GC skips +binding-owned primary PVs and their encrypted StorageClasses. + +## Encryption + +Encryption is part of the durable selection and binding decision. A later feature-gate change does not change it. +Only the `roxReadOnly` transition supports encryption. The `rwxReadOnly` transition rejects it. + +NVMesh encryption uses existing sharing-domain-scoped resources: + +- regular: StorageClass `-sc` and Secret ``; +- Helm: StorageClass `sc-` and Secret `scsec-`. + +The derived StorageClass must match the expected NVMesh provisioner, `Retain`, binding mode, expansion setting, and +parameters. The Secret must contain a non-empty `dmcryptKey`. Existing conflicting objects fail closed, and existing +key material is preserved. + +These objects are shared by multiple bindings in one domain, so they do not carry one binding UID. The binding records +their deterministic names, never Secret contents. + +## Failure and cleanup rules + +| Event | Result for a new persisted request | |---|---| -| Identity | Version, workflow, sharing-domain digest, and cache-handle digest | -| Decision | Provider, provisioner, transition, required access modes, reader-PV mount options, and catalog payload digest | -| StorageClass snapshot | Name, UID, `Retain`, and configuration digest | -| Resource intent | Deterministic names for NVCA-created PVCs, static PVs, Jobs, and Leases | -| Lifecycle | `Active` or `Retiring`, request namespace/name/UID references, and finalizer | -| Realized state | Bound PV and provider data identity plus population state as they become known | - -The target API is a namespaced `ModelCacheBinding` in the model-cache control namespace. Its immutable `spec` contains -identity, decision, StorageClass snapshot, and shared resource intent. Its `status` contains lifecycle, request -references, realized state, and conditions; a finalizer protects cleanup. `StorageRequest.status.modelCache` gains an -immutable request selection containing mode, binding name/UID, and deterministic namespace-local reader intent, plus a -request finalizer. The UID is the API-assigned `ModelCacheBinding.metadata.uid`; it is not part of binding `spec`. API -validation rejects binding-spec or request-selection mutation after persistence. - -The versioned StorageClass digest is SHA-256 over canonical JSON containing provisioner, sorted parameters, reclaim -policy, binding mode, mount options, and allowed topologies; list order is preserved. It excludes object metadata and -volume expansion. The catalog digest is SHA-256 over the exact ConfigMap payload. The request records mode `none`, -`ephemeral`, or `durable` and the binding reference when durable. - -Binding rules: - -- Persist the binding and request reference before the first storage side effect. -- All requests for one cache key converge through optimistic concurrency. -- A Lease keyed by the binding serializes at most one active writer. -- Retries and agent restarts reuse the binding. -- Creation uses the recorded deterministic name and Get-before-Create. A retry adopts an object only when its binding - UID and immutable spec match the intent; a mismatch is terminal. -- Record provider data identity only after it exists; do not change it afterward. -- Catalog, feature-gate, and StorageClass updates do not mutate an existing binding. -- Before any resource exists, input drift fails without side effects. -- After a resource exists, reconcile only the binding's resources; never switch providers or delete data because of - drift. -- Do not include Secret contents. - -Cleanup is a state transition, not a list-then-delete check. The reconciler marks the binding `Retiring`, which blocks -new references, while its finalizer prevents deletion. A stale reference is one whose request is absent, has a different -UID, or does not point back to the binding. Request deletion first removes reader resources, then removes the binding -reference, then releases the request finalizer. Provider cleanup starts only at zero references; the binding finalizer -is released only after owned Kubernetes resources are gone. Deleting retained backend data, if required, is an explicit -transition operation, not a response to configuration drift. - -## Model-cache contract - -PVCs are namespace-scoped. One PVC cannot be mounted by Pods in different namespaces. A same-namespace multi-Pod -read-only test is primitive evidence only. - -A regular or Helm model-cache transition can be enabled only if it provides: - -- at most one active writer per cache key, serialized by the binding Lease; -- one provider-backed data identity per cache key; -- no source-sized clone or extra data copy; -- namespace-local reader objects that resolve to the same data identity through a documented provider mechanism; -- no duplicate PVs with one CSI `volumeHandle`; -- `readOnly: true` on both the PVC volume source and each matching `volumeMount`; -- a read-only flag on the filesystem mount observed inside each reader container; -- failed write attempts from every reader Pod; -- a durable populated marker that survives writer cleanup and NVCA restart; -- idempotent creation, observation, retry, and deletion; -- cleanup that cannot delete data while another namespace references it. - -The current webhook sets `PersistentVolumeClaimVolumeSource.readOnly: true` but sets both model-cache `volumeMount` -values to `false`. Runtime implementation must set and test both mount values as `true` before external qualification. - -Kubernetes and generic CSI do not provide a cross-namespace PVC-sharing primitive. Each transition owns its -provider-specific reader-view mapping. A registered transition must document a driver-supported alias or rebind method -that produces namespace-local PVC/PV pairs with distinct CSI `volumeHandle` values resolving to the same backend data. -The generic planner must not synthesize those PVs or assume two PVCs from one class expose the same data. A provider -remains disabled if it cannot meet this contract without a source-sized copy. - -Cross-workflow and cross-domain reuse are not assumed. Either requires its own qualified transition and authorization -contract. - -## Failure, migration, and rollback - -After runtime enforcement is wired, durable selection fails closed for a missing or invalid catalog, a missing or -non-`Retain` `nvcf-sc`, an unknown provisioner or transition, a disabled workflow, or drift before the first side -effect. -Stale resources from the presence-selected sharedfs path are not adopted. - -Failure before a binding or request fallback is persisted creates no storage objects. Failure afterward follows the -recorded transition's cleanup rules. `none` or `ephemeral` is used only when recorded on the request. - -Legacy requests may have no backend or only a coarse backend value. Runtime migration must enforce: - -- requests with existing resources are not rebound to another provider; -- requests without storage side effects may establish or reference a binding; -- ambiguous shared-filesystem resources are not adopted from class presence or labels alone; -- NVMesh retained-primary-PV, sharedfs writer-PVC, and Samba backing-PVC markers are inspected explicitly; -- a migrated request remains stable across retry and restart. - -The runtime change must derive the exact legacy conversion from existing resources and cover it with tests. - -New unencrypted cache requests must reject the legacy `Agent.ModelCache.StorageClassName` override unless it is empty -or `nvcf-sc`. NVMesh encryption is the exception: selection still uses `nvcf-sc`, while the `roxReadOnly` transition records -and validates its deterministic per-NCA derived StorageClass before creating the encrypted writer PVC. The NCA sharing -domain is part of the cache key, and the derived class cannot select a provider. - -Provider-defining StorageClass fields, including the provisioner and parameters, cannot be changed in place. A -provider change is a maintenance operation: - -1. Stop new cache PVC creation. -2. Mark every binding for the current `nvcf-sc` UID `Retiring` so it rejects new references. -3. Drain requests, reach zero references, complete cleanup, and inventory retained data. -4. Verify no active binding references the old UID, then delete and recreate `nvcf-sc`. -5. Verify the new UID, provisioner, configuration digest, provisioning, and mounts. -6. Run functional qualification for the exact deployed configuration. -7. Resume new requests only after catalog and runtime support are deployed. - -Rollback repeats the process with the prior definition. `Retain` prevents automatic deletion; it does not migrate data. - -## Provider qualification - -1. Verify CSI controller health and node-plugin readiness on every eligible CPU and GPU node. -2. Render and verify the exact `nvcf-sc` definition. -3. Run dynamic provisioning and basic mount tests. -4. Prove each cataloged access mode for the exact deployed configuration. -5. Add the catalog entry with both transitions `disabled`. -6. Implement an explicit transition, including its provider-specific reader-view mapping. -7. Run the full workflow suite across namespaces and eligible pools. -8. Record CSI, backend, StorageClass, Kubernetes, node, and dated evidence details. -9. Set only a passing workflow to its implemented transition. -10. Deploy catalog and runtime changes together, then verify new bindings. - -A disabled entry does not enable cache traffic. - -## Test plan - -### Catalog and deployment - -- Missing namespace, ConfigMap, key, payload, and chart file. -- Malformed YAML, unknown fields, missing or invalid access modes, missing reader mount options, and missing transitions. -- Whitespace-only IDs, duplicate modes or options, conflicting options, unknown transitions, and transition misuse. -- `roxReadOnly` requires the NVMesh provisioner and provider, RWO, ROX, `ro`, `norecovery`, and `nouuid`. -- `rwxReadOnly` requires RWX, an empty reader-mount-option array, and the regular workflow. -- Source/release catalog, schema, and template parity. -- Deployment-tooling tests for exact Weka, OCI FSS, OCI Lustre, and generic provider rendering. -- Fixed `nvcf-sc` name and `Retain` policy. -- Deployment-tooling tests for exact parameters, mount options, binding mode, expansion, and topology. -- Deployment tooling preserves NVMesh expansion; the catalog selects `roxReadOnly` for both workflows. -- Malformed deployment input fails rendering. -- `nvcf-function-storage-sc` remains unchanged. - -### Binding and migration - -- Feature gates select the request fallback without creating a durable binding. -- The NVMesh provisioner entry selects `roxReadOnly` for both workflows. -- StorageClass and catalog digests are deterministic, sensitive to included fields, and captured with UIDs. -- Unknown provisioner, disabled transition, and non-`Retain` class. -- Concurrent namespaces in one sharing domain converge on one binding; other workflows or domains use different keys. -- Retry, leader change, and agent restart reuse the binding and Lease. -- Drift before the first side effect fails without resources; drift afterward never switches providers. -- A crash after object creation adopts only an exact deterministic intent and binding UID; a mismatch fails closed. -- `Active` to `Retiring` blocks new references; cleanup waits for zero references and honors the finalizer. -- Request deletion removes reader resources and the binding reference before releasing its finalizer. -- Legacy empty-backend, NVMesh PV, sharedfs PVC, and Samba PVC marker migration. -- Legacy class-override rejection and NVMesh encrypted derived-class validation. -- Provider replacement retires every binding for the old StorageClass UID before new requests resume. -- Stale shared-filesystem guard and cleanup. - -### Workflow - -- The Lease permits one active writer, including when competing writer Pods share a node. -- Readers in multiple namespaces mount the same provider data identity. -- CPU and GPU readers resolve the same provider data identity. -- Full-file and tree checksum equality. -- Denied create, append, rename, chmod, truncate, and delete. -- Both volume source and volume mount are read-only. -- A same-provider baseline mount is writable by the test UID/GID; the reader reports `ro` in `/proc/self/mountinfo`. -- Each cataloged PVC access mode is created and mounted explicitly. -- An RWX claim with a read-only Pod mount is not reported as ROX evidence. -- Writer and reader restart. -- Reader recreation and rescheduling to another eligible node. -- Agent restart at every lifecycle phase. -- Cancellation and injected provision, mount, writer, and cleanup failures. -- Idempotent retries with no duplicate writer or leaked resources. -- Cleanup with active and inactive readers. -- Upgrade and rollback with existing bindings. -- No source-sized clone, extra copy, or duplicate CSI handle. - -Performance testing is a separate follow-up after this functional suite passes. - -## Security and observability - -Security requirements: - -- Store no credentials, Secret contents, tokens, or private endpoints; use CSI-supported Secret references. -- Grant least-privilege read access to the StorageClass and catalog. -- Set read-only intent at every Kubernetes layer and verify write denial. -- Reject unknown fields and transitions. -- Qualification evidence records the exact deployed configuration; published evidence redacts credentials and private - deployment values while preserving the configuration digest. - -Observability requirements: - -- Log and trace binding creation, reuse, fallback, drift, transition, and cleanup. -- Count binding outcomes with bounded labels and publish persistence, population, reader, and cleanup conditions. -- Do not use cache handles, PVC names, backend IDs, or binding digests as metric labels. - -## Implementation order - -Keep external transitions disabled. Add the binding API and strict-loader call site, migrate legacy state, fix read-only -mounts, implement provider-specific no-copy transitions, remove presence selection, then qualify each workflow. +| Feature gate changes after selection | Recorded mode remains authoritative | +| StorageClass or catalog drifts before binding creation | Terminal failure, no binding or cache object | +| Catalog changes after binding creation | Existing binding remains authoritative | +| Legacy mount-option ConfigMap changes after binding creation | Persisted provider-required options remain authoritative | +| Missing writer requires dynamic provisioning after class replacement | Terminal failure before PVC creation | +| Runtime binding is missing, replaced, `Retiring`, or lacks the exact request reference | Terminal failure; the exact sole regular request may resume cleanup against `Retiring`, and Helm cleanup has the validated tombstone exception | +| Object has missing or foreign ownership | Never adopt or delete that object; runtime fails terminal and cleanup refuses that target | +| `rwxReadOnly` writer PVC is missing while its Job exists | Terminal failure; do not recreate or publish the claim | +| `rwxReadOnly` writer contains environment input or a Secret reference | Terminal failure before writer resources are created | +| `rwxReadOnly` Job records another PVC UID | Terminal failure; do not publish or recreate the writer | +| `rwxReadOnly` populated label lacks an exact completed Job fence | Terminal failure; do not publish the claim | +| `rwxReadOnly` PVC, PV, or Job is terminating; or the PVC/PV is not Bound | Terminal failure before publication | +| `rwxReadOnly` bound PV identity or volume mode does not match | Terminal failure before workload publication | +| Missing unencrypted writer PVC and Job before initialization | Recreate only after live `nvcf-sc` matches the binding snapshot | +| An artifact that should already exist in the persisted runtime phase is missing or invalid | Terminal failure | +| Deterministic selection, binding, or ownership data does not match | Terminal failure before the conflicting object is used or changed | +| Recorded cleanup Job or PVC is already absent | Idempotent skip; a bound PVC that references a missing PV still fails cleanup | +| Cleanup retry finds its exact PV already changed to `Delete` | Revalidate every other identity field and continue cleanup | +| Durable cache execution reports failure | Terminal failure, no uncached fallback | +| Transient Kubernetes API error | Requeue without changing selection or phase | +| Forbidden, Unauthorized, Invalid, or Gone API response | Surface the API error without converting deterministic state to a terminal cache failure | +| Annotation-free legacy request fails caching | Existing legacy fallback behavior remains | + +On request deletion, NVCA waits until instances are terminated, removes the exact request UID reference from the +binding, and then removes its general request finalizer. Reference removal is idempotent. + +Request references are live references, not tombstones. `ModelCacheBinding` has no tombstone field. Helm +StorageRequest cleanup may run after reference release and then uses the persisted request identity described above. +Zero references do not trigger `Retiring`, binding deletion, or cache-data deletion. + +Annotated Helm cleanup deletes validated snapshots with UID and resourceVersion preconditions. Binding-scoped regular +Job and PVC deletes use UID and resourceVersion preconditions. + +This change does not delete a zero-reference binding or its retained cache data. Operator uninstall explicitly removes +binding finalizers before deleting the model-cache control namespace. + +Configuration drift never authorizes data deletion. + +## Compatibility and limitations + +Requests with no storage-selection annotation are legacy requests. They keep existing feature-gated regular behavior +and existing Helm backend state. When no Helm StorageRequest exists, the compatibility selector may still inspect the +legacy NVMesh marker, shared-filesystem marker, or Samba configuration. New annotated requests do not use those +presence checks. + +There is no automatic conversion of legacy cache objects into bindings. In particular, NVCA does not adopt an +unlabeled legacy PVC, PV, Job, Secret, or Lease into a new binding. + +Current limitations: + +- `roxReadOnly` supports regular and Helm model cache; `rwxReadOnly` supports only regular model cache, and no shipped + provider entry enables it; +- Weka, OCI FSS, and OCI Lustre remain disabled pending live NVCA qualification; +- current translated writer artifacts contain inputs rejected by the binding-safe `rwxReadOnly` contract, so + end-to-end NVCA population is not yet supported; +- binding-scoped writer input identity, Secret creation/adoption/rotation/cleanup, and removal of raw credentials from + the retained fence Job are not implemented; +- a failed shared writer with multiple binding references has no binding-level failure state or recovery transition; +- the NVMesh regular writer is not serialized by a cluster-wide Lease; +- binding names cannot separate the same handle across workflows or sharing domains; +- realized state and binding conditions are not populated; +- only regular sole-request failure cleanup marks a binding `Retiring`; +- no general retirement or provider-data garbage-collection controller exists; +- successful `rwxReadOnly` PVCs and completed fence Jobs remain until a future binding lifecycle and garbage-collection + controller removes them; +- no provider replacement controller exists; +- binding-specific metrics are not implemented; +- backend functional qualification is outside the unit test suite. + +Do not replace `nvcf-sc` while Active bindings may need to provision a writer. Stop new requests and drain the affected +cache state first. General drain, zero-reference retirement, and retained-data garbage collection remain future work. + +## Test contract + +### Automated tests in this change + +- strict catalog and annotation parsing; +- provider-required reader-PV mount-option validation, persistence, defensive copying, and legacy ConfigMap isolation; +- NVMesh regular and Helm reader-PV option application, conflict filtering, retry repair, and RWX no-mutation behavior; +- executable JSON Schema acceptance and rejection for regular `rwxReadOnly`, required RWX, and Helm refusal; +- exact provisioner lookup and workflow transition selection; +- regular-only `rwxReadOnly` schema and selection, including rejection for Helm and encryption; +- `Retain`, UID, StorageClass digest, and catalog digest checks; +- missing class, unknown provider, disabled transition, and feature-gate outcomes; +- binding API schema, immutable spec, status subresource, and generated clients; +- binding create, retry adoption, collision, request-reference add/release, and drift handling; +- finalizer-before-side-effect ordering; +- regular and Helm binding UID propagation, Helm request UID propagation, and stale-generation rejection; +- sole-reference retirement, concurrent-join refusal, and interrupted regular cleanup retry; +- immutable regular and Helm writer-object adoption, drift refusal, and create-race validation; +- one provider-neutral RWX writer PVC and Job with no reader PVC; +- writer-Job proof of an exact writable mount or volume device for that PVC; +- removal of PVC request metadata and all preexisting Job and Pod-template metadata, credential-input refusal, + controlled two-request reuse, and unsafe existing-object refusal; +- exact RWX PV claim UID, CSI driver, access mode, volume mode, StorageClass, reclaim policy, and volume-handle + validation; +- retained completed-Job publication fencing, exact Job-to-PVC UID witness, missing-fence refusal, terminating-object + refusal, non-Bound PV refusal, Job-spec drift refusal, and retry behavior; +- publication-race refusal when the PV, completed Job, or Active binding changes during a marker-update retry; +- fake-client action traces showing no second PVC, PV mutation, VolumeAttachment access, writer PVC deletion, or + completed Job deletion on tested publication paths; +- same-claim worker injection with both the PVC source and every matching mount read-only; +- setup-failure cleanup that changes an exact bound `Retain` PV before deleting its claim; +- failure-cleanup retry after the exact PV is already changed from `Retain` to `Delete`, with other identity drift + still refused; +- transient and non-transient Kubernetes API error classification; +- Helm tombstone authorization and refusal, including reader-only cleanup after reference release; +- Helm reader PV/PVC and regular Job/PVC UID and resourceVersion delete preconditions; +- binding-owned primary-PV and encrypted-StorageClass idle-GC exclusion with legacy idle GC preserved; +- encrypted StorageClass and Secret conflict detection; +- read-only PVC volume sources and every model-cache mount path; +- annotation-free compatibility behavior; +- operator catalog mirroring, RBAC, CRD installation, and uninstall cleanup. + +These tests use fake clients, unit tests, and Kubernetes `envtest`. They prove control-plane decisions, state +transitions, API actions, and generated objects only. Provider names in fixtures exercise selection data; the tests do +not contact those CSI drivers or storage backends. Repeat reconcile is tested, but an actual NVCA process restart +remains part of provider qualification. The two-reference and writer-publication fixtures are credential-free; they do +not prove that current translated production writer artifacts can execute this transition. + +### Required provider qualification + +A workflow stays `disabled` until an exact provider configuration passes all of these tests: + +1. Provision and mount every cataloged PVC access mode. +2. Populate one cache and prove every reader resolves the same provider data identity. +3. Mount from multiple Pods and from every required namespace pattern. +4. Run readers on eligible CPU and GPU nodes. +5. Compare full-file and tree checksums. +6. Verify `ro` in `/proc/self/mountinfo` inside each reader. +7. Deny create, append, rename, chmod, truncate, and delete from each reader. +8. Prove a same-provider baseline mount is writable by the test UID/GID. +9. Restart an interrupted writer before publication, restart NVCA and readers, and reschedule readers to another + eligible node. Prove no writer starts after publication. +10. Inject provisioning, mount, writer, cancellation, and cleanup failures. +11. Retry every lifecycle step and verify no duplicate writer, leaked object, or data loss. +12. Prove cleanup cannot remove data while another reader uses it. +13. Prove the transition performs no source-sized clone or extra copy. + +Record the CSI provisioner and images, backend version and configuration, StorageClass digest, Kubernetes version, +eligible node matrix, catalog version, date, and evidence reference. Performance testing follows functional acceptance. + +## Provider enablement + +1. Install the CSI driver and render exact `StorageClass/nvcf-sc` parameters through deployment tooling. +2. Add or update the public catalog entry with both workflows `disabled`. +3. Record only access modes and reader-PV mount options proven on the exact configuration. +4. Implement a named regular or Helm transition. Do not infer a transition from access modes. +5. Add unit, failure, retry, ownership, cleanup, and read-only object tests. +6. Implement binding-scoped writer input and Secret identity, adoption, rotation, and cleanup when the transition + retains or shares a writer Job. +7. Implement a binding-level failure and recovery path for a shared writer. +8. Run the complete functional qualification above with translated NVCF writer artifacts. +9. Enable only the passing workflow in the catalog. +10. Deploy transition code and the enabling catalog together. +11. Verify new requests persist the expected provisioner, transition, digests, and binding. + +For Weka and OCI FSS regular model cache, the provider-neutral `rwxReadOnly` storage path exists, but the shipped +entries remain disabled. Enablement requires binding-safe translated writer inputs, shared-writer failure recovery, +exact provider functional qualification, and a production decision for zero-reference retirement and retained-data +cleanup. + +Helm still requires a tested no-copy mapping from one populated data identity to namespace-local read-only readers, +together with ownership, retry, and cleanup implementation. + +## Rollout and rollback + +Rollout order: + +1. Install the `ModelCacheBinding` CRD and RBAC. +2. Install and mirror the catalog. +3. Deploy NVCA with persisted selection, binding support, the NVMesh transitions, and the regular `rwxReadOnly` + transition. +4. Verify one new NVMesh regular request and one new NVMesh Helm request. +5. Verify selection annotations, Active bindings, exact UID labels, read-only object intent, and cleanup refusal tests. +6. Keep every external transition disabled until qualification passes. + +Existing annotation-free requests continue through compatibility code. New annotated requests must not be downgraded +to an NVCA version that does not understand bindings while they are active. Drain those requests before rollback. The +`Retain` policy preserves backend data, but it does not migrate or make an older controller binding-aware. + +The legacy `nvca-cache-mount-options` ConfigMap remains a compatibility input only for annotation-free requests. New +bound requests read provider-required options from their persisted selection and binding. The separate operator +`cacheMountOptions` setting remains an additive cluster override; an option that negates a provider requirement is +ignored. ## Public source references - [Storage catalog](https://github.com/NVIDIA/nvcf/blob/main/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml) - [Catalog JSON Schema](https://github.com/NVIDIA/nvcf/blob/main/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.schema.json) -- [Catalog loader and validator](https://github.com/NVIDIA/nvcf/blob/main/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go) -- [Current Helm cache selection](https://github.com/NVIDIA/nvcf/blob/main/src/compute-plane-services/nvca/pkg/storage/cachebackend.go) -- [Current model-cache webhook](https://github.com/NVIDIA/nvcf/blob/main/src/compute-plane-services/nvca/pkg/webhook/helm_storage_webhook.go) +- [Catalog selection](https://github.com/NVIDIA/nvcf/blob/main/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go) +- [Persisted selection](https://github.com/NVIDIA/nvcf/blob/main/src/compute-plane-services/nvca/pkg/storage/modelcache_selection.go) +- [Binding API](https://github.com/NVIDIA/nvcf/blob/main/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1/modelcachebinding_types.go) +- [Binding lifecycle](https://github.com/NVIDIA/nvcf/blob/main/src/compute-plane-services/nvca/pkg/nvca/modelcache_binding.go) +- [RWX read-only runtime](https://github.com/NVIDIA/nvcf/blob/main/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend_modelcache_rwx_readonly.go) - [Kubernetes persistent-volume access modes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes) -- [Runtime integration issue](https://github.com/NVIDIA/nvcf/issues/1326) diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml b/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml index 220e0c033..bbbdb427c 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml @@ -20,8 +20,8 @@ # A transition value is a closed enum, not free text. A strategy name means # enabled; disabled means the workflow is unsupported. # A disabled transition means that the end-to-end workflow is not qualified. -# The strict loader validates this catalog. Runtime reconciliation does not -# consume it until the storage-selection change is released. +# NVCA reads this catalog when it creates a new model-cache storage selection. +# Only a registered, non-disabled transition can select durable caching. apiVersion: storage.nvcf.nvidia.com/v1alpha1 kind: StorageCapabilityCatalog drivers: diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/pre-delete-cleanup-rbac.yaml b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/pre-delete-cleanup-rbac.yaml index 51c2e07ca..b573f3096 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/pre-delete-cleanup-rbac.yaml +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/pre-delete-cleanup-rbac.yaml @@ -80,7 +80,7 @@ rules: resources: ["icmsrequests/status"] verbs: ["get", "update", "patch"] - apiGroups: ["nvca.nvcf.nvidia.io"] - resources: ["storagerequests", "storagerequests/status"] + resources: ["modelcachebindings", "modelcachebindings/status", "storagerequests", "storagerequests/status"] verbs: ["get", "list", "watch", "patch", "create", "update", "delete"] - apiGroups: ["batch"] resources: ["jobs", "cronjobs"] diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/role.yaml b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/role.yaml index 9817e0c28..e565043fd 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/role.yaml +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/role.yaml @@ -63,7 +63,7 @@ rules: resources: ["icmsrequests/status"] verbs: ["get", "update", "patch"] - apiGroups: ["nvca.nvcf.nvidia.io"] - resources: ["storagerequests", "storagerequests/status"] + resources: ["modelcachebindings", "modelcachebindings/status", "storagerequests", "storagerequests/status"] verbs: ["get", "list", "watch", "patch", "create", "update", "delete"] - apiGroups: ["batch"] resources: ["jobs", "cronjobs"] diff --git a/src/compute-plane-services/nvca/internal/envtest/crds/nvca.nvcf.nvidia.io_modelcachebindings.yaml b/src/compute-plane-services/nvca/internal/envtest/crds/nvca.nvcf.nvidia.io_modelcachebindings.yaml new file mode 100644 index 000000000..6fb784428 --- /dev/null +++ b/src/compute-plane-services/nvca/internal/envtest/crds/nvca.nvcf.nvidia.io_modelcachebindings.yaml @@ -0,0 +1,232 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: modelcachebindings.nvca.nvcf.nvidia.io +spec: + group: nvca.nvcf.nvidia.io + names: + kind: ModelCacheBinding + listKind: ModelCacheBindingList + plural: modelcachebindings + singular: modelcachebinding + shortNames: [mcb] + scope: Namespaced + versions: + - name: v2beta1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - name: Workflow + jsonPath: .spec.identity.workflow + type: string + - name: Provider + jsonPath: .spec.decision.provider + type: string + - name: Phase + jsonPath: .status.phase + type: string + - name: Age + jsonPath: .metadata.creationTimestamp + type: date + schema: + openAPIV3Schema: + type: object + required: [spec] + properties: + spec: + type: object + required: [identity, decision, storageClass, resources] + x-kubernetes-validations: + - rule: self == oldSelf + message: spec is immutable + properties: + identity: + type: object + required: [version, workflow, sharingDomainDigest, cacheHandleDigest] + properties: + version: + type: string + minLength: 1 + workflow: + type: string + enum: [regularModelCache, helmModelCache] + sharingDomainDigest: + type: string + minLength: 1 + cacheHandleDigest: + type: string + minLength: 1 + decision: + type: object + required: + - provider + - provisioner + - transition + - requiredAccessModes + - catalogDigest + - encryptionRequired + properties: + provider: + type: string + minLength: 1 + provisioner: + type: string + minLength: 1 + transition: + type: string + minLength: 1 + requiredAccessModes: + type: array + minItems: 1 + x-kubernetes-list-type: set + items: + type: string + enum: [ReadWriteOnce, ReadOnlyMany, ReadWriteMany, ReadWriteOncePod] + requiredMountOptions: + type: array + x-kubernetes-list-type: atomic + items: + type: string + minLength: 1 + catalogDigest: + type: string + pattern: ^sha256:[a-f0-9]{64}$ + encryptionRequired: + type: boolean + storageClass: + type: object + required: [name, uid, reclaimPolicy, configurationDigest] + properties: + name: + type: string + minLength: 1 + uid: + type: string + minLength: 1 + reclaimPolicy: + type: string + enum: [Retain] + configurationDigest: + type: string + pattern: ^v1:sha256:[a-f0-9]{64}$ + resources: + type: object + required: [writerNamespace] + properties: + writerNamespace: + type: string + minLength: 1 + persistentVolumeClaimNames: + type: array + x-kubernetes-list-type: set + items: + type: string + minLength: 1 + persistentVolumeNames: + type: array + x-kubernetes-list-type: set + items: + type: string + minLength: 1 + jobNames: + type: array + x-kubernetes-list-type: set + items: + type: string + minLength: 1 + storageClassNames: + type: array + x-kubernetes-list-type: set + items: + type: string + minLength: 1 + secretNames: + type: array + x-kubernetes-list-type: set + items: + type: string + minLength: 1 + leaseName: + type: string + minLength: 1 + status: + type: object + x-kubernetes-validations: + - rule: "!has(oldSelf.phase) || oldSelf.phase != 'Retiring' || self.phase == 'Retiring'" + message: a Retiring binding cannot become Active + - rule: >- + !has(oldSelf.realized) || !has(oldSelf.realized.providerDataIdentity) || + (has(self.realized) && has(self.realized.providerDataIdentity) && + self.realized.providerDataIdentity == oldSelf.realized.providerDataIdentity) + message: providerDataIdentity is immutable once recorded + properties: + phase: + type: string + enum: [Active, Retiring] + lastPhaseTransitionTime: + type: string + format: date-time + requestReferences: + type: array + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: [uid] + items: + type: object + required: [namespace, name, uid] + properties: + namespace: + type: string + minLength: 1 + name: + type: string + minLength: 1 + uid: + type: string + minLength: 1 + realized: + type: object + properties: + boundPersistentVolumeName: + type: string + minLength: 1 + providerDataIdentity: + type: string + minLength: 1 + populationState: + type: string + enum: [Pending, Populating, Ready, Failed] + conditions: + type: array + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: [type] + items: + type: object + required: [lastTransitionTime, message, reason, status, type] + properties: + lastTransitionTime: + type: string + format: date-time + message: + type: string + maxLength: 32768 + observedGeneration: + type: integer + format: int64 + minimum: 0 + reason: + type: string + minLength: 1 + maxLength: 1024 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + status: + type: string + enum: ["True", "False", Unknown] + type: + type: string + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9_.]*[a-z0-9])?\.)*[A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?$ diff --git a/src/compute-plane-services/nvca/internal/miniservice/BUILD.bazel b/src/compute-plane-services/nvca/internal/miniservice/BUILD.bazel index b903ca03b..b5986447e 100644 --- a/src/compute-plane-services/nvca/internal/miniservice/BUILD.bazel +++ b/src/compute-plane-services/nvca/internal/miniservice/BUILD.bazel @@ -10,6 +10,7 @@ go_library( "decode.go", "gvkcache.go", "metadata_configmap.go", + "modelcache_storage_selection.go", "mutate.go", "prereqs.go", "reconcile.go", @@ -126,6 +127,7 @@ go_test( "controller_test.go", "gvkcache_test.go", "metadata_configmap_test.go", + "modelcache_storage_selection_test.go", "mutate_test.go", "prereqs_test.go", "reconcile_modelcache_test.go", @@ -199,6 +201,7 @@ go_test( "//src/compute-plane-services/nvca/vendor/k8s.io/api/core/v1:core", "//src/compute-plane-services/nvca/vendor/k8s.io/api/networking/v1:networking", "//src/compute-plane-services/nvca/vendor/k8s.io/api/rbac/v1:rbac", + "//src/compute-plane-services/nvca/vendor/k8s.io/api/storage/v1:storage", "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/api/errors", "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/api/meta", "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/api/resource", diff --git a/src/compute-plane-services/nvca/internal/miniservice/modelcache_storage_selection.go b/src/compute-plane-services/nvca/internal/miniservice/modelcache_storage_selection.go new file mode 100644 index 000000000..06a52f775 --- /dev/null +++ b/src/compute-plane-services/nvca/internal/miniservice/modelcache_storage_selection.go @@ -0,0 +1,166 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mscontroller + +import ( + "context" + "fmt" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + nvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1" + nvcastorage "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/storage" +) + +func (r *Reconciler) selectHelmCacheBackend( + ctx context.Context, + icmsReq *nvcav2beta1.ICMSRequest, + instanceNamespace string, +) (nvcastorage.HelmCacheBackend, error) { + raw := icmsReq.Annotations[nvcastorage.ModelCacheStorageSelectionAnnotationKey] + if raw != "" { + selection, err := nvcastorage.ParsePersistedModelCacheStorageSelection(raw) + if err != nil { + return "", reconcile.TerminalError(err) + } + backend, err := nvcastorage.HelmCacheBackendFromSelection(selection) + if err != nil { + return "", reconcile.TerminalError(err) + } + if err := r.validateExistingModelCacheStorageRequest( + ctx, icmsReq, instanceNamespace, raw, backend); err != nil { + return "", err + } + return backend, nil + } + + existing := &nvcav2beta1.StorageRequest{} + key := client.ObjectKey{ + Namespace: instanceNamespace, + Name: nvcav2beta1.ModelCacheRequest.Name(), + } + switch err := r.Client.Get(ctx, key, existing); { + case err == nil: + if existing.Spec.ModelCache == nil { + return "", fmt.Errorf("existing model cache StorageRequest %s/%s has no modelCache spec", + key.Namespace, key.Name) + } + return nvcastorage.PersistedHelmCacheBackend(existing.Spec.ModelCache.Backend) + case !apierrors.IsNotFound(err): + return "", fmt.Errorf("get existing model cache StorageRequest %s/%s: %w", + key.Namespace, key.Name, err) + } + + return nvcastorage.SelectLegacyHelmCacheBackend( + ctx, r.Client, r.FeatureFlagFetcher, r.cfg.Agent.ModelCache.StorageClassName) +} + +func (r *Reconciler) validateExistingModelCacheStorageRequest( + ctx context.Context, + icmsReq *nvcav2beta1.ICMSRequest, + instanceNamespace string, + selectionPayload string, + backend nvcastorage.HelmCacheBackend, +) error { + existing := &nvcav2beta1.StorageRequest{} + key := client.ObjectKey{Namespace: instanceNamespace, Name: nvcav2beta1.ModelCacheRequest.Name()} + if err := r.Client.Get(ctx, key, existing); err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return fmt.Errorf("get existing model cache StorageRequest %s/%s: %w", + key.Namespace, key.Name, err) + } + if err := validatePersistedModelCacheStorageRequest( + existing, icmsReq, selectionPayload, backend); err != nil { + return reconcile.TerminalError(err) + } + return nil +} + +func validatePersistedModelCacheStorageRequest( + existing *nvcav2beta1.StorageRequest, + icmsReq *nvcav2beta1.ICMSRequest, + selectionPayload string, + backend nvcastorage.HelmCacheBackend, +) error { + conflict := func(reason string) error { + return fmt.Errorf( + "existing model cache StorageRequest %s/%s conflicts with persisted selection: %s", + existing.Namespace, existing.Name, reason) + } + if existing == nil { + return fmt.Errorf("existing model cache StorageRequest is nil") + } + if existing.Name != nvcav2beta1.ModelCacheRequest.Name() || + existing.Spec.Type != nvcav2beta1.ModelCacheRequest { + return conflict("name or type does not match") + } + if backend != nvcastorage.HelmCacheBackendNVMesh { + return conflict(fmt.Sprintf("backend %q does not create a StorageRequest", backend)) + } + if existing.Spec.ModelCache == nil { + return conflict("modelCache spec is missing") + } + if existing.Spec.RequestName != icmsReq.Name || existing.Spec.RequestNamespace != icmsReq.Namespace { + return conflict("request identity does not match") + } + if existing.Annotations[nvcastorage.ICMSRequestUIDAnnotationKey] != string(icmsReq.UID) { + return conflict("request UID does not match") + } + if existing.Spec.ModelCache.CacheHandle != helmModelCacheHandle(icmsReq) { + return conflict("cache handle does not match") + } + if existing.Spec.ModelCache.Backend != string(backend) { + return conflict("backend does not match") + } + if existing.Annotations[nvcastorage.ModelCacheStorageSelectionAnnotationKey] != selectionPayload { + return conflict("storage selection does not match") + } + return nil +} + +func helmModelCacheHandle(req *nvcav2beta1.ICMSRequest) string { + switch { + case req.Spec.CreationMsgInfo.FunctionLaunchSpecification != nil && + req.Spec.CreationMsgInfo.FunctionLaunchSpecification.CacheLaunchSpecification != nil: + return req.Spec.CreationMsgInfo.FunctionLaunchSpecification.CacheLaunchSpecification.CacheHandle + case req.Spec.CreationMsgInfo.TaskLaunchSpecification != nil && + req.Spec.CreationMsgInfo.TaskLaunchSpecification.CacheLaunchSpecification != nil: + return req.Spec.CreationMsgInfo.TaskLaunchSpecification.CacheLaunchSpecification.CacheHandle + default: + return "" + } +} + +func persistedDurableHelmCacheSelection(req *nvcav2beta1.ICMSRequest) (bool, error) { + raw := req.Annotations[nvcastorage.ModelCacheStorageSelectionAnnotationKey] + if raw == "" { + return false, nil + } + selection, err := nvcastorage.ParsePersistedModelCacheStorageSelection(raw) + if err != nil { + return false, fmt.Errorf("parse persisted model cache storage selection: %w", err) + } + if selection.Workflow != nvcastorage.ModelCacheWorkflowHelm { + return false, fmt.Errorf("persisted model cache workflow %q is not Helm", selection.Workflow) + } + return selection.Mode == nvcastorage.ModelCacheSelectionDurable, nil +} diff --git a/src/compute-plane-services/nvca/internal/miniservice/modelcache_storage_selection_test.go b/src/compute-plane-services/nvca/internal/miniservice/modelcache_storage_selection_test.go new file mode 100644 index 000000000..f9d37a6b6 --- /dev/null +++ b/src/compute-plane-services/nvca/internal/miniservice/modelcache_storage_selection_test.go @@ -0,0 +1,367 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mscontroller + +import ( + "context" + "errors" + "testing" + + "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common" + "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + nvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/featureflag" + featureflagmock "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/featureflag/mock" + nvcastorage "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/storage" +) + +func TestSelectHelmCacheBackend(t *testing.T) { + const instanceNamespace = "instance-ns" + + t.Run("persisted selection wins over a conflicting existing StorageRequest", func(t *testing.T) { + existing := &nvcav2beta1.StorageRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: nvcav2beta1.ModelCacheRequest.Name(), + Namespace: instanceNamespace, + }, + Spec: nvcav2beta1.StorageRequestSpec{ + ModelCache: &nvcav2beta1.ModelCacheSpec{Backend: string(nvcastorage.HelmCacheBackendSamba)}, + }, + } + r := newModelCacheSelectionReconciler(t, existing) + request := &nvcav2beta1.ICMSRequest{ + ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ + nvcastorage.ModelCacheStorageSelectionAnnotationKey: "not-json", + }}, + } + + _, err := r.selectHelmCacheBackend(t.Context(), request, instanceNamespace) + + require.Error(t, err) + assert.Contains(t, err.Error(), "parse model cache storage selection") + assert.True(t, errors.Is(err, reconcile.TerminalError(nil))) + }) + + t.Run("matching persisted StorageRequest is adopted", func(t *testing.T) { + request := requestWithModelCacheSelection( + t, nvcastorage.ModelCacheWorkflowHelm, nvcastorage.ModelCacheSelectionDurable) + raw := request.Annotations[nvcastorage.ModelCacheStorageSelectionAnnotationKey] + existing := &nvcav2beta1.StorageRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: nvcav2beta1.ModelCacheRequest.Name(), + Namespace: instanceNamespace, + Annotations: map[string]string{ + nvcastorage.ModelCacheStorageSelectionAnnotationKey: raw, + nvcastorage.ICMSRequestUIDAnnotationKey: string(request.UID), + }, + }, + Spec: nvcav2beta1.StorageRequestSpec{ + Type: nvcav2beta1.ModelCacheRequest, + RequestName: request.Name, + RequestNamespace: request.Namespace, + ModelCache: &nvcav2beta1.ModelCacheSpec{ + Backend: string(nvcastorage.HelmCacheBackendNVMesh), + CacheHandle: helmModelCacheHandle(request), + }, + }, + } + r := newModelCacheSelectionReconciler(t, existing) + + got, err := r.selectHelmCacheBackend(t.Context(), request, instanceNamespace) + + require.NoError(t, err) + assert.Equal(t, nvcastorage.HelmCacheBackendNVMesh, got) + }) + + t.Run("ephemeral selection rejects a stale durable StorageRequest", func(t *testing.T) { + existing := &nvcav2beta1.StorageRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: nvcav2beta1.ModelCacheRequest.Name(), + Namespace: instanceNamespace, + }, + Spec: nvcav2beta1.StorageRequestSpec{ + Type: nvcav2beta1.ModelCacheRequest, + ModelCache: &nvcav2beta1.ModelCacheSpec{Backend: string(nvcastorage.HelmCacheBackendNVMesh)}, + }, + } + r := newModelCacheSelectionReconciler(t, existing) + request := requestWithModelCacheSelection( + t, nvcastorage.ModelCacheWorkflowHelm, nvcastorage.ModelCacheSelectionEphemeral) + + _, err := r.selectHelmCacheBackend(t.Context(), request, instanceNamespace) + + require.Error(t, err) + assert.Contains(t, err.Error(), "does not create a StorageRequest") + assert.True(t, errors.Is(err, reconcile.TerminalError(nil))) + }) + + t.Run("existing StorageRequest without model cache spec fails", func(t *testing.T) { + existing := &nvcav2beta1.StorageRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: nvcav2beta1.ModelCacheRequest.Name(), + Namespace: instanceNamespace, + }, + } + r := newModelCacheSelectionReconciler(t, existing) + + _, err := r.selectHelmCacheBackend(t.Context(), &nvcav2beta1.ICMSRequest{}, instanceNamespace) + + require.Error(t, err) + assert.Contains(t, err.Error(), "has no modelCache spec") + }) + + for _, tt := range []struct { + name string + mode nvcastorage.ModelCacheSelectionMode + want nvcastorage.HelmCacheBackend + }{ + { + name: "persisted durable selection uses NVMesh", + mode: nvcastorage.ModelCacheSelectionDurable, + want: nvcastorage.HelmCacheBackendNVMesh, + }, + { + name: "persisted ephemeral selection uses ephemeral cache", + mode: nvcastorage.ModelCacheSelectionEphemeral, + want: nvcastorage.HelmCacheBackendEphemeral, + }, + { + name: "persisted none selection disables cache", + mode: nvcastorage.ModelCacheSelectionNone, + want: nvcastorage.HelmCacheBackendNone, + }, + } { + t.Run(tt.name, func(t *testing.T) { + r := newModelCacheSelectionReconciler(t) + request := requestWithModelCacheSelection(t, nvcastorage.ModelCacheWorkflowHelm, tt.mode) + + got, err := r.selectHelmCacheBackend(t.Context(), request, instanceNamespace) + + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } + + t.Run("malformed persisted selection fails without legacy fallback", func(t *testing.T) { + r := newModelCacheSelectionReconciler(t) + request := &nvcav2beta1.ICMSRequest{ + ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ + nvcastorage.ModelCacheStorageSelectionAnnotationKey: "{", + }}, + } + + _, err := r.selectHelmCacheBackend(t.Context(), request, instanceNamespace) + + require.Error(t, err) + assert.Contains(t, err.Error(), "parse model cache storage selection") + assert.True(t, errors.Is(err, reconcile.TerminalError(nil))) + }) + + t.Run("persisted regular-workflow selection fails", func(t *testing.T) { + r := newModelCacheSelectionReconciler(t) + request := requestWithModelCacheSelection( + t, nvcastorage.ModelCacheWorkflowRegular, nvcastorage.ModelCacheSelectionNone) + + _, err := r.selectHelmCacheBackend(t.Context(), request, instanceNamespace) + + require.Error(t, err) + assert.Contains(t, err.Error(), `workflow "regularModelCache" is not Helm`) + assert.True(t, errors.Is(err, reconcile.TerminalError(nil))) + }) + + t.Run("request without selection uses legacy fallback", func(t *testing.T) { + nvmeshClass := &storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{Name: nvcastorage.NVMeshStorageClassName}, + Provisioner: "nvmesh-csi-driver", + } + r := newModelCacheSelectionReconciler(t, nvmeshClass) + r.FeatureFlagFetcher = &featureflagmock.Fetcher{EnabledFFs: []*featureflag.FeatureFlag{ + featureflag.CachingSupport, + featureflag.HelmModelCaching, + }} + + got, err := r.selectHelmCacheBackend( + t.Context(), &nvcav2beta1.ICMSRequest{}, instanceNamespace) + + require.NoError(t, err) + assert.Equal(t, nvcastorage.HelmCacheBackendNVMesh, got) + }) + + t.Run("StorageRequest get error is returned", func(t *testing.T) { + wantErr := errors.New("storage API unavailable") + r := newModelCacheSelectionReconciler(t) + r.Client = &getErrorClient{Client: r.Client, err: wantErr} + + _, err := r.selectHelmCacheBackend(t.Context(), &nvcav2beta1.ICMSRequest{}, instanceNamespace) + + require.Error(t, err) + assert.ErrorIs(t, err, wantErr) + assert.Contains(t, err.Error(), "get existing model cache StorageRequest") + }) +} + +func TestPersistedDurableHelmCacheSelection(t *testing.T) { + for _, tt := range []struct { + name string + req func(*testing.T) *nvcav2beta1.ICMSRequest + want bool + wantErr string + }{ + { + name: "legacy request", + req: func(*testing.T) *nvcav2beta1.ICMSRequest { return &nvcav2beta1.ICMSRequest{} }, + }, + { + name: "durable Helm decision", + req: func(t *testing.T) *nvcav2beta1.ICMSRequest { + return requestWithModelCacheSelection( + t, nvcastorage.ModelCacheWorkflowHelm, nvcastorage.ModelCacheSelectionDurable) + }, + want: true, + }, + { + name: "ephemeral Helm decision", + req: func(t *testing.T) *nvcav2beta1.ICMSRequest { + return requestWithModelCacheSelection( + t, nvcastorage.ModelCacheWorkflowHelm, nvcastorage.ModelCacheSelectionEphemeral) + }, + }, + { + name: "wrong workflow", + req: func(t *testing.T) *nvcav2beta1.ICMSRequest { + return requestWithModelCacheSelection( + t, nvcastorage.ModelCacheWorkflowRegular, nvcastorage.ModelCacheSelectionNone) + }, + wantErr: "is not Helm", + }, + { + name: "malformed decision", + req: func(*testing.T) *nvcav2beta1.ICMSRequest { + return &nvcav2beta1.ICMSRequest{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ + nvcastorage.ModelCacheStorageSelectionAnnotationKey: "{", + }}} + }, + wantErr: "parse persisted model cache storage selection", + }, + } { + t.Run(tt.name, func(t *testing.T) { + got, err := persistedDurableHelmCacheSelection(tt.req(t)) + if tt.wantErr != "" { + require.ErrorContains(t, err, tt.wantErr) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +func newModelCacheSelectionReconciler(t *testing.T, objects ...client.Object) *Reconciler { + t.Helper() + scheme := runtime.NewScheme() + require.NoError(t, nvcav2beta1.AddToScheme(scheme)) + require.NoError(t, storagev1.AddToScheme(scheme)) + + return &Reconciler{ + ControllerOptions: ControllerOptions{FeatureFlagFetcher: &featureflagmock.Fetcher{}}, + Client: clientfake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(objects...). + Build(), + } +} + +func requestWithModelCacheSelection( + t *testing.T, + workflow nvcastorage.ModelCacheWorkflow, + mode nvcastorage.ModelCacheSelectionMode, +) *nvcav2beta1.ICMSRequest { + t.Helper() + var resolved *nvcastorage.ModelCacheStorageSelection + if mode == nvcastorage.ModelCacheSelectionDurable { + resolved = &nvcastorage.ModelCacheStorageSelection{ + StorageClassName: nvcastorage.DefaultModelCacheStorageClassName, + StorageClassUID: types.UID("storage-class-uid"), + StorageClassDigest: "storage-class-digest", + CatalogDigest: "catalog-digest", + Provider: "nvmesh", + Provisioner: nvcastorage.NVMeshStorageClassProvisioner, + Transition: nvcastorage.ModelCacheTransitionROXReadOnly, + RequiredAccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + corev1.ReadOnlyMany, + }, + RequiredMountOptions: []string{"ro", "norecovery", "nouuid"}, + } + } + selection, err := nvcastorage.NewPersistedModelCacheStorageSelection(workflow, mode, resolved) + require.NoError(t, err) + if mode == nvcastorage.ModelCacheSelectionDurable { + selection.BindingName = "model-cache-binding" + selection.BindingUID = types.UID("binding-uid") + } + raw, err := selection.Marshal() + require.NoError(t, err) + + return &nvcav2beta1.ICMSRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "request", + Namespace: "requests", + UID: types.UID("request-uid"), + Annotations: map[string]string{ + nvcastorage.ModelCacheStorageSelectionAnnotationKey: raw, + }, + }, + Spec: nvcav2beta1.ICMSRequestSpec{ + CreationMsgInfo: nvcav2beta1.ICMSCreationMessageInfo{ + FunctionLaunchSpecification: &function.LaunchSpecification{ + CacheLaunchSpecification: &common.CacheLaunchSpecification{ + CacheHandle: "cache-handle", + CacheSize: 1, + }, + HelmChartLaunchSpecification: &common.HelmChartLaunchSpecification{}, + }, + }, + }, + } +} + +type getErrorClient struct { + client.Client + err error +} + +func (c *getErrorClient) Get( + _ context.Context, + _ client.ObjectKey, + _ client.Object, + _ ...client.GetOption, +) error { + return c.err +} diff --git a/src/compute-plane-services/nvca/internal/miniservice/reconcile.go b/src/compute-plane-services/nvca/internal/miniservice/reconcile.go index d0d5625fa..8da0b3de5 100644 --- a/src/compute-plane-services/nvca/internal/miniservice/reconcile.go +++ b/src/compute-plane-services/nvca/internal/miniservice/reconcile.go @@ -746,15 +746,14 @@ func (r *Reconciler) doInstall(ctx context.Context, if cacheLaunchRequested(icmsReq) { // Same config value the storage controller provisions cache volumes // with, so the class checked here is the class they land on. - cacheBackend, err = nvcastorage.SelectHelmCacheBackend(ctx, r.Client, r.FeatureFlagFetcher, - r.cfg.Agent.ModelCache.StorageClassName) + cacheBackend, err = r.selectHelmCacheBackend(ctx, icmsReq, ms.Spec.Namespace) if err != nil { return reconcile.Result{}, fmt.Errorf("select helm cache backend: %w", err) } } // Create storage requests if configured for the cluster. - stDone, err := r.doStorageRequests(ctx, + stDone, readyStorageRequests, err := r.doStorageRequests(ctx, ms, icmsReq, infraObjectMutators, workerPullSecrets, cacheInitJob, cacheInitPVC, cacheBackend, ) @@ -762,11 +761,7 @@ func (r *Reconciler) doInstall(ctx context.Context, return reconcile.Result{}, err } - stList := &nvcav2beta1.StorageRequestList{} - if err := r.Client.List(ctx, stList, client.InNamespace(ms.Spec.Namespace)); err != nil { - return reconcile.Result{}, (err) - } - instanceStorageAnnos, utilsStorageAnnos := getAnnotationsForReadyStorageRequests(stList) + instanceStorageAnnos, utilsStorageAnnos := getAnnotationsForReadyStorageRequests(readyStorageRequests) if len(instanceStorageAnnos) != 0 { maps.Copy(metaInput.PodAnnotations, instanceStorageAnnos) } diff --git a/src/compute-plane-services/nvca/internal/miniservice/reconcile_storagerequests.go b/src/compute-plane-services/nvca/internal/miniservice/reconcile_storagerequests.go index 49d5760b2..4f67ccab0 100644 --- a/src/compute-plane-services/nvca/internal/miniservice/reconcile_storagerequests.go +++ b/src/compute-plane-services/nvca/internal/miniservice/reconcile_storagerequests.go @@ -44,7 +44,7 @@ func (r *Reconciler) doStorageRequests(ctx context.Context, cacheInitJob *batchv1.Job, cacheInitPVC *corev1.PersistentVolumeClaim, cacheBackend nvcastorage.HelmCacheBackend, -) (allReady bool, err error) { +) (allReady bool, ready *nvcav2beta1.StorageRequestList, err error) { log := logf.FromContext(ctx) sts, err := r.makeStorageRequests(icmsReq, workerImagePullSecrets, cacheInitJob, cacheInitPVC, cacheBackend) @@ -52,13 +52,13 @@ func (r *Reconciler) doStorageRequests(ctx context.Context, log.Error(err, "Failed to make StorageRequests") // makeStorageRequests marks genuinely terminal errors (invalid spec) // with reconcile.TerminalError itself; anything else is retried. - return false, err + return false, nil, err } stNames := sets.New[string]() for _, st := range sts { if err := r.create(ctx, ms, objMutators, nil, r.Client, st); err != nil { log.Error(err, "Failed to create StorageRequest") - return false, err + return false, nil, err } // Mark the miniservice as caching in progress for agent to handle. if st.Spec.Type == nvcav2beta1.ModelCacheRequest { @@ -69,15 +69,35 @@ func (r *Reconciler) doStorageRequests(ctx context.Context, stList := &nvcav2beta1.StorageRequestList{} if err := r.Client.List(ctx, stList, client.InNamespace(ms.Spec.Namespace)); err != nil { - return false, err + return false, nil, err } + ready = &nvcav2beta1.StorageRequestList{} - for _, st := range stList.Items { + for _, st := range storageRequestsWithNames(stList, stNames) { + if st.Spec.Type == nvcav2beta1.ModelCacheRequest && + icmsReq.Annotations[nvcastorage.ModelCacheStorageSelectionAnnotationKey] != "" { + if err := validatePersistedModelCacheStorageRequest( + &st, + icmsReq, + icmsReq.Annotations[nvcastorage.ModelCacheStorageSelectionAnnotationKey], + cacheBackend, + ); err != nil { + return false, nil, reconcile.TerminalError(err) + } + } switch st.Status.Phase { case nvcav2beta1.StorageFailed, nvcav2beta1.StorageRuntimeError: lerr := fmt.Errorf("storage request %s has failed", st.Spec.Type) switch st.Spec.Type { case nvcav2beta1.ModelCacheRequest: + durable, err := persistedDurableHelmCacheSelection(icmsReq) + if err != nil { + return false, nil, reconcile.TerminalError(err) + } + if durable { + log.Error(lerr, "Durable model caching failed", "phase", st.Status.Phase) + return false, nil, reconcile.TerminalError(lerr) + } log.Error(lerr, "Storage failed, model caching will be disabled", "phase", st.Status.Phase) meta.SetStatusCondition(&ms.Status.Conditions, metav1.Condition{ Type: v1alpha1.MiniServiceConditionCacheSuccessful, @@ -89,14 +109,23 @@ func (r *Reconciler) doStorageRequests(ctx context.Context, stNames.Delete(st.Name) case nvcav2beta1.SharedStorageRequest: log.Error(lerr, "Shared storage failed", "phase", st.Status.Phase) - return false, reconcile.TerminalError(lerr) + return false, nil, reconcile.TerminalError(lerr) case nvcav2beta1.InternalPersistentStorageRequest: log.Error(lerr, "Internal persistent storage failed", "phase", st.Status.Phase) - return false, reconcile.TerminalError(lerr) + return false, nil, reconcile.TerminalError(lerr) } case nvcav2beta1.StorageReady: switch st.Spec.Type { case nvcav2beta1.ModelCacheRequest: + durable, selectionErr := persistedDurableHelmCacheSelection(icmsReq) + if selectionErr != nil { + return false, nil, reconcile.TerminalError(selectionErr) + } + if durable { + if err := validateReadyPersistedModelCacheStorageRequest(&st, icmsReq); err != nil { + return false, nil, reconcile.TerminalError(err) + } + } meta.SetStatusCondition(&ms.Status.Conditions, metav1.Condition{ Type: v1alpha1.MiniServiceConditionCacheSuccessful, Status: metav1.ConditionTrue, @@ -107,6 +136,7 @@ func (r *Reconciler) doStorageRequests(ctx context.Context, } stNames.Delete(st.Name) + ready.Items = append(ready.Items, *st.DeepCopy()) log.V(1).Info("StorageRequest succeeded", "type", st.Spec.Type) default: @@ -125,7 +155,56 @@ func (r *Reconciler) doStorageRequests(ctx context.Context, } } - return stNames.Len() == 0, nil + return stNames.Len() == 0, ready, nil +} + +func storageRequestsWithNames( + stList *nvcav2beta1.StorageRequestList, + names sets.Set[string], +) []nvcav2beta1.StorageRequest { + if stList == nil || names.Len() == 0 { + return nil + } + result := make([]nvcav2beta1.StorageRequest, 0, names.Len()) + for i := range stList.Items { + if names.Has(stList.Items[i].Name) { + result = append(result, *stList.Items[i].DeepCopy()) + } + } + return result +} + +func validateReadyPersistedModelCacheStorageRequest( + st *nvcav2beta1.StorageRequest, + icmsReq *nvcav2beta1.ICMSRequest, +) error { + selection, err := nvcastorage.ParsePersistedModelCacheStorageSelection( + icmsReq.Annotations[nvcastorage.ModelCacheStorageSelectionAnnotationKey]) + if err != nil { + return err + } + if selection.BindingName == "" || selection.BindingUID == "" { + return fmt.Errorf("durable Helm model cache selection has no committed binding reference") + } + expectedROPVCName := "ro-pvc-" + helmModelCacheHandle(icmsReq) + if st.Status.ModelCache == nil || st.Status.ModelCache.ROPVCName != expectedROPVCName { + return fmt.Errorf( + "durable model cache StorageRequest %s/%s reported reader PVC %q, want %q", + st.Namespace, st.Name, modelCacheROPVCName(st), expectedROPVCName) + } + if got := st.Labels[nvcastorage.ModelCacheBindingUIDLabelKey]; got != string(selection.BindingUID) { + return fmt.Errorf( + "durable model cache StorageRequest %s/%s has binding UID %q, want %q", + st.Namespace, st.Name, got, selection.BindingUID) + } + return nil +} + +func modelCacheROPVCName(st *nvcav2beta1.StorageRequest) string { + if st == nil || st.Status.ModelCache == nil { + return "" + } + return st.Status.ModelCache.ROPVCName } func (r *Reconciler) makeStorageRequests( @@ -136,7 +215,7 @@ func (r *Reconciler) makeStorageRequests( backend nvcastorage.HelmCacheBackend, ) (sts []*nvcav2beta1.StorageRequest, err error) { // The caching backend is selected ONCE per reconcile by the caller - // (SelectHelmCacheBackend in doInstall, gated on cacheLaunchRequested) and + // (selectHelmCacheBackend in doInstall, gated on cacheLaunchRequested) and // passed in, so StorageRequest creation and the ephemeral annotation // decision always agree even if the cluster's storage classes change // mid-reconcile. diff --git a/src/compute-plane-services/nvca/internal/miniservice/reconcile_storagerequests_test.go b/src/compute-plane-services/nvca/internal/miniservice/reconcile_storagerequests_test.go index 29de4835c..0a7236d68 100644 --- a/src/compute-plane-services/nvca/internal/miniservice/reconcile_storagerequests_test.go +++ b/src/compute-plane-services/nvca/internal/miniservice/reconcile_storagerequests_test.go @@ -25,11 +25,16 @@ import ( "github.com/stretchr/testify/require" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" + clientfake "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common" "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v1alpha1" nvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1" featureflagmock "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/featureflag/mock" nvcastorage "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/storage" @@ -98,7 +103,7 @@ func TestMakeStorageRequests_BackendHandling(t *testing.T) { }) t.Run("none backend emits no ModelCacheRequest", func(t *testing.T) { - // SelectHelmCacheBackend returns None when CachingSupport or the + // selectHelmCacheBackend returns None when CachingSupport or the // HelmModelCaching sub-gate is off; a valid cache spec must still // produce no ModelCacheRequest. icmsReq := &nvcav2beta1.ICMSRequest{} @@ -123,3 +128,112 @@ func TestMakeStorageRequests_BackendHandling(t *testing.T) { assert.Empty(t, sts) }) } + +func TestStorageRequestsWithNamesIgnoresForeignModelCacheRequest(t *testing.T) { + canonical := nvcav2beta1.StorageRequest{ + ObjectMeta: metav1.ObjectMeta{Name: nvcav2beta1.ModelCacheRequest.Name()}, + Spec: nvcav2beta1.StorageRequestSpec{Type: nvcav2beta1.ModelCacheRequest}, + Status: nvcav2beta1.StorageRequestStatus{ + Phase: nvcav2beta1.StorageReady, + ModelCache: &nvcav2beta1.ModelCacheStatus{ROPVCName: "ro-pvc-canonical"}, + }, + } + foreign := nvcav2beta1.StorageRequest{ + ObjectMeta: metav1.ObjectMeta{Name: "foreign-model-cache"}, + Spec: nvcav2beta1.StorageRequestSpec{Type: nvcav2beta1.ModelCacheRequest}, + Status: nvcav2beta1.StorageRequestStatus{ + Phase: nvcav2beta1.StorageFailed, + ModelCache: &nvcav2beta1.ModelCacheStatus{ROPVCName: "ro-pvc-foreign"}, + }, + } + + filtered := storageRequestsWithNames( + &nvcav2beta1.StorageRequestList{Items: []nvcav2beta1.StorageRequest{canonical, foreign}}, + sets.New(nvcav2beta1.ModelCacheRequest.Name()), + ) + + require.Len(t, filtered, 1) + assert.Equal(t, canonical.Name, filtered[0].Name) + instanceAnnotations, utilsAnnotations := getAnnotationsForReadyStorageRequests( + &nvcav2beta1.StorageRequestList{Items: filtered}) + assert.Equal(t, "ro-pvc-canonical", + instanceAnnotations[nvcastorage.WebhookModelCachePVCNameAnnotationKey]) + assert.Equal(t, "ro-pvc-canonical", + utilsAnnotations[nvcastorage.WebhookModelCachePVCNameAnnotationKey]) +} + +func TestDoStorageRequestsIgnoresFailedForeignModelCacheRequest(t *testing.T) { + const instanceNamespace = "instance-ns" + request := requestWithModelCacheSelection( + t, nvcastorage.ModelCacheWorkflowHelm, nvcastorage.ModelCacheSelectionDurable) + request.Spec.Action = common.FunctionCreationAction + canonical, err := nvcastorage.NewModelCacheStorageRequest(request, &featureflagmock.Fetcher{}) + require.NoError(t, err) + canonical.Namespace = instanceNamespace + canonical.Spec.ModelCache.Backend = string(nvcastorage.HelmCacheBackendNVMesh) + canonical.Labels[nvcastorage.ModelCacheBindingUIDLabelKey] = "binding-uid" + canonical.Status = nvcav2beta1.StorageRequestStatus{ + Phase: nvcav2beta1.StorageReady, + ModelCache: &nvcav2beta1.ModelCacheStatus{ROPVCName: "ro-pvc-cache-handle"}, + } + foreign := canonical.DeepCopy() + foreign.Name = "foreign-model-cache" + foreign.Status.Phase = nvcav2beta1.StorageFailed + foreign.Status.ModelCache.ROPVCName = "ro-pvc-foreign" + + scheme := runtime.NewScheme() + require.NoError(t, v1alpha1.AddToScheme(scheme)) + require.NoError(t, nvcav2beta1.AddToScheme(scheme)) + r := &Reconciler{ + ControllerOptions: ControllerOptions{ + FeatureFlagFetcher: &featureflagmock.Fetcher{}, + }, + newPermissionsChecker: newFakePermissionsChecker, + Client: clientfake.NewClientBuilder().WithScheme(scheme). + WithRESTMapper(newTestRESTMapper(scheme)). + WithObjects(canonical, foreign).Build(), + } + ms := &v1alpha1.MiniService{Spec: v1alpha1.MiniServiceSpec{Namespace: instanceNamespace}} + + allReady, ready, err := r.doStorageRequests( + t.Context(), ms, request, nil, nil, + &batchv1.Job{}, &corev1.PersistentVolumeClaim{}, nvcastorage.HelmCacheBackendNVMesh) + + require.NoError(t, err) + assert.True(t, allReady) + require.NotNil(t, ready) + require.Len(t, ready.Items, 1) + assert.Equal(t, canonical.Name, ready.Items[0].Name) + assert.Equal(t, "ro-pvc-cache-handle", ready.Items[0].Status.ModelCache.ROPVCName) +} + +func TestValidateReadyPersistedModelCacheStorageRequest(t *testing.T) { + request := requestWithModelCacheSelection( + t, nvcastorage.ModelCacheWorkflowHelm, nvcastorage.ModelCacheSelectionDurable) + base := &nvcav2beta1.StorageRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: nvcav2beta1.ModelCacheRequest.Name(), + Labels: map[string]string{ + nvcastorage.ModelCacheBindingUIDLabelKey: "binding-uid", + }, + }, + Status: nvcav2beta1.StorageRequestStatus{ + Phase: nvcav2beta1.StorageReady, + ModelCache: &nvcav2beta1.ModelCacheStatus{ROPVCName: "ro-pvc-cache-handle"}, + }, + } + + require.NoError(t, validateReadyPersistedModelCacheStorageRequest(base, request)) + + wrongReader := base.DeepCopy() + wrongReader.Status.ModelCache.ROPVCName = "ro-pvc-foreign" + require.ErrorContains(t, + validateReadyPersistedModelCacheStorageRequest(wrongReader, request), + "reported reader PVC") + + wrongBinding := base.DeepCopy() + wrongBinding.Labels[nvcastorage.ModelCacheBindingUIDLabelKey] = "foreign-binding" + require.ErrorContains(t, + validateReadyPersistedModelCacheStorageRequest(wrongBinding, request), + "has binding UID") +} diff --git a/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1/BUILD.bazel b/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1/BUILD.bazel index 259e4bd10..09418d4bb 100644 --- a/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1/BUILD.bazel +++ b/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1/BUILD.bazel @@ -8,7 +8,9 @@ go_library( srcs = [ "conversion.go", "doc.go", + "generated.openapi.go", "icmsrequest_types.go", + "modelcachebinding_types.go", "register.go", "storage_types.go", "zz_generated.deepcopy.go", @@ -25,6 +27,9 @@ go_library( "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/apis/meta/v1:meta", "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/runtime", "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/runtime/schema", + "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/types", + "//src/compute-plane-services/nvca/vendor/k8s.io/kube-openapi/pkg/common", + "//src/compute-plane-services/nvca/vendor/k8s.io/kube-openapi/pkg/validation/spec", ], ) @@ -39,6 +44,7 @@ go_test( srcs = [ "conversion_test.go", "icmsrequest_types_test.go", + "modelcachebinding_types_test.go", ], embed = [":v2beta1"], deps = [ @@ -47,5 +53,8 @@ go_test( "//src/compute-plane-services/nvca/vendor/github.com/stretchr/testify/assert", "//src/compute-plane-services/nvca/vendor/github.com/stretchr/testify/require", "//src/compute-plane-services/nvca/vendor/k8s.io/api/core/v1:core", + "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/apis/meta/v1:meta", + "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/runtime", + "//src/compute-plane-services/nvca/vendor/k8s.io/kube-openapi/pkg/validation/spec", ], ) diff --git a/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1/doc.go b/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1/doc.go index 7523d499a..d4d79d9b5 100644 --- a/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1/doc.go +++ b/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1/doc.go @@ -19,7 +19,7 @@ limitations under the License. // +k8s:defaulter-gen=TypeMeta // +groupName=nvca.nvcf.nvidia.io // -// Package v2beta1 contains the NVCA storage request API (StorageRequest). +// Package v2beta1 contains the NVCA request, storage, and model-cache binding APIs. // Prefer v2beta1 for new code; it uses RequestName/RequestNamespace instead of // the legacy prefixed fields, with ICMS request semantics in comments. diff --git a/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1/generated.openapi.go b/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1/generated.openapi.go new file mode 100644 index 000000000..7a0a05918 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1/generated.openapi.go @@ -0,0 +1,3778 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by openapi-gen. DO NOT EDIT. + +package v2beta1 + +import ( + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + common "k8s.io/kube-openapi/pkg/common" + spec "k8s.io/kube-openapi/pkg/validation/spec" +) + +func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenAPIDefinition { + return map[string]common.OpenAPIDefinition{ + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ICMSCreationMessageInfo": schema_pkg_apis_nvca_v2beta1_ICMSCreationMessageInfo(ref), + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ICMSRequest": schema_pkg_apis_nvca_v2beta1_ICMSRequest(ref), + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ICMSRequestList": schema_pkg_apis_nvca_v2beta1_ICMSRequestList(ref), + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ICMSRequestSpec": schema_pkg_apis_nvca_v2beta1_ICMSRequestSpec(ref), + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ICMSRequestStatus": schema_pkg_apis_nvca_v2beta1_ICMSRequestStatus(ref), + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ICMSTerminationMessageInfo": schema_pkg_apis_nvca_v2beta1_ICMSTerminationMessageInfo(ref), + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ModelCacheBinding": schema_pkg_apis_nvca_v2beta1_ModelCacheBinding(ref), + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ModelCacheBindingDecision": schema_pkg_apis_nvca_v2beta1_ModelCacheBindingDecision(ref), + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ModelCacheBindingIdentity": schema_pkg_apis_nvca_v2beta1_ModelCacheBindingIdentity(ref), + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ModelCacheBindingList": schema_pkg_apis_nvca_v2beta1_ModelCacheBindingList(ref), + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ModelCacheBindingRealizedState": schema_pkg_apis_nvca_v2beta1_ModelCacheBindingRealizedState(ref), + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ModelCacheBindingRequestReference": schema_pkg_apis_nvca_v2beta1_ModelCacheBindingRequestReference(ref), + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ModelCacheBindingResourceIntent": schema_pkg_apis_nvca_v2beta1_ModelCacheBindingResourceIntent(ref), + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ModelCacheBindingSpec": schema_pkg_apis_nvca_v2beta1_ModelCacheBindingSpec(ref), + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ModelCacheBindingStatus": schema_pkg_apis_nvca_v2beta1_ModelCacheBindingStatus(ref), + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ModelCacheStorageClassSnapshot": schema_pkg_apis_nvca_v2beta1_ModelCacheStorageClassSnapshot(ref), + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.StorageRequest": schema_pkg_apis_nvca_v2beta1_StorageRequest(ref), + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.StorageRequestList": schema_pkg_apis_nvca_v2beta1_StorageRequestList(ref), + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.StorageRequestSpec": schema_pkg_apis_nvca_v2beta1_StorageRequestSpec(ref), + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.StorageRequestStatus": schema_pkg_apis_nvca_v2beta1_StorageRequestStatus(ref), + v1.APIGroup{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIGroup(ref), + v1.APIGroupList{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIGroupList(ref), + v1.APIResource{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIResource(ref), + v1.APIResourceList{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIResourceList(ref), + v1.APIVersions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_APIVersions(ref), + v1.ApplyOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ApplyOptions(ref), + v1.Condition{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Condition(ref), + v1.CreateOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_CreateOptions(ref), + v1.DeleteOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_DeleteOptions(ref), + v1.Duration{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Duration(ref), + v1.FieldSelectorRequirement{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_FieldSelectorRequirement(ref), + v1.FieldsV1{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_FieldsV1(ref), + v1.GetOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GetOptions(ref), + v1.GroupKind{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupKind(ref), + v1.GroupResource{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupResource(ref), + v1.GroupVersion{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupVersion(ref), + v1.GroupVersionForDiscovery{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupVersionForDiscovery(ref), + v1.GroupVersionKind{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupVersionKind(ref), + v1.GroupVersionResource{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_GroupVersionResource(ref), + v1.InternalEvent{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_InternalEvent(ref), + v1.LabelSelector{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_LabelSelector(ref), + v1.LabelSelectorRequirement{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_LabelSelectorRequirement(ref), + v1.List{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_List(ref), + v1.ListMeta{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ListMeta(ref), + v1.ListOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ListOptions(ref), + v1.ManagedFieldsEntry{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ManagedFieldsEntry(ref), + v1.MicroTime{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_MicroTime(ref), + v1.ObjectMeta{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ObjectMeta(ref), + v1.OwnerReference{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_OwnerReference(ref), + v1.PartialObjectMetadata{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_PartialObjectMetadata(ref), + v1.PartialObjectMetadataList{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_PartialObjectMetadataList(ref), + v1.Patch{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Patch(ref), + v1.PatchOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_PatchOptions(ref), + v1.Preconditions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Preconditions(ref), + v1.RootPaths{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_RootPaths(ref), + v1.ServerAddressByClientCIDR{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_ServerAddressByClientCIDR(ref), + v1.Status{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Status(ref), + v1.StatusCause{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_StatusCause(ref), + v1.StatusDetails{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_StatusDetails(ref), + v1.Table{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Table(ref), + v1.TableColumnDefinition{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TableColumnDefinition(ref), + v1.TableOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TableOptions(ref), + v1.TableRow{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TableRow(ref), + v1.TableRowCondition{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TableRowCondition(ref), + v1.Time{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Time(ref), + v1.Timestamp{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_Timestamp(ref), + v1.TypeMeta{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_TypeMeta(ref), + v1.UpdateOptions{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_UpdateOptions(ref), + v1.WatchEvent{}.OpenAPIModelName(): schema_pkg_apis_meta_v1_WatchEvent(ref), + } +} + +func schema_pkg_apis_nvca_v2beta1_ICMSCreationMessageInfo(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "clusterGroup": { + SchemaProps: spec.SchemaProps{ + Description: "Deprecated: this no longer will be set since cluster group is static", + Type: []string{"string"}, + Format: "", + }, + }, + "gpuName": { + SchemaProps: spec.SchemaProps{ + Description: "Deprecated: use gpuType", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "queueURL": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "requestId": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "sub": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "messageBatchId": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "ncaId": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "accountName": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "action": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "instanceCount": { + SchemaProps: spec.SchemaProps{ + Default: 0, + Type: []string{"integer"}, + Format: "int64", + }, + }, + "instanceTypeName": { + SchemaProps: spec.SchemaProps{ + Description: "InstanceTypeName is the full instance type name, with GPU number suffix \"_{d}x\". Use this for metrics/instance identification.", + Type: []string{"string"}, + Format: "", + }, + }, + "instanceTypeValue": { + SchemaProps: spec.SchemaProps{ + Description: "InstanceTypeValue is the instance type name without GPU number suffix. Use this for label selectors.", + Type: []string{"string"}, + Format: "", + }, + }, + "instanceType": { + SchemaProps: spec.SchemaProps{ + Description: "Deprecated: use InstanceTypeName.", + Type: []string{"string"}, + Format: "", + }, + }, + "traceParent": { + SchemaProps: spec.SchemaProps{ + Description: "OTel fields", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "traceState": { + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Allows: true, + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "gpuType": { + SchemaProps: spec.SchemaProps{ + Description: "These fields will not be set for GFN.", + Type: []string{"string"}, + Format: "", + }, + }, + "requestedGPUCount": { + SchemaProps: spec.SchemaProps{ + Type: []string{"integer"}, + Format: "int64", + }, + }, + "deploymentId": { + SchemaProps: spec.SchemaProps{ + Description: "DeploymentID is the unique identifier for the deployment (function ID and function version ID can be re-used)", + Type: []string{"string"}, + Format: "", + }, + }, + "gpuSpecificationId": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "launchArtifacts": { + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function.LaunchArtifact"), + }, + }, + }, + }, + }, + "functionLaunchSpecification": { + SchemaProps: spec.SchemaProps{ + Ref: ref("github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function.LaunchSpecification"), + }, + }, + "taskLaunchSpecification": { + SchemaProps: spec.SchemaProps{ + Ref: ref("github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/task.LaunchSpecification"), + }, + }, + }, + Required: []string{"gpuName", "requestId", "sub", "messageBatchId", "ncaId", "accountName", "action", "instanceCount", "traceParent", "traceState"}, + }, + }, + Dependencies: []string{ + "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function.LaunchArtifact", "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function.LaunchSpecification", "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/task.LaunchSpecification"}, + } +} + +func schema_pkg_apis_nvca_v2beta1_ICMSRequest(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ICMSRequest represents Request Object as obtained from ICMS to be applied by NVCA and the progress of the request.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref(v1.ObjectMeta{}.OpenAPIModelName()), + }, + }, + "spec": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ICMSRequestSpec"), + }, + }, + "status": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ICMSRequestStatus"), + }, + }, + }, + }, + }, + Dependencies: []string{ + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ICMSRequestSpec", "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ICMSRequestStatus", v1.ObjectMeta{}.OpenAPIModelName()}, + } +} + +func schema_pkg_apis_nvca_v2beta1_ICMSRequestList(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref(v1.ListMeta{}.OpenAPIModelName()), + }, + }, + "items": { + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ICMSRequest"), + }, + }, + }, + }, + }, + }, + Required: []string{"items"}, + }, + }, + Dependencies: []string{ + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ICMSRequest", v1.ListMeta{}.OpenAPIModelName()}, + } +} + +func schema_pkg_apis_nvca_v2beta1_ICMSRequestSpec(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ICMSRequestSpec defines the desired request to be handled by NVCA", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "requestId": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "sqsMessageReceipt": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "ncaId": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "messageBatchId": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "action": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "creationMsgInfo": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ICMSCreationMessageInfo"), + }, + }, + "terminationMsgInfo": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ICMSTerminationMessageInfo"), + }, + }, + "functionDetails": { + SchemaProps: spec.SchemaProps{ + Description: "FunctionDetails will be set if the message is for creating a function.", + Default: map[string]interface{}{}, + Ref: ref("github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function.Details"), + }, + }, + "taskDetails": { + SchemaProps: spec.SchemaProps{ + Description: "TaskDetails will be set if the message is for creating a task.", + Default: map[string]interface{}{}, + Ref: ref("github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/task.Details"), + }, + }, + "functionId": { + SchemaProps: spec.SchemaProps{ + Description: "Deprecated: use functionDetails.functionId", + Type: []string{"string"}, + Format: "", + }, + }, + "functionVersionId": { + SchemaProps: spec.SchemaProps{ + Description: "Deprecated: use functionDetails.functionVersionId", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"creationMsgInfo", "terminationMsgInfo"}, + }, + }, + Dependencies: []string{ + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ICMSCreationMessageInfo", "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ICMSTerminationMessageInfo", "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function.Details", "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/task.Details"}, + } +} + +func schema_pkg_apis_nvca_v2beta1_ICMSRequestStatus(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ICMSRequestStatus is the NVCA managed status of all requests in the backend.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "requestStatus": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "lastStatusUpdated": { + SchemaProps: spec.SchemaProps{ + Ref: ref(v1.Time{}.OpenAPIModelName()), + }, + }, + "lastObservedAllTerminatedInstances": { + SchemaProps: spec.SchemaProps{ + Ref: ref(v1.Time{}.OpenAPIModelName()), + }, + }, + "lastACKTimestamp": { + SchemaProps: spec.SchemaProps{ + Ref: ref(v1.Time{}.OpenAPIModelName()), + }, + }, + "cacheReferenceName": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "instances": { + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Allows: true, + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.InstanceStatus"), + }, + }, + }, + }, + }, + "reconcileErrors": { + SchemaProps: spec.SchemaProps{ + Type: []string{"integer"}, + Format: "int64", + }, + }, + "lastReconcileError": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "requestStatusTraceContexts": { + SchemaProps: spec.SchemaProps{ + Description: "RequestStatusTraceContexts represents a of map of traces to be closed when the state represented by the the RequestStatus is reached.", + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Allows: true, + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ICMSRequestSpanContextConfig"), + }, + }, + }, + }, + }, + }, + }, + }, + Dependencies: []string{ + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ICMSRequestSpanContextConfig", "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.InstanceStatus", v1.Time{}.OpenAPIModelName()}, + } +} + +func schema_pkg_apis_nvca_v2beta1_ICMSTerminationMessageInfo(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "availabilityZone": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "instanceIds": { + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + }, + }, + }, + } +} + +func schema_pkg_apis_nvca_v2beta1_ModelCacheBinding(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ModelCacheBinding records one immutable provider decision for a shared model-cache key.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref(v1.ObjectMeta{}.OpenAPIModelName()), + }, + }, + "spec": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ModelCacheBindingSpec"), + }, + }, + "status": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ModelCacheBindingStatus"), + }, + }, + }, + Required: []string{"spec"}, + }, + }, + Dependencies: []string{ + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ModelCacheBindingSpec", "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ModelCacheBindingStatus", v1.ObjectMeta{}.OpenAPIModelName()}, + } +} + +func schema_pkg_apis_nvca_v2beta1_ModelCacheBindingDecision(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ModelCacheBindingDecision snapshots the selected provider transition.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "provider": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "provisioner": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "transition": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "requiredAccessModes": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "set", + }, + }, + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "requiredMountOptions": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "catalogDigest": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "encryptionRequired": { + SchemaProps: spec.SchemaProps{ + Default: false, + Type: []string{"boolean"}, + Format: "", + }, + }, + }, + Required: []string{"provider", "provisioner", "transition", "requiredAccessModes", "catalogDigest", "encryptionRequired"}, + }, + }, + } +} + +func schema_pkg_apis_nvca_v2beta1_ModelCacheBindingIdentity(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ModelCacheBindingIdentity is the stable cache key represented by a binding.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "version": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "workflow": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "sharingDomainDigest": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "cacheHandleDigest": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"version", "workflow", "sharingDomainDigest", "cacheHandleDigest"}, + }, + }, + } +} + +func schema_pkg_apis_nvca_v2beta1_ModelCacheBindingList(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ModelCacheBindingList is a list of ModelCacheBinding objects.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref(v1.ListMeta{}.OpenAPIModelName()), + }, + }, + "items": { + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ModelCacheBinding"), + }, + }, + }, + }, + }, + }, + Required: []string{"items"}, + }, + }, + Dependencies: []string{ + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ModelCacheBinding", v1.ListMeta{}.OpenAPIModelName()}, + } +} + +func schema_pkg_apis_nvca_v2beta1_ModelCacheBindingRealizedState(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ModelCacheBindingRealizedState records provider resources only after they exist.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "boundPersistentVolumeName": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "providerDataIdentity": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "populationState": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + } +} + +func schema_pkg_apis_nvca_v2beta1_ModelCacheBindingRequestReference(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ModelCacheBindingRequestReference identifies a request currently using a binding.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "namespace": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "name": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "uid": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"namespace", "name", "uid"}, + }, + }, + } +} + +func schema_pkg_apis_nvca_v2beta1_ModelCacheBindingResourceIntent(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ModelCacheBindingResourceIntent records deterministic names for shared resources.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "writerNamespace": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "persistentVolumeClaimNames": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "set", + }, + }, + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "persistentVolumeNames": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "set", + }, + }, + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "jobNames": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "set", + }, + }, + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "storageClassNames": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "set", + }, + }, + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "secretNames": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "set", + }, + }, + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "leaseName": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"writerNamespace"}, + }, + }, + } +} + +func schema_pkg_apis_nvca_v2beta1_ModelCacheBindingSpec(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ModelCacheBindingSpec contains the immutable identity, provider decision, and resource intent.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "identity": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ModelCacheBindingIdentity"), + }, + }, + "decision": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ModelCacheBindingDecision"), + }, + }, + "storageClass": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ModelCacheStorageClassSnapshot"), + }, + }, + "resources": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ModelCacheBindingResourceIntent"), + }, + }, + }, + Required: []string{"identity", "decision", "storageClass", "resources"}, + }, + }, + Dependencies: []string{ + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ModelCacheBindingDecision", "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ModelCacheBindingIdentity", "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ModelCacheBindingResourceIntent", "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ModelCacheStorageClassSnapshot"}, + } +} + +func schema_pkg_apis_nvca_v2beta1_ModelCacheBindingStatus(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ModelCacheBindingStatus contains mutable lifecycle and realized-resource state.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "phase": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "lastPhaseTransitionTime": { + SchemaProps: spec.SchemaProps{ + Ref: ref(v1.Time{}.OpenAPIModelName()), + }, + }, + "requestReferences": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "uid", + }, + "x-kubernetes-list-type": "map", + }, + }, + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ModelCacheBindingRequestReference"), + }, + }, + }, + }, + }, + "realized": { + SchemaProps: spec.SchemaProps{ + Ref: ref("github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ModelCacheBindingRealizedState"), + }, + }, + "conditions": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "type", + }, + "x-kubernetes-list-type": "map", + }, + }, + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref(v1.Condition{}.OpenAPIModelName()), + }, + }, + }, + }, + }, + }, + }, + }, + Dependencies: []string{ + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ModelCacheBindingRealizedState", "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ModelCacheBindingRequestReference", v1.Condition{}.OpenAPIModelName(), v1.Time{}.OpenAPIModelName()}, + } +} + +func schema_pkg_apis_nvca_v2beta1_ModelCacheStorageClassSnapshot(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ModelCacheStorageClassSnapshot identifies the exact retained StorageClass selected by the binding.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "name": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "uid": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "reclaimPolicy": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "configurationDigest": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"name", "uid", "reclaimPolicy", "configurationDigest"}, + }, + }, + } +} + +func schema_pkg_apis_nvca_v2beta1_StorageRequest(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref(v1.ObjectMeta{}.OpenAPIModelName()), + }, + }, + "spec": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.StorageRequestSpec"), + }, + }, + "status": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.StorageRequestStatus"), + }, + }, + }, + }, + }, + Dependencies: []string{ + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.StorageRequestSpec", "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.StorageRequestStatus", v1.ObjectMeta{}.OpenAPIModelName()}, + } +} + +func schema_pkg_apis_nvca_v2beta1_StorageRequestList(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref(v1.ListMeta{}.OpenAPIModelName()), + }, + }, + "items": { + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.StorageRequest"), + }, + }, + }, + }, + }, + }, + Required: []string{"items"}, + }, + }, + Dependencies: []string{ + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.StorageRequest", v1.ListMeta{}.OpenAPIModelName()}, + } +} + +func schema_pkg_apis_nvca_v2beta1_StorageRequestSpec(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "type": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "requestName": { + SchemaProps: spec.SchemaProps{ + Description: "RequestName is the name of the ICMS request that requested this storage.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "requestNamespace": { + SchemaProps: spec.SchemaProps{ + Description: "RequestNamespace is the namespace of the ICMS request.", + Type: []string{"string"}, + Format: "", + }, + }, + "modelCache": { + SchemaProps: spec.SchemaProps{ + Ref: ref("github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ModelCacheSpec"), + }, + }, + "sharedStorage": { + SchemaProps: spec.SchemaProps{ + Ref: ref("github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.SharedStorageSpec"), + }, + }, + "internalPersistentStorage": { + SchemaProps: spec.SchemaProps{ + Ref: ref("github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.InternalPersistentStorageSpec"), + }, + }, + }, + Required: []string{"type", "requestName"}, + }, + }, + Dependencies: []string{ + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.InternalPersistentStorageSpec", "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ModelCacheSpec", "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.SharedStorageSpec"}, + } +} + +func schema_pkg_apis_nvca_v2beta1_StorageRequestStatus(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "lastPhaseTransitionTime": { + SchemaProps: spec.SchemaProps{ + Ref: ref(v1.Time{}.OpenAPIModelName()), + }, + }, + "phase": { + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + "conditions": { + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref(v1.Condition{}.OpenAPIModelName()), + }, + }, + }, + }, + }, + "modelCache": { + SchemaProps: spec.SchemaProps{ + Ref: ref("github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ModelCacheStatus"), + }, + }, + "sharedStorage": { + SchemaProps: spec.SchemaProps{ + Ref: ref("github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.SharedStorageStatus"), + }, + }, + "internalPersistentStorage": { + SchemaProps: spec.SchemaProps{ + Ref: ref("github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.InternalPersistentStorageStatus"), + }, + }, + }, + }, + }, + Dependencies: []string{ + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.InternalPersistentStorageStatus", "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.ModelCacheStatus", "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1.SharedStorageStatus", v1.Condition{}.OpenAPIModelName(), v1.Time{}.OpenAPIModelName()}, + } +} + +func schema_pkg_apis_meta_v1_APIGroup(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "APIGroup contains the name, the supported versions, and the preferred version of a group.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "name": { + SchemaProps: spec.SchemaProps{ + Description: "name is the name of the group.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "versions": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "versions are the versions supported in this group.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref(v1.GroupVersionForDiscovery{}.OpenAPIModelName()), + }, + }, + }, + }, + }, + "preferredVersion": { + SchemaProps: spec.SchemaProps{ + Description: "preferredVersion is the version preferred by the API server, which probably is the storage version.", + Default: map[string]interface{}{}, + Ref: ref(v1.GroupVersionForDiscovery{}.OpenAPIModelName()), + }, + }, + "serverAddressByClientCIDRs": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "a map of client CIDR to server address that is serving this group. This is to help clients reach servers in the most network-efficient way possible. Clients can use the appropriate server address as per the CIDR that they match. In case of multiple matches, clients should use the longest matching CIDR. The server returns only those CIDRs that it thinks that the client can match. For example: the master will return an internal IP CIDR only, if the client reaches the server using an internal IP. Server looks at X-Forwarded-For header or X-Real-Ip header or request.RemoteAddr (in that order) to get the client IP.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref(v1.ServerAddressByClientCIDR{}.OpenAPIModelName()), + }, + }, + }, + }, + }, + }, + Required: []string{"name", "versions"}, + }, + }, + Dependencies: []string{ + v1.GroupVersionForDiscovery{}.OpenAPIModelName(), v1.ServerAddressByClientCIDR{}.OpenAPIModelName()}, + } +} + +func schema_pkg_apis_meta_v1_APIGroupList(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "APIGroupList is a list of APIGroup, to allow clients to discover the API at /apis.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "groups": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "groups is a list of APIGroup.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref(v1.APIGroup{}.OpenAPIModelName()), + }, + }, + }, + }, + }, + }, + Required: []string{"groups"}, + }, + }, + Dependencies: []string{ + v1.APIGroup{}.OpenAPIModelName()}, + } +} + +func schema_pkg_apis_meta_v1_APIResource(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "APIResource specifies the name of a resource and whether it is namespaced.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "name": { + SchemaProps: spec.SchemaProps{ + Description: "name is the plural name of the resource.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "singularName": { + SchemaProps: spec.SchemaProps{ + Description: "singularName is the singular name of the resource. This allows clients to handle plural and singular opaquely. The singularName is more correct for reporting status on a single item and both singular and plural are allowed from the kubectl CLI interface.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "namespaced": { + SchemaProps: spec.SchemaProps{ + Description: "namespaced indicates if a resource is namespaced or not.", + Default: false, + Type: []string{"boolean"}, + Format: "", + }, + }, + "group": { + SchemaProps: spec.SchemaProps{ + Description: "group is the preferred group of the resource. Empty implies the group of the containing resource list. For subresources, this may have a different value, for example: Scale\".", + Type: []string{"string"}, + Format: "", + }, + }, + "version": { + SchemaProps: spec.SchemaProps{ + Description: "version is the preferred version of the resource. Empty implies the version of the containing resource list For subresources, this may have a different value, for example: v1 (while inside a v1beta1 version of the core resource's group)\".", + Type: []string{"string"}, + Format: "", + }, + }, + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "kind is the kind for the resource (e.g. 'Foo' is the kind for a resource 'foo')", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "verbs": { + SchemaProps: spec.SchemaProps{ + Description: "verbs is a list of supported kube verbs (this includes get, list, watch, create, update, patch, delete, deletecollection, and proxy)", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "shortNames": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "shortNames is a list of suggested short names of the resource.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "categories": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "categories is a list of the grouped resources this resource belongs to (e.g. 'all')", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "storageVersionHash": { + SchemaProps: spec.SchemaProps{ + Description: "The hash value of the storage version, the version this resource is converted to when written to the data store. Value must be treated as opaque by clients. Only equality comparison on the value is valid. This is an alpha feature and may change or be removed in the future. The field is populated by the apiserver only if the StorageVersionHash feature gate is enabled. This field will remain optional even if it graduates.", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"name", "singularName", "namespaced", "kind", "verbs"}, + }, + }, + } +} + +func schema_pkg_apis_meta_v1_APIResourceList(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "APIResourceList is a list of APIResource, it is used to expose the name of the resources supported in a specific group and version, and if the resource is namespaced.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "groupVersion": { + SchemaProps: spec.SchemaProps{ + Description: "groupVersion is the group and version this APIResourceList is for.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "resources": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "resources contains the name of the resources and if they are namespaced.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref(v1.APIResource{}.OpenAPIModelName()), + }, + }, + }, + }, + }, + }, + Required: []string{"groupVersion", "resources"}, + }, + }, + Dependencies: []string{ + v1.APIResource{}.OpenAPIModelName()}, + } +} + +func schema_pkg_apis_meta_v1_APIVersions(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "APIVersions lists the versions that are available, to allow clients to discover the API at /api, which is the root path of the legacy v1 API.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "versions": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "versions are the api versions that are available.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "serverAddressByClientCIDRs": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "a map of client CIDR to server address that is serving this group. This is to help clients reach servers in the most network-efficient way possible. Clients can use the appropriate server address as per the CIDR that they match. In case of multiple matches, clients should use the longest matching CIDR. The server returns only those CIDRs that it thinks that the client can match. For example: the master will return an internal IP CIDR only, if the client reaches the server using an internal IP. Server looks at X-Forwarded-For header or X-Real-Ip header or request.RemoteAddr (in that order) to get the client IP.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref(v1.ServerAddressByClientCIDR{}.OpenAPIModelName()), + }, + }, + }, + }, + }, + }, + Required: []string{"versions", "serverAddressByClientCIDRs"}, + }, + }, + Dependencies: []string{ + v1.ServerAddressByClientCIDR{}.OpenAPIModelName()}, + } +} + +func schema_pkg_apis_meta_v1_ApplyOptions(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ApplyOptions may be provided when applying an API object. FieldManager is required for apply requests. ApplyOptions is equivalent to PatchOptions. It is provided as a convenience with documentation that speaks specifically to how the options fields relate to apply.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "dryRun": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "force": { + SchemaProps: spec.SchemaProps{ + Description: "Force is going to \"force\" Apply requests. It means user will re-acquire conflicting fields owned by other people.", + Default: false, + Type: []string{"boolean"}, + Format: "", + }, + }, + "fieldManager": { + SchemaProps: spec.SchemaProps{ + Description: "fieldManager is a name associated with the actor or entity that is making these changes. The value must be less than or 128 characters long, and only contain printable characters, as defined by https://golang.org/pkg/unicode/#IsPrint. This field is required.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"force", "fieldManager"}, + }, + }, + } +} + +func schema_pkg_apis_meta_v1_Condition(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "Condition contains details for one aspect of the current state of this API Resource.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "type": { + SchemaProps: spec.SchemaProps{ + Description: "type of condition in CamelCase or in foo.example.com/CamelCase.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "status": { + SchemaProps: spec.SchemaProps{ + Description: "status of the condition, one of True, False, Unknown.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "observedGeneration": { + SchemaProps: spec.SchemaProps{ + Description: "observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance.", + Type: []string{"integer"}, + Format: "int64", + }, + }, + "lastTransitionTime": { + SchemaProps: spec.SchemaProps{ + Description: "lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable.", + Ref: ref(v1.Time{}.OpenAPIModelName()), + }, + }, + "reason": { + SchemaProps: spec.SchemaProps{ + Description: "reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "message": { + SchemaProps: spec.SchemaProps{ + Description: "message is a human readable message indicating details about the transition. This may be an empty string.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"type", "status", "lastTransitionTime", "reason", "message"}, + }, + }, + Dependencies: []string{ + v1.Time{}.OpenAPIModelName()}, + } +} + +func schema_pkg_apis_meta_v1_CreateOptions(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "CreateOptions may be provided when creating an API object.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "dryRun": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "fieldManager": { + SchemaProps: spec.SchemaProps{ + Description: "fieldManager is a name associated with the actor or entity that is making these changes. The value must be less than or 128 characters long, and only contain printable characters, as defined by https://golang.org/pkg/unicode/#IsPrint.", + Type: []string{"string"}, + Format: "", + }, + }, + "fieldValidation": { + SchemaProps: spec.SchemaProps{ + Description: "fieldValidation instructs the server on how to handle objects in the request (POST/PUT/PATCH) containing unknown or duplicate fields. Valid values are: - Ignore: This will ignore any unknown fields that are silently dropped from the object, and will ignore all but the last duplicate field that the decoder encounters. This is the default behavior prior to v1.23. - Warn: This will send a warning via the standard warning response header for each unknown field that is dropped from the object, and for each duplicate field that is encountered. The request will still succeed if there are no other errors, and will only persist the last of any duplicate fields. This is the default in v1.23+ - Strict: This will fail the request with a BadRequest error if any unknown fields would be dropped from the object, or if any duplicate fields are present. The error returned from the server will contain all unknown and duplicate fields encountered.", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + } +} + +func schema_pkg_apis_meta_v1_DeleteOptions(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "DeleteOptions may be provided when deleting an API object.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "gracePeriodSeconds": { + SchemaProps: spec.SchemaProps{ + Description: "The duration in seconds before the object should be deleted. Value must be non-negative integer. The value zero indicates delete immediately. If this value is nil, the default grace period for the specified type will be used. Defaults to a per object value if not specified. zero means delete immediately.", + Type: []string{"integer"}, + Format: "int64", + }, + }, + "preconditions": { + SchemaProps: spec.SchemaProps{ + Description: "Must be fulfilled before a deletion is carried out. If not possible, a 409 Conflict status will be returned.", + Ref: ref(v1.Preconditions{}.OpenAPIModelName()), + }, + }, + "orphanDependents": { + SchemaProps: spec.SchemaProps{ + Description: "Deprecated: please use the PropagationPolicy, this field will be deprecated in 1.7. Should the dependent objects be orphaned. If true/false, the \"orphan\" finalizer will be added to/removed from the object's finalizers list. Either this field or PropagationPolicy may be set, but not both.", + Type: []string{"boolean"}, + Format: "", + }, + }, + "propagationPolicy": { + SchemaProps: spec.SchemaProps{ + Description: "Whether and how garbage collection will be performed. Either this field or OrphanDependents may be set, but not both. The default policy is decided by the existing finalizer set in the metadata.finalizers and the resource-specific default policy. Acceptable values are: 'Orphan' - orphan the dependents; 'Background' - allow the garbage collector to delete the dependents in the background; 'Foreground' - a cascading policy that deletes all dependents in the foreground.", + Type: []string{"string"}, + Format: "", + }, + }, + "dryRun": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "ignoreStoreReadErrorWithClusterBreakingPotential": { + SchemaProps: spec.SchemaProps{ + Description: "if set to true, it will trigger an unsafe deletion of the resource in case the normal deletion flow fails with a corrupt object error. A resource is considered corrupt if it can not be retrieved from the underlying storage successfully because of a) its data can not be transformed e.g. decryption failure, or b) it fails to decode into an object. NOTE: unsafe deletion ignores finalizer constraints, skips precondition checks, and removes the object from the storage. WARNING: This may potentially break the cluster if the workload associated with the resource being unsafe-deleted relies on normal deletion flow. Use only if you REALLY know what you are doing. The default value is false, and the user must opt in to enable it", + Type: []string{"boolean"}, + Format: "", + }, + }, + }, + }, + }, + Dependencies: []string{ + v1.Preconditions{}.OpenAPIModelName()}, + } +} + +func schema_pkg_apis_meta_v1_Duration(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "Duration is a wrapper around time.Duration which supports correct marshaling to YAML and JSON. In particular, it marshals into strings, which can be used as map keys in json.", + Type: v1.Duration{}.OpenAPISchemaType(), + Format: v1.Duration{}.OpenAPISchemaFormat(), + }, + }, + } +} + +func schema_pkg_apis_meta_v1_FieldSelectorRequirement(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "FieldSelectorRequirement is a selector that contains values, a key, and an operator that relates the key and values.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "key": { + SchemaProps: spec.SchemaProps{ + Description: "key is the field selector key that the requirement applies to.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "operator": { + SchemaProps: spec.SchemaProps{ + Description: "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. The list of operators may grow in the future.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "values": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + }, + Required: []string{"key", "operator"}, + }, + }, + } +} + +func schema_pkg_apis_meta_v1_FieldsV1(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "FieldsV1 stores a set of fields in a data structure like a Trie, in JSON format.\n\nEach key is either a '.' representing the field itself, and will always map to an empty set, or a string representing a sub-field or item. The string will follow one of these four formats: 'f:', where is the name of a field in a struct, or key in a map 'v:', where is the exact json formatted value of a list item 'i:', where is position of a item in a list 'k:', where is a map of a list item's key fields to their unique values If a key maps to an empty Fields value, the field that key represents is part of the set.\n\nThe exact format is defined in sigs.k8s.io/structured-merge-diff", + Type: []string{"object"}, + }, + }, + } +} + +func schema_pkg_apis_meta_v1_GetOptions(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "GetOptions is the standard query options to the standard REST get call.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "resourceVersion": { + SchemaProps: spec.SchemaProps{ + Description: "resourceVersion sets a constraint on what resource versions a request may be served from. See https://kubernetes.io/docs/reference/using-api/api-concepts/#resource-versions for details.\n\nDefaults to unset", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + } +} + +func schema_pkg_apis_meta_v1_GroupKind(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "GroupKind specifies a Group and a Kind, but does not force a version. This is useful for identifying concepts during lookup stages without having partially valid types", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "group": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "kind": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"group", "kind"}, + }, + }, + } +} + +func schema_pkg_apis_meta_v1_GroupResource(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "GroupResource specifies a Group and a Resource, but does not force a version. This is useful for identifying concepts during lookup stages without having partially valid types", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "group": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "resource": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"group", "resource"}, + }, + }, + } +} + +func schema_pkg_apis_meta_v1_GroupVersion(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "GroupVersion contains the \"group\" and the \"version\", which uniquely identifies the API.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "group": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "version": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"group", "version"}, + }, + }, + } +} + +func schema_pkg_apis_meta_v1_GroupVersionForDiscovery(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "GroupVersion contains the \"group/version\" and \"version\" string of a version. It is made a struct to keep extensibility.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "groupVersion": { + SchemaProps: spec.SchemaProps{ + Description: "groupVersion specifies the API group and version in the form \"group/version\"", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "version": { + SchemaProps: spec.SchemaProps{ + Description: "version specifies the version in the form of \"version\". This is to save the clients the trouble of splitting the GroupVersion.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"groupVersion", "version"}, + }, + }, + } +} + +func schema_pkg_apis_meta_v1_GroupVersionKind(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "GroupVersionKind unambiguously identifies a kind. It doesn't anonymously include GroupVersion to avoid automatic coercion. It doesn't use a GroupVersion to avoid custom marshalling", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "group": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "version": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "kind": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"group", "version", "kind"}, + }, + }, + } +} + +func schema_pkg_apis_meta_v1_GroupVersionResource(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "GroupVersionResource unambiguously identifies a resource. It doesn't anonymously include GroupVersion to avoid automatic coercion. It doesn't use a GroupVersion to avoid custom marshalling", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "group": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "version": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "resource": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"group", "version", "resource"}, + }, + }, + } +} + +func schema_pkg_apis_meta_v1_InternalEvent(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "InternalEvent makes watch.Event versioned", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "Type": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "Object": { + SchemaProps: spec.SchemaProps{ + Description: "Object is:\n * If Type is Added or Modified: the new state of the object.\n * If Type is Deleted: the state of the object immediately before deletion.\n * If Type is Bookmark: the object (instance of a type being watched) where\n only ResourceVersion field is set. On successful restart of watch from a\n bookmark resourceVersion, client is guaranteed to not get repeat event\n nor miss any events.\n * If Type is Error: *api.Status is recommended; other types may make sense\n depending on context.", + }, + }, + }, + Required: []string{"Type", "Object"}, + }, + }, + } +} + +func schema_pkg_apis_meta_v1_LabelSelector(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "matchLabels": { + SchemaProps: spec.SchemaProps{ + Description: "matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is \"key\", the operator is \"In\", and the values array contains only \"value\". The requirements are ANDed.", + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Allows: true, + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "matchExpressions": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "matchExpressions is a list of label selector requirements. The requirements are ANDed.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref(v1.LabelSelectorRequirement{}.OpenAPIModelName()), + }, + }, + }, + }, + }, + }, + }, + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-map-type": "atomic", + }, + }, + }, + Dependencies: []string{ + v1.LabelSelectorRequirement{}.OpenAPIModelName()}, + } +} + +func schema_pkg_apis_meta_v1_LabelSelectorRequirement(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "key": { + SchemaProps: spec.SchemaProps{ + Description: "key is the label key that the selector applies to.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "operator": { + SchemaProps: spec.SchemaProps{ + Description: "operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "values": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + }, + Required: []string{"key", "operator"}, + }, + }, + } +} + +func schema_pkg_apis_meta_v1_List(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "List holds a list of objects, which may not be known by the server.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Description: "Standard list metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Default: map[string]interface{}{}, + Ref: ref(v1.ListMeta{}.OpenAPIModelName()), + }, + }, + "items": { + SchemaProps: spec.SchemaProps{ + Description: "List of objects", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/apimachinery/pkg/runtime.RawExtension"), + }, + }, + }, + }, + }, + }, + Required: []string{"items"}, + }, + }, + Dependencies: []string{ + v1.ListMeta{}.OpenAPIModelName(), "k8s.io/apimachinery/pkg/runtime.RawExtension"}, + } +} + +func schema_pkg_apis_meta_v1_ListMeta(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ListMeta describes metadata that synthetic resources must have, including lists and various status objects. A resource may have only one of {ObjectMeta, ListMeta}.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "selfLink": { + SchemaProps: spec.SchemaProps{ + Description: "Deprecated: selfLink is a legacy read-only field that is no longer populated by the system.", + Type: []string{"string"}, + Format: "", + }, + }, + "resourceVersion": { + SchemaProps: spec.SchemaProps{ + Description: "String that identifies the server's internal version of this object that can be used by clients to determine when objects have changed. Value must be treated as opaque by clients and passed unmodified back to the server. Populated by the system. Read-only. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency", + Type: []string{"string"}, + Format: "", + }, + }, + "continue": { + SchemaProps: spec.SchemaProps{ + Description: "continue may be set if the user set a limit on the number of items returned, and indicates that the server has more data available. The value is opaque and may be used to issue another request to the endpoint that served this list to retrieve the next set of available objects. Continuing a consistent list may not be possible if the server configuration has changed or more than a few minutes have passed. The resourceVersion field returned when using this continue value will be identical to the value in the first response, unless you have received this token from an error message.", + Type: []string{"string"}, + Format: "", + }, + }, + "remainingItemCount": { + SchemaProps: spec.SchemaProps{ + Description: "remainingItemCount is the number of subsequent items in the list which are not included in this list response. If the list request contained label or field selectors, then the number of remaining items is unknown and the field will be left unset and omitted during serialization. If the list is complete (either because it is not chunking or because this is the last chunk), then there are no more remaining items and this field will be left unset and omitted during serialization. Servers older than v1.15 do not set this field. The intended use of the remainingItemCount is *estimating* the size of a collection. Clients should not rely on the remainingItemCount to be set or to be exact.", + Type: []string{"integer"}, + Format: "int64", + }, + }, + }, + }, + }, + } +} + +func schema_pkg_apis_meta_v1_ListOptions(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ListOptions is the query options to a standard REST list call.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "labelSelector": { + SchemaProps: spec.SchemaProps{ + Description: "A selector to restrict the list of returned objects by their labels. Defaults to everything.", + Type: []string{"string"}, + Format: "", + }, + }, + "fieldSelector": { + SchemaProps: spec.SchemaProps{ + Description: "A selector to restrict the list of returned objects by their fields. Defaults to everything.", + Type: []string{"string"}, + Format: "", + }, + }, + "watch": { + SchemaProps: spec.SchemaProps{ + Description: "Watch for changes to the described resources and return them as a stream of add, update, and remove notifications. Specify resourceVersion.", + Type: []string{"boolean"}, + Format: "", + }, + }, + "allowWatchBookmarks": { + SchemaProps: spec.SchemaProps{ + Description: "allowWatchBookmarks requests watch events with type \"BOOKMARK\". Servers that do not implement bookmarks may ignore this flag and bookmarks are sent at the server's discretion. Clients should not assume bookmarks are returned at any specific interval, nor may they assume the server will send any BOOKMARK event during a session. If this is not a watch, this field is ignored.", + Type: []string{"boolean"}, + Format: "", + }, + }, + "resourceVersion": { + SchemaProps: spec.SchemaProps{ + Description: "resourceVersion sets a constraint on what resource versions a request may be served from. See https://kubernetes.io/docs/reference/using-api/api-concepts/#resource-versions for details.\n\nDefaults to unset", + Type: []string{"string"}, + Format: "", + }, + }, + "resourceVersionMatch": { + SchemaProps: spec.SchemaProps{ + Description: "resourceVersionMatch determines how resourceVersion is applied to list calls. It is highly recommended that resourceVersionMatch be set for list calls where resourceVersion is set See https://kubernetes.io/docs/reference/using-api/api-concepts/#resource-versions for details.\n\nDefaults to unset", + Type: []string{"string"}, + Format: "", + }, + }, + "timeoutSeconds": { + SchemaProps: spec.SchemaProps{ + Description: "Timeout for the list/watch call. This limits the duration of the call, regardless of any activity or inactivity.", + Type: []string{"integer"}, + Format: "int64", + }, + }, + "limit": { + SchemaProps: spec.SchemaProps{ + Description: "limit is a maximum number of responses to return for a list call. If more items exist, the server will set the `continue` field on the list metadata to a value that can be used with the same initial query to retrieve the next set of results. Setting a limit may return fewer than the requested amount of items (up to zero items) in the event all requested objects are filtered out and clients should only use the presence of the continue field to determine whether more results are available. Servers may choose not to support the limit argument and will return all of the available results. If limit is specified and the continue field is empty, clients may assume that no more results are available. This field is not supported if watch is true.\n\nThe server guarantees that the objects returned when using continue will be identical to issuing a single list call without a limit - that is, no objects created, modified, or deleted after the first request is issued will be included in any subsequent continued requests. This is sometimes referred to as a consistent snapshot, and ensures that a client that is using limit to receive smaller chunks of a very large result can ensure they see all possible objects. If objects are updated during a chunked list the version of the object that was present at the time the first list result was calculated is returned.", + Type: []string{"integer"}, + Format: "int64", + }, + }, + "continue": { + SchemaProps: spec.SchemaProps{ + Description: "The continue option should be set when retrieving more results from the server. Since this value is server defined, clients may only use the continue value from a previous query result with identical query parameters (except for the value of continue) and the server may reject a continue value it does not recognize. If the specified continue value is no longer valid whether due to expiration (generally five to fifteen minutes) or a configuration change on the server, the server will respond with a 410 ResourceExpired error together with a continue token. If the client needs a consistent list, it must restart their list without the continue field. Otherwise, the client may send another list request with the token received with the 410 error, the server will respond with a list starting from the next key, but from the latest snapshot, which is inconsistent from the previous list results - objects that are created, modified, or deleted after the first list request will be included in the response, as long as their keys are after the \"next key\".\n\nThis field is not supported when watch is true. Clients may start a watch from the last resourceVersion value returned by the server and not miss any modifications.", + Type: []string{"string"}, + Format: "", + }, + }, + "sendInitialEvents": { + SchemaProps: spec.SchemaProps{ + Description: "`sendInitialEvents=true` may be set together with `watch=true`. In that case, the watch stream will begin with synthetic events to produce the current state of objects in the collection. Once all such events have been sent, a synthetic \"Bookmark\" event will be sent. The bookmark will report the ResourceVersion (RV) corresponding to the set of objects, and be marked with `\"k8s.io/initial-events-end\": \"true\"` annotation. Afterwards, the watch stream will proceed as usual, sending watch events corresponding to changes (subsequent to the RV) to objects watched.\n\nWhen `sendInitialEvents` option is set, we require `resourceVersionMatch` option to also be set. The semantic of the watch request is as following: - `resourceVersionMatch` = NotOlderThan\n is interpreted as \"data at least as new as the provided `resourceVersion`\"\n and the bookmark event is send when the state is synced\n to a `resourceVersion` at least as fresh as the one provided by the ListOptions.\n If `resourceVersion` is unset, this is interpreted as \"consistent read\" and the\n bookmark event is send when the state is synced at least to the moment\n when request started being processed.\n- `resourceVersionMatch` set to any other value or unset\n Invalid error is returned.\n\nDefaults to true if `resourceVersion=\"\"` or `resourceVersion=\"0\"` (for backward compatibility reasons) and to false otherwise.", + Type: []string{"boolean"}, + Format: "", + }, + }, + }, + }, + }, + } +} + +func schema_pkg_apis_meta_v1_ManagedFieldsEntry(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ManagedFieldsEntry is a workflow-id, a FieldSet and the group version of the resource that the fieldset applies to.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "manager": { + SchemaProps: spec.SchemaProps{ + Description: "Manager is an identifier of the workflow managing these fields.", + Type: []string{"string"}, + Format: "", + }, + }, + "operation": { + SchemaProps: spec.SchemaProps{ + Description: "Operation is the type of operation which lead to this ManagedFieldsEntry being created. The only valid values for this field are 'Apply' and 'Update'.", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the version of this resource that this field set applies to. The format is \"group/version\" just like the top-level APIVersion field. It is necessary to track the version of a field set because it cannot be automatically converted.", + Type: []string{"string"}, + Format: "", + }, + }, + "time": { + SchemaProps: spec.SchemaProps{ + Description: "Time is the timestamp of when the ManagedFields entry was added. The timestamp will also be updated if a field is added, the manager changes any of the owned fields value or removes a field. The timestamp does not update when a field is removed from the entry because another manager took it over.", + Ref: ref(v1.Time{}.OpenAPIModelName()), + }, + }, + "fieldsType": { + SchemaProps: spec.SchemaProps{ + Description: "FieldsType is the discriminator for the different fields format and version. There is currently only one possible value: \"FieldsV1\"", + Type: []string{"string"}, + Format: "", + }, + }, + "fieldsV1": { + SchemaProps: spec.SchemaProps{ + Description: "FieldsV1 holds the first JSON version format as described in the \"FieldsV1\" type.", + Ref: ref(v1.FieldsV1{}.OpenAPIModelName()), + }, + }, + "subresource": { + SchemaProps: spec.SchemaProps{ + Description: "Subresource is the name of the subresource used to update that object, or empty string if the object was updated through the main resource. The value of this field is used to distinguish between managers, even if they share the same name. For example, a status update will be distinct from a regular update using the same manager name. Note that the APIVersion field is not related to the Subresource field and it always corresponds to the version of the main resource.", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + Dependencies: []string{ + v1.FieldsV1{}.OpenAPIModelName(), v1.Time{}.OpenAPIModelName()}, + } +} + +func schema_pkg_apis_meta_v1_MicroTime(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "MicroTime is version of Time with microsecond level precision.", + Type: v1.MicroTime{}.OpenAPISchemaType(), + Format: v1.MicroTime{}.OpenAPISchemaFormat(), + }, + }, + } +} + +func schema_pkg_apis_meta_v1_ObjectMeta(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ObjectMeta is metadata that all persisted resources must have, which includes all objects users must create.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "name": { + SchemaProps: spec.SchemaProps{ + Description: "Name must be unique within a namespace. Is required when creating resources, although some resources may allow a client to request the generation of an appropriate name automatically. Name is primarily intended for creation idempotence and configuration definition. Cannot be updated. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names#names", + Type: []string{"string"}, + Format: "", + }, + }, + "generateName": { + SchemaProps: spec.SchemaProps{ + Description: "GenerateName is an optional prefix, used by the server, to generate a unique name ONLY IF the Name field has not been provided. If this field is used, the name returned to the client will be different than the name passed. This value will also be combined with a unique suffix. The provided value has the same validation rules as the Name field, and may be truncated by the length of the suffix required to make the value unique on the server.\n\nIf this field is specified and the generated name exists, the server will return a 409.\n\nApplied only if Name is not specified. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#idempotency", + Type: []string{"string"}, + Format: "", + }, + }, + "namespace": { + SchemaProps: spec.SchemaProps{ + Description: "Namespace defines the space within which each name must be unique. An empty namespace is equivalent to the \"default\" namespace, but \"default\" is the canonical representation. Not all objects are required to be scoped to a namespace - the value of this field for those objects will be empty.\n\nMust be a DNS_LABEL. Cannot be updated. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces", + Type: []string{"string"}, + Format: "", + }, + }, + "selfLink": { + SchemaProps: spec.SchemaProps{ + Description: "Deprecated: selfLink is a legacy read-only field that is no longer populated by the system.", + Type: []string{"string"}, + Format: "", + }, + }, + "uid": { + SchemaProps: spec.SchemaProps{ + Description: "UID is the unique in time and space value for this object. It is typically generated by the server on successful creation of a resource and is not allowed to change on PUT operations.\n\nPopulated by the system. Read-only. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names#uids", + Type: []string{"string"}, + Format: "", + }, + }, + "resourceVersion": { + SchemaProps: spec.SchemaProps{ + Description: "An opaque value that represents the internal version of this object that can be used by clients to determine when objects have changed. May be used for optimistic concurrency, change detection, and the watch operation on a resource or set of resources. Clients must treat these values as opaque and passed unmodified back to the server. They may only be valid for a particular resource or set of resources.\n\nPopulated by the system. Read-only. Value must be treated as opaque by clients and . More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency", + Type: []string{"string"}, + Format: "", + }, + }, + "generation": { + SchemaProps: spec.SchemaProps{ + Description: "A sequence number representing a specific generation of the desired state. Populated by the system. Read-only.", + Type: []string{"integer"}, + Format: "int64", + }, + }, + "creationTimestamp": { + SchemaProps: spec.SchemaProps{ + Description: "CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC.\n\nPopulated by the system. Read-only. Null for lists. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata", + Ref: ref(v1.Time{}.OpenAPIModelName()), + }, + }, + "deletionTimestamp": { + SchemaProps: spec.SchemaProps{ + Description: "DeletionTimestamp is RFC 3339 date and time at which this resource will be deleted. This field is set by the server when a graceful deletion is requested by the user, and is not directly settable by a client. The resource is expected to be deleted (no longer visible from resource lists, and not reachable by name) after the time in this field, once the finalizers list is empty. As long as the finalizers list contains items, deletion is blocked. Once the deletionTimestamp is set, this value may not be unset or be set further into the future, although it may be shortened or the resource may be deleted prior to this time. For example, a user may request that a pod is deleted in 30 seconds. The Kubelet will react by sending a graceful termination signal to the containers in the pod. After that 30 seconds, the Kubelet will send a hard termination signal (SIGKILL) to the container and after cleanup, remove the pod from the API. In the presence of network partitions, this object may still exist after this timestamp, until an administrator or automated process can determine the resource is fully terminated. If not set, graceful deletion of the object has not been requested.\n\nPopulated by the system when a graceful deletion is requested. Read-only. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata", + Ref: ref(v1.Time{}.OpenAPIModelName()), + }, + }, + "deletionGracePeriodSeconds": { + SchemaProps: spec.SchemaProps{ + Description: "Number of seconds allowed for this object to gracefully terminate before it will be removed from the system. Only set when deletionTimestamp is also set. May only be shortened. Read-only.", + Type: []string{"integer"}, + Format: "int64", + }, + }, + "labels": { + SchemaProps: spec.SchemaProps{ + Description: "Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels", + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Allows: true, + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "annotations": { + SchemaProps: spec.SchemaProps{ + Description: "Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations", + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Allows: true, + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "ownerReferences": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "uid", + }, + "x-kubernetes-list-type": "map", + "x-kubernetes-patch-merge-key": "uid", + "x-kubernetes-patch-strategy": "merge", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "List of objects depended by this object. If ALL objects in the list have been deleted, this object will be garbage collected. If this object is managed by a controller, then an entry in this list will point to this controller, with the controller field set to true. There cannot be more than one managing controller.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref(v1.OwnerReference{}.OpenAPIModelName()), + }, + }, + }, + }, + }, + "finalizers": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "set", + "x-kubernetes-patch-strategy": "merge", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "Must be empty before the object is deleted from the registry. Each entry is an identifier for the responsible component that will remove the entry from the list. If the deletionTimestamp of the object is non-nil, entries in this list can only be removed. Finalizers may be processed and removed in any order. Order is NOT enforced because it introduces significant risk of stuck finalizers. finalizers is a shared field, any actor with permission can reorder it. If the finalizer list is processed in order, then this can lead to a situation in which the component responsible for the first finalizer in the list is waiting for a signal (field value, external system, or other) produced by a component responsible for a finalizer later in the list, resulting in a deadlock. Without enforced ordering finalizers are free to order amongst themselves and are not vulnerable to ordering changes in the list.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "managedFields": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "ManagedFields maps workflow-id and version to the set of fields that are managed by that workflow. This is mostly for internal housekeeping, and users typically shouldn't need to set or understand this field. A workflow can be the user's name, a controller's name, or the name of a specific apply path like \"ci-cd\". The set of fields is always in the version that the workflow used when modifying the object.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref(v1.ManagedFieldsEntry{}.OpenAPIModelName()), + }, + }, + }, + }, + }, + }, + }, + }, + Dependencies: []string{ + v1.ManagedFieldsEntry{}.OpenAPIModelName(), v1.OwnerReference{}.OpenAPIModelName(), v1.Time{}.OpenAPIModelName()}, + } +} + +func schema_pkg_apis_meta_v1_OwnerReference(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "OwnerReference contains enough information to let you identify an owning object. An owning object must be in the same namespace as the dependent, or be cluster-scoped, so there is no namespace field.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "API version of the referent.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "name": { + SchemaProps: spec.SchemaProps{ + Description: "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names#names", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "uid": { + SchemaProps: spec.SchemaProps{ + Description: "UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names#uids", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "controller": { + SchemaProps: spec.SchemaProps{ + Description: "If true, this reference points to the managing controller.", + Type: []string{"boolean"}, + Format: "", + }, + }, + "blockOwnerDeletion": { + SchemaProps: spec.SchemaProps{ + Description: "If true, AND if the owner has the \"foregroundDeletion\" finalizer, then the owner cannot be deleted from the key-value store until this reference is removed. See https://kubernetes.io/docs/concepts/architecture/garbage-collection/#foreground-deletion for how the garbage collector interacts with this field and enforces the foreground deletion. Defaults to false. To set this field, a user needs \"delete\" permission of the owner, otherwise 422 (Unprocessable Entity) will be returned.", + Type: []string{"boolean"}, + Format: "", + }, + }, + }, + Required: []string{"apiVersion", "kind", "name", "uid"}, + }, + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-map-type": "atomic", + }, + }, + }, + } +} + +func schema_pkg_apis_meta_v1_PartialObjectMetadata(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "PartialObjectMetadata is a generic representation of any object with ObjectMeta. It allows clients to get access to a particular ObjectMeta schema without knowing the details of the version.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Description: "Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata", + Default: map[string]interface{}{}, + Ref: ref(v1.ObjectMeta{}.OpenAPIModelName()), + }, + }, + }, + }, + }, + Dependencies: []string{ + v1.ObjectMeta{}.OpenAPIModelName()}, + } +} + +func schema_pkg_apis_meta_v1_PartialObjectMetadataList(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "PartialObjectMetadataList contains a list of objects containing only their metadata", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Description: "Standard list metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Default: map[string]interface{}{}, + Ref: ref(v1.ListMeta{}.OpenAPIModelName()), + }, + }, + "items": { + SchemaProps: spec.SchemaProps{ + Description: "items contains each of the included items.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref(v1.PartialObjectMetadata{}.OpenAPIModelName()), + }, + }, + }, + }, + }, + }, + Required: []string{"items"}, + }, + }, + Dependencies: []string{ + v1.ListMeta{}.OpenAPIModelName(), v1.PartialObjectMetadata{}.OpenAPIModelName()}, + } +} + +func schema_pkg_apis_meta_v1_Patch(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "Patch is provided to give a concrete name and type to the Kubernetes PATCH request body.", + Type: []string{"object"}, + }, + }, + } +} + +func schema_pkg_apis_meta_v1_PatchOptions(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "PatchOptions may be provided when patching an API object. PatchOptions is meant to be a superset of UpdateOptions.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "dryRun": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "force": { + SchemaProps: spec.SchemaProps{ + Description: "Force is going to \"force\" Apply requests. It means user will re-acquire conflicting fields owned by other people. Force flag must be unset for non-apply patch requests.", + Type: []string{"boolean"}, + Format: "", + }, + }, + "fieldManager": { + SchemaProps: spec.SchemaProps{ + Description: "fieldManager is a name associated with the actor or entity that is making these changes. The value must be less than or 128 characters long, and only contain printable characters, as defined by https://golang.org/pkg/unicode/#IsPrint. This field is required for apply requests (application/apply-patch) but optional for non-apply patch types (JsonPatch, MergePatch, StrategicMergePatch).", + Type: []string{"string"}, + Format: "", + }, + }, + "fieldValidation": { + SchemaProps: spec.SchemaProps{ + Description: "fieldValidation instructs the server on how to handle objects in the request (POST/PUT/PATCH) containing unknown or duplicate fields. Valid values are: - Ignore: This will ignore any unknown fields that are silently dropped from the object, and will ignore all but the last duplicate field that the decoder encounters. This is the default behavior prior to v1.23. - Warn: This will send a warning via the standard warning response header for each unknown field that is dropped from the object, and for each duplicate field that is encountered. The request will still succeed if there are no other errors, and will only persist the last of any duplicate fields. This is the default in v1.23+ - Strict: This will fail the request with a BadRequest error if any unknown fields would be dropped from the object, or if any duplicate fields are present. The error returned from the server will contain all unknown and duplicate fields encountered.", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + } +} + +func schema_pkg_apis_meta_v1_Preconditions(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "Preconditions must be fulfilled before an operation (update, delete, etc.) is carried out.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "uid": { + SchemaProps: spec.SchemaProps{ + Description: "Specifies the target UID.", + Type: []string{"string"}, + Format: "", + }, + }, + "resourceVersion": { + SchemaProps: spec.SchemaProps{ + Description: "Specifies the target ResourceVersion", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + } +} + +func schema_pkg_apis_meta_v1_RootPaths(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "RootPaths lists the paths available at root. For example: \"/healthz\", \"/apis\".", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "paths": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "paths are the paths available at root.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + }, + Required: []string{"paths"}, + }, + }, + } +} + +func schema_pkg_apis_meta_v1_ServerAddressByClientCIDR(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ServerAddressByClientCIDR helps the client to determine the server address that they should use, depending on the clientCIDR that they match.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "clientCIDR": { + SchemaProps: spec.SchemaProps{ + Description: "The CIDR with which clients can match their IP to figure out the server address that they should use.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "serverAddress": { + SchemaProps: spec.SchemaProps{ + Description: "Address of this server, suitable for a client that matches the above CIDR. This can be a hostname, hostname:port, IP or IP:port.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"clientCIDR", "serverAddress"}, + }, + }, + } +} + +func schema_pkg_apis_meta_v1_Status(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "Status is a return value for calls that don't return other objects.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Description: "Standard list metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Default: map[string]interface{}{}, + Ref: ref(v1.ListMeta{}.OpenAPIModelName()), + }, + }, + "status": { + SchemaProps: spec.SchemaProps{ + Description: "Status of the operation. One of: \"Success\" or \"Failure\". More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status", + Type: []string{"string"}, + Format: "", + }, + }, + "message": { + SchemaProps: spec.SchemaProps{ + Description: "A human-readable description of the status of this operation.", + Type: []string{"string"}, + Format: "", + }, + }, + "reason": { + SchemaProps: spec.SchemaProps{ + Description: "A machine-readable description of why this operation is in the \"Failure\" status. If this value is empty there is no information available. A Reason clarifies an HTTP status code but does not override it.", + Type: []string{"string"}, + Format: "", + }, + }, + "details": { + SchemaProps: spec.SchemaProps{ + Description: "Extended data associated with the reason. Each reason may define its own extended details. This field is optional and the data returned is not guaranteed to conform to any schema except that defined by the reason type.", + Ref: ref(v1.StatusDetails{}.OpenAPIModelName()), + }, + }, + "code": { + SchemaProps: spec.SchemaProps{ + Description: "Suggested HTTP return code for this status, 0 if not set.", + Type: []string{"integer"}, + Format: "int32", + }, + }, + }, + }, + }, + Dependencies: []string{ + v1.ListMeta{}.OpenAPIModelName(), v1.StatusDetails{}.OpenAPIModelName()}, + } +} + +func schema_pkg_apis_meta_v1_StatusCause(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "StatusCause provides more information about an api.Status failure, including cases when multiple errors are encountered.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "reason": { + SchemaProps: spec.SchemaProps{ + Description: "A machine-readable description of the cause of the error. If this value is empty there is no information available.", + Type: []string{"string"}, + Format: "", + }, + }, + "message": { + SchemaProps: spec.SchemaProps{ + Description: "A human-readable description of the cause of the error. This field may be presented as-is to a reader.", + Type: []string{"string"}, + Format: "", + }, + }, + "field": { + SchemaProps: spec.SchemaProps{ + Description: "The field of the resource that has caused this error, as named by its JSON serialization. May include dot and postfix notation for nested attributes. Arrays are zero-indexed. Fields may appear more than once in an array of causes due to fields having multiple errors. Optional.\n\nExamples:\n \"name\" - the field \"name\" on the current resource\n \"items[0].name\" - the field \"name\" on the first array entry in \"items\"", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + } +} + +func schema_pkg_apis_meta_v1_StatusDetails(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "StatusDetails is a set of additional properties that MAY be set by the server to provide additional information about a response. The Reason field of a Status object defines what attributes will be set. Clients must ignore fields that do not match the defined type of each attribute, and should assume that any attribute may be empty, invalid, or under defined.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "name": { + SchemaProps: spec.SchemaProps{ + Description: "The name attribute of the resource associated with the status StatusReason (when there is a single name which can be described).", + Type: []string{"string"}, + Format: "", + }, + }, + "group": { + SchemaProps: spec.SchemaProps{ + Description: "The group attribute of the resource associated with the status StatusReason.", + Type: []string{"string"}, + Format: "", + }, + }, + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "The kind attribute of the resource associated with the status StatusReason. On some operations may differ from the requested resource Kind. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "uid": { + SchemaProps: spec.SchemaProps{ + Description: "UID of the resource. (when there is a single resource which can be described). More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names#uids", + Type: []string{"string"}, + Format: "", + }, + }, + "causes": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "The Causes array includes more details associated with the StatusReason failure. Not all StatusReasons may provide detailed causes.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref(v1.StatusCause{}.OpenAPIModelName()), + }, + }, + }, + }, + }, + "retryAfterSeconds": { + SchemaProps: spec.SchemaProps{ + Description: "If specified, the time in seconds before the operation should be retried. Some errors may indicate the client must take an alternate action - for those errors this field may indicate how long to wait before taking the alternate action.", + Type: []string{"integer"}, + Format: "int32", + }, + }, + }, + }, + }, + Dependencies: []string{ + v1.StatusCause{}.OpenAPIModelName()}, + } +} + +func schema_pkg_apis_meta_v1_Table(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "Table is a tabular representation of a set of API resources. The server transforms the object into a set of preferred columns for quickly reviewing the objects.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Description: "Standard list metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Default: map[string]interface{}{}, + Ref: ref(v1.ListMeta{}.OpenAPIModelName()), + }, + }, + "columnDefinitions": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "columnDefinitions describes each column in the returned items array. The number of cells per row will always match the number of column definitions.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref(v1.TableColumnDefinition{}.OpenAPIModelName()), + }, + }, + }, + }, + }, + "rows": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "rows is the list of items in the table.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref(v1.TableRow{}.OpenAPIModelName()), + }, + }, + }, + }, + }, + }, + Required: []string{"columnDefinitions", "rows"}, + }, + }, + Dependencies: []string{ + v1.ListMeta{}.OpenAPIModelName(), v1.TableColumnDefinition{}.OpenAPIModelName(), v1.TableRow{}.OpenAPIModelName()}, + } +} + +func schema_pkg_apis_meta_v1_TableColumnDefinition(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "TableColumnDefinition contains information about a column returned in the Table.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "name": { + SchemaProps: spec.SchemaProps{ + Description: "name is a human readable name for the column.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "type": { + SchemaProps: spec.SchemaProps{ + Description: "type is an OpenAPI type definition for this column, such as number, integer, string, or array. See https://github.com/OAI/OpenAPI-Specification/blob/master/versions/2.0.md#data-types for more.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "format": { + SchemaProps: spec.SchemaProps{ + Description: "format is an optional OpenAPI type modifier for this column. A format modifies the type and imposes additional rules, like date or time formatting for a string. The 'name' format is applied to the primary identifier column which has type 'string' to assist in clients identifying column is the resource name. See https://github.com/OAI/OpenAPI-Specification/blob/master/versions/2.0.md#data-types for more.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "description": { + SchemaProps: spec.SchemaProps{ + Description: "description is a human readable description of this column.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "priority": { + SchemaProps: spec.SchemaProps{ + Description: "priority is an integer defining the relative importance of this column compared to others. Lower numbers are considered higher priority. Columns that may be omitted in limited space scenarios should be given a higher priority.", + Default: 0, + Type: []string{"integer"}, + Format: "int32", + }, + }, + }, + Required: []string{"name", "type", "format", "description", "priority"}, + }, + }, + } +} + +func schema_pkg_apis_meta_v1_TableOptions(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "TableOptions are used when a Table is requested by the caller.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "includeObject": { + SchemaProps: spec.SchemaProps{ + Description: "includeObject decides whether to include each object along with its columnar information. Specifying \"None\" will return no object, specifying \"Object\" will return the full object contents, and specifying \"Metadata\" (the default) will return the object's metadata in the PartialObjectMetadata kind in version v1beta1 of the meta.k8s.io API group.", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + } +} + +func schema_pkg_apis_meta_v1_TableRow(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "TableRow is an individual row in a table.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "cells": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "cells will be as wide as the column definitions array and may contain strings, numbers (float64 or int64), booleans, simple maps, lists, or null. See the type field of the column definition for a more detailed description.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Format: "", + }, + }, + }, + }, + }, + "conditions": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "conditions describe additional status of a row that are relevant for a human user. These conditions apply to the row, not to the object, and will be specific to table output. The only defined condition type is 'Completed', for a row that indicates a resource that has run to completion and can be given less visual priority.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref(v1.TableRowCondition{}.OpenAPIModelName()), + }, + }, + }, + }, + }, + "object": { + SchemaProps: spec.SchemaProps{ + Description: "This field contains the requested additional information about each object based on the includeObject policy when requesting the Table. If \"None\", this field is empty, if \"Object\" this will be the default serialization of the object for the current API version, and if \"Metadata\" (the default) will contain the object metadata. Check the returned kind and apiVersion of the object before parsing. The media type of the object will always match the enclosing list - if this as a JSON table, these will be JSON encoded objects.", + Ref: ref("k8s.io/apimachinery/pkg/runtime.RawExtension"), + }, + }, + }, + Required: []string{"cells"}, + }, + }, + Dependencies: []string{ + v1.TableRowCondition{}.OpenAPIModelName(), "k8s.io/apimachinery/pkg/runtime.RawExtension"}, + } +} + +func schema_pkg_apis_meta_v1_TableRowCondition(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "TableRowCondition allows a row to be marked with additional information.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "type": { + SchemaProps: spec.SchemaProps{ + Description: "Type of row condition. The only defined value is 'Completed' indicating that the object this row represents has reached a completed state and may be given less visual priority than other rows. Clients are not required to honor any conditions but should be consistent where possible about handling the conditions.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "status": { + SchemaProps: spec.SchemaProps{ + Description: "Status of the condition, one of True, False, Unknown.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "reason": { + SchemaProps: spec.SchemaProps{ + Description: "(brief) machine readable reason for the condition's last transition.", + Type: []string{"string"}, + Format: "", + }, + }, + "message": { + SchemaProps: spec.SchemaProps{ + Description: "Human readable message indicating details about last transition.", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"type", "status"}, + }, + }, + } +} + +func schema_pkg_apis_meta_v1_Time(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "Time is a wrapper around time.Time which supports correct marshaling to YAML and JSON. Wrappers are provided for many of the factory methods that the time package offers.", + Type: v1.Time{}.OpenAPISchemaType(), + Format: v1.Time{}.OpenAPISchemaFormat(), + }, + }, + } +} + +func schema_pkg_apis_meta_v1_Timestamp(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "Timestamp is a struct that is equivalent to Time, but intended for protobuf marshalling/unmarshalling. It is generated into a serialization that matches Time. Do not use in Go structs.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "seconds": { + SchemaProps: spec.SchemaProps{ + Description: "Represents seconds of UTC time since Unix epoch 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to 9999-12-31T23:59:59Z inclusive.", + Default: 0, + Type: []string{"integer"}, + Format: "int64", + }, + }, + "nanos": { + SchemaProps: spec.SchemaProps{ + Description: "Non-negative fractions of a second at nanosecond resolution. Negative second values with fractions must still have non-negative nanos values that count forward in time. Must be from 0 to 999,999,999 inclusive. This field may be limited in precision depending on context.", + Default: 0, + Type: []string{"integer"}, + Format: "int32", + }, + }, + }, + Required: []string{"seconds", "nanos"}, + }, + }, + } +} + +func schema_pkg_apis_meta_v1_TypeMeta(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "TypeMeta describes an individual object in an API response or request with strings representing the type of the object and its API schema version. Structures that are versioned or persisted should inline TypeMeta.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + } +} + +func schema_pkg_apis_meta_v1_UpdateOptions(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "UpdateOptions may be provided when updating an API object. All fields in UpdateOptions should also be present in PatchOptions.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "dryRun": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "fieldManager": { + SchemaProps: spec.SchemaProps{ + Description: "fieldManager is a name associated with the actor or entity that is making these changes. The value must be less than or 128 characters long, and only contain printable characters, as defined by https://golang.org/pkg/unicode/#IsPrint.", + Type: []string{"string"}, + Format: "", + }, + }, + "fieldValidation": { + SchemaProps: spec.SchemaProps{ + Description: "fieldValidation instructs the server on how to handle objects in the request (POST/PUT/PATCH) containing unknown or duplicate fields. Valid values are: - Ignore: This will ignore any unknown fields that are silently dropped from the object, and will ignore all but the last duplicate field that the decoder encounters. This is the default behavior prior to v1.23. - Warn: This will send a warning via the standard warning response header for each unknown field that is dropped from the object, and for each duplicate field that is encountered. The request will still succeed if there are no other errors, and will only persist the last of any duplicate fields. This is the default in v1.23+ - Strict: This will fail the request with a BadRequest error if any unknown fields would be dropped from the object, or if any duplicate fields are present. The error returned from the server will contain all unknown and duplicate fields encountered.", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + } +} + +func schema_pkg_apis_meta_v1_WatchEvent(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "Event represents a single event to a watched resource.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "type": { + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "object": { + SchemaProps: spec.SchemaProps{ + Description: "Object is:\n * If Type is Added or Modified: the new state of the object.\n * If Type is Deleted: the state of the object immediately before deletion.\n * If Type is Error: *Status is recommended; other types may make sense\n depending on context.", + Ref: ref("k8s.io/apimachinery/pkg/runtime.RawExtension"), + }, + }, + }, + Required: []string{"type", "object"}, + }, + }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/runtime.RawExtension"}, + } +} diff --git a/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1/modelcachebinding_types.go b/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1/modelcachebinding_types.go new file mode 100644 index 000000000..62eba5500 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1/modelcachebinding_types.go @@ -0,0 +1,172 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v2beta1 + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +const ( + // ModelCacheBindingFinalizer protects provider resources while a binding is in use or retiring. + ModelCacheBindingFinalizer = "nvca.nvcf.nvidia.io/model-cache-binding-finalizer" +) + +// ModelCacheWorkflow identifies the model-cache workflow sharing a binding. +type ModelCacheWorkflow string + +const ( + // ModelCacheWorkflowRegular identifies the regular container model-cache workflow. + ModelCacheWorkflowRegular ModelCacheWorkflow = "regularModelCache" + // ModelCacheWorkflowHelm identifies the Helm model-cache workflow. + ModelCacheWorkflowHelm ModelCacheWorkflow = "helmModelCache" +) + +// ModelCacheBindingPhase is the lifecycle state of a cache binding. +type ModelCacheBindingPhase string + +const ( + // ModelCacheBindingPhaseActive accepts new request references. + ModelCacheBindingPhaseActive ModelCacheBindingPhase = "Active" + // ModelCacheBindingPhaseRetiring rejects new references while resources are released. + ModelCacheBindingPhaseRetiring ModelCacheBindingPhase = "Retiring" +) + +// ModelCachePopulationState is the state of the shared cache data identity. +type ModelCachePopulationState string + +const ( + // ModelCachePopulationPending means population has not started. + ModelCachePopulationPending ModelCachePopulationState = "Pending" + // ModelCachePopulationPopulating means a writer is populating the cache. + ModelCachePopulationPopulating ModelCachePopulationState = "Populating" + // ModelCachePopulationReady means the cache is ready for readers. + ModelCachePopulationReady ModelCachePopulationState = "Ready" + // ModelCachePopulationFailed means the latest population attempt failed. + ModelCachePopulationFailed ModelCachePopulationState = "Failed" +) + +// ModelCacheBinding records one immutable provider decision for a shared model-cache key. +// +genclient +// +k8s:openapi-gen=true +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type ModelCacheBinding struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ModelCacheBindingSpec `json:"spec"` + Status ModelCacheBindingStatus `json:"status,omitempty"` +} + +// ModelCacheBindingSpec contains the immutable identity, provider decision, and resource intent. +// +k8s:openapi-gen=true +type ModelCacheBindingSpec struct { + Identity ModelCacheBindingIdentity `json:"identity"` + Decision ModelCacheBindingDecision `json:"decision"` + StorageClass ModelCacheStorageClassSnapshot `json:"storageClass"` + Resources ModelCacheBindingResourceIntent `json:"resources"` +} + +// ModelCacheBindingIdentity is the stable cache key represented by a binding. +// +k8s:openapi-gen=true +type ModelCacheBindingIdentity struct { + Version string `json:"version"` + Workflow ModelCacheWorkflow `json:"workflow"` + SharingDomainDigest string `json:"sharingDomainDigest"` + CacheHandleDigest string `json:"cacheHandleDigest"` +} + +// ModelCacheBindingDecision snapshots the selected provider transition. +// +k8s:openapi-gen=true +type ModelCacheBindingDecision struct { + Provider string `json:"provider"` + Provisioner string `json:"provisioner"` + Transition string `json:"transition"` + // +listType=set + RequiredAccessModes []corev1.PersistentVolumeAccessMode `json:"requiredAccessModes"` + // +listType=atomic + RequiredMountOptions []string `json:"requiredMountOptions,omitempty"` + CatalogDigest string `json:"catalogDigest"` + EncryptionRequired bool `json:"encryptionRequired"` +} + +// ModelCacheStorageClassSnapshot identifies the exact retained StorageClass selected by the binding. +// +k8s:openapi-gen=true +type ModelCacheStorageClassSnapshot struct { + Name string `json:"name"` + UID types.UID `json:"uid"` + ReclaimPolicy corev1.PersistentVolumeReclaimPolicy `json:"reclaimPolicy"` + ConfigurationDigest string `json:"configurationDigest"` +} + +// ModelCacheBindingResourceIntent records deterministic names for shared resources. +// +k8s:openapi-gen=true +type ModelCacheBindingResourceIntent struct { + WriterNamespace string `json:"writerNamespace"` + // +listType=set + PersistentVolumeClaimNames []string `json:"persistentVolumeClaimNames,omitempty"` + // +listType=set + PersistentVolumeNames []string `json:"persistentVolumeNames,omitempty"` + // +listType=set + JobNames []string `json:"jobNames,omitempty"` + // +listType=set + StorageClassNames []string `json:"storageClassNames,omitempty"` + // +listType=set + SecretNames []string `json:"secretNames,omitempty"` + LeaseName string `json:"leaseName,omitempty"` +} + +// ModelCacheBindingStatus contains mutable lifecycle and realized-resource state. +// +k8s:openapi-gen=true +type ModelCacheBindingStatus struct { + Phase ModelCacheBindingPhase `json:"phase,omitempty"` + LastPhaseTransitionTime *metav1.Time `json:"lastPhaseTransitionTime,omitempty"` + // +listType=map + // +listMapKey=uid + RequestReferences []ModelCacheBindingRequestReference `json:"requestReferences,omitempty"` + Realized *ModelCacheBindingRealizedState `json:"realized,omitempty"` + // +listType=map + // +listMapKey=type + Conditions []metav1.Condition `json:"conditions,omitempty"` +} + +// ModelCacheBindingRequestReference identifies a request currently using a binding. +// +k8s:openapi-gen=true +type ModelCacheBindingRequestReference struct { + Namespace string `json:"namespace"` + Name string `json:"name"` + UID types.UID `json:"uid"` +} + +// ModelCacheBindingRealizedState records provider resources only after they exist. +// +k8s:openapi-gen=true +type ModelCacheBindingRealizedState struct { + BoundPersistentVolumeName string `json:"boundPersistentVolumeName,omitempty"` + ProviderDataIdentity string `json:"providerDataIdentity,omitempty"` + PopulationState ModelCachePopulationState `json:"populationState,omitempty"` +} + +// ModelCacheBindingList is a list of ModelCacheBinding objects. +// +k8s:openapi-gen=true +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +type ModelCacheBindingList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []ModelCacheBinding `json:"items"` +} diff --git a/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1/modelcachebinding_types_test.go b/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1/modelcachebinding_types_test.go new file mode 100644 index 000000000..794d4b660 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1/modelcachebinding_types_test.go @@ -0,0 +1,114 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v2beta1 + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/kube-openapi/pkg/validation/spec" +) + +func TestModelCacheBindingSchemeRegistration(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, AddToScheme(scheme)) + + gvks, unversioned, err := scheme.ObjectKinds(&ModelCacheBinding{}) + require.NoError(t, err) + assert.False(t, unversioned) + assert.Contains(t, gvks, SchemeGroupVersion.WithKind("ModelCacheBinding")) + + gvks, unversioned, err = scheme.ObjectKinds(&ModelCacheBindingList{}) + require.NoError(t, err) + assert.False(t, unversioned) + assert.Contains(t, gvks, SchemeGroupVersion.WithKind("ModelCacheBindingList")) +} + +func TestModelCacheBindingDeepCopy(t *testing.T) { + now := metav1.Now() + original := &ModelCacheBinding{ + Spec: ModelCacheBindingSpec{ + Decision: ModelCacheBindingDecision{ + RequiredAccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + RequiredMountOptions: []string{"ro", "norecovery", "nouuid"}, + }, + Resources: ModelCacheBindingResourceIntent{ + PersistentVolumeClaimNames: []string{"writer-pvc"}, + PersistentVolumeNames: []string{"reader-pv"}, + JobNames: []string{"writer-job"}, + StorageClassNames: []string{"encrypted-sc"}, + SecretNames: []string{"encrypted-secret"}, + }, + }, + Status: ModelCacheBindingStatus{ + LastPhaseTransitionTime: &now, + RequestReferences: []ModelCacheBindingRequestReference{ + {Namespace: "request-ns", Name: "request", UID: "request-uid"}, + }, + Realized: &ModelCacheBindingRealizedState{ProviderDataIdentity: "provider-id"}, + Conditions: []metav1.Condition{ + {Type: "Ready", Status: metav1.ConditionTrue, Reason: "Populated", Message: "ready"}, + }, + }, + } + + copy := original.DeepCopy() + copy.Spec.Decision.RequiredAccessModes[0] = corev1.ReadOnlyMany + copy.Spec.Decision.RequiredMountOptions[0] = "rw" + copy.Spec.Resources.PersistentVolumeClaimNames[0] = "changed-pvc" + copy.Spec.Resources.PersistentVolumeNames[0] = "changed-pv" + copy.Spec.Resources.JobNames[0] = "changed-job" + copy.Spec.Resources.StorageClassNames[0] = "changed-sc" + copy.Spec.Resources.SecretNames[0] = "changed-secret" + copy.Status.RequestReferences[0].Name = "changed-request" + copy.Status.Realized.ProviderDataIdentity = "changed-provider-id" + copy.Status.Conditions[0].Message = "changed" + + assert.Equal(t, corev1.ReadWriteOnce, original.Spec.Decision.RequiredAccessModes[0]) + assert.Equal(t, "ro", original.Spec.Decision.RequiredMountOptions[0]) + assert.Equal(t, "writer-pvc", original.Spec.Resources.PersistentVolumeClaimNames[0]) + assert.Equal(t, "reader-pv", original.Spec.Resources.PersistentVolumeNames[0]) + assert.Equal(t, "writer-job", original.Spec.Resources.JobNames[0]) + assert.Equal(t, "encrypted-sc", original.Spec.Resources.StorageClassNames[0]) + assert.Equal(t, "encrypted-secret", original.Spec.Resources.SecretNames[0]) + assert.Equal(t, "request", original.Status.RequestReferences[0].Name) + assert.Equal(t, "provider-id", original.Status.Realized.ProviderDataIdentity) + assert.Equal(t, "ready", original.Status.Conditions[0].Message) +} + +func TestModelCacheBindingOpenAPIDefinitions(t *testing.T) { + definitions := GetOpenAPIDefinitions(func(string) spec.Ref { return spec.Ref{} }) + for _, typeName := range []string{ + "ModelCacheBinding", + "ModelCacheBindingDecision", + "ModelCacheBindingIdentity", + "ModelCacheBindingList", + "ModelCacheBindingRealizedState", + "ModelCacheBindingRequestReference", + "ModelCacheBindingResourceIntent", + "ModelCacheBindingSpec", + "ModelCacheBindingStatus", + } { + assert.Contains(t, definitions, + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1."+typeName) + } +} diff --git a/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1/register.go b/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1/register.go index d13b4ab8e..bea635dcf 100644 --- a/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1/register.go +++ b/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1/register.go @@ -47,6 +47,8 @@ func Resource(resource string) schema.GroupResource { func addKnownTypes(scheme *runtime.Scheme) error { scheme.AddKnownTypes( SchemeGroupVersion, + &ModelCacheBinding{}, + &ModelCacheBindingList{}, &StorageRequest{}, &StorageRequestList{}, &ICMSRequest{}, diff --git a/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1/zz_generated.deepcopy.go b/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1/zz_generated.deepcopy.go index 1334d7dd0..69b0c80c8 100644 --- a/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1/zz_generated.deepcopy.go +++ b/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1/zz_generated.deepcopy.go @@ -329,6 +329,239 @@ func (in *InternalPersistentStorageStatus) DeepCopy() *InternalPersistentStorage return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModelCacheBinding) DeepCopyInto(out *ModelCacheBinding) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelCacheBinding. +func (in *ModelCacheBinding) DeepCopy() *ModelCacheBinding { + if in == nil { + return nil + } + out := new(ModelCacheBinding) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ModelCacheBinding) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModelCacheBindingDecision) DeepCopyInto(out *ModelCacheBindingDecision) { + *out = *in + if in.RequiredAccessModes != nil { + in, out := &in.RequiredAccessModes, &out.RequiredAccessModes + *out = make([]v1.PersistentVolumeAccessMode, len(*in)) + copy(*out, *in) + } + if in.RequiredMountOptions != nil { + in, out := &in.RequiredMountOptions, &out.RequiredMountOptions + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelCacheBindingDecision. +func (in *ModelCacheBindingDecision) DeepCopy() *ModelCacheBindingDecision { + if in == nil { + return nil + } + out := new(ModelCacheBindingDecision) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModelCacheBindingIdentity) DeepCopyInto(out *ModelCacheBindingIdentity) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelCacheBindingIdentity. +func (in *ModelCacheBindingIdentity) DeepCopy() *ModelCacheBindingIdentity { + if in == nil { + return nil + } + out := new(ModelCacheBindingIdentity) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModelCacheBindingList) DeepCopyInto(out *ModelCacheBindingList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ModelCacheBinding, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelCacheBindingList. +func (in *ModelCacheBindingList) DeepCopy() *ModelCacheBindingList { + if in == nil { + return nil + } + out := new(ModelCacheBindingList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ModelCacheBindingList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModelCacheBindingRealizedState) DeepCopyInto(out *ModelCacheBindingRealizedState) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelCacheBindingRealizedState. +func (in *ModelCacheBindingRealizedState) DeepCopy() *ModelCacheBindingRealizedState { + if in == nil { + return nil + } + out := new(ModelCacheBindingRealizedState) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModelCacheBindingRequestReference) DeepCopyInto(out *ModelCacheBindingRequestReference) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelCacheBindingRequestReference. +func (in *ModelCacheBindingRequestReference) DeepCopy() *ModelCacheBindingRequestReference { + if in == nil { + return nil + } + out := new(ModelCacheBindingRequestReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModelCacheBindingResourceIntent) DeepCopyInto(out *ModelCacheBindingResourceIntent) { + *out = *in + if in.PersistentVolumeClaimNames != nil { + in, out := &in.PersistentVolumeClaimNames, &out.PersistentVolumeClaimNames + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.PersistentVolumeNames != nil { + in, out := &in.PersistentVolumeNames, &out.PersistentVolumeNames + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.JobNames != nil { + in, out := &in.JobNames, &out.JobNames + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.StorageClassNames != nil { + in, out := &in.StorageClassNames, &out.StorageClassNames + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.SecretNames != nil { + in, out := &in.SecretNames, &out.SecretNames + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelCacheBindingResourceIntent. +func (in *ModelCacheBindingResourceIntent) DeepCopy() *ModelCacheBindingResourceIntent { + if in == nil { + return nil + } + out := new(ModelCacheBindingResourceIntent) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModelCacheBindingSpec) DeepCopyInto(out *ModelCacheBindingSpec) { + *out = *in + out.Identity = in.Identity + in.Decision.DeepCopyInto(&out.Decision) + out.StorageClass = in.StorageClass + in.Resources.DeepCopyInto(&out.Resources) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelCacheBindingSpec. +func (in *ModelCacheBindingSpec) DeepCopy() *ModelCacheBindingSpec { + if in == nil { + return nil + } + out := new(ModelCacheBindingSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModelCacheBindingStatus) DeepCopyInto(out *ModelCacheBindingStatus) { + *out = *in + if in.LastPhaseTransitionTime != nil { + in, out := &in.LastPhaseTransitionTime, &out.LastPhaseTransitionTime + *out = (*in).DeepCopy() + } + if in.RequestReferences != nil { + in, out := &in.RequestReferences, &out.RequestReferences + *out = make([]ModelCacheBindingRequestReference, len(*in)) + copy(*out, *in) + } + if in.Realized != nil { + in, out := &in.Realized, &out.Realized + *out = new(ModelCacheBindingRealizedState) + **out = **in + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]metav1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelCacheBindingStatus. +func (in *ModelCacheBindingStatus) DeepCopy() *ModelCacheBindingStatus { + if in == nil { + return nil + } + out := new(ModelCacheBindingStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ModelCacheEncryption) DeepCopyInto(out *ModelCacheEncryption) { *out = *in @@ -382,6 +615,22 @@ func (in *ModelCacheStatus) DeepCopy() *ModelCacheStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ModelCacheStorageClassSnapshot) DeepCopyInto(out *ModelCacheStorageClassSnapshot) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ModelCacheStorageClassSnapshot. +func (in *ModelCacheStorageClassSnapshot) DeepCopy() *ModelCacheStorageClassSnapshot { + if in == nil { + return nil + } + out := new(ModelCacheStorageClassSnapshot) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SharedStorageServerSpec) DeepCopyInto(out *SharedStorageServerSpec) { *out = *in diff --git a/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1/zz_generated.deepcopy.go b/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1/zz_generated.deepcopy.go index 1623ddac5..651b51707 100644 --- a/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1/zz_generated.deepcopy.go +++ b/src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1/zz_generated.deepcopy.go @@ -2,7 +2,8 @@ // +build !ignore_autogenerated /* -Copyright The Kubernetes Authors. +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/compute-plane-services/nvca/pkg/client/clientset/versioned/typed/nvca/v2beta1/BUILD.bazel b/src/compute-plane-services/nvca/pkg/client/clientset/versioned/typed/nvca/v2beta1/BUILD.bazel index fa4cf7cc8..70ef5bab1 100644 --- a/src/compute-plane-services/nvca/pkg/client/clientset/versioned/typed/nvca/v2beta1/BUILD.bazel +++ b/src/compute-plane-services/nvca/pkg/client/clientset/versioned/typed/nvca/v2beta1/BUILD.bazel @@ -9,6 +9,7 @@ go_library( "doc.go", "generated_expansion.go", "icmsrequest.go", + "modelcachebinding.go", "nvca_client.go", "storagerequest.go", ], diff --git a/src/compute-plane-services/nvca/pkg/client/clientset/versioned/typed/nvca/v2beta1/fake/BUILD.bazel b/src/compute-plane-services/nvca/pkg/client/clientset/versioned/typed/nvca/v2beta1/fake/BUILD.bazel index ebb66700e..dfc85b67c 100644 --- a/src/compute-plane-services/nvca/pkg/client/clientset/versioned/typed/nvca/v2beta1/fake/BUILD.bazel +++ b/src/compute-plane-services/nvca/pkg/client/clientset/versioned/typed/nvca/v2beta1/fake/BUILD.bazel @@ -8,6 +8,7 @@ go_library( srcs = [ "doc.go", "fake_icmsrequest.go", + "fake_modelcachebinding.go", "fake_nvca_client.go", "fake_storagerequest.go", ], diff --git a/src/compute-plane-services/nvca/pkg/client/clientset/versioned/typed/nvca/v2beta1/fake/fake_modelcachebinding.go b/src/compute-plane-services/nvca/pkg/client/clientset/versioned/typed/nvca/v2beta1/fake/fake_modelcachebinding.go new file mode 100644 index 000000000..222cb692d --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/client/clientset/versioned/typed/nvca/v2beta1/fake/fake_modelcachebinding.go @@ -0,0 +1,53 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package fake + +import ( + v2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1" + nvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/client/clientset/versioned/typed/nvca/v2beta1" + gentype "k8s.io/client-go/gentype" +) + +// fakeModelCacheBindings implements ModelCacheBindingInterface +type fakeModelCacheBindings struct { + *gentype.FakeClientWithList[*v2beta1.ModelCacheBinding, *v2beta1.ModelCacheBindingList] + Fake *FakeNvcaV2beta1 +} + +func newFakeModelCacheBindings(fake *FakeNvcaV2beta1, namespace string) nvcav2beta1.ModelCacheBindingInterface { + return &fakeModelCacheBindings{ + gentype.NewFakeClientWithList[*v2beta1.ModelCacheBinding, *v2beta1.ModelCacheBindingList]( + fake.Fake, + namespace, + v2beta1.SchemeGroupVersion.WithResource("modelcachebindings"), + v2beta1.SchemeGroupVersion.WithKind("ModelCacheBinding"), + func() *v2beta1.ModelCacheBinding { return &v2beta1.ModelCacheBinding{} }, + func() *v2beta1.ModelCacheBindingList { return &v2beta1.ModelCacheBindingList{} }, + func(dst, src *v2beta1.ModelCacheBindingList) { dst.ListMeta = src.ListMeta }, + func(list *v2beta1.ModelCacheBindingList) []*v2beta1.ModelCacheBinding { + return gentype.ToPointerSlice(list.Items) + }, + func(list *v2beta1.ModelCacheBindingList, items []*v2beta1.ModelCacheBinding) { + list.Items = gentype.FromPointerSlice(items) + }, + ), + fake, + } +} diff --git a/src/compute-plane-services/nvca/pkg/client/clientset/versioned/typed/nvca/v2beta1/fake/fake_nvca_client.go b/src/compute-plane-services/nvca/pkg/client/clientset/versioned/typed/nvca/v2beta1/fake/fake_nvca_client.go index 5bf8ae5d4..981e83660 100644 --- a/src/compute-plane-services/nvca/pkg/client/clientset/versioned/typed/nvca/v2beta1/fake/fake_nvca_client.go +++ b/src/compute-plane-services/nvca/pkg/client/clientset/versioned/typed/nvca/v2beta1/fake/fake_nvca_client.go @@ -33,6 +33,10 @@ func (c *FakeNvcaV2beta1) ICMSRequests(namespace string) v2beta1.ICMSRequestInte return newFakeICMSRequests(c, namespace) } +func (c *FakeNvcaV2beta1) ModelCacheBindings(namespace string) v2beta1.ModelCacheBindingInterface { + return newFakeModelCacheBindings(c, namespace) +} + func (c *FakeNvcaV2beta1) StorageRequests(namespace string) v2beta1.StorageRequestInterface { return newFakeStorageRequests(c, namespace) } diff --git a/src/compute-plane-services/nvca/pkg/client/clientset/versioned/typed/nvca/v2beta1/generated_expansion.go b/src/compute-plane-services/nvca/pkg/client/clientset/versioned/typed/nvca/v2beta1/generated_expansion.go index b1719c9d7..29eb8c59f 100644 --- a/src/compute-plane-services/nvca/pkg/client/clientset/versioned/typed/nvca/v2beta1/generated_expansion.go +++ b/src/compute-plane-services/nvca/pkg/client/clientset/versioned/typed/nvca/v2beta1/generated_expansion.go @@ -21,4 +21,6 @@ package v2beta1 type ICMSRequestExpansion interface{} +type ModelCacheBindingExpansion interface{} + type StorageRequestExpansion interface{} diff --git a/src/compute-plane-services/nvca/pkg/client/clientset/versioned/typed/nvca/v2beta1/modelcachebinding.go b/src/compute-plane-services/nvca/pkg/client/clientset/versioned/typed/nvca/v2beta1/modelcachebinding.go new file mode 100644 index 000000000..fa86f04c2 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/client/clientset/versioned/typed/nvca/v2beta1/modelcachebinding.go @@ -0,0 +1,71 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by client-gen. DO NOT EDIT. + +package v2beta1 + +import ( + context "context" + + nvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1" + scheme "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/client/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + gentype "k8s.io/client-go/gentype" +) + +// ModelCacheBindingsGetter has a method to return a ModelCacheBindingInterface. +// A group's client should implement this interface. +type ModelCacheBindingsGetter interface { + ModelCacheBindings(namespace string) ModelCacheBindingInterface +} + +// ModelCacheBindingInterface has methods to work with ModelCacheBinding resources. +type ModelCacheBindingInterface interface { + Create(ctx context.Context, modelCacheBinding *nvcav2beta1.ModelCacheBinding, opts v1.CreateOptions) (*nvcav2beta1.ModelCacheBinding, error) + Update(ctx context.Context, modelCacheBinding *nvcav2beta1.ModelCacheBinding, opts v1.UpdateOptions) (*nvcav2beta1.ModelCacheBinding, error) + // Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + UpdateStatus(ctx context.Context, modelCacheBinding *nvcav2beta1.ModelCacheBinding, opts v1.UpdateOptions) (*nvcav2beta1.ModelCacheBinding, error) + Delete(ctx context.Context, name string, opts v1.DeleteOptions) error + DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error + Get(ctx context.Context, name string, opts v1.GetOptions) (*nvcav2beta1.ModelCacheBinding, error) + List(ctx context.Context, opts v1.ListOptions) (*nvcav2beta1.ModelCacheBindingList, error) + Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) + Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *nvcav2beta1.ModelCacheBinding, err error) + ModelCacheBindingExpansion +} + +// modelCacheBindings implements ModelCacheBindingInterface +type modelCacheBindings struct { + *gentype.ClientWithList[*nvcav2beta1.ModelCacheBinding, *nvcav2beta1.ModelCacheBindingList] +} + +// newModelCacheBindings returns a ModelCacheBindings +func newModelCacheBindings(c *NvcaV2beta1Client, namespace string) *modelCacheBindings { + return &modelCacheBindings{ + gentype.NewClientWithList[*nvcav2beta1.ModelCacheBinding, *nvcav2beta1.ModelCacheBindingList]( + "modelcachebindings", + c.RESTClient(), + scheme.ParameterCodec, + namespace, + func() *nvcav2beta1.ModelCacheBinding { return &nvcav2beta1.ModelCacheBinding{} }, + func() *nvcav2beta1.ModelCacheBindingList { return &nvcav2beta1.ModelCacheBindingList{} }, + ), + } +} diff --git a/src/compute-plane-services/nvca/pkg/client/clientset/versioned/typed/nvca/v2beta1/nvca_client.go b/src/compute-plane-services/nvca/pkg/client/clientset/versioned/typed/nvca/v2beta1/nvca_client.go index ef717b659..5425afe1b 100644 --- a/src/compute-plane-services/nvca/pkg/client/clientset/versioned/typed/nvca/v2beta1/nvca_client.go +++ b/src/compute-plane-services/nvca/pkg/client/clientset/versioned/typed/nvca/v2beta1/nvca_client.go @@ -30,6 +30,7 @@ import ( type NvcaV2beta1Interface interface { RESTClient() rest.Interface ICMSRequestsGetter + ModelCacheBindingsGetter StorageRequestsGetter } @@ -42,6 +43,10 @@ func (c *NvcaV2beta1Client) ICMSRequests(namespace string) ICMSRequestInterface return newICMSRequests(c, namespace) } +func (c *NvcaV2beta1Client) ModelCacheBindings(namespace string) ModelCacheBindingInterface { + return newModelCacheBindings(c, namespace) +} + func (c *NvcaV2beta1Client) StorageRequests(namespace string) StorageRequestInterface { return newStorageRequests(c, namespace) } diff --git a/src/compute-plane-services/nvca/pkg/client/informers/externalversions/generic.go b/src/compute-plane-services/nvca/pkg/client/informers/externalversions/generic.go index 42c725ca5..525e8c23f 100644 --- a/src/compute-plane-services/nvca/pkg/client/informers/externalversions/generic.go +++ b/src/compute-plane-services/nvca/pkg/client/informers/externalversions/generic.go @@ -67,6 +67,8 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource // Group=nvca.nvcf.nvidia.io, Version=v2beta1 case v2beta1.SchemeGroupVersion.WithResource("icmsrequests"): return &genericInformer{resource: resource.GroupResource(), informer: f.Nvca().V2beta1().ICMSRequests().Informer()}, nil + case v2beta1.SchemeGroupVersion.WithResource("modelcachebindings"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Nvca().V2beta1().ModelCacheBindings().Informer()}, nil case v2beta1.SchemeGroupVersion.WithResource("storagerequests"): return &genericInformer{resource: resource.GroupResource(), informer: f.Nvca().V2beta1().StorageRequests().Informer()}, nil diff --git a/src/compute-plane-services/nvca/pkg/client/informers/externalversions/nvca/v2beta1/BUILD.bazel b/src/compute-plane-services/nvca/pkg/client/informers/externalversions/nvca/v2beta1/BUILD.bazel index 3cc0b1da1..cc86baedb 100644 --- a/src/compute-plane-services/nvca/pkg/client/informers/externalversions/nvca/v2beta1/BUILD.bazel +++ b/src/compute-plane-services/nvca/pkg/client/informers/externalversions/nvca/v2beta1/BUILD.bazel @@ -8,6 +8,7 @@ go_library( srcs = [ "icmsrequest.go", "interface.go", + "modelcachebinding.go", "storagerequest.go", ], importpath = "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/client/informers/externalversions/nvca/v2beta1", diff --git a/src/compute-plane-services/nvca/pkg/client/informers/externalversions/nvca/v2beta1/interface.go b/src/compute-plane-services/nvca/pkg/client/informers/externalversions/nvca/v2beta1/interface.go index 87114c17d..1f43d5ced 100644 --- a/src/compute-plane-services/nvca/pkg/client/informers/externalversions/nvca/v2beta1/interface.go +++ b/src/compute-plane-services/nvca/pkg/client/informers/externalversions/nvca/v2beta1/interface.go @@ -27,6 +27,8 @@ import ( type Interface interface { // ICMSRequests returns a ICMSRequestInformer. ICMSRequests() ICMSRequestInformer + // ModelCacheBindings returns a ModelCacheBindingInformer. + ModelCacheBindings() ModelCacheBindingInformer // StorageRequests returns a StorageRequestInformer. StorageRequests() StorageRequestInformer } @@ -47,6 +49,11 @@ func (v *version) ICMSRequests() ICMSRequestInformer { return &iCMSRequestInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} } +// ModelCacheBindings returns a ModelCacheBindingInformer. +func (v *version) ModelCacheBindings() ModelCacheBindingInformer { + return &modelCacheBindingInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} + // StorageRequests returns a StorageRequestInformer. func (v *version) StorageRequests() StorageRequestInformer { return &storageRequestInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} diff --git a/src/compute-plane-services/nvca/pkg/client/informers/externalversions/nvca/v2beta1/modelcachebinding.go b/src/compute-plane-services/nvca/pkg/client/informers/externalversions/nvca/v2beta1/modelcachebinding.go new file mode 100644 index 000000000..2c564939a --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/client/informers/externalversions/nvca/v2beta1/modelcachebinding.go @@ -0,0 +1,103 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by informer-gen. DO NOT EDIT. + +package v2beta1 + +import ( + context "context" + time "time" + + apisnvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1" + versioned "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/client/clientset/versioned" + internalinterfaces "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/client/informers/externalversions/internalinterfaces" + nvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/client/listers/nvca/v2beta1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// ModelCacheBindingInformer provides access to a shared informer and lister for +// ModelCacheBindings. +type ModelCacheBindingInformer interface { + Informer() cache.SharedIndexInformer + Lister() nvcav2beta1.ModelCacheBindingLister +} + +type modelCacheBindingInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewModelCacheBindingInformer constructs a new informer for ModelCacheBinding type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewModelCacheBindingInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredModelCacheBindingInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredModelCacheBindingInformer constructs a new informer for ModelCacheBinding type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredModelCacheBindingInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + cache.ToListWatcherWithWatchListSemantics(&cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.NvcaV2beta1().ModelCacheBindings(namespace).List(context.Background(), options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.NvcaV2beta1().ModelCacheBindings(namespace).Watch(context.Background(), options) + }, + ListWithContextFunc: func(ctx context.Context, options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.NvcaV2beta1().ModelCacheBindings(namespace).List(ctx, options) + }, + WatchFuncWithContext: func(ctx context.Context, options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.NvcaV2beta1().ModelCacheBindings(namespace).Watch(ctx, options) + }, + }, client), + &apisnvcav2beta1.ModelCacheBinding{}, + resyncPeriod, + indexers, + ) +} + +func (f *modelCacheBindingInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredModelCacheBindingInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *modelCacheBindingInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&apisnvcav2beta1.ModelCacheBinding{}, f.defaultInformer) +} + +func (f *modelCacheBindingInformer) Lister() nvcav2beta1.ModelCacheBindingLister { + return nvcav2beta1.NewModelCacheBindingLister(f.Informer().GetIndexer()) +} diff --git a/src/compute-plane-services/nvca/pkg/client/listers/nvca/v2beta1/BUILD.bazel b/src/compute-plane-services/nvca/pkg/client/listers/nvca/v2beta1/BUILD.bazel index 1574761d0..ccaad4b3a 100644 --- a/src/compute-plane-services/nvca/pkg/client/listers/nvca/v2beta1/BUILD.bazel +++ b/src/compute-plane-services/nvca/pkg/client/listers/nvca/v2beta1/BUILD.bazel @@ -8,6 +8,7 @@ go_library( srcs = [ "expansion_generated.go", "icmsrequest.go", + "modelcachebinding.go", "storagerequest.go", ], importpath = "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/client/listers/nvca/v2beta1", diff --git a/src/compute-plane-services/nvca/pkg/client/listers/nvca/v2beta1/expansion_generated.go b/src/compute-plane-services/nvca/pkg/client/listers/nvca/v2beta1/expansion_generated.go index 052c70863..0f349bd8e 100644 --- a/src/compute-plane-services/nvca/pkg/client/listers/nvca/v2beta1/expansion_generated.go +++ b/src/compute-plane-services/nvca/pkg/client/listers/nvca/v2beta1/expansion_generated.go @@ -27,6 +27,14 @@ type ICMSRequestListerExpansion interface{} // ICMSRequestNamespaceLister. type ICMSRequestNamespaceListerExpansion interface{} +// ModelCacheBindingListerExpansion allows custom methods to be added to +// ModelCacheBindingLister. +type ModelCacheBindingListerExpansion interface{} + +// ModelCacheBindingNamespaceListerExpansion allows custom methods to be added to +// ModelCacheBindingNamespaceLister. +type ModelCacheBindingNamespaceListerExpansion interface{} + // StorageRequestListerExpansion allows custom methods to be added to // StorageRequestLister. type StorageRequestListerExpansion interface{} diff --git a/src/compute-plane-services/nvca/pkg/client/listers/nvca/v2beta1/modelcachebinding.go b/src/compute-plane-services/nvca/pkg/client/listers/nvca/v2beta1/modelcachebinding.go new file mode 100644 index 000000000..e6cf9d626 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/client/listers/nvca/v2beta1/modelcachebinding.go @@ -0,0 +1,71 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by lister-gen. DO NOT EDIT. + +package v2beta1 + +import ( + nvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1" + labels "k8s.io/apimachinery/pkg/labels" + listers "k8s.io/client-go/listers" + cache "k8s.io/client-go/tools/cache" +) + +// ModelCacheBindingLister helps list ModelCacheBindings. +// All objects returned here must be treated as read-only. +type ModelCacheBindingLister interface { + // List lists all ModelCacheBindings in the indexer. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*nvcav2beta1.ModelCacheBinding, err error) + // ModelCacheBindings returns an object that can list and get ModelCacheBindings. + ModelCacheBindings(namespace string) ModelCacheBindingNamespaceLister + ModelCacheBindingListerExpansion +} + +// modelCacheBindingLister implements the ModelCacheBindingLister interface. +type modelCacheBindingLister struct { + listers.ResourceIndexer[*nvcav2beta1.ModelCacheBinding] +} + +// NewModelCacheBindingLister returns a new ModelCacheBindingLister. +func NewModelCacheBindingLister(indexer cache.Indexer) ModelCacheBindingLister { + return &modelCacheBindingLister{listers.New[*nvcav2beta1.ModelCacheBinding](indexer, nvcav2beta1.Resource("modelcachebinding"))} +} + +// ModelCacheBindings returns an object that can list and get ModelCacheBindings. +func (s *modelCacheBindingLister) ModelCacheBindings(namespace string) ModelCacheBindingNamespaceLister { + return modelCacheBindingNamespaceLister{listers.NewNamespaced[*nvcav2beta1.ModelCacheBinding](s.ResourceIndexer, namespace)} +} + +// ModelCacheBindingNamespaceLister helps list and get ModelCacheBindings. +// All objects returned here must be treated as read-only. +type ModelCacheBindingNamespaceLister interface { + // List lists all ModelCacheBindings in the indexer for a given namespace. + // Objects returned here must be treated as read-only. + List(selector labels.Selector) (ret []*nvcav2beta1.ModelCacheBinding, err error) + // Get retrieves the ModelCacheBinding from the indexer for a given namespace and name. + // Objects returned here must be treated as read-only. + Get(name string) (*nvcav2beta1.ModelCacheBinding, error) + ModelCacheBindingNamespaceListerExpansion +} + +// modelCacheBindingNamespaceLister implements the ModelCacheBindingNamespaceLister +// interface. +type modelCacheBindingNamespaceLister struct { + listers.ResourceIndexer[*nvcav2beta1.ModelCacheBinding] +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/BUILD.bazel b/src/compute-plane-services/nvca/pkg/nvca/BUILD.bazel index f7364dd10..4c412eb41 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/BUILD.bazel +++ b/src/compute-plane-services/nvca/pkg/nvca/BUILD.bazel @@ -21,9 +21,14 @@ go_library( "k8scomputebackend.go", "k8scomputebackend_miniservice.go", "k8scomputebackend_modelcache.go", + "k8scomputebackend_modelcache_rwx_readonly.go", "k8scomputebackend_task_container.go", "ledger_event_correlator.go", "ledger_events.go", + "modelcache_binding.go", + "modelcache_binding_resources.go", + "modelcache_shared_writer.go", + "modelcache_storage_selection.go", "nvsnap_coldstart_gate.go", "nvsnap_coldstart_metrics.go", "nvsnap_controller_start.go", @@ -60,6 +65,7 @@ go_library( "//src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1", "//src/compute-plane-services/nvca/pkg/apis/nvsnap/v1alpha1", "//src/compute-plane-services/nvca/pkg/client/clientset/versioned/scheme", + "//src/compute-plane-services/nvca/pkg/client/clientset/versioned/typed/nvca/v2beta1", "//src/compute-plane-services/nvca/pkg/client/informers/externalversions", "//src/compute-plane-services/nvca/pkg/client/listers/nvca/v2beta1", "//src/compute-plane-services/nvca/pkg/featureflag", @@ -184,11 +190,16 @@ go_test( "icms_client_test.go", "jwks_updater_test.go", "k8scomputebackend_miniservice_test.go", + "k8scomputebackend_modelcache_rwx_readonly_test.go", "k8scomputebackend_modelcache_test.go", "k8scomputebackend_task_container_test.go", "k8scomputebackend_test.go", "ledger_event_correlator_test.go", "ledger_events_test.go", + "modelcache_binding_test.go", + "modelcache_cleanup_binding_test.go", + "modelcache_shared_writer_test.go", + "modelcache_storage_selection_test.go", "nvsnap_hook_lookup_test.go", "nvsnap_hook_test.go", "queue_manager_test.go", @@ -294,6 +305,8 @@ go_test( "//src/compute-plane-services/nvca/vendor/k8s.io/api/core/v1:core", "//src/compute-plane-services/nvca/vendor/k8s.io/api/networking/v1:networking", "//src/compute-plane-services/nvca/vendor/k8s.io/api/rbac/v1:rbac", + "//src/compute-plane-services/nvca/vendor/k8s.io/api/storage/v1:storage", + "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/api/equality", "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/api/errors", "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/api/resource", "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/apis/meta/v1:meta", @@ -312,6 +325,7 @@ go_test( "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/kubernetes/fake", "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/listers/core/v1:core", "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/rest", + "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/testing", "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/tools/cache", "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/tools/record", "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/util/workqueue", diff --git a/src/compute-plane-services/nvca/pkg/nvca/backendk8scache.go b/src/compute-plane-services/nvca/pkg/nvca/backendk8scache.go index 09a6e78a9..a004353ec 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/backendk8scache.go +++ b/src/compute-plane-services/nvca/pkg/nvca/backendk8scache.go @@ -2261,6 +2261,9 @@ func (c *BackendK8sCache) CreateICMSCreationMessageRequest(ctx context.Context, TaskID: mt.Details.TaskID, }) } + if err := c.persistModelCacheStorageSelection(ctx, &o); err != nil { + return nil, err + } obj, err := c.clients.BART.NvcaV2beta1().ICMSRequests(c.requestsNamespace).Create(ctx, &o, metav1.CreateOptions{}) if err != nil { return nil, fmt.Errorf("failed to persist the ICMS request on the backend, err: %v", err) @@ -2582,6 +2585,9 @@ func (c *BackendK8sCache) syncICMSRequest(ctx context.Context, req *nvcav2beta1n if retryErr != nil && !k8serrors.IsNotFound(retryErr) { return fmt.Errorf("failed to add finalizer to ICMS request %s, err: %w", req.Name, retryErr) } + // Persist protection in its own reconcile. No workload or storage + // side effect may race ahead of the request finalizer. + return nil } } else { // The object is being deleted @@ -2590,6 +2596,14 @@ func (c *BackendK8sCache) syncICMSRequest(ctx context.Context, req *nvcav2beta1n if !c.icmsRequestHelper.AllInstancesTerminatedAndReported(ctx, req) { return fmt.Errorf("%w: instances are not terminated and reported for %s", errICMSRequestFinalizerRetained, req.Name) } + if err := c.resumeRetiringRegularModelCacheCleanup(ctx, req); err != nil { + return fmt.Errorf("finish Retiring model cache cleanup before removing finalizer from %s: %w", + req.Name, err) + } + if err := c.releaseModelCacheBindingReference(ctx, req); err != nil { + return fmt.Errorf("release model cache binding before removing finalizer from %s: %w", + req.Name, err) + } retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { // Retrieve the latest version of ICMS request before attempting update @@ -2645,6 +2659,13 @@ func (c *BackendK8sCache) syncICMSRequest(ctx context.Context, req *nvcav2beta1n // stuck InProgress err = c.CleanupCreationRequestResources(ctx, req) } else { + bindingUpdated, bindingErr := c.ensureModelCacheBinding(ctx, req) + if bindingErr != nil { + return bindingErr + } + if bindingUpdated { + return nil + } err = c.icmsRequestHelper.ApplyCreationMessage(ctx, req) } case common.TerminationAction: diff --git a/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_test.go b/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_test.go index efa33f3f1..a5770616c 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_test.go +++ b/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_test.go @@ -4426,9 +4426,9 @@ func TestGetGPUUsageStats_FallbackToNonSuffixSingleType(t *testing.T) { Status: corev1.NodeStatus{ Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}, Allocatable: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("5"), - corev1.ResourceMemory: resource.MustParse("32Gi"), - corev1.ResourceEphemeralStorage: resource.MustParse("256Gi"), + corev1.ResourceCPU: resource.MustParse("5"), + corev1.ResourceMemory: resource.MustParse("32Gi"), + corev1.ResourceEphemeralStorage: resource.MustParse("256Gi"), corev1.ResourceName(nodefeatures.GPUResourceKey): resource.MustParse("4"), }, }, @@ -5073,6 +5073,14 @@ func TestSyncICMSRequestNormalizesLegacyCreationActions(t *testing.T) { } require.NoError(t, bc.syncICMSRequest(ctx, req.DeepCopy())) + assert.Zero(t, helper.creationCalls, + "the finalizer must be persisted before creation is attempted") + persisted, err := bc.clients.BART.NvcaV2beta1().ICMSRequests(req.Namespace). + Get(ctx, req.Name, metav1.GetOptions{}) + require.NoError(t, err) + assert.Contains(t, persisted.Finalizers, NVCAFinalizer) + + require.NoError(t, bc.syncICMSRequest(ctx, persisted.DeepCopy())) assert.Equal(t, 1, helper.creationCalls) assert.Equal(t, action, helper.lastAction) }) diff --git a/src/compute-plane-services/nvca/pkg/nvca/encryption/BUILD.bazel b/src/compute-plane-services/nvca/pkg/nvca/encryption/BUILD.bazel index e2768163f..300db62b4 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/encryption/BUILD.bazel +++ b/src/compute-plane-services/nvca/pkg/nvca/encryption/BUILD.bazel @@ -34,6 +34,8 @@ go_test( "//src/compute-plane-services/nvca/internal/kubeclients", "//src/compute-plane-services/nvca/vendor/github.com/stretchr/testify/assert", "//src/compute-plane-services/nvca/vendor/github.com/stretchr/testify/mock", + "//src/compute-plane-services/nvca/vendor/k8s.io/api/core/v1:core", + "//src/compute-plane-services/nvca/vendor/k8s.io/api/storage/v1:storage", "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/apis/meta/v1:meta", "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/kubernetes/fake", ], diff --git a/src/compute-plane-services/nvca/pkg/nvca/encryption/encrypt_modelcache.go b/src/compute-plane-services/nvca/pkg/nvca/encryption/encrypt_modelcache.go index d80cd64af..6e2eee1ee 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/encryption/encrypt_modelcache.go +++ b/src/compute-plane-services/nvca/pkg/nvca/encryption/encrypt_modelcache.go @@ -23,6 +23,8 @@ import ( "crypto/rand" "encoding/base64" "encoding/hex" + "fmt" + "maps" "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/core" "github.com/sirupsen/logrus" @@ -80,72 +82,123 @@ func SetupEncryption(ctx context.Context, clients *kubeclients.KubeClients, ncaI func ensureNVMeshEncryptionStorageClass(ctx context.Context, clients *kubeclients.KubeClients, secretName, namespace, scName string) error { logger := core.GetLogger(ctx) - _, err := clients.K8s.StorageV1().StorageClasses().Get(ctx, scName, metav1.GetOptions{}) + existing, err := clients.K8s.StorageV1().StorageClasses().Get(ctx, scName, metav1.GetOptions{}) // Track K8s API call metrics if metrics := nvcametrics.FromContext(ctx); metrics != nil { metrics.TrackK8sAPICall("storageclass", err) } - if errors.IsNotFound(err) { - logger.WithFields(logrus.Fields{ - "storageclass": scName, - "namespace": namespace, - }).Debugf("Creating StorageClass") - - //create storage class - allowedExpansion := true - reclaimPolicy := corev1.PersistentVolumeReclaimRetain - bindingMode := StorageClassBindMode - - scReq := &storagev1.StorageClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: scName, - }, - Provisioner: StorageClassProvisioner, - AllowVolumeExpansion: &allowedExpansion, - VolumeBindingMode: (*storagev1.VolumeBindingMode)(&bindingMode), - ReclaimPolicy: &reclaimPolicy, - Parameters: map[string]string{ - StorageClassVPG: StorageClassVPGType, - StorageClassCSIFS: StorageClassFS, - StorageClassCSISecret: secretName, - StorageClassCSINS: namespace, - }, - } - _, err = clients.K8s.StorageV1().StorageClasses().Create(ctx, scReq, metav1.CreateOptions{}) + if err == nil { + return validateNVMeshEncryptionStorageClass(existing, expectedNVMeshEncryptionStorageClass( + secretName, namespace, scName)) + } + if !errors.IsNotFound(err) { + return fmt.Errorf("get NVMesh encryption StorageClass %q: %w", scName, err) + } + + logger.WithFields(logrus.Fields{ + "storageclass": scName, + "namespace": namespace, + }).Debugf("Creating StorageClass") + + _, err = clients.K8s.StorageV1().StorageClasses().Create(ctx, + expectedNVMeshEncryptionStorageClass(secretName, namespace, scName), metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("create NVMesh encryption StorageClass %q: %w", scName, err) + } + return nil +} + +func expectedNVMeshEncryptionStorageClass(secretName, namespace, scName string) *storagev1.StorageClass { + allowedExpansion := true + reclaimPolicy := corev1.PersistentVolumeReclaimRetain + bindingMode := storagev1.VolumeBindingMode(StorageClassBindMode) + + return &storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: scName, + }, + Provisioner: StorageClassProvisioner, + AllowVolumeExpansion: &allowedExpansion, + VolumeBindingMode: &bindingMode, + ReclaimPolicy: &reclaimPolicy, + Parameters: map[string]string{ + StorageClassVPG: StorageClassVPGType, + StorageClassCSIFS: StorageClassFS, + StorageClassCSISecret: secretName, + StorageClassCSINS: namespace, + }, } - return err +} + +func validateNVMeshEncryptionStorageClass(actual, expected *storagev1.StorageClass) error { + invalid := func(field string) error { + return fmt.Errorf("existing NVMesh encryption StorageClass %q has unexpected %s", actual.Name, field) + } + if actual.Provisioner != expected.Provisioner { + return invalid("provisioner") + } + if actual.ReclaimPolicy == nil || expected.ReclaimPolicy == nil || + *actual.ReclaimPolicy != *expected.ReclaimPolicy { + return invalid("reclaimPolicy") + } + if actual.VolumeBindingMode == nil || expected.VolumeBindingMode == nil || + *actual.VolumeBindingMode != *expected.VolumeBindingMode { + return invalid("volumeBindingMode") + } + if actual.AllowVolumeExpansion == nil || expected.AllowVolumeExpansion == nil || + *actual.AllowVolumeExpansion != *expected.AllowVolumeExpansion { + return invalid("allowVolumeExpansion") + } + if !maps.Equal(actual.Parameters, expected.Parameters) { + return invalid("parameters") + } + if len(actual.MountOptions) != 0 { + return invalid("mountOptions") + } + if len(actual.AllowedTopologies) != 0 { + return invalid("allowedTopologies") + } + return nil } func ensureNVMeshEncryptionSecret(ctx context.Context, clients *kubeclients.KubeClients, name, namespace string) error { logger := core.GetLogger(ctx) - _, err := clients.K8s.CoreV1().Secrets(namespace).Get(ctx, name, metav1.GetOptions{}) + secret, err := clients.K8s.CoreV1().Secrets(namespace).Get(ctx, name, metav1.GetOptions{}) // Track K8s API call metrics if metrics := nvcametrics.FromContext(ctx); metrics != nil { metrics.TrackK8sAPICall("secret", err) } - if errors.IsNotFound(err) { - logger.WithFields(logrus.Fields{ - "nca_hash": name, - "namespace": namespace, - }).Debugf("Creating secret") - - //Secret not found. Create it. - // Metadata for creating secret - secretRequest := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - }, - Data: map[string][]byte{ - "dmcryptKey": []byte(generateToken(ctx, name)), - }, + if err == nil { + if len(secret.Data["dmcryptKey"]) == 0 { + return fmt.Errorf("existing NVMesh encryption Secret %s/%s has no dmcryptKey", namespace, name) } - _, err = clients.K8s.CoreV1().Secrets(namespace).Create(ctx, secretRequest, metav1.CreateOptions{}) + return nil + } + if !errors.IsNotFound(err) { + return fmt.Errorf("get NVMesh encryption Secret %s/%s: %w", namespace, name, err) + } + + logger.WithFields(logrus.Fields{ + "nca_hash": name, + "namespace": namespace, + }).Debugf("Creating secret") + + secretRequest := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Data: map[string][]byte{ + "dmcryptKey": []byte(generateToken(ctx, name)), + }, + } + if _, err := clients.K8s.CoreV1().Secrets(namespace).Create(ctx, secretRequest, metav1.CreateOptions{}); err != nil { + return fmt.Errorf("create NVMesh encryption Secret %s/%s: %w", namespace, name, err) } - return err + return nil } // Generate the Random KEY_BYTES byte token diff --git a/src/compute-plane-services/nvca/pkg/nvca/encryption/encrypt_modelcache_test.go b/src/compute-plane-services/nvca/pkg/nvca/encryption/encrypt_modelcache_test.go index a5388dca7..f47fc9f42 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/encryption/encrypt_modelcache_test.go +++ b/src/compute-plane-services/nvca/pkg/nvca/encryption/encrypt_modelcache_test.go @@ -23,6 +23,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" @@ -157,6 +159,133 @@ func TestEnsureNVMeshEncryptionStorageClass(t *testing.T) { assert.Equal(t, namespace, sc.Parameters[StorageClassCSINS]) } +func TestEnsureNVMeshEncryptionStorageClassValidatesExisting(t *testing.T) { + const ( + secretName = "test-secret" + namespace = "test-namespace" + scName = "test-sc" + ) + + tests := []struct { + name string + mutate func(*storagev1.StorageClass) + wantField string + }{ + { + name: "provisioner", + mutate: func(sc *storagev1.StorageClass) { + sc.Provisioner = "unexpected.example.com" + }, + wantField: "provisioner", + }, + { + name: "reclaim policy", + mutate: func(sc *storagev1.StorageClass) { + policy := corev1.PersistentVolumeReclaimDelete + sc.ReclaimPolicy = &policy + }, + wantField: "reclaimPolicy", + }, + { + name: "binding mode", + mutate: func(sc *storagev1.StorageClass) { + mode := storagev1.VolumeBindingWaitForFirstConsumer + sc.VolumeBindingMode = &mode + }, + wantField: "volumeBindingMode", + }, + { + name: "volume expansion", + mutate: func(sc *storagev1.StorageClass) { + allowed := false + sc.AllowVolumeExpansion = &allowed + }, + wantField: "allowVolumeExpansion", + }, + { + name: "parameters", + mutate: func(sc *storagev1.StorageClass) { + sc.Parameters["unexpected"] = "value" + }, + wantField: "parameters", + }, + { + name: "mount options", + mutate: func(sc *storagev1.StorageClass) { + sc.MountOptions = []string{"unexpected"} + }, + wantField: "mountOptions", + }, + { + name: "allowed topologies", + mutate: func(sc *storagev1.StorageClass) { + sc.AllowedTopologies = []corev1.TopologySelectorTerm{{}} + }, + wantField: "allowedTopologies", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sc := expectedNVMeshEncryptionStorageClass(secretName, namespace, scName) + tt.mutate(sc) + clients := &kubeclients.KubeClients{K8s: fake.NewSimpleClientset(sc)} + + err := ensureNVMeshEncryptionStorageClass( + context.Background(), clients, secretName, namespace, scName) + + assert.EqualError(t, err, + "existing NVMesh encryption StorageClass \"test-sc\" has unexpected "+tt.wantField) + }) + } +} + +func TestSetupEncryptionPreservesExistingSecret(t *testing.T) { + const ( + ncaID = "test-nca" + namespace = "test-namespace" + ) + secretName := BuildMD5Hash(ncaID) + scName := BuildStorageClassName(secretName) + wantData := map[string][]byte{ + "dmcryptKey": []byte("existing-key"), + "extra": []byte("preserve-me"), + } + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: secretName, Namespace: namespace}, + Data: wantData, + } + clients := &kubeclients.KubeClients{K8s: fake.NewSimpleClientset( + secret, expectedNVMeshEncryptionStorageClass(secretName, namespace, scName), + )} + + gotSCName, err := SetupEncryption(context.Background(), clients, ncaID, namespace) + + assert.NoError(t, err) + assert.Equal(t, scName, gotSCName) + actual, err := clients.K8s.CoreV1().Secrets(namespace).Get( + context.Background(), secretName, metav1.GetOptions{}) + assert.NoError(t, err) + assert.Equal(t, wantData, actual.Data) +} + +func TestEnsureNVMeshEncryptionSecretRejectsMissingKey(t *testing.T) { + const ( + name = "test-secret" + namespace = "test-namespace" + ) + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, + Data: map[string][]byte{"unrelated": []byte("value")}, + } + clients := &kubeclients.KubeClients{K8s: fake.NewSimpleClientset(secret)} + + err := ensureNVMeshEncryptionSecret(context.Background(), clients, name, namespace) + + assert.EqualError(t, err, + "existing NVMesh encryption Secret test-namespace/test-secret has no dmcryptKey") +} + func TestEnsureNVMeshEncryptionSecret(t *testing.T) { ctx := context.Background() clients := &kubeclients.KubeClients{ diff --git a/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend.go b/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend.go index 48a8a36ed..d152d026b 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend.go +++ b/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend.go @@ -79,11 +79,41 @@ const ( ImagePullIssueAlternateReason = "ImagePullBackOff" InferenceContainerName = "inference" InitContainerName = "init" - RWPVCSuffix = "rw-pvc" - ROPVCSuffix = "ro-pvc" + RWPVCPrefix = "rw-pvc-" + ROPVCPrefix = "ro-pvc-" ModelVolumeName = "model-data" ) +func regularModelCachePVCNameHandle(name, prefix string) (string, bool) { + handle, found := strings.CutPrefix(name, prefix) + return handle, found && handle != "" +} + +func regularModelCacheReaderPVCName(writerPVCName string) (string, error) { + handle, ok := regularModelCachePVCNameHandle(writerPVCName, RWPVCPrefix) + if !ok { + return "", fmt.Errorf("regular model cache writer PVC name %q must start with %q and include a cache handle", + writerPVCName, RWPVCPrefix) + } + return ROPVCPrefix + handle, nil +} + +func classifyRegularModelCachePVCName(name string) (reader bool, err error) { + if _, ok := regularModelCachePVCNameHandle(name, RWPVCPrefix); ok { + return false, nil + } + if _, ok := regularModelCachePVCNameHandle(name, ROPVCPrefix); ok { + return true, nil + } + return false, fmt.Errorf("regular model cache PVC name %q must start with %q or %q and include a cache handle", + name, RWPVCPrefix, ROPVCPrefix) +} + +func isRegularModelCacheWriterPVCName(name string) bool { + _, ok := regularModelCachePVCNameHandle(name, RWPVCPrefix) + return ok +} + type K8sComputeBackend struct { clients *kubeclients.KubeClients bk8s *BackendK8sCache @@ -557,8 +587,20 @@ func (c K8sComputeBackend) applyFunctionCreationMessage(ctx context.Context, req } } + // A new request's persisted selection is authoritative even if the live + // feature gate changes before this reconcile. Requests without the + // annotation retain the legacy static-gate behavior. + persistedCacheSelection, selectionErr := persistedRegularModelCacheSelection(req) + if selectionErr != nil { + return nvcaerrors.TerminalError(selectionErr) + } + if err := validatePersistedRegularModelCacheArtifacts( + persistedCacheSelection, bdCreate, initCacheJob); err != nil { + return err + } + cacheRuntimeEnabled := persistedCacheSelection != nil || c.bk8s.cachingSupportEnabled // setup the InitCacheJob & BlockDevice if requested - if c.bk8s.cachingSupportEnabled && (initCacheJob.Specification != "" && bdCreate.Specification != "") { + if cacheRuntimeEnabled && (initCacheJob.Specification != "" && bdCreate.Specification != "") { cacheMF, cachePVCName, err := c.setupContainerFunctionModelCaching(ctx, req, bdCreate, initCacheJob, func(obj client.Object) { // for caching mf we need to skip OwnerRefs @@ -571,7 +613,7 @@ func (c K8sComputeBackend) applyFunctionCreationMessage(ctx context.Context, req } roPVCName = cachePVCName cacheMF(pod) - } else if !c.bk8s.cachingSupportEnabled { + } else if !cacheRuntimeEnabled { log.Debugf("ModelCaching support is disabled, creating instance without caching") } else { log.Debug("InitCacheJob / BDCreate spec was not specified, skipping model caching") @@ -655,14 +697,72 @@ func (c K8sComputeBackend) setupContainerFunctionModelCaching(ctx context.Contex cachemf mutateFunc, ) (func(*corev1.Pod), string, error) { log := core.GetLogger(ctx) + selection, selectionErr := persistedRegularModelCacheSelection(req) + if selectionErr != nil { + return nil, "", nvcaerrors.TerminalError(selectionErr) + } rwPVC, initJob, err := getModelCacheK8sArtifacts(ctx, bdCreate, initCacheJob, cachemf) if err != nil { + if selection != nil && selection.Mode == nvcastorage.ModelCacheSelectionDurable { + return nil, "", nvcaerrors.TerminalError( + fmt.Errorf("decode artifacts for persisted durable regular model cache: %w", err)) + } log.WithError(err).Error("failed getModelCacheK8sArtifacts, model caching will be disabled") return func(*corev1.Pod) {}, "", nil } return c.setupContainerModelCaching(ctx, req, rwPVC, initJob, cachemf) } +func persistedRegularModelCacheSelection( + req *nvcav2beta1.ICMSRequest, +) (*nvcastorage.PersistedModelCacheStorageSelection, error) { + raw := req.Annotations[nvcastorage.ModelCacheStorageSelectionAnnotationKey] + if raw == "" { + return nil, nil + } + selection, err := nvcastorage.ParsePersistedModelCacheStorageSelection(raw) + if err != nil { + return nil, fmt.Errorf("parse persisted model cache storage selection: %w", err) + } + if selection.Workflow != nvcastorage.ModelCacheWorkflowRegular { + return nil, fmt.Errorf("persisted model cache workflow %q is not regularModelCache", selection.Workflow) + } + switch selection.Mode { + case nvcastorage.ModelCacheSelectionNone, nvcastorage.ModelCacheSelectionDurable: + return selection, nil + default: + return nil, fmt.Errorf("unsupported regular model cache mode %q", selection.Mode) + } +} + +func regularModelCacheRuntimeDecision( + req *nvcav2beta1.ICMSRequest, + legacyEnabled bool, +) (enabled bool, persisted bool, err error) { + selection, err := persistedRegularModelCacheSelection(req) + if err != nil { + return false, false, err + } + if selection == nil { + return legacyEnabled, false, nil + } + return selection.Mode == nvcastorage.ModelCacheSelectionDurable, true, nil +} + +func validatePersistedRegularModelCacheArtifacts( + selection *nvcastorage.PersistedModelCacheStorageSelection, + rwPVC, initJob function.LaunchArtifact, +) error { + if selection == nil || selection.Mode != nvcastorage.ModelCacheSelectionDurable { + return nil + } + if rwPVC.Specification == "" || initJob.Specification == "" { + return nvcaerrors.TerminalError(fmt.Errorf( + "persisted durable regular model cache requires both PVC and init Job artifacts")) + } + return nil +} + func (c K8sComputeBackend) setupContainerModelCaching(ctx context.Context, req *nvcav2beta1.ICMSRequest, rwPVC *corev1.PersistentVolumeClaim, @@ -670,8 +770,60 @@ func (c K8sComputeBackend) setupContainerModelCaching(ctx context.Context, cachemf mutateFunc, ) (mf func(*corev1.Pod), roPVCName string, err error) { log := core.GetLogger(ctx) + selection, selectionErr := persistedRegularModelCacheSelection(req) + if selectionErr != nil { + return nil, "", nvcaerrors.TerminalError(selectionErr) + } + persistedDurable := false + encryptionRequired := c.bk8s.nvmeshEncryptionEnabled + if selection != nil { + switch selection.Mode { + case nvcastorage.ModelCacheSelectionNone: + log.Debug("Persisted model cache selection disables durable caching") + return func(*corev1.Pod) {}, "", nil + case nvcastorage.ModelCacheSelectionDurable: + persistedDurable = true + } + binding, err := c.bk8s.activeModelCacheBindingForRuntime(ctx, req) + if err != nil { + return nil, "", err + } + encryptionRequired = selection.EncryptionRequired + switch selection.Transition { + case nvcastorage.ModelCacheTransitionROXReadOnly: + case nvcastorage.ModelCacheTransitionRWXReadOnly: + if encryptionRequired { + return nil, "", nvcaerrors.TerminalError(fmt.Errorf( + "rwxReadOnly model cache transition cannot require NVMesh encryption")) + } + rwPVC.Spec.AccessModes = append( + []corev1.PersistentVolumeAccessMode(nil), RWXAccessMode...) + default: + return nil, "", nvcaerrors.TerminalError(fmt.Errorf( + "unsupported regular model cache transition %q", selection.Transition)) + } + expectedStorageClass, err := regularModelCacheExpectedStorageClassName(binding) + if err != nil { + return nil, "", nvcaerrors.TerminalError(err) + } + if rwPVC.Spec.StorageClassName != nil && *rwPVC.Spec.StorageClassName != "" && + *rwPVC.Spec.StorageClassName != selection.StorageClassName && + *rwPVC.Spec.StorageClassName != expectedStorageClass { + return nil, "", nvcaerrors.TerminalError(fmt.Errorf( + "regular model cache StorageClass override %q conflicts with persisted intent %q", + *rwPVC.Spec.StorageClassName, expectedStorageClass)) + } + rwPVC.Spec.StorageClassName = &expectedStorageClass + if err := c.prepareRegularModelCacheBindingResources(ctx, binding, rwPVC, initJob); err != nil { + return nil, "", err + } + } // setup init cache Job writer and RWMany PVC - mc, roPVCName := c.SetupModelCachingForRequest(ctx, rwPVC, initJob, req, cachemf) + mc, roPVCName, modelCacheErr := c.SetupModelCachingForRequest( + ctx, rwPVC, initJob, req, encryptionRequired, cachemf) + if modelCacheErr != nil { + return nil, "", modelCacheErr + } switch mc { case ModelCachingCompleted: log.Infof("model caching completed, starting worker creation") @@ -688,6 +840,8 @@ func (c K8sComputeBackend) setupContainerModelCaching(ctx context.Context, } } } + setModelCacheVolumeMountsReadOnly(pod.Spec.InitContainers) + setModelCacheVolumeMountsReadOnly(pod.Spec.Containers) } return mf, roPVCName, nil case ModelCachingInProgress: @@ -703,6 +857,10 @@ func (c K8sComputeBackend) setupContainerModelCaching(ctx context.Context, } return nil, "", fmt.Errorf("model caching is still in progress") case ModelCachingFailed: + if persistedDurable { + return nil, "", nvcaerrors.TerminalError( + fmt.Errorf("model caching failed for persisted durable regular cache selection")) + } c.bk8s.EmitICMSEvent(req, corev1.EventTypeWarning, string(types.EventCategoryModelCaching), "Caching setup failed, resort to non-cached workers", nil) log.Warnf("model caching failed, NVCA will create non-cached workers") @@ -710,6 +868,16 @@ func (c K8sComputeBackend) setupContainerModelCaching(ctx context.Context, return func(*corev1.Pod) {}, "", nil } +func setModelCacheVolumeMountsReadOnly(containers []corev1.Container) { + for containerIndex := range containers { + for mountIndex := range containers[containerIndex].VolumeMounts { + if containers[containerIndex].VolumeMounts[mountIndex].Name == ModelVolumeName { + containers[containerIndex].VolumeMounts[mountIndex].ReadOnly = true + } + } + } +} + func getPullSecretsFromArtifacts(ctx context.Context, req *nvcav2beta1.ICMSRequest, ) (workerImagePullSecrets, workloadImagePullSecrets []*corev1.Secret, err error) { diff --git a/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend_modelcache.go b/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend_modelcache.go index a36f84cf1..6b1c0be33 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend_modelcache.go +++ b/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend_modelcache.go @@ -19,8 +19,10 @@ package nvca import ( "context" + stderrors "errors" "fmt" "reflect" + "slices" "strconv" "strings" "time" @@ -33,6 +35,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" + k8stypes "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" @@ -42,6 +45,8 @@ import ( nvcav1new "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v1" nvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/encryption" + nvcaerrors "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/errors" + nvcastorage "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/storage" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/types" ) @@ -49,6 +54,7 @@ import ( var ( skipVolumeDetachCheck = false ROAccessMode = []v1.PersistentVolumeAccessMode{v1.ReadOnlyMany} + RWXAccessMode = []v1.PersistentVolumeAccessMode{v1.ReadWriteMany} ) type PVCState string @@ -100,8 +106,23 @@ func (c K8sComputeBackend) CleanupModelCachingSetupArtifacts(ctx context.Context _, _, _, _, icjDecoded, bdDecode := getArtifactsFromReq(req) isMiniServiceType := req.Spec.CreationMsgInfo.FunctionLaunchSpecification != nil && req.Spec.CreationMsgInfo.FunctionLaunchSpecification.HelmChartLaunchSpecification != nil - - if !c.bk8s.cachingSupportEnabled || isMiniServiceType || icjDecoded.Specification == "" { + if isMiniServiceType { + return nil + } + binding, cleanupSharedResources, err := c.regularModelCacheCleanupBinding(ctx, req) + if err != nil { + return fmt.Errorf("resolve regular model cache binding for setup artifact cleanup: %w", err) + } + if !cleanupSharedResources { + return nil + } + if binding == nil && !c.bk8s.cachingSupportEnabled { + return nil + } + if icjDecoded.Specification == "" || bdDecode.Specification == "" { + if binding != nil { + return fmt.Errorf("persisted durable regular model cache has incomplete cleanup artifacts") + } return nil } @@ -112,22 +133,34 @@ func (c K8sComputeBackend) CleanupModelCachingSetupArtifacts(ctx context.Context log.WithError(err).Error("failed getModelCacheK8sArtifacts, model caching will be disabled") return fmt.Errorf("failed to cleanup in-flight cache job: %w", err) } + if binding != nil { + // Binding-scoped cleanup must use the transition-aware path. It proves PV + // ownership, changes an exact bound Retain PV to Delete, and then removes + // the Job and PVC with identity preconditions. The same request may resume + // this cleanup after the binding entered Retiring above. + return c.CleanupModelCachingResources(ctx, req, rwPVC, initJob.Name) + } - // cleanup InitJob & its pods, this will clear the rw-pvc also - backgroundDeletion := metav1.DeletePropagationBackground - err = c.clients.K8s.BatchV1().Jobs(c.bk8s.podInstanceNamespace).Delete(ctx, initJob.Name, metav1.DeleteOptions{ - PropagationPolicy: &backgroundDeletion, - }) - if err != nil && !errors.IsNotFound(err) { - log.WithError(err).Warnf("failed to cleanup initCacheJob %v/%v in SetupPVCForReaders, needs manual cleanup", - c.bk8s.podInstanceNamespace, initJob.Name) - return fmt.Errorf("failed to cleanup in-flight cache job: %w", err) + // Preserve the annotation-free legacy cleanup behavior. + namespace := c.bk8s.podInstanceNamespace + if initJob != nil { + backgroundDeletion := metav1.DeletePropagationBackground + deleteOptions := metav1.DeleteOptions{PropagationPolicy: &backgroundDeletion} + err = c.clients.K8s.BatchV1().Jobs(namespace). + Delete(ctx, initJob.Name, deleteOptions) + if err != nil && !errors.IsNotFound(err) { + log.WithError(err).Warnf("failed to cleanup initCacheJob %v/%v, needs manual cleanup", + namespace, initJob.Name) + return fmt.Errorf("failed to cleanup in-flight cache job: %w", err) + } } - // now purge the RWPVC - err = c.clients.K8s.CoreV1().PersistentVolumeClaims(c.bk8s.podInstanceNamespace).Delete(ctx, rwPVC.Name, metav1.DeleteOptions{}) - if err != nil && !errors.IsNotFound(err) { - return fmt.Errorf("failed to delete pvc %v, err: %v", rwPVC.Name, err) + if rwPVC != nil { + err = c.clients.K8s.CoreV1().PersistentVolumeClaims(namespace). + Delete(ctx, rwPVC.Name, metav1.DeleteOptions{}) + if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("failed to delete pvc %v: %w", rwPVC.Name, err) + } } return nil } @@ -136,8 +169,9 @@ func (c K8sComputeBackend) SetupModelCachingForRequest(ctx context.Context, rwPVC *v1.PersistentVolumeClaim, initJob *batchv1.Job, req *nvcav2beta1.ICMSRequest, + encryptionRequired bool, mf mutateFunc, -) (ModelCachingState, string) { +) (ModelCachingState, string, error) { c.bk8s.modelCacheMtx.Lock() defer c.bk8s.modelCacheMtx.Unlock() @@ -145,131 +179,261 @@ func (c K8sComputeBackend) SetupModelCachingForRequest(ctx context.Context, log.Debugf("decoding caching artifacts") metrics := nvcametrics.FromContext(ctx) - if c.bk8s.nvmeshEncryptionEnabled { + bindingUID, bindingScoped, err := regularModelCacheBindingUID(req) + if err != nil { + return ModelCachingFailed, "", nvcaerrors.TerminalError( + fmt.Errorf("resolve regular model cache binding ownership: %w", err)) + } + + var binding *nvcav2beta1.ModelCacheBinding + if bindingScoped { + binding, err = c.bk8s.activeModelCacheBindingForRuntime(ctx, req) + if err != nil { + if stderrors.Is(err, errRegularModelCacheBindingRetiring) { + cleanupErr := c.CleanupModelCachingResources(ctx, req, rwPVC, initJob.Name) + if cleanupErr != nil { + return regularModelCacheResultForError(true, cleanupErr) + } + } + return regularModelCacheResultForError(true, err) + } + if bindingScoped && binding != nil && + binding.Spec.Decision.Transition == nvcastorage.ModelCacheTransitionRWXReadOnly { + if encryptionRequired { + return regularModelCacheResultForError(true, nvcaerrors.TerminalError( + fmt.Errorf("rwxReadOnly model cache transition does not support NVMesh encryption"))) + } + return c.setupRWXReadOnlyModelCachingForRequest( + ctx, rwPVC, initJob, req, binding, bindingUID) + } + } + if encryptionRequired { //If encryption is used, then we need to update StorageClass name in PVC. storageClassName, err := encryption.SetupEncryption(ctx, c.clients, req.Spec.NCAId, req.Namespace) if err != nil { log.WithError(err).Error("failed to set up cache encryption, resort to non-caching") - return ModelCachingFailed, "" + return regularModelCacheResultForError(bindingScoped, + fmt.Errorf("set up regular model cache encryption: %w", err)) } - *rwPVC.Spec.StorageClassName = storageClassName + if bindingScoped { + expectedStorageClass, expectedErr := regularModelCacheExpectedStorageClassName(binding) + if expectedErr != nil { + return regularModelCacheResultForError(true, expectedErr) + } + if storageClassName != expectedStorageClass { + return regularModelCacheResultForError(true, fmt.Errorf( + "derived encrypted StorageClass %q does not match binding intent %q", storageClassName, expectedStorageClass)) + } + } + rwPVC.Spec.StorageClassName = &storageClassName } - roPVCName := strings.ReplaceAll(rwPVC.Name, RWPVCSuffix, ROPVCSuffix) - roPVCState, err := c.CheckPVCState(ctx, roPVCName) + roPVCName, err := regularModelCacheReaderPVCName(rwPVC.Name) + if err != nil { + return regularModelCacheResultForError(bindingScoped, nvcaerrors.TerminalError(err)) + } + roPVCState, err := c.checkPVCState(ctx, roPVCName, bindingUID, bindingScoped, true, binding) switch roPVCState { case PVCNotFound: - jS := c.CheckInitCacheJobState(ctx, rwPVC.Name, initJob) + jS, jobStateErr := c.CheckInitCacheJobState(ctx, rwPVC.Name, initJob, bindingUID, bindingScoped) + if jobStateErr != nil && bindingScoped { + return regularModelCacheResultForError(true, jobStateErr) + } switch jS { case InitCacheJobNotFound: pvLabelSel, err := makePVLabelSelectorForCacheRequest(req) if err != nil { log.WithError(err).Error("failed to create label requirement for cache PV, resort to non-caching") - return ModelCachingFailed, "" + return regularModelCacheResultForError(bindingScoped, err) } // check if PV for the function exists, if so, continue as ModelCachingInProgress // lets find the underlying PV for this function/task pvObjList, err := c.clients.K8s.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{ LabelSelector: pvLabelSel}) - if (err != nil && errors.IsNotFound(err)) || (pvObjList != nil && len(pvObjList.Items) == 0) { + if err != nil && !errors.IsNotFound(err) { + if bindingScoped { + return regularModelCacheResultForError(true, + fmt.Errorf("list regular model cache PVs: %w", err)) + } + return ModelCachingFailed, "", nil + } + if errors.IsNotFound(err) || (pvObjList != nil && len(pvObjList.Items) == 0) { err = c.SetupInitCacheJobBlockDevice(ctx, rwPVC, initJob, req) if err != nil { c.bk8s.EmitICMSEvent(req, v1.EventTypeWarning, string(types.EventCategoryModelCaching), "failed caching setup, resort to non-caching", nil) log.WithError(err).Error("failed SetupInitCacheJobBlockDevice, model caching will be disabled") - return ModelCachingFailed, "" + return regularModelCacheResultForError(bindingScoped, err) } - return ModelCachingInProgress, "" - } else if pvObjList != nil && len(pvObjList.Items) == 1 { + return ModelCachingInProgress, "", nil + } + if pvObjList != nil && len(pvObjList.Items) == 1 { // let it reconcile again - mc := ModelCachingInProgress - roPVCState, err := c.SetupPVCForReaders(ctx, rwPVC, initJob.Name, req, mf) - if err != nil { - log.WithError(err).Errorf("failed to SetupPVCForReaders at %v, model caching will be disabled", roPVCState) - err = c.CleanupModelCachingResources(ctx, rwPVC, initJob.Name) - if err != nil { - log.WithError(err).Error("failed to cleanup ModelCaching resources, needs manual cleanup") + roPVCState, setupErr := c.SetupPVCForReaders(ctx, rwPVC, initJob.Name, req, mf) + if setupErr == nil { + return ModelCachingInProgress, "", nil + } + if bindingScoped && regularModelCacheErrorIsRetryable(setupErr) { + return ModelCachingInProgress, "", setupErr + } + log.WithError(setupErr).Errorf( + "failed to SetupPVCForReaders at %v, model caching will be disabled", roPVCState) + cleanupErr := c.CleanupModelCachingResources(ctx, req, rwPVC, initJob.Name) + if cleanupErr != nil { + log.WithError(cleanupErr).Error( + "failed to cleanup ModelCaching resources, needs manual cleanup") + if bindingScoped { + return regularModelCacheResultForError(true, cleanupErr) } - c.bk8s.EmitICMSEvent(req, v1.EventTypeWarning, - string(types.EventCategoryModelCaching), "failed pvc setup, resort to non-caching", nil) - metrics.EventErrorTotal.WithLabelValues(metrics.WithDefaultLabelValues(EventModelCachingFailed)...).Inc() - mc = ModelCachingFailed } - return mc, "" + c.bk8s.EmitICMSEvent(req, v1.EventTypeWarning, + string(types.EventCategoryModelCaching), "failed pvc setup, resort to non-caching", nil) + metrics.EventErrorTotal.WithLabelValues( + metrics.WithDefaultLabelValues(EventModelCachingFailed)...).Inc() + if bindingScoped { + return regularModelCacheResultForError(true, setupErr) + } + return ModelCachingFailed, "", nil } + return ModelCachingFailed, "", nil case InitCacheJobFailed: // this is an irrecoverable error on InitCacheJob, NVCA will switch to - // No Caching Workflow - // Caller will need to use the PodSpec without ROPVC VolumeMount - err = c.CleanupModelCachingResources(ctx, rwPVC, initJob.Name) - if err != nil { - log.WithError(err).Error("failed to cleanup ModelCaching resources, needs manual cleanup") + // No Caching Workflow. + cleanupErr := c.CleanupModelCachingResources(ctx, req, rwPVC, initJob.Name) + if cleanupErr != nil { + log.WithError(cleanupErr).Error( + "failed to cleanup ModelCaching resources, needs manual cleanup") + if bindingScoped { + return regularModelCacheResultForError(true, cleanupErr) + } } c.bk8s.EmitICMSEventf(req, v1.EventTypeWarning, string(types.EventCategoryModelCaching), "%v failed, resort to non-caching", nil, initJob.Name) reason := c.getInitCacheJobFailureReason(ctx, initJob) - metrics.RecordModelCacheResult(modelcachetypes.ResultFailure, reason, string(types.HelmCacheBackendNVMesh)) - return ModelCachingFailed, "" + metrics.RecordModelCacheResult( + modelcachetypes.ResultFailure, reason, string(types.HelmCacheBackendNVMesh)) + if bindingScoped { + return regularModelCacheResultForError(true, + fmt.Errorf("init cache Job %s failed", initJob.Name)) + } + return ModelCachingFailed, "", nil case InitCacheJobCompleted: - mc := ModelCachingInProgress - roPVCState, err := c.SetupPVCForReaders(ctx, rwPVC, initJob.Name, req, mf) - if err != nil { - log.WithError(err).Errorf("failed to SetupPVCForReaders at %v, model caching will be disabled", roPVCState) - err = c.CleanupModelCachingResources(ctx, rwPVC, initJob.Name) - if err != nil { - log.WithError(err).Error("failed to cleanup ModelCaching resources, needs manual cleanup") + roPVCState, setupErr := c.SetupPVCForReaders(ctx, rwPVC, initJob.Name, req, mf) + if setupErr == nil { + return ModelCachingInProgress, "", nil + } + if bindingScoped && regularModelCacheErrorIsRetryable(setupErr) { + return ModelCachingInProgress, "", setupErr + } + log.WithError(setupErr).Errorf( + "failed to SetupPVCForReaders at %v, model caching will be disabled", roPVCState) + cleanupErr := c.CleanupModelCachingResources(ctx, req, rwPVC, initJob.Name) + if cleanupErr != nil { + log.WithError(cleanupErr).Error( + "failed to cleanup ModelCaching resources, needs manual cleanup") + if bindingScoped { + return regularModelCacheResultForError(true, cleanupErr) } - c.bk8s.EmitICMSEvent(req, v1.EventTypeWarning, - string(types.EventCategoryModelCaching), "failed pvc setup, resort to non-caching", nil) - metrics.EventErrorTotal.WithLabelValues(metrics.WithDefaultLabelValues(EventModelCachingFailed)...).Inc() - metrics.RecordModelCacheResult(modelcachetypes.ResultFailure, modelcachetypes.ReasonPVCSetupFailed, string(types.HelmCacheBackendNVMesh)) - mc = ModelCachingFailed } - return mc, "" + c.bk8s.EmitICMSEvent(req, v1.EventTypeWarning, + string(types.EventCategoryModelCaching), "failed pvc setup, resort to non-caching", nil) + metrics.EventErrorTotal.WithLabelValues( + metrics.WithDefaultLabelValues(EventModelCachingFailed)...).Inc() + metrics.RecordModelCacheResult(modelcachetypes.ResultFailure, + modelcachetypes.ReasonPVCSetupFailed, string(types.HelmCacheBackendNVMesh)) + if bindingScoped { + return regularModelCacheResultForError(true, setupErr) + } + return ModelCachingFailed, "", nil case InitCacheJobInProgress: - return ModelCachingInProgress, "" + return ModelCachingInProgress, "", nil } case PVCQueryError: log.WithError(err).Error("failed to query ROPVC") - // this is a transient error, will reattempt - return ModelCachingInProgress, "" + if bindingScoped { + return regularModelCacheResultForError(true, err) + } + // Preserve the annotation-free legacy retry behavior. + return ModelCachingInProgress, "", nil case PVCFoundUnBound: - // if it has been more than 2 mins since PVC was created - // Clear RWPVC, ROPVC, InitCacheJob and Disable Model Caching on the Request log.Debugf("ROPVC is still unbound, continue wait") - return ModelCachingInProgress, "" + return ModelCachingInProgress, "", nil case PVCFoundBindFailed: - log.WithError(err).Errorf("ROPVC is not getting bound, cleanup Modelcaching resource and deploy without caching") - err := c.CleanupModelCachingResources(ctx, rwPVC, initJob.Name) - if err != nil { - // TODO: Perform Deeper Cleanup on reconciliation - log.WithError(err).Errorf("failed to cleanup ModelCaching resources, needs manual cleanup") + log.WithError(err).Errorf( + "ROPVC is not getting bound, cleanup Modelcaching resource and deploy without caching") + cleanupErr := c.CleanupModelCachingResources(ctx, req, rwPVC, initJob.Name) + if cleanupErr != nil { + log.WithError(cleanupErr).Errorf( + "failed to cleanup ModelCaching resources, needs manual cleanup") + if bindingScoped { + return regularModelCacheResultForError(true, cleanupErr) + } } c.bk8s.EmitICMSEventf(req, v1.EventTypeWarning, string(types.EventCategoryModelCaching), "%v bind failed, resort to non-caching", nil, roPVCName) - metrics.EventErrorTotal.WithLabelValues(metrics.WithDefaultLabelValues(EventPVCModelCachingError)...).Inc() - metrics.EventErrorTotal.WithLabelValues(metrics.WithDefaultLabelValues(EventModelCachingFailed)...).Inc() - metrics.RecordModelCacheResult(modelcachetypes.ResultFailure, modelcachetypes.ReasonPVCBindFailed, string(types.HelmCacheBackendNVMesh)) - return ModelCachingFailed, "" + metrics.EventErrorTotal.WithLabelValues( + metrics.WithDefaultLabelValues(EventPVCModelCachingError)...).Inc() + metrics.EventErrorTotal.WithLabelValues( + metrics.WithDefaultLabelValues(EventModelCachingFailed)...).Inc() + metrics.RecordModelCacheResult(modelcachetypes.ResultFailure, + modelcachetypes.ReasonPVCBindFailed, string(types.HelmCacheBackendNVMesh)) + if bindingScoped { + if err == nil { + err = fmt.Errorf("reader PVC %s bind failed", roPVCName) + } + return regularModelCacheResultForError(true, err) + } + return ModelCachingFailed, "", nil case PVCFoundBound: - log.Infof("ROPVC %v setup completed, Modelcaching will be enabled for request %v/%v", roPVCName, req.Namespace, req.Name) - // cleanup InitJob & its pods - // rw-pvc is deleted in setup of ro-pvc - backgroundDeletion := metav1.DeletePropagationBackground - err = c.clients.K8s.BatchV1().Jobs(c.bk8s.podInstanceNamespace).Delete(ctx, initJob.Name, metav1.DeleteOptions{ - PropagationPolicy: &backgroundDeletion, - }) - if err != nil && !errors.IsNotFound(err) { - log.WithError(err).Warnf("failed to cleanup initCacheJob %v/%v, needs manual cleanup", - c.bk8s.podInstanceNamespace, initJob.Name) + log.Infof("ROPVC %v setup completed, Modelcaching will be enabled for request %v/%v", + roPVCName, req.Namespace, req.Name) + transitionTargets, _, transitionErr := c.regularModelCacheTransitionTargets( + ctx, req, rwPVC.Name, initJob.Name) + if transitionErr != nil { + log.WithError(transitionErr).Error("refusing unverified model cache transition cleanup") + return regularModelCacheResultForError(bindingScoped, transitionErr) } - metrics.EventErrorTotal.WithLabelValues(metrics.WithDefaultLabelValues(EventModelCachingSuccess)...).Inc() - metrics.RecordModelCacheResult(modelcachetypes.ResultSuccess, "", string(types.HelmCacheBackendNVMesh)) - return ModelCachingCompleted, roPVCName + // The successful writer-to-reader transition publishes the shared cache, + // so another request reference does not block this exact Job cleanup. + jobNamespace := c.bk8s.podInstanceNamespace + jobToDelete := initJob + if bindingScoped { + jobNamespace = transitionTargets.namespace + jobToDelete = transitionTargets.initJob + } + if jobToDelete != nil { + backgroundDeletion := metav1.DeletePropagationBackground + deleteOptions := metav1.DeleteOptions{PropagationPolicy: &backgroundDeletion} + if bindingScoped { + deleteOptions, err = modelCacheDeleteOptions(jobToDelete, &backgroundDeletion) + if err != nil { + return regularModelCacheResultForError(true, err) + } + } + err = c.clients.K8s.BatchV1().Jobs(jobNamespace). + Delete(ctx, jobToDelete.Name, deleteOptions) + if err != nil && !errors.IsNotFound(err) { + log.WithError(err).Warnf( + "failed to cleanup initCacheJob %v/%v, needs manual cleanup", + jobNamespace, jobToDelete.Name) + if bindingScoped { + return regularModelCacheResultForError(true, + fmt.Errorf("delete binding-owned init cache Job: %w", err)) + } + } + } + metrics.EventErrorTotal.WithLabelValues( + metrics.WithDefaultLabelValues(EventModelCachingSuccess)...).Inc() + metrics.RecordModelCacheResult( + modelcachetypes.ResultSuccess, "", string(types.HelmCacheBackendNVMesh)) + return ModelCachingCompleted, roPVCName, nil + } + if bindingScoped { + return regularModelCacheResultForError(true, + fmt.Errorf("unexpected regular model cache state %q", roPVCState)) } - // Never reached - return ModelCachingFailed, "" + return ModelCachingFailed, "", nil } // references for K8sComputeBackend are that of PVCNames @@ -287,6 +451,11 @@ func (c K8sComputeBackend) ComputeCleanupCacheReferences(ctx context.Context, ca log.WithError(err).Errorf("failed to cleanup PVC %v/%v and backing PV, needs manual cleanup", c.bk8s.podInstanceNamespace, pvc) continue } + if bindingUID := pvcObj.Labels[nvcastorage.ModelCacheBindingUIDLabelKey]; bindingUID != "" { + log.Infof("skipping periodic cleanup of PVC %v/%v owned by model cache binding %v", + c.bk8s.podInstanceNamespace, pvc, bindingUID) + continue + } pvName := pvcObj.Spec.VolumeName if pvName != "" { retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { @@ -316,52 +485,196 @@ func (c K8sComputeBackend) ComputeCleanupCacheReferences(ctx context.Context, ca } return nil } +func regularModelCacheErrorIsRetryable(err error) bool { + if err == nil || nvcaerrors.IsTerminal(err) { + return false + } + return nvcak8sutil.IsTransientK8sError(err) +} + +func regularModelCacheErrorIsKubernetesAPI(err error) bool { + var apiStatus errors.APIStatus + return stderrors.As(err, &apiStatus) +} + +func regularModelCacheResultForError( + bindingScoped bool, + err error, +) (ModelCachingState, string, error) { + if !bindingScoped { + return ModelCachingFailed, "", nil + } + if err == nil { + err = fmt.Errorf("regular model cache operation failed without an error") + } + if nvcaerrors.IsTerminal(err) { + return ModelCachingFailed, "", err + } + if regularModelCacheErrorIsRetryable(err) || + regularModelCacheErrorIsKubernetesAPI(err) { + return ModelCachingInProgress, "", err + } + return ModelCachingFailed, "", nvcaerrors.TerminalError(err) +} + +func modelCacheDeleteOptions( + obj metav1.Object, + propagationPolicy *metav1.DeletionPropagation, +) (metav1.DeleteOptions, error) { + options := metav1.DeleteOptions{PropagationPolicy: propagationPolicy} + if obj == nil || reflect.ValueOf(obj).IsNil() { + return options, fmt.Errorf("binding-owned model cache delete target is nil") + } + if obj.GetUID() == "" || obj.GetResourceVersion() == "" { + return options, fmt.Errorf( + "binding-owned model cache object %s/%s has incomplete delete identity (UID %q, resourceVersion %q)", + obj.GetNamespace(), obj.GetName(), obj.GetUID(), obj.GetResourceVersion()) + } + uid := obj.GetUID() + resourceVersion := obj.GetResourceVersion() + options.Preconditions = &metav1.Preconditions{ + UID: &uid, + ResourceVersion: &resourceVersion, + } + return options, nil +} + +func validateRegularModelCachePVClaimForSetup( + pv *v1.PersistentVolume, + binding *nvcav2beta1.ModelCacheBinding, + pvc *v1.PersistentVolumeClaim, + namespace string, + rwPVCName string, + roPVCName string, +) error { + if pv == nil || pv.Spec.ClaimRef == nil { + return fmt.Errorf("regular model cache PV has no claimRef") + } + claimRef := pv.Spec.ClaimRef + if pvc != nil { + return validateRegularModelCachePVForPVC(binding, pvc, pv, + []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}) + } + if claimRef.Namespace != namespace || (claimRef.Name != rwPVCName && claimRef.Name != roPVCName) { + return fmt.Errorf("PV %s claimRef does not match binding-owned PVC %s/%s or %s", + pv.Name, namespace, rwPVCName, roPVCName) + } + expectedModes := []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce} + if claimRef.Name == roPVCName { + expectedModes = ROAccessMode + } + return validateRegularModelCachePVIdentity(binding, pv, expectedModes) +} func (c K8sComputeBackend) CleanupModelCachingResources(ctx context.Context, - rwPVC *v1.PersistentVolumeClaim, initJobName string) error { + req *nvcav2beta1.ICMSRequest, + rwPVC *v1.PersistentVolumeClaim, + initJobName string, +) error { log := core.GetLogger(ctx) - var pvcObj *v1.PersistentVolumeClaim - var err error + if rwPVC == nil { + return fmt.Errorf("regular model cache cleanup PVC is nil") + } - // cleanup InitJob & its pods - backgroundDeletion := metav1.DeletePropagationBackground - err = c.clients.K8s.BatchV1().Jobs(c.bk8s.podInstanceNamespace).Delete(ctx, initJobName, metav1.DeleteOptions{ - PropagationPolicy: &backgroundDeletion, - }) - if err != nil && !errors.IsNotFound(err) { - log.WithError(err).Warnf("failed to cleanup initCacheJob %v/%v in SetupPVCForReaders, needs manual cleanup", - c.bk8s.podInstanceNamespace, initJobName) + binding, cleanupSharedResources, err := c.regularModelCacheCleanupBinding(ctx, req) + if err != nil { + return fmt.Errorf("resolve regular model cache binding for cleanup: %w", err) + } + if !cleanupSharedResources { + return nil + } + if binding == nil { + // Preserve the annotation-free legacy ordering and best-effort Job cleanup. + backgroundDeletion := metav1.DeletePropagationBackground + err = c.clients.K8s.BatchV1().Jobs(c.bk8s.podInstanceNamespace).Delete( + ctx, initJobName, metav1.DeleteOptions{PropagationPolicy: &backgroundDeletion}) + if err != nil && !errors.IsNotFound(err) { + log.WithError(err).Warnf("failed to cleanup initCacheJob %v/%v, needs manual cleanup", + c.bk8s.podInstanceNamespace, initJobName) + } } - // ROPVC - roPVCName := strings.ReplaceAll(rwPVC.Name, RWPVCSuffix, ROPVCSuffix) - // Get the BackedPV Object - pvcObj, err = c.clients.K8s.CoreV1().PersistentVolumeClaims(c.bk8s.podInstanceNamespace).Get(ctx, roPVCName, metav1.GetOptions{}) + roPVCName, err := regularModelCacheReaderPVCName(rwPVC.Name) if err != nil { + return err + } + namespace := c.bk8s.podInstanceNamespace + var targets *regularModelCacheCleanupTargets + if binding != nil { + targets, err = c.validateRegularModelCacheCleanupTargets(ctx, binding, rwPVC.Name, initJobName) + if err != nil { + return fmt.Errorf("validate regular model cache cleanup targets: %w", err) + } + namespace = targets.namespace + } + + var pvcObj *v1.PersistentVolumeClaim + if binding != nil { + pvcObj = targets.roPVC + if pvcObj == nil { + pvcObj = targets.rwPVC + } + } else { + pvcObj, err = c.clients.K8s.CoreV1().PersistentVolumeClaims(namespace). + Get(ctx, roPVCName, metav1.GetOptions{}) if errors.IsNotFound(err) { log.Debugf("ROPVC was never setup, try obtaining the RWPVC") - pvcObj, err = c.clients.K8s.CoreV1().PersistentVolumeClaims(c.bk8s.podInstanceNamespace).Get(ctx, rwPVC.Name, metav1.GetOptions{}) - if err != nil { - if errors.IsNotFound(err) { - log.Warnf("RWPVC was also never setup, no PV to update and no PVCs to cleanup") - return nil - } - return fmt.Errorf("failed to get ROPVC, err: %v", err) - } - } else { + pvcObj, err = c.clients.K8s.CoreV1().PersistentVolumeClaims(namespace). + Get(ctx, rwPVC.Name, metav1.GetOptions{}) + } + if err != nil && !errors.IsNotFound(err) { return fmt.Errorf("failed to get ROPVC, err: %v", err) } } - // get the pvName - pvName := pvcObj.Spec.VolumeName + var pvName string + if pvcObj != nil { + pvName = pvcObj.Spec.VolumeName + } + // For binding-scoped cleanup, prove PV ownership before the first write. + if binding != nil && pvName != "" { + pvObj, getErr := c.clients.K8s.CoreV1().PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}) + if getErr != nil { + return fmt.Errorf("get regular model cache PV %v before cleanup: %w", pvName, getErr) + } + if err := validateRegularModelCacheCleanupPV(binding, pvcObj, pvObj); err != nil { + return fmt.Errorf("validate regular model cache PV before cleanup: %w", err) + } + } + + // cleanup InitJob & its pods only after all binding-owned targets have been validated. + if binding != nil && targets.initJob != nil { + backgroundDeletion := metav1.DeletePropagationBackground + deleteOptions, optionsErr := modelCacheDeleteOptions(targets.initJob, &backgroundDeletion) + if optionsErr != nil { + return fmt.Errorf("build binding-owned init Job delete preconditions: %w", optionsErr) + } + err = c.clients.K8s.BatchV1().Jobs(namespace). + Delete(ctx, targets.initJob.Name, deleteOptions) + if err != nil && !errors.IsNotFound(err) { + log.WithError(err).Warnf("failed to cleanup initCacheJob %v/%v, needs manual cleanup", + namespace, initJobName) + return fmt.Errorf("delete binding-owned init cache Job: %w", err) + } + } + + if pvcObj == nil { + log.Warnf("RWPVC and ROPVC were never setup, no PV or PVC to cleanup") + return nil + } + if pvName != "" { retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { // Retrieve the latest version of PV before attempting update // RetryOnConflict uses exponential backoff to avoid exhausting the apiserver pvObj, err := c.clients.K8s.CoreV1().PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}) if err != nil { - return fmt.Errorf("failed to get PV %v to update with PersistentVolumeReclaimPolicy:Delete: %v", pvName, err) + return fmt.Errorf("failed to get PV %v to update with PersistentVolumeReclaimPolicy:Delete: %w", pvName, err) + } + if binding != nil { + if err := validateRegularModelCacheCleanupPV(binding, pvcObj, pvObj); err != nil { + return err + } } // update policy @@ -371,22 +684,42 @@ func (c K8sComputeBackend) CleanupModelCachingResources(ctx context.Context, return updateErr }) if retryErr != nil { - return fmt.Errorf("failed to update PV %v with PersistentVolumeReclaimPolicy:Delete, err: %v", pvName, err) + return fmt.Errorf("failed to update PV %v with PersistentVolumeReclaimPolicy:Delete: %w", pvName, retryErr) } } else { - log.WithError(err).Errorf("unabled to set PersistentVolumeReclaimPolicy because PV name is unknown") + log.Errorf("unable to set PersistentVolumeReclaimPolicy because PV name is unknown") } - // deleting the RWPVC - err = c.clients.K8s.CoreV1().PersistentVolumeClaims(c.bk8s.podInstanceNamespace).Delete(ctx, rwPVC.Name, metav1.DeleteOptions{}) - if err != nil && !errors.IsNotFound(err) { - return fmt.Errorf("failed to delete RWPVC, err:%v", err) + pvcsToDelete := []struct { + kind string + name string + obj *v1.PersistentVolumeClaim + }{ + {kind: "RWPVC", name: rwPVC.Name}, + {kind: "ROPVC", name: roPVCName}, } - - // delete the ROPVC - err = c.clients.K8s.CoreV1().PersistentVolumeClaims(c.bk8s.podInstanceNamespace).Delete(ctx, roPVCName, metav1.DeleteOptions{}) - if err != nil && !errors.IsNotFound(err) { - return fmt.Errorf("failed to delete ROPVC, err:%v", err) + if binding != nil { + pvcsToDelete[0].obj = targets.rwPVC + pvcsToDelete[1].obj = targets.roPVC + } + for _, target := range pvcsToDelete { + if binding != nil && target.obj == nil { + continue + } + deleteOptions := metav1.DeleteOptions{} + if binding != nil { + var optionsErr error + deleteOptions, optionsErr = modelCacheDeleteOptions(target.obj, nil) + if optionsErr != nil { + return fmt.Errorf("build binding-owned %s delete preconditions: %w", + target.kind, optionsErr) + } + } + err = c.clients.K8s.CoreV1().PersistentVolumeClaims(namespace). + Delete(ctx, target.name, deleteOptions) + if err != nil && !errors.IsNotFound(err) { + return fmt.Errorf("failed to delete %s: %w", target.kind, err) + } } return nil } @@ -425,28 +758,91 @@ Returns: PVCFoundBound -> ROPVCFound and Usable, Workers Can be created with this PVC Name for volume Name */ -func (c K8sComputeBackend) CheckPVCState(ctx context.Context, roPVCName string) (PVCState, error) { +func (c K8sComputeBackend) CheckPVCState( + ctx context.Context, + roPVCName string, +) (PVCState, error) { + return c.checkPVCState(ctx, roPVCName, "", false, false, nil) +} + +func (c K8sComputeBackend) checkPVCState( + ctx context.Context, + roPVCName string, + bindingUID k8stypes.UID, + bindingScoped bool, + bindingScopedReader bool, + binding *nvcav2beta1.ModelCacheBinding, +) (PVCState, error) { log := core.GetLogger(ctx) - roPVCObj, err := c.clients.K8s.CoreV1().PersistentVolumeClaims(c.bk8s.podInstanceNamespace).Get(ctx, roPVCName, metav1.GetOptions{}) + roPVCObj, err := c.clients.K8s.CoreV1(). + PersistentVolumeClaims(c.bk8s.podInstanceNamespace). + Get(ctx, roPVCName, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { log.Debugf("PVC %v/%v doesn't exist", c.bk8s.podInstanceNamespace, roPVCName) return PVCNotFound, nil } - log.WithError(err).Errorf("failed to query for ROPVC %v/%v", c.bk8s.podInstanceNamespace, roPVCName) + log.WithError(err).Errorf( + "failed to query for ROPVC %v/%v", c.bk8s.podInstanceNamespace, roPVCName) return PVCQueryError, err } - log.Debugf("PVC %v/%v exists", c.bk8s.podInstanceNamespace, roPVCName) - if reflect.DeepEqual(roPVCObj.Spec.AccessModes, ROAccessMode) { - pvObj, err := c.clients.K8s.CoreV1().PersistentVolumes().Get(ctx, roPVCObj.Spec.VolumeName, metav1.GetOptions{}) - if err != nil && !errors.IsNotFound(err) { - return PVCQueryError, fmt.Errorf("failed to get PV %v to check volume attachment status: %v", roPVCObj.Spec.VolumeName, err) + if bindingScoped { + if err := requireRegularModelCacheBindingUID(roPVCObj, bindingUID); err != nil { + return PVCFoundBindFailed, err + } + } + if bindingScoped && bindingScopedReader { + if !reflect.DeepEqual(roPVCObj.Spec.AccessModes, ROAccessMode) { + return PVCFoundBindFailed, fmt.Errorf( + "binding-owned reader PVC %s/%s does not use ReadOnlyMany", + roPVCObj.Namespace, roPVCObj.Name) + } + expectedStorageClass, expectedErr := regularModelCacheExpectedStorageClassName(binding) + if expectedErr != nil { + return PVCFoundBindFailed, expectedErr + } + if roPVCObj.Spec.StorageClassName == nil || *roPVCObj.Spec.StorageClassName != expectedStorageClass { + return PVCFoundBindFailed, fmt.Errorf( + "binding-owned reader PVC %s/%s StorageClass does not match %q", + roPVCObj.Namespace, roPVCObj.Name, expectedStorageClass) + } + // A pending reader has not yet acquired its claim UID in the PV. Validate + // the complete PVC-to-PV ownership chain before declaring it Bound. + if isPVCBound(roPVCObj) { + if roPVCObj.Spec.VolumeName == "" { + return PVCFoundBindFailed, fmt.Errorf( + "binding-owned reader PVC %s/%s has no bound PV", + roPVCObj.Namespace, roPVCObj.Name) + } + pvObj, pvGetErr := c.clients.K8s.CoreV1().PersistentVolumes(). + Get(ctx, roPVCObj.Spec.VolumeName, metav1.GetOptions{}) + if pvGetErr != nil { + return PVCQueryError, fmt.Errorf( + "get binding-owned reader PV %s: %w", roPVCObj.Spec.VolumeName, pvGetErr) + } + if err := validateRegularModelCachePVForPVC(binding, roPVCObj, pvObj, ROAccessMode); err != nil { + return PVCFoundBindFailed, err + } + if err := validateRegularModelCacheReaderPVMountOptions(binding, pvObj); err != nil { + return PVCFoundBindFailed, err + } + } + } else if !bindingScoped && reflect.DeepEqual(roPVCObj.Spec.AccessModes, ROAccessMode) { + pvObj, pvGetErr := c.clients.K8s.CoreV1().PersistentVolumes(). + Get(ctx, roPVCObj.Spec.VolumeName, metav1.GetOptions{}) + if pvGetErr != nil && !errors.IsNotFound(pvGetErr) { + return PVCQueryError, fmt.Errorf( + "failed to get PV %v to check volume attachment status: %w", + roPVCObj.Spec.VolumeName, pvGetErr) } if pvObj != nil && pvObj.Spec.PersistentVolumeReclaimPolicy == v1.PersistentVolumeReclaimDelete { - err = c.clients.K8s.CoreV1().PersistentVolumeClaims(c.bk8s.podInstanceNamespace).Delete(ctx, roPVCName, metav1.DeleteOptions{}) + // Preserve the annotation-free legacy dangling-PVC behavior. + err = c.clients.K8s.CoreV1(). + PersistentVolumeClaims(c.bk8s.podInstanceNamespace). + Delete(ctx, roPVCName, metav1.DeleteOptions{}) if err != nil && errors.IsNotFound(err) { - // error out PVCBind to resort to non-cache - return PVCFoundBindFailed, fmt.Errorf("failed to delete dangling ROPVC %v, modelcaching setup failed", roPVCObj.Name) + return PVCFoundBindFailed, + fmt.Errorf("failed to delete dangling ROPVC %v, modelcaching setup failed", roPVCObj.Name) } return PVCNotFound, nil } @@ -462,11 +858,14 @@ func (c K8sComputeBackend) CheckPVCState(ctx context.Context, roPVCName string) return ps, err } - if time.Since(roPVCObj.ObjectMeta.CreationTimestamp.Time) > c.bk8s.k8sTimeConfig.ModelCacheROPVCBindTimeGracePeriod { + if time.Since(roPVCObj.ObjectMeta.CreationTimestamp.Time) > + c.bk8s.k8sTimeConfig.ModelCacheROPVCBindTimeGracePeriod { return PVCFoundBindFailed, - fmt.Errorf("pvc %v didn't bind within %v", roPVCName, c.bk8s.k8sTimeConfig.ModelCacheROPVCBindTimeGracePeriod) + fmt.Errorf("pvc %v didn't bind within %v", + roPVCName, c.bk8s.k8sTimeConfig.ModelCacheROPVCBindTimeGracePeriod) } - log.Warnf("PVC %v is still unbound, continue to wait, phase: %v", roPVCName, roPVCObj.Status.Phase) + log.Warnf("PVC %v is still unbound, continue to wait, phase: %v", + roPVCName, roPVCObj.Status.Phase) return PVCFoundUnBound, nil } @@ -486,7 +885,7 @@ func (c K8sComputeBackend) handleLostPVC(ctx context.Context, roPVCObj *v1.Persi roPVCObjNewLocal, err := c.clients.K8s.CoreV1().PersistentVolumeClaims(roPVCObj.Namespace).Get(ctx, roPVCObj.Name, metav1.GetOptions{}) if err != nil { - return fmt.Errorf("failed to get PVC %v to update with RebindRequestedAnnotation: %v", roPVCObj.Name, err) + return fmt.Errorf("failed to get PVC %v to update with RebindRequestedAnnotation: %w", roPVCObj.Name, err) } // add NVCARebindAttemptedAnnotationKey @@ -503,7 +902,7 @@ func (c K8sComputeBackend) handleLostPVC(ctx context.Context, roPVCObj *v1.Persi }) if retryErr != nil { return PVCFoundBindFailed, - fmt.Errorf("failed to update PVC %v with RebindRequestedAnnotation, err: %v", roPVCObj.Name, retryErr) + fmt.Errorf("failed to update PVC %v with RebindRequestedAnnotation: %w", roPVCObj.Name, retryErr) } } else { return PVCFoundBindFailed, fmt.Errorf("pvc %v lost again, with rebind-request", roPVCObj.Name) @@ -541,7 +940,7 @@ func (c K8sComputeBackend) waitForVolumeDetach(ctx context.Context, volumeName s strings.Compare(*attachment.Spec.Source.PersistentVolumeName, volumeName) == 0 { pvObj, err := c.clients.K8s.CoreV1().PersistentVolumes().Get(ctx, volumeName, metav1.GetOptions{}) if err != nil { - return fmt.Errorf("failed to get PV %v to check volume attachment status: %v", volumeName, err) + return fmt.Errorf("failed to get PV %v to check volume attachment status: %w", volumeName, err) } if len(pvObj.Spec.AccessModes) == 1 && pvObj.Spec.AccessModes[0] == v1.ReadOnlyMany { attachedInRwMode = false @@ -564,6 +963,48 @@ func (c K8sComputeBackend) waitForVolumeDetach(ctx context.Context, volumeName s } } +// regularModelCacheReaderMountOptions makes provider requirements authoritative +// for a binding-scoped NVMesh reader PV. Cluster options remain additive, but +// an option that negates a provider requirement is ignored. +func regularModelCacheReaderMountOptions( + binding *nvcav2beta1.ModelCacheBinding, + configured []string, +) ([]string, error) { + if binding == nil { + return append([]string(nil), configured...), nil + } + if binding.Spec.Decision.Transition != nvcastorage.ModelCacheTransitionROXReadOnly { + return nil, fmt.Errorf("transition %q does not create a read-only reader PV", + binding.Spec.Decision.Transition) + } + + required := binding.Spec.Decision.RequiredMountOptions + effective := append([]string(nil), required...) + for _, option := range configured { + conflicts := slices.ContainsFunc(required, func(requiredOption string) bool { + return regularModelCacheMountOptionsConflict(requiredOption, option) + }) + if conflicts || slices.Contains(effective, option) { + continue + } + effective = append(effective, option) + } + return effective, nil +} + +func regularModelCacheMountOptionsConflict(left, right string) bool { + switch { + case left == "ro" && right == "rw", left == "rw" && right == "ro": + return true + case strings.HasPrefix(left, "no") && left[2:] == right: + return true + case strings.HasPrefix(right, "no") && right[2:] == left: + return true + default: + return false + } +} + // This function will setup the PVC as follows /* 1. Get the PV Name from the LaunchArtifact.CacheHanle-rw-pvc in bdArt.Specification @@ -573,7 +1014,7 @@ func (c K8sComputeBackend) waitForVolumeDetach(ctx context.Context, volumeName s 2. Remove the /spec/claimRef/resourceVersion 3. Remove the /spec/claimRef/uid 4. Change the /spec/accessModes -> ReadOnlyMany - 5. Set the /spec/mountOptions -> ["ro","norecovery","nouuid"] + 5. Set the /spec/mountOptions from the persisted provider requirements 4. Update the PV Object 5. Once Updated, create a new PVC from bdArt.Specification updating the following 1. Name -> $LaunchSpecification.CacheHandle-ro-pvc @@ -582,12 +1023,46 @@ func (c K8sComputeBackend) waitForVolumeDetach(ctx context.Context, volumeName s func (c K8sComputeBackend) SetupPVCForReaders(ctx context.Context, rwPVC *v1.PersistentVolumeClaim, initJobName string, req *nvcav2beta1.ICMSRequest, mf mutateFunc) (ROPVCSetupPhase, error) { log := core.GetLogger(ctx) - roPVCName := strings.ReplaceAll(rwPVC.Name, RWPVCSuffix, ROPVCSuffix) + roPVCName, err := regularModelCacheReaderPVCName(rwPVC.Name) + if err != nil { + return ROPVCSetupFailed, err + } + bindingUID, bindingScoped, err := regularModelCacheBindingUID(req) + if err != nil { + return ROPVCSetupFailed, fmt.Errorf("resolve regular model cache binding ownership: %w", err) + } + var transitionTargets *regularModelCacheCleanupTargets + readerMountOptions := append([]string(nil), c.bk8s.csiVolumeMountOptions...) + if bindingScoped { + transitionTargets, _, err = c.regularModelCacheTransitionTargets( + ctx, req, rwPVC.Name, initJobName) + if err != nil { + phase := ROPVCSetupFailed + if regularModelCacheErrorIsRetryable(err) { + phase = ROPVCSetupQueryFailed + } + return phase, fmt.Errorf("validate regular model cache transition targets: %w", err) + } + readerMountOptions, err = regularModelCacheReaderMountOptions( + transitionTargets.binding, c.bk8s.csiVolumeMountOptions) + if err != nil { + return ROPVCSetupFailed, err + } + } var pvName string var pvObj *v1.PersistentVolume - var err error - pvcCur, _ := c.clients.K8s.CoreV1().PersistentVolumeClaims(c.bk8s.podInstanceNamespace).Get(ctx, rwPVC.Name, metav1.GetOptions{}) + pvcCur, pvcGetErr := c.clients.K8s.CoreV1().PersistentVolumeClaims(c.bk8s.podInstanceNamespace). + Get(ctx, rwPVC.Name, metav1.GetOptions{}) + if errors.IsNotFound(pvcGetErr) { + pvcCur = nil + } else if pvcGetErr != nil { + if bindingScoped { + return ROPVCSetupQueryFailed, fmt.Errorf("get binding-owned writer PVC %s/%s: %w", + c.bk8s.podInstanceNamespace, rwPVC.Name, pvcGetErr) + } + pvcCur = nil + } if pvcCur == nil { // this would mean the RWPVC has been successfully purged, // lets find the underlying PV for this function/task @@ -599,7 +1074,8 @@ func (c K8sComputeBackend) SetupPVCForReaders(ctx context.Context, pvObjList, err := c.clients.K8s.CoreV1().PersistentVolumes().List(ctx, metav1.ListOptions{ LabelSelector: pvLabelSel}) if err != nil && !errors.IsNotFound(err) { - return ROPVCSetupQueryFailed, fmt.Errorf("failed to query PV list for selector %s", pvLabelSel) + return ROPVCSetupQueryFailed, + fmt.Errorf("failed to query PV list for selector %s: %w", pvLabelSel, err) } if len(pvObjList.Items) > 1 { return ROPVCSetupQueryFailed, fmt.Errorf("found %v PVs for functionVersionId", len(pvObjList.Items)) @@ -608,6 +1084,11 @@ func (c K8sComputeBackend) SetupPVCForReaders(ctx context.Context, pvName = pvObjList.Items[0].Name } } else { + if bindingScoped { + if err := validateRegularModelCachePVC(pvcCur, rwPVC, transitionTargets.binding, false); err != nil { + return ROPVCSetupFailed, err + } + } pvName = pvcCur.Spec.VolumeName } @@ -618,7 +1099,25 @@ func (c K8sComputeBackend) SetupPVCForReaders(ctx context.Context, pvObj, err = c.clients.K8s.CoreV1().PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}) if err != nil { - return ROPVCSetupQueryFailed, fmt.Errorf("failed to get PV %v in SetupPVCForReaders", pvName) + return ROPVCSetupQueryFailed, + fmt.Errorf("failed to get PV %v in SetupPVCForReaders: %w", pvName, err) + } + if bindingScoped { + gotBindingUID := pvObj.Labels[nvcastorage.ModelCacheBindingUIDLabelKey] + if pvcCur == nil && gotBindingUID != string(bindingUID) { + return ROPVCSetupFailed, fmt.Errorf( + "writer PVC is absent and PV %s has binding UID %q, want %q", + pvObj.Name, gotBindingUID, bindingUID) + } + if gotBindingUID != "" && gotBindingUID != string(bindingUID) { + return ROPVCSetupFailed, fmt.Errorf( + "PV %s belongs to model cache binding UID %q, not %q", + pvObj.Name, gotBindingUID, bindingUID) + } + if err := validateRegularModelCachePVClaimForSetup( + pvObj, transitionTargets.binding, pvcCur, c.bk8s.podInstanceNamespace, rwPVC.Name, roPVCName); err != nil { + return ROPVCSetupFailed, err + } } // update the PV with an identifying label. @@ -628,36 +1127,79 @@ func (c K8sComputeBackend) SetupPVCForReaders(ctx context.Context, } else { labelKey, labelVal = taskIDLabelString, req.Spec.TaskDetails.TaskID } - if _, ok := pvObj.Labels[labelKey]; !ok { + _, hasRequestLabel := pvObj.Labels[labelKey] + hasBindingLabel := !bindingScoped || + pvObj.Labels[nvcastorage.ModelCacheBindingUIDLabelKey] == string(bindingUID) + if !hasRequestLabel || !hasBindingLabel { retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { // Retrieve the latest version of PV before attempting update // RetryOnConflict uses exponential backoff to avoid exhausting the apiserver pvObj, err := c.clients.K8s.CoreV1().PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}) if err != nil { - return fmt.Errorf("failed to get PV %v to update with PersistentVolumeReclaimPolicy:Delete: %v", pvName, err) + return fmt.Errorf( + "failed to get PV %v to update with PersistentVolumeReclaimPolicy:Delete: %w", + pvName, err) } if pvObj.Labels == nil { pvObj.Labels = make(map[string]string) } + if bindingScoped { + gotBindingUID := pvObj.Labels[nvcastorage.ModelCacheBindingUIDLabelKey] + if pvcCur == nil && gotBindingUID != string(bindingUID) { + return fmt.Errorf( + "writer PVC is absent and PV %s has binding UID %q, want %q", + pvObj.Name, gotBindingUID, bindingUID) + } + if gotBindingUID != "" && gotBindingUID != string(bindingUID) { + return fmt.Errorf( + "PV %s belongs to model cache binding UID %q, not %q", + pvObj.Name, gotBindingUID, bindingUID) + } + if err := validateRegularModelCachePVClaimForSetup( + pvObj, transitionTargets.binding, pvcCur, c.bk8s.podInstanceNamespace, rwPVC.Name, roPVCName); err != nil { + return err + } + pvObj.Labels[nvcastorage.ModelCacheBindingUIDLabelKey] = string(bindingUID) + } pvObj.Labels[labelKey] = labelVal _, updateErr := c.clients.K8s.CoreV1().PersistentVolumes().Update(ctx, pvObj, metav1.UpdateOptions{}) return updateErr }) if retryErr != nil { - return ROPVUpdateFailed, fmt.Errorf("failed to update PV for ReadOnlyPVC binding: err: %v", err) + return ROPVUpdateFailed, + fmt.Errorf("failed to update PV for ReadOnlyPVC binding: %w", retryErr) } } // cleanup initJob & its pods - backgroundDeletion := metav1.DeletePropagationBackground - err = c.clients.K8s.BatchV1().Jobs(c.bk8s.podInstanceNamespace).Delete(ctx, initJobName, metav1.DeleteOptions{ - PropagationPolicy: &backgroundDeletion, - }) - if err != nil && !errors.IsNotFound(err) { - log.WithError(err).Warnf("failed to cleanup initCacheJob %v/%v, needs manual cleanup", - c.bk8s.podInstanceNamespace, initJobName) + jobNamespace := c.bk8s.podInstanceNamespace + jobToDelete := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: initJobName}} + jobDeleteOptions := metav1.DeleteOptions{} + if bindingScoped { + jobNamespace = transitionTargets.namespace + jobToDelete = transitionTargets.initJob + } + if jobToDelete != nil { + backgroundDeletion := metav1.DeletePropagationBackground + jobDeleteOptions.PropagationPolicy = &backgroundDeletion + if bindingScoped { + jobDeleteOptions, err = modelCacheDeleteOptions(jobToDelete, &backgroundDeletion) + if err != nil { + return ROPVCSetupFailed, + fmt.Errorf("build binding-owned init Job delete preconditions: %w", err) + } + } + err = c.clients.K8s.BatchV1().Jobs(jobNamespace). + Delete(ctx, jobToDelete.Name, jobDeleteOptions) + if err != nil && !errors.IsNotFound(err) { + log.WithError(err).Warnf("failed to cleanup initCacheJob %v/%v, needs manual cleanup", + jobNamespace, jobToDelete.Name) + if bindingScoped { + return ROPVCSetupFailed, fmt.Errorf("delete binding-owned init cache Job: %w", err) + } + } } // wait for volumeDetach if fails, skip caching @@ -669,39 +1211,79 @@ func (c K8sComputeBackend) SetupPVCForReaders(ctx context.Context, } // cleanup RWPVC - err = c.clients.K8s.CoreV1().PersistentVolumeClaims(c.bk8s.podInstanceNamespace).Delete(ctx, rwPVC.Name, metav1.DeleteOptions{}) - if err != nil && errors.IsNotFound(err) { - log.Infof("RWPVC %v/%v cleaned-up, setup ROPVC", c.bk8s.podInstanceNamespace, rwPVC.Name) + rwPVCToDelete := rwPVC + rwPVCDeleteOptions := metav1.DeleteOptions{} + if bindingScoped { + rwPVCToDelete = transitionTargets.rwPVC + } + if rwPVCToDelete != nil { + if bindingScoped { + rwPVCDeleteOptions, err = modelCacheDeleteOptions(rwPVCToDelete, nil) + if err != nil { + return ROPVCSetupFailed, + fmt.Errorf("build binding-owned writer PVC delete preconditions: %w", err) + } + } + err = c.clients.K8s.CoreV1().PersistentVolumeClaims(c.bk8s.podInstanceNamespace). + Delete(ctx, rwPVCToDelete.Name, rwPVCDeleteOptions) + if err != nil && errors.IsNotFound(err) { + log.Infof("RWPVC %v/%v cleaned-up, setup ROPVC", + c.bk8s.podInstanceNamespace, rwPVCToDelete.Name) + } else if err != nil && bindingScoped { + return ROPVCSetupFailed, fmt.Errorf("delete binding-owned writer PVC: %w", err) + } } - // if the ClaimRef was already Updated to the ROPVCName, skip update - if pvObj.Spec.ClaimRef.Name != roPVCName { + if bindingScoped { + pvObj, err = c.clients.K8s.CoreV1().PersistentVolumes().Get(ctx, pvName, metav1.GetOptions{}) + if err != nil { + return ROPVCSetupQueryFailed, fmt.Errorf("get binding-owned PV %v after writer deletion: %w", pvName, err) + } + if err := c.validateRegularModelCachePVClaimAfterWriterDelete( + ctx, pvObj, pvcCur, transitionTargets.binding, c.bk8s.podInstanceNamespace, rwPVC.Name, roPVCName); err != nil { + return ROPVCSetupFailed, err + } + } + + // A retry may find the reader identity already published but its mount + // options drifted. Repair only persisted, binding-scoped requirements; + // preserve the annotation-free legacy retry behavior. + if pvObj.Spec.ClaimRef.Name != roPVCName || + (bindingScoped && !slices.Equal(pvObj.Spec.MountOptions, readerMountOptions)) { retryErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { // Retrieve the latest version of PV before attempting update // RetryOnConflict uses exponential backoff to avoid exhausting the apiserver pvObj, err := c.clients.K8s.CoreV1().PersistentVolumes().Get(ctx, pvObj.Name, metav1.GetOptions{}) if err != nil { - return fmt.Errorf("failed to get PV %v to update with PersistentVolumeReclaimPolicy:Delete: %v", pvName, err) + return fmt.Errorf( + "failed to get PV %v to prepare for read-only binding: %w", + pvName, err) } - var newPVCRef v1.ObjectReference - // prepare PV for ReadOnly Mode - // Copy the current claimRef - pvObj.Spec.ClaimRef.DeepCopyInto(&newPVCRef) - - newPVCRef.UID = "" - newPVCRef.ResourceVersion = "" - newPVCRef.Name = roPVCName - - // set the new PVCRef - pvObj.Spec.ClaimRef = &newPVCRef - pvObj.Spec.AccessModes = ROAccessMode - pvObj.Spec.MountOptions = c.bk8s.csiVolumeMountOptions + if bindingScoped { + if err := c.validateRegularModelCachePVClaimAfterWriterDelete( + ctx, pvObj, pvcCur, transitionTargets.binding, c.bk8s.podInstanceNamespace, rwPVC.Name, roPVCName); err != nil { + return err + } + } + if pvObj.Spec.ClaimRef.Name != roPVCName { + var newPVCRef v1.ObjectReference + // Prepare the PV for ReadOnlyMany and let the reader PVC bind + // with its API-assigned UID. + pvObj.Spec.ClaimRef.DeepCopyInto(&newPVCRef) + newPVCRef.UID = "" + newPVCRef.ResourceVersion = "" + newPVCRef.Name = roPVCName + pvObj.Spec.ClaimRef = &newPVCRef + pvObj.Spec.AccessModes = ROAccessMode + } + pvObj.Spec.MountOptions = append([]string(nil), readerMountOptions...) _, updateErr := c.clients.K8s.CoreV1().PersistentVolumes().Update(ctx, pvObj, metav1.UpdateOptions{}) return updateErr }) if retryErr != nil { - return ROPVUpdateFailed, fmt.Errorf("failed to update PV for ReadOnlyPVC binding: err: %v", err) + return ROPVUpdateFailed, + fmt.Errorf("failed to update PV for ReadOnlyPVC binding: %w", retryErr) } } @@ -714,33 +1296,58 @@ func (c K8sComputeBackend) SetupPVCForReaders(ctx context.Context, mf(rwPVC) - _, err = c.clients.K8s.CoreV1().PersistentVolumeClaims(c.bk8s.podInstanceNamespace).Create(ctx, rwPVC, metav1.CreateOptions{}) + createdReader, err := c.clients.K8s.CoreV1().PersistentVolumeClaims(c.bk8s.podInstanceNamespace). + Create(ctx, rwPVC, metav1.CreateOptions{}) + if errors.IsAlreadyExists(err) && bindingScoped { + createdReader, err = c.clients.K8s.CoreV1().PersistentVolumeClaims(c.bk8s.podInstanceNamespace). + Get(ctx, roPVCName, metav1.GetOptions{}) + } if err != nil && !errors.IsAlreadyExists(err) { - return ROPVCSetupFailed, fmt.Errorf("failed to create ROPVC for readers, %v", err) + return ROPVCSetupFailed, fmt.Errorf("failed to create ROPVC for readers: %w", err) + } + if bindingScoped { + if err := validateRegularModelCachePVC(createdReader, rwPVC, transitionTargets.binding, true); err != nil { + return ROPVCSetupFailed, err + } } return ROPVCSetupCompleted, nil } -func (c K8sComputeBackend) CheckInitCacheJobState(ctx context.Context, rwPVCName string, job *batchv1.Job) InitCacheJobState { +func (c K8sComputeBackend) CheckInitCacheJobState( + ctx context.Context, + rwPVCName string, + job *batchv1.Job, + bindingUID k8stypes.UID, + bindingScoped bool, +) (InitCacheJobState, error) { log := core.GetLogger(ctx) - jS, err := c.clients.K8s.BatchV1().Jobs(c.bk8s.podInstanceNamespace).Get(ctx, job.Name, metav1.GetOptions{}) + jS, err := c.clients.K8s.BatchV1().Jobs(c.bk8s.podInstanceNamespace). + Get(ctx, job.Name, metav1.GetOptions{}) if err != nil { if errors.IsNotFound(err) { - log.WithField("job", job.Name).Errorf("initCacheJob not found, it may have just been created, but it should be running") - } else { - log.WithError(err).Errorf("failed to query the initCacheJob %v/%v", job.Namespace, job.Name) + log.WithField("job", job.Name).Errorf( + "initCacheJob not found, it may have just been created, but it should be running") + return InitCacheJobNotFound, nil + } + log.WithError(err).Errorf( + "failed to query the initCacheJob %v/%v", job.Namespace, job.Name) + return InitCacheJobNotFound, err + } + if bindingScoped { + if err := requireRegularModelCacheBindingUID(jS, bindingUID); err != nil { + return InitCacheJobFailed, err } - return InitCacheJobNotFound } if jS.Status.CompletionTime != nil && jS.Status.Succeeded > 0 { - log.Infof("init job %v/%v completed at %v", jS.Namespace, jS.Name, jS.Status.CompletionTime.ToUnstructured()) - return InitCacheJobCompleted + log.Infof("init job %v/%v completed at %v", + jS.Namespace, jS.Name, jS.Status.CompletionTime.ToUnstructured()) + return InitCacheJobCompleted, nil } // check the RWPVC state - rwPVCState, err := c.CheckPVCState(ctx, rwPVCName) + rwPVCState, err := c.checkPVCState(ctx, rwPVCName, bindingUID, bindingScoped, false, nil) switch rwPVCState { case PVCFoundBound: // no action @@ -748,9 +1355,15 @@ func (c K8sComputeBackend) CheckInitCacheJobState(ctx context.Context, rwPVCName log.WithError(err).Debugf("rwpvc %v is unbound", rwPVCName) case PVCQueryError: log.WithError(err).Errorf("transient failure to query the %v", rwPVCName) + if bindingScoped { + return InitCacheJobInProgress, err + } case PVCNotFound, PVCFoundBindFailed: log.WithError(err).Errorf("rwpvc %v bind failed, caching will be skipped", rwPVCName) - return InitCacheJobFailed + if bindingScoped && err != nil { + return InitCacheJobFailed, err + } + return InitCacheJobFailed, nil } // Use the job's configured backoff limit, defaulting to K8s default of 6 @@ -760,19 +1373,22 @@ func (c K8sComputeBackend) CheckInitCacheJobState(ctx context.Context, rwPVCName } if jS.Status.Failed > backoffLimit || (jS.Status.Active != 0 && - time.Since(jS.ObjectMeta.CreationTimestamp.Time) >= c.bk8s.k8sTimeConfig.InitCacheJobFailureThreshold) { + time.Since(jS.ObjectMeta.CreationTimestamp.Time) >= + c.bk8s.k8sTimeConfig.InitCacheJobFailureThreshold) { if jS.Status.Failed > backoffLimit { - log.WithError(err).Errorf("initCache job %v/%v has failed more than backoff limit (%d)", + log.WithError(err).Errorf( + "initCache job %v/%v has failed more than backoff limit (%d)", jS.Namespace, jS.Name, backoffLimit) } else { - log.WithError(err).Errorf("initCache job %v/%v has not completed within %v duration since launch", + log.WithError(err).Errorf( + "initCache job %v/%v has not completed within %v duration since launch", jS.Namespace, jS.Name, c.bk8s.k8sTimeConfig.InitCacheJobFailureThreshold) } - return InitCacheJobFailed + return InitCacheJobFailed, nil } log.Debugf("init cache job is still running") - return InitCacheJobInProgress + return InitCacheJobInProgress, nil } // getInitCacheJobFailureReason returns the failure reason for a failed init cache job. @@ -795,31 +1411,193 @@ func (c K8sComputeBackend) getInitCacheJobFailureReason(ctx context.Context, job func (c K8sComputeBackend) SetupInitCacheJobBlockDevice(ctx context.Context, rwPVCObj *v1.PersistentVolumeClaim, initJob *batchv1.Job, - _ *nvcav2beta1.ICMSRequest) error { + req *nvcav2beta1.ICMSRequest) error { log := core.GetLogger(ctx) var pvcCur *v1.PersistentVolumeClaim var err error + var binding *nvcav2beta1.ModelCacheBinding log.Debug("SetupInitCacheJobBlockDevice for ModelCaching") + bindingUID, bindingScoped, err := regularModelCacheBindingUID(req) + if err != nil { + return fmt.Errorf("resolve regular model cache binding ownership: %w", err) + } + if bindingScoped { + for _, obj := range []metav1.Object{rwPVCObj, initJob, &initJob.Spec.Template.ObjectMeta} { + if err := requireRegularModelCacheBindingUID(obj, bindingUID); err != nil { + return err + } + } + } + if bindingScoped { + binding, err = c.bk8s.activeModelCacheBindingForRuntime(ctx, req) + if err != nil { + return err + } + if err := validateRegularModelCachePVC(rwPVCObj, rwPVCObj, binding, false); err != nil { + return nvcaerrors.TerminalError(err) + } + } - pvcCur, err = c.clients.K8s.CoreV1().PersistentVolumeClaims(c.bk8s.podInstanceNamespace).Create(ctx, rwPVCObj, metav1.CreateOptions{}) - if err != nil && !errors.IsAlreadyExists(err) { - return fmt.Errorf("failed to create PVC %s/%s from artifact: %v", c.bk8s.podInstanceNamespace, rwPVCObj.Name, err) + pvcs := c.clients.K8s.CoreV1().PersistentVolumeClaims(c.bk8s.podInstanceNamespace) + if bindingScoped { + pvcCur, err = pvcs.Get(ctx, rwPVCObj.Name, metav1.GetOptions{}) + switch { + case err == nil: + if err := validateRegularModelCachePVC(pvcCur, rwPVCObj, binding, false); err != nil { + return err + } + case errors.IsNotFound(err): + selection, selectionErr := persistedRegularModelCacheSelection(req) + if selectionErr != nil { + return nvcaerrors.TerminalError(selectionErr) + } + if selection == nil || selection.Mode != nvcastorage.ModelCacheSelectionDurable { + return nvcaerrors.TerminalError(fmt.Errorf( + "binding-scoped writer PVC creation requires a durable persisted selection")) + } + if !selection.EncryptionRequired { + if rwPVCObj.Spec.StorageClassName == nil || + *rwPVCObj.Spec.StorageClassName != selection.StorageClassName { + return nvcaerrors.TerminalError(fmt.Errorf( + "writer PVC StorageClass does not match persisted selection %q", + selection.StorageClassName)) + } + if validationErr := nvcastorage.ValidateModelCacheStorageSelectionLiveWithClientset( + ctx, c.clients.K8s, selection); validationErr != nil { + wrapped := fmt.Errorf( + "validate persisted StorageClass before writer PVC creation: %w", validationErr) + if stderrors.Is(validationErr, nvcastorage.ErrModelCacheStorageSelectionDrift) || + errors.IsNotFound(validationErr) { + return nvcaerrors.TerminalError(wrapped) + } + return wrapped + } + } + pvcCur, err = pvcs.Create(ctx, rwPVCObj, metav1.CreateOptions{}) + if errors.IsAlreadyExists(err) { + pvcCur, err = pvcs.Get(ctx, rwPVCObj.Name, metav1.GetOptions{}) + } + if err != nil { + return fmt.Errorf("create binding-owned PVC %s/%s: %w", + c.bk8s.podInstanceNamespace, rwPVCObj.Name, err) + } + if err := validateRegularModelCachePVC(pvcCur, rwPVCObj, binding, false); err != nil { + return err + } + default: + return fmt.Errorf("get binding-owned PVC %s/%s: %w", + c.bk8s.podInstanceNamespace, rwPVCObj.Name, err) + } + } else { + pvcCur, err = pvcs.Create(ctx, rwPVCObj, metav1.CreateOptions{}) + if errors.IsAlreadyExists(err) { + pvcCur, err = pvcs.Get(ctx, rwPVCObj.Name, metav1.GetOptions{}) + } + if err != nil { + return fmt.Errorf("failed to create PVC %s/%s from artifact: %v", + c.bk8s.podInstanceNamespace, rwPVCObj.Name, err) + } } + if bindingScoped { + if err := bindRegularModelCacheWriterJobToPVC(initJob, pvcCur, binding); err != nil { + return nvcaerrors.TerminalError(err) + } + if err := validateRegularModelCacheJob(initJob, initJob, binding); err != nil { + return nvcaerrors.TerminalError(err) + } + if err := c.revalidateRegularModelCacheBindingAfterCreate( + ctx, req, rwPVCObj, initJob.Name, "writer PVC creation or adoption"); err != nil { + return err + } + } + if bindingScoped && binding != nil && + binding.Spec.Decision.Transition == nvcastorage.ModelCacheTransitionRWXReadOnly { + if err := c.validatePersistedStorageClassBeforeRWXWriterJob(ctx, req); err != nil { + return err + } + } log.Debugf("Created PVC %v/%v", c.bk8s.podInstanceNamespace, rwPVCObj.Name) // ICMS request gets purged if pvcCur != nil { _, err := c.clients.K8s.BatchV1().Jobs(c.bk8s.podInstanceNamespace).Create(ctx, initJob, metav1.CreateOptions{}) - if err != nil && !errors.IsAlreadyExists(err) { + if errors.IsAlreadyExists(err) && bindingScoped { + existing, getErr := c.clients.K8s.BatchV1().Jobs(c.bk8s.podInstanceNamespace). + Get(ctx, initJob.Name, metav1.GetOptions{}) + if getErr != nil { + return fmt.Errorf("get existing Job %s/%s: %w", + c.bk8s.podInstanceNamespace, initJob.Name, getErr) + } + if err := validateRegularModelCacheJob(existing, initJob, binding); err != nil { + return err + } + } else if err != nil && !errors.IsAlreadyExists(err) { // the job need not be created again if another ICMS request references it - return fmt.Errorf("failed to create Job %s/%s from artifact: %v", c.bk8s.podInstanceNamespace, rwPVCObj.Name, err) + return fmt.Errorf("failed to create Job %s/%s from artifact: %w", c.bk8s.podInstanceNamespace, initJob.Name, err) } log.Debugf("Created Job %v/%v", c.bk8s.podInstanceNamespace, initJob.Name) + if bindingScoped { + if err := c.revalidateRegularModelCacheBindingAfterCreate( + ctx, req, rwPVCObj, initJob.Name, "writer Job creation or adoption"); err != nil { + return err + } + } + } + return nil +} + +func (c K8sComputeBackend) validatePersistedStorageClassBeforeRWXWriterJob( + ctx context.Context, + req *nvcav2beta1.ICMSRequest, +) error { + selection, err := persistedRegularModelCacheSelection(req) + if err != nil { + return nvcaerrors.TerminalError(err) + } + if selection == nil || selection.Mode != nvcastorage.ModelCacheSelectionDurable || + selection.Transition != nvcastorage.ModelCacheTransitionRWXReadOnly { + return nvcaerrors.TerminalError(fmt.Errorf( + "rwxReadOnly writer Job requires a durable persisted rwxReadOnly selection")) + } + if selection.EncryptionRequired { + return nvcaerrors.TerminalError(fmt.Errorf( + "rwxReadOnly writer Job selection cannot require NVMesh encryption")) + } + if err := nvcastorage.ValidateModelCacheStorageSelectionLiveWithClientset( + ctx, c.clients.K8s, selection); err != nil { + wrapped := fmt.Errorf("validate persisted StorageClass before rwxReadOnly writer Job creation: %w", err) + if stderrors.Is(err, nvcastorage.ErrModelCacheStorageSelectionDrift) || errors.IsNotFound(err) { + return nvcaerrors.TerminalError(wrapped) + } + return wrapped } return nil } +func (c K8sComputeBackend) revalidateRegularModelCacheBindingAfterCreate( + ctx context.Context, + req *nvcav2beta1.ICMSRequest, + rwPVC *v1.PersistentVolumeClaim, + initJobName string, + operation string, +) error { + err := c.bk8s.validateModelCacheBindingForRuntime(ctx, req) + if err == nil { + return nil + } + if !stderrors.Is(err, errRegularModelCacheBindingRetiring) && + !stderrors.Is(err, errRegularModelCacheBindingReferenceReleased) { + return fmt.Errorf("revalidate model cache binding after %s: %w", operation, err) + } + if cleanupErr := c.CleanupModelCachingResources( + ctx, req, rwPVC, initJobName); cleanupErr != nil { + return fmt.Errorf("revalidate model cache binding after %s: %w; compensating cleanup failed: %v", + operation, err, cleanupErr) + } + return fmt.Errorf("revalidate model cache binding after %s: %w", operation, err) +} + var ( fnVersionIDLabelString = fmt.Sprintf("%s/%s", nvcav1new.SchemeGroupVersion.Group, types.FunctionVersionIDKey) taskIDLabelString = fmt.Sprintf("%s/%s", nvcav1new.SchemeGroupVersion.Group, types.TaskIDKey) @@ -828,7 +1606,14 @@ var ( func makePVLabelSelectorForCacheRequest(req *nvcav2beta1.ICMSRequest) (string, error) { var vals []string var key string - if req.Spec.FunctionDetails.FunctionVersionID != "" { + bindingUID, bindingScoped, err := regularModelCacheBindingUID(req) + if err != nil { + return "", err + } + if bindingScoped { + key = nvcastorage.ModelCacheBindingUIDLabelKey + vals = []string{string(bindingUID)} + } else if req.Spec.FunctionDetails.FunctionVersionID != "" { key = fnVersionIDLabelString vals = []string{req.Spec.FunctionDetails.FunctionVersionID} } else { diff --git a/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend_modelcache_rwx_readonly.go b/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend_modelcache_rwx_readonly.go new file mode 100644 index 000000000..a2f6a64c0 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend_modelcache_rwx_readonly.go @@ -0,0 +1,502 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nvca + +import ( + "context" + "fmt" + + "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/core" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + + nvcametrics "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/metrics" + modelcachetypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/metrics/modelcachetypes" + nvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1" + nvcaerrors "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/errors" + nvcastorage "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/storage" + nvcatypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/types" +) + +// setupRWXReadOnlyModelCachingForRequest implements the regular model-cache +// transition that publishes the populated writer claim itself to readers. The +// caller holds modelCacheMtx. Writer and worker Pods are in the same namespace, +// so the transition requires no second PVC, PV rewrite, volume detach, or data +// copy. Workload construction applies read-only intent to the PVC source and +// every matching mount after this method returns the writer claim name. +func (c K8sComputeBackend) setupRWXReadOnlyModelCachingForRequest( + ctx context.Context, + rwPVC *corev1.PersistentVolumeClaim, + initJob *batchv1.Job, + req *nvcav2beta1.ICMSRequest, + binding *nvcav2beta1.ModelCacheBinding, + bindingUID types.UID, +) (ModelCachingState, string, error) { + if rwPVC == nil || initJob == nil || binding == nil { + return regularModelCacheResultForError(true, nvcaerrors.TerminalError( + fmt.Errorf("rwxReadOnly model cache intent is incomplete"))) + } + if binding.Spec.Decision.Transition != nvcastorage.ModelCacheTransitionRWXReadOnly { + return regularModelCacheResultForError(true, nvcaerrors.TerminalError( + fmt.Errorf("model cache binding transition is %q, want %q", + binding.Spec.Decision.Transition, nvcastorage.ModelCacheTransitionRWXReadOnly))) + } + if initJob.Spec.TTLSecondsAfterFinished != nil { + return regularModelCacheResultForError(true, nvcaerrors.TerminalError( + fmt.Errorf("rwxReadOnly writer Job must not use ttlSecondsAfterFinished"))) + } + if err := validateRWXReadOnlyWriterJobPVC(initJob, rwPVC.Name); err != nil { + return regularModelCacheResultForError(true, + nvcaerrors.TerminalError(err)) + } + + current, err := c.clients.K8s.CoreV1(). + PersistentVolumeClaims(c.bk8s.podInstanceNamespace). + Get(ctx, rwPVC.Name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + job, jobErr := c.getRWXReadOnlyWriterJob(ctx, initJob, binding, nil) + if jobErr != nil { + return regularModelCacheResultForError(true, jobErr) + } + if job != nil { + return regularModelCacheResultForError(true, nvcaerrors.TerminalError( + fmt.Errorf("rwxReadOnly writer PVC %s/%s is missing while writer Job %s/%s exists", + c.bk8s.podInstanceNamespace, rwPVC.Name, job.Namespace, job.Name))) + } + if err := c.SetupInitCacheJobBlockDevice(ctx, rwPVC, initJob, req); err != nil { + return regularModelCacheResultForError(true, err) + } + return ModelCachingInProgress, "", nil + } + if err != nil { + return regularModelCacheResultForError(true, + fmt.Errorf("get rwxReadOnly writer PVC %s/%s: %w", + c.bk8s.podInstanceNamespace, rwPVC.Name, err)) + } + if err := validateRegularModelCachePVC(current, rwPVC, binding, false); err != nil { + return regularModelCacheResultForError(true, nvcaerrors.TerminalError(err)) + } + if current.DeletionTimestamp != nil { + return regularModelCacheResultForError(true, nvcaerrors.TerminalError( + fmt.Errorf("rwxReadOnly writer PVC %s/%s is terminating", + current.Namespace, current.Name))) + } + if err := bindRegularModelCacheWriterJobToPVC(initJob, current, binding); err != nil { + return regularModelCacheResultForError(true, nvcaerrors.TerminalError(err)) + } + + pvcState, stateErr := c.checkPVCState( + ctx, rwPVC.Name, bindingUID, true, false, binding) + switch pvcState { + case PVCQueryError: + return regularModelCacheResultForError(true, stateErr) + case PVCFoundUnBound: + job, jobErr := c.getRWXReadOnlyWriterJob(ctx, initJob, binding, current) + if jobErr != nil { + return regularModelCacheResultForError(true, jobErr) + } + if job == nil { + if err := c.SetupInitCacheJobBlockDevice(ctx, rwPVC, initJob, req); err != nil { + return regularModelCacheResultForError(true, err) + } + } + return ModelCachingInProgress, "", nil + case PVCFoundBindFailed, PVCNotFound: + if stateErr == nil { + stateErr = fmt.Errorf("rwxReadOnly writer PVC %s/%s is not usable", + c.bk8s.podInstanceNamespace, rwPVC.Name) + } + return c.failRWXReadOnlyModelCache(ctx, req, rwPVC, initJob, stateErr) + case PVCFoundBound: + default: + return regularModelCacheResultForError(true, nvcaerrors.TerminalError( + fmt.Errorf("unexpected rwxReadOnly writer PVC state %q", pvcState))) + } + current, err = c.getValidatedRWXReadOnlyBoundClaim(ctx, rwPVC, binding) + if err != nil { + return regularModelCacheResultForError(true, err) + } + if _, err := c.getRWXReadOnlyWriterJob(ctx, initJob, binding, current); err != nil { + return regularModelCacheResultForError(true, err) + } + + if current.Labels[nvcastorage.ModelCachePopulatedLabelKey] == + nvcastorage.ModelCachePopulatedLabelValue { + published, err := c.validateRWXReadOnlyPublication( + ctx, req, rwPVC, initJob, binding) + if err != nil { + return regularModelCacheResultForError(true, err) + } + return ModelCachingCompleted, published.Name, nil + } + + jobState, jobErr := c.CheckInitCacheJobState( + ctx, rwPVC.Name, initJob, bindingUID, true) + if jobErr != nil { + return regularModelCacheResultForError(true, jobErr) + } + switch jobState { + case InitCacheJobNotFound: + if err := c.SetupInitCacheJobBlockDevice(ctx, rwPVC, initJob, req); err != nil { + return regularModelCacheResultForError(true, err) + } + return ModelCachingInProgress, "", nil + case InitCacheJobInProgress: + return ModelCachingInProgress, "", nil + case InitCacheJobFailed: + return c.failRWXReadOnlyModelCache( + ctx, req, rwPVC, initJob, fmt.Errorf("init cache Job %s failed", initJob.Name)) + case InitCacheJobCompleted: + if _, err := c.requireCompletedRWXReadOnlyWriterJob( + ctx, initJob, binding, current); err != nil { + return regularModelCacheResultForError(true, err) + } + if err := c.markRWXReadOnlyModelCachePopulated( + ctx, req, rwPVC, initJob, binding); err != nil { + return regularModelCacheResultForError(true, err) + } + published, err := c.validateRWXReadOnlyPublication( + ctx, req, rwPVC, initJob, binding) + if err != nil { + return regularModelCacheResultForError(true, err) + } + nvcametrics.FromContext(ctx).RecordModelCacheResult( + modelcachetypes.ResultSuccess, "", string(nvcatypes.HelmCacheBackendSharedFS)) + return ModelCachingCompleted, published.Name, nil + default: + return regularModelCacheResultForError(true, nvcaerrors.TerminalError( + fmt.Errorf("unexpected rwxReadOnly writer Job state %q", jobState))) + } +} + +func validateRWXReadOnlyWriterJobPVC(job *batchv1.Job, pvcName string) error { + if job == nil || pvcName == "" { + return fmt.Errorf("rwxReadOnly writer Job or PVC name is missing") + } + modelVolumeCount := 0 + for _, volume := range job.Spec.Template.Spec.Volumes { + if volume.Name != ModelVolumeName { + continue + } + modelVolumeCount++ + claim := volume.PersistentVolumeClaim + if claim == nil || claim.ClaimName != pvcName { + return fmt.Errorf( + "rwxReadOnly writer Job volume %q does not reference PVC %q", + ModelVolumeName, pvcName) + } + if claim.ReadOnly { + return fmt.Errorf("rwxReadOnly writer Job PVC volume %q is read-only", ModelVolumeName) + } + } + if modelVolumeCount == 0 { + return fmt.Errorf("rwxReadOnly writer Job has no %q volume", ModelVolumeName) + } + if modelVolumeCount != 1 { + return fmt.Errorf("rwxReadOnly writer Job has %d %q volumes, want exactly one", + modelVolumeCount, ModelVolumeName) + } + containers := append( + append([]corev1.Container(nil), job.Spec.Template.Spec.InitContainers...), + job.Spec.Template.Spec.Containers...) + for _, container := range containers { + for _, mount := range container.VolumeMounts { + if mount.Name == ModelVolumeName && !mount.ReadOnly { + return nil + } + } + for _, device := range container.VolumeDevices { + if device.Name == ModelVolumeName { + return nil + } + } + } + return fmt.Errorf( + "rwxReadOnly writer Job does not mount PVC %q writable", pvcName) +} +func validateRWXReadOnlyWriterJobPVCWitness( + job *batchv1.Job, + pvc *corev1.PersistentVolumeClaim, +) error { + if job == nil || pvc == nil || pvc.UID == "" { + return fmt.Errorf("rwxReadOnly writer Job PVC witness input is incomplete") + } + recorded := job.Spec.Template.Annotations[nvcastorage.ModelCacheWriterPVCUIDAnnotationKey] + if recorded != string(pvc.UID) { + return fmt.Errorf("rwxReadOnly writer Job records PVC UID %q, want %q", + recorded, pvc.UID) + } + return nil +} + +func (c K8sComputeBackend) currentRWXReadOnlyBinding( + ctx context.Context, + req *nvcav2beta1.ICMSRequest, + wanted *nvcav2beta1.ModelCacheBinding, +) (*nvcav2beta1.ModelCacheBinding, error) { + current, err := c.bk8s.activeModelCacheBindingForRuntime(ctx, req) + if err != nil { + return nil, err + } + if current == nil || wanted == nil || current.UID != wanted.UID { + return nil, nvcaerrors.TerminalError(fmt.Errorf( + "active rwxReadOnly model cache binding identity changed")) + } + if current.Spec.Decision.Transition != nvcastorage.ModelCacheTransitionRWXReadOnly { + return nil, nvcaerrors.TerminalError(fmt.Errorf( + "model cache binding transition is %q, want %q", + current.Spec.Decision.Transition, + nvcastorage.ModelCacheTransitionRWXReadOnly)) + } + return current, nil +} + +// validateRWXReadOnlyPublication re-reads every durable publication witness +// before returning the writer claim to a reader. The repeated binding and +// claim reads close the gaps between independent Kubernetes API operations; +// they do not claim transactional consistency across those objects. +func (c K8sComputeBackend) validateRWXReadOnlyPublication( + ctx context.Context, + req *nvcav2beta1.ICMSRequest, + wantedPVC *corev1.PersistentVolumeClaim, + wantedJob *batchv1.Job, + wantedBinding *nvcav2beta1.ModelCacheBinding, +) (*corev1.PersistentVolumeClaim, error) { + liveBinding, err := c.currentRWXReadOnlyBinding(ctx, req, wantedBinding) + if err != nil { + return nil, err + } + claim, err := c.getValidatedRWXReadOnlyBoundClaim(ctx, wantedPVC, liveBinding) + if err != nil { + return nil, err + } + if claim.Labels[nvcastorage.ModelCachePopulatedLabelKey] != + nvcastorage.ModelCachePopulatedLabelValue { + return nil, nvcaerrors.TerminalError(fmt.Errorf( + "rwxReadOnly writer PVC %s/%s is not marked populated", + claim.Namespace, claim.Name)) + } + if _, err := c.requireCompletedRWXReadOnlyWriterJob( + ctx, wantedJob, liveBinding, claim); err != nil { + return nil, err + } + + // Re-read after validating the Job so a concurrently retired binding or + // replaced claim cannot be published from the earlier snapshot. + liveBinding, err = c.currentRWXReadOnlyBinding(ctx, req, wantedBinding) + if err != nil { + return nil, err + } + finalClaim, err := c.getValidatedRWXReadOnlyBoundClaim( + ctx, wantedPVC, liveBinding) + if err != nil { + return nil, err + } + if finalClaim.UID != claim.UID { + return nil, nvcaerrors.TerminalError(fmt.Errorf( + "rwxReadOnly writer PVC %s/%s UID changed from %q to %q during publication", + claim.Namespace, claim.Name, claim.UID, finalClaim.UID)) + } + if finalClaim.Labels[nvcastorage.ModelCachePopulatedLabelKey] != + nvcastorage.ModelCachePopulatedLabelValue { + return nil, nvcaerrors.TerminalError(fmt.Errorf( + "rwxReadOnly writer PVC %s/%s lost its populated marker during publication", + finalClaim.Namespace, finalClaim.Name)) + } + if _, err := c.requireCompletedRWXReadOnlyWriterJob( + ctx, wantedJob, liveBinding, finalClaim); err != nil { + return nil, err + } + if _, err := c.currentRWXReadOnlyBinding(ctx, req, wantedBinding); err != nil { + return nil, err + } + return finalClaim, nil +} +func (c K8sComputeBackend) getValidatedRWXReadOnlyBoundClaim( + ctx context.Context, + wanted *corev1.PersistentVolumeClaim, + binding *nvcav2beta1.ModelCacheBinding, +) (*corev1.PersistentVolumeClaim, error) { + current, err := c.clients.K8s.CoreV1(). + PersistentVolumeClaims(c.bk8s.podInstanceNamespace). + Get(ctx, wanted.Name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("get rwxReadOnly writer PVC before publication: %w", err) + } + if err := validateRegularModelCachePVC(current, wanted, binding, false); err != nil { + return nil, nvcaerrors.TerminalError(err) + } + if current.DeletionTimestamp != nil { + return nil, nvcaerrors.TerminalError(fmt.Errorf( + "rwxReadOnly writer PVC %s/%s is terminating", + current.Namespace, current.Name)) + } + if !isPVCBound(current) { + return nil, nvcaerrors.TerminalError(fmt.Errorf( + "rwxReadOnly writer PVC %s/%s is not Bound", + current.Namespace, current.Name)) + } + if current.Spec.VolumeName == "" { + return nil, nvcaerrors.TerminalError(fmt.Errorf( + "Bound rwxReadOnly writer PVC %s/%s has no PV name", + current.Namespace, current.Name)) + } + pv, err := c.clients.K8s.CoreV1().PersistentVolumes(). + Get(ctx, current.Spec.VolumeName, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil, nvcaerrors.TerminalError(fmt.Errorf( + "Bound rwxReadOnly writer PVC %s/%s references missing PV %q", + current.Namespace, current.Name, current.Spec.VolumeName)) + } + if err != nil { + return nil, fmt.Errorf("get rwxReadOnly writer PV %s: %w", + current.Spec.VolumeName, err) + } + if pv.DeletionTimestamp != nil { + return nil, nvcaerrors.TerminalError(fmt.Errorf( + "rwxReadOnly writer PV %s is terminating", pv.Name)) + } + if pv.Status.Phase != corev1.VolumeBound { + return nil, nvcaerrors.TerminalError(fmt.Errorf( + "rwxReadOnly writer PV %s phase is %q, want %q", + pv.Name, pv.Status.Phase, corev1.VolumeBound)) + } + if err := validateRegularModelCachePVForPVC( + binding, current, pv, RWXAccessMode); err != nil { + return nil, nvcaerrors.TerminalError(err) + } + return current, nil +} + +func (c K8sComputeBackend) getRWXReadOnlyWriterJob( + ctx context.Context, + wanted *batchv1.Job, + binding *nvcav2beta1.ModelCacheBinding, + pvc *corev1.PersistentVolumeClaim, +) (*batchv1.Job, error) { + job, err := c.clients.K8s.BatchV1().Jobs(c.bk8s.podInstanceNamespace). + Get(ctx, wanted.Name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("get rwxReadOnly writer Job %s/%s: %w", + c.bk8s.podInstanceNamespace, wanted.Name, err) + } + if job.DeletionTimestamp != nil { + return nil, nvcaerrors.TerminalError(fmt.Errorf( + "rwxReadOnly writer Job %s/%s is terminating", job.Namespace, job.Name)) + } + if pvc != nil { + if err := validateRWXReadOnlyWriterJobPVCWitness(job, pvc); err != nil { + return nil, nvcaerrors.TerminalError(err) + } + } + if err := validateRegularModelCacheJob(job, wanted, binding); err != nil { + return nil, nvcaerrors.TerminalError(err) + } + return job, nil +} + +func (c K8sComputeBackend) markRWXReadOnlyModelCachePopulated( + ctx context.Context, + req *nvcav2beta1.ICMSRequest, + wanted *corev1.PersistentVolumeClaim, + wantedJob *batchv1.Job, + binding *nvcav2beta1.ModelCacheBinding, +) error { + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + liveBinding, err := c.currentRWXReadOnlyBinding(ctx, req, binding) + if err != nil { + return err + } + current, err := c.getValidatedRWXReadOnlyBoundClaim(ctx, wanted, liveBinding) + if err != nil { + return err + } + if _, err := c.requireCompletedRWXReadOnlyWriterJob( + ctx, wantedJob, liveBinding, current); err != nil { + return err + } + if current.Labels == nil { + current.Labels = map[string]string{} + } + if current.Labels[nvcastorage.ModelCachePopulatedLabelKey] == + nvcastorage.ModelCachePopulatedLabelValue { + return nil + } + current.Labels[nvcastorage.ModelCachePopulatedLabelKey] = + nvcastorage.ModelCachePopulatedLabelValue + _, err = c.clients.K8s.CoreV1(). + PersistentVolumeClaims(c.bk8s.podInstanceNamespace). + Update(ctx, current, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("mark rwxReadOnly writer PVC populated: %w", err) + } + return nil + }) +} + +// requireCompletedRWXReadOnlyWriterJob keeps the completed Job as the durable +// publication fence. NVCA never deletes it while the binding is Active. This +// prevents another NVCA replica with a stale pre-publication read from +// recreating a writer after readers have started. +func (c K8sComputeBackend) requireCompletedRWXReadOnlyWriterJob( + ctx context.Context, + wanted *batchv1.Job, + binding *nvcav2beta1.ModelCacheBinding, + pvc *corev1.PersistentVolumeClaim, +) (*batchv1.Job, error) { + job, err := c.getRWXReadOnlyWriterJob(ctx, wanted, binding, pvc) + if err != nil { + return nil, err + } + if job == nil { + return nil, nvcaerrors.TerminalError(fmt.Errorf( + "rwxReadOnly publication fence Job %s/%s is missing", + c.bk8s.podInstanceNamespace, wanted.Name)) + } + if job.Status.CompletionTime == nil || job.Status.Succeeded == 0 { + return nil, nvcaerrors.TerminalError(fmt.Errorf( + "populated rwxReadOnly writer PVC has non-completed Job %s/%s", + job.Namespace, job.Name)) + } + return job, nil +} + +func (c K8sComputeBackend) failRWXReadOnlyModelCache( + ctx context.Context, + req *nvcav2beta1.ICMSRequest, + rwPVC *corev1.PersistentVolumeClaim, + initJob *batchv1.Job, + cause error, +) (ModelCachingState, string, error) { + log := core.GetLogger(ctx) + if err := c.CleanupModelCachingResources(ctx, req, rwPVC, initJob.Name); err != nil { + log.WithError(err).Error("failed to clean up rwxReadOnly model cache resources") + return regularModelCacheResultForError(true, err) + } + nvcametrics.FromContext(ctx).RecordModelCacheResult( + modelcachetypes.ResultFailure, modelcachetypes.ReasonPVCSetupFailed, + string(nvcatypes.HelmCacheBackendSharedFS)) + return regularModelCacheResultForError(true, nvcaerrors.TerminalError(cause)) +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend_modelcache_rwx_readonly_test.go b/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend_modelcache_rwx_readonly_test.go new file mode 100644 index 000000000..c9f426459 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend_modelcache_rwx_readonly_test.go @@ -0,0 +1,1394 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nvca + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + fakek8sclient "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + + nvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1" + nvcaerrors "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/errors" + nvcastorage "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/storage" + nvcatypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/types" +) + +type rwxReadOnlyRuntimeFixture struct { + backend K8sComputeBackend + req *nvcav2beta1.ICMSRequest + binding *nvcav2beta1.ModelCacheBinding + rwPVC *corev1.PersistentVolumeClaim + job *batchv1.Job +} + +func addRWXReadOnlyRequestOwnership(obj metav1.Object, requestID string) { + labels := obj.GetLabels() + if labels == nil { + labels = map[string]string{} + } + labels[nvcatypes.ICMSRequestIDKey] = requestID + labels[nvcatypes.MessageBatchIDKey] = "batch-" + requestID + labels[nvcatypes.FunctionVersionIDKey] = "version-" + requestID + obj.SetLabels(labels) + annotations := obj.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[nvcatypes.ICMSRequestIDKey] = requestID + annotations[nvcatypes.InstanceCountKey] = "1" + obj.SetAnnotations(annotations) + obj.SetOwnerReferences([]metav1.OwnerReference{{ + APIVersion: nvcav2beta1.SchemeGroupVersion.String(), + Kind: "ICMSRequest", + Name: requestID, + UID: types.UID("uid-" + requestID), + }}) +} + +func assertRWXReadOnlyBindingScopedMetadata(t *testing.T, obj metav1.Object) { + t.Helper() + assert.Empty(t, obj.GetOwnerReferences()) + assert.NotContains(t, obj.GetLabels(), nvcatypes.ICMSRequestIDKey) + assert.NotContains(t, obj.GetLabels(), nvcatypes.MessageBatchIDKey) + assert.NotContains(t, obj.GetLabels(), nvcatypes.FunctionVersionIDKey) + assert.NotContains(t, obj.GetAnnotations(), nvcatypes.ICMSRequestIDKey) + assert.NotContains(t, obj.GetAnnotations(), nvcatypes.InstanceCountKey) +} + +func newRWXReadOnlyRuntimeFixture( + t *testing.T, + provider string, + provisioner string, +) *rwxReadOnlyRuntimeFixture { + t.Helper() + + resolved := &nvcastorage.ModelCacheStorageSelection{ + StorageClassName: nvcastorage.DefaultModelCacheStorageClassName, + StorageClassUID: types.UID("storage-class-uid"), + StorageClassDigest: "v1:sha256:storage-class-digest", + CatalogDigest: "sha256:catalog-digest", + Provider: provider, + Provisioner: provisioner, + Transition: nvcastorage.ModelCacheTransitionRWXReadOnly, + RequiredAccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + } + return newRWXReadOnlyRuntimeFixtureFromSelection( + t, resolved, fakek8sclient.NewSimpleClientset()) +} + +func newResolvedWekaRWXReadOnlyRuntimeFixture(t *testing.T) *rwxReadOnlyRuntimeFixture { + t.Helper() + storageClass := selectionStorageClassForProvisioner("csi.weka.io") + catalog := selectionCatalogConfigMap(selectionCatalogRWXReadOnly) + k8sClient := fakek8sclient.NewSimpleClientset(storageClass, catalog) + resolved, err := nvcastorage.ResolveModelCacheStorageWithClientset( + t.Context(), k8sClient, selectionCatalogNamespace, + nvcastorage.ModelCacheWorkflowRegular) + require.NoError(t, err) + require.Equal(t, nvcastorage.ModelCacheTransitionRWXReadOnly, resolved.Transition) + return newRWXReadOnlyRuntimeFixtureFromSelection(t, resolved, k8sClient) +} + +func newRWXReadOnlyRuntimeFixtureFromSelection( + t *testing.T, + resolved *nvcastorage.ModelCacheStorageSelection, + k8sClient *fakek8sclient.Clientset, +) *rwxReadOnlyRuntimeFixture { + t.Helper() + raw := persistedSelectionAnnotation( + t, + nvcastorage.ModelCacheWorkflowRegular, + nvcastorage.ModelCacheSelectionDurable, + resolved, + ) + + backend := testContainerModelCacheBackend(k8sClient) + req := &nvcav2beta1.ICMSRequest{ + ObjectMeta: metav1.ObjectMeta{Name: "request", Namespace: RequestsNamespace}, + } + binding := installActiveRegularModelCacheBinding(t, &backend, req, raw) + require.Equal(t, nvcastorage.ModelCacheTransitionRWXReadOnly, binding.Spec.Decision.Transition) + require.Equal(t, []string{"rw-pvc-model-cache-handle"}, + binding.Spec.Resources.PersistentVolumeClaimNames) + + bindingLabels := map[string]string{ + nvcastorage.ModelCacheBindingUIDLabelKey: string(binding.UID), + } + automountServiceAccountToken := false + storageClassName := nvcastorage.DefaultModelCacheStorageClassName + rwPVC := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: binding.Spec.Resources.PersistentVolumeClaimNames[0], + Namespace: binding.Spec.Resources.WriterNamespace, + Labels: bindingLabels, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + StorageClassName: &storageClassName, + }, + } + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: binding.Spec.Resources.JobNames[0], + Namespace: binding.Spec.Resources.WriterNamespace, + Labels: bindingLabels, + }, + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: bindingLabels}, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + AutomountServiceAccountToken: &automountServiceAccountToken, + Volumes: []corev1.Volume{{ + Name: ModelVolumeName, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: rwPVC.Name, + }, + }, + }}, + Containers: []corev1.Container{{ + Name: "writer", + Image: "example.invalid/model-cache-writer:test", + VolumeMounts: []corev1.VolumeMount{{ + Name: ModelVolumeName, MountPath: "/models"}}, + }}, + }, + }, + }, + } + + return &rwxReadOnlyRuntimeFixture{ + backend: backend, + req: req, + binding: binding, + rwPVC: rwPVC, + job: job, + } +} + +func TestRWXReadOnlyCreatesWriterFromResolvedProvider(t *testing.T) { + fixture := newResolvedWekaRWXReadOnlyRuntimeFixture(t) + k8sClient := fixture.backend.clients.K8s.(*fakek8sclient.Clientset) + for _, obj := range []metav1.Object{ + fixture.rwPVC, fixture.job, &fixture.job.Spec.Template.ObjectMeta, + } { + addRWXReadOnlyRequestOwnership(obj, "request-a") + } + require.NoError(t, fixture.backend.prepareRegularModelCacheBindingResources( + t.Context(), fixture.binding, fixture.rwPVC, fixture.job)) + for _, obj := range []metav1.Object{ + fixture.rwPVC, fixture.job, &fixture.job.Spec.Template.ObjectMeta, + } { + assertRWXReadOnlyBindingScopedMetadata(t, obj) + } + installRWXReadOnlyCreateIdentityReactors(t, k8sClient) + k8sClient.ClearActions() + + state, claimName, err := runRWXReadOnlySetup(t, fixture) + require.NoError(t, err) + assert.Equal(t, ModelCachingInProgress, state) + assert.Empty(t, claimName, "NVCA must not publish the claim before the writer completes") + + createdPVC, err := k8sClient.CoreV1().PersistentVolumeClaims(fixture.rwPVC.Namespace). + Get(t.Context(), fixture.rwPVC.Name, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + createdPVC.Spec.AccessModes) + require.NotNil(t, createdPVC.Spec.StorageClassName) + assert.Equal(t, nvcastorage.DefaultModelCacheStorageClassName, + *createdPVC.Spec.StorageClassName) + assert.Equal(t, string(fixture.binding.UID), + createdPVC.Labels[nvcastorage.ModelCacheBindingUIDLabelKey]) + assertRWXReadOnlyBindingScopedMetadata(t, createdPVC) + assert.NotContains(t, createdPVC.Labels, nvcastorage.ModelCachePopulatedLabelKey) + + createdJob, err := k8sClient.BatchV1().Jobs(fixture.job.Namespace). + Get(t.Context(), fixture.job.Name, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, string(fixture.binding.UID), + createdJob.Labels[nvcastorage.ModelCacheBindingUIDLabelKey]) + assert.Equal(t, string(fixture.binding.UID), + createdJob.Spec.Template.Labels[nvcastorage.ModelCacheBindingUIDLabelKey]) + assertRWXReadOnlyBindingScopedMetadata(t, createdJob) + assertRWXReadOnlyBindingScopedMetadata(t, &createdJob.Spec.Template.ObjectMeta) + assert.Equal(t, string(createdPVC.UID), + createdJob.Spec.Template.Annotations[nvcastorage.ModelCacheWriterPVCUIDAnnotationKey]) + + actions := k8sClient.Actions() + assert.Equal(t, 1, countRWXReadOnlyAction(actions, "create", "persistentvolumeclaims")) + assert.Equal(t, 1, countRWXReadOnlyAction(actions, "create", "jobs")) + for _, action := range actions { + resource := action.GetResource().Resource + name := rwxReadOnlyActionName(action) + switch resource { + case "persistentvolumes", "volumeattachments": + t.Errorf("rwxReadOnly must not access %s: %s %s", + resource, action.GetVerb(), name) + case "persistentvolumeclaims": + assert.Equal(t, fixture.rwPVC.Name, name, + "writer setup must not address a second PVC") + case "jobs": + assert.Equal(t, fixture.job.Name, name, + "writer setup must only address its recorded Job") + } + } +} + +func TestPrepareRWXReadOnlyResourcesSupportsSharedRequestReuse(t *testing.T) { + fixture := newRWXReadOnlyRuntimeFixture(t, "weka", "csi.weka.io") + firstPVC := fixture.rwPVC.DeepCopy() + firstJob := fixture.job.DeepCopy() + for _, obj := range []metav1.Object{ + firstPVC, firstJob, &firstJob.Spec.Template.ObjectMeta, + } { + addRWXReadOnlyRequestOwnership(obj, "request-a") + } + require.NoError(t, fixture.backend.prepareRegularModelCacheBindingResources( + t.Context(), fixture.binding, firstPVC, firstJob)) + for _, obj := range []metav1.Object{ + firstPVC, firstJob, &firstJob.Spec.Template.ObjectMeta, + } { + assertRWXReadOnlyBindingScopedMetadata(t, obj) + } + + firstPVC.UID = types.UID("shared-pvc-uid") + firstPVC.ResourceVersion = "shared-pvc-rv" + firstPVC.Spec.VolumeName = "shared-pv" + firstPVC.Status.Phase = corev1.ClaimBound + firstPVC.Labels[nvcastorage.ModelCachePopulatedLabelKey] = + nvcastorage.ModelCachePopulatedLabelValue + firstJob.UID = types.UID("shared-job-uid") + firstJob.ResourceVersion = "shared-job-rv" + now := metav1.Now() + firstJob.Status.Succeeded = 1 + firstJob.Status.CompletionTime = &now + require.NoError(t, bindRegularModelCacheWriterJobToPVC( + firstJob, firstPVC, fixture.binding)) + sharedPV := fixture.boundPV(firstPVC) + k8sClient := fixture.useK8sObjects(firstPVC.DeepCopy(), sharedPV, firstJob.DeepCopy()) + secondPVC := fixture.rwPVC.DeepCopy() + secondJob := fixture.job.DeepCopy() + for _, obj := range []metav1.Object{ + secondPVC, secondJob, &secondJob.Spec.Template.ObjectMeta, + } { + addRWXReadOnlyRequestOwnership(obj, "request-b") + } + k8sClient.ClearActions() + require.NoError(t, fixture.backend.prepareRegularModelCacheBindingResources( + t.Context(), fixture.binding, secondPVC, secondJob)) + for _, obj := range []metav1.Object{ + secondPVC, secondJob, &secondJob.Spec.Template.ObjectMeta, + } { + assertRWXReadOnlyBindingScopedMetadata(t, obj) + } + storedBinding, err := fixture.backend.clients.BART.NvcaV2beta1(). + ModelCacheBindings(fixture.binding.Namespace). + Get(t.Context(), fixture.binding.Name, metav1.GetOptions{}) + require.NoError(t, err) + secondReq := fixture.req.DeepCopy() + secondReq.Name = "request-b" + secondReq.UID = types.UID("request-b-uid") + storedBinding.Status.RequestReferences = append( + storedBinding.Status.RequestReferences, + nvcav2beta1.ModelCacheBindingRequestReference{ + Namespace: secondReq.Namespace, + Name: secondReq.Name, + UID: secondReq.UID, + }) + _, err = fixture.backend.clients.BART.NvcaV2beta1(). + ModelCacheBindings(storedBinding.Namespace). + UpdateStatus(t.Context(), storedBinding, metav1.UpdateOptions{}) + require.NoError(t, err) + _, err = fixture.backend.clients.BART.NvcaV2beta1(). + ICMSRequests(secondReq.Namespace).Create(t.Context(), secondReq, metav1.CreateOptions{}) + require.NoError(t, err) + + mutate, claimName, runtimeErr := fixture.backend.setupContainerModelCaching( + newTestContext(), secondReq, secondPVC.DeepCopy(), secondJob.DeepCopy(), nil) + require.NoError(t, runtimeErr) + require.NotNil(t, mutate) + assert.Equal(t, firstPVC.Name, claimName) + assertNoKubernetesWrites(t, k8sClient.Actions()) +} + +func TestPrepareRWXReadOnlyResourcesRejectsRequestOwnedExistingObject(t *testing.T) { + fixture := newRWXReadOnlyRuntimeFixture(t, "weka", "csi.weka.io") + foreignPVC := fixture.rwPVC.DeepCopy() + addRWXReadOnlyRequestOwnership(foreignPVC, "request-a") + fixture.useK8sObjects(foreignPVC) + + err := fixture.backend.prepareRegularModelCacheBindingResources( + t.Context(), fixture.binding, fixture.rwPVC.DeepCopy(), fixture.job.DeepCopy()) + require.ErrorContains(t, err, "request owner references") + assert.True(t, nvcaerrors.IsTerminal(err)) +} + +func TestPrepareRWXReadOnlyResourcesRejectsRequestScopedJobPodTemplate(t *testing.T) { + tests := []struct { + name string + mutate func(*metav1.ObjectMeta) + wantErr string + }{ + { + name: "owner reference", + mutate: func(meta *metav1.ObjectMeta) { + meta.OwnerReferences = []metav1.OwnerReference{{ + APIVersion: nvcav2beta1.SchemeGroupVersion.String(), + Kind: "ICMSRequest", + Name: "request-a", + UID: types.UID("request-a-uid"), + }} + }, + wantErr: "request owner references", + }, + { + name: "request label", + mutate: func(meta *metav1.ObjectMeta) { + meta.Labels[nvcatypes.ICMSRequestIDKey] = "request-a" + }, + wantErr: "request-scoped label", + }, + { + name: "request annotation", + mutate: func(meta *metav1.ObjectMeta) { + meta.Annotations = map[string]string{ + nvcatypes.InstanceCountKey: "1", + } + }, + wantErr: "request-scoped annotation", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fixture := newRWXReadOnlyRuntimeFixture(t, "weka", "csi.weka.io") + existingJob := fixture.job.DeepCopy() + tt.mutate(&existingJob.Spec.Template.ObjectMeta) + k8sClient := fixture.useK8sObjects(existingJob) + + err := fixture.backend.prepareRegularModelCacheBindingResources( + t.Context(), fixture.binding, fixture.rwPVC.DeepCopy(), fixture.job.DeepCopy()) + require.ErrorContains(t, err, "writer Job Pod template") + require.ErrorContains(t, err, tt.wantErr) + assert.True(t, nvcaerrors.IsTerminal(err)) + assertNoKubernetesWrites(t, k8sClient.Actions()) + }) + } +} + +func TestRWXReadOnlyRejectsMissingPVCWithExistingJob(t *testing.T) { + fixture := newRWXReadOnlyRuntimeFixture(t, "weka", "csi.weka.io") + job := fixture.writerJob() + k8sClient := fixture.useK8sObjects(job) + + state, claimName, err := runRWXReadOnlySetup(t, fixture) + assert.Equal(t, ModelCachingFailed, state) + assert.Empty(t, claimName) + require.ErrorContains(t, err, "is missing while writer Job") + assert.True(t, nvcaerrors.IsTerminal(err)) + assertNoKubernetesWrites(t, k8sClient.Actions()) +} + +func TestRWXReadOnlyAdoptsExactCreateRaceWinners(t *testing.T) { + fixture := newResolvedWekaRWXReadOnlyRuntimeFixture(t) + k8sClient := fixture.backend.clients.K8s.(*fakek8sclient.Clientset) + pvcRaceWon, jobRaceWon := false, false + k8sClient.Fake.PrependReactor( + "create", "persistentvolumeclaims", + func(action k8stesting.Action) (bool, runtime.Object, error) { + created := action.(k8stesting.CreateAction).GetObject().(*corev1.PersistentVolumeClaim).DeepCopy() + created.UID = types.UID("race-winner-pvc-uid") + created.ResourceVersion = "race-winner-pvc-rv" + require.NoError(t, k8sClient.Tracker().Create( + corev1.SchemeGroupVersion.WithResource("persistentvolumeclaims"), + created, created.Namespace)) + pvcRaceWon = true + return true, nil, apierrors.NewAlreadyExists( + corev1.Resource("persistentvolumeclaims"), created.Name) + }) + k8sClient.Fake.PrependReactor( + "create", "jobs", + func(action k8stesting.Action) (bool, runtime.Object, error) { + created := action.(k8stesting.CreateAction).GetObject().(*batchv1.Job).DeepCopy() + created.UID = types.UID("race-winner-job-uid") + created.ResourceVersion = "race-winner-job-rv" + require.NoError(t, k8sClient.Tracker().Create( + batchv1.SchemeGroupVersion.WithResource("jobs"), + created, created.Namespace)) + jobRaceWon = true + return true, nil, apierrors.NewAlreadyExists( + corev1.Resource("jobs"), created.Name) + }) + k8sClient.ClearActions() + + state, claimName, err := runRWXReadOnlySetup(t, fixture) + require.NoError(t, err) + assert.Equal(t, ModelCachingInProgress, state) + assert.Empty(t, claimName) + assert.True(t, pvcRaceWon) + assert.True(t, jobRaceWon) + storedPVC, err := k8sClient.CoreV1().PersistentVolumeClaims(fixture.rwPVC.Namespace). + Get(t.Context(), fixture.rwPVC.Name, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, string(fixture.binding.UID), + storedPVC.Labels[nvcastorage.ModelCacheBindingUIDLabelKey]) + storedJob, err := k8sClient.BatchV1().Jobs(fixture.job.Namespace). + Get(t.Context(), fixture.job.Name, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, string(fixture.binding.UID), + storedJob.Labels[nvcastorage.ModelCacheBindingUIDLabelKey]) +} + +func (f *rwxReadOnlyRuntimeFixture) boundPVC(populated bool) *corev1.PersistentVolumeClaim { + pvc := f.rwPVC.DeepCopy() + pvc.UID = types.UID("writer-pvc-uid") + pvc.ResourceVersion = "writer-pvc-rv" + pvc.Spec.VolumeName = "pv-model-cache-handle" + pvc.Status.Phase = corev1.ClaimBound + if populated { + pvc.Labels[nvcastorage.ModelCachePopulatedLabelKey] = + nvcastorage.ModelCachePopulatedLabelValue + } + return pvc +} + +func (f *rwxReadOnlyRuntimeFixture) boundPV( + pvc *corev1.PersistentVolumeClaim, +) *corev1.PersistentVolume { + return &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{Name: pvc.Spec.VolumeName}, + Spec: corev1.PersistentVolumeSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + StorageClassName: nvcastorage.DefaultModelCacheStorageClassName, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + ClaimRef: &corev1.ObjectReference{ + Namespace: pvc.Namespace, + Name: pvc.Name, + UID: pvc.UID, + }, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + Driver: f.binding.Spec.Decision.Provisioner, + VolumeHandle: "model-cache-volume-handle", + }, + }, + }, + Status: corev1.PersistentVolumeStatus{Phase: corev1.VolumeBound}, + } +} + +func (f *rwxReadOnlyRuntimeFixture) writerJob() *batchv1.Job { + job := f.job.DeepCopy() + job.UID = types.UID("writer-job-uid") + job.ResourceVersion = "writer-job-rv" + if job.Spec.Template.Annotations == nil { + job.Spec.Template.Annotations = map[string]string{} + } + job.Spec.Template.Annotations[nvcastorage.ModelCacheWriterPVCUIDAnnotationKey] = "writer-pvc-uid" + return job +} + +func (f *rwxReadOnlyRuntimeFixture) completedWriterJob() *batchv1.Job { + job := f.writerJob() + now := metav1.Now() + job.Status.Succeeded = 1 + job.Status.CompletionTime = &now + return job +} + +func (f *rwxReadOnlyRuntimeFixture) useK8sObjects( + objects ...runtime.Object, +) *fakek8sclient.Clientset { + k8sClient := fakek8sclient.NewSimpleClientset(objects...) + f.backend.clients.K8s = k8sClient + f.backend.bk8s.clients.K8s = k8sClient + return k8sClient +} + +func installRWXReadOnlyCreateIdentityReactors( + t *testing.T, + k8sClient *fakek8sclient.Clientset, +) { + t.Helper() + k8sClient.Fake.PrependReactor( + "create", "persistentvolumeclaims", + func(action k8stesting.Action) (bool, runtime.Object, error) { + created := action.(k8stesting.CreateAction). + GetObject().(*corev1.PersistentVolumeClaim).DeepCopy() + created.UID = types.UID("api-created-writer-pvc-uid") + created.ResourceVersion = "api-created-writer-pvc-rv" + require.NoError(t, k8sClient.Tracker().Create( + corev1.SchemeGroupVersion.WithResource("persistentvolumeclaims"), + created, created.Namespace)) + return true, created, nil + }) + k8sClient.Fake.PrependReactor( + "create", "jobs", + func(action k8stesting.Action) (bool, runtime.Object, error) { + created := action.(k8stesting.CreateAction). + GetObject().(*batchv1.Job).DeepCopy() + created.UID = types.UID("api-created-writer-job-uid") + created.ResourceVersion = "api-created-writer-job-rv" + require.NoError(t, k8sClient.Tracker().Create( + batchv1.SchemeGroupVersion.WithResource("jobs"), + created, created.Namespace)) + return true, created, nil + }) +} + +func runRWXReadOnlySetup( + t *testing.T, + fixture *rwxReadOnlyRuntimeFixture, +) (ModelCachingState, string, error) { + t.Helper() + return fixture.backend.SetupModelCachingForRequest( + newTestContext(), + fixture.rwPVC.DeepCopy(), + fixture.job.DeepCopy(), + fixture.req, + false, + nil, + ) +} + +func rwxReadOnlyActionName(action k8stesting.Action) string { + if named, ok := action.(interface{ GetName() string }); ok { + return named.GetName() + } + if objectAction, ok := action.(interface{ GetObject() runtime.Object }); ok { + if object, ok := objectAction.GetObject().(metav1.Object); ok { + return object.GetName() + } + } + return "" +} + +func assertRWXReadOnlyActionTrace( + t *testing.T, + actions []k8stesting.Action, + wantPVCName string, + wantJobName string, +) { + t.Helper() + for _, action := range actions { + resource := action.GetResource().Resource + verb := action.GetVerb() + name := rwxReadOnlyActionName(action) + switch resource { + case "persistentvolumes": + assert.Equal(t, "get", verb, + "rwxReadOnly may validate a PV but must not modify it") + case "volumeattachments": + t.Errorf("rwxReadOnly must not access %s: %s %s", resource, verb, name) + case "persistentvolumeclaims": + assert.Equal(t, wantPVCName, name, + "rwxReadOnly must not address a second PVC") + assert.NotEqual(t, "create", verb, + "an existing populated/unpopulated writer claim must not be replaced") + assert.NotEqual(t, "delete", verb, + "rwxReadOnly publication must retain the writer claim") + assert.NotEqual(t, "delete-collection", verb, + "rwxReadOnly publication must retain the writer claim") + case "jobs": + assert.Equal(t, wantJobName, name, + "rwxReadOnly must only address its recorded writer Job") + } + } +} + +func countRWXReadOnlyAction( + actions []k8stesting.Action, + verb string, + resource string, +) int { + count := 0 + for _, action := range actions { + if action.GetVerb() == verb && action.GetResource().Resource == resource { + count++ + } + } + return count +} + +func TestRWXReadOnlyCompletedWriterPublishesSameClaim(t *testing.T) { + providers := []struct { + name string + provider string + provisioner string + }{ + {name: "Weka", provider: "weka", provisioner: "csi.weka.io"}, + {name: "OCI FSS", provider: "ociFss", provisioner: "fss.csi.oraclecloud.com"}, + } + for _, tt := range providers { + t.Run(tt.name, func(t *testing.T) { + fixture := newRWXReadOnlyRuntimeFixture(t, tt.provider, tt.provisioner) + pvc := fixture.boundPVC(false) + pv := fixture.boundPV(pvc) + job := fixture.completedWriterJob() + k8sClient := fixture.useK8sObjects(pvc, pv, job) + + state, claimName, err := runRWXReadOnlySetup(t, fixture) + require.NoError(t, err) + assert.Equal(t, ModelCachingCompleted, state) + assert.Equal(t, fixture.rwPVC.Name, claimName) + + actions := k8sClient.Actions() + assertRWXReadOnlyActionTrace(t, actions, fixture.rwPVC.Name, fixture.job.Name) + assert.Equal(t, 1, countRWXReadOnlyAction(actions, "update", "persistentvolumeclaims")) + assert.Equal(t, 0, countRWXReadOnlyAction(actions, "delete", "jobs")) + assert.Equal(t, 0, countRWXReadOnlyAction(actions, "create", "jobs")) + + storedPVC, getErr := k8sClient.CoreV1().PersistentVolumeClaims(pvc.Namespace). + Get(t.Context(), pvc.Name, metav1.GetOptions{}) + require.NoError(t, getErr) + assert.Equal(t, nvcastorage.ModelCachePopulatedLabelValue, + storedPVC.Labels[nvcastorage.ModelCachePopulatedLabelKey]) + storedJob, getErr := k8sClient.BatchV1().Jobs(job.Namespace). + Get(t.Context(), job.Name, metav1.GetOptions{}) + require.NoError(t, getErr) + assert.Equal(t, job.UID, storedJob.UID, + "the completed Job must remain as the publication fence") + + k8sClient.ClearActions() + state, claimName, err = runRWXReadOnlySetup(t, fixture) + require.NoError(t, err) + assert.Equal(t, ModelCachingCompleted, state) + assert.Equal(t, fixture.rwPVC.Name, claimName) + assert.Equal(t, 0, countRWXReadOnlyAction( + k8sClient.Actions(), "create", "jobs")) + assert.Equal(t, 0, countRWXReadOnlyAction( + k8sClient.Actions(), "delete", "jobs")) + }) + } +} + +func TestRWXReadOnlyRejectsMissingPublicationFence(t *testing.T) { + fixture := newRWXReadOnlyRuntimeFixture(t, "weka", "csi.weka.io") + pvc := fixture.boundPVC(true) + pv := fixture.boundPV(pvc) + k8sClient := fixture.useK8sObjects(pvc, pv) + + state, claimName, err := runRWXReadOnlySetup(t, fixture) + assert.Equal(t, ModelCachingFailed, state) + assert.Empty(t, claimName) + require.ErrorContains(t, err, "publication fence Job") + assert.True(t, nvcaerrors.IsTerminal(err)) + + actions := k8sClient.Actions() + assertRWXReadOnlyActionTrace(t, actions, pvc.Name, fixture.job.Name) + assert.Equal(t, 0, countRWXReadOnlyAction(actions, "create", "jobs")) + assert.Equal(t, 0, countRWXReadOnlyAction(actions, "delete", "jobs")) + assert.Equal(t, 0, countRWXReadOnlyAction(actions, "update", "persistentvolumeclaims")) +} + +func TestRWXReadOnlyUnpopulatedClaimRestartsWriter(t *testing.T) { + fixture := newResolvedWekaRWXReadOnlyRuntimeFixture(t) + pvc := fixture.boundPVC(false) + pv := fixture.boundPV(pvc) + k8sClient := fixture.useK8sObjects( + pvc, pv, selectionStorageClassForProvisioner("csi.weka.io")) + + state, claimName, err := runRWXReadOnlySetup(t, fixture) + require.NoError(t, err) + assert.Equal(t, ModelCachingInProgress, state) + assert.Empty(t, claimName, "an unpopulated claim must never be published") + + actions := k8sClient.Actions() + assertRWXReadOnlyActionTrace(t, actions, pvc.Name, fixture.job.Name) + assert.Equal(t, 1, countRWXReadOnlyAction(actions, "create", "jobs")) + assert.Equal(t, 0, countRWXReadOnlyAction(actions, "update", "persistentvolumeclaims")) + createdJob, getErr := k8sClient.BatchV1().Jobs(fixture.job.Namespace). + Get(t.Context(), fixture.job.Name, metav1.GetOptions{}) + require.NoError(t, getErr) + assert.Equal(t, string(fixture.binding.UID), + createdJob.Labels[nvcastorage.ModelCacheBindingUIDLabelKey]) +} + +func TestRWXReadOnlyRejectsForeignOwnership(t *testing.T) { + t.Run("writer PVC", func(t *testing.T) { + fixture := newRWXReadOnlyRuntimeFixture(t, "weka", "csi.weka.io") + pvc := fixture.boundPVC(false) + pvc.Labels[nvcastorage.ModelCacheBindingUIDLabelKey] = "foreign-binding" + k8sClient := fixture.useK8sObjects(pvc) + + state, claimName, err := runRWXReadOnlySetup(t, fixture) + assert.Equal(t, ModelCachingFailed, state) + assert.Empty(t, claimName) + require.ErrorContains(t, err, "foreign-binding") + assert.True(t, nvcaerrors.IsTerminal(err)) + assertRWXReadOnlyActionTrace(t, k8sClient.Actions(), pvc.Name, fixture.job.Name) + assertNoKubernetesWrites(t, k8sClient.Actions()) + }) + + t.Run("writer Job", func(t *testing.T) { + fixture := newRWXReadOnlyRuntimeFixture(t, "weka", "csi.weka.io") + pvc := fixture.boundPVC(false) + pv := fixture.boundPV(pvc) + job := fixture.writerJob() + job.Labels[nvcastorage.ModelCacheBindingUIDLabelKey] = "foreign-binding" + k8sClient := fixture.useK8sObjects(pvc, pv, job) + + state, claimName, err := runRWXReadOnlySetup(t, fixture) + assert.Equal(t, ModelCachingFailed, state) + assert.Empty(t, claimName) + require.ErrorContains(t, err, "foreign-binding") + assert.True(t, nvcaerrors.IsTerminal(err)) + assertRWXReadOnlyActionTrace(t, k8sClient.Actions(), pvc.Name, fixture.job.Name) + assertNoKubernetesWrites(t, k8sClient.Actions()) + }) +} + +func TestRWXReadOnlyRejectsActiveWriterBehindPopulatedMarker(t *testing.T) { + fixture := newRWXReadOnlyRuntimeFixture(t, "weka", "csi.weka.io") + pvc := fixture.boundPVC(true) + pv := fixture.boundPV(pvc) + job := fixture.writerJob() + job.Status.Active = 1 + k8sClient := fixture.useK8sObjects(pvc, pv, job) + + state, claimName, err := runRWXReadOnlySetup(t, fixture) + assert.Equal(t, ModelCachingFailed, state) + assert.Empty(t, claimName) + require.ErrorContains(t, err, "populated rwxReadOnly writer PVC has non-completed Job") + assert.True(t, nvcaerrors.IsTerminal(err)) + assertRWXReadOnlyActionTrace(t, k8sClient.Actions(), pvc.Name, fixture.job.Name) + assertNoKubernetesWrites(t, k8sClient.Actions()) +} + +func TestRWXReadOnlyRejectsDriftedCompletedPublicationFence(t *testing.T) { + tests := []struct { + name string + mutate func(*batchv1.Job) + wantErr string + }{ + { + name: "foreign binding UID", + mutate: func(job *batchv1.Job) { + job.Labels[nvcastorage.ModelCacheBindingUIDLabelKey] = "foreign-binding" + }, + wantErr: "foreign-binding", + }, + { + name: "immutable writer spec", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.Containers[0].Image = "example.invalid/foreign:latest" + }, + wantErr: "immutable spec does not match intent", + }, + { + name: "recreated PVC UID witness", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Annotations[nvcastorage.ModelCacheWriterPVCUIDAnnotationKey] = + "previous-writer-pvc-uid" + }, + wantErr: "records PVC UID", + }, + { + name: "terminating Job", + mutate: func(job *batchv1.Job) { + now := metav1.Now() + job.DeletionTimestamp = &now + }, + wantErr: "is terminating", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fixture := newRWXReadOnlyRuntimeFixture(t, "weka", "csi.weka.io") + pvc := fixture.boundPVC(true) + pv := fixture.boundPV(pvc) + job := fixture.completedWriterJob() + tt.mutate(job) + k8sClient := fixture.useK8sObjects(pvc, pv, job) + + state, claimName, err := runRWXReadOnlySetup(t, fixture) + assert.Equal(t, ModelCachingFailed, state) + assert.Empty(t, claimName) + require.ErrorContains(t, err, tt.wantErr) + assert.True(t, nvcaerrors.IsTerminal(err)) + assertNoKubernetesWrites(t, k8sClient.Actions()) + }) + } +} + +func TestRWXReadOnlyRejectsBoundPVIdentityDrift(t *testing.T) { + tests := []struct { + name string + mutate func(*corev1.PersistentVolumeClaim, *corev1.PersistentVolume) + omitPV bool + wantErr string + }{ + { + name: "empty volume name", + mutate: func(pvc *corev1.PersistentVolumeClaim, _ *corev1.PersistentVolume) { + pvc.Spec.VolumeName = "" + }, + wantErr: "has no PV name", + }, + {name: "missing PV", omitPV: true, wantErr: "references missing PV"}, + { + name: "missing claimRef", + mutate: func(_ *corev1.PersistentVolumeClaim, pv *corev1.PersistentVolume) { + pv.Spec.ClaimRef = nil + }, + wantErr: "has no claimRef", + }, + { + name: "wrong claimRef namespace", + mutate: func(_ *corev1.PersistentVolumeClaim, pv *corev1.PersistentVolume) { + pv.Spec.ClaimRef.Namespace = "foreign-namespace" + }, + wantErr: "claimRef does not match exact PVC", + }, + { + name: "wrong claimRef name", + mutate: func(_ *corev1.PersistentVolumeClaim, pv *corev1.PersistentVolume) { + pv.Spec.ClaimRef.Name = "foreign-claim" + }, + wantErr: "claimRef does not match exact PVC", + }, + { + name: "wrong claimRef UID", + mutate: func(_ *corev1.PersistentVolumeClaim, pv *corev1.PersistentVolume) { + pv.Spec.ClaimRef.UID = types.UID("foreign-pvc-uid") + }, + wantErr: "claimRef UID", + }, + { + name: "wrong CSI driver", + mutate: func(_ *corev1.PersistentVolumeClaim, pv *corev1.PersistentVolume) { + pv.Spec.CSI.Driver = "foreign.csi.example.com" + }, + wantErr: "CSI driver", + }, + { + name: "missing CSI source", + mutate: func(_ *corev1.PersistentVolumeClaim, pv *corev1.PersistentVolume) { + pv.Spec.CSI = nil + }, + wantErr: "CSI driver", + }, + { + name: "wrong access mode", + mutate: func(_ *corev1.PersistentVolumeClaim, pv *corev1.PersistentVolume) { + pv.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + }, + wantErr: "access modes", + }, + { + name: "wrong StorageClass", + mutate: func(_ *corev1.PersistentVolumeClaim, pv *corev1.PersistentVolume) { + pv.Spec.StorageClassName = "foreign-sc" + }, + wantErr: "StorageClass", + }, + { + name: "wrong reclaim policy", + mutate: func(_ *corev1.PersistentVolumeClaim, pv *corev1.PersistentVolume) { + pv.Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete + }, + wantErr: "reclaim policy", + }, + { + name: "empty CSI volume handle", + mutate: func(_ *corev1.PersistentVolumeClaim, pv *corev1.PersistentVolume) { + pv.Spec.CSI.VolumeHandle = "" + }, + wantErr: "empty CSI volume handle", + }, + { + name: "wrong volume mode", + mutate: func(_ *corev1.PersistentVolumeClaim, pv *corev1.PersistentVolume) { + mode := corev1.PersistentVolumeBlock + pv.Spec.VolumeMode = &mode + }, + wantErr: "volume mode", + }, + { + name: "terminating PVC", + mutate: func(pvc *corev1.PersistentVolumeClaim, _ *corev1.PersistentVolume) { + now := metav1.Now() + pvc.DeletionTimestamp = &now + }, + wantErr: "is terminating", + }, + { + name: "terminating PV", + mutate: func(_ *corev1.PersistentVolumeClaim, pv *corev1.PersistentVolume) { + now := metav1.Now() + pv.DeletionTimestamp = &now + }, + wantErr: "is terminating", + }, + { + name: "PV Available", + mutate: func(_ *corev1.PersistentVolumeClaim, pv *corev1.PersistentVolume) { + pv.Status.Phase = corev1.VolumeAvailable + }, + wantErr: "phase", + }, + { + name: "PV Released", + mutate: func(_ *corev1.PersistentVolumeClaim, pv *corev1.PersistentVolume) { + pv.Status.Phase = corev1.VolumeReleased + }, + wantErr: "phase", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fixture := newRWXReadOnlyRuntimeFixture(t, "weka", "csi.weka.io") + pvc := fixture.boundPVC(true) + pv := fixture.boundPV(pvc) + job := fixture.completedWriterJob() + if tt.mutate != nil { + tt.mutate(pvc, pv) + } + objects := []runtime.Object{pvc, job} + if !tt.omitPV { + objects = append(objects, pv) + } + k8sClient := fixture.useK8sObjects(objects...) + + state, claimName, err := runRWXReadOnlySetup(t, fixture) + assert.Equal(t, ModelCachingFailed, state) + assert.Empty(t, claimName) + require.ErrorContains(t, err, tt.wantErr) + assert.True(t, nvcaerrors.IsTerminal(err)) + assertRWXReadOnlyActionTrace(t, k8sClient.Actions(), pvc.Name, fixture.job.Name) + assertNoKubernetesWrites(t, k8sClient.Actions()) + }) + } +} + +func TestRWXReadOnlyRejectsCompletedWriterJobDriftBeforeMarker(t *testing.T) { + fixture := newRWXReadOnlyRuntimeFixture(t, "weka", "csi.weka.io") + pvc := fixture.boundPVC(false) + pv := fixture.boundPV(pvc) + job := fixture.completedWriterJob() + job.Spec.Template.Spec.Containers = []corev1.Container{{ + Name: "foreign-writer", + Image: "example.invalid/foreign:latest", + }} + k8sClient := fixture.useK8sObjects(pvc, pv, job) + + state, claimName, err := runRWXReadOnlySetup(t, fixture) + assert.Equal(t, ModelCachingFailed, state) + assert.Empty(t, claimName) + require.ErrorContains(t, err, "immutable spec does not match intent") + assert.True(t, nvcaerrors.IsTerminal(err)) + assertNoKubernetesWrites(t, k8sClient.Actions()) + + storedPVC, getErr := k8sClient.CoreV1().PersistentVolumeClaims(pvc.Namespace). + Get(t.Context(), pvc.Name, metav1.GetOptions{}) + require.NoError(t, getErr) + assert.NotContains(t, storedPVC.Labels, nvcastorage.ModelCachePopulatedLabelKey) +} + +func TestRWXReadOnlyRejectsWriterJobTTL(t *testing.T) { + fixture := newRWXReadOnlyRuntimeFixture(t, "weka", "csi.weka.io") + ttl := int32(60) + fixture.job.Spec.TTLSecondsAfterFinished = &ttl + + state, claimName, err := runRWXReadOnlySetup(t, fixture) + assert.Equal(t, ModelCachingFailed, state) + assert.Empty(t, claimName) + require.ErrorContains(t, err, "must not use ttlSecondsAfterFinished") + assert.True(t, nvcaerrors.IsTerminal(err)) + assert.Empty(t, fixture.backend.clients.K8s.(*fakek8sclient.Clientset).Actions()) +} + +func TestRWXReadOnlyRejectsWriterWithoutWritableClaimMount(t *testing.T) { + tests := []struct { + name string + mutate func(*batchv1.Job) + wantErr string + }{ + { + name: "missing claim volume", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.Volumes = nil + }, + wantErr: "has no", + }, + { + name: "different claim", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = "other-claim" + }, + wantErr: "does not reference PVC", + }, + { + name: "read-only claim source", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ReadOnly = true + }, + wantErr: "PVC volume", + }, + { + name: "read-only mount", + mutate: func(job *batchv1.Job) { + job.Spec.Template.Spec.Containers[0].VolumeMounts[0].ReadOnly = true + }, + wantErr: "does not mount PVC", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fixture := newRWXReadOnlyRuntimeFixture(t, "weka", "csi.weka.io") + tt.mutate(fixture.job) + state, claimName, err := runRWXReadOnlySetup(t, fixture) + assert.Equal(t, ModelCachingFailed, state) + assert.Empty(t, claimName) + require.ErrorContains(t, err, tt.wantErr) + assert.True(t, nvcaerrors.IsTerminal(err)) + assert.Empty(t, fixture.backend.clients.K8s.(*fakek8sclient.Clientset).Actions()) + }) + } +} + +func TestRWXReadOnlyRejectsDecoyWritablePVCMount(t *testing.T) { + fixture := newRWXReadOnlyRuntimeFixture(t, "weka", "csi.weka.io") + fixture.job.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = "other-claim" + fixture.job.Spec.Template.Spec.Volumes = append( + fixture.job.Spec.Template.Spec.Volumes, + corev1.Volume{ + Name: "decoy-model-cache", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: fixture.rwPVC.Name, + }, + }, + }) + fixture.job.Spec.Template.Spec.Containers[0].VolumeMounts = append( + fixture.job.Spec.Template.Spec.Containers[0].VolumeMounts, + corev1.VolumeMount{Name: "decoy-model-cache", MountPath: "/decoy"}) + + state, claimName, err := runRWXReadOnlySetup(t, fixture) + assert.Equal(t, ModelCachingFailed, state) + assert.Empty(t, claimName) + require.Error(t, err) + assert.True(t, nvcaerrors.IsTerminal(err)) + assert.Empty(t, fixture.backend.clients.K8s.(*fakek8sclient.Clientset).Actions()) +} + +func TestRWXReadOnlyCleansUpObjectsWhenBindingRetiresAfterCreate(t *testing.T) { + fixture := newResolvedWekaRWXReadOnlyRuntimeFixture(t) + k8sClient := fixture.backend.clients.K8s.(*fakek8sclient.Clientset) + pvcCreated := false + jobCreated := false + k8sClient.Fake.PrependReactor( + "create", "persistentvolumeclaims", + func(action k8stesting.Action) (bool, runtime.Object, error) { + created := action.(k8stesting.CreateAction). + GetObject().(*corev1.PersistentVolumeClaim).DeepCopy() + created.UID = types.UID("created-writer-pvc-uid") + created.ResourceVersion = "created-writer-pvc-rv" + require.NoError(t, k8sClient.Tracker().Create( + corev1.SchemeGroupVersion.WithResource("persistentvolumeclaims"), + created, created.Namespace)) + pvcCreated = true + return true, created, nil + }) + k8sClient.Fake.PrependReactor( + "create", "jobs", + func(action k8stesting.Action) (bool, runtime.Object, error) { + created := action.(k8stesting.CreateAction).GetObject().(*batchv1.Job).DeepCopy() + created.UID = types.UID("created-writer-job-uid") + created.ResourceVersion = "created-writer-job-rv" + require.NoError(t, k8sClient.Tracker().Create( + batchv1.SchemeGroupVersion.WithResource("jobs"), + created, created.Namespace)) + jobCreated = true + + binding, err := fixture.backend.clients.BART.NvcaV2beta1(). + ModelCacheBindings(fixture.binding.Namespace). + Get(t.Context(), fixture.binding.Name, metav1.GetOptions{}) + require.NoError(t, err) + binding.Status.Phase = nvcav2beta1.ModelCacheBindingPhaseRetiring + _, err = fixture.backend.clients.BART.NvcaV2beta1(). + ModelCacheBindings(binding.Namespace). + UpdateStatus(t.Context(), binding, metav1.UpdateOptions{}) + require.NoError(t, err) + return true, created, nil + }) + k8sClient.ClearActions() + + state, claimName, err := runRWXReadOnlySetup(t, fixture) + assert.Equal(t, ModelCachingFailed, state) + assert.Empty(t, claimName) + require.ErrorIs(t, err, errRegularModelCacheBindingRetiring) + assert.True(t, nvcaerrors.IsTerminal(err)) + assert.True(t, pvcCreated) + assert.True(t, jobCreated) + + _, getErr := k8sClient.CoreV1().PersistentVolumeClaims(fixture.rwPVC.Namespace). + Get(t.Context(), fixture.rwPVC.Name, metav1.GetOptions{}) + assert.True(t, apierrors.IsNotFound(getErr), "binding-owned PVC must be removed") + _, getErr = k8sClient.BatchV1().Jobs(fixture.job.Namespace). + Get(t.Context(), fixture.job.Name, metav1.GetOptions{}) + assert.True(t, apierrors.IsNotFound(getErr), "binding-owned Job must be removed") + assert.Equal(t, 1, countRWXReadOnlyAction( + k8sClient.Actions(), "delete", "persistentvolumeclaims")) + assert.Equal(t, 1, countRWXReadOnlyAction(k8sClient.Actions(), "delete", "jobs")) +} + +func TestRWXReadOnlyRetriesPopulatedMarkerConflict(t *testing.T) { + fixture := newRWXReadOnlyRuntimeFixture(t, "weka", "csi.weka.io") + pvc := fixture.boundPVC(false) + pv := fixture.boundPV(pvc) + job := fixture.completedWriterJob() + k8sClient := fixture.useK8sObjects(pvc, pv, job) + updateAttempts := 0 + k8sClient.Fake.PrependReactor( + "update", "persistentvolumeclaims", + func(k8stesting.Action) (bool, runtime.Object, error) { + updateAttempts++ + if updateAttempts == 1 { + return true, nil, apierrors.NewConflict( + corev1.Resource("persistentvolumeclaims"), pvc.Name, + assert.AnError) + } + return false, nil, nil + }) + + state, claimName, err := runRWXReadOnlySetup(t, fixture) + require.NoError(t, err) + assert.Equal(t, ModelCachingCompleted, state) + assert.Equal(t, pvc.Name, claimName) + assert.Equal(t, 2, updateAttempts) + storedPVC, getErr := k8sClient.CoreV1().PersistentVolumeClaims(pvc.Namespace). + Get(t.Context(), pvc.Name, metav1.GetOptions{}) + require.NoError(t, getErr) + assert.Equal(t, nvcastorage.ModelCachePopulatedLabelValue, + storedPVC.Labels[nvcastorage.ModelCachePopulatedLabelKey]) +} + +func TestRWXReadOnlyRefusesPublicationStateRace(t *testing.T) { + tests := []struct { + name string + mutate func(*testing.T, *rwxReadOnlyRuntimeFixture, *fakek8sclient.Clientset, + *corev1.PersistentVolume) error + wantErr string + }{ + { + name: "PV identity changes", + mutate: func(_ *testing.T, _ *rwxReadOnlyRuntimeFixture, + k8sClient *fakek8sclient.Clientset, pv *corev1.PersistentVolume) error { + drifted := pv.DeepCopy() + drifted.Spec.CSI.Driver = "foreign.csi.example.com" + return k8sClient.Tracker().Update( + corev1.SchemeGroupVersion.WithResource("persistentvolumes"), + drifted, "") + }, + wantErr: "CSI driver", + }, + { + name: "completed Job disappears", + mutate: func(_ *testing.T, fixture *rwxReadOnlyRuntimeFixture, + k8sClient *fakek8sclient.Clientset, _ *corev1.PersistentVolume) error { + return k8sClient.Tracker().Delete( + batchv1.SchemeGroupVersion.WithResource("jobs"), + fixture.job.Namespace, fixture.job.Name) + }, + wantErr: "publication fence Job", + }, + { + name: "binding retires", + mutate: func(t *testing.T, fixture *rwxReadOnlyRuntimeFixture, + _ *fakek8sclient.Clientset, _ *corev1.PersistentVolume) error { + binding, err := fixture.backend.clients.BART.NvcaV2beta1(). + ModelCacheBindings(fixture.binding.Namespace). + Get(t.Context(), fixture.binding.Name, metav1.GetOptions{}) + if err != nil { + return err + } + binding.Status.Phase = nvcav2beta1.ModelCacheBindingPhaseRetiring + _, err = fixture.backend.clients.BART.NvcaV2beta1(). + ModelCacheBindings(binding.Namespace). + UpdateStatus(t.Context(), binding, metav1.UpdateOptions{}) + return err + }, + wantErr: "Retiring", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fixture := newRWXReadOnlyRuntimeFixture(t, "weka", "csi.weka.io") + pvc := fixture.boundPVC(false) + pv := fixture.boundPV(pvc) + job := fixture.completedWriterJob() + k8sClient := fixture.useK8sObjects(pvc, pv, job) + raced := false + k8sClient.Fake.PrependReactor( + "update", "persistentvolumeclaims", + func(k8stesting.Action) (bool, runtime.Object, error) { + if raced { + return false, nil, nil + } + raced = true + require.NoError(t, tt.mutate(t, fixture, k8sClient, pv)) + return true, nil, apierrors.NewConflict( + corev1.Resource("persistentvolumeclaims"), pvc.Name, assert.AnError) + }) + + state, claimName, err := runRWXReadOnlySetup(t, fixture) + assert.Equal(t, ModelCachingFailed, state) + assert.Empty(t, claimName) + require.ErrorContains(t, err, tt.wantErr) + assert.True(t, nvcaerrors.IsTerminal(err)) + storedPVC, getErr := k8sClient.CoreV1().PersistentVolumeClaims(pvc.Namespace). + Get(t.Context(), pvc.Name, metav1.GetOptions{}) + require.NoError(t, getErr) + assert.NotContains(t, storedPVC.Labels, nvcastorage.ModelCachePopulatedLabelKey) + }) + } +} + +func TestRWXReadOnlyPropagatesTransientPVRead(t *testing.T) { + fixture := newRWXReadOnlyRuntimeFixture(t, "weka", "csi.weka.io") + pvc := fixture.boundPVC(false) + pv := fixture.boundPV(pvc) + k8sClient := fixture.useK8sObjects(pvc, pv) + k8sClient.Fake.PrependReactor( + "get", "persistentvolumes", + func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewServiceUnavailable("PV API unavailable") + }) + + state, claimName, err := runRWXReadOnlySetup(t, fixture) + assert.Equal(t, ModelCachingInProgress, state) + assert.Empty(t, claimName) + require.Error(t, err) + assert.True(t, apierrors.IsServiceUnavailable(err)) + assert.False(t, nvcaerrors.IsTerminal(err)) + assertNoKubernetesWrites(t, k8sClient.Actions()) +} + +func TestCleanupModelCachingSetupArtifactsRWXReadOnlyHandlesRetainedPV(t *testing.T) { + fixture := newRWXReadOnlyRuntimeFixture(t, "weka", "csi.weka.io") + pvc := fixture.boundPVC(false) + pv := fixture.boundPV(pvc) + job := fixture.writerJob() + pvc.APIVersion = "v1" + pvc.Kind = "PersistentVolumeClaim" + job.APIVersion = "batch/v1" + job.Kind = "Job" + setCleanupArtifacts(t, fixture.req, pvc, job) + k8sClient := fixture.useK8sObjects(pvc, pv, job) + + require.NoError(t, fixture.backend.CleanupModelCachingSetupArtifacts( + newTestContext(), fixture.req)) + storedBinding, err := fixture.backend.clients.BART.NvcaV2beta1(). + ModelCacheBindings(fixture.binding.Namespace). + Get(t.Context(), fixture.binding.Name, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, nvcav2beta1.ModelCacheBindingPhaseRetiring, storedBinding.Status.Phase) + updatedPV, err := k8sClient.CoreV1().PersistentVolumes().Get( + t.Context(), pv.Name, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, corev1.PersistentVolumeReclaimDelete, + updatedPV.Spec.PersistentVolumeReclaimPolicy) + _, err = k8sClient.CoreV1().PersistentVolumeClaims(pvc.Namespace). + Get(t.Context(), pvc.Name, metav1.GetOptions{}) + assert.True(t, apierrors.IsNotFound(err)) + _, err = k8sClient.BatchV1().Jobs(job.Namespace). + Get(t.Context(), job.Name, metav1.GetOptions{}) + assert.True(t, apierrors.IsNotFound(err)) + + pvUpdateIndex, pvcDeleteIndex := -1, -1 + for index, action := range k8sClient.Actions() { + switch { + case action.GetVerb() == "update" && action.GetResource().Resource == "persistentvolumes": + pvUpdateIndex = index + case action.GetVerb() == "delete" && + action.GetResource().Resource == "persistentvolumeclaims": + pvcDeleteIndex = index + } + } + require.NotEqual(t, -1, pvUpdateIndex) + require.NotEqual(t, -1, pvcDeleteIndex) + assert.Less(t, pvUpdateIndex, pvcDeleteIndex, + "the exact retained PV must change to Delete before the claim is removed") +} + +func TestSetupContainerModelCachingRWXReadOnlyMountsSameClaimReadOnly(t *testing.T) { + fixture := newRWXReadOnlyRuntimeFixture(t, "weka", "csi.weka.io") + pvc := fixture.boundPVC(true) + pv := fixture.boundPV(pvc) + job := fixture.completedWriterJob() + k8sClient := fixture.useK8sObjects(pvc, pv, job) + + mutate, claimName, err := fixture.backend.setupContainerModelCaching( + newTestContext(), + fixture.req, + fixture.rwPVC.DeepCopy(), + fixture.job.DeepCopy(), + nil, + ) + require.NoError(t, err) + require.NotNil(t, mutate) + assert.Equal(t, pvc.Name, claimName) + + pod := &corev1.Pod{Spec: corev1.PodSpec{ + Volumes: []corev1.Volume{ + {Name: ModelVolumeName, VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + {Name: "unrelated", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}, + }, + InitContainers: []corev1.Container{{ + Name: "init", + VolumeMounts: []corev1.VolumeMount{ + {Name: ModelVolumeName, MountPath: "/models"}, + {Name: "unrelated", MountPath: "/config"}, + }, + }}, + Containers: []corev1.Container{{ + Name: "worker", + VolumeMounts: []corev1.VolumeMount{ + {Name: ModelVolumeName, MountPath: "/models"}, + {Name: "unrelated", MountPath: "/config"}, + }, + }}, + }} + mutate(pod) + + require.NotNil(t, pod.Spec.Volumes[0].PersistentVolumeClaim) + assert.Equal(t, pvc.Name, pod.Spec.Volumes[0].PersistentVolumeClaim.ClaimName) + assert.True(t, pod.Spec.Volumes[0].PersistentVolumeClaim.ReadOnly) + assert.True(t, pod.Spec.InitContainers[0].VolumeMounts[0].ReadOnly) + assert.True(t, pod.Spec.Containers[0].VolumeMounts[0].ReadOnly) + assert.False(t, pod.Spec.InitContainers[0].VolumeMounts[1].ReadOnly) + assert.False(t, pod.Spec.Containers[0].VolumeMounts[1].ReadOnly) + assert.NotNil(t, pod.Spec.Volumes[1].EmptyDir) + + assertRWXReadOnlyActionTrace(t, k8sClient.Actions(), pvc.Name, fixture.job.Name) + assert.Equal(t, 0, countRWXReadOnlyAction(k8sClient.Actions(), "create", "jobs")) + assert.Equal(t, 0, countRWXReadOnlyAction(k8sClient.Actions(), "delete", "jobs")) +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend_modelcache_test.go b/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend_modelcache_test.go index cec4192dc..357613ba7 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend_modelcache_test.go +++ b/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend_modelcache_test.go @@ -34,6 +34,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" nvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/encryption" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/types" ) @@ -41,6 +42,48 @@ const ( TestPVCName = "ropvc-test-name" ) +func TestSetupModelCachingForRequestSetsEncryptedStorageClassWhenUnset(t *testing.T) { + ctx := newTestContext() + clients := mockKubeClients() + const testNamespace = "test-encrypted-cache" + backend := K8sComputeBackend{ + clients: clients, + bk8s: &BackendK8sCache{ + podInstanceNamespace: testNamespace, + nvmeshEncryptionEnabled: true, + }, + } + rwPVC := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: "rw-pvc-encrypted", Namespace: testNamespace}, + } + initJob := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Name: "writer-encrypted", Namespace: testNamespace}, + } + req := &nvcav2beta1.ICMSRequest{ + ObjectMeta: metav1.ObjectMeta{Namespace: testNamespace}, + Spec: nvcav2beta1.ICMSRequestSpec{ + NCAId: "test-nca", + FunctionDetails: function.Details{ + FunctionVersionID: "test-function-version", + }, + }, + } + + state, _, setupErr := backend.SetupModelCachingForRequest( + ctx, rwPVC, initJob, req, true, func(client.Object) {}) + + require.NoError(t, setupErr) + require.Equal(t, ModelCachingInProgress, state) + require.NotNil(t, rwPVC.Spec.StorageClassName) + wantStorageClass := encryption.BuildStorageClassName(encryption.BuildMD5Hash(req.Spec.NCAId)) + assert.Equal(t, wantStorageClass, *rwPVC.Spec.StorageClassName) + createdPVC, err := clients.K8s.CoreV1().PersistentVolumeClaims(testNamespace).Get( + ctx, rwPVC.Name, metav1.GetOptions{}) + require.NoError(t, err) + require.NotNil(t, createdPVC.Spec.StorageClassName) + assert.Equal(t, wantStorageClass, *createdPVC.Spec.StorageClassName) +} + func Test_pvcReclaimDeleteCheck(t *testing.T) { ctx, cancel := context.WithCancel(newTestContext()) t.Cleanup(cancel) diff --git a/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend_task_container.go b/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend_task_container.go index a9fbcb212..9f6f0c600 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend_task_container.go +++ b/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend_task_container.go @@ -157,7 +157,7 @@ func (c K8sComputeBackend) applyContainerTaskCreationMessage(ctx context.Context obj.SetOwnerReferences(ownerRefsForReq) switch typedObj := obj.(type) { case *corev1.PersistentVolumeClaim: - if strings.HasPrefix(typedObj.Name, "rw-pvc-") { + if isRegularModelCacheWriterPVCName(typedObj.Name) { bdCreateCachePVC = typedObj } case *batchv1.Job: @@ -180,11 +180,19 @@ func (c K8sComputeBackend) applyContainerTaskCreationMessage(ctx context.Context // which likely mean the cache is still being initialized. cacheMF := func(*corev1.Pod) {} var cachePVCName string - if c.bk8s.cachingSupportEnabled && (initCacheJob != nil && bdCreateCachePVC != nil) { + modelCachingEnabled, persistedSelection, selectionErr := regularModelCacheRuntimeDecision( + req, c.bk8s.cachingSupportEnabled) + if selectionErr != nil { + return nvcaerrors.TerminalError(selectionErr) + } + if modelCachingEnabled && (initCacheJob != nil && bdCreateCachePVC != nil) { if cacheMF, cachePVCName, err = c.setupContainerModelCaching(ctx, req, bdCreateCachePVC, initCacheJob, mf); err != nil { return err } - } else if !c.bk8s.cachingSupportEnabled { + } else if modelCachingEnabled && persistedSelection { + return nvcaerrors.TerminalError(fmt.Errorf( + "persisted durable regular model cache requires both PVC and init Job artifacts")) + } else if !modelCachingEnabled { log.Debugf("ModelCaching support is disabled, creating task instance without caching") } else { log.Debug("InitCacheJob / BDCreate spec was not specified, skipping task model caching") diff --git a/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend_test.go b/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend_test.go index fd0c81d3d..a8c9202d8 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend_test.go +++ b/src/compute-plane-services/nvca/pkg/nvca/k8scomputebackend_test.go @@ -2422,3 +2422,24 @@ func TestFunctionTranslateInjectsLegacyStargateAddressFromLLMRequestRouterEnv(t assert.Equal(t, testRouterAddress, initEnvMap["LLM_REQUEST_ROUTER_ADDRESS"]) assert.Equal(t, testRouterAddress, initEnvMap["STARGATE_ADDRESS"]) } + +func TestRegularModelCachePVCNamesUseExactLeadingPrefix(t *testing.T) { + name, err := regularModelCacheReaderPVCName("rw-pvc-cache-rw-pvc-segment") + require.NoError(t, err) + assert.Equal(t, "ro-pvc-cache-rw-pvc-segment", name) + + reader, err := classifyRegularModelCachePVCName("ro-pvc-cache-rw-pvc-segment") + require.NoError(t, err) + assert.True(t, reader) + + reader, err = classifyRegularModelCachePVCName("rw-pvc-cache-ro-pvc-segment") + require.NoError(t, err) + assert.False(t, reader) + + for _, invalid := range []string{"cache-rw-pvc-handle", "rw-pvc-", "ro-pvc-"} { + _, err := regularModelCacheReaderPVCName(invalid) + require.Error(t, err) + _, err = classifyRegularModelCachePVCName(invalid) + require.Error(t, err) + } +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/modelcache_binding.go b/src/compute-plane-services/nvca/pkg/nvca/modelcache_binding.go new file mode 100644 index 000000000..6b09ca444 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/modelcache_binding.go @@ -0,0 +1,715 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nvca + +import ( + "context" + "errors" + "fmt" + "reflect" + "sort" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + + nvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1" + nvcaclientv2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/client/clientset/versioned/typed/nvca/v2beta1" + nvcaerrors "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/errors" + nvcastorage "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/storage" +) + +var ( + errICMSRequestDeletingDuringBinding = errors.New( + "ICMS request began deleting while persisting model cache binding") + errICMSRequestReplacedDuringBinding = errors.New( + "ICMS request was replaced while persisting model cache binding") + errRegularModelCacheBindingRetiring = errors.New( + "regular model cache binding is Retiring") + errRegularModelCacheBindingReferenceReleased = errors.New( + "regular model cache binding request reference was released") +) + +type modelCacheBindingInput struct { + selection *nvcastorage.PersistedModelCacheStorageSelection + sharingDomain string + cacheHandle string + writerNamespace string +} + +func (c *BackendK8sCache) modelCacheBindingInput( + req *nvcav2beta1.ICMSRequest, +) (*modelCacheBindingInput, error) { + raw := req.Annotations[nvcastorage.ModelCacheStorageSelectionAnnotationKey] + if raw == "" { + return nil, nil + } + selection, err := nvcastorage.ParsePersistedModelCacheStorageSelection(raw) + if err != nil { + return nil, fmt.Errorf("parse persisted model cache storage selection: %w", err) + } + if selection.Mode != nvcastorage.ModelCacheSelectionDurable { + return nil, nil + } + + cacheSpec, workflow := cacheSelectionInput(req) + if cacheSpec == nil || cacheSpec.CacheHandle == "" { + return nil, fmt.Errorf("durable model cache selection has no cache handle") + } + if workflow != selection.Workflow { + return nil, fmt.Errorf("durable model cache workflow changed from %q to %q", + selection.Workflow, workflow) + } + if req.Spec.NCAId == "" { + return nil, fmt.Errorf("durable model cache selection has no sharing domain") + } + if req.Namespace == "" || req.Name == "" || req.UID == "" { + return nil, fmt.Errorf("durable model cache request requires namespace, name, and UID") + } + + writerNamespace := nvcastorage.ModelCacheInitNamespace + if workflow == nvcastorage.ModelCacheWorkflowRegular { + writerNamespace = c.podInstanceNamespace + } + if writerNamespace == "" { + return nil, fmt.Errorf("durable model cache writer namespace is empty") + } + return &modelCacheBindingInput{ + selection: selection, + sharingDomain: req.Spec.NCAId, + cacheHandle: cacheSpec.CacheHandle, + writerNamespace: writerNamespace, + }, nil +} + +// ensureModelCacheBinding creates or adopts the immutable binding and adds the +// exact ICMSRequest UID as a reference before persisting the binding reference +// on the request. A true return value means the request annotation changed and +// the caller must stop this reconcile before creating any side effect. +func (c *BackendK8sCache) ensureModelCacheBinding( + ctx context.Context, + req *nvcav2beta1.ICMSRequest, +) (bool, error) { + input, err := c.modelCacheBindingInput(req) + if err != nil { + return false, nvcaerrors.TerminalError(err) + } + if input == nil { + return false, nil + } + if c.clients == nil || c.clients.BART == nil || c.clients.K8s == nil { + return false, fmt.Errorf("model cache binding clients are not configured") + } + + expected, err := nvcastorage.NewModelCacheBinding( + input.selection, input.sharingDomain, input.cacheHandle, input.writerNamespace) + if err != nil { + return false, nvcaerrors.TerminalError(fmt.Errorf("build model cache binding intent: %w", err)) + } + bindings := c.clients.BART.NvcaV2beta1().ModelCacheBindings(expected.Namespace) + + var binding *nvcav2beta1.ModelCacheBinding + if input.selection.BindingName == "" { + binding, err = bindings.Get(ctx, expected.Name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + if err := nvcastorage.ValidateModelCacheStorageSelectionInputsWithClientset( + ctx, c.clients.K8s, c.systemNamespace, input.selection); err != nil { + if errors.Is(err, nvcastorage.ErrModelCacheStorageSelectionDrift) || apierrors.IsNotFound(err) { + return false, nvcaerrors.TerminalError( + fmt.Errorf("validate model cache selection before binding creation: %w", err)) + } + return false, fmt.Errorf("validate model cache selection before binding creation: %w", err) + } + binding, err = bindings.Create(ctx, expected, metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(err) { + binding, err = bindings.Get(ctx, expected.Name, metav1.GetOptions{}) + } + } + if err != nil { + return false, fmt.Errorf("get or create model cache binding %s/%s: %w", + expected.Namespace, expected.Name, err) + } + } else { + if input.selection.BindingName != expected.Name { + return false, nvcaerrors.TerminalError(fmt.Errorf( + "persisted model cache binding name %q does not match expected %q", + input.selection.BindingName, expected.Name)) + } + binding, err = bindings.Get(ctx, input.selection.BindingName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return false, nvcaerrors.TerminalError(fmt.Errorf( + "persisted model cache binding %s/%s is missing", + expected.Namespace, input.selection.BindingName)) + } + return false, fmt.Errorf("get persisted model cache binding %s/%s: %w", + expected.Namespace, input.selection.BindingName, err) + } + } + + if binding.UID == "" { + return false, nvcaerrors.TerminalError(fmt.Errorf( + "model cache binding %s/%s has no API-assigned UID", binding.Namespace, binding.Name)) + } + if err := nvcastorage.ValidateModelCacheBindingIntent( + binding, input.selection, input.sharingDomain, input.cacheHandle, input.writerNamespace); err != nil { + return false, nvcaerrors.TerminalError(err) + } + if input.selection.BindingName != "" { + if err := validateModelCacheBindingForEnsure(binding, input, req); err != nil { + return false, nvcaerrors.TerminalError(err) + } + return false, nil + } + + binding, err = c.ensureActiveModelCacheBindingReference(ctx, bindings, input, req) + if err != nil { + return false, err + } + changed, err := c.persistModelCacheBindingReference(ctx, req, input.selection, binding) + if errors.Is(err, errICMSRequestDeletingDuringBinding) || + errors.Is(err, errICMSRequestReplacedDuringBinding) { + if releaseErr := c.removeModelCacheBindingReference( + ctx, bindings, binding.Name, input, req.Namespace, req.Name, req.UID); releaseErr != nil { + return false, fmt.Errorf("%w; release newly-added binding reference: %v", err, releaseErr) + } + } + return changed, err +} + +func (c *BackendK8sCache) ensureActiveModelCacheBindingReference( + ctx context.Context, + bindings nvcaclientv2beta1.ModelCacheBindingInterface, + input *modelCacheBindingInput, + req *nvcav2beta1.ICMSRequest, +) (*nvcav2beta1.ModelCacheBinding, error) { + var result *nvcav2beta1.ModelCacheBinding + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + binding, err := bindings.Get(ctx, nvcastorage.ModelCacheBindingName(input.cacheHandle), metav1.GetOptions{}) + if err != nil { + return err + } + if err := nvcastorage.ValidateModelCacheBindingIntent( + binding, input.selection, input.sharingDomain, input.cacheHandle, input.writerNamespace); err != nil { + return nvcaerrors.TerminalError(err) + } + + changed := false + switch binding.Status.Phase { + case "": + if binding.Status.LastPhaseTransitionTime != nil || len(binding.Status.RequestReferences) != 0 || + binding.Status.Realized != nil || len(binding.Status.Conditions) != 0 { + return nvcaerrors.TerminalError(fmt.Errorf( + "model cache binding %s/%s has a partially initialized status", + binding.Namespace, binding.Name)) + } + now := metav1.Now() + binding.Status.Phase = nvcav2beta1.ModelCacheBindingPhaseActive + binding.Status.LastPhaseTransitionTime = &now + changed = true + case nvcav2beta1.ModelCacheBindingPhaseActive: + case nvcav2beta1.ModelCacheBindingPhaseRetiring: + return nvcaerrors.TerminalError(fmt.Errorf( + "model cache binding %s/%s is Retiring", binding.Namespace, binding.Name)) + default: + return nvcaerrors.TerminalError(fmt.Errorf( + "model cache binding %s/%s has unknown phase %q", + binding.Namespace, binding.Name, binding.Status.Phase)) + } + + originalRefs := append([]nvcav2beta1.ModelCacheBindingRequestReference(nil), + binding.Status.RequestReferences...) + refs, exactFound, err := c.referencesWithCurrentRequest(ctx, binding, req) + if err != nil { + return err + } + if !exactFound { + refs = append(refs, nvcav2beta1.ModelCacheBindingRequestReference{ + Namespace: req.Namespace, + Name: req.Name, + UID: req.UID, + }) + changed = true + } + sort.Slice(refs, func(i, j int) bool { + if refs[i].Namespace != refs[j].Namespace { + return refs[i].Namespace < refs[j].Namespace + } + if refs[i].Name != refs[j].Name { + return refs[i].Name < refs[j].Name + } + return refs[i].UID < refs[j].UID + }) + if !reflect.DeepEqual(refs, originalRefs) { + changed = true + } + binding.Status.RequestReferences = refs + if !changed { + result = binding + return nil + } + result, err = bindings.UpdateStatus(ctx, binding, metav1.UpdateOptions{}) + return err + }) + if err != nil { + return nil, fmt.Errorf("activate model cache binding for request %s/%s: %w", + req.Namespace, req.Name, err) + } + if result == nil { + return nil, fmt.Errorf("activate model cache binding for request %s/%s returned no binding", + req.Namespace, req.Name) + } + if err := validateActiveModelCacheBindingForRequest(result, input, req); err != nil { + return nil, nvcaerrors.TerminalError(err) + } + return result, nil +} + +func (c *BackendK8sCache) referencesWithCurrentRequest( + ctx context.Context, + binding *nvcav2beta1.ModelCacheBinding, + req *nvcav2beta1.ICMSRequest, +) ([]nvcav2beta1.ModelCacheBindingRequestReference, bool, error) { + refs := make([]nvcav2beta1.ModelCacheBindingRequestReference, 0, + len(binding.Status.RequestReferences)+1) + exactFound := false + for _, ref := range binding.Status.RequestReferences { + if ref.Namespace != req.Namespace || ref.Name != req.Name { + refs = append(refs, ref) + continue + } + if ref.UID == req.UID { + if exactFound { + return nil, false, nvcaerrors.TerminalError(fmt.Errorf( + "model cache binding %s/%s contains a duplicate request reference for %s/%s", + binding.Namespace, binding.Name, req.Namespace, req.Name)) + } + exactFound = true + refs = append(refs, ref) + continue + } + + live, err := c.clients.BART.NvcaV2beta1().ICMSRequests(ref.Namespace). + Get(ctx, ref.Name, metav1.GetOptions{}) + switch { + case apierrors.IsNotFound(err): + continue + case err != nil: + return nil, false, fmt.Errorf("validate stale model cache binding reference %s/%s: %w", + ref.Namespace, ref.Name, err) + case live.UID == ref.UID: + return nil, false, nvcaerrors.TerminalError(fmt.Errorf( + "model cache binding %s/%s is already referenced by live request %s/%s UID %s", + binding.Namespace, binding.Name, ref.Namespace, ref.Name, ref.UID)) + default: + // The namespaced request name was reused after the recorded UID + // disappeared. The stale reference is safe to replace. + continue + } + } + return refs, exactFound, nil +} + +func (c *BackendK8sCache) persistModelCacheBindingReference( + ctx context.Context, + req *nvcav2beta1.ICMSRequest, + selection *nvcastorage.PersistedModelCacheStorageSelection, + binding *nvcav2beta1.ModelCacheBinding, +) (bool, error) { + boundSelection := *selection + boundSelection.RequiredAccessModes = append( + []corev1.PersistentVolumeAccessMode(nil), selection.RequiredAccessModes...) + boundSelection.BindingName = binding.Name + boundSelection.BindingUID = binding.UID + payload, err := boundSelection.Marshal() + if err != nil { + return false, nvcaerrors.TerminalError(fmt.Errorf("marshal bound model cache selection: %w", err)) + } + + changed := false + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + latest, err := c.clients.BART.NvcaV2beta1().ICMSRequests(req.Namespace). + Get(ctx, req.Name, metav1.GetOptions{}) + if err != nil { + return err + } + if latest.UID != req.UID { + return fmt.Errorf("%w: expected UID %s, found %s", + errICMSRequestReplacedDuringBinding, req.UID, latest.UID) + } + if !latest.DeletionTimestamp.IsZero() { + return errICMSRequestDeletingDuringBinding + } + current, err := nvcastorage.ParsePersistedModelCacheStorageSelection( + latest.Annotations[nvcastorage.ModelCacheStorageSelectionAnnotationKey]) + if err != nil { + return nvcaerrors.TerminalError(fmt.Errorf( + "parse latest persisted model cache storage selection: %w", err)) + } + if reflect.DeepEqual(current, &boundSelection) { + // Another reconcile may have committed the binding while this + // caller still holds the unbound request snapshot. Force that + // stale caller to stop before runtime side effects. + if !reflect.DeepEqual(selection, &boundSelection) { + changed = true + } + return nil + } + if !reflect.DeepEqual(current, selection) { + return nvcaerrors.TerminalError(fmt.Errorf( + "model cache storage selection changed while binding request %s/%s", + req.Namespace, req.Name)) + } + if latest.Annotations == nil { + latest.Annotations = map[string]string{} + } + latest.Annotations[nvcastorage.ModelCacheStorageSelectionAnnotationKey] = payload + _, err = c.clients.BART.NvcaV2beta1().ICMSRequests(req.Namespace). + Update(ctx, latest, metav1.UpdateOptions{}) + if err == nil { + changed = true + } + return err + }) + if err != nil { + return false, fmt.Errorf("persist model cache binding reference on request %s/%s: %w", + req.Namespace, req.Name, err) + } + return changed, nil +} + +// validateModelCacheBindingForRuntime makes the binding, not the mutable live +// StorageClass or catalog, authoritative after selection has been committed. +func (c *BackendK8sCache) validateModelCacheBindingForRuntime( + ctx context.Context, + req *nvcav2beta1.ICMSRequest, +) error { + _, err := c.activeModelCacheBindingForRuntime(ctx, req) + return err +} + +func (c *BackendK8sCache) activeModelCacheBindingForRuntime( + ctx context.Context, + req *nvcav2beta1.ICMSRequest, +) (*nvcav2beta1.ModelCacheBinding, error) { + input, err := c.modelCacheBindingInput(req) + if err != nil { + return nil, nvcaerrors.TerminalError(err) + } + if input == nil { + return nil, nil + } + if input.selection.BindingName == "" || input.selection.BindingUID == "" { + return nil, nvcaerrors.TerminalError(fmt.Errorf( + "durable model cache selection has no committed binding reference")) + } + if c.clients == nil || c.clients.BART == nil { + return nil, fmt.Errorf("model cache binding client is not configured") + } + binding, err := c.clients.BART.NvcaV2beta1(). + ModelCacheBindings(nvcastorage.ModelCacheInitNamespace). + Get(ctx, input.selection.BindingName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return nil, nvcaerrors.TerminalError(fmt.Errorf( + "persisted model cache binding %s/%s is missing", + nvcastorage.ModelCacheInitNamespace, input.selection.BindingName)) + } + return nil, fmt.Errorf("get persisted model cache binding %s/%s: %w", + nvcastorage.ModelCacheInitNamespace, input.selection.BindingName, err) + } + if binding.Status.Phase == nvcav2beta1.ModelCacheBindingPhaseRetiring && + input.selection.Workflow == nvcastorage.ModelCacheWorkflowRegular { + if err := validateModelCacheBindingForEnsure(binding, input, req); err != nil { + return nil, nvcaerrors.TerminalError(err) + } + return nil, nvcaerrors.TerminalError(fmt.Errorf("%w: %s/%s", + errRegularModelCacheBindingRetiring, binding.Namespace, binding.Name)) + } + if err := validateActiveModelCacheBindingForRequest(binding, input, req); err != nil { + return nil, nvcaerrors.TerminalError(err) + } + return binding, nil +} + +func validateModelCacheBindingForEnsure( + binding *nvcav2beta1.ModelCacheBinding, + input *modelCacheBindingInput, + req *nvcav2beta1.ICMSRequest, +) error { + if binding.Status.Phase != nvcav2beta1.ModelCacheBindingPhaseRetiring { + return validateActiveModelCacheBindingForRequest(binding, input, req) + } + if input.selection.Workflow != nvcastorage.ModelCacheWorkflowRegular { + return fmt.Errorf("model cache binding %s/%s is Retiring and cannot serve %q workflow", + binding.Namespace, binding.Name, input.selection.Workflow) + } + if err := nvcastorage.ValidateModelCacheBindingIntent( + binding, input.selection, input.sharingDomain, input.cacheHandle, input.writerNamespace); err != nil { + return err + } + sole, err := validateExactModelCacheBindingRequestReference(binding, req) + if err != nil { + return fmt.Errorf("model cache binding %s/%s is Retiring: %w", + binding.Namespace, binding.Name, err) + } + if !sole { + return fmt.Errorf("model cache binding %s/%s is Retiring with other request references", + binding.Namespace, binding.Name) + } + return nil +} + +func validateActiveModelCacheBindingForRequest( + binding *nvcav2beta1.ModelCacheBinding, + input *modelCacheBindingInput, + req *nvcav2beta1.ICMSRequest, +) error { + if err := nvcastorage.ValidateModelCacheBinding( + binding, input.selection, input.sharingDomain, input.cacheHandle, input.writerNamespace); err != nil { + return err + } + if !nvcastorage.ModelCacheBindingHasRequestReference( + binding, req.Namespace, req.Name, req.UID) { + return fmt.Errorf("%w: model cache binding %s/%s has no reference to request %s/%s UID %s", + errRegularModelCacheBindingReferenceReleased, binding.Namespace, binding.Name, + req.Namespace, req.Name, req.UID) + } + return nil +} + +// beginRegularModelCacheBindingRetirement atomically closes a durable regular +// cache binding to new users before destructive cleanup. It authorizes cleanup +// only when the exact request UID is the sole reference. An already-Retiring +// binding is adopted only by that same request so interrupted cleanup can +// resume without exposing the binding to the Active runtime path. +func (c *BackendK8sCache) beginRegularModelCacheBindingRetirement( + ctx context.Context, + req *nvcav2beta1.ICMSRequest, +) (*nvcav2beta1.ModelCacheBinding, bool, error) { + input, err := c.modelCacheBindingInput(req) + if err != nil { + return nil, false, nvcaerrors.TerminalError(err) + } + if input == nil || input.selection.Workflow != nvcastorage.ModelCacheWorkflowRegular || + input.selection.BindingName == "" || input.selection.BindingUID == "" { + return nil, false, nvcaerrors.TerminalError(fmt.Errorf( + "regular model cache retirement requires a committed durable regular binding reference")) + } + if c.clients == nil || c.clients.BART == nil { + return nil, false, fmt.Errorf("model cache binding client is not configured") + } + + bindings := c.clients.BART.NvcaV2beta1().ModelCacheBindings(nvcastorage.ModelCacheInitNamespace) + var result *nvcav2beta1.ModelCacheBinding + authorized := false + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + result = nil + authorized = false + binding, err := bindings.Get(ctx, input.selection.BindingName, metav1.GetOptions{}) + if err != nil { + return err + } + if err := nvcastorage.ValidateModelCacheBindingIntent( + binding, input.selection, input.sharingDomain, input.cacheHandle, input.writerNamespace); err != nil { + return nvcaerrors.TerminalError(err) + } + releasedWithNoUsers := len(binding.Status.RequestReferences) == 0 + sole := false + if !releasedWithNoUsers { + sole, err = validateExactModelCacheBindingRequestReference(binding, req) + if err != nil { + return nvcaerrors.TerminalError(err) + } + } + + switch binding.Status.Phase { + case nvcav2beta1.ModelCacheBindingPhaseActive: + if releasedWithNoUsers { + return nvcaerrors.TerminalError(fmt.Errorf( + "Active model cache binding %s/%s has no reference to any request", binding.Namespace, binding.Name)) + } + if !sole { + result = binding + return nil + } + now := metav1.Now() + binding.Status.Phase = nvcav2beta1.ModelCacheBindingPhaseRetiring + binding.Status.LastPhaseTransitionTime = &now + updated, updateErr := bindings.UpdateStatus(ctx, binding, metav1.UpdateOptions{}) + if updateErr != nil { + return updateErr + } + result = updated + authorized = true + return nil + case nvcav2beta1.ModelCacheBindingPhaseRetiring: + if !sole && !releasedWithNoUsers { + return nvcaerrors.TerminalError(fmt.Errorf( + "Retiring model cache binding %s/%s has other request references", + binding.Namespace, binding.Name)) + } + result = binding + authorized = true + return nil + default: + return nvcaerrors.TerminalError(fmt.Errorf( + "model cache binding %s/%s cannot retire from phase %q", + binding.Namespace, binding.Name, binding.Status.Phase)) + } + }) + if err != nil { + return nil, false, fmt.Errorf("retire regular model cache binding %s for request %s/%s: %w", + input.selection.BindingName, req.Namespace, req.Name, err) + } + if result == nil { + return nil, false, fmt.Errorf("retire regular model cache binding %s returned no binding", + input.selection.BindingName) + } + return result, authorized, nil +} + +func (c *BackendK8sCache) resumeRetiringRegularModelCacheCleanup( + ctx context.Context, + req *nvcav2beta1.ICMSRequest, +) error { + input, err := c.modelCacheBindingInput(req) + if err != nil { + return fmt.Errorf("resolve model cache binding before deletion cleanup: %w", err) + } + if input == nil || input.selection.Workflow != nvcastorage.ModelCacheWorkflowRegular || + input.selection.BindingName == "" || input.selection.BindingUID == "" { + return nil + } + binding, err := c.clients.BART.NvcaV2beta1(). + ModelCacheBindings(nvcastorage.ModelCacheInitNamespace). + Get(ctx, input.selection.BindingName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("get model cache binding before deletion cleanup: %w", err) + } + if err := nvcastorage.ValidateModelCacheBindingIntent( + binding, input.selection, input.sharingDomain, input.cacheHandle, input.writerNamespace); err != nil { + return nvcaerrors.TerminalError(err) + } + if binding.Status.Phase != nvcav2beta1.ModelCacheBindingPhaseRetiring { + return nil + } + helper, ok := c.k8sArtifactHelper.(K8sComputeBackend) + if !ok { + return fmt.Errorf("resume Retiring model cache cleanup: expected K8sComputeBackend, got %T", + c.k8sArtifactHelper) + } + if err := helper.CleanupModelCachingSetupArtifacts(ctx, req); err != nil { + return fmt.Errorf("resume Retiring model cache cleanup: %w", err) + } + return nil +} +func validateExactModelCacheBindingRequestReference( + binding *nvcav2beta1.ModelCacheBinding, + req *nvcav2beta1.ICMSRequest, +) (bool, error) { + exact := 0 + for _, ref := range binding.Status.RequestReferences { + if ref.Namespace == req.Namespace && ref.Name == req.Name && ref.UID == req.UID { + exact++ + } + } + if exact == 0 { + return false, fmt.Errorf("model cache binding %s/%s has no reference to request %s/%s UID %s", + binding.Namespace, binding.Name, req.Namespace, req.Name, req.UID) + } + if exact != 1 { + return false, fmt.Errorf("model cache binding %s/%s has %d references to request %s/%s UID %s", + binding.Namespace, binding.Name, exact, req.Namespace, req.Name, req.UID) + } + return len(binding.Status.RequestReferences) == 1, nil +} + +func (c *BackendK8sCache) releaseModelCacheBindingReference( + ctx context.Context, + req *nvcav2beta1.ICMSRequest, +) error { + input, err := c.modelCacheBindingInput(req) + if err != nil { + // Deletion errors must remain retryable so the request finalizer is not + // silently removed while binding ownership is ambiguous. + return fmt.Errorf("resolve model cache binding during request deletion: %w", err) + } + if input == nil { + return nil + } + name := input.selection.BindingName + if name == "" { + name = nvcastorage.ModelCacheBindingName(input.cacheHandle) + } + bindings := c.clients.BART.NvcaV2beta1().ModelCacheBindings(nvcastorage.ModelCacheInitNamespace) + return c.removeModelCacheBindingReference( + ctx, bindings, name, input, req.Namespace, req.Name, req.UID) +} + +func (c *BackendK8sCache) removeModelCacheBindingReference( + ctx context.Context, + bindings nvcaclientv2beta1.ModelCacheBindingInterface, + bindingName string, + input *modelCacheBindingInput, + requestNamespace string, + requestName string, + requestUID types.UID, +) error { + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + binding, err := bindings.Get(ctx, bindingName, metav1.GetOptions{}) + if apierrors.IsNotFound(err) && input.selection.BindingName == "" { + return nil + } + if err != nil { + return err + } + if err := nvcastorage.ValidateModelCacheBindingIntent( + binding, input.selection, input.sharingDomain, input.cacheHandle, input.writerNamespace); err != nil { + if input.selection.BindingName == "" { + // An unbound request may collide with an existing handle-scoped + // binding owned by another immutable sharing domain. It never + // acquired that binding and must not modify it during deletion. + return nil + } + return err + } + refs := binding.Status.RequestReferences[:0] + found := false + for _, ref := range binding.Status.RequestReferences { + if ref.Namespace == requestNamespace && ref.Name == requestName && ref.UID == requestUID { + found = true + continue + } + refs = append(refs, ref) + } + if !found { + return nil + } + binding.Status.RequestReferences = refs + _, err = bindings.UpdateStatus(ctx, binding, metav1.UpdateOptions{}) + return err + }) + if err != nil { + return fmt.Errorf("release model cache binding %s reference for request %s/%s: %w", + bindingName, requestNamespace, requestName, err) + } + return nil +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/modelcache_binding_resources.go b/src/compute-plane-services/nvca/pkg/nvca/modelcache_binding_resources.go new file mode 100644 index 000000000..e1d259d6f --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/modelcache_binding_resources.go @@ -0,0 +1,918 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nvca + +import ( + "context" + "fmt" + "slices" + "strings" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + nvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1" + nvcaerrors "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/errors" + nvcastorage "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/storage" + nvcatypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/types" +) + +func (c K8sComputeBackend) prepareRegularModelCacheBindingResources( + ctx context.Context, + binding *nvcav2beta1.ModelCacheBinding, + rwPVC *corev1.PersistentVolumeClaim, + initJob *batchv1.Job, +) error { + if binding == nil { + return nvcaerrors.TerminalError(fmt.Errorf("regular model cache binding is nil")) + } + if rwPVC == nil || initJob == nil { + return nvcaerrors.TerminalError(fmt.Errorf("regular model cache PVC or init Job is nil")) + } + roPVCName, err := regularModelCacheReaderPVCName(rwPVC.Name) + if err != nil { + return nvcaerrors.TerminalError(err) + } + writerModes, _, separateReader, err := regularModelCacheAccessModePlan(binding) + if err != nil { + return nvcaerrors.TerminalError(err) + } + if !slices.Contains(binding.Spec.Resources.PersistentVolumeClaimNames, rwPVC.Name) { + return nvcaerrors.TerminalError(fmt.Errorf( + "regular model cache writer PVC name %q does not match binding intent %v", + rwPVC.Name, binding.Spec.Resources.PersistentVolumeClaimNames)) + } + if separateReader != slices.Contains(binding.Spec.Resources.PersistentVolumeClaimNames, roPVCName) { + return nvcaerrors.TerminalError(fmt.Errorf( + "regular model cache reader PVC name %q does not match transition %q intent %v", + roPVCName, binding.Spec.Decision.Transition, + binding.Spec.Resources.PersistentVolumeClaimNames)) + } + if !slices.Contains(binding.Spec.Resources.JobNames, initJob.Name) { + return nvcaerrors.TerminalError(fmt.Errorf( + "regular model cache init Job name %q does not match binding intent %v", + initJob.Name, binding.Spec.Resources.JobNames)) + } + for kind, namespace := range map[string]string{ + "PVC": rwPVC.Namespace, + "Job": initJob.Namespace, + } { + if namespace != binding.Spec.Resources.WriterNamespace { + return nvcaerrors.TerminalError(fmt.Errorf( + "regular model cache %s namespace %q does not match binding writer namespace %q", + kind, namespace, binding.Spec.Resources.WriterNamespace)) + } + } + if binding.Spec.Decision.Transition == nvcastorage.ModelCacheTransitionRWXReadOnly { + if err := prepareRWXReadOnlySharedWriterJob(initJob); err != nil { + return nvcaerrors.TerminalError(err) + } + } + + for _, obj := range []metav1.Object{rwPVC, initJob, &initJob.Spec.Template.ObjectMeta} { + canonicalizeRegularModelCacheSharedMetadata(obj) + if err := nvcastorage.SetModelCacheBindingUIDLabel(obj, binding.UID); err != nil { + return nvcaerrors.TerminalError(err) + } + } + if !slices.Equal(rwPVC.Spec.AccessModes, writerModes) { + return nvcaerrors.TerminalError(fmt.Errorf( + "regular model cache writer PVC access modes are %v, want %v for transition %q", + rwPVC.Spec.AccessModes, writerModes, binding.Spec.Decision.Transition)) + } + + pvcTargets := []struct { + wanted *corev1.PersistentVolumeClaim + reader bool + }{ + {wanted: rwPVC}, + } + if separateReader { + roPVCIntent := rwPVC.DeepCopy() + roPVCIntent.Name = roPVCName + roPVCIntent.Spec.AccessModes = ROAccessMode + pvcTargets = append(pvcTargets, struct { + wanted *corev1.PersistentVolumeClaim + reader bool + }{wanted: roPVCIntent, reader: true}) + } + for _, target := range pvcTargets { + if err := validateRegularModelCachePVC(target.wanted, target.wanted, binding, target.reader); err != nil { + return nvcaerrors.TerminalError(err) + } + } + if err := validateRegularModelCacheJob(initJob, initJob, binding); err != nil { + return nvcaerrors.TerminalError(err) + } + + for _, target := range pvcTargets { + existing, err := c.clients.K8s.CoreV1().PersistentVolumeClaims( + binding.Spec.Resources.WriterNamespace).Get(ctx, target.wanted.Name, metav1.GetOptions{}) + switch { + case apierrors.IsNotFound(err): + case err != nil: + return fmt.Errorf("get existing regular model cache PVC %s/%s: %w", + binding.Spec.Resources.WriterNamespace, target.wanted.Name, err) + default: + if err := validateRegularModelCachePVC(existing, target.wanted, binding, target.reader); err != nil { + return nvcaerrors.TerminalError(err) + } + } + } + existingJob, err := c.clients.K8s.BatchV1().Jobs(binding.Spec.Resources.WriterNamespace). + Get(ctx, initJob.Name, metav1.GetOptions{}) + switch { + case apierrors.IsNotFound(err): + case err != nil: + return fmt.Errorf("get existing regular model cache Job %s/%s: %w", + binding.Spec.Resources.WriterNamespace, initJob.Name, err) + default: + if err := validateRegularModelCacheJob(existingJob, initJob, binding); err != nil { + return nvcaerrors.TerminalError(err) + } + } + return nil +} + +func regularModelCacheBindingUID( + req *nvcav2beta1.ICMSRequest, +) (types.UID, bool, error) { + selection, err := persistedRegularModelCacheSelection(req) + if err != nil { + return "", false, err + } + if selection == nil || selection.Mode != nvcastorage.ModelCacheSelectionDurable { + return "", false, nil + } + if selection.BindingName == "" || selection.BindingUID == "" { + return "", true, fmt.Errorf("durable regular model cache selection has no binding reference") + } + return selection.BindingUID, true, nil +} + +func requireRegularModelCacheBindingUID(obj metav1.Object, bindingUID types.UID) error { + if obj == nil { + return fmt.Errorf("regular model cache object is nil") + } + if got := obj.GetLabels()[nvcastorage.ModelCacheBindingUIDLabelKey]; got != string(bindingUID) { + return fmt.Errorf("regular model cache object %s/%s has binding UID %q, want %q", + obj.GetNamespace(), obj.GetName(), got, bindingUID) + } + return nil +} + +func regularModelCacheExpectedStorageClassName(binding *nvcav2beta1.ModelCacheBinding) (string, error) { + if binding == nil { + return "", fmt.Errorf("regular model cache binding is nil") + } + if binding.Spec.Decision.EncryptionRequired { + if len(binding.Spec.Resources.StorageClassNames) != 1 { + return "", fmt.Errorf("encrypted regular model cache binding must record exactly one derived StorageClass") + } + return binding.Spec.Resources.StorageClassNames[0], nil + } + if binding.Spec.StorageClass.Name == "" { + return "", fmt.Errorf("regular model cache binding has no selected StorageClass") + } + return binding.Spec.StorageClass.Name, nil +} + +func regularModelCacheAccessModePlan( + binding *nvcav2beta1.ModelCacheBinding, +) ( + writer []corev1.PersistentVolumeAccessMode, + reader []corev1.PersistentVolumeAccessMode, + separateReader bool, + err error, +) { + if binding == nil { + return nil, nil, false, fmt.Errorf("regular model cache binding is nil") + } + switch binding.Spec.Decision.Transition { + case nvcastorage.ModelCacheTransitionROXReadOnly: + return []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + append([]corev1.PersistentVolumeAccessMode(nil), ROAccessMode...), true, nil + case nvcastorage.ModelCacheTransitionRWXReadOnly: + return append([]corev1.PersistentVolumeAccessMode(nil), RWXAccessMode...), nil, false, nil + default: + return nil, nil, false, fmt.Errorf( + "unsupported regular model cache transition %q", binding.Spec.Decision.Transition) + } +} + +func regularModelCacheExpectedPVCModes( + binding *nvcav2beta1.ModelCacheBinding, + reader bool, +) ([]corev1.PersistentVolumeAccessMode, error) { + writerModes, readerModes, separateReader, err := regularModelCacheAccessModePlan(binding) + if err != nil { + return nil, err + } + if !reader { + return writerModes, nil + } + if !separateReader { + return nil, fmt.Errorf( + "regular model cache transition %q has no separate reader PVC", + binding.Spec.Decision.Transition) + } + return readerModes, nil +} + +func regularModelCachePVCVolumeMode(pvc *corev1.PersistentVolumeClaim) corev1.PersistentVolumeMode { + if pvc != nil && pvc.Spec.VolumeMode != nil { + return *pvc.Spec.VolumeMode + } + return corev1.PersistentVolumeFilesystem +} + +func regularModelCachePVVolumeMode(pv *corev1.PersistentVolume) corev1.PersistentVolumeMode { + if pv != nil && pv.Spec.VolumeMode != nil { + return *pv.Spec.VolumeMode + } + return corev1.PersistentVolumeFilesystem +} + +func regularModelCacheRequestLabelKeys() []string { + return []string{ + nvcatypes.ICMSRequestIDKey, + nvcatypes.NCAIDKey, + nvcatypes.NCAIDUpperKey, + nvcatypes.MessageBatchIDKey, + nvcatypes.GPUNameKey, + nvcatypes.FunctionIDKey, + nvcatypes.FunctionIDUpperKey, + nvcatypes.FunctionVersionIDKey, + nvcatypes.FunctionVersionIDUpperKey, + nvcatypes.TaskIDKey, + nvcatypes.TaskIDUpperKey, + nvcatypes.ShaderCacheLabelKey, + } +} + +func regularModelCacheRequestAnnotationKeys() []string { + return []string{ + nvcatypes.ICMSRequestIDKey, + nvcatypes.NCAIDKey, + nvcatypes.ClusterGroupKey, + nvcatypes.InstanceCountKey, + } +} + +func canonicalizeRegularModelCacheSharedMetadata(obj metav1.Object) { + if obj == nil { + return + } + obj.SetOwnerReferences(nil) + labels := obj.GetLabels() + for _, key := range regularModelCacheRequestLabelKeys() { + delete(labels, key) + } + obj.SetLabels(labels) + annotations := obj.GetAnnotations() + for _, key := range regularModelCacheRequestAnnotationKeys() { + delete(annotations, key) + } + obj.SetAnnotations(annotations) +} + +func validateRegularModelCacheSharedMetadata(obj metav1.Object) error { + if obj == nil { + return fmt.Errorf("regular model cache object is nil") + } + if len(obj.GetOwnerReferences()) != 0 { + return fmt.Errorf("regular model cache object %s/%s has request owner references", + obj.GetNamespace(), obj.GetName()) + } + for _, key := range regularModelCacheRequestLabelKeys() { + if _, found := obj.GetLabels()[key]; found { + return fmt.Errorf("regular model cache object %s/%s retains request-scoped label %q", + obj.GetNamespace(), obj.GetName(), key) + } + } + for _, key := range regularModelCacheRequestAnnotationKeys() { + if _, found := obj.GetAnnotations()[key]; found { + return fmt.Errorf("regular model cache object %s/%s retains request-scoped annotation %q", + obj.GetNamespace(), obj.GetName(), key) + } + } + return nil +} + +func bindRegularModelCacheWriterJobToPVC( + job *batchv1.Job, + pvc *corev1.PersistentVolumeClaim, + binding *nvcav2beta1.ModelCacheBinding, +) error { + if binding == nil || binding.Spec.Decision.Transition != + nvcastorage.ModelCacheTransitionRWXReadOnly { + return nil + } + if job == nil || pvc == nil || pvc.UID == "" { + return fmt.Errorf("rwxReadOnly writer Job cannot record an empty PVC UID") + } + if job.Spec.Template.Annotations == nil { + job.Spec.Template.Annotations = map[string]string{} + } + recorded := job.Spec.Template.Annotations[nvcastorage.ModelCacheWriterPVCUIDAnnotationKey] + if recorded != "" && recorded != string(pvc.UID) { + return fmt.Errorf("rwxReadOnly writer Job records PVC UID %q, want %q", + recorded, pvc.UID) + } + job.Spec.Template.Annotations[nvcastorage.ModelCacheWriterPVCUIDAnnotationKey] = string(pvc.UID) + return nil +} +func validateRegularModelCacheObjectMeta(existing, wanted metav1.Object, bindingUID types.UID) error { + if existing == nil || wanted == nil { + return fmt.Errorf( + "regular model cache object intent is incomplete") + } + if err := validateRegularModelCacheSharedMetadata(existing); err != nil { + return err + } + if err := validateRegularModelCacheSharedMetadata(wanted); err != nil { + return fmt.Errorf("regular model cache intended metadata is not binding-scoped: %w", err) + } + if existing.GetNamespace() != wanted.GetNamespace() || existing.GetName() != wanted.GetName() { + return fmt.Errorf( + "regular model cache object %s/%s does not match intended %s/%s", + existing.GetNamespace(), existing.GetName(), wanted.GetNamespace(), wanted.GetName()) + } + if err := requireRegularModelCacheBindingUID(existing, bindingUID); err != nil { + return err + } + for key, want := range wanted.GetLabels() { + if existing.GetLabels()[key] != want { + return fmt.Errorf( + "regular model cache object %s/%s label %q is %q, want %q", + existing.GetNamespace(), existing.GetName(), key, existing.GetLabels()[key], want) + } + } + for key, want := range wanted.GetAnnotations() { + if existing.GetAnnotations()[key] != want { + return fmt.Errorf( + "regular model cache object %s/%s annotation %q changed", + existing.GetNamespace(), existing.GetName(), key) + } + } + return nil +} + +func validateRegularModelCachePVC( + existing *corev1.PersistentVolumeClaim, + wanted *corev1.PersistentVolumeClaim, + binding *nvcav2beta1.ModelCacheBinding, + reader bool, +) error { + if existing == nil || wanted == nil || binding == nil { + return fmt.Errorf("regular model cache PVC intent is incomplete") + } + if err := validateRegularModelCacheObjectMeta(existing, wanted, binding.UID); err != nil { + return err + } + expectedStorageClass, err := regularModelCacheExpectedStorageClassName(binding) + if err != nil { + return err + } + if wanted.Spec.StorageClassName == nil || *wanted.Spec.StorageClassName != expectedStorageClass { + return fmt.Errorf( + "regular model cache PVC %s/%s intent StorageClass is not %q", + wanted.Namespace, wanted.Name, expectedStorageClass) + } + expectedModes, err := regularModelCacheExpectedPVCModes(binding, reader) + if err != nil { + return err + } + if !slices.Equal(wanted.Spec.AccessModes, expectedModes) { + return fmt.Errorf( + "regular model cache PVC %s/%s intent access modes are %v, want %v", + wanted.Namespace, wanted.Name, wanted.Spec.AccessModes, expectedModes) + } + if !slices.Equal(existing.Spec.AccessModes, wanted.Spec.AccessModes) || + !apiequality.Semantic.DeepEqual(existing.Spec.StorageClassName, wanted.Spec.StorageClassName) || + regularModelCachePVCVolumeMode(existing) != regularModelCachePVCVolumeMode(wanted) || + !apiequality.Semantic.DeepEqual(existing.Spec.Resources, wanted.Spec.Resources) || + !apiequality.Semantic.DeepEqual(existing.Spec.Selector, wanted.Spec.Selector) || + !apiequality.Semantic.DeepEqual(existing.Spec.DataSource, wanted.Spec.DataSource) || + !apiequality.Semantic.DeepEqual(existing.Spec.DataSourceRef, wanted.Spec.DataSourceRef) || + !apiequality.Semantic.DeepEqual( + existing.Spec.VolumeAttributesClassName, wanted.Spec.VolumeAttributesClassName) { + return fmt.Errorf( + "regular model cache PVC %s/%s immutable spec does not match intent", + existing.Namespace, existing.Name) + } + if wanted.Spec.VolumeName != "" && existing.Spec.VolumeName != wanted.Spec.VolumeName { + return fmt.Errorf("regular model cache PVC %s/%s volumeName %q does not match intent %q", + existing.Namespace, existing.Name, existing.Spec.VolumeName, wanted.Spec.VolumeName) + } + if reader && existing.Status.Phase == corev1.ClaimBound && existing.Spec.VolumeName == "" { + return fmt.Errorf("bound regular model cache reader PVC %s/%s has no volumeName", + existing.Namespace, existing.Name) + } + if reader && binding.Status.Realized != nil && + binding.Status.Realized.BoundPersistentVolumeName != "" && + existing.Spec.VolumeName != binding.Status.Realized.BoundPersistentVolumeName { + + return fmt.Errorf("regular model cache reader PVC %s/%s volumeName %q does not match binding PV %q", + existing.Namespace, existing.Name, existing.Spec.VolumeName, + binding.Status.Realized.BoundPersistentVolumeName) + } + return nil +} + +func normalizeRegularModelCacheContainerDefaults(container *corev1.Container) { + if container == nil { + return + } + defaultPullPolicy := corev1.PullIfNotPresent + lastSlash := strings.LastIndex(container.Image, "/") + lastColon := strings.LastIndex(container.Image, ":") + if lastColon <= lastSlash || container.Image[lastColon+1:] == "latest" { + defaultPullPolicy = corev1.PullAlways + } + if container.ImagePullPolicy == defaultPullPolicy { + container.ImagePullPolicy = "" + } + if container.TerminationMessagePath == corev1.TerminationMessagePathDefault { + container.TerminationMessagePath = "" + } + if container.TerminationMessagePolicy == corev1.TerminationMessageReadFile { + container.TerminationMessagePolicy = "" + } + for i := range container.Ports { + if container.Ports[i].Protocol == corev1.ProtocolTCP { + container.Ports[i].Protocol = "" + } + } +} + +func normalizeRegularModelCachePodSpec(spec *corev1.PodSpec) { + if spec == nil { + return + } + if spec.DNSPolicy == corev1.DNSClusterFirst { + spec.DNSPolicy = "" + } + if spec.SchedulerName == corev1.DefaultSchedulerName { + spec.SchedulerName = "" + } + if spec.TerminationGracePeriodSeconds != nil && *spec.TerminationGracePeriodSeconds == 30 { + spec.TerminationGracePeriodSeconds = nil + } + if spec.EnableServiceLinks != nil && *spec.EnableServiceLinks { + spec.EnableServiceLinks = nil + } + if spec.PreemptionPolicy != nil && *spec.PreemptionPolicy == corev1.PreemptLowerPriority { + spec.PreemptionPolicy = nil + } + for i := range spec.Containers { + normalizeRegularModelCacheContainerDefaults(&spec.Containers[i]) + } + for i := range spec.InitContainers { + normalizeRegularModelCacheContainerDefaults(&spec.InitContainers[i]) + } + for i := range spec.Volumes { + volume := &spec.Volumes[i] + defaultModes := make([]**int32, 0, 4) + if volume.Secret != nil { + defaultModes = append(defaultModes, &volume.Secret.DefaultMode) + } + if volume.ConfigMap != nil { + defaultModes = append(defaultModes, &volume.ConfigMap.DefaultMode) + } + if volume.DownwardAPI != nil { + defaultModes = append(defaultModes, &volume.DownwardAPI.DefaultMode) + } + if volume.Projected != nil { + defaultModes = append(defaultModes, &volume.Projected.DefaultMode) + } + for _, defaultMode := range defaultModes { + if *defaultMode != nil && **defaultMode == 0o644 { + *defaultMode = nil + } + } + } + +} +func normalizeRegularModelCacheJobSpec(spec *batchv1.JobSpec) { + if spec == nil { + return + } + if spec.Parallelism != nil && *spec.Parallelism == 1 { + spec.Parallelism = nil + } + if spec.Completions != nil && *spec.Completions == 1 { + spec.Completions = nil + } + if spec.BackoffLimit != nil && *spec.BackoffLimit == 6 { + spec.BackoffLimit = nil + } + if spec.CompletionMode != nil && *spec.CompletionMode == batchv1.NonIndexedCompletion { + spec.CompletionMode = nil + } + if spec.Suspend != nil && !*spec.Suspend { + spec.Suspend = nil + } + if spec.ManualSelector != nil && !*spec.ManualSelector { + spec.ManualSelector = nil + } +} + +func validateRegularModelCacheJob( + existing *batchv1.Job, + wanted *batchv1.Job, + binding *nvcav2beta1.ModelCacheBinding, +) error { + if existing == nil || wanted == nil || binding == nil { + return fmt.Errorf("regular model cache Job intent is incomplete") + } + if err := validateRegularModelCacheObjectMeta(existing, wanted, binding.UID); err != nil { + return err + } + if err := validateRegularModelCacheSharedMetadata(&existing.Spec.Template.ObjectMeta); err != nil { + return fmt.Errorf("regular model cache writer Job Pod template: %w", err) + } + if err := validateRegularModelCacheSharedMetadata(&wanted.Spec.Template.ObjectMeta); err != nil { + return fmt.Errorf("regular model cache intended writer Job Pod template is not binding-scoped: %w", err) + } + if err := requireRegularModelCacheBindingUID(&existing.Spec.Template.ObjectMeta, binding.UID); err != nil { + return fmt.Errorf("regular model cache writer Job Pod template: %w", err) + } + for key, want := range wanted.Spec.Template.Labels { + if existing.Spec.Template.Labels[key] != want { + return fmt.Errorf("regular model cache Job %s/%s Pod-template label %q changed", + existing.Namespace, existing.Name, key) + } + } + for key, want := range wanted.Spec.Template.Annotations { + if existing.Spec.Template.Annotations[key] != want { + return fmt.Errorf("regular model cache Job %s/%s Pod-template annotation %q changed", + existing.Namespace, existing.Name, key) + } + } + + existingSpec := existing.Spec.DeepCopy() + wantedSpec := wanted.Spec.DeepCopy() + existingTemplate := existingSpec.Template.Spec.DeepCopy() + wantedTemplate := wantedSpec.Template.Spec.DeepCopy() + normalizeRegularModelCachePodSpec(existingTemplate) + normalizeRegularModelCachePodSpec(wantedTemplate) + existingSpec.Template = corev1.PodTemplateSpec{} + wantedSpec.Template = corev1.PodTemplateSpec{} + if wantedSpec.Selector == nil { + existingSpec.Selector = nil + } + normalizeRegularModelCacheJobSpec(existingSpec) + normalizeRegularModelCacheJobSpec(wantedSpec) + if !apiequality.Semantic.DeepEqual(existingSpec, wantedSpec) || + !apiequality.Semantic.DeepEqual(existingTemplate, wantedTemplate) { + return fmt.Errorf("regular model cache Job %s/%s immutable spec does not match intent", + existing.Namespace, existing.Name) + } + return nil +} + +func validateRegularModelCachePVIdentity( + binding *nvcav2beta1.ModelCacheBinding, + pv *corev1.PersistentVolume, + expectedModes []corev1.PersistentVolumeAccessMode, +) error { + if binding == nil || pv == nil { + return fmt.Errorf("regular model cache PV identity is incomplete") + } + if binding.Spec.Decision.Transition == nvcastorage.ModelCacheTransitionROXReadOnly { + if err := requireRegularModelCacheBindingUID(pv, binding.UID); err != nil { + return err + } + } + expectedStorageClass, err := regularModelCacheExpectedStorageClassName(binding) + if err != nil { + return err + } + if pv.Spec.StorageClassName != expectedStorageClass { + return fmt.Errorf( + "regular model cache PV %s StorageClass is %q, want %q", + pv.Name, pv.Spec.StorageClassName, expectedStorageClass) + } + if pv.Spec.PersistentVolumeReclaimPolicy != corev1.PersistentVolumeReclaimRetain { + return fmt.Errorf( + "regular model cache PV %s reclaim policy is %q, want Retain", + pv.Name, pv.Spec.PersistentVolumeReclaimPolicy) + } + if !slices.Equal(pv.Spec.AccessModes, expectedModes) { + return fmt.Errorf( + "regular model cache PV %s access modes are %v, want %v", + pv.Name, pv.Spec.AccessModes, expectedModes) + } + if pv.Spec.CSI == nil || pv.Spec.CSI.Driver != binding.Spec.Decision.Provisioner { + return fmt.Errorf( + "regular model cache PV %s CSI driver does not match persisted provisioner %q", + pv.Name, binding.Spec.Decision.Provisioner) + } + if pv.Spec.CSI.VolumeHandle == "" { + return fmt.Errorf("regular model cache PV %s has an empty CSI volume handle", pv.Name) + } + return nil +} + +func validateRegularModelCacheReaderPVMountOptions( + binding *nvcav2beta1.ModelCacheBinding, + pv *corev1.PersistentVolume, +) error { + if binding == nil || pv == nil { + return fmt.Errorf("regular model cache reader mount-option identity is incomplete") + } + if binding.Spec.Decision.Transition != nvcastorage.ModelCacheTransitionROXReadOnly { + return nil + } + for _, required := range binding.Spec.Decision.RequiredMountOptions { + if !slices.Contains(pv.Spec.MountOptions, required) { + return fmt.Errorf("regular model cache reader PV %s is missing required mount option %q", + pv.Name, required) + } + for _, actual := range pv.Spec.MountOptions { + if regularModelCacheMountOptionsConflict(required, actual) { + return fmt.Errorf( + "regular model cache reader PV %s mount option %q conflicts with required option %q", + pv.Name, actual, required) + } + } + } + return nil +} + +func validateRegularModelCachePVForPVC( + binding *nvcav2beta1.ModelCacheBinding, + pvc *corev1.PersistentVolumeClaim, + pv *corev1.PersistentVolume, + expectedModes []corev1.PersistentVolumeAccessMode, +) error { + if pvc == nil || pvc.UID == "" { + return fmt.Errorf("regular model cache PVC identity is incomplete") + } + if err := validateRegularModelCachePVIdentity(binding, pv, expectedModes); err != nil { + return err + } + if regularModelCachePVVolumeMode(pv) != regularModelCachePVCVolumeMode(pvc) { + return fmt.Errorf("regular model cache PV %s volume mode %q does not match PVC %s/%s volume mode %q", + pv.Name, regularModelCachePVVolumeMode(pv), + pvc.Namespace, pvc.Name, regularModelCachePVCVolumeMode(pvc)) + } + expectedStorageClass, err := regularModelCacheExpectedStorageClassName(binding) + if err != nil { + return err + } + if pvc.Spec.StorageClassName == nil || *pvc.Spec.StorageClassName != expectedStorageClass { + return fmt.Errorf("regular model cache PVC %s/%s StorageClass does not match %q", + pvc.Namespace, pvc.Name, expectedStorageClass) + } + claimRef := pv.Spec.ClaimRef + if claimRef == nil { + return fmt.Errorf("regular model cache PV %s has no claimRef", pv.Name) + } + if claimRef.Namespace != pvc.Namespace || claimRef.Name != pvc.Name { + return fmt.Errorf("regular model cache PV %s claimRef does not match exact PVC %s/%s UID %q", + pv.Name, pvc.Namespace, pvc.Name, pvc.UID) + } + if claimRef.UID != pvc.UID { + return fmt.Errorf("regular model cache PV %s claimRef UID %q does not match PVC UID %q", + pv.Name, claimRef.UID, pvc.UID) + } + return nil +} + +type regularModelCacheCleanupTargets struct { + namespace string + binding *nvcav2beta1.ModelCacheBinding + rwPVC *corev1.PersistentVolumeClaim + roPVC *corev1.PersistentVolumeClaim + initJob *batchv1.Job +} + +func (c K8sComputeBackend) regularModelCacheCleanupBinding( + ctx context.Context, + req *nvcav2beta1.ICMSRequest, +) (*nvcav2beta1.ModelCacheBinding, bool, error) { + selection, err := persistedRegularModelCacheSelection(req) + if err != nil { + return nil, false, fmt.Errorf("parse regular model cache selection for cleanup: %w", err) + } + if selection == nil { + return nil, true, nil + } + if selection.Mode != nvcastorage.ModelCacheSelectionDurable { + return nil, false, nil + } + binding, authorized, err := c.bk8s.beginRegularModelCacheBindingRetirement(ctx, req) + if err != nil { + return nil, false, err + } + return binding, authorized, nil +} + +func (c K8sComputeBackend) regularModelCacheTransitionTargets( + ctx context.Context, + req *nvcav2beta1.ICMSRequest, + rwPVCName string, + initJobName string, +) (*regularModelCacheCleanupTargets, bool, error) { + selection, err := persistedRegularModelCacheSelection(req) + if err != nil { + return nil, false, fmt.Errorf("parse regular model cache selection for transition: %w", err) + } + if selection == nil || selection.Mode != nvcastorage.ModelCacheSelectionDurable { + return nil, false, nil + } + binding, err := c.bk8s.activeModelCacheBindingForRuntime(ctx, req) + if err != nil { + return nil, true, err + } + targets, err := c.validateRegularModelCacheCleanupTargets( + ctx, binding, rwPVCName, initJobName) + if err != nil { + return nil, true, err + } + return targets, true, nil +} + +func (c K8sComputeBackend) validateRegularModelCacheCleanupTargets( + ctx context.Context, + binding *nvcav2beta1.ModelCacheBinding, + rwPVCName string, + initJobName string, +) (*regularModelCacheCleanupTargets, error) { + if binding == nil { + return nil, nil + } + _, _, separateReader, err := regularModelCacheAccessModePlan(binding) + if err != nil { + return nil, err + } + roPVCName, err := regularModelCacheReaderPVCName(rwPVCName) + if err != nil { + return nil, err + } + if !slices.Contains(binding.Spec.Resources.PersistentVolumeClaimNames, rwPVCName) || + !slices.Contains(binding.Spec.Resources.JobNames, initJobName) { + return nil, fmt.Errorf("refusing regular model cache cleanup outside binding resource intent") + } + if separateReader != slices.Contains(binding.Spec.Resources.PersistentVolumeClaimNames, roPVCName) { + return nil, fmt.Errorf( + "refusing regular model cache cleanup with reader inventory outside transition %q intent", + binding.Spec.Decision.Transition) + } + targets := ®ularModelCacheCleanupTargets{ + namespace: binding.Spec.Resources.WriterNamespace, + binding: binding, + } + pvcTargets := []struct { + name string + set func(*corev1.PersistentVolumeClaim) + }{ + {name: rwPVCName, set: func(pvc *corev1.PersistentVolumeClaim) { targets.rwPVC = pvc }}, + } + if separateReader { + pvcTargets = append(pvcTargets, struct { + name string + set func(*corev1.PersistentVolumeClaim) + }{name: roPVCName, set: func(pvc *corev1.PersistentVolumeClaim) { targets.roPVC = pvc }}) + } + for _, target := range pvcTargets { + pvc, err := c.clients.K8s.CoreV1().PersistentVolumeClaims(binding.Spec.Resources.WriterNamespace). + Get(ctx, target.name, metav1.GetOptions{}) + switch { + case apierrors.IsNotFound(err): + case err != nil: + return nil, fmt.Errorf("get regular model cache cleanup PVC %s/%s: %w", + binding.Spec.Resources.WriterNamespace, target.name, err) + default: + if err := requireRegularModelCacheBindingUID(pvc, binding.UID); err != nil { + return nil, err + } + target.set(pvc) + } + } + job, err := c.clients.K8s.BatchV1().Jobs(binding.Spec.Resources.WriterNamespace). + Get(ctx, initJobName, metav1.GetOptions{}) + switch { + case apierrors.IsNotFound(err): + case err != nil: + return nil, fmt.Errorf("get regular model cache cleanup Job %s/%s: %w", + binding.Spec.Resources.WriterNamespace, initJobName, err) + default: + if err := requireRegularModelCacheBindingUID(job, binding.UID); err != nil { + return nil, err + } + targets.initJob = job + } + return targets, nil +} + +func validateRegularModelCacheCleanupPV( + binding *nvcav2beta1.ModelCacheBinding, + pvc *corev1.PersistentVolumeClaim, + pv *corev1.PersistentVolume, +) error { + if binding == nil || pvc == nil || pv == nil { + return fmt.Errorf("regular model cache cleanup binding, PVC, and PV must be present") + } + reader, err := classifyRegularModelCachePVCName(pvc.Name) + if err != nil { + return fmt.Errorf("regular model cache cleanup PVC %s/%s is outside writer/reader intent: %w", + pvc.Namespace, pvc.Name, err) + } + expectedModes, err := regularModelCacheExpectedPVCModes(binding, reader) + if err != nil { + return err + } + // Cleanup changes an exact Retain PV to Delete before deleting its PVC. A + // retry must accept that already-applied state while validating every other + // part of the PV/PVC identity. + pvForValidation := pv + if pv.Spec.PersistentVolumeReclaimPolicy == corev1.PersistentVolumeReclaimDelete { + pvForValidation = pv.DeepCopy() + pvForValidation.Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimRetain + } + if err := validateRegularModelCachePVForPVC( + binding, pvc, pvForValidation, expectedModes); err != nil { + return err + } + if reader { + if err := validateRegularModelCacheReaderPVMountOptions(binding, pvForValidation); err != nil { + return err + } + } + if len(binding.Spec.Resources.PersistentVolumeNames) != 0 && + !slices.Contains(binding.Spec.Resources.PersistentVolumeNames, pv.Name) { + return fmt.Errorf("regular model cache PV %q does not match binding intent %v", + pv.Name, binding.Spec.Resources.PersistentVolumeNames) + } + if binding.Status.Realized != nil && binding.Status.Realized.BoundPersistentVolumeName != "" && + binding.Status.Realized.BoundPersistentVolumeName != pv.Name { + return fmt.Errorf("regular model cache PV %q does not match binding realized PV %q", + pv.Name, binding.Status.Realized.BoundPersistentVolumeName) + } + return nil +} + +func (c K8sComputeBackend) validateRegularModelCachePVClaimAfterWriterDelete( + ctx context.Context, + pv *corev1.PersistentVolume, + writerPVC *corev1.PersistentVolumeClaim, + binding *nvcav2beta1.ModelCacheBinding, + namespace string, + rwPVCName string, + roPVCName string, +) error { + if pv.Spec.ClaimRef == nil || pv.Spec.ClaimRef.Namespace != namespace { + return fmt.Errorf("regular model cache PV %q has no binding-owned claimRef", pv.Name) + } + claimRef := pv.Spec.ClaimRef + switch claimRef.Name { + case rwPVCName: + if err := validateRegularModelCachePVForPVC(binding, writerPVC, pv, + []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}); err != nil { + return fmt.Errorf("validate regular model cache writer PV: %w", err) + } + case roPVCName: + readerPVC, err := c.clients.K8s.CoreV1().PersistentVolumeClaims(namespace). + Get(ctx, roPVCName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("get existing regular model cache reader PVC %s/%s: %w", + namespace, roPVCName, err) + } + if err := requireRegularModelCacheBindingUID(readerPVC, binding.UID); err != nil { + return err + } + if !slices.Equal(readerPVC.Spec.AccessModes, ROAccessMode) { + return fmt.Errorf("regular model cache reader PVC %s/%s does not use ReadOnlyMany", + readerPVC.Namespace, readerPVC.Name) + } + if err := validateRegularModelCachePVForPVC(binding, readerPVC, pv, ROAccessMode); err != nil { + return fmt.Errorf("validate regular model cache reader PV: %w", err) + } + default: + return fmt.Errorf("regular model cache PV %q claimRef name %q is outside binding intent", + pv.Name, claimRef.Name) + } + return nil +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/modelcache_binding_test.go b/src/compute-plane-services/nvca/pkg/nvca/modelcache_binding_test.go new file mode 100644 index 000000000..e45257b8b --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/modelcache_binding_test.go @@ -0,0 +1,773 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nvca + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + fakek8sclient "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/kubeclients" + nvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1" + fakebartclient "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/client/clientset/versioned/fake" + nvcaerrors "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/errors" + nvcastorage "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/storage" +) + +func durableModelCacheBindingRequest( + t *testing.T, + workflow nvcastorage.ModelCacheWorkflow, +) (*nvcav2beta1.ICMSRequest, *nvcastorage.PersistedModelCacheStorageSelection) { + t.Helper() + req := selectionRequest(workflow == nvcastorage.ModelCacheWorkflowHelm) + req.UID = types.UID("request-uid") + req.Spec.NCAId = "nca-a" + req.Finalizers = []string{NVCAFinalizer} + selection, err := nvcastorage.NewPersistedModelCacheStorageSelection( + workflow, + nvcastorage.ModelCacheSelectionDurable, + resolvedSelectionForSetup(t), + ) + require.NoError(t, err) + raw, err := selection.Marshal() + require.NoError(t, err) + req.Annotations = map[string]string{ + nvcastorage.ModelCacheStorageSelectionAnnotationKey: raw, + } + return req, selection +} + +func bindingTestBackend( + req *nvcav2beta1.ICMSRequest, + bartObjects ...runtime.Object, +) (*BackendK8sCache, *fakebartclient.Clientset, *fakek8sclient.Clientset) { + objects := []runtime.Object{req.DeepCopy()} + objects = append(objects, bartObjects...) + bart := fakebartclient.NewSimpleClientset(objects...) + k8s := fakek8sclient.NewSimpleClientset( + selectionStorageClass(), + selectionCatalogConfigMap(selectionCatalogNVMesh), + ) + return &BackendK8sCache{ + clients: &kubeclients.KubeClients{ + BART: bart, + K8s: k8s, + }, + podInstanceNamespace: "pod-instances", + systemNamespace: selectionCatalogNamespace, + }, bart, k8s +} + +func assignCreatedBindingUID(bart *fakebartclient.Clientset, uid types.UID) { + bart.PrependReactor("create", "modelcachebindings", func(action k8stesting.Action) (bool, runtime.Object, error) { + created := action.(k8stesting.CreateAction).GetObject().(*nvcav2beta1.ModelCacheBinding) + created.UID = uid + return false, nil, nil + }) +} + +func boundSelectionPayload( + t *testing.T, + selection *nvcastorage.PersistedModelCacheStorageSelection, + binding *nvcav2beta1.ModelCacheBinding, +) string { + t.Helper() + bound := *selection + bound.BindingName = binding.Name + bound.BindingUID = binding.UID + raw, err := bound.Marshal() + require.NoError(t, err) + return raw +} + +func boundModelCacheBindingFixture( + t *testing.T, + workflow nvcastorage.ModelCacheWorkflow, + phase nvcav2beta1.ModelCacheBindingPhase, + refs []nvcav2beta1.ModelCacheBindingRequestReference, +) (*nvcav2beta1.ICMSRequest, *nvcav2beta1.ModelCacheBinding, *BackendK8sCache, *fakebartclient.Clientset) { + t.Helper() + req, selection := durableModelCacheBindingRequest(t, workflow) + writerNamespace := "pod-instances" + if workflow == nvcastorage.ModelCacheWorkflowHelm { + writerNamespace = nvcastorage.ModelCacheInitNamespace + } + binding, err := nvcastorage.NewModelCacheBinding( + selection, req.Spec.NCAId, "model-cache-handle", writerNamespace) + require.NoError(t, err) + binding.UID = types.UID("binding-uid") + binding.ResourceVersion = "1" + binding.Status.Phase = phase + for i := range refs { + if refs[i].Namespace == "requests" { + refs[i].Namespace = req.Namespace + } + } + binding.Status.RequestReferences = append( + []nvcav2beta1.ModelCacheBindingRequestReference(nil), refs...) + req.Annotations[nvcastorage.ModelCacheStorageSelectionAnnotationKey] = + boundSelectionPayload(t, selection, binding) + backend, bart, _ := bindingTestBackend(req, binding) + return req, binding, backend, bart +} + +func countModelCacheBindingStatusUpdates(actions []k8stesting.Action) int { + updates := 0 + for _, action := range actions { + if action.GetVerb() == "update" && action.GetResource().Resource == "modelcachebindings" && + action.GetSubresource() == "status" { + updates++ + } + } + return updates +} + +func installActiveRegularModelCacheBinding( + t *testing.T, + backend *K8sComputeBackend, + req *nvcav2beta1.ICMSRequest, + rawSelection string, +) *nvcav2beta1.ModelCacheBinding { + t.Helper() + selection, err := nvcastorage.ParsePersistedModelCacheStorageSelection(rawSelection) + require.NoError(t, err) + require.Equal(t, nvcastorage.ModelCacheWorkflowRegular, selection.Workflow) + require.Equal(t, nvcastorage.ModelCacheSelectionDurable, selection.Mode) + + template := selectionRequest(false) + req.Spec.CreationMsgInfo = template.Spec.CreationMsgInfo + req.Spec.NCAId = "nca-a" + req.UID = types.UID("request-uid") + binding, err := nvcastorage.NewModelCacheBinding( + selection, + req.Spec.NCAId, + req.Spec.CreationMsgInfo.FunctionLaunchSpecification.CacheLaunchSpecification.CacheHandle, + backend.bk8s.podInstanceNamespace, + ) + require.NoError(t, err) + binding.UID = types.UID("binding-uid") + binding.Status.Phase = nvcav2beta1.ModelCacheBindingPhaseActive + binding.Status.RequestReferences = []nvcav2beta1.ModelCacheBindingRequestReference{{ + Namespace: req.Namespace, + Name: req.Name, + UID: req.UID, + }} + if req.Annotations == nil { + req.Annotations = map[string]string{} + } + req.Annotations[nvcastorage.ModelCacheStorageSelectionAnnotationKey] = + boundSelectionPayload(t, selection, binding) + backend.clients.BART = fakebartclient.NewSimpleClientset(req.DeepCopy(), binding.DeepCopy()) + return binding +} + +func TestEnsureModelCacheBindingCreatesBeforeRuntimeAndSurvivesLiveDrift(t *testing.T) { + req, _ := durableModelCacheBindingRequest(t, nvcastorage.ModelCacheWorkflowRegular) + backend, bart, k8s := bindingTestBackend(req) + assignCreatedBindingUID(bart, types.UID("binding-uid")) + + changed, err := backend.ensureModelCacheBinding(t.Context(), req.DeepCopy()) + require.NoError(t, err) + assert.True(t, changed) + + persisted, err := bart.NvcaV2beta1().ICMSRequests(req.Namespace). + Get(t.Context(), req.Name, metav1.GetOptions{}) + require.NoError(t, err) + selection, err := nvcastorage.ParsePersistedModelCacheStorageSelection( + persisted.Annotations[nvcastorage.ModelCacheStorageSelectionAnnotationKey]) + require.NoError(t, err) + assert.Equal(t, types.UID("binding-uid"), selection.BindingUID) + assert.Equal(t, nvcastorage.ModelCacheBindingName("model-cache-handle"), selection.BindingName) + + binding, err := bart.NvcaV2beta1().ModelCacheBindings(nvcastorage.ModelCacheInitNamespace). + Get(t.Context(), selection.BindingName, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, nvcav2beta1.ModelCacheBindingPhaseActive, binding.Status.Phase) + assert.True(t, nvcastorage.ModelCacheBindingHasRequestReference( + binding, req.Namespace, req.Name, req.UID)) + + // Once committed, the immutable binding is authoritative. A live class + // deletion must not cause reselection or block adoption on a later reconcile. + require.NoError(t, k8s.StorageV1().StorageClasses().Delete( + t.Context(), nvcastorage.DefaultModelCacheStorageClassName, metav1.DeleteOptions{})) + k8s.ClearActions() + changed, err = backend.ensureModelCacheBinding(t.Context(), persisted.DeepCopy()) + require.NoError(t, err) + assert.False(t, changed) + assert.Empty(t, k8s.Actions(), "a committed binding must not re-read the live StorageClass") +} + +func TestEnsureModelCacheBindingRejectsPreCreateStorageClassDrift(t *testing.T) { + req, _ := durableModelCacheBindingRequest(t, nvcastorage.ModelCacheWorkflowRegular) + backend, bart, k8s := bindingTestBackend(req) + drifted := selectionStorageClass() + drifted.Parameters["pool"] = "changed" + k8s = fakek8sclient.NewSimpleClientset(drifted) + backend.clients.K8s = k8s + + changed, err := backend.ensureModelCacheBinding(t.Context(), req.DeepCopy()) + assert.False(t, changed) + require.ErrorContains(t, err, "configuration digest changed") + assert.True(t, nvcaerrors.IsTerminal(err)) + + bindings, listErr := bart.NvcaV2beta1().ModelCacheBindings(nvcastorage.ModelCacheInitNamespace). + List(t.Context(), metav1.ListOptions{}) + require.NoError(t, listErr) + assert.Empty(t, bindings.Items) +} + +func TestEnsureModelCacheBindingRejectsPreCreateCatalogDrift(t *testing.T) { + req, _ := durableModelCacheBindingRequest(t, nvcastorage.ModelCacheWorkflowRegular) + backend, bart, _ := bindingTestBackend(req) + backend.clients.K8s = fakek8sclient.NewSimpleClientset( + selectionStorageClass(), + selectionCatalogConfigMap(selectionCatalogNVMesh+"\n"), + ) + + changed, err := backend.ensureModelCacheBinding(t.Context(), req.DeepCopy()) + assert.False(t, changed) + require.ErrorContains(t, err, "catalog digest changed") + assert.True(t, nvcaerrors.IsTerminal(err)) + + bindings, listErr := bart.NvcaV2beta1().ModelCacheBindings(nvcastorage.ModelCacheInitNamespace). + List(t.Context(), metav1.ListOptions{}) + require.NoError(t, listErr) + assert.Empty(t, bindings.Items) +} + +func TestEnsureModelCacheBindingConvergesAcrossRequestNamespaces(t *testing.T) { + reqA, _ := durableModelCacheBindingRequest(t, nvcastorage.ModelCacheWorkflowRegular) + reqB := reqA.DeepCopy() + reqB.Namespace = "requests-b" + reqB.Name = "request-b" + reqB.UID = types.UID("request-b-uid") + backend, bart, _ := bindingTestBackend(reqA, reqB.DeepCopy()) + assignCreatedBindingUID(bart, types.UID("binding-uid")) + + changed, err := backend.ensureModelCacheBinding(t.Context(), reqA.DeepCopy()) + require.NoError(t, err) + assert.True(t, changed) + changed, err = backend.ensureModelCacheBinding(t.Context(), reqB.DeepCopy()) + require.NoError(t, err) + assert.True(t, changed) + + bindings, err := bart.NvcaV2beta1().ModelCacheBindings(nvcastorage.ModelCacheInitNamespace). + List(t.Context(), metav1.ListOptions{}) + require.NoError(t, err) + require.Len(t, bindings.Items, 1) + binding := &bindings.Items[0] + assert.True(t, nvcastorage.ModelCacheBindingHasRequestReference( + binding, reqA.Namespace, reqA.Name, reqA.UID)) + assert.True(t, nvcastorage.ModelCacheBindingHasRequestReference( + binding, reqB.Namespace, reqB.Name, reqB.UID)) + assert.Len(t, binding.Status.RequestReferences, 2) + + for _, req := range []*nvcav2beta1.ICMSRequest{reqA, reqB} { + persisted, getErr := bart.NvcaV2beta1().ICMSRequests(req.Namespace). + Get(t.Context(), req.Name, metav1.GetOptions{}) + require.NoError(t, getErr) + selection, parseErr := nvcastorage.ParsePersistedModelCacheStorageSelection( + persisted.Annotations[nvcastorage.ModelCacheStorageSelectionAnnotationKey]) + require.NoError(t, parseErr) + assert.Equal(t, binding.Name, selection.BindingName) + assert.Equal(t, binding.UID, selection.BindingUID) + } +} + +func TestEnsureModelCacheBindingReplacesOnlyStaleRequestUIDReference(t *testing.T) { + t.Run("replaced request UID", func(t *testing.T) { + req, selection := durableModelCacheBindingRequest(t, nvcastorage.ModelCacheWorkflowRegular) + binding, err := nvcastorage.NewModelCacheBinding( + selection, req.Spec.NCAId, "model-cache-handle", "pod-instances") + require.NoError(t, err) + binding.UID = types.UID("binding-uid") + binding.Status.Phase = nvcav2beta1.ModelCacheBindingPhaseActive + binding.Status.RequestReferences = []nvcav2beta1.ModelCacheBindingRequestReference{{ + Namespace: req.Namespace, + Name: req.Name, + UID: types.UID("old-request-uid"), + }} + backend, bart, _ := bindingTestBackend(req, binding) + + changed, ensureErr := backend.ensureModelCacheBinding(t.Context(), req.DeepCopy()) + require.NoError(t, ensureErr) + assert.True(t, changed) + updated, getErr := bart.NvcaV2beta1().ModelCacheBindings(binding.Namespace). + Get(t.Context(), binding.Name, metav1.GetOptions{}) + require.NoError(t, getErr) + assert.True(t, nvcastorage.ModelCacheBindingHasRequestReference( + updated, req.Namespace, req.Name, req.UID)) + assert.False(t, nvcastorage.ModelCacheBindingHasRequestReference( + updated, req.Namespace, req.Name, types.UID("old-request-uid"))) + }) + + t.Run("old request UID is still live", func(t *testing.T) { + req, selection := durableModelCacheBindingRequest(t, nvcastorage.ModelCacheWorkflowRegular) + oldReq := req.DeepCopy() + oldReq.UID = types.UID("old-request-uid") + binding, err := nvcastorage.NewModelCacheBinding( + selection, req.Spec.NCAId, "model-cache-handle", "pod-instances") + require.NoError(t, err) + binding.UID = types.UID("binding-uid") + binding.Status.Phase = nvcav2beta1.ModelCacheBindingPhaseActive + binding.Status.RequestReferences = []nvcav2beta1.ModelCacheBindingRequestReference{{ + Namespace: oldReq.Namespace, + Name: oldReq.Name, + UID: oldReq.UID, + }} + bart := fakebartclient.NewSimpleClientset(oldReq, binding) + backend, _, k8s := bindingTestBackend(req) + backend.clients.BART = bart + backend.clients.K8s = k8s + + changed, ensureErr := backend.ensureModelCacheBinding(t.Context(), req.DeepCopy()) + assert.False(t, changed) + require.ErrorContains(t, ensureErr, "already referenced by live request") + assert.True(t, nvcaerrors.IsTerminal(ensureErr)) + }) +} + +func TestEnsureModelCacheBindingReleasesReferenceWhenRequestStartsDeleting(t *testing.T) { + req, _ := durableModelCacheBindingRequest(t, nvcastorage.ModelCacheWorkflowRegular) + backend, bart, _ := bindingTestBackend(req) + assignCreatedBindingUID(bart, types.UID("binding-uid")) + deleting := req.DeepCopy() + now := metav1.Now() + deleting.DeletionTimestamp = &now + bart.PrependReactor("get", "icmsrequests", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, deleting.DeepCopy(), nil + }) + + changed, err := backend.ensureModelCacheBinding(t.Context(), req.DeepCopy()) + assert.False(t, changed) + require.ErrorContains(t, err, "began deleting") + binding, getErr := bart.NvcaV2beta1().ModelCacheBindings(nvcastorage.ModelCacheInitNamespace). + Get(t.Context(), nvcastorage.ModelCacheBindingName("model-cache-handle"), metav1.GetOptions{}) + require.NoError(t, getErr) + assert.Empty(t, binding.Status.RequestReferences) +} + +func TestEnsureModelCacheBindingReleasesReferenceWhenRequestIsReplaced(t *testing.T) { + req, _ := durableModelCacheBindingRequest(t, nvcastorage.ModelCacheWorkflowRegular) + backend, bart, _ := bindingTestBackend(req) + assignCreatedBindingUID(bart, types.UID("binding-uid")) + replacement := req.DeepCopy() + replacement.UID = types.UID("replacement-request-uid") + bart.PrependReactor("get", "icmsrequests", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, replacement.DeepCopy(), nil + }) + + changed, err := backend.ensureModelCacheBinding(t.Context(), req.DeepCopy()) + assert.False(t, changed) + require.ErrorContains(t, err, "was replaced") + binding, getErr := bart.NvcaV2beta1().ModelCacheBindings(nvcastorage.ModelCacheInitNamespace). + Get(t.Context(), nvcastorage.ModelCacheBindingName("model-cache-handle"), metav1.GetOptions{}) + require.NoError(t, getErr) + assert.Empty(t, binding.Status.RequestReferences) +} + +func TestEnsureModelCacheBindingRetriesTransientCatalogRead(t *testing.T) { + req, _ := durableModelCacheBindingRequest(t, nvcastorage.ModelCacheWorkflowRegular) + backend, bart, k8s := bindingTestBackend(req) + k8s.PrependReactor("get", "configmaps", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewServiceUnavailable("catalog API unavailable") + }) + + changed, err := backend.ensureModelCacheBinding(t.Context(), req.DeepCopy()) + assert.False(t, changed) + require.ErrorContains(t, err, "catalog API unavailable") + assert.False(t, nvcaerrors.IsTerminal(err)) + bindings, listErr := bart.NvcaV2beta1().ModelCacheBindings(nvcastorage.ModelCacheInitNamespace). + List(t.Context(), metav1.ListOptions{}) + require.NoError(t, listErr) + assert.Empty(t, bindings.Items) +} + +func TestEnsureModelCacheBindingAdoptsExactEmptyStatus(t *testing.T) { + req, selection := durableModelCacheBindingRequest(t, nvcastorage.ModelCacheWorkflowRegular) + binding, err := nvcastorage.NewModelCacheBinding( + selection, req.Spec.NCAId, "model-cache-handle", "pod-instances") + require.NoError(t, err) + binding.UID = types.UID("binding-uid") + backend, bart, k8s := bindingTestBackend(req, binding) + + changed, err := backend.ensureModelCacheBinding(t.Context(), req.DeepCopy()) + require.NoError(t, err) + assert.True(t, changed) + assert.Empty(t, k8s.Actions(), "exact binding adoption must not consult the live StorageClass") + + adopted, err := bart.NvcaV2beta1().ModelCacheBindings(binding.Namespace). + Get(t.Context(), binding.Name, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, nvcav2beta1.ModelCacheBindingPhaseActive, adopted.Status.Phase) + assert.True(t, nvcastorage.ModelCacheBindingHasRequestReference( + adopted, req.Namespace, req.Name, req.UID)) +} + +func TestEnsureModelCacheBindingStopsStaleReconcileAfterConcurrentCommit(t *testing.T) { + req, selection := durableModelCacheBindingRequest(t, nvcastorage.ModelCacheWorkflowRegular) + binding, err := nvcastorage.NewModelCacheBinding( + selection, req.Spec.NCAId, "model-cache-handle", "pod-instances") + require.NoError(t, err) + binding.UID = types.UID("binding-uid") + binding.Status.Phase = nvcav2beta1.ModelCacheBindingPhaseActive + binding.Status.RequestReferences = []nvcav2beta1.ModelCacheBindingRequestReference{{ + Namespace: req.Namespace, + Name: req.Name, + UID: req.UID, + }} + backend, bart, _ := bindingTestBackend(req, binding) + + latest := req.DeepCopy() + latest.Annotations[nvcastorage.ModelCacheStorageSelectionAnnotationKey] = + boundSelectionPayload(t, selection, binding) + _, err = bart.NvcaV2beta1().ICMSRequests(latest.Namespace). + Update(t.Context(), latest, metav1.UpdateOptions{}) + require.NoError(t, err) + + changed, err := backend.ensureModelCacheBinding(t.Context(), req.DeepCopy()) + require.NoError(t, err) + assert.True(t, changed, "a stale unbound reconcile must stop after another reconcile commits the binding") +} + +func TestEnsureModelCacheBindingFailsClosedOnRetiringOrCollision(t *testing.T) { + for _, tt := range []struct { + name string + bindingNCA string + phase nvcav2beta1.ModelCacheBindingPhase + want string + }{ + {name: "retiring", bindingNCA: "nca-a", phase: nvcav2beta1.ModelCacheBindingPhaseRetiring, want: "Retiring"}, + {name: "handle collision", bindingNCA: "nca-b", phase: nvcav2beta1.ModelCacheBindingPhaseActive, want: "immutable spec"}, + } { + t.Run(tt.name, func(t *testing.T) { + req, selection := durableModelCacheBindingRequest(t, nvcastorage.ModelCacheWorkflowRegular) + binding, err := nvcastorage.NewModelCacheBinding( + selection, tt.bindingNCA, "model-cache-handle", "pod-instances") + require.NoError(t, err) + binding.UID = types.UID("binding-uid") + binding.Status.Phase = tt.phase + backend, _, _ := bindingTestBackend(req, binding) + + changed, err := backend.ensureModelCacheBinding(t.Context(), req.DeepCopy()) + assert.False(t, changed) + require.ErrorContains(t, err, tt.want) + assert.True(t, nvcaerrors.IsTerminal(err)) + }) + } +} + +func TestValidateModelCacheBindingForRuntimeRequiresExactRequestReference(t *testing.T) { + req, selection := durableModelCacheBindingRequest(t, nvcastorage.ModelCacheWorkflowRegular) + binding, err := nvcastorage.NewModelCacheBinding( + selection, req.Spec.NCAId, "model-cache-handle", "pod-instances") + require.NoError(t, err) + binding.UID = types.UID("binding-uid") + binding.Status.Phase = nvcav2beta1.ModelCacheBindingPhaseActive + req.Annotations[nvcastorage.ModelCacheStorageSelectionAnnotationKey] = + boundSelectionPayload(t, selection, binding) + backend, _, _ := bindingTestBackend(req, binding) + + err = backend.validateModelCacheBindingForRuntime(t.Context(), req.DeepCopy()) + require.ErrorContains(t, err, "has no reference") + assert.True(t, nvcaerrors.IsTerminal(err)) + + binding.Status.RequestReferences = []nvcav2beta1.ModelCacheBindingRequestReference{{ + Namespace: req.Namespace, + Name: req.Name, + UID: req.UID, + }} + backend, _, _ = bindingTestBackend(req, binding) + require.NoError(t, backend.validateModelCacheBindingForRuntime(t.Context(), req.DeepCopy())) +} + +func TestBeginRegularModelCacheBindingRetirementTransitionsSoleReference(t *testing.T) { + req, binding, backend, bart := boundModelCacheBindingFixture( + t, + nvcastorage.ModelCacheWorkflowRegular, + nvcav2beta1.ModelCacheBindingPhaseActive, + []nvcav2beta1.ModelCacheBindingRequestReference{{ + Namespace: "requests", + Name: "request", + UID: types.UID("request-uid"), + }}, + ) + + retiring, authorized, err := backend.beginRegularModelCacheBindingRetirement(t.Context(), req.DeepCopy()) + require.NoError(t, err) + assert.True(t, authorized) + require.NotNil(t, retiring) + assert.Equal(t, nvcav2beta1.ModelCacheBindingPhaseRetiring, retiring.Status.Phase) + assert.NotNil(t, retiring.Status.LastPhaseTransitionTime) + assert.Equal(t, binding.Status.RequestReferences, retiring.Status.RequestReferences) + assert.Equal(t, 1, countModelCacheBindingStatusUpdates(bart.Actions())) + + firstTransitionTime := retiring.Status.LastPhaseTransitionTime.DeepCopy() + bart.ClearActions() + retiring, authorized, err = backend.beginRegularModelCacheBindingRetirement(t.Context(), req.DeepCopy()) + require.NoError(t, err) + assert.True(t, authorized) + require.NotNil(t, retiring) + assert.Equal(t, nvcav2beta1.ModelCacheBindingPhaseRetiring, retiring.Status.Phase) + assert.Equal(t, firstTransitionTime, retiring.Status.LastPhaseTransitionTime) + assert.Zero(t, countModelCacheBindingStatusUpdates(bart.Actions()), + "an interrupted cleanup must adopt Retiring without returning it to Active") + + _, runtimeErr := backend.activeModelCacheBindingForRuntime(t.Context(), req.DeepCopy()) + require.ErrorIs(t, runtimeErr, errRegularModelCacheBindingRetiring) + assert.True(t, nvcaerrors.IsTerminal(runtimeErr)) +} +func TestBeginRegularModelCacheBindingRetirementResumesAfterReferenceRelease(t *testing.T) { + req, binding, backend, bart := boundModelCacheBindingFixture( + t, + nvcastorage.ModelCacheWorkflowRegular, + nvcav2beta1.ModelCacheBindingPhaseRetiring, + nil, + ) + bart.ClearActions() + + got, authorized, err := backend.beginRegularModelCacheBindingRetirement( + t.Context(), req.DeepCopy()) + require.NoError(t, err) + assert.True(t, authorized) + require.NotNil(t, got) + assert.Equal(t, binding.UID, got.UID) + assert.Equal(t, nvcav2beta1.ModelCacheBindingPhaseRetiring, got.Status.Phase) + assert.Zero(t, countModelCacheBindingStatusUpdates(bart.Actions())) +} + +func TestBeginRegularModelCacheBindingRetirementOtherReferenceBlocksWithoutMutation(t *testing.T) { + req, _, backend, bart := boundModelCacheBindingFixture( + t, + nvcastorage.ModelCacheWorkflowRegular, + nvcav2beta1.ModelCacheBindingPhaseActive, + []nvcav2beta1.ModelCacheBindingRequestReference{ + {Namespace: "requests", Name: "request", UID: types.UID("request-uid")}, + {Namespace: "requests", Name: "other", UID: types.UID("other-uid")}, + }, + ) + + got, authorized, err := backend.beginRegularModelCacheBindingRetirement(t.Context(), req.DeepCopy()) + require.NoError(t, err) + assert.False(t, authorized) + require.NotNil(t, got) + assert.Equal(t, nvcav2beta1.ModelCacheBindingPhaseActive, got.Status.Phase) + assert.Nil(t, got.Status.LastPhaseTransitionTime) + assert.Zero(t, countModelCacheBindingStatusUpdates(bart.Actions())) +} + +func TestBeginRegularModelCacheBindingRetirementConcurrentReferenceWinsConflict(t *testing.T) { + req, binding, backend, bart := boundModelCacheBindingFixture( + t, + nvcastorage.ModelCacheWorkflowRegular, + nvcav2beta1.ModelCacheBindingPhaseActive, + []nvcav2beta1.ModelCacheBindingRequestReference{{ + Namespace: "requests", + Name: "request", + UID: types.UID("request-uid"), + }}, + ) + gvr := nvcav2beta1.SchemeGroupVersion.WithResource("modelcachebindings") + updateAttempts := 0 + bart.PrependReactor("update", "modelcachebindings", func(action k8stesting.Action) (bool, runtime.Object, error) { + if action.GetSubresource() != "status" { + return false, nil, nil + } + updateAttempts++ + if updateAttempts != 1 { + return false, nil, nil + } + tracked, err := bart.Tracker().Get(gvr, binding.Namespace, binding.Name) + if err != nil { + return true, nil, err + } + concurrent := tracked.(*nvcav2beta1.ModelCacheBinding).DeepCopy() + concurrent.ResourceVersion = "2" + concurrent.Status.RequestReferences = append( + concurrent.Status.RequestReferences, + nvcav2beta1.ModelCacheBindingRequestReference{ + Namespace: "requests", + Name: "concurrent", + UID: types.UID("concurrent-uid"), + }, + ) + if err := bart.Tracker().Update(gvr, concurrent, binding.Namespace); err != nil { + return true, nil, err + } + return true, nil, apierrors.NewConflict( + nvcav2beta1.Resource("modelcachebindings"), binding.Name, errors.New("stale status resource version")) + }) + + got, authorized, err := backend.beginRegularModelCacheBindingRetirement(t.Context(), req.DeepCopy()) + require.NoError(t, err) + assert.False(t, authorized) + require.NotNil(t, got) + assert.Equal(t, nvcav2beta1.ModelCacheBindingPhaseActive, got.Status.Phase) + assert.Len(t, got.Status.RequestReferences, 2) + assert.Equal(t, 1, updateAttempts, "the conflict retry must observe the new reference and stop writing") + + stored, err := bart.NvcaV2beta1().ModelCacheBindings(binding.Namespace). + Get(t.Context(), binding.Name, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, nvcav2beta1.ModelCacheBindingPhaseActive, stored.Status.Phase) + assert.Len(t, stored.Status.RequestReferences, 2) +} + +func TestBeginRegularModelCacheBindingRetirementRejectsForeignOrMissingReference(t *testing.T) { + for _, tt := range []struct { + name string + refs []nvcav2beta1.ModelCacheBindingRequestReference + }{ + {name: "missing"}, + { + name: "foreign UID", + refs: []nvcav2beta1.ModelCacheBindingRequestReference{{ + Namespace: "requests", + Name: "request", + UID: types.UID("foreign-uid"), + }}, + }, + } { + t.Run(tt.name, func(t *testing.T) { + req, _, backend, bart := boundModelCacheBindingFixture( + t, nvcastorage.ModelCacheWorkflowRegular, + nvcav2beta1.ModelCacheBindingPhaseActive, tt.refs) + + got, authorized, err := backend.beginRegularModelCacheBindingRetirement(t.Context(), req.DeepCopy()) + assert.Nil(t, got) + assert.False(t, authorized) + require.ErrorContains(t, err, "has no reference") + assert.True(t, nvcaerrors.IsTerminal(err)) + assert.Zero(t, countModelCacheBindingStatusUpdates(bart.Actions())) + }) + } +} + +func TestEnsureModelCacheBindingAllowsOnlySoleRegularRequestToResumeRetiringCleanup(t *testing.T) { + for _, tt := range []struct { + name string + workflow nvcastorage.ModelCacheWorkflow + refs []nvcav2beta1.ModelCacheBindingRequestReference + wantError string + }{ + { + name: "sole regular request", + workflow: nvcastorage.ModelCacheWorkflowRegular, + refs: []nvcav2beta1.ModelCacheBindingRequestReference{{ + Namespace: "requests", + Name: "request", + UID: types.UID("request-uid"), + }}, + }, + { + name: "Helm request", + workflow: nvcastorage.ModelCacheWorkflowHelm, + refs: []nvcav2beta1.ModelCacheBindingRequestReference{{ + Namespace: "requests", + Name: "request", + UID: types.UID("request-uid"), + }}, + wantError: "cannot serve", + }, + { + name: "missing regular reference", + workflow: nvcastorage.ModelCacheWorkflowRegular, + wantError: "has no reference", + }, + { + name: "regular request with another reference", + workflow: nvcastorage.ModelCacheWorkflowRegular, + refs: []nvcav2beta1.ModelCacheBindingRequestReference{ + {Namespace: "requests", Name: "request", UID: types.UID("request-uid")}, + {Namespace: "requests", Name: "other", UID: types.UID("other-uid")}, + }, + wantError: "other request references", + }, + } { + t.Run(tt.name, func(t *testing.T) { + req, _, backend, bart := boundModelCacheBindingFixture( + t, tt.workflow, nvcav2beta1.ModelCacheBindingPhaseRetiring, tt.refs) + bart.ClearActions() + + changed, err := backend.ensureModelCacheBinding(t.Context(), req.DeepCopy()) + assert.False(t, changed) + if tt.wantError == "" { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, tt.wantError) + assert.True(t, nvcaerrors.IsTerminal(err)) + } + assert.Zero(t, countModelCacheBindingStatusUpdates(bart.Actions()), + "ensure must never transition Retiring back to Active") + }) + } +} + +func TestBeginRegularModelCacheBindingRetirementRejectsHelm(t *testing.T) { + req, _, backend, bart := boundModelCacheBindingFixture( + t, + nvcastorage.ModelCacheWorkflowHelm, + nvcav2beta1.ModelCacheBindingPhaseActive, + []nvcav2beta1.ModelCacheBindingRequestReference{{ + Namespace: "requests", + Name: "request", + UID: types.UID("request-uid"), + }}, + ) + + got, authorized, err := backend.beginRegularModelCacheBindingRetirement(t.Context(), req.DeepCopy()) + assert.Nil(t, got) + assert.False(t, authorized) + require.ErrorContains(t, err, "durable regular binding") + assert.True(t, nvcaerrors.IsTerminal(err)) + assert.Zero(t, countModelCacheBindingStatusUpdates(bart.Actions())) +} + +func TestReleaseModelCacheBindingReferenceIsExactAndIdempotent(t *testing.T) { + req, selection := durableModelCacheBindingRequest(t, nvcastorage.ModelCacheWorkflowRegular) + binding, err := nvcastorage.NewModelCacheBinding( + selection, req.Spec.NCAId, "model-cache-handle", "pod-instances") + require.NoError(t, err) + binding.UID = types.UID("binding-uid") + binding.Status.Phase = nvcav2beta1.ModelCacheBindingPhaseActive + binding.Status.RequestReferences = []nvcav2beta1.ModelCacheBindingRequestReference{ + {Namespace: req.Namespace, Name: req.Name, UID: req.UID}, + {Namespace: req.Namespace, Name: "other", UID: types.UID("other-uid")}, + } + req.Annotations[nvcastorage.ModelCacheStorageSelectionAnnotationKey] = + boundSelectionPayload(t, selection, binding) + backend, bart, _ := bindingTestBackend(req, binding) + + require.NoError(t, backend.releaseModelCacheBindingReference(t.Context(), req.DeepCopy())) + require.NoError(t, backend.releaseModelCacheBindingReference(t.Context(), req.DeepCopy())) + updated, err := bart.NvcaV2beta1().ModelCacheBindings(binding.Namespace). + Get(t.Context(), binding.Name, metav1.GetOptions{}) + require.NoError(t, err) + assert.False(t, nvcastorage.ModelCacheBindingHasRequestReference( + updated, req.Namespace, req.Name, req.UID)) + assert.True(t, nvcastorage.ModelCacheBindingHasRequestReference( + updated, req.Namespace, "other", types.UID("other-uid"))) +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/modelcache_cleanup_binding_test.go b/src/compute-plane-services/nvca/pkg/nvca/modelcache_cleanup_binding_test.go new file mode 100644 index 000000000..0e1a4be19 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/modelcache_cleanup_binding_test.go @@ -0,0 +1,1234 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nvca + +import ( + "encoding/base64" + "fmt" + "strings" + "testing" + + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/util/k8sutil" + "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + fakek8sclient "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + "sigs.k8s.io/yaml" + + nvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1" + nvcaerrors "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/errors" + nvcastorage "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/storage" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type regularCleanupTestFixture struct { + backend K8sComputeBackend + req *nvcav2beta1.ICMSRequest + binding *nvcav2beta1.ModelCacheBinding + rwPVC *corev1.PersistentVolumeClaim + job *batchv1.Job + pv *corev1.PersistentVolume +} + +func newRegularCleanupTestFixture(t *testing.T) *regularCleanupTestFixture { + t.Helper() + resolved := resolvedSelectionForSetup(t) + raw := persistedSelectionAnnotation( + t, nvcastorage.ModelCacheWorkflowRegular, nvcastorage.ModelCacheSelectionDurable, resolved) + backend := testContainerModelCacheBackend(fakek8sclient.NewSimpleClientset()) + backend.bk8s.k8sTimeConfig = (&k8sutil.TimeConfig{}).Complete() + req := &nvcav2beta1.ICMSRequest{ + ObjectMeta: metav1.ObjectMeta{Name: "request", Namespace: RequestsNamespace}, + } + binding := installActiveRegularModelCacheBinding(t, &backend, req, raw) + labels := map[string]string{nvcastorage.ModelCacheBindingUIDLabelKey: string(binding.UID)} + storageClassName := nvcastorage.DefaultModelCacheStorageClassName + rwPVC := &corev1.PersistentVolumeClaim{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "PersistentVolumeClaim"}, + ObjectMeta: metav1.ObjectMeta{ + Name: binding.Spec.Resources.PersistentVolumeClaimNames[0], + Namespace: binding.Spec.Resources.WriterNamespace, + UID: types.UID("rw-pvc-uid"), + ResourceVersion: "rw-pvc-rv", + Labels: labels, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + StorageClassName: &storageClassName, + VolumeName: "cache-pv", + }, + } + job := &batchv1.Job{ + TypeMeta: metav1.TypeMeta{APIVersion: "batch/v1", Kind: "Job"}, + ObjectMeta: metav1.ObjectMeta{ + Name: binding.Spec.Resources.JobNames[0], + Namespace: binding.Spec.Resources.WriterNamespace, + UID: types.UID("writer-job-uid"), + ResourceVersion: "writer-job-rv", + Labels: labels, + }, + Spec: batchv1.JobSpec{Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + Spec: corev1.PodSpec{RestartPolicy: corev1.RestartPolicyNever}, + }}, + } + pv := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cache-pv", + UID: types.UID("cache-pv-uid"), + ResourceVersion: "cache-pv-rv", + Labels: labels, + }, + Spec: corev1.PersistentVolumeSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + StorageClassName: storageClassName, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + Driver: binding.Spec.Decision.Provisioner, + VolumeHandle: "cache-volume-handle", + }, + }, + ClaimRef: &corev1.ObjectReference{ + Namespace: rwPVC.Namespace, + Name: rwPVC.Name, + UID: rwPVC.UID, + }, + }, + } + return ®ularCleanupTestFixture{ + backend: backend, + req: req, + binding: binding, + rwPVC: rwPVC, + job: job, + pv: pv, + } +} + +func newRWXRegularCleanupTestFixture(t *testing.T) *regularCleanupTestFixture { + t.Helper() + runtimeFixture := newRWXReadOnlyRuntimeFixture(t, "weka", "csi.weka.io") + runtimeFixture.backend.bk8s.k8sTimeConfig = (&k8sutil.TimeConfig{}).Complete() + rwPVC := runtimeFixture.boundPVC(false) + job := runtimeFixture.writerJob() + pv := runtimeFixture.boundPV(rwPVC) + pv.UID = types.UID("cache-pv-uid") + pv.ResourceVersion = "cache-pv-rv" + return ®ularCleanupTestFixture{ + backend: runtimeFixture.backend, + req: runtimeFixture.req, + binding: runtimeFixture.binding, + rwPVC: rwPVC, + job: job, + pv: pv, + } +} + +func (f *regularCleanupTestFixture) useK8sObjects(objects ...runtime.Object) *fakek8sclient.Clientset { + k8sClient := fakek8sclient.NewSimpleClientset(objects...) + f.backend.clients.K8s = k8sClient + return k8sClient +} + +func encodeCleanupObject(t *testing.T, obj runtime.Object) string { + t.Helper() + raw, err := yaml.Marshal(obj) + require.NoError(t, err) + return base64.StdEncoding.EncodeToString(raw) +} + +func setCleanupArtifacts( + t *testing.T, + req *nvcav2beta1.ICMSRequest, + rwPVC *corev1.PersistentVolumeClaim, + job *batchv1.Job, +) { + t.Helper() + req.Spec.CreationMsgInfo.LaunchArtifacts = function.LaunchArtifacts{ + {Type: function.LaunchArtifactTypeBlockDevice, Specification: encodeCleanupObject(t, rwPVC)}, + {Type: function.LaunchArtifactTypeInitCacheJob, Specification: encodeCleanupObject(t, job)}, + } +} + +func requireDeleteIdentity( + t *testing.T, + actions []k8stesting.Action, + resource string, + name string, + wantUID types.UID, + wantResourceVersion string, +) { + t.Helper() + for _, action := range actions { + if action.GetVerb() != "delete" || action.GetResource().Resource != resource { + continue + } + deleteAction := action.(k8stesting.DeleteAction) + if deleteAction.GetName() != name { + continue + } + preconditions := deleteAction.GetDeleteOptions().Preconditions + require.NotNil(t, preconditions) + require.NotNil(t, preconditions.UID) + assert.Equal(t, wantUID, *preconditions.UID) + require.NotNil(t, preconditions.ResourceVersion) + assert.Equal(t, wantResourceVersion, *preconditions.ResourceVersion) + return + } + t.Fatalf("delete action for %s %s not found", resource, name) +} + +func TestCleanupModelCachingResourcesRejectsUnownedTargetsBeforeWrites(t *testing.T) { + tests := []struct { + name string + mutate func(*regularCleanupTestFixture) + wantErr string + }{ + { + name: "missing PVC binding label", + mutate: func(f *regularCleanupTestFixture) { + f.rwPVC.Labels = nil + }, + wantErr: `has binding UID "", want "binding-uid"`, + }, + { + name: "wrong Job binding label", + mutate: func(f *regularCleanupTestFixture) { + f.job.Labels = map[string]string{nvcastorage.ModelCacheBindingUIDLabelKey: "other-binding"} + }, + wantErr: `has binding UID "other-binding", want "binding-uid"`, + }, + { + name: "missing PV binding label", + mutate: func(f *regularCleanupTestFixture) { + f.pv.Labels = nil + }, + wantErr: `has binding UID "", want "binding-uid"`, + }, + { + name: "stale PV claim owner UID", + mutate: func(f *regularCleanupTestFixture) { + f.pv.Spec.ClaimRef.UID = types.UID("stale-pvc-uid") + }, + wantErr: `claimRef UID "stale-pvc-uid" does not match PVC UID "rw-pvc-uid"`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + tt.mutate(fixture) + k8sClient := fixture.useK8sObjects(fixture.rwPVC, fixture.job, fixture.pv) + + err := fixture.backend.CleanupModelCachingResources( + newTestContext(), fixture.req, fixture.rwPVC.DeepCopy(), fixture.job.Name) + require.ErrorContains(t, err, tt.wantErr) + assertNoKubernetesWrites(t, k8sClient.Actions()) + }) + } +} + +func TestCleanupModelCachingResourcesSkipsSharedBinding(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + stored, err := fixture.backend.clients.BART.NvcaV2beta1().ModelCacheBindings(fixture.binding.Namespace). + Get(t.Context(), fixture.binding.Name, metav1.GetOptions{}) + require.NoError(t, err) + stored.Status.RequestReferences = append(stored.Status.RequestReferences, + nvcav2beta1.ModelCacheBindingRequestReference{ + Namespace: fixture.req.Namespace, + Name: "other-request", + UID: types.UID("other-request-uid"), + }) + _, err = fixture.backend.clients.BART.NvcaV2beta1().ModelCacheBindings(stored.Namespace). + UpdateStatus(t.Context(), stored, metav1.UpdateOptions{}) + require.NoError(t, err) + k8sClient := fixture.useK8sObjects(fixture.rwPVC, fixture.job, fixture.pv) + + require.NoError(t, fixture.backend.CleanupModelCachingResources( + newTestContext(), fixture.req, fixture.rwPVC.DeepCopy(), fixture.job.Name)) + assert.Empty(t, k8sClient.Actions(), "another exact request reference must preserve the shared writer") +} + +func TestCleanupModelCachingResourcesResumesExactRetiringBinding(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + stored, err := fixture.backend.clients.BART.NvcaV2beta1().ModelCacheBindings(fixture.binding.Namespace). + Get(t.Context(), fixture.binding.Name, metav1.GetOptions{}) + require.NoError(t, err) + stored.Status.Phase = nvcav2beta1.ModelCacheBindingPhaseRetiring + _, err = fixture.backend.clients.BART.NvcaV2beta1().ModelCacheBindings(stored.Namespace). + UpdateStatus(t.Context(), stored, metav1.UpdateOptions{}) + require.NoError(t, err) + k8sClient := fixture.useK8sObjects(fixture.rwPVC, fixture.job, fixture.pv) + + require.NoError(t, fixture.backend.CleanupModelCachingResources( + newTestContext(), fixture.req, fixture.rwPVC.DeepCopy(), fixture.job.Name)) + _, err = k8sClient.CoreV1().PersistentVolumeClaims(fixture.rwPVC.Namespace). + Get(t.Context(), fixture.rwPVC.Name, metav1.GetOptions{}) + assert.True(t, apierrors.IsNotFound(err)) + _, err = k8sClient.BatchV1().Jobs(fixture.job.Namespace). + Get(t.Context(), fixture.job.Name, metav1.GetOptions{}) + assert.True(t, apierrors.IsNotFound(err)) +} + +func TestCleanupModelCachingResourcesResumesAfterPVPolicyUpdate(t *testing.T) { + tests := []struct { + name string + fixture func(*testing.T) *regularCleanupTestFixture + }{ + {name: "NVMesh", fixture: newRegularCleanupTestFixture}, + {name: "rwxReadOnly", fixture: newRWXRegularCleanupTestFixture}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fixture := tt.fixture(t) + k8sClient := fixture.useK8sObjects(fixture.rwPVC, fixture.job, fixture.pv) + pvcDeleteAttempts := 0 + k8sClient.Fake.PrependReactor( + "delete", "persistentvolumeclaims", + func(k8stesting.Action) (bool, runtime.Object, error) { + pvcDeleteAttempts++ + if pvcDeleteAttempts == 1 { + return true, nil, apierrors.NewServiceUnavailable( + "transient PVC delete failure") + } + return false, nil, nil + }) + + err := fixture.backend.CleanupModelCachingResources( + newTestContext(), fixture.req, fixture.rwPVC.DeepCopy(), fixture.job.Name) + require.Error(t, err) + assert.True(t, apierrors.IsServiceUnavailable(err)) + + updatedPV, getErr := k8sClient.CoreV1().PersistentVolumes(). + Get(t.Context(), fixture.pv.Name, metav1.GetOptions{}) + require.NoError(t, getErr) + assert.Equal(t, corev1.PersistentVolumeReclaimDelete, + updatedPV.Spec.PersistentVolumeReclaimPolicy) + _, getErr = k8sClient.CoreV1().PersistentVolumeClaims(fixture.rwPVC.Namespace). + Get(t.Context(), fixture.rwPVC.Name, metav1.GetOptions{}) + require.NoError(t, getErr, "the first transient delete must leave the PVC for retry") + + require.NoError(t, fixture.backend.CleanupModelCachingResources( + newTestContext(), fixture.req, fixture.rwPVC.DeepCopy(), fixture.job.Name)) + assert.Equal(t, 2, pvcDeleteAttempts) + _, getErr = k8sClient.CoreV1().PersistentVolumeClaims(fixture.rwPVC.Namespace). + Get(t.Context(), fixture.rwPVC.Name, metav1.GetOptions{}) + assert.True(t, apierrors.IsNotFound(getErr)) + }) + } +} + +func TestCleanupModelCachingResourcesRejectsIdentityDriftAfterPVPolicyUpdate(t *testing.T) { + fixture := newRWXRegularCleanupTestFixture(t) + stored, err := fixture.backend.clients.BART.NvcaV2beta1(). + ModelCacheBindings(fixture.binding.Namespace). + Get(t.Context(), fixture.binding.Name, metav1.GetOptions{}) + require.NoError(t, err) + stored.Status.Phase = nvcav2beta1.ModelCacheBindingPhaseRetiring + _, err = fixture.backend.clients.BART.NvcaV2beta1(). + ModelCacheBindings(stored.Namespace). + UpdateStatus(t.Context(), stored, metav1.UpdateOptions{}) + require.NoError(t, err) + fixture.pv.Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete + fixture.pv.Spec.CSI.Driver = "foreign.csi.example.com" + k8sClient := fixture.useK8sObjects(fixture.rwPVC, fixture.job, fixture.pv) + + err = fixture.backend.CleanupModelCachingResources( + newTestContext(), fixture.req, fixture.rwPVC.DeepCopy(), fixture.job.Name) + require.ErrorContains(t, err, "CSI driver") + assertNoKubernetesWrites(t, k8sClient.Actions()) + _, getErr := k8sClient.CoreV1().PersistentVolumeClaims(fixture.rwPVC.Namespace). + Get(t.Context(), fixture.rwPVC.Name, metav1.GetOptions{}) + require.NoError(t, getErr) + _, getErr = k8sClient.BatchV1().Jobs(fixture.job.Namespace). + Get(t.Context(), fixture.job.Name, metav1.GetOptions{}) + require.NoError(t, getErr) +} + +func TestSetupModelCachingForRequestResumesRetiringCleanupWithoutServingReaders(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + stored, err := fixture.backend.clients.BART.NvcaV2beta1().ModelCacheBindings(fixture.binding.Namespace). + Get(t.Context(), fixture.binding.Name, metav1.GetOptions{}) + require.NoError(t, err) + stored.Status.Phase = nvcav2beta1.ModelCacheBindingPhaseRetiring + _, err = fixture.backend.clients.BART.NvcaV2beta1().ModelCacheBindings(stored.Namespace). + UpdateStatus(t.Context(), stored, metav1.UpdateOptions{}) + require.NoError(t, err) + k8sClient := fixture.useK8sObjects(fixture.rwPVC, fixture.job, fixture.pv) + + state, roPVCName, err := fixture.backend.SetupModelCachingForRequest( + newTestContext(), fixture.rwPVC.DeepCopy(), fixture.job.DeepCopy(), + fixture.req, false, noOpRegularModelCacheMutation) + assert.Equal(t, ModelCachingFailed, state) + assert.Empty(t, roPVCName) + require.ErrorIs(t, err, errRegularModelCacheBindingRetiring) + assert.True(t, nvcaerrors.IsTerminal(err)) + _, getErr := k8sClient.CoreV1().PersistentVolumeClaims(fixture.rwPVC.Namespace). + Get(t.Context(), fixture.rwPVC.Name, metav1.GetOptions{}) + assert.True(t, apierrors.IsNotFound(getErr)) + _, getErr = k8sClient.BatchV1().Jobs(fixture.job.Namespace). + Get(t.Context(), fixture.job.Name, metav1.GetOptions{}) + assert.True(t, apierrors.IsNotFound(getErr)) +} + +func TestCleanupModelCachingResourcesDeletesOnlyExactOwner(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + k8sClient := fixture.useK8sObjects(fixture.rwPVC, fixture.job, fixture.pv) + + require.NoError(t, fixture.backend.CleanupModelCachingResources( + newTestContext(), fixture.req, fixture.rwPVC.DeepCopy(), fixture.job.Name)) + stored, err := fixture.backend.clients.BART.NvcaV2beta1().ModelCacheBindings(fixture.binding.Namespace). + Get(t.Context(), fixture.binding.Name, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, nvcav2beta1.ModelCacheBindingPhaseRetiring, stored.Status.Phase) + + _, err = k8sClient.CoreV1().PersistentVolumeClaims(fixture.rwPVC.Namespace). + Get(t.Context(), fixture.rwPVC.Name, metav1.GetOptions{}) + assert.True(t, apierrors.IsNotFound(err)) + _, err = k8sClient.BatchV1().Jobs(fixture.job.Namespace). + Get(t.Context(), fixture.job.Name, metav1.GetOptions{}) + assert.True(t, apierrors.IsNotFound(err)) + pv, err := k8sClient.CoreV1().PersistentVolumes().Get(t.Context(), fixture.pv.Name, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, corev1.PersistentVolumeReclaimDelete, pv.Spec.PersistentVolumeReclaimPolicy) + requireDeleteIdentity(t, k8sClient.Actions(), "jobs", fixture.job.Name, + fixture.job.UID, fixture.job.ResourceVersion) + requireDeleteIdentity(t, k8sClient.Actions(), "persistentvolumeclaims", fixture.rwPVC.Name, + fixture.rwPVC.UID, fixture.rwPVC.ResourceVersion) +} + +func TestCleanupModelCachingSetupArtifactsUsesPersistedBindingAcrossGateDrift(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + fixture.backend.bk8s.cachingSupportEnabled = false + setCleanupArtifacts(t, fixture.req, fixture.rwPVC, fixture.job) + k8sClient := fixture.useK8sObjects(fixture.rwPVC, fixture.job, fixture.pv) + + require.NoError(t, fixture.backend.CleanupModelCachingSetupArtifacts(newTestContext(), fixture.req)) + updatedPV, err := k8sClient.CoreV1().PersistentVolumes().Get( + t.Context(), fixture.pv.Name, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, corev1.PersistentVolumeReclaimDelete, updatedPV.Spec.PersistentVolumeReclaimPolicy) + requireDeleteIdentity(t, k8sClient.Actions(), "jobs", fixture.job.Name, + fixture.job.UID, fixture.job.ResourceVersion) + requireDeleteIdentity(t, k8sClient.Actions(), "persistentvolumeclaims", fixture.rwPVC.Name, + fixture.rwPVC.UID, fixture.rwPVC.ResourceVersion) +} + +func TestCleanupModelCachingSetupArtifactsRejectsNameOutsideBindingIntent(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + wrongPVC := fixture.rwPVC.DeepCopy() + wrongPVC.Name = "rw-pvc-wrong-cache" + setCleanupArtifacts(t, fixture.req, wrongPVC, fixture.job) + k8sClient := fixture.useK8sObjects(fixture.rwPVC, fixture.job) + + err := fixture.backend.CleanupModelCachingSetupArtifacts(newTestContext(), fixture.req) + require.ErrorContains(t, err, "outside binding resource intent") + assertNoKubernetesWrites(t, k8sClient.Actions()) +} + +func TestCleanupModelCachingResourcesLegacyRequestRemainsCompatible(t *testing.T) { + rwPVC := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: "rw-pvc-legacy", Namespace: RequestsNamespace}, + Spec: corev1.PersistentVolumeClaimSpec{VolumeName: "legacy-pv"}, + } + job := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "writer-job-legacy", Namespace: RequestsNamespace}} + pv := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{Name: "legacy-pv"}, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + }, + } + k8sClient := fakek8sclient.NewSimpleClientset(rwPVC, job, pv) + backend := testContainerModelCacheBackend(k8sClient) + req := &nvcav2beta1.ICMSRequest{ObjectMeta: metav1.ObjectMeta{ + Name: "legacy-request", Namespace: RequestsNamespace, + }} + + require.NoError(t, backend.CleanupModelCachingResources( + newTestContext(), req, rwPVC.DeepCopy(), job.Name)) + updatedPV, err := k8sClient.CoreV1().PersistentVolumes().Get(t.Context(), pv.Name, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, corev1.PersistentVolumeReclaimDelete, updatedPV.Spec.PersistentVolumeReclaimPolicy) + _, err = k8sClient.CoreV1().PersistentVolumeClaims(rwPVC.Namespace). + Get(t.Context(), rwPVC.Name, metav1.GetOptions{}) + assert.True(t, apierrors.IsNotFound(err)) +} + +func TestComputeCleanupCacheReferencesLeavesBindingOwnedPVCToBindingLifecycle(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + k8sClient := fixture.useK8sObjects(fixture.rwPVC, fixture.pv) + + require.NoError(t, fixture.backend.ComputeCleanupCacheReferences( + newTestContext(), []string{fixture.rwPVC.Name})) + assertNoKubernetesWrites(t, k8sClient.Actions()) + gotPVC, err := k8sClient.CoreV1().PersistentVolumeClaims(fixture.rwPVC.Namespace). + Get(t.Context(), fixture.rwPVC.Name, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, fixture.rwPVC.UID, gotPVC.UID) + gotPV, err := k8sClient.CoreV1().PersistentVolumes().Get(t.Context(), fixture.pv.Name, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, corev1.PersistentVolumeReclaimRetain, gotPV.Spec.PersistentVolumeReclaimPolicy) +} + +func TestMakePVLabelSelectorUsesPersistedBindingUID(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + + selector, err := makePVLabelSelectorForCacheRequest(fixture.req) + require.NoError(t, err) + assert.Equal(t, fmt.Sprintf("%s=%s", nvcastorage.ModelCacheBindingUIDLabelKey, fixture.binding.UID), selector) + + legacy := &nvcav2beta1.ICMSRequest{Spec: nvcav2beta1.ICMSRequestSpec{ + FunctionDetails: function.Details{FunctionVersionID: "legacy-version"}, + }} + selector, err = makePVLabelSelectorForCacheRequest(legacy) + require.NoError(t, err) + assert.Equal(t, fmt.Sprintf("%s=legacy-version", fnVersionIDLabelString), selector) +} + +func TestSetupInitCacheJobBlockDeviceValidatesClassOnlyBeforeFirstWriterCreate(t *testing.T) { + t.Run("missing writer and drifted class fails terminal before create", func(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + drifted := selectionStorageClass() + drifted.Parameters["pool"] = "drifted" + k8sClient := fixture.useK8sObjects(drifted) + className := nvcastorage.DefaultModelCacheStorageClassName + fixture.rwPVC.Spec.StorageClassName = &className + fixture.job.Spec.Template.Labels = map[string]string{ + nvcastorage.ModelCacheBindingUIDLabelKey: string(fixture.binding.UID), + } + + err := fixture.backend.SetupInitCacheJobBlockDevice( + newTestContext(), fixture.rwPVC.DeepCopy(), fixture.job.DeepCopy(), fixture.req) + require.ErrorContains(t, err, "configuration digest changed") + assert.True(t, nvcaerrors.IsTerminal(err)) + assertNoKubernetesWrites(t, k8sClient.Actions()) + }) + + t.Run("missing writer and missing class fails terminal before create", func(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + k8sClient := fixture.useK8sObjects() + className := nvcastorage.DefaultModelCacheStorageClassName + fixture.rwPVC.Spec.StorageClassName = &className + fixture.job.Spec.Template.Labels = map[string]string{ + nvcastorage.ModelCacheBindingUIDLabelKey: string(fixture.binding.UID), + } + + err := fixture.backend.SetupInitCacheJobBlockDevice( + newTestContext(), fixture.rwPVC.DeepCopy(), fixture.job.DeepCopy(), fixture.req) + require.ErrorContains(t, err, "get selected model cache StorageClass") + assert.True(t, apierrors.IsNotFound(err)) + assert.True(t, nvcaerrors.IsTerminal(err)) + assertNoKubernetesWrites(t, k8sClient.Actions()) + }) + + t.Run("existing exact writer survives live class deletion", func(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + className := nvcastorage.DefaultModelCacheStorageClassName + fixture.rwPVC.Spec.StorageClassName = &className + fixture.job.Spec.Template.Labels = map[string]string{ + nvcastorage.ModelCacheBindingUIDLabelKey: string(fixture.binding.UID), + } + k8sClient := fixture.useK8sObjects(fixture.rwPVC) + + require.NoError(t, fixture.backend.SetupInitCacheJobBlockDevice( + newTestContext(), fixture.rwPVC.DeepCopy(), fixture.job.DeepCopy(), fixture.req)) + for _, action := range k8sClient.Actions() { + assert.NotEqual(t, "storageclasses", action.GetResource().Resource, + "an existing exact writer must not re-read the live StorageClass") + } + _, err := k8sClient.BatchV1().Jobs(fixture.job.Namespace). + Get(t.Context(), fixture.job.Name, metav1.GetOptions{}) + require.NoError(t, err) + }) + + t.Run("encrypted derived writer does not read base class", func(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + selection, err := nvcastorage.ParsePersistedModelCacheStorageSelection( + fixture.req.Annotations[nvcastorage.ModelCacheStorageSelectionAnnotationKey]) + require.NoError(t, err) + selection.EncryptionRequired = true + selection.BindingName = "" + selection.BindingUID = "" + raw, err := selection.Marshal() + require.NoError(t, err) + fixture.binding = installActiveRegularModelCacheBinding(t, &fixture.backend, fixture.req, raw) + derivedClassName := fixture.binding.Spec.Resources.StorageClassNames[0] + fixture.rwPVC.Spec.StorageClassName = &derivedClassName + fixture.job.Spec.Template.Labels = map[string]string{ + nvcastorage.ModelCacheBindingUIDLabelKey: string(fixture.binding.UID), + } + k8sClient := fixture.useK8sObjects() + + require.NoError(t, fixture.backend.SetupInitCacheJobBlockDevice( + newTestContext(), fixture.rwPVC.DeepCopy(), fixture.job.DeepCopy(), fixture.req)) + for _, action := range k8sClient.Actions() { + assert.NotEqual(t, "storageclasses", action.GetResource().Resource, + "an encrypted derived writer must not read the base StorageClass") + } + }) +} + +func TestSetupPVCForReadersRejectsStaleSameNamePVCUID(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + fixture.req.Spec.FunctionDetails.FunctionVersionID = "function-version" + fixture.pv.Spec.ClaimRef.UID = types.UID("stale-pvc-uid") + k8sClient := fixture.useK8sObjects(fixture.rwPVC, fixture.pv) + + phase, err := fixture.backend.SetupPVCForReaders( + newTestContext(), fixture.rwPVC.DeepCopy(), fixture.job.Name, fixture.req, nil) + assert.Equal(t, ROPVCSetupFailed, phase) + require.ErrorContains(t, err, `claimRef UID "stale-pvc-uid" does not match PVC UID "rw-pvc-uid"`) + assertNoKubernetesWrites(t, k8sClient.Actions()) +} +func boundReaderForRegularCleanupFixture( + fixture *regularCleanupTestFixture, +) *corev1.PersistentVolumeClaim { + reader := fixture.rwPVC.DeepCopy() + reader.Name = ROPVCPrefix + strings.TrimPrefix(reader.Name, RWPVCPrefix) + reader.UID = types.UID("ro-pvc-uid") + reader.ResourceVersion = "ro-pvc-rv" + reader.Spec.AccessModes = ROAccessMode + reader.Status.Phase = corev1.ClaimBound + return reader +} + +func boundReaderPVForRegularCleanupFixture( + fixture *regularCleanupTestFixture, + reader *corev1.PersistentVolumeClaim, +) *corev1.PersistentVolume { + pv := fixture.pv.DeepCopy() + pv.Spec.AccessModes = ROAccessMode + pv.Spec.MountOptions = append([]string(nil), fixture.binding.Spec.Decision.RequiredMountOptions...) + pv.Spec.ClaimRef = &corev1.ObjectReference{ + Namespace: reader.Namespace, + Name: reader.Name, + UID: reader.UID, + } + return pv +} + +func noOpRegularModelCacheMutation(client.Object) {} + +func TestRegularModelCacheReaderMountOptions(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + configured := []string{"rw", "noatime", "nouuid", "recovery"} + + got, err := regularModelCacheReaderMountOptions(fixture.binding, configured) + require.NoError(t, err) + assert.Equal(t, []string{"ro", "norecovery", "nouuid", "noatime"}, got) + assert.Equal(t, []string{"rw", "noatime", "nouuid", "recovery"}, configured) + assert.Equal(t, []string{"ro", "norecovery", "nouuid"}, + fixture.binding.Spec.Decision.RequiredMountOptions) + + wrongTransition := fixture.binding.DeepCopy() + wrongTransition.Spec.Decision.Transition = nvcastorage.ModelCacheTransitionRWXReadOnly + _, err = regularModelCacheReaderMountOptions(wrongTransition, configured) + require.ErrorContains(t, err, "does not create a read-only reader PV") +} + +func TestSetupPVCForReadersUsesPersistedReaderMountOptions(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + fixture.req.Spec.FunctionDetails.FunctionVersionID = "function-version" + fixture.pv.Labels[fnVersionIDLabelString] = "function-version" + fixture.backend.bk8s.csiVolumeMountOptions = []string{"rw", "noatime", "nouuid", "recovery"} + k8sClient := fixture.useK8sObjects(fixture.rwPVC, fixture.job, fixture.pv) + + previousSkip := skipVolumeDetachCheck + skipVolumeDetachCheck = true + t.Cleanup(func() { skipVolumeDetachCheck = previousSkip }) + + phase, err := fixture.backend.SetupPVCForReaders( + newTestContext(), fixture.rwPVC.DeepCopy(), fixture.job.Name, + fixture.req, noOpRegularModelCacheMutation) + require.NoError(t, err) + assert.Equal(t, ROPVCSetupCompleted, phase) + + got, err := k8sClient.CoreV1().PersistentVolumes(). + Get(t.Context(), fixture.pv.Name, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, []string{"ro", "norecovery", "nouuid", "noatime"}, got.Spec.MountOptions) +} + +func TestSetupPVCForReadersRepairsMountOptionsWithoutChangingReaderIdentity(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + fixture.req.Spec.FunctionDetails.FunctionVersionID = "function-version" + reader := boundReaderForRegularCleanupFixture(fixture) + readerPV := boundReaderPVForRegularCleanupFixture(fixture, reader) + readerPV.Labels[fnVersionIDLabelString] = "function-version" + readerPV.Spec.MountOptions = []string{"rw", "recovery", "uuid"} + fixture.backend.bk8s.csiVolumeMountOptions = []string{"noatime"} + k8sClient := fixture.useK8sObjects(reader, readerPV) + + previousSkip := skipVolumeDetachCheck + skipVolumeDetachCheck = true + t.Cleanup(func() { skipVolumeDetachCheck = previousSkip }) + + phase, err := fixture.backend.SetupPVCForReaders( + newTestContext(), fixture.rwPVC.DeepCopy(), fixture.job.Name, + fixture.req, noOpRegularModelCacheMutation) + require.NoError(t, err) + assert.Equal(t, ROPVCSetupCompleted, phase) + + got, err := k8sClient.CoreV1().PersistentVolumes(). + Get(t.Context(), readerPV.Name, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, reader.UID, got.Spec.ClaimRef.UID) + assert.Equal(t, reader.Name, got.Spec.ClaimRef.Name) + assert.Equal(t, []string{"ro", "norecovery", "nouuid", "noatime"}, got.Spec.MountOptions) +} + +func TestSetupInitCacheJobBlockDevicePreservesTransientStorageClassError(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + className := nvcastorage.DefaultModelCacheStorageClassName + fixture.rwPVC.Spec.StorageClassName = &className + fixture.job.Spec.Template.Labels = map[string]string{ + nvcastorage.ModelCacheBindingUIDLabelKey: string(fixture.binding.UID), + } + k8sClient := fixture.useK8sObjects() + k8sClient.Fake.PrependReactor( + "get", "storageclasses", + func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewServiceUnavailable("storage API unavailable") + }) + + err := fixture.backend.SetupInitCacheJobBlockDevice( + newTestContext(), fixture.rwPVC.DeepCopy(), fixture.job.DeepCopy(), fixture.req) + require.Error(t, err) + assert.True(t, apierrors.IsServiceUnavailable(err)) + assert.False(t, nvcaerrors.IsTerminal(err)) + assertNoKubernetesWrites(t, k8sClient.Actions()) +} + +func TestSetupModelCachingForRequestPropagatesTransientReads(t *testing.T) { + t.Run("reader PVC Get", func(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + k8sClient := fixture.useK8sObjects() + roPVCName := ROPVCPrefix + strings.TrimPrefix(fixture.rwPVC.Name, RWPVCPrefix) + k8sClient.Fake.PrependReactor( + "get", "persistentvolumeclaims", + func(action k8stesting.Action) (bool, runtime.Object, error) { + if action.(k8stesting.GetAction).GetName() != roPVCName { + return false, nil, nil + } + return true, nil, apierrors.NewServiceUnavailable("PVC API unavailable") + }) + + state, _, err := fixture.backend.SetupModelCachingForRequest( + newTestContext(), fixture.rwPVC.DeepCopy(), fixture.job.DeepCopy(), + fixture.req, false, noOpRegularModelCacheMutation) + assert.Equal(t, ModelCachingInProgress, state) + require.Error(t, err) + assert.True(t, apierrors.IsServiceUnavailable(err)) + assert.False(t, nvcaerrors.IsTerminal(err)) + assertNoKubernetesWrites(t, k8sClient.Actions()) + }) + + t.Run("init Job Get", func(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + k8sClient := fixture.useK8sObjects() + k8sClient.Fake.PrependReactor( + "get", "jobs", + func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewServiceUnavailable("Job API unavailable") + }) + + state, _, err := fixture.backend.SetupModelCachingForRequest( + newTestContext(), fixture.rwPVC.DeepCopy(), fixture.job.DeepCopy(), + fixture.req, false, noOpRegularModelCacheMutation) + assert.Equal(t, ModelCachingInProgress, state) + require.Error(t, err) + assert.True(t, apierrors.IsServiceUnavailable(err)) + assert.False(t, nvcaerrors.IsTerminal(err)) + assertNoKubernetesWrites(t, k8sClient.Actions()) + }) + + t.Run("PV List", func(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + k8sClient := fixture.useK8sObjects() + k8sClient.Fake.PrependReactor( + "list", "persistentvolumes", + func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewServiceUnavailable("PV API unavailable") + }) + + state, _, err := fixture.backend.SetupModelCachingForRequest( + newTestContext(), fixture.rwPVC.DeepCopy(), fixture.job.DeepCopy(), + fixture.req, false, noOpRegularModelCacheMutation) + assert.Equal(t, ModelCachingInProgress, state) + require.Error(t, err) + assert.True(t, apierrors.IsServiceUnavailable(err)) + assert.False(t, nvcaerrors.IsTerminal(err)) + assertNoKubernetesWrites(t, k8sClient.Actions()) + }) + + t.Run("PV Get during reader transition", func(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + reader := boundReaderForRegularCleanupFixture(fixture) + k8sClient := fixture.useK8sObjects(reader) + k8sClient.Fake.PrependReactor( + "get", "persistentvolumes", + func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewServiceUnavailable("PV API unavailable") + }) + + state, _, err := fixture.backend.SetupModelCachingForRequest( + newTestContext(), fixture.rwPVC.DeepCopy(), fixture.job.DeepCopy(), + fixture.req, false, noOpRegularModelCacheMutation) + assert.Equal(t, ModelCachingInProgress, state) + require.Error(t, err) + assert.True(t, apierrors.IsServiceUnavailable(err)) + assert.False(t, nvcaerrors.IsTerminal(err)) + assertNoKubernetesWrites(t, k8sClient.Actions()) + }) + + t.Run("cleanup PV Get", func(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + fixture.rwPVC.Status.Phase = corev1.ClaimBound + fixture.job.Status.Failed = 7 + k8sClient := fixture.useK8sObjects(fixture.rwPVC, fixture.job, fixture.pv) + k8sClient.Fake.PrependReactor( + "get", "persistentvolumes", + func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewServiceUnavailable("PV API unavailable") + }) + + state, _, err := fixture.backend.SetupModelCachingForRequest( + newTestContext(), fixture.rwPVC.DeepCopy(), fixture.job.DeepCopy(), + fixture.req, false, noOpRegularModelCacheMutation) + assert.Equal(t, ModelCachingInProgress, state) + require.Error(t, err) + assert.True(t, apierrors.IsServiceUnavailable(err)) + assert.False(t, nvcaerrors.IsTerminal(err)) + assertNoKubernetesWrites(t, k8sClient.Actions()) + }) + + t.Run("encryption API", func(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + k8sClient := fixture.useK8sObjects() + k8sClient.Fake.PrependReactor( + "get", "secrets", + func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewServiceUnavailable("Secret API unavailable") + }) + + state, _, err := fixture.backend.SetupModelCachingForRequest( + newTestContext(), fixture.rwPVC.DeepCopy(), fixture.job.DeepCopy(), + fixture.req, true, noOpRegularModelCacheMutation) + assert.Equal(t, ModelCachingInProgress, state) + require.Error(t, err) + assert.True(t, apierrors.IsServiceUnavailable(err)) + assert.False(t, nvcaerrors.IsTerminal(err)) + assertNoKubernetesWrites(t, k8sClient.Actions()) + }) + + t.Run("Forbidden reader PVC Get remains a reconcile error", func(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + k8sClient := fixture.useK8sObjects() + roPVCName := ROPVCPrefix + strings.TrimPrefix(fixture.rwPVC.Name, RWPVCPrefix) + k8sClient.Fake.PrependReactor( + "get", "persistentvolumeclaims", + func(action k8stesting.Action) (bool, runtime.Object, error) { + if action.(k8stesting.GetAction).GetName() != roPVCName { + return false, nil, nil + } + return true, nil, apierrors.NewForbidden( + corev1.Resource("persistentvolumeclaims"), roPVCName, fmt.Errorf("forbidden")) + }) + + state, _, err := fixture.backend.SetupModelCachingForRequest( + newTestContext(), fixture.rwPVC.DeepCopy(), fixture.job.DeepCopy(), + fixture.req, false, noOpRegularModelCacheMutation) + assert.Equal(t, ModelCachingInProgress, state) + require.Error(t, err) + assert.True(t, apierrors.IsForbidden(err)) + assert.False(t, nvcaerrors.IsTerminal(err)) + assertNoKubernetesWrites(t, k8sClient.Actions()) + }) +} + +func TestSetupPVCForReadersPreservesTransientWriterGet(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + k8sClient := fixture.useK8sObjects() + k8sClient.Fake.PrependReactor( + "get", "persistentvolumeclaims", + func(action k8stesting.Action) (bool, runtime.Object, error) { + if action.(k8stesting.GetAction).GetName() != fixture.rwPVC.Name { + return false, nil, nil + } + return true, nil, apierrors.NewServiceUnavailable("PVC API unavailable") + }) + + phase, err := fixture.backend.SetupPVCForReaders( + newTestContext(), fixture.rwPVC.DeepCopy(), fixture.job.Name, + fixture.req, noOpRegularModelCacheMutation) + assert.Equal(t, ROPVCSetupQueryFailed, phase) + require.Error(t, err) + assert.True(t, apierrors.IsServiceUnavailable(err)) + assertNoKubernetesWrites(t, k8sClient.Actions()) +} + +func TestSetupPVCForReadersRejectsUnlabeledPVWhenWriterIsAbsent(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + fixture.req.Spec.FunctionDetails.FunctionVersionID = "function-version" + unlabeledPV := fixture.pv.DeepCopy() + unlabeledPV.Labels = map[string]string{fnVersionIDLabelString: "function-version"} + listedPV := fixture.pv.DeepCopy() + k8sClient := fixture.useK8sObjects(listedPV) + k8sClient.Fake.PrependReactor( + "get", "persistentvolumes", + func(action k8stesting.Action) (bool, runtime.Object, error) { + if action.(k8stesting.GetAction).GetName() != unlabeledPV.Name { + return false, nil, nil + } + return true, unlabeledPV, nil + }) + + phase, err := fixture.backend.SetupPVCForReaders( + newTestContext(), fixture.rwPVC.DeepCopy(), fixture.job.Name, + fixture.req, noOpRegularModelCacheMutation) + assert.Equal(t, ROPVCSetupFailed, phase) + require.ErrorContains(t, err, "writer PVC is absent") + assertNoKubernetesWrites(t, k8sClient.Actions()) +} + +func TestSetupModelCachingForRequestRejectsForeignTransitionArtifacts(t *testing.T) { + t.Run("foreign same-name Job", func(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + reader := boundReaderForRegularCleanupFixture(fixture) + readerPV := boundReaderPVForRegularCleanupFixture(fixture, reader) + foreignJob := fixture.job.DeepCopy() + foreignJob.Labels = map[string]string{ + nvcastorage.ModelCacheBindingUIDLabelKey: "foreign-binding", + } + k8sClient := fixture.useK8sObjects(reader, foreignJob, readerPV) + + state, _, err := fixture.backend.SetupModelCachingForRequest( + newTestContext(), fixture.rwPVC.DeepCopy(), fixture.job.DeepCopy(), + fixture.req, false, noOpRegularModelCacheMutation) + assert.Equal(t, ModelCachingFailed, state) + require.ErrorContains(t, err, "foreign-binding") + assert.True(t, nvcaerrors.IsTerminal(err)) + assertNoKubernetesWrites(t, k8sClient.Actions()) + }) + + t.Run("missing Job resourceVersion", func(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + reader := boundReaderForRegularCleanupFixture(fixture) + readerPV := boundReaderPVForRegularCleanupFixture(fixture, reader) + fixture.job.ResourceVersion = "" + k8sClient := fixture.useK8sObjects(reader, fixture.job, readerPV) + + state, _, err := fixture.backend.SetupModelCachingForRequest( + newTestContext(), fixture.rwPVC.DeepCopy(), fixture.job.DeepCopy(), + fixture.req, false, noOpRegularModelCacheMutation) + assert.Equal(t, ModelCachingFailed, state) + require.ErrorContains(t, err, "incomplete delete identity") + assert.True(t, nvcaerrors.IsTerminal(err)) + assertNoKubernetesWrites(t, k8sClient.Actions()) + }) +} + +func TestSuccessfulWriterTransitionIgnoresOtherBindingReferences(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + stored, err := fixture.backend.clients.BART.NvcaV2beta1(). + ModelCacheBindings(fixture.binding.Namespace). + Get(t.Context(), fixture.binding.Name, metav1.GetOptions{}) + require.NoError(t, err) + stored.Status.RequestReferences = append(stored.Status.RequestReferences, + nvcav2beta1.ModelCacheBindingRequestReference{ + Namespace: fixture.req.Namespace, + Name: "other-request", + UID: types.UID("other-request-uid"), + }) + _, err = fixture.backend.clients.BART.NvcaV2beta1(). + ModelCacheBindings(stored.Namespace). + UpdateStatus(t.Context(), stored, metav1.UpdateOptions{}) + require.NoError(t, err) + + reader := boundReaderForRegularCleanupFixture(fixture) + readerPV := boundReaderPVForRegularCleanupFixture(fixture, reader) + k8sClient := fixture.useK8sObjects(reader, fixture.job, readerPV) + state, roPVCName, err := fixture.backend.SetupModelCachingForRequest( + newTestContext(), fixture.rwPVC.DeepCopy(), fixture.job.DeepCopy(), + fixture.req, false, noOpRegularModelCacheMutation) + require.NoError(t, err) + assert.Equal(t, ModelCachingCompleted, state) + assert.Equal(t, reader.Name, roPVCName) + requireDeleteIdentity(t, k8sClient.Actions(), "jobs", fixture.job.Name, + fixture.job.UID, fixture.job.ResourceVersion) +} + +func TestSuccessfulWriterTransitionDeleteConflictIsRetryable(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + reader := boundReaderForRegularCleanupFixture(fixture) + readerPV := boundReaderPVForRegularCleanupFixture(fixture, reader) + k8sClient := fixture.useK8sObjects(reader, fixture.job, readerPV) + k8sClient.Fake.PrependReactor( + "delete", "jobs", + func(action k8stesting.Action) (bool, runtime.Object, error) { + deleteAction := action.(k8stesting.DeleteAction) + preconditions := deleteAction.GetDeleteOptions().Preconditions + require.NotNil(t, preconditions) + require.NotNil(t, preconditions.UID) + assert.Equal(t, fixture.job.UID, *preconditions.UID) + require.NotNil(t, preconditions.ResourceVersion) + assert.Equal(t, fixture.job.ResourceVersion, *preconditions.ResourceVersion) + return true, nil, apierrors.NewConflict( + corev1.Resource("jobs"), fixture.job.Name, fmt.Errorf("delete race")) + }) + + state, _, err := fixture.backend.SetupModelCachingForRequest( + newTestContext(), fixture.rwPVC.DeepCopy(), fixture.job.DeepCopy(), + fixture.req, false, noOpRegularModelCacheMutation) + assert.Equal(t, ModelCachingInProgress, state) + require.Error(t, err) + assert.True(t, apierrors.IsConflict(err)) + assert.False(t, nvcaerrors.IsTerminal(err)) + _, getErr := k8sClient.BatchV1().Jobs(fixture.job.Namespace). + Get(t.Context(), fixture.job.Name, metav1.GetOptions{}) + require.NoError(t, getErr) +} + +func TestSetupPVCForReadersRejectsForeignSameNameArtifacts(t *testing.T) { + t.Run("writer PVC", func(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + foreignWriter := fixture.rwPVC.DeepCopy() + foreignWriter.Labels = map[string]string{ + nvcastorage.ModelCacheBindingUIDLabelKey: "foreign-binding", + } + k8sClient := fixture.useK8sObjects(foreignWriter, fixture.job, fixture.pv) + + phase, err := fixture.backend.SetupPVCForReaders( + newTestContext(), fixture.rwPVC.DeepCopy(), fixture.job.Name, + fixture.req, noOpRegularModelCacheMutation) + assert.Equal(t, ROPVCSetupFailed, phase) + require.ErrorContains(t, err, "foreign-binding") + assertNoKubernetesWrites(t, k8sClient.Actions()) + }) + + t.Run("reader PVC wins AlreadyExists race", func(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + fixture.req.Spec.FunctionDetails.FunctionVersionID = "function-version" + fixture.pv.Labels[fnVersionIDLabelString] = "function-version" + foreignReader := boundReaderForRegularCleanupFixture(fixture) + foreignReader.Labels = map[string]string{ + nvcastorage.ModelCacheBindingUIDLabelKey: "foreign-binding", + } + foreignReader.UID = types.UID("foreign-reader-uid") + foreignReader.ResourceVersion = "foreign-reader-rv" + k8sClient := fixture.useK8sObjects( + fixture.rwPVC, fixture.job, fixture.pv, foreignReader) + + roPVCGets := 0 + k8sClient.Fake.PrependReactor( + "get", "persistentvolumeclaims", + func(action k8stesting.Action) (bool, runtime.Object, error) { + if action.(k8stesting.GetAction).GetName() != foreignReader.Name { + return false, nil, nil + } + roPVCGets++ + if roPVCGets == 1 { + return true, nil, apierrors.NewNotFound( + corev1.Resource("persistentvolumeclaims"), foreignReader.Name) + } + return false, nil, nil + }) + + previousSkip := skipVolumeDetachCheck + skipVolumeDetachCheck = true + t.Cleanup(func() { skipVolumeDetachCheck = previousSkip }) + + phase, err := fixture.backend.SetupPVCForReaders( + newTestContext(), fixture.rwPVC.DeepCopy(), fixture.job.Name, + fixture.req, noOpRegularModelCacheMutation) + assert.Equal(t, ROPVCSetupFailed, phase) + require.ErrorContains(t, err, "foreign-binding") + got, getErr := k8sClient.CoreV1().PersistentVolumeClaims(foreignReader.Namespace). + Get(t.Context(), foreignReader.Name, metav1.GetOptions{}) + require.NoError(t, getErr) + assert.Equal(t, foreignReader.UID, got.UID) + for _, action := range k8sClient.Actions() { + if action.GetVerb() == "delete" && + action.GetResource().Resource == "persistentvolumeclaims" { + assert.NotEqual(t, foreignReader.Name, + action.(k8stesting.DeleteAction).GetName()) + } + } + }) +} +func TestSetupPVCForReadersRevalidatesPVClaimInsideUpdateRetry(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + fixture.req.Spec.FunctionDetails.FunctionVersionID = "function-version" + fixture.pv.Labels[fnVersionIDLabelString] = "function-version" + k8sClient := fixture.useK8sObjects(fixture.rwPVC, fixture.job, fixture.pv) + + pvGets := 0 + k8sClient.Fake.PrependReactor( + "get", "persistentvolumes", + func(k8stesting.Action) (bool, runtime.Object, error) { + pvGets++ + if pvGets != 3 { + return false, nil, nil + } + racedPV := fixture.pv.DeepCopy() + racedPV.Spec.ClaimRef.Name = "foreign-reader" + return true, racedPV, nil + }) + + previousSkip := skipVolumeDetachCheck + skipVolumeDetachCheck = true + t.Cleanup(func() { skipVolumeDetachCheck = previousSkip }) + + phase, err := fixture.backend.SetupPVCForReaders( + newTestContext(), fixture.rwPVC.DeepCopy(), fixture.job.Name, + fixture.req, noOpRegularModelCacheMutation) + assert.Equal(t, ROPVUpdateFailed, phase) + require.ErrorContains(t, err, "outside binding intent") + for _, action := range k8sClient.Actions() { + if action.GetVerb() == "update" && + action.GetResource().Resource == "persistentvolumes" { + t.Fatalf("PV ownership changed during retry; refusing update was required") + } + } +} + +func TestSetupModelCachingForRequestRejectsUnownedBoundReaderPV(t *testing.T) { + tests := []struct { + name string + mutate func(*corev1.PersistentVolume) + wantErr string + }{ + { + name: "foreign PV binding", + mutate: func(pv *corev1.PersistentVolume) { + pv.Labels[nvcastorage.ModelCacheBindingUIDLabelKey] = "foreign-binding" + }, + wantErr: "foreign-binding", + }, + { + name: "stale reader claim UID", + mutate: func(pv *corev1.PersistentVolume) { + pv.Spec.ClaimRef.UID = types.UID("stale-reader-uid") + }, + wantErr: "claimRef", + }, + { + name: "missing required reader mount option", + mutate: func(pv *corev1.PersistentVolume) { + pv.Spec.MountOptions = []string{"ro", "norecovery"} + }, + wantErr: "missing required mount option", + }, + { + name: "conflicting reader mount option", + mutate: func(pv *corev1.PersistentVolume) { + pv.Spec.MountOptions = append(pv.Spec.MountOptions, "rw") + }, + wantErr: "conflicts with required option", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + reader := boundReaderForRegularCleanupFixture(fixture) + readerPV := boundReaderPVForRegularCleanupFixture(fixture, reader) + tt.mutate(readerPV) + k8sClient := fixture.useK8sObjects(reader, readerPV) + + state, _, err := fixture.backend.SetupModelCachingForRequest( + newTestContext(), fixture.rwPVC.DeepCopy(), fixture.job.DeepCopy(), + fixture.req, false, noOpRegularModelCacheMutation) + assert.Equal(t, ModelCachingFailed, state) + require.ErrorContains(t, err, tt.wantErr) + assert.True(t, nvcaerrors.IsTerminal(err)) + assertNoKubernetesWrites(t, k8sClient.Actions()) + }) + } +} + +func TestSetupModelCachingForRequestAllowsPendingOwnedReader(t *testing.T) { + fixture := newRegularCleanupTestFixture(t) + reader := boundReaderForRegularCleanupFixture(fixture) + reader.Status.Phase = corev1.ClaimPending + reader.CreationTimestamp = metav1.Now() + k8sClient := fixture.useK8sObjects(reader) + + state, _, err := fixture.backend.SetupModelCachingForRequest( + newTestContext(), fixture.rwPVC.DeepCopy(), fixture.job.DeepCopy(), + fixture.req, false, noOpRegularModelCacheMutation) + require.NoError(t, err) + assert.Equal(t, ModelCachingInProgress, state) + for _, action := range k8sClient.Actions() { + assert.NotEqual(t, "persistentvolumes", action.GetResource().Resource, + "pending reader validation must wait for Kubernetes to bind the PV") + } + assertNoKubernetesWrites(t, k8sClient.Actions()) +} + +func TestRequestDeletionResumesRetiringCleanupAfterReferenceRelease(t *testing.T) { + fixture := newRWXRegularCleanupTestFixture(t) + fixture.rwPVC.APIVersion = "v1" + fixture.rwPVC.Kind = "PersistentVolumeClaim" + fixture.job.APIVersion = "batch/v1" + fixture.job.Kind = "Job" + setCleanupArtifacts(t, fixture.req, fixture.rwPVC, fixture.job) + fixture.backend.bk8s.k8sArtifactHelper = fixture.backend + k8sClient := fixture.useK8sObjects(fixture.rwPVC, fixture.job, fixture.pv) + pvcDeleteAttempts := 0 + k8sClient.Fake.PrependReactor( + "delete", "persistentvolumeclaims", + func(k8stesting.Action) (bool, runtime.Object, error) { + pvcDeleteAttempts++ + if pvcDeleteAttempts == 1 { + return true, nil, apierrors.NewServiceUnavailable( + "transient PVC delete failure") + } + return false, nil, nil + }) + + err := fixture.backend.CleanupModelCachingResources( + newTestContext(), fixture.req, fixture.rwPVC.DeepCopy(), fixture.job.Name) + require.Error(t, err) + assert.True(t, apierrors.IsServiceUnavailable(err)) + require.NoError(t, fixture.backend.bk8s.releaseModelCacheBindingReference( + t.Context(), fixture.req.DeepCopy())) + + stored, getErr := fixture.backend.clients.BART.NvcaV2beta1(). + ModelCacheBindings(fixture.binding.Namespace). + Get(t.Context(), fixture.binding.Name, metav1.GetOptions{}) + require.NoError(t, getErr) + assert.Equal(t, nvcav2beta1.ModelCacheBindingPhaseRetiring, stored.Status.Phase) + assert.Empty(t, stored.Status.RequestReferences) + + require.NoError(t, fixture.backend.bk8s.resumeRetiringRegularModelCacheCleanup( + newTestContext(), fixture.req.DeepCopy())) + assert.Equal(t, 2, pvcDeleteAttempts) + _, getErr = k8sClient.CoreV1().PersistentVolumeClaims(fixture.rwPVC.Namespace). + Get(t.Context(), fixture.rwPVC.Name, metav1.GetOptions{}) + assert.True(t, apierrors.IsNotFound(getErr)) +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/modelcache_shared_writer.go b/src/compute-plane-services/nvca/pkg/nvca/modelcache_shared_writer.go new file mode 100644 index 000000000..67681f1b1 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/modelcache_shared_writer.go @@ -0,0 +1,122 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nvca + +import ( + "fmt" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" +) + +// prepareRWXReadOnlySharedWriterJob strips all existing Job and Pod-template +// labels, annotations, and owner references, then rejects inputs that a +// retained, binding-owned Job cannot safely keep. The caller adds the binding +// identity after this function returns. +// A future credential indirection must record every Secret in the binding and +// implement exact create, adoption, rotation, and cleanup before relaxing +// these restrictions. +func prepareRWXReadOnlySharedWriterJob(job *batchv1.Job) error { + if job == nil { + return fmt.Errorf("rwxReadOnly writer Job is nil") + } + + // Only binding metadata is allowed to survive on the shared Job. The caller + // adds the binding UID label after this function returns. + job.OwnerReferences = nil + job.Labels = nil + job.Annotations = nil + job.Spec.Template.OwnerReferences = nil + job.Spec.Template.Labels = nil + job.Spec.Template.Annotations = nil + + podSpec := &job.Spec.Template.Spec + if len(podSpec.ImagePullSecrets) != 0 { + return fmt.Errorf( + "rwxReadOnly writer Job %s/%s references imagePullSecrets; "+ + "binding-scoped pull Secret identity and cleanup are not implemented", + job.Namespace, job.Name) + } + + for i := range podSpec.InitContainers { + if err := validateRWXReadOnlySharedWriterContainer( + job, "init container", &podSpec.InitContainers[i]); err != nil { + return err + } + } + for i := range podSpec.Containers { + if err := validateRWXReadOnlySharedWriterContainer( + job, "container", &podSpec.Containers[i]); err != nil { + return err + } + } + for i := range podSpec.Volumes { + if rwxReadOnlyWriterVolumeReferencesSecret(&podSpec.Volumes[i]) { + return fmt.Errorf( + "rwxReadOnly writer Job %s/%s volume %q references a Secret; "+ + "binding-scoped Secret identity and cleanup are not implemented", + job.Namespace, job.Name, podSpec.Volumes[i].Name) + } + } + + // The cache writer does not use the Kubernetes API. Avoid giving a retained + // shared Job an implicit, namespace-scoped service-account credential. + automountServiceAccountToken := false + podSpec.AutomountServiceAccountToken = &automountServiceAccountToken + return nil +} + +func validateRWXReadOnlySharedWriterContainer( + job *batchv1.Job, + kind string, + container *corev1.Container, +) error { + if len(container.Env) != 0 || len(container.EnvFrom) != 0 { + return fmt.Errorf( + "rwxReadOnly writer Job %s/%s %s %q retains environment input; "+ + "binding-scoped credential indirection and immutable input identity are not implemented", + job.Namespace, job.Name, kind, container.Name) + } + return nil +} + +func rwxReadOnlyWriterVolumeReferencesSecret(volume *corev1.Volume) bool { + if volume == nil { + return false + } + if volume.Secret != nil || + (volume.CSI != nil && volume.CSI.NodePublishSecretRef != nil) || + (volume.FlexVolume != nil && volume.FlexVolume.SecretRef != nil) || + (volume.CephFS != nil && volume.CephFS.SecretRef != nil) || + (volume.Cinder != nil && volume.Cinder.SecretRef != nil) || + (volume.RBD != nil && volume.RBD.SecretRef != nil) || + (volume.ScaleIO != nil && volume.ScaleIO.SecretRef != nil) || + (volume.StorageOS != nil && volume.StorageOS.SecretRef != nil) || + (volume.AzureFile != nil && volume.AzureFile.SecretName != "") { + return true + } + if volume.Projected == nil { + return false + } + for _, source := range volume.Projected.Sources { + if source.Secret != nil { + return true + } + } + return false +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/modelcache_shared_writer_test.go b/src/compute-plane-services/nvca/pkg/nvca/modelcache_shared_writer_test.go new file mode 100644 index 000000000..5f9c45aec --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/modelcache_shared_writer_test.go @@ -0,0 +1,131 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nvca + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + + nvcaerrors "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/errors" + nvcastorage "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/storage" +) + +func TestPrepareRWXReadOnlySharedWriterJobCanonicalizesMetadata(t *testing.T) { + fixture := newRWXReadOnlyRuntimeFixture(t, "weka", "csi.weka.io") + fixture.job.Labels["request.example/label"] = "request-a" + fixture.job.Annotations = map[string]string{"request.example/annotation": "request-a"} + fixture.job.Spec.Template.Labels["request.example/label"] = "request-a" + fixture.job.Spec.Template.Annotations = map[string]string{} + fixture.job.Spec.Template.Annotations["request.example/annotation"] = "request-a" + + require.NoError(t, fixture.backend.prepareRegularModelCacheBindingResources( + t.Context(), fixture.binding, fixture.rwPVC, fixture.job)) + + assert.Equal(t, map[string]string{ + nvcastorage.ModelCacheBindingUIDLabelKey: string(fixture.binding.UID), + }, fixture.job.Labels) + assert.Empty(t, fixture.job.Annotations) + assert.Equal(t, map[string]string{ + nvcastorage.ModelCacheBindingUIDLabelKey: string(fixture.binding.UID), + }, fixture.job.Spec.Template.Labels) + assert.Empty(t, fixture.job.Spec.Template.Annotations) + require.NotNil(t, fixture.job.Spec.Template.Spec.AutomountServiceAccountToken) + assert.False(t, *fixture.job.Spec.Template.Spec.AutomountServiceAccountToken) +} + +func TestPrepareRWXReadOnlySharedWriterJobRejectsRequestScopedInputs(t *testing.T) { + tests := []struct { + name string + mutate func(*rwxReadOnlyRuntimeFixture) + wantErr string + }{ + { + name: "raw worker token", + mutate: func(fixture *rwxReadOnlyRuntimeFixture) { + fixture.job.Spec.Template.Spec.Containers[0].Env = []corev1.EnvVar{{ + Name: "NVCF_WORKER_TOKEN", Value: "request-token", + }} + }, + wantErr: "retains environment input", + }, + { + name: "environment Secret reference", + mutate: func(fixture *rwxReadOnlyRuntimeFixture) { + fixture.job.Spec.Template.Spec.Containers[0].Env = []corev1.EnvVar{{ + Name: "NVCF_WORKER_TOKEN", + ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "request-secret"}, + Key: "token", + }}, + }} + }, + wantErr: "binding-scoped credential indirection", + }, + { + name: "environment source", + mutate: func(fixture *rwxReadOnlyRuntimeFixture) { + fixture.job.Spec.Template.Spec.Containers[0].EnvFrom = []corev1.EnvFromSource{{ + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{Name: "request-secret"}, + }, + }} + }, + wantErr: "immutable input identity", + }, + { + name: "image pull Secret", + mutate: func(fixture *rwxReadOnlyRuntimeFixture) { + fixture.job.Spec.Template.Spec.ImagePullSecrets = []corev1.LocalObjectReference{{ + Name: "request-pull-secret", + }} + }, + wantErr: "binding-scoped pull Secret identity and cleanup are not implemented", + }, + { + name: "Secret volume", + mutate: func(fixture *rwxReadOnlyRuntimeFixture) { + fixture.job.Spec.Template.Spec.Volumes = append( + fixture.job.Spec.Template.Spec.Volumes, + corev1.Volume{ + Name: "request-secret", + VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{ + SecretName: "request-secret", + }}, + }) + }, + wantErr: "binding-scoped Secret identity and cleanup are not implemented", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fixture := newRWXReadOnlyRuntimeFixture(t, "weka", "csi.weka.io") + fixture.job.Labels = map[string]string{} + fixture.job.Spec.Template.Labels = map[string]string{} + tt.mutate(fixture) + + err := fixture.backend.prepareRegularModelCacheBindingResources( + t.Context(), fixture.binding, fixture.rwPVC, fixture.job) + require.ErrorContains(t, err, tt.wantErr) + assert.True(t, nvcaerrors.IsTerminal(err)) + }) + } +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection.go b/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection.go new file mode 100644 index 000000000..d6ec44f9d --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection.go @@ -0,0 +1,116 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nvca + +import ( + "context" + "errors" + "fmt" + + "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common" + + nvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/featureflag" + nvcastorage "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/storage" +) + +func cacheSelectionInput( + req *nvcav2beta1.ICMSRequest, +) (*common.CacheLaunchSpecification, nvcastorage.ModelCacheWorkflow) { + switch { + case req.Spec.CreationMsgInfo.FunctionLaunchSpecification != nil: + spec := req.Spec.CreationMsgInfo.FunctionLaunchSpecification + workflow := nvcastorage.ModelCacheWorkflowRegular + if spec.HelmChartLaunchSpecification != nil { + workflow = nvcastorage.ModelCacheWorkflowHelm + } + return spec.CacheLaunchSpecification, workflow + case req.Spec.CreationMsgInfo.TaskLaunchSpecification != nil: + spec := req.Spec.CreationMsgInfo.TaskLaunchSpecification + workflow := nvcastorage.ModelCacheWorkflowRegular + if spec.HelmChartLaunchSpecification != nil { + workflow = nvcastorage.ModelCacheWorkflowHelm + } + return spec.CacheLaunchSpecification, workflow + default: + return nil, "" + } +} + +// persistModelCacheStorageSelection resolves and records the cache decision +// before the ICMSRequest is created. An absent annotation therefore identifies +// a request created by legacy NVCA. +func (c *BackendK8sCache) persistModelCacheStorageSelection( + ctx context.Context, + req *nvcav2beta1.ICMSRequest, +) error { + cacheSpec, workflow := cacheSelectionInput(req) + if cacheSpec == nil || cacheSpec.CacheSize <= 0 { + return nil + } + + mode := nvcastorage.ModelCacheSelectionNone + var resolved *nvcastorage.ModelCacheStorageSelection + cachingEnabled := c.featureFlagFetcher.IsFeatureFlagEnabled(featureflag.CachingSupport) + if workflow == nvcastorage.ModelCacheWorkflowHelm { + cachingEnabled = cachingEnabled && + c.featureFlagFetcher.IsFeatureFlagEnabled(featureflag.HelmModelCaching) + } + if cachingEnabled { + var err error + resolved, err = nvcastorage.ResolveModelCacheStorageWithClientset( + ctx, c.clients.K8s, c.systemNamespace, workflow) + switch { + case errors.Is(err, nvcastorage.ErrModelCacheStorageClassNotFound): + if workflow == nvcastorage.ModelCacheWorkflowHelm { + mode = nvcastorage.ModelCacheSelectionEphemeral + } + case err != nil: + return fmt.Errorf("resolve model cache storage: %w", err) + case resolved.Transition == nvcastorage.ModelCacheTransitionDisabled: + if workflow == nvcastorage.ModelCacheWorkflowHelm { + mode = nvcastorage.ModelCacheSelectionEphemeral + } + case resolved.Transition == nvcastorage.ModelCacheTransitionROXReadOnly: + mode = nvcastorage.ModelCacheSelectionDurable + case resolved.Transition == nvcastorage.ModelCacheTransitionRWXReadOnly && + workflow == nvcastorage.ModelCacheWorkflowRegular: + mode = nvcastorage.ModelCacheSelectionDurable + default: + return fmt.Errorf("unsupported model cache transition %q", resolved.Transition) + } + } + + selection, err := nvcastorage.NewPersistedModelCacheStorageSelection(workflow, mode, resolved) + if err != nil { + return fmt.Errorf("build model cache storage selection: %w", err) + } + if selection.Mode == nvcastorage.ModelCacheSelectionDurable && + selection.Transition == nvcastorage.ModelCacheTransitionROXReadOnly { + selection.EncryptionRequired = c.featureFlagFetcher.IsFeatureFlagEnabled(featureflag.NVMeshEncryption) + } + payload, err := selection.Marshal() + if err != nil { + return err + } + if req.Annotations == nil { + req.Annotations = map[string]string{} + } + req.Annotations[nvcastorage.ModelCacheStorageSelectionAnnotationKey] = payload + return nil +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection_test.go b/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection_test.go new file mode 100644 index 000000000..299fd4e76 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/nvca/modelcache_storage_selection_test.go @@ -0,0 +1,759 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nvca + +import ( + "fmt" + "testing" + + "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/common" + "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/icms-translate/translate/function" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + fakek8sclient "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/kubeclients" + nvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/featureflag" + featureflagmock "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/featureflag/mock" + nvcaerrors "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/nvca/errors" + nvcastorage "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/storage" +) + +const ( + selectionCatalogNamespace = "nvca-system" + selectionCatalogNVMesh = `apiVersion: storage.nvcf.nvidia.com/v1alpha1 +kind: StorageCapabilityCatalog +drivers: + nvmesh-csi.excelero.com: + provider: nvmesh + accessModes: [ReadWriteOnce, ReadOnlyMany] + readerMountOptions: [ro, norecovery, nouuid] + transitions: + regularModelCache: roxReadOnly + helmModelCache: roxReadOnly +` + selectionCatalogDisabled = `apiVersion: storage.nvcf.nvidia.com/v1alpha1 +kind: StorageCapabilityCatalog +drivers: + nvmesh-csi.excelero.com: + provider: nvmesh + accessModes: [ReadWriteOnce, ReadOnlyMany] + readerMountOptions: [] + transitions: + regularModelCache: disabled + helmModelCache: disabled +` + selectionCatalogRWXReadOnly = `apiVersion: storage.nvcf.nvidia.com/v1alpha1 +kind: StorageCapabilityCatalog +drivers: + csi.weka.io: + provider: weka + accessModes: [ReadWriteMany, ReadOnlyMany] + readerMountOptions: [] + transitions: + regularModelCache: rwxReadOnly + helmModelCache: disabled +` +) + +func selectionStorageClass() *storagev1.StorageClass { + retain := corev1.PersistentVolumeReclaimRetain + wait := storagev1.VolumeBindingWaitForFirstConsumer + return &storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: nvcastorage.DefaultModelCacheStorageClassName, + UID: types.UID("nvcf-sc-uid"), + }, + Provisioner: nvcastorage.NVMeshStorageClassProvisioner, + Parameters: map[string]string{"pool": "model-cache"}, + ReclaimPolicy: &retain, + VolumeBindingMode: &wait, + MountOptions: []string{"nouuid", "noatime"}, + } +} + +func selectionStorageClassForProvisioner(provisioner string) *storagev1.StorageClass { + storageClass := selectionStorageClass() + storageClass.Provisioner = provisioner + return storageClass +} + +func selectionCatalogConfigMap(raw string) *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: nvcastorage.StorageCapabilityConfigMapName, + Namespace: selectionCatalogNamespace, + }, + Data: map[string]string{nvcastorage.StorageCapabilityConfigMapKey: raw}, + } +} + +func selectionRequest(helm bool) *nvcav2beta1.ICMSRequest { + launchSpec := &function.LaunchSpecification{ + CacheLaunchSpecification: &common.CacheLaunchSpecification{ + CacheArtifacts: true, + CacheHandle: "model-cache-handle", + CacheSize: 1 << 30, + }, + } + if helm { + launchSpec.HelmChartLaunchSpecification = &common.HelmChartLaunchSpecification{ + HelmChartURL: "https://example.invalid/chart.tgz", + } + } + return &nvcav2beta1.ICMSRequest{ + ObjectMeta: metav1.ObjectMeta{Name: "request", Namespace: RequestsNamespace}, + Spec: nvcav2beta1.ICMSRequestSpec{ + CreationMsgInfo: nvcav2beta1.ICMSCreationMessageInfo{ + FunctionLaunchSpecification: launchSpec, + }, + }, + } +} + +func selectionBackendCache( + objects []runtime.Object, + flags ...*featureflag.FeatureFlag, +) (*BackendK8sCache, *fakek8sclient.Clientset) { + k8sClient := fakek8sclient.NewSimpleClientset(objects...) + clients := &kubeclients.KubeClients{K8s: k8sClient} + return &BackendK8sCache{ + clients: clients, + systemNamespace: selectionCatalogNamespace, + requestsNamespace: RequestsNamespace, + featureFlagFetcher: &featureflagmock.Fetcher{EnabledFFs: flags}, + }, k8sClient +} + +func parseRequestStorageSelection( + t *testing.T, + req *nvcav2beta1.ICMSRequest, +) *nvcastorage.PersistedModelCacheStorageSelection { + t.Helper() + raw := req.Annotations[nvcastorage.ModelCacheStorageSelectionAnnotationKey] + require.NotEmpty(t, raw) + selection, err := nvcastorage.ParsePersistedModelCacheStorageSelection(raw) + require.NoError(t, err) + return selection +} + +func TestPersistModelCacheStorageSelection(t *testing.T) { + tests := []struct { + name string + helm bool + objects func() []runtime.Object + flags []*featureflag.FeatureFlag + wantWorkflow nvcastorage.ModelCacheWorkflow + wantMode nvcastorage.ModelCacheSelectionMode + wantTransition string + wantResolvedState bool + wantProvider string + wantProvisioner string + wantEncryption bool + }{ + { + name: "regular durable NVMesh", + objects: func() []runtime.Object { + return []runtime.Object{selectionStorageClass(), selectionCatalogConfigMap(selectionCatalogNVMesh)} + }, + flags: []*featureflag.FeatureFlag{featureflag.CachingSupport}, + wantWorkflow: nvcastorage.ModelCacheWorkflowRegular, + wantMode: nvcastorage.ModelCacheSelectionDurable, + wantTransition: nvcastorage.ModelCacheTransitionROXReadOnly, + wantResolvedState: true, + wantProvider: nvcastorage.ModelCacheProviderNVMesh, + wantProvisioner: nvcastorage.NVMeshStorageClassProvisioner, + }, + { + name: "Helm durable NVMesh", + helm: true, + objects: func() []runtime.Object { + return []runtime.Object{selectionStorageClass(), selectionCatalogConfigMap(selectionCatalogNVMesh)} + }, + flags: []*featureflag.FeatureFlag{ + featureflag.CachingSupport, + featureflag.HelmModelCaching, + }, + wantWorkflow: nvcastorage.ModelCacheWorkflowHelm, + wantMode: nvcastorage.ModelCacheSelectionDurable, + wantTransition: nvcastorage.ModelCacheTransitionROXReadOnly, + wantResolvedState: true, + wantProvider: nvcastorage.ModelCacheProviderNVMesh, + wantProvisioner: nvcastorage.NVMeshStorageClassProvisioner, + }, + { + name: "regular durable provider-neutral RWX", + objects: func() []runtime.Object { + return []runtime.Object{ + selectionStorageClassForProvisioner("csi.weka.io"), + selectionCatalogConfigMap(selectionCatalogRWXReadOnly), + } + }, + flags: []*featureflag.FeatureFlag{ + featureflag.CachingSupport, + featureflag.NVMeshEncryption, + }, + wantWorkflow: nvcastorage.ModelCacheWorkflowRegular, + wantMode: nvcastorage.ModelCacheSelectionDurable, + wantTransition: nvcastorage.ModelCacheTransitionRWXReadOnly, + wantResolvedState: true, + wantProvider: "weka", + wantProvisioner: "csi.weka.io", + }, + { + name: "disabled regular cache persists none", + objects: func() []runtime.Object { + return []runtime.Object{selectionStorageClass(), selectionCatalogConfigMap(selectionCatalogDisabled)} + }, + flags: []*featureflag.FeatureFlag{featureflag.CachingSupport}, + wantWorkflow: nvcastorage.ModelCacheWorkflowRegular, + wantMode: nvcastorage.ModelCacheSelectionNone, + wantTransition: nvcastorage.ModelCacheTransitionDisabled, + wantResolvedState: true, + wantProvider: nvcastorage.ModelCacheProviderNVMesh, + wantProvisioner: nvcastorage.NVMeshStorageClassProvisioner, + }, + { + name: "disabled Helm cache persists ephemeral", + helm: true, + objects: func() []runtime.Object { + return []runtime.Object{selectionStorageClass(), selectionCatalogConfigMap(selectionCatalogDisabled)} + }, + flags: []*featureflag.FeatureFlag{ + featureflag.CachingSupport, + featureflag.HelmModelCaching, + }, + wantWorkflow: nvcastorage.ModelCacheWorkflowHelm, + wantMode: nvcastorage.ModelCacheSelectionEphemeral, + wantTransition: nvcastorage.ModelCacheTransitionDisabled, + wantResolvedState: true, + wantProvider: nvcastorage.ModelCacheProviderNVMesh, + wantProvisioner: nvcastorage.NVMeshStorageClassProvisioner, + }, + { + name: "missing StorageClass disables regular cache", + objects: func() []runtime.Object { + return []runtime.Object{selectionCatalogConfigMap(selectionCatalogNVMesh)} + }, + flags: []*featureflag.FeatureFlag{featureflag.CachingSupport}, + wantWorkflow: nvcastorage.ModelCacheWorkflowRegular, + wantMode: nvcastorage.ModelCacheSelectionNone, + }, + { + name: "missing StorageClass falls Helm back to ephemeral", + helm: true, + objects: func() []runtime.Object { + return []runtime.Object{selectionCatalogConfigMap(selectionCatalogNVMesh)} + }, + flags: []*featureflag.FeatureFlag{ + featureflag.CachingSupport, + featureflag.HelmModelCaching, + }, + wantWorkflow: nvcastorage.ModelCacheWorkflowHelm, + wantMode: nvcastorage.ModelCacheSelectionEphemeral, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cache, _ := selectionBackendCache(tt.objects(), tt.flags...) + req := selectionRequest(tt.helm) + + require.NoError(t, cache.persistModelCacheStorageSelection(t.Context(), req)) + selection := parseRequestStorageSelection(t, req) + assert.Equal(t, tt.wantWorkflow, selection.Workflow) + assert.Equal(t, tt.wantMode, selection.Mode) + assert.Equal(t, tt.wantTransition, selection.Transition) + if !tt.wantResolvedState { + assert.Empty(t, selection.StorageClassName) + assert.Empty(t, selection.StorageClassUID) + assert.Empty(t, selection.StorageClassDigest) + assert.Empty(t, selection.CatalogDigest) + assert.Empty(t, selection.Provider) + assert.Empty(t, selection.Provisioner) + return + } + assert.Equal(t, nvcastorage.DefaultModelCacheStorageClassName, selection.StorageClassName) + assert.Equal(t, types.UID("nvcf-sc-uid"), selection.StorageClassUID) + assert.NotEmpty(t, selection.StorageClassDigest) + assert.NotEmpty(t, selection.CatalogDigest) + assert.Equal(t, tt.wantProvider, selection.Provider) + assert.Equal(t, tt.wantProvisioner, selection.Provisioner) + assert.Equal(t, tt.wantEncryption, selection.EncryptionRequired) + }) + } +} + +func TestCreateICMSCreationMessageRequestInvalidCatalogFailsBeforeCreate(t *testing.T) { + objects := []runtime.Object{ + selectionStorageClass(), + selectionCatalogConfigMap("drivers: ["), + } + cache, _ := selectionBackendCache(objects, featureflag.CachingSupport) + cache.clients = mockKubeClients(objects...) + cache.requestsNamespace = RequestsNamespace + + msg := function.CreationQueueMessage{ + CreationQueueMessageMetadata: common.CreationQueueMessageMetadata{ + RequestID: "invalid-catalog-request", + NCAID: "test-nca", + Action: common.FunctionCreationAction, + }, + Details: function.Details{ + FunctionID: "function-id", + FunctionVersionID: "function-version-id", + }, + LaunchSpecification: selectionRequest(false).Spec.CreationMsgInfo.FunctionLaunchSpecification, + } + + created, err := cache.CreateICMSCreationMessageRequest( + newTestContext(), msg, "receipt", "message-id", "queue") + require.ErrorContains(t, err, "resolve model cache storage: parse storage capability catalog") + assert.Nil(t, created) + + requests, listErr := cache.clients.BART.NvcaV2beta1().ICMSRequests(RequestsNamespace). + List(t.Context(), metav1.ListOptions{}) + require.NoError(t, listErr) + assert.Empty(t, requests.Items, "an invalid catalog must fail before the ICMSRequest Create call") +} + +func persistedSelectionAnnotation( + t *testing.T, + workflow nvcastorage.ModelCacheWorkflow, + mode nvcastorage.ModelCacheSelectionMode, + resolved *nvcastorage.ModelCacheStorageSelection, +) string { + t.Helper() + selection, err := nvcastorage.NewPersistedModelCacheStorageSelection(workflow, mode, resolved) + require.NoError(t, err) + raw, err := selection.Marshal() + require.NoError(t, err) + return raw +} + +func resolvedSelectionForSetup(t *testing.T) *nvcastorage.ModelCacheStorageSelection { + t.Helper() + k8sClient := fakek8sclient.NewSimpleClientset( + selectionStorageClass(), + selectionCatalogConfigMap(selectionCatalogNVMesh), + ) + selection, err := nvcastorage.ResolveModelCacheStorageWithClientset( + t.Context(), k8sClient, selectionCatalogNamespace, nvcastorage.ModelCacheWorkflowRegular) + require.NoError(t, err) + return selection +} + +func testContainerModelCacheBackend( + k8sClient *fakek8sclient.Clientset, +) K8sComputeBackend { + clients := &kubeclients.KubeClients{K8s: k8sClient} + bk8s := &BackendK8sCache{ + clients: clients, + podInstanceNamespace: RequestsNamespace, + featureFlagFetcher: &featureflagmock.Fetcher{}, + } + return K8sComputeBackend{clients: clients, bk8s: bk8s} +} + +func assertNoKubernetesWrites(t *testing.T, actions []k8stesting.Action) { + t.Helper() + for _, action := range actions { + switch action.GetVerb() { + case "create", "update", "patch", "delete", "delete-collection": + t.Errorf("unexpected Kubernetes write before model-cache selection validation: %s %s", + action.GetVerb(), action.GetResource().Resource) + } + } +} + +func TestSetupContainerModelCachingSelectionContract(t *testing.T) { + resolved := resolvedSelectionForSetup(t) + durableRaw := persistedSelectionAnnotation( + t, nvcastorage.ModelCacheWorkflowRegular, nvcastorage.ModelCacheSelectionDurable, resolved) + noneRaw := persistedSelectionAnnotation( + t, nvcastorage.ModelCacheWorkflowRegular, nvcastorage.ModelCacheSelectionNone, nil) + helmRaw := persistedSelectionAnnotation( + t, nvcastorage.ModelCacheWorkflowHelm, nvcastorage.ModelCacheSelectionNone, nil) + ephemeralRaw := `{"version":"v1alpha1","workflow":"regularModelCache","mode":"ephemeral"}` + + defaultClass := nvcastorage.DefaultModelCacheStorageClassName + customClass := "custom-model-cache-class" + + tests := []struct { + name string + raw string + storageClass *string + objects []runtime.Object + wantErr string + wantTerminal bool + wantNoOp bool + wantReadCount int + }{ + { + name: "none is a no-op", + raw: noneRaw, + storageClass: &defaultClass, + wantNoOp: true, + }, + { + name: "malformed selection", + raw: "{", + storageClass: &defaultClass, + wantErr: "parse persisted model cache storage selection", + wantTerminal: true, + }, + { + name: "wrong workflow", + raw: helmRaw, + storageClass: &defaultClass, + wantErr: "is not regularModelCache", + wantTerminal: true, + }, + { + name: "ephemeral mode is unsupported for regular cache", + raw: ephemeralRaw, + storageClass: &defaultClass, + wantErr: "ephemeral model cache selection requires Helm workflow", + wantTerminal: true, + }, + { + name: "custom StorageClass is rejected", + raw: durableRaw, + storageClass: &customClass, + wantErr: "StorageClass override", + wantTerminal: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + k8sClient := fakek8sclient.NewSimpleClientset(tt.objects...) + backend := testContainerModelCacheBackend(k8sClient) + req := &nvcav2beta1.ICMSRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "request", + Namespace: RequestsNamespace, + Annotations: map[string]string{ + nvcastorage.ModelCacheStorageSelectionAnnotationKey: tt.raw, + }, + }, + } + if tt.raw == durableRaw { + installActiveRegularModelCacheBinding(t, &backend, req, tt.raw) + } + rwPVC := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: "rw-pvc-cache", Namespace: RequestsNamespace}, + Spec: corev1.PersistentVolumeClaimSpec{ + StorageClassName: tt.storageClass, + }, + } + initJob := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Name: "writer-job-cache", Namespace: RequestsNamespace}, + } + + mf, roPVCName, err := backend.setupContainerModelCaching( + newTestContext(), req, rwPVC, initJob, nil) + if tt.wantErr != "" { + require.ErrorContains(t, err, tt.wantErr) + assert.Equal(t, tt.wantTerminal, nvcaerrors.IsTerminal(err)) + assert.Nil(t, mf) + assert.Empty(t, roPVCName) + } else { + require.NoError(t, err) + require.NotNil(t, mf) + assert.Empty(t, roPVCName) + pod := &corev1.Pod{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + VolumeMounts: []corev1.VolumeMount{{Name: "unrelated", ReadOnly: false}}, + }}, + }} + before := pod.DeepCopy() + mf(pod) + assert.Equal(t, before, pod, "none selection must return a true no-op mutator") + } + + assertNoKubernetesWrites(t, k8sClient.Actions()) + assert.Len(t, k8sClient.Actions(), tt.wantReadCount) + }) + } +} + +func TestSetModelCacheVolumeMountsReadOnly(t *testing.T) { + containers := []corev1.Container{ + { + Name: "inference", + VolumeMounts: []corev1.VolumeMount{ + {Name: ModelVolumeName, MountPath: "/model", ReadOnly: false}, + {Name: "config", MountPath: "/config", ReadOnly: false}, + }, + }, + { + Name: "sidecar", + VolumeMounts: []corev1.VolumeMount{ + {Name: ModelVolumeName, MountPath: "/models", ReadOnly: true}, + }, + }, + {Name: "no-mounts"}, + } + + setModelCacheVolumeMountsReadOnly(containers) + + assert.True(t, containers[0].VolumeMounts[0].ReadOnly) + assert.False(t, containers[0].VolumeMounts[1].ReadOnly, "unrelated mounts must not be changed") + assert.True(t, containers[1].VolumeMounts[0].ReadOnly) + assert.NotPanics(t, func() { setModelCacheVolumeMountsReadOnly(nil) }) +} + +func TestSetupContainerFunctionModelCachingArtifactDecodeFailure(t *testing.T) { + resolved := resolvedSelectionForSetup(t) + durableRaw := persistedSelectionAnnotation( + t, nvcastorage.ModelCacheWorkflowRegular, nvcastorage.ModelCacheSelectionDurable, resolved) + + tests := []struct { + name string + annotation string + wantErr string + wantTerminal bool + wantLegacyNoOp bool + }{ + { + name: "persisted durable selection fails closed", + annotation: durableRaw, + wantErr: "decode artifacts for persisted durable regular model cache", + wantTerminal: true, + }, + { + name: "legacy request falls back to uncached", + wantLegacyNoOp: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + k8sClient := fakek8sclient.NewSimpleClientset() + backend := testContainerModelCacheBackend(k8sClient) + req := &nvcav2beta1.ICMSRequest{ + ObjectMeta: metav1.ObjectMeta{Name: "request", Namespace: RequestsNamespace}, + } + if tt.annotation != "" { + req.Annotations = map[string]string{ + nvcastorage.ModelCacheStorageSelectionAnnotationKey: tt.annotation, + } + } + invalidArtifact := function.LaunchArtifact{Specification: "%%%"} + + mf, roPVCName, err := backend.setupContainerFunctionModelCaching( + newTestContext(), req, invalidArtifact, invalidArtifact, func(_ client.Object) {}) + if tt.wantErr != "" { + require.ErrorContains(t, err, tt.wantErr) + assert.Equal(t, tt.wantTerminal, nvcaerrors.IsTerminal(err)) + assert.Nil(t, mf) + assert.Empty(t, roPVCName) + } else { + require.NoError(t, err) + require.True(t, tt.wantLegacyNoOp) + require.NotNil(t, mf) + assert.Empty(t, roPVCName) + pod := &corev1.Pod{} + before := pod.DeepCopy() + mf(pod) + assert.Equal(t, before, pod) + } + assert.Empty(t, k8sClient.Actions(), "artifact decode failure must happen before Kubernetes access") + }) + } +} + +func TestValidatePersistedRegularModelCacheArtifacts(t *testing.T) { + resolved := resolvedSelectionForSetup(t) + durableRaw := persistedSelectionAnnotation( + t, nvcastorage.ModelCacheWorkflowRegular, nvcastorage.ModelCacheSelectionDurable, resolved) + durable, err := nvcastorage.ParsePersistedModelCacheStorageSelection(durableRaw) + require.NoError(t, err) + noneRaw := persistedSelectionAnnotation( + t, nvcastorage.ModelCacheWorkflowRegular, nvcastorage.ModelCacheSelectionNone, nil) + none, err := nvcastorage.ParsePersistedModelCacheStorageSelection(noneRaw) + require.NoError(t, err) + present := function.LaunchArtifact{Specification: "present"} + missing := function.LaunchArtifact{} + + tests := []struct { + name string + selection *nvcastorage.PersistedModelCacheStorageSelection + rwPVC function.LaunchArtifact + initJob function.LaunchArtifact + wantErr bool + }{ + {name: "durable selection with both artifacts", selection: durable, rwPVC: present, initJob: present}, + {name: "durable selection missing PVC", selection: durable, rwPVC: missing, initJob: present, wantErr: true}, + {name: "durable selection missing init Job", selection: durable, rwPVC: present, initJob: missing, wantErr: true}, + {name: "durable selection missing both", selection: durable, rwPVC: missing, initJob: missing, wantErr: true}, + {name: "legacy request preserves missing-artifact fallback", rwPVC: missing, initJob: missing}, + {name: "none selection does not require artifacts", selection: none, rwPVC: missing, initJob: missing}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validatePersistedRegularModelCacheArtifacts(tt.selection, tt.rwPVC, tt.initJob) + if !tt.wantErr { + require.NoError(t, err) + return + } + require.ErrorContains(t, err, + "persisted durable regular model cache requires both PVC and init Job artifacts") + assert.True(t, nvcaerrors.IsTerminal(err)) + }) + } +} + +func TestSetupContainerModelCachingFailedOutcome(t *testing.T) { + resolved := resolvedSelectionForSetup(t) + durableRaw := persistedSelectionAnnotation( + t, nvcastorage.ModelCacheWorkflowRegular, nvcastorage.ModelCacheSelectionDurable, resolved) + defaultClass := nvcastorage.DefaultModelCacheStorageClassName + + tests := []struct { + name string + annotation string + wantErr string + wantTerminal bool + wantLegacyNoOp bool + }{ + { + name: "persisted durable execution error fails closed", + annotation: durableRaw, + wantErr: "forced PVC create failure", + wantTerminal: true, + }, + { + name: "legacy request falls back to uncached", + wantLegacyNoOp: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + k8sClient := fakek8sclient.NewSimpleClientset(selectionStorageClass()) + k8sClient.PrependReactor("create", "persistentvolumeclaims", + func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, fmt.Errorf("forced PVC create failure") + }) + backend := testContainerModelCacheBackend(k8sClient) + req := &nvcav2beta1.ICMSRequest{ + ObjectMeta: metav1.ObjectMeta{Name: "request", Namespace: RequestsNamespace}, + Spec: nvcav2beta1.ICMSRequestSpec{ + FunctionDetails: function.Details{FunctionVersionID: "function-version-id"}, + }, + } + var binding *nvcav2beta1.ModelCacheBinding + if tt.annotation != "" { + req.Annotations = map[string]string{ + nvcastorage.ModelCacheStorageSelectionAnnotationKey: tt.annotation, + } + binding = installActiveRegularModelCacheBinding(t, &backend, req, tt.annotation) + } + rwPVCName := "rw-pvc-cache" + initJobName := "writer-job-cache" + if binding != nil { + rwPVCName = binding.Spec.Resources.PersistentVolumeClaimNames[0] + initJobName = binding.Spec.Resources.JobNames[0] + } + rwPVC := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: rwPVCName, Namespace: RequestsNamespace}, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + StorageClassName: &defaultClass, + }, + } + initJob := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Name: initJobName, Namespace: RequestsNamespace}, + } + + mf, roPVCName, err := backend.setupContainerModelCaching( + newTestContext(), req, rwPVC, initJob, func(_ client.Object) {}) + if tt.wantErr != "" { + require.ErrorContains(t, err, tt.wantErr) + assert.Equal(t, tt.wantTerminal, nvcaerrors.IsTerminal(err)) + assert.Nil(t, mf) + assert.Empty(t, roPVCName) + } else { + require.NoError(t, err) + require.True(t, tt.wantLegacyNoOp) + require.NotNil(t, mf) + assert.Empty(t, roPVCName) + } + + createAttempts := 0 + for _, action := range k8sClient.Actions() { + if action.GetVerb() == "create" && action.GetResource().Resource == "persistentvolumeclaims" { + createAttempts++ + } + } + assert.Equal(t, 1, createAttempts, "the forced failure must produce ModelCachingFailed") + }) + } +} + +func TestRegularModelCacheRuntimeDecisionHonorsPersistedSelection(t *testing.T) { + resolved := resolvedSelectionForSetup(t) + durableRaw := persistedSelectionAnnotation( + t, nvcastorage.ModelCacheWorkflowRegular, nvcastorage.ModelCacheSelectionDurable, resolved) + noneRaw := persistedSelectionAnnotation( + t, nvcastorage.ModelCacheWorkflowRegular, nvcastorage.ModelCacheSelectionNone, nil) + + tests := []struct { + name string + raw string + legacyEnabled bool + wantEnabled bool + wantPersisted bool + wantErr bool + }{ + {name: "legacy gate enabled", legacyEnabled: true, wantEnabled: true}, + {name: "legacy gate disabled"}, + {name: "persisted durable survives gate disable", raw: durableRaw, wantEnabled: true, wantPersisted: true}, + {name: "persisted none survives gate enable", raw: noneRaw, legacyEnabled: true, wantPersisted: true}, + {name: "malformed selection fails closed", raw: "{", wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := &nvcav2beta1.ICMSRequest{} + if tt.raw != "" { + req.Annotations = map[string]string{nvcastorage.ModelCacheStorageSelectionAnnotationKey: tt.raw} + } + enabled, persisted, err := regularModelCacheRuntimeDecision(req, tt.legacyEnabled) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantEnabled, enabled) + assert.Equal(t, tt.wantPersisted, persisted) + }) + } +} diff --git a/src/compute-plane-services/nvca/pkg/operator/cleanup/BUILD.bazel b/src/compute-plane-services/nvca/pkg/operator/cleanup/BUILD.bazel index 6034f166c..c2e9817b0 100644 --- a/src/compute-plane-services/nvca/pkg/operator/cleanup/BUILD.bazel +++ b/src/compute-plane-services/nvca/pkg/operator/cleanup/BUILD.bazel @@ -60,5 +60,6 @@ go_test( "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/runtime/schema", "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/dynamic/fake", "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/kubernetes/fake", + "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/testing", ], ) diff --git a/src/compute-plane-services/nvca/pkg/operator/cleanup/cleanup.go b/src/compute-plane-services/nvca/pkg/operator/cleanup/cleanup.go index 1ad1189a9..a5ac2962a 100644 --- a/src/compute-plane-services/nvca/pkg/operator/cleanup/cleanup.go +++ b/src/compute-plane-services/nvca/pkg/operator/cleanup/cleanup.go @@ -22,6 +22,7 @@ package cleanup import ( "context" + "errors" "fmt" "strings" "time" @@ -122,6 +123,13 @@ func CleanupBackendResources( //nolint:revive // exported name is intentional return fmt.Errorf("failed to cleanup namespace %v, err: %v", requestsNS, err) } + // Strip binding finalizers before deleting the control namespace. The agent + // may already be stopped during uninstall, so no reconciler remains to do it. + if err := deleteModelCacheBindings(ctx, dynamicClient, DefaultModelCacheInitNamespace); err != nil { + return fmt.Errorf("failed to delete model-cache bindings in namespace %s: %w", + DefaultModelCacheInitNamespace, err) + } + // Cleanup the shared model-cache initialization namespace created by NVCA. err = k8sClient.CoreV1().Namespaces().Delete(ctx, DefaultModelCacheInitNamespace, metav1.DeleteOptions{}) if err != nil && !k8serrors.IsNotFound(err) { @@ -635,6 +643,63 @@ func deleteICMSRequests(ctx context.Context, dynamicClient dynamic.Interface, na return nil } +// deleteModelCacheBindings removes finalizers from every binding and deletes it during uninstall. +func deleteModelCacheBindings(ctx context.Context, dynamicClient dynamic.Interface, namespace string) error { + log := core.GetLogger(ctx) + var errs []error + bindingGVR := schema.GroupVersionResource{ + Group: "nvca.nvcf.nvidia.io", + Version: "v2beta1", + Resource: "modelcachebindings", + } + + list, err := dynamicClient.Resource(bindingGVR).Namespace(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + if k8serrors.IsNotFound(err) { + return nil + } + return fmt.Errorf("failed to list model-cache bindings: %w", err) + } + + for _, item := range list.Items { + name := item.GetName() + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + latest, err := dynamicClient.Resource(bindingGVR).Namespace(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + if k8serrors.IsNotFound(err) { + return nil + } + return err + } + if len(latest.GetFinalizers()) == 0 { + return nil + } + + latest.SetGroupVersionKind(schema.GroupVersionKind{ + Group: bindingGVR.Group, Version: bindingGVR.Version, Kind: "ModelCacheBinding", + }) + latest.SetFinalizers(nil) + _, err = dynamicClient.Resource(bindingGVR).Namespace(namespace).Update(ctx, latest, metav1.UpdateOptions{}) + return err + }) + if err != nil { + log.WithError(err).Warnf("failed to remove finalizers from model-cache binding %s/%s", namespace, name) + errs = append(errs, fmt.Errorf( + "remove finalizers from model-cache binding %s/%s: %w", namespace, name, err)) + continue + } + + err = dynamicClient.Resource(bindingGVR).Namespace(namespace).Delete(ctx, name, metav1.DeleteOptions{}) + if err != nil && !k8serrors.IsNotFound(err) { + log.WithError(err).Warnf("failed to delete model-cache binding %s/%s", namespace, name) + errs = append(errs, fmt.Errorf( + "delete model-cache binding %s/%s: %w", namespace, name, err)) + } + } + + return errors.Join(errs...) +} + // workloadNamespaceLabelSelector selects namespaces created by NVCA for workload instances. const workloadNamespaceLabelSelector = "nvca.nvcf.nvidia.io/workload-instance-type" diff --git a/src/compute-plane-services/nvca/pkg/operator/cleanup/cleanup_test.go b/src/compute-plane-services/nvca/pkg/operator/cleanup/cleanup_test.go index a2a0c3aa1..071dbb715 100644 --- a/src/compute-plane-services/nvca/pkg/operator/cleanup/cleanup_test.go +++ b/src/compute-plane-services/nvca/pkg/operator/cleanup/cleanup_test.go @@ -19,6 +19,7 @@ package cleanup import ( "context" + "errors" "testing" "time" @@ -33,11 +34,21 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" fakedynamic "k8s.io/client-go/dynamic/fake" "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" nvidiaiov1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1" fakenvcaop "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/client/clientset/versioned/fake" ) +var ( + testICMSRequestGVR = schema.GroupVersionResource{ + Group: "nvca.nvcf.nvidia.io", Version: "v2beta1", Resource: "icmsrequests", + } + testModelCacheBindingGVR = schema.GroupVersionResource{ + Group: "nvca.nvcf.nvidia.io", Version: "v2beta1", Resource: "modelcachebindings", + } +) + func TestBackendNamespaces(t *testing.T) { tests := []struct { name string @@ -1173,7 +1184,8 @@ func TestCleanupBackendResources(t *testing.T) { k8sClient := fake.NewSimpleClientset(objs...) dynamicClient := fakedynamic.NewSimpleDynamicClientWithCustomListKinds(scheme, map[schema.GroupVersionResource]string{ - icmsGVR: "ICMSRequestList", + icmsGVR: "ICMSRequestList", + testModelCacheBindingGVR: "ModelCacheBindingList", }) err := CleanupBackendResources(ctx, k8sClient, dynamicClient, tt.backend) @@ -1235,7 +1247,8 @@ func TestCleanupBackendResources_WithICMSRequests(t *testing.T) { ) dynamicClient := fakedynamic.NewSimpleDynamicClientWithCustomListKinds(scheme, map[schema.GroupVersionResource]string{ - icmsGVR: "ICMSRequestList", + icmsGVR: "ICMSRequestList", + testModelCacheBindingGVR: "ModelCacheBindingList", }, dynamicObjs...) @@ -1247,6 +1260,121 @@ func TestCleanupBackendResources_WithICMSRequests(t *testing.T) { assert.Empty(t, remaining.Items) } +func TestCleanupBackendResources_WithModelCacheBindings(t *testing.T) { + ctx := context.Background() + scheme := runtime.NewScheme() + icmsGVR := schema.GroupVersionResource{ + Group: "nvca.nvcf.nvidia.io", Version: "v2beta1", Resource: "icmsrequests", + } + bindingGVR := schema.GroupVersionResource{ + Group: "nvca.nvcf.nvidia.io", Version: "v2beta1", Resource: "modelcachebindings", + } + binding := &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": "nvca.nvcf.nvidia.io/v2beta1", + "kind": "ModelCacheBinding", + "metadata": map[string]interface{}{ + "name": "model-cache-test", + "namespace": DefaultModelCacheInitNamespace, + "finalizers": []interface{}{"nvca.nvcf.nvidia.io/model-cache-binding-finalizer"}, + }, + }} + k8sClient := fake.NewSimpleClientset( + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: DefaultNVCASystemNamespace}}, + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: DefaultNVCARequestsNamespace}}, + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: DefaultModelCacheInitNamespace}}, + ) + dynamicClient := fakedynamic.NewSimpleDynamicClientWithCustomListKinds( + scheme, + map[schema.GroupVersionResource]string{ + icmsGVR: "ICMSRequestList", + bindingGVR: "ModelCacheBindingList", + }, + binding, + ) + backend := &nvidiaiov1.NVCFBackend{ObjectMeta: metav1.ObjectMeta{Name: "test-backend"}} + + err := CleanupBackendResources(ctx, k8sClient, dynamicClient, backend) + require.NoError(t, err) + remaining, err := dynamicClient.Resource(bindingGVR).Namespace(DefaultModelCacheInitNamespace).List( + ctx, metav1.ListOptions{}) + require.NoError(t, err) + assert.Empty(t, remaining.Items) +} + +func TestCleanupBackendResources_PropagatesModelCacheBindingFailure(t *testing.T) { + for _, tt := range []struct { + name string + verb string + finalizers []interface{} + wantError string + }{ + { + name: "finalizer update fails", + verb: "update", + finalizers: []interface{}{"nvca.nvcf.nvidia.io/model-cache-binding-finalizer"}, + wantError: "remove finalizers from model-cache binding", + }, + { + name: "binding delete fails", + verb: "delete", + wantError: "delete model-cache binding", + }, + } { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + injected := errors.New("injected binding cleanup failure") + binding := &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": "nvca.nvcf.nvidia.io/v2beta1", + "kind": "ModelCacheBinding", + "metadata": map[string]interface{}{ + "name": "model-cache-test", + "namespace": DefaultModelCacheInitNamespace, + "finalizers": tt.finalizers, + }, + }} + dynamicClient := fakedynamic.NewSimpleDynamicClientWithCustomListKinds( + runtime.NewScheme(), + map[schema.GroupVersionResource]string{ + testICMSRequestGVR: "ICMSRequestList", + testModelCacheBindingGVR: "ModelCacheBindingList", + }, + binding, + ) + dynamicClient.PrependReactor( + tt.verb, + "modelcachebindings", + func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, injected + }, + ) + k8sClient := fake.NewSimpleClientset( + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: DefaultNVCASystemNamespace}}, + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: DefaultNVCARequestsNamespace}}, + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: DefaultModelCacheInitNamespace}}, + ) + + err := CleanupBackendResources( + ctx, + k8sClient, + dynamicClient, + &nvidiaiov1.NVCFBackend{ObjectMeta: metav1.ObjectMeta{Name: "test-backend"}}, + ) + + require.Error(t, err) + assert.ErrorIs(t, err, injected) + assert.ErrorContains(t, err, "failed to delete model-cache bindings in namespace") + assert.ErrorContains(t, err, tt.wantError) + _, getErr := dynamicClient.Resource(testModelCacheBindingGVR). + Namespace(DefaultModelCacheInitNamespace). + Get(ctx, binding.GetName(), metav1.GetOptions{}) + require.NoError(t, getErr, "failed cleanup must leave the binding for retry") + _, getErr = k8sClient.CoreV1().Namespaces().Get( + ctx, DefaultModelCacheInitNamespace, metav1.GetOptions{}) + require.NoError(t, getErr, "failed cleanup must retain the binding namespace") + }) + } +} + func TestCleanupBackendResources_WithWebhooks(t *testing.T) { ctx := context.Background() @@ -1271,7 +1399,8 @@ func TestCleanupBackendResources_WithWebhooks(t *testing.T) { ) dynamicClient := fakedynamic.NewSimpleDynamicClientWithCustomListKinds(scheme, map[schema.GroupVersionResource]string{ - icmsGVR: "ICMSRequestList", + icmsGVR: "ICMSRequestList", + testModelCacheBindingGVR: "ModelCacheBindingList", }) err := CleanupBackendResources(ctx, k8sClient, dynamicClient, backend) @@ -1515,7 +1644,8 @@ func TestCountICMSRequests_EmptyNamespace(t *testing.T) { dynamicClient := fakedynamic.NewSimpleDynamicClientWithCustomListKinds(scheme, map[schema.GroupVersionResource]string{ - icmsGVR: "ICMSRequestList", + icmsGVR: "ICMSRequestList", + testModelCacheBindingGVR: "ModelCacheBindingList", }) count, err := CountICMSRequests(ctx, dynamicClient, DefaultNVCARequestsNamespace) @@ -1718,7 +1848,8 @@ func TestCleanupBackendResources_WithWorkloadNamespaces(t *testing.T) { ) dynamicClient := fakedynamic.NewSimpleDynamicClientWithCustomListKinds(scheme, map[schema.GroupVersionResource]string{ - icmsGVR: "ICMSRequestList", + icmsGVR: "ICMSRequestList", + testModelCacheBindingGVR: "ModelCacheBindingList", }) err := CleanupBackendResources(ctx, k8sClient, dynamicClient, backend) diff --git a/src/compute-plane-services/nvca/pkg/operator/cleanup/shutdown.go b/src/compute-plane-services/nvca/pkg/operator/cleanup/shutdown.go index c2cb0fe31..16b9912f5 100644 --- a/src/compute-plane-services/nvca/pkg/operator/cleanup/shutdown.go +++ b/src/compute-plane-services/nvca/pkg/operator/cleanup/shutdown.go @@ -223,6 +223,11 @@ func RunShutdownCleanup(ctx context.Context, opts ShutdownHandlerOptions) Shutdo // Use shared cleanup to delete all managed resources if err := CleanupBackendResources(ctx, opts.K8sClient, opts.DynamicClient, nb); err != nil { log.WithError(err).Errorf("Failed to cleanup resources for NVCFBackend %s/%s", nb.Namespace, nb.Name) + return ShutdownResponse{ + Cleanup: true, + Message: "failed to cleanup NVCFBackend resources", + Error: err.Error(), + } } // Remove the finalizer from the NVCFBackend to allow it to be garbage collected diff --git a/src/compute-plane-services/nvca/pkg/operator/cleanup/shutdown_test.go b/src/compute-plane-services/nvca/pkg/operator/cleanup/shutdown_test.go index c889a0061..b99b62f04 100644 --- a/src/compute-plane-services/nvca/pkg/operator/cleanup/shutdown_test.go +++ b/src/compute-plane-services/nvca/pkg/operator/cleanup/shutdown_test.go @@ -20,6 +20,7 @@ package cleanup import ( "context" "encoding/json" + "errors" "net/http" "net/http/httptest" "testing" @@ -37,12 +38,13 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" fakedynamic "k8s.io/client-go/dynamic/fake" "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" nvidiaiov1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1" fakenvcaop "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/client/clientset/versioned/fake" ) -// newTestDynamicClient creates a fake dynamic client with ICMSRequest list kind registered +// newTestDynamicClient creates a fake dynamic client with NVCA list kinds registered. func newTestDynamicClient() *fakedynamic.FakeDynamicClient { scheme := runtime.NewScheme() icmsGVR := schema.GroupVersionResource{ @@ -52,7 +54,8 @@ func newTestDynamicClient() *fakedynamic.FakeDynamicClient { } return fakedynamic.NewSimpleDynamicClientWithCustomListKinds(scheme, map[schema.GroupVersionResource]string{ - icmsGVR: "ICMSRequestList", + icmsGVR: "ICMSRequestList", + testModelCacheBindingGVR: "ModelCacheBindingList", }) } @@ -277,6 +280,129 @@ func TestRunShutdownCleanup_RemovesManagedResources(t *testing.T) { assert.NotContains(t, crb.Finalizers, SentinelFinalizer) } +func TestRunShutdownCleanup_StopsBeforeFinalizerRemovalWhenBindingCleanupFails(t *testing.T) { + for _, tt := range []struct { + name string + verb string + finalizers []interface{} + }{ + { + name: "binding finalizer update fails", + verb: "update", + finalizers: []interface{}{"nvca.nvcf.nvidia.io/model-cache-binding-finalizer"}, + }, + { + name: "binding delete fails", + verb: "delete", + }, + } { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + injected := errors.New("injected binding cleanup failure") + sentinel := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + Name: ShutdownSentinelConfigMapName, + Namespace: "test-namespace", + Finalizers: []string{SentinelFinalizer}, + DeletionTimestamp: &metav1.Time{Time: time.Now()}, + }} + clusterRole := &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{ + Name: "test-operator", Finalizers: []string{SentinelFinalizer}, + }} + clusterRoleBinding := &rbacv1.ClusterRoleBinding{ObjectMeta: metav1.ObjectMeta{ + Name: "test-operator", Finalizers: []string{SentinelFinalizer}, + }} + backend := &nvidiaiov1.NVCFBackend{ObjectMeta: metav1.ObjectMeta{ + Name: "test-backend", Namespace: "test-namespace", Finalizers: []string{NVCAOperatorFinalizer}, + }} + binding := &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": "nvca.nvcf.nvidia.io/v2beta1", + "kind": "ModelCacheBinding", + "metadata": map[string]interface{}{ + "name": "model-cache-test", + "namespace": DefaultModelCacheInitNamespace, + "finalizers": tt.finalizers, + }, + }} + + k8sClient := fake.NewSimpleClientset( + sentinel, + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: DefaultNVCASystemNamespace}}, + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: DefaultNVCARequestsNamespace}}, + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: DefaultModelCacheInitNamespace}}, + clusterRole, + clusterRoleBinding, + ) + k8sClient.Fake.PrependReactor( + "delete", "configmaps", func(action k8stesting.Action) (bool, runtime.Object, error) { + deleteAction, ok := action.(k8stesting.DeleteAction) + if ok && deleteAction.GetName() == ShutdownSentinelConfigMapName { + return true, nil, nil + } + return false, nil, nil + }, + ) + nvcaClient := fakenvcaop.NewSimpleClientset(backend) + nvcaClient.Fake.PrependReactor( + "delete", "nvcfbackends", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, nil + }, + ) + dynamicClient := fakedynamic.NewSimpleDynamicClientWithCustomListKinds( + runtime.NewScheme(), + map[schema.GroupVersionResource]string{ + testICMSRequestGVR: "ICMSRequestList", + testModelCacheBindingGVR: "ModelCacheBindingList", + }, + binding, + ) + dynamicClient.PrependReactor( + tt.verb, + "modelcachebindings", + func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, injected + }, + ) + + resp := RunShutdownCleanup(ctx, ShutdownHandlerOptions{ + K8sClient: k8sClient, + NVCAClient: nvcaClient, + DynamicClient: dynamicClient, + Namespace: "test-namespace", + ClusterRoleName: clusterRole.Name, + ClusterRoleBindingName: clusterRoleBinding.Name, + SetGracefulShutdown: func(bool) {}, + }) + + require.True(t, resp.Cleanup) + assert.Equal(t, "failed to cleanup NVCFBackend resources", resp.Message) + assert.Contains(t, resp.Error, injected.Error()) + + storedBackend, err := nvcaClient.NvcfV1().NVCFBackends(backend.Namespace). + Get(ctx, backend.Name, metav1.GetOptions{}) + require.NoError(t, err) + assert.Contains(t, storedBackend.Finalizers, NVCAOperatorFinalizer) + storedSentinel, err := k8sClient.CoreV1().ConfigMaps(sentinel.Namespace). + Get(ctx, sentinel.Name, metav1.GetOptions{}) + require.NoError(t, err) + assert.Contains(t, storedSentinel.Finalizers, SentinelFinalizer) + storedRole, err := k8sClient.RbacV1().ClusterRoles().Get(ctx, clusterRole.Name, metav1.GetOptions{}) + require.NoError(t, err) + assert.Contains(t, storedRole.Finalizers, SentinelFinalizer) + storedRoleBinding, err := k8sClient.RbacV1().ClusterRoleBindings().Get( + ctx, clusterRoleBinding.Name, metav1.GetOptions{}) + require.NoError(t, err) + assert.Contains(t, storedRoleBinding.Finalizers, SentinelFinalizer) + _, err = dynamicClient.Resource(testModelCacheBindingGVR). + Namespace(DefaultModelCacheInitNamespace). + Get(ctx, binding.GetName(), metav1.GetOptions{}) + require.NoError(t, err) + _, err = k8sClient.CoreV1().Namespaces().Get( + ctx, DefaultModelCacheInitNamespace, metav1.GetOptions{}) + require.NoError(t, err) + }) + } +} + func TestNewShutdownHandler_DefaultTimeouts(t *testing.T) { ctx := context.Background() @@ -594,7 +720,8 @@ func TestNewShutdownHandler_WithV1ICMSRequests(t *testing.T) { } dynamicClient := fakedynamic.NewSimpleDynamicClientWithCustomListKinds(scheme, map[schema.GroupVersionResource]string{ - icmsGVR: "ICMSRequestList", + icmsGVR: "ICMSRequestList", + testModelCacheBindingGVR: "ModelCacheBindingList", }, icmsRequest) opts := ShutdownHandlerOptions{ @@ -683,7 +810,10 @@ func TestRunShutdownCleanup_StripsICMSRequestFinalizersAfterDrain(t *testing.T) }, } dynamicClient := fakedynamic.NewSimpleDynamicClientWithCustomListKinds(scheme, - map[schema.GroupVersionResource]string{icmsGVR: "ICMSRequestList"}, icmsRequest) + map[schema.GroupVersionResource]string{ + icmsGVR: "ICMSRequestList", + testModelCacheBindingGVR: "ModelCacheBindingList", + }, icmsRequest) // A drain timeout shorter than the fixed 5s poll interval in drainWorkloads // forces the "timeout reached, proceeding with forced cleanup" branch on diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/BUILD.bazel b/src/compute-plane-services/nvca/pkg/operator/reconcile/BUILD.bazel index bb8762e2c..c591e2dd1 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/BUILD.bazel +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/BUILD.bazel @@ -39,6 +39,7 @@ go_library( "manifests/netpol/allow-ingress-monitoring.yaml", "manifests/nvcf.nvidia.io_icmsrequests_crd.yaml", "manifests/nvcf.nvidia.io_miniservices_crd.yaml", + "manifests/nvcf.nvidia.io_modelcachebindings_crd.yaml", "manifests/nvcf.nvidia.io_storagerequests_crd.yaml", "manifests/otel_collector_config.yaml", "manifests/rbacTemplate.yaml", @@ -141,6 +142,7 @@ go_test( "sanitize_test.go", "setup_auth_client_secrets_extended_test.go", "setup_auth_client_secrets_test.go", + "storage_capabilities_configmap_test.go", "transport_tls_config_test.go", "vault_config_data_test.go", "version_detection_test.go", @@ -154,6 +156,7 @@ go_test( "//src/compute-plane-services/nvca/internal/envtest:crds", ], embed = [":reconcile"], + env_inherit = ["KUBEBUILDER_ASSETS"], embedsrcs = [ "testdata/netpols.yaml", "testdata/netpols_with_ddcs.yaml", @@ -172,6 +175,7 @@ go_test( "//src/compute-plane-services/nvca/internal/envtest", "//src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1", "//src/compute-plane-services/nvca/pkg/apis/nvcf/v1:nvcf", + "//src/compute-plane-services/nvca/pkg/client/clientset/versioned", "//src/compute-plane-services/nvca/pkg/client/clientset/versioned/fake", "//src/compute-plane-services/nvca/pkg/client/clientset/versioned/scheme", "//src/compute-plane-services/nvca/pkg/client/informers/externalversions", diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/backendk8scache.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/backendk8scache.go index 399095457..0d77c77c5 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/backendk8scache.go +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/backendk8scache.go @@ -61,6 +61,7 @@ import ( "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/metrics" nvcaopotel "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/otel" nvcaoptypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/types" + nvcastorage "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/storage" ) const ( @@ -576,7 +577,8 @@ func configMapUpdateForcesNVCAReconcile(name string) bool { nvcfGPUProfilingConfigMapName, nvcfBackendChartDefaultsConfigMapName, agentConfigMergeConfigMapName, - nvcaOperatorConfigMapName: + nvcaOperatorConfigMapName, + nvcastorage.StorageCapabilityConfigMapName: return true default: return false diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/backendk8scache_test.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/backendk8scache_test.go index dbb90ea7b..ddd36b133 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/backendk8scache_test.go +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/backendk8scache_test.go @@ -63,9 +63,15 @@ import ( "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/reconcile/clustermgmt" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/types" nvcaoptypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/types" + nvcastorage "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/storage" ) -var icmsGVK schema.GroupVersionKind +var ( + icmsGVK schema.GroupVersionKind + testModelCacheBindingGVR = schema.GroupVersionResource{ + Group: "nvca.nvcf.nvidia.io", Version: "v2beta1", Resource: "modelcachebindings", + } +) func init() { icmsCRD := makeICMSRequestCRD() @@ -85,7 +91,12 @@ func init() { &nvidiaiov1.NVCFBackendList{}, ) newDynamicClient = func(_ *runtime.Scheme, _ *rest.Config) (dynamic.Interface, error) { - return fakedynamic.NewSimpleDynamicClient(testScheme), nil + return fakedynamic.NewSimpleDynamicClientWithCustomListKinds( + testScheme, + map[schema.GroupVersionResource]string{ + testModelCacheBindingGVR: "ModelCacheBindingList", + }, + ), nil } newDiscoverClient = func(_ kubernetes.Interface, _ *rest.Config) (discovery.DiscoveryInterface, error) { @@ -473,6 +484,10 @@ func newTestScheme() *runtime.Scheme { s := runtime.NewScheme() s.AddKnownTypeWithName(icmsGVK, &nvcav2beta1.ICMSRequest{}) s.AddKnownTypeWithName(icmsGVK.GroupVersion().WithKind(icmsGVK.Kind+"List"), &nvcav2beta1.ICMSRequestList{}) + s.AddKnownTypeWithName( + nvcav2beta1.SchemeGroupVersion.WithKind("ModelCacheBinding"), &nvcav2beta1.ModelCacheBinding{}) + s.AddKnownTypeWithName( + nvcav2beta1.SchemeGroupVersion.WithKind("ModelCacheBindingList"), &nvcav2beta1.ModelCacheBindingList{}) return s } @@ -503,11 +518,16 @@ func mockKubeClientsForIntegrationTests() *kubeclients.KubeClients { panic(err) } return &kubeclients.KubeClients{ - Config: newRESTConfig(), - NVCAOP: fakenvcaopclient.NewSimpleClientset(), - K8s: k8sClient, - APIExtV1: fakeapiextensionclient.NewSimpleClientset().ApiextensionsV1(), - DynamicClient: fakedynamic.NewSimpleDynamicClient(scheme), + Config: newRESTConfig(), + NVCAOP: fakenvcaopclient.NewSimpleClientset(), + K8s: k8sClient, + APIExtV1: fakeapiextensionclient.NewSimpleClientset().ApiextensionsV1(), + DynamicClient: fakedynamic.NewSimpleDynamicClientWithCustomListKinds( + scheme, + map[schema.GroupVersionResource]string{ + testModelCacheBindingGVR: "ModelCacheBindingList", + }, + ), DiscoveryClient: discClient, DiscoveryRESTMapper: restmapper.NewDiscoveryRESTMapper(grs), } @@ -517,6 +537,13 @@ func mockKubeClients() *kubeclients.KubeClients { scheme := newTestScheme() k8sClient := fakek8sclient.NewSimpleClientset( newStaticGPUSConfigMap(), + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: nvcastorage.StorageCapabilityConfigMapName, + Namespace: NVCAOperatorNamespace, + }, + Data: map[string]string{nvcastorage.StorageCapabilityConfigMapKey: "test-catalog"}, + }, &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: nvcfCustomAnnotationsConfigMapName, @@ -549,11 +576,16 @@ func mockKubeClients() *kubeclients.KubeClients { panic(err) } return &kubeclients.KubeClients{ - Config: newRESTConfig(), - NVCAOP: fakenvcaopclient.NewSimpleClientset(), - K8s: k8sClient, - APIExtV1: fakeapiextensionclient.NewSimpleClientset().ApiextensionsV1(), - DynamicClient: fakedynamic.NewSimpleDynamicClient(scheme), + Config: newRESTConfig(), + NVCAOP: fakenvcaopclient.NewSimpleClientset(), + K8s: k8sClient, + APIExtV1: fakeapiextensionclient.NewSimpleClientset().ApiextensionsV1(), + DynamicClient: fakedynamic.NewSimpleDynamicClientWithCustomListKinds( + scheme, + map[schema.GroupVersionResource]string{ + testModelCacheBindingGVR: "ModelCacheBindingList", + }, + ), DiscoveryClient: discClient, DiscoveryRESTMapper: restmapper.NewDiscoveryRESTMapper(grs), } diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/crd_reconcile.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/crd_reconcile.go index 8c6233863..2ffe23637 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/crd_reconcile.go +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/crd_reconcile.go @@ -31,6 +31,8 @@ import ( ) var ( + //go:embed manifests/nvcf.nvidia.io_modelcachebindings_crd.yaml + modelCacheBindingsCRDData []byte //go:embed manifests/nvcf.nvidia.io_storagerequests_crd.yaml storageRequestsCRDData []byte //go:embed manifests/nvcf.nvidia.io_miniservices_crd.yaml @@ -40,6 +42,7 @@ var ( ) const ( + ModelCacheBindingCRDName = "modelcachebindings.nvca.nvcf.nvidia.io" StorageRequestCRDName = "storagerequests.nvca.nvcf.nvidia.io" MiniServicesCRDName = "miniservices.nvca.nvcf.nvidia.io" ICMSRequestCRDName = "icmsrequests.nvca.nvcf.nvidia.io" @@ -50,6 +53,10 @@ const ( func (c *BackendK8sCache) setupCRDs(ctx context.Context) error { log := core.GetLogger(ctx) + modelCacheBindingCRD, err := decodeCRD(modelCacheBindingsCRDData) + if err != nil { + return fmt.Errorf("make ModelCacheBinding CRD: %v", err) + } storageReqCRD, err := decodeCRD(storageRequestsCRDData) if err != nil { return fmt.Errorf("make StorageRequest CRD: %v", err) @@ -91,6 +98,7 @@ func (c *BackendK8sCache) setupCRDs(ctx context.Context) error { } } for _, crdObj := range []*apiextv1.CustomResourceDefinition{ + modelCacheBindingCRD, storageReqCRD, miniserviceCRD, makeICMSRequestCRD(), diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/crd_reconcile_test.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/crd_reconcile_test.go index f5571864f..ba1ea4599 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/crd_reconcile_test.go +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/crd_reconcile_test.go @@ -19,11 +19,17 @@ package operator import ( "context" + "strings" "testing" + nvcaenvtest "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/envtest" + nvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1" + nvcaclientset "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/client/clientset/versioned" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -38,6 +44,8 @@ func Test_setupCRDs(t *testing.T) { _, err = b.clients.APIExtV1.CustomResourceDefinitions().Get(ctx, ICMSRequestCRDName, metav1.GetOptions{}) require.NoError(t, err) + _, err = b.clients.APIExtV1.CustomResourceDefinitions().Get(ctx, ModelCacheBindingCRDName, metav1.GetOptions{}) + require.NoError(t, err) _, err = b.clients.APIExtV1.CustomResourceDefinitions().Get(ctx, "storagerequests.nvca.nvcf.nvidia.io", metav1.GetOptions{}) require.NoError(t, err) _, err = b.clients.APIExtV1.CustomResourceDefinitions().Get(ctx, "miniservices.nvca.nvcf.nvidia.io", metav1.GetOptions{}) @@ -49,12 +57,162 @@ func Test_setupCRDs(t *testing.T) { _, err = b.clients.APIExtV1.CustomResourceDefinitions().Get(ctx, ICMSRequestCRDName, metav1.GetOptions{}) require.NoError(t, err) + _, err = b.clients.APIExtV1.CustomResourceDefinitions().Get(ctx, ModelCacheBindingCRDName, metav1.GetOptions{}) + require.NoError(t, err) _, err = b.clients.APIExtV1.CustomResourceDefinitions().Get(ctx, "storagerequests.nvca.nvcf.nvidia.io", metav1.GetOptions{}) require.NoError(t, err) _, err = b.clients.APIExtV1.CustomResourceDefinitions().Get(ctx, "miniservices.nvca.nvcf.nvidia.io", metav1.GetOptions{}) require.NoError(t, err) } +func TestModelCacheBindingCRDContract(t *testing.T) { + crd, err := decodeCRD(modelCacheBindingsCRDData) + require.NoError(t, err) + assert.Equal(t, ModelCacheBindingCRDName, crd.Name) + assert.Equal(t, apiextv1.NamespaceScoped, crd.Spec.Scope) + require.Len(t, crd.Spec.Versions, 1) + + version := crd.Spec.Versions[0] + assert.Equal(t, "v2beta1", version.Name) + assert.True(t, version.Served) + assert.True(t, version.Storage) + require.NotNil(t, version.Subresources) + require.NotNil(t, version.Subresources.Status) + require.NotNil(t, version.Schema) + require.NotNil(t, version.Schema.OpenAPIV3Schema) + + root := version.Schema.OpenAPIV3Schema + specSchema := root.Properties["spec"] + require.Len(t, specSchema.XValidations, 1) + assert.Equal(t, "self == oldSelf", specSchema.XValidations[0].Rule) + + decisionSchema := specSchema.Properties["decision"] + assert.Equal(t, "^sha256:[a-f0-9]{64}$", decisionSchema.Properties["catalogDigest"].Pattern) + assert.Contains(t, decisionSchema.Required, "encryptionRequired") + requiredAccessModesSchema := decisionSchema.Properties["requiredAccessModes"] + require.NotNil(t, requiredAccessModesSchema.XListType) + assert.Equal(t, "set", *requiredAccessModesSchema.XListType) + + storageClassSchema := specSchema.Properties["storageClass"] + assert.Equal(t, "^v1:sha256:[a-f0-9]{64}$", + storageClassSchema.Properties["configurationDigest"].Pattern) + reclaimPolicySchema := storageClassSchema.Properties["reclaimPolicy"] + require.Len(t, reclaimPolicySchema.Enum, 1) + assert.JSONEq(t, `"Retain"`, string(reclaimPolicySchema.Enum[0].Raw)) + + resourceSchema := specSchema.Properties["resources"] + assert.Contains(t, resourceSchema.Required, "writerNamespace") + assert.NotContains(t, resourceSchema.Required, "leaseName") + assert.Contains(t, resourceSchema.Properties, "storageClassNames") + assert.Contains(t, resourceSchema.Properties, "secretNames") + for _, field := range []string{"persistentVolumeClaimNames", "persistentVolumeNames", "jobNames", "storageClassNames", "secretNames"} { + schema := resourceSchema.Properties[field] + require.NotNil(t, schema.XListType, field) + assert.Equal(t, "set", *schema.XListType, field) + } + + statusSchema := root.Properties["status"] + require.Len(t, statusSchema.XValidations, 2) + assert.Contains(t, statusSchema.XValidations[0].Rule, "Retiring") + assert.Contains(t, statusSchema.XValidations[1].Rule, "providerDataIdentity") + requestReferencesSchema := statusSchema.Properties["requestReferences"] + require.NotNil(t, requestReferencesSchema.XListType) + assert.Equal(t, "map", *requestReferencesSchema.XListType) +} + +func TestModelCacheBindingCRDEnforcement(t *testing.T) { + cfg, k8sClient, cleanup, err := nvcaenvtest.SetupEnvtest() + require.NoError(t, err) + t.Cleanup(cleanup) + + ctx := context.Background() + const namespace = "model-cache-binding-test" + _, err = k8sClient.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: namespace}, + }, metav1.CreateOptions{}) + require.NoError(t, err) + + nvcaClient, err := nvcaclientset.NewForConfig(cfg) + require.NoError(t, err) + bindings := nvcaClient.NvcaV2beta1().ModelCacheBindings(namespace) + + catalogDigest := "sha256:" + strings.Repeat("0", 64) + binding := &nvcav2beta1.ModelCacheBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "regular-cache", Namespace: namespace}, + Spec: nvcav2beta1.ModelCacheBindingSpec{ + Identity: nvcav2beta1.ModelCacheBindingIdentity{ + Version: "v1", + Workflow: nvcav2beta1.ModelCacheWorkflowRegular, + SharingDomainDigest: "sharing-domain", + CacheHandleDigest: "cache-handle", + }, + Decision: nvcav2beta1.ModelCacheBindingDecision{ + Provider: "nvmesh", + Provisioner: "nvmesh-csi", + Transition: "regular-rox", + RequiredAccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany}, + CatalogDigest: catalogDigest, + EncryptionRequired: false, + }, + StorageClass: nvcav2beta1.ModelCacheStorageClassSnapshot{ + Name: "nvcf-sc", + UID: "storage-class-uid", + ReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + ConfigurationDigest: "v1:" + catalogDigest, + }, + Resources: nvcav2beta1.ModelCacheBindingResourceIntent{ + WriterNamespace: "writer", + PersistentVolumeClaimNames: []string{"writer-pvc", "reader-pvc"}, + JobNames: []string{"writer-job"}, + }, + }, + } + + created, err := bindings.Create(ctx, binding, metav1.CreateOptions{}) + require.NoError(t, err) + assert.Empty(t, created.Spec.Resources.LeaseName, + "regular cache bindings must not claim a nonexistent Lease") + + specChange := created.DeepCopy() + specChange.Spec.Decision.Provider = "different-provider" + _, err = bindings.Update(ctx, specChange, metav1.UpdateOptions{}) + require.True(t, k8serrors.IsInvalid(err), "immutable spec update returned %v", err) + + statusWrite := created.DeepCopy() + statusWrite.Spec.Decision.Provider = "ignored-by-status-subresource" + statusWrite.Status.Phase = nvcav2beta1.ModelCacheBindingPhaseRetiring + statusWrite.Status.Realized = &nvcav2beta1.ModelCacheBindingRealizedState{ + ProviderDataIdentity: "provider-data-1", + PopulationState: nvcav2beta1.ModelCachePopulationReady, + } + statusUpdated, err := bindings.UpdateStatus(ctx, statusWrite, metav1.UpdateOptions{}) + require.NoError(t, err) + assert.Equal(t, "nvmesh", statusUpdated.Spec.Decision.Provider, + "status updates must not mutate spec") + + phaseReversal := statusUpdated.DeepCopy() + phaseReversal.Status.Phase = nvcav2beta1.ModelCacheBindingPhaseActive + _, err = bindings.UpdateStatus(ctx, phaseReversal, metav1.UpdateOptions{}) + require.True(t, k8serrors.IsInvalid(err), "Retiring-to-Active update returned %v", err) + + providerIdentityChange := statusUpdated.DeepCopy() + providerIdentityChange.Status.Realized.ProviderDataIdentity = "provider-data-2" + _, err = bindings.UpdateStatus(ctx, providerIdentityChange, metav1.UpdateOptions{}) + require.True(t, k8serrors.IsInvalid(err), "provider identity update returned %v", err) + + bareCatalogDigest := binding.DeepCopy() + bareCatalogDigest.Name = "bare-catalog-digest" + bareCatalogDigest.Spec.Decision.CatalogDigest = strings.Repeat("0", 64) + _, err = bindings.Create(ctx, bareCatalogDigest, metav1.CreateOptions{}) + require.True(t, k8serrors.IsInvalid(err), "bare catalog digest returned %v", err) + + bareStorageClassDigest := binding.DeepCopy() + bareStorageClassDigest.Name = "bare-storage-class-digest" + bareStorageClassDigest.Spec.StorageClass.ConfigurationDigest = strings.Repeat("0", 64) + _, err = bindings.Create(ctx, bareStorageClassDigest, metav1.CreateOptions{}) + require.True(t, k8serrors.IsInvalid(err), "bare StorageClass digest returned %v", err) +} + func Test_setupCRDs_migrateMiniService(t *testing.T) { b := &BackendK8sCache{ clients: mockKubeClients(), diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/manifests/nvcf.nvidia.io_modelcachebindings_crd.yaml b/src/compute-plane-services/nvca/pkg/operator/reconcile/manifests/nvcf.nvidia.io_modelcachebindings_crd.yaml new file mode 100644 index 000000000..6fb784428 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/manifests/nvcf.nvidia.io_modelcachebindings_crd.yaml @@ -0,0 +1,232 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: modelcachebindings.nvca.nvcf.nvidia.io +spec: + group: nvca.nvcf.nvidia.io + names: + kind: ModelCacheBinding + listKind: ModelCacheBindingList + plural: modelcachebindings + singular: modelcachebinding + shortNames: [mcb] + scope: Namespaced + versions: + - name: v2beta1 + served: true + storage: true + subresources: + status: {} + additionalPrinterColumns: + - name: Workflow + jsonPath: .spec.identity.workflow + type: string + - name: Provider + jsonPath: .spec.decision.provider + type: string + - name: Phase + jsonPath: .status.phase + type: string + - name: Age + jsonPath: .metadata.creationTimestamp + type: date + schema: + openAPIV3Schema: + type: object + required: [spec] + properties: + spec: + type: object + required: [identity, decision, storageClass, resources] + x-kubernetes-validations: + - rule: self == oldSelf + message: spec is immutable + properties: + identity: + type: object + required: [version, workflow, sharingDomainDigest, cacheHandleDigest] + properties: + version: + type: string + minLength: 1 + workflow: + type: string + enum: [regularModelCache, helmModelCache] + sharingDomainDigest: + type: string + minLength: 1 + cacheHandleDigest: + type: string + minLength: 1 + decision: + type: object + required: + - provider + - provisioner + - transition + - requiredAccessModes + - catalogDigest + - encryptionRequired + properties: + provider: + type: string + minLength: 1 + provisioner: + type: string + minLength: 1 + transition: + type: string + minLength: 1 + requiredAccessModes: + type: array + minItems: 1 + x-kubernetes-list-type: set + items: + type: string + enum: [ReadWriteOnce, ReadOnlyMany, ReadWriteMany, ReadWriteOncePod] + requiredMountOptions: + type: array + x-kubernetes-list-type: atomic + items: + type: string + minLength: 1 + catalogDigest: + type: string + pattern: ^sha256:[a-f0-9]{64}$ + encryptionRequired: + type: boolean + storageClass: + type: object + required: [name, uid, reclaimPolicy, configurationDigest] + properties: + name: + type: string + minLength: 1 + uid: + type: string + minLength: 1 + reclaimPolicy: + type: string + enum: [Retain] + configurationDigest: + type: string + pattern: ^v1:sha256:[a-f0-9]{64}$ + resources: + type: object + required: [writerNamespace] + properties: + writerNamespace: + type: string + minLength: 1 + persistentVolumeClaimNames: + type: array + x-kubernetes-list-type: set + items: + type: string + minLength: 1 + persistentVolumeNames: + type: array + x-kubernetes-list-type: set + items: + type: string + minLength: 1 + jobNames: + type: array + x-kubernetes-list-type: set + items: + type: string + minLength: 1 + storageClassNames: + type: array + x-kubernetes-list-type: set + items: + type: string + minLength: 1 + secretNames: + type: array + x-kubernetes-list-type: set + items: + type: string + minLength: 1 + leaseName: + type: string + minLength: 1 + status: + type: object + x-kubernetes-validations: + - rule: "!has(oldSelf.phase) || oldSelf.phase != 'Retiring' || self.phase == 'Retiring'" + message: a Retiring binding cannot become Active + - rule: >- + !has(oldSelf.realized) || !has(oldSelf.realized.providerDataIdentity) || + (has(self.realized) && has(self.realized.providerDataIdentity) && + self.realized.providerDataIdentity == oldSelf.realized.providerDataIdentity) + message: providerDataIdentity is immutable once recorded + properties: + phase: + type: string + enum: [Active, Retiring] + lastPhaseTransitionTime: + type: string + format: date-time + requestReferences: + type: array + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: [uid] + items: + type: object + required: [namespace, name, uid] + properties: + namespace: + type: string + minLength: 1 + name: + type: string + minLength: 1 + uid: + type: string + minLength: 1 + realized: + type: object + properties: + boundPersistentVolumeName: + type: string + minLength: 1 + providerDataIdentity: + type: string + minLength: 1 + populationState: + type: string + enum: [Pending, Populating, Ready, Failed] + conditions: + type: array + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: [type] + items: + type: object + required: [lastTransitionTime, message, reason, status, type] + properties: + lastTransitionTime: + type: string + format: date-time + message: + type: string + maxLength: 32768 + observedGeneration: + type: integer + format: int64 + minimum: 0 + reason: + type: string + minLength: 1 + maxLength: 1024 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + status: + type: string + enum: ["True", "False", Unknown] + type: + type: string + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9_.]*[a-z0-9])?\.)*[A-Za-z0-9]([-A-Za-z0-9_.]*[A-Za-z0-9])?$ diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/nvcaagent_reconcile.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/nvcaagent_reconcile.go index 113ba3901..5208c2c20 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/nvcaagent_reconcile.go +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/nvcaagent_reconcile.go @@ -58,6 +58,7 @@ import ( nvcaoperatorerrors "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/internal/errors" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/reconcile/clustermgmt" nvcaoptypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/types" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/storage" nvcatypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/types" ) @@ -575,6 +576,12 @@ func (bc *BackendK8sCache) setupNVCAAgentInfra( nb.Namespace, nb.Name, err) } + err = bc.mirrorConfigMap(ctx, nb, storage.StorageCapabilityConfigMapName) + if err != nil { + return fmt.Errorf("failed to setup %v for NVCFBackend %v/%v, err: %w", storage.StorageCapabilityConfigMapName, + nb.Namespace, nb.Name, err) + } + err = bc.mirrorConfigMap(ctx, nb, nvcfCustomAnnotationsConfigMapName) if err != nil { return fmt.Errorf("failed to setup %v for NVCFBackend %v/%v, err: %w", nvcfCustomAnnotationsConfigMapName, @@ -776,8 +783,11 @@ func (bc *BackendK8sCache) setupNVCARBAC(ctx context.Context, nb *nvidiaiov1.NVC }, { APIGroups: []string{"nvca.nvcf.nvidia.io"}, - Resources: []string{"storagerequests", "storagerequests/status"}, - Verbs: crudVerbs, + Resources: []string{ + "modelcachebindings", "modelcachebindings/status", + "storagerequests", "storagerequests/status", + }, + Verbs: crudVerbs, }, { APIGroups: []string{"storage.k8s.io"}, @@ -986,7 +996,11 @@ func (bc *BackendK8sCache) setupNVCARBAC(ctx context.Context, nb *nvidiaiov1.NVC func (bc *BackendK8sCache) mirrorConfigMap(ctx context.Context, nb *nvidiaiov1.NVCFBackend, srcName string) error { log := core.GetLogger(ctx) - srcCM, err := bc.clients.K8s.CoreV1().ConfigMaps(NVCAOperatorNamespace).Get(ctx, srcName, metav1.GetOptions{}) + operatorNamespace := bc.operatorNamespace + if operatorNamespace == "" { + operatorNamespace = NVCAOperatorNamespace + } + srcCM, err := bc.clients.K8s.CoreV1().ConfigMaps(operatorNamespace).Get(ctx, srcName, metav1.GetOptions{}) if err != nil { log.Errorf("failed to get source configmap %v", srcName) return err diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/nvcaagent_reconcile_test.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/nvcaagent_reconcile_test.go index 145510c09..74a3204d3 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/nvcaagent_reconcile_test.go +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/nvcaagent_reconcile_test.go @@ -1588,6 +1588,8 @@ func Test_setupNVCARBAC(t *testing.T) { { APIGroups: []string{"nvca.nvcf.nvidia.io"}, Resources: []string{ + "modelcachebindings", + "modelcachebindings/status", "storagerequests", "storagerequests/status", }, @@ -2008,6 +2010,8 @@ func Test_setupNVCARBAC_ValidationPolicy(t *testing.T) { { APIGroups: []string{"nvca.nvcf.nvidia.io"}, Resources: []string{ + "modelcachebindings", + "modelcachebindings/status", "storagerequests", "storagerequests/status", }, @@ -2242,6 +2246,8 @@ func Test_NVLinkOptimized(t *testing.T) { { APIGroups: []string{"nvca.nvcf.nvidia.io"}, Resources: []string{ + "modelcachebindings", + "modelcachebindings/status", "storagerequests", "storagerequests/status", }, diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/storage_capabilities_configmap_test.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/storage_capabilities_configmap_test.go new file mode 100644 index 000000000..413580c90 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/storage_capabilities_configmap_test.go @@ -0,0 +1,98 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package operator + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + k8serr "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + nvidiaiov1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvcf/v1" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/storage" +) + +func TestMirrorStorageCapabilitiesConfigMap(t *testing.T) { + const operatorNamespace = "custom-operator-system" + nb := &nvidiaiov1.NVCFBackend{ + ObjectMeta: metav1.ObjectMeta{Name: "test-backend", Namespace: operatorNamespace}, + Spec: nvidiaiov1.NVCFBackendSpec{ + NVCFBackendSpecT: nvidiaiov1.NVCFBackendSpecT{ + ClusterConfig: nvidiaiov1.ClusterConfig{SystemNamespace: "custom-agent-system"}, + }, + }, + } + + t.Run("mirrors from the configured operator namespace and updates the agent copy", func(t *testing.T) { + ctx := newTestContext() + clients := mockKubeClientsForIntegrationTests() + bc := &BackendK8sCache{clients: clients, operatorNamespace: operatorNamespace} + + src := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: storage.StorageCapabilityConfigMapName, + Namespace: operatorNamespace, + }, + Data: map[string]string{storage.StorageCapabilityConfigMapKey: "version: v1alpha1"}, + } + _, err := clients.K8s.CoreV1().ConfigMaps(operatorNamespace).Create(ctx, src, metav1.CreateOptions{}) + require.NoError(t, err) + + err = bc.mirrorConfigMap(ctx, nb, storage.StorageCapabilityConfigMapName) + require.NoError(t, err) + + agentNamespace := getSystemNamespace(nb) + mirrored, err := clients.K8s.CoreV1().ConfigMaps(agentNamespace).Get( + ctx, storage.StorageCapabilityConfigMapName, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, src.Data, mirrored.Data) + + src.Data[storage.StorageCapabilityConfigMapKey] = "version: v1alpha1\nproviders: []" + _, err = clients.K8s.CoreV1().ConfigMaps(operatorNamespace).Update(ctx, src, metav1.UpdateOptions{}) + require.NoError(t, err) + + err = bc.mirrorConfigMap(ctx, nb, storage.StorageCapabilityConfigMapName) + require.NoError(t, err) + mirrored, err = clients.K8s.CoreV1().ConfigMaps(agentNamespace).Get( + ctx, storage.StorageCapabilityConfigMapName, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, src.Data, mirrored.Data) + + lastGood := mirrored.Data + require.NoError(t, clients.K8s.CoreV1().ConfigMaps(operatorNamespace).Delete( + ctx, storage.StorageCapabilityConfigMapName, metav1.DeleteOptions{})) + err = bc.mirrorConfigMap(ctx, nb, storage.StorageCapabilityConfigMapName) + assert.True(t, k8serr.IsNotFound(err)) + mirrored, err = clients.K8s.CoreV1().ConfigMaps(agentNamespace).Get( + ctx, storage.StorageCapabilityConfigMapName, metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, lastGood, mirrored.Data) + }) + + t.Run("missing source fails closed", func(t *testing.T) { + ctx := newTestContext() + clients := mockKubeClientsForIntegrationTests() + bc := &BackendK8sCache{clients: clients, operatorNamespace: operatorNamespace} + + err := bc.mirrorConfigMap(ctx, nb, storage.StorageCapabilityConfigMapName) + assert.True(t, k8serr.IsNotFound(err)) + }) +} diff --git a/src/compute-plane-services/nvca/pkg/operator/reconcile/transport_tls_config_test.go b/src/compute-plane-services/nvca/pkg/operator/reconcile/transport_tls_config_test.go index 2b6b063c8..dc164f20d 100644 --- a/src/compute-plane-services/nvca/pkg/operator/reconcile/transport_tls_config_test.go +++ b/src/compute-plane-services/nvca/pkg/operator/reconcile/transport_tls_config_test.go @@ -32,6 +32,7 @@ import ( "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/internal/kubeclients" nvcaopotel "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/otel" nvcaoptypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/operator/types" + nvcastorage "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/storage" nvcaconfig "github.com/NVIDIA/nvcf/src/libraries/go/lib/pkg/types/nvca/config" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" @@ -898,9 +899,91 @@ func TestConfigMapChangesForceNVCAReconcile(t *testing.T) { assert.True(t, configMapUpdateForcesNVCAReconcile(nvcaOperatorConfigMapName)) assert.True(t, configMapUpdateForcesNVCAReconcile(nvcfBackendChartDefaultsConfigMapName)) assert.True(t, configMapUpdateForcesNVCAReconcile(agentConfigMergeConfigMapName)) + assert.True(t, configMapUpdateForcesNVCAReconcile(nvcastorage.StorageCapabilityConfigMapName)) assert.False(t, configMapUpdateForcesNVCAReconcile("unrelated-configmap")) } +func TestStorageCapabilityConfigMapEventsForceNVCAReconcile(t *testing.T) { + for _, tt := range []struct { + name string + handle func(context.Context, *BackendK8sCache) error + wantForce bool + }{ + { + name: "add after informer sync", + handle: func(ctx context.Context, bc *BackendK8sCache) error { + bc.syncedFuncs = []cache.InformerSynced{func() bool { return true }} + bc.configMapHandlerRegistration = testResourceEventHandlerRegistration{synced: true} + return bc.handleConfigMapAdd(ctx, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: nvcastorage.StorageCapabilityConfigMapName}, + }) + }, + wantForce: true, + }, + { + name: "changed update", + handle: func(ctx context.Context, bc *BackendK8sCache) error { + return bc.handleConfigMapUpdate(ctx, + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: nvcastorage.StorageCapabilityConfigMapName}, + Data: map[string]string{nvcastorage.StorageCapabilityConfigMapKey: "before"}, + }, + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: nvcastorage.StorageCapabilityConfigMapName}, + Data: map[string]string{nvcastorage.StorageCapabilityConfigMapKey: "after"}, + }, + ) + }, + wantForce: true, + }, + { + name: "delete", + handle: func(ctx context.Context, bc *BackendK8sCache) error { + return bc.handleConfigMapDelete(ctx, cache.DeletedFinalStateUnknown{ + Obj: &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: nvcastorage.StorageCapabilityConfigMapName}, + }, + }) + }, + wantForce: true, + }, + { + name: "unchanged update", + handle: func(ctx context.Context, bc *BackendK8sCache) error { + unchanged := map[string]string{nvcastorage.StorageCapabilityConfigMapKey: "same"} + return bc.handleConfigMapUpdate(ctx, + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: nvcastorage.StorageCapabilityConfigMapName}, + Data: unchanged, + }, + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: nvcastorage.StorageCapabilityConfigMapName}, + Data: map[string]string{nvcastorage.StorageCapabilityConfigMapKey: "same"}, + }, + ) + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + ctx := newTestContext() + bc, backend := newConfigMapEventTestCache(t, ctx) + + err := tt.handle(ctx, bc) + if tt.wantForce { + require.ErrorContains(t, err, "version cannot be empty") + } else { + require.NoError(t, err) + } + + stored, getErr := bc.clients.NVCAOP.NvcfV1().NVCFBackends(NVCAOperatorNamespace). + Get(ctx, backend.Name, metav1.GetOptions{}) + require.NoError(t, getErr) + assert.Equal(t, tt.wantForce, + containsString(stored.Finalizers, cleanup.NVCAOperatorFinalizer)) + }) + } +} + func newConfigMapEventTestCache(t *testing.T, ctx context.Context) (*BackendK8sCache, *nvidiaiov1.NVCFBackend) { t.Helper() clients := mockKubeClientsForIntegrationTests() diff --git a/src/compute-plane-services/nvca/pkg/operator/types/types.go b/src/compute-plane-services/nvca/pkg/operator/types/types.go index ed72286ee..19ee90de9 100644 --- a/src/compute-plane-services/nvca/pkg/operator/types/types.go +++ b/src/compute-plane-services/nvca/pkg/operator/types/types.go @@ -46,8 +46,9 @@ const ( DefaultNVCARequestsNamespace = "nvcf-backend" // CRD names - StorageRequestCRDName = "storagerequests.nvca.nvcf.nvidia.io" - MiniServicesCRDName = "miniservices.nvca.nvcf.nvidia.io" + ModelCacheBindingCRDName = "modelcachebindings.nvca.nvcf.nvidia.io" + StorageRequestCRDName = "storagerequests.nvca.nvcf.nvidia.io" + MiniServicesCRDName = "miniservices.nvca.nvcf.nvidia.io" ) const ( diff --git a/src/compute-plane-services/nvca/pkg/storage/BUILD.bazel b/src/compute-plane-services/nvca/pkg/storage/BUILD.bazel index d923e8571..083417669 100644 --- a/src/compute-plane-services/nvca/pkg/storage/BUILD.bazel +++ b/src/compute-plane-services/nvca/pkg/storage/BUILD.bazel @@ -11,9 +11,12 @@ go_library( "controller.go", "controller_modelcache.go", "modelcache.go", + "modelcache_binding.go", + "modelcache_binding_helm.go", "modelcache_cleanup.go", "modelcache_nvmesh_client_status.go", "modelcache_nvmesh_encrypt.go", + "modelcache_selection.go", "persistentstorage.go", "reconcile.go", "sharedstorage.go", @@ -62,6 +65,7 @@ go_library( "//src/compute-plane-services/nvca/vendor/k8s.io/api/networking/v1:networking", "//src/compute-plane-services/nvca/vendor/k8s.io/api/rbac/v1:rbac", "//src/compute-plane-services/nvca/vendor/k8s.io/api/storage/v1:storage", + "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/api/equality", "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/api/errors", "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/api/meta", "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/api/resource", @@ -71,10 +75,13 @@ go_library( "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/runtime/schema", "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/runtime/serializer", "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/selection", + "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/types", "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/util/intstr", "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/util/runtime", "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/util/sets", "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/util/strategicpatch", + "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/util/validation", + "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/kubernetes", "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/tools/record", "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/util/retry", "//src/compute-plane-services/nvca/vendor/sigs.k8s.io/controller-runtime", @@ -102,8 +109,12 @@ go_test( "cachebackend_samba_test.go", "cachebackend_test.go", "controller_test.go", + "modelcache_binding_helm_test.go", + "modelcache_binding_test.go", "modelcache_cleanup_test.go", "modelcache_nvmesh_encrypt_test.go", + "modelcache_reprovision_test.go", + "modelcache_selection_test.go", "modelcache_test.go", "persistentstorage_test.go", "reconcile_modelcache_test.go", @@ -111,6 +122,7 @@ go_test( "sharedstorage_test.go", "smbcsidriver_test.go", "storage_capabilities_test.go", + "storage_resolution_test.go", "storage_request_api_test.go", "storagerequest_test.go", "translate_workload_test.go", @@ -174,6 +186,7 @@ go_test( "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/util/runtime", "//src/compute-plane-services/nvca/vendor/k8s.io/apimachinery/pkg/util/sets", "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/rest", + "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/kubernetes/fake", "//src/compute-plane-services/nvca/vendor/k8s.io/client-go/testing", "//src/compute-plane-services/nvca/vendor/sigs.k8s.io/controller-runtime", "//src/compute-plane-services/nvca/vendor/sigs.k8s.io/controller-runtime/pkg/client", diff --git a/src/compute-plane-services/nvca/pkg/storage/cachebackend.go b/src/compute-plane-services/nvca/pkg/storage/cachebackend.go index 1dcdde29e..5328fb343 100644 --- a/src/compute-plane-services/nvca/pkg/storage/cachebackend.go +++ b/src/compute-plane-services/nvca/pkg/storage/cachebackend.go @@ -79,7 +79,50 @@ func ModelCacheStorageClassName(override string) string { return DefaultModelCacheStorageClassName } -// SelectHelmCacheBackend resolves the Helm model-cache storage backend. All +// HelmCacheBackendFromSelection maps an immutable request selection to the +// existing Helm execution path. +func HelmCacheBackendFromSelection( + selection *PersistedModelCacheStorageSelection, +) (HelmCacheBackend, error) { + if err := selection.Validate(); err != nil { + return "", err + } + if selection.Workflow != ModelCacheWorkflowHelm { + return "", fmt.Errorf("model cache selection workflow %q is not Helm", selection.Workflow) + } + switch selection.Mode { + case ModelCacheSelectionNone: + return HelmCacheBackendNone, nil + case ModelCacheSelectionEphemeral: + return HelmCacheBackendEphemeral, nil + case ModelCacheSelectionDurable: + switch selection.Transition { + case ModelCacheTransitionROXReadOnly: + return HelmCacheBackendNVMesh, nil + default: + return "", fmt.Errorf("unsupported durable Helm model cache transition %q", selection.Transition) + } + default: + return "", fmt.Errorf("unsupported Helm model cache selection mode %q", selection.Mode) + } +} + +// PersistedHelmCacheBackend parses the coarse backend stored on a legacy +// StorageRequest. Empty means NVMesh for backward compatibility. +func PersistedHelmCacheBackend(raw string) (HelmCacheBackend, error) { + backend := HelmCacheBackend(raw) + if backend == "" { + return HelmCacheBackendNVMesh, nil + } + switch backend { + case HelmCacheBackendNVMesh, HelmCacheBackendSharedFS, HelmCacheBackendSamba: + return backend, nil + default: + return "", fmt.Errorf("unsupported persisted Helm model cache backend %q", raw) + } +} + +// SelectLegacyHelmCacheBackend resolves the legacy Helm model-cache storage backend. All // caching is gated on CachingSupport plus the HelmModelCaching sub-gate; the // mechanism is then chosen by which storage class the cluster provides, // falling back to Samba (when HelmSharedStorage is enabled and the block class @@ -93,7 +136,7 @@ func ModelCacheStorageClassName(override string) string { // modelCacheStorageClass is Agent.ModelCache.StorageClassName, the same config // value the storage controller provisions model cache volumes with; empty // resolves to the default. -func SelectHelmCacheBackend( +func SelectLegacyHelmCacheBackend( ctx context.Context, c client.Client, ff featureflag.Fetcher, diff --git a/src/compute-plane-services/nvca/pkg/storage/cachebackend_samba.go b/src/compute-plane-services/nvca/pkg/storage/cachebackend_samba.go index 7a6e1facc..404714812 100644 --- a/src/compute-plane-services/nvca/pkg/storage/cachebackend_samba.go +++ b/src/compute-plane-services/nvca/pkg/storage/cachebackend_samba.go @@ -146,7 +146,7 @@ type SambaModelCacheInfraState struct { // intentionally creates NO StorageClass; cache volumes are static SMB PVs bound // to the per-handle share (see newSambaModelCachePV). // -// dataStorageClass must be the class SelectHelmCacheBackend verified exists +// dataStorageClass must be the class SelectLegacyHelmCacheBackend verified exists // before choosing this backend; empty resolves to the default. func EnsureSambaModelCacheInfra( ctx context.Context, diff --git a/src/compute-plane-services/nvca/pkg/storage/cachebackend_samba_test.go b/src/compute-plane-services/nvca/pkg/storage/cachebackend_samba_test.go index 1ffceccbb..206fe94b9 100644 --- a/src/compute-plane-services/nvca/pkg/storage/cachebackend_samba_test.go +++ b/src/compute-plane-services/nvca/pkg/storage/cachebackend_samba_test.go @@ -146,7 +146,7 @@ func TestEnsureSambaModelCacheInfra_ReportsDeploymentCreationTime(t *testing.T) // TestEnsureSambaModelCacheInfra_BackingPVCStorageClass proves the backing PVC // lands on the configured model cache class, so the class -// SelectHelmCacheBackend checked for existence is the one the volume needs, and +// SelectLegacyHelmCacheBackend checked for existence is the one the volume needs, and // that an empty override resolves to the default rather than leaving the field // empty (which would silently pick the cluster's default StorageClass). func TestEnsureSambaModelCacheInfra_BackingPVCStorageClass(t *testing.T) { diff --git a/src/compute-plane-services/nvca/pkg/storage/cachebackend_test.go b/src/compute-plane-services/nvca/pkg/storage/cachebackend_test.go index 9c9dc0a1f..18718ecc5 100644 --- a/src/compute-plane-services/nvca/pkg/storage/cachebackend_test.go +++ b/src/compute-plane-services/nvca/pkg/storage/cachebackend_test.go @@ -50,7 +50,7 @@ func cacheBackendClient(t *testing.T, scs ...*storagev1.StorageClass) *fake.Clie return b } -func TestSelectHelmCacheBackend(t *testing.T) { +func TestSelectLegacyHelmCacheBackend(t *testing.T) { cachingOnly := []*featureflag.FeatureFlag{ featureflag.CachingSupport, featureflag.HelmModelCaching, @@ -178,7 +178,7 @@ func TestSelectHelmCacheBackend(t *testing.T) { c := cacheBackendClient(t, tt.storageClasses...).Build() ff := &featureflagmock.Fetcher{EnabledFFs: tt.flags} - got, err := SelectHelmCacheBackend(t.Context(), c, ff, tt.modelCacheClass) + got, err := SelectLegacyHelmCacheBackend(t.Context(), c, ff, tt.modelCacheClass) require.NoError(t, err) assert.Equal(t, tt.want, got) }) @@ -190,11 +190,39 @@ func TestModelCacheStorageClassNameResolution(t *testing.T) { assert.Equal(t, "custom-block-sc", ModelCacheStorageClassName("custom-block-sc")) } -// TestSelectHelmCacheBackend_SambaClassLookupError proves a failed lookup of the +func TestPersistedHelmCacheBackend(t *testing.T) { + for _, tt := range []struct { + name string + raw string + want HelmCacheBackend + wantErr bool + }{ + {name: "empty legacy value", want: HelmCacheBackendNVMesh}, + {name: "NVMesh", raw: string(HelmCacheBackendNVMesh), want: HelmCacheBackendNVMesh}, + {name: "shared filesystem", raw: string(HelmCacheBackendSharedFS), want: HelmCacheBackendSharedFS}, + {name: "Samba", raw: string(HelmCacheBackendSamba), want: HelmCacheBackendSamba}, + {name: "none is not durable", raw: string(HelmCacheBackendNone), wantErr: true}, + {name: "ephemeral is not durable", raw: string(HelmCacheBackendEphemeral), wantErr: true}, + {name: "unknown", raw: "invented", wantErr: true}, + } { + t.Run(tt.name, func(t *testing.T) { + got, err := PersistedHelmCacheBackend(tt.raw) + if tt.wantErr { + require.Error(t, err) + assert.Empty(t, got) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +// TestSelectLegacyHelmCacheBackend_SambaClassLookupError proves a failed lookup of the // Samba backing class surfaces as an error rather than silently degrading to the // ephemeral cache: a transient API error must be retried, not treated as an // absent StorageClass. -func TestSelectHelmCacheBackend_SambaClassLookupError(t *testing.T) { +func TestSelectLegacyHelmCacheBackend_SambaClassLookupError(t *testing.T) { sch := runtime.NewScheme() require.NoError(t, storagev1.AddToScheme(sch)) c := fake.NewClientBuilder().WithScheme(sch). @@ -214,7 +242,7 @@ func TestSelectHelmCacheBackend_SambaClassLookupError(t *testing.T) { &featureflag.HelmSharedStorage.FeatureFlag, }} - _, err := SelectHelmCacheBackend(t.Context(), c, ff, "") + _, err := SelectLegacyHelmCacheBackend(t.Context(), c, ff, "") require.Error(t, err) assert.Contains(t, err.Error(), DefaultModelCacheStorageClassName) } diff --git a/src/compute-plane-services/nvca/pkg/storage/modelcache.go b/src/compute-plane-services/nvca/pkg/storage/modelcache.go index 1e44c88cc..84a577c80 100644 --- a/src/compute-plane-services/nvca/pkg/storage/modelcache.go +++ b/src/compute-plane-services/nvca/pkg/storage/modelcache.go @@ -38,6 +38,7 @@ import ( "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + apitypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -86,6 +87,19 @@ const ( primaryPVLabelKey = fqdnPrefix + "/modelcache-primary-pv" primaryPVLabelValue = "true" + // ModelCachePopulatedLabelKey marks a binding-owned PVC as holding a fully + // populated model cache. Both model-cache runtimes use this public package + // contract when the retained PVC, rather than a rewritten PV, is the data + // identity. + ModelCachePopulatedLabelKey = fqdnPrefix + "/modelcache-populated" + // ModelCachePopulatedLabelValue is the ready value for ModelCachePopulatedLabelKey. + ModelCachePopulatedLabelValue = "true" + + // ModelCacheWriterPVCUIDAnnotationKey records the exact PVC instance populated + // by an immutable writer Job. Readers reject a completed Job whose witness + // does not match the current claim UID. + ModelCacheWriterPVCUIDAnnotationKey = fqdnPrefix + "/modelcache-writer-pvc-uid" + // cachePopulatedLabelKey marks a per-handle backing PVC as holding a fully // populated cache. It is the durable, restart-safe "cache populated" signal // for the backends that key reuse on a PVC rather than an NVMesh primary PV: @@ -94,8 +108,8 @@ const ( // init, so a cold reconcile (e.g. after an agent restart, when the in-memory // init-status fan-out is empty) never re-runs the writer or hangs on the // lease. It is the PVC analogue of the primaryPVLabelKey marker. - cachePopulatedLabelKey = fqdnPrefix + "/modelcache-populated" - cachePopulatedLabelValue = "true" + cachePopulatedLabelKey = ModelCachePopulatedLabelKey + cachePopulatedLabelValue = ModelCachePopulatedLabelValue // The annotation applied to primary PV's that denotes the last time // a function or task referenced it. @@ -124,6 +138,15 @@ func (r *Reconciler) terminalErrorWithMetricErr(reason string, err error) error return reconcile.TerminalError(err) } +// shouldSurfaceModelCacheK8sError reports Kubernetes API failures that are not +// deterministic cache-spec failures. Transient failures are requeued by the +// reconcile boundary; authorization and validation failures are returned so +// controller-runtime can surface them without marking the request terminal. +func shouldSurfaceModelCacheK8sError(err error) bool { + return k8sutil.IsTransientK8sError(err) || apierrors.IsForbidden(err) || + apierrors.IsUnauthorized(err) || apierrors.IsInvalid(err) || apierrors.IsGone(err) +} + // mapPodIssuesToFailureReason maps pod issues to a failure reason for metrics. // Returns the most specific reason based on priority order. func mapPodIssuesToFailureReason(podIssues sets.Set[string]) string { @@ -170,8 +193,21 @@ func (r *Reconciler) doModelCacheRouted(ctx context.Context, st nvcav1new.StorageRequest, stCopy *nvcav1new.StorageRequest, icmsReq *nvcav2beta1.ICMSRequest, ) (reconcile.Result, error) { + if err := r.validatePersistedHelmCacheSelection(ctx, stCopy, icmsReq); err != nil { + // A persisted selection is the immutable request contract. Reject an + // invalid or conflicting request without invoking broad legacy cleanup, + // which could otherwise delete resources belonging to another cache. + if k8sutil.IsTransientK8sError(err) { + return reconcile.Result{Requeue: true}, nil + } + if isTerminal(err) { + stCopy.Status.Phase = nvcav1new.StorageFailed + } + return reconcile.Result{}, err + } + // Select the populate path by the backend stamped on the request by the - // miniservice reconciler (SelectHelmCacheBackend). Empty backend defaults to + // miniservice reconciler. Empty backend defaults to // NVMesh for backward compatibility. var res reconcile.Result var err error @@ -227,6 +263,66 @@ func (r *Reconciler) doModelCacheRouted(ctx context.Context, return res, err } +func (r *Reconciler) validatePersistedHelmCacheSelection( + ctx context.Context, + stCopy *nvcav1new.StorageRequest, + icmsReq *nvcav2beta1.ICMSRequest, +) error { + raw := stCopy.Annotations[ModelCacheStorageSelectionAnnotationKey] + if raw == "" { + return nil + } + if icmsReq == nil { + return r.terminalErrorWithMetric( + modelcachetypes.ReasonCacheSpecInvalid, + "annotated Helm model cache StorageRequest has no ICMSRequest") + } + icmsRaw := icmsReq.Annotations[ModelCacheStorageSelectionAnnotationKey] + if icmsRaw == "" || icmsRaw != raw { + return r.terminalErrorWithMetric( + modelcachetypes.ReasonCacheSpecInvalid, + "StorageRequest model cache selection does not match the live ICMSRequest selection") + } + selection, err := ParsePersistedModelCacheStorageSelection(raw) + if err != nil { + return r.terminalErrorWithMetricErr(modelcachetypes.ReasonCacheSpecInvalid, + fmt.Errorf("parse persisted model cache storage selection: %w", err)) + } + expectedBackend, err := HelmCacheBackendFromSelection(selection) + if err != nil { + return r.terminalErrorWithMetricErr(modelcachetypes.ReasonCacheSpecInvalid, err) + } + backend := "" + if stCopy.Spec.ModelCache != nil { + backend = stCopy.Spec.ModelCache.Backend + } + if selection.Mode != ModelCacheSelectionDurable || expectedBackend != HelmCacheBackendNVMesh || + backend != string(expectedBackend) { + return r.terminalErrorWithMetric(modelcachetypes.ReasonCacheSpecInvalid, + fmt.Sprintf("persisted model cache selection %q conflicts with StorageRequest backend %q", + selection.Mode, backend)) + } + encryptionRequired := stCopy.Spec.ModelCache != nil && stCopy.Spec.ModelCache.Encryption != nil && + stCopy.Spec.ModelCache.Encryption.Required + if encryptionRequired != selection.EncryptionRequired { + return r.terminalErrorWithMetric(modelcachetypes.ReasonCacheSpecInvalid, + fmt.Sprintf("persisted model cache encryption requirement %t conflicts with StorageRequest value %t", + selection.EncryptionRequired, encryptionRequired)) + } + binding, err := r.validatePersistedHelmModelCacheBinding(ctx, stCopy, icmsReq, selection) + if err != nil { + wrapped := fmt.Errorf("validate persisted Helm model cache binding: %w", err) + if shouldSurfaceModelCacheK8sError(err) { + return wrapped + } + return r.terminalErrorWithMetricErr(modelcachetypes.ReasonCacheSpecInvalid, wrapped) + } + if err := SetModelCacheBindingUIDLabel(stCopy, binding.UID); err != nil { + return r.terminalErrorWithMetricErr(modelcachetypes.ReasonCacheSpecInvalid, err) + } + return nil +} + // sharedFSProbeTTLSeconds is how long a cached CSI ROX/RWX probe result for // nvcf-miniservice-sc is reused before re-probing. const sharedFSProbeTTLSeconds = 3600 @@ -619,12 +715,18 @@ var accessModesRO = []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany} // after the agent starts is picked up on a later reconcile rather than being // written off forever. func (r *Reconciler) provisionerDefaultMountOptions(ctx context.Context) ([]string, bool) { - log := logf.FromContext(ctx) - provisioner, ok := r.modelCacheProvisionerName(ctx) if !ok { return nil, false } + return r.defaultMountOptionsForProvisioner(ctx, provisioner) +} + +func (r *Reconciler) defaultMountOptionsForProvisioner( + ctx context.Context, + provisioner string, +) ([]string, bool) { + log := logf.FromContext(ctx) cmName := r.cacheMountOptionsConfigMap if cmName == "" { @@ -773,6 +875,26 @@ func negatesMountOption(required, configured string) bool { // a default. Otherwise the configured options are used unchanged. func (r *Reconciler) resolveCacheMountOptions(ctx context.Context, pv *corev1.PersistentVolume) []string { defaults, found := r.provisionerDefaultMountOptions(ctx) + return r.resolveCacheMountOptionsWithDefaults(ctx, pv, defaults, found) +} + +// resolveCacheMountOptionsWithRequired merges the provider requirements +// persisted on a storage selection with the operator's additive options. It +// deliberately does not reread the legacy per-provisioner ConfigMap. +func (r *Reconciler) resolveCacheMountOptionsWithRequired( + ctx context.Context, + pv *corev1.PersistentVolume, + required []string, +) []string { + return r.resolveCacheMountOptionsWithDefaults(ctx, pv, required, true) +} + +func (r *Reconciler) resolveCacheMountOptionsWithDefaults( + ctx context.Context, + pv *corev1.PersistentVolume, + defaults []string, + found bool, +) []string { if !found { return r.csiVolumeMountOptions } @@ -817,9 +939,25 @@ func mergeMountOptions(lists ...[]string) []string { func (r *Reconciler) reconcileSecondaryPVMountOptions(ctx context.Context, secondaryPV *corev1.PersistentVolume, ) error { - log := logf.FromContext(ctx) - want := r.resolveCacheMountOptions(ctx, secondaryPV) + return r.reconcileSecondaryPVMountOptionsTo(ctx, secondaryPV, want) +} + +func (r *Reconciler) reconcileSecondaryPVMountOptionsWithRequired( + ctx context.Context, + secondaryPV *corev1.PersistentVolume, + required []string, +) error { + want := r.resolveCacheMountOptionsWithRequired(ctx, secondaryPV, required) + return r.reconcileSecondaryPVMountOptionsTo(ctx, secondaryPV, want) +} + +func (r *Reconciler) reconcileSecondaryPVMountOptionsTo( + ctx context.Context, + secondaryPV *corev1.PersistentVolume, + want []string, +) error { + log := logf.FromContext(ctx) if slices.Equal(secondaryPV.Spec.MountOptions, want) { return nil } @@ -828,7 +966,8 @@ func (r *Reconciler) reconcileSecondaryPVMountOptions(ctx context.Context, secondaryPV.Spec.MountOptions = want if err := nvcaotel.InvokeWithSpan(ctx, modelCacheTracer, "nvca.modelcache.reconcile_mount_options", func(ctx context.Context) error { - return r.Client.Patch(ctx, secondaryPV, client.MergeFrom(secondaryPVOld)) + return r.Client.Patch(ctx, secondaryPV, + client.MergeFromWithOptions(secondaryPVOld, client.MergeFromWithOptimisticLock{})) }, oteltrace.WithAttributes(otelattr.String("nvcf.modelcache.pv", secondaryPV.Name)), ); err != nil { @@ -868,23 +1007,103 @@ func (r *Reconciler) doModelCacheNVMesh(ctx context.Context, //nolint:gocyclo if err != nil { return reconcile.Result{}, r.terminalErrorWithMetricErr(modelcachetypes.ReasonCacheSpecInvalid, fmt.Errorf("find and decode artifacts: %w", err)) } + // The translated artifacts may carry the workload namespace, but Helm cache + // writers are shared and always created in the binding's writer namespace. + // Normalize before validating immutable intent or checking existing objects. + rwPVC.Namespace = ModelCacheInitNamespace + initJob.Namespace = ModelCacheInitNamespace + if err := propagateModelCacheBindingUIDLabel(stCopy, rwPVC, initJob, &initJob.Spec.Template.ObjectMeta); err != nil { + return reconcile.Result{}, r.terminalErrorWithMetricErr(modelcachetypes.ReasonCacheSpecInvalid, err) + } - r.applyModelCacheStorageClass(ctx, rwPVC) + rawSelection := stCopy.Annotations[ModelCacheStorageSelectionAnnotationKey] + var selection *PersistedModelCacheStorageSelection + var binding *nvcav2beta1.ModelCacheBinding + if rawSelection == "" { + r.applyModelCacheStorageClass(ctx, rwPVC) + } else { + selection, err = ParsePersistedModelCacheStorageSelection(rawSelection) + if err != nil { + return reconcile.Result{}, r.terminalErrorWithMetricErr( + modelcachetypes.ReasonCacheSpecInvalid, fmt.Errorf("parse persisted model cache storage selection: %w", err)) + } + if !selection.EncryptionRequired { + if rwPVC.Spec.StorageClassName != nil && *rwPVC.Spec.StorageClassName != "" && + *rwPVC.Spec.StorageClassName != selection.StorageClassName { + return reconcile.Result{}, r.terminalErrorWithMetric(modelcachetypes.ReasonCacheSpecInvalid, + fmt.Sprintf("model cache writer StorageClass %q conflicts with persisted selection %q", + *rwPVC.Spec.StorageClassName, selection.StorageClassName)) + } + rwPVC.Spec.StorageClassName = &selection.StorageClassName + } + binding, err = r.validatePersistedHelmModelCacheBinding(ctx, stCopy, icmsReq, selection) + if err != nil { + wrapped := fmt.Errorf("validate Helm model cache binding resource intent: %w", err) + if k8sutil.IsTransientK8sError(err) { + return reconcile.Result{Requeue: true}, nil + } + if shouldSurfaceModelCacheK8sError(err) { + return reconcile.Result{}, wrapped + } + return reconcile.Result{}, r.terminalErrorWithMetricErr( + modelcachetypes.ReasonCacheSpecInvalid, wrapped) + } + } - if enc := stCopy.Spec.ModelCache.Encryption; enc != nil { - scName, err := r.doEncryptedStorageClassNVMesh(ctx, stCopy, icmsReq.Spec.CreationMsgInfo.NCAID) + if stCopy.Spec.ModelCache.Encryption != nil && stCopy.Spec.ModelCache.Encryption.Required { + encryptionSharingDomain := icmsReq.Spec.CreationMsgInfo.NCAID + if rawSelection != "" { + encryptionSharingDomain = icmsReq.Spec.NCAId + } + scName, err := r.doEncryptedStorageClassNVMesh(ctx, stCopy, encryptionSharingDomain) if err != nil { return reconcile.Result{}, err } rwPVC.Spec.StorageClassName = &scName } + if err := prepareModelCacheInitObjects( + stCopy, rwPVC, initJob, workerPullSecrets, HelmCacheBackendNVMesh); err != nil { + return reconcile.Result{}, r.terminalErrorWithMetricErr(modelcachetypes.ReasonCacheSpecInvalid, err) + } + if selection != nil { + intentLease := newInitLease(stCopy) + if err := propagateModelCacheBindingUIDLabel(stCopy, intentLease); err != nil { + return reconcile.Result{}, r.terminalErrorWithMetricErr(modelcachetypes.ReasonCacheSpecInvalid, err) + } + if err := r.prepareHelmModelCacheBindingResources( + ctx, binding, rwPVC, initJob, intentLease); err != nil { + if k8sutil.IsTransientK8sError(err) { + return reconcile.Result{Requeue: true}, nil + } + if errors.Is(err, errModelCacheBindingOwnership) { + return reconcile.Result{}, r.terminalErrorWithMetricErr( + modelcachetypes.ReasonCacheSpecInvalid, err) + } + return reconcile.Result{}, err + } + } // The presence or absence of the primary PV will depend on which stage model caching is in. primaryPV, ppvErr := r.getPrimaryPV(ctx, stCopy) switch st.Status.Phase { case nvcav1new.StorageUnknown, nvcav1new.StoragePending, nvcav1new.StorageInitRunning: if apierrors.IsNotFound(ppvErr) { - return r.doInitModelCacheNVMesh(ctx, st, stCopy, rwPVC, initJob, workerPullSecrets, HelmCacheBackendNVMesh) + writerPVCAlreadyExists, err := r.validateDynamicModelCacheWriterStorageClass( + ctx, st, stCopy, rwPVC, selection) + if err != nil { + if apierrors.IsNotFound(err) || errors.Is(err, ErrModelCacheStorageSelectionDrift) || + errors.Is(err, errModelCacheBindingOwnership) { + return reconcile.Result{}, r.terminalErrorWithMetricErr( + modelcachetypes.ReasonCacheSpecInvalid, err) + } + if k8sutil.IsTransientK8sError(err) { + return reconcile.Result{Requeue: true}, nil + } + return reconcile.Result{}, err + } + return r.doInitModelCacheNVMeshWithExistingWriter( + ctx, st, stCopy, rwPVC, initJob, workerPullSecrets, + HelmCacheBackendNVMesh, writerPVCAlreadyExists) } else if ppvErr != nil { return reconcile.Result{}, ppvErr } @@ -906,6 +1125,22 @@ func (r *Reconciler) doModelCacheNVMesh(ctx context.Context, //nolint:gocyclo return r.doCleanupModelCacheNVMesh(ctx, stCopy) } + var bindingUID, requestUID apitypes.UID + if selection != nil { + bindingUID = apitypes.UID(stCopy.Labels[ModelCacheBindingUIDLabelKey]) + requestUID = apitypes.UID(stCopy.Annotations[ICMSRequestUIDAnnotationKey]) + if bindingUID == "" || requestUID == "" { + return reconcile.Result{}, r.terminalErrorWithMetric( + modelcachetypes.ReasonCacheSpecInvalid, + "annotated Helm model cache reader has incomplete binding or request ownership") + } + if err := validateHelmModelCachePrimaryPVForReuse( + stCopy, selection, rwPVC, primaryPV, bindingUID); err != nil { + return reconcile.Result{}, r.terminalErrorWithMetricErr( + modelcachetypes.ReasonCacheSpecInvalid, err) + } + } + switch primaryPV.Status.Phase { case corev1.VolumeFailed: log.Info("Primary PV is failed", "pv", primaryPV.Name) @@ -948,9 +1183,9 @@ func (r *Reconciler) doModelCacheNVMesh(ctx context.Context, //nolint:gocyclo // The name must be unique and related to the storage request that owns it. secondaryPVName := "secondary-pv-" + stCopy.Spec.ICMSRequestName roPVCName := "ro-pvc-" + stCopy.Spec.ModelCache.CacheHandle - // Create the PV first, which will be locked to the PVC by claim ref. secondaryPV := &corev1.PersistentVolume{} + secondaryPVAlreadyExists := false if err := r.Client.Get(ctx, client.ObjectKey{Name: secondaryPVName}, secondaryPV); err != nil { if !apierrors.IsNotFound(err) { return reconcile.Result{}, err @@ -963,8 +1198,22 @@ func (r *Reconciler) doModelCacheNVMesh(ctx context.Context, //nolint:gocyclo Annotations: types.GetAnnotationsForRequest(icmsReq), } maps.Copy(secondaryPV.Labels, getClusterWideResourceLabels(stCopy)) + if err := propagateModelCacheBindingUIDLabel(stCopy, secondaryPV); err != nil { + return reconcile.Result{}, r.terminalErrorWithMetricErr(modelcachetypes.ReasonCacheSpecInvalid, err) + } + if selection != nil { + if err := SetModelCacheRequestUIDLabel(secondaryPV, requestUID); err != nil { + return reconcile.Result{}, r.terminalErrorWithMetricErr( + modelcachetypes.ReasonCacheSpecInvalid, err) + } + } secondaryPV.Spec.AccessModes = accessModesRO - secondaryPV.Spec.MountOptions = r.resolveCacheMountOptions(ctx, secondaryPV) + if selection == nil { + secondaryPV.Spec.MountOptions = r.resolveCacheMountOptions(ctx, secondaryPV) + } else { + secondaryPV.Spec.MountOptions = r.resolveCacheMountOptionsWithRequired( + ctx, secondaryPV, selection.RequiredMountOptions) + } secondaryPV.Spec.ClaimRef = &corev1.ObjectReference{ APIVersion: "v1", Kind: "PersistentVolumeClaim", @@ -982,14 +1231,39 @@ func (r *Reconciler) doModelCacheNVMesh(ctx context.Context, //nolint:gocyclo return reconcile.Result{}, err } if err := r.Client.Create(ctx, secondaryPV); err != nil { - return reconcile.Result{}, err + if !apierrors.IsAlreadyExists(err) { + return reconcile.Result{}, err + } + secondaryPVAlreadyExists = true + secondaryPV = &corev1.PersistentVolume{} + if err := r.Client.Get(ctx, client.ObjectKey{Name: secondaryPVName}, secondaryPV); err != nil { + return reconcile.Result{}, fmt.Errorf("get secondary PV after create race: %w", err) + } + } else { + log.Info("Secondary PV created", "pv", secondaryPV.Name) } - log.Info("Secondary PV created", "pv", secondaryPV.Name) } else { + secondaryPVAlreadyExists = true log.V(1).Info("Secondary PV already exists, checking status", "pv", secondaryPV.Name) + } + if selection != nil { + if err := validateHelmModelCacheSecondaryPV( + stCopy, selection, primaryPV, secondaryPV, roPVCName, bindingUID, requestUID); err != nil { + return reconcile.Result{}, r.terminalErrorWithMetricErr( + modelcachetypes.ReasonCacheSpecInvalid, err) + } + } + if secondaryPVAlreadyExists { // Mount options are mutable via NGC/NVCFBackend, so an existing PV can be // left behind when the configuration changes. - if err := r.reconcileSecondaryPVMountOptions(ctx, secondaryPV); err != nil { + var err error + if selection == nil { + err = r.reconcileSecondaryPVMountOptions(ctx, secondaryPV) + } else { + err = r.reconcileSecondaryPVMountOptionsWithRequired( + ctx, secondaryPV, selection.RequiredMountOptions) + } + if err != nil { if k8sutil.IsTransientK8sError(err) { log.V(1).Info("Transient error reconciling secondary PV mount options, will retry", "pv", secondaryPV.Name) @@ -1005,6 +1279,12 @@ func (r *Reconciler) doModelCacheNVMesh(ctx context.Context, //nolint:gocyclo if !apierrors.IsNotFound(err) { return reconcile.Result{}, err } + if selection != nil && secondaryPV.Spec.ClaimRef != nil && secondaryPV.Spec.ClaimRef.UID != "" { + return reconcile.Result{}, r.terminalErrorWithMetric( + modelcachetypes.ReasonCacheSpecInvalid, + fmt.Sprintf("secondary PV %q claimRef UID %q identifies a reader PVC generation that no longer exists", + secondaryPV.Name, secondaryPV.Spec.ClaimRef.UID)) + } roPVC = rwPVC.DeepCopy() roPVC.ObjectMeta = metav1.ObjectMeta{ @@ -1014,6 +1294,15 @@ func (r *Reconciler) doModelCacheNVMesh(ctx context.Context, //nolint:gocyclo Annotations: types.GetAnnotationsForRequest(icmsReq), } maps.Copy(roPVC.Labels, getClusterWideResourceLabels(stCopy)) + if err := propagateModelCacheBindingUIDLabel(stCopy, roPVC); err != nil { + return reconcile.Result{}, r.terminalErrorWithMetricErr(modelcachetypes.ReasonCacheSpecInvalid, err) + } + if selection != nil { + if err := SetModelCacheRequestUIDLabel(roPVC, requestUID); err != nil { + return reconcile.Result{}, r.terminalErrorWithMetricErr( + modelcachetypes.ReasonCacheSpecInvalid, err) + } + } roPVC.Spec.AccessModes = accessModesRO // Set VolumeName to specify this particular PV to bind. roPVC.Spec.VolumeName = secondaryPVName @@ -1023,13 +1312,27 @@ func (r *Reconciler) doModelCacheNVMesh(ctx context.Context, //nolint:gocyclo return reconcile.Result{}, err } if err := r.Client.Create(ctx, roPVC); err != nil { - return reconcile.Result{}, err + if !apierrors.IsAlreadyExists(err) { + return reconcile.Result{}, err + } + roPVC = &corev1.PersistentVolumeClaim{} + if err := r.Client.Get(ctx, + client.ObjectKey{Name: roPVCName, Namespace: stCopy.Namespace}, roPVC); err != nil { + return reconcile.Result{}, fmt.Errorf("get reader PVC after create race: %w", err) + } + } else { + log.Info("RO PVC created", "pvc", roPVC.Name) } - log.Info("RO PVC created", "pvc", roPVC.Name) } else { log.V(1).Info("RO PVC already exists, checking status", "pvc", roPVC.Name) } - + if selection != nil { + if err := validateHelmModelCacheReaderPVC( + stCopy, rwPVC, secondaryPV, roPVC, bindingUID, requestUID); err != nil { + return reconcile.Result{}, r.terminalErrorWithMetricErr( + modelcachetypes.ReasonCacheSpecInvalid, err) + } + } pvcState := r.getPVCState(roPVC) switch pvcState { case pvcBound: @@ -1055,6 +1358,45 @@ func (r *Reconciler) doModelCacheNVMesh(ctx context.Context, //nolint:gocyclo return reconcile.Result{}, nil } +// validateDynamicModelCacheWriterStorageClass verifies the persisted base +// StorageClass immediately before an annotated request can create a missing +// unencrypted writer PVC. An existing binding-owned writer and the encrypted +// derived-class path already have immutable resource identities, so neither +// depends on the continued presence of nvcf-sc. +func (r *Reconciler) validateDynamicModelCacheWriterStorageClass( + ctx context.Context, + st nvcav1new.StorageRequest, + stCopy *nvcav1new.StorageRequest, + rwPVC *corev1.PersistentVolumeClaim, + selection *PersistedModelCacheStorageSelection, +) (bool, error) { + if st.Status.Phase != nvcav1new.StorageUnknown || selection == nil || selection.EncryptionRequired { + return false, nil + } + + bindingUID := apitypes.UID(stCopy.Labels[ModelCacheBindingUIDLabelKey]) + if bindingUID == "" { + return false, fmt.Errorf( + "%w: annotated model cache writer has no binding UID", errModelCacheBindingOwnership) + } + + existing := &corev1.PersistentVolumeClaim{} + key := client.ObjectKey{Namespace: ModelCacheInitNamespace, Name: rwPVC.Name} + if err := r.Client.Get(ctx, key, existing); err == nil { + if err := validateHelmModelCacheWriterPVC(existing, rwPVC, bindingUID); err != nil { + return false, err + } + return true, nil + } else if !apierrors.IsNotFound(err) { + return false, fmt.Errorf("get model cache writer PVC %s/%s: %w", key.Namespace, key.Name, err) + } + + if err := ValidateModelCacheStorageSelectionLive(ctx, r.Client, selection); err != nil { + return false, fmt.Errorf("validate selected StorageClass before creating model cache writer PVC: %w", err) + } + return false, nil +} + func (r *Reconciler) findAndDecodeCacheArtifacts( icmsReq *nvcav2beta1.ICMSRequest, namespace string, @@ -1066,7 +1408,7 @@ func (r *Reconciler) findAndDecodeCacheArtifacts( ) { objs, err := r.translateWorkload(namespace, icmsReq) if err != nil { - return nil, nil, nil, fmt.Errorf("translate request: %v", err) + return nil, nil, nil, fmt.Errorf("translate request: %w", err) } job, pvc, pullSecrets = findModelCacheObjects(objs) if pvc == nil { @@ -1097,11 +1439,117 @@ func (r *Reconciler) doInitModelCacheNVMesh(ctx context.Context, initJob *batchv1.Job, pullSecrets []*corev1.Secret, backend HelmCacheBackend, +) (res reconcile.Result, err error) { + return r.doInitModelCacheNVMeshWithExistingWriter( + ctx, st, stCopy, rwPVC, initJob, pullSecrets, backend, false) +} + +func prepareModelCacheInitObjects( + st *nvcav1new.StorageRequest, + rwPVC *corev1.PersistentVolumeClaim, + initJob *batchv1.Job, + pullSecrets []*corev1.Secret, + backend HelmCacheBackend, +) error { + if st == nil || st.Spec.ModelCache == nil || rwPVC == nil || initJob == nil { + return fmt.Errorf("model cache init object intent is incomplete") + } + initJob.Spec.Template.Spec.ImagePullSecrets = make([]corev1.LocalObjectReference, len(pullSecrets)) + for i, pullSecret := range pullSecrets { + if pullSecret == nil { + return fmt.Errorf("model cache init pull Secret %d is nil", i) + } + pullSecret.Name = fmt.Sprintf("%s-%d-pull-worker", initJob.Name, i) + if err := propagateModelCacheBindingUIDLabel(st, pullSecret); err != nil { + return err + } + initJob.Spec.Template.Spec.ImagePullSecrets[i].Name = pullSecret.Name + } + if initJob.Spec.Template.Labels == nil { + initJob.Spec.Template.Labels = map[string]string{} + } + initJob.Spec.Template.Labels[modelCacheHandleLabelKey] = st.Spec.ModelCache.CacheHandle + if bindingUID := st.Labels[ModelCacheBindingUIDLabelKey]; bindingUID != "" { + initJob.Spec.Template.Labels[ModelCacheBindingUIDLabelKey] = bindingUID + } + if backend == HelmCacheBackendNVMesh { + SetNVMeshClientStatusSchedulingRequirement(&initJob.Spec.Template.Spec) + } + for _, obj := range append( + []client.Object{rwPVC, initJob}, clientObjectsFromSecrets(pullSecrets)..., + ) { + obj.SetNamespace(ModelCacheInitNamespace) + labels := obj.GetLabels() + if labels == nil { + labels = map[string]string{} + obj.SetLabels(labels) + } + labels[modelCacheHandleLabelKey] = st.Spec.ModelCache.CacheHandle + } + return nil +} + +func clientObjectsFromSecrets(secrets []*corev1.Secret) []client.Object { + objects := make([]client.Object, 0, len(secrets)) + for _, secret := range secrets { + objects = append(objects, secret) + } + return objects +} + +func (r *Reconciler) doInitModelCacheNVMeshWithExistingWriter(ctx context.Context, + st nvcav1new.StorageRequest, stCopy *nvcav1new.StorageRequest, + rwPVC *corev1.PersistentVolumeClaim, + initJob *batchv1.Job, + pullSecrets []*corev1.Secret, + backend HelmCacheBackend, + writerPVCAlreadyExists bool, ) (res reconcile.Result, err error) { logf.IntoContext(ctx, logf.FromContext(ctx, "namespace", ModelCacheInitNamespace)) + if err := propagateModelCacheBindingUIDLabel(stCopy, rwPVC, initJob, &initJob.Spec.Template.ObjectMeta); err != nil { + return reconcile.Result{}, r.terminalErrorWithMetricErr(modelcachetypes.ReasonCacheSpecInvalid, err) + } + if err := prepareModelCacheInitObjects(stCopy, rwPVC, initJob, pullSecrets, backend); err != nil { + return reconcile.Result{}, r.terminalErrorWithMetricErr(modelcachetypes.ReasonCacheSpecInvalid, err) + } + if bindingUID := apitypes.UID(stCopy.Labels[ModelCacheBindingUIDLabelKey]); bindingUID != "" { + intentLease := newInitLease(stCopy) + if err := propagateModelCacheBindingUIDLabel(stCopy, intentLease); err != nil { + return reconcile.Result{}, r.terminalErrorWithMetricErr(modelcachetypes.ReasonCacheSpecInvalid, err) + } + wantedObjects := []client.Object{rwPVC, initJob, intentLease} + wantedObjects = append(wantedObjects, clientObjectsFromSecrets(pullSecrets)...) + for _, wanted := range wantedObjects { + existing, ok := wanted.DeepCopyObject().(client.Object) + if !ok { + return reconcile.Result{}, r.terminalErrorWithMetric( + modelcachetypes.ReasonCacheSpecInvalid, + fmt.Sprintf("model cache object %T is not a controller-runtime object", wanted)) + } + err := r.Client.Get(ctx, client.ObjectKeyFromObject(wanted), existing) + switch { + case apierrors.IsNotFound(err): + case err != nil: + if k8sutil.IsTransientK8sError(err) { + return reconcile.Result{Requeue: true}, nil + } + return reconcile.Result{}, err + default: + if err := validateHelmModelCacheBindingOwnedObjectIntent( + existing, wanted, bindingUID); err != nil { + return reconcile.Result{}, r.terminalErrorWithMetricErr( + modelcachetypes.ReasonCacheSpecInvalid, err) + } + } + } + } // Use a lease to lock initialization. - lres, holdsLease, err := r.handleLease(ctx, newInitLease(stCopy)) + lease := newInitLease(stCopy) + if err := propagateModelCacheBindingUIDLabel(stCopy, lease); err != nil { + return reconcile.Result{}, r.terminalErrorWithMetricErr(modelcachetypes.ReasonCacheSpecInvalid, err) + } + lres, holdsLease, err := r.handleLease(ctx, lease) if err != nil { return reconcile.Result{}, err } @@ -1128,7 +1576,8 @@ func (r *Reconciler) doInitModelCacheNVMesh(ctx context.Context, r.initStatuses.Lock() defer r.initStatuses.Unlock() - res, err = r.reconcileInitModelCacheNVMesh(ctx, st, stCopy, rwPVC, initJob, pullSecrets, backend) + res, err = r.reconcileInitModelCacheNVMesh( + ctx, st, stCopy, rwPVC, initJob, pullSecrets, backend, writerPVCAlreadyExists) // The lease holder updates the status for all non-holders (fan-out). if existingStatus, ok := r.initStatuses.get(cacheHandle); !ok || @@ -1153,6 +1602,7 @@ func (r *Reconciler) reconcileInitModelCacheNVMesh(ctx context.Context, initJob *batchv1.Job, pullSecrets []*corev1.Secret, backend HelmCacheBackend, + writerPVCAlreadyExists bool, ) (reconcile.Result, error) { log := logf.FromContext(ctx) @@ -1160,44 +1610,30 @@ func (r *Reconciler) reconcileInitModelCacheNVMesh(ctx context.Context, case nvcav1new.StorageUnknown: log.V(1).Info("Creating objects for pending model cache") - objsToCreate := []client.Object{rwPVC} - initJob.Spec.Template.Spec.ImagePullSecrets = make([]corev1.LocalObjectReference, len(pullSecrets)) + objsToCreate := []client.Object{} + if !writerPVCAlreadyExists { + objsToCreate = append(objsToCreate, rwPVC) + } for i, pullSecret := range pullSecrets { - // Rename the secret so it is consistent across all storage requests for the volume handle. - pullSecret.Name = fmt.Sprintf("%s-%d-pull-worker", initJob.Name, i) objsToCreate = append(objsToCreate, pullSecret) initJob.Spec.Template.Spec.ImagePullSecrets[i].Name = pullSecret.Name } - // Add labels to Job pods for scheduling/observability. - if initJob.Spec.Template.Labels == nil { - initJob.Spec.Template.Labels = map[string]string{} - } - initJob.Spec.Template.Labels[modelCacheHandleLabelKey] = stCopy.Spec.ModelCache.CacheHandle - // NVMesh client readiness scheduling applies only to the NVMesh backend; - // the shared-FS backend (nvcf-miniservice-sc) has no NVMesh node requirement. - if backend == HelmCacheBackendNVMesh { - SetNVMeshClientStatusSchedulingRequirement(&initJob.Spec.Template.Spec) - } objsToCreate = append(objsToCreate, initJob) for _, obj := range objsToCreate { - obj.SetNamespace(ModelCacheInitNamespace) - labels := obj.GetLabels() - if labels == nil { - labels = map[string]string{} - obj.SetLabels(labels) - } - // Set the model cache handle label for association and deletion. - labels[modelCacheHandleLabelKey] = stCopy.Spec.ModelCache.CacheHandle - - if err := r.Client.Create(ctx, obj); err != nil { - if apierrors.IsAlreadyExists(err) { - log.V(1).Error(err, "Init cache object already exists, either prior create failed "+ - "or model cache not cleaned up on previous run") - continue + alreadyExists, err := r.createOrValidateModelCacheBindingOwnedObject(ctx, obj) + if err != nil { + log.Error(err, "Failed to create or validate cache init object") + if errors.Is(err, errModelCacheBindingOwnership) { + return reconcile.Result{}, r.terminalErrorWithMetricErr( + modelcachetypes.ReasonCacheSpecInvalid, err) } - log.Error(err, "Failed to create cache init object") return reconcile.Result{}, err } + if alreadyExists { + log.V(1).Info("Init cache object already exists, either prior create failed " + + "or model cache was retained from a previous run") + continue + } } stCopy.Status.Phase = nvcav1new.StoragePending return reconcile.Result{Requeue: true}, nil @@ -1208,6 +1644,7 @@ func (r *Reconciler) reconcileInitModelCacheNVMesh(ctx context.Context, case nvcav1new.StorageInitRunning: log.V(1).Info("Handling initializing model cache objects") + wantedJob := initJob.DeepCopy() jobKey := client.ObjectKey{Namespace: ModelCacheInitNamespace, Name: initJob.Name} if err := r.Client.Get(ctx, jobKey, initJob); err != nil { if apierrors.IsNotFound(err) { @@ -1215,6 +1652,12 @@ func (r *Reconciler) reconcileInitModelCacheNVMesh(ctx context.Context, } return reconcile.Result{}, err } + if bindingUID := apitypes.UID(stCopy.Labels[ModelCacheBindingUIDLabelKey]); bindingUID != "" { + if err := validateHelmModelCacheWriterJob(initJob, wantedJob, bindingUID); err != nil { + return reconcile.Result{}, r.terminalErrorWithMetricErr( + modelcachetypes.ReasonCacheSpecInvalid, err) + } + } switch r.getInitCacheJobState(ctx, initJob) { case initCacheJobCompleted: // check the RWPVC state next @@ -1228,6 +1671,7 @@ func (r *Reconciler) reconcileInitModelCacheNVMesh(ctx context.Context, return reconcile.Result{}, nil } + wantedRWPVC := rwPVC.DeepCopy() rwPVCKey := client.ObjectKey{Namespace: ModelCacheInitNamespace, Name: rwPVC.Name} if err := r.Client.Get(ctx, rwPVCKey, rwPVC); err != nil { if apierrors.IsNotFound(err) { @@ -1235,8 +1679,15 @@ func (r *Reconciler) reconcileInitModelCacheNVMesh(ctx context.Context, } return reconcile.Result{}, err } + if bindingUID := apitypes.UID(stCopy.Labels[ModelCacheBindingUIDLabelKey]); bindingUID != "" { + if err := validateHelmModelCacheWriterPVC(rwPVC, wantedRWPVC, bindingUID); err != nil { + return reconcile.Result{}, r.terminalErrorWithMetricErr( + modelcachetypes.ReasonCacheSpecInvalid, err) + } + } switch r.getPVCState(rwPVC) { case pvcBound: + var cleanupErrs []error switch backend { case HelmCacheBackendNVMesh: // NVMesh: retain the bound dynamic PV as the cross-namespace @@ -1247,7 +1698,7 @@ func (r *Reconciler) reconcileInitModelCacheNVMesh(ctx context.Context, log.Error(err, "Failed to finalize primary PV") return reconcile.Result{}, err } - _ = r.cleanupInitModelCache(ctx, stCopy, false) + cleanupErrs = r.cleanupInitModelCache(ctx, stCopy, false) case HelmCacheBackendSamba: // Samba: the cache data lives in the per-handle backing PVC // (samba-), populated via the writer's SMB mount. Stamp @@ -1258,7 +1709,7 @@ func (r *Reconciler) reconcileInitModelCacheNVMesh(ctx context.Context, log.Error(err, "Failed to mark Samba cache populated") return reconcile.Result{}, err } - _ = r.cleanupInitModelCache(ctx, stCopy, false) + cleanupErrs = r.cleanupInitModelCache(ctx, stCopy, false) default: // Shared-FS: the cache data lives on the shared backend // (nvcf-miniservice-sc). Keep the writer RW PVC/job as the persistent @@ -1276,7 +1727,11 @@ func (r *Reconciler) reconcileInitModelCacheNVMesh(ctx context.Context, // without this, one completed Job and one Lease would accumulate // per cache handle with no other cleanup path (idle GC only // reclaims writer PVCs). - _ = r.cleanupInitModelCache(ctx, stCopy, true) + cleanupErrs = r.cleanupInitModelCache(ctx, stCopy, true) + } + cleanupResult, cleanupErr := modelCacheSuccessfulInitCleanupResult(cleanupErrs) + if cleanupErr != nil || cleanupResult != (reconcile.Result{}) { + return cleanupResult, cleanupErr } // The single-writer download completed: count one populate for the backend. @@ -1299,6 +1754,28 @@ func (r *Reconciler) reconcileInitModelCacheNVMesh(ctx context.Context, return reconcile.Result{}, nil } +func modelCacheSuccessfulInitCleanupResult(errs []error) (reconcile.Result, error) { + if len(errs) == 0 { + return reconcile.Result{}, nil + } + realErrs := make([]error, 0, len(errs)) + pendingDetach := false + for _, err := range errs { + if errors.Is(err, errVolumeStillAttached) { + pendingDetach = true + continue + } + realErrs = append(realErrs, err) + } + if len(realErrs) == 0 && pendingDetach { + return reconcile.Result{RequeueAfter: volumeDetachRequeueInterval}, nil + } + if k8sutil.AnyNonTransientK8sError(realErrs) == nil { + return reconcile.Result{Requeue: true}, nil + } + return reconcile.Result{}, errors.Join(realErrs...) +} + // handleLease will attempt to create lease for a cache handle. // If the lease already exists and is held by this storage request, it will renew the lease. // Else if it is expired, it will attempt to acquire it, and return true if so. @@ -1315,6 +1792,11 @@ func (r *Reconciler) handleLease(ctx context.Context, switch err := r.Client.Get(ctx, leaseKey, currLease); { case err == nil: // The lease was already created by another thread, proceed with handler. + if bindingUID := apitypes.UID(lease.Labels[ModelCacheBindingUIDLabelKey]); bindingUID != "" { + if err := ValidateModelCacheBindingUIDLabel(currLease, bindingUID); err != nil { + return reconcile.Result{}, false, reconcile.TerminalError(err) + } + } case apierrors.IsNotFound(err): // The lease can be acquired. log.Info("Creating lease, starting model cache initialization") @@ -1322,11 +1804,11 @@ func (r *Reconciler) handleLease(ctx context.Context, lease.Spec.AcquireTime = &metav1.MicroTime{Time: now} if err := r.Client.Create(ctx, lease); err != nil { log.Error(err, "Failed to create lease") - return reconcile.Result{}, false, err + return modelCacheLeaseAPIErrorResult("create model cache Lease", err) } return reconcile.Result{}, true, nil default: - return reconcile.Result{}, false, err + return modelCacheLeaseAPIErrorResult("get model cache Lease", err) } if currLease.Spec.HolderIdentity == nil { @@ -1343,18 +1825,32 @@ func (r *Reconciler) handleLease(ctx context.Context, // The least may exist but after a controller restart and/or ICMS request cleanup, // the owner might not exist anymore to proceed with caching. // Instead of waiting for timeout, acquire the lease and continue. + holderName, holderUID := parseModelCacheLeaseHolderIdentity(*currLease.Spec.HolderIdentity) icmsReq := &nvcav2beta1.ICMSRequest{} srerr := r.Client.Get(ctx, client.ObjectKey{ Namespace: r.ICMSRequestNamespace, - Name: *currLease.Spec.HolderIdentity, + Name: holderName, }, icmsReq) + if srerr != nil && !apierrors.IsNotFound(srerr) { + wrapped := fmt.Errorf("get model cache Lease holder ICMSRequest %q: %w", holderName, srerr) + if k8sutil.IsTransientK8sError(srerr) { + log.V(1).Info("Transient Lease holder lookup failed, will retry", "error", wrapped) + return reconcile.Result{Requeue: true}, false, nil + } + return reconcile.Result{}, false, wrapped + } // Some other thread is initializing the cache. // Check if lease is expired. leaseDur := time.Duration(*currLease.Spec.LeaseDurationSeconds) * time.Second - if apierrors.IsNotFound(srerr) || - (currLease.Spec.RenewTime == nil && currLease.Spec.AcquireTime.Add(leaseDur).Before(now)) || - currLease.Spec.RenewTime.Add(leaseDur).Before(now) { + leaseExpired := false + if currLease.Spec.RenewTime != nil { + leaseExpired = currLease.Spec.RenewTime.Add(leaseDur).Before(now) + } else { + leaseExpired = currLease.Spec.AcquireTime.Add(leaseDur).Before(now) + } + holderGone := apierrors.IsNotFound(srerr) || (holderUID != "" && icmsReq.UID != holderUID) + if holderGone || leaseExpired { log.V(1).Info("Lease has expired or holder is gone, attempting to acquire", "oldHolder", *currLease.Spec.HolderIdentity, "newHolder", *lease.Spec.HolderIdentity, @@ -1367,7 +1863,7 @@ func (r *Reconciler) handleLease(ctx context.Context, currLease.Spec.RenewTime = &metav1.MicroTime{Time: now} currLease.Spec.AcquireTime = &metav1.MicroTime{Time: now} if err := r.Client.Update(ctx, currLease); err != nil { - return reconcile.Result{}, false, err + return modelCacheLeaseAPIErrorResult("acquire model cache Lease", err) } // No conflict, this storage request has acquired the lease. log.Info("Acquired lease from old holder", @@ -1406,7 +1902,7 @@ func (r *Reconciler) handleLease(ctx context.Context, log.V(1).Info("Renewing lease", "holder", *currLease.Spec.HolderIdentity) currLease.Spec.RenewTime = &metav1.MicroTime{Time: now} if err := r.Client.Update(ctx, currLease); err != nil { - return reconcile.Result{}, false, err + return modelCacheLeaseAPIErrorResult("renew model cache Lease", err) } } res.RequeueAfter = leaseDurHalf @@ -1415,6 +1911,16 @@ func (r *Reconciler) handleLease(ctx context.Context, return res, holdsLease, nil } +func modelCacheLeaseAPIErrorResult( + operation string, + err error, +) (reconcile.Result, bool, error) { + if k8sutil.IsTransientK8sError(err) || apierrors.IsAlreadyExists(err) { + return reconcile.Result{Requeue: true}, false, nil + } + return reconcile.Result{}, false, fmt.Errorf("%s: %w", operation, err) +} + func (r *Reconciler) getPrimaryPV(ctx context.Context, st *nvcav1new.StorageRequest) (*corev1.PersistentVolume, error) { log := logf.FromContext(ctx) // No primary PV will be found for a cache handle unless finalizePrimaryPVOnSuccessfulInit @@ -1423,6 +1929,9 @@ func (r *Reconciler) getPrimaryPV(ctx context.Context, st *nvcav1new.StorageRequ primaryPVLabelKey: primaryPVLabelValue, modelCacheHandleLabelKey: st.Spec.ModelCache.CacheHandle, } + if bindingUID := st.Labels[ModelCacheBindingUIDLabelKey]; bindingUID != "" { + ppvLabels[ModelCacheBindingUIDLabelKey] = bindingUID + } pvs := &corev1.PersistentVolumeList{} if err := r.Client.List(ctx, pvs, &client.ListOptions{ LabelSelector: labels.SelectorFromSet(ppvLabels), @@ -1454,6 +1963,20 @@ func (r *Reconciler) finalizePrimaryPVOnSuccessfulInit(ctx context.Context, if err := r.Client.Get(ctx, client.ObjectKey{Name: primaryPVName}, primaryPV); err != nil { return err } + if bindingUID := apitypes.UID(stCopy.Labels[ModelCacheBindingUIDLabelKey]); bindingUID != "" { + if err := validateHelmModelCachePrimaryPVClaim(rwPVC, primaryPV, bindingUID); err != nil { + return reconcile.TerminalError(err) + } + selection, err := ParsePersistedModelCacheStorageSelection( + stCopy.Annotations[ModelCacheStorageSelectionAnnotationKey]) + if err != nil { + return reconcile.TerminalError(fmt.Errorf( + "parse persisted model cache selection before finalizing primary PV: %w", err)) + } + if err := validateHelmModelCachePrimaryPVForFinalize(selection, rwPVC, primaryPV); err != nil { + return reconcile.TerminalError(err) + } + } primaryPVOld := primaryPV.DeepCopy() if primaryPV.Labels == nil { @@ -1464,12 +1987,16 @@ func (r *Reconciler) finalizePrimaryPVOnSuccessfulInit(ctx context.Context, } primaryPV.Labels[primaryPVLabelKey] = primaryPVLabelValue primaryPV.Labels[modelCacheHandleLabelKey] = stCopy.Spec.ModelCache.CacheHandle + if err := propagateModelCacheBindingUIDLabel(stCopy, primaryPV); err != nil { + return err + } timeStr := r.nowFunc().Format(primaryPVLastReferencedTimeFormat) primaryPV.Annotations[primaryPVLastReferencedAnnotationKey] = timeStr // Ensure PV data is retained for reuse by secondary PV's. primaryPV.Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimRetain - if err := r.Client.Patch(ctx, primaryPV, client.MergeFrom(primaryPVOld)); err != nil { - return fmt.Errorf("patch primary PV on successful init: %v", err) + if err := r.Client.Patch(ctx, primaryPV, + client.MergeFromWithOptions(primaryPVOld, client.MergeFromWithOptimisticLock{})); err != nil { + return fmt.Errorf("patch primary PV on successful init: %w", err) } return nil @@ -1548,7 +2075,7 @@ func (r *Reconciler) sharedFSCachePopulated(ctx context.Context, rwPVCName strin func newInitLease(st *nvcav1new.StorageRequest) *coordv1.Lease { // Multiple ICMS requests may be trying to initialize the cache. - leaseHolderID := st.Spec.ICMSRequestName + leaseHolderID := modelCacheLeaseHolderIdentity(st) // Set to an hour so the model has time to download, // and the reconciler can sufficiently back off the request. var leaseDurSeconds int32 = 3600 @@ -1569,6 +2096,25 @@ func newInitLease(st *nvcav1new.StorageRequest) *coordv1.Lease { return lease } +func modelCacheLeaseHolderIdentity(st *nvcav1new.StorageRequest) string { + if st == nil { + return "" + } + requestName := st.Spec.ICMSRequestName + if requestUID := st.Annotations[ICMSRequestUIDAnnotationKey]; requestUID != "" { + return requestName + "@" + requestUID + } + return requestName +} + +func parseModelCacheLeaseHolderIdentity(identity string) (string, apitypes.UID) { + name, uid, found := strings.Cut(identity, "@") + if !found || name == "" || uid == "" { + return identity, "" + } + return name, apitypes.UID(uid) +} + func buildInitLeaseName(cacheHandle string) string { return "modelcache-init-" + cacheHandle } @@ -1606,6 +2152,7 @@ func (r *Reconciler) handlePendingModelCache( ) (reconcile.Result, error) { log := logf.FromContext(ctx) + wantedJob := initJob.DeepCopy() jobKey := client.ObjectKey{Namespace: ModelCacheInitNamespace, Name: initJob.Name} if err := r.Client.Get(ctx, jobKey, initJob); err != nil { if apierrors.IsNotFound(err) { @@ -1621,6 +2168,12 @@ func (r *Reconciler) handlePendingModelCache( log.Error(err, "Failed to get cache init job for pending request") return reconcile.Result{}, err } + if bindingUID := apitypes.UID(stCopy.Labels[ModelCacheBindingUIDLabelKey]); bindingUID != "" { + if err := validateHelmModelCacheWriterJob(initJob, wantedJob, bindingUID); err != nil { + return reconcile.Result{}, r.terminalErrorWithMetricErr( + modelcachetypes.ReasonCacheSpecInvalid, err) + } + } jobState := r.getInitCacheJobState(ctx, initJob) if jobState == initCacheJobCompleted { log.V(1).Info("Init job completed, transition to init running") diff --git a/src/compute-plane-services/nvca/pkg/storage/modelcache_binding.go b/src/compute-plane-services/nvca/pkg/storage/modelcache_binding.go new file mode 100644 index 000000000..790500dd9 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/storage/modelcache_binding.go @@ -0,0 +1,226 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package storage + +import ( + "crypto/sha256" + "fmt" + "reflect" + "slices" + "strings" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation" + + nvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1" +) + +const modelCacheBindingNamePrefix = "model-cache-" + +// NewModelCacheBinding builds the immutable binding intent for a durable +// request. Current NVMesh resource names are cache-handle scoped, so the +// binding name is also handle-scoped. A second workflow or sharing domain with +// the same handle therefore collides with, and must not adopt, the first +// binding instead of addressing the same resources through a different spec. +func NewModelCacheBinding( + selection *PersistedModelCacheStorageSelection, + sharingDomain string, + cacheHandle string, + writerNamespace string, +) (*nvcav2beta1.ModelCacheBinding, error) { + if err := selection.Validate(); err != nil { + return nil, err + } + if selection.Mode != ModelCacheSelectionDurable { + return nil, fmt.Errorf("model cache binding requires a durable selection") + } + if strings.TrimSpace(sharingDomain) == "" { + return nil, fmt.Errorf("model cache binding sharing domain is empty") + } + if strings.TrimSpace(cacheHandle) == "" { + return nil, fmt.Errorf("model cache binding cache handle is empty") + } + if errs := validation.IsDNS1123Label(writerNamespace); len(errs) != 0 { + return nil, fmt.Errorf("model cache binding writer namespace %q is invalid: %s", + writerNamespace, strings.Join(errs, "; ")) + } + + rwPVCName := "rw-pvc-" + cacheHandle + jobName := "writer-job-" + cacheHandle + for kind, name := range map[string]string{ + "writer PVC": rwPVCName, + "writer Job": jobName, + } { + if errs := validation.IsDNS1123Subdomain(name); len(errs) != 0 { + return nil, fmt.Errorf("model cache binding %s name %q is invalid: %s", + kind, name, strings.Join(errs, "; ")) + } + } + + resources := nvcav2beta1.ModelCacheBindingResourceIntent{ + WriterNamespace: writerNamespace, + PersistentVolumeClaimNames: []string{rwPVCName}, + JobNames: []string{jobName}, + } + if selection.Workflow == ModelCacheWorkflowRegular && selection.Transition == ModelCacheTransitionROXReadOnly { + resources.PersistentVolumeClaimNames = append( + resources.PersistentVolumeClaimNames, "ro-pvc-"+cacheHandle) + } else if selection.Workflow == ModelCacheWorkflowHelm { + resources.LeaseName = buildInitLeaseName(cacheHandle) + } + if selection.EncryptionRequired { + switch selection.Workflow { + case ModelCacheWorkflowRegular: + // Keep this snapshot aligned with pkg/nvca/encryption, which + // predates the Helm controller's resource naming convention. + domainHash := hashNCAID(sharingDomain) + resources.StorageClassNames = []string{domainHash + "-sc"} + resources.SecretNames = []string{domainHash} + case ModelCacheWorkflowHelm: + resources.StorageClassNames = []string{buildStorageClassName(sharingDomain)} + resources.SecretNames = []string{buildStorageClassSecretName(sharingDomain)} + } + } + + binding := &nvcav2beta1.ModelCacheBinding{ + TypeMeta: metav1.TypeMeta{ + APIVersion: nvcav2beta1.SchemeGroupVersion.String(), + Kind: "ModelCacheBinding", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: ModelCacheBindingName(cacheHandle), + Namespace: ModelCacheInitNamespace, + Finalizers: []string{nvcav2beta1.ModelCacheBindingFinalizer}, + }, + Spec: nvcav2beta1.ModelCacheBindingSpec{ + Identity: nvcav2beta1.ModelCacheBindingIdentity{ + Version: ModelCacheStorageSelectionVersion, + Workflow: nvcav2beta1.ModelCacheWorkflow(selection.Workflow), + SharingDomainDigest: digestBindingValue(sharingDomain), + CacheHandleDigest: digestBindingValue(cacheHandle), + }, + Decision: nvcav2beta1.ModelCacheBindingDecision{ + Provider: selection.Provider, + Provisioner: selection.Provisioner, + Transition: selection.Transition, + RequiredAccessModes: append([]corev1.PersistentVolumeAccessMode(nil), selection.RequiredAccessModes...), + RequiredMountOptions: append([]string(nil), selection.RequiredMountOptions...), + CatalogDigest: selection.CatalogDigest, + EncryptionRequired: selection.EncryptionRequired, + }, + StorageClass: nvcav2beta1.ModelCacheStorageClassSnapshot{ + Name: selection.StorageClassName, + UID: selection.StorageClassUID, + ReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + ConfigurationDigest: selection.StorageClassDigest, + }, + Resources: resources, + }, + } + return binding, nil +} + +// ModelCacheBindingName returns the deterministic, non-sensitive name used by +// the current handle-scoped NVMesh resource layout. +func ModelCacheBindingName(cacheHandle string) string { + sum := sha256.Sum256([]byte(cacheHandle)) + return fmt.Sprintf("%s%x", modelCacheBindingNamePrefix, sum[:24]) +} + +func digestBindingValue(value string) string { + sum := sha256.Sum256([]byte(value)) + return fmt.Sprintf("sha256:%x", sum) +} + +// ValidateModelCacheBinding proves that an existing binding is the exact +// immutable object this request intended to create. +func ValidateModelCacheBinding( + binding *nvcav2beta1.ModelCacheBinding, + selection *PersistedModelCacheStorageSelection, + sharingDomain string, + cacheHandle string, + writerNamespace string, +) error { + if err := ValidateModelCacheBindingIntent( + binding, selection, sharingDomain, cacheHandle, writerNamespace); err != nil { + return err + } + if binding.Status.Phase != nvcav2beta1.ModelCacheBindingPhaseActive { + return fmt.Errorf("model cache binding %s/%s is not Active: %q", + binding.Namespace, binding.Name, binding.Status.Phase) + } + return nil +} + +// ValidateModelCacheBindingIntent validates identity and immutable spec without +// requiring status to be initialized. It is used only while recovering the +// create-before-status-update window. +func ValidateModelCacheBindingIntent( + binding *nvcav2beta1.ModelCacheBinding, + selection *PersistedModelCacheStorageSelection, + sharingDomain string, + cacheHandle string, + writerNamespace string, +) error { + if binding == nil { + return fmt.Errorf("model cache binding is nil") + } + expected, err := NewModelCacheBinding(selection, sharingDomain, cacheHandle, writerNamespace) + if err != nil { + return err + } + if binding.Name != expected.Name || binding.Namespace != expected.Namespace { + return fmt.Errorf("model cache binding identity %s/%s does not match expected %s/%s", + binding.Namespace, binding.Name, expected.Namespace, expected.Name) + } + if !binding.DeletionTimestamp.IsZero() { + return fmt.Errorf("model cache binding %s/%s is being deleted", binding.Namespace, binding.Name) + } + if !slices.Contains(binding.Finalizers, nvcav2beta1.ModelCacheBindingFinalizer) { + return fmt.Errorf("model cache binding %s/%s has no protection finalizer", binding.Namespace, binding.Name) + } + if !reflect.DeepEqual(binding.Spec, expected.Spec) { + return fmt.Errorf("model cache binding %s/%s immutable spec does not match the request intent", + binding.Namespace, binding.Name) + } + if selection.BindingName != "" { + if binding.Name != selection.BindingName || binding.UID != selection.BindingUID { + return fmt.Errorf("model cache binding reference changed from %s/%s to %s/%s", + selection.BindingName, selection.BindingUID, binding.Name, binding.UID) + } + } + return nil +} + +// ModelCacheBindingHasRequestReference reports whether status contains the +// exact request object, including its API-assigned UID. +func ModelCacheBindingHasRequestReference( + binding *nvcav2beta1.ModelCacheBinding, + namespace string, + name string, + uid types.UID, +) bool { + for _, ref := range binding.Status.RequestReferences { + if ref.Namespace == namespace && ref.Name == name && ref.UID == uid { + return true + } + } + return false +} diff --git a/src/compute-plane-services/nvca/pkg/storage/modelcache_binding_helm.go b/src/compute-plane-services/nvca/pkg/storage/modelcache_binding_helm.go new file mode 100644 index 000000000..ebb760c24 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/storage/modelcache_binding_helm.go @@ -0,0 +1,1066 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package storage + +import ( + "context" + "errors" + "fmt" + "slices" + + batchv1 "k8s.io/api/batch/v1" + coordv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation" + "sigs.k8s.io/controller-runtime/pkg/client" + + nvcav1new "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v1" + nvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1" + nvcatypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/types" +) + +var errModelCacheBindingOwnership = errors.New("model cache binding ownership mismatch") + +const ( + // ModelCacheBindingUIDLabelKey identifies resources owned by one immutable + // model-cache binding. Cache-handle labels remain for event fan-out only. + ModelCacheBindingUIDLabelKey = "nvca.nvcf.nvidia.io/model-cache-binding-uid" + // ModelCacheRequestUIDLabelKey identifies per-request reader resources. A + // binding UID is intentionally shared by every request using one cache, so it + // cannot by itself distinguish a stale same-name reader left by an earlier + // ICMSRequest generation. + ModelCacheRequestUIDLabelKey = "nvca.nvcf.nvidia.io/model-cache-request-uid" +) + +// SetModelCacheBindingUIDLabel stamps an object with an immutable binding UID. +// It rejects an existing label owned by another binding. +func SetModelCacheBindingUIDLabel(obj metav1.Object, bindingUID types.UID) error { + if obj == nil { + return fmt.Errorf("cannot label a nil object with a model cache binding UID") + } + value := string(bindingUID) + if value == "" { + return fmt.Errorf("model cache binding UID is empty") + } + if errs := validation.IsValidLabelValue(value); len(errs) != 0 { + return fmt.Errorf("model cache binding UID %q is not a valid label value: %v", value, errs) + } + labels := obj.GetLabels() + if labels == nil { + labels = map[string]string{} + obj.SetLabels(labels) + } + if existing := labels[ModelCacheBindingUIDLabelKey]; existing != "" && existing != value { + return fmt.Errorf("%w: object %s/%s belongs to model cache binding UID %q, not %q", + errModelCacheBindingOwnership, + obj.GetNamespace(), obj.GetName(), existing, value) + } + labels[ModelCacheBindingUIDLabelKey] = value + return nil +} + +// ValidateModelCacheBindingUIDLabel requires an object's existing ownership +// label to match the exact binding UID. It never adopts an unlabeled object. +func ValidateModelCacheBindingUIDLabel(obj metav1.Object, bindingUID types.UID) error { + if obj == nil { + return fmt.Errorf("cannot validate model cache binding ownership on a nil object") + } + want := string(bindingUID) + if got := obj.GetLabels()[ModelCacheBindingUIDLabelKey]; got != want { + return fmt.Errorf("%w: object %s/%s has model cache binding UID %q, want %q", + errModelCacheBindingOwnership, + obj.GetNamespace(), obj.GetName(), got, want) + } + return nil +} + +// SetModelCacheRequestUIDLabel stamps a per-request reader object with the +// exact API-assigned ICMSRequest UID. Shared writer objects must not use this +// label because several requests can legitimately share them. +func SetModelCacheRequestUIDLabel(obj metav1.Object, requestUID types.UID) error { + if obj == nil { + return fmt.Errorf("cannot label a nil object with a model cache request UID") + } + value := string(requestUID) + if value == "" { + return fmt.Errorf("model cache request UID is empty") + } + if errs := validation.IsValidLabelValue(value); len(errs) != 0 { + return fmt.Errorf("model cache request UID %q is not a valid label value: %v", value, errs) + } + labels := obj.GetLabels() + if labels == nil { + labels = map[string]string{} + obj.SetLabels(labels) + } + if existing := labels[ModelCacheRequestUIDLabelKey]; existing != "" && existing != value { + return fmt.Errorf("%w: object %s/%s belongs to model cache request UID %q, not %q", + errModelCacheBindingOwnership, + obj.GetNamespace(), obj.GetName(), existing, value) + } + labels[ModelCacheRequestUIDLabelKey] = value + return nil +} + +// ValidateModelCacheRequestUIDLabel requires exact per-request ownership and +// never adopts an unlabeled reader object. +func ValidateModelCacheRequestUIDLabel(obj metav1.Object, requestUID types.UID) error { + if obj == nil { + return fmt.Errorf("cannot validate model cache request ownership on a nil object") + } + want := string(requestUID) + if got := obj.GetLabels()[ModelCacheRequestUIDLabelKey]; got != want { + return fmt.Errorf("%w: object %s/%s has model cache request UID %q, want %q", + errModelCacheBindingOwnership, + obj.GetNamespace(), obj.GetName(), got, want) + } + return nil +} + +// ValidateModelCacheReaderOwnership is shared by reader reconciliation and +// cleanup. Both the shared binding identity and the exact request generation +// must match before a per-request PV or PVC can be adopted or deleted. +func ValidateModelCacheReaderOwnership( + obj metav1.Object, + bindingUID types.UID, + requestUID types.UID, +) error { + if err := ValidateModelCacheBindingUIDLabel(obj, bindingUID); err != nil { + return err + } + return ValidateModelCacheRequestUIDLabel(obj, requestUID) +} + +func propagateModelCacheBindingUIDLabel(owner metav1.Object, objects ...metav1.Object) error { + if owner == nil { + return fmt.Errorf("model cache binding label owner is nil") + } + bindingUID := types.UID(owner.GetLabels()[ModelCacheBindingUIDLabelKey]) + if bindingUID == "" { + return nil + } + for _, obj := range objects { + if err := SetModelCacheBindingUIDLabel(obj, bindingUID); err != nil { + return err + } + } + return nil +} + +func validateHelmModelCacheWriterObjectMeta( + existing metav1.Object, + wanted metav1.Object, + bindingUID types.UID, +) error { + if existing == nil || wanted == nil { + return fmt.Errorf("%w: Helm model cache writer object identity is incomplete", + errModelCacheBindingOwnership) + } + if existing.GetNamespace() != wanted.GetNamespace() || existing.GetName() != wanted.GetName() { + return fmt.Errorf("%w: Helm model cache writer object %s/%s does not match %s/%s", + errModelCacheBindingOwnership, + existing.GetNamespace(), existing.GetName(), wanted.GetNamespace(), wanted.GetName()) + } + if err := ValidateModelCacheBindingUIDLabel(existing, bindingUID); err != nil { + return err + } + for key, want := range wanted.GetLabels() { + if existing.GetLabels()[key] != want { + return fmt.Errorf("%w: Helm model cache writer object %s/%s label %q is %q, want %q", + errModelCacheBindingOwnership, existing.GetNamespace(), existing.GetName(), + key, existing.GetLabels()[key], want) + } + } + return nil +} + +func modelCachePVCVolumeMode(pvc *corev1.PersistentVolumeClaim) corev1.PersistentVolumeMode { + if pvc != nil && pvc.Spec.VolumeMode != nil { + return *pvc.Spec.VolumeMode + } + return corev1.PersistentVolumeFilesystem +} + +func validateHelmModelCacheWriterPVC( + existing *corev1.PersistentVolumeClaim, + wanted *corev1.PersistentVolumeClaim, + bindingUID types.UID, +) error { + if existing == nil || wanted == nil { + return fmt.Errorf("%w: Helm model cache writer PVC intent is incomplete", + errModelCacheBindingOwnership) + } + if err := validateHelmModelCacheWriterObjectMeta(existing, wanted, bindingUID); err != nil { + return err + } + if !slices.Equal(existing.Spec.AccessModes, wanted.Spec.AccessModes) || + !apiequality.Semantic.DeepEqual(existing.Spec.StorageClassName, wanted.Spec.StorageClassName) || + modelCachePVCVolumeMode(existing) != modelCachePVCVolumeMode(wanted) || + !apiequality.Semantic.DeepEqual(existing.Spec.Resources, wanted.Spec.Resources) || + !apiequality.Semantic.DeepEqual(existing.Spec.Selector, wanted.Spec.Selector) || + !apiequality.Semantic.DeepEqual(existing.Spec.DataSource, wanted.Spec.DataSource) || + !apiequality.Semantic.DeepEqual(existing.Spec.DataSourceRef, wanted.Spec.DataSourceRef) || + !apiequality.Semantic.DeepEqual( + existing.Spec.VolumeAttributesClassName, wanted.Spec.VolumeAttributesClassName) { + return fmt.Errorf("%w: Helm model cache writer PVC %s/%s immutable spec does not match intent", + errModelCacheBindingOwnership, existing.Namespace, existing.Name) + } + return nil +} + +func normalizeHelmWriterContainerDefaults(container *corev1.Container) { + if container == nil { + return + } + defaultPullPolicy := corev1.PullIfNotPresent + image := container.Image + lastSlash := -1 + for i := len(image) - 1; i >= 0; i-- { + if image[i] == '/' { + lastSlash = i + break + } + } + lastColon := -1 + for i := len(image) - 1; i > lastSlash; i-- { + if image[i] == ':' { + lastColon = i + break + } + } + if lastColon < 0 || image[lastColon+1:] == "latest" { + defaultPullPolicy = corev1.PullAlways + } + if container.ImagePullPolicy == defaultPullPolicy { + container.ImagePullPolicy = "" + } + if container.TerminationMessagePath == corev1.TerminationMessagePathDefault { + container.TerminationMessagePath = "" + } + if container.TerminationMessagePolicy == corev1.TerminationMessageReadFile { + container.TerminationMessagePolicy = "" + } + for i := range container.Ports { + if container.Ports[i].Protocol == corev1.ProtocolTCP { + container.Ports[i].Protocol = "" + } + } +} + +func normalizeHelmWriterPodSpec(spec *corev1.PodSpec) { + if spec == nil { + return + } + if spec.DNSPolicy == corev1.DNSClusterFirst { + spec.DNSPolicy = "" + } + if spec.SchedulerName == corev1.DefaultSchedulerName { + spec.SchedulerName = "" + } + if spec.TerminationGracePeriodSeconds != nil && *spec.TerminationGracePeriodSeconds == 30 { + spec.TerminationGracePeriodSeconds = nil + } + if spec.EnableServiceLinks != nil && *spec.EnableServiceLinks { + spec.EnableServiceLinks = nil + } + if spec.PreemptionPolicy != nil && *spec.PreemptionPolicy == corev1.PreemptLowerPriority { + spec.PreemptionPolicy = nil + } + for i := range spec.Containers { + normalizeHelmWriterContainerDefaults(&spec.Containers[i]) + } + for i := range spec.InitContainers { + normalizeHelmWriterContainerDefaults(&spec.InitContainers[i]) + } + for i := range spec.Volumes { + volume := &spec.Volumes[i] + defaultModes := make([]**int32, 0, 4) + if volume.Secret != nil { + defaultModes = append(defaultModes, &volume.Secret.DefaultMode) + } + if volume.ConfigMap != nil { + defaultModes = append(defaultModes, &volume.ConfigMap.DefaultMode) + } + if volume.DownwardAPI != nil { + defaultModes = append(defaultModes, &volume.DownwardAPI.DefaultMode) + } + if volume.Projected != nil { + defaultModes = append(defaultModes, &volume.Projected.DefaultMode) + } + for _, defaultMode := range defaultModes { + if *defaultMode != nil && **defaultMode == 0o644 { + *defaultMode = nil + } + } + } +} + +func normalizeHelmWriterJobSpec(spec *batchv1.JobSpec) { + if spec == nil { + return + } + if spec.Parallelism != nil && *spec.Parallelism == 1 { + spec.Parallelism = nil + } + if spec.Completions != nil && *spec.Completions == 1 { + spec.Completions = nil + } + if spec.BackoffLimit != nil && *spec.BackoffLimit == 6 { + spec.BackoffLimit = nil + } + if spec.CompletionMode != nil && *spec.CompletionMode == batchv1.NonIndexedCompletion { + spec.CompletionMode = nil + } + if spec.Suspend != nil && !*spec.Suspend { + spec.Suspend = nil + } + if spec.ManualSelector != nil && !*spec.ManualSelector { + spec.ManualSelector = nil + } +} + +func validateHelmModelCacheWriterJob( + existing *batchv1.Job, + wanted *batchv1.Job, + bindingUID types.UID, +) error { + if existing == nil || wanted == nil { + return fmt.Errorf("%w: Helm model cache writer Job intent is incomplete", + errModelCacheBindingOwnership) + } + if err := validateHelmModelCacheWriterObjectMeta(existing, wanted, bindingUID); err != nil { + return err + } + if err := ValidateModelCacheBindingUIDLabel(&existing.Spec.Template.ObjectMeta, bindingUID); err != nil { + return fmt.Errorf("writer Job Pod template: %w", err) + } + for key, want := range wanted.Spec.Template.Labels { + if existing.Spec.Template.Labels[key] != want { + return fmt.Errorf("%w: Helm model cache writer Job %s/%s Pod-template label %q is %q, want %q", + errModelCacheBindingOwnership, existing.Namespace, existing.Name, + key, existing.Spec.Template.Labels[key], want) + } + } + for key, want := range wanted.Spec.Template.Annotations { + if existing.Spec.Template.Annotations[key] != want { + return fmt.Errorf("%w: Helm model cache writer Job %s/%s Pod-template annotation %q changed", + errModelCacheBindingOwnership, existing.Namespace, existing.Name, key) + } + } + + existingSpec := existing.Spec.DeepCopy() + wantedSpec := wanted.Spec.DeepCopy() + existingTemplate := existingSpec.Template.Spec.DeepCopy() + wantedTemplate := wantedSpec.Template.Spec.DeepCopy() + normalizeHelmWriterPodSpec(existingTemplate) + normalizeHelmWriterPodSpec(wantedTemplate) + existingSpec.Template = corev1.PodTemplateSpec{} + wantedSpec.Template = corev1.PodTemplateSpec{} + if wantedSpec.Selector == nil { + existingSpec.Selector = nil + } + normalizeHelmWriterJobSpec(existingSpec) + normalizeHelmWriterJobSpec(wantedSpec) + if !apiequality.Semantic.DeepEqual(existingSpec, wantedSpec) || + !apiequality.Semantic.DeepEqual(existingTemplate, wantedTemplate) { + return fmt.Errorf("%w: Helm model cache writer Job %s/%s immutable spec does not match intent", + errModelCacheBindingOwnership, existing.Namespace, existing.Name) + } + return nil +} + +func validateHelmModelCacheWriterLease( + existing *coordv1.Lease, + wanted *coordv1.Lease, + bindingUID types.UID, +) error { + if existing == nil || wanted == nil { + return fmt.Errorf("%w: Helm model cache writer Lease intent is incomplete", + errModelCacheBindingOwnership) + } + if err := validateHelmModelCacheWriterObjectMeta(existing, wanted, bindingUID); err != nil { + return err + } + if existing.Spec.LeaseDurationSeconds == nil || wanted.Spec.LeaseDurationSeconds == nil || + *existing.Spec.LeaseDurationSeconds != *wanted.Spec.LeaseDurationSeconds { + return fmt.Errorf("%w: Helm model cache writer Lease %s/%s duration does not match intent", + errModelCacheBindingOwnership, existing.Namespace, existing.Name) + } + if existing.Spec.HolderIdentity == nil || *existing.Spec.HolderIdentity == "" { + return fmt.Errorf("%w: Helm model cache writer Lease %s/%s has no holder", + errModelCacheBindingOwnership, existing.Namespace, existing.Name) + } + return nil +} + +func modelCacheSecretData(secret *corev1.Secret) map[string][]byte { + if secret == nil { + return nil + } + data := make(map[string][]byte, len(secret.Data)+len(secret.StringData)) + for key, value := range secret.Data { + data[key] = append([]byte(nil), value...) + } + for key, value := range secret.StringData { + data[key] = []byte(value) + } + return data +} + +func validateHelmModelCacheWriterSecret( + existing *corev1.Secret, + wanted *corev1.Secret, + bindingUID types.UID, +) error { + if existing == nil || wanted == nil { + return fmt.Errorf("%w: Helm model cache writer Secret intent is incomplete", + errModelCacheBindingOwnership) + } + if err := validateHelmModelCacheWriterObjectMeta(existing, wanted, bindingUID); err != nil { + return err + } + for key, want := range wanted.Annotations { + if existing.Annotations[key] != want { + return fmt.Errorf("%w: Helm model cache writer Secret %s/%s annotation %q changed", + errModelCacheBindingOwnership, existing.Namespace, existing.Name, key) + } + } + if existing.Type != wanted.Type || + !apiequality.Semantic.DeepEqual(existing.Immutable, wanted.Immutable) || + !apiequality.Semantic.DeepEqual(modelCacheSecretData(existing), modelCacheSecretData(wanted)) { + return fmt.Errorf("%w: Helm model cache writer Secret %s/%s immutable intent does not match", + errModelCacheBindingOwnership, existing.Namespace, existing.Name) + } + return nil +} + +func validateHelmModelCacheBindingOwnedObjectIntent( + existing client.Object, + wanted client.Object, + bindingUID types.UID, +) error { + switch wanted := wanted.(type) { + case *corev1.PersistentVolumeClaim: + got, ok := existing.(*corev1.PersistentVolumeClaim) + if !ok { + return fmt.Errorf("%w: existing Helm writer object %T is not a PVC", + errModelCacheBindingOwnership, existing) + } + return validateHelmModelCacheWriterPVC(got, wanted, bindingUID) + case *batchv1.Job: + got, ok := existing.(*batchv1.Job) + if !ok { + return fmt.Errorf("%w: existing Helm writer object %T is not a Job", + errModelCacheBindingOwnership, existing) + } + return validateHelmModelCacheWriterJob(got, wanted, bindingUID) + case *coordv1.Lease: + got, ok := existing.(*coordv1.Lease) + if !ok { + return fmt.Errorf("%w: existing Helm writer object %T is not a Lease", + errModelCacheBindingOwnership, existing) + } + return validateHelmModelCacheWriterLease(got, wanted, bindingUID) + case *corev1.Secret: + got, ok := existing.(*corev1.Secret) + if !ok { + return fmt.Errorf("%w: existing Helm writer object %T is not a Secret", + errModelCacheBindingOwnership, existing) + } + return validateHelmModelCacheWriterSecret(got, wanted, bindingUID) + default: + return ValidateModelCacheBindingUIDLabel(existing, bindingUID) + } +} + +// prepareHelmModelCacheBindingResources proves that translated writer +// artifacts and the ownership Lease match the immutable binding intent before +// any create. Existing shared objects are accepted only with the exact binding +// UID; per-request ownership is deliberately not applied to shared writers. +func (r *Reconciler) prepareHelmModelCacheBindingResources( + ctx context.Context, + binding *nvcav2beta1.ModelCacheBinding, + rwPVC *corev1.PersistentVolumeClaim, + initJob *batchv1.Job, + lease *coordv1.Lease, +) error { + if binding == nil { + return fmt.Errorf("model cache binding is nil") + } + if rwPVC == nil || initJob == nil || lease == nil { + return fmt.Errorf("Helm model cache writer PVC, Job, and Lease must be present") + } + resources := binding.Spec.Resources + if resources.WriterNamespace != ModelCacheInitNamespace { + return fmt.Errorf("%w: Helm model cache writer namespace %q is not %q", + errModelCacheBindingOwnership, resources.WriterNamespace, ModelCacheInitNamespace) + } + if len(resources.PersistentVolumeClaimNames) != 1 || + resources.PersistentVolumeClaimNames[0] != rwPVC.Name { + return fmt.Errorf("%w: Helm model cache writer PVC %q does not match binding intent %v", + errModelCacheBindingOwnership, rwPVC.Name, resources.PersistentVolumeClaimNames) + } + if len(resources.JobNames) != 1 || resources.JobNames[0] != initJob.Name { + return fmt.Errorf("%w: Helm model cache writer Job %q does not match binding intent %v", + errModelCacheBindingOwnership, initJob.Name, resources.JobNames) + } + if resources.LeaseName == "" || resources.LeaseName != lease.Name { + return fmt.Errorf("%w: Helm model cache Lease %q does not match binding intent %q", + errModelCacheBindingOwnership, lease.Name, resources.LeaseName) + } + for kind, obj := range map[string]metav1.Object{ + "writer PVC": rwPVC, + "writer Job": initJob, + "Lease": lease, + } { + if obj.GetNamespace() != resources.WriterNamespace { + return fmt.Errorf("%w: Helm model cache %s namespace %q does not match binding intent %q", + errModelCacheBindingOwnership, kind, obj.GetNamespace(), resources.WriterNamespace) + } + if err := ValidateModelCacheBindingUIDLabel(obj, binding.UID); err != nil { + return err + } + } + if err := ValidateModelCacheBindingUIDLabel(&initJob.Spec.Template.ObjectMeta, binding.UID); err != nil { + return fmt.Errorf("writer Job Pod template: %w", err) + } + + for _, wanted := range []client.Object{rwPVC, initJob, lease} { + existing, ok := wanted.DeepCopyObject().(client.Object) + if !ok { + return fmt.Errorf("model cache object %T is not a controller-runtime client object", wanted) + } + err := r.Client.Get(ctx, client.ObjectKeyFromObject(wanted), existing) + switch { + case apierrors.IsNotFound(err): + continue + case err != nil: + return fmt.Errorf("get existing Helm model cache object %s/%s: %w", + wanted.GetNamespace(), wanted.GetName(), err) + default: + if err := validateHelmModelCacheBindingOwnedObjectIntent( + existing, wanted, binding.UID); err != nil { + return err + } + } + } + return nil +} + +func validateHelmModelCacheSecondaryPV( + st *nvcav1new.StorageRequest, + selection *PersistedModelCacheStorageSelection, + primaryPV *corev1.PersistentVolume, + secondaryPV *corev1.PersistentVolume, + roPVCName string, + bindingUID types.UID, + requestUID types.UID, +) error { + if st == nil || st.Spec.ModelCache == nil || selection == nil || primaryPV == nil || secondaryPV == nil { + return fmt.Errorf("%w: incomplete Helm model cache secondary PV identity", + errModelCacheBindingOwnership) + } + if err := ValidateModelCacheReaderOwnership(secondaryPV, bindingUID, requestUID); err != nil { + return err + } + if err := ValidateModelCacheBindingUIDLabel(primaryPV, bindingUID); err != nil { + return err + } + expectedName := "secondary-pv-" + st.Spec.ICMSRequestName + if secondaryPV.Name != expectedName || secondaryPV.Namespace != "" { + return fmt.Errorf("%w: secondary PV identity %q/%q does not match %q", + errModelCacheBindingOwnership, secondaryPV.Namespace, secondaryPV.Name, expectedName) + } + if primaryPV.Spec.CSI == nil || secondaryPV.Spec.CSI == nil { + return fmt.Errorf("%w: primary or secondary PV has no CSI source", errModelCacheBindingOwnership) + } + if primaryPV.Spec.CSI.Driver != selection.Provisioner || secondaryPV.Spec.CSI.Driver != selection.Provisioner { + return fmt.Errorf("%w: primary/secondary PV provisioner does not match persisted provisioner %q", + errModelCacheBindingOwnership, selection.Provisioner) + } + expectedHandle, err := updateSecondaryPVVolumeHandle(primaryPV.Spec.CSI.VolumeHandle, st.Namespace) + if err != nil { + return fmt.Errorf("%w: derive secondary PV volume identity: %w", errModelCacheBindingOwnership, err) + } + if secondaryPV.Spec.CSI.VolumeHandle != expectedHandle { + return fmt.Errorf("%w: secondary PV volume identity %q does not match %q", + errModelCacheBindingOwnership, secondaryPV.Spec.CSI.VolumeHandle, expectedHandle) + } + if secondaryPV.Spec.StorageClassName != primaryPV.Spec.StorageClassName { + return fmt.Errorf("%w: secondary PV StorageClass %q does not match primary PV %q", + errModelCacheBindingOwnership, secondaryPV.Spec.StorageClassName, primaryPV.Spec.StorageClassName) + } + expectedCSI := primaryPV.Spec.CSI.DeepCopy() + expectedCSI.VolumeHandle = expectedHandle + if !apiequality.Semantic.DeepEqual(secondaryPV.Spec.CSI, expectedCSI) { + return fmt.Errorf("%w: secondary PV CSI source does not match the primary PV intent", + errModelCacheBindingOwnership) + } + if secondaryPV.Spec.PersistentVolumeReclaimPolicy != corev1.PersistentVolumeReclaimRetain { + return fmt.Errorf("%w: secondary PV %q reclaim policy is %q, want Retain", + errModelCacheBindingOwnership, secondaryPV.Name, + secondaryPV.Spec.PersistentVolumeReclaimPolicy) + } + if !apiequality.Semantic.DeepEqual(secondaryPV.Spec.Capacity, primaryPV.Spec.Capacity) || + !apiequality.Semantic.DeepEqual(secondaryPV.Spec.VolumeMode, primaryPV.Spec.VolumeMode) || + !apiequality.Semantic.DeepEqual(secondaryPV.Spec.NodeAffinity, primaryPV.Spec.NodeAffinity) { + return fmt.Errorf("%w: secondary PV %q capacity, volume mode, or node affinity changed", + errModelCacheBindingOwnership, secondaryPV.Name) + } + if !slices.Equal(secondaryPV.Spec.AccessModes, accessModesRO) { + return fmt.Errorf("%w: secondary PV access modes %v do not match %v", + errModelCacheBindingOwnership, secondaryPV.Spec.AccessModes, accessModesRO) + } + claimRef := secondaryPV.Spec.ClaimRef + if claimRef == nil || claimRef.APIVersion != "v1" || claimRef.Kind != "PersistentVolumeClaim" || + claimRef.Namespace != st.Namespace || claimRef.Name != roPVCName { + return fmt.Errorf("%w: secondary PV claimRef does not match reader PVC %s/%s", + errModelCacheBindingOwnership, st.Namespace, roPVCName) + } + return nil +} + +func validateHelmModelCacheReaderPVC( + st *nvcav1new.StorageRequest, + rwPVC *corev1.PersistentVolumeClaim, + secondaryPV *corev1.PersistentVolume, + roPVC *corev1.PersistentVolumeClaim, + bindingUID types.UID, + requestUID types.UID, +) error { + if st == nil || st.Spec.ModelCache == nil || rwPVC == nil || secondaryPV == nil || roPVC == nil { + return fmt.Errorf("%w: incomplete Helm model cache reader PVC identity", + errModelCacheBindingOwnership) + } + if err := ValidateModelCacheReaderOwnership(roPVC, bindingUID, requestUID); err != nil { + return err + } + expectedName := "ro-pvc-" + st.Spec.ModelCache.CacheHandle + expectedPVName := "secondary-pv-" + st.Spec.ICMSRequestName + if roPVC.Name != expectedName || roPVC.Namespace != st.Namespace { + return fmt.Errorf("%w: reader PVC identity %s/%s does not match %s/%s", + errModelCacheBindingOwnership, roPVC.Namespace, roPVC.Name, st.Namespace, expectedName) + } + if roPVC.Spec.VolumeName != expectedPVName || secondaryPV.Name != expectedPVName { + return fmt.Errorf("%w: reader PVC volumeName %q does not match secondary PV %q", + errModelCacheBindingOwnership, roPVC.Spec.VolumeName, expectedPVName) + } + if !slices.Equal(roPVC.Spec.AccessModes, accessModesRO) { + return fmt.Errorf("%w: reader PVC access modes %v do not match %v", + errModelCacheBindingOwnership, roPVC.Spec.AccessModes, accessModesRO) + } + if (roPVC.Spec.StorageClassName == nil) != (rwPVC.Spec.StorageClassName == nil) || + (roPVC.Spec.StorageClassName != nil && *roPVC.Spec.StorageClassName != *rwPVC.Spec.StorageClassName) { + return fmt.Errorf("%w: reader PVC StorageClass does not match writer PVC", + errModelCacheBindingOwnership) + } + claimRef := secondaryPV.Spec.ClaimRef + if claimRef == nil || claimRef.Namespace != roPVC.Namespace || claimRef.Name != roPVC.Name { + return fmt.Errorf("%w: secondary PV claimRef does not match reader PVC identity", + errModelCacheBindingOwnership) + } + if roPVC.Status.Phase == corev1.ClaimBound && (roPVC.UID == "" || claimRef.UID != roPVC.UID) { + return fmt.Errorf("%w: bound reader PVC %s/%s UID %q does not match secondary PV claimRef UID %q", + errModelCacheBindingOwnership, roPVC.Namespace, roPVC.Name, roPVC.UID, claimRef.UID) + } + if roPVC.Status.Phase != corev1.ClaimBound && claimRef.UID != "" && claimRef.UID != roPVC.UID { + return fmt.Errorf("%w: secondary PV claimRef UID %q does not match reader PVC UID %q", + errModelCacheBindingOwnership, claimRef.UID, roPVC.UID) + } + return nil +} + +// validateHelmModelCachePrimaryPVForReuse proves that a discovered primary PV +// is the exact immutable backing volume finalized for this binding. In +// particular, a matching cache-handle label alone is not authority to mutate +// or clone a PV. +func validateHelmModelCachePrimaryPVForReuse( + st *nvcav1new.StorageRequest, + selection *PersistedModelCacheStorageSelection, + rwPVC *corev1.PersistentVolumeClaim, + primaryPV *corev1.PersistentVolume, + bindingUID types.UID, +) error { + if st == nil || st.Spec.ModelCache == nil || selection == nil || rwPVC == nil || primaryPV == nil { + return fmt.Errorf("%w: incomplete Helm model cache primary PV identity", + errModelCacheBindingOwnership) + } + if err := ValidateModelCacheBindingUIDLabel(primaryPV, bindingUID); err != nil { + return err + } + if primaryPV.Labels[primaryPVLabelKey] != primaryPVLabelValue || + primaryPV.Labels[modelCacheHandleLabelKey] != st.Spec.ModelCache.CacheHandle { + return fmt.Errorf("%w: primary PV %q labels do not match cache handle %q", + errModelCacheBindingOwnership, primaryPV.Name, st.Spec.ModelCache.CacheHandle) + } + if primaryPV.Spec.CSI == nil || primaryPV.Spec.CSI.Driver != selection.Provisioner { + return fmt.Errorf("%w: primary PV %q provisioner does not match persisted provisioner %q", + errModelCacheBindingOwnership, primaryPV.Name, selection.Provisioner) + } + if primaryPV.Spec.CSI.VolumeHandle == "" { + return fmt.Errorf("%w: primary PV %q has an empty volume handle", + errModelCacheBindingOwnership, primaryPV.Name) + } + expectedHandle, err := updateSecondaryPVVolumeHandle( + primaryPV.Spec.CSI.VolumeHandle, ModelCacheInitNamespace) + if err != nil || expectedHandle != primaryPV.Spec.CSI.VolumeHandle { + return fmt.Errorf("%w: primary PV %q volume handle %q does not identify writer namespace %q", + errModelCacheBindingOwnership, primaryPV.Name, + primaryPV.Spec.CSI.VolumeHandle, ModelCacheInitNamespace) + } + if !selection.EncryptionRequired && primaryPV.Spec.StorageClassName != selection.StorageClassName { + return fmt.Errorf("%w: primary PV %q StorageClass %q does not match persisted StorageClass %q", + errModelCacheBindingOwnership, primaryPV.Name, + primaryPV.Spec.StorageClassName, selection.StorageClassName) + } + if primaryPV.Spec.PersistentVolumeReclaimPolicy != corev1.PersistentVolumeReclaimRetain { + return fmt.Errorf("%w: primary PV %q reclaim policy is %q, want Retain", + errModelCacheBindingOwnership, primaryPV.Name, + primaryPV.Spec.PersistentVolumeReclaimPolicy) + } + if !slices.Equal(primaryPV.Spec.AccessModes, + []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}) { + return fmt.Errorf("%w: primary PV %q access modes %v do not match writer RWO intent", + errModelCacheBindingOwnership, primaryPV.Name, primaryPV.Spec.AccessModes) + } + claimRef := primaryPV.Spec.ClaimRef + if claimRef == nil || claimRef.APIVersion != "v1" || claimRef.Kind != "PersistentVolumeClaim" || + claimRef.Namespace != ModelCacheInitNamespace || claimRef.Name != rwPVC.Name || claimRef.UID == "" { + return fmt.Errorf("%w: primary PV %q claimRef does not identify writer PVC %s/%s", + errModelCacheBindingOwnership, primaryPV.Name, ModelCacheInitNamespace, rwPVC.Name) + } + if rwPVC.UID != "" && claimRef.UID != rwPVC.UID { + return fmt.Errorf("%w: primary PV %q claimRef UID %q does not match live writer PVC UID %q", + errModelCacheBindingOwnership, primaryPV.Name, claimRef.UID, rwPVC.UID) + } + return nil +} + +func validateHelmModelCachePrimaryPVForFinalize( + selection *PersistedModelCacheStorageSelection, + rwPVC *corev1.PersistentVolumeClaim, + primaryPV *corev1.PersistentVolume, +) error { + if selection == nil || rwPVC == nil || primaryPV == nil || primaryPV.Spec.CSI == nil { + return fmt.Errorf("%w: primary PV provisioning identity is incomplete", + errModelCacheBindingOwnership) + } + if primaryPV.Spec.CSI.Driver != selection.Provisioner { + return fmt.Errorf("%w: primary PV %q provisioner %q does not match persisted provisioner %q", + errModelCacheBindingOwnership, primaryPV.Name, + primaryPV.Spec.CSI.Driver, selection.Provisioner) + } + if primaryPV.Spec.CSI.VolumeHandle == "" { + return fmt.Errorf("%w: primary PV %q has an empty volume handle", + errModelCacheBindingOwnership, primaryPV.Name) + } + expectedHandle, err := updateSecondaryPVVolumeHandle( + primaryPV.Spec.CSI.VolumeHandle, ModelCacheInitNamespace) + if err != nil || expectedHandle != primaryPV.Spec.CSI.VolumeHandle { + return fmt.Errorf("%w: primary PV %q volume handle %q does not identify writer namespace %q", + errModelCacheBindingOwnership, primaryPV.Name, + primaryPV.Spec.CSI.VolumeHandle, ModelCacheInitNamespace) + } + if rwPVC.Spec.StorageClassName == nil || *rwPVC.Spec.StorageClassName == "" || + primaryPV.Spec.StorageClassName != *rwPVC.Spec.StorageClassName { + return fmt.Errorf("%w: primary PV %q StorageClass %q does not match writer PVC", + errModelCacheBindingOwnership, primaryPV.Name, primaryPV.Spec.StorageClassName) + } + if primaryPV.Spec.PersistentVolumeReclaimPolicy != corev1.PersistentVolumeReclaimRetain { + return fmt.Errorf("%w: primary PV %q reclaim policy is %q, want Retain", + errModelCacheBindingOwnership, primaryPV.Name, primaryPV.Spec.PersistentVolumeReclaimPolicy) + } + if !slices.Equal(primaryPV.Spec.AccessModes, rwPVC.Spec.AccessModes) { + return fmt.Errorf("%w: primary PV %q access modes %v do not match writer PVC %v", + errModelCacheBindingOwnership, primaryPV.Name, + primaryPV.Spec.AccessModes, rwPVC.Spec.AccessModes) + } + return nil +} + +func validateHelmModelCachePrimaryPVClaim( + rwPVC *corev1.PersistentVolumeClaim, + primaryPV *corev1.PersistentVolume, + bindingUID types.UID, +) error { + if rwPVC == nil || primaryPV == nil { + return fmt.Errorf("%w: writer PVC and primary PV must be present", errModelCacheBindingOwnership) + } + if rwPVC.Namespace != ModelCacheInitNamespace || rwPVC.UID == "" { + return fmt.Errorf("%w: writer PVC %s/%s has no exact API identity", + errModelCacheBindingOwnership, rwPVC.Namespace, rwPVC.Name) + } + if err := ValidateModelCacheBindingUIDLabel(rwPVC, bindingUID); err != nil { + return err + } + if rwPVC.Spec.VolumeName != primaryPV.Name { + return fmt.Errorf("%w: writer PVC volumeName %q does not match primary PV %q", + errModelCacheBindingOwnership, rwPVC.Spec.VolumeName, primaryPV.Name) + } + claimRef := primaryPV.Spec.ClaimRef + if claimRef == nil || claimRef.APIVersion != "v1" || claimRef.Kind != "PersistentVolumeClaim" || + claimRef.Namespace != rwPVC.Namespace || claimRef.Name != rwPVC.Name || claimRef.UID != rwPVC.UID { + return fmt.Errorf("%w: primary PV claimRef does not match writer PVC %s/%s UID %q", + errModelCacheBindingOwnership, rwPVC.Namespace, rwPVC.Name, rwPVC.UID) + } + return nil +} + +func (r *Reconciler) createOrValidateModelCacheBindingOwnedObject( + ctx context.Context, + obj client.Object, +) (bool, error) { + err := r.Client.Create(ctx, obj) + if err == nil { + return false, nil + } + if !apierrors.IsAlreadyExists(err) { + return false, err + } + bindingUID := types.UID(obj.GetLabels()[ModelCacheBindingUIDLabelKey]) + if bindingUID == "" { + return true, nil + } + existing, ok := obj.DeepCopyObject().(client.Object) + if !ok { + return true, fmt.Errorf("model cache object %T is not a controller-runtime client object", obj) + } + if err := r.Client.Get(ctx, client.ObjectKeyFromObject(obj), existing); err != nil { + return true, fmt.Errorf("get existing model cache object %s/%s: %w", + obj.GetNamespace(), obj.GetName(), err) + } + if err := validateHelmModelCacheBindingOwnedObjectIntent(existing, obj, bindingUID); err != nil { + return true, err + } + return true, nil +} + +func (r *Reconciler) validatePersistedHelmModelCacheBinding( + ctx context.Context, + st *nvcav1new.StorageRequest, + icmsReq *nvcav2beta1.ICMSRequest, + selection *PersistedModelCacheStorageSelection, +) (*nvcav2beta1.ModelCacheBinding, error) { + if st == nil || st.Spec.ModelCache == nil { + return nil, fmt.Errorf("model cache StorageRequest is missing its modelCache spec") + } + if icmsReq == nil { + return nil, fmt.Errorf("ICMSRequest is nil") + } + if selection == nil { + return nil, fmt.Errorf("persisted model cache storage selection is nil") + } + if selection.Workflow != ModelCacheWorkflowHelm || selection.Mode != ModelCacheSelectionDurable { + return nil, fmt.Errorf("model cache binding requires a durable Helm selection") + } + if selection.BindingName == "" || selection.BindingUID == "" { + return nil, fmt.Errorf("durable Helm model cache selection has no binding reference") + } + if icmsReq.UID == "" { + return nil, fmt.Errorf("ICMSRequest %s/%s has no UID", icmsReq.Namespace, icmsReq.Name) + } + if st.Spec.ICMSRequestName != icmsReq.Name || st.Spec.ICMSRequestNamespace != icmsReq.Namespace { + return nil, fmt.Errorf("StorageRequest ICMS identity %s/%s does not match %s/%s", + st.Spec.ICMSRequestNamespace, st.Spec.ICMSRequestName, icmsReq.Namespace, icmsReq.Name) + } + if got := st.Annotations[ICMSRequestUIDAnnotationKey]; got != string(icmsReq.UID) { + return nil, fmt.Errorf("StorageRequest ICMS UID %q does not match %q", got, icmsReq.UID) + } + + binding := &nvcav2beta1.ModelCacheBinding{} + key := client.ObjectKey{Namespace: ModelCacheInitNamespace, Name: selection.BindingName} + if err := r.Client.Get(ctx, key, binding); err != nil { + return nil, fmt.Errorf("get model cache binding %s/%s: %w", key.Namespace, key.Name, err) + } + if binding.UID != selection.BindingUID { + return nil, fmt.Errorf("model cache binding %s/%s UID %q does not match persisted UID %q", + binding.Namespace, binding.Name, binding.UID, selection.BindingUID) + } + if err := ValidateModelCacheBinding( + binding, + selection, + icmsReq.Spec.NCAId, + st.Spec.ModelCache.CacheHandle, + ModelCacheInitNamespace, + ); err != nil { + return nil, err + } + if !ModelCacheBindingHasRequestReference(binding, icmsReq.Namespace, icmsReq.Name, icmsReq.UID) { + return nil, fmt.Errorf("model cache binding %s/%s has no reference to ICMSRequest %s/%s UID %q", + binding.Namespace, binding.Name, icmsReq.Namespace, icmsReq.Name, icmsReq.UID) + } + return binding, nil +} + +func (r *Reconciler) validateModelCacheBindingForCleanup( + ctx context.Context, + st *nvcav1new.StorageRequest, +) (types.UID, bool, bool, error) { + raw := st.Annotations[ModelCacheStorageSelectionAnnotationKey] + if raw == "" { + return "", false, false, nil + } + selection, err := ParsePersistedModelCacheStorageSelection(raw) + if err != nil { + return "", true, false, fmt.Errorf("parse persisted model cache storage selection for cleanup: %w", err) + } + if selection.Workflow != ModelCacheWorkflowHelm || selection.Mode != ModelCacheSelectionDurable || + selection.BindingName == "" || selection.BindingUID == "" { + return "", true, false, fmt.Errorf("annotated model cache cleanup requires a durable Helm binding reference") + } + requestUID := types.UID(st.Annotations[ICMSRequestUIDAnnotationKey]) + if requestUID == "" || st.Spec.ICMSRequestName == "" || st.Spec.ICMSRequestNamespace == "" { + return "", true, false, fmt.Errorf("annotated model cache cleanup has an incomplete ICMSRequest identity") + } + sharingDomain, ok := nvcatypes.GetNCAIDLabelVal(st.Labels) + if !ok { + return "", true, false, fmt.Errorf("annotated model cache cleanup has no sharing-domain label") + } + + binding := &nvcav2beta1.ModelCacheBinding{} + key := client.ObjectKey{Namespace: ModelCacheInitNamespace, Name: selection.BindingName} + if err := r.Client.Get(ctx, key, binding); err != nil { + return "", true, false, fmt.Errorf("get model cache binding for cleanup %s/%s: %w", key.Namespace, key.Name, err) + } + if err := ValidateModelCacheBinding( + binding, selection, sharingDomain, st.Spec.ModelCache.CacheHandle, ModelCacheInitNamespace); err != nil { + return "", true, false, fmt.Errorf("refusing model cache cleanup: %w", err) + } + referencePresent := ModelCacheBindingHasRequestReference( + binding, st.Spec.ICMSRequestNamespace, st.Spec.ICMSRequestName, requestUID) + if !referencePresent { + // The ICMSRequest finalizer releases its binding reference before the + // request disappears. The persisted StorageRequest identity is the + // cleanup tombstone, but it is valid only after the exact request is + // deleting or gone. A live request without its reference fails closed. + request := &nvcav2beta1.ICMSRequest{} + requestKey := client.ObjectKey{ + Namespace: st.Spec.ICMSRequestNamespace, + Name: st.Spec.ICMSRequestName, + } + err := r.Client.Get(ctx, requestKey, request) + switch { + case apierrors.IsNotFound(err): + case err != nil: + return "", true, false, fmt.Errorf( + "verify ICMSRequest deletion before model cache cleanup %s/%s: %w", + requestKey.Namespace, requestKey.Name, err) + case request.UID != requestUID: + return "", true, false, fmt.Errorf( + "refusing model cache cleanup: live ICMSRequest %s/%s UID %q does not match recorded UID %q", + request.Namespace, request.Name, request.UID, requestUID) + case request.DeletionTimestamp.IsZero(): + return "", true, false, fmt.Errorf( + "refusing model cache cleanup: live ICMSRequest %s/%s has no exact binding reference", + request.Namespace, request.Name) + } + } + return binding.UID, true, referencePresent, nil +} + +func (r *Reconciler) validateModelCacheInitCleanupOwnership( + ctx context.Context, + cacheHandle string, + bindingUID types.UID, + holderIdentity string, +) (bool, *coordv1.Lease, error) { + listOpts := []client.ListOption{ + client.MatchingLabels(map[string]string{modelCacheHandleLabelKey: cacheHandle}), + client.InNamespace(ModelCacheInitNamespace), + } + + jobs := &batchv1.JobList{} + if err := r.Client.List(ctx, jobs, listOpts...); err != nil { + return false, nil, fmt.Errorf("list model cache Jobs before cleanup: %w", err) + } + pods := &corev1.PodList{} + if err := r.Client.List(ctx, pods, listOpts...); err != nil { + return false, nil, fmt.Errorf("list model cache Pods before cleanup: %w", err) + } + pvcs := &corev1.PersistentVolumeClaimList{} + if err := r.Client.List(ctx, pvcs, listOpts...); err != nil { + return false, nil, fmt.Errorf("list model cache PVCs before cleanup: %w", err) + } + if len(jobs.Items) > 1 { + return false, nil, fmt.Errorf("refusing model cache cleanup: found %d writer Jobs for cache handle %q", + len(jobs.Items), cacheHandle) + } + if len(jobs.Items) == 1 && jobs.Items[0].Name != "writer-job-"+cacheHandle { + return false, nil, fmt.Errorf("refusing model cache cleanup: writer Job %q does not match cache handle %q", + jobs.Items[0].Name, cacheHandle) + } + if len(pvcs.Items) > 1 { + return false, nil, fmt.Errorf("refusing model cache cleanup: found %d writer PVCs for cache handle %q", + len(pvcs.Items), cacheHandle) + } + if len(pvcs.Items) == 1 && pvcs.Items[0].Name != "rw-pvc-"+cacheHandle { + return false, nil, fmt.Errorf("refusing model cache cleanup: writer PVC %q does not match cache handle %q", + pvcs.Items[0].Name, cacheHandle) + } + secrets := &corev1.SecretList{} + if err := r.Client.List(ctx, secrets, listOpts...); err != nil { + return false, nil, fmt.Errorf("list model cache Secrets before cleanup: %w", err) + } + + lease := &coordv1.Lease{} + key := client.ObjectKey{Namespace: ModelCacheInitNamespace, Name: buildInitLeaseName(cacheHandle)} + if err := r.Client.Get(ctx, key, lease); err != nil { + if apierrors.IsNotFound(err) { + if len(jobs.Items)+len(pods.Items)+len(pvcs.Items)+len(secrets.Items) != 0 { + return false, nil, fmt.Errorf("refusing model cache cleanup: writer artifacts exist without their ownership Lease") + } + return false, nil, nil + } + return false, nil, fmt.Errorf("get model cache Lease before cleanup: %w", err) + } + if err := ValidateModelCacheBindingUIDLabel(lease, bindingUID); err != nil { + return false, nil, err + } + if lease.Spec.HolderIdentity == nil || *lease.Spec.HolderIdentity == "" { + return false, nil, fmt.Errorf("refusing model cache cleanup: ownership Lease has no holder") + } + if *lease.Spec.HolderIdentity != holderIdentity { + return false, lease, nil + } + + for i := range jobs.Items { + if err := ValidateModelCacheBindingUIDLabel(&jobs.Items[i], bindingUID); err != nil { + return false, nil, err + } + } + for i := range pods.Items { + if err := ValidateModelCacheBindingUIDLabel(&pods.Items[i], bindingUID); err != nil { + return false, nil, err + } + } + for i := range pvcs.Items { + if err := ValidateModelCacheBindingUIDLabel(&pvcs.Items[i], bindingUID); err != nil { + return false, nil, err + } + } + for i := range secrets.Items { + if err := ValidateModelCacheBindingUIDLabel(&secrets.Items[i], bindingUID); err != nil { + return false, nil, err + } + } + return true, lease, nil +} diff --git a/src/compute-plane-services/nvca/pkg/storage/modelcache_binding_helm_test.go b/src/compute-plane-services/nvca/pkg/storage/modelcache_binding_helm_test.go new file mode 100644 index 000000000..ca8cb587d --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/storage/modelcache_binding_helm_test.go @@ -0,0 +1,1339 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package storage + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + batchv1 "k8s.io/api/batch/v1" + coordv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + nvcav1new "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v1" + nvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1" + nvcatypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/types" +) + +const ( + helmBindingTestCacheHandle = "cache-handle" + helmBindingTestNCAID = "nca-a" + helmBindingTestRequestNS = "requests" + helmBindingTestRequestName = "request-a" + helmBindingTestRequestUID = types.UID("request-uid") + helmBindingTestBindingUID = types.UID("binding-uid") +) + +func newHelmBindingTestFixture(t *testing.T) ( + *PersistedModelCacheStorageSelection, + *nvcav2beta1.ModelCacheBinding, + *nvcav1new.StorageRequest, + *nvcav2beta1.ICMSRequest, +) { + t.Helper() + selection := durableBindingSelection(t) + binding, err := NewModelCacheBinding( + selection, helmBindingTestNCAID, helmBindingTestCacheHandle, ModelCacheInitNamespace) + require.NoError(t, err) + binding.UID = helmBindingTestBindingUID + binding.Status.Phase = nvcav2beta1.ModelCacheBindingPhaseActive + binding.Status.RequestReferences = []nvcav2beta1.ModelCacheBindingRequestReference{{ + Namespace: helmBindingTestRequestNS, + Name: helmBindingTestRequestName, + UID: helmBindingTestRequestUID, + }} + selection.BindingName = binding.Name + selection.BindingUID = binding.UID + raw, err := selection.Marshal() + require.NoError(t, err) + + st := &nvcav1new.StorageRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: nvcav1new.ModelCacheRequest.Name(), + Namespace: "workload-a", + Labels: map[string]string{ + nvcatypes.NCAIDKey: nvcatypes.MakeNCAIDLabelValue(helmBindingTestNCAID), + }, + Annotations: map[string]string{ + ModelCacheStorageSelectionAnnotationKey: raw, + ICMSRequestUIDAnnotationKey: string(helmBindingTestRequestUID), + }, + }, + Spec: nvcav1new.StorageRequestSpec{ + Type: nvcav1new.ModelCacheRequest, + ICMSRequestName: helmBindingTestRequestName, + ICMSRequestNamespace: helmBindingTestRequestNS, + ModelCache: &nvcav1new.ModelCacheSpec{ + CacheHandle: helmBindingTestCacheHandle, + Backend: string(HelmCacheBackendNVMesh), + }, + }, + } + icmsReq := &nvcav2beta1.ICMSRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: helmBindingTestRequestName, + Namespace: helmBindingTestRequestNS, + UID: helmBindingTestRequestUID, + Annotations: map[string]string{ + ModelCacheStorageSelectionAnnotationKey: raw, + }, + }, + Spec: newModelCacheICMSSpec(helmBindingTestCacheHandle), + } + icmsReq.Spec.NCAId = helmBindingTestNCAID + icmsReq.Spec.CreationMsgInfo.NCAID = helmBindingTestNCAID + return selection, binding, st, icmsReq +} + +func TestValidatePersistedHelmCacheSelectionRequiresExactBinding(t *testing.T) { + for _, tt := range []struct { + name string + mutate func(*nvcav2beta1.ModelCacheBinding) + omit bool + wantErr string + wantValid bool + }{ + {name: "valid", wantValid: true}, + {name: "missing", omit: true, wantErr: "get model cache binding"}, + {name: "wrong UID", mutate: func(binding *nvcav2beta1.ModelCacheBinding) { + binding.UID = types.UID("other-binding") + }, wantErr: "does not match persisted UID"}, + {name: "retiring", mutate: func(binding *nvcav2beta1.ModelCacheBinding) { + binding.Status.Phase = nvcav2beta1.ModelCacheBindingPhaseRetiring + }, wantErr: "is not Active"}, + {name: "missing request reference", mutate: func(binding *nvcav2beta1.ModelCacheBinding) { + binding.Status.RequestReferences = nil + }, wantErr: "has no reference to ICMSRequest"}, + {name: "spec drift", mutate: func(binding *nvcav2beta1.ModelCacheBinding) { + binding.Spec.Decision.Provider = "other" + }, wantErr: "immutable spec does not match"}, + } { + t.Run(tt.name, func(t *testing.T) { + _, binding, st, icmsReq := newHelmBindingTestFixture(t) + if tt.mutate != nil { + tt.mutate(binding) + } + builder := fake.NewClientBuilder().WithScheme(mgrScheme) + if !tt.omit { + builder = builder.WithObjects(binding) + } + r := &Reconciler{Client: builder.Build(), metrics: newTestMetrics()} + stCopy := st.DeepCopy() + err := r.validatePersistedHelmCacheSelection(t.Context(), stCopy, icmsReq) + if tt.wantValid { + require.NoError(t, err) + assert.Equal(t, string(helmBindingTestBindingUID), + stCopy.Labels[ModelCacheBindingUIDLabelKey]) + return + } + require.ErrorContains(t, err, tt.wantErr) + assert.ErrorIs(t, err, reconcile.TerminalError(nil)) + assert.Empty(t, stCopy.Labels[ModelCacheBindingUIDLabelKey]) + }) + } +} + +func TestValidatePersistedHelmCacheSelectionRequiresMatchingICMSSelection(t *testing.T) { + for _, tt := range []struct { + name string + mutate func(*testing.T, *PersistedModelCacheStorageSelection, *nvcav2beta1.ICMSRequest) + }{ + { + name: "missing ICMS selection", + mutate: func(_ *testing.T, _ *PersistedModelCacheStorageSelection, request *nvcav2beta1.ICMSRequest) { + delete(request.Annotations, ModelCacheStorageSelectionAnnotationKey) + }, + }, + { + name: "different valid ICMS selection", + mutate: func(t *testing.T, selection *PersistedModelCacheStorageSelection, request *nvcav2beta1.ICMSRequest) { + t.Helper() + selection.CatalogDigest = "sha256:different" + raw, err := selection.Marshal() + require.NoError(t, err) + request.Annotations[ModelCacheStorageSelectionAnnotationKey] = raw + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + selection, binding, st, request := newHelmBindingTestFixture(t) + tt.mutate(t, selection, request) + r := &Reconciler{ + Client: fake.NewClientBuilder().WithScheme(mgrScheme).WithObjects(binding).Build(), + metrics: newTestMetrics(), + } + + err := r.validatePersistedHelmCacheSelection(t.Context(), st.DeepCopy(), request) + require.ErrorContains(t, err, "does not match the live ICMSRequest selection") + assert.True(t, isTerminal(err)) + }) + } +} + +func TestDoModelCacheRoutedClassifiesBindingReadErrors(t *testing.T) { + for _, tt := range []struct { + name string + readErr error + wantError bool + wantRequeue bool + wantTerminal bool + wantFinalPhase nvcav1new.StoragePhase + }{ + { + name: "temporary API outage requeues", + readErr: apierrors.NewServiceUnavailable("temporary binding read failure"), + wantRequeue: true, + wantFinalPhase: nvcav1new.StoragePending, + }, + { + name: "authorization failure surfaces without changing state", + readErr: apierrors.NewForbidden( + corev1.Resource("modelcachebindings"), helmBindingTestRequestName, errors.New("denied")), + wantError: true, + wantFinalPhase: nvcav1new.StoragePending, + }, + { + name: "missing persisted binding is terminal", + readErr: apierrors.NewNotFound( + corev1.Resource("modelcachebindings"), helmBindingTestRequestName), + wantError: true, + wantTerminal: true, + wantFinalPhase: nvcav1new.StorageFailed, + }, + } { + t.Run(tt.name, func(t *testing.T) { + _, binding, st, icmsReq := newHelmBindingTestFixture(t) + st.Status.Phase = nvcav1new.StoragePending + c := fake.NewClientBuilder().WithScheme(mgrScheme).WithObjects(binding). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, + obj client.Object, opts ...client.GetOption, + ) error { + if _, ok := obj.(*nvcav2beta1.ModelCacheBinding); ok { + return tt.readErr + } + return cl.Get(ctx, key, obj, opts...) + }, + }).Build() + r := &Reconciler{Client: c, metrics: newTestMetrics()} + stCopy := st.DeepCopy() + + res, err := r.doModelCacheRouted(t.Context(), *st, stCopy, icmsReq) + if tt.wantError { + require.Error(t, err) + } else { + require.NoError(t, err) + } + assert.Equal(t, tt.wantRequeue, res.Requeue) + assert.Equal(t, tt.wantTerminal, isTerminal(err)) + assert.Equal(t, tt.wantFinalPhase, stCopy.Status.Phase) + }) + } +} + +func TestModelCacheBindingUIDLabelHelpers(t *testing.T) { + obj := &corev1.PersistentVolumeClaim{} + require.ErrorContains(t, + ValidateModelCacheBindingUIDLabel(obj, helmBindingTestBindingUID), "ownership mismatch") + require.NoError(t, SetModelCacheBindingUIDLabel(obj, helmBindingTestBindingUID)) + require.NoError(t, ValidateModelCacheBindingUIDLabel(obj, helmBindingTestBindingUID)) + require.ErrorContains(t, + SetModelCacheBindingUIDLabel(obj, types.UID("other-binding")), "ownership mismatch") + assert.Equal(t, string(helmBindingTestBindingUID), obj.Labels[ModelCacheBindingUIDLabelKey]) + + require.ErrorContains(t, + ValidateModelCacheRequestUIDLabel(obj, helmBindingTestRequestUID), "ownership mismatch") + require.NoError(t, SetModelCacheRequestUIDLabel(obj, helmBindingTestRequestUID)) + require.NoError(t, ValidateModelCacheReaderOwnership( + obj, helmBindingTestBindingUID, helmBindingTestRequestUID)) + require.ErrorContains(t, + SetModelCacheRequestUIDLabel(obj, types.UID("replacement-request")), "ownership mismatch") + assert.Equal(t, string(helmBindingTestRequestUID), obj.Labels[ModelCacheRequestUIDLabelKey]) +} + +func TestCreateOrValidateModelCacheBindingOwnedObjectRejectsForeignObject(t *testing.T) { + key := client.ObjectKey{Name: "rw-pvc-" + helmBindingTestCacheHandle, Namespace: ModelCacheInitNamespace} + foreign := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{ + Name: key.Name, Namespace: key.Namespace, + Labels: map[string]string{ModelCacheBindingUIDLabelKey: "other-binding"}, + }} + wanted := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{ + Name: key.Name, Namespace: key.Namespace, + Labels: map[string]string{ModelCacheBindingUIDLabelKey: string(helmBindingTestBindingUID)}, + }} + c := fake.NewClientBuilder().WithScheme(mgrScheme).WithObjects(foreign).Build() + r := &Reconciler{Client: c} + + alreadyExists, err := r.createOrValidateModelCacheBindingOwnedObject(t.Context(), wanted) + assert.True(t, alreadyExists) + require.ErrorContains(t, err, "ownership mismatch") + got := &corev1.PersistentVolumeClaim{} + require.NoError(t, c.Get(t.Context(), key, got)) + assert.Equal(t, "other-binding", got.Labels[ModelCacheBindingUIDLabelKey]) +} + +func TestCreateOrValidateModelCacheBindingOwnedSecretRequiresExactIntent(t *testing.T) { + immutable := true + wanted := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "writer-job-" + helmBindingTestCacheHandle + "-0-pull-worker", + Namespace: ModelCacheInitNamespace, + Labels: map[string]string{ + modelCacheHandleLabelKey: helmBindingTestCacheHandle, + ModelCacheBindingUIDLabelKey: string(helmBindingTestBindingUID), + "stable.test/label": "expected", + }, + Annotations: map[string]string{"stable.test/annotation": "expected"}, + }, + Type: corev1.SecretTypeDockerConfigJson, + Immutable: &immutable, + StringData: map[string]string{corev1.DockerConfigJsonKey: `{"auths":{"registry.test":{}}}`}, + } + for _, tt := range []struct { + name string + mutate func(*corev1.Secret) + wantErr bool + }{ + {name: "exact server representation"}, + {name: "type drift", wantErr: true, mutate: func(secret *corev1.Secret) { + secret.Type = corev1.SecretTypeOpaque + }}, + {name: "data drift", wantErr: true, mutate: func(secret *corev1.Secret) { + secret.Data[corev1.DockerConfigJsonKey] = []byte(`{"auths":{"other.test":{}}}`) + }}, + {name: "immutable drift", wantErr: true, mutate: func(secret *corev1.Secret) { + value := false + secret.Immutable = &value + }}, + {name: "stable label drift", wantErr: true, mutate: func(secret *corev1.Secret) { + delete(secret.Labels, "stable.test/label") + }}, + } { + t.Run(tt.name, func(t *testing.T) { + existing := wanted.DeepCopy() + existing.Data = modelCacheSecretData(wanted) + existing.StringData = nil + if tt.mutate != nil { + tt.mutate(existing) + } + c := fake.NewClientBuilder().WithScheme(mgrScheme).WithObjects(existing).Build() + r := &Reconciler{Client: c} + + alreadyExists, err := r.createOrValidateModelCacheBindingOwnedObject( + t.Context(), wanted.DeepCopy()) + assert.True(t, alreadyExists) + if tt.wantErr { + require.ErrorIs(t, err, errModelCacheBindingOwnership) + return + } + require.NoError(t, err) + }) + } +} + +func TestHelmInitResourcesCarryBindingUID(t *testing.T) { + selection, _, st, _ := newHelmBindingTestFixture(t) + st.Labels = map[string]string{ModelCacheBindingUIDLabelKey: string(helmBindingTestBindingUID)} + rwPVC := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: "rw-pvc-" + helmBindingTestCacheHandle}} + initJob := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Name: "writer-job-" + helmBindingTestCacheHandle}, + Spec: batchv1.JobSpec{Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "writer-job-" + helmBindingTestCacheHandle}, + }}, + } + pullSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "worker-pull"}} + c := fake.NewClientBuilder().WithScheme(mgrScheme).Build() + r := &Reconciler{ + Client: c, + nowFunc: time.Now, + metrics: newTestMetrics(), + initStatuses: newInitStatusCache(c), + } + + res, err := r.doInitModelCacheNVMesh( + t.Context(), *st, st.DeepCopy(), rwPVC, initJob, []*corev1.Secret{pullSecret}, HelmCacheBackendNVMesh) + require.NoError(t, err) + assert.True(t, res.Requeue) + + wantUID := string(helmBindingTestBindingUID) + for _, obj := range []client.Object{ + &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{ + Name: "rw-pvc-" + helmBindingTestCacheHandle, Namespace: ModelCacheInitNamespace}}, + &batchv1.Job{ObjectMeta: metav1.ObjectMeta{ + Name: "writer-job-" + helmBindingTestCacheHandle, Namespace: ModelCacheInitNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{ + Name: "writer-job-" + helmBindingTestCacheHandle + "-0-pull-worker", Namespace: ModelCacheInitNamespace}}, + &coordv1.Lease{ObjectMeta: metav1.ObjectMeta{ + Name: buildInitLeaseName(helmBindingTestCacheHandle), Namespace: ModelCacheInitNamespace}}, + } { + require.NoError(t, c.Get(t.Context(), client.ObjectKeyFromObject(obj), obj)) + assert.Equal(t, wantUID, obj.GetLabels()[ModelCacheBindingUIDLabelKey], "%T", obj) + } + createdJob := &batchv1.Job{} + require.NoError(t, c.Get(t.Context(), client.ObjectKey{ + Name: "writer-job-" + helmBindingTestCacheHandle, Namespace: ModelCacheInitNamespace}, createdJob)) + assert.Equal(t, wantUID, createdJob.Spec.Template.Labels[ModelCacheBindingUIDLabelKey]) + + writerUID := types.UID("writer-pvc-uid") + primaryPV := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{Name: "primary-pv"}, + Spec: corev1.PersistentVolumeSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + StorageClassName: selection.StorageClassName, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + ClaimRef: &corev1.ObjectReference{ + APIVersion: "v1", + Kind: "PersistentVolumeClaim", + Namespace: ModelCacheInitNamespace, + Name: "rw-pvc-" + helmBindingTestCacheHandle, + UID: writerUID, + }, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + Driver: selection.Provisioner, + VolumeHandle: "cluster:csi:volume:" + ModelCacheInitNamespace, + }, + }, + }, + } + require.NoError(t, c.Create(t.Context(), primaryPV)) + boundWriterPVC := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rw-pvc-" + helmBindingTestCacheHandle, + Namespace: ModelCacheInitNamespace, + UID: writerUID, + Labels: map[string]string{ + ModelCacheBindingUIDLabelKey: string(helmBindingTestBindingUID), + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + StorageClassName: &selection.StorageClassName, + VolumeName: primaryPV.Name, + }, + } + require.NoError(t, r.finalizePrimaryPVOnSuccessfulInit(t.Context(), st, boundWriterPVC)) + require.NoError(t, c.Get(t.Context(), client.ObjectKeyFromObject(primaryPV), primaryPV)) + assert.Equal(t, wantUID, primaryPV.Labels[ModelCacheBindingUIDLabelKey]) +} + +func TestPrepareHelmModelCacheBindingResourcesRequiresExactIntent(t *testing.T) { + _, binding, _, _ := newHelmBindingTestFixture(t) + newResources := func(t *testing.T) (*corev1.PersistentVolumeClaim, *batchv1.Job, *coordv1.Lease) { + t.Helper() + rwPVC := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{ + Name: binding.Spec.Resources.PersistentVolumeClaimNames[0], Namespace: ModelCacheInitNamespace, + }} + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: binding.Spec.Resources.JobNames[0], Namespace: ModelCacheInitNamespace, + }, + Spec: batchv1.JobSpec{Template: corev1.PodTemplateSpec{}}, + } + lease := &coordv1.Lease{ObjectMeta: metav1.ObjectMeta{ + Name: binding.Spec.Resources.LeaseName, Namespace: ModelCacheInitNamespace, + }} + for _, obj := range []metav1.Object{rwPVC, job, &job.Spec.Template.ObjectMeta, lease} { + require.NoError(t, SetModelCacheBindingUIDLabel(obj, binding.UID)) + } + return rwPVC, job, lease + } + + for _, tt := range []struct { + name string + mutate func(*corev1.PersistentVolumeClaim, *batchv1.Job, *coordv1.Lease) + }{ + {name: "valid"}, + {name: "wrong writer PVC name", mutate: func(pvc *corev1.PersistentVolumeClaim, _ *batchv1.Job, _ *coordv1.Lease) { + pvc.Name = "other-writer" + }}, + {name: "wrong writer Job name", mutate: func(_ *corev1.PersistentVolumeClaim, job *batchv1.Job, _ *coordv1.Lease) { + job.Name = "other-job" + }}, + {name: "wrong Lease name", mutate: func(_ *corev1.PersistentVolumeClaim, _ *batchv1.Job, lease *coordv1.Lease) { + lease.Name = "other-lease" + }}, + {name: "wrong writer namespace", mutate: func(pvc *corev1.PersistentVolumeClaim, _ *batchv1.Job, _ *coordv1.Lease) { + pvc.Namespace = "other-namespace" + }}, + {name: "unowned Job Pod template", mutate: func(_ *corev1.PersistentVolumeClaim, job *batchv1.Job, _ *coordv1.Lease) { + delete(job.Spec.Template.Labels, ModelCacheBindingUIDLabelKey) + }}, + } { + t.Run(tt.name, func(t *testing.T) { + rwPVC, job, lease := newResources(t) + if tt.mutate != nil { + tt.mutate(rwPVC, job, lease) + } + r := &Reconciler{Client: fake.NewClientBuilder().WithScheme(mgrScheme).Build()} + err := r.prepareHelmModelCacheBindingResources(t.Context(), binding, rwPVC, job, lease) + if tt.mutate == nil { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, errModelCacheBindingOwnership) + }) + } + + t.Run("existing foreign writer is not adopted", func(t *testing.T) { + rwPVC, job, lease := newResources(t) + foreign := rwPVC.DeepCopy() + foreign.Labels[ModelCacheBindingUIDLabelKey] = "other-binding" + r := &Reconciler{Client: fake.NewClientBuilder().WithScheme(mgrScheme).WithObjects(foreign).Build()} + err := r.prepareHelmModelCacheBindingResources(t.Context(), binding, rwPVC, job, lease) + require.ErrorIs(t, err, errModelCacheBindingOwnership) + }) +} + +func TestFinalizePrimaryPVRequiresExactWriterClaim(t *testing.T) { + for _, tt := range []struct { + name string + mutate func(*corev1.PersistentVolumeClaim, *corev1.PersistentVolume) + }{ + {name: "valid"}, + {name: "missing claimRef", mutate: func(_ *corev1.PersistentVolumeClaim, pv *corev1.PersistentVolume) { + pv.Spec.ClaimRef = nil + }}, + {name: "wrong claim namespace", mutate: func(_ *corev1.PersistentVolumeClaim, pv *corev1.PersistentVolume) { + pv.Spec.ClaimRef.Namespace = "other-namespace" + }}, + {name: "wrong claim name", mutate: func(_ *corev1.PersistentVolumeClaim, pv *corev1.PersistentVolume) { + pv.Spec.ClaimRef.Name = "other-writer" + }}, + {name: "wrong claim UID", mutate: func(_ *corev1.PersistentVolumeClaim, pv *corev1.PersistentVolume) { + pv.Spec.ClaimRef.UID = "other-writer-uid" + }}, + {name: "wrong claim kind", mutate: func(_ *corev1.PersistentVolumeClaim, pv *corev1.PersistentVolume) { + pv.Spec.ClaimRef.Kind = "Secret" + }}, + {name: "empty volume handle", mutate: func(_ *corev1.PersistentVolumeClaim, pv *corev1.PersistentVolume) { + pv.Spec.CSI.VolumeHandle = "" + }}, + {name: "wrong volume handle namespace", mutate: func(_ *corev1.PersistentVolumeClaim, pv *corev1.PersistentVolume) { + pv.Spec.CSI.VolumeHandle = "cluster:csi:volume:other-namespace" + }}, + {name: "wrong reclaim policy", mutate: func(_ *corev1.PersistentVolumeClaim, pv *corev1.PersistentVolume) { + pv.Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete + }}, + } { + t.Run(tt.name, func(t *testing.T) { + selection, _, st, _ := newHelmBindingTestFixture(t) + st.Labels[ModelCacheBindingUIDLabelKey] = string(helmBindingTestBindingUID) + writerUID := types.UID("writer-pvc-uid") + writer := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "rw-pvc-" + helmBindingTestCacheHandle, Namespace: ModelCacheInitNamespace, + UID: writerUID, + Labels: map[string]string{ModelCacheBindingUIDLabelKey: string(helmBindingTestBindingUID)}, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + StorageClassName: &selection.StorageClassName, + VolumeName: "primary-pv", + }, + } + primary := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{Name: "primary-pv"}, + Spec: corev1.PersistentVolumeSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + StorageClassName: selection.StorageClassName, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + ClaimRef: &corev1.ObjectReference{ + APIVersion: "v1", Kind: "PersistentVolumeClaim", + Namespace: writer.Namespace, Name: writer.Name, UID: writer.UID, + }, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + Driver: selection.Provisioner, + VolumeHandle: "cluster:csi:volume:" + ModelCacheInitNamespace, + }, + }, + }, + } + if tt.mutate != nil { + tt.mutate(writer, primary) + } + c := fake.NewClientBuilder().WithScheme(mgrScheme).Build() + require.NoError(t, c.Create(t.Context(), primary)) + r := &Reconciler{Client: c, nowFunc: time.Now} + err := r.finalizePrimaryPVOnSuccessfulInit(t.Context(), st, writer) + got := &corev1.PersistentVolume{} + require.NoError(t, c.Get(t.Context(), client.ObjectKey{Name: primary.Name}, got)) + if tt.mutate == nil { + require.NoError(t, err) + assert.Equal(t, string(helmBindingTestBindingUID), got.Labels[ModelCacheBindingUIDLabelKey]) + return + } + require.Error(t, err) + assert.True(t, isTerminal(err)) + assert.Empty(t, got.Labels[ModelCacheBindingUIDLabelKey]) + assert.Empty(t, got.Annotations[primaryPVLastReferencedAnnotationKey]) + }) + } +} + +func TestHelmReaderValidatorsRejectStaleRequestGeneration(t *testing.T) { + selection, _, st, _ := newHelmBindingTestFixture(t) + className := selection.StorageClassName + primary := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{ + ModelCacheBindingUIDLabelKey: string(helmBindingTestBindingUID), + }}, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + StorageClassName: className, + PersistentVolumeSource: corev1.PersistentVolumeSource{CSI: &corev1.CSIPersistentVolumeSource{ + Driver: selection.Provisioner, VolumeHandle: "cluster:csi:volume:" + ModelCacheInitNamespace, + }}, + }, + } + rwPVC := &corev1.PersistentVolumeClaim{Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: &className}} + roPVCName := "ro-pvc-" + helmBindingTestCacheHandle + secondary := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secondary-pv-" + helmBindingTestRequestName, + Labels: map[string]string{ + ModelCacheBindingUIDLabelKey: string(helmBindingTestBindingUID), + ModelCacheRequestUIDLabelKey: string(helmBindingTestRequestUID), + }, + }, + Spec: corev1.PersistentVolumeSpec{ + AccessModes: accessModesRO, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + StorageClassName: className, + ClaimRef: &corev1.ObjectReference{ + APIVersion: "v1", Kind: "PersistentVolumeClaim", + Namespace: st.Namespace, Name: roPVCName, + }, + PersistentVolumeSource: corev1.PersistentVolumeSource{CSI: &corev1.CSIPersistentVolumeSource{ + Driver: selection.Provisioner, VolumeHandle: "cluster:csi:volume:" + st.Namespace, + }}, + }, + } + roPVC := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: roPVCName, Namespace: st.Namespace, UID: "reader-pvc-uid", + Labels: map[string]string{ + ModelCacheBindingUIDLabelKey: string(helmBindingTestBindingUID), + ModelCacheRequestUIDLabelKey: string(helmBindingTestRequestUID), + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: accessModesRO, StorageClassName: &className, VolumeName: secondary.Name, + }, + Status: corev1.PersistentVolumeClaimStatus{Phase: corev1.ClaimPending}, + } + + require.NoError(t, validateHelmModelCacheSecondaryPV( + st, selection, primary, secondary, roPVCName, helmBindingTestBindingUID, helmBindingTestRequestUID)) + require.NoError(t, validateHelmModelCacheReaderPVC( + st, rwPVC, secondary, roPVC, helmBindingTestBindingUID, helmBindingTestRequestUID)) + + stalePV := secondary.DeepCopy() + stalePV.Labels[ModelCacheRequestUIDLabelKey] = "stale-request-uid" + require.ErrorContains(t, validateHelmModelCacheSecondaryPV( + st, selection, primary, stalePV, roPVCName, helmBindingTestBindingUID, helmBindingTestRequestUID), + "request UID") + stalePVC := roPVC.DeepCopy() + stalePVC.Labels[ModelCacheRequestUIDLabelKey] = "stale-request-uid" + require.ErrorContains(t, validateHelmModelCacheReaderPVC( + st, rwPVC, secondary, stalePVC, helmBindingTestBindingUID, helmBindingTestRequestUID), + "request UID") + + secondaryWithStaleClaim := secondary.DeepCopy() + secondaryWithStaleClaim.Spec.ClaimRef.UID = "deleted-reader-pvc-uid" + require.ErrorContains(t, validateHelmModelCacheReaderPVC( + st, rwPVC, secondaryWithStaleClaim, roPVC, + helmBindingTestBindingUID, helmBindingTestRequestUID), "claimRef UID") + + deletePolicy := secondary.DeepCopy() + deletePolicy.Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete + require.ErrorContains(t, validateHelmModelCacheSecondaryPV( + st, selection, primary, deletePolicy, roPVCName, + helmBindingTestBindingUID, helmBindingTestRequestUID), "reclaim policy") + csiDrift := secondary.DeepCopy() + csiDrift.Spec.CSI.FSType = "other-fs" + require.ErrorContains(t, validateHelmModelCacheSecondaryPV( + st, selection, primary, csiDrift, roPVCName, + helmBindingTestBindingUID, helmBindingTestRequestUID), "CSI source") + + boundReader := roPVC.DeepCopy() + boundReader.Status.Phase = corev1.ClaimBound + require.ErrorContains(t, validateHelmModelCacheReaderPVC( + st, rwPVC, secondary, boundReader, + helmBindingTestBindingUID, helmBindingTestRequestUID), "claimRef UID") + boundSecondary := secondary.DeepCopy() + boundSecondary.Spec.ClaimRef.UID = boundReader.UID + require.NoError(t, validateHelmModelCacheReaderPVC( + st, rwPVC, boundSecondary, boundReader, + helmBindingTestBindingUID, helmBindingTestRequestUID)) + boundSecondary.Spec.ClaimRef.UID = "stale-reader-pvc-uid" + require.ErrorContains(t, validateHelmModelCacheReaderPVC( + st, rwPVC, boundSecondary, boundReader, + helmBindingTestBindingUID, helmBindingTestRequestUID), "claimRef UID") +} + +func TestHelmPrimaryPVReuseRequiresExactImmutableIdentity(t *testing.T) { + for _, tt := range []struct { + name string + mutate func(*corev1.PersistentVolume) + }{ + {name: "valid"}, + {name: "wrong binding", mutate: func(pv *corev1.PersistentVolume) { + pv.Labels[ModelCacheBindingUIDLabelKey] = "other-binding" + }}, + {name: "wrong provisioner", mutate: func(pv *corev1.PersistentVolume) { + pv.Spec.CSI.Driver = "other.csi.example.com" + }}, + {name: "wrong handle namespace", mutate: func(pv *corev1.PersistentVolume) { + pv.Spec.CSI.VolumeHandle = "cluster:csi:volume:other-namespace" + }}, + {name: "wrong StorageClass", mutate: func(pv *corev1.PersistentVolume) { + pv.Spec.StorageClassName = "other-class" + }}, + {name: "missing writer claim UID", mutate: func(pv *corev1.PersistentVolume) { + pv.Spec.ClaimRef.UID = "" + }}, + {name: "wrong reclaim policy", mutate: func(pv *corev1.PersistentVolume) { + pv.Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete + }}, + {name: "wrong access modes", mutate: func(pv *corev1.PersistentVolume) { + pv.Spec.AccessModes = accessModesRO + }}, + {name: "stale live writer UID", mutate: func(pv *corev1.PersistentVolume) { + pv.Spec.ClaimRef.UID = "stale-writer-pvc-uid" + }}, + } { + t.Run(tt.name, func(t *testing.T) { + selection, _, st, _ := newHelmBindingTestFixture(t) + rwPVC := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{ + Name: "rw-pvc-" + helmBindingTestCacheHandle, Namespace: ModelCacheInitNamespace, + UID: "writer-pvc-uid", + }} + primary := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{Name: "primary-pv", Labels: map[string]string{ + primaryPVLabelKey: primaryPVLabelValue, modelCacheHandleLabelKey: helmBindingTestCacheHandle, + ModelCacheBindingUIDLabelKey: string(helmBindingTestBindingUID), + }}, + Spec: corev1.PersistentVolumeSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + StorageClassName: selection.StorageClassName, + ClaimRef: &corev1.ObjectReference{ + APIVersion: "v1", Kind: "PersistentVolumeClaim", + Namespace: ModelCacheInitNamespace, Name: rwPVC.Name, UID: "writer-pvc-uid", + }, + PersistentVolumeSource: corev1.PersistentVolumeSource{CSI: &corev1.CSIPersistentVolumeSource{ + Driver: selection.Provisioner, VolumeHandle: "cluster:csi:volume:" + ModelCacheInitNamespace, + }}, + }, + } + if tt.mutate != nil { + tt.mutate(primary) + } + err := validateHelmModelCachePrimaryPVForReuse( + st, selection, rwPVC, primary, helmBindingTestBindingUID) + if tt.mutate == nil { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, errModelCacheBindingOwnership) + }) + } + + t.Run("valid after writer PVC deletion", func(t *testing.T) { + selection, _, st, _ := newHelmBindingTestFixture(t) + rwPVC := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{ + Name: "rw-pvc-" + helmBindingTestCacheHandle, Namespace: ModelCacheInitNamespace, + }} + primary := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{Name: "primary-pv", Labels: map[string]string{ + primaryPVLabelKey: primaryPVLabelValue, modelCacheHandleLabelKey: helmBindingTestCacheHandle, + ModelCacheBindingUIDLabelKey: string(helmBindingTestBindingUID), + }}, + Spec: corev1.PersistentVolumeSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + StorageClassName: selection.StorageClassName, + ClaimRef: &corev1.ObjectReference{ + APIVersion: "v1", Kind: "PersistentVolumeClaim", + Namespace: ModelCacheInitNamespace, Name: rwPVC.Name, UID: "historical-writer-pvc-uid", + }, + PersistentVolumeSource: corev1.PersistentVolumeSource{CSI: &corev1.CSIPersistentVolumeSource{ + Driver: selection.Provisioner, VolumeHandle: "cluster:csi:volume:" + ModelCacheInitNamespace, + }}, + }, + } + require.NoError(t, validateHelmModelCachePrimaryPVForReuse( + st, selection, rwPVC, primary, helmBindingTestBindingUID)) + }) +} + +func TestAnnotatedCleanupRefusesOwnershipMismatch(t *testing.T) { + _, binding, st, _ := newHelmBindingTestFixture(t) + wrongJob := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{ + Name: "writer-job-" + helmBindingTestCacheHandle, + Namespace: ModelCacheInitNamespace, + Labels: map[string]string{ + modelCacheHandleLabelKey: helmBindingTestCacheHandle, + ModelCacheBindingUIDLabelKey: "other-binding", + }, + }} + lease := bindingOwnedTestLease(helmBindingTestRequestName) + c := fake.NewClientBuilder().WithScheme(mgrScheme).WithObjects(binding, wrongJob, lease).Build() + r := &Reconciler{Client: c} + + errs := r.cleanupInitModelCache(t.Context(), st, false) + require.Len(t, errs, 1) + require.ErrorContains(t, errs[0], "ownership mismatch") + require.NoError(t, c.Get(t.Context(), client.ObjectKeyFromObject(wrongJob), &batchv1.Job{})) + require.NoError(t, c.Get(t.Context(), client.ObjectKeyFromObject(lease), &coordv1.Lease{})) +} + +func TestAnnotatedCleanupRefusesAmbiguousWriterResourcesBeforeDeletion(t *testing.T) { + for _, tt := range []struct { + name string + extra client.Object + wantErr string + includeJob bool + }{ + { + name: "multiple writer Jobs", + extra: &batchv1.Job{ObjectMeta: metav1.ObjectMeta{ + Name: "writer-job-duplicate", Namespace: ModelCacheInitNamespace, + UID: types.UID("duplicate-job-uid"), ResourceVersion: "1", + Labels: map[string]string{ + modelCacheHandleLabelKey: helmBindingTestCacheHandle, + ModelCacheBindingUIDLabelKey: string(helmBindingTestBindingUID), + }, + }}, + wantErr: "found 2 writer Jobs", + includeJob: true, + }, + { + name: "non-canonical writer PVC", + extra: &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{ + Name: "rw-pvc-other", Namespace: ModelCacheInitNamespace, + UID: types.UID("other-pvc-uid"), ResourceVersion: "1", + Labels: map[string]string{ + modelCacheHandleLabelKey: helmBindingTestCacheHandle, + ModelCacheBindingUIDLabelKey: string(helmBindingTestBindingUID), + }, + }}, + wantErr: "does not match cache handle", + }, + } { + t.Run(tt.name, func(t *testing.T) { + _, binding, st, _ := newHelmBindingTestFixture(t) + job := bindingOwnedTestJob() + lease := bindingOwnedTestLease(helmBindingTestRequestName) + objects := []client.Object{binding, lease, tt.extra} + if tt.includeJob { + objects = append(objects, job) + } + c := fake.NewClientBuilder().WithScheme(mgrScheme).WithObjects(objects...).Build() + r := &Reconciler{Client: c} + + errs := r.cleanupInitModelCache(t.Context(), st, false) + require.Len(t, errs, 1) + require.ErrorContains(t, errs[0], tt.wantErr) + for _, obj := range objects { + got, ok := obj.DeepCopyObject().(client.Object) + require.True(t, ok) + require.NoError(t, c.Get(t.Context(), client.ObjectKeyFromObject(obj), got)) + } + }) + } +} + +func TestAnnotatedCleanupDoesNotDeleteAnotherLeaseHoldersWriter(t *testing.T) { + _, binding, st, _ := newHelmBindingTestFixture(t) + job := bindingOwnedTestJob() + lease := bindingOwnedTestLease("request-b") + c := fake.NewClientBuilder().WithScheme(mgrScheme).WithObjects(binding, job, lease).Build() + r := &Reconciler{Client: c} + + require.Empty(t, r.cleanupInitModelCache(t.Context(), st, false)) + require.NoError(t, c.Get(t.Context(), client.ObjectKeyFromObject(job), &batchv1.Job{})) + require.NoError(t, c.Get(t.Context(), client.ObjectKeyFromObject(lease), &coordv1.Lease{})) +} + +func TestAnnotatedCleanupDoesNotAuthorizeRecreatedSameNameRequest(t *testing.T) { + _, binding, st, _ := newHelmBindingTestFixture(t) + job := bindingOwnedTestJob() + lease := bindingOwnedTestLease( + helmBindingTestRequestName + "@replacement-request-uid") + c := fake.NewClientBuilder().WithScheme(mgrScheme).WithObjects(binding, job, lease).Build() + r := &Reconciler{Client: c} + + require.Empty(t, r.cleanupInitModelCache(t.Context(), st, false)) + require.NoError(t, c.Get(t.Context(), client.ObjectKeyFromObject(job), &batchv1.Job{})) + require.NoError(t, c.Get(t.Context(), client.ObjectKeyFromObject(lease), &coordv1.Lease{})) +} + +func TestAnnotatedCleanupStopsAfterInconclusiveTargetList(t *testing.T) { + _, binding, st, _ := newHelmBindingTestFixture(t) + job := bindingOwnedTestJob() + lease := bindingOwnedTestLease(helmBindingTestRequestName) + jobLists := 0 + deletes := 0 + c := fake.NewClientBuilder().WithScheme(mgrScheme).WithObjects(binding, job, lease). + WithInterceptorFuncs(interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, + opts ...client.ListOption, + ) error { + if _, ok := list.(*batchv1.JobList); ok { + jobLists++ + if jobLists == 2 { + return apierrors.NewServiceUnavailable("inconclusive writer Job list") + } + } + return cl.List(ctx, list, opts...) + }, + Delete: func(ctx context.Context, cl client.WithWatch, obj client.Object, + opts ...client.DeleteOption, + ) error { + deletes++ + return cl.Delete(ctx, obj, opts...) + }, + }).Build() + r := &Reconciler{Client: c} + + errs := r.cleanupInitModelCache(t.Context(), st, false) + require.Len(t, errs, 1) + assert.True(t, apierrors.IsServiceUnavailable(errs[0])) + assert.Equal(t, 0, deletes) + require.NoError(t, c.Get(t.Context(), client.ObjectKeyFromObject(job), &batchv1.Job{})) +} + +func TestAnnotatedCleanupRevalidatesLeaseBeforeEveryDelete(t *testing.T) { + _, binding, st, _ := newHelmBindingTestFixture(t) + job := bindingOwnedTestJob() + lease := bindingOwnedTestLease(helmBindingTestRequestName) + deletes := 0 + c := fake.NewClientBuilder().WithScheme(mgrScheme).WithObjects(binding, job, lease). + WithInterceptorFuncs(interceptor.Funcs{ + Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, + patch client.Patch, opts ...client.PatchOption, + ) error { + if err := cl.Patch(ctx, obj, patch, opts...); err != nil { + return err + } + if _, ok := obj.(*coordv1.Lease); !ok { + return nil + } + current := &coordv1.Lease{} + if err := cl.Get(ctx, client.ObjectKeyFromObject(obj), current); err != nil { + return err + } + replacement := "other-request@other-request-uid" + current.Spec.HolderIdentity = &replacement + return cl.Update(ctx, current) + }, + Delete: func(ctx context.Context, cl client.WithWatch, obj client.Object, + opts ...client.DeleteOption, + ) error { + deletes++ + return cl.Delete(ctx, obj, opts...) + }, + }).Build() + r := &Reconciler{Client: c} + + errs := r.cleanupInitModelCache(t.Context(), st, false) + require.Len(t, errs, 1) + require.ErrorIs(t, errs[0], errModelCacheBindingOwnership) + assert.Equal(t, 0, deletes) + require.NoError(t, c.Get(t.Context(), client.ObjectKeyFromObject(job), &batchv1.Job{})) +} + +func TestHandleLeaseUsesAcquireTimeWhenRenewTimeIsNil(t *testing.T) { + _, _, st, _ := newHelmBindingTestFixture(t) + now := time.Now().UTC() + holder := "other-request@other-request-uid" + lease := bindingOwnedTestLease(holder) + lease.Spec.RenewTime = nil + lease.Spec.AcquireTime = &metav1.MicroTime{Time: now.Add(-time.Minute)} + otherRequest := &nvcav2beta1.ICMSRequest{ObjectMeta: metav1.ObjectMeta{ + Name: "other-request", Namespace: helmBindingTestRequestNS, UID: "other-request-uid", + }} + c := fake.NewClientBuilder().WithScheme(mgrScheme).WithObjects(lease, otherRequest).Build() + r := &Reconciler{ + Client: c, ICMSRequestNamespace: helmBindingTestRequestNS, + nowFunc: func() time.Time { return now }, + } + wanted := newInitLease(st) + require.NoError(t, propagateModelCacheBindingUIDLabel(st, wanted)) + + res, holds, err := r.handleLease(t.Context(), wanted) + require.NoError(t, err) + assert.False(t, holds) + assert.Greater(t, res.RequeueAfter, 58*time.Minute) + assert.LessOrEqual(t, res.RequeueAfter, 59*time.Minute) +} + +func TestHandleLeaseClassifiesHolderLookupErrors(t *testing.T) { + for _, tt := range []struct { + name string + readErr error + wantRequeue bool + wantError bool + }{ + { + name: "transient", + readErr: apierrors.NewServiceUnavailable( + "temporary Lease holder lookup failure"), + wantRequeue: true, + }, + { + name: "forbidden", + readErr: apierrors.NewForbidden( + corev1.Resource("icmsrequests"), "other-request", errors.New("denied")), + wantError: true, + }, + } { + t.Run(tt.name, func(t *testing.T) { + _, _, st, _ := newHelmBindingTestFixture(t) + lease := bindingOwnedTestLease("other-request@other-request-uid") + acquire := metav1.NowMicro() + lease.Spec.AcquireTime = &acquire + c := fake.NewClientBuilder().WithScheme(mgrScheme).WithObjects(lease). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, + obj client.Object, opts ...client.GetOption, + ) error { + if _, ok := obj.(*nvcav2beta1.ICMSRequest); ok { + return tt.readErr + } + return cl.Get(ctx, key, obj, opts...) + }, + }).Build() + r := &Reconciler{ + Client: c, ICMSRequestNamespace: helmBindingTestRequestNS, nowFunc: time.Now, + } + wanted := newInitLease(st) + require.NoError(t, propagateModelCacheBindingUIDLabel(st, wanted)) + + res, holds, err := r.handleLease(t.Context(), wanted) + assert.False(t, holds) + assert.Equal(t, tt.wantRequeue, res.Requeue) + if tt.wantError { + require.Error(t, err) + assert.True(t, apierrors.IsForbidden(err)) + return + } + require.NoError(t, err) + }) + } +} + +func TestHandleLeaseClassifiesLeaseAPIErrors(t *testing.T) { + for _, tt := range []struct { + name string + getErr error + createErr error + wantRequeue bool + wantError bool + }{ + { + name: "transient Lease read", + getErr: apierrors.NewServiceUnavailable( + "temporary Lease read failure"), + wantRequeue: true, + }, + { + name: "forbidden Lease read", + getErr: apierrors.NewForbidden( + corev1.Resource("leases"), "lease", errors.New("denied")), + wantError: true, + }, + { + name: "concurrent Lease create", + createErr: apierrors.NewAlreadyExists( + corev1.Resource("leases"), "lease"), + wantRequeue: true, + }, + } { + t.Run(tt.name, func(t *testing.T) { + _, _, st, _ := newHelmBindingTestFixture(t) + c := fake.NewClientBuilder().WithScheme(mgrScheme). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, + obj client.Object, opts ...client.GetOption, + ) error { + if _, ok := obj.(*coordv1.Lease); ok && tt.getErr != nil { + return tt.getErr + } + return cl.Get(ctx, key, obj, opts...) + }, + Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, + opts ...client.CreateOption, + ) error { + if _, ok := obj.(*coordv1.Lease); ok && tt.createErr != nil { + return tt.createErr + } + return cl.Create(ctx, obj, opts...) + }, + }).Build() + r := &Reconciler{Client: c, nowFunc: time.Now} + wanted := newInitLease(st) + require.NoError(t, propagateModelCacheBindingUIDLabel(st, wanted)) + + res, holds, err := r.handleLease(t.Context(), wanted) + assert.False(t, holds) + assert.Equal(t, tt.wantRequeue, res.Requeue) + if tt.wantError { + require.Error(t, err) + assert.True(t, apierrors.IsForbidden(err)) + return + } + require.NoError(t, err) + }) + } +} + +func TestModelCacheSuccessfulInitCleanupResult(t *testing.T) { + for _, tt := range []struct { + name string + errs []error + want reconcile.Result + wantErr bool + wantForbidden bool + }{ + {name: "success"}, + { + name: "volume detach pending", errs: []error{errVolumeStillAttached}, + want: reconcile.Result{RequeueAfter: volumeDetachRequeueInterval}, + }, + { + name: "transient API failure", errs: []error{ + fmt.Errorf("wrapped cleanup read: %w", apierrors.NewServiceUnavailable("temporary")), + }, want: reconcile.Result{Requeue: true}, + }, + { + name: "non-transient API failure", errs: []error{ + apierrors.NewForbidden(corev1.Resource("jobs"), "writer", errors.New("denied")), + }, wantErr: true, wantForbidden: true, + }, + } { + t.Run(tt.name, func(t *testing.T) { + res, err := modelCacheSuccessfulInitCleanupResult(tt.errs) + assert.Equal(t, tt.want, res) + if tt.wantErr { + require.Error(t, err) + if tt.wantForbidden { + assert.True(t, apierrors.IsForbidden(err)) + } + return + } + require.NoError(t, err) + }) + } +} + +func TestAnnotatedCleanupDeletesExactBindingOwnersWriter(t *testing.T) { + _, binding, st, _ := newHelmBindingTestFixture(t) + job := bindingOwnedTestJob() + lease := bindingOwnedTestLease(helmBindingTestRequestName) + c := fake.NewClientBuilder().WithScheme(mgrScheme).WithObjects(binding, job, lease).Build() + r := &Reconciler{Client: c} + + require.Empty(t, r.cleanupInitModelCache(t.Context(), st, false)) + assert.True(t, apierrors.IsNotFound(c.Get( + t.Context(), client.ObjectKeyFromObject(job), &batchv1.Job{}))) + assert.True(t, apierrors.IsNotFound(c.Get( + t.Context(), client.ObjectKeyFromObject(lease), &coordv1.Lease{}))) +} + +func TestAnnotatedSharedWriterCleanupUsesExactDeletePreconditions(t *testing.T) { + _, binding, st, _ := newHelmBindingTestFixture(t) + job := bindingOwnedTestJob() + secretName := job.Name + "-0-pull-worker" + job.Spec.Template.Spec.ImagePullSecrets = []corev1.LocalObjectReference{{Name: secretName}} + lease := bindingOwnedTestLease(helmBindingTestRequestName) + writerPVC := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{ + Name: "rw-pvc-" + helmBindingTestCacheHandle, Namespace: ModelCacheInitNamespace, + UID: types.UID("writer-pvc-uid"), ResourceVersion: "1", + Labels: map[string]string{ + modelCacheHandleLabelKey: helmBindingTestCacheHandle, + ModelCacheBindingUIDLabelKey: string(helmBindingTestBindingUID), + }, + }} + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: job.Name + "-pod", Namespace: ModelCacheInitNamespace, + UID: types.UID("writer-pod-uid"), ResourceVersion: "1", + Labels: map[string]string{ + modelCacheHandleLabelKey: helmBindingTestCacheHandle, + ModelCacheBindingUIDLabelKey: string(helmBindingTestBindingUID), + }, + }} + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{ + Name: secretName, Namespace: ModelCacheInitNamespace, + UID: types.UID("pull-secret-uid"), ResourceVersion: "1", + Labels: map[string]string{ + modelCacheHandleLabelKey: helmBindingTestCacheHandle, + ModelCacheBindingUIDLabelKey: string(helmBindingTestBindingUID), + }, + }} + objects := []client.Object{binding, job, lease, writerPVC, pod, secret} + validated := map[string]bool{} + c := fake.NewClientBuilder().WithScheme(mgrScheme).WithObjects(objects...). + WithInterceptorFuncs(interceptor.Funcs{ + Delete: func(ctx context.Context, cl client.WithWatch, obj client.Object, + opts ...client.DeleteOption, + ) error { + deleteOptions := &client.DeleteOptions{} + for _, opt := range opts { + opt.ApplyToDelete(deleteOptions) + } + require.NotNil(t, deleteOptions.Preconditions, obj.GetName()) + require.NotNil(t, deleteOptions.Preconditions.UID, obj.GetName()) + require.NotNil(t, deleteOptions.Preconditions.ResourceVersion, obj.GetName()) + assert.Equal(t, obj.GetUID(), *deleteOptions.Preconditions.UID) + assert.Equal(t, obj.GetResourceVersion(), *deleteOptions.Preconditions.ResourceVersion) + validated[obj.GetName()] = true + return cl.Delete(ctx, obj, opts...) + }, + }).Build() + r := &Reconciler{Client: c} + + require.Empty(t, r.cleanupInitModelCache(t.Context(), st, false)) + for _, obj := range []client.Object{job, lease, writerPVC, pod, secret} { + assert.True(t, validated[obj.GetName()], obj.GetName()) + } +} + +func TestValidateModelCacheBindingForCleanupUsesDeletedRequestTombstone(t *testing.T) { + for _, tt := range []struct { + name string + request *nvcav2beta1.ICMSRequest + wantErr string + wantReference bool + transientGetError error + }{ + {name: "request is gone"}, + { + name: "request is deleting", + request: &nvcav2beta1.ICMSRequest{ObjectMeta: metav1.ObjectMeta{ + Name: helmBindingTestRequestName, + Namespace: helmBindingTestRequestNS, + UID: helmBindingTestRequestUID, + DeletionTimestamp: &metav1.Time{Time: time.Now()}, + Finalizers: []string{"test.nvcf.nvidia.io/finalizer"}, + }}, + }, + { + name: "live request without reference", + request: &nvcav2beta1.ICMSRequest{ObjectMeta: metav1.ObjectMeta{ + Name: helmBindingTestRequestName, Namespace: helmBindingTestRequestNS, UID: helmBindingTestRequestUID, + }}, + wantErr: "live ICMSRequest", + }, + { + name: "request name was reused", + request: &nvcav2beta1.ICMSRequest{ObjectMeta: metav1.ObjectMeta{ + Name: helmBindingTestRequestName, Namespace: helmBindingTestRequestNS, UID: types.UID("replacement-uid"), + }}, + wantErr: "does not match recorded UID", + }, + { + name: "transient request read", + transientGetError: apierrors.NewServiceUnavailable("temporary ICMSRequest read failure"), + wantErr: "temporary ICMSRequest read failure", + }, + } { + t.Run(tt.name, func(t *testing.T) { + _, binding, st, _ := newHelmBindingTestFixture(t) + binding.Status.RequestReferences = nil + objects := []client.Object{binding} + if tt.request != nil { + objects = append(objects, tt.request) + } + builder := fake.NewClientBuilder().WithScheme(mgrScheme).WithObjects(objects...) + if tt.transientGetError != nil { + builder = builder.WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, + obj client.Object, opts ...client.GetOption, + ) error { + if _, ok := obj.(*nvcav2beta1.ICMSRequest); ok { + return tt.transientGetError + } + return cl.Get(ctx, key, obj, opts...) + }, + }) + } + r := &Reconciler{Client: builder.Build()} + + uid, annotated, referencePresent, err := + r.validateModelCacheBindingForCleanup(t.Context(), st) + assert.True(t, annotated) + assert.Equal(t, tt.wantReference, referencePresent) + if tt.wantErr != "" { + require.ErrorContains(t, err, tt.wantErr) + assert.Empty(t, uid) + return + } + require.NoError(t, err) + assert.Equal(t, helmBindingTestBindingUID, uid) + }) + } +} + +func bindingOwnedTestJob() *batchv1.Job { + return &batchv1.Job{ObjectMeta: metav1.ObjectMeta{ + Name: "writer-job-" + helmBindingTestCacheHandle, + Namespace: ModelCacheInitNamespace, + UID: types.UID("writer-job-uid"), + ResourceVersion: "1", + Labels: map[string]string{ + modelCacheHandleLabelKey: helmBindingTestCacheHandle, + ModelCacheBindingUIDLabelKey: string(helmBindingTestBindingUID), + }, + }} +} + +func bindingOwnedTestLease(holder string) *coordv1.Lease { + if holder == helmBindingTestRequestName { + holder += "@" + string(helmBindingTestRequestUID) + } + duration := int32(3600) + return &coordv1.Lease{ + ObjectMeta: metav1.ObjectMeta{ + Name: buildInitLeaseName(helmBindingTestCacheHandle), + Namespace: ModelCacheInitNamespace, + UID: types.UID("writer-lease-uid"), + ResourceVersion: "1", + Labels: map[string]string{ + modelCacheHandleLabelKey: helmBindingTestCacheHandle, + ModelCacheBindingUIDLabelKey: string(helmBindingTestBindingUID), + }, + }, + Spec: coordv1.LeaseSpec{HolderIdentity: &holder, LeaseDurationSeconds: &duration}, + } +} diff --git a/src/compute-plane-services/nvca/pkg/storage/modelcache_binding_test.go b/src/compute-plane-services/nvca/pkg/storage/modelcache_binding_test.go new file mode 100644 index 000000000..16b00a567 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/storage/modelcache_binding_test.go @@ -0,0 +1,227 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package storage + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + + nvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1" +) + +func durableBindingSelection(t *testing.T) *PersistedModelCacheStorageSelection { + t.Helper() + selection, err := NewPersistedModelCacheStorageSelection( + ModelCacheWorkflowHelm, + ModelCacheSelectionDurable, + testResolvedModelCacheStorage(ModelCacheTransitionROXReadOnly), + ) + require.NoError(t, err) + return selection +} + +func durableRWXReadOnlyBindingSelection(t *testing.T) *PersistedModelCacheStorageSelection { + t.Helper() + selection, err := NewPersistedModelCacheStorageSelection( + ModelCacheWorkflowRegular, + ModelCacheSelectionDurable, + testResolvedModelCacheStorage(ModelCacheTransitionRWXReadOnly), + ) + require.NoError(t, err) + return selection +} + +func TestNewModelCacheBinding(t *testing.T) { + selection := durableBindingSelection(t) + binding, err := NewModelCacheBinding(selection, "nca-a", "cache-handle", ModelCacheInitNamespace) + require.NoError(t, err) + + assert.Equal(t, ModelCacheBindingName("cache-handle"), binding.Name) + assert.Equal(t, ModelCacheInitNamespace, binding.Namespace) + assert.Equal(t, []string{nvcav2beta1.ModelCacheBindingFinalizer}, binding.Finalizers) + assert.Equal(t, nvcav2beta1.ModelCacheWorkflowHelm, binding.Spec.Identity.Workflow) + assert.Equal(t, digestBindingValue("nca-a"), binding.Spec.Identity.SharingDomainDigest) + assert.Equal(t, digestBindingValue("cache-handle"), binding.Spec.Identity.CacheHandleDigest) + assert.Equal(t, NVMeshStorageClassProvisioner, binding.Spec.Decision.Provisioner) + assert.Equal(t, + []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce, corev1.ReadOnlyMany}, + binding.Spec.Decision.RequiredAccessModes) + assert.Equal(t, []string{"ro", "norecovery", "nouuid"}, + binding.Spec.Decision.RequiredMountOptions) + assert.Equal(t, corev1.PersistentVolumeReclaimRetain, binding.Spec.StorageClass.ReclaimPolicy) + assert.Equal(t, []string{"rw-pvc-cache-handle"}, binding.Spec.Resources.PersistentVolumeClaimNames) + assert.Equal(t, []string{"writer-job-cache-handle"}, binding.Spec.Resources.JobNames) + assert.Equal(t, "modelcache-init-cache-handle", binding.Spec.Resources.LeaseName) + assert.Empty(t, binding.Spec.Resources.StorageClassNames) + assert.Empty(t, binding.Spec.Resources.SecretNames) +} + +func TestNewRWXReadOnlyModelCacheBindingRecordsOneClaim(t *testing.T) { + selection := durableRWXReadOnlyBindingSelection(t) + binding, err := NewModelCacheBinding(selection, "nca-a", "cache-handle", "pod-instances") + require.NoError(t, err) + + assert.Equal(t, ModelCacheTransitionRWXReadOnly, binding.Spec.Decision.Transition) + assert.Equal(t, + []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + binding.Spec.Decision.RequiredAccessModes) + assert.Empty(t, binding.Spec.Decision.RequiredMountOptions) + assert.False(t, binding.Spec.Decision.EncryptionRequired) + assert.Equal(t, + []string{"rw-pvc-cache-handle"}, + binding.Spec.Resources.PersistentVolumeClaimNames) + assert.Equal(t, []string{"writer-job-cache-handle"}, binding.Spec.Resources.JobNames) + assert.Empty(t, binding.Spec.Resources.LeaseName) + assert.Empty(t, binding.Spec.Resources.StorageClassNames) + assert.Empty(t, binding.Spec.Resources.SecretNames) +} + +func TestNewEncryptedModelCacheBindingRecordsNamesOnly(t *testing.T) { + selection := durableBindingSelection(t) + selection.EncryptionRequired = true + binding, err := NewModelCacheBinding(selection, "nca-a", "cache-handle", ModelCacheInitNamespace) + require.NoError(t, err) + + assert.True(t, binding.Spec.Decision.EncryptionRequired) + assert.Equal(t, []string{buildStorageClassName("nca-a")}, binding.Spec.Resources.StorageClassNames) + assert.Equal(t, []string{buildStorageClassSecretName("nca-a")}, binding.Spec.Resources.SecretNames) +} + +func TestNewEncryptedRegularModelCacheBindingRecordsLegacyNames(t *testing.T) { + selection := durableBindingSelection(t) + selection.Workflow = ModelCacheWorkflowRegular + selection.EncryptionRequired = true + binding, err := NewModelCacheBinding(selection, "nca-a", "cache-handle", "pod-instances") + require.NoError(t, err) + + domainHash := hashNCAID("nca-a") + assert.Equal(t, + []string{"rw-pvc-cache-handle", "ro-pvc-cache-handle"}, + binding.Spec.Resources.PersistentVolumeClaimNames) + assert.Empty(t, binding.Spec.Resources.LeaseName) + assert.Equal(t, []string{domainHash + "-sc"}, binding.Spec.Resources.StorageClassNames) + assert.Equal(t, []string{domainHash}, binding.Spec.Resources.SecretNames) +} + +func TestModelCacheBindingHandleCollisionFailsIntentValidation(t *testing.T) { + selection := durableBindingSelection(t) + binding, err := NewModelCacheBinding(selection, "nca-a", "same-handle", ModelCacheInitNamespace) + require.NoError(t, err) + binding.Status.Phase = nvcav2beta1.ModelCacheBindingPhaseActive + + err = ValidateModelCacheBinding(binding, selection, "nca-b", "same-handle", ModelCacheInitNamespace) + require.ErrorContains(t, err, "immutable spec does not match") + assert.Equal(t, ModelCacheBindingName("same-handle"), binding.Name, + "the handle-scoped name forces another sharing domain to collide and fail closed") +} + +func TestValidateModelCacheBinding(t *testing.T) { + selection := durableBindingSelection(t) + binding, err := NewModelCacheBinding(selection, "nca-a", "cache-handle", ModelCacheInitNamespace) + require.NoError(t, err) + binding.UID = types.UID("binding-uid") + binding.Status.Phase = nvcav2beta1.ModelCacheBindingPhaseActive + + require.NoError(t, ValidateModelCacheBinding( + binding, selection, "nca-a", "cache-handle", ModelCacheInitNamespace)) + + t.Run("request reference", func(t *testing.T) { + selected := *selection + selected.BindingName = binding.Name + selected.BindingUID = binding.UID + require.NoError(t, ValidateModelCacheBinding( + binding, &selected, "nca-a", "cache-handle", ModelCacheInitNamespace)) + + selected.BindingUID = types.UID("other") + err := ValidateModelCacheBinding( + binding, &selected, "nca-a", "cache-handle", ModelCacheInitNamespace) + require.ErrorContains(t, err, "binding reference changed") + }) + + t.Run("retiring", func(t *testing.T) { + changed := binding.DeepCopy() + changed.Status.Phase = nvcav2beta1.ModelCacheBindingPhaseRetiring + err := ValidateModelCacheBinding( + changed, selection, "nca-a", "cache-handle", ModelCacheInitNamespace) + require.ErrorContains(t, err, "is not Active") + }) + + t.Run("missing finalizer", func(t *testing.T) { + changed := binding.DeepCopy() + changed.Finalizers = nil + err := ValidateModelCacheBinding( + changed, selection, "nca-a", "cache-handle", ModelCacheInitNamespace) + require.ErrorContains(t, err, "has no protection finalizer") + }) + + t.Run("spec drift", func(t *testing.T) { + changed := binding.DeepCopy() + changed.Spec.Decision.Provider = "other" + err := ValidateModelCacheBinding( + changed, selection, "nca-a", "cache-handle", ModelCacheInitNamespace) + require.ErrorContains(t, err, "immutable spec does not match") + }) + + t.Run("required mount option drift", func(t *testing.T) { + changed := binding.DeepCopy() + changed.Spec.Decision.RequiredMountOptions = append( + changed.Spec.Decision.RequiredMountOptions, "noatime") + err := ValidateModelCacheBinding( + changed, selection, "nca-a", "cache-handle", ModelCacheInitNamespace) + require.ErrorContains(t, err, "immutable spec does not match") + }) +} + +func TestModelCacheBindingHasRequestReference(t *testing.T) { + binding := &nvcav2beta1.ModelCacheBinding{ + Status: nvcav2beta1.ModelCacheBindingStatus{ + RequestReferences: []nvcav2beta1.ModelCacheBindingRequestReference{ + {Namespace: "requests", Name: "request", UID: types.UID("uid")}, + }, + }, + } + assert.True(t, ModelCacheBindingHasRequestReference( + binding, "requests", "request", types.UID("uid"))) + assert.False(t, ModelCacheBindingHasRequestReference( + binding, "requests", "request", types.UID("other"))) +} + +func TestNewModelCacheBindingRejectsInvalidInput(t *testing.T) { + selection := durableBindingSelection(t) + for _, tt := range []struct { + name string + domain string + handle string + namespace string + want string + }{ + {name: "domain", handle: "cache", namespace: "writers", want: "sharing domain is empty"}, + {name: "handle", domain: "nca", namespace: "writers", want: "cache handle is empty"}, + {name: "namespace", domain: "nca", handle: "cache", namespace: "INVALID", want: "writer namespace"}, + {name: "resource name", domain: "nca", handle: "INVALID", namespace: "writers", want: "name"}, + } { + t.Run(tt.name, func(t *testing.T) { + _, err := NewModelCacheBinding(selection, tt.domain, tt.handle, tt.namespace) + require.ErrorContains(t, err, tt.want) + }) + } +} diff --git a/src/compute-plane-services/nvca/pkg/storage/modelcache_cleanup.go b/src/compute-plane-services/nvca/pkg/storage/modelcache_cleanup.go index ce7b5a1eb..43fd02b5e 100644 --- a/src/compute-plane-services/nvca/pkg/storage/modelcache_cleanup.go +++ b/src/compute-plane-services/nvca/pkg/storage/modelcache_cleanup.go @@ -21,6 +21,7 @@ import ( "context" "errors" "fmt" + "slices" "time" batchv1 "k8s.io/api/batch/v1" @@ -32,11 +33,13 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" + apitypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/util/k8sutil" nvcav1new "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v1" ) @@ -66,6 +69,105 @@ func init() { } } +func (r *Reconciler) deleteModelCacheCleanupObject( + ctx context.Context, + obj client.Object, + annotated bool, + opts ...client.DeleteOption, +) error { + if annotated { + if obj == nil || obj.GetUID() == "" || obj.GetResourceVersion() == "" { + return fmt.Errorf("refusing model cache cleanup without exact UID and resourceVersion for %T", obj) + } + uid := obj.GetUID() + resourceVersion := obj.GetResourceVersion() + opts = append(opts, client.Preconditions(metav1.Preconditions{ + UID: &uid, + ResourceVersion: &resourceVersion, + })) + } + return r.Client.Delete(ctx, obj, opts...) +} + +type modelCacheInitCleanupLeaseGuard struct { + lease *coordv1.Lease + bindingUID apitypes.UID + holder string +} + +func (r *Reconciler) lockModelCacheInitCleanupLease( + ctx context.Context, + lease *coordv1.Lease, + bindingUID apitypes.UID, + holder string, +) (*modelCacheInitCleanupLeaseGuard, error) { + if lease == nil || lease.UID == "" || lease.ResourceVersion == "" { + return nil, fmt.Errorf("refusing model cache init cleanup without an exact ownership Lease identity") + } + if err := ValidateModelCacheBindingUIDLabel(lease, bindingUID); err != nil { + return nil, err + } + if lease.Spec.HolderIdentity == nil || *lease.Spec.HolderIdentity != holder { + return nil, fmt.Errorf("%w: model cache cleanup Lease holder changed from %q", + errModelCacheBindingOwnership, holder) + } + old := lease.DeepCopy() + now := metav1.NowMicro() + lease.Spec.RenewTime = &now + if lease.Spec.AcquireTime == nil { + lease.Spec.AcquireTime = &now + } + if err := r.Client.Patch(ctx, lease, + client.MergeFromWithOptions(old, client.MergeFromWithOptimisticLock{})); err != nil { + return nil, fmt.Errorf("lock model cache init cleanup Lease %s/%s: %w", + lease.Namespace, lease.Name, err) + } + return &modelCacheInitCleanupLeaseGuard{ + lease: lease.DeepCopy(), bindingUID: bindingUID, holder: holder, + }, nil +} + +func (r *Reconciler) validateModelCacheInitCleanupLeaseGuard( + ctx context.Context, + guard *modelCacheInitCleanupLeaseGuard, +) error { + if guard == nil || guard.lease == nil { + return fmt.Errorf("refusing annotated model cache init cleanup without a Lease guard") + } + current := &coordv1.Lease{} + if err := r.Client.Get(ctx, client.ObjectKeyFromObject(guard.lease), current); err != nil { + return fmt.Errorf("revalidate model cache init cleanup Lease %s/%s: %w", + guard.lease.Namespace, guard.lease.Name, err) + } + if current.UID != guard.lease.UID || current.ResourceVersion != guard.lease.ResourceVersion { + return fmt.Errorf("%w: model cache init cleanup Lease changed after authorization", + errModelCacheBindingOwnership) + } + if err := ValidateModelCacheBindingUIDLabel(current, guard.bindingUID); err != nil { + return err + } + if current.Spec.HolderIdentity == nil || *current.Spec.HolderIdentity != guard.holder { + return fmt.Errorf("%w: model cache init cleanup Lease holder changed from %q", + errModelCacheBindingOwnership, guard.holder) + } + return nil +} + +func (r *Reconciler) deleteModelCacheInitCleanupObject( + ctx context.Context, + obj client.Object, + annotated bool, + guard *modelCacheInitCleanupLeaseGuard, + opts ...client.DeleteOption, +) error { + if annotated { + if err := r.validateModelCacheInitCleanupLeaseGuard(ctx, guard); err != nil { + return err + } + } + return r.deleteModelCacheCleanupObject(ctx, obj, annotated, opts...) +} + func (r *Reconciler) doCleanupModelCacheNVMesh(ctx context.Context, st *nvcav1new.StorageRequest) (reconcile.Result, error) { //nolint log := logf.FromContext(ctx) @@ -80,33 +182,66 @@ func (r *Reconciler) doCleanupModelCacheNVMesh(ctx context.Context, st *nvcav1ne }) } - var errs []error - errs = append(errs, r.cleanupInitModelCache(ctx, st, false)...) - + bindingUID, annotated, _, err := r.validateModelCacheBindingForCleanup(ctx, st) + if err != nil { + return modelCacheCleanupErrorResult(err) + } cwResourceLabels := getClusterWideResourceLabels(st) pvList := &corev1.PersistentVolumeList{} if err := r.Client.List(ctx, pvList, client.MatchingLabels(cwResourceLabels)); err != nil { log.Error(err, "Failed to list PVs for storage request") - return reconcile.Result{}, errors.Join(append(errs, err)...) + return modelCacheCleanupErrorResult(fmt.Errorf("list reader PVs for model cache cleanup: %w", err)) } - pvcList := &corev1.PersistentVolumeClaimList{} if err := r.Client.List(ctx, pvcList, client.MatchingLabels(cwResourceLabels), client.InNamespace(st.Namespace), ); err != nil { log.Error(err, "Failed to list PVCs for storage request") - return reconcile.Result{}, errors.Join(append(errs, err)...) + return modelCacheCleanupErrorResult(fmt.Errorf("list reader PVCs for model cache cleanup: %w", err)) + } + if annotated { + if err := validateAnnotatedModelCacheReaderCleanup(st, bindingUID, &pvList.Items, &pvcList.Items); err != nil { + return reconcile.Result{}, err + } + for i := range pvList.Items { + if err := ValidateModelCacheReaderOwnership( + &pvList.Items[i], bindingUID, apitypes.UID(st.Annotations[ICMSRequestUIDAnnotationKey])); err != nil { + return reconcile.Result{}, fmt.Errorf( + "refusing per-request model cache cleanup for PV %q: %w", pvList.Items[i].Name, err) + } + } + for i := range pvcList.Items { + if err := ValidateModelCacheReaderOwnership( + &pvcList.Items[i], bindingUID, apitypes.UID(st.Annotations[ICMSRequestUIDAnnotationKey])); err != nil { + return reconcile.Result{}, fmt.Errorf( + "refusing per-request model cache cleanup for PVC %s/%s: %w", + pvcList.Items[i].Namespace, pvcList.Items[i].Name, err) + } + } + } + + initCleanupErrs := r.cleanupInitModelCache(ctx, st, false) + if len(initCleanupErrs) != 0 { + meta.SetStatusCondition(&st.Status.Conditions, metav1.Condition{ + Type: ConditionTypeCleanupSuccessful, + Status: metav1.ConditionFalse, + Reason: ConditionReasonSomeObjectsPendingDeletion, + Message: fmt.Sprintf("errors encountered while cleaning up init objects: %+q", initCleanupErrs), + }) + return modelCacheSuccessfulInitCleanupResult(initCleanupErrs) } // PVC's can be deleted before pods are, and will be finalized once the pod is deleted. for _, pvc := range pvcList.Items { if pvc.DeletionTimestamp != nil { log.V(1).Info("PVC has already been deleted", "pvc", pvc.Name) - } else if err := r.Client.Delete(ctx, &pvc); err != nil && !apierrors.IsNotFound(err) { + } else if err := r.deleteModelCacheCleanupObject(ctx, &pvc, annotated); err != nil && + !apierrors.IsNotFound(err) { log.Error(err, "Failed to delete PVC, manual cleanup needed", "pvc", pvc.Name) - errs = append(errs, err) + return modelCacheCleanupErrorResult(fmt.Errorf( + "delete reader PVC %s/%s: %w", pvc.Namespace, pvc.Name, err)) } } @@ -115,53 +250,107 @@ func (r *Reconciler) doCleanupModelCacheNVMesh(ctx context.Context, st *nvcav1ne for _, pv := range pvList.Items { if pv.DeletionTimestamp != nil { log.V(1).Info("PV has already been deleted", "pv", pv.Name) - } else if err := r.Client.Delete(ctx, &pv); err != nil && !apierrors.IsNotFound(err) { + } else if err := r.deleteModelCacheCleanupObject(ctx, &pv, annotated); err != nil && + !apierrors.IsNotFound(err) { log.Error(err, "Failed to delete PV, manual cleanup needed", "pv", pv.Name) - errs = append(errs, err) + return modelCacheCleanupErrorResult(fmt.Errorf("delete reader PV %q: %w", pv.Name, err)) } } - // A still-attaching writer volume is not a failure: requeue and retry without - // blocking the single reconcile worker (the previous in-reconcile poll could - // starve every other StorageRequest, including the terminating-namespace - // finalizer escape hatch). Separate that sentinel from real errors. - var realErrs []error - pendingDetach := false - for _, e := range errs { - if errors.Is(e, errVolumeStillAttached) { - pendingDetach = true - continue - } - realErrs = append(realErrs, e) - } + meta.SetStatusCondition(&st.Status.Conditions, metav1.Condition{ + Type: ConditionTypeCleanupSuccessful, + Status: metav1.ConditionTrue, + Reason: ConditionReasonAllObjectsDeleted, + Message: "All init and secondary model cache objects were cleaned up", + }) + return reconcile.Result{}, nil +} - if len(realErrs) == 0 && pendingDetach { - meta.SetStatusCondition(&st.Status.Conditions, metav1.Condition{ - Type: ConditionTypeCleanupSuccessful, - Status: metav1.ConditionFalse, - Reason: ConditionReasonSomeObjectsPendingDeletion, - Message: "waiting for cache init volume to detach before deleting the writer PVC", - }) - return reconcile.Result{RequeueAfter: volumeDetachRequeueInterval}, nil +func modelCacheCleanupErrorResult(err error) (reconcile.Result, error) { + if err == nil { + return reconcile.Result{}, nil } - - if len(realErrs) == 0 { - meta.SetStatusCondition(&st.Status.Conditions, metav1.Condition{ - Type: ConditionTypeCleanupSuccessful, - Status: metav1.ConditionTrue, - Reason: ConditionReasonAllObjectsDeleted, - Message: "All init and secondary model cache objects were cleaned up", - }) - } else { - meta.SetStatusCondition(&st.Status.Conditions, metav1.Condition{ - Type: ConditionTypeCleanupSuccessful, - Status: metav1.ConditionFalse, - Reason: ConditionReasonSomeObjectsPendingDeletion, - Message: fmt.Sprintf("errors encountered while cleaning up: %+q", realErrs), - }) + if k8sutil.IsTransientK8sError(err) { + return reconcile.Result{Requeue: true}, nil } + return reconcile.Result{}, err +} - return reconcile.Result{}, errors.Join(realErrs...) +func validateAnnotatedModelCacheReaderCleanup( + st *nvcav1new.StorageRequest, + bindingUID apitypes.UID, + pvs *[]corev1.PersistentVolume, + pvcs *[]corev1.PersistentVolumeClaim, +) error { + if st == nil || st.Spec.ModelCache == nil || pvs == nil || pvcs == nil { + return fmt.Errorf("refusing annotated model cache cleanup with incomplete reader identity") + } + selection, err := ParsePersistedModelCacheStorageSelection( + st.Annotations[ModelCacheStorageSelectionAnnotationKey]) + if err != nil { + return fmt.Errorf("parse persisted model cache selection for reader cleanup: %w", err) + } + requestUID := apitypes.UID(st.Annotations[ICMSRequestUIDAnnotationKey]) + expectedPVName := "secondary-pv-" + st.Spec.ICMSRequestName + expectedPVCName := "ro-pvc-" + st.Spec.ModelCache.CacheHandle + if len(*pvs) > 1 || len(*pvcs) > 1 { + return fmt.Errorf("refusing annotated model cache cleanup with %d reader PVs and %d reader PVCs", + len(*pvs), len(*pvcs)) + } + for i := range *pvs { + pv := &(*pvs)[i] + if pv.Name != expectedPVName || pv.Namespace != "" { + return fmt.Errorf("refusing model cache cleanup for unexpected reader PV %q/%q", + pv.Namespace, pv.Name) + } + if err := ValidateModelCacheReaderOwnership(pv, bindingUID, requestUID); err != nil { + return err + } + if pv.Spec.CSI == nil || pv.Spec.CSI.Driver != selection.Provisioner { + return fmt.Errorf("refusing reader PV %q with provisioner other than %q", + pv.Name, selection.Provisioner) + } + if pv.Spec.PersistentVolumeReclaimPolicy != corev1.PersistentVolumeReclaimRetain { + return fmt.Errorf("refusing reader PV %q with reclaim policy %q; want Retain", + pv.Name, pv.Spec.PersistentVolumeReclaimPolicy) + } + if !slices.Equal(pv.Spec.AccessModes, accessModesRO) { + return fmt.Errorf("refusing reader PV %q with access modes %v", pv.Name, pv.Spec.AccessModes) + } + claimRef := pv.Spec.ClaimRef + if claimRef == nil || claimRef.APIVersion != "v1" || claimRef.Kind != "PersistentVolumeClaim" || + claimRef.Namespace != st.Namespace || claimRef.Name != expectedPVCName { + return fmt.Errorf("refusing reader PV %q with unexpected claimRef", pv.Name) + } + } + for i := range *pvcs { + pvc := &(*pvcs)[i] + if pvc.Name != expectedPVCName || pvc.Namespace != st.Namespace { + return fmt.Errorf("refusing model cache cleanup for unexpected reader PVC %s/%s", + pvc.Namespace, pvc.Name) + } + if err := ValidateModelCacheReaderOwnership(pvc, bindingUID, requestUID); err != nil { + return err + } + if pvc.Spec.VolumeName != expectedPVName || !slices.Equal(pvc.Spec.AccessModes, accessModesRO) { + return fmt.Errorf("refusing reader PVC %s/%s with unexpected volume or access modes", + pvc.Namespace, pvc.Name) + } + } + if len(*pvs) == 1 && len(*pvcs) == 1 { + pv := &(*pvs)[0] + pvc := &(*pvcs)[0] + if pvc.Status.Phase == corev1.ClaimBound && + (pvc.UID == "" || pv.Spec.ClaimRef.UID != pvc.UID) { + return fmt.Errorf("refusing bound reader PVC %s/%s UID %q with PV claimRef UID %q", + pvc.Namespace, pvc.Name, pvc.UID, pv.Spec.ClaimRef.UID) + } + if pv.Spec.ClaimRef.UID != "" && pv.Spec.ClaimRef.UID != pvc.UID { + return fmt.Errorf("refusing reader PV %q claimRef UID %q for PVC UID %q", + pv.Name, pv.Spec.ClaimRef.UID, pvc.UID) + } + } + return nil } // cleanupInitModelCache deletes the init objects (job, pods, lease, pull @@ -173,143 +362,216 @@ func (r *Reconciler) cleanupInitModelCache(ctx context.Context, st *nvcav1new.St log := logf.FromContext(ctx) log.V(1).Info("Cleaning up model cache init objects", "retainWriterPVC", retainWriterPVC) + if st == nil || st.Spec.ModelCache == nil || st.Spec.ModelCache.CacheHandle == "" { + return []error{fmt.Errorf("cannot clean model cache init objects without a cache handle")} + } + bindingUID, annotated, requestReferencePresent, err := r.validateModelCacheBindingForCleanup(ctx, st) + if err != nil { + return []error{err} + } + var cleanupLeaseGuard *modelCacheInitCleanupLeaseGuard + if annotated { + if !requestReferencePresent { + // Per-request reader cleanup remains authorized by the persisted + // tombstone, but shared writer cleanup requires the live binding + // reference and is skipped after that reference is released. + log.V(1).Info("Skipping shared model cache init cleanup after request reference release") + return nil + } + holderIdentity := modelCacheLeaseHolderIdentity(st) + authorized, ownershipLease, err := r.validateModelCacheInitCleanupOwnership( + ctx, st.Spec.ModelCache.CacheHandle, bindingUID, holderIdentity) + if err != nil { + return []error{err} + } + if !authorized { + log.V(1).Info("Skipping shared model cache init cleanup owned by another request") + return nil + } + cleanupLeaseGuard, err = r.lockModelCacheInitCleanupLease( + ctx, ownershipLease, bindingUID, holderIdentity) + if err != nil { + return []error{err} + } + } + matchLabels := map[string]string{ + modelCacheHandleLabelKey: st.Spec.ModelCache.CacheHandle, + } + if annotated { + matchLabels[ModelCacheBindingUIDLabelKey] = string(bindingUID) + } listOpts := []client.ListOption{ - client.MatchingLabels(map[string]string{ - modelCacheHandleLabelKey: st.Spec.ModelCache.CacheHandle, - }), + client.MatchingLabels(matchLabels), client.InNamespace(ModelCacheInitNamespace), } - // Delete Job and its Pods first. + // Discover every target before the first delete. An inconclusive API read is + // never authority to continue destructive cleanup. jobList := &batchv1.JobList{} if err := r.Client.List(ctx, jobList, listOpts...); err != nil { - log.Error(err, "Init job list failed, manual cleanup needed") - errs = append(errs, err) - } - switch l := len(jobList.Items); l { - case 0: - case 1: - job := jobList.Items[0] - if job.DeletionTimestamp == nil { - log.V(1).Info("Deleting model cache init job", "job", job.Name) - if err := r.Client.Delete(ctx, &job, - client.PropagationPolicy(metav1.DeletePropagationForeground), - ); err != nil && !apierrors.IsNotFound(err) { - log.Error(err, "Init job delete failed, manual cleanup needed") - errs = append(errs, err) + return append(errs, fmt.Errorf("list model cache init Jobs: %w", err)) + } + podList := &corev1.PodList{} + if err := r.Client.List(ctx, podList, listOpts...); err != nil { + return append(errs, fmt.Errorf("list model cache init Pods: %w", err)) + } + pvcList := &corev1.PersistentVolumeClaimList{} + if !retainWriterPVC { + if err := r.Client.List(ctx, pvcList, listOpts...); err != nil { + return append(errs, fmt.Errorf("list model cache init PVCs: %w", err)) + } + } + secretList := &corev1.SecretList{} + if annotated { + if err := r.Client.List(ctx, secretList, listOpts...); err != nil { + return append(errs, fmt.Errorf("list model cache init pull Secrets: %w", err)) + } + } else { + seen := map[string]struct{}{} + for _, job := range jobList.Items { + for _, ref := range job.Spec.Template.Spec.ImagePullSecrets { + if _, ok := seen[ref.Name]; ok { + continue + } + seen[ref.Name] = struct{}{} + secret := corev1.Secret{} + err := r.Client.Get(ctx, client.ObjectKey{Name: ref.Name, Namespace: job.Namespace}, &secret) + if apierrors.IsNotFound(err) { + continue + } + if err != nil { + return append(errs, fmt.Errorf("get model cache init pull Secret %s/%s: %w", + job.Namespace, ref.Name, err)) + } + secretList.Items = append(secretList.Items, secret) } } - default: - // This should never happen, but log it in case there's a bug. - log.Error(fmt.Errorf("unexpected number of init jobs"), - "Found more than one init job to delete", "found", l) } - // Attempt to explicitly delete pods since they may have a grace period. - podList := &corev1.PodList{} - if err := r.Client.List(ctx, podList, listOpts...); err != nil { - log.Error(err, "Init job pod list failed, manual cleanup needed") - errs = append(errs, err) - } - gracePeriod := 0 - // There could be some failed pods. - for _, pod := range podList.Items { - log.V(1).Info("Deleting model cache init job pod", "pod", pod.Name) - if err := r.Client.Delete(ctx, &pod, - client.GracePeriodSeconds(gracePeriod), - ); err != nil && !apierrors.IsNotFound(err) { - log.Error(err, "Init job pod delete failed, manual cleanup needed") - errs = append(errs, err) + if annotated { + if len(jobList.Items) > 1 || len(pvcList.Items) > 1 { + return append(errs, fmt.Errorf("refusing annotated cleanup with %d writer Jobs and %d writer PVCs", + len(jobList.Items), len(pvcList.Items))) + } + for _, obj := range append( + []client.Object{}, modelCacheCleanupObjects(jobList, podList, pvcList, secretList)..., + ) { + if err := ValidateModelCacheBindingUIDLabel(obj, bindingUID); err != nil { + return append(errs, err) + } } } - if !retainWriterPVC { - // Delete the RW PVC once all Pods are deleted. - writerVolumeSettled := true - pvcList := &corev1.PersistentVolumeClaimList{} - if err := r.Client.List(ctx, pvcList, listOpts...); err != nil { - log.Error(err, "RW PVC list failed, manual cleanup needed") - errs = append(errs, err) + deleteTarget := func(obj client.Object, opts ...client.DeleteOption) error { + err := r.deleteModelCacheInitCleanupObject(ctx, obj, annotated, cleanupLeaseGuard, opts...) + if err == nil || apierrors.IsNotFound(err) { + return nil + } + return err + } + if len(jobList.Items) == 1 && jobList.Items[0].DeletionTimestamp == nil { + job := &jobList.Items[0] + if err := deleteTarget(job, client.PropagationPolicy(metav1.DeletePropagationForeground)); err != nil { + return append(errs, fmt.Errorf("delete model cache init Job %s/%s: %w", + job.Namespace, job.Name, err)) + } + } + gracePeriod := int64(0) + for i := range podList.Items { + pod := &podList.Items[i] + if err := deleteTarget(pod, client.GracePeriodSeconds(gracePeriod)); err != nil { + return append(errs, fmt.Errorf("delete model cache init Pod %s/%s: %w", + pod.Namespace, pod.Name, err)) } - switch l := len(pvcList.Items); l { - case 0: - case 1: - pvc := pvcList.Items[0] - if pvc.DeletionTimestamp == nil { - // Delete the RW PVC only once its volume has detached, but NEVER block - // the single reconcile worker polling for it: a single-shot check that - // returns a requeue sentinel when still attached (see isVolumeDetached). - deletePVC := true - if pvc.Spec.VolumeName != "" { - detached, err := r.isVolumeDetached(ctx, pvc.Spec.VolumeName) - switch { - case err != nil: - log.Error(err, "Failed to check PV detachment", "pv", pvc.Spec.VolumeName) - errs = append(errs, err) - deletePVC = false - case !detached: - log.V(1).Info("Init RW PVC volume still attached, will requeue", "pv", pvc.Spec.VolumeName) - errs = append(errs, errVolumeStillAttached) - deletePVC = false - } + } + + writerVolumeSettled := true + if !retainWriterPVC && len(pvcList.Items) == 1 { + pvc := &pvcList.Items[0] + if pvc.DeletionTimestamp == nil { + if pvc.Spec.VolumeName != "" { + detached, err := r.isVolumeDetached(ctx, pvc.Spec.VolumeName) + if err != nil { + return append(errs, fmt.Errorf("check model cache writer PV %q detachment: %w", + pvc.Spec.VolumeName, err)) } - if !deletePVC { - writerVolumeSettled = false - } else { - log.V(1).Info("Deleting model cache init RW PVC", "pvc", pvc.Name) - if err := r.Client.Delete(ctx, &pvc); err != nil && !apierrors.IsNotFound(err) { - log.Error(err, "Init RW PVC delete failed, manual cleanup needed", "pvc", pvc.Name) - errs = append(errs, err) - } + if !detached { + return append(errs, errVolumeStillAttached) } } - default: - // This should never happen, but log it in case there's a bug. - log.Error(fmt.Errorf("unexpected number of init RW PVCs"), - "Found more than one PVC to delete", "found", l) - } - - // The Samba writer binds a static plumbing SMB PV; delete it with the writer - // PVC. Static PVs have no provisioner, so a Released one leaks forever, and - // its stale claimRef would block a later writer PVC from re-binding. - // NotFound is the normal case for the other backends. While the writer - // volume is still attached the PVC is retained above, so retain the PV - // with it; the requeued cleanup deletes both after detachment. - if writerVolumeSettled { - sambaWriterPV := &corev1.PersistentVolume{} - sambaWriterPV.Name = sambaModelCacheWriterPVName(st.Spec.ModelCache.CacheHandle) - if err := r.Client.Delete(ctx, sambaWriterPV); err != nil && !apierrors.IsNotFound(err) { - log.Error(err, "Samba writer PV delete failed, manual cleanup needed", "pv", sambaWriterPV.Name) - errs = append(errs, err) + if err := deleteTarget(pvc); err != nil { + return append(errs, fmt.Errorf("delete model cache init PVC %s/%s: %w", + pvc.Namespace, pvc.Name, err)) } } } + if !retainWriterPVC && len(pvcList.Items) > 1 { + writerVolumeSettled = false + } + + if writerVolumeSettled && !retainWriterPVC && !annotated { + sambaWriterPV := &corev1.PersistentVolume{} + sambaWriterPV.Name = sambaModelCacheWriterPVName(st.Spec.ModelCache.CacheHandle) + if err := r.Client.Delete(ctx, sambaWriterPV); err != nil && !apierrors.IsNotFound(err) { + return append(errs, fmt.Errorf("delete Samba model cache writer PV %q: %w", + sambaWriterPV.Name, err)) + } + } + for i := range secretList.Items { + secret := &secretList.Items[i] + if err := deleteTarget(secret); err != nil { + return append(errs, fmt.Errorf("delete model cache init pull Secret %s/%s: %w", + secret.Namespace, secret.Name, err)) + } + } - // Finally delete the rest. Do not fail on these. lease := &coordv1.Lease{} - lease.Name = buildInitLeaseName(st.Spec.ModelCache.CacheHandle) - lease.Namespace = ModelCacheInitNamespace - if err := r.Client.Delete(ctx, lease); err != nil && !apierrors.IsNotFound(err) { - log.Error(err, "Lease deletion failed, manual cleanup needed") - errs = append(errs, err) - } - for _, job := range jobList.Items { - for _, pullSecretName := range job.Spec.Template.Spec.ImagePullSecrets { - secret := &corev1.Secret{} - secret.Name = pullSecretName.Name - secret.Namespace = job.Namespace - if err := r.Client.Delete(ctx, secret); err != nil && !apierrors.IsNotFound(err) { - log.Error(err, "Image pull secret deletion failed, manual cleanup needed", - "secret_name", pullSecretName) - errs = append(errs, err) + if annotated { + lease = cleanupLeaseGuard.lease.DeepCopy() + } else { + leaseKey := client.ObjectKey{ + Name: buildInitLeaseName(st.Spec.ModelCache.CacheHandle), Namespace: ModelCacheInitNamespace, + } + if err := r.Client.Get(ctx, leaseKey, lease); err != nil { + if apierrors.IsNotFound(err) { + return errs } + return append(errs, fmt.Errorf("get model cache init Lease %s/%s: %w", + leaseKey.Namespace, leaseKey.Name, err)) } } + if err := deleteTarget(lease); err != nil { + return append(errs, fmt.Errorf("delete model cache init Lease %s/%s: %w", + lease.Namespace, lease.Name, err)) + } return errs } +func modelCacheCleanupObjects( + jobs *batchv1.JobList, + pods *corev1.PodList, + pvcs *corev1.PersistentVolumeClaimList, + secrets *corev1.SecretList, +) []client.Object { + objects := make([]client.Object, 0, + len(jobs.Items)+len(pods.Items)+len(pvcs.Items)+len(secrets.Items)) + for i := range jobs.Items { + objects = append(objects, &jobs.Items[i]) + } + for i := range pods.Items { + objects = append(objects, &pods.Items[i]) + } + for i := range pvcs.Items { + objects = append(objects, &pvcs.Items[i]) + } + for i := range secrets.Items { + objects = append(objects, &secrets.Items[i]) + } + return objects +} + func (r *Reconciler) cleanupIdleModelCaches(ctx context.Context) error { //nolint log := logf.FromContext(ctx) @@ -345,6 +607,13 @@ func (r *Reconciler) cleanupIdleModelCaches(ctx context.Context) error { //nolin foundCacheHandles := sets.New[string]() storageClassesToDelete := sets.New[string]() for _, pv := range pvs.Items { + if pv.Labels != nil && pv.Labels[ModelCacheBindingUIDLabelKey] != "" { + cacheHandle := pv.Labels[modelCacheHandleLabelKey] + if _, ok := r.initStatuses.get(cacheHandle); cacheHandle != "" && ok { + foundCacheHandles.Insert(cacheHandle) + } + continue + } if pv.Annotations == nil { continue } @@ -408,10 +677,11 @@ func (r *Reconciler) cleanupIdleModelCaches(ctx context.Context) error { //nolin } r.initStatuses.Unlock() - // Storage classes are shared between PV's, and should be removed from the set to delete - // if at least one PV is bound that references a storage class. + // Storage classes are shared between PVs. A binding-owned PV is outside this + // legacy GC's authority, so it protects its class regardless of PV phase. for _, pv := range pvs.Items { - if pv.Status.Phase == corev1.VolumeBound { + if pv.Status.Phase == corev1.VolumeBound || + pv.Labels[ModelCacheBindingUIDLabelKey] != "" { storageClassesToDelete = storageClassesToDelete.Delete(pv.Spec.StorageClassName) } } diff --git a/src/compute-plane-services/nvca/pkg/storage/modelcache_cleanup_test.go b/src/compute-plane-services/nvca/pkg/storage/modelcache_cleanup_test.go index ac384cab5..bf1de884f 100644 --- a/src/compute-plane-services/nvca/pkg/storage/modelcache_cleanup_test.go +++ b/src/compute-plane-services/nvca/pkg/storage/modelcache_cleanup_test.go @@ -19,18 +19,27 @@ package storage import ( "context" + stderrors "errors" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + batchv1 "k8s.io/api/batch/v1" + coordv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + k8stypes "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/util/k8sutil" nvcav1new "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v1" "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/types" ) @@ -202,3 +211,336 @@ func TestCleanupModelCaches(t *testing.T) { require.NoError(t, err) assert.Equal(t, pvCopy.Name, "test-pv") } + +func TestAnnotatedPerRequestCleanupRefusesMixedBindingOwnership(t *testing.T) { + for _, tt := range []struct { + name string + pvBinding string + pvcBinding string + mutate func(*corev1.PersistentVolume, *corev1.PersistentVolumeClaim) + }{ + { + name: "missing PV binding label", + pvcBinding: string(helmBindingTestBindingUID), + }, + { + name: "foreign PVC binding label", + pvBinding: string(helmBindingTestBindingUID), + pvcBinding: "other-binding", + }, + { + name: "missing PV request UID label", + pvBinding: string(helmBindingTestBindingUID), + pvcBinding: string(helmBindingTestBindingUID), + mutate: func(pv *corev1.PersistentVolume, _ *corev1.PersistentVolumeClaim) { + delete(pv.Labels, ModelCacheRequestUIDLabelKey) + }, + }, + { + name: "foreign PVC request UID label", + pvBinding: string(helmBindingTestBindingUID), + pvcBinding: string(helmBindingTestBindingUID), + mutate: func(_ *corev1.PersistentVolume, pvc *corev1.PersistentVolumeClaim) { + pvc.Labels[ModelCacheRequestUIDLabelKey] = "replacement-request-uid" + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + _, binding, st, _ := newHelmBindingTestFixture(t) + pv, pvc := perRequestBindingOwnedVolumes(st, tt.pvBinding, tt.pvcBinding) + if tt.mutate != nil { + tt.mutate(pv, pvc) + } + job := bindingOwnedTestJob() + lease := bindingOwnedTestLease(helmBindingTestRequestName) + c := fake.NewClientBuilder().WithScheme(mgrScheme). + WithObjects(binding, pv, pvc, job, lease).Build() + r := &Reconciler{Client: c} + + res, err := r.doCleanupModelCacheNVMesh(t.Context(), st) + assert.Equal(t, reconcile.Result{}, res) + require.ErrorContains(t, err, "ownership mismatch") + require.NoError(t, c.Get(t.Context(), client.ObjectKeyFromObject(pv), &corev1.PersistentVolume{})) + require.NoError(t, c.Get( + t.Context(), client.ObjectKeyFromObject(pvc), &corev1.PersistentVolumeClaim{})) + require.NoError(t, c.Get(t.Context(), client.ObjectKeyFromObject(job), &batchv1.Job{})) + require.NoError(t, c.Get(t.Context(), client.ObjectKeyFromObject(lease), &coordv1.Lease{})) + condition := meta.FindStatusCondition(st.Status.Conditions, ConditionTypeCleanupSuccessful) + require.NotNil(t, condition) + assert.Equal(t, metav1.ConditionFalse, condition.Status) + }) + } +} + +func TestAnnotatedPerRequestCleanupDeletesExactBindingOwnership(t *testing.T) { + _, binding, st, _ := newHelmBindingTestFixture(t) + pv, pvc := perRequestBindingOwnedVolumes( + st, string(helmBindingTestBindingUID), string(helmBindingTestBindingUID)) + c := fake.NewClientBuilder().WithScheme(mgrScheme).WithObjects(binding, pv, pvc).Build() + r := &Reconciler{Client: c} + + res, err := r.doCleanupModelCacheNVMesh(t.Context(), st) + require.NoError(t, err) + assert.Equal(t, reconcile.Result{}, res) + assert.True(t, errors.IsNotFound(c.Get( + t.Context(), client.ObjectKeyFromObject(pv), &corev1.PersistentVolume{}))) + assert.True(t, errors.IsNotFound(c.Get( + t.Context(), client.ObjectKeyFromObject(pvc), &corev1.PersistentVolumeClaim{}))) + condition := meta.FindStatusCondition(st.Status.Conditions, ConditionTypeCleanupSuccessful) + require.NotNil(t, condition) + assert.Equal(t, metav1.ConditionTrue, condition.Status) +} + +func TestAnnotatedCleanupTombstoneDeletesReadersButPreservesSharedWriter(t *testing.T) { + _, binding, st, _ := newHelmBindingTestFixture(t) + binding.Status.RequestReferences = nil + pv, pvc := perRequestBindingOwnedVolumes( + st, string(helmBindingTestBindingUID), string(helmBindingTestBindingUID)) + job := bindingOwnedTestJob() + lease := bindingOwnedTestLease(helmBindingTestRequestName) + c := fake.NewClientBuilder().WithScheme(mgrScheme). + WithObjects(binding, pv, pvc, job, lease).Build() + r := &Reconciler{Client: c} + + res, err := r.doCleanupModelCacheNVMesh(t.Context(), st) + require.NoError(t, err) + assert.Equal(t, reconcile.Result{}, res) + assert.True(t, errors.IsNotFound(c.Get( + t.Context(), client.ObjectKeyFromObject(pv), &corev1.PersistentVolume{}))) + assert.True(t, errors.IsNotFound(c.Get( + t.Context(), client.ObjectKeyFromObject(pvc), &corev1.PersistentVolumeClaim{}))) + require.NoError(t, c.Get(t.Context(), client.ObjectKeyFromObject(job), &batchv1.Job{})) + require.NoError(t, c.Get(t.Context(), client.ObjectKeyFromObject(lease), &coordv1.Lease{})) + condition := meta.FindStatusCondition(st.Status.Conditions, ConditionTypeCleanupSuccessful) + require.NotNil(t, condition) + assert.Equal(t, metav1.ConditionTrue, condition.Status) +} + +func TestAnnotatedReaderCleanupUsesExactDeletePreconditions(t *testing.T) { + _, binding, st, _ := newHelmBindingTestFixture(t) + pv, pvc := perRequestBindingOwnedVolumes( + st, string(helmBindingTestBindingUID), string(helmBindingTestBindingUID)) + validated := map[string]bool{} + c := fake.NewClientBuilder().WithScheme(mgrScheme).WithObjects(binding, pv, pvc). + WithInterceptorFuncs(interceptor.Funcs{ + Delete: func(ctx context.Context, cl client.WithWatch, obj client.Object, + opts ...client.DeleteOption, + ) error { + deleteOptions := &client.DeleteOptions{} + for _, opt := range opts { + opt.ApplyToDelete(deleteOptions) + } + require.NotNil(t, deleteOptions.Preconditions, obj.GetName()) + require.NotNil(t, deleteOptions.Preconditions.UID, obj.GetName()) + require.NotNil(t, deleteOptions.Preconditions.ResourceVersion, obj.GetName()) + assert.Equal(t, obj.GetUID(), *deleteOptions.Preconditions.UID) + assert.Equal(t, obj.GetResourceVersion(), *deleteOptions.Preconditions.ResourceVersion) + validated[obj.GetName()] = true + return cl.Delete(ctx, obj, opts...) + }, + }).Build() + r := &Reconciler{Client: c} + + _, err := r.doCleanupModelCacheNVMesh(t.Context(), st) + require.NoError(t, err) + assert.True(t, validated[pv.Name]) + assert.True(t, validated[pvc.Name]) +} + +func TestAnnotatedReaderCleanupRefusesDeletePolicyPVBeforeAnyDelete(t *testing.T) { + _, binding, st, _ := newHelmBindingTestFixture(t) + pv, pvc := perRequestBindingOwnedVolumes( + st, string(helmBindingTestBindingUID), string(helmBindingTestBindingUID)) + pv.Spec.PersistentVolumeReclaimPolicy = corev1.PersistentVolumeReclaimDelete + deletes := 0 + c := fake.NewClientBuilder().WithScheme(mgrScheme).WithObjects(binding, pv, pvc). + WithInterceptorFuncs(interceptor.Funcs{ + Delete: func(ctx context.Context, cl client.WithWatch, obj client.Object, + opts ...client.DeleteOption, + ) error { + deletes++ + return cl.Delete(ctx, obj, opts...) + }, + }).Build() + r := &Reconciler{Client: c} + + res, err := r.doCleanupModelCacheNVMesh(t.Context(), st) + assert.Equal(t, reconcile.Result{}, res) + require.ErrorContains(t, err, "reclaim policy") + assert.Equal(t, 0, deletes) + require.NoError(t, c.Get(t.Context(), client.ObjectKeyFromObject(pv), &corev1.PersistentVolume{})) + require.NoError(t, c.Get( + t.Context(), client.ObjectKeyFromObject(pvc), &corev1.PersistentVolumeClaim{})) +} + +func TestAnnotatedReaderCleanupClassifiesListErrorsWithoutDeleting(t *testing.T) { + for _, tt := range []struct { + name string + readErr error + wantRequeue bool + wantError bool + }{ + { + name: "transient API failure", + readErr: errors.NewServiceUnavailable( + "temporary reader inventory failure"), + wantRequeue: true, + }, + { + name: "authorization failure", + readErr: errors.NewForbidden( + corev1.Resource("persistentvolumes"), "", stderrors.New("denied")), + wantError: true, + }, + } { + t.Run(tt.name, func(t *testing.T) { + _, binding, st, _ := newHelmBindingTestFixture(t) + deletes := 0 + c := fake.NewClientBuilder().WithScheme(mgrScheme).WithObjects(binding). + WithInterceptorFuncs(interceptor.Funcs{ + List: func(_ context.Context, _ client.WithWatch, list client.ObjectList, + _ ...client.ListOption, + ) error { + if _, ok := list.(*corev1.PersistentVolumeList); ok { + return tt.readErr + } + return nil + }, + Delete: func(ctx context.Context, cl client.WithWatch, obj client.Object, + opts ...client.DeleteOption, + ) error { + deletes++ + return cl.Delete(ctx, obj, opts...) + }, + }).Build() + r := &Reconciler{Client: c} + + res, err := r.doCleanupModelCacheNVMesh(t.Context(), st) + assert.Equal(t, tt.wantRequeue, res.Requeue) + assert.Equal(t, 0, deletes) + if tt.wantError { + require.Error(t, err) + assert.True(t, errors.IsForbidden(err)) + return + } + require.NoError(t, err) + }) + } +} + +func TestCleanupIdleModelCachesSkipsBindingOwnedPrimaryPV(t *testing.T) { + now := time.Now() + timeConfig := (&k8sutil.TimeConfig{}).Complete() + oldReference := now.Add(-timeConfig.ModelCacheIdlePeriod - time.Minute). + Format(primaryPVLastReferencedTimeFormat) + bindingHandle := "binding-cache" + bindingPV := idlePrimaryPV("binding-primary", bindingHandle, oldReference) + bindingPV.Labels[ModelCacheBindingUIDLabelKey] = string(helmBindingTestBindingUID) + legacyPV := idlePrimaryPV("legacy-primary", "legacy-cache", oldReference) + encryptedSC := &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{ + Name: "binding-encrypted-sc", + Annotations: map[string]string{ + encryptedModelCacheStorageClassAnnotation: encryptedModelCacheStorageClassAnnotationValue, + }, + }} + bindingPV.Spec.StorageClassName = encryptedSC.Name + legacyPV.Spec.StorageClassName = encryptedSC.Name + + c := fake.NewClientBuilder(). + WithScheme(mgrScheme). + WithObjects(bindingPV, legacyPV, encryptedSC). + WithIndex(&nvcav1new.StorageRequest{}, objectNameFieldPath, objectNameExtractValues). + Build() + statuses := newInitStatusCache(c) + statuses.put(bindingHandle, nvcav1new.StorageRequestStatus{Phase: nvcav1new.StorageReady}) + r := &Reconciler{ + Client: c, + nowFunc: func() time.Time { return now }, + k8sTimeConfig: timeConfig, + initStatuses: statuses, + metrics: newTestMetrics(), + } + + require.NoError(t, r.cleanupIdleModelCaches(t.Context())) + gotBindingPV := &corev1.PersistentVolume{} + require.NoError(t, c.Get(t.Context(), client.ObjectKeyFromObject(bindingPV), gotBindingPV)) + assert.Equal(t, corev1.PersistentVolumeReclaimRetain, gotBindingPV.Spec.PersistentVolumeReclaimPolicy) + require.NoError(t, c.Get(t.Context(), client.ObjectKeyFromObject(encryptedSC), &storagev1.StorageClass{})) + _, found := statuses.get(bindingHandle) + assert.True(t, found, "binding-owned primary PV must preserve its init-status entry") + assert.True(t, errors.IsNotFound(c.Get( + t.Context(), client.ObjectKeyFromObject(legacyPV), &corev1.PersistentVolume{})), + "legacy idle GC must continue reclaiming unlabeled primary PVs") +} + +func perRequestBindingOwnedVolumes( + st *nvcav1new.StorageRequest, + pvBinding string, + pvcBinding string, +) (*corev1.PersistentVolume, *corev1.PersistentVolumeClaim) { + labels := getClusterWideResourceLabels(st) + pvLabels := map[string]string{ + StorageRequestOwnerKey: labels[StorageRequestOwnerKey], + StorageRequestNamespaceKey: labels[StorageRequestNamespaceKey], + } + if pvBinding != "" { + pvLabels[ModelCacheBindingUIDLabelKey] = pvBinding + } + pvLabels[ModelCacheRequestUIDLabelKey] = string(helmBindingTestRequestUID) + pvcLabels := map[string]string{ + StorageRequestOwnerKey: labels[StorageRequestOwnerKey], + StorageRequestNamespaceKey: labels[StorageRequestNamespaceKey], + } + if pvcBinding != "" { + pvcLabels[ModelCacheBindingUIDLabelKey] = pvcBinding + } + pvcLabels[ModelCacheRequestUIDLabelKey] = string(helmBindingTestRequestUID) + pvName := "secondary-pv-" + st.Spec.ICMSRequestName + pvcName := "ro-pvc-" + st.Spec.ModelCache.CacheHandle + pvcUID := k8stypes.UID("reader-pvc-uid") + return &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvName, UID: k8stypes.UID("secondary-pv-uid"), ResourceVersion: "1", Labels: pvLabels, + }, + Spec: corev1.PersistentVolumeSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany}, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + PersistentVolumeSource: corev1.PersistentVolumeSource{CSI: &corev1.CSIPersistentVolumeSource{ + Driver: NVMeshStorageClassProvisioner, VolumeHandle: "test-volume-handle", + }}, + ClaimRef: &corev1.ObjectReference{ + APIVersion: "v1", Kind: "PersistentVolumeClaim", Namespace: st.Namespace, + Name: pvcName, UID: pvcUID, + }, + }, + }, &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvcName, Namespace: st.Namespace, UID: pvcUID, + ResourceVersion: "1", Labels: pvcLabels, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadOnlyMany}, + VolumeName: pvName, + }, + } +} + +func idlePrimaryPV(name, cacheHandle, lastReference string) *corev1.PersistentVolume { + return &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + primaryPVLabelKey: primaryPVLabelValue, + modelCacheHandleLabelKey: cacheHandle, + }, + Annotations: map[string]string{ + primaryPVLastReferencedAnnotationKey: lastReference, + }, + }, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + StorageClassName: DefaultModelCacheStorageClassName, + }, + Status: corev1.PersistentVolumeStatus{Phase: corev1.VolumeAvailable}, + } +} diff --git a/src/compute-plane-services/nvca/pkg/storage/modelcache_nvmesh_encrypt.go b/src/compute-plane-services/nvca/pkg/storage/modelcache_nvmesh_encrypt.go index 3c9c2d364..d6f58892b 100644 --- a/src/compute-plane-services/nvca/pkg/storage/modelcache_nvmesh_encrypt.go +++ b/src/compute-plane-services/nvca/pkg/storage/modelcache_nvmesh_encrypt.go @@ -21,6 +21,7 @@ import ( "context" "fmt" "io" + "maps" //nolint:gosec "crypto/md5" @@ -29,8 +30,10 @@ import ( corev1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -144,11 +147,50 @@ func (r *Reconciler) ensureNVMeshEncryptionStorageClass( NVMeshStorageClassCSINS: ModelCacheInitNamespace, }, } - op, err := controllerutil.CreateOrUpdate(ctx, r.Client, sc, noMutF) - if op == controllerutil.OperationResultCreated { + existing := &storagev1.StorageClass{} + if err := r.Client.Get(ctx, client.ObjectKey{Name: scName}, existing); err != nil { + if !apierrors.IsNotFound(err) { + return "", fmt.Errorf("get NVMesh encryption StorageClass %q: %w", scName, err) + } + if err := r.Client.Create(ctx, sc); err != nil { + return "", fmt.Errorf("create NVMesh encryption StorageClass %q: %w", scName, err) + } log.Info("Created NVMesh encryption StorageClass", "storageclass", sc.Name) + return scName, nil + } + + if err := validateNVMeshEncryptionStorageClass(existing, sc); err != nil { + return "", err + } + return scName, nil +} + +func validateNVMeshEncryptionStorageClass(actual, expected *storagev1.StorageClass) error { + invalid := func(field string) error { + return fmt.Errorf("existing NVMesh encryption StorageClass %q has unexpected %s", actual.Name, field) + } + if actual.Provisioner != expected.Provisioner { + return invalid("provisioner") } - return scName, err + if actual.ReclaimPolicy == nil || expected.ReclaimPolicy == nil || + *actual.ReclaimPolicy != *expected.ReclaimPolicy { + return invalid("reclaimPolicy") + } + if actual.VolumeBindingMode == nil || expected.VolumeBindingMode == nil || + *actual.VolumeBindingMode != *expected.VolumeBindingMode { + return invalid("volumeBindingMode") + } + if actual.AllowVolumeExpansion == nil || expected.AllowVolumeExpansion == nil || + *actual.AllowVolumeExpansion != *expected.AllowVolumeExpansion { + return invalid("allowVolumeExpansion") + } + if !maps.Equal(actual.Parameters, expected.Parameters) { + return invalid("parameters") + } + if !isStorageClassEncrypted(actual) { + return invalid("encryption annotation") + } + return nil } func (r *Reconciler) ensureNVMeshEncryptionSecret( @@ -175,7 +217,14 @@ func (r *Reconciler) ensureNVMeshEncryptionSecret( if op == controllerutil.OperationResultCreated { log.Info("Created NVMesh encryption Secret", "secret_name", secret.Name) } - return secretName, err + if err != nil { + return "", err + } + if len(secret.Data["dmcryptKey"]) == 0 { + return "", fmt.Errorf("existing NVMesh encryption Secret %s/%s has no dmcryptKey", + secret.Namespace, secret.Name) + } + return secretName, nil } // Generate the Random NVMeshKeyBytes byte token diff --git a/src/compute-plane-services/nvca/pkg/storage/modelcache_reprovision_test.go b/src/compute-plane-services/nvca/pkg/storage/modelcache_reprovision_test.go new file mode 100644 index 000000000..7546fd90b --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/storage/modelcache_reprovision_test.go @@ -0,0 +1,785 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package storage + +import ( + "bytes" + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + batchv1 "k8s.io/api/batch/v1" + coordv1 "k8s.io/api/coordination/v1" + corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/internal/util/k8sutil" + nvcav1new "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v1" + nvcav2beta1 "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/apis/nvca/v2beta1" + featureflagmock "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/featureflag/mock" + nvcatypes "github.com/NVIDIA/nvcf/src/compute-plane-services/nvca/pkg/types" +) + +const ( + reprovisionCacheHandle = "reprovision-cache" + reprovisionNCAID = "reprovision-nca" + reprovisionRequestNS = "requests" + reprovisionRequestName = "reprovision-request" + reprovisionWorkloadNS = "workload" + reprovisionRequestUID = types.UID("reprovision-request-uid") + reprovisionBindingUID = types.UID("reprovision-binding-uid") +) + +type helmReprovisionFixture struct { + binding *nvcav2beta1.ModelCacheBinding + request *nvcav1new.StorageRequest + icms *nvcav2beta1.ICMSRequest + class *storagev1.StorageClass +} + +func newHelmReprovisionFixture(t *testing.T, encrypted bool) *helmReprovisionFixture { + t.Helper() + reclaimPolicy := corev1.PersistentVolumeReclaimRetain + bindingMode := storagev1.VolumeBindingImmediate + sc := &storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: DefaultModelCacheStorageClassName, + UID: types.UID("model-cache-storage-class-uid"), + }, + Provisioner: NVMeshStorageClassProvisioner, + ReclaimPolicy: &reclaimPolicy, + VolumeBindingMode: &bindingMode, + Parameters: map[string]string{ + NVMeshStorageClassVPG: NVMeshStorageClassVPGType, + NVMeshStorageClassCSIFS: NVMeshStorageClassFS, + }, + } + selection, err := NewPersistedModelCacheStorageSelection( + ModelCacheWorkflowHelm, + ModelCacheSelectionDurable, + &ModelCacheStorageSelection{ + StorageClassName: sc.Name, + StorageClassUID: sc.UID, + StorageClassDigest: digestStorageClass(sc), + CatalogDigest: "sha256:" + strings.Repeat("a", 64), + Provider: "nvmesh", + Provisioner: sc.Provisioner, + Transition: ModelCacheTransitionROXReadOnly, + RequiredAccessModes: requiredAccessModesForTransition(ModelCacheTransitionROXReadOnly), + RequiredMountOptions: []string{"ro", "norecovery", "nouuid"}, + }, + ) + require.NoError(t, err) + selection.EncryptionRequired = encrypted + + binding, err := NewModelCacheBinding( + selection, reprovisionNCAID, reprovisionCacheHandle, ModelCacheInitNamespace) + require.NoError(t, err) + binding.UID = reprovisionBindingUID + binding.Status.Phase = nvcav2beta1.ModelCacheBindingPhaseActive + binding.Status.RequestReferences = []nvcav2beta1.ModelCacheBindingRequestReference{{ + Namespace: reprovisionRequestNS, + Name: reprovisionRequestName, + UID: reprovisionRequestUID, + }} + selection.BindingName = binding.Name + selection.BindingUID = binding.UID + rawSelection, err := selection.Marshal() + require.NoError(t, err) + + request := &nvcav1new.StorageRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: nvcav1new.ModelCacheRequest.Name(), + Namespace: reprovisionWorkloadNS, + Labels: map[string]string{ + nvcatypes.NCAIDKey: nvcatypes.MakeNCAIDLabelValue(reprovisionNCAID), + }, + Annotations: map[string]string{ + ModelCacheStorageSelectionAnnotationKey: rawSelection, + ICMSRequestUIDAnnotationKey: string(reprovisionRequestUID), + }, + }, + Spec: nvcav1new.StorageRequestSpec{ + Type: nvcav1new.ModelCacheRequest, + ICMSRequestName: reprovisionRequestName, + ICMSRequestNamespace: reprovisionRequestNS, + ModelCache: &nvcav1new.ModelCacheSpec{ + CacheHandle: reprovisionCacheHandle, + Backend: string(HelmCacheBackendNVMesh), + Encryption: &nvcav1new.ModelCacheEncryption{Required: encrypted}, + }, + }, + } + icms := &nvcav2beta1.ICMSRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: reprovisionRequestName, + Namespace: reprovisionRequestNS, + UID: reprovisionRequestUID, + Annotations: map[string]string{ + ModelCacheStorageSelectionAnnotationKey: rawSelection, + }, + }, + Spec: newModelCacheICMSSpec(reprovisionCacheHandle), + } + icms.Spec.NCAId = reprovisionNCAID + icms.Spec.CreationMsgInfo.NCAID = reprovisionNCAID + + return &helmReprovisionFixture{ + binding: binding, + request: request, + icms: icms, + class: sc, + } +} + +type helmReprovisionCalls struct { + storageClassGets int + creates []string +} + +func newHelmReprovisionClient( + t *testing.T, + objects []client.Object, + storageClassGetError error, +) (client.Client, *helmReprovisionCalls) { + t.Helper() + calls := &helmReprovisionCalls{} + c := fake.NewClientBuilder().WithScheme(mgrScheme).WithRESTMapper(newTestRESTMapper(mgrScheme)). + WithObjects(objects...). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, + obj client.Object, opts ...client.GetOption, + ) error { + if _, ok := obj.(*storagev1.StorageClass); ok && key.Name == DefaultModelCacheStorageClassName { + calls.storageClassGets++ + if storageClassGetError != nil { + return storageClassGetError + } + } + return cl.Get(ctx, key, obj, opts...) + }, + Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, + opts ...client.CreateOption, + ) error { + calls.creates = append(calls.creates, + fmt.Sprintf("%T %s/%s", obj, obj.GetNamespace(), obj.GetName())) + if _, ok := obj.(*corev1.PersistentVolumeClaim); ok && obj.GetCreationTimestamp().Time.IsZero() { + obj.SetCreationTimestamp(metav1.Now()) + } + return cl.Create(ctx, obj, opts...) + }, + }).Build() + return c, calls +} + +func newHelmReprovisionReconciler(c client.Client) *Reconciler { + return &Reconciler{ + Client: c, + modelCacheStorageClass: DefaultModelCacheStorageClassName, + fff: &featureflagmock.Fetcher{}, + metrics: newTestMetrics(), + nowFunc: time.Now, + randReader: bytes.NewReader(bytes.Repeat([]byte("x"), NVMeshKeyBytes)), + initStatuses: newInitStatusCache(c), + k8sTimeConfig: (&k8sutil.TimeConfig{}).Complete(), + } +} + +func TestDoModelCacheNVMeshClassifiesBindingRereadErrors(t *testing.T) { + for _, tt := range []struct { + name string + readErr error + wantError bool + wantRequeue bool + }{ + { + name: "temporary API outage requeues", + readErr: apierrors.NewServiceUnavailable("temporary binding reread failure"), + wantRequeue: true, + }, + { + name: "authorization failure surfaces", + readErr: apierrors.NewForbidden( + schema.GroupResource{Group: "nvca.nvcf.nvidia.io", Resource: "modelcachebindings"}, + "binding", fmt.Errorf("denied")), + wantError: true, + }, + } { + t.Run(tt.name, func(t *testing.T) { + fixture := newHelmReprovisionFixture(t, false) + fixture.request.Status.Phase = nvcav1new.StoragePending + c := fake.NewClientBuilder().WithScheme(mgrScheme).WithRESTMapper(newTestRESTMapper(mgrScheme)). + WithObjects(fixture.binding). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, cl client.WithWatch, key client.ObjectKey, + obj client.Object, opts ...client.GetOption, + ) error { + if _, ok := obj.(*nvcav2beta1.ModelCacheBinding); ok { + return tt.readErr + } + return cl.Get(ctx, key, obj, opts...) + }, + }).Build() + r := newHelmReprovisionReconciler(c) + stCopy := fixture.request.DeepCopy() + + res, err := r.doModelCacheNVMesh( + t.Context(), *fixture.request, stCopy, fixture.icms) + if tt.wantError { + require.Error(t, err) + } else { + require.NoError(t, err) + } + assert.Equal(t, tt.wantRequeue, res.Requeue) + assert.False(t, isTerminal(err)) + assert.Equal(t, nvcav1new.StoragePending, stCopy.Status.Phase) + }) + } +} + +func TestHelmWriterReprovisionRejectsMissingOrDriftedStorageClassBeforeCreate(t *testing.T) { + deletePolicy := corev1.PersistentVolumeReclaimDelete + for _, tt := range []struct { + name string + omitClass bool + mutate func(*storagev1.StorageClass) + want string + wantDrift bool + wantMissing bool + }{ + {name: "missing", omitClass: true, want: "not found", wantMissing: true}, + {name: "UID drift", mutate: func(sc *storagev1.StorageClass) { + sc.UID = types.UID("replacement-storage-class-uid") + }, want: "UID changed", wantDrift: true}, + {name: "provisioner drift", mutate: func(sc *storagev1.StorageClass) { + sc.Provisioner = "other.csi.example.com" + }, want: "provisioner changed", wantDrift: true}, + {name: "reclaim policy drift", mutate: func(sc *storagev1.StorageClass) { + sc.ReclaimPolicy = &deletePolicy + }, want: "reclaimPolicy Retain", wantDrift: true}, + {name: "configuration digest drift", mutate: func(sc *storagev1.StorageClass) { + sc.Parameters["changed"] = "true" + }, want: "configuration digest changed", wantDrift: true}, + } { + t.Run(tt.name, func(t *testing.T) { + fixture := newHelmReprovisionFixture(t, false) + objects := []client.Object{fixture.binding} + if !tt.omitClass { + liveClass := fixture.class.DeepCopy() + if tt.mutate != nil { + tt.mutate(liveClass) + } + objects = append(objects, liveClass) + } + c, calls := newHelmReprovisionClient(t, objects, nil) + r := newHelmReprovisionReconciler(c) + stCopy := fixture.request.DeepCopy() + + _, err := r.doModelCacheRouted(t.Context(), *fixture.request, stCopy, fixture.icms) + require.ErrorContains(t, err, tt.want) + assert.True(t, isTerminal(err)) + assert.Equal(t, 1, calls.storageClassGets) + assert.Empty(t, calls.creates, "no writer object may be created before the precondition passes") + if tt.wantDrift { + assert.ErrorIs(t, err, ErrModelCacheStorageSelectionDrift) + } + if tt.wantMissing { + assert.True(t, apierrors.IsNotFound(err)) + } + }) + } +} + +func TestHelmWriterReprovisionRetriesTransientStorageClassRead(t *testing.T) { + fixture := newHelmReprovisionFixture(t, false) + c, calls := newHelmReprovisionClient(t, []client.Object{fixture.binding}, + apierrors.NewServiceUnavailable("temporary StorageClass read failure")) + r := newHelmReprovisionReconciler(c) + + res, err := r.doModelCacheRouted( + t.Context(), *fixture.request, fixture.request.DeepCopy(), fixture.icms) + require.NoError(t, err) + assert.True(t, res.Requeue, "transient API failures must remain retryable") + assert.Equal(t, 1, calls.storageClassGets) + assert.Empty(t, calls.creates) +} + +func TestHelmWriterReprovisionWithExactStorageClassProceeds(t *testing.T) { + fixture := newHelmReprovisionFixture(t, false) + c, calls := newHelmReprovisionClient( + t, []client.Object{fixture.binding, fixture.class}, nil) + r := newHelmReprovisionReconciler(c) + stCopy := fixture.request.DeepCopy() + + res, err := r.doModelCacheRouted(t.Context(), *fixture.request, stCopy, fixture.icms) + require.NoError(t, err) + assert.True(t, res.Requeue) + assert.Equal(t, nvcav1new.StoragePending, stCopy.Status.Phase) + assert.Equal(t, 1, calls.storageClassGets) + assert.NotEmpty(t, calls.creates) + + writer := &corev1.PersistentVolumeClaim{} + require.NoError(t, c.Get(t.Context(), client.ObjectKey{ + Namespace: ModelCacheInitNamespace, + Name: "rw-pvc-" + reprovisionCacheHandle, + }, writer)) + require.NotNil(t, writer.Spec.StorageClassName) + assert.Equal(t, DefaultModelCacheStorageClassName, *writer.Spec.StorageClassName) + assert.Equal(t, string(reprovisionBindingUID), writer.Labels[ModelCacheBindingUIDLabelKey]) +} + +func TestHelmExistingBindingOwnedWriterDoesNotRereadStorageClass(t *testing.T) { + fixture := newHelmReprovisionFixture(t, false) + c, calls := newHelmReprovisionClient( + t, []client.Object{fixture.binding, fixture.class}, nil) + r := newHelmReprovisionReconciler(c) + stCopy := fixture.request.DeepCopy() + + res, err := r.doModelCacheRouted(t.Context(), *fixture.request, stCopy, fixture.icms) + require.NoError(t, err) + assert.True(t, res.Requeue) + assert.Equal(t, nvcav1new.StoragePending, stCopy.Status.Phase) + existingWriter := &corev1.PersistentVolumeClaim{} + require.NoError(t, c.Get(t.Context(), client.ObjectKey{ + Namespace: ModelCacheInitNamespace, + Name: "rw-pvc-" + reprovisionCacheHandle, + }, existingWriter)) + require.NoError(t, c.Delete(t.Context(), fixture.class)) + calls.storageClassGets = 0 + calls.creates = nil + + next := stCopy.DeepCopy() + res, err = r.doModelCacheRouted(t.Context(), *stCopy, next, fixture.icms) + require.NoError(t, err) + assert.True(t, res.Requeue) + assert.Equal(t, 0, calls.storageClassGets) + assert.Empty(t, calls.creates) + + unknown := stCopy.DeepCopy() + unknown.Status.Phase = nvcav1new.StorageUnknown + unknownOut := unknown.DeepCopy() + calls.creates = nil + res, err = r.doModelCacheRouted(t.Context(), *unknown, unknownOut, fixture.icms) + require.NoError(t, err) + assert.True(t, res.Requeue) + assert.Equal(t, nvcav1new.StoragePending, unknownOut.Status.Phase) + assert.Equal(t, 0, calls.storageClassGets) + assert.NotContains(t, calls.creates, + "*v1.PersistentVolumeClaim "+ModelCacheInitNamespace+"/rw-pvc-"+reprovisionCacheHandle) + + got := &corev1.PersistentVolumeClaim{} + require.NoError(t, c.Get(t.Context(), client.ObjectKeyFromObject(existingWriter), got)) + assert.Equal(t, existingWriter.UID, got.UID) + assert.Equal(t, string(reprovisionBindingUID), got.Labels[ModelCacheBindingUIDLabelKey]) +} + +func TestHelmWriterPreflightRejectsSameBindingDriftBeforeAnyCreate(t *testing.T) { + for _, tt := range []struct { + name string + missingWriter bool + mutate func(*testing.T, context.Context, client.Client) + }{ + { + name: "writer PVC immutable spec drift", + mutate: func(t *testing.T, ctx context.Context, c client.Client) { + t.Helper() + pvc := &corev1.PersistentVolumeClaim{} + require.NoError(t, c.Get(ctx, client.ObjectKey{ + Namespace: ModelCacheInitNamespace, Name: "rw-pvc-" + reprovisionCacheHandle, + }, pvc)) + pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + require.NoError(t, c.Update(ctx, pvc)) + }, + }, + { + name: "writer Job immutable spec drift", missingWriter: true, + mutate: func(t *testing.T, ctx context.Context, c client.Client) { + t.Helper() + job := &batchv1.Job{} + require.NoError(t, c.Get(ctx, client.ObjectKey{ + Namespace: ModelCacheInitNamespace, Name: "writer-job-" + reprovisionCacheHandle, + }, job)) + require.NotEmpty(t, job.Spec.Template.Spec.Containers) + job.Spec.Template.Spec.Containers[0].Image = "tampered.invalid/writer:latest" + require.NoError(t, c.Update(ctx, job)) + }, + }, + { + name: "writer Lease immutable duration drift", + mutate: func(t *testing.T, ctx context.Context, c client.Client) { + t.Helper() + lease := &coordv1.Lease{} + require.NoError(t, c.Get(ctx, client.ObjectKey{ + Namespace: ModelCacheInitNamespace, Name: buildInitLeaseName(reprovisionCacheHandle), + }, lease)) + duration := int32(1) + lease.Spec.LeaseDurationSeconds = &duration + require.NoError(t, c.Update(ctx, lease)) + }, + }, + { + name: "writer pull Secret immutable data drift", missingWriter: true, + mutate: func(t *testing.T, ctx context.Context, c client.Client) { + t.Helper() + secret := &corev1.Secret{} + require.NoError(t, c.Get(ctx, client.ObjectKey{ + Namespace: ModelCacheInitNamespace, + Name: "writer-job-" + reprovisionCacheHandle + "-0-pull-worker", + }, secret)) + if secret.Data == nil { + secret.Data = map[string][]byte{} + } + secret.Data["tampered"] = []byte("true") + require.NoError(t, c.Update(ctx, secret)) + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + fixture := newHelmReprovisionFixture(t, false) + c, calls := newHelmReprovisionClient( + t, []client.Object{fixture.binding, fixture.class}, nil) + r := newHelmReprovisionReconciler(c) + first := fixture.request.DeepCopy() + _, err := r.doModelCacheRouted( + t.Context(), *fixture.request, first, fixture.icms) + require.NoError(t, err) + tt.mutate(t, t.Context(), c) + if tt.missingWriter { + writer := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{ + Namespace: ModelCacheInitNamespace, Name: "rw-pvc-" + reprovisionCacheHandle, + }} + require.NoError(t, c.Delete(t.Context(), writer)) + } + calls.creates = nil + + _, err = r.doModelCacheRouted( + t.Context(), *fixture.request, fixture.request.DeepCopy(), fixture.icms) + require.Error(t, err) + assert.True(t, isTerminal(err)) + require.ErrorIs(t, err, errModelCacheBindingOwnership) + assert.Empty(t, calls.creates, + "the complete writer-object preflight must finish before the first create") + }) + } +} + +func TestHelmSuccessfulInitRetriesCleanupFailureWithoutPhaseAdvance(t *testing.T) { + fixture := newHelmReprovisionFixture(t, false) + seedClient, _ := newHelmReprovisionClient( + t, []client.Object{fixture.binding, fixture.class}, nil) + seedReconciler := newHelmReprovisionReconciler(seedClient) + request := fixture.request.DeepCopy() + _, err := seedReconciler.doModelCacheRouted( + t.Context(), *fixture.request, request, fixture.icms) + require.NoError(t, err) + + writer := &corev1.PersistentVolumeClaim{} + require.NoError(t, seedClient.Get(t.Context(), client.ObjectKey{ + Namespace: ModelCacheInitNamespace, Name: "rw-pvc-" + reprovisionCacheHandle, + }, writer)) + writer.UID = "writer-pvc-uid" + writer.Spec.VolumeName = "primary-pv-" + reprovisionCacheHandle + writer.Status.Phase = corev1.ClaimBound + job := &batchv1.Job{} + require.NoError(t, seedClient.Get(t.Context(), client.ObjectKey{ + Namespace: ModelCacheInitNamespace, Name: "writer-job-" + reprovisionCacheHandle, + }, job)) + completion := metav1.Now() + job.Status.CompletionTime = &completion + job.Status.Succeeded = 1 + lease := &coordv1.Lease{} + require.NoError(t, seedClient.Get(t.Context(), client.ObjectKey{ + Namespace: ModelCacheInitNamespace, Name: buildInitLeaseName(reprovisionCacheHandle), + }, lease)) + secret := &corev1.Secret{} + require.NoError(t, seedClient.Get(t.Context(), client.ObjectKey{ + Namespace: ModelCacheInitNamespace, + Name: "writer-job-" + reprovisionCacheHandle + "-0-pull-worker", + }, secret)) + + primary := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{Name: writer.Spec.VolumeName}, + Spec: corev1.PersistentVolumeSpec{ + AccessModes: append([]corev1.PersistentVolumeAccessMode(nil), writer.Spec.AccessModes...), + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + StorageClassName: *writer.Spec.StorageClassName, + ClaimRef: &corev1.ObjectReference{ + APIVersion: "v1", Kind: "PersistentVolumeClaim", + Namespace: writer.Namespace, Name: writer.Name, UID: writer.UID, + }, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + Driver: fixture.class.Provisioner, + VolumeHandle: "cluster:csi:volume:" + ModelCacheInitNamespace, + }, + }, + }, + } + cleanupLists := 0 + c := fake.NewClientBuilder().WithScheme(mgrScheme).WithRESTMapper(newTestRESTMapper(mgrScheme)). + WithObjects(fixture.binding.DeepCopy(), writer, job, lease, secret, primary). + WithInterceptorFuncs(interceptor.Funcs{ + List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, + opts ...client.ListOption, + ) error { + if _, ok := list.(*batchv1.JobList); ok { + cleanupLists++ + return apierrors.NewServiceUnavailable("temporary successful-init cleanup read") + } + return cl.List(ctx, list, opts...) + }, + }).Build() + r := newHelmReprovisionReconciler(c) + request.Status.Phase = nvcav1new.StorageInitRunning + out := request.DeepCopy() + + res, err := r.reconcileInitModelCacheNVMesh( + t.Context(), *request, out, writer.DeepCopy(), job.DeepCopy(), + []*corev1.Secret{secret.DeepCopy()}, HelmCacheBackendNVMesh, true) + require.NoError(t, err) + assert.True(t, res.Requeue) + assert.Equal(t, nvcav1new.StorageInitRunning, out.Status.Phase) + assert.Equal(t, 1, cleanupLists) + gotPrimary := &corev1.PersistentVolume{} + require.NoError(t, c.Get(t.Context(), client.ObjectKey{Name: primary.Name}, gotPrimary)) + assert.Equal(t, string(reprovisionBindingUID), + gotPrimary.Labels[ModelCacheBindingUIDLabelKey]) +} + +func TestHelmEncryptedWriterDoesNotRereadBaseStorageClass(t *testing.T) { + fixture := newHelmReprovisionFixture(t, true) + c, calls := newHelmReprovisionClient(t, []client.Object{fixture.binding}, nil) + r := newHelmReprovisionReconciler(c) + stCopy := fixture.request.DeepCopy() + + res, err := r.doModelCacheRouted(t.Context(), *fixture.request, stCopy, fixture.icms) + require.NoError(t, err) + assert.True(t, res.Requeue) + assert.Equal(t, 0, calls.storageClassGets) + + derivedClassName := buildStorageClassName(reprovisionNCAID) + derivedClass := &storagev1.StorageClass{} + require.NoError(t, c.Get(t.Context(), client.ObjectKey{Name: derivedClassName}, derivedClass)) + writer := &corev1.PersistentVolumeClaim{} + require.NoError(t, c.Get(t.Context(), client.ObjectKey{ + Namespace: ModelCacheInitNamespace, + Name: "rw-pvc-" + reprovisionCacheHandle, + }, writer)) + require.NotNil(t, writer.Spec.StorageClassName) + assert.Equal(t, derivedClassName, *writer.Spec.StorageClassName) +} + +func TestHelmStaticReaderCreationDoesNotRereadStorageClass(t *testing.T) { + fixture := newHelmReprovisionFixture(t, false) + primaryPV := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "primary-reprovision-cache", + Labels: map[string]string{ + primaryPVLabelKey: primaryPVLabelValue, + modelCacheHandleLabelKey: reprovisionCacheHandle, + ModelCacheBindingUIDLabelKey: string(reprovisionBindingUID), + }, + }, + Spec: corev1.PersistentVolumeSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + ClaimRef: &corev1.ObjectReference{ + APIVersion: "v1", + Kind: "PersistentVolumeClaim", + Namespace: ModelCacheInitNamespace, + Name: "rw-pvc-" + reprovisionCacheHandle, + UID: types.UID("reprovision-writer-pvc-uid"), + }, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + Driver: NVMeshStorageClassProvisioner, + VolumeHandle: "cluster:csi:volume:" + ModelCacheInitNamespace, + }, + }, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + StorageClassName: DefaultModelCacheStorageClassName, + }, + Status: corev1.PersistentVolumeStatus{Phase: corev1.VolumeAvailable}, + } + c, calls := newHelmReprovisionClient( + t, []client.Object{fixture.binding, primaryPV}, nil) + r := newHelmReprovisionReconciler(c) + stCopy := fixture.request.DeepCopy() + + _, err := r.doModelCacheRouted(t.Context(), *fixture.request, stCopy, fixture.icms) + require.NoError(t, err) + assert.Equal(t, 0, calls.storageClassGets) + assert.Equal(t, nvcav1new.StorageCreating, stCopy.Status.Phase) + + secondaryPV := &corev1.PersistentVolume{} + require.NoError(t, c.Get(t.Context(), client.ObjectKey{ + Name: "secondary-pv-" + reprovisionRequestName, + }, secondaryPV)) + assert.Equal(t, string(reprovisionBindingUID), secondaryPV.Labels[ModelCacheBindingUIDLabelKey]) + assert.Equal(t, string(reprovisionRequestUID), secondaryPV.Labels[ModelCacheRequestUIDLabelKey]) + roPVC := &corev1.PersistentVolumeClaim{} + require.NoError(t, c.Get(t.Context(), client.ObjectKey{ + Namespace: reprovisionWorkloadNS, + Name: "ro-pvc-" + reprovisionCacheHandle, + }, roPVC)) + assert.Equal(t, string(reprovisionBindingUID), roPVC.Labels[ModelCacheBindingUIDLabelKey]) + assert.Equal(t, string(reprovisionRequestUID), roPVC.Labels[ModelCacheRequestUIDLabelKey]) + + _, err = r.doModelCacheRouted( + t.Context(), *fixture.request, fixture.request.DeepCopy(), fixture.icms) + require.NoError(t, err) + assert.Equal(t, 0, calls.storageClassGets, + "reconciling existing static reader objects must also use the persisted provisioner") +} + +func TestHelmReaderCreateRaceRejectsStaleRequestGeneration(t *testing.T) { + for _, target := range []string{"secondary PV", "reader PVC"} { + t.Run(target, func(t *testing.T) { + fixture := newHelmReprovisionFixture(t, false) + primary := newHelmReaderPrimaryPV() + injected := false + c := fake.NewClientBuilder().WithScheme(mgrScheme).WithRESTMapper(newTestRESTMapper(mgrScheme)). + WithObjects(fixture.binding, primary). + WithInterceptorFuncs(interceptor.Funcs{ + Create: func(ctx context.Context, cl client.WithWatch, obj client.Object, + opts ...client.CreateOption, + ) error { + resource := "" + switch typed := obj.(type) { + case *corev1.PersistentVolume: + if target == "secondary PV" && typed.Name == "secondary-pv-"+reprovisionRequestName { + resource = "persistentvolumes" + } + case *corev1.PersistentVolumeClaim: + if target == "reader PVC" && typed.Namespace == reprovisionWorkloadNS && + typed.Name == "ro-pvc-"+reprovisionCacheHandle { + resource = "persistentvolumeclaims" + } + } + if resource == "" || injected { + return cl.Create(ctx, obj, opts...) + } + stale, ok := obj.DeepCopyObject().(client.Object) + if !ok { + return fmt.Errorf("race object %T is not a client object", obj) + } + stale.GetLabels()[ModelCacheRequestUIDLabelKey] = "stale-request-uid" + if err := cl.Create(ctx, stale, opts...); err != nil { + return err + } + injected = true + return apierrors.NewAlreadyExists( + schema.GroupResource{Resource: resource}, obj.GetName()) + }, + }).Build() + r := newHelmReprovisionReconciler(c) + stCopy := fixture.request.DeepCopy() + stCopy.Labels[ModelCacheBindingUIDLabelKey] = string(reprovisionBindingUID) + + _, err := r.doModelCacheNVMesh(t.Context(), *fixture.request, stCopy, fixture.icms) + require.ErrorContains(t, err, "request UID") + assert.True(t, isTerminal(err)) + assert.True(t, injected) + + var stale client.Object + if target == "secondary PV" { + stale = &corev1.PersistentVolume{ObjectMeta: metav1.ObjectMeta{ + Name: "secondary-pv-" + reprovisionRequestName, + }} + } else { + stale = &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{ + Name: "ro-pvc-" + reprovisionCacheHandle, Namespace: reprovisionWorkloadNS, + }} + } + require.NoError(t, c.Get(t.Context(), client.ObjectKeyFromObject(stale), stale)) + assert.Equal(t, "stale-request-uid", stale.GetLabels()[ModelCacheRequestUIDLabelKey]) + }) + } +} + +func TestHelmReaderDoesNotRecreatePVCForStaleClaimUID(t *testing.T) { + fixture := newHelmReprovisionFixture(t, false) + primary := newHelmReaderPrimaryPV() + secondary := primary.DeepCopy() + secondary.ObjectMeta = metav1.ObjectMeta{ + Name: "secondary-pv-" + reprovisionRequestName, + Labels: map[string]string{ + ModelCacheBindingUIDLabelKey: string(reprovisionBindingUID), + ModelCacheRequestUIDLabelKey: string(reprovisionRequestUID), + }, + } + secondary.Spec.AccessModes = accessModesRO + secondary.Spec.ClaimRef = &corev1.ObjectReference{ + APIVersion: "v1", Kind: "PersistentVolumeClaim", + Namespace: reprovisionWorkloadNS, Name: "ro-pvc-" + reprovisionCacheHandle, + UID: types.UID("deleted-reader-pvc-uid"), + } + secondary.Spec.CSI.VolumeHandle = "cluster:csi:volume:" + reprovisionWorkloadNS + secondary.Status = corev1.PersistentVolumeStatus{} + c, calls := newHelmReprovisionClient( + t, []client.Object{fixture.binding, primary, secondary}, nil) + r := newHelmReprovisionReconciler(c) + stCopy := fixture.request.DeepCopy() + stCopy.Labels[ModelCacheBindingUIDLabelKey] = string(reprovisionBindingUID) + + _, err := r.doModelCacheNVMesh(t.Context(), *fixture.request, stCopy, fixture.icms) + require.ErrorContains(t, err, "reader PVC generation that no longer exists") + assert.True(t, isTerminal(err)) + assert.Empty(t, calls.creates, "a stale claimRef UID must be rejected before reader PVC creation") + reader := &corev1.PersistentVolumeClaim{} + assert.True(t, apierrors.IsNotFound(c.Get(t.Context(), client.ObjectKey{ + Namespace: reprovisionWorkloadNS, Name: "ro-pvc-" + reprovisionCacheHandle, + }, reader))) +} + +func newHelmReaderPrimaryPV() *corev1.PersistentVolume { + return &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "primary-" + reprovisionCacheHandle, + Labels: map[string]string{ + primaryPVLabelKey: primaryPVLabelValue, + modelCacheHandleLabelKey: reprovisionCacheHandle, + ModelCacheBindingUIDLabelKey: string(reprovisionBindingUID), + }, + }, + Spec: corev1.PersistentVolumeSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + ClaimRef: &corev1.ObjectReference{ + APIVersion: "v1", Kind: "PersistentVolumeClaim", + Namespace: ModelCacheInitNamespace, Name: "rw-pvc-" + reprovisionCacheHandle, + UID: types.UID("reprovision-writer-pvc-uid"), + }, + PersistentVolumeSource: corev1.PersistentVolumeSource{CSI: &corev1.CSIPersistentVolumeSource{ + Driver: NVMeshStorageClassProvisioner, VolumeHandle: "cluster:csi:volume:" + ModelCacheInitNamespace, + }}, + PersistentVolumeReclaimPolicy: corev1.PersistentVolumeReclaimRetain, + StorageClassName: DefaultModelCacheStorageClassName, + }, + Status: corev1.PersistentVolumeStatus{Phase: corev1.VolumeAvailable}, + } +} diff --git a/src/compute-plane-services/nvca/pkg/storage/modelcache_selection.go b/src/compute-plane-services/nvca/pkg/storage/modelcache_selection.go new file mode 100644 index 000000000..623125571 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/storage/modelcache_selection.go @@ -0,0 +1,260 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package storage + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "slices" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" +) + +const ( + // ModelCacheStorageSelectionAnnotationKey persists the storage decision on + // the ICMSRequest before any cache storage side effect. + ModelCacheStorageSelectionAnnotationKey = "nvca.nvcf.nvidia.io/model-cache-storage-selection" + + // ModelCacheStorageSelectionVersion is the annotation contract version. + ModelCacheStorageSelectionVersion = "v1alpha1" + modelCacheStorageSelectionVersion = ModelCacheStorageSelectionVersion +) + +// ModelCacheSelectionMode is the request-level cache behavior. +type ModelCacheSelectionMode string + +const ( + ModelCacheSelectionNone ModelCacheSelectionMode = "none" + ModelCacheSelectionEphemeral ModelCacheSelectionMode = "ephemeral" + ModelCacheSelectionDurable ModelCacheSelectionMode = "durable" +) + +// PersistedModelCacheStorageSelection is intentionally small. It is an +// immutable request snapshot, not a general CSI capability matrix. +type PersistedModelCacheStorageSelection struct { + Version string `json:"version"` + Workflow ModelCacheWorkflow `json:"workflow"` + Mode ModelCacheSelectionMode `json:"mode"` + StorageClassName string `json:"storageClassName,omitempty"` + StorageClassUID types.UID `json:"storageClassUID,omitempty"` + StorageClassDigest string `json:"storageClassDigest,omitempty"` + CatalogDigest string `json:"catalogDigest,omitempty"` + Provider string `json:"provider,omitempty"` + Provisioner string `json:"provisioner,omitempty"` + Transition string `json:"transition,omitempty"` + RequiredAccessModes []corev1.PersistentVolumeAccessMode `json:"requiredAccessModes,omitempty"` + RequiredMountOptions []string `json:"requiredMountOptions,omitempty"` + EncryptionRequired bool `json:"encryptionRequired,omitempty"` + BindingName string `json:"bindingName,omitempty"` + BindingUID types.UID `json:"bindingUID,omitempty"` +} + +// NewPersistedModelCacheStorageSelection creates and validates a request +// snapshot. resolved may be nil for a non-durable fallback caused by an absent +// nvcf-sc. +func NewPersistedModelCacheStorageSelection( + workflow ModelCacheWorkflow, + mode ModelCacheSelectionMode, + resolved *ModelCacheStorageSelection, +) (*PersistedModelCacheStorageSelection, error) { + selection := &PersistedModelCacheStorageSelection{ + Version: ModelCacheStorageSelectionVersion, + Workflow: workflow, + Mode: mode, + } + if resolved != nil { + selection.StorageClassName = resolved.StorageClassName + selection.StorageClassUID = resolved.StorageClassUID + selection.StorageClassDigest = resolved.StorageClassDigest + selection.CatalogDigest = resolved.CatalogDigest + selection.Provider = resolved.Provider + selection.Provisioner = resolved.Provisioner + selection.Transition = resolved.Transition + selection.RequiredAccessModes = append( + []corev1.PersistentVolumeAccessMode(nil), resolved.RequiredAccessModes...) + selection.RequiredMountOptions = append([]string(nil), resolved.RequiredMountOptions...) + } + if err := selection.Validate(); err != nil { + return nil, err + } + return selection, nil +} + +// Marshal returns the canonical annotation payload. +func (s *PersistedModelCacheStorageSelection) Marshal() (string, error) { + if err := s.Validate(); err != nil { + return "", err + } + raw, err := json.Marshal(s) + if err != nil { + return "", fmt.Errorf("marshal model cache storage selection: %w", err) + } + return string(raw), nil +} + +// ParsePersistedModelCacheStorageSelection strictly parses an annotation. +func ParsePersistedModelCacheStorageSelection(raw string) (*PersistedModelCacheStorageSelection, error) { + decoder := json.NewDecoder(bytes.NewBufferString(raw)) + decoder.DisallowUnknownFields() + + selection := &PersistedModelCacheStorageSelection{} + if err := decoder.Decode(selection); err != nil { + return nil, fmt.Errorf("parse model cache storage selection: %w", err) + } + if err := ensureJSONEOF(decoder); err != nil { + return nil, err + } + if err := selection.Validate(); err != nil { + return nil, err + } + return selection, nil +} + +func ensureJSONEOF(decoder *json.Decoder) error { + var extra any + if err := decoder.Decode(&extra); err != io.EOF { + if err == nil { + return fmt.Errorf("parse model cache storage selection: multiple JSON values") + } + return fmt.Errorf("parse model cache storage selection: %w", err) + } + return nil +} + +// Validate rejects partial or invented decisions. +func (s *PersistedModelCacheStorageSelection) Validate() error { + if s == nil { + return fmt.Errorf("model cache storage selection is nil") + } + if s.Version != ModelCacheStorageSelectionVersion { + return fmt.Errorf("unsupported model cache storage selection version %q", s.Version) + } + switch s.Workflow { + case ModelCacheWorkflowRegular, ModelCacheWorkflowHelm: + default: + return fmt.Errorf("invalid model cache workflow %q", s.Workflow) + } + switch s.Mode { + case ModelCacheSelectionNone, ModelCacheSelectionEphemeral, ModelCacheSelectionDurable: + default: + return fmt.Errorf("invalid model cache selection mode %q", s.Mode) + } + if s.Mode == ModelCacheSelectionEphemeral && s.Workflow != ModelCacheWorkflowHelm { + return fmt.Errorf("ephemeral model cache selection requires Helm workflow") + } + if (s.BindingName == "") != (s.BindingUID == "") { + return fmt.Errorf("model cache selection has incomplete binding reference") + } + if s.Mode != ModelCacheSelectionDurable { + if s.BindingName != "" { + return fmt.Errorf("non-durable model cache selection has a binding reference") + } + if s.EncryptionRequired { + return fmt.Errorf("non-durable model cache selection requires encryption") + } + } + + hasResolvedFields := s.StorageClassName != "" || s.StorageClassUID != "" || s.StorageClassDigest != "" || + s.CatalogDigest != "" || s.Provider != "" || s.Provisioner != "" || s.Transition != "" || + len(s.RequiredAccessModes) != 0 || len(s.RequiredMountOptions) != 0 + if !hasResolvedFields { + if s.Mode == ModelCacheSelectionDurable { + return fmt.Errorf("durable model cache selection has no resolved storage") + } + return nil + } + if s.StorageClassName != DefaultModelCacheStorageClassName { + return fmt.Errorf("model cache selection StorageClass must be %q", DefaultModelCacheStorageClassName) + } + if s.StorageClassUID == "" || strings.TrimSpace(s.StorageClassDigest) == "" || + strings.TrimSpace(s.CatalogDigest) == "" || strings.TrimSpace(s.Provider) == "" || + strings.TrimSpace(s.Provisioner) == "" || strings.TrimSpace(s.Transition) == "" { + return fmt.Errorf("model cache selection has incomplete resolved storage") + } + + switch s.Mode { + case ModelCacheSelectionDurable: + switch s.Transition { + case ModelCacheTransitionROXReadOnly: + if s.Provider != ModelCacheProviderNVMesh { + return fmt.Errorf("model cache transition %q requires provider %q, got %q", + s.Transition, ModelCacheProviderNVMesh, s.Provider) + } + if s.Provisioner != NVMeshStorageClassProvisioner { + return fmt.Errorf("model cache transition %q requires provisioner %q, got %q", + s.Transition, NVMeshStorageClassProvisioner, s.Provisioner) + } + for _, required := range []string{"ro", "norecovery", "nouuid"} { + if !slices.Contains(s.RequiredMountOptions, required) { + return fmt.Errorf("model cache transition %q requires mount option %q", + s.Transition, required) + } + } + case ModelCacheTransitionRWXReadOnly: + if s.Workflow != ModelCacheWorkflowRegular { + return fmt.Errorf("model cache transition %q requires regular workflow, got %q", + s.Transition, s.Workflow) + } + if s.EncryptionRequired { + return fmt.Errorf("model cache transition %q does not support encryption", + s.Transition) + } + if len(s.RequiredMountOptions) != 0 { + return fmt.Errorf("model cache transition %q does not create a reader PV and cannot require mount options", + s.Transition) + } + default: + return fmt.Errorf("durable model cache selection has unsupported transition %q", s.Transition) + } + if !slices.Equal(s.RequiredAccessModes, requiredAccessModesForTransition(s.Transition)) { + return fmt.Errorf("model cache transition %q requires access modes %v, got %v", + s.Transition, requiredAccessModesForTransition(s.Transition), s.RequiredAccessModes) + } + case ModelCacheSelectionNone, ModelCacheSelectionEphemeral: + if s.Transition != ModelCacheTransitionDisabled { + return fmt.Errorf("non-durable model cache selection has transition %q", s.Transition) + } + if len(s.RequiredAccessModes) != 0 { + return fmt.Errorf("non-durable model cache selection has required access modes") + } + if len(s.RequiredMountOptions) != 0 { + return fmt.Errorf("non-durable model cache selection has required mount options") + } + } + seenMountOptions := make(map[string]struct{}, len(s.RequiredMountOptions)) + for i, option := range s.RequiredMountOptions { + if strings.TrimSpace(option) == "" || strings.TrimSpace(option) != option { + return fmt.Errorf("model cache selection has invalid required mount option %q", option) + } + if _, found := seenMountOptions[option]; found { + return fmt.Errorf("model cache selection has duplicate required mount option %q", option) + } + for _, previous := range s.RequiredMountOptions[:i] { + if negatesMountOption(previous, option) { + return fmt.Errorf("model cache selection required mount options %q and %q conflict", + previous, option) + } + } + seenMountOptions[option] = struct{}{} + } + return nil +} diff --git a/src/compute-plane-services/nvca/pkg/storage/modelcache_selection_test.go b/src/compute-plane-services/nvca/pkg/storage/modelcache_selection_test.go new file mode 100644 index 000000000..86e6b6260 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/storage/modelcache_selection_test.go @@ -0,0 +1,433 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package storage + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" +) + +func testResolvedModelCacheStorage(transition string) *ModelCacheStorageSelection { + selection := &ModelCacheStorageSelection{ + StorageClassName: DefaultModelCacheStorageClassName, + StorageClassUID: types.UID("storage-class-uid"), + StorageClassDigest: "v1:sha256:storage-class-digest", + CatalogDigest: "sha256:catalog-digest", + Provider: "nvmesh", + Provisioner: NVMeshStorageClassProvisioner, + Transition: transition, + RequiredAccessModes: requiredAccessModesForTransition(transition), + } + if transition == ModelCacheTransitionROXReadOnly { + selection.RequiredMountOptions = []string{"ro", "norecovery", "nouuid"} + } + if transition == ModelCacheTransitionRWXReadOnly { + selection.Provider = "sharedFilesystem" + selection.Provisioner = "shared.csi.example.com" + } + return selection +} + +func testPersistedModelCacheStorageSelection( + mode ModelCacheSelectionMode, + transition string, +) *PersistedModelCacheStorageSelection { + resolved := testResolvedModelCacheStorage(transition) + return &PersistedModelCacheStorageSelection{ + Version: modelCacheStorageSelectionVersion, + Workflow: ModelCacheWorkflowRegular, + Mode: mode, + StorageClassName: resolved.StorageClassName, + StorageClassUID: resolved.StorageClassUID, + StorageClassDigest: resolved.StorageClassDigest, + CatalogDigest: resolved.CatalogDigest, + Provider: resolved.Provider, + Provisioner: resolved.Provisioner, + Transition: resolved.Transition, + RequiredAccessModes: append([]corev1.PersistentVolumeAccessMode(nil), resolved.RequiredAccessModes...), + RequiredMountOptions: append([]string(nil), resolved.RequiredMountOptions...), + } +} + +func TestNewPersistedModelCacheStorageSelection(t *testing.T) { + tests := []struct { + name string + workflow ModelCacheWorkflow + mode ModelCacheSelectionMode + resolved *ModelCacheStorageSelection + wantErr string + wantFields bool + }{ + { + name: "regular cache disabled without a StorageClass", + workflow: ModelCacheWorkflowRegular, + mode: ModelCacheSelectionNone, + }, + { + name: "Helm cache falls back without a StorageClass", + workflow: ModelCacheWorkflowHelm, + mode: ModelCacheSelectionEphemeral, + }, + { + name: "durable NVMesh", + workflow: ModelCacheWorkflowRegular, + mode: ModelCacheSelectionDurable, + resolved: testResolvedModelCacheStorage(ModelCacheTransitionROXReadOnly), + wantFields: true, + }, + { + name: "durable regular rwxReadOnly", + workflow: ModelCacheWorkflowRegular, + mode: ModelCacheSelectionDurable, + resolved: testResolvedModelCacheStorage(ModelCacheTransitionRWXReadOnly), + wantFields: true, + }, + { + name: "Helm rwxReadOnly is rejected", + workflow: ModelCacheWorkflowHelm, + mode: ModelCacheSelectionDurable, + resolved: testResolvedModelCacheStorage(ModelCacheTransitionRWXReadOnly), + wantErr: "requires regular workflow", + wantFields: true, + }, + { + name: "catalog-disabled regular cache", + workflow: ModelCacheWorkflowRegular, + mode: ModelCacheSelectionNone, + resolved: testResolvedModelCacheStorage(ModelCacheTransitionDisabled), + wantFields: true, + }, + { + name: "catalog-disabled Helm cache", + workflow: ModelCacheWorkflowHelm, + mode: ModelCacheSelectionEphemeral, + resolved: testResolvedModelCacheStorage(ModelCacheTransitionDisabled), + wantFields: true, + }, + { + name: "durable selection without resolved storage", + workflow: ModelCacheWorkflowRegular, + mode: ModelCacheSelectionDurable, + wantErr: "has no resolved storage", + }, + { + name: "durable selection with disabled transition", + workflow: ModelCacheWorkflowRegular, + mode: ModelCacheSelectionDurable, + resolved: testResolvedModelCacheStorage(ModelCacheTransitionDisabled), + wantErr: "unsupported transition", + wantFields: true, + }, + { + name: "non-durable selection with NVMesh transition", + workflow: ModelCacheWorkflowRegular, + mode: ModelCacheSelectionNone, + resolved: testResolvedModelCacheStorage(ModelCacheTransitionROXReadOnly), + wantErr: "non-durable model cache selection has transition", + wantFields: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + selection, err := NewPersistedModelCacheStorageSelection(tt.workflow, tt.mode, tt.resolved) + if tt.wantErr != "" { + require.ErrorContains(t, err, tt.wantErr) + assert.Nil(t, selection) + return + } + + require.NoError(t, err) + assert.Equal(t, modelCacheStorageSelectionVersion, selection.Version) + assert.Equal(t, tt.workflow, selection.Workflow) + assert.Equal(t, tt.mode, selection.Mode) + if !tt.wantFields { + assert.Empty(t, selection.StorageClassName) + assert.Empty(t, selection.StorageClassUID) + assert.Empty(t, selection.StorageClassDigest) + assert.Empty(t, selection.CatalogDigest) + assert.Empty(t, selection.Provider) + assert.Empty(t, selection.Provisioner) + assert.Empty(t, selection.Transition) + return + } + assert.Equal(t, tt.resolved.StorageClassName, selection.StorageClassName) + assert.Equal(t, tt.resolved.StorageClassUID, selection.StorageClassUID) + assert.Equal(t, tt.resolved.StorageClassDigest, selection.StorageClassDigest) + assert.Equal(t, tt.resolved.CatalogDigest, selection.CatalogDigest) + assert.Equal(t, tt.resolved.Provider, selection.Provider) + assert.Equal(t, tt.resolved.Provisioner, selection.Provisioner) + assert.Equal(t, tt.resolved.Transition, selection.Transition) + assert.Equal(t, tt.resolved.RequiredMountOptions, selection.RequiredMountOptions) + }) + } +} + +func TestPersistedModelCacheStorageSelectionRWXReadOnlyContract(t *testing.T) { + selection := testPersistedModelCacheStorageSelection( + ModelCacheSelectionDurable, ModelCacheTransitionRWXReadOnly) + require.NoError(t, selection.Validate()) + assert.False(t, selection.EncryptionRequired) + assert.Equal(t, + []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + selection.RequiredAccessModes) + assert.Empty(t, selection.RequiredMountOptions) + + raw, err := selection.Marshal() + require.NoError(t, err) + parsed, err := ParsePersistedModelCacheStorageSelection(raw) + require.NoError(t, err) + assert.Equal(t, selection, parsed) + + encrypted := *selection + encrypted.EncryptionRequired = true + require.ErrorContains(t, encrypted.Validate(), "does not support encryption") + + helm := *selection + helm.Workflow = ModelCacheWorkflowHelm + require.ErrorContains(t, helm.Validate(), "requires regular workflow") + + extraMode := *selection + extraMode.RequiredAccessModes = []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteMany, + corev1.ReadOnlyMany, + } + require.ErrorContains(t, extraMode.Validate(), "requires access modes [ReadWriteMany]") + + withReaderOptions := *selection + withReaderOptions.RequiredMountOptions = []string{"ro"} + require.ErrorContains(t, withReaderOptions.Validate(), "does not create a reader PV") +} + +func TestPersistedModelCacheStorageSelectionMarshalParseRoundTrip(t *testing.T) { + want := testPersistedModelCacheStorageSelection(ModelCacheSelectionDurable, ModelCacheTransitionROXReadOnly) + want.Workflow = ModelCacheWorkflowHelm + + raw, err := want.Marshal() + require.NoError(t, err) + assert.True(t, json.Valid([]byte(raw))) + + got, err := ParsePersistedModelCacheStorageSelection(raw + "\n\t") + require.NoError(t, err) + assert.Equal(t, want, got) + + remarshaled, err := got.Marshal() + require.NoError(t, err) + assert.Equal(t, raw, remarshaled, "the persisted annotation must have stable field ordering") +} + +func TestPersistedModelCacheStorageSelectionRejectsProviderTransitionMismatch(t *testing.T) { + selection := testPersistedModelCacheStorageSelection( + ModelCacheSelectionDurable, ModelCacheTransitionROXReadOnly) + selection.Provider = "weka" + + err := selection.Validate() + require.ErrorContains(t, err, "requires provider") + assert.Empty(t, selection.BindingName) +} + +func TestParsePersistedModelCacheStorageSelectionStrict(t *testing.T) { + const minimal = `{"version":"v1alpha1","workflow":"regularModelCache","mode":"none"}` + tests := []struct { + name string + raw string + want string + }{ + {name: "empty", raw: "", want: "EOF"}, + {name: "malformed", raw: "{", want: "unexpected EOF"}, + { + name: "unknown field", + raw: `{"version":"v1alpha1","workflow":"regularModelCache","mode":"none","backend":"nvmesh"}`, + want: `unknown field "backend"`, + }, + {name: "multiple values", raw: minimal + ` {}`, want: "multiple JSON values"}, + {name: "trailing junk", raw: minimal + " x", want: "invalid character"}, + {name: "wrong JSON type", raw: `[]`, want: "cannot unmarshal array"}, + {name: "null", raw: `null`, want: "unsupported model cache storage selection version"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + selection, err := ParsePersistedModelCacheStorageSelection(tt.raw) + require.ErrorContains(t, err, tt.want) + assert.Nil(t, selection) + }) + } +} + +func TestPersistedModelCacheStorageSelectionValidate(t *testing.T) { + validDurable := func() *PersistedModelCacheStorageSelection { + return testPersistedModelCacheStorageSelection(ModelCacheSelectionDurable, ModelCacheTransitionROXReadOnly) + } + validDisabled := func() *PersistedModelCacheStorageSelection { + return testPersistedModelCacheStorageSelection(ModelCacheSelectionNone, ModelCacheTransitionDisabled) + } + tests := []struct { + name string + selection func() *PersistedModelCacheStorageSelection + want string + }{ + {name: "nil", selection: func() *PersistedModelCacheStorageSelection { return nil }, want: "is nil"}, + {name: "unsupported version", selection: func() *PersistedModelCacheStorageSelection { + s := validDurable() + s.Version = "v2" + return s + }, want: "unsupported model cache storage selection version"}, + {name: "empty workflow", selection: func() *PersistedModelCacheStorageSelection { + s := validDurable() + s.Workflow = "" + return s + }, want: "invalid model cache workflow"}, + {name: "unknown workflow", selection: func() *PersistedModelCacheStorageSelection { + s := validDurable() + s.Workflow = "containerCache" + return s + }, want: "invalid model cache workflow"}, + {name: "empty mode", selection: func() *PersistedModelCacheStorageSelection { + s := validDurable() + s.Mode = "" + return s + }, want: "invalid model cache selection mode"}, + {name: "unknown mode", selection: func() *PersistedModelCacheStorageSelection { + s := validDurable() + s.Mode = "shared" + return s + }, want: "invalid model cache selection mode"}, + {name: "regular workflow with ephemeral mode", selection: func() *PersistedModelCacheStorageSelection { + s := validDisabled() + s.Mode = ModelCacheSelectionEphemeral + return s + }, want: "ephemeral model cache selection requires Helm workflow"}, + {name: "durable without resolved storage", selection: func() *PersistedModelCacheStorageSelection { + return &PersistedModelCacheStorageSelection{ + Version: modelCacheStorageSelectionVersion, + Workflow: ModelCacheWorkflowRegular, + Mode: ModelCacheSelectionDurable, + } + }, want: "has no resolved storage"}, + {name: "wrong StorageClass", selection: func() *PersistedModelCacheStorageSelection { + s := validDurable() + s.StorageClassName = "another-class" + return s + }, want: `StorageClass must be "nvcf-sc"`}, + {name: "missing StorageClass UID", selection: func() *PersistedModelCacheStorageSelection { + s := validDurable() + s.StorageClassUID = "" + return s + }, want: "incomplete resolved storage"}, + {name: "missing StorageClass digest", selection: func() *PersistedModelCacheStorageSelection { + s := validDurable() + s.StorageClassDigest = "" + return s + }, want: "incomplete resolved storage"}, + {name: "missing catalog digest", selection: func() *PersistedModelCacheStorageSelection { + s := validDurable() + s.CatalogDigest = "" + return s + }, want: "incomplete resolved storage"}, + {name: "missing provider", selection: func() *PersistedModelCacheStorageSelection { + s := validDurable() + s.Provider = "" + return s + }, want: "incomplete resolved storage"}, + {name: "whitespace provider", selection: func() *PersistedModelCacheStorageSelection { + s := validDurable() + s.Provider = " \t" + return s + }, want: "incomplete resolved storage"}, + {name: "missing provisioner", selection: func() *PersistedModelCacheStorageSelection { + s := validDurable() + s.Provisioner = "" + return s + }, want: "incomplete resolved storage"}, + {name: "NVMesh transition with another provisioner", selection: func() *PersistedModelCacheStorageSelection { + s := validDurable() + s.Provisioner = "another.csi.example.com" + return s + }, want: "requires provisioner"}, + {name: "missing transition", selection: func() *PersistedModelCacheStorageSelection { + s := validDurable() + s.Transition = "" + return s + }, want: "incomplete resolved storage"}, + {name: "durable disabled transition", selection: func() *PersistedModelCacheStorageSelection { + s := validDurable() + s.Transition = ModelCacheTransitionDisabled + return s + }, want: "unsupported transition"}, + {name: "durable unknown transition", selection: func() *PersistedModelCacheStorageSelection { + s := validDurable() + s.Transition = "shared-filesystem" + return s + }, want: "unsupported transition"}, + {name: "NVMesh missing required reader mount option", selection: func() *PersistedModelCacheStorageSelection { + s := validDurable() + s.RequiredMountOptions = []string{"ro", "norecovery"} + return s + }, want: `requires mount option "nouuid"`}, + {name: "duplicate required reader mount option", selection: func() *PersistedModelCacheStorageSelection { + s := validDurable() + s.RequiredMountOptions = append(s.RequiredMountOptions, "ro") + return s + }, want: `duplicate required mount option "ro"`}, + {name: "conflicting required reader mount options", selection: func() *PersistedModelCacheStorageSelection { + s := validDurable() + s.RequiredMountOptions = append(s.RequiredMountOptions, "rw") + return s + }, want: "required mount options \"ro\" and \"rw\" conflict"}, + {name: "blank required reader mount option", selection: func() *PersistedModelCacheStorageSelection { + s := validDurable() + s.RequiredMountOptions = append(s.RequiredMountOptions, " ") + return s + }, want: "invalid required mount option"}, + {name: "non-durable with required mount options", selection: func() *PersistedModelCacheStorageSelection { + s := validDisabled() + s.RequiredMountOptions = []string{"ro"} + return s + }, want: "non-durable model cache selection has required mount options"}, + {name: "none with durable transition", selection: func() *PersistedModelCacheStorageSelection { + s := validDisabled() + s.Transition = ModelCacheTransitionROXReadOnly + return s + }, want: "non-durable model cache selection has transition"}, + {name: "ephemeral with unknown transition", selection: func() *PersistedModelCacheStorageSelection { + s := validDisabled() + s.Workflow = ModelCacheWorkflowHelm + s.Mode = ModelCacheSelectionEphemeral + s.Transition = "shared-filesystem" + return s + }, want: "non-durable model cache selection has transition"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + selection := tt.selection() + require.ErrorContains(t, selection.Validate(), tt.want) + + if selection != nil { + _, err := selection.Marshal() + require.ErrorContains(t, err, tt.want) + } + }) + } +} diff --git a/src/compute-plane-services/nvca/pkg/storage/modelcache_test.go b/src/compute-plane-services/nvca/pkg/storage/modelcache_test.go index f508a00bd..93eb6670c 100644 --- a/src/compute-plane-services/nvca/pkg/storage/modelcache_test.go +++ b/src/compute-plane-services/nvca/pkg/storage/modelcache_test.go @@ -744,6 +744,34 @@ func TestResolveCacheMountOptions_ConfigMapEditTakesEffect(t *testing.T) { } } +func TestResolveCacheMountOptionsWithRequiredIgnoresLegacyConfigMap(t *testing.T) { + ctx := context.Background() + c := fake.NewClientBuilder(). + WithScheme(mgrScheme). + WithObjects(newMountOptionDefaultsObjects(NVMeshStorageClassProvisioner, map[string]string{ + NVMeshStorageClassProvisioner: "rw,recovery,uuid", + })...). + Build() + r := newMountOptionsReconciler(t, c, []string{"rw", "noatime"}) + pv := &corev1.PersistentVolume{ObjectMeta: metav1.ObjectMeta{Name: "secondary-pv-test"}} + required := []string{"ro", "norecovery", "nouuid"} + want := []string{"ro", "norecovery", "nouuid", "noatime"} + + if got := r.resolveCacheMountOptionsWithRequired(ctx, pv, required); !slices.Equal(got, want) { + t.Fatalf("resolved mount options = %v, want %v", got, want) + } + + cm := &corev1.ConfigMap{} + key := client.ObjectKey{Name: DefaultCacheMountOptionsConfigMapName, Namespace: ModelCacheInitNamespace} + require.NoError(t, c.Get(ctx, key, cm)) + cm.Data[NVMeshStorageClassProvisioner] = "ro,norecovery,nouuid,dirsync" + require.NoError(t, c.Update(ctx, cm)) + + if got := r.resolveCacheMountOptionsWithRequired(ctx, pv, required); !slices.Equal(got, want) { + t.Errorf("resolved options changed after legacy ConfigMap edit: got %v, want %v", got, want) + } +} + // TestModelCacheStorageClassResolvedOnce covers the storage class NewReconciler // resolves for the life of the reconciler: the option override first (tests), // then the agent config value, then the default. The config value is the single @@ -997,7 +1025,7 @@ func TestReconcileSecondaryPVMountOptions(t *testing.T) { } rvBefore := stored.ResourceVersion - if err := r.reconcileSecondaryPVMountOptions(context.Background(), pv); err != nil { + if err := r.reconcileSecondaryPVMountOptions(context.Background(), stored); err != nil { t.Fatalf("reconcileSecondaryPVMountOptions() error = %v", err) } @@ -1018,6 +1046,68 @@ func TestReconcileSecondaryPVMountOptions(t *testing.T) { } } +func TestReconcileSecondaryPVMountOptionsRejectsStaleObject(t *testing.T) { + pv := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{Name: "secondary-pv-test"}, + Spec: corev1.PersistentVolumeSpec{ + MountOptions: []string{"ro", "norecovery", "nouuid"}, + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + Driver: NVMeshStorageClassProvisioner, + VolumeHandle: "handle", + }, + }, + }, + } + objects := append(newMountOptionDefaultsObjects( + NVMeshStorageClassProvisioner, nvmeshMountOptionDefaults), pv) + c := fake.NewClientBuilder().WithScheme(mgrScheme).WithObjects(objects...).Build() + r := newMountOptionsReconciler(t, c, []string{"noatime"}) + + stale := &corev1.PersistentVolume{} + require.NoError(t, c.Get(t.Context(), client.ObjectKeyFromObject(pv), stale)) + live := stale.DeepCopy() + live.Labels = map[string]string{"replacement": "true"} + require.NoError(t, c.Update(t.Context(), live)) + + err := r.reconcileSecondaryPVMountOptions(t.Context(), stale) + require.Error(t, err) + assert.True(t, apierrors.IsConflict(err), err) + + got := &corev1.PersistentVolume{} + require.NoError(t, c.Get(t.Context(), client.ObjectKeyFromObject(pv), got)) + assert.Equal(t, []string{"ro", "norecovery", "nouuid"}, got.Spec.MountOptions) + assert.Equal(t, "true", got.Labels["replacement"]) +} + +func TestReconcileSecondaryPVMountOptionsWithRequired(t *testing.T) { + pv := &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{Name: "secondary-pv-test"}, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + Driver: NVMeshStorageClassProvisioner, + VolumeHandle: "handle", + }, + }, + }, + } + objects := append(newMountOptionDefaultsObjects( + NVMeshStorageClassProvisioner, + map[string]string{NVMeshStorageClassProvisioner: "rw,recovery,uuid"}), pv) + c := fake.NewClientBuilder().WithScheme(mgrScheme).WithObjects(objects...).Build() + r := newMountOptionsReconciler(t, c, []string{"rw", "noatime"}) + + stored := &corev1.PersistentVolume{} + require.NoError(t, c.Get(t.Context(), client.ObjectKeyFromObject(pv), stored)) + require.NoError(t, r.reconcileSecondaryPVMountOptionsWithRequired( + t.Context(), stored, []string{"ro", "norecovery", "nouuid"})) + + got := &corev1.PersistentVolume{} + require.NoError(t, c.Get(t.Context(), client.ObjectKeyFromObject(pv), got)) + assert.Equal(t, []string{"ro", "norecovery", "nouuid", "noatime"}, got.Spec.MountOptions) +} + func Test_updateSecondaryPVVolumeHandle(t *testing.T) { namespace := "sr-fd7d88ab-6e18-4442-8a94-344da5f7341e" tests := []struct { diff --git a/src/compute-plane-services/nvca/pkg/storage/reconcile.go b/src/compute-plane-services/nvca/pkg/storage/reconcile.go index 45309cae5..521f2b74a 100644 --- a/src/compute-plane-services/nvca/pkg/storage/reconcile.go +++ b/src/compute-plane-services/nvca/pkg/storage/reconcile.go @@ -304,6 +304,13 @@ func (r *Reconciler) tryRemoveFinalizerInTerminatingNamespace(ctx context.Contex if !controllerutil.ContainsFinalizer(st, StorageRequestFinalizer) { return false, reconcile.Result{}, nil } + // A persisted durable model cache owns objects in the global init + // namespace and cluster-scoped PVs. Namespace termination does not prove + // those resources are gone, so it cannot bypass cleanup. Annotation-free + // legacy requests keep the historical escape hatch. + if requiresStrictModelCacheCleanup(st) { + return false, reconcile.Result{}, nil + } ns := &corev1.Namespace{} if err := r.Client.Get(ctx, client.ObjectKey{Name: namespace}, ns); err != nil || ns.DeletionTimestamp == nil { return false, reconcile.Result{}, nil @@ -323,6 +330,22 @@ func (r *Reconciler) tryRemoveFinalizerInTerminatingNamespace(ctx context.Contex return true, reconcile.Result{}, nil } +func requiresStrictModelCacheCleanup(st *nvcav1new.StorageRequest) bool { + if st == nil || st.Spec.Type != nvcav1new.ModelCacheRequest { + return false + } + raw := st.Annotations[ModelCacheStorageSelectionAnnotationKey] + if raw == "" { + return false + } + selection, err := ParsePersistedModelCacheStorageSelection(raw) + if err != nil { + // An invalid persisted decision must not bypass cleanup protection. + return true + } + return selection.Mode == ModelCacheSelectionDurable +} + // getICMSRequestName returns the ICMS request name from the StorageRequest. // It prefers RequestName from the spec (storage API field); for instance namespaces // (where the namespace name equals the ICMS request name), falls back to st.Namespace. @@ -494,9 +517,12 @@ func (r *Reconciler) doReconcile( res reconcile.Result rerr error ) - if stCopy.DeletionTimestamp == nil { + if stCopy.DeletionTimestamp == nil && !controllerutil.ContainsFinalizer(stCopy, StorageRequestFinalizer) { + // Persist cleanup ownership before any provider resource is created. The + // next reconcile performs provisioning only after observing the finalizer. controllerutil.AddFinalizer(stCopy, StorageRequestFinalizer) - + res = reconcile.Result{Requeue: true} + } else if stCopy.DeletionTimestamp == nil { switch stCopy.Spec.Type { case nvcav1new.ModelCacheRequest: res, rerr = r.doModelCache(ctx, *st, stCopy, icmsReq) diff --git a/src/compute-plane-services/nvca/pkg/storage/reconcile_test.go b/src/compute-plane-services/nvca/pkg/storage/reconcile_test.go index eebd0d814..0ca5b4243 100644 --- a/src/compute-plane-services/nvca/pkg/storage/reconcile_test.go +++ b/src/compute-plane-services/nvca/pkg/storage/reconcile_test.go @@ -243,6 +243,36 @@ func TestRequeueDeletingStorageRequestWithFinalizer(t *testing.T) { } } +func TestRequiresStrictModelCacheCleanupPreservesLegacyEscape(t *testing.T) { + legacy := &nvcav1new.StorageRequest{ + Spec: nvcav1new.StorageRequestSpec{Type: nvcav1new.ModelCacheRequest}, + } + assert.False(t, requiresStrictModelCacheCleanup(legacy)) + + _, _, durable, _ := newHelmBindingTestFixture(t) + assert.True(t, requiresStrictModelCacheCleanup(durable)) + + nonDurable := durable.DeepCopy() + selection, err := ParsePersistedModelCacheStorageSelection( + nonDurable.Annotations[ModelCacheStorageSelectionAnnotationKey]) + require.NoError(t, err) + selection.Mode = ModelCacheSelectionEphemeral + selection.BindingName = "" + selection.BindingUID = "" + selection.EncryptionRequired = false + selection.Transition = ModelCacheTransitionDisabled + selection.RequiredAccessModes = nil + selection.RequiredMountOptions = nil + raw, err := selection.Marshal() + require.NoError(t, err) + nonDurable.Annotations[ModelCacheStorageSelectionAnnotationKey] = raw + assert.False(t, requiresStrictModelCacheCleanup(nonDurable)) + + invalid := durable.DeepCopy() + invalid.Annotations[ModelCacheStorageSelectionAnnotationKey] = "{" + assert.True(t, requiresStrictModelCacheCleanup(invalid)) +} + func TestDoCleanupSharedStorage(t *testing.T) { // Setup shared storage request, and perform a standard cleanup on happy path tests := []struct { diff --git a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go index e86c97688..3c0e175e3 100644 --- a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go +++ b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go @@ -19,10 +19,19 @@ package storage import ( "context" + "crypto/sha256" + "encoding/json" + "errors" "fmt" + "sort" "strings" corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/yaml" ) @@ -43,14 +52,48 @@ const ( // ModelCacheTransitionROXReadOnly populates a writer claim and publishes a // separate ReadOnlyMany reader claim with read-only Pod mounts. ModelCacheTransitionROXReadOnly = "roxReadOnly" - // ModelCacheTransitionRWXReadOnly populates one ReadWriteMany claim and - // serves that same claim through read-only Pod mounts. + // ModelCacheTransitionRWXReadOnly selects a regular model-cache transition + // that populates one ReadWriteMany claim and serves that same claim through + // read-only Pod mounts. ModelCacheTransitionRWXReadOnly = "rwxReadOnly" // ModelCacheProviderNVMesh is the only provider currently allowed to select // the ROX read-only transition. ModelCacheProviderNVMesh = "nvmesh" ) +var ( + // ErrModelCacheStorageClassNotFound means nvcf-sc is absent. Callers may map + // this to a documented non-durable fallback. + ErrModelCacheStorageClassNotFound = errors.New("model cache StorageClass not found") + // ErrModelCacheStorageSelectionDrift marks a deterministic mismatch between + // a persisted selection and its live StorageClass or catalog input. Callers + // can fail the request without treating transient API errors as drift. + ErrModelCacheStorageSelectionDrift = errors.New("model cache storage selection drift") +) + +// ModelCacheWorkflow selects one transition column from the catalog. +type ModelCacheWorkflow string + +const ( + ModelCacheWorkflowRegular ModelCacheWorkflow = "regularModelCache" + ModelCacheWorkflowHelm ModelCacheWorkflow = "helmModelCache" +) + +// ModelCacheStorageSelection is the durable-storage decision derived from the +// live nvcf-sc object and the public capability catalog. It deliberately does +// not infer behavior from a provider name or access mode. +type ModelCacheStorageSelection struct { + StorageClassName string + StorageClassUID types.UID + StorageClassDigest string + CatalogDigest string + Provider string + Provisioner string + Transition string + RequiredAccessModes []corev1.PersistentVolumeAccessMode + RequiredMountOptions []string +} + type storageCapabilityCatalog struct { APIVersion string `json:"apiVersion"` Kind string `json:"kind"` @@ -90,6 +133,36 @@ func loadStorageCapabilityCatalog( namespace, StorageCapabilityConfigMapName, StorageCapabilityConfigMapKey) } + return parseStorageCapabilityCatalog(raw) +} + +func loadStorageCapabilityCatalogSnapshot( + ctx context.Context, + c client.Client, + namespace string, +) (*storageCapabilityCatalog, string, error) { + if namespace == "" { + return nil, "", fmt.Errorf("storage capability ConfigMap namespace is empty") + } + + cm := &corev1.ConfigMap{} + if err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: StorageCapabilityConfigMapName}, cm); err != nil { + return nil, "", fmt.Errorf("get storage capability ConfigMap %s/%s: %w", + namespace, StorageCapabilityConfigMapName, err) + } + raw, ok := cm.Data[StorageCapabilityConfigMapKey] + if !ok || raw == "" { + return nil, "", fmt.Errorf("storage capability ConfigMap %s/%s has no %q data", + namespace, StorageCapabilityConfigMapName, StorageCapabilityConfigMapKey) + } + catalog, err := parseStorageCapabilityCatalog(raw) + if err != nil { + return nil, "", err + } + return catalog, digestCatalogPayload(raw), nil +} + +func parseStorageCapabilityCatalog(raw string) (*storageCapabilityCatalog, error) { catalog := &storageCapabilityCatalog{} if err := yaml.UnmarshalStrict([]byte(raw), catalog); err != nil { return nil, fmt.Errorf("parse storage capability catalog: %w", err) @@ -100,6 +173,323 @@ func loadStorageCapabilityCatalog( return catalog, nil } +// ResolveModelCacheStorage resolves the exact transition for a workflow. A +// missing nvcf-sc is returned as a sentinel so callers that support an +// ephemeral cache can choose that fallback. Invalid or unsafe configuration is +// an error and must not silently select another durable provider. +func ResolveModelCacheStorage( + ctx context.Context, + c client.Client, + catalogNamespace string, + workflow ModelCacheWorkflow, +) (*ModelCacheStorageSelection, error) { + sc := &storagev1.StorageClass{} + if err := c.Get(ctx, client.ObjectKey{Name: DefaultModelCacheStorageClassName}, sc); err != nil { + if apierrors.IsNotFound(err) { + return nil, ErrModelCacheStorageClassNotFound + } + return nil, fmt.Errorf("get model cache StorageClass %q: %w", DefaultModelCacheStorageClassName, err) + } + + catalog, catalogDigest, err := loadStorageCapabilityCatalogSnapshot(ctx, c, catalogNamespace) + if err != nil { + return nil, err + } + return selectModelCacheStorageFromObjects(sc, catalog, catalogDigest, workflow) +} + +// ResolveModelCacheStorageWithClientset provides the same decision to the +// regular container model-cache path, which uses client-go rather than a +// controller-runtime client. +func ResolveModelCacheStorageWithClientset( + ctx context.Context, + k8sClient kubernetes.Interface, + catalogNamespace string, + workflow ModelCacheWorkflow, +) (*ModelCacheStorageSelection, error) { + if catalogNamespace == "" { + return nil, fmt.Errorf("storage capability ConfigMap namespace is empty") + } + + sc, err := k8sClient.StorageV1().StorageClasses().Get( + ctx, DefaultModelCacheStorageClassName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + return nil, ErrModelCacheStorageClassNotFound + } + return nil, fmt.Errorf("get model cache StorageClass %q: %w", DefaultModelCacheStorageClassName, err) + } + cm, err := k8sClient.CoreV1().ConfigMaps(catalogNamespace).Get( + ctx, StorageCapabilityConfigMapName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("get storage capability ConfigMap %s/%s: %w", + catalogNamespace, StorageCapabilityConfigMapName, err) + } + raw, ok := cm.Data[StorageCapabilityConfigMapKey] + if !ok || raw == "" { + return nil, fmt.Errorf("storage capability ConfigMap %s/%s has no %q data", + catalogNamespace, StorageCapabilityConfigMapName, StorageCapabilityConfigMapKey) + } + catalog, err := parseStorageCapabilityCatalog(raw) + if err != nil { + return nil, err + } + return selectModelCacheStorageFromObjects(sc, catalog, digestCatalogPayload(raw), workflow) +} + +func selectModelCacheStorageFromObjects( + sc *storagev1.StorageClass, + catalog *storageCapabilityCatalog, + catalogDigest string, + workflow ModelCacheWorkflow, +) (*ModelCacheStorageSelection, error) { + if sc.ReclaimPolicy == nil || *sc.ReclaimPolicy != corev1.PersistentVolumeReclaimRetain { + return nil, fmt.Errorf("model cache StorageClass %q must use reclaimPolicy Retain", + DefaultModelCacheStorageClassName) + } + if strings.TrimSpace(sc.Provisioner) == "" { + return nil, fmt.Errorf("model cache StorageClass %q has an empty provisioner", + DefaultModelCacheStorageClassName) + } + + driver, ok := catalog.Drivers[sc.Provisioner] + if !ok { + return nil, fmt.Errorf("model cache StorageClass %q uses provisioner %q with no catalog entry", + DefaultModelCacheStorageClassName, sc.Provisioner) + } + + var transition string + switch workflow { + case ModelCacheWorkflowRegular: + transition = driver.Transitions.RegularModelCache + case ModelCacheWorkflowHelm: + transition = driver.Transitions.HelmModelCache + default: + return nil, fmt.Errorf("unknown model cache workflow %q", workflow) + } + var requiredMountOptions []string + if transition == ModelCacheTransitionROXReadOnly { + requiredMountOptions = append([]string(nil), (*driver.ReaderMountOptions)...) + } + + return &ModelCacheStorageSelection{ + StorageClassName: sc.Name, + StorageClassUID: sc.UID, + StorageClassDigest: digestStorageClass(sc), + CatalogDigest: catalogDigest, + Provider: driver.Provider, + Provisioner: sc.Provisioner, + Transition: transition, + RequiredAccessModes: requiredAccessModesForTransition(transition), + RequiredMountOptions: requiredMountOptions, + }, nil +} + +func requiredAccessModesForTransition(transition string) []corev1.PersistentVolumeAccessMode { + switch transition { + case ModelCacheTransitionROXReadOnly: + return []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce, corev1.ReadOnlyMany} + case ModelCacheTransitionRWXReadOnly: + return []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + default: + return nil + } +} + +type canonicalStorageClassParameter struct { + Name string `json:"name"` + Value string `json:"value"` +} + +type canonicalStorageClass struct { + Provisioner string `json:"provisioner"` + Parameters []canonicalStorageClassParameter `json:"parameters"` + ReclaimPolicy string `json:"reclaimPolicy"` + VolumeBindingMode string `json:"volumeBindingMode"` + MountOptions []string `json:"mountOptions"` + AllowedTopologies []corev1.TopologySelectorTerm `json:"allowedTopologies"` +} + +func digestStorageClass(sc *storagev1.StorageClass) string { + parameterNames := make([]string, 0, len(sc.Parameters)) + for name := range sc.Parameters { + parameterNames = append(parameterNames, name) + } + sort.Strings(parameterNames) + parameters := make([]canonicalStorageClassParameter, 0, len(parameterNames)) + for _, name := range parameterNames { + parameters = append(parameters, canonicalStorageClassParameter{Name: name, Value: sc.Parameters[name]}) + } + + reclaimPolicy := "" + if sc.ReclaimPolicy != nil { + reclaimPolicy = string(*sc.ReclaimPolicy) + } + volumeBindingMode := "" + if sc.VolumeBindingMode != nil { + volumeBindingMode = string(*sc.VolumeBindingMode) + } + canonical := canonicalStorageClass{ + Provisioner: sc.Provisioner, + Parameters: parameters, + ReclaimPolicy: reclaimPolicy, + VolumeBindingMode: volumeBindingMode, + MountOptions: append([]string{}, sc.MountOptions...), + AllowedTopologies: append([]corev1.TopologySelectorTerm{}, sc.AllowedTopologies...), + } + raw, err := json.Marshal(canonical) + if err != nil { + panic(fmt.Sprintf("marshal canonical StorageClass: %v", err)) + } + sum := sha256.Sum256(raw) + return fmt.Sprintf("v1:sha256:%x", sum) +} + +func digestCatalogPayload(raw string) string { + sum := sha256.Sum256([]byte(raw)) + return fmt.Sprintf("sha256:%x", sum) +} + +// ValidateModelCacheStorageSelectionLive verifies that a persisted durable +// decision still points at the same immutable StorageClass before its first +// storage side effect. It does not reselect from the current catalog. +func ValidateModelCacheStorageSelectionLive( + ctx context.Context, + c client.Client, + selection *PersistedModelCacheStorageSelection, +) error { + if err := selection.Validate(); err != nil { + return err + } + if selection.Mode != ModelCacheSelectionDurable { + return nil + } + + sc := &storagev1.StorageClass{} + if err := c.Get(ctx, client.ObjectKey{Name: selection.StorageClassName}, sc); err != nil { + return fmt.Errorf("get selected model cache StorageClass %q: %w", selection.StorageClassName, err) + } + return validateSelectedStorageClass(sc, selection) +} + +// ValidateModelCacheStorageSelectionLiveWithClientset is the client-go +// equivalent used by the regular container model-cache path. +func ValidateModelCacheStorageSelectionLiveWithClientset( + ctx context.Context, + k8sClient kubernetes.Interface, + selection *PersistedModelCacheStorageSelection, +) error { + if err := selection.Validate(); err != nil { + return err + } + if selection.Mode != ModelCacheSelectionDurable { + return nil + } + sc, err := k8sClient.StorageV1().StorageClasses().Get( + ctx, selection.StorageClassName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("get selected model cache StorageClass %q: %w", selection.StorageClassName, err) + } + return validateSelectedStorageClass(sc, selection) +} + +// ValidateModelCacheStorageSelectionInputsWithClientset verifies every live +// input captured by a durable selection immediately before its first binding +// is created. Once the binding exists, callers use the immutable binding and +// must not reselect from a later catalog revision. +func ValidateModelCacheStorageSelectionInputsWithClientset( + ctx context.Context, + k8sClient kubernetes.Interface, + catalogNamespace string, + selection *PersistedModelCacheStorageSelection, +) error { + if err := ValidateModelCacheStorageSelectionLiveWithClientset(ctx, k8sClient, selection); err != nil { + return err + } + if selection.Mode != ModelCacheSelectionDurable { + return nil + } + if strings.TrimSpace(catalogNamespace) == "" { + return fmt.Errorf("%w: storage capability ConfigMap namespace is empty", + ErrModelCacheStorageSelectionDrift) + } + + cm, err := k8sClient.CoreV1().ConfigMaps(catalogNamespace).Get( + ctx, StorageCapabilityConfigMapName, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("get storage capability ConfigMap %s/%s before binding creation: %w", + catalogNamespace, StorageCapabilityConfigMapName, err) + } + raw, ok := cm.Data[StorageCapabilityConfigMapKey] + if !ok || raw == "" { + return fmt.Errorf("%w: storage capability ConfigMap %s/%s has no %q data", + ErrModelCacheStorageSelectionDrift, + catalogNamespace, StorageCapabilityConfigMapName, StorageCapabilityConfigMapKey) + } + if digest := digestCatalogPayload(raw); digest != selection.CatalogDigest { + return fmt.Errorf("%w: storage capability catalog digest changed from %q to %q", + ErrModelCacheStorageSelectionDrift, selection.CatalogDigest, digest) + } + catalog, err := parseStorageCapabilityCatalog(raw) + if err != nil { + return fmt.Errorf("%w: selected storage capability catalog is invalid: %v", + ErrModelCacheStorageSelectionDrift, err) + } + driver, ok := catalog.Drivers[selection.Provisioner] + if !ok { + return fmt.Errorf("%w: selected provisioner %q has no catalog entry", + ErrModelCacheStorageSelectionDrift, selection.Provisioner) + } + if driver.Provider != selection.Provider { + return fmt.Errorf("%w: selected provisioner %q provider changed from %q to %q", + ErrModelCacheStorageSelectionDrift, + selection.Provisioner, selection.Provider, driver.Provider) + } + var transition string + switch selection.Workflow { + case ModelCacheWorkflowRegular: + transition = driver.Transitions.RegularModelCache + case ModelCacheWorkflowHelm: + transition = driver.Transitions.HelmModelCache + default: + return fmt.Errorf("%w: unknown model cache workflow %q", + ErrModelCacheStorageSelectionDrift, selection.Workflow) + } + if transition != selection.Transition { + return fmt.Errorf("%w: selected provisioner %q transition for %s changed from %q to %q", + ErrModelCacheStorageSelectionDrift, + selection.Provisioner, selection.Workflow, selection.Transition, transition) + } + return nil +} + +func validateSelectedStorageClass( + sc *storagev1.StorageClass, + selection *PersistedModelCacheStorageSelection, +) error { + if sc.UID != selection.StorageClassUID { + return fmt.Errorf("%w: selected model cache StorageClass %q UID changed from %q to %q", + ErrModelCacheStorageSelectionDrift, + selection.StorageClassName, selection.StorageClassUID, sc.UID) + } + if sc.Provisioner != selection.Provisioner { + return fmt.Errorf("%w: selected model cache StorageClass %q provisioner changed from %q to %q", + ErrModelCacheStorageSelectionDrift, + selection.StorageClassName, selection.Provisioner, sc.Provisioner) + } + if sc.ReclaimPolicy == nil || *sc.ReclaimPolicy != corev1.PersistentVolumeReclaimRetain { + return fmt.Errorf("%w: selected model cache StorageClass %q no longer uses reclaimPolicy Retain", + ErrModelCacheStorageSelectionDrift, + selection.StorageClassName) + } + if digest := digestStorageClass(sc); digest != selection.StorageClassDigest { + return fmt.Errorf("%w: selected model cache StorageClass %q configuration digest changed from %q to %q", + ErrModelCacheStorageSelectionDrift, + selection.StorageClassName, selection.StorageClassDigest, digest) + } + return nil +} + func validateStorageCapabilityCatalog(catalog *storageCapabilityCatalog) error { if catalog.APIVersion != storageCapabilityCatalogAPIVersion { return fmt.Errorf("unsupported storage capability apiVersion %q", catalog.APIVersion) @@ -184,9 +574,9 @@ func validateStorageCapabilityCatalog(catalog *storageCapabilityCatalog) error { } } case ModelCacheTransitionRWXReadOnly: - if workflow != "regularModelCache" { - return fmt.Errorf("driver %q transition %s strategy %s is only supported for regularModelCache", - provisioner, workflow, strategy) + if workflow != string(ModelCacheWorkflowRegular) { + return fmt.Errorf("driver %q transition %s strategy %s is only supported for %s", + provisioner, workflow, strategy, ModelCacheWorkflowRegular) } if !accessModes[string(corev1.ReadWriteMany)] { return fmt.Errorf("driver %q transition %s strategy %s requires ReadWriteMany access mode", diff --git a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go index ad9471871..42d61de5a 100644 --- a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go +++ b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go @@ -74,7 +74,7 @@ func validStorageCapabilityCatalog() *storageCapabilityCatalog { Kind: storageCapabilityCatalogKind, Drivers: map[string]storageDriverSpec{ NVMeshStorageClassProvisioner: { - Provider: ModelCacheProviderNVMesh, + Provider: "nvmesh", AccessModes: accessModes("ReadWriteOnce", "ReadOnlyMany"), ReaderMountOptions: readerMountOptions("ro", "norecovery", "nouuid"), Transitions: storageTransitions{ @@ -271,17 +271,17 @@ func TestValidateStorageCapabilityCatalog(t *testing.T) { d.ReaderMountOptions = readerMountOptions("ro", "norecovery", "uuid", "nouuid") c.Drivers[NVMeshStorageClassProvisioner] = d }, want: `readerMountOptions "uuid" and "nouuid" conflict`}, - {name: "ROX transition lacks ro reader mount option", mutate: func(c *storageCapabilityCatalog) { + {name: "NVMesh transition lacks ro reader mount option", mutate: func(c *storageCapabilityCatalog) { d := c.Drivers[NVMeshStorageClassProvisioner] d.ReaderMountOptions = readerMountOptions("norecovery", "nouuid") c.Drivers[NVMeshStorageClassProvisioner] = d }, want: `requires readerMountOption "ro"`}, - {name: "ROX transition lacks norecovery reader mount option", mutate: func(c *storageCapabilityCatalog) { + {name: "NVMesh transition lacks norecovery reader mount option", mutate: func(c *storageCapabilityCatalog) { d := c.Drivers[NVMeshStorageClassProvisioner] d.ReaderMountOptions = readerMountOptions("ro", "nouuid") c.Drivers[NVMeshStorageClassProvisioner] = d }, want: `requires readerMountOption "norecovery"`}, - {name: "ROX transition lacks nouuid reader mount option", mutate: func(c *storageCapabilityCatalog) { + {name: "NVMesh transition lacks nouuid reader mount option", mutate: func(c *storageCapabilityCatalog) { d := c.Drivers[NVMeshStorageClassProvisioner] d.ReaderMountOptions = readerMountOptions("ro", "norecovery") c.Drivers[NVMeshStorageClassProvisioner] = d @@ -296,33 +296,33 @@ func TestValidateStorageCapabilityCatalog(t *testing.T) { d.Transitions.HelmModelCache = "samba" c.Drivers[NVMeshStorageClassProvisioner] = d }, want: "invalid strategy"}, - {name: "ROX transition is provisioner-specific", mutate: func(c *storageCapabilityCatalog) { + {name: "NVMesh transition is provisioner-specific", mutate: func(c *storageCapabilityCatalog) { d := c.Drivers[NVMeshStorageClassProvisioner] delete(c.Drivers, NVMeshStorageClassProvisioner) c.Drivers["example.csi.test"] = d }, want: "restricted to provisioner"}, - {name: "ROX transition is provider-specific", mutate: func(c *storageCapabilityCatalog) { + {name: "NVMesh transition is provider-specific", mutate: func(c *storageCapabilityCatalog) { d := c.Drivers[NVMeshStorageClassProvisioner] d.Provider = "weka" c.Drivers[NVMeshStorageClassProvisioner] = d }, want: "requires provider"}, - {name: "ROX transition lacks ReadWriteOnce", mutate: func(c *storageCapabilityCatalog) { + {name: "NVMesh transition lacks ReadWriteOnce", mutate: func(c *storageCapabilityCatalog) { d := c.Drivers[NVMeshStorageClassProvisioner] d.AccessModes = accessModes("ReadOnlyMany") c.Drivers[NVMeshStorageClassProvisioner] = d }, want: "requires ReadWriteOnce and ReadOnlyMany"}, - {name: "ROX transition lacks ReadOnlyMany", mutate: func(c *storageCapabilityCatalog) { + {name: "NVMesh transition lacks ReadOnlyMany", mutate: func(c *storageCapabilityCatalog) { d := c.Drivers[NVMeshStorageClassProvisioner] d.AccessModes = accessModes("ReadWriteOnce") c.Drivers[NVMeshStorageClassProvisioner] = d }, want: "requires ReadWriteOnce and ReadOnlyMany"}, - {name: "RWX transition is rejected for Helm", mutate: func(c *storageCapabilityCatalog) { + {name: "rwxReadOnly transition is rejected for Helm", mutate: func(c *storageCapabilityCatalog) { d := c.Drivers[NVMeshStorageClassProvisioner] d.AccessModes = accessModes("ReadWriteOnce", "ReadOnlyMany", "ReadWriteMany") d.Transitions.HelmModelCache = ModelCacheTransitionRWXReadOnly c.Drivers[NVMeshStorageClassProvisioner] = d }, want: "only supported for regularModelCache"}, - {name: "RWX transition lacks ReadWriteMany", mutate: func(c *storageCapabilityCatalog) { + {name: "rwxReadOnly transition lacks ReadWriteMany", mutate: func(c *storageCapabilityCatalog) { d := c.Drivers[NVMeshStorageClassProvisioner] d.Transitions.RegularModelCache = ModelCacheTransitionRWXReadOnly d.Transitions.HelmModelCache = ModelCacheTransitionDisabled @@ -353,6 +353,20 @@ func TestValidateStorageCapabilityCatalogAllowsRegularRWXReadOnly(t *testing.T) } require.NoError(t, validateStorageCapabilityCatalog(catalog)) + assert.Equal(t, + []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + requiredAccessModesForTransition(ModelCacheTransitionRWXReadOnly)) + + sc := testModelCacheStorageClass() + sc.Provisioner = provisioner + selection, err := selectModelCacheStorageFromObjects( + sc, catalog, "sha256:catalog", ModelCacheWorkflowRegular) + require.NoError(t, err) + assert.Equal(t, ModelCacheTransitionRWXReadOnly, selection.Transition) + assert.Equal(t, + []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + selection.RequiredAccessModes) + assert.Empty(t, selection.RequiredMountOptions) driver := catalog.Drivers[provisioner] driver.ReaderMountOptions = readerMountOptions("ro") @@ -367,8 +381,8 @@ func TestValidateStorageCapabilityCatalogAllowsDisabledTransitionsWithEmptyModes driver.AccessModes = accessModes() driver.ReaderMountOptions = readerMountOptions() driver.Transitions = storageTransitions{ - RegularModelCache: ModelCacheTransitionDisabled, - HelmModelCache: ModelCacheTransitionDisabled, + RegularModelCache: "disabled", + HelmModelCache: "disabled", } catalog.Drivers[NVMeshStorageClassProvisioner] = driver diff --git a/src/compute-plane-services/nvca/pkg/storage/storage_resolution_test.go b/src/compute-plane-services/nvca/pkg/storage/storage_resolution_test.go new file mode 100644 index 000000000..566372a01 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/storage/storage_resolution_test.go @@ -0,0 +1,594 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package storage + +import ( + "crypto/sha256" + "errors" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + kubernetesfake "k8s.io/client-go/kubernetes/fake" + "sigs.k8s.io/controller-runtime/pkg/client" + controllerfake "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +const splitTransitionCatalog = `apiVersion: storage.nvcf.nvidia.com/v1alpha1 +kind: StorageCapabilityCatalog +drivers: + nvmesh-csi.excelero.com: + provider: nvmesh + accessModes: [ReadWriteOnce, ReadOnlyMany] + readerMountOptions: [ro, norecovery, nouuid] + transitions: + regularModelCache: roxReadOnly + helmModelCache: disabled +` + +func testModelCacheStorageClass() *storagev1.StorageClass { + retain := corev1.PersistentVolumeReclaimRetain + wait := storagev1.VolumeBindingWaitForFirstConsumer + expand := true + return &storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: DefaultModelCacheStorageClassName, + UID: types.UID("nvcf-sc-uid"), + Labels: map[string]string{"test-metadata": "excluded-from-digest"}, + Annotations: map[string]string{"test-metadata": "excluded-from-digest"}, + }, + Provisioner: NVMeshStorageClassProvisioner, + Parameters: map[string]string{ + "z-option": "last", + "a-option": "first", + }, + ReclaimPolicy: &retain, + MountOptions: []string{"nouuid", "noatime"}, + AllowVolumeExpansion: &expand, + VolumeBindingMode: &wait, + AllowedTopologies: []corev1.TopologySelectorTerm{{ + MatchLabelExpressions: []corev1.TopologySelectorLabelRequirement{{ + Key: "topology.kubernetes.io/zone", + Values: []string{"zone-b", "zone-a"}, + }}, + }}, + } +} + +func storageResolutionClient(t *testing.T, objects ...client.Object) client.Client { + t.Helper() + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + require.NoError(t, storagev1.AddToScheme(scheme)) + return controllerfake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).Build() +} + +func TestResolveModelCacheStorageSelectsWorkflowTransition(t *testing.T) { + sc := testModelCacheStorageClass() + cm := capabilityCatalogConfigMap(splitTransitionCatalog) + c := storageResolutionClient(t, sc, cm) + sum := sha256.Sum256([]byte(splitTransitionCatalog)) + wantCatalogDigest := fmt.Sprintf("sha256:%x", sum) + + tests := []struct { + name string + workflow ModelCacheWorkflow + wantTransition string + }{ + {name: "regular NVMesh transition", workflow: ModelCacheWorkflowRegular, wantTransition: ModelCacheTransitionROXReadOnly}, + {name: "disabled Helm transition", workflow: ModelCacheWorkflowHelm, wantTransition: ModelCacheTransitionDisabled}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + selection, err := ResolveModelCacheStorage(t.Context(), c, testCatalogNamespace, tt.workflow) + require.NoError(t, err) + assert.Equal(t, DefaultModelCacheStorageClassName, selection.StorageClassName) + assert.Equal(t, sc.UID, selection.StorageClassUID) + assert.Equal(t, digestStorageClass(sc), selection.StorageClassDigest) + assert.Equal(t, wantCatalogDigest, selection.CatalogDigest) + assert.Equal(t, "nvmesh", selection.Provider) + assert.Equal(t, NVMeshStorageClassProvisioner, selection.Provisioner) + assert.Equal(t, tt.wantTransition, selection.Transition) + if tt.wantTransition == ModelCacheTransitionROXReadOnly { + assert.Equal(t, []string{"ro", "norecovery", "nouuid"}, selection.RequiredMountOptions) + } else { + assert.Empty(t, selection.RequiredMountOptions) + } + }) + } +} + +func TestResolveModelCacheStorageErrors(t *testing.T) { + deletePolicy := corev1.PersistentVolumeReclaimDelete + tests := []struct { + name string + namespace string + workflow ModelCacheWorkflow + objects func() []client.Object + want string + notFound bool + }{ + { + name: "missing exact StorageClass", + namespace: testCatalogNamespace, + workflow: ModelCacheWorkflowRegular, + objects: func() []client.Object { + return []client.Object{capabilityCatalogConfigMap(validCatalog)} + }, + want: "model cache StorageClass not found", + notFound: true, + }, + { + name: "different StorageClass name is not selected", + namespace: testCatalogNamespace, + workflow: ModelCacheWorkflowRegular, + objects: func() []client.Object { + sc := testModelCacheStorageClass() + sc.Name = "another-class" + return []client.Object{sc, capabilityCatalogConfigMap(validCatalog)} + }, + want: "model cache StorageClass not found", + notFound: true, + }, + { + name: "missing catalog ConfigMap", + namespace: testCatalogNamespace, + workflow: ModelCacheWorkflowRegular, + objects: func() []client.Object { return []client.Object{testModelCacheStorageClass()} }, + want: "get storage capability ConfigMap", + }, + { + name: "catalog ConfigMap missing data key", + namespace: testCatalogNamespace, + workflow: ModelCacheWorkflowRegular, + objects: func() []client.Object { + return []client.Object{ + testModelCacheStorageClass(), + capabilityCatalogConfigMap(""), + } + }, + want: "has no", + }, + { + name: "empty catalog namespace", + namespace: "", + workflow: ModelCacheWorkflowRegular, + objects: func() []client.Object { + return []client.Object{testModelCacheStorageClass(), capabilityCatalogConfigMap(validCatalog)} + }, + want: "namespace is empty", + }, + { + name: "Delete reclaim policy", + namespace: testCatalogNamespace, + workflow: ModelCacheWorkflowRegular, + objects: func() []client.Object { + sc := testModelCacheStorageClass() + sc.ReclaimPolicy = &deletePolicy + return []client.Object{sc, capabilityCatalogConfigMap(validCatalog)} + }, + want: "must use reclaimPolicy Retain", + }, + { + name: "missing reclaim policy", + namespace: testCatalogNamespace, + workflow: ModelCacheWorkflowRegular, + objects: func() []client.Object { + sc := testModelCacheStorageClass() + sc.ReclaimPolicy = nil + return []client.Object{sc, capabilityCatalogConfigMap(validCatalog)} + }, + want: "must use reclaimPolicy Retain", + }, + { + name: "empty provisioner", + namespace: testCatalogNamespace, + workflow: ModelCacheWorkflowRegular, + objects: func() []client.Object { + sc := testModelCacheStorageClass() + sc.Provisioner = " \t" + return []client.Object{sc, capabilityCatalogConfigMap(validCatalog)} + }, + want: "has an empty provisioner", + }, + { + name: "unknown provisioner", + namespace: testCatalogNamespace, + workflow: ModelCacheWorkflowRegular, + objects: func() []client.Object { + sc := testModelCacheStorageClass() + sc.Provisioner = "unknown.csi.example.com" + return []client.Object{sc, capabilityCatalogConfigMap(validCatalog)} + }, + want: "with no catalog entry", + }, + { + name: "unknown workflow", + namespace: testCatalogNamespace, + workflow: "containerCache", + objects: func() []client.Object { + return []client.Object{testModelCacheStorageClass(), capabilityCatalogConfigMap(validCatalog)} + }, + want: "unknown model cache workflow", + }, + { + name: "malformed catalog", + namespace: testCatalogNamespace, + workflow: ModelCacheWorkflowRegular, + objects: func() []client.Object { + return []client.Object{testModelCacheStorageClass(), capabilityCatalogConfigMap("drivers: [")} + }, + want: "parse storage capability catalog", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := storageResolutionClient(t, tt.objects()...) + selection, err := ResolveModelCacheStorage(t.Context(), c, tt.namespace, tt.workflow) + require.ErrorContains(t, err, tt.want) + assert.Nil(t, selection) + assert.Equal(t, tt.notFound, errors.Is(err, ErrModelCacheStorageClassNotFound)) + }) + } +} + +func TestResolveModelCacheStorageWithClientset(t *testing.T) { + sc := testModelCacheStorageClass() + cm := capabilityCatalogConfigMap(splitTransitionCatalog) + k8sClient := kubernetesfake.NewSimpleClientset(sc, cm) + + for _, tt := range []struct { + workflow ModelCacheWorkflow + wantTransition string + }{ + {workflow: ModelCacheWorkflowRegular, wantTransition: ModelCacheTransitionROXReadOnly}, + {workflow: ModelCacheWorkflowHelm, wantTransition: ModelCacheTransitionDisabled}, + } { + selection, err := ResolveModelCacheStorageWithClientset( + t.Context(), k8sClient, testCatalogNamespace, tt.workflow) + require.NoError(t, err) + assert.Equal(t, sc.UID, selection.StorageClassUID) + assert.Equal(t, digestStorageClass(sc), selection.StorageClassDigest) + assert.Equal(t, digestCatalogPayload(splitTransitionCatalog), selection.CatalogDigest) + assert.Equal(t, tt.wantTransition, selection.Transition) + } +} + +func TestResolveModelCacheStorageWithClientsetErrors(t *testing.T) { + deletePolicy := corev1.PersistentVolumeReclaimDelete + tests := []struct { + name string + namespace string + workflow ModelCacheWorkflow + objects func() []runtime.Object + want string + notFound bool + }{ + { + name: "missing exact StorageClass", + namespace: testCatalogNamespace, + workflow: ModelCacheWorkflowRegular, + objects: func() []runtime.Object { + return []runtime.Object{capabilityCatalogConfigMap(validCatalog)} + }, + want: "model cache StorageClass not found", + notFound: true, + }, + { + name: "different StorageClass name is not selected", + namespace: testCatalogNamespace, + workflow: ModelCacheWorkflowRegular, + objects: func() []runtime.Object { + sc := testModelCacheStorageClass() + sc.Name = "another-class" + return []runtime.Object{sc, capabilityCatalogConfigMap(validCatalog)} + }, + want: "model cache StorageClass not found", + notFound: true, + }, + { + name: "missing catalog ConfigMap", + namespace: testCatalogNamespace, + workflow: ModelCacheWorkflowRegular, + objects: func() []runtime.Object { return []runtime.Object{testModelCacheStorageClass()} }, + want: "get storage capability ConfigMap", + }, + { + name: "empty catalog namespace", + namespace: "", + workflow: ModelCacheWorkflowRegular, + objects: func() []runtime.Object { + return []runtime.Object{testModelCacheStorageClass(), capabilityCatalogConfigMap(validCatalog)} + }, + want: "namespace is empty", + }, + { + name: "Delete reclaim policy", + namespace: testCatalogNamespace, + workflow: ModelCacheWorkflowRegular, + objects: func() []runtime.Object { + sc := testModelCacheStorageClass() + sc.ReclaimPolicy = &deletePolicy + return []runtime.Object{sc, capabilityCatalogConfigMap(validCatalog)} + }, + want: "must use reclaimPolicy Retain", + }, + { + name: "unknown provisioner", + namespace: testCatalogNamespace, + workflow: ModelCacheWorkflowRegular, + objects: func() []runtime.Object { + sc := testModelCacheStorageClass() + sc.Provisioner = "unknown.csi.example.com" + return []runtime.Object{sc, capabilityCatalogConfigMap(validCatalog)} + }, + want: "with no catalog entry", + }, + { + name: "unknown workflow", + namespace: testCatalogNamespace, + workflow: "containerCache", + objects: func() []runtime.Object { + return []runtime.Object{testModelCacheStorageClass(), capabilityCatalogConfigMap(validCatalog)} + }, + want: "unknown model cache workflow", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + k8sClient := kubernetesfake.NewSimpleClientset(tt.objects()...) + selection, err := ResolveModelCacheStorageWithClientset( + t.Context(), k8sClient, tt.namespace, tt.workflow) + require.ErrorContains(t, err, tt.want) + assert.Nil(t, selection) + assert.Equal(t, tt.notFound, errors.Is(err, ErrModelCacheStorageClassNotFound)) + }) + } +} + +func TestStorageSnapshotDigests(t *testing.T) { + base := testModelCacheStorageClass() + baseDigest := digestStorageClass(base) + assert.True(t, strings.HasPrefix(baseDigest, "v1:sha256:")) + assert.Len(t, strings.TrimPrefix(baseDigest, "v1:sha256:"), sha256.Size*2) + + parameterOrder := base.DeepCopy() + parameterOrder.Parameters = map[string]string{ + "a-option": "first", + "z-option": "last", + } + assert.Equal(t, baseDigest, digestStorageClass(parameterOrder), "parameter map order must not affect the digest") + + metadataAndExpansion := base.DeepCopy() + metadataAndExpansion.ResourceVersion = "new-resource-version" + metadataAndExpansion.Labels = map[string]string{"changed": "metadata"} + metadataAndExpansion.Annotations = map[string]string{"changed": "metadata"} + expand := false + metadataAndExpansion.AllowVolumeExpansion = &expand + assert.Equal(t, baseDigest, digestStorageClass(metadataAndExpansion)) + + deletePolicy := corev1.PersistentVolumeReclaimDelete + immediate := storagev1.VolumeBindingImmediate + mutations := []struct { + name string + mutate func(*storagev1.StorageClass) + }{ + {name: "provisioner", mutate: func(sc *storagev1.StorageClass) { sc.Provisioner = "other.csi.example.com" }}, + {name: "parameter value", mutate: func(sc *storagev1.StorageClass) { sc.Parameters["a-option"] = "changed" }}, + {name: "parameter name", mutate: func(sc *storagev1.StorageClass) { + delete(sc.Parameters, "a-option") + sc.Parameters["b-option"] = "first" + }}, + {name: "reclaim policy", mutate: func(sc *storagev1.StorageClass) { sc.ReclaimPolicy = &deletePolicy }}, + {name: "binding mode", mutate: func(sc *storagev1.StorageClass) { sc.VolumeBindingMode = &immediate }}, + {name: "mount option", mutate: func(sc *storagev1.StorageClass) { sc.MountOptions[0] = "ro" }}, + {name: "mount option order", mutate: func(sc *storagev1.StorageClass) { + sc.MountOptions[0], sc.MountOptions[1] = sc.MountOptions[1], sc.MountOptions[0] + }}, + {name: "topology key", mutate: func(sc *storagev1.StorageClass) { + sc.AllowedTopologies[0].MatchLabelExpressions[0].Key = "topology.example.com/rack" + }}, + {name: "topology value order", mutate: func(sc *storagev1.StorageClass) { + values := sc.AllowedTopologies[0].MatchLabelExpressions[0].Values + values[0], values[1] = values[1], values[0] + }}, + } + for _, tt := range mutations { + t.Run(tt.name, func(t *testing.T) { + changed := base.DeepCopy() + tt.mutate(changed) + assert.NotEqual(t, baseDigest, digestStorageClass(changed)) + }) + } + + catalogDigest := digestCatalogPayload(validCatalog) + assert.True(t, strings.HasPrefix(catalogDigest, "sha256:")) + assert.NotEqual(t, catalogDigest, digestCatalogPayload(validCatalog+"\n"), + "the catalog digest must cover the exact payload") +} + +func durableSelectionForStorageClass(t *testing.T, sc *storagev1.StorageClass) *PersistedModelCacheStorageSelection { + t.Helper() + selection, err := NewPersistedModelCacheStorageSelection( + ModelCacheWorkflowRegular, + ModelCacheSelectionDurable, + &ModelCacheStorageSelection{ + StorageClassName: sc.Name, + StorageClassUID: sc.UID, + StorageClassDigest: digestStorageClass(sc), + CatalogDigest: digestCatalogPayload(validCatalog), + Provider: "nvmesh", + Provisioner: sc.Provisioner, + Transition: ModelCacheTransitionROXReadOnly, + RequiredAccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + corev1.ReadOnlyMany, + }, + RequiredMountOptions: []string{"ro", "norecovery", "nouuid"}, + }, + ) + require.NoError(t, err) + return selection +} + +func TestValidateModelCacheStorageSelectionLive(t *testing.T) { + base := testModelCacheStorageClass() + selection := durableSelectionForStorageClass(t, base) + deletePolicy := corev1.PersistentVolumeReclaimDelete + tests := []struct { + name string + object func() *storagev1.StorageClass + want string + }{ + {name: "unchanged", object: func() *storagev1.StorageClass { return base.DeepCopy() }}, + {name: "metadata and expansion changes are ignored", object: func() *storagev1.StorageClass { + sc := base.DeepCopy() + sc.Labels = map[string]string{"new": "metadata"} + expand := false + sc.AllowVolumeExpansion = &expand + return sc + }}, + {name: "UID replacement", object: func() *storagev1.StorageClass { + sc := base.DeepCopy() + sc.UID = "replacement-uid" + return sc + }, want: "UID changed"}, + {name: "provisioner replacement", object: func() *storagev1.StorageClass { + sc := base.DeepCopy() + sc.Provisioner = "other.csi.example.com" + return sc + }, want: "provisioner changed"}, + {name: "reclaim policy drift", object: func() *storagev1.StorageClass { + sc := base.DeepCopy() + sc.ReclaimPolicy = &deletePolicy + return sc + }, want: "no longer uses reclaimPolicy Retain"}, + {name: "configuration drift", object: func() *storagev1.StorageClass { + sc := base.DeepCopy() + sc.Parameters["a-option"] = "changed" + return sc + }, want: "configuration digest changed"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := storageResolutionClient(t, tt.object()) + err := ValidateModelCacheStorageSelectionLive(t.Context(), c, selection) + if tt.want == "" { + require.NoError(t, err) + return + } + require.ErrorContains(t, err, tt.want) + }) + } + + t.Run("missing selected StorageClass", func(t *testing.T) { + c := storageResolutionClient(t) + err := ValidateModelCacheStorageSelectionLive(t.Context(), c, selection) + require.ErrorContains(t, err, "get selected model cache StorageClass") + }) + + t.Run("invalid selection fails before API access", func(t *testing.T) { + invalid := *selection + invalid.Version = "v2" + err := ValidateModelCacheStorageSelectionLive(t.Context(), nil, &invalid) + require.ErrorContains(t, err, "unsupported model cache storage selection version") + }) +} + +func TestValidateModelCacheStorageSelectionLiveWithClientset(t *testing.T) { + base := testModelCacheStorageClass() + selection := durableSelectionForStorageClass(t, base) + + k8sClient := kubernetesfake.NewSimpleClientset(base.DeepCopy()) + require.NoError(t, ValidateModelCacheStorageSelectionLiveWithClientset(t.Context(), k8sClient, selection)) + + drifted := base.DeepCopy() + drifted.MountOptions = append(drifted.MountOptions, "ro") + k8sClient = kubernetesfake.NewSimpleClientset(drifted) + err := ValidateModelCacheStorageSelectionLiveWithClientset(t.Context(), k8sClient, selection) + require.ErrorContains(t, err, "configuration digest changed") + + k8sClient = kubernetesfake.NewSimpleClientset() + err = ValidateModelCacheStorageSelectionLiveWithClientset(t.Context(), k8sClient, selection) + require.ErrorContains(t, err, "get selected model cache StorageClass") +} + +func TestValidateModelCacheStorageSelectionInputsWithClientset(t *testing.T) { + base := testModelCacheStorageClass() + selection := durableSelectionForStorageClass(t, base) + + t.Run("exact inputs", func(t *testing.T) { + k8sClient := kubernetesfake.NewSimpleClientset( + base.DeepCopy(), capabilityCatalogConfigMap(validCatalog)) + require.NoError(t, ValidateModelCacheStorageSelectionInputsWithClientset( + t.Context(), k8sClient, testCatalogNamespace, selection)) + }) + + t.Run("catalog payload drift", func(t *testing.T) { + k8sClient := kubernetesfake.NewSimpleClientset( + base.DeepCopy(), capabilityCatalogConfigMap(validCatalog+"\n")) + err := ValidateModelCacheStorageSelectionInputsWithClientset( + t.Context(), k8sClient, testCatalogNamespace, selection) + require.ErrorContains(t, err, "catalog digest changed") + assert.ErrorIs(t, err, ErrModelCacheStorageSelectionDrift) + }) + + t.Run("matching digest cannot invent transition", func(t *testing.T) { + disabled := strings.Replace( + validCatalog, "regularModelCache: roxReadOnly", "regularModelCache: disabled", 1) + forged := *selection + forged.CatalogDigest = digestCatalogPayload(disabled) + k8sClient := kubernetesfake.NewSimpleClientset( + base.DeepCopy(), capabilityCatalogConfigMap(disabled)) + err := ValidateModelCacheStorageSelectionInputsWithClientset( + t.Context(), k8sClient, testCatalogNamespace, &forged) + require.ErrorContains(t, err, "transition for regularModelCache changed") + assert.ErrorIs(t, err, ErrModelCacheStorageSelectionDrift) + }) + + t.Run("missing catalog", func(t *testing.T) { + k8sClient := kubernetesfake.NewSimpleClientset(base.DeepCopy()) + err := ValidateModelCacheStorageSelectionInputsWithClientset( + t.Context(), k8sClient, testCatalogNamespace, selection) + require.ErrorContains(t, err, "get storage capability ConfigMap") + assert.True(t, apierrors.IsNotFound(err)) + }) +} + +func TestValidateNonDurableSelectionDoesNotReadKubernetes(t *testing.T) { + for _, mode := range []ModelCacheSelectionMode{ModelCacheSelectionNone, ModelCacheSelectionEphemeral} { + selection, err := NewPersistedModelCacheStorageSelection(ModelCacheWorkflowHelm, mode, nil) + require.NoError(t, err) + require.NoError(t, ValidateModelCacheStorageSelectionLive(t.Context(), nil, selection)) + require.NoError(t, ValidateModelCacheStorageSelectionLiveWithClientset(t.Context(), nil, selection)) + require.NoError(t, ValidateModelCacheStorageSelectionInputsWithClientset( + t.Context(), nil, "", selection)) + } +} diff --git a/src/compute-plane-services/nvca/pkg/storage/storagerequest.go b/src/compute-plane-services/nvca/pkg/storage/storagerequest.go index d8e0729cb..21c2ad988 100644 --- a/src/compute-plane-services/nvca/pkg/storage/storagerequest.go +++ b/src/compute-plane-services/nvca/pkg/storage/storagerequest.go @@ -36,6 +36,9 @@ import ( const ( requeableStorageErrorToken = "RequeableStorageError" + // ICMSRequestUIDAnnotationKey binds a namespaced StorageRequest to the + // exact ICMSRequest generation that created it. + ICMSRequestUIDAnnotationKey = "nvca.nvcf.nvidia.io/icms-request-uid" ) func NewModelCacheStorageRequest(req *nvcav2beta1.ICMSRequest, fff featureflag.Fetcher) (*nvcav2beta1.StorageRequest, error) { @@ -66,7 +69,27 @@ func NewModelCacheStorageRequest(req *nvcav2beta1.ICMSRequest, fff featureflag.F st.Labels = map[string]string{} } st.Labels[modelCacheHandleLabelKey] = cacheLaunchSpec.CacheHandle - if fff.IsFeatureFlagEnabled(featureflag.NVMeshEncryption) { + if raw := req.Annotations[ModelCacheStorageSelectionAnnotationKey]; raw != "" { + if st.Annotations == nil { + st.Annotations = map[string]string{} + } + st.Annotations[ModelCacheStorageSelectionAnnotationKey] = raw + } + if req.UID != "" { + if st.Annotations == nil { + st.Annotations = map[string]string{} + } + st.Annotations[ICMSRequestUIDAnnotationKey] = string(req.UID) + } + encryptionRequired := fff.IsFeatureFlagEnabled(featureflag.NVMeshEncryption) + if raw := req.Annotations[ModelCacheStorageSelectionAnnotationKey]; raw != "" { + selection, err := ParsePersistedModelCacheStorageSelection(raw) + if err != nil { + return nil, fmt.Errorf("parse persisted model cache storage selection: %w", err) + } + encryptionRequired = selection.EncryptionRequired + } + if encryptionRequired { st.Spec.ModelCache.Encryption = &nvcav2beta1.ModelCacheEncryption{ Required: true, } diff --git a/src/compute-plane-services/nvca/pkg/webhook/helm_storage_webhook.go b/src/compute-plane-services/nvca/pkg/webhook/helm_storage_webhook.go index 1fb7a722f..529f336ca 100644 --- a/src/compute-plane-services/nvca/pkg/webhook/helm_storage_webhook.go +++ b/src/compute-plane-services/nvca/pkg/webhook/helm_storage_webhook.go @@ -244,17 +244,31 @@ func getModelCachePVCVolumeAppendFunc(pvcName string) podMutateFunc { func getModelCachePVCVolumeMountAppendFunc() podMutateFunc { return func(_ context.Context, ps *corev1.PodSpec) (mod bool) { for _, containers := range [][]corev1.Container{ps.InitContainers, ps.Containers} { + mod = forceVolumeMountsReadOnly(containers, cmnnvcastorage.ModelCachePodVolumeName) || mod modModels := addVolumeMount(containers, cmnnvcastorage.ModelCachePodVolumeName, - cmnnvcastorage.ModelCachePodModelMountPath, false) + cmnnvcastorage.ModelCachePodModelMountPath, true) mod = mod || modModels modResources := addVolumeMount(containers, cmnnvcastorage.ModelCachePodVolumeName, - cmnnvcastorage.ModelCachePodResourcesMountPath, false) + cmnnvcastorage.ModelCachePodResourcesMountPath, true) mod = mod || modResources } return mod } } +func forceVolumeMountsReadOnly(containers []corev1.Container, volumeName string) (mod bool) { + for i := range containers { + for j := range containers[i].VolumeMounts { + mount := &containers[i].VolumeMounts[j] + if mount.Name == volumeName && !mount.ReadOnly { + mount.ReadOnly = true + mod = true + } + } + } + return mod +} + // getEphemeralModelCacheInitAppendFunc injects a model-cache-init init // container that downloads the model into an emptyDir shared with the workload // containers. Used as the per-pod fallback (backend "ephemeral") when no shared @@ -438,9 +452,13 @@ func addVolumeFunc(pvcName, volumeName string) podMutateFunc { func addVolumeMount(containers []corev1.Container, volumeName, mountPath string, readOnly bool) (mod bool) { for i, c := range containers { foundVolumeMount := false - for _, v := range c.VolumeMounts { - if v.Name == volumeName && v.MountPath == mountPath && v.ReadOnly == readOnly { + for j, v := range c.VolumeMounts { + if v.Name == volumeName && v.MountPath == mountPath { foundVolumeMount = true + if v.ReadOnly != readOnly { + containers[i].VolumeMounts[j].ReadOnly = readOnly + mod = true + } break } } diff --git a/src/compute-plane-services/nvca/pkg/webhook/helm_storage_webhook_test.go b/src/compute-plane-services/nvca/pkg/webhook/helm_storage_webhook_test.go index 275845beb..3ec1b5581 100644 --- a/src/compute-plane-services/nvca/pkg/webhook/helm_storage_webhook_test.go +++ b/src/compute-plane-services/nvca/pkg/webhook/helm_storage_webhook_test.go @@ -74,8 +74,16 @@ func TestGetModelCachePVCAppend(t *testing.T) { { Name: "foo-init", VolumeMounts: []corev1.VolumeMount{ - {Name: cmnnvcastorage.ModelCachePodVolumeName, MountPath: cmnnvcastorage.ModelCachePodModelMountPath}, - {Name: cmnnvcastorage.ModelCachePodVolumeName, MountPath: cmnnvcastorage.ModelCachePodResourcesMountPath}, + { + Name: cmnnvcastorage.ModelCachePodVolumeName, + MountPath: cmnnvcastorage.ModelCachePodModelMountPath, + ReadOnly: true, + }, + { + Name: cmnnvcastorage.ModelCachePodVolumeName, + MountPath: cmnnvcastorage.ModelCachePodResourcesMountPath, + ReadOnly: true, + }, }, }, }, @@ -83,8 +91,16 @@ func TestGetModelCachePVCAppend(t *testing.T) { { Name: "foo", VolumeMounts: []corev1.VolumeMount{ - {Name: cmnnvcastorage.ModelCachePodVolumeName, MountPath: cmnnvcastorage.ModelCachePodModelMountPath}, - {Name: cmnnvcastorage.ModelCachePodVolumeName, MountPath: cmnnvcastorage.ModelCachePodResourcesMountPath}, + { + Name: cmnnvcastorage.ModelCachePodVolumeName, + MountPath: cmnnvcastorage.ModelCachePodModelMountPath, + ReadOnly: true, + }, + { + Name: cmnnvcastorage.ModelCachePodVolumeName, + MountPath: cmnnvcastorage.ModelCachePodResourcesMountPath, + ReadOnly: true, + }, }, }, }, @@ -138,10 +154,18 @@ func TestGetModelCachePVCAppend(t *testing.T) { { Name: "foo-init", VolumeMounts: []corev1.VolumeMount{ - {Name: cmnnvcastorage.ModelCachePodVolumeName, MountPath: "/my-models"}, - {Name: cmnnvcastorage.ModelCachePodVolumeName, MountPath: "/my-resources"}, - {Name: cmnnvcastorage.ModelCachePodVolumeName, MountPath: cmnnvcastorage.ModelCachePodModelMountPath}, - {Name: cmnnvcastorage.ModelCachePodVolumeName, MountPath: cmnnvcastorage.ModelCachePodResourcesMountPath}, + {Name: cmnnvcastorage.ModelCachePodVolumeName, MountPath: "/my-models", ReadOnly: true}, + {Name: cmnnvcastorage.ModelCachePodVolumeName, MountPath: "/my-resources", ReadOnly: true}, + { + Name: cmnnvcastorage.ModelCachePodVolumeName, + MountPath: cmnnvcastorage.ModelCachePodModelMountPath, + ReadOnly: true, + }, + { + Name: cmnnvcastorage.ModelCachePodVolumeName, + MountPath: cmnnvcastorage.ModelCachePodResourcesMountPath, + ReadOnly: true, + }, }, }, }, @@ -149,10 +173,18 @@ func TestGetModelCachePVCAppend(t *testing.T) { { Name: "foo", VolumeMounts: []corev1.VolumeMount{ - {Name: cmnnvcastorage.ModelCachePodVolumeName, MountPath: "/my-models"}, - {Name: cmnnvcastorage.ModelCachePodVolumeName, MountPath: "/my-resources"}, - {Name: cmnnvcastorage.ModelCachePodVolumeName, MountPath: cmnnvcastorage.ModelCachePodModelMountPath}, - {Name: cmnnvcastorage.ModelCachePodVolumeName, MountPath: cmnnvcastorage.ModelCachePodResourcesMountPath}, + {Name: cmnnvcastorage.ModelCachePodVolumeName, MountPath: "/my-models", ReadOnly: true}, + {Name: cmnnvcastorage.ModelCachePodVolumeName, MountPath: "/my-resources", ReadOnly: true}, + { + Name: cmnnvcastorage.ModelCachePodVolumeName, + MountPath: cmnnvcastorage.ModelCachePodModelMountPath, + ReadOnly: true, + }, + { + Name: cmnnvcastorage.ModelCachePodVolumeName, + MountPath: cmnnvcastorage.ModelCachePodResourcesMountPath, + ReadOnly: true, + }, }, }, }, @@ -186,8 +218,16 @@ func TestGetModelCachePVCAppend(t *testing.T) { { Name: "foo-init", VolumeMounts: []corev1.VolumeMount{ - {Name: cmnnvcastorage.ModelCachePodVolumeName, MountPath: cmnnvcastorage.ModelCachePodModelMountPath}, - {Name: cmnnvcastorage.ModelCachePodVolumeName, MountPath: cmnnvcastorage.ModelCachePodResourcesMountPath}, + { + Name: cmnnvcastorage.ModelCachePodVolumeName, + MountPath: cmnnvcastorage.ModelCachePodModelMountPath, + ReadOnly: true, + }, + { + Name: cmnnvcastorage.ModelCachePodVolumeName, + MountPath: cmnnvcastorage.ModelCachePodResourcesMountPath, + ReadOnly: true, + }, }, }, }, @@ -195,8 +235,16 @@ func TestGetModelCachePVCAppend(t *testing.T) { { Name: "foo", VolumeMounts: []corev1.VolumeMount{ - {Name: cmnnvcastorage.ModelCachePodVolumeName, MountPath: cmnnvcastorage.ModelCachePodModelMountPath}, - {Name: cmnnvcastorage.ModelCachePodVolumeName, MountPath: cmnnvcastorage.ModelCachePodResourcesMountPath}, + { + Name: cmnnvcastorage.ModelCachePodVolumeName, + MountPath: cmnnvcastorage.ModelCachePodModelMountPath, + ReadOnly: true, + }, + { + Name: cmnnvcastorage.ModelCachePodVolumeName, + MountPath: cmnnvcastorage.ModelCachePodResourcesMountPath, + ReadOnly: true, + }, }, }, }, @@ -235,8 +283,16 @@ func TestGetModelCachePVCAppend(t *testing.T) { { Name: "foo-init", VolumeMounts: []corev1.VolumeMount{ - {Name: cmnnvcastorage.ModelCachePodVolumeName, MountPath: cmnnvcastorage.ModelCachePodModelMountPath}, - {Name: cmnnvcastorage.ModelCachePodVolumeName, MountPath: cmnnvcastorage.ModelCachePodResourcesMountPath}, + { + Name: cmnnvcastorage.ModelCachePodVolumeName, + MountPath: cmnnvcastorage.ModelCachePodModelMountPath, + ReadOnly: true, + }, + { + Name: cmnnvcastorage.ModelCachePodVolumeName, + MountPath: cmnnvcastorage.ModelCachePodResourcesMountPath, + ReadOnly: true, + }, }, }, }, @@ -244,8 +300,16 @@ func TestGetModelCachePVCAppend(t *testing.T) { { Name: "foo", VolumeMounts: []corev1.VolumeMount{ - {Name: cmnnvcastorage.ModelCachePodVolumeName, MountPath: cmnnvcastorage.ModelCachePodModelMountPath}, - {Name: cmnnvcastorage.ModelCachePodVolumeName, MountPath: cmnnvcastorage.ModelCachePodResourcesMountPath}, + { + Name: cmnnvcastorage.ModelCachePodVolumeName, + MountPath: cmnnvcastorage.ModelCachePodModelMountPath, + ReadOnly: true, + }, + { + Name: cmnnvcastorage.ModelCachePodVolumeName, + MountPath: cmnnvcastorage.ModelCachePodResourcesMountPath, + ReadOnly: true, + }, }, }, }, @@ -303,6 +367,9 @@ func TestGetEphemeralModelCacheInitAppendFunc(t *testing.T) { {Name: "MODEL_URL", Value: "https://models.example/m1"}, }, ic.Env) assert.Len(t, ic.VolumeMounts, 3) + for _, mount := range ic.VolumeMounts { + assert.False(t, mount.ReadOnly, "ephemeral model-cache init mounts must remain writable") + } } assert.True(t, hasVolumeNamed(ps.Volumes, cmnnvcastorage.ModelCachePodVolumeName)) @@ -311,6 +378,9 @@ func TestGetEphemeralModelCacheInitAppendFunc(t *testing.T) { assert.NotNilf(t, v.VolumeSource.EmptyDir, "volume %s should be emptyDir", v.Name) } assert.Len(t, ps.Containers[0].VolumeMounts, 2) + for _, mount := range ps.Containers[0].VolumeMounts { + assert.False(t, mount.ReadOnly, "ephemeral model-cache workload mounts must remain writable") + } // Idempotent: a second pass does not duplicate the container, volumes, or mounts. getEphemeralModelCacheInitAppendFunc("init-image:1", envSet)(t.Context(), &ps) @@ -385,9 +455,12 @@ func TestMutate_StableVolumeOrderAcrossReadmission(t *testing.T) { // First admission (CREATE). _, _, err := v.mutate(ctx, pod, nvcatypes.MiniserviceMetadata{}) require.NoError(t, err) + assertModelCacheMountsReadOnly(t, pod.Spec.InitContainers) + assertModelCacheMountsReadOnly(t, pod.Spec.Containers) wantVols := volNames(pod.Spec.Volumes) wantMounts := mountSig(pod.Spec.Containers[0].VolumeMounts) wantInitMounts := mountSig(pod.Spec.InitContainers[0].VolumeMounts) + wantPodSpec := pod.Spec.DeepCopy() // Sanity: all three injected volumes are present. assert.Contains(t, wantVols, cmnnvcastorage.ModelCachePodVolumeName) @@ -395,7 +468,26 @@ func TestMutate_StableVolumeOrderAcrossReadmission(t *testing.T) { // order must be byte-for-byte identical, or the UPDATE is rejected. _, _, err = v.mutate(ctx, pod, nvcatypes.MiniserviceMetadata{}) require.NoError(t, err) + assertModelCacheMountsReadOnly(t, pod.Spec.InitContainers) + assertModelCacheMountsReadOnly(t, pod.Spec.Containers) + assert.Equal(t, *wantPodSpec, pod.Spec, "pod spec must be stable across re-admission") assert.Equal(t, wantVols, volNames(pod.Spec.Volumes), "volume order must be stable across re-admission") assert.Equal(t, wantMounts, mountSig(pod.Spec.Containers[0].VolumeMounts), "container mount order must be stable") assert.Equal(t, wantInitMounts, mountSig(pod.Spec.InitContainers[0].VolumeMounts), "init mount order must be stable") } + +func assertModelCacheMountsReadOnly(t *testing.T, containers []corev1.Container) { + t.Helper() + for _, container := range containers { + count := 0 + for _, mount := range container.VolumeMounts { + if mount.Name != cmnnvcastorage.ModelCachePodVolumeName { + continue + } + count++ + assert.Truef(t, mount.ReadOnly, "model-cache mount %s in container %s must be read-only", + mount.MountPath, container.Name) + } + assert.Equalf(t, 2, count, "container %s must have both model-cache mounts", container.Name) + } +} diff --git a/src/compute-plane-services/nvca/pkg/webhook/miniservice_mutating_webhook_test.go b/src/compute-plane-services/nvca/pkg/webhook/miniservice_mutating_webhook_test.go index 28359185a..24197e468 100644 --- a/src/compute-plane-services/nvca/pkg/webhook/miniservice_mutating_webhook_test.go +++ b/src/compute-plane-services/nvca/pkg/webhook/miniservice_mutating_webhook_test.go @@ -627,6 +627,7 @@ func TestMiniserviceOperatorWebhook_PodSpecCreateThenUpdate_IsIdempotentOrderPre createMutatedRaw := applyPatches(t, createRaw, createResp.Patches) var createdPod corev1.Pod require.NoError(t, json.Unmarshal(createMutatedRaw, &createdPod)) + assertModelCacheMountsReadOnly(t, createdPod.Spec.Containers) require.Equal(t, []string{ cmnnvcastorage.SharedStorageVolumeKNSTokenVolumeName, cmnnvcastorage.SharedStorageSecretsVolumeName, @@ -668,6 +669,7 @@ func TestMiniserviceOperatorWebhook_PodSpecCreateThenUpdate_IsIdempotentOrderPre updateMutatedRaw := applyPatches(t, updateRaw, updateResp.Patches) var gotUpdatedPod corev1.Pod require.NoError(t, json.Unmarshal(updateMutatedRaw, &gotUpdatedPod)) + assertModelCacheMountsReadOnly(t, gotUpdatedPod.Spec.Containers) assert.Equal(t, createdPod.Spec.Volumes, gotUpdatedPod.Spec.Volumes, "volume order must remain stable on update") assert.Equal( @@ -710,6 +712,7 @@ func TestMiniserviceOperatorWebhook_PodSpecCreateThenUpdate_IsIdempotentOrderPre createMutatedRaw := applyPatches(t, createRaw, createResp.Patches) var createdPod corev1.Pod require.NoError(t, json.Unmarshal(createMutatedRaw, &createdPod)) + assertModelCacheMountsReadOnly(t, createdPod.Spec.Containers) require.Equal(t, []string{ "kube-api-access-rmxfs", cmnnvcastorage.SharedStorageVolumeKNSTokenVolumeName, @@ -740,6 +743,7 @@ func TestMiniserviceOperatorWebhook_PodSpecCreateThenUpdate_IsIdempotentOrderPre updateMutatedRaw := applyPatches(t, updateRaw, updateResp.Patches) var gotUpdatedPod corev1.Pod require.NoError(t, json.Unmarshal(updateMutatedRaw, &gotUpdatedPod)) + assertModelCacheMountsReadOnly(t, gotUpdatedPod.Spec.Containers) assert.Equal(t, createdPod.Spec.Volumes, gotUpdatedPod.Spec.Volumes, "volume order must remain stable on update") assert.Equal( diff --git a/src/compute-plane-services/nvca/scripts/openapigen_update b/src/compute-plane-services/nvca/scripts/openapigen_update index 907c74b45..a48588c43 100755 --- a/src/compute-plane-services/nvca/scripts/openapigen_update +++ b/src/compute-plane-services/nvca/scripts/openapigen_update @@ -25,6 +25,7 @@ openapi_gen() { main() { openapi_gen nvcf/v1 openapi_gen nvca/v1alpha1 + openapi_gen nvca/v2beta1 } main "$@"