diff --git a/dependencies.md b/dependencies.md index 24b448570..247385fea 100644 --- a/dependencies.md +++ b/dependencies.md @@ -1843,7 +1843,8 @@ Generated by `go run -C ./tools/collect-dependencies .`. Refresh: `go run -C ./t - `Node.js`: `zod@4.3.6` - `Node.js`: `zwitch@2.0.4` - `Python`: `fastapi==0.110.0` -- `Python`: `pyyaml` (`PyYAML>=5.4.1`, `pyyaml>=6.0.1`) +- `Python`: `jsonschema==4.10.3` +- `Python`: `pyyaml` (`PyYAML==6.0.2`, `PyYAML>=5.4.1`, `pyyaml>=6.0.1`) - `Python`: `rich>=13.7.0` - `Rust`: `async-stream` (`async-stream 0.3`, `async-stream =0.3.6`) - `Rust`: `axum` (`axum 0.8`, `axum =0.8.9`) diff --git a/deploy/helm/nvca-operator/nvca-operator/README.md b/deploy/helm/nvca-operator/nvca-operator/README.md index 19406bc7a..99843ad41 100644 --- a/deploy/helm/nvca-operator/nvca-operator/README.md +++ b/deploy/helm/nvca-operator/nvca-operator/README.md @@ -3,6 +3,12 @@ NVCF Cluster Agent (NVCA) Operator installs and manages reconfiguration, upgrades, and health checks of NVCA used in Kubernetes Clusters to run NVCF Workloads. +## Storage capability catalog + +The chart installs the versioned `nvcf-storage-capabilities` ConfigMap in the Helm release namespace. For each exact CSI provisioner the catalog records only the PVC access modes qualified end to end in an NVCF cache workflow, plus the mount options for reader volumes NVCA creates. Nothing about the flow is declared: NVCA derives it from those modes. An empty `accessModes` list means nothing is qualified yet, so caching stays off for that driver. Container cache is outside NVCA and is not part of this catalog. + +This release does not wire the catalog into backend selection. Runtime use requires a durable cache plan and safe legacy-request migration so retries and agent restarts cannot change backends. Managed deployments will inspect the exact `nvcf-sc` StorageClass when that follow-up is implemented. Editing this ConfigMap does not enable a storage backend today. + ## Parameters ### NVCA Operator parameters diff --git a/deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.schema.json b/deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.schema.json new file mode 100644 index 000000000..6d55f32f1 --- /dev/null +++ b/deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.schema.json @@ -0,0 +1,199 @@ +{ + "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://nvcf.nvidia.com/schemas/storage-capability-catalog-v1alpha1.json", + "title": "NVCF storage capability catalog", + "description": "PVC access modes qualified per exact CSI provisioner. NVCA derives the cache flow from these modes; nothing about the flow is declared here.", + "type": "object", + "additionalProperties": false, + "required": [ + "apiVersion", + "kind", + "drivers" + ], + "properties": { + "apiVersion": { + "const": "storage.nvcf.nvidia.com/v1alpha1" + }, + "kind": { + "const": "StorageCapabilityCatalog" + }, + "drivers": { + "type": "object", + "minProperties": 1, + "propertyNames": { + "type": "string", + "minLength": 1, + "pattern": "\\S" + }, + "additionalProperties": { + "$ref": "#/$defs/driver" + } + } + }, + "$defs": { + "accessMode": { + "type": "string", + "enum": [ + "ReadWriteOnce", + "ReadOnlyMany", + "ReadWriteMany" + ] + }, + "driver": { + "type": "object", + "additionalProperties": false, + "required": [ + "provider", + "accessModes", + "readerMountOptions" + ], + "properties": { + "provider": { + "type": "string", + "minLength": 1, + "pattern": "\\S" + }, + "accessModes": { + "description": "Modes qualified end to end in an NVCF cache workflow, not modes the driver accepts. Empty means the driver is not enabled.", + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/$defs/accessMode" + } + }, + "readerMountOptions": { + "description": "Mount options for reader PVs NVCA creates. Only the ReadWriteOnce plus ReadOnlyMany shape creates them, and it must include \"ro\".", + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/$defs/readerMountOption" + }, + "allOf": [ + { + "not": { + "allOf": [ + { + "contains": { + "const": "ro" + } + }, + { + "contains": { + "const": "rw" + } + } + ] + } + }, + { + "not": { + "allOf": [ + { + "contains": { + "const": "recovery" + } + }, + { + "contains": { + "const": "norecovery" + } + } + ] + } + }, + { + "not": { + "allOf": [ + { + "contains": { + "const": "uuid" + } + }, + { + "contains": { + "const": "nouuid" + } + } + ] + } + } + ] + } + }, + "allOf": [ + { + "$comment": "A driver qualified for the ReadOnlyMany reader shape creates reader PVs, so it must mount them read-only.", + "if": { + "properties": { + "accessModes": { + "allOf": [ + { + "contains": { + "const": "ReadWriteOnce" + } + }, + { + "contains": { + "const": "ReadOnlyMany" + } + } + ] + } + }, + "required": [ + "accessModes" + ] + }, + "then": { + "properties": { + "readerMountOptions": { + "contains": { + "const": "ro" + } + } + } + } + }, + { + "$comment": "ReadOnlyMany describes readers. Without a writer mode alongside it there is nothing to populate the cache.", + "if": { + "properties": { + "accessModes": { + "contains": { + "const": "ReadOnlyMany" + } + } + }, + "required": [ + "accessModes" + ] + }, + "then": { + "properties": { + "accessModes": { + "anyOf": [ + { + "contains": { + "const": "ReadWriteOnce" + } + }, + { + "contains": { + "const": "ReadWriteMany" + } + } + ] + } + } + } + } + ] + }, + "readerMountOption": { + "type": "string", + "minLength": 1, + "pattern": "^\\S(?:.*\\S)?$" + } + } +} diff --git a/deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml b/deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml new file mode 100644 index 000000000..377d9c3e5 --- /dev/null +++ b/deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NVCA owns this catalog and installs it with the NVCA chart. It records, for +# each exact CSI provisioner, the PVC access modes qualified end to end in an +# NVCF cache workflow. Nothing else is declared: NVCA derives how caching runs +# from these modes. +# +# ReadWriteMany one shared claim; readers mount it read-only +# ReadWriteOnce+ReadOnlyMany writer takes the claim; readers get their own +# +# An access mode a driver merely accepts is not a qualification. A claim that +# binds is not either. An empty accessModes list means nothing is qualified +# yet, so caching stays off for that driver, and a provisioner absent from this +# file is unsupported. Enabling a backend is an edit here, backed by a +# qualification run. +apiVersion: storage.nvcf.nvidia.com/v1alpha1 +kind: StorageCapabilityCatalog +drivers: + nvmesh-csi.excelero.com: + provider: nvmesh + accessModes: + - ReadWriteOnce + - ReadOnlyMany + # The reader PV is XFS on the same filesystem as the writer, so it needs + # nouuid and norecovery or the mount fails outright. + readerMountOptions: + - ro + - norecovery + - nouuid + csi.weka.io: + provider: weka + # Fresh ReadWriteMany and ReadOnlyMany claims were tested, but a cache + # workflow was not, so nothing is qualified yet. + accessModes: [] + readerMountOptions: [] + fss.csi.oraclecloud.com: + provider: ociFss + # A ReadWriteMany claim was tested; its readers used read-only Pod mounts, + # which is not evidence for a ReadOnlyMany claim. No cache workflow was + # qualified. + accessModes: [] + readerMountOptions: [] + lustre.csi.oraclecloud.com: + provider: ociLustre + # No PVC access mode has been qualified in an NVCF cache workflow. + accessModes: [] + readerMountOptions: [] diff --git a/deploy/helm/nvca-operator/nvca-operator/templates/storage-capabilities-configmap.yaml b/deploy/helm/nvca-operator/nvca-operator/templates/storage-capabilities-configmap.yaml new file mode 100644 index 000000000..83bb433c0 --- /dev/null +++ b/deploy/helm/nvca-operator/nvca-operator/templates/storage-capabilities-configmap.yaml @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- $catalogPath := "files/nvcf-storage-capabilities-v1alpha1.yaml" -}} +{{- $catalog := .Files.Get $catalogPath -}} +{{- if not $catalog -}} +{{- fail (printf "required NVCF storage capability catalog %s is missing" $catalogPath) -}} +{{- end }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: nvcf-storage-capabilities + namespace: {{ .Release.Namespace }} + labels: + {{- include "nvcaop.labels" . | nindent 4 }} +data: + storage-provider-capabilities.yaml: | +{{- $catalog | nindent 4 }} diff --git a/docs/dev/sdd-central-model-cache-service.md b/docs/dev/sdd-central-model-cache-service.md index 448b28334..5a5ec5698 100644 --- a/docs/dev/sdd-central-model-cache-service.md +++ b/docs/dev/sdd-central-model-cache-service.md @@ -6,13 +6,22 @@ Implemented in NVCA. This document describes the model cache design as built in `pkg/storage` (the model cache StorageRequest controller), `pkg/webhook` (workload volume injection), and `internal/metrics` (observability). +The target provider-neutral design is documented in +[Storage-Agnostic Cache Architecture](sdd-storage-agnostic-cache-architecture.md). +The catalog is not wired into runtime selection, so this document remains the +current implementation reference until that migration is complete. The current +webhook does not set every model-cache volume mount read-only; the target design +defines the required fix and qualification test. The stable deployment contract +is `StorageClass/nvcf-sc`; the extra class-name checks below are legacy runtime +behavior, not the target provider-selection contract. + ## Goal A function or task can declare a model cache by `cacheHandle`. The first workload that needs a given handle downloads the model once; every later workload for the -same handle, in any namespace, attaches the already-downloaded copy read-only -instead of downloading again. Caching is best-effort: if the cache cannot be -provisioned the workload still runs, just without a shared cache. +same handle, in any namespace, attaches the already-downloaded copy through a +reader volume instead of downloading again. Caching is best-effort: if the cache +cannot be provisioned the workload still runs, just without a shared cache. The service is folded into NVCA rather than run as a standalone operator. It reuses NVCA's existing machinery: a control namespace, a per-cacheHandle @@ -28,29 +37,37 @@ and the mutating webhook that mounts the cache into workload pods. - Backend: the storage mechanism used to hold and share the cache. One of `nvmesh`, `sharedfs`, `samba`, `ephemeral`. - Writer: the single init Job that downloads the model into the backend. -- Reader: the per-namespace read-only volume a workload mounts. -- Durable marker: a cluster-scoped object whose existence means "this handle is +- Reader: the per-namespace volume intended to expose populated cache data. The + current webhook marks the PVC source read-only but not every matching + `volumeMount`. +- Durable marker: a durable Kubernetes object whose existence means "this handle is populated", used so any namespace and any agent restart can detect a populated cache without depending on in-memory state. ## Backend selection -The miniservice reconciler selects a backend once per request via -`SelectHelmCacheBackend` (`pkg/storage/cachebackend.go`) and stamps it on the -`StorageRequest.spec.modelCache.backend`. Selection is deterministic on cluster -state and feature flags: +The current miniservice reconciler calls `SelectHelmCacheBackend` +(`pkg/storage/cachebackend.go`) during each install reconcile and stamps the +result on `StorageRequest.spec.modelCache.backend`. It does not persist an +immutable plan before side effects. Selection uses cluster state, feature flags, +and legacy class-name sentinels: -1. `CachingSupport` disabled: no cache (`none`). -2. `nvcf-sc-30` StorageClass exists (NVMesh 3.x): `nvmesh`. +1. `CachingSupport` or `HelmModelCaching` disabled: no cache (`none`). +2. Legacy `nvcf-sc-30` sentinel exists: `nvmesh`. 3. `nvcf-miniservice-sc` StorageClass exists (operator-provided third-party shared storage): `sharedfs`. -4. `HelmSharedStorage` flag enabled: `samba`. +4. `HelmSharedStorage` enabled and the configured model-cache backing class exists: `samba`. 5. Otherwise: `ephemeral`. NVCA never creates `nvcf-miniservice-sc`. That StorageClass is exclusively the operator's signal that third-party shared storage is present (branch 3 above). The Samba backend (branch 4) is self-contained and creates no StorageClass. +This decision tree describes compatibility code in public NVCA main. Target +selection reads `nvcf-sc`, finds its provisioner in the catalog, and persists a +shared cache binding. NVMesh uses transition `nvmesh` for both regular and Helm +model cache. Samba remains only a legacy fallback; it is not an NVMesh transition. + ## Lifecycle shared by all shared backends The shared backends (nvmesh, sharedfs, samba) share one lifecycle; only the @@ -93,15 +110,16 @@ the init lease or any in-memory fan-out, both of which are lost on restart. namespace) plus a read-only PVC. - samba: a static SMB CSI PV pointing at the per-handle Samba share, read-only, plus a read-only PVC. -- sharedfs: a read-only PVC on the shared class; the class itself shares data - across namespaces, so no per-namespace PV plumbing is needed. +- sharedfs: a separately provisioned read-only PVC on the shared class. Current + code assumes it resolves to the writer's data, which is not guaranteed by a + StorageClass or provisioner alone. -## Samba backend (Samba over NVMesh) +## Legacy Samba fallback -Selected when no NVMesh 3.x or operator shared storage exists but -`HelmSharedStorage` is on. NVMesh block storage is ReadWriteOnce, so it cannot be -shared read-only across namespaces directly. The Samba backend re-exports an -`nvcf-sc` volume over SMB, which supports ReadWriteMany and ReadOnlyMany. +Current compatibility code selects this backend when neither legacy class +sentinel is present and `HelmSharedStorage` is on. It re-exports an `nvcf-sc` +volume over SMB for cross-namespace readers. It is not the target NVMesh +transition. Each cacheHandle gets its own Samba server and its own backing volume. There is no single shared server or global data PVC: a single fixed-size volume cannot be @@ -131,8 +149,9 @@ character name limit a stable hashed suffix is used. The mutating webhook (`pkg/webhook/helm_storage_webhook.go`) injects the cache into workload pods. When a pod carries the `nvca.nvcf.nvidia.io/storage-modelcache-pvc-name` annotation the webhook adds a -`model-data` volume bound to the read-only cache PVC and mounts it at the model -paths. +`model-data` volume bound to the cache PVC and mounts it at the model paths. The +PVC volume source is marked read-only; current matching `volumeMount` entries +are not, which the target design requires NVCA to fix before qualification. The shared-storage volumes use a drop-and-re-add pattern on every admission. The model cache volume participates in the same drop-and-re-add @@ -198,11 +217,10 @@ child span around per-handle Samba provisioning. separately provisioned PVCs on the same StorageClass do not share data on provisioners that create per-claim access points, subvolumes, or directories (EFS access points, CephFS subvolumes, some NFS provisioners). - Classes backed by a single share (for example an SMB class pointing at one - server share) work. The capability probe validates bindability, not - data-sharing; the fast-follow is a write-through-one-claim / - read-through-another probe, and deriving reader PVs from the writer's bound - volume where the driver supports it. + A class backed by one share may expose the same data, but class presence does + not prove that identity. The capability probe validates bindability, not + data-sharing; target qualification requires a no-copy cross-namespace data + identity and read-only enforcement tests. - The per-handle Samba infrastructure is create-once: image, resource, and cache-size changes do not reconcile onto an existing server or expand its backing PVC. diff --git a/docs/dev/sdd-storage-agnostic-cache-architecture.md b/docs/dev/sdd-storage-agnostic-cache-architecture.md new file mode 100644 index 000000000..8d8f6d806 --- /dev/null +++ b/docs/dev/sdd-storage-agnostic-cache-architecture.md @@ -0,0 +1,455 @@ +# SDD: Storage-Agnostic Cache Architecture + +## Summary + +This document separates implemented foundation from target runtime behavior. + +Implemented foundation: + +- The NVCA Operator chart installs a public `v1alpha1` storage catalog and packages its JSON Schema. +- The catalog uses closed transition enums and records required reader-PV mount options. +- NVCA has a strict catalog loader and semantic validator. Tests call them; runtime reconciliation does not. +- Current public NVCA still uses legacy StorageClass-presence checks and feature flags for backend selection. +- Weka, OCI File Storage (FSS), and OCI Lustre model-cache transitions remain `disabled`. + +Target: + +1. Deployment tooling renders exactly one selected provider as `StorageClass/nvcf-sc`. +2. The public NVCA catalog declares qualified access modes, reader-PV mount options, and regular/Helm cache + transitions. +3. NVCA resolves the live class and catalog entry, then persists a shared cache binding before storage side effects. +4. A provider transition remains disabled until its complete functional contract passes. + +Deployment tooling owns CSI-specific StorageClass parameters. NVCA owns model-cache transitions. A provider name, +provisioner, or access mode alone never enables a workflow. + +Editing the catalog ConfigMap does not enable a provider today. Runtime work is tracked in +[NVIDIA/nvcf#1326](https://github.com/NVIDIA/nvcf/issues/1326); this SDD defines the stable `nvcf-sc` target contract. + +## Table of contents + +- [Summary](#summary) +- [Scope](#scope) +- [Terms](#terms) +- [Deployment configuration](#deployment-configuration) +- [Storage catalog](#storage-catalog) +- [Target runtime design](#target-runtime-design) +- [Model-cache contract](#model-cache-contract) +- [Failure, migration, and rollback](#failure-migration-and-rollback) +- [Provider qualification](#provider-qualification) +- [Test plan](#test-plan) +- [Security and observability](#security-and-observability) +- [Implementation order](#implementation-order) +- [Public source references](#public-source-references) + +## Scope + +This design covers regular and Helm/MiniService model cache across managed and self-managed deployments, including +NVMesh, Weka, OCI FSS, OCI Lustre, future CSI providers, functional qualification, migration, and rollback. + +It excludes container cache; function, internal, and database storage; `nvcf-function-storage-sc`; CSI driver +installation/lifecycle; and performance qualification, which follows functional acceptance. + +## Terms + +| Term | Meaning | +|---|---| +| Provider | Product or integration ID, such as `ociFss` or `weka` | +| Provisioner | Exact CSI string in `StorageClass.provisioner` | +| Access mode | Kubernetes PVC mode declared only after qualification for an exact provider configuration | +| Workflow | `regularModelCache` or `helmModelCache` | +| Transition | Named NVCA implementation that moves a cache from writer state to reusable reader state | +| Qualification record | Versioned evidence for the exact tested storage and cluster configuration | +| Sharing domain | Stable authorization and encryption scope, such as an NCA ID | +| Cache key | Tuple of workflow, sharing domain, and cache handle | +| Cache binding | Immutable provider and transition choice shared by all requests for one cache key | +| Request selection | Per-request mode and optional cache-binding reference persisted before side effects | +| Data identity | Provider-backed object containing one populated cache handle | +| Writer | Workload holding the binding Lease while it populates a data identity | +| Reader view | Provider-specific, namespace-local, read-only access to the same data identity | + +`ReadOnlyMany` (ROX) is a PVC access mode. Mounting a `ReadWriteMany` (RWX) PVC with `readOnly: true` is not evidence +that the driver provisions or binds a ROX PVC. Neither condition alone proves backend-enforced write denial or +cross-namespace data identity. + +`ReadWriteOnce` (RWO) limits a volume to one node, not one Pod or writer. Kubernetes access modes describe attachment +and mount intent; NVCA must separately serialize writers and test reader write denial. + +## Deployment configuration + +Managed input selects a provider through `spec.nvcfStorage.provider`, resolves +`spec.storage.drivers..storageClasses.nvcf-sc`, and renders `StorageClass/nvcf-sc`. + +The selected provider supplies its exact provisioner, parameters, mount options, binding mode, expansion setting, +and topology. Provider values must match the installed CSI driver. + +The class name is always `nvcf-sc`, its reclaim policy is always `Retain`, and exactly one installed provider is +primary even when multiple CSI drivers coexist. Rendering rejects an empty provisioner, invalid class fields, unknown +provider fields, and provider input for `reclaimPolicy`. + +When no provider is selected, managed output remains compatible with NVMesh. Self-managed deployments may use +different tooling, but NVCA sees the same live `StorageClass/nvcf-sc` contract. + +NVMesh supports volume expansion, so its `nvcf-sc` definition and regression tests must preserve +`allowVolumeExpansion: true`. Expansion is deployment behavior, not a field in the NVCA catalog. + +## Storage catalog + +The chart installs ConfigMap `nvcf-storage-capabilities` in its release namespace. Data key +`storage-provider-capabilities.yaml` contains a `storage.nvcf.nvidia.com/v1alpha1` `StorageCapabilityCatalog`. Source +and release charts contain byte-identical catalog and schema files; rendering fails when the catalog file is absent. + +### Minimal shape + +```yaml +drivers: + : + provider: + accessModes: + - + readerMountOptions: + - + transitions: + regularModelCache: + helmModelCache: +``` + +This is a shape illustration, not a valid provider entry. Actual entries must use exact provisioner strings, +Kubernetes access-mode names, and registered transition names. + +The exact provisioner is the lookup key. `provider` is a label, and `accessModes` cites externally qualified PVC +modes. Transition values are a closed enum, not free text. A strategy name means enabled; `disabled` means +unsupported. + +| Workflow field | Allowed values | +|---|---| +| `regularModelCache` | `disabled`, `roxReadOnly`, `rwxReadOnly` | +| `helmModelCache` | `disabled`, `roxReadOnly` | + +Transition code, not the flat access-mode list, defines state A to state B: + +| Transition | Writer to reader contract | Catalog restriction | +|---|---|---| +| `disabled` | No durable transition | Any provider | +| `roxReadOnly` | Populate an RWO writer, then publish ROX reader storage | Exact NVMesh provisioner and provider | +| `rwxReadOnly` | Populate one RWX claim, then serve it through read-only Pod mounts | Regular model cache only | + +Adding a transition requires dispatcher code, schema and semantic validation, a declared writer-to-reader mode pair, +and workflow tests. Access modes only show which required modes were qualified. + +`disabled` is the complete workflow-disabled state; a driver may still list qualified access modes. There is no +separate qualification field. Enabling a transition is a release decision allowed only after implementation and exact +workflow qualification; the schema cannot prove that evidence exists. + +`readerMountOptions` is a required array for options NVCA must apply when it creates or rewrites a reader PV. It is +not `StorageClass.mountOptions`, a Pod `volumeMount`, or the Pod-level read-only flag. `roxReadOnly` requires +`ro`, `norecovery`, and `nouuid`. `rwxReadOnly` uses the dynamically provisioned RWX claim and therefore +requires an empty array. An empty array is not provider qualification evidence. + +The catalog intentionally does not contain a general CSI capability matrix. StorageClass rendering and provider +documentation own expansion, snapshots, clones, topology, and other CSI settings. + +Access-mode rules: + +- Record only Kubernetes access modes exercised by functional evidence for the exact provider configuration. +- Do not infer ROX from a read-only Pod mount of an RWX claim. +- Do not infer RWX or ROX from CSI driver documentation alone. +- Do not infer cross-namespace sharing or backend write denial from an access mode. + +NVMesh uses transition `roxReadOnly` for both regular and Helm model cache. Samba is not an NVMesh transition. Weka, OCI +FSS, and OCI Lustre transitions remain `disabled` until their exact workflow qualification passes. + +### What operators set + +Use the exact installed CSI provisioner. Do not infer or enable a strategy from access modes alone. + +| Provider state | `regularModelCache` | `helmModelCache` | +|---|---|---| +| Shipped NVMesh configuration | `roxReadOnly` | `roxReadOnly` | +| New or unqualified provider | `disabled` | `disabled` | +| Qualified shared-RWX regular workflow | `rwxReadOnly` | `disabled` | + +The last row is a target configuration, not current enablement. Set it only after the exact CSI and StorageClass +configuration passes the provider-qualification contract and its NVCA runtime transition is released. + +The target catalog does not register a generic `sharedfs` transition. Current NVCA retains a legacy, presence-selected +`sharedfs` route. Separate dynamic PVCs may expose different data, so class or driver presence cannot qualify it. + +### Validation contract + +CI validates structure, required fields, closed transition values, and workflow constraints with the packaged JSON +Schema; Helm only checks that the file exists. When called, the Go loader independently performs strict decoding and +semantic validation. It also rejects duplicate, blank, whitespace-padded, or conflicting reader mount options. The +test plan covers both validation layers. + +Catalog entries are configuration metadata, not credentials. The security requirements below govern their content. + +### Current runtime gap + +The loader has no runtime call site, and NVCA has no complete provider-neutral cache binding. Current requests persist +only a coarse backend value. Both gaps must be closed before the catalog can control runtime behavior. Until then, a +`disabled` catalog entry does not disable a legacy path selected by StorageClass presence. + +For Helm caching, current public NVCA first requires `CachingSupport` and `HelmModelCaching`. It then checks the legacy +`nvcf-sc-30` marker, the `nvcf-miniservice-sc` sharedfs sentinel, and a gated Samba fallback with a usable backing +class; otherwise it selects the per-Pod ephemeral cache. These are compatibility paths, not target provider selection. +The presence-only sharedfs branch cannot prove a cross-namespace backend identity. The target reads only `nvcf-sc`, +finds the exact provisioner entry, and selects its recorded transition. + +## Target runtime design + +### Selection algorithm + +For each request, NVCA must: + +1. Evaluate the applicable feature gates and workflow. Persist `none` or `ephemeral` on the request when selected; do + not create a durable binding. +2. For durable caching, derive `(workflow, sharingDomain, cacheHandle)` and read its binding from the model-cache + control namespace. +3. If it is `Active`, use it. If it is `Retiring`, retry or record a supported request fallback; never rebind it. +4. If no binding exists, read `StorageClass/nvcf-sc`, require `Retain`, and strictly load the catalog. +5. Find the exact provisioner entry and require a non-disabled transition with its required access modes and reader + mount options. +6. Create the binding by deterministic name and optimistic concurrency. A losing creator reads the winning binding. +7. Conditionally add the request namespace, name, and UID while the binding is `Active`. Then persist its name, UID, + and a request finalizer before any storage side effect. +8. Re-read the binding, confirm the reference and request finalizer are present, and execute only its transition. + +Unknown provisioners, invalid catalogs, non-`Retain` classes, and disabled transitions cannot select durable storage. +The only generic fallbacks are `none` and, where the workflow supports it, `ephemeral`; NVCA never switches to a +second durable provider. + +### Release qualification + +Provisioner matching selects code; it does not prove that a deployment was qualified. Before a transition is enabled, +dated evidence must identify the CSI provisioner and images, backend version/configuration, `nvcf-sc` digest, +Kubernetes version, eligible node/OS/architecture matrix, catalog/schema version, and evidence reference. + +Some backend fields are not discoverable through Kubernetes or CSI. The release or deployment qualification process, +not NVCA, owns this record and verifies those fields. NVCA snapshots only live Kubernetes data it can read: the +StorageClass name, UID, provisioner, configuration digest, and catalog digest. That snapshot detects drift for a cache +binding; it does not prove the backend product or version. + +Setting a transition to a non-disabled value is therefore an operator or release assertion that the qualification +record applies to the cluster. NVCA validates the catalog and live StorageClass, but cannot verify non-observable +fields. + +### Cache binding and realized state + +Recomputing per request could mix providers for one cache key. The binding is one durable Kubernetes object in the +model-cache control namespace; every namespace-local request for that key references it. + +| Group | Required binding data | +|---|---| +| Identity | Version, workflow, sharing-domain digest, and cache-handle digest | +| Decision | Provider, provisioner, transition, required access modes, reader-PV mount options, and catalog payload digest | +| StorageClass snapshot | Name, UID, `Retain`, and configuration digest | +| Resource intent | Deterministic names for NVCA-created PVCs, static PVs, Jobs, and Leases | +| Lifecycle | `Active` or `Retiring`, request namespace/name/UID references, and finalizer | +| Realized state | Bound PV and provider data identity plus population state as they become known | + +The target API is a namespaced `ModelCacheBinding` in the model-cache control namespace. Its immutable `spec` contains +identity, decision, StorageClass snapshot, and shared resource intent. Its `status` contains lifecycle, request +references, realized state, and conditions; a finalizer protects cleanup. `StorageRequest.status.modelCache` gains an +immutable request selection containing mode, binding name/UID, and deterministic namespace-local reader intent, plus a +request finalizer. The UID is the API-assigned `ModelCacheBinding.metadata.uid`; it is not part of binding `spec`. API +validation rejects binding-spec or request-selection mutation after persistence. + +The versioned StorageClass digest is SHA-256 over canonical JSON containing provisioner, sorted parameters, reclaim +policy, binding mode, mount options, and allowed topologies; list order is preserved. It excludes object metadata and +volume expansion. The catalog digest is SHA-256 over the exact ConfigMap payload. The request records mode `none`, +`ephemeral`, or `durable` and the binding reference when durable. + +Binding rules: + +- Persist the binding and request reference before the first storage side effect. +- All requests for one cache key converge through optimistic concurrency. +- A Lease keyed by the binding serializes at most one active writer. +- Retries and agent restarts reuse the binding. +- Creation uses the recorded deterministic name and Get-before-Create. A retry adopts an object only when its binding + UID and immutable spec match the intent; a mismatch is terminal. +- Record provider data identity only after it exists; do not change it afterward. +- Catalog, feature-gate, and StorageClass updates do not mutate an existing binding. +- Before any resource exists, input drift fails without side effects. +- After a resource exists, reconcile only the binding's resources; never switch providers or delete data because of + drift. +- Do not include Secret contents. + +Cleanup is a state transition, not a list-then-delete check. The reconciler marks the binding `Retiring`, which blocks +new references, while its finalizer prevents deletion. A stale reference is one whose request is absent, has a different +UID, or does not point back to the binding. Request deletion first removes reader resources, then removes the binding +reference, then releases the request finalizer. Provider cleanup starts only at zero references; the binding finalizer +is released only after owned Kubernetes resources are gone. Deleting retained backend data, if required, is an explicit +transition operation, not a response to configuration drift. + +## Model-cache contract + +PVCs are namespace-scoped. One PVC cannot be mounted by Pods in different namespaces. A same-namespace multi-Pod +read-only test is primitive evidence only. + +A regular or Helm model-cache transition can be enabled only if it provides: + +- at most one active writer per cache key, serialized by the binding Lease; +- one provider-backed data identity per cache key; +- no source-sized clone or extra data copy; +- namespace-local reader objects that resolve to the same data identity through a documented provider mechanism; +- no duplicate PVs with one CSI `volumeHandle`; +- `readOnly: true` on both the PVC volume source and each matching `volumeMount`; +- a read-only flag on the filesystem mount observed inside each reader container; +- failed write attempts from every reader Pod; +- a durable populated marker that survives writer cleanup and NVCA restart; +- idempotent creation, observation, retry, and deletion; +- cleanup that cannot delete data while another namespace references it. + +The current webhook sets `PersistentVolumeClaimVolumeSource.readOnly: true` but sets both model-cache `volumeMount` +values to `false`. Runtime implementation must set and test both mount values as `true` before external qualification. + +Kubernetes and generic CSI do not provide a cross-namespace PVC-sharing primitive. Each transition owns its +provider-specific reader-view mapping. A registered transition must document a driver-supported alias or rebind method +that produces namespace-local PVC/PV pairs with distinct CSI `volumeHandle` values resolving to the same backend data. +The generic planner must not synthesize those PVs or assume two PVCs from one class expose the same data. A provider +remains disabled if it cannot meet this contract without a source-sized copy. + +Cross-workflow and cross-domain reuse are not assumed. Either requires its own qualified transition and authorization +contract. + +## Failure, migration, and rollback + +After runtime enforcement is wired, durable selection fails closed for a missing or invalid catalog, a missing or +non-`Retain` `nvcf-sc`, an unknown provisioner or transition, a disabled workflow, or drift before the first side +effect. +Stale resources from the presence-selected sharedfs path are not adopted. + +Failure before a binding or request fallback is persisted creates no storage objects. Failure afterward follows the +recorded transition's cleanup rules. `none` or `ephemeral` is used only when recorded on the request. + +Legacy requests may have no backend or only a coarse backend value. Runtime migration must enforce: + +- requests with existing resources are not rebound to another provider; +- requests without storage side effects may establish or reference a binding; +- ambiguous shared-filesystem resources are not adopted from class presence or labels alone; +- NVMesh retained-primary-PV, sharedfs writer-PVC, and Samba backing-PVC markers are inspected explicitly; +- a migrated request remains stable across retry and restart. + +The runtime change must derive the exact legacy conversion from existing resources and cover it with tests. + +New unencrypted cache requests must reject the legacy `Agent.ModelCache.StorageClassName` override unless it is empty +or `nvcf-sc`. NVMesh encryption is the exception: selection still uses `nvcf-sc`, while the `roxReadOnly` transition records +and validates its deterministic per-NCA derived StorageClass before creating the encrypted writer PVC. The NCA sharing +domain is part of the cache key, and the derived class cannot select a provider. + +Provider-defining StorageClass fields, including the provisioner and parameters, cannot be changed in place. A +provider change is a maintenance operation: + +1. Stop new cache PVC creation. +2. Mark every binding for the current `nvcf-sc` UID `Retiring` so it rejects new references. +3. Drain requests, reach zero references, complete cleanup, and inventory retained data. +4. Verify no active binding references the old UID, then delete and recreate `nvcf-sc`. +5. Verify the new UID, provisioner, configuration digest, provisioning, and mounts. +6. Run functional qualification for the exact deployed configuration. +7. Resume new requests only after catalog and runtime support are deployed. + +Rollback repeats the process with the prior definition. `Retain` prevents automatic deletion; it does not migrate data. + +## Provider qualification + +1. Verify CSI controller health and node-plugin readiness on every eligible CPU and GPU node. +2. Render and verify the exact `nvcf-sc` definition. +3. Run dynamic provisioning and basic mount tests. +4. Prove each cataloged access mode for the exact deployed configuration. +5. Add the catalog entry with both transitions `disabled`. +6. Implement an explicit transition, including its provider-specific reader-view mapping. +7. Run the full workflow suite across namespaces and eligible pools. +8. Record CSI, backend, StorageClass, Kubernetes, node, and dated evidence details. +9. Set only a passing workflow to its implemented transition. +10. Deploy catalog and runtime changes together, then verify new bindings. + +A disabled entry does not enable cache traffic. + +## Test plan + +### Catalog and deployment + +- Missing namespace, ConfigMap, key, payload, and chart file. +- Malformed YAML, unknown fields, missing or invalid access modes, missing reader mount options, and missing transitions. +- Whitespace-only IDs, duplicate modes or options, conflicting options, unknown transitions, and transition misuse. +- `roxReadOnly` requires the NVMesh provisioner and provider, RWO, ROX, `ro`, `norecovery`, and `nouuid`. +- `rwxReadOnly` requires RWX, an empty reader-mount-option array, and the regular workflow. +- Source/release catalog, schema, and template parity. +- Deployment-tooling tests for exact Weka, OCI FSS, OCI Lustre, and generic provider rendering. +- Fixed `nvcf-sc` name and `Retain` policy. +- Deployment-tooling tests for exact parameters, mount options, binding mode, expansion, and topology. +- Deployment tooling preserves NVMesh expansion; the catalog selects `roxReadOnly` for both workflows. +- Malformed deployment input fails rendering. +- `nvcf-function-storage-sc` remains unchanged. + +### Binding and migration + +- Feature gates select the request fallback without creating a durable binding. +- The NVMesh provisioner entry selects `roxReadOnly` for both workflows. +- StorageClass and catalog digests are deterministic, sensitive to included fields, and captured with UIDs. +- Unknown provisioner, disabled transition, and non-`Retain` class. +- Concurrent namespaces in one sharing domain converge on one binding; other workflows or domains use different keys. +- Retry, leader change, and agent restart reuse the binding and Lease. +- Drift before the first side effect fails without resources; drift afterward never switches providers. +- A crash after object creation adopts only an exact deterministic intent and binding UID; a mismatch fails closed. +- `Active` to `Retiring` blocks new references; cleanup waits for zero references and honors the finalizer. +- Request deletion removes reader resources and the binding reference before releasing its finalizer. +- Legacy empty-backend, NVMesh PV, sharedfs PVC, and Samba PVC marker migration. +- Legacy class-override rejection and NVMesh encrypted derived-class validation. +- Provider replacement retires every binding for the old StorageClass UID before new requests resume. +- Stale shared-filesystem guard and cleanup. + +### Workflow + +- The Lease permits one active writer, including when competing writer Pods share a node. +- Readers in multiple namespaces mount the same provider data identity. +- CPU and GPU readers resolve the same provider data identity. +- Full-file and tree checksum equality. +- Denied create, append, rename, chmod, truncate, and delete. +- Both volume source and volume mount are read-only. +- A same-provider baseline mount is writable by the test UID/GID; the reader reports `ro` in `/proc/self/mountinfo`. +- Each cataloged PVC access mode is created and mounted explicitly. +- An RWX claim with a read-only Pod mount is not reported as ROX evidence. +- Writer and reader restart. +- Reader recreation and rescheduling to another eligible node. +- Agent restart at every lifecycle phase. +- Cancellation and injected provision, mount, writer, and cleanup failures. +- Idempotent retries with no duplicate writer or leaked resources. +- Cleanup with active and inactive readers. +- Upgrade and rollback with existing bindings. +- No source-sized clone, extra copy, or duplicate CSI handle. + +Performance testing is a separate follow-up after this functional suite passes. + +## Security and observability + +Security requirements: + +- Store no credentials, Secret contents, tokens, or private endpoints; use CSI-supported Secret references. +- Grant least-privilege read access to the StorageClass and catalog. +- Set read-only intent at every Kubernetes layer and verify write denial. +- Reject unknown fields and transitions. +- Qualification evidence records the exact deployed configuration; published evidence redacts credentials and private + deployment values while preserving the configuration digest. + +Observability requirements: + +- Log and trace binding creation, reuse, fallback, drift, transition, and cleanup. +- Count binding outcomes with bounded labels and publish persistence, population, reader, and cleanup conditions. +- Do not use cache handles, PVC names, backend IDs, or binding digests as metric labels. + +## Implementation order + +Keep external transitions disabled. Add the binding API and strict-loader call site, migrate legacy state, fix read-only +mounts, implement provider-specific no-copy transitions, remove presence selection, then qualify each workflow. + +## Public source references + +- [Storage catalog](https://github.com/NVIDIA/nvcf/blob/main/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml) +- [Catalog JSON Schema](https://github.com/NVIDIA/nvcf/blob/main/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.schema.json) +- [Catalog loader and validator](https://github.com/NVIDIA/nvcf/blob/main/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go) +- [Current Helm cache selection](https://github.com/NVIDIA/nvcf/blob/main/src/compute-plane-services/nvca/pkg/storage/cachebackend.go) +- [Current model-cache webhook](https://github.com/NVIDIA/nvcf/blob/main/src/compute-plane-services/nvca/pkg/webhook/helm_storage_webhook.go) +- [Kubernetes persistent-volume access modes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes) +- [Runtime integration issue](https://github.com/NVIDIA/nvcf/issues/1326) diff --git a/fern/versions/dev.yml b/fern/versions/dev.yml index 4aaee4976..e05048c70 100644 --- a/fern/versions/dev.yml +++ b/fern/versions/dev.yml @@ -264,6 +264,8 @@ navigation: contents: - page: Architecture Overview path: ../../docs/dev/architecture.md + - page: Storage-Agnostic Cache Architecture + path: ../../docs/dev/sdd-storage-agnostic-cache-architecture.md - section: Local Development skip-slug: true contents: diff --git a/src/compute-plane-services/nvca/BUILD.bazel b/src/compute-plane-services/nvca/BUILD.bazel index 207e5c40a..ac051be48 100644 --- a/src/compute-plane-services/nvca/BUILD.bazel +++ b/src/compute-plane-services/nvca/BUILD.bazel @@ -15,6 +15,15 @@ load("@gazelle//:def.bzl", "gazelle") +filegroup( + name = "storage-capability-catalog", + srcs = [ + "deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.schema.json", + "deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml", + ], + visibility = ["//src/compute-plane-services/nvca/pkg/storage:__pkg__"], +) + # gazelle:prefix github.com/NVIDIA/nvcf/src/compute-plane-services/nvca # gazelle:proto disable_global # gazelle:go_naming_convention import_alias diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/README.md b/src/compute-plane-services/nvca/deployments/nvca-operator/README.md index f1135082c..457e105a7 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/README.md +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/README.md @@ -3,6 +3,12 @@ NVCF Cluster Agent (NVCA) Operator installs and manages reconfiguration, upgrades, and health checks of NVCA used in Kubernetes Clusters to run NVCF Workloads. +## Storage capability catalog + +The chart installs the versioned `nvcf-storage-capabilities` ConfigMap in the Helm release namespace. For each exact CSI provisioner the catalog records only the PVC access modes qualified end to end in an NVCF cache workflow, plus the mount options for reader volumes NVCA creates. Nothing about the flow is declared: NVCA derives it from those modes. An empty `accessModes` list means nothing is qualified yet, so caching stays off for that driver. Container cache is outside NVCA and is not part of this catalog. + +This release does not wire the catalog into backend selection. Runtime use requires a durable cache plan and safe legacy-request migration so retries and agent restarts cannot change backends. Managed deployments will inspect the exact `nvcf-sc` StorageClass when that follow-up is implemented. Editing this ConfigMap does not enable a storage backend today. + ## Parameters ### NVCA Operator parameters diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.schema.json b/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.schema.json new file mode 100644 index 000000000..6d55f32f1 --- /dev/null +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.schema.json @@ -0,0 +1,199 @@ +{ + "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://nvcf.nvidia.com/schemas/storage-capability-catalog-v1alpha1.json", + "title": "NVCF storage capability catalog", + "description": "PVC access modes qualified per exact CSI provisioner. NVCA derives the cache flow from these modes; nothing about the flow is declared here.", + "type": "object", + "additionalProperties": false, + "required": [ + "apiVersion", + "kind", + "drivers" + ], + "properties": { + "apiVersion": { + "const": "storage.nvcf.nvidia.com/v1alpha1" + }, + "kind": { + "const": "StorageCapabilityCatalog" + }, + "drivers": { + "type": "object", + "minProperties": 1, + "propertyNames": { + "type": "string", + "minLength": 1, + "pattern": "\\S" + }, + "additionalProperties": { + "$ref": "#/$defs/driver" + } + } + }, + "$defs": { + "accessMode": { + "type": "string", + "enum": [ + "ReadWriteOnce", + "ReadOnlyMany", + "ReadWriteMany" + ] + }, + "driver": { + "type": "object", + "additionalProperties": false, + "required": [ + "provider", + "accessModes", + "readerMountOptions" + ], + "properties": { + "provider": { + "type": "string", + "minLength": 1, + "pattern": "\\S" + }, + "accessModes": { + "description": "Modes qualified end to end in an NVCF cache workflow, not modes the driver accepts. Empty means the driver is not enabled.", + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/$defs/accessMode" + } + }, + "readerMountOptions": { + "description": "Mount options for reader PVs NVCA creates. Only the ReadWriteOnce plus ReadOnlyMany shape creates them, and it must include \"ro\".", + "type": "array", + "uniqueItems": true, + "items": { + "$ref": "#/$defs/readerMountOption" + }, + "allOf": [ + { + "not": { + "allOf": [ + { + "contains": { + "const": "ro" + } + }, + { + "contains": { + "const": "rw" + } + } + ] + } + }, + { + "not": { + "allOf": [ + { + "contains": { + "const": "recovery" + } + }, + { + "contains": { + "const": "norecovery" + } + } + ] + } + }, + { + "not": { + "allOf": [ + { + "contains": { + "const": "uuid" + } + }, + { + "contains": { + "const": "nouuid" + } + } + ] + } + } + ] + } + }, + "allOf": [ + { + "$comment": "A driver qualified for the ReadOnlyMany reader shape creates reader PVs, so it must mount them read-only.", + "if": { + "properties": { + "accessModes": { + "allOf": [ + { + "contains": { + "const": "ReadWriteOnce" + } + }, + { + "contains": { + "const": "ReadOnlyMany" + } + } + ] + } + }, + "required": [ + "accessModes" + ] + }, + "then": { + "properties": { + "readerMountOptions": { + "contains": { + "const": "ro" + } + } + } + } + }, + { + "$comment": "ReadOnlyMany describes readers. Without a writer mode alongside it there is nothing to populate the cache.", + "if": { + "properties": { + "accessModes": { + "contains": { + "const": "ReadOnlyMany" + } + } + }, + "required": [ + "accessModes" + ] + }, + "then": { + "properties": { + "accessModes": { + "anyOf": [ + { + "contains": { + "const": "ReadWriteOnce" + } + }, + { + "contains": { + "const": "ReadWriteMany" + } + } + ] + } + } + } + } + ] + }, + "readerMountOption": { + "type": "string", + "minLength": 1, + "pattern": "^\\S(?:.*\\S)?$" + } + } +} diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml b/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml new file mode 100644 index 000000000..377d9c3e5 --- /dev/null +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NVCA owns this catalog and installs it with the NVCA chart. It records, for +# each exact CSI provisioner, the PVC access modes qualified end to end in an +# NVCF cache workflow. Nothing else is declared: NVCA derives how caching runs +# from these modes. +# +# ReadWriteMany one shared claim; readers mount it read-only +# ReadWriteOnce+ReadOnlyMany writer takes the claim; readers get their own +# +# An access mode a driver merely accepts is not a qualification. A claim that +# binds is not either. An empty accessModes list means nothing is qualified +# yet, so caching stays off for that driver, and a provisioner absent from this +# file is unsupported. Enabling a backend is an edit here, backed by a +# qualification run. +apiVersion: storage.nvcf.nvidia.com/v1alpha1 +kind: StorageCapabilityCatalog +drivers: + nvmesh-csi.excelero.com: + provider: nvmesh + accessModes: + - ReadWriteOnce + - ReadOnlyMany + # The reader PV is XFS on the same filesystem as the writer, so it needs + # nouuid and norecovery or the mount fails outright. + readerMountOptions: + - ro + - norecovery + - nouuid + csi.weka.io: + provider: weka + # Fresh ReadWriteMany and ReadOnlyMany claims were tested, but a cache + # workflow was not, so nothing is qualified yet. + accessModes: [] + readerMountOptions: [] + fss.csi.oraclecloud.com: + provider: ociFss + # A ReadWriteMany claim was tested; its readers used read-only Pod mounts, + # which is not evidence for a ReadOnlyMany claim. No cache workflow was + # qualified. + accessModes: [] + readerMountOptions: [] + lustre.csi.oraclecloud.com: + provider: ociLustre + # No PVC access mode has been qualified in an NVCF cache workflow. + accessModes: [] + readerMountOptions: [] diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/storage-capabilities-configmap.yaml b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/storage-capabilities-configmap.yaml new file mode 100644 index 000000000..83bb433c0 --- /dev/null +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/storage-capabilities-configmap.yaml @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- $catalogPath := "files/nvcf-storage-capabilities-v1alpha1.yaml" -}} +{{- $catalog := .Files.Get $catalogPath -}} +{{- if not $catalog -}} +{{- fail (printf "required NVCF storage capability catalog %s is missing" $catalogPath) -}} +{{- end }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: nvcf-storage-capabilities + namespace: {{ .Release.Namespace }} + labels: + {{- include "nvcaop.labels" . | nindent 4 }} +data: + storage-provider-capabilities.yaml: | +{{- $catalog | nindent 4 }} diff --git a/src/compute-plane-services/nvca/pkg/storage/BUILD.bazel b/src/compute-plane-services/nvca/pkg/storage/BUILD.bazel index eb31ee50c..d923e8571 100644 --- a/src/compute-plane-services/nvca/pkg/storage/BUILD.bazel +++ b/src/compute-plane-services/nvca/pkg/storage/BUILD.bazel @@ -18,6 +18,7 @@ go_library( "reconcile.go", "sharedstorage.go", "smbcsidriver.go", + "storage_capabilities.go", "storage_request_api.go", "storagerequest.go", "translate_workload.go", @@ -85,6 +86,7 @@ go_library( "//src/compute-plane-services/nvca/vendor/sigs.k8s.io/controller-runtime/pkg/manager", "//src/compute-plane-services/nvca/vendor/sigs.k8s.io/controller-runtime/pkg/predicate", "//src/compute-plane-services/nvca/vendor/sigs.k8s.io/controller-runtime/pkg/reconcile", + "//src/compute-plane-services/nvca/vendor/sigs.k8s.io/yaml", ], ) @@ -108,12 +110,16 @@ go_test( "reconcile_test.go", "sharedstorage_test.go", "smbcsidriver_test.go", + "storage_capabilities_test.go", "storage_request_api_test.go", "storagerequest_test.go", "translate_workload_test.go", "types_test.go", ], - data = ["//src/compute-plane-services/nvca/internal/envtest:crds"], + data = [ + "//src/compute-plane-services/nvca:storage-capability-catalog", + "//src/compute-plane-services/nvca/internal/envtest:crds", + ], embed = [":storage"], env_inherit = ["KUBEBUILDER_ASSETS"], rundir = ".", diff --git a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go new file mode 100644 index 000000000..4b3d81503 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go @@ -0,0 +1,185 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package storage + +import ( + "context" + "fmt" + "strings" + + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" +) + +const ( + storageCapabilityCatalogAPIVersion = "storage.nvcf.nvidia.com/v1alpha1" + storageCapabilityCatalogKind = "StorageCapabilityCatalog" + + // StorageCapabilityConfigMapName is the stable name of the ConfigMap that + // contains NVCA's public CSI provider capability catalog. + StorageCapabilityConfigMapName = "nvcf-storage-capabilities" + // StorageCapabilityConfigMapKey is the ConfigMap data key containing the + // serialized storage capability catalog. + StorageCapabilityConfigMapKey = "storage-provider-capabilities.yaml" + + // ModelCacheTransitionDisabled means no durable cache for a workflow. + ModelCacheTransitionDisabled = "disabled" + // ModelCacheTransitionROXReadOnly populates a writer claim and publishes a + // separate ReadOnlyMany reader claim with read-only Pod mounts. + ModelCacheTransitionROXReadOnly = "roxReadOnly" + // ModelCacheTransitionRWXReadOnly populates one ReadWriteMany claim and + // serves that same claim through read-only Pod mounts. + ModelCacheTransitionRWXReadOnly = "rwxReadOnly" + // ModelCacheProviderNVMesh is the provider id the catalog uses for NVMesh. + // It names a driver family; it does not gate what a driver may run. + ModelCacheProviderNVMesh = "nvmesh" +) + +type storageCapabilityCatalog struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Drivers map[string]storageDriverSpec `json:"drivers"` +} + +// storageDriverSpec records what a qualification run established for one exact +// CSI provisioner, and nothing more. How NVCA caches on that driver is derived +// from the access modes, not declared here: a ReadWriteMany claim is shared and +// mounted read-only, a ReadWriteOnce plus ReadOnlyMany pair gives readers their +// own claim, and neither means no durable cache. +type storageDriverSpec struct { + Provider string `json:"provider"` + // AccessModes are the modes qualified end to end in a cache workflow, not + // the modes the driver will accept. An empty list means nothing is + // qualified yet and caching stays off for that driver. A pointer so that an + // absent field is rejected rather than read as empty. + AccessModes *[]string `json:"accessModes"` + // ReaderMountOptions apply to reader PVs NVCA creates, which only the + // ReadWriteOnce plus ReadOnlyMany shape does. Vendor specific options + // belong here rather than in code: norecovery and nouuid are NVMesh XFS + // requirements and apply to no other driver. + ReaderMountOptions *[]string `json:"readerMountOptions"` +} + +func loadStorageCapabilityCatalog( + ctx context.Context, + c client.Client, + namespace string, +) (*storageCapabilityCatalog, error) { + if namespace == "" { + return nil, fmt.Errorf("storage capability ConfigMap namespace is empty") + } + + cm := &corev1.ConfigMap{} + if err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: StorageCapabilityConfigMapName}, cm); err != nil { + return nil, fmt.Errorf("get storage capability ConfigMap %s/%s: %w", + namespace, StorageCapabilityConfigMapName, err) + } + + raw, ok := cm.Data[StorageCapabilityConfigMapKey] + if !ok || raw == "" { + return nil, fmt.Errorf("storage capability ConfigMap %s/%s has no %q data", + namespace, StorageCapabilityConfigMapName, StorageCapabilityConfigMapKey) + } + + catalog := &storageCapabilityCatalog{} + if err := yaml.UnmarshalStrict([]byte(raw), catalog); err != nil { + return nil, fmt.Errorf("parse storage capability catalog: %w", err) + } + if err := validateStorageCapabilityCatalog(catalog); err != nil { + return nil, err + } + return catalog, nil +} + +func validateStorageCapabilityCatalog(catalog *storageCapabilityCatalog) error { + if catalog.APIVersion != storageCapabilityCatalogAPIVersion { + return fmt.Errorf("unsupported storage capability apiVersion %q", catalog.APIVersion) + } + if catalog.Kind != storageCapabilityCatalogKind { + return fmt.Errorf("unsupported storage capability kind %q", catalog.Kind) + } + if len(catalog.Drivers) == 0 { + return fmt.Errorf("storage capability catalog has no drivers") + } + + for provisioner, driver := range catalog.Drivers { + if strings.TrimSpace(provisioner) == "" || strings.TrimSpace(driver.Provider) == "" { + return fmt.Errorf("storage capability catalog has an empty provisioner or provider") + } + if driver.AccessModes == nil { + return fmt.Errorf("driver %q has no accessModes", provisioner) + } + if driver.ReaderMountOptions == nil { + return fmt.Errorf("driver %q has no readerMountOptions", provisioner) + } + readerMountOptions := make(map[string]bool, len(*driver.ReaderMountOptions)) + for i, option := range *driver.ReaderMountOptions { + if strings.TrimSpace(option) == "" { + return fmt.Errorf("driver %q has blank readerMountOption", provisioner) + } + if strings.TrimSpace(option) != option { + return fmt.Errorf("driver %q has readerMountOption %q with surrounding whitespace", provisioner, option) + } + if readerMountOptions[option] { + return fmt.Errorf("driver %q has duplicate readerMountOption %q", provisioner, option) + } + for _, previous := range (*driver.ReaderMountOptions)[:i] { + if negatesMountOption(previous, option) { + return fmt.Errorf("driver %q readerMountOptions %q and %q conflict", + provisioner, previous, option) + } + } + readerMountOptions[option] = true + } + accessModes := make(map[string]bool, len(*driver.AccessModes)) + for _, mode := range *driver.AccessModes { + switch mode { + case string(corev1.ReadWriteOnce), string(corev1.ReadOnlyMany), string(corev1.ReadWriteMany): + default: + return fmt.Errorf("driver %q has invalid accessMode %q", provisioner, mode) + } + if accessModes[mode] { + return fmt.Errorf("driver %q has duplicate accessMode %q", provisioner, mode) + } + accessModes[mode] = true + } + + // A driver qualified for the ReadWriteOnce plus ReadOnlyMany shape gets + // reader PVs that NVCA creates, so it must say how to mount them + // read-only. The shared claim shape creates no reader PV, so it needs + // no options. + // ReadOnlyMany describes readers. Without a writer mode alongside it + // there is nothing to populate the cache. + if accessModes[string(corev1.ReadOnlyMany)] && + !accessModes[string(corev1.ReadWriteOnce)] && !accessModes[string(corev1.ReadWriteMany)] { + return fmt.Errorf( + "driver %q qualifies ReadOnlyMany with no writer mode, "+ + "it needs ReadWriteOnce or ReadWriteMany", + provisioner) + } + if accessModes[string(corev1.ReadWriteOnce)] && accessModes[string(corev1.ReadOnlyMany)] && + !readerMountOptions["ro"] { + return fmt.Errorf( + "driver %q qualifies for the ReadOnlyMany reader shape and must list readerMountOption %q", + provisioner, "ro") + } + } + + return nil +} diff --git a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go new file mode 100644 index 000000000..41137a672 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go @@ -0,0 +1,408 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package storage + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +const testCatalogNamespace = "nvca-system" + +func capabilityCatalogConfigMap(raw string) *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: StorageCapabilityConfigMapName, Namespace: testCatalogNamespace}, + Data: map[string]string{StorageCapabilityConfigMapKey: raw}, + } +} + +func capabilityClient(t *testing.T, cm *corev1.ConfigMap) *fake.ClientBuilder { + t.Helper() + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + return fake.NewClientBuilder().WithScheme(scheme).WithObjects(cm) +} + +const validCatalog = `apiVersion: storage.nvcf.nvidia.com/v1alpha1 +kind: StorageCapabilityCatalog +drivers: + nvmesh-csi.excelero.com: + provider: nvmesh + accessModes: [ReadWriteOnce, ReadOnlyMany] + readerMountOptions: [ro, norecovery, nouuid] +` + +func accessModes(modes ...string) *[]string { + return &modes +} + +func readerMountOptions(options ...string) *[]string { + return &options +} + +func validStorageCapabilityCatalog() *storageCapabilityCatalog { + return &storageCapabilityCatalog{ + APIVersion: storageCapabilityCatalogAPIVersion, + Kind: storageCapabilityCatalogKind, + Drivers: map[string]storageDriverSpec{ + NVMeshStorageClassProvisioner: { + Provider: ModelCacheProviderNVMesh, + AccessModes: accessModes("ReadWriteOnce", "ReadOnlyMany"), + ReaderMountOptions: readerMountOptions("ro", "norecovery", "nouuid"), + }, + }, + } +} + +func TestLoadStorageCapabilityCatalogStrict(t *testing.T) { + c := capabilityClient(t, capabilityCatalogConfigMap(validCatalog)).Build() + + catalog, err := loadStorageCapabilityCatalog(t.Context(), c, testCatalogNamespace) + require.NoError(t, err) + nvmesh := catalog.Drivers[NVMeshStorageClassProvisioner] + require.NotNil(t, nvmesh.AccessModes) + assert.ElementsMatch(t, []string{"ReadWriteOnce", "ReadOnlyMany"}, *nvmesh.AccessModes) + require.NotNil(t, nvmesh.ReaderMountOptions) + assert.Equal(t, []string{"ro", "norecovery", "nouuid"}, *nvmesh.ReaderMountOptions) +} + +func TestLoadStorageCapabilityCatalogErrors(t *testing.T) { + missingConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "other", Namespace: testCatalogNamespace}, + } + missingData := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: StorageCapabilityConfigMapName, Namespace: testCatalogNamespace}, + } + missingAccessModes := strings.Replace( + validCatalog, " accessModes: [ReadWriteOnce, ReadOnlyMany]\n", "", 1) + nullAccessModes := strings.Replace( + validCatalog, " accessModes: [ReadWriteOnce, ReadOnlyMany]", " accessModes: null", 1) + missingReaderMountOptions := strings.Replace( + validCatalog, " readerMountOptions: [ro, norecovery, nouuid]\n", "", 1) + nullReaderMountOptions := strings.Replace( + validCatalog, " readerMountOptions: [ro, norecovery, nouuid]", " readerMountOptions: null", 1) + + tests := []struct { + name string + namespace string + configMap *corev1.ConfigMap + want string + }{ + { + name: "empty namespace", namespace: "", + configMap: capabilityCatalogConfigMap(validCatalog), want: "namespace is empty", + }, + { + name: "missing ConfigMap", namespace: testCatalogNamespace, + configMap: missingConfigMap, want: "get storage capability ConfigMap", + }, + { + name: "missing data key", namespace: testCatalogNamespace, + configMap: missingData, want: "has no", + }, + { + name: "empty data", namespace: testCatalogNamespace, + configMap: capabilityCatalogConfigMap(""), want: "has no", + }, + { + name: "malformed YAML", namespace: testCatalogNamespace, + configMap: capabilityCatalogConfigMap("drivers: ["), want: "parse storage capability catalog", + }, + { + name: "missing accessModes", namespace: testCatalogNamespace, + configMap: capabilityCatalogConfigMap(missingAccessModes), want: "has no accessModes", + }, + { + name: "null accessModes", namespace: testCatalogNamespace, + configMap: capabilityCatalogConfigMap(nullAccessModes), want: "has no accessModes", + }, + { + name: "missing readerMountOptions", namespace: testCatalogNamespace, + configMap: capabilityCatalogConfigMap(missingReaderMountOptions), want: "has no readerMountOptions", + }, + { + name: "null readerMountOptions", namespace: testCatalogNamespace, + configMap: capabilityCatalogConfigMap(nullReaderMountOptions), want: "has no readerMountOptions", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := capabilityClient(t, tt.configMap).Build() + _, err := loadStorageCapabilityCatalog(t.Context(), c, tt.namespace) + require.ErrorContains(t, err, tt.want) + }) + } +} + +func TestLoadStorageCapabilityCatalogRejectsUnknownFields(t *testing.T) { + tests := []struct { + name string + raw string + want string + }{ + { + name: "driver field", + raw: strings.Replace(validCatalog, " provider: nvmesh", " provider: nvmesh\n surprise: true", 1), + want: "unknown field \"surprise\"", + }, + { + name: "declared transition", + raw: strings.Replace(validCatalog, " provider: nvmesh", + " provider: nvmesh\n transitions:\n helmModelCache: roxReadOnly", 1), + want: "unknown field \"transitions\"", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := capabilityClient(t, capabilityCatalogConfigMap(tt.raw)).Build() + _, err := loadStorageCapabilityCatalog(t.Context(), c, testCatalogNamespace) + require.ErrorContains(t, err, tt.want) + }) + } +} + +func TestValidateStorageCapabilityCatalog(t *testing.T) { + tests := []struct { + name string + mutate func(*storageCapabilityCatalog) + want string + }{ + {name: "bad apiVersion", mutate: func(c *storageCapabilityCatalog) { c.APIVersion = "v1" }, want: "apiVersion"}, + {name: "bad kind", mutate: func(c *storageCapabilityCatalog) { c.Kind = "ConfigMap" }, want: "kind"}, + {name: "empty drivers", mutate: func(c *storageCapabilityCatalog) { c.Drivers = nil }, want: "no drivers"}, + {name: "empty provider", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.Provider = "" + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: "empty provisioner or provider"}, + {name: "whitespace provider", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.Provider = " \t" + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: "empty provisioner or provider"}, + {name: "whitespace provisioner", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + delete(c.Drivers, NVMeshStorageClassProvisioner) + c.Drivers[" \t"] = d + }, want: "empty provisioner or provider"}, + {name: "missing access modes", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.AccessModes = nil + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: "has no accessModes"}, + {name: "bad access mode", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.AccessModes = accessModes(append(*d.AccessModes, "ReadWriteEverywhere")...) + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: "invalid accessMode"}, + {name: "duplicate access mode", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.AccessModes = accessModes(append(*d.AccessModes, "ReadWriteOnce")...) + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: "duplicate accessMode"}, + {name: "missing reader mount options", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.ReaderMountOptions = nil + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: "has no readerMountOptions"}, + {name: "blank reader mount option", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.ReaderMountOptions = readerMountOptions(append(*d.ReaderMountOptions, " \t")...) + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: "blank readerMountOption"}, + {name: "reader mount option with surrounding whitespace", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.ReaderMountOptions = readerMountOptions("ro", " norecovery", "nouuid") + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: "surrounding whitespace"}, + {name: "duplicate reader mount option", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.ReaderMountOptions = readerMountOptions(append(*d.ReaderMountOptions, "ro")...) + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: "duplicate readerMountOption"}, + {name: "conflicting reader mount options", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.ReaderMountOptions = readerMountOptions("ro", "norecovery", "nouuid", "rw") + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: `readerMountOptions "ro" and "rw" conflict`}, + {name: "conflicting recovery reader mount options", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.ReaderMountOptions = readerMountOptions("ro", "recovery", "norecovery", "nouuid") + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: `readerMountOptions "recovery" and "norecovery" conflict`}, + {name: "conflicting UUID reader mount options", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.ReaderMountOptions = readerMountOptions("ro", "norecovery", "uuid", "nouuid") + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: `readerMountOptions "uuid" and "nouuid" conflict`}, + {name: "ReadOnlyMany reader shape lacks ro reader mount option", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.ReaderMountOptions = readerMountOptions("norecovery", "nouuid") + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: `must list readerMountOption "ro"`}, + {name: "ReadOnlyMany without ReadWriteOnce has no writer", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.AccessModes = accessModes("ReadOnlyMany") + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: "ReadOnlyMany with no writer mode"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + catalog := validStorageCapabilityCatalog() + tt.mutate(catalog) + require.ErrorContains(t, validateStorageCapabilityCatalog(catalog), tt.want) + }) + } +} + +// TestValidateStorageCapabilityCatalogAccepts covers shapes that are valid +// because the catalog records qualified access modes only. NVCA derives the +// flow, so nothing here is restricted to a known provisioner or provider, and +// filesystem specific reader options are per driver data rather than a rule. +func TestValidateStorageCapabilityCatalogAccepts(t *testing.T) { + tests := []struct { + name string + mutate func(*storageCapabilityCatalog) + }{ + {name: "reader shape needs only ro", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.ReaderMountOptions = readerMountOptions("ro") + c.Drivers[NVMeshStorageClassProvisioner] = d + }}, + {name: "any provisioner may qualify for the reader shape", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.Provider = "example" + delete(c.Drivers, NVMeshStorageClassProvisioner) + c.Drivers["example.csi.test"] = d + }}, + {name: "ReadWriteOnce alone qualifies for nothing", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.AccessModes = accessModes("ReadWriteOnce") + d.ReaderMountOptions = readerMountOptions() + c.Drivers[NVMeshStorageClassProvisioner] = d + }}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + catalog := validStorageCapabilityCatalog() + tt.mutate(catalog) + require.NoError(t, validateStorageCapabilityCatalog(catalog)) + }) + } +} + +// TestValidateStorageCapabilityCatalogAcceptsSharedClaimDriver covers a driver +// NVCA has no special knowledge of. A shared claim creates no reader PV, so it +// needs no reader mount options, and listing some is not an error either: the +// options simply go unused. +func TestValidateStorageCapabilityCatalogAcceptsSharedClaimDriver(t *testing.T) { + const provisioner = "shared.csi.example.com" + catalog := validStorageCapabilityCatalog() + catalog.Drivers[provisioner] = storageDriverSpec{ + Provider: "someVendor", + AccessModes: accessModes("ReadWriteMany", "ReadOnlyMany"), + ReaderMountOptions: readerMountOptions(), + } + require.NoError(t, validateStorageCapabilityCatalog(catalog)) +} + +// TestValidateStorageCapabilityCatalogRequiresReadOnlyReaderMount pins the one +// rule the reader shape still carries: a driver qualified for ReadWriteOnce +// plus ReadOnlyMany gets reader PVs that NVCA creates, so it has to say how to +// mount them read-only. +func TestValidateStorageCapabilityCatalogRequiresReadOnlyReaderMount(t *testing.T) { + catalog := validStorageCapabilityCatalog() + driver := catalog.Drivers[NVMeshStorageClassProvisioner] + driver.ReaderMountOptions = readerMountOptions("norecovery", "nouuid") + catalog.Drivers[NVMeshStorageClassProvisioner] = driver + + require.ErrorContains(t, validateStorageCapabilityCatalog(catalog), + `must list readerMountOption "ro"`) +} + +// TestValidateStorageCapabilityCatalogAllowsNothingQualified covers how an +// unqualified driver is recorded: present in the catalog with no access modes, +// rather than absent or rejected. +func TestValidateStorageCapabilityCatalogAllowsNothingQualified(t *testing.T) { + catalog := validStorageCapabilityCatalog() + driver := catalog.Drivers[NVMeshStorageClassProvisioner] + driver.AccessModes = accessModes() + driver.ReaderMountOptions = readerMountOptions() + catalog.Drivers[NVMeshStorageClassProvisioner] = driver + + require.NoError(t, validateStorageCapabilityCatalog(catalog)) +} + +func TestShippedStorageCapabilityCatalog(t *testing.T) { + chartDir := filepath.Join("src", "compute-plane-services", "nvca", "deployments", "nvca-operator") + if _, err := os.Stat(chartDir); os.IsNotExist(err) { + chartDir = filepath.Join("..", "..", "deployments", "nvca-operator") + } + raw, err := os.ReadFile(filepath.Join(chartDir, "files", "nvcf-storage-capabilities-v1alpha1.yaml")) + require.NoError(t, err) + c := capabilityClient(t, capabilityCatalogConfigMap(string(raw))).Build() + catalog, err := loadStorageCapabilityCatalog(t.Context(), c, testCatalogNamespace) + require.NoError(t, err) + + nvmesh := catalog.Drivers[NVMeshStorageClassProvisioner] + require.NotNil(t, nvmesh.AccessModes) + assert.ElementsMatch(t, []string{"ReadWriteOnce", "ReadOnlyMany"}, *nvmesh.AccessModes) + require.NotNil(t, nvmesh.ReaderMountOptions) + assert.Equal(t, []string{"ro", "norecovery", "nouuid"}, *nvmesh.ReaderMountOptions) + + // Weka, FSS and Lustre are recorded but not qualified for a cache workflow, + // so they carry no access modes and caching stays off for them. Enabling + // one is an edit to its accessModes, backed by a qualification run. + for _, provisioner := range []string{"csi.weka.io", "fss.csi.oraclecloud.com", "lustre.csi.oraclecloud.com"} { + driver, ok := catalog.Drivers[provisioner] + require.True(t, ok, provisioner) + require.NotNil(t, driver.AccessModes, provisioner) + assert.Empty(t, *driver.AccessModes, provisioner) + require.NotNil(t, driver.ReaderMountOptions, provisioner) + assert.Empty(t, *driver.ReaderMountOptions, provisioner) + } + + schemaRaw, err := os.ReadFile(filepath.Join(chartDir, "files", "nvcf-storage-capabilities-v1alpha1.schema.json")) + require.NoError(t, err) + assert.True(t, json.Valid(schemaRaw), "shipped JSON schema must be valid JSON") + var schema map[string]any + require.NoError(t, json.Unmarshal(schemaRaw, &schema)) + definitions, ok := schema["$defs"].(map[string]any) + require.True(t, ok) + accessMode, ok := definitions["accessMode"].(map[string]any) + require.True(t, ok) + assert.ElementsMatch(t, + []any{"ReadWriteOnce", "ReadOnlyMany", "ReadWriteMany"}, accessMode["enum"], + "the schema constrains qualified access modes, and declares no cache flow") + assert.NotContains(t, definitions, "transitions", + "the flow is derived from access modes, never declared") +} diff --git a/src/compute-plane-services/nvca/scripts/lint_helm.sh b/src/compute-plane-services/nvca/scripts/lint_helm.sh index 7ae22ca94..d4084346b 100755 --- a/src/compute-plane-services/nvca/scripts/lint_helm.sh +++ b/src/compute-plane-services/nvca/scripts/lint_helm.sh @@ -51,10 +51,10 @@ assert_eq() { local message=${3} if [[ "${got}" != "${want}" ]]; then - printf '✗ %s: got %q, want %q\n' "${message}" "${got}" "${want}" >&2 + printf 'FAIL %s: got %q, want %q\n' "${message}" "${got}" "${want}" >&2 return 1 fi - printf '✓ %s\n' "${message}" + printf 'ok %s\n' "${message}" } assert_pre_delete_cleanup_rbac() { @@ -91,6 +91,155 @@ assert_pre_delete_cleanup_rbac() { "pre-delete cleanup hook RBAC is kept for the running Job" } +assert_storage_capability_catalog() ( + local service_chart="${repo_root}/deployments/nvca-operator" + local release_chart="${repo_root}/../../../deploy/helm/nvca-operator/nvca-operator" + local catalog="files/nvcf-storage-capabilities-v1alpha1.yaml" + local schema="files/nvcf-storage-capabilities-v1alpha1.schema.json" + local template="templates/storage-capabilities-configmap.yaml" + local rendered missing_chart schema_python tmpdir invalid_catalog schema_check + tmpdir="$(mktemp -d)" + trap 'rm -rf "${tmpdir}"' EXIT + missing_chart="${tmpdir}/missing-chart" + schema_python="${tmpdir}/venv/bin/python" + python3 -m venv --system-site-packages "${tmpdir}/venv" + "${schema_python}" -m pip install --disable-pip-version-check --quiet \ + --requirement "${repo_root}/scripts/requirements-lint.txt" + schema_check='import json,sys,yaml,jsonschema; schema=json.load(open(sys.argv[1])); jsonschema.Draft202012Validator.check_schema(schema); jsonschema.Draft202012Validator(schema).validate(yaml.safe_load(open(sys.argv[2])))' + + for relative in "${catalog}" "${schema}" "${template}"; do + diff -u "${service_chart}/${relative}" "${release_chart}/${relative}" + done + + rendered="${tmpdir}/rendered.yaml" + helm template test-release "${service_chart}" --namespace nvca-system \ + --set "ngcConfig.serviceKey=fakekey" \ + --show-only "${template}" >"${rendered}" + assert_eq "nvcf-storage-capabilities" "$(yq -r ".metadata.name" "${rendered}")" \ + "storage capability ConfigMap uses the stable name" + assert_eq "nvca-system" "$(yq -r ".metadata.namespace" "${rendered}")" \ + "storage capability ConfigMap is owned by the chart release namespace" + assert_eq "$(<"${service_chart}/${catalog}")" \ + "$(yq -r ".data.\"storage-provider-capabilities.yaml\"" "${rendered}")" \ + "storage capability ConfigMap embeds the exact catalog payload" + "${schema_python}" -c "${schema_check}" \ + "${service_chart}/${schema}" "${service_chart}/${catalog}" + + invalid_catalog="${tmpdir}/invalid-catalog.yaml" + for mutation in \ + 'del(.drivers."csi.weka.io".accessModes)' \ + '.drivers."csi.weka.io".accessModes = null'; do + yq "${mutation}" "${service_chart}/${catalog}" >"${invalid_catalog}" + if "${schema_python}" -c "${schema_check}" \ + "${service_chart}/${schema}" "${invalid_catalog}" 2>/dev/null; then + echo "Expected schema to reject missing or null accessModes" >&2 + return 1 + fi + done + echo "PASS: schema rejects missing and null accessModes" + + for mutation in \ + 'del(.drivers."csi.weka.io".readerMountOptions)' \ + '.drivers."csi.weka.io".readerMountOptions = null'; do + yq "${mutation}" "${service_chart}/${catalog}" >"${invalid_catalog}" + if "${schema_python}" -c "${schema_check}" \ + "${service_chart}/${schema}" "${invalid_catalog}" 2>/dev/null; then + echo "Expected schema to reject missing or null readerMountOptions" >&2 + return 1 + fi + done + echo "PASS: schema rejects missing and null readerMountOptions" + + yq '.drivers."nvmesh-csi.excelero.com".readerMountOptions = ["ro", " norecovery", "nouuid"]' \ + "${service_chart}/${catalog}" >"${invalid_catalog}" + if "${schema_python}" -c "${schema_check}" \ + "${service_chart}/${schema}" "${invalid_catalog}" 2>/dev/null; then + echo "Expected schema to reject readerMountOptions with surrounding whitespace" >&2 + return 1 + fi + echo "PASS: schema rejects readerMountOptions with surrounding whitespace" + + yq '.drivers."csi.weka.io".transitions.regularModelCache = "roxReadOnly"' \ + "${service_chart}/${catalog}" >"${invalid_catalog}" + if "${schema_python}" -c "${schema_check}" \ + "${service_chart}/${schema}" "${invalid_catalog}" 2>/dev/null; then + echo "Expected schema to reject a declared transition" >&2 + return 1 + fi + echo "PASS: schema rejects a declared transition, the flow is derived" + + for options in \ + '["ro", "rw", "norecovery", "nouuid"]' \ + '["ro", "recovery", "norecovery", "nouuid"]' \ + '["ro", "norecovery", "uuid", "nouuid"]'; do + yq ".drivers.\"nvmesh-csi.excelero.com\".readerMountOptions = ${options}" \ + "${service_chart}/${catalog}" >"${invalid_catalog}" + if "${schema_python}" -c "${schema_check}" \ + "${service_chart}/${schema}" "${invalid_catalog}" 2>/dev/null; then + echo "Expected schema to reject conflicting readerMountOptions" >&2 + return 1 + fi + done + echo "PASS: schema rejects conflicting readerMountOptions" + + yq '(.drivers."csi.weka.io".accessModes = ["ReadWriteOnce", "ReadOnlyMany"]) | + (.drivers."csi.weka.io".readerMountOptions = [])' \ + "${service_chart}/${catalog}" >"${invalid_catalog}" + if "${schema_python}" -c "${schema_check}" \ + "${service_chart}/${schema}" "${invalid_catalog}" 2>/dev/null; then + echo "Expected schema to reject a ReadOnlyMany reader shape without ro" >&2 + return 1 + fi + echo "PASS: schema rejects a ReadOnlyMany reader shape without a read-only mount" + + yq '.drivers."csi.weka.io".accessModes = ["ReadOnlyMany"]' \ + "${service_chart}/${catalog}" >"${invalid_catalog}" + if "${schema_python}" -c "${schema_check}" \ + "${service_chart}/${schema}" "${invalid_catalog}" 2>/dev/null; then + echo "Expected schema to reject ReadOnlyMany with no writer mode" >&2 + return 1 + fi + echo "PASS: schema rejects ReadOnlyMany with no writer mode" + + yq '.drivers."csi.weka.io".accessModes = ["ReadWriteMany"]' \ + "${service_chart}/${catalog}" >"${invalid_catalog}" + if ! "${schema_python}" -c "${schema_check}" \ + "${service_chart}/${schema}" "${invalid_catalog}"; then + echo "Expected schema to accept a shared claim driver with no reader options" >&2 + return 1 + fi + echo "PASS: schema accepts a shared claim driver with no reader options" + + yq '.drivers."csi.weka.io".unexpected = true' \ + "${service_chart}/${catalog}" >"${invalid_catalog}" + if "${schema_python}" -c "${schema_check}" \ + "${service_chart}/${schema}" "${invalid_catalog}" 2>/dev/null; then + echo "Expected schema to reject an unknown driver field" >&2 + return 1 + fi + echo "PASS: schema rejects an unknown driver field" + + helm template test-release "${release_chart}" --namespace nvca-system \ + --show-only "${template}" >"${rendered}" + assert_eq "nvca-system" "$(yq -r ".metadata.namespace" "${rendered}")" \ + "release-chart storage capability ConfigMap is owned by the release namespace" + assert_eq "$(<"${service_chart}/${catalog}")" \ + "$(yq -r ".data.\"storage-provider-capabilities.yaml\"" "${rendered}")" \ + "release chart embeds the exact catalog payload" + + mkdir -p "${missing_chart}" + cp -a "${service_chart}/." "${missing_chart}/" + rm -f "${missing_chart}/${catalog}" + if helm template test-release "${missing_chart}" --set "ngcConfig.serviceKey=fakekey" >"${rendered}" 2>&1; then + echo "Expected rendering without the storage capability catalog to fail" >&2 + return 1 + fi + grep -q "required NVCF storage capability catalog" "${rendered}" + echo "PASS: storage capability catalog schema, render, payload, and chart parity" +) + +assert_storage_capability_catalog + # The Bazel-built NVCA operator image is distroless, so the rendered workload # commands must execute the packaged binaries directly. A /tini wrapper would # fail at container startup because that binary is not present in the image. @@ -184,7 +333,7 @@ assert_service_oauth_nil_safe() { exit 1 fi rm -f "${render_output}" - echo "✓ ${label} cluster-dto renders nil-safely without agent.serviceOAuth defaults" + echo "ok ${label} cluster-dto renders nil-safely without agent.serviceOAuth defaults" } assert_service_oauth_nil_safe "helm-managed" \ @@ -290,7 +439,7 @@ assert_transport_trust_config() { exit 1 fi rm -f "${render_output}" - echo "✓ ${label} workload transport trust ConfigMap renders" + echo "ok ${label} workload transport trust ConfigMap renders" } assert_transport_trust_config "default" "" "" "" --set "ngcConfig.serviceKey=fakekey" @@ -335,22 +484,22 @@ helm template test-release "${repo_root}/deployments/nvca-operator" \ --values "${repo_root}/test/test-network-policies.yaml" \ --set "ngcConfig.serviceKey=fakekey" \ --show-only templates/custom-network-policies-configmap.yaml \ - | grep -q "nvcf-custom-network-policies" && echo "✓ Network policies ConfigMap created" || echo "✗ Network policies ConfigMap missing" + | grep -q "nvcf-custom-network-policies" && echo "ok Network policies ConfigMap created" || echo "FAIL Network policies ConfigMap missing" helm template test-release "${repo_root}/deployments/nvca-operator" \ --values "${repo_root}/test/test-custom-annotations.yaml" \ --set "ngcConfig.serviceKey=fakekey" \ --show-only templates/custom-annotations-configmap.yaml \ - | grep -q "nvca-namespace-pod-annotations" && echo "✓ Custom annotations ConfigMap created" || echo "✗ Custom annotations ConfigMap missing" + | grep -q "nvca-namespace-pod-annotations" && echo "ok Custom annotations ConfigMap created" || echo "FAIL Custom annotations ConfigMap missing" # Test ConfigMaps are created even without custom values (always created behavior) echo "Testing ConfigMaps are always created..." helm template test-release "${repo_root}/deployments/nvca-operator" \ --set "ngcConfig.serviceKey=fakekey" \ --show-only templates/custom-annotations-configmap.yaml \ - | grep -q "nvca-namespace-pod-annotations" && echo "✓ Annotations ConfigMap always created" || echo "✗ Annotations ConfigMap not created" + | grep -q "nvca-namespace-pod-annotations" && echo "ok Annotations ConfigMap always created" || echo "FAIL Annotations ConfigMap not created" helm template test-release "${repo_root}/deployments/nvca-operator" \ --set "ngcConfig.serviceKey=fakekey" \ --show-only templates/custom-network-policies-configmap.yaml \ - | grep -q "nvcf-custom-network-policies" && echo "✓ Network policies ConfigMap always created" || echo "✗ Network policies ConfigMap not created" + | grep -q "nvcf-custom-network-policies" && echo "ok Network policies ConfigMap always created" || echo "FAIL Network policies ConfigMap not created" diff --git a/src/compute-plane-services/nvca/scripts/requirements-lint.txt b/src/compute-plane-services/nvca/scripts/requirements-lint.txt new file mode 100644 index 000000000..5b48b7529 --- /dev/null +++ b/src/compute-plane-services/nvca/scripts/requirements-lint.txt @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +jsonschema==4.10.3 +PyYAML==6.0.2