From c8bc197fa954d692c6a5b19173119d2e776bfa36 Mon Sep 17 00:00:00 2001 From: balaji Date: Fri, 28 Aug 2026 15:14:42 -0700 Subject: [PATCH 1/3] feat(nvca): publish storage capability catalog Signed-off-by: balaji --- dependencies.md | 3 +- .../nvca-operator/nvca-operator/README.md | 6 + ...-storage-capabilities-v1alpha1.schema.json | 101 +++++ .../nvcf-storage-capabilities-v1alpha1.yaml | 56 +++ .../storage-capabilities-configmap.yaml | 30 ++ docs/dev/sdd-central-model-cache-service.md | 70 +-- ...sdd-storage-agnostic-cache-architecture.md | 423 ++++++++++++++++++ fern/versions/dev.yml | 2 + src/compute-plane-services/nvca/BUILD.bazel | 9 + .../nvca/deployments/nvca-operator/README.md | 6 + ...-storage-capabilities-v1alpha1.schema.json | 101 +++++ .../nvcf-storage-capabilities-v1alpha1.yaml | 56 +++ .../storage-capabilities-configmap.yaml | 30 ++ .../nvca/pkg/storage/BUILD.bazel | 8 +- .../nvca/pkg/storage/storage_capabilities.go | 143 ++++++ .../pkg/storage/storage_capabilities_test.go | 304 +++++++++++++ .../nvca/scripts/lint_helm.sh | 105 +++++ .../nvca/scripts/requirements-lint.txt | 4 + 18 files changed, 1429 insertions(+), 28 deletions(-) create mode 100644 deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.schema.json create mode 100644 deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml create mode 100644 deploy/helm/nvca-operator/nvca-operator/templates/storage-capabilities-configmap.yaml create mode 100644 docs/dev/sdd-storage-agnostic-cache-architecture.md create mode 100644 src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.schema.json create mode 100644 src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml create mode 100644 src/compute-plane-services/nvca/deployments/nvca-operator/templates/storage-capabilities-configmap.yaml create mode 100644 src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go create mode 100644 src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go create mode 100644 src/compute-plane-services/nvca/scripts/requirements-lint.txt diff --git a/dependencies.md b/dependencies.md index 24b448570..247385fea 100644 --- a/dependencies.md +++ b/dependencies.md @@ -1843,7 +1843,8 @@ Generated by `go run -C ./tools/collect-dependencies .`. Refresh: `go run -C ./t - `Node.js`: `zod@4.3.6` - `Node.js`: `zwitch@2.0.4` - `Python`: `fastapi==0.110.0` -- `Python`: `pyyaml` (`PyYAML>=5.4.1`, `pyyaml>=6.0.1`) +- `Python`: `jsonschema==4.10.3` +- `Python`: `pyyaml` (`PyYAML==6.0.2`, `PyYAML>=5.4.1`, `pyyaml>=6.0.1`) - `Python`: `rich>=13.7.0` - `Rust`: `async-stream` (`async-stream 0.3`, `async-stream =0.3.6`) - `Rust`: `axum` (`axum 0.8`, `axum =0.8.9`) diff --git a/deploy/helm/nvca-operator/nvca-operator/README.md b/deploy/helm/nvca-operator/nvca-operator/README.md index 19406bc7a..bd41cf217 100644 --- a/deploy/helm/nvca-operator/nvca-operator/README.md +++ b/deploy/helm/nvca-operator/nvca-operator/README.md @@ -3,6 +3,12 @@ NVCF Cluster Agent (NVCA) Operator installs and manages reconfiguration, upgrades, and health checks of NVCA used in Kubernetes Clusters to run NVCF Workloads. +## Storage capability catalog + +The chart installs the versioned `nvcf-storage-capabilities` ConfigMap in the Helm release namespace. The public catalog and JSON Schema contain only the PVC access modes demonstrated for each CSI provisioner and the transition strategy for regular and Helm model cache. A `disabled` transition means that workflow is not qualified. Container cache is outside NVCA and is not part of this catalog. + +This release does not wire the catalog into backend selection. Runtime use requires a durable cache plan and safe legacy-request migration so retries and agent restarts cannot change backends. Managed deployments will inspect the exact `nvcf-sc` StorageClass when that follow-up is implemented. Editing this ConfigMap does not enable a storage backend today. + ## Parameters ### NVCA Operator parameters diff --git a/deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.schema.json b/deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.schema.json new file mode 100644 index 000000000..f44134625 --- /dev/null +++ b/deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.schema.json @@ -0,0 +1,101 @@ +{ + "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://nvcf.nvidia.com/schemas/storage-capability-catalog-v1alpha1.json", + "title": "NVCF storage capability catalog", + "type": "object", + "additionalProperties": false, + "required": ["apiVersion", "kind", "drivers"], + "properties": { + "apiVersion": {"const": "storage.nvcf.nvidia.com/v1alpha1"}, + "kind": {"const": "StorageCapabilityCatalog"}, + "drivers": { + "type": "object", + "minProperties": 1, + "propertyNames": {"type": "string", "minLength": 1, "pattern": "\\S"}, + "properties": { + "nvmesh-csi.excelero.com": {"$ref": "#/$defs/driver"} + }, + "additionalProperties": {"$ref": "#/$defs/disabledDriver"} + } + }, + "$defs": { + "accessMode": { + "type": "string", + "enum": ["ReadWriteOnce", "ReadOnlyMany", "ReadWriteMany"] + }, + "transitionStrategy": { + "type": "string", + "enum": ["disabled", "nvmesh"] + }, + "transitions": { + "type": "object", + "additionalProperties": false, + "required": ["regularModelCache", "helmModelCache"], + "properties": { + "regularModelCache": {"$ref": "#/$defs/transitionStrategy"}, + "helmModelCache": {"$ref": "#/$defs/transitionStrategy"} + } + }, + "driver": { + "type": "object", + "additionalProperties": false, + "required": ["provider", "accessModes", "transitions"], + "properties": { + "provider": {"type": "string", "minLength": 1, "pattern": "\\S"}, + "accessModes": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/$defs/accessMode"} + }, + "transitions": {"$ref": "#/$defs/transitions"} + }, + "allOf": [ + { + "if": { + "properties": { + "transitions": { + "anyOf": [ + { + "properties": {"regularModelCache": {"const": "nvmesh"}}, + "required": ["regularModelCache"] + }, + { + "properties": {"helmModelCache": {"const": "nvmesh"}}, + "required": ["helmModelCache"] + } + ] + } + }, + "required": ["transitions"] + }, + "then": { + "properties": { + "accessModes": { + "allOf": [ + {"contains": {"const": "ReadWriteOnce"}}, + {"contains": {"const": "ReadOnlyMany"}} + ] + } + } + } + } + ] + }, + "disabledDriver": { + "allOf": [ + {"$ref": "#/$defs/driver"}, + { + "properties": { + "transitions": { + "properties": { + "regularModelCache": {"const": "disabled"}, + "helmModelCache": {"const": "disabled"} + } + } + } + } + ] + } + } +} diff --git a/deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml b/deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml new file mode 100644 index 000000000..7adbc72f9 --- /dev/null +++ b/deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This public catalog is owned by NVCA and installed with the NVCA chart. +# accessModes lists only modes demonstrated for the exact tested provider. +# A disabled transition means that the end-to-end workflow is not qualified. +# The catalog is not yet wired into NVCA reconciliation or backend selection. +apiVersion: storage.nvcf.nvidia.com/v1alpha1 +kind: StorageCapabilityCatalog +drivers: + nvmesh-csi.excelero.com: + provider: nvmesh + accessModes: + - ReadWriteOnce + - ReadOnlyMany + transitions: + regularModelCache: nvmesh + helmModelCache: nvmesh + csi.weka.io: + provider: weka + # Fresh RWX and ROX claims were tested. Cross-namespace cache lifecycle is + # not yet qualified, so both transitions remain disabled. + accessModes: + - ReadWriteMany + - ReadOnlyMany + transitions: + regularModelCache: disabled + helmModelCache: disabled + fss.csi.oraclecloud.com: + provider: ociFss + # Only an RWX claim was tested. Its readers used read-only Pod mounts; that + # is not evidence that a ReadOnlyMany claim is supported. + accessModes: + - ReadWriteMany + transitions: + regularModelCache: disabled + helmModelCache: disabled + lustre.csi.oraclecloud.com: + provider: ociLustre + # No PVC access mode has been qualified in an NVCF cache workflow. + accessModes: [] + transitions: + regularModelCache: disabled + helmModelCache: disabled diff --git a/deploy/helm/nvca-operator/nvca-operator/templates/storage-capabilities-configmap.yaml b/deploy/helm/nvca-operator/nvca-operator/templates/storage-capabilities-configmap.yaml new file mode 100644 index 000000000..83bb433c0 --- /dev/null +++ b/deploy/helm/nvca-operator/nvca-operator/templates/storage-capabilities-configmap.yaml @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- $catalogPath := "files/nvcf-storage-capabilities-v1alpha1.yaml" -}} +{{- $catalog := .Files.Get $catalogPath -}} +{{- if not $catalog -}} +{{- fail (printf "required NVCF storage capability catalog %s is missing" $catalogPath) -}} +{{- end }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: nvcf-storage-capabilities + namespace: {{ .Release.Namespace }} + labels: + {{- include "nvcaop.labels" . | nindent 4 }} +data: + storage-provider-capabilities.yaml: | +{{- $catalog | nindent 4 }} diff --git a/docs/dev/sdd-central-model-cache-service.md b/docs/dev/sdd-central-model-cache-service.md index 448b28334..5a5ec5698 100644 --- a/docs/dev/sdd-central-model-cache-service.md +++ b/docs/dev/sdd-central-model-cache-service.md @@ -6,13 +6,22 @@ Implemented in NVCA. This document describes the model cache design as built in `pkg/storage` (the model cache StorageRequest controller), `pkg/webhook` (workload volume injection), and `internal/metrics` (observability). +The target provider-neutral design is documented in +[Storage-Agnostic Cache Architecture](sdd-storage-agnostic-cache-architecture.md). +The catalog is not wired into runtime selection, so this document remains the +current implementation reference until that migration is complete. The current +webhook does not set every model-cache volume mount read-only; the target design +defines the required fix and qualification test. The stable deployment contract +is `StorageClass/nvcf-sc`; the extra class-name checks below are legacy runtime +behavior, not the target provider-selection contract. + ## Goal A function or task can declare a model cache by `cacheHandle`. The first workload that needs a given handle downloads the model once; every later workload for the -same handle, in any namespace, attaches the already-downloaded copy read-only -instead of downloading again. Caching is best-effort: if the cache cannot be -provisioned the workload still runs, just without a shared cache. +same handle, in any namespace, attaches the already-downloaded copy through a +reader volume instead of downloading again. Caching is best-effort: if the cache +cannot be provisioned the workload still runs, just without a shared cache. The service is folded into NVCA rather than run as a standalone operator. It reuses NVCA's existing machinery: a control namespace, a per-cacheHandle @@ -28,29 +37,37 @@ and the mutating webhook that mounts the cache into workload pods. - Backend: the storage mechanism used to hold and share the cache. One of `nvmesh`, `sharedfs`, `samba`, `ephemeral`. - Writer: the single init Job that downloads the model into the backend. -- Reader: the per-namespace read-only volume a workload mounts. -- Durable marker: a cluster-scoped object whose existence means "this handle is +- Reader: the per-namespace volume intended to expose populated cache data. The + current webhook marks the PVC source read-only but not every matching + `volumeMount`. +- Durable marker: a durable Kubernetes object whose existence means "this handle is populated", used so any namespace and any agent restart can detect a populated cache without depending on in-memory state. ## Backend selection -The miniservice reconciler selects a backend once per request via -`SelectHelmCacheBackend` (`pkg/storage/cachebackend.go`) and stamps it on the -`StorageRequest.spec.modelCache.backend`. Selection is deterministic on cluster -state and feature flags: +The current miniservice reconciler calls `SelectHelmCacheBackend` +(`pkg/storage/cachebackend.go`) during each install reconcile and stamps the +result on `StorageRequest.spec.modelCache.backend`. It does not persist an +immutable plan before side effects. Selection uses cluster state, feature flags, +and legacy class-name sentinels: -1. `CachingSupport` disabled: no cache (`none`). -2. `nvcf-sc-30` StorageClass exists (NVMesh 3.x): `nvmesh`. +1. `CachingSupport` or `HelmModelCaching` disabled: no cache (`none`). +2. Legacy `nvcf-sc-30` sentinel exists: `nvmesh`. 3. `nvcf-miniservice-sc` StorageClass exists (operator-provided third-party shared storage): `sharedfs`. -4. `HelmSharedStorage` flag enabled: `samba`. +4. `HelmSharedStorage` enabled and the configured model-cache backing class exists: `samba`. 5. Otherwise: `ephemeral`. NVCA never creates `nvcf-miniservice-sc`. That StorageClass is exclusively the operator's signal that third-party shared storage is present (branch 3 above). The Samba backend (branch 4) is self-contained and creates no StorageClass. +This decision tree describes compatibility code in public NVCA main. Target +selection reads `nvcf-sc`, finds its provisioner in the catalog, and persists a +shared cache binding. NVMesh uses transition `nvmesh` for both regular and Helm +model cache. Samba remains only a legacy fallback; it is not an NVMesh transition. + ## Lifecycle shared by all shared backends The shared backends (nvmesh, sharedfs, samba) share one lifecycle; only the @@ -93,15 +110,16 @@ the init lease or any in-memory fan-out, both of which are lost on restart. namespace) plus a read-only PVC. - samba: a static SMB CSI PV pointing at the per-handle Samba share, read-only, plus a read-only PVC. -- sharedfs: a read-only PVC on the shared class; the class itself shares data - across namespaces, so no per-namespace PV plumbing is needed. +- sharedfs: a separately provisioned read-only PVC on the shared class. Current + code assumes it resolves to the writer's data, which is not guaranteed by a + StorageClass or provisioner alone. -## Samba backend (Samba over NVMesh) +## Legacy Samba fallback -Selected when no NVMesh 3.x or operator shared storage exists but -`HelmSharedStorage` is on. NVMesh block storage is ReadWriteOnce, so it cannot be -shared read-only across namespaces directly. The Samba backend re-exports an -`nvcf-sc` volume over SMB, which supports ReadWriteMany and ReadOnlyMany. +Current compatibility code selects this backend when neither legacy class +sentinel is present and `HelmSharedStorage` is on. It re-exports an `nvcf-sc` +volume over SMB for cross-namespace readers. It is not the target NVMesh +transition. Each cacheHandle gets its own Samba server and its own backing volume. There is no single shared server or global data PVC: a single fixed-size volume cannot be @@ -131,8 +149,9 @@ character name limit a stable hashed suffix is used. The mutating webhook (`pkg/webhook/helm_storage_webhook.go`) injects the cache into workload pods. When a pod carries the `nvca.nvcf.nvidia.io/storage-modelcache-pvc-name` annotation the webhook adds a -`model-data` volume bound to the read-only cache PVC and mounts it at the model -paths. +`model-data` volume bound to the cache PVC and mounts it at the model paths. The +PVC volume source is marked read-only; current matching `volumeMount` entries +are not, which the target design requires NVCA to fix before qualification. The shared-storage volumes use a drop-and-re-add pattern on every admission. The model cache volume participates in the same drop-and-re-add @@ -198,11 +217,10 @@ child span around per-handle Samba provisioning. separately provisioned PVCs on the same StorageClass do not share data on provisioners that create per-claim access points, subvolumes, or directories (EFS access points, CephFS subvolumes, some NFS provisioners). - Classes backed by a single share (for example an SMB class pointing at one - server share) work. The capability probe validates bindability, not - data-sharing; the fast-follow is a write-through-one-claim / - read-through-another probe, and deriving reader PVs from the writer's bound - volume where the driver supports it. + A class backed by one share may expose the same data, but class presence does + not prove that identity. The capability probe validates bindability, not + data-sharing; target qualification requires a no-copy cross-namespace data + identity and read-only enforcement tests. - The per-handle Samba infrastructure is create-once: image, resource, and cache-size changes do not reconcile onto an existing server or expand its backing PVC. diff --git a/docs/dev/sdd-storage-agnostic-cache-architecture.md b/docs/dev/sdd-storage-agnostic-cache-architecture.md new file mode 100644 index 000000000..71f9dee9a --- /dev/null +++ b/docs/dev/sdd-storage-agnostic-cache-architecture.md @@ -0,0 +1,423 @@ +# SDD: Storage-Agnostic Cache Architecture + +## Summary + +This document separates implemented foundation from target runtime behavior. + +Implemented foundation: + +- The NVCA Operator chart installs a public `v1alpha1` storage catalog and packages its JSON Schema. +- NVCA has a strict catalog loader and semantic validator. Tests call them; runtime reconciliation does not. +- Current public NVCA still uses legacy StorageClass-presence checks and feature flags for backend selection. +- Weka, OCI File Storage (FSS), and OCI Lustre model-cache transitions remain `disabled`. + +Target: + +1. Deployment tooling renders exactly one selected provider as `StorageClass/nvcf-sc`. +2. The public NVCA catalog declares qualified access modes and regular/Helm cache transitions. +3. NVCA resolves the live class and catalog entry, then persists a shared cache binding before storage side effects. +4. A provider transition remains disabled until its complete functional contract passes. + +Deployment tooling owns CSI-specific StorageClass parameters. NVCA owns model-cache transitions. A provider name, +provisioner, or access mode alone never enables a workflow. + +Editing the catalog ConfigMap does not enable a provider today. Runtime work is tracked in +[NVIDIA/nvcf#1326](https://github.com/NVIDIA/nvcf/issues/1326); this SDD defines the stable `nvcf-sc` target contract. + +## Table of contents + +- [Summary](#summary) +- [Scope](#scope) +- [Terms](#terms) +- [Deployment configuration](#deployment-configuration) +- [Storage catalog](#storage-catalog) +- [Target runtime design](#target-runtime-design) +- [Model-cache contract](#model-cache-contract) +- [Failure, migration, and rollback](#failure-migration-and-rollback) +- [Provider qualification](#provider-qualification) +- [Test plan](#test-plan) +- [Security and observability](#security-and-observability) +- [Implementation order](#implementation-order) +- [Public source references](#public-source-references) + +## Scope + +This design covers regular and Helm/MiniService model cache across managed and self-managed deployments, including +NVMesh, Weka, OCI FSS, OCI Lustre, future CSI providers, functional qualification, migration, and rollback. + +It excludes container cache; function, internal, and database storage; `nvcf-function-storage-sc`; CSI driver +installation/lifecycle; and performance qualification, which follows functional acceptance. + +## Terms + +| Term | Meaning | +|---|---| +| Provider | Product or integration ID, such as `ociFss` or `weka` | +| Provisioner | Exact CSI string in `StorageClass.provisioner` | +| Access mode | Kubernetes PVC mode declared only after qualification for an exact provider configuration | +| Workflow | `regularModelCache` or `helmModelCache` | +| Transition | Named NVCA implementation that moves a cache from writer state to reusable reader state | +| Qualification record | Versioned evidence for the exact tested storage and cluster configuration | +| Sharing domain | Stable authorization and encryption scope, such as an NCA ID | +| Cache key | Tuple of workflow, sharing domain, and cache handle | +| Cache binding | Immutable provider and transition choice shared by all requests for one cache key | +| Request selection | Per-request mode and optional cache-binding reference persisted before side effects | +| Data identity | Provider-backed object containing one populated cache handle | +| Writer | Workload holding the binding Lease while it populates a data identity | +| Reader view | Provider-specific, namespace-local, read-only access to the same data identity | + +`ReadOnlyMany` (ROX) is a PVC access mode. Mounting a `ReadWriteMany` (RWX) PVC with `readOnly: true` is not evidence +that the driver provisions or binds a ROX PVC. Neither condition alone proves backend-enforced write denial or +cross-namespace data identity. + +`ReadWriteOnce` (RWO) limits a volume to one node, not one Pod or writer. Kubernetes access modes describe attachment +and mount intent; NVCA must separately serialize writers and test reader write denial. + +## Deployment configuration + +Managed input selects a provider through `spec.nvcfStorage.provider`, resolves +`spec.storage.drivers..storageClasses.nvcf-sc`, and renders `StorageClass/nvcf-sc`. + +The selected provider supplies its exact provisioner, parameters, mount options, binding mode, expansion setting, +and topology. Provider values must match the installed CSI driver. + +The class name is always `nvcf-sc`, its reclaim policy is always `Retain`, and exactly one installed provider is +primary even when multiple CSI drivers coexist. Rendering rejects an empty provisioner, invalid class fields, unknown +provider fields, and provider input for `reclaimPolicy`. + +When no provider is selected, managed output remains compatible with NVMesh. Self-managed deployments may use +different tooling, but NVCA sees the same live `StorageClass/nvcf-sc` contract. + +NVMesh supports volume expansion, so its `nvcf-sc` definition and regression tests must preserve +`allowVolumeExpansion: true`. Expansion is deployment behavior, not a field in the NVCA catalog. + +## Storage catalog + +The chart installs ConfigMap `nvcf-storage-capabilities` in its release namespace. Data key +`storage-provider-capabilities.yaml` contains a `storage.nvcf.nvidia.com/v1alpha1` `StorageCapabilityCatalog`. Source +and release charts contain byte-identical catalog and schema files; rendering fails when the catalog file is absent. + +### Minimal shape + +```yaml +drivers: + : + provider: + accessModes: + - + transitions: + regularModelCache: + helmModelCache: +``` + +This is a shape illustration, not a valid provider entry. Actual entries must use exact provisioner strings, +Kubernetes access-mode names, and registered transition names. + +The exact provisioner is the lookup key. `provider` is a label, `accessModes` cites externally qualified PVC modes, and +each workflow names an implemented transition or `disabled`. + +Transition code, not the flat access-mode list, defines state A to state B: + +| Transition | Provisioner | Writer to reader contract | +|---|---|---| +| `disabled` | Any | No durable transition | +| `nvmesh` | `nvmesh-csi.excelero.com` | RWO writer to ROX reader views | + +Adding a transition requires dispatcher code, schema and semantic validation, a declared writer-to-reader mode pair, +and workflow tests. Access modes only show which required modes were qualified. + +`disabled` is the complete workflow-disabled state; a driver may still list qualified access modes. There is no +separate qualification field. Enabling a transition is a release decision allowed only after implementation and exact +workflow qualification; the schema cannot prove that evidence exists. + +The catalog intentionally does not contain a general CSI capability matrix. StorageClass rendering and provider +documentation own expansion, snapshots, clones, topology, and other CSI settings. + +Access-mode rules: + +- Record only Kubernetes access modes exercised by functional evidence for the exact provider configuration. +- Do not infer ROX from a read-only Pod mount of an RWX claim. +- Do not infer RWX or ROX from CSI driver documentation alone. +- Do not infer cross-namespace sharing or backend write denial from an access mode. + +NVMesh uses transition `nvmesh` for both regular and Helm model cache. Samba is not an NVMesh transition. Weka, OCI +FSS, and OCI Lustre transitions remain `disabled` until their exact workflow qualification passes. + +The target catalog does not register a generic `sharedfs` transition. Current NVCA retains a legacy, presence-selected +`sharedfs` route. Separate dynamic PVCs may expose different data, so class or driver presence cannot qualify it. + +### Validation contract + +CI validates structure and required fields with the packaged JSON Schema; Helm only checks that the file exists. When +called, the Go loader independently performs strict decoding and semantic validation, including ID, access-mode, +transition, and provisioner-transition checks. The test plan covers every rejection case. + +Catalog entries are configuration metadata, not credentials. The security requirements below govern their content. + +### Current runtime gap + +The loader has no runtime call site, and NVCA has no complete provider-neutral cache binding. Current requests persist +only a coarse backend value. Both gaps must be closed before the catalog can control runtime behavior. Until then, a +`disabled` catalog entry does not disable a legacy path selected by StorageClass presence. + +For Helm caching, current public NVCA first requires `CachingSupport` and `HelmModelCaching`. It then checks the legacy +`nvcf-sc-30` marker, the `nvcf-miniservice-sc` sharedfs sentinel, and a gated Samba fallback with a usable backing +class; otherwise it selects the per-Pod ephemeral cache. These are compatibility paths, not target provider selection. +The presence-only sharedfs branch cannot prove a cross-namespace backend identity. The target reads only `nvcf-sc`, +finds the exact provisioner entry, and selects its recorded transition. + +## Target runtime design + +### Selection algorithm + +For each request, NVCA must: + +1. Evaluate the applicable feature gates and workflow. Persist `none` or `ephemeral` on the request when selected; do + not create a durable binding. +2. For durable caching, derive `(workflow, sharingDomain, cacheHandle)` and read its binding from the model-cache + control namespace. +3. If it is `Active`, use it. If it is `Retiring`, retry or record a supported request fallback; never rebind it. +4. If no binding exists, read `StorageClass/nvcf-sc`, require `Retain`, and strictly load the catalog. +5. Find the exact provisioner entry and require a non-disabled transition with its required access modes. +6. Create the binding by deterministic name and optimistic concurrency. A losing creator reads the winning binding. +7. Conditionally add the request namespace, name, and UID while the binding is `Active`. Then persist its name, UID, + and a request finalizer before any storage side effect. +8. Re-read the binding, confirm the reference and request finalizer are present, and execute only its transition. + +Unknown provisioners, invalid catalogs, non-`Retain` classes, and disabled transitions cannot select durable storage. +The only generic fallbacks are `none` and, where the workflow supports it, `ephemeral`; NVCA never switches to a +second durable provider. + +### Release qualification + +Provisioner matching selects code; it does not prove that a deployment was qualified. Before a transition is enabled, +dated evidence must identify the CSI provisioner and images, backend version/configuration, `nvcf-sc` digest, +Kubernetes version, eligible node/OS/architecture matrix, catalog/schema version, and evidence reference. + +Some backend fields are not discoverable through Kubernetes or CSI. The release or deployment qualification process, +not NVCA, owns this record and verifies those fields. NVCA snapshots only live Kubernetes data it can read: the +StorageClass name, UID, provisioner, configuration digest, and catalog digest. That snapshot detects drift for a cache +binding; it does not prove the backend product or version. + +Setting a transition to a non-disabled value is therefore an operator or release assertion that the qualification +record applies to the cluster. NVCA validates the catalog and live StorageClass, but cannot verify non-observable +fields. + +### Cache binding and realized state + +Recomputing per request could mix providers for one cache key. The binding is one durable Kubernetes object in the +model-cache control namespace; every namespace-local request for that key references it. + +| Group | Required binding data | +|---|---| +| Identity | Version, workflow, sharing-domain digest, and cache-handle digest | +| Decision | Provider, provisioner, transition, required access modes, and catalog payload digest | +| StorageClass snapshot | Name, UID, `Retain`, and configuration digest | +| Resource intent | Deterministic names for NVCA-created PVCs, static PVs, Jobs, and Leases | +| Lifecycle | `Active` or `Retiring`, request namespace/name/UID references, and finalizer | +| Realized state | Bound PV and provider data identity plus population state as they become known | + +The target API is a namespaced `ModelCacheBinding` in the model-cache control namespace. Its immutable `spec` contains +identity, decision, StorageClass snapshot, and shared resource intent. Its `status` contains lifecycle, request +references, realized state, and conditions; a finalizer protects cleanup. `StorageRequest.status.modelCache` gains an +immutable request selection containing mode, binding name/UID, and deterministic namespace-local reader intent, plus a +request finalizer. The UID is the API-assigned `ModelCacheBinding.metadata.uid`; it is not part of binding `spec`. API +validation rejects binding-spec or request-selection mutation after persistence. + +The versioned StorageClass digest is SHA-256 over canonical JSON containing provisioner, sorted parameters, reclaim +policy, binding mode, mount options, and allowed topologies; list order is preserved. It excludes object metadata and +volume expansion. The catalog digest is SHA-256 over the exact ConfigMap payload. The request records mode `none`, +`ephemeral`, or `durable` and the binding reference when durable. + +Binding rules: + +- Persist the binding and request reference before the first storage side effect. +- All requests for one cache key converge through optimistic concurrency. +- A Lease keyed by the binding serializes at most one active writer. +- Retries and agent restarts reuse the binding. +- Creation uses the recorded deterministic name and Get-before-Create. A retry adopts an object only when its binding + UID and immutable spec match the intent; a mismatch is terminal. +- Record provider data identity only after it exists; do not change it afterward. +- Catalog, feature-gate, and StorageClass updates do not mutate an existing binding. +- Before any resource exists, input drift fails without side effects. +- After a resource exists, reconcile only the binding's resources; never switch providers or delete data because of + drift. +- Do not include Secret contents. + +Cleanup is a state transition, not a list-then-delete check. The reconciler marks the binding `Retiring`, which blocks +new references, while its finalizer prevents deletion. A stale reference is one whose request is absent, has a different +UID, or does not point back to the binding. Request deletion first removes reader resources, then removes the binding +reference, then releases the request finalizer. Provider cleanup starts only at zero references; the binding finalizer +is released only after owned Kubernetes resources are gone. Deleting retained backend data, if required, is an explicit +transition operation, not a response to configuration drift. + +## Model-cache contract + +PVCs are namespace-scoped. One PVC cannot be mounted by Pods in different namespaces. A same-namespace multi-Pod +read-only test is primitive evidence only. + +A regular or Helm model-cache transition can be enabled only if it provides: + +- at most one active writer per cache key, serialized by the binding Lease; +- one provider-backed data identity per cache key; +- no source-sized clone or extra data copy; +- namespace-local reader objects that resolve to the same data identity through a documented provider mechanism; +- no duplicate PVs with one CSI `volumeHandle`; +- `readOnly: true` on both the PVC volume source and each matching `volumeMount`; +- a read-only flag on the filesystem mount observed inside each reader container; +- failed write attempts from every reader Pod; +- a durable populated marker that survives writer cleanup and NVCA restart; +- idempotent creation, observation, retry, and deletion; +- cleanup that cannot delete data while another namespace references it. + +The current webhook sets `PersistentVolumeClaimVolumeSource.readOnly: true` but sets both model-cache `volumeMount` +values to `false`. Runtime implementation must set and test both mount values as `true` before external qualification. + +Kubernetes and generic CSI do not provide a cross-namespace PVC-sharing primitive. Each transition owns its +provider-specific reader-view mapping. A registered transition must document a driver-supported alias or rebind method +that produces namespace-local PVC/PV pairs with distinct CSI `volumeHandle` values resolving to the same backend data. +The generic planner must not synthesize those PVs or assume two PVCs from one class expose the same data. A provider +remains disabled if it cannot meet this contract without a source-sized copy. + +Cross-workflow and cross-domain reuse are not assumed. Either requires its own qualified transition and authorization +contract. + +## Failure, migration, and rollback + +After runtime enforcement is wired, durable selection fails closed for a missing or invalid catalog, a missing or +non-`Retain` `nvcf-sc`, an unknown provisioner or transition, a disabled workflow, or drift before the first side +effect. +Stale resources from the presence-selected sharedfs path are not adopted. + +Failure before a binding or request fallback is persisted creates no storage objects. Failure afterward follows the +recorded transition's cleanup rules. `none` or `ephemeral` is used only when recorded on the request. + +Legacy requests may have no backend or only a coarse backend value. Runtime migration must enforce: + +- requests with existing resources are not rebound to another provider; +- requests without storage side effects may establish or reference a binding; +- ambiguous shared-filesystem resources are not adopted from class presence or labels alone; +- NVMesh retained-primary-PV, sharedfs writer-PVC, and Samba backing-PVC markers are inspected explicitly; +- a migrated request remains stable across retry and restart. + +The runtime change must derive the exact legacy conversion from existing resources and cover it with tests. + +New unencrypted cache requests must reject the legacy `Agent.ModelCache.StorageClassName` override unless it is empty +or `nvcf-sc`. NVMesh encryption is the exception: selection still uses `nvcf-sc`, while the `nvmesh` transition records +and validates its deterministic per-NCA derived StorageClass before creating the encrypted writer PVC. The NCA sharing +domain is part of the cache key, and the derived class cannot select a provider. + +Provider-defining StorageClass fields, including the provisioner and parameters, cannot be changed in place. A +provider change is a maintenance operation: + +1. Stop new cache PVC creation. +2. Mark every binding for the current `nvcf-sc` UID `Retiring` so it rejects new references. +3. Drain requests, reach zero references, complete cleanup, and inventory retained data. +4. Verify no active binding references the old UID, then delete and recreate `nvcf-sc`. +5. Verify the new UID, provisioner, configuration digest, provisioning, and mounts. +6. Run functional qualification for the exact deployed configuration. +7. Resume new requests only after catalog and runtime support are deployed. + +Rollback repeats the process with the prior definition. `Retain` prevents automatic deletion; it does not migrate data. + +## Provider qualification + +1. Verify CSI controller health and node-plugin readiness on every eligible CPU and GPU node. +2. Render and verify the exact `nvcf-sc` definition. +3. Run dynamic provisioning and basic mount tests. +4. Prove each cataloged access mode for the exact deployed configuration. +5. Add the catalog entry with both transitions `disabled`. +6. Implement an explicit transition, including its provider-specific reader-view mapping. +7. Run the full workflow suite across namespaces and eligible pools. +8. Record CSI, backend, StorageClass, Kubernetes, node, and dated evidence details. +9. Set only a passing workflow to its implemented transition. +10. Deploy catalog and runtime changes together, then verify new bindings. + +A disabled entry does not enable cache traffic. + +## Test plan + +### Catalog and deployment + +- Missing namespace, ConfigMap, key, payload, and chart file. +- Malformed YAML, unknown fields, missing or invalid access modes, and missing transitions. +- Whitespace-only IDs, duplicate modes, unknown transitions, and provider-specific transition misuse. +- `nvmesh` requires the NVMesh provisioner plus RWO and ROX; external providers remain `disabled`. +- Source/release catalog, schema, and template parity. +- Deployment-tooling tests for exact Weka, OCI FSS, OCI Lustre, and generic provider rendering. +- Fixed `nvcf-sc` name and `Retain` policy. +- Deployment-tooling tests for exact parameters, mount options, binding mode, expansion, and topology. +- Deployment tooling preserves NVMesh expansion; the catalog selects `nvmesh` for both workflows. +- Malformed deployment input fails rendering. +- `nvcf-function-storage-sc` remains unchanged. + +### Binding and migration + +- Feature gates select the request fallback without creating a durable binding. +- The NVMesh provisioner entry selects `nvmesh` for both workflows. +- StorageClass and catalog digests are deterministic, sensitive to included fields, and captured with UIDs. +- Unknown provisioner, disabled transition, and non-`Retain` class. +- Concurrent namespaces in one sharing domain converge on one binding; other workflows or domains use different keys. +- Retry, leader change, and agent restart reuse the binding and Lease. +- Drift before the first side effect fails without resources; drift afterward never switches providers. +- A crash after object creation adopts only an exact deterministic intent and binding UID; a mismatch fails closed. +- `Active` to `Retiring` blocks new references; cleanup waits for zero references and honors the finalizer. +- Request deletion removes reader resources and the binding reference before releasing its finalizer. +- Legacy empty-backend, NVMesh PV, sharedfs PVC, and Samba PVC marker migration. +- Legacy class-override rejection and NVMesh encrypted derived-class validation. +- Provider replacement retires every binding for the old StorageClass UID before new requests resume. +- Stale shared-filesystem guard and cleanup. + +### Workflow + +- The Lease permits one active writer, including when competing writer Pods share a node. +- Readers in multiple namespaces mount the same provider data identity. +- CPU and GPU readers resolve the same provider data identity. +- Full-file and tree checksum equality. +- Denied create, append, rename, chmod, truncate, and delete. +- Both volume source and volume mount are read-only. +- A same-provider baseline mount is writable by the test UID/GID; the reader reports `ro` in `/proc/self/mountinfo`. +- Each cataloged PVC access mode is created and mounted explicitly. +- An RWX claim with a read-only Pod mount is not reported as ROX evidence. +- Writer and reader restart. +- Reader recreation and rescheduling to another eligible node. +- Agent restart at every lifecycle phase. +- Cancellation and injected provision, mount, writer, and cleanup failures. +- Idempotent retries with no duplicate writer or leaked resources. +- Cleanup with active and inactive readers. +- Upgrade and rollback with existing bindings. +- No source-sized clone, extra copy, or duplicate CSI handle. + +Performance testing is a separate follow-up after this functional suite passes. + +## Security and observability + +Security requirements: + +- Store no credentials, Secret contents, tokens, or private endpoints; use CSI-supported Secret references. +- Grant least-privilege read access to the StorageClass and catalog. +- Set read-only intent at every Kubernetes layer and verify write denial. +- Reject unknown fields and transitions. +- Qualification evidence records the exact deployed configuration; published evidence redacts credentials and private + deployment values while preserving the configuration digest. + +Observability requirements: + +- Log and trace binding creation, reuse, fallback, drift, transition, and cleanup. +- Count binding outcomes with bounded labels and publish persistence, population, reader, and cleanup conditions. +- Do not use cache handles, PVC names, backend IDs, or binding digests as metric labels. + +## Implementation order + +Keep external transitions disabled. Add the binding API and strict-loader call site, migrate legacy state, fix read-only +mounts, implement provider-specific no-copy transitions, remove presence selection, then qualify each workflow. + +## Public source references + +- [Storage catalog](https://github.com/NVIDIA/nvcf/blob/main/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml) +- [Catalog JSON Schema](https://github.com/NVIDIA/nvcf/blob/main/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.schema.json) +- [Catalog loader and validator](https://github.com/NVIDIA/nvcf/blob/main/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go) +- [Current Helm cache selection](https://github.com/NVIDIA/nvcf/blob/main/src/compute-plane-services/nvca/pkg/storage/cachebackend.go) +- [Current model-cache webhook](https://github.com/NVIDIA/nvcf/blob/main/src/compute-plane-services/nvca/pkg/webhook/helm_storage_webhook.go) +- [Kubernetes persistent-volume access modes](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes) +- [Runtime integration issue](https://github.com/NVIDIA/nvcf/issues/1326) diff --git a/fern/versions/dev.yml b/fern/versions/dev.yml index 4aaee4976..e05048c70 100644 --- a/fern/versions/dev.yml +++ b/fern/versions/dev.yml @@ -264,6 +264,8 @@ navigation: contents: - page: Architecture Overview path: ../../docs/dev/architecture.md + - page: Storage-Agnostic Cache Architecture + path: ../../docs/dev/sdd-storage-agnostic-cache-architecture.md - section: Local Development skip-slug: true contents: diff --git a/src/compute-plane-services/nvca/BUILD.bazel b/src/compute-plane-services/nvca/BUILD.bazel index 207e5c40a..ac051be48 100644 --- a/src/compute-plane-services/nvca/BUILD.bazel +++ b/src/compute-plane-services/nvca/BUILD.bazel @@ -15,6 +15,15 @@ load("@gazelle//:def.bzl", "gazelle") +filegroup( + name = "storage-capability-catalog", + srcs = [ + "deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.schema.json", + "deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml", + ], + visibility = ["//src/compute-plane-services/nvca/pkg/storage:__pkg__"], +) + # gazelle:prefix github.com/NVIDIA/nvcf/src/compute-plane-services/nvca # gazelle:proto disable_global # gazelle:go_naming_convention import_alias diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/README.md b/src/compute-plane-services/nvca/deployments/nvca-operator/README.md index f1135082c..9dc656e33 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/README.md +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/README.md @@ -3,6 +3,12 @@ NVCF Cluster Agent (NVCA) Operator installs and manages reconfiguration, upgrades, and health checks of NVCA used in Kubernetes Clusters to run NVCF Workloads. +## Storage capability catalog + +The chart installs the versioned `nvcf-storage-capabilities` ConfigMap in the Helm release namespace. The public catalog and JSON Schema contain only the PVC access modes demonstrated for each CSI provisioner and the transition strategy for regular and Helm model cache. A `disabled` transition means that workflow is not qualified. Container cache is outside NVCA and is not part of this catalog. + +This release does not wire the catalog into backend selection. Runtime use requires a durable cache plan and safe legacy-request migration so retries and agent restarts cannot change backends. Managed deployments will inspect the exact `nvcf-sc` StorageClass when that follow-up is implemented. Editing this ConfigMap does not enable a storage backend today. + ## Parameters ### NVCA Operator parameters diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.schema.json b/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.schema.json new file mode 100644 index 000000000..f44134625 --- /dev/null +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.schema.json @@ -0,0 +1,101 @@ +{ + "$comment": "SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://nvcf.nvidia.com/schemas/storage-capability-catalog-v1alpha1.json", + "title": "NVCF storage capability catalog", + "type": "object", + "additionalProperties": false, + "required": ["apiVersion", "kind", "drivers"], + "properties": { + "apiVersion": {"const": "storage.nvcf.nvidia.com/v1alpha1"}, + "kind": {"const": "StorageCapabilityCatalog"}, + "drivers": { + "type": "object", + "minProperties": 1, + "propertyNames": {"type": "string", "minLength": 1, "pattern": "\\S"}, + "properties": { + "nvmesh-csi.excelero.com": {"$ref": "#/$defs/driver"} + }, + "additionalProperties": {"$ref": "#/$defs/disabledDriver"} + } + }, + "$defs": { + "accessMode": { + "type": "string", + "enum": ["ReadWriteOnce", "ReadOnlyMany", "ReadWriteMany"] + }, + "transitionStrategy": { + "type": "string", + "enum": ["disabled", "nvmesh"] + }, + "transitions": { + "type": "object", + "additionalProperties": false, + "required": ["regularModelCache", "helmModelCache"], + "properties": { + "regularModelCache": {"$ref": "#/$defs/transitionStrategy"}, + "helmModelCache": {"$ref": "#/$defs/transitionStrategy"} + } + }, + "driver": { + "type": "object", + "additionalProperties": false, + "required": ["provider", "accessModes", "transitions"], + "properties": { + "provider": {"type": "string", "minLength": 1, "pattern": "\\S"}, + "accessModes": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/$defs/accessMode"} + }, + "transitions": {"$ref": "#/$defs/transitions"} + }, + "allOf": [ + { + "if": { + "properties": { + "transitions": { + "anyOf": [ + { + "properties": {"regularModelCache": {"const": "nvmesh"}}, + "required": ["regularModelCache"] + }, + { + "properties": {"helmModelCache": {"const": "nvmesh"}}, + "required": ["helmModelCache"] + } + ] + } + }, + "required": ["transitions"] + }, + "then": { + "properties": { + "accessModes": { + "allOf": [ + {"contains": {"const": "ReadWriteOnce"}}, + {"contains": {"const": "ReadOnlyMany"}} + ] + } + } + } + } + ] + }, + "disabledDriver": { + "allOf": [ + {"$ref": "#/$defs/driver"}, + { + "properties": { + "transitions": { + "properties": { + "regularModelCache": {"const": "disabled"}, + "helmModelCache": {"const": "disabled"} + } + } + } + } + ] + } + } +} diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml b/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml new file mode 100644 index 000000000..7adbc72f9 --- /dev/null +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This public catalog is owned by NVCA and installed with the NVCA chart. +# accessModes lists only modes demonstrated for the exact tested provider. +# A disabled transition means that the end-to-end workflow is not qualified. +# The catalog is not yet wired into NVCA reconciliation or backend selection. +apiVersion: storage.nvcf.nvidia.com/v1alpha1 +kind: StorageCapabilityCatalog +drivers: + nvmesh-csi.excelero.com: + provider: nvmesh + accessModes: + - ReadWriteOnce + - ReadOnlyMany + transitions: + regularModelCache: nvmesh + helmModelCache: nvmesh + csi.weka.io: + provider: weka + # Fresh RWX and ROX claims were tested. Cross-namespace cache lifecycle is + # not yet qualified, so both transitions remain disabled. + accessModes: + - ReadWriteMany + - ReadOnlyMany + transitions: + regularModelCache: disabled + helmModelCache: disabled + fss.csi.oraclecloud.com: + provider: ociFss + # Only an RWX claim was tested. Its readers used read-only Pod mounts; that + # is not evidence that a ReadOnlyMany claim is supported. + accessModes: + - ReadWriteMany + transitions: + regularModelCache: disabled + helmModelCache: disabled + lustre.csi.oraclecloud.com: + provider: ociLustre + # No PVC access mode has been qualified in an NVCF cache workflow. + accessModes: [] + transitions: + regularModelCache: disabled + helmModelCache: disabled diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/templates/storage-capabilities-configmap.yaml b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/storage-capabilities-configmap.yaml new file mode 100644 index 000000000..83bb433c0 --- /dev/null +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/templates/storage-capabilities-configmap.yaml @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- $catalogPath := "files/nvcf-storage-capabilities-v1alpha1.yaml" -}} +{{- $catalog := .Files.Get $catalogPath -}} +{{- if not $catalog -}} +{{- fail (printf "required NVCF storage capability catalog %s is missing" $catalogPath) -}} +{{- end }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: nvcf-storage-capabilities + namespace: {{ .Release.Namespace }} + labels: + {{- include "nvcaop.labels" . | nindent 4 }} +data: + storage-provider-capabilities.yaml: | +{{- $catalog | nindent 4 }} diff --git a/src/compute-plane-services/nvca/pkg/storage/BUILD.bazel b/src/compute-plane-services/nvca/pkg/storage/BUILD.bazel index eb31ee50c..d923e8571 100644 --- a/src/compute-plane-services/nvca/pkg/storage/BUILD.bazel +++ b/src/compute-plane-services/nvca/pkg/storage/BUILD.bazel @@ -18,6 +18,7 @@ go_library( "reconcile.go", "sharedstorage.go", "smbcsidriver.go", + "storage_capabilities.go", "storage_request_api.go", "storagerequest.go", "translate_workload.go", @@ -85,6 +86,7 @@ go_library( "//src/compute-plane-services/nvca/vendor/sigs.k8s.io/controller-runtime/pkg/manager", "//src/compute-plane-services/nvca/vendor/sigs.k8s.io/controller-runtime/pkg/predicate", "//src/compute-plane-services/nvca/vendor/sigs.k8s.io/controller-runtime/pkg/reconcile", + "//src/compute-plane-services/nvca/vendor/sigs.k8s.io/yaml", ], ) @@ -108,12 +110,16 @@ go_test( "reconcile_test.go", "sharedstorage_test.go", "smbcsidriver_test.go", + "storage_capabilities_test.go", "storage_request_api_test.go", "storagerequest_test.go", "translate_workload_test.go", "types_test.go", ], - data = ["//src/compute-plane-services/nvca/internal/envtest:crds"], + data = [ + "//src/compute-plane-services/nvca:storage-capability-catalog", + "//src/compute-plane-services/nvca/internal/envtest:crds", + ], embed = [":storage"], env_inherit = ["KUBEBUILDER_ASSETS"], rundir = ".", diff --git a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go new file mode 100644 index 000000000..1ad53d5b6 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go @@ -0,0 +1,143 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package storage + +import ( + "context" + "fmt" + "strings" + + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" +) + +const ( + storageCapabilityCatalogAPIVersion = "storage.nvcf.nvidia.com/v1alpha1" + storageCapabilityCatalogKind = "StorageCapabilityCatalog" + + // StorageCapabilityConfigMapName is the stable name of the ConfigMap that + // contains NVCA's public CSI provider capability catalog. + StorageCapabilityConfigMapName = "nvcf-storage-capabilities" + // StorageCapabilityConfigMapKey is the ConfigMap data key containing the + // serialized storage capability catalog. + StorageCapabilityConfigMapKey = "storage-provider-capabilities.yaml" +) + +type storageCapabilityCatalog struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Drivers map[string]storageDriverSpec `json:"drivers"` +} + +type storageDriverSpec struct { + Provider string `json:"provider"` + AccessModes *[]string `json:"accessModes"` + Transitions storageTransitions `json:"transitions"` +} + +type storageTransitions struct { + RegularModelCache string `json:"regularModelCache"` + HelmModelCache string `json:"helmModelCache"` +} + +func loadStorageCapabilityCatalog( + ctx context.Context, + c client.Client, + namespace string, +) (*storageCapabilityCatalog, error) { + if namespace == "" { + return nil, fmt.Errorf("storage capability ConfigMap namespace is empty") + } + + cm := &corev1.ConfigMap{} + if err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: StorageCapabilityConfigMapName}, cm); err != nil { + return nil, fmt.Errorf("get storage capability ConfigMap %s/%s: %w", + namespace, StorageCapabilityConfigMapName, err) + } + + raw, ok := cm.Data[StorageCapabilityConfigMapKey] + if !ok || raw == "" { + return nil, fmt.Errorf("storage capability ConfigMap %s/%s has no %q data", + namespace, StorageCapabilityConfigMapName, StorageCapabilityConfigMapKey) + } + + catalog := &storageCapabilityCatalog{} + if err := yaml.UnmarshalStrict([]byte(raw), catalog); err != nil { + return nil, fmt.Errorf("parse storage capability catalog: %w", err) + } + if err := validateStorageCapabilityCatalog(catalog); err != nil { + return nil, err + } + return catalog, nil +} + +func validateStorageCapabilityCatalog(catalog *storageCapabilityCatalog) error { + if catalog.APIVersion != storageCapabilityCatalogAPIVersion { + return fmt.Errorf("unsupported storage capability apiVersion %q", catalog.APIVersion) + } + if catalog.Kind != storageCapabilityCatalogKind { + return fmt.Errorf("unsupported storage capability kind %q", catalog.Kind) + } + if len(catalog.Drivers) == 0 { + return fmt.Errorf("storage capability catalog has no drivers") + } + + for provisioner, driver := range catalog.Drivers { + if strings.TrimSpace(provisioner) == "" || strings.TrimSpace(driver.Provider) == "" { + return fmt.Errorf("storage capability catalog has an empty provisioner or provider") + } + if driver.AccessModes == nil { + return fmt.Errorf("driver %q has no accessModes", provisioner) + } + accessModes := make(map[string]bool, len(*driver.AccessModes)) + for _, mode := range *driver.AccessModes { + switch mode { + case string(corev1.ReadWriteOnce), string(corev1.ReadOnlyMany), string(corev1.ReadWriteMany): + default: + return fmt.Errorf("driver %q has invalid accessMode %q", provisioner, mode) + } + if accessModes[mode] { + return fmt.Errorf("driver %q has duplicate accessMode %q", provisioner, mode) + } + accessModes[mode] = true + } + + for workflow, strategy := range map[string]string{ + "regularModelCache": driver.Transitions.RegularModelCache, + "helmModelCache": driver.Transitions.HelmModelCache, + } { + if strategy != "disabled" && strategy != "nvmesh" { + return fmt.Errorf("driver %q transition %s has invalid strategy %q", provisioner, workflow, strategy) + } + if strategy != "nvmesh" { + continue + } + if provisioner != NVMeshStorageClassProvisioner { + return fmt.Errorf("driver %q transition %s strategy %s is restricted to provisioner %q", + provisioner, workflow, strategy, NVMeshStorageClassProvisioner) + } + if !accessModes[string(corev1.ReadWriteOnce)] || !accessModes[string(corev1.ReadOnlyMany)] { + return fmt.Errorf("driver %q transition %s strategy %s requires ReadWriteOnce and ReadOnlyMany access modes", + provisioner, workflow, strategy) + } + } + } + + return nil +} diff --git a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go new file mode 100644 index 000000000..4ce3300d6 --- /dev/null +++ b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go @@ -0,0 +1,304 @@ +/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package storage + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +const testCatalogNamespace = "nvca-system" + +func capabilityCatalogConfigMap(raw string) *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: StorageCapabilityConfigMapName, Namespace: testCatalogNamespace}, + Data: map[string]string{StorageCapabilityConfigMapKey: raw}, + } +} + +func capabilityClient(t *testing.T, cm *corev1.ConfigMap) *fake.ClientBuilder { + t.Helper() + scheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(scheme)) + return fake.NewClientBuilder().WithScheme(scheme).WithObjects(cm) +} + +const validCatalog = `apiVersion: storage.nvcf.nvidia.com/v1alpha1 +kind: StorageCapabilityCatalog +drivers: + nvmesh-csi.excelero.com: + provider: nvmesh + accessModes: [ReadWriteOnce, ReadOnlyMany] + transitions: + regularModelCache: nvmesh + helmModelCache: nvmesh +` + +func accessModes(modes ...string) *[]string { + return &modes +} + +func validStorageCapabilityCatalog() *storageCapabilityCatalog { + return &storageCapabilityCatalog{ + APIVersion: storageCapabilityCatalogAPIVersion, + Kind: storageCapabilityCatalogKind, + Drivers: map[string]storageDriverSpec{ + NVMeshStorageClassProvisioner: { + Provider: "nvmesh", + AccessModes: accessModes("ReadWriteOnce", "ReadOnlyMany"), + Transitions: storageTransitions{ + RegularModelCache: "nvmesh", + HelmModelCache: "nvmesh", + }, + }, + }, + } +} + +func TestLoadStorageCapabilityCatalogStrict(t *testing.T) { + c := capabilityClient(t, capabilityCatalogConfigMap(validCatalog)).Build() + + catalog, err := loadStorageCapabilityCatalog(t.Context(), c, testCatalogNamespace) + require.NoError(t, err) + nvmesh := catalog.Drivers[NVMeshStorageClassProvisioner] + assert.Equal(t, "nvmesh", nvmesh.Transitions.RegularModelCache) + assert.Equal(t, "nvmesh", nvmesh.Transitions.HelmModelCache) + require.NotNil(t, nvmesh.AccessModes) + assert.ElementsMatch(t, []string{"ReadWriteOnce", "ReadOnlyMany"}, *nvmesh.AccessModes) +} + +func TestLoadStorageCapabilityCatalogErrors(t *testing.T) { + missingConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "other", Namespace: testCatalogNamespace}, + } + missingData := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: StorageCapabilityConfigMapName, Namespace: testCatalogNamespace}, + } + missingAccessModes := strings.Replace( + validCatalog, " accessModes: [ReadWriteOnce, ReadOnlyMany]\n", "", 1) + nullAccessModes := strings.Replace( + validCatalog, " accessModes: [ReadWriteOnce, ReadOnlyMany]", " accessModes: null", 1) + + tests := []struct { + name string + namespace string + configMap *corev1.ConfigMap + want string + }{ + { + name: "empty namespace", namespace: "", + configMap: capabilityCatalogConfigMap(validCatalog), want: "namespace is empty", + }, + { + name: "missing ConfigMap", namespace: testCatalogNamespace, + configMap: missingConfigMap, want: "get storage capability ConfigMap", + }, + { + name: "missing data key", namespace: testCatalogNamespace, + configMap: missingData, want: "has no", + }, + { + name: "empty data", namespace: testCatalogNamespace, + configMap: capabilityCatalogConfigMap(""), want: "has no", + }, + { + name: "malformed YAML", namespace: testCatalogNamespace, + configMap: capabilityCatalogConfigMap("drivers: ["), want: "parse storage capability catalog", + }, + { + name: "missing accessModes", namespace: testCatalogNamespace, + configMap: capabilityCatalogConfigMap(missingAccessModes), want: "has no accessModes", + }, + { + name: "null accessModes", namespace: testCatalogNamespace, + configMap: capabilityCatalogConfigMap(nullAccessModes), want: "has no accessModes", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := capabilityClient(t, tt.configMap).Build() + _, err := loadStorageCapabilityCatalog(t.Context(), c, tt.namespace) + require.ErrorContains(t, err, tt.want) + }) + } +} + +func TestLoadStorageCapabilityCatalogRejectsUnknownFields(t *testing.T) { + tests := []struct { + name string + raw string + want string + }{ + { + name: "driver field", + raw: strings.Replace(validCatalog, " provider: nvmesh", " provider: nvmesh\n surprise: true", 1), + want: "unknown field \"surprise\"", + }, + { + name: "container cache transition", + raw: strings.Replace(validCatalog, " helmModelCache: nvmesh", + " helmModelCache: nvmesh\n containerCache: disabled", 1), + want: "unknown field \"containerCache\"", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := capabilityClient(t, capabilityCatalogConfigMap(tt.raw)).Build() + _, err := loadStorageCapabilityCatalog(t.Context(), c, testCatalogNamespace) + require.ErrorContains(t, err, tt.want) + }) + } +} + +func TestValidateStorageCapabilityCatalog(t *testing.T) { + tests := []struct { + name string + mutate func(*storageCapabilityCatalog) + want string + }{ + {name: "bad apiVersion", mutate: func(c *storageCapabilityCatalog) { c.APIVersion = "v1" }, want: "apiVersion"}, + {name: "bad kind", mutate: func(c *storageCapabilityCatalog) { c.Kind = "ConfigMap" }, want: "kind"}, + {name: "empty drivers", mutate: func(c *storageCapabilityCatalog) { c.Drivers = nil }, want: "no drivers"}, + {name: "empty provider", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.Provider = "" + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: "empty provisioner or provider"}, + {name: "whitespace provider", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.Provider = " \t" + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: "empty provisioner or provider"}, + {name: "whitespace provisioner", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + delete(c.Drivers, NVMeshStorageClassProvisioner) + c.Drivers[" \t"] = d + }, want: "empty provisioner or provider"}, + {name: "missing access modes", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.AccessModes = nil + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: "has no accessModes"}, + {name: "bad access mode", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.AccessModes = accessModes(append(*d.AccessModes, "ReadWriteEverywhere")...) + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: "invalid accessMode"}, + {name: "duplicate access mode", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.AccessModes = accessModes(append(*d.AccessModes, "ReadWriteOnce")...) + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: "duplicate accessMode"}, + {name: "bad regular transition", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.Transitions.RegularModelCache = "shared-pvc-readonly-fanout" + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: "invalid strategy"}, + {name: "bad helm transition", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.Transitions.HelmModelCache = "samba" + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: "invalid strategy"}, + {name: "NVMesh transition is provisioner-specific", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + delete(c.Drivers, NVMeshStorageClassProvisioner) + c.Drivers["example.csi.test"] = d + }, want: "restricted to provisioner"}, + {name: "NVMesh transition lacks ReadWriteOnce", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.AccessModes = accessModes("ReadOnlyMany") + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: "requires ReadWriteOnce and ReadOnlyMany"}, + {name: "NVMesh transition lacks ReadOnlyMany", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.AccessModes = accessModes("ReadWriteOnce") + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: "requires ReadWriteOnce and ReadOnlyMany"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + catalog := validStorageCapabilityCatalog() + tt.mutate(catalog) + require.ErrorContains(t, validateStorageCapabilityCatalog(catalog), tt.want) + }) + } +} + +func TestValidateStorageCapabilityCatalogAllowsDisabledTransitionsWithEmptyModes(t *testing.T) { + catalog := validStorageCapabilityCatalog() + driver := catalog.Drivers[NVMeshStorageClassProvisioner] + driver.AccessModes = accessModes() + driver.Transitions = storageTransitions{ + RegularModelCache: "disabled", + HelmModelCache: "disabled", + } + catalog.Drivers[NVMeshStorageClassProvisioner] = driver + + require.NoError(t, validateStorageCapabilityCatalog(catalog)) +} + +func TestShippedStorageCapabilityCatalog(t *testing.T) { + chartDir := filepath.Join("src", "compute-plane-services", "nvca", "deployments", "nvca-operator") + if _, err := os.Stat(chartDir); os.IsNotExist(err) { + chartDir = filepath.Join("..", "..", "deployments", "nvca-operator") + } + raw, err := os.ReadFile(filepath.Join(chartDir, "files", "nvcf-storage-capabilities-v1alpha1.yaml")) + require.NoError(t, err) + c := capabilityClient(t, capabilityCatalogConfigMap(string(raw))).Build() + catalog, err := loadStorageCapabilityCatalog(t.Context(), c, testCatalogNamespace) + require.NoError(t, err) + + nvmesh := catalog.Drivers[NVMeshStorageClassProvisioner] + assert.Equal(t, "nvmesh", nvmesh.Transitions.RegularModelCache) + assert.Equal(t, "nvmesh", nvmesh.Transitions.HelmModelCache) + require.NotNil(t, nvmesh.AccessModes) + assert.ElementsMatch(t, []string{"ReadWriteOnce", "ReadOnlyMany"}, *nvmesh.AccessModes) + + for _, provisioner := range []string{"csi.weka.io", "fss.csi.oraclecloud.com", "lustre.csi.oraclecloud.com"} { + driver, ok := catalog.Drivers[provisioner] + require.True(t, ok, provisioner) + assert.Equal(t, "disabled", driver.Transitions.RegularModelCache) + assert.Equal(t, "disabled", driver.Transitions.HelmModelCache) + } + weka := catalog.Drivers["csi.weka.io"] + require.NotNil(t, weka.AccessModes) + assert.ElementsMatch(t, []string{"ReadWriteMany", "ReadOnlyMany"}, *weka.AccessModes) + fss := catalog.Drivers["fss.csi.oraclecloud.com"] + require.NotNil(t, fss.AccessModes) + assert.Equal(t, []string{"ReadWriteMany"}, *fss.AccessModes) + lustre := catalog.Drivers["lustre.csi.oraclecloud.com"] + require.NotNil(t, lustre.AccessModes) + assert.Empty(t, *lustre.AccessModes) + + schemaRaw, err := os.ReadFile(filepath.Join(chartDir, "files", "nvcf-storage-capabilities-v1alpha1.schema.json")) + require.NoError(t, err) + assert.True(t, json.Valid(schemaRaw), "shipped JSON schema must be valid JSON") +} diff --git a/src/compute-plane-services/nvca/scripts/lint_helm.sh b/src/compute-plane-services/nvca/scripts/lint_helm.sh index 7ae22ca94..ffdb16bb3 100755 --- a/src/compute-plane-services/nvca/scripts/lint_helm.sh +++ b/src/compute-plane-services/nvca/scripts/lint_helm.sh @@ -91,6 +91,111 @@ assert_pre_delete_cleanup_rbac() { "pre-delete cleanup hook RBAC is kept for the running Job" } +assert_storage_capability_catalog() ( + local service_chart="${repo_root}/deployments/nvca-operator" + local release_chart="${repo_root}/../../../deploy/helm/nvca-operator/nvca-operator" + local catalog="files/nvcf-storage-capabilities-v1alpha1.yaml" + local schema="files/nvcf-storage-capabilities-v1alpha1.schema.json" + local template="templates/storage-capabilities-configmap.yaml" + local rendered missing_chart schema_python tmpdir invalid_catalog schema_check + tmpdir="$(mktemp -d)" + trap 'rm -rf "${tmpdir}"' EXIT + missing_chart="${tmpdir}/missing-chart" + schema_python="${tmpdir}/venv/bin/python" + python3 -m venv --system-site-packages "${tmpdir}/venv" + "${schema_python}" -m pip install --disable-pip-version-check --quiet \ + --requirement "${repo_root}/scripts/requirements-lint.txt" + schema_check='import json,sys,yaml,jsonschema; schema=json.load(open(sys.argv[1])); jsonschema.Draft202012Validator.check_schema(schema); jsonschema.Draft202012Validator(schema).validate(yaml.safe_load(open(sys.argv[2])))' + + for relative in "${catalog}" "${schema}" "${template}"; do + diff -u "${service_chart}/${relative}" "${release_chart}/${relative}" + done + + rendered="${tmpdir}/rendered.yaml" + helm template test-release "${service_chart}" --namespace nvca-system \ + --set "ngcConfig.serviceKey=fakekey" \ + --show-only "${template}" >"${rendered}" + assert_eq "nvcf-storage-capabilities" "$(yq -r ".metadata.name" "${rendered}")" \ + "storage capability ConfigMap uses the stable name" + assert_eq "nvca-system" "$(yq -r ".metadata.namespace" "${rendered}")" \ + "storage capability ConfigMap is owned by the chart release namespace" + assert_eq "$(<"${service_chart}/${catalog}")" \ + "$(yq -r ".data.\"storage-provider-capabilities.yaml\"" "${rendered}")" \ + "storage capability ConfigMap embeds the exact catalog payload" + "${schema_python}" -c "${schema_check}" \ + "${service_chart}/${schema}" "${service_chart}/${catalog}" + + invalid_catalog="${tmpdir}/invalid-catalog.yaml" + for mutation in \ + 'del(.drivers."csi.weka.io".accessModes)' \ + '.drivers."csi.weka.io".accessModes = null'; do + yq "${mutation}" "${service_chart}/${catalog}" >"${invalid_catalog}" + if "${schema_python}" -c "${schema_check}" \ + "${service_chart}/${schema}" "${invalid_catalog}" 2>/dev/null; then + echo "Expected schema to reject missing or null accessModes" >&2 + return 1 + fi + done + echo "PASS: schema rejects missing and null accessModes" + + yq '.drivers."nvmesh-csi.excelero.com".accessModes = ["ReadWriteOnce"]' \ + "${service_chart}/${catalog}" >"${invalid_catalog}" + if "${schema_python}" -c "${schema_check}" \ + "${service_chart}/${schema}" "${invalid_catalog}" 2>/dev/null; then + echo "Expected schema to reject an NVMesh transition without ReadOnlyMany" >&2 + return 1 + fi + echo "PASS: schema rejects an NVMesh transition without ReadOnlyMany" + + yq '(.drivers."csi.weka.io".accessModes = ["ReadWriteOnce", "ReadOnlyMany"]) | + (.drivers."csi.weka.io".transitions.regularModelCache = "nvmesh")' \ + "${service_chart}/${catalog}" >"${invalid_catalog}" + if "${schema_python}" -c "${schema_check}" \ + "${service_chart}/${schema}" "${invalid_catalog}" 2>/dev/null; then + echo "Expected schema to reject an NVMesh transition on another provisioner" >&2 + return 1 + fi + echo "PASS: schema rejects an NVMesh transition on another provisioner" + + yq '.drivers."csi.weka.io".transitions.containerCache = "disabled"' \ + "${service_chart}/${catalog}" >"${invalid_catalog}" + if "${schema_python}" -c "${schema_check}" \ + "${service_chart}/${schema}" "${invalid_catalog}" 2>/dev/null; then + echo "Expected schema to reject a container-cache transition" >&2 + return 1 + fi + echo "PASS: schema rejects a container-cache transition" + + yq '.drivers."csi.weka.io".unexpected = true' \ + "${service_chart}/${catalog}" >"${invalid_catalog}" + if "${schema_python}" -c "${schema_check}" \ + "${service_chart}/${schema}" "${invalid_catalog}" 2>/dev/null; then + echo "Expected schema to reject an unknown driver field" >&2 + return 1 + fi + echo "PASS: schema rejects an unknown driver field" + + helm template test-release "${release_chart}" --namespace nvca-system \ + --show-only "${template}" >"${rendered}" + assert_eq "nvca-system" "$(yq -r ".metadata.namespace" "${rendered}")" \ + "release-chart storage capability ConfigMap is owned by the release namespace" + assert_eq "$(<"${service_chart}/${catalog}")" \ + "$(yq -r ".data.\"storage-provider-capabilities.yaml\"" "${rendered}")" \ + "release chart embeds the exact catalog payload" + + mkdir -p "${missing_chart}" + cp -a "${service_chart}/." "${missing_chart}/" + rm -f "${missing_chart}/${catalog}" + if helm template test-release "${missing_chart}" --set "ngcConfig.serviceKey=fakekey" >"${rendered}" 2>&1; then + echo "Expected rendering without the storage capability catalog to fail" >&2 + return 1 + fi + grep -q "required NVCF storage capability catalog" "${rendered}" + echo "PASS: storage capability catalog schema, render, payload, and chart parity" +) + +assert_storage_capability_catalog + # The Bazel-built NVCA operator image is distroless, so the rendered workload # commands must execute the packaged binaries directly. A /tini wrapper would # fail at container startup because that binary is not present in the image. diff --git a/src/compute-plane-services/nvca/scripts/requirements-lint.txt b/src/compute-plane-services/nvca/scripts/requirements-lint.txt new file mode 100644 index 000000000..5b48b7529 --- /dev/null +++ b/src/compute-plane-services/nvca/scripts/requirements-lint.txt @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +jsonschema==4.10.3 +PyYAML==6.0.2 From 68fae0f15cb4a714519dfd7c0473e5ac0a368c9a Mon Sep 17 00:00:00 2001 From: balaji Date: Sun, 30 Aug 2026 21:27:44 -0700 Subject: [PATCH 2/3] feat(nvca): define cache reader strategies Signed-off-by: balaji --- ...-storage-capabilities-v1alpha1.schema.json | 96 +++++++++-- .../nvcf-storage-capabilities-v1alpha1.yaml | 18 +- ...sdd-storage-agnostic-cache-architecture.md | 72 +++++--- ...-storage-capabilities-v1alpha1.schema.json | 96 +++++++++-- .../nvcf-storage-capabilities-v1alpha1.yaml | 18 +- .../nvca/pkg/storage/storage_capabilities.go | 87 ++++++++-- .../pkg/storage/storage_capabilities_test.go | 162 ++++++++++++++++-- .../nvca/scripts/lint_helm.sh | 105 +++++++++++- 8 files changed, 575 insertions(+), 79 deletions(-) diff --git a/deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.schema.json b/deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.schema.json index f44134625..6ca798961 100644 --- a/deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.schema.json +++ b/deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.schema.json @@ -14,9 +14,9 @@ "minProperties": 1, "propertyNames": {"type": "string", "minLength": 1, "pattern": "\\S"}, "properties": { - "nvmesh-csi.excelero.com": {"$ref": "#/$defs/driver"} + "nvmesh-csi.excelero.com": {"$ref": "#/$defs/nvmeshDriver"} }, - "additionalProperties": {"$ref": "#/$defs/disabledDriver"} + "additionalProperties": {"$ref": "#/$defs/nonNVMeshDriver"} } }, "$defs": { @@ -24,23 +24,32 @@ "type": "string", "enum": ["ReadWriteOnce", "ReadOnlyMany", "ReadWriteMany"] }, - "transitionStrategy": { + "readerMountOption": { "type": "string", - "enum": ["disabled", "nvmesh"] + "minLength": 1, + "pattern": "^\\S(?:.*\\S)?$" + }, + "regularTransitionStrategy": { + "type": "string", + "enum": ["disabled", "roxReadOnly", "rwxReadOnly"] + }, + "helmTransitionStrategy": { + "type": "string", + "enum": ["disabled", "roxReadOnly"] }, "transitions": { "type": "object", "additionalProperties": false, "required": ["regularModelCache", "helmModelCache"], "properties": { - "regularModelCache": {"$ref": "#/$defs/transitionStrategy"}, - "helmModelCache": {"$ref": "#/$defs/transitionStrategy"} + "regularModelCache": {"$ref": "#/$defs/regularTransitionStrategy"}, + "helmModelCache": {"$ref": "#/$defs/helmTransitionStrategy"} } }, "driver": { "type": "object", "additionalProperties": false, - "required": ["provider", "accessModes", "transitions"], + "required": ["provider", "accessModes", "readerMountOptions", "transitions"], "properties": { "provider": {"type": "string", "minLength": 1, "pattern": "\\S"}, "accessModes": { @@ -48,6 +57,37 @@ "uniqueItems": true, "items": {"$ref": "#/$defs/accessMode"} }, + "readerMountOptions": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/$defs/readerMountOption"}, + "allOf": [ + { + "not": { + "allOf": [ + {"contains": {"const": "ro"}}, + {"contains": {"const": "rw"}} + ] + } + }, + { + "not": { + "allOf": [ + {"contains": {"const": "recovery"}}, + {"contains": {"const": "norecovery"}} + ] + } + }, + { + "not": { + "allOf": [ + {"contains": {"const": "uuid"}}, + {"contains": {"const": "nouuid"}} + ] + } + } + ] + }, "transitions": {"$ref": "#/$defs/transitions"} }, "allOf": [ @@ -57,11 +97,11 @@ "transitions": { "anyOf": [ { - "properties": {"regularModelCache": {"const": "nvmesh"}}, + "properties": {"regularModelCache": {"const": "roxReadOnly"}}, "required": ["regularModelCache"] }, { - "properties": {"helmModelCache": {"const": "nvmesh"}}, + "properties": {"helmModelCache": {"const": "roxReadOnly"}}, "required": ["helmModelCache"] } ] @@ -76,20 +116,54 @@ {"contains": {"const": "ReadWriteOnce"}}, {"contains": {"const": "ReadOnlyMany"}} ] + }, + "readerMountOptions": { + "allOf": [ + {"contains": {"const": "ro"}}, + {"contains": {"const": "norecovery"}}, + {"contains": {"const": "nouuid"}} + ] } } } + }, + { + "if": { + "properties": { + "transitions": { + "properties": {"regularModelCache": {"const": "rwxReadOnly"}}, + "required": ["regularModelCache"] + } + }, + "required": ["transitions"] + }, + "then": { + "properties": { + "accessModes": {"contains": {"const": "ReadWriteMany"}}, + "readerMountOptions": {"maxItems": 0} + } + } + } + ] + }, + "nvmeshDriver": { + "allOf": [ + {"$ref": "#/$defs/driver"}, + { + "properties": { + "provider": {"const": "nvmesh"} + } } ] }, - "disabledDriver": { + "nonNVMeshDriver": { "allOf": [ {"$ref": "#/$defs/driver"}, { "properties": { "transitions": { "properties": { - "regularModelCache": {"const": "disabled"}, + "regularModelCache": {"enum": ["disabled", "rwxReadOnly"]}, "helmModelCache": {"const": "disabled"} } } diff --git a/deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml b/deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml index 7adbc72f9..220e0c033 100644 --- a/deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml +++ b/deploy/helm/nvca-operator/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml @@ -15,8 +15,13 @@ # This public catalog is owned by NVCA and installed with the NVCA chart. # accessModes lists only modes demonstrated for the exact tested provider. +# readerMountOptions lists options for read-only reader PVs that NVCA creates +# or rewrites. rwxReadOnly mounts the shared claim read-only and uses none. +# A transition value is a closed enum, not free text. A strategy name means +# enabled; disabled means the workflow is unsupported. # A disabled transition means that the end-to-end workflow is not qualified. -# The catalog is not yet wired into NVCA reconciliation or backend selection. +# The strict loader validates this catalog. Runtime reconciliation does not +# consume it until the storage-selection change is released. apiVersion: storage.nvcf.nvidia.com/v1alpha1 kind: StorageCapabilityCatalog drivers: @@ -25,9 +30,13 @@ drivers: accessModes: - ReadWriteOnce - ReadOnlyMany + readerMountOptions: + - ro + - norecovery + - nouuid transitions: - regularModelCache: nvmesh - helmModelCache: nvmesh + regularModelCache: roxReadOnly + helmModelCache: roxReadOnly csi.weka.io: provider: weka # Fresh RWX and ROX claims were tested. Cross-namespace cache lifecycle is @@ -35,6 +44,7 @@ drivers: accessModes: - ReadWriteMany - ReadOnlyMany + readerMountOptions: [] transitions: regularModelCache: disabled helmModelCache: disabled @@ -44,6 +54,7 @@ drivers: # is not evidence that a ReadOnlyMany claim is supported. accessModes: - ReadWriteMany + readerMountOptions: [] transitions: regularModelCache: disabled helmModelCache: disabled @@ -51,6 +62,7 @@ drivers: provider: ociLustre # No PVC access mode has been qualified in an NVCF cache workflow. accessModes: [] + readerMountOptions: [] transitions: regularModelCache: disabled helmModelCache: disabled diff --git a/docs/dev/sdd-storage-agnostic-cache-architecture.md b/docs/dev/sdd-storage-agnostic-cache-architecture.md index 71f9dee9a..8d8f6d806 100644 --- a/docs/dev/sdd-storage-agnostic-cache-architecture.md +++ b/docs/dev/sdd-storage-agnostic-cache-architecture.md @@ -7,6 +7,7 @@ This document separates implemented foundation from target runtime behavior. Implemented foundation: - The NVCA Operator chart installs a public `v1alpha1` storage catalog and packages its JSON Schema. +- The catalog uses closed transition enums and records required reader-PV mount options. - NVCA has a strict catalog loader and semantic validator. Tests call them; runtime reconciliation does not. - Current public NVCA still uses legacy StorageClass-presence checks and feature flags for backend selection. - Weka, OCI File Storage (FSS), and OCI Lustre model-cache transitions remain `disabled`. @@ -14,7 +15,8 @@ Implemented foundation: Target: 1. Deployment tooling renders exactly one selected provider as `StorageClass/nvcf-sc`. -2. The public NVCA catalog declares qualified access modes and regular/Helm cache transitions. +2. The public NVCA catalog declares qualified access modes, reader-PV mount options, and regular/Helm cache + transitions. 3. NVCA resolves the live class and catalog entry, then persists a shared cache binding before storage side effects. 4. A provider transition remains disabled until its complete functional contract passes. @@ -105,23 +107,32 @@ drivers: provider: accessModes: - + readerMountOptions: + - transitions: - regularModelCache: - helmModelCache: + regularModelCache: + helmModelCache: ``` This is a shape illustration, not a valid provider entry. Actual entries must use exact provisioner strings, Kubernetes access-mode names, and registered transition names. -The exact provisioner is the lookup key. `provider` is a label, `accessModes` cites externally qualified PVC modes, and -each workflow names an implemented transition or `disabled`. +The exact provisioner is the lookup key. `provider` is a label, and `accessModes` cites externally qualified PVC +modes. Transition values are a closed enum, not free text. A strategy name means enabled; `disabled` means +unsupported. + +| Workflow field | Allowed values | +|---|---| +| `regularModelCache` | `disabled`, `roxReadOnly`, `rwxReadOnly` | +| `helmModelCache` | `disabled`, `roxReadOnly` | Transition code, not the flat access-mode list, defines state A to state B: -| Transition | Provisioner | Writer to reader contract | +| Transition | Writer to reader contract | Catalog restriction | |---|---|---| -| `disabled` | Any | No durable transition | -| `nvmesh` | `nvmesh-csi.excelero.com` | RWO writer to ROX reader views | +| `disabled` | No durable transition | Any provider | +| `roxReadOnly` | Populate an RWO writer, then publish ROX reader storage | Exact NVMesh provisioner and provider | +| `rwxReadOnly` | Populate one RWX claim, then serve it through read-only Pod mounts | Regular model cache only | Adding a transition requires dispatcher code, schema and semantic validation, a declared writer-to-reader mode pair, and workflow tests. Access modes only show which required modes were qualified. @@ -130,6 +141,11 @@ and workflow tests. Access modes only show which required modes were qualified. separate qualification field. Enabling a transition is a release decision allowed only after implementation and exact workflow qualification; the schema cannot prove that evidence exists. +`readerMountOptions` is a required array for options NVCA must apply when it creates or rewrites a reader PV. It is +not `StorageClass.mountOptions`, a Pod `volumeMount`, or the Pod-level read-only flag. `roxReadOnly` requires +`ro`, `norecovery`, and `nouuid`. `rwxReadOnly` uses the dynamically provisioned RWX claim and therefore +requires an empty array. An empty array is not provider qualification evidence. + The catalog intentionally does not contain a general CSI capability matrix. StorageClass rendering and provider documentation own expansion, snapshots, clones, topology, and other CSI settings. @@ -140,17 +156,31 @@ Access-mode rules: - Do not infer RWX or ROX from CSI driver documentation alone. - Do not infer cross-namespace sharing or backend write denial from an access mode. -NVMesh uses transition `nvmesh` for both regular and Helm model cache. Samba is not an NVMesh transition. Weka, OCI +NVMesh uses transition `roxReadOnly` for both regular and Helm model cache. Samba is not an NVMesh transition. Weka, OCI FSS, and OCI Lustre transitions remain `disabled` until their exact workflow qualification passes. +### What operators set + +Use the exact installed CSI provisioner. Do not infer or enable a strategy from access modes alone. + +| Provider state | `regularModelCache` | `helmModelCache` | +|---|---|---| +| Shipped NVMesh configuration | `roxReadOnly` | `roxReadOnly` | +| New or unqualified provider | `disabled` | `disabled` | +| Qualified shared-RWX regular workflow | `rwxReadOnly` | `disabled` | + +The last row is a target configuration, not current enablement. Set it only after the exact CSI and StorageClass +configuration passes the provider-qualification contract and its NVCA runtime transition is released. + The target catalog does not register a generic `sharedfs` transition. Current NVCA retains a legacy, presence-selected `sharedfs` route. Separate dynamic PVCs may expose different data, so class or driver presence cannot qualify it. ### Validation contract -CI validates structure and required fields with the packaged JSON Schema; Helm only checks that the file exists. When -called, the Go loader independently performs strict decoding and semantic validation, including ID, access-mode, -transition, and provisioner-transition checks. The test plan covers every rejection case. +CI validates structure, required fields, closed transition values, and workflow constraints with the packaged JSON +Schema; Helm only checks that the file exists. When called, the Go loader independently performs strict decoding and +semantic validation. It also rejects duplicate, blank, whitespace-padded, or conflicting reader mount options. The +test plan covers both validation layers. Catalog entries are configuration metadata, not credentials. The security requirements below govern their content. @@ -178,7 +208,8 @@ For each request, NVCA must: control namespace. 3. If it is `Active`, use it. If it is `Retiring`, retry or record a supported request fallback; never rebind it. 4. If no binding exists, read `StorageClass/nvcf-sc`, require `Retain`, and strictly load the catalog. -5. Find the exact provisioner entry and require a non-disabled transition with its required access modes. +5. Find the exact provisioner entry and require a non-disabled transition with its required access modes and reader + mount options. 6. Create the binding by deterministic name and optimistic concurrency. A losing creator reads the winning binding. 7. Conditionally add the request namespace, name, and UID while the binding is `Active`. Then persist its name, UID, and a request finalizer before any storage side effect. @@ -211,7 +242,7 @@ model-cache control namespace; every namespace-local request for that key refere | Group | Required binding data | |---|---| | Identity | Version, workflow, sharing-domain digest, and cache-handle digest | -| Decision | Provider, provisioner, transition, required access modes, and catalog payload digest | +| Decision | Provider, provisioner, transition, required access modes, reader-PV mount options, and catalog payload digest | | StorageClass snapshot | Name, UID, `Retain`, and configuration digest | | Resource intent | Deterministic names for NVCA-created PVCs, static PVs, Jobs, and Leases | | Lifecycle | `Active` or `Retiring`, request namespace/name/UID references, and finalizer | @@ -303,7 +334,7 @@ Legacy requests may have no backend or only a coarse backend value. Runtime migr The runtime change must derive the exact legacy conversion from existing resources and cover it with tests. New unencrypted cache requests must reject the legacy `Agent.ModelCache.StorageClassName` override unless it is empty -or `nvcf-sc`. NVMesh encryption is the exception: selection still uses `nvcf-sc`, while the `nvmesh` transition records +or `nvcf-sc`. NVMesh encryption is the exception: selection still uses `nvcf-sc`, while the `roxReadOnly` transition records and validates its deterministic per-NCA derived StorageClass before creating the encrypted writer PVC. The NCA sharing domain is part of the cache key, and the derived class cannot select a provider. @@ -340,21 +371,22 @@ A disabled entry does not enable cache traffic. ### Catalog and deployment - Missing namespace, ConfigMap, key, payload, and chart file. -- Malformed YAML, unknown fields, missing or invalid access modes, and missing transitions. -- Whitespace-only IDs, duplicate modes, unknown transitions, and provider-specific transition misuse. -- `nvmesh` requires the NVMesh provisioner plus RWO and ROX; external providers remain `disabled`. +- Malformed YAML, unknown fields, missing or invalid access modes, missing reader mount options, and missing transitions. +- Whitespace-only IDs, duplicate modes or options, conflicting options, unknown transitions, and transition misuse. +- `roxReadOnly` requires the NVMesh provisioner and provider, RWO, ROX, `ro`, `norecovery`, and `nouuid`. +- `rwxReadOnly` requires RWX, an empty reader-mount-option array, and the regular workflow. - Source/release catalog, schema, and template parity. - Deployment-tooling tests for exact Weka, OCI FSS, OCI Lustre, and generic provider rendering. - Fixed `nvcf-sc` name and `Retain` policy. - Deployment-tooling tests for exact parameters, mount options, binding mode, expansion, and topology. -- Deployment tooling preserves NVMesh expansion; the catalog selects `nvmesh` for both workflows. +- Deployment tooling preserves NVMesh expansion; the catalog selects `roxReadOnly` for both workflows. - Malformed deployment input fails rendering. - `nvcf-function-storage-sc` remains unchanged. ### Binding and migration - Feature gates select the request fallback without creating a durable binding. -- The NVMesh provisioner entry selects `nvmesh` for both workflows. +- The NVMesh provisioner entry selects `roxReadOnly` for both workflows. - StorageClass and catalog digests are deterministic, sensitive to included fields, and captured with UIDs. - Unknown provisioner, disabled transition, and non-`Retain` class. - Concurrent namespaces in one sharing domain converge on one binding; other workflows or domains use different keys. diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.schema.json b/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.schema.json index f44134625..6ca798961 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.schema.json +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.schema.json @@ -14,9 +14,9 @@ "minProperties": 1, "propertyNames": {"type": "string", "minLength": 1, "pattern": "\\S"}, "properties": { - "nvmesh-csi.excelero.com": {"$ref": "#/$defs/driver"} + "nvmesh-csi.excelero.com": {"$ref": "#/$defs/nvmeshDriver"} }, - "additionalProperties": {"$ref": "#/$defs/disabledDriver"} + "additionalProperties": {"$ref": "#/$defs/nonNVMeshDriver"} } }, "$defs": { @@ -24,23 +24,32 @@ "type": "string", "enum": ["ReadWriteOnce", "ReadOnlyMany", "ReadWriteMany"] }, - "transitionStrategy": { + "readerMountOption": { "type": "string", - "enum": ["disabled", "nvmesh"] + "minLength": 1, + "pattern": "^\\S(?:.*\\S)?$" + }, + "regularTransitionStrategy": { + "type": "string", + "enum": ["disabled", "roxReadOnly", "rwxReadOnly"] + }, + "helmTransitionStrategy": { + "type": "string", + "enum": ["disabled", "roxReadOnly"] }, "transitions": { "type": "object", "additionalProperties": false, "required": ["regularModelCache", "helmModelCache"], "properties": { - "regularModelCache": {"$ref": "#/$defs/transitionStrategy"}, - "helmModelCache": {"$ref": "#/$defs/transitionStrategy"} + "regularModelCache": {"$ref": "#/$defs/regularTransitionStrategy"}, + "helmModelCache": {"$ref": "#/$defs/helmTransitionStrategy"} } }, "driver": { "type": "object", "additionalProperties": false, - "required": ["provider", "accessModes", "transitions"], + "required": ["provider", "accessModes", "readerMountOptions", "transitions"], "properties": { "provider": {"type": "string", "minLength": 1, "pattern": "\\S"}, "accessModes": { @@ -48,6 +57,37 @@ "uniqueItems": true, "items": {"$ref": "#/$defs/accessMode"} }, + "readerMountOptions": { + "type": "array", + "uniqueItems": true, + "items": {"$ref": "#/$defs/readerMountOption"}, + "allOf": [ + { + "not": { + "allOf": [ + {"contains": {"const": "ro"}}, + {"contains": {"const": "rw"}} + ] + } + }, + { + "not": { + "allOf": [ + {"contains": {"const": "recovery"}}, + {"contains": {"const": "norecovery"}} + ] + } + }, + { + "not": { + "allOf": [ + {"contains": {"const": "uuid"}}, + {"contains": {"const": "nouuid"}} + ] + } + } + ] + }, "transitions": {"$ref": "#/$defs/transitions"} }, "allOf": [ @@ -57,11 +97,11 @@ "transitions": { "anyOf": [ { - "properties": {"regularModelCache": {"const": "nvmesh"}}, + "properties": {"regularModelCache": {"const": "roxReadOnly"}}, "required": ["regularModelCache"] }, { - "properties": {"helmModelCache": {"const": "nvmesh"}}, + "properties": {"helmModelCache": {"const": "roxReadOnly"}}, "required": ["helmModelCache"] } ] @@ -76,20 +116,54 @@ {"contains": {"const": "ReadWriteOnce"}}, {"contains": {"const": "ReadOnlyMany"}} ] + }, + "readerMountOptions": { + "allOf": [ + {"contains": {"const": "ro"}}, + {"contains": {"const": "norecovery"}}, + {"contains": {"const": "nouuid"}} + ] } } } + }, + { + "if": { + "properties": { + "transitions": { + "properties": {"regularModelCache": {"const": "rwxReadOnly"}}, + "required": ["regularModelCache"] + } + }, + "required": ["transitions"] + }, + "then": { + "properties": { + "accessModes": {"contains": {"const": "ReadWriteMany"}}, + "readerMountOptions": {"maxItems": 0} + } + } + } + ] + }, + "nvmeshDriver": { + "allOf": [ + {"$ref": "#/$defs/driver"}, + { + "properties": { + "provider": {"const": "nvmesh"} + } } ] }, - "disabledDriver": { + "nonNVMeshDriver": { "allOf": [ {"$ref": "#/$defs/driver"}, { "properties": { "transitions": { "properties": { - "regularModelCache": {"const": "disabled"}, + "regularModelCache": {"enum": ["disabled", "rwxReadOnly"]}, "helmModelCache": {"const": "disabled"} } } diff --git a/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml b/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml index 7adbc72f9..220e0c033 100644 --- a/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml +++ b/src/compute-plane-services/nvca/deployments/nvca-operator/files/nvcf-storage-capabilities-v1alpha1.yaml @@ -15,8 +15,13 @@ # This public catalog is owned by NVCA and installed with the NVCA chart. # accessModes lists only modes demonstrated for the exact tested provider. +# readerMountOptions lists options for read-only reader PVs that NVCA creates +# or rewrites. rwxReadOnly mounts the shared claim read-only and uses none. +# A transition value is a closed enum, not free text. A strategy name means +# enabled; disabled means the workflow is unsupported. # A disabled transition means that the end-to-end workflow is not qualified. -# The catalog is not yet wired into NVCA reconciliation or backend selection. +# The strict loader validates this catalog. Runtime reconciliation does not +# consume it until the storage-selection change is released. apiVersion: storage.nvcf.nvidia.com/v1alpha1 kind: StorageCapabilityCatalog drivers: @@ -25,9 +30,13 @@ drivers: accessModes: - ReadWriteOnce - ReadOnlyMany + readerMountOptions: + - ro + - norecovery + - nouuid transitions: - regularModelCache: nvmesh - helmModelCache: nvmesh + regularModelCache: roxReadOnly + helmModelCache: roxReadOnly csi.weka.io: provider: weka # Fresh RWX and ROX claims were tested. Cross-namespace cache lifecycle is @@ -35,6 +44,7 @@ drivers: accessModes: - ReadWriteMany - ReadOnlyMany + readerMountOptions: [] transitions: regularModelCache: disabled helmModelCache: disabled @@ -44,6 +54,7 @@ drivers: # is not evidence that a ReadOnlyMany claim is supported. accessModes: - ReadWriteMany + readerMountOptions: [] transitions: regularModelCache: disabled helmModelCache: disabled @@ -51,6 +62,7 @@ drivers: provider: ociLustre # No PVC access mode has been qualified in an NVCF cache workflow. accessModes: [] + readerMountOptions: [] transitions: regularModelCache: disabled helmModelCache: disabled diff --git a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go index 1ad53d5b6..e86c97688 100644 --- a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go +++ b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go @@ -37,6 +37,18 @@ const ( // StorageCapabilityConfigMapKey is the ConfigMap data key containing the // serialized storage capability catalog. StorageCapabilityConfigMapKey = "storage-provider-capabilities.yaml" + + // ModelCacheTransitionDisabled prevents durable storage for a workflow. + ModelCacheTransitionDisabled = "disabled" + // ModelCacheTransitionROXReadOnly populates a writer claim and publishes a + // separate ReadOnlyMany reader claim with read-only Pod mounts. + ModelCacheTransitionROXReadOnly = "roxReadOnly" + // ModelCacheTransitionRWXReadOnly populates one ReadWriteMany claim and + // serves that same claim through read-only Pod mounts. + ModelCacheTransitionRWXReadOnly = "rwxReadOnly" + // ModelCacheProviderNVMesh is the only provider currently allowed to select + // the ROX read-only transition. + ModelCacheProviderNVMesh = "nvmesh" ) type storageCapabilityCatalog struct { @@ -46,9 +58,10 @@ type storageCapabilityCatalog struct { } type storageDriverSpec struct { - Provider string `json:"provider"` - AccessModes *[]string `json:"accessModes"` - Transitions storageTransitions `json:"transitions"` + Provider string `json:"provider"` + AccessModes *[]string `json:"accessModes"` + ReaderMountOptions *[]string `json:"readerMountOptions"` + Transitions storageTransitions `json:"transitions"` } type storageTransitions struct { @@ -105,6 +118,28 @@ func validateStorageCapabilityCatalog(catalog *storageCapabilityCatalog) error { if driver.AccessModes == nil { return fmt.Errorf("driver %q has no accessModes", provisioner) } + if driver.ReaderMountOptions == nil { + return fmt.Errorf("driver %q has no readerMountOptions", provisioner) + } + readerMountOptions := make(map[string]bool, len(*driver.ReaderMountOptions)) + for i, option := range *driver.ReaderMountOptions { + if strings.TrimSpace(option) == "" { + return fmt.Errorf("driver %q has blank readerMountOption", provisioner) + } + if strings.TrimSpace(option) != option { + return fmt.Errorf("driver %q has readerMountOption %q with surrounding whitespace", provisioner, option) + } + if readerMountOptions[option] { + return fmt.Errorf("driver %q has duplicate readerMountOption %q", provisioner, option) + } + for _, previous := range (*driver.ReaderMountOptions)[:i] { + if negatesMountOption(previous, option) { + return fmt.Errorf("driver %q readerMountOptions %q and %q conflict", + provisioner, previous, option) + } + } + readerMountOptions[option] = true + } accessModes := make(map[string]bool, len(*driver.AccessModes)) for _, mode := range *driver.AccessModes { switch mode { @@ -122,19 +157,45 @@ func validateStorageCapabilityCatalog(catalog *storageCapabilityCatalog) error { "regularModelCache": driver.Transitions.RegularModelCache, "helmModelCache": driver.Transitions.HelmModelCache, } { - if strategy != "disabled" && strategy != "nvmesh" { + if strategy != ModelCacheTransitionDisabled && strategy != ModelCacheTransitionROXReadOnly && + strategy != ModelCacheTransitionRWXReadOnly { return fmt.Errorf("driver %q transition %s has invalid strategy %q", provisioner, workflow, strategy) } - if strategy != "nvmesh" { + switch strategy { + case ModelCacheTransitionDisabled: continue - } - if provisioner != NVMeshStorageClassProvisioner { - return fmt.Errorf("driver %q transition %s strategy %s is restricted to provisioner %q", - provisioner, workflow, strategy, NVMeshStorageClassProvisioner) - } - if !accessModes[string(corev1.ReadWriteOnce)] || !accessModes[string(corev1.ReadOnlyMany)] { - return fmt.Errorf("driver %q transition %s strategy %s requires ReadWriteOnce and ReadOnlyMany access modes", - provisioner, workflow, strategy) + case ModelCacheTransitionROXReadOnly: + if provisioner != NVMeshStorageClassProvisioner { + return fmt.Errorf("driver %q transition %s strategy %s is restricted to provisioner %q", + provisioner, workflow, strategy, NVMeshStorageClassProvisioner) + } + if driver.Provider != ModelCacheProviderNVMesh { + return fmt.Errorf("driver %q transition %s strategy %s requires provider %q", + provisioner, workflow, strategy, ModelCacheProviderNVMesh) + } + if !accessModes[string(corev1.ReadWriteOnce)] || !accessModes[string(corev1.ReadOnlyMany)] { + return fmt.Errorf("driver %q transition %s strategy %s requires ReadWriteOnce and ReadOnlyMany access modes", + provisioner, workflow, strategy) + } + for _, requiredOption := range []string{"ro", "norecovery", "nouuid"} { + if !readerMountOptions[requiredOption] { + return fmt.Errorf("driver %q transition %s strategy %s requires readerMountOption %q", + provisioner, workflow, strategy, requiredOption) + } + } + case ModelCacheTransitionRWXReadOnly: + if workflow != "regularModelCache" { + return fmt.Errorf("driver %q transition %s strategy %s is only supported for regularModelCache", + provisioner, workflow, strategy) + } + if !accessModes[string(corev1.ReadWriteMany)] { + return fmt.Errorf("driver %q transition %s strategy %s requires ReadWriteMany access mode", + provisioner, workflow, strategy) + } + if len(*driver.ReaderMountOptions) != 0 { + return fmt.Errorf("driver %q transition %s strategy %s does not create a reader PV and requires empty readerMountOptions", + provisioner, workflow, strategy) + } } } } diff --git a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go index 4ce3300d6..ad9471871 100644 --- a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go +++ b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go @@ -54,26 +54,32 @@ drivers: nvmesh-csi.excelero.com: provider: nvmesh accessModes: [ReadWriteOnce, ReadOnlyMany] + readerMountOptions: [ro, norecovery, nouuid] transitions: - regularModelCache: nvmesh - helmModelCache: nvmesh + regularModelCache: roxReadOnly + helmModelCache: roxReadOnly ` func accessModes(modes ...string) *[]string { return &modes } +func readerMountOptions(options ...string) *[]string { + return &options +} + func validStorageCapabilityCatalog() *storageCapabilityCatalog { return &storageCapabilityCatalog{ APIVersion: storageCapabilityCatalogAPIVersion, Kind: storageCapabilityCatalogKind, Drivers: map[string]storageDriverSpec{ NVMeshStorageClassProvisioner: { - Provider: "nvmesh", - AccessModes: accessModes("ReadWriteOnce", "ReadOnlyMany"), + Provider: ModelCacheProviderNVMesh, + AccessModes: accessModes("ReadWriteOnce", "ReadOnlyMany"), + ReaderMountOptions: readerMountOptions("ro", "norecovery", "nouuid"), Transitions: storageTransitions{ - RegularModelCache: "nvmesh", - HelmModelCache: "nvmesh", + RegularModelCache: ModelCacheTransitionROXReadOnly, + HelmModelCache: ModelCacheTransitionROXReadOnly, }, }, }, @@ -86,10 +92,12 @@ func TestLoadStorageCapabilityCatalogStrict(t *testing.T) { catalog, err := loadStorageCapabilityCatalog(t.Context(), c, testCatalogNamespace) require.NoError(t, err) nvmesh := catalog.Drivers[NVMeshStorageClassProvisioner] - assert.Equal(t, "nvmesh", nvmesh.Transitions.RegularModelCache) - assert.Equal(t, "nvmesh", nvmesh.Transitions.HelmModelCache) + assert.Equal(t, ModelCacheTransitionROXReadOnly, nvmesh.Transitions.RegularModelCache) + assert.Equal(t, ModelCacheTransitionROXReadOnly, nvmesh.Transitions.HelmModelCache) require.NotNil(t, nvmesh.AccessModes) assert.ElementsMatch(t, []string{"ReadWriteOnce", "ReadOnlyMany"}, *nvmesh.AccessModes) + require.NotNil(t, nvmesh.ReaderMountOptions) + assert.Equal(t, []string{"ro", "norecovery", "nouuid"}, *nvmesh.ReaderMountOptions) } func TestLoadStorageCapabilityCatalogErrors(t *testing.T) { @@ -103,6 +111,10 @@ func TestLoadStorageCapabilityCatalogErrors(t *testing.T) { validCatalog, " accessModes: [ReadWriteOnce, ReadOnlyMany]\n", "", 1) nullAccessModes := strings.Replace( validCatalog, " accessModes: [ReadWriteOnce, ReadOnlyMany]", " accessModes: null", 1) + missingReaderMountOptions := strings.Replace( + validCatalog, " readerMountOptions: [ro, norecovery, nouuid]\n", "", 1) + nullReaderMountOptions := strings.Replace( + validCatalog, " readerMountOptions: [ro, norecovery, nouuid]", " readerMountOptions: null", 1) tests := []struct { name string @@ -138,6 +150,14 @@ func TestLoadStorageCapabilityCatalogErrors(t *testing.T) { name: "null accessModes", namespace: testCatalogNamespace, configMap: capabilityCatalogConfigMap(nullAccessModes), want: "has no accessModes", }, + { + name: "missing readerMountOptions", namespace: testCatalogNamespace, + configMap: capabilityCatalogConfigMap(missingReaderMountOptions), want: "has no readerMountOptions", + }, + { + name: "null readerMountOptions", namespace: testCatalogNamespace, + configMap: capabilityCatalogConfigMap(nullReaderMountOptions), want: "has no readerMountOptions", + }, } for _, tt := range tests { @@ -162,8 +182,8 @@ func TestLoadStorageCapabilityCatalogRejectsUnknownFields(t *testing.T) { }, { name: "container cache transition", - raw: strings.Replace(validCatalog, " helmModelCache: nvmesh", - " helmModelCache: nvmesh\n containerCache: disabled", 1), + raw: strings.Replace(validCatalog, " helmModelCache: roxReadOnly", + " helmModelCache: roxReadOnly\n containerCache: disabled", 1), want: "unknown field \"containerCache\"", }, } @@ -216,6 +236,56 @@ func TestValidateStorageCapabilityCatalog(t *testing.T) { d.AccessModes = accessModes(append(*d.AccessModes, "ReadWriteOnce")...) c.Drivers[NVMeshStorageClassProvisioner] = d }, want: "duplicate accessMode"}, + {name: "missing reader mount options", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.ReaderMountOptions = nil + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: "has no readerMountOptions"}, + {name: "blank reader mount option", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.ReaderMountOptions = readerMountOptions(append(*d.ReaderMountOptions, " \t")...) + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: "blank readerMountOption"}, + {name: "reader mount option with surrounding whitespace", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.ReaderMountOptions = readerMountOptions("ro", " norecovery", "nouuid") + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: "surrounding whitespace"}, + {name: "duplicate reader mount option", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.ReaderMountOptions = readerMountOptions(append(*d.ReaderMountOptions, "ro")...) + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: "duplicate readerMountOption"}, + {name: "conflicting reader mount options", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.ReaderMountOptions = readerMountOptions("ro", "norecovery", "nouuid", "rw") + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: `readerMountOptions "ro" and "rw" conflict`}, + {name: "conflicting recovery reader mount options", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.ReaderMountOptions = readerMountOptions("ro", "recovery", "norecovery", "nouuid") + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: `readerMountOptions "recovery" and "norecovery" conflict`}, + {name: "conflicting UUID reader mount options", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.ReaderMountOptions = readerMountOptions("ro", "norecovery", "uuid", "nouuid") + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: `readerMountOptions "uuid" and "nouuid" conflict`}, + {name: "ROX transition lacks ro reader mount option", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.ReaderMountOptions = readerMountOptions("norecovery", "nouuid") + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: `requires readerMountOption "ro"`}, + {name: "ROX transition lacks norecovery reader mount option", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.ReaderMountOptions = readerMountOptions("ro", "nouuid") + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: `requires readerMountOption "norecovery"`}, + {name: "ROX transition lacks nouuid reader mount option", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.ReaderMountOptions = readerMountOptions("ro", "norecovery") + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: `requires readerMountOption "nouuid"`}, {name: "bad regular transition", mutate: func(c *storageCapabilityCatalog) { d := c.Drivers[NVMeshStorageClassProvisioner] d.Transitions.RegularModelCache = "shared-pvc-readonly-fanout" @@ -226,21 +296,38 @@ func TestValidateStorageCapabilityCatalog(t *testing.T) { d.Transitions.HelmModelCache = "samba" c.Drivers[NVMeshStorageClassProvisioner] = d }, want: "invalid strategy"}, - {name: "NVMesh transition is provisioner-specific", mutate: func(c *storageCapabilityCatalog) { + {name: "ROX transition is provisioner-specific", mutate: func(c *storageCapabilityCatalog) { d := c.Drivers[NVMeshStorageClassProvisioner] delete(c.Drivers, NVMeshStorageClassProvisioner) c.Drivers["example.csi.test"] = d }, want: "restricted to provisioner"}, - {name: "NVMesh transition lacks ReadWriteOnce", mutate: func(c *storageCapabilityCatalog) { + {name: "ROX transition is provider-specific", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.Provider = "weka" + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: "requires provider"}, + {name: "ROX transition lacks ReadWriteOnce", mutate: func(c *storageCapabilityCatalog) { d := c.Drivers[NVMeshStorageClassProvisioner] d.AccessModes = accessModes("ReadOnlyMany") c.Drivers[NVMeshStorageClassProvisioner] = d }, want: "requires ReadWriteOnce and ReadOnlyMany"}, - {name: "NVMesh transition lacks ReadOnlyMany", mutate: func(c *storageCapabilityCatalog) { + {name: "ROX transition lacks ReadOnlyMany", mutate: func(c *storageCapabilityCatalog) { d := c.Drivers[NVMeshStorageClassProvisioner] d.AccessModes = accessModes("ReadWriteOnce") c.Drivers[NVMeshStorageClassProvisioner] = d }, want: "requires ReadWriteOnce and ReadOnlyMany"}, + {name: "RWX transition is rejected for Helm", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.AccessModes = accessModes("ReadWriteOnce", "ReadOnlyMany", "ReadWriteMany") + d.Transitions.HelmModelCache = ModelCacheTransitionRWXReadOnly + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: "only supported for regularModelCache"}, + {name: "RWX transition lacks ReadWriteMany", mutate: func(c *storageCapabilityCatalog) { + d := c.Drivers[NVMeshStorageClassProvisioner] + d.Transitions.RegularModelCache = ModelCacheTransitionRWXReadOnly + d.Transitions.HelmModelCache = ModelCacheTransitionDisabled + c.Drivers[NVMeshStorageClassProvisioner] = d + }, want: "requires ReadWriteMany"}, } for _, tt := range tests { @@ -252,13 +339,36 @@ func TestValidateStorageCapabilityCatalog(t *testing.T) { } } +func TestValidateStorageCapabilityCatalogAllowsRegularRWXReadOnly(t *testing.T) { + const provisioner = "shared.csi.example.com" + catalog := validStorageCapabilityCatalog() + catalog.Drivers[provisioner] = storageDriverSpec{ + Provider: "sharedFilesystem", + AccessModes: accessModes("ReadWriteMany", "ReadOnlyMany"), + ReaderMountOptions: readerMountOptions(), + Transitions: storageTransitions{ + RegularModelCache: ModelCacheTransitionRWXReadOnly, + HelmModelCache: ModelCacheTransitionDisabled, + }, + } + + require.NoError(t, validateStorageCapabilityCatalog(catalog)) + + driver := catalog.Drivers[provisioner] + driver.ReaderMountOptions = readerMountOptions("ro") + catalog.Drivers[provisioner] = driver + require.ErrorContains(t, validateStorageCapabilityCatalog(catalog), + "does not create a reader PV and requires empty readerMountOptions") +} + func TestValidateStorageCapabilityCatalogAllowsDisabledTransitionsWithEmptyModes(t *testing.T) { catalog := validStorageCapabilityCatalog() driver := catalog.Drivers[NVMeshStorageClassProvisioner] driver.AccessModes = accessModes() + driver.ReaderMountOptions = readerMountOptions() driver.Transitions = storageTransitions{ - RegularModelCache: "disabled", - HelmModelCache: "disabled", + RegularModelCache: ModelCacheTransitionDisabled, + HelmModelCache: ModelCacheTransitionDisabled, } catalog.Drivers[NVMeshStorageClassProvisioner] = driver @@ -277,16 +387,20 @@ func TestShippedStorageCapabilityCatalog(t *testing.T) { require.NoError(t, err) nvmesh := catalog.Drivers[NVMeshStorageClassProvisioner] - assert.Equal(t, "nvmesh", nvmesh.Transitions.RegularModelCache) - assert.Equal(t, "nvmesh", nvmesh.Transitions.HelmModelCache) + assert.Equal(t, ModelCacheTransitionROXReadOnly, nvmesh.Transitions.RegularModelCache) + assert.Equal(t, ModelCacheTransitionROXReadOnly, nvmesh.Transitions.HelmModelCache) require.NotNil(t, nvmesh.AccessModes) assert.ElementsMatch(t, []string{"ReadWriteOnce", "ReadOnlyMany"}, *nvmesh.AccessModes) + require.NotNil(t, nvmesh.ReaderMountOptions) + assert.Equal(t, []string{"ro", "norecovery", "nouuid"}, *nvmesh.ReaderMountOptions) for _, provisioner := range []string{"csi.weka.io", "fss.csi.oraclecloud.com", "lustre.csi.oraclecloud.com"} { driver, ok := catalog.Drivers[provisioner] require.True(t, ok, provisioner) assert.Equal(t, "disabled", driver.Transitions.RegularModelCache) assert.Equal(t, "disabled", driver.Transitions.HelmModelCache) + require.NotNil(t, driver.ReaderMountOptions) + assert.Empty(t, *driver.ReaderMountOptions) } weka := catalog.Drivers["csi.weka.io"] require.NotNil(t, weka.AccessModes) @@ -301,4 +415,18 @@ func TestShippedStorageCapabilityCatalog(t *testing.T) { schemaRaw, err := os.ReadFile(filepath.Join(chartDir, "files", "nvcf-storage-capabilities-v1alpha1.schema.json")) require.NoError(t, err) assert.True(t, json.Valid(schemaRaw), "shipped JSON schema must be valid JSON") + var schema map[string]any + require.NoError(t, json.Unmarshal(schemaRaw, &schema)) + definitions, ok := schema["$defs"].(map[string]any) + require.True(t, ok) + regularStrategy, ok := definitions["regularTransitionStrategy"].(map[string]any) + require.True(t, ok) + helmStrategy, ok := definitions["helmTransitionStrategy"].(map[string]any) + require.True(t, ok) + assert.ElementsMatch(t, + []any{ModelCacheTransitionDisabled, ModelCacheTransitionROXReadOnly, ModelCacheTransitionRWXReadOnly}, + regularStrategy["enum"]) + assert.ElementsMatch(t, + []any{ModelCacheTransitionDisabled, ModelCacheTransitionROXReadOnly}, + helmStrategy["enum"]) } diff --git a/src/compute-plane-services/nvca/scripts/lint_helm.sh b/src/compute-plane-services/nvca/scripts/lint_helm.sh index ffdb16bb3..8bc4edab1 100755 --- a/src/compute-plane-services/nvca/scripts/lint_helm.sh +++ b/src/compute-plane-services/nvca/scripts/lint_helm.sh @@ -138,6 +138,62 @@ assert_storage_capability_catalog() ( done echo "PASS: schema rejects missing and null accessModes" + for mutation in \ + 'del(.drivers."csi.weka.io".readerMountOptions)' \ + '.drivers."csi.weka.io".readerMountOptions = null'; do + yq "${mutation}" "${service_chart}/${catalog}" >"${invalid_catalog}" + if "${schema_python}" -c "${schema_check}" \ + "${service_chart}/${schema}" "${invalid_catalog}" 2>/dev/null; then + echo "Expected schema to reject missing or null readerMountOptions" >&2 + return 1 + fi + done + echo "PASS: schema rejects missing and null readerMountOptions" + + yq '.drivers."nvmesh-csi.excelero.com".readerMountOptions = ["ro", " norecovery", "nouuid"]' \ + "${service_chart}/${catalog}" >"${invalid_catalog}" + if "${schema_python}" -c "${schema_check}" \ + "${service_chart}/${schema}" "${invalid_catalog}" 2>/dev/null; then + echo "Expected schema to reject readerMountOptions with surrounding whitespace" >&2 + return 1 + fi + echo "PASS: schema rejects readerMountOptions with surrounding whitespace" + + for mutation in \ + '.drivers."nvmesh-csi.excelero.com".transitions.regularModelCache = "custom"' \ + '.drivers."nvmesh-csi.excelero.com".transitions.helmModelCache = "custom"'; do + yq "${mutation}" "${service_chart}/${catalog}" >"${invalid_catalog}" + if "${schema_python}" -c "${schema_check}" \ + "${service_chart}/${schema}" "${invalid_catalog}" 2>/dev/null; then + echo "Expected schema to reject an unregistered transition value" >&2 + return 1 + fi + done + echo "PASS: schema rejects unregistered transition values" + + yq '.drivers."nvmesh-csi.excelero.com".provider = "weka"' \ + "${service_chart}/${catalog}" >"${invalid_catalog}" + if "${schema_python}" -c "${schema_check}" \ + "${service_chart}/${schema}" "${invalid_catalog}" 2>/dev/null; then + echo "Expected schema to reject the NVMesh strategy with another provider" >&2 + return 1 + fi + echo "PASS: schema rejects the NVMesh strategy with another provider" + + for options in \ + '["ro", "rw", "norecovery", "nouuid"]' \ + '["ro", "recovery", "norecovery", "nouuid"]' \ + '["ro", "norecovery", "uuid", "nouuid"]'; do + yq ".drivers.\"nvmesh-csi.excelero.com\".readerMountOptions = ${options}" \ + "${service_chart}/${catalog}" >"${invalid_catalog}" + if "${schema_python}" -c "${schema_check}" \ + "${service_chart}/${schema}" "${invalid_catalog}" 2>/dev/null; then + echo "Expected schema to reject conflicting readerMountOptions" >&2 + return 1 + fi + done + echo "PASS: schema rejects conflicting readerMountOptions" + yq '.drivers."nvmesh-csi.excelero.com".accessModes = ["ReadWriteOnce"]' \ "${service_chart}/${catalog}" >"${invalid_catalog}" if "${schema_python}" -c "${schema_check}" \ @@ -147,8 +203,17 @@ assert_storage_capability_catalog() ( fi echo "PASS: schema rejects an NVMesh transition without ReadOnlyMany" + yq '.drivers."nvmesh-csi.excelero.com".readerMountOptions = ["ro", "norecovery"]' \ + "${service_chart}/${catalog}" >"${invalid_catalog}" + if "${schema_python}" -c "${schema_check}" \ + "${service_chart}/${schema}" "${invalid_catalog}" 2>/dev/null; then + echo "Expected schema to reject an NVMesh transition without nouuid" >&2 + return 1 + fi + echo "PASS: schema rejects an NVMesh transition without nouuid" + yq '(.drivers."csi.weka.io".accessModes = ["ReadWriteOnce", "ReadOnlyMany"]) | - (.drivers."csi.weka.io".transitions.regularModelCache = "nvmesh")' \ + (.drivers."csi.weka.io".transitions.regularModelCache = "roxReadOnly")' \ "${service_chart}/${catalog}" >"${invalid_catalog}" if "${schema_python}" -c "${schema_check}" \ "${service_chart}/${schema}" "${invalid_catalog}" 2>/dev/null; then @@ -157,6 +222,44 @@ assert_storage_capability_catalog() ( fi echo "PASS: schema rejects an NVMesh transition on another provisioner" + yq '.drivers."csi.weka.io".transitions.regularModelCache = "rwxReadOnly"' \ + "${service_chart}/${catalog}" >"${invalid_catalog}" + if ! "${schema_python}" -c "${schema_check}" \ + "${service_chart}/${schema}" "${invalid_catalog}"; then + echo "Expected schema to accept regular rwxReadOnly with ReadWriteMany" >&2 + return 1 + fi + echo "PASS: schema accepts regular rwxReadOnly with ReadWriteMany" + + yq '(.drivers."csi.weka.io".transitions.regularModelCache = "rwxReadOnly") | + (.drivers."csi.weka.io".readerMountOptions = ["ro"])' \ + "${service_chart}/${catalog}" >"${invalid_catalog}" + if "${schema_python}" -c "${schema_check}" \ + "${service_chart}/${schema}" "${invalid_catalog}" 2>/dev/null; then + echo "Expected schema to reject reader-PV mount options for rwxReadOnly" >&2 + return 1 + fi + echo "PASS: schema rejects reader-PV mount options for rwxReadOnly" + + yq '(.drivers."csi.weka.io".accessModes = ["ReadOnlyMany"]) | + (.drivers."csi.weka.io".transitions.regularModelCache = "rwxReadOnly")' \ + "${service_chart}/${catalog}" >"${invalid_catalog}" + if "${schema_python}" -c "${schema_check}" \ + "${service_chart}/${schema}" "${invalid_catalog}" 2>/dev/null; then + echo "Expected schema to reject rwxReadOnly without ReadWriteMany" >&2 + return 1 + fi + echo "PASS: schema rejects rwxReadOnly without ReadWriteMany" + + yq '.drivers."csi.weka.io".transitions.helmModelCache = "rwxReadOnly"' \ + "${service_chart}/${catalog}" >"${invalid_catalog}" + if "${schema_python}" -c "${schema_check}" \ + "${service_chart}/${schema}" "${invalid_catalog}" 2>/dev/null; then + echo "Expected schema to reject rwxReadOnly for Helm" >&2 + return 1 + fi + echo "PASS: schema rejects rwxReadOnly for Helm" + yq '.drivers."csi.weka.io".transitions.containerCache = "disabled"' \ "${service_chart}/${catalog}" >"${invalid_catalog}" if "${schema_python}" -c "${schema_check}" \ From 119b23e162d877bbb29c767a48f3af63c7edcec3 Mon Sep 17 00:00:00 2001 From: balaji Date: Mon, 31 Aug 2026 17:40:29 -0700 Subject: [PATCH 3/3] fix(nvca): address review feedback on the storage capability catalog Three items, all against the repo's own standards. lint_helm.sh printed check marks and crosses. AGENTS.md requires standard ASCII in committed text, so they are now "ok" and "FAIL". Twelve non-ASCII characters removed; the script still passes. One error in storage_capabilities.go exceeded the 120 character limit. Wrapped. Catalog validation iterated the two workflows as a map. Go randomises map iteration, so a driver with both transitions invalid reported whichever one it happened to reach first, and an operator fixing a catalog would see the error change between runs. It is a slice now, so the failure is reported in declaration order every time, with a test that runs the same invalid catalog twenty times and requires an identical message. Relates to #1326 Co-Authored-By: Balaji Ganesan --- .../nvca/pkg/storage/storage_capabilities.go | 17 ++++++++--- .../pkg/storage/storage_capabilities_test.go | 28 +++++++++++++++++++ .../nvca/scripts/lint_helm.sh | 16 +++++------ 3 files changed, 49 insertions(+), 12 deletions(-) diff --git a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go index e86c97688..060e645a1 100644 --- a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go +++ b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities.go @@ -153,10 +153,17 @@ func validateStorageCapabilityCatalog(catalog *storageCapabilityCatalog) error { accessModes[mode] = true } - for workflow, strategy := range map[string]string{ - "regularModelCache": driver.Transitions.RegularModelCache, - "helmModelCache": driver.Transitions.HelmModelCache, + // A slice, not a map: Go randomises map iteration, so a driver with + // both transitions invalid would report whichever one it happened to + // reach first. An operator fixing a catalog needs the same error twice. + for _, wf := range []struct { + workflow string + strategy string + }{ + {"regularModelCache", driver.Transitions.RegularModelCache}, + {"helmModelCache", driver.Transitions.HelmModelCache}, } { + workflow, strategy := wf.workflow, wf.strategy if strategy != ModelCacheTransitionDisabled && strategy != ModelCacheTransitionROXReadOnly && strategy != ModelCacheTransitionRWXReadOnly { return fmt.Errorf("driver %q transition %s has invalid strategy %q", provisioner, workflow, strategy) @@ -193,7 +200,9 @@ func validateStorageCapabilityCatalog(catalog *storageCapabilityCatalog) error { provisioner, workflow, strategy) } if len(*driver.ReaderMountOptions) != 0 { - return fmt.Errorf("driver %q transition %s strategy %s does not create a reader PV and requires empty readerMountOptions", + return fmt.Errorf( + "driver %q transition %s strategy %s does not create a reader PV "+ + "and requires empty readerMountOptions", provisioner, workflow, strategy) } } diff --git a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go index ad9471871..311a403ec 100644 --- a/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go +++ b/src/compute-plane-services/nvca/pkg/storage/storage_capabilities_test.go @@ -430,3 +430,31 @@ func TestShippedStorageCapabilityCatalog(t *testing.T) { []any{ModelCacheTransitionDisabled, ModelCacheTransitionROXReadOnly}, helmStrategy["enum"]) } + +// TestValidateStorageCapabilityCatalogErrorsAreDeterministic pins the error an +// operator sees. The workflows were iterated as a map, and Go randomises map +// iteration, so a driver with both transitions invalid reported whichever one +// it happened to reach first. Fixing a catalog against an error that changes +// between runs is guesswork. +func TestValidateStorageCapabilityCatalogErrorsAreDeterministic(t *testing.T) { + catalog := func() *storageCapabilityCatalog { + c := validStorageCapabilityCatalog() + d := c.Drivers[NVMeshStorageClassProvisioner] + d.Transitions.RegularModelCache = "notAStrategy" + d.Transitions.HelmModelCache = "alsoNotAStrategy" + c.Drivers[NVMeshStorageClassProvisioner] = d + return c + } + + first := validateStorageCapabilityCatalog(catalog()) + require.Error(t, first) + assert.Contains(t, first.Error(), "regularModelCache", + "the first workflow in declaration order is the one reported") + + for i := 0; i < 20; i++ { + got := validateStorageCapabilityCatalog(catalog()) + require.Error(t, got) + assert.Equal(t, first.Error(), got.Error(), + "the same invalid catalog must produce the same error every time") + } +} diff --git a/src/compute-plane-services/nvca/scripts/lint_helm.sh b/src/compute-plane-services/nvca/scripts/lint_helm.sh index 8bc4edab1..68b6efc79 100755 --- a/src/compute-plane-services/nvca/scripts/lint_helm.sh +++ b/src/compute-plane-services/nvca/scripts/lint_helm.sh @@ -51,10 +51,10 @@ assert_eq() { local message=${3} if [[ "${got}" != "${want}" ]]; then - printf '✗ %s: got %q, want %q\n' "${message}" "${got}" "${want}" >&2 + printf 'FAIL %s: got %q, want %q\n' "${message}" "${got}" "${want}" >&2 return 1 fi - printf '✓ %s\n' "${message}" + printf 'ok %s\n' "${message}" } assert_pre_delete_cleanup_rbac() { @@ -392,7 +392,7 @@ assert_service_oauth_nil_safe() { exit 1 fi rm -f "${render_output}" - echo "✓ ${label} cluster-dto renders nil-safely without agent.serviceOAuth defaults" + echo "ok ${label} cluster-dto renders nil-safely without agent.serviceOAuth defaults" } assert_service_oauth_nil_safe "helm-managed" \ @@ -498,7 +498,7 @@ assert_transport_trust_config() { exit 1 fi rm -f "${render_output}" - echo "✓ ${label} workload transport trust ConfigMap renders" + echo "ok ${label} workload transport trust ConfigMap renders" } assert_transport_trust_config "default" "" "" "" --set "ngcConfig.serviceKey=fakekey" @@ -543,22 +543,22 @@ helm template test-release "${repo_root}/deployments/nvca-operator" \ --values "${repo_root}/test/test-network-policies.yaml" \ --set "ngcConfig.serviceKey=fakekey" \ --show-only templates/custom-network-policies-configmap.yaml \ - | grep -q "nvcf-custom-network-policies" && echo "✓ Network policies ConfigMap created" || echo "✗ Network policies ConfigMap missing" + | grep -q "nvcf-custom-network-policies" && echo "ok Network policies ConfigMap created" || echo "FAIL Network policies ConfigMap missing" helm template test-release "${repo_root}/deployments/nvca-operator" \ --values "${repo_root}/test/test-custom-annotations.yaml" \ --set "ngcConfig.serviceKey=fakekey" \ --show-only templates/custom-annotations-configmap.yaml \ - | grep -q "nvca-namespace-pod-annotations" && echo "✓ Custom annotations ConfigMap created" || echo "✗ Custom annotations ConfigMap missing" + | grep -q "nvca-namespace-pod-annotations" && echo "ok Custom annotations ConfigMap created" || echo "FAIL Custom annotations ConfigMap missing" # Test ConfigMaps are created even without custom values (always created behavior) echo "Testing ConfigMaps are always created..." helm template test-release "${repo_root}/deployments/nvca-operator" \ --set "ngcConfig.serviceKey=fakekey" \ --show-only templates/custom-annotations-configmap.yaml \ - | grep -q "nvca-namespace-pod-annotations" && echo "✓ Annotations ConfigMap always created" || echo "✗ Annotations ConfigMap not created" + | grep -q "nvca-namespace-pod-annotations" && echo "ok Annotations ConfigMap always created" || echo "FAIL Annotations ConfigMap not created" helm template test-release "${repo_root}/deployments/nvca-operator" \ --set "ngcConfig.serviceKey=fakekey" \ --show-only templates/custom-network-policies-configmap.yaml \ - | grep -q "nvcf-custom-network-policies" && echo "✓ Network policies ConfigMap always created" || echo "✗ Network policies ConfigMap not created" + | grep -q "nvcf-custom-network-policies" && echo "ok Network policies ConfigMap always created" || echo "FAIL Network policies ConfigMap not created"