fix(validator): reject incompatible StorageClass before model-cache PVC create

validators/performance/inference_perf_constraint.go

@@ -405,6 +405,7 @@ type inferenceWorkloadConfig struct {
 	deployedByUs           bool   // true if we (or a prior run we own) created the workload
 	modelCacheSize         string // PVC size (e.g. "100Gi") enabling the model-weights cache; empty = disabled
 	modelCacheStorageClass string // StorageClass for the cache PVC; empty = cluster default
+	gpuNodeInstanceType    string // chosen node's node.kubernetes.io/instance-type; empty if unlabeled
 	routingMode            inferenceRoutingMode
 	routerMode             string // Dynamo frontend DYN_ROUTER_MODE (dynamo-router path only); env > default (see resolveRouterMode)
 
@@ -770,6 +771,7 @@ func buildInferenceConfig(ctx *validators.Context, mode *allocmode.Mode) (*infer
 		model:                  model,
 		modelCacheSize:         cacheSize,
 		modelCacheStorageClass: strings.TrimSpace(os.Getenv(envModelCacheStorageClass)),
+		gpuNodeInstanceType:    chosen.Labels[instanceTypeLabel],
 		routingMode:            routingMode,
 		routerMode:             routerMode,
 		gpuAllocMode:           mode,

validators/performance/model_cache.go

@@ -28,6 +28,7 @@ import (
 	"github.com/NVIDIA/aicr/validators"
 	batchv1 "k8s.io/api/batch/v1"
 	v1 "k8s.io/api/core/v1"
+	storagev1 "k8s.io/api/storage/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -125,6 +126,36 @@ const (
 	cacheWorkerFSGroup = int64(1000)
 )
 
+// storageCompatibilityRule declares that on a given CSI provisioner, the
+// listed machine families can only attach a StorageClass whose
+// parameters.type either carries compatibleTypePrefix or exactly matches
+// autoSelectType (a driver-specific value that resolves to a compatible disk
+// per-node rather than naming one directly, e.g. GKE's "dynamic"; leave empty
+// for a provisioner with no such value). Anything else provisioned by that
+// driver is rejected at attach time. Add a rule here for any other
+// cloud/provisioner with the same shape of restriction; nothing else in this
+// file needs to change.
+type storageCompatibilityRule struct {
+	provisioner          string
+	families             map[string]bool // node.kubernetes.io/instance-type family segment, e.g. "a4x" for "a4x-highgpu-4g"
+	compatibleTypePrefix string
+	autoSelectType       string
+	docsRef              string
+}
+
+var storageCompatibilityRules = []storageCompatibilityRule{
+	{
+		provisioner:          "pd.csi.storage.gke.io", // GKE Persistent Disk CSI driver (also provisions Hyperdisk)
+		families:             map[string]bool{"a4x": true},
+		compatibleTypePrefix: "hyperdisk-",
+		// "dynamic" auto-selects Hyperdisk vs Persistent Disk per the node's
+		// machine type (GKE 1.35.3-gke.1290000+); a4x can't attach Persistent
+		// Disk at all, so on a4x nodes it always resolves to Hyperdisk.
+		autoSelectType: "dynamic",
+		docsRef:        "docs/integrator/gke-gb200-networking.md#storage-prerequisites",
+	},
+}
+
 // modelCacheEnabled reports whether the PVC cache is active for this run.
 // config.modelCacheSize is "" only when the operator explicitly disabled it
 // (see parseModelCacheSize); the unset case has already been defaulted on.
@@ -162,23 +193,71 @@ func parseModelCacheSize(raw string) (string, bool, error) {
 	return raw, true, nil
 }
 
-// clusterHasDefaultStorageClass reports whether any StorageClass on the cluster
-// is annotated as the default. Used to fail fast before provisioning a cache
-// PVC with no StorageClass on a cluster that has no default.
-func clusterHasDefaultStorageClass(ctx *validators.Context) (bool, error) {
+// defaultStorageClass returns the cluster's effective default StorageClass,
+// or nil if none is annotated default. Kubernetes tolerates more than one
+// StorageClass annotated default; its own DefaultStorageClass admission
+// controller resolves the ambiguity by picking the most recently created one
+// (https://kubernetes.io/docs/concepts/storage/storage-classes/#default-storageclass).
+// Matching that here means this pre-flight checks the same StorageClass a PVC
+// with no storageClassName would actually bind to.
+func defaultStorageClass(ctx *validators.Context) (*storagev1.StorageClass, error) {
 	listCtx, cancel := context.WithTimeout(ctx.Ctx, defaults.DiagnosticTimeout)
 	defer cancel()
 	scs, err := ctx.Clientset.StorageV1().StorageClasses().List(listCtx, metav1.ListOptions{})
 	if err != nil {
-		return false, errors.Wrap(errors.ErrCodeInternal, "failed to list StorageClasses for cache pre-flight", err)
+		return nil, errors.Wrap(errors.ErrCodeInternal, "failed to list StorageClasses for cache pre-flight", err)
 	}
+	var best *storagev1.StorageClass
 	for i := range scs.Items {
-		ann := scs.Items[i].Annotations
-		if ann[defaultStorageClassAnnotation] == defaultStorageClassAnnotationValue || ann[defaultStorageClassAnnotationBeta] == defaultStorageClassAnnotationValue {
-			return true, nil
+		sc := &scs.Items[i]
+		ann := sc.Annotations
+		if ann[defaultStorageClassAnnotation] != defaultStorageClassAnnotationValue && ann[defaultStorageClassAnnotationBeta] != defaultStorageClassAnnotationValue {
+			continue
+		}
+		if best == nil || sc.CreationTimestamp.After(best.CreationTimestamp.Time) {
+			best = sc
 		}
 	}
-	return false, nil
+	// nil, nil means no default StorageClass is set, not an error.
+	//nolint:nilnil
+	return best, nil
+}
+
+// machineFamily returns the leading segment of a node.kubernetes.io/instance-type
+// value, e.g. "a4x" for "a4x-highgpu-4g". Empty for an empty or family-less input.
+func machineFamily(instanceType string) string {
+	family, _, _ := strings.Cut(instanceType, "-")
+	return family
+}
+
+// checkStorageClassNodeCompatibility reports an error when sc's disk type
+// can't attach to the worker node's machine family, per
+// storageCompatibilityRules. A nil sc (not found, e.g. a typo in the
+// explicit override) is not an error here; that surfaces via the normal
+// PVC-create path instead.
+func checkStorageClassNodeCompatibility(instanceType string, sc *storagev1.StorageClass) error {
+	if sc == nil {
+		return nil
+	}
+	family := machineFamily(instanceType)
+	for _, rule := range storageCompatibilityRules {
+		if sc.Provisioner != rule.provisioner || !rule.families[family] {
+			continue
+		}
+		typ := sc.Parameters["type"]
+		if strings.HasPrefix(typ, rule.compatibleTypePrefix) || (rule.autoSelectType != "" && typ == rule.autoSelectType) {
+			continue
+		}
+		typeGuidance := fmt.Sprintf("parameters.type starts with %q", rule.compatibleTypePrefix)
+		if rule.autoSelectType != "" {
+			typeGuidance += fmt.Sprintf(" (or is %q)", rule.autoSelectType)
+		}
+		return errors.New(errors.ErrCodeInvalidRequest, fmt.Sprintf(
+			"model-weights cache PVC would bind to StorageClass %q (provisioner %s), which node machine family %q can't attach; "+
+				"set %s to a StorageClass whose %s, or disable the cache with %s=off; see %s",
+			sc.Name, sc.Provisioner, family, envModelCacheStorageClass, typeGuidance, envModelCacheSize, rule.docsRef))
+	}
+	return nil
 }
 
 // ensureModelCache provisions the model-weights cache when enabled: an RWO PVC
@@ -207,26 +286,53 @@ func ensureModelCache(ctx *validators.Context, config *inferenceWorkloadConfig)
 			fmt.Sprintf("invalid %s=%q: must be a Kubernetes quantity (e.g. 100Gi)", envModelCacheSize, config.modelCacheSize), err)
 	}
 
-	// Fail fast when there is no StorageClass to bind the cache PVC to: with no
-	// explicit MODEL_CACHE_STORAGE_CLASS, the PVC relies on a cluster default,
-	// and without one it sits Pending until the populate Job times out (minutes).
-	// Surface an actionable error immediately instead.
-	if strings.TrimSpace(config.modelCacheStorageClass) == "" {
-		hasDefault, derr := clusterHasDefaultStorageClass(ctx)
+	// Resolve the StorageClass the cache PVC will bind to (explicit name, or
+	// the cluster default) for the checks below.
+	explicitSC := strings.TrimSpace(config.modelCacheStorageClass)
+	var resolvedSC *storagev1.StorageClass
+	if explicitSC == "" {
+		sc, derr := defaultStorageClass(ctx)
 		if derr != nil {
 			return derr
 		}
-		if !hasDefault {
+		if sc == nil {
 			return errors.New(errors.ErrCodeInvalidRequest,
 				fmt.Sprintf("model-weights cache is enabled but the cluster has no default StorageClass and %s is unset; "+
 					"set %s=<name> (e.g. gp2/gp3 on EKS, standard-rwo on GKE) or disable the cache with %s=off",
 					envModelCacheStorageClass, envModelCacheStorageClass, envModelCacheSize))
 		}
+		resolvedSC = sc
+	} else {
+		getCtx, getCancel := context.WithTimeout(ctx.Ctx, defaults.DiagnosticTimeout)
+		sc, gerr := ctx.Clientset.StorageV1().StorageClasses().Get(getCtx, explicitSC, metav1.GetOptions{})
+		getCancel()
+		switch {
+		case gerr == nil:
+			resolvedSC = sc
+		case apierrors.IsNotFound(gerr):
+			// Leave resolvedSC nil: fall through to the existing PVC-create
+			// path, which surfaces a nonexistent StorageClass the same way
+			// it always has.
+		default:
+			return errors.Wrap(errors.ErrCodeInternal, "failed to get StorageClass for cache pre-flight", gerr)
+		}
+	}
+	if cerr := checkStorageClassNodeCompatibility(config.gpuNodeInstanceType, resolvedSC); cerr != nil {
+		return cerr
+	}
+
+	// Pin the StorageClass we just validated onto the PVC: if we resolved an
+	// implicit cluster default above, use its name explicitly rather than
+	// leaving StorageClassName nil, so a default that changes between this
+	// check and PVC admission can't silently bind an unvalidated StorageClass.
+	pvcStorageClass := explicitSC
+	if pvcStorageClass == "" && resolvedSC != nil {
+		pvcStorageClass = resolvedSC.Name
 	}
 
 	// Bound the create calls so a slow/wedged apiserver can't burn the check
 	// budget before the (separately bounded) populate-Job wait even starts.
-	pvc := buildModelCachePVC(config.namespace, qty, strings.TrimSpace(config.modelCacheStorageClass))
+	pvc := buildModelCachePVC(config.namespace, qty, pvcStorageClass)
 	pvcCtx, pvcCancel := context.WithTimeout(ctx.Ctx, defaults.DiagnosticTimeout)
 	_, err = ctx.Clientset.CoreV1().PersistentVolumeClaims(config.namespace).Create(pvcCtx, pvc, metav1.CreateOptions{})
 	pvcCancel()