diff --git a/src/compute-plane-services/nvca/pkg/nvca/backendk8scache.go b/src/compute-plane-services/nvca/pkg/nvca/backendk8scache.go index 09a6e78a9..c3107211c 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/backendk8scache.go +++ b/src/compute-plane-services/nvca/pkg/nvca/backendk8scache.go @@ -734,6 +734,13 @@ func (b *BackendK8sCacheBuilder) Start(ctx context.Context) (*BackendK8sCache, < if err != nil && !k8serrors.IsAlreadyExists(err) { return nil, nil, fmt.Errorf("failed to create model cache init namespace: %w", err) } + // Patch WorkloadInstanceTypeLabel onto the namespace so the Kyverno + // add-unbound-dns policy injects nvcf-unbound nameservers into writer + // job pods. Done here (not only in Create) so pre-existing namespaces + // on upgraded clusters receive the label immediately at startup. + if err := ensureModelCacheNamespaceLabel(ctx, c.clients.K8s.CoreV1().Namespaces(), mcInitNamespace.Name); err != nil { + return nil, nil, fmt.Errorf("failed to patch model cache init namespace labels: %w", err) + } // Network policies must exist in all workload namespaces; // the Helm handler methods will do this for each new namespace. diff --git a/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_gxcache.go b/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_gxcache.go index 5570dcc5a..2208dac56 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_gxcache.go +++ b/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_gxcache.go @@ -46,3 +46,17 @@ func ensureGXCacheNamespaceLabels(ctx context.Context, nsPatcher k8sNamespacePat _, err := nsPatcher.Patch(ctx, namespace, k8sapitypes.JSONPatchType, patchData, metav1.PatchOptions{}) return err } + +// ensureModelCacheNamespaceLabel patches WorkloadInstanceTypeLabel onto the +// model-cache init namespace so the Kyverno add-unbound-dns policy injects +// the nvcf-unbound nameserver into writer job pods. Called at NVCA startup so +// the label is applied immediately on upgrade, before any model cache reconcile +// runs. JSON patch "add" is idempotent: it inserts the key when absent and +// updates it when present, so re-running on an already-labelled namespace is safe. +func ensureModelCacheNamespaceLabel(ctx context.Context, nsPatcher k8sNamespacePatcher, namespace string) error { + key := strings.ReplaceAll(nvcatypes.WorkloadInstanceTypeLabel, "/", "~1") + patchData := []byte(fmt.Sprintf(`[{"op": "add", "path": "/metadata/labels/%s", "value": %q}]`, + key, nvcatypes.WorkloadInstanceTypeValueMiniService)) + _, err := nsPatcher.Patch(ctx, namespace, k8sapitypes.JSONPatchType, patchData, metav1.PatchOptions{}) + return err +} diff --git a/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_test.go b/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_test.go index efa33f3f1..d4a338985 100644 --- a/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_test.go +++ b/src/compute-plane-services/nvca/pkg/nvca/backendk8scache_test.go @@ -40,6 +40,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/propagation" @@ -4426,9 +4427,9 @@ func TestGetGPUUsageStats_FallbackToNonSuffixSingleType(t *testing.T) { Status: corev1.NodeStatus{ Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}, Allocatable: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("5"), - corev1.ResourceMemory: resource.MustParse("32Gi"), - corev1.ResourceEphemeralStorage: resource.MustParse("256Gi"), + corev1.ResourceCPU: resource.MustParse("5"), + corev1.ResourceMemory: resource.MustParse("32Gi"), + corev1.ResourceEphemeralStorage: resource.MustParse("256Gi"), corev1.ResourceName(nodefeatures.GPUResourceKey): resource.MustParse("4"), }, }, @@ -5460,3 +5461,56 @@ func TestUpdateSchedulerWorkloadMetrics(t *testing.T) { assert.Equal(t, float64(1), vals[gaugeKey{"kai-scheduler", "function"}]) }) } + +func TestEnsureModelCacheNamespaceLabel_PatchesWithCorrectPayload(t *testing.T) { + namespace := "nvca-modelcache-init" + expectedPatch := []byte(fmt.Sprintf(`[{"op": "add", "path": "/metadata/labels/%s", "value": %q}]`, + strings.ReplaceAll(nvcatypes.WorkloadInstanceTypeLabel, "/", "~1"), + nvcatypes.WorkloadInstanceTypeValueMiniService)) + + nsPatcher := &mockNamespacePatcher{} + nsPatcher.On("Patch", mock.Anything, namespace, apitypes.JSONPatchType, expectedPatch, metav1.PatchOptions{}). + Return(&corev1.Namespace{}, nil) + + err := ensureModelCacheNamespaceLabel(context.Background(), nsPatcher, namespace) + assert.NoError(t, err) + nsPatcher.AssertExpectations(t) +} + +func TestEnsureModelCacheNamespaceLabel_PatchError(t *testing.T) { + nsPatcher := &mockNamespacePatcher{} + nsPatcher.On("Patch", mock.Anything, mock.Anything, apitypes.JSONPatchType, mock.Anything, metav1.PatchOptions{}). + Return(nil, fmt.Errorf("patch error")) + + err := ensureModelCacheNamespaceLabel(context.Background(), nsPatcher, "nvca-modelcache-init") + assert.Error(t, err) +} + +// TestEnsureModelCacheNamespaceLabel_IdempotentWhenLabelPresent confirms that +// ensureModelCacheNamespaceLabel always issues the JSON patch "add" operation, +// even when the label is already set. RFC 6902 §4.1 specifies that "add" on an +// existing object key replaces its value, so the call is safe and idempotent +// regardless of whether the namespace was freshly created or already labelled. +func TestEnsureModelCacheNamespaceLabel_IdempotentWhenLabelPresent(t *testing.T) { + namespace := "nvca-modelcache-init" + expectedPatch := []byte(fmt.Sprintf(`[{"op": "add", "path": "/metadata/labels/%s", "value": %q}]`, + strings.ReplaceAll(nvcatypes.WorkloadInstanceTypeLabel, "/", "~1"), + nvcatypes.WorkloadInstanceTypeValueMiniService)) + + // Simulate a namespace that already carries the correct label; the API + // server accepts the patch (replace is a no-op at the state level). + alreadyLabelled := &corev1.Namespace{} + alreadyLabelled.Labels = map[string]string{ + nvcatypes.WorkloadInstanceTypeLabel: nvcatypes.WorkloadInstanceTypeValueMiniService, + } + + nsPatcher := &mockNamespacePatcher{} + nsPatcher.On("Patch", mock.Anything, namespace, apitypes.JSONPatchType, expectedPatch, metav1.PatchOptions{}). + Return(alreadyLabelled, nil) + + err := ensureModelCacheNamespaceLabel(context.Background(), nsPatcher, namespace) + assert.NoError(t, err) + // Patch must have been called exactly once — not skipped because the label + // was already present. + nsPatcher.AssertNumberOfCalls(t, "Patch", 1) +}