From 20849c77ff9d3c1f34002b53fad95ecf99dbc957 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 12:26:55 -0700 Subject: [PATCH 1/6] fix(recipes): raise K8s floors to clear the DRA chart's kubeVersion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every recipe inherits nvidia-dra-driver-gpu from base.yaml, and the pinned chart declares kubeVersion: ">=1.32.0-0". Helm refuses the install below that, so 29 overlays declaring a lower K8s.server.version admitted clusters that pass every recipe-time check and then fail at `helm install`. recipes/overlays/ocp.yaml already carried >= 1.32 for exactly this reason; its comment recorded the diagnosis but the rest of the catalog was never reconciled. Every declaration is raised, not just base.yaml: constraints merge by name with the later overlay winning and no max comparison, so a leaf declaring ">= 1.30" silently overwrites a higher floor inherited from base. That is visible in the golden churn — the 16 leaves that moved are those inheriting a raised floor, while leaves declaring their own >= 1.34 or >= 1.32.4 were already clear and are unchanged. Adds a guard asserting no overlay or mixin declares a floor below the chart's, so the reconciliation cannot drift back. Control verified: reverting one leaf to 1.31 fails the guard. It also fails closed when no floors match, so it cannot go vacuous. Signed-off-by: Yuan Chen --- pkg/bundler/testdata/stock_render_golden.yaml | 32 ++--- pkg/recipe/dra_k8s_floor_test.go | 114 ++++++++++++++++++ .../testdata/catalog_parity_golden.yaml | 32 ++--- recipes/overlays/a100-eks-training.yaml | 2 +- .../a100-eks-ubuntu-training-kubeflow.yaml | 2 +- .../overlays/a100-eks-ubuntu-training.yaml | 2 +- .../a100-gke-cos-training-kubeflow.yaml | 2 +- recipes/overlays/a100-gke-cos-training.yaml | 2 +- recipes/overlays/a100-oke-training.yaml | 2 +- .../a100-oke-ubuntu-training-kubeflow.yaml | 2 +- .../overlays/a100-oke-ubuntu-training.yaml | 2 +- recipes/overlays/base.yaml | 2 +- recipes/overlays/eks-inference.yaml | 2 +- recipes/overlays/eks-training.yaml | 2 +- recipes/overlays/eks.yaml | 2 +- recipes/overlays/gke-cos-inference.yaml | 2 +- recipes/overlays/gke-cos-training.yaml | 2 +- recipes/overlays/gke-cos.yaml | 2 +- recipes/overlays/kind-inference.yaml | 2 +- recipes/overlays/kind.yaml | 2 +- recipes/overlays/l40s-oke-inference.yaml | 2 +- recipes/overlays/l40s-oke-training.yaml | 2 +- recipes/overlays/lke-inference.yaml | 2 +- recipes/overlays/lke-training.yaml | 2 +- recipes/overlays/lke.yaml | 2 +- recipes/overlays/oke-ol-inference.yaml | 2 +- recipes/overlays/oke-ol-training.yaml | 2 +- recipes/overlays/oke-ol.yaml | 2 +- .../overlays/rtx-pro-6000-lke-inference.yaml | 2 +- .../overlays/rtx-pro-6000-lke-training.yaml | 2 +- .../rtx-pro-6000-lke-ubuntu-inference.yaml | 2 +- .../rtx-pro-6000-lke-ubuntu-training.yaml | 2 +- 32 files changed, 175 insertions(+), 61 deletions(-) create mode 100644 pkg/recipe/dra_k8s_floor_test.go diff --git a/pkg/bundler/testdata/stock_render_golden.yaml b/pkg/bundler/testdata/stock_render_golden.yaml index d43d9f13f..2efcc80c9 100644 --- a/pkg/bundler/testdata/stock_render_golden.yaml +++ b/pkg/bundler/testdata/stock_render_golden.yaml @@ -4,27 +4,27 @@ # One entry per leaf overlay: a digest over its fully rendered helm-deployer # bundle tree (sorted relative paths paired with per-file content hashes). a100-aks-ubuntu-training-kubeflow: 265940a7ef6f82bfe9752e5e488b416d5d54aa0245b29f0262796978547af68d -a100-any: 457dd24ba4f8b9d44af7f68ddc1f4e9d46d0e6eac30b0eceb2e979e1e9c2d2db -a100-eks-ubuntu-training-kubeflow: 9a77e69e7ea639be6100c4c2c0cb49bf87059348020c98f5c561158a22d452e7 -a100-gke-cos-training-kubeflow: 0fb904af67e4ad85cb1b71efed94c8cd3966fc36f14cc80d9b50d5ffe7babde5 -a100-oke-ubuntu-training-kubeflow: 92c1a420bccca84c7ca7e74d8960af6c3662bce4c708c45ccaa386f209dc1c9e -b200-any: b8e028ff78b7f142c3e157143a258510c4f44579e31f13c183796767de53d862 +a100-any: 9e6f720c91e3d1f15e0fb930f8aed364a657a874340823868e056676d635bb78 +a100-eks-ubuntu-training-kubeflow: 180f190dd61f516c11a0161e402d49a5582ebfb8d21249932d5314ee9c27b58a +a100-gke-cos-training-kubeflow: 05929baa06d96fb5a0667bf3bfddc79c2b39bba9c54974214c269ecd403be07f +a100-oke-ubuntu-training-kubeflow: 94ca5d0debe6285457fed3be501cd1791a160dbde3f9976b77aef78ae728687b +b200-any: e9db5b5573fee7df43427a04fd3ac059c14e2cc9a565b0a11f1ebcca7b642981 b200-gke-cos-inference-dynamo: cd3c475c9bb709b8164571be18ec625b38263a762d91d16f89049e75fc934eb4 b200-gke-cos-training-kubeflow: f35da200bec023578aaaeebe337775d9b62718cfd2ac0c568c4a56f69575fbb3 bcm-inference: 2a7d525459b585c3b1639eb556de8feb682b27f6bb8be3a7015de7773ba7c235 -gb200-any: ff2c8128bc91fce4a48d650d160659b08272a346a044f7955d3b23109fde6e8d +gb200-any: 4e467ff5dda2a59082d13cfcfddf15b7763655fffab5128cbad959fb1c9c08ca gb200-eks-ubuntu-inference-dynamo: 637a4fd32933c220453d3732363c9a56bd75dd91cdd886a9b0571b666670744d gb200-eks-ubuntu-training-kubeflow: e4d2b1cc86cf33b7741e881a984f6943353bf2e1b85d918b8d3e82cf50c8fa60 gb200-eks-ubuntu-training-slurm: d356fec2562b9d3f8adcccb2344402f41270ca301f353affd6529aaa86e430d4 gb200-oke-ubuntu-inference-dynamo: 0304fffcd27d04827ae561b32ced55b9ff87e0f6049bda30a075afd49f24ad11 gb200-oke-ubuntu-training-kubeflow: 3ed8582775ca602b48eb88ff05d228db3f4189f4230f31f72cf355be8d5241b0 -gb300-any: a67ce96e16a0a22d85f22c2803f029e49b1befb4226a60cdf9820fb6b6e77ea2 +gb300-any: da939cd77c1e1331f4d98d8081225ac427407196389570670051bc5482fbc049 gb300-eks-ubuntu-inference-dynamo: c725636167448e64ef1d567613aded4bcd7ede0137a3d1e4b9c70be1ec20c61a gb300-eks-ubuntu-training-kubeflow: e84019cb236bae67eef79a8166214295b066d1514ebb869619898925be48522d h100-aks-ubuntu-inference-dynamo: 54061313326c300b83a0627f415002bd080c7c0e338e19bcce7996446a791065 h100-aks-ubuntu-training-kubeflow: ea0180c91126bdda49e3641538cfef03961358ebe4d883e893d53a82b26d4bd7 h100-aks-ubuntu-training-slurm: 7f9c80295d770456adca24b66bc7681e42fd9db54fcb97d2541c657c939645c9 -h100-any: 7e98352317f4d2a753255ec776e7b7bc605ad288f3856b4a467a15f8844aa903 +h100-any: 449a706634ce8ca3407bbc8a566f41db9b66bc8a3e35883f5052e3ba0ec9569e h100-bcm-ubuntu-training: c73ab59f00b5425a0be7c8aaac6107469c8fb59e54eecce9cf4937704821611d h100-eks-ubuntu-inference-dynamo: 6a3e47afa2018591b22f2fd96105d75ec23b631fafb4c0f30f42ba77af6f537a h100-eks-ubuntu-inference-nim: 3b9b4fe53c967fa95e56e2cf6e631a55f044d50f76423e11ac176d839e005d2d @@ -36,18 +36,18 @@ h100-gke-cos-training-slurm: 0efee10e13589930111b9f5064430d2dcac463d2d2cf6032a6a h100-kind-inference-dynamo: 8cea3d208001c5d7bb542d83a75e2ab641722981b279db9391a300b8f9d1f20c h100-kind-training-kubeflow: 1203d2e475a51a3e07d3a62a87ad2dca91c24eda3aea800aca86bf2dfee5a80a h100-kind-training-slurm: 8a27b1ae4e91cda80c4a32d456552952f2ec0a28af10e9e0317219b551ddd133 -h200-any: ceee05a6ed9dd0f218919a90187de851e16e93cf89293c9746d5f0e46bbda222 +h200-any: 8eb6e4073c0acaeb9a6eaf3c26e82c6f0cfad17c1eb5dc2d906a4dfeebebcaf4 h200-eks-inference: 2900b7d96bc48a27eb5fedda75b99b010b9165bc6058372d2de3b991f64c4982 h200-eks-training: 4b3074954d10a81d057ac746c7bddf3664637dfd002cd0955348ec3dee11eb20 -l40s-any: dabbb23da635f2b6db7803be55841a7305f884ce7798dbf79d952686004f09d8 -l40s-oke-inference: 8a8884dcf64b9feaa708b152cc4eaf1b9c09d790de295cf2b0e5a5ca464154f1 -l40s-oke-training: f9b741b390f9f29451c7ad16da56602bbf6497bec067be6b19242cffdf1e1de7 -monitoring-hpa: 32917a470d982044c3d49960dbd5e62ab56cd870375bfac23c35d7a4b3f4298a +l40s-any: 18fa876e004f2507d2610047ac78f0265c7214336ff3538ed4bb78fda479a709 +l40s-oke-inference: 367a7ba248841c1a4cae620dd446712007d869f007e1e1f61f6a1f799316f8d1 +l40s-oke-training: 7430568438061aede469c5ca3779ecc34a78aaf27786e2626ca6264d49b18f6a +monitoring-hpa: b5377927b96805237ba93800bbf4c963dabcf2ed61948b7427e07f35df05832f ocp-inference-nim: 016fc59c13e901de5556f09f3cb3d5a90f91b9826d34f126fde1e066b311605f ocp-training: 652f7ccdf52009adafe993da111857aa67390678f382b4c0caf0aecd0d613635 -rtx-pro-6000-any: 9115c5d61d9941130e448f29b9fbcd834f6d15a46fa4edf3239afc44fcdd36f0 +rtx-pro-6000-any: 59bfd011fe436f7a8785df6227d35dea4bcceb5590650cd4063c9f4573389d98 rtx-pro-6000-eks-ubuntu-inference-dynamo: 8c35c21289dc4a56f3e035da9d35427c9a1f2402c643febda2c6b397692b2a49 rtx-pro-6000-eks-ubuntu-inference-nim: feb5f1d72ebd95459c9df0891420d20fac42c6b8c16f0e8fa551da61a19aef1d rtx-pro-6000-eks-ubuntu-training-kubeflow: 1dfe905ac8d891fba4ce797de6532320024fe2616653e4703be5fc8e29716d77 -rtx-pro-6000-lke-ubuntu-inference: c5d53e0adbe9c500ecd5eb45d2482e632f1c414657a08ebef053a8aab90fc3cb -rtx-pro-6000-lke-ubuntu-training: 39e9cc1636a11d2568d9d50ecf3744f5773c4870772727661c40be63bac33280 +rtx-pro-6000-lke-ubuntu-inference: df7a09c3a1038beba8ba33b675c02e80c5cfa377eee5b89e550096c56d1ea6e0 +rtx-pro-6000-lke-ubuntu-training: bd5efe51684a82b4532c5173b1d0a23169be7f0bf69ba105c6129604e8589952 diff --git a/pkg/recipe/dra_k8s_floor_test.go b/pkg/recipe/dra_k8s_floor_test.go new file mode 100644 index 000000000..195f0c1d7 --- /dev/null +++ b/pkg/recipe/dra_k8s_floor_test.go @@ -0,0 +1,114 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package recipe + +import ( + "io/fs" + "regexp" + "strconv" + "strings" + "testing" +) + +// draChartKubeVersionMinor is the minor version floor the pinned +// nvidia-dra-driver-gpu chart declares: +// +// kubeVersion: ">=1.32.0-0" +// +// Helm REJECTS the install outright when the cluster is below it, so a recipe +// declaring a lower K8s.server.version admits clusters that pass every +// recipe-time check and then fail at `helm install`. +// +// Sourced from the chart, not from a running cluster. If the DRA chart pin in +// recipes/registry.yaml moves to a version with a different kubeVersion, this +// constant and the affected overlays must move together — that coupling is the +// point of the guard. +const draChartKubeVersionMinor = 32 + +// k8sFloorRE captures the minor version from a K8s.server.version constraint +// expressed as a floor. Only `>=` forms are checked: an exact pin or a range is +// a deliberate statement that should be reviewed on its own terms. +var k8sFloorRE = regexp.MustCompile(`(?s)- name: K8s\.server\.version\s*\n\s*value: "\s*>=\s*1\.(\d+)`) + +// TestOverlayK8sFloorsClearDRAChartFloor asserts no overlay or mixin declares a +// Kubernetes floor below the DRA chart's own kubeVersion. +// +// Every recipe inherits nvidia-dra-driver-gpu from base.yaml — no overlay +// removes or disables it — so the chart's floor applies catalog-wide. +// +// Why every declaration and not just base.yaml: constraints merge by name with +// the LATER overlay winning and no max comparison (see mergeValidation in +// validation.go). A leaf declaring ">= 1.30" silently overwrites a higher floor +// inherited from base, so raising base alone would not hold. This is the same +// last-wins hazard documented for driver floors in #2438. +// +// recipes/overlays/ocp.yaml already carried >= 1.32 for exactly this reason +// before the rest of the catalog was reconciled; its comment records the +// diagnosis. +func TestOverlayK8sFloorsClearDRAChartFloor(t *testing.T) { + t.Parallel() + + efs := GetEmbeddedFS() + + var checked int + err := fs.WalkDir(efs, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".yaml") { + return nil + } + if !strings.Contains(path, "overlays/") && !strings.Contains(path, "mixins/") { + return nil + } + raw, readErr := efs.ReadFile(path) + if readErr != nil { + return readErr + } + m := k8sFloorRE.FindStringSubmatch(string(raw)) + if m == nil { + return nil + } + checked++ + minor, convErr := strconv.Atoi(m[1]) + if convErr != nil { + t.Errorf("%s: could not parse K8s.server.version minor %q: %v", path, m[1], convErr) + return nil + } + if minor < draChartKubeVersionMinor { + t.Errorf("%s declares K8s.server.version \">= 1.%d\", below the pinned "+ + "nvidia-dra-driver-gpu chart's kubeVersion \">=1.%d.0-0\".\n"+ + " Every recipe inherits the DRA driver from base.yaml, and Helm refuses the\n"+ + " install below the chart floor — so this recipe validates clean and then\n"+ + " fails at `helm install`. Raise it to \">= 1.%d\".\n"+ + " Raising base.yaml alone does NOT fix a leaf: constraints merge last-wins\n"+ + " with no max comparison, so a lower leaf value overwrites a higher\n"+ + " inherited one. See #2402.", + path, minor, draChartKubeVersionMinor, draChartKubeVersionMinor) + } + return nil + }) + if err != nil { + t.Fatalf("walking embedded recipes: %v", err) + } + + // Fail closed on a vacuous pass: if nothing matched, the guard is inert. + if checked == 0 { + t.Fatal("no K8s.server.version floors found in overlays or mixins — this guard " + + "is vacuous. Either the constraint name changed or the embed pattern no " + + "longer covers the recipe tree.") + } + t.Logf("verified %d K8s.server.version floor(s) clear the DRA chart floor", checked) +} diff --git a/pkg/recipe/testdata/catalog_parity_golden.yaml b/pkg/recipe/testdata/catalog_parity_golden.yaml index 95fe07389..26601b5a9 100644 --- a/pkg/recipe/testdata/catalog_parity_golden.yaml +++ b/pkg/recipe/testdata/catalog_parity_golden.yaml @@ -4,27 +4,27 @@ # One entry per leaf overlay: sha256 of its deterministically-marshalled # resolved recipe. A moved digest means that recipe's resolved bytes changed. a100-aks-ubuntu-training-kubeflow: 3d8d77ac7ac29bf13253410e0b0486bc322d69d66d8778884df3375619566d58 -a100-any: 62d1581b21fecd3465b69659bac3dc9e5438eb2612c98c1acdd7b19cc9237084 -a100-eks-ubuntu-training-kubeflow: 0196adfa0c77f230d95ed3ebee2bdade8ac720f631cd3bbe2b8cec7d55094a10 -a100-gke-cos-training-kubeflow: 514f996360b251a6ef657ba1a715b5705be03d6b6dccb53887ee96554a83ffb6 -a100-oke-ubuntu-training-kubeflow: 310a6ee2abdd7f8508882429d657c01bd23458ba635236c410934825e4f617dc -b200-any: 63dbc4fe27b84321e8b1991cd98c13f85395a3919d13e0c6891599eaa1f1e729 +a100-any: 31bc0466b3d53eb0aadeb9b14b185b99b69732c4d3c900dfcf7f5d9db521e806 +a100-eks-ubuntu-training-kubeflow: 28b246bb2fa8a6d2494f02e8322e470a662d979a43a4dfddc68e18230537dfc6 +a100-gke-cos-training-kubeflow: e889ee9a78c119c4ef852bc59241193d92d602c9b8f468b5245a66a8b80969a0 +a100-oke-ubuntu-training-kubeflow: e26e364447f30f32f827a515d6cd79a8172d4fe82fc636337734c6bdc645305b +b200-any: 67d10aeb8f4260df9e2439942f75a905d169e21098944c545e4dcb123487495e b200-gke-cos-inference-dynamo: 8174db8270ea447d9b307f21be9945b919ed3bcdacc702d2961d2ff75fb60c35 b200-gke-cos-training-kubeflow: 96723738cb0baaaec300c95d5cfbc08a18d999fd0adb52aef3c88436561da1e7 bcm-inference: 92fa64d3af61026891b37cbf692d2663be7b2484078ab65e2653d18fc2bfe6f8 -gb200-any: 9ee6ab187b4c1b93cd859d86ada35e922d0873ffc6fe892cb084ad98baf09727 +gb200-any: 9377312866b4e8596235ab8959ffe6368eb800c2cb9f9e12410c16b50eb4149a gb200-eks-ubuntu-inference-dynamo: 8babae6413d104da16060f18d919c80fd97ff983bd32f9876bd5f72da7fd6be6 gb200-eks-ubuntu-training-kubeflow: 63b2fbd3a195bac682bb45263e5d5d0ec479a6571dd38ef0505eeb4f287b29e9 gb200-eks-ubuntu-training-slurm: c35e33618c0a8bb548bbe3c5d6765faf90cebe1427478ef97089f7a30bb37a6c gb200-oke-ubuntu-inference-dynamo: 6f01848824dcbefb0f58ec56ee9e911fff1470c9f67c0d0a9ba0de1dd1e648d2 gb200-oke-ubuntu-training-kubeflow: 3959f47474edfac9b53a65939117d4364551fde0c9324246f195f144fe3beb12 -gb300-any: 7f68607dffbbfc912f19b00ffa833caa81ca675d25bc22cf612dbd0b76f39475 +gb300-any: eb4c4d7016820afe6c6c9524cd96d4e3f3022cfb6091133354308d1f13f11c35 gb300-eks-ubuntu-inference-dynamo: b14a3d99503949b8bfcdc81455da4af389a339deb24ea784637a7901297d70ad gb300-eks-ubuntu-training-kubeflow: 5303e5ebf8dc2ec13c2cff60c7caba2ef53d1a65bfd69d06f4f78cbb93835c49 h100-aks-ubuntu-inference-dynamo: 9fe4d82412d250d4004efc56fa4ddd6b8b1a9166387455e96ac2c2284d323107 h100-aks-ubuntu-training-kubeflow: 72a8ca777c333e0db82717f53a706ae9155163ff0c73fd4e5ed8d71badda5811 h100-aks-ubuntu-training-slurm: ef48d77171817bb5b24815d128457c246e0e5dc0fe850aa6881276952e590055 -h100-any: f5a55f03948075adde9b1b24fd650d98fa4d35b85f394d071ad3365ba05588a5 +h100-any: bc038c07e521f598b08ac12929e9e9be3e877b516a74db0323077d42c9757469 h100-bcm-ubuntu-training: 76746f45add1fb6b0d7b6441b4d9581acceb3b5d3d4800b943c1cf460c92fbc5 h100-eks-ubuntu-inference-dynamo: 0a31f03010849f76b385ce81c21e499af884f41bee89de47b9bacf03f8b6e311 h100-eks-ubuntu-inference-nim: 9c6a4839bf5b59f620093427de0e138a9fafaf1f55f3fa7f555b1ee33b914a3a @@ -36,18 +36,18 @@ h100-gke-cos-training-slurm: 414f95e1a231b1c86bcbe70729b820a95533367dec0c9e3f7e3 h100-kind-inference-dynamo: 0e2d552b62b55b91f960c9721ed128b1d07f68be8517799473fbaa3baa1cc6e5 h100-kind-training-kubeflow: 4bb03659faaa6a9a66d58dfc95572bdb54b75010c219c4bbefbd0eafeb8ed867 h100-kind-training-slurm: 0245c786638ade1023ba4dc49e0f2efd56d2f789bdf447e946bbc3a3e652e734 -h200-any: acf1986d76037dddd025125eaa34eb7ecf3d31fe62cb80e7dc89dd042ebb42f5 +h200-any: a836e39e3500493e588ec46df019e29f064b325066c5f33b574b034c28c32a00 h200-eks-inference: d1ff148001722e9d5e23606bd7770f8120506f08c10f0c132e0c379f535609c5 h200-eks-training: 98d04b3c81a059a0321451a39772af8d0d3310ad1fb861837f87d82dd997c4a1 -l40s-any: 89210ba5815f93ce3d7b1a2a78f7ba75d083556e940603a1bc812eac9aeccd0a -l40s-oke-inference: 6c75ef93e86fb21dbdf73ac97089b894ca8da832164dc807e94592493ab7f7cc -l40s-oke-training: 2a35011bd59cf3c727cea158d504c51bd59cba069620d3d6a4988d68972dee74 -monitoring-hpa: 15e93304e7e68997e7b15a0be4af360a29f10c869502e6dccb6a6cefb6519fd5 +l40s-any: d66c22b36824a9336f76b061e1944f1fcb5eb4845cbf2e8235731a2b1e5c0883 +l40s-oke-inference: b10497f38c3468134740f9968ecf072be198b5ca302470ae75621bc1fbb50a36 +l40s-oke-training: 796b1b1763233ea258d3b766798bec8b91a61775bdca7969b91f54ac392dfe4e +monitoring-hpa: ffe345da6067c5a6a6dd19051cf9186fd11cce93070c55a8f679a961bcbe60b1 ocp-inference-nim: 49245ecfc91f67fcc2ed0b305b00046a4164628cafc75ab98542e6dafa626821 ocp-training: aaa8cd7f45ef42235b075936461d715e2e017ea2802df9e8017de38304a17915 -rtx-pro-6000-any: 9639d67639256a44be8566f3f8eda49ceb4025b6b140a3bc2a4b6d4768d07ffa +rtx-pro-6000-any: b5644bcf8dac4a1752b956b64b671b231152ce6644b81184e4dda79331bb30ac rtx-pro-6000-eks-ubuntu-inference-dynamo: cbe11051d979ae20e83471142c4582e1504fd24f55898a00afcededbe143bba6 rtx-pro-6000-eks-ubuntu-inference-nim: 08b73d58d0d60681be55ee1402bdec5bf1367a0cb137a273ca27bec0788cc020 rtx-pro-6000-eks-ubuntu-training-kubeflow: c2595e2994d56a735714103eedd30c6afd45921f458bf026c3432017b5eba60e -rtx-pro-6000-lke-ubuntu-inference: 28cdab0531f8ed6018391aba7cab1fbfad8ba551392de1b9138f3b88042f5340 -rtx-pro-6000-lke-ubuntu-training: 312c4cf49d0e8cec0377cd9858338d7106226a2252d81828a75c1d426a194a2f +rtx-pro-6000-lke-ubuntu-inference: 5658ed36024798b11f4e253b72c842f3686161508a72e54e58a8ba22c9c90f44 +rtx-pro-6000-lke-ubuntu-training: 44b9ed14fe8940fac82b7aafca7f97f6619ce872b3e53adc4b0e4af08c8648f6 diff --git a/recipes/overlays/a100-eks-training.yaml b/recipes/overlays/a100-eks-training.yaml index c2e2c1159..f9fc790c7 100644 --- a/recipes/overlays/a100-eks-training.yaml +++ b/recipes/overlays/a100-eks-training.yaml @@ -32,7 +32,7 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: # A100-specific GPU Operator overrides (inherits valuesFile from eks-training). diff --git a/recipes/overlays/a100-eks-ubuntu-training-kubeflow.yaml b/recipes/overlays/a100-eks-ubuntu-training-kubeflow.yaml index 95cdd1eed..368c2afa9 100644 --- a/recipes/overlays/a100-eks-ubuntu-training-kubeflow.yaml +++ b/recipes/overlays/a100-eks-ubuntu-training-kubeflow.yaml @@ -37,6 +37,6 @@ spec: # A100 + EKS specific constraints (not covered by mixin) constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/a100-eks-ubuntu-training.yaml b/recipes/overlays/a100-eks-ubuntu-training.yaml index d9a461d80..c6625021e 100644 --- a/recipes/overlays/a100-eks-ubuntu-training.yaml +++ b/recipes/overlays/a100-eks-ubuntu-training.yaml @@ -35,7 +35,7 @@ spec: # A100 + EKS specific constraints (not covered by mixin) constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/a100-gke-cos-training-kubeflow.yaml b/recipes/overlays/a100-gke-cos-training-kubeflow.yaml index 6eb940a08..0df5add20 100644 --- a/recipes/overlays/a100-gke-cos-training-kubeflow.yaml +++ b/recipes/overlays/a100-gke-cos-training-kubeflow.yaml @@ -32,7 +32,7 @@ spec: # Constraints for A100 on GKE COS for Kubeflow training workloads constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" # Kubeflow Training Operator for TrainJob support. # Declared inline (not via the platform-kubeflow mixin) to match the GKE COS diff --git a/recipes/overlays/a100-gke-cos-training.yaml b/recipes/overlays/a100-gke-cos-training.yaml index 0f8f716a4..ae92d247d 100644 --- a/recipes/overlays/a100-gke-cos-training.yaml +++ b/recipes/overlays/a100-gke-cos-training.yaml @@ -32,7 +32,7 @@ spec: # the GKE COS training baseline rather than the H100 1.32 floor. constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: # A100-specific GPU Operator overrides (inherits valuesFile from gke-cos-training). diff --git a/recipes/overlays/a100-oke-training.yaml b/recipes/overlays/a100-oke-training.yaml index e929c2ce5..4b6a8ddab 100644 --- a/recipes/overlays/a100-oke-training.yaml +++ b/recipes/overlays/a100-oke-training.yaml @@ -34,7 +34,7 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: # A100-specific GPU Operator overrides (inherits valuesFile from oke-training). diff --git a/recipes/overlays/a100-oke-ubuntu-training-kubeflow.yaml b/recipes/overlays/a100-oke-ubuntu-training-kubeflow.yaml index ca77b4cda..d7ed0c31c 100644 --- a/recipes/overlays/a100-oke-ubuntu-training-kubeflow.yaml +++ b/recipes/overlays/a100-oke-ubuntu-training-kubeflow.yaml @@ -37,6 +37,6 @@ spec: # A100 + OKE specific constraints (not covered by mixin) constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/a100-oke-ubuntu-training.yaml b/recipes/overlays/a100-oke-ubuntu-training.yaml index dcb5f5867..bb298d2da 100644 --- a/recipes/overlays/a100-oke-ubuntu-training.yaml +++ b/recipes/overlays/a100-oke-ubuntu-training.yaml @@ -35,7 +35,7 @@ spec: # A100 + OKE specific constraints (not covered by mixin) constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/base.yaml b/recipes/overlays/base.yaml index 473b2629a..67fc3dbb8 100644 --- a/recipes/overlays/base.yaml +++ b/recipes/overlays/base.yaml @@ -22,7 +22,7 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.25" + value: ">= 1.32" componentRefs: - name: nfd diff --git a/recipes/overlays/eks-inference.yaml b/recipes/overlays/eks-inference.yaml index b5fb62a03..c85e8e890 100644 --- a/recipes/overlays/eks-inference.yaml +++ b/recipes/overlays/eks-inference.yaml @@ -33,6 +33,6 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/eks-training.yaml b/recipes/overlays/eks-training.yaml index fe436bcf7..d7014a611 100644 --- a/recipes/overlays/eks-training.yaml +++ b/recipes/overlays/eks-training.yaml @@ -29,7 +29,7 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: # Training workloads use the training-optimized GPU Operator values diff --git a/recipes/overlays/eks.yaml b/recipes/overlays/eks.yaml index 3b9459d06..195186014 100644 --- a/recipes/overlays/eks.yaml +++ b/recipes/overlays/eks.yaml @@ -28,7 +28,7 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.28" + value: ">= 1.32" # EKS-specific components componentRefs: diff --git a/recipes/overlays/gke-cos-inference.yaml b/recipes/overlays/gke-cos-inference.yaml index d659c931f..1240b9304 100644 --- a/recipes/overlays/gke-cos-inference.yaml +++ b/recipes/overlays/gke-cos-inference.yaml @@ -33,6 +33,6 @@ spec: # Inference specific constraints for GKE workloads constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/gke-cos-training.yaml b/recipes/overlays/gke-cos-training.yaml index 3cea0c4bd..e0e1fb19d 100644 --- a/recipes/overlays/gke-cos-training.yaml +++ b/recipes/overlays/gke-cos-training.yaml @@ -29,7 +29,7 @@ spec: # Training specific constraints for GKE workloads constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: # Training workloads use the GKE-COS training-optimized GPU Operator values diff --git a/recipes/overlays/gke-cos.yaml b/recipes/overlays/gke-cos.yaml index 29e42499c..2dab8f638 100644 --- a/recipes/overlays/gke-cos.yaml +++ b/recipes/overlays/gke-cos.yaml @@ -28,7 +28,7 @@ spec: # GKE-specific constraints constraints: - name: K8s.server.version - value: ">= 1.28" + value: ">= 1.32" # The GPU stack shape is a configuration profile (ADR-015, issue # #1761): one qualified GKE cluster state per value — how the driver is diff --git a/recipes/overlays/kind-inference.yaml b/recipes/overlays/kind-inference.yaml index 0ebbf84b1..3467d5b17 100644 --- a/recipes/overlays/kind-inference.yaml +++ b/recipes/overlays/kind-inference.yaml @@ -33,6 +33,6 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/kind.yaml b/recipes/overlays/kind.yaml index e8a20d5a7..08afd5c5b 100644 --- a/recipes/overlays/kind.yaml +++ b/recipes/overlays/kind.yaml @@ -27,7 +27,7 @@ spec: # Kind-specific constraints constraints: - name: K8s.server.version - value: ">= 1.25" + value: ">= 1.32" # Kind-specific component overrides componentRefs: diff --git a/recipes/overlays/l40s-oke-inference.yaml b/recipes/overlays/l40s-oke-inference.yaml index dc9830e97..710394771 100644 --- a/recipes/overlays/l40s-oke-inference.yaml +++ b/recipes/overlays/l40s-oke-inference.yaml @@ -29,7 +29,7 @@ spec: constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: - name: gpu-operator diff --git a/recipes/overlays/l40s-oke-training.yaml b/recipes/overlays/l40s-oke-training.yaml index 6314819d2..957e43b5a 100644 --- a/recipes/overlays/l40s-oke-training.yaml +++ b/recipes/overlays/l40s-oke-training.yaml @@ -32,7 +32,7 @@ spec: # requirement, so the recipe keeps the OKE training baseline K8s floor. constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: # L40S-specific GPU Operator overrides (inherits valuesFile from oke-training). diff --git a/recipes/overlays/lke-inference.yaml b/recipes/overlays/lke-inference.yaml index d4b34a107..54d8b0e97 100644 --- a/recipes/overlays/lke-inference.yaml +++ b/recipes/overlays/lke-inference.yaml @@ -32,6 +32,6 @@ spec: # Inference specific constraints for LKE workloads constraints: - name: K8s.server.version - value: ">= 1.31" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/lke-training.yaml b/recipes/overlays/lke-training.yaml index 7e4812c8c..0e7ee9f45 100644 --- a/recipes/overlays/lke-training.yaml +++ b/recipes/overlays/lke-training.yaml @@ -29,6 +29,6 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.31" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/lke.yaml b/recipes/overlays/lke.yaml index da811a074..795883ace 100644 --- a/recipes/overlays/lke.yaml +++ b/recipes/overlays/lke.yaml @@ -28,7 +28,7 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.31" + value: ">= 1.32" # LKE-specific components componentRefs: diff --git a/recipes/overlays/oke-ol-inference.yaml b/recipes/overlays/oke-ol-inference.yaml index 24b212de5..c46c6295c 100644 --- a/recipes/overlays/oke-ol-inference.yaml +++ b/recipes/overlays/oke-ol-inference.yaml @@ -30,6 +30,6 @@ spec: constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/oke-ol-training.yaml b/recipes/overlays/oke-ol-training.yaml index 0d26f3dd5..097819f05 100644 --- a/recipes/overlays/oke-ol-training.yaml +++ b/recipes/overlays/oke-ol-training.yaml @@ -27,7 +27,7 @@ spec: constraints: - name: K8s.server.version - value: ">= 1.30" + value: ">= 1.32" componentRefs: - name: gpu-operator diff --git a/recipes/overlays/oke-ol.yaml b/recipes/overlays/oke-ol.yaml index 57f4687d9..0944a5bdd 100644 --- a/recipes/overlays/oke-ol.yaml +++ b/recipes/overlays/oke-ol.yaml @@ -26,7 +26,7 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.28" + value: ">= 1.32" # OKE-specific components componentRefs: diff --git a/recipes/overlays/rtx-pro-6000-lke-inference.yaml b/recipes/overlays/rtx-pro-6000-lke-inference.yaml index bdeb89112..2fba32e06 100644 --- a/recipes/overlays/rtx-pro-6000-lke-inference.yaml +++ b/recipes/overlays/rtx-pro-6000-lke-inference.yaml @@ -30,7 +30,7 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.31" + value: ">= 1.32" componentRefs: # RTX PRO 6000 GPU Operator dependencies diff --git a/recipes/overlays/rtx-pro-6000-lke-training.yaml b/recipes/overlays/rtx-pro-6000-lke-training.yaml index e397e2b3f..94fb6e0fa 100644 --- a/recipes/overlays/rtx-pro-6000-lke-training.yaml +++ b/recipes/overlays/rtx-pro-6000-lke-training.yaml @@ -30,7 +30,7 @@ spec: # Constraint names use fully qualified measurement paths: {type}.{subtype}.{key} constraints: - name: K8s.server.version - value: ">= 1.31" + value: ">= 1.32" componentRefs: # RTX PRO 6000 GPU Operator dependencies diff --git a/recipes/overlays/rtx-pro-6000-lke-ubuntu-inference.yaml b/recipes/overlays/rtx-pro-6000-lke-ubuntu-inference.yaml index 5b8d518a2..f74fbe601 100644 --- a/recipes/overlays/rtx-pro-6000-lke-ubuntu-inference.yaml +++ b/recipes/overlays/rtx-pro-6000-lke-ubuntu-inference.yaml @@ -35,7 +35,7 @@ spec: # Constraints for RTX PRO 6000 on LKE with Ubuntu for inference workloads constraints: - name: K8s.server.version - value: ">= 1.31" + value: ">= 1.32" componentRefs: [] diff --git a/recipes/overlays/rtx-pro-6000-lke-ubuntu-training.yaml b/recipes/overlays/rtx-pro-6000-lke-ubuntu-training.yaml index d90aeba8a..1554736d2 100644 --- a/recipes/overlays/rtx-pro-6000-lke-ubuntu-training.yaml +++ b/recipes/overlays/rtx-pro-6000-lke-ubuntu-training.yaml @@ -35,7 +35,7 @@ spec: # Constraints for RTX PRO 6000 on LKE with Ubuntu for training workloads constraints: - name: K8s.server.version - value: ">= 1.31" + value: ">= 1.32" componentRefs: [] From a913b63337c3bee8d1d3693fdb61a9488e4978e0 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 14:11:56 -0700 Subject: [PATCH 2/6] test(recipe): check every K8s floor declaration and fail closed on unknown forms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The guard used FindStringSubmatch, so only the first K8s.server.version declaration in a file was checked, and its regex matched only ">= 1.", so an exact pin or a range was skipped entirely. Both would admit clusters below the DRA chart's kubeVersion just as effectively. The original comment rationalised the second hole — "an exact pin or a range is a deliberate statement that should be reviewed on its own terms" — which is the wrong instinct for a guard whose only job is catching a future author deviating from the established shape. Now iterates every declaration and fails closed on any form it cannot interpret, naming the value and asking for either a >= floor or an extension to the guard. Controls verified: an exact pin of "== 1.30" fails as uninterpretable, and a second declaration of ">= 1.29" appended after a valid one fails as below the floor. Neither was caught before. Signed-off-by: Yuan Chen --- pkg/recipe/dra_k8s_floor_test.go | 68 +++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 24 deletions(-) diff --git a/pkg/recipe/dra_k8s_floor_test.go b/pkg/recipe/dra_k8s_floor_test.go index 195f0c1d7..054ad7377 100644 --- a/pkg/recipe/dra_k8s_floor_test.go +++ b/pkg/recipe/dra_k8s_floor_test.go @@ -37,10 +37,14 @@ import ( // point of the guard. const draChartKubeVersionMinor = 32 -// k8sFloorRE captures the minor version from a K8s.server.version constraint -// expressed as a floor. Only `>=` forms are checked: an exact pin or a range is -// a deliberate statement that should be reviewed on its own terms. -var k8sFloorRE = regexp.MustCompile(`(?s)- name: K8s\.server\.version\s*\n\s*value: "\s*>=\s*1\.(\d+)`) +// k8sConstraintRE captures EVERY K8s.server.version declaration in a file and +// its raw value. FindAllStringSubmatch, not FindStringSubmatch: a file may carry +// more than one declaration, and checking only the first would let a later, +// lower one through. +var k8sConstraintRE = regexp.MustCompile(`- name: K8s\.server\.version\s*\n\s*value: "([^"]*)"`) + +// geFloorRE matches the `>= 1.` form this guard can reason about. +var geFloorRE = regexp.MustCompile(`^\s*>=\s*1\.(\d+)`) // TestOverlayK8sFloorsClearDRAChartFloor asserts no overlay or mixin declares a // Kubernetes floor below the DRA chart's own kubeVersion. @@ -77,26 +81,42 @@ func TestOverlayK8sFloorsClearDRAChartFloor(t *testing.T) { if readErr != nil { return readErr } - m := k8sFloorRE.FindStringSubmatch(string(raw)) - if m == nil { - return nil - } - checked++ - minor, convErr := strconv.Atoi(m[1]) - if convErr != nil { - t.Errorf("%s: could not parse K8s.server.version minor %q: %v", path, m[1], convErr) - return nil - } - if minor < draChartKubeVersionMinor { - t.Errorf("%s declares K8s.server.version \">= 1.%d\", below the pinned "+ - "nvidia-dra-driver-gpu chart's kubeVersion \">=1.%d.0-0\".\n"+ - " Every recipe inherits the DRA driver from base.yaml, and Helm refuses the\n"+ - " install below the chart floor — so this recipe validates clean and then\n"+ - " fails at `helm install`. Raise it to \">= 1.%d\".\n"+ - " Raising base.yaml alone does NOT fix a leaf: constraints merge last-wins\n"+ - " with no max comparison, so a lower leaf value overwrites a higher\n"+ - " inherited one. See #2402.", - path, minor, draChartKubeVersionMinor, draChartKubeVersionMinor) + for _, m := range k8sConstraintRE.FindAllStringSubmatch(string(raw), -1) { + checked++ + value := m[1] + + g := geFloorRE.FindStringSubmatch(value) + if g == nil { + // Fail closed on any form this guard cannot interpret — an exact + // pin (== 1.30), a range, or a bare version would each be just as + // capable of admitting a sub-floor cluster, and silently skipping + // them would make the guard weakest exactly where a future author + // deviates from the established shape. + t.Errorf("%s declares K8s.server.version %q, a form this guard cannot verify.\n"+ + " It can only reason about \">= 1.\". Any other form may admit clusters\n"+ + " below the pinned nvidia-dra-driver-gpu chart's kubeVersion \">=1.%d.0-0\",\n"+ + " where Helm refuses the install. Either express it as a >= floor, or extend\n"+ + " this guard to understand the new form. See #2402.", + path, value, draChartKubeVersionMinor) + continue + } + + minor, convErr := strconv.Atoi(g[1]) + if convErr != nil { + t.Errorf("%s: could not parse K8s.server.version minor from %q: %v", path, value, convErr) + continue + } + if minor < draChartKubeVersionMinor { + t.Errorf("%s declares K8s.server.version %q, below the pinned "+ + "nvidia-dra-driver-gpu chart's kubeVersion \">=1.%d.0-0\".\n"+ + " Every recipe inherits the DRA driver from base.yaml, and Helm refuses the\n"+ + " install below the chart floor — so this recipe validates clean and then\n"+ + " fails at `helm install`. Raise it to \">= 1.%d\".\n"+ + " Raising base.yaml alone does NOT fix a leaf: constraints merge last-wins\n"+ + " with no max comparison, so a lower leaf value overwrites a higher\n"+ + " inherited one. See #2402.", + path, value, draChartKubeVersionMinor, draChartKubeVersionMinor) + } } return nil }) From d36c010491af6fbab1e9511f8ff5d962ef1106dd Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 14:16:55 -0700 Subject: [PATCH 3/6] test(recipe): couple the K8s floor guard to the registry pins and finish the sweep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three gaps from review. The guard's value regex required double quotes, so a single-quoted or plain scalar was not misparsed but INVISIBLE — the declaration was never counted or checked. It now matches any YAML scalar style and trims quoting. The floor was a hardcoded constant that never read the registry, so a DRA chart bump raising kubeVersion would leave the guard green at a stale 1.32 despite a comment claiming the two move together. Replaced with an audited component/version/floor table plus TestDRAChartFloorAuditIsCurrent, following the ownsCRDs version-audit pattern. Both DRA components are enrolled: the earlier comment wrongly claimed no overlay disables the generic one, but ocp.yaml sets enabled: false and substitutes nvidia-dra-driver-gpu-ocp, so covering only the generic entry left the OCP chain unguarded. Controls verified: a single-quoted ">= 1.29" now fails where it was previously invisible, and pointing an audited entry at a version the registry does not pin fails the audit test. Also finishes the floor-reference sweep — the CLI reference constraint examples, the component-catalog Topology Updater note, the OKE L40S demo claim that the floor drops to 1.30, and two test comments naming the old kind 1.25 floor. Left alone: the GB200 table in demos/images/recipe.md claimed >= 1.28 before this PR while GB200 already required 1.34, so it is pre-existing drift rather than this change's to correct. Signed-off-by: Yuan Chen --- demos/query.md | 6 +- docs/user/cli-reference.md | 4 +- docs/user/component-catalog.md | 2 +- pkg/cli/touched_invariant_test.go | 2 +- pkg/client/v1/relax_test.go | 2 +- pkg/recipe/dra_k8s_floor_test.go | 91 ++++++++++++++++++++++++------- 6 files changed, 81 insertions(+), 26 deletions(-) diff --git a/demos/query.md b/demos/query.md index 17cb7bddd..f98c23bcd 100644 --- a/demos/query.md +++ b/demos/query.md @@ -110,14 +110,16 @@ aicr query --service gke --accelerator h100 --intent training --os cos \ value: '>= 1.32' ``` -L40S (on OKE) relaxes K8s further (older accelerators run on older clusters): +L40S (on OKE) resolves a different component set; its K8s floor now matches the +catalog-wide minimum (every recipe carries the DRA driver, whose chart requires +1.32): ```shell aicr query --service oke --accelerator l40s --intent training --os ol \ --selector constraints ``` -> K8s minimum drops from `1.32.4` to `1.30`. +> K8s minimum relaxes from `1.32.4` to `1.32`. ### Component set differs by intent diff --git a/docs/user/cli-reference.md b/docs/user/cli-reference.md index 73e6c5c1e..a1fd3651d 100644 --- a/docs/user/cli-reference.md +++ b/docs/user/cli-reference.md @@ -996,7 +996,7 @@ aicr query --service eks --accelerator gb200 --intent training \ --selector components.nodewright-customizations.values # Watch constraints tighten as you add specificity -# Just "EKS" → 1 constraint (K8s >= 1.28) +# Just "EKS" → 1 constraint (K8s >= 1.32) aicr query --service eks --selector constraints # Add GPU + intent + OS → 4 constraints (K8s >= 1.32.4, Ubuntu 24.04, kernel >= 6.8) aicr query --service eks --accelerator h100 --intent training --os ubuntu \ @@ -1121,7 +1121,7 @@ Supported operators: | Operator | Example | Description | |----------|---------|-------------| -| `>=` | `>= 1.30` | Greater than or equal (version comparison) | +| `>=` | `>= 1.32` | Greater than or equal (version comparison) | | `<=` | `<= 1.33` | Less than or equal (version comparison) | | `>` | `> 1.30` | Greater than (version comparison) | | `<` | `< 2.0` | Less than (version comparison) | diff --git a/docs/user/component-catalog.md b/docs/user/component-catalog.md index e62bce2a2..c9e7897b2 100644 --- a/docs/user/component-catalog.md +++ b/docs/user/component-catalog.md @@ -72,7 +72,7 @@ Not every component appears in every recipe. The recipe engine selects component Production GPU leaf recipes (H100, GB200, RTX Pro 6000 on EKS / AKS / GKE / OKE / LKE) enable the NFD Topology Updater. It publishes per-node `NodeResourceTopology` CRDs that describe NUMA zones, GPU-to-NUMA affinity, and NIC-to-NUMA affinity. Runtime consumers (NUMA-aware schedulers, debugging via `kubectl get noderesourcetopologies`) can read these CRDs without further configuration. -The Topology Updater requires the kubelet `podResources` gRPC socket. The `KubeletPodResources` feature gate has been on by default since Kubernetes 1.15 (Beta) and reached GA in Kubernetes 1.28; AICR's recipe constraints on the affected leaves require K8s ≥ 1.30 or higher, so this is satisfied in practice. Recipes targeting Kubernetes `< 1.15` must enable the feature gate explicitly. Kind / KWOK simulated clusters do not run a real kubelet and therefore leave the Topology Updater disabled — kind-based recipes will not see `NodeResourceTopology` CRDs. +The Topology Updater requires the kubelet `podResources` gRPC socket. The `KubeletPodResources` feature gate has been on by default since Kubernetes 1.15 (Beta) and reached GA in Kubernetes 1.28; AICR's recipe constraints require K8s ≥ 1.32 or higher, so this is satisfied in practice. Recipes targeting Kubernetes `< 1.15` must enable the feature gate explicitly. Kind / KWOK simulated clusters do not run a real kubelet and therefore leave the Topology Updater disabled — kind-based recipes will not see `NodeResourceTopology` CRDs. See the upstream [Topology Updater docs](https://kubernetes-sigs.github.io/node-feature-discovery/stable/usage/nfd-topology-updater.html) for runtime consumer examples. diff --git a/pkg/cli/touched_invariant_test.go b/pkg/cli/touched_invariant_test.go index 6cb84eddc..bae5635d0 100644 --- a/pkg/cli/touched_invariant_test.go +++ b/pkg/cli/touched_invariant_test.go @@ -244,7 +244,7 @@ func TestRecipeCmd_Snapshot_StatedDimensionNotRelaxed(t *testing.T) { } // constraintFailingKindSnapshotYAML fingerprints to service=kind on a -// Kubernetes version below the kind overlay's `K8s.server.version >= 1.25` +// Kubernetes version below the kind overlay's `K8s.server.version >= 1.32` // constraint, so the only overlay covering service=kind is excluded by // constraint evaluation rather than absent from the catalog. const constraintFailingKindSnapshotYAML = `kind: Snapshot diff --git a/pkg/client/v1/relax_test.go b/pkg/client/v1/relax_test.go index 23555c4a7..bd26aae7e 100644 --- a/pkg/client/v1/relax_test.go +++ b/pkg/client/v1/relax_test.go @@ -486,7 +486,7 @@ func kindUbuntuSnapshot() *snapshotter.Snapshot { } // constraintFailingKindSnapshot fingerprints to service=kind on a Kubernetes -// version below the kind overlay's `K8s.server.version >= 1.25` constraint, so +// version below the kind overlay's `K8s.server.version >= 1.32` constraint, so // constraint evaluation excludes the only overlay that covers service=kind. func constraintFailingKindSnapshot() *snapshotter.Snapshot { return &snapshotter.Snapshot{ diff --git a/pkg/recipe/dra_k8s_floor_test.go b/pkg/recipe/dra_k8s_floor_test.go index 054ad7377..c1464cf93 100644 --- a/pkg/recipe/dra_k8s_floor_test.go +++ b/pkg/recipe/dra_k8s_floor_test.go @@ -22,26 +22,77 @@ import ( "testing" ) -// draChartKubeVersionMinor is the minor version floor the pinned -// nvidia-dra-driver-gpu chart declares: +// auditedDRAChartFloors records, per DRA component, the chart version whose +// kubeVersion was read and the Kubernetes minor it declares. // -// kubeVersion: ">=1.32.0-0" +// Both catalog DRA components are enrolled. OCP disables the generic +// nvidia-dra-driver-gpu (recipes/overlays/ocp.yaml sets enabled: false) and +// substitutes nvidia-dra-driver-gpu-ocp, so covering only the generic one would +// leave the OCP chain unguarded. // -// Helm REJECTS the install outright when the cluster is below it, so a recipe -// declaring a lower K8s.server.version admits clusters that pass every -// recipe-time check and then fail at `helm install`. -// -// Sourced from the chart, not from a running cluster. If the DRA chart pin in -// recipes/registry.yaml moves to a version with a different kubeVersion, this -// constant and the affected overlays must move together — that coupling is the -// point of the guard. -const draChartKubeVersionMinor = 32 +// Re-audit procedure when a pin moves: read `kubeVersion` from the chart at the +// new version and update the entry. TestDRAChartFloorAuditIsCurrent fails until +// you do, which is what couples this guard to the registry rather than letting +// it sit green at a stale floor. +var auditedDRAChartFloors = map[string]struct { + version string + minor int +}{ + // oci://registry.k8s.io/dra-driver-nvidia/charts/dra-driver-nvidia-gpu + // kubeVersion: ">=1.32.0-0" + "nvidia-dra-driver-gpu": {version: "0.4.1", minor: 32}, + "nvidia-dra-driver-gpu-ocp": {version: "0.4.1", minor: 32}, +} + +// draChartKubeVersionMinor is the highest audited floor across the enrolled DRA +// components — the value every recipe must clear, since each recipe carries one +// of them. +func draChartKubeVersionMinorFn() int { + highest := 0 + for _, a := range auditedDRAChartFloors { + if a.minor > highest { + highest = a.minor + } + } + return highest +} + +// TestDRAChartFloorAuditIsCurrent fails when a DRA chart pin in registry.yaml +// moves away from the version whose kubeVersion was audited, so a chart bump +// cannot silently leave the floor guard asserting a stale minor. +func TestDRAChartFloorAuditIsCurrent(t *testing.T) { + t.Parallel() + + registry, err := GetComponentRegistry() + if err != nil { + t.Fatalf("GetComponentRegistry: %v", err) + } + + for name, audited := range auditedDRAChartFloors { + cfg := registry.Get(name) + if cfg == nil { + t.Errorf("audited DRA component %q is not in the registry; remove it from "+ + "auditedDRAChartFloors or restore the component", name) + continue + } + if cfg.Helm.DefaultVersion != audited.version { + t.Errorf("%s is pinned at %q but its kubeVersion was audited at %q.\n"+ + " Read `kubeVersion` from the chart at %s, update auditedDRAChartFloors,\n"+ + " and raise the affected overlay floors if it moved. See #2402.", + name, cfg.Helm.DefaultVersion, audited.version, cfg.Helm.DefaultVersion) + } + } +} // k8sConstraintRE captures EVERY K8s.server.version declaration in a file and // its raw value. FindAllStringSubmatch, not FindStringSubmatch: a file may carry // more than one declaration, and checking only the first would let a later, // lower one through. -var k8sConstraintRE = regexp.MustCompile(`- name: K8s\.server\.version\s*\n\s*value: "([^"]*)"`) +// The value is matched irrespective of quoting: YAML accepts double-quoted, +// single-quoted and plain scalars, and a guard that only sees one style would +// not merely misparse the others — it would not see the declaration at all. +var k8sConstraintRE = regexp.MustCompile( + `- name: K8s\.server\.version\s*\n\s*value:[ \t]*("[^"]*"|'[^']*'|[^\n#]*)`) // geFloorRE matches the `>= 1.` form this guard can reason about. var geFloorRE = regexp.MustCompile(`^\s*>=\s*1\.(\d+)`) @@ -49,8 +100,10 @@ var geFloorRE = regexp.MustCompile(`^\s*>=\s*1\.(\d+)`) // TestOverlayK8sFloorsClearDRAChartFloor asserts no overlay or mixin declares a // Kubernetes floor below the DRA chart's own kubeVersion. // -// Every recipe inherits nvidia-dra-driver-gpu from base.yaml — no overlay -// removes or disables it — so the chart's floor applies catalog-wide. +// Every recipe carries a DRA driver: base.yaml declares nvidia-dra-driver-gpu, +// and OCP disables that one and substitutes nvidia-dra-driver-gpu-ocp. Both +// resolve to the same upstream chart and the same kubeVersion, so the floor +// applies catalog-wide either way. // // Why every declaration and not just base.yaml: constraints merge by name with // the LATER overlay winning and no max comparison (see mergeValidation in @@ -83,7 +136,7 @@ func TestOverlayK8sFloorsClearDRAChartFloor(t *testing.T) { } for _, m := range k8sConstraintRE.FindAllStringSubmatch(string(raw), -1) { checked++ - value := m[1] + value := strings.Trim(strings.TrimSpace(m[1]), `"'`) g := geFloorRE.FindStringSubmatch(value) if g == nil { @@ -97,7 +150,7 @@ func TestOverlayK8sFloorsClearDRAChartFloor(t *testing.T) { " below the pinned nvidia-dra-driver-gpu chart's kubeVersion \">=1.%d.0-0\",\n"+ " where Helm refuses the install. Either express it as a >= floor, or extend\n"+ " this guard to understand the new form. See #2402.", - path, value, draChartKubeVersionMinor) + path, value, draChartKubeVersionMinorFn()) continue } @@ -106,7 +159,7 @@ func TestOverlayK8sFloorsClearDRAChartFloor(t *testing.T) { t.Errorf("%s: could not parse K8s.server.version minor from %q: %v", path, value, convErr) continue } - if minor < draChartKubeVersionMinor { + if minor < draChartKubeVersionMinorFn() { t.Errorf("%s declares K8s.server.version %q, below the pinned "+ "nvidia-dra-driver-gpu chart's kubeVersion \">=1.%d.0-0\".\n"+ " Every recipe inherits the DRA driver from base.yaml, and Helm refuses the\n"+ @@ -115,7 +168,7 @@ func TestOverlayK8sFloorsClearDRAChartFloor(t *testing.T) { " Raising base.yaml alone does NOT fix a leaf: constraints merge last-wins\n"+ " with no max comparison, so a lower leaf value overwrites a higher\n"+ " inherited one. See #2402.", - path, value, draChartKubeVersionMinor, draChartKubeVersionMinor) + path, value, draChartKubeVersionMinorFn(), draChartKubeVersionMinorFn()) } } return nil From dfea839e14d86c168434902bacff35da98a24ff8 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 14:39:58 -0700 Subject: [PATCH 4/6] test(recipe): evaluate K8s floors typed instead of pattern-matching YAML text Decode each overlay and mixin into RecipeMetadata and evaluate every K8s.server.version constraint with the shipping parser and evaluator, so the guard no longer depends on YAML key order, quoting, or the expression's surface form. Also correct two stale comments: the MirrorDefaultKubeVersion note naming the old ">= 1.25" base floor, and the A100 GKE contrast with an H100 floor the recipe now shares. Signed-off-by: Yuan Chen --- pkg/defaults/timeouts.go | 2 +- pkg/recipe/dra_k8s_floor_test.go | 252 ++++++++++++++++---- recipes/overlays/a100-gke-cos-training.yaml | 5 +- 3 files changed, 203 insertions(+), 56 deletions(-) diff --git a/pkg/defaults/timeouts.go b/pkg/defaults/timeouts.go index daeea3b9b..aff750f12 100644 --- a/pkg/defaults/timeouts.go +++ b/pkg/defaults/timeouts.go @@ -1257,7 +1257,7 @@ const ( // // This is a render-safe floor, not a support floor. This constant must // stay at or above the strictest kubeVersion any bundled chart declares. - // Do NOT lower it to match the ">= 1.25" recipe floor in + // Do NOT lower it to match the ">= 1.32" recipe floor in // recipes/overlays/base.yaml: recipes are validated against their own // constraints, while mirror discovery raises lower versions to this // value solely for Helm rendering (see mirror.KubeVersionFromConstraints). diff --git a/pkg/recipe/dra_k8s_floor_test.go b/pkg/recipe/dra_k8s_floor_test.go index c1464cf93..e06cfa352 100644 --- a/pkg/recipe/dra_k8s_floor_test.go +++ b/pkg/recipe/dra_k8s_floor_test.go @@ -12,16 +12,28 @@ // See the License for the specific language governing permissions and // limitations under the License. -package recipe +// This guard lives in the external recipe_test package so it can import +// pkg/constraints — the production constraint parser and evaluator — which +// itself imports pkg/recipe. An in-package test could not, and would be forced +// to reimplement comparison logic that the shipping evaluator already owns. +package recipe_test import ( + "fmt" "io/fs" - "regexp" - "strconv" "strings" "testing" + + "gopkg.in/yaml.v3" + + "github.com/NVIDIA/aicr/pkg/constraints" + "github.com/NVIDIA/aicr/pkg/recipe" ) +// k8sServerVersionConstraint is the measurement path whose floor this guard +// audits. +const k8sServerVersionConstraint = "K8s.server.version" + // auditedDRAChartFloors records, per DRA component, the chart version whose // kubeVersion was read and the Kubernetes minor it declares. // @@ -47,7 +59,7 @@ var auditedDRAChartFloors = map[string]struct { // draChartKubeVersionMinor is the highest audited floor across the enrolled DRA // components — the value every recipe must clear, since each recipe carries one // of them. -func draChartKubeVersionMinorFn() int { +func draChartKubeVersionMinor() int { highest := 0 for _, a := range auditedDRAChartFloors { if a.minor > highest { @@ -63,7 +75,7 @@ func draChartKubeVersionMinorFn() int { func TestDRAChartFloorAuditIsCurrent(t *testing.T) { t.Parallel() - registry, err := GetComponentRegistry() + registry, err := recipe.GetComponentRegistry() if err != nil { t.Fatalf("GetComponentRegistry: %v", err) } @@ -84,21 +96,110 @@ func TestDRAChartFloorAuditIsCurrent(t *testing.T) { } } -// k8sConstraintRE captures EVERY K8s.server.version declaration in a file and -// its raw value. FindAllStringSubmatch, not FindStringSubmatch: a file may carry -// more than one declaration, and checking only the first would let a later, -// lower one through. -// The value is matched irrespective of quoting: YAML accepts double-quoted, -// single-quoted and plain scalars, and a guard that only sees one style would -// not merely misparse the others — it would not see the declaration at all. -var k8sConstraintRE = regexp.MustCompile( - `- name: K8s\.server\.version\s*\n\s*value:[ \t]*("[^"]*"|'[^']*'|[^\n#]*)`) +// subFloorProbeVersions returns the Kubernetes version strings that MUST NOT +// satisfy any catalog floor: every minor below the DRA chart's kubeVersion, +// rendered in each shape a real cluster reading or a recipe author's exact pin +// can take. +// +// Probing the production evaluator with concrete readings — rather than +// pattern-matching the expression text — is what makes this guard independent +// of the expression's *form*. A prefix match on ">= 1." is defeated by a +// compound expression (">= 1.32 || >= 1.29" begins with a safe floor and is +// still satisfied by 1.29.7, because the shipping parser treats "||" as OR), +// and a bare-string exact pin ("1.30") carries no operator to match at all. +// Both are caught here because both admit a sub-floor reading. +func subFloorProbeVersions(floorMinor int) []string { + probes := make([]string, 0, 2+4*floorMinor) + probes = append(probes, "0.99", "0.99.99") + for minor := range floorMinor { + probes = append(probes, + fmt.Sprintf("1.%d", minor), + fmt.Sprintf("1.%d.0", minor), + fmt.Sprintf("1.%d.99", minor), + fmt.Sprintf("v1.%d.0", minor), + ) + } + return probes +} + +// supportedProbeVersions returns readings at or above the floor, used only to +// prove a constraint is not vacuous. An expression satisfied by nothing admits +// no sub-floor cluster and would pass the sub-floor sweep trivially, but it +// also rejects every cluster the catalog claims to support — a typo, not a +// floor. Failing on it keeps the guard closed against expressions it cannot +// show are meaningful. +func supportedProbeVersions(floorMinor int) []string { + var probes []string + for minor := floorMinor; minor <= 60; minor++ { + probes = append(probes, fmt.Sprintf("1.%d.0", minor)) + } + return probes +} -// geFloorRE matches the `>= 1.` form this guard can reason about. -var geFloorRE = regexp.MustCompile(`^\s*>=\s*1\.(\d+)`) +// k8sFloorDeclaration is one typed K8s.server.version constraint located in the +// catalog, with the structural path it was found at for error reporting. +type k8sFloorDeclaration struct { + file string + location string + value string +} + +// collectK8sFloorDeclarations decodes one recipe metadata document and returns +// every K8s.server.version constraint it declares, from every field that can +// carry one: spec.constraints, each validation phase, and each profile value's +// constraints and readinessConstraints. +// +// Decoding to the typed form is what closes the YAML-layout hole: a mapping +// written "value:" before "name:" is the same Constraint after unmarshalling, +// so key order, quoting style, comments, and indentation are all invisible to +// this guard by construction rather than by widening a regex. +func collectK8sFloorDeclarations(file string, raw []byte) ([]k8sFloorDeclaration, error) { + var metadata recipe.RecipeMetadata + if err := yaml.Unmarshal(raw, &metadata); err != nil { + return nil, err + } + + var found []k8sFloorDeclaration + collect := func(location string, cs []recipe.Constraint) { + for _, c := range cs { + if c.Name != k8sServerVersionConstraint { + continue + } + found = append(found, k8sFloorDeclaration{file: file, location: location, value: c.Value}) + } + } + + spec := metadata.Spec + collect("spec.constraints", spec.Constraints) + + if v := spec.Validation; v != nil { + for _, phase := range []struct { + name string + p *recipe.ValidationPhase + }{ + {"readiness", v.Readiness}, + {"deployment", v.Deployment}, + {"performance", v.Performance}, + {"conformance", v.Conformance}, + } { + if phase.p != nil { + collect("spec.validation."+phase.name+".constraints", phase.p.Constraints) + } + } + } + + if p := spec.Profile; p != nil { + for valueName, pv := range p.Values { + collect(fmt.Sprintf("spec.profile.values[%s].constraints", valueName), pv.Constraints) + collect(fmt.Sprintf("spec.profile.values[%s].readinessConstraints", valueName), pv.ReadinessConstraints) + } + } + + return found, nil +} // TestOverlayK8sFloorsClearDRAChartFloor asserts no overlay or mixin declares a -// Kubernetes floor below the DRA chart's own kubeVersion. +// Kubernetes floor that admits a cluster below the DRA chart's own kubeVersion. // // Every recipe carries a DRA driver: base.yaml declares nvidia-dra-driver-gpu, // and OCP disables that one and substitutes nvidia-dra-driver-gpu-ocp. Both @@ -114,10 +215,23 @@ var geFloorRE = regexp.MustCompile(`^\s*>=\s*1\.(\d+)`) // recipes/overlays/ocp.yaml already carried >= 1.32 for exactly this reason // before the rest of the catalog was reconciled; its comment records the // diagnosis. +// +// How it checks, and why not by reading the expression: each declaration is +// decoded typed, parsed with the shipping parser +// (constraints.ParseCompoundConstraint), and then EVALUATED against concrete +// sub-floor readings with the shipping evaluator. The guard therefore asserts +// the property that actually matters — "no supported-but-too-old cluster +// satisfies this" — instead of asserting the expression is spelled a +// particular way. It fails closed on any expression the parser rejects and on +// any expression no supported reading satisfies. func TestOverlayK8sFloorsClearDRAChartFloor(t *testing.T) { t.Parallel() - efs := GetEmbeddedFS() + floorMinor := draChartKubeVersionMinor() + subFloor := subFloorProbeVersions(floorMinor) + supported := supportedProbeVersions(floorMinor) + + efs := recipe.GetEmbeddedFS() var checked int err := fs.WalkDir(efs, ".", func(path string, d fs.DirEntry, err error) error { @@ -134,42 +248,15 @@ func TestOverlayK8sFloorsClearDRAChartFloor(t *testing.T) { if readErr != nil { return readErr } - for _, m := range k8sConstraintRE.FindAllStringSubmatch(string(raw), -1) { - checked++ - value := strings.Trim(strings.TrimSpace(m[1]), `"'`) - - g := geFloorRE.FindStringSubmatch(value) - if g == nil { - // Fail closed on any form this guard cannot interpret — an exact - // pin (== 1.30), a range, or a bare version would each be just as - // capable of admitting a sub-floor cluster, and silently skipping - // them would make the guard weakest exactly where a future author - // deviates from the established shape. - t.Errorf("%s declares K8s.server.version %q, a form this guard cannot verify.\n"+ - " It can only reason about \">= 1.\". Any other form may admit clusters\n"+ - " below the pinned nvidia-dra-driver-gpu chart's kubeVersion \">=1.%d.0-0\",\n"+ - " where Helm refuses the install. Either express it as a >= floor, or extend\n"+ - " this guard to understand the new form. See #2402.", - path, value, draChartKubeVersionMinorFn()) - continue - } + decls, decodeErr := collectK8sFloorDeclarations(path, raw) + if decodeErr != nil { + t.Errorf("%s: could not decode recipe metadata: %v", path, decodeErr) + return nil + } - minor, convErr := strconv.Atoi(g[1]) - if convErr != nil { - t.Errorf("%s: could not parse K8s.server.version minor from %q: %v", path, value, convErr) - continue - } - if minor < draChartKubeVersionMinorFn() { - t.Errorf("%s declares K8s.server.version %q, below the pinned "+ - "nvidia-dra-driver-gpu chart's kubeVersion \">=1.%d.0-0\".\n"+ - " Every recipe inherits the DRA driver from base.yaml, and Helm refuses the\n"+ - " install below the chart floor — so this recipe validates clean and then\n"+ - " fails at `helm install`. Raise it to \">= 1.%d\".\n"+ - " Raising base.yaml alone does NOT fix a leaf: constraints merge last-wins\n"+ - " with no max comparison, so a lower leaf value overwrites a higher\n"+ - " inherited one. See #2402.", - path, value, draChartKubeVersionMinorFn(), draChartKubeVersionMinorFn()) - } + for _, decl := range decls { + checked++ + verifyK8sFloorDeclaration(t, decl, floorMinor, subFloor, supported) } return nil }) @@ -185,3 +272,62 @@ func TestOverlayK8sFloorsClearDRAChartFloor(t *testing.T) { } t.Logf("verified %d K8s.server.version floor(s) clear the DRA chart floor", checked) } + +// verifyK8sFloorDeclaration checks one declaration against the DRA chart floor +// using the production parser and evaluator. +func verifyK8sFloorDeclaration(t *testing.T, decl k8sFloorDeclaration, floorMinor int, subFloor, supported []string) { + t.Helper() + + parsed, err := constraints.ParseCompoundConstraint(decl.value) + if err != nil { + t.Errorf("%s (%s) declares K8s.server.version %q, which the shipping constraint\n"+ + " parser rejects: %v\n"+ + " An expression aicr cannot parse cannot be shown to clear the pinned\n"+ + " nvidia-dra-driver-gpu chart's kubeVersion \">=1.%d.0-0\". See #2402.", + decl.file, decl.location, decl.value, err, floorMinor) + return + } + + for _, reading := range subFloor { + satisfied, evalErr := parsed.Evaluate(reading) + if evalErr != nil { + t.Errorf("%s (%s) declares K8s.server.version %q, which the shipping evaluator\n"+ + " could not evaluate against the Kubernetes reading %q: %v\n"+ + " The guard fails closed: an expression whose result is unknown may admit a\n"+ + " cluster below the chart floor \">=1.%d.0-0\". See #2402.", + decl.file, decl.location, decl.value, reading, evalErr, floorMinor) + return + } + if satisfied { + t.Errorf("%s (%s) declares K8s.server.version %q, which is SATISFIED by a\n"+ + " Kubernetes %s cluster — below the pinned nvidia-dra-driver-gpu chart's\n"+ + " kubeVersion \">=1.%d.0-0\".\n"+ + " Every recipe inherits the DRA driver from base.yaml, and Helm refuses the\n"+ + " install below the chart floor — so this recipe validates clean and then\n"+ + " fails at `helm install`. Raise it to \">= 1.%d\".\n"+ + " Raising base.yaml alone does NOT fix a leaf: constraints merge last-wins\n"+ + " with no max comparison, so a lower leaf value overwrites a higher\n"+ + " inherited one. See #2402.", + decl.file, decl.location, decl.value, reading, floorMinor, floorMinor) + return + } + } + + for _, reading := range supported { + satisfied, evalErr := parsed.Evaluate(reading) + if evalErr != nil { + t.Errorf("%s (%s) declares K8s.server.version %q, which the shipping evaluator\n"+ + " could not evaluate against the Kubernetes reading %q: %v. See #2402.", + decl.file, decl.location, decl.value, reading, evalErr) + return + } + if satisfied { + return + } + } + + t.Errorf("%s (%s) declares K8s.server.version %q, which no Kubernetes release from\n"+ + " 1.%d through 1.60 satisfies. It admits no cluster the catalog supports, so this\n"+ + " guard cannot show it is a floor rather than a typo, and fails closed. See #2402.", + decl.file, decl.location, decl.value, floorMinor) +} diff --git a/recipes/overlays/a100-gke-cos-training.yaml b/recipes/overlays/a100-gke-cos-training.yaml index ae92d247d..c3c6c69c9 100644 --- a/recipes/overlays/a100-gke-cos-training.yaml +++ b/recipes/overlays/a100-gke-cos-training.yaml @@ -28,8 +28,9 @@ spec: intent: training # Specific constraints for A100 on GKE COS training workloads. - # A100 has no IMEX/NVLink ComputeDomain requirement, so the recipe keeps - # the GKE COS training baseline rather than the H100 1.32 floor. + # A100 has no IMEX/NVLink ComputeDomain requirement, so this floor is not + # driven by a ComputeDomain need as the H100 recipes' is; it simply carries + # the catalog-wide 1.32 baseline the DRA driver chart's kubeVersion sets. constraints: - name: K8s.server.version value: ">= 1.32" From 36adee0eef98baa51a68f9b3ceccde480dee35c1 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 15:16:21 -0700 Subject: [PATCH 5/6] test(recipe): prove the K8s floor symbolically instead of sampling versions The guard proved 'no too-old cluster satisfies this floor' by evaluating each declared expression against a fixed list of probe versions. That is sampling, and the production grammar supports arbitrary OR-of-AND ranges, so no finite probe list can cover it. '>= 1.32 || > 1.31.0 < 1.31.2' is a supported shape that passed the guard while the production evaluator accepts a Kubernetes 1.31.1 cluster that Helm's '>=1.32.0-0' rejects. Walk the parsed structure from constraints.ParseCompoundConstraint instead and prove the effective lower bound. An AND group's satisfying set is the intersection of its terms, so the group clears the floor as soon as any one term does; a compound's satisfying set is the union of its groups, so every group must clear it. Only >=, >, ==, and bare exact match place a lower bound; <, <=, and != place none. Anything else - an unparseable value, a major-only precision, an unknown operator - fails closed rather than being waved through. Symbolic proof was chosen over restricting the catalog to a simple '>= X.Y' form because it keeps the per-track GKE range expressions the parser already supports (see #1985) provable rather than banned, and it is exact where a grammar restriction is merely conservative. The defeating expression is kept as a permanent regression control in TestProveExpressionClearsFloor, with an adversarial control asserting the production evaluator really does admit 1.31.1 for it - so a prover bug that rejected everything cannot make the table green. Also corrects the comment attributing top-level constraint last-wins merging to mergeValidation in validation.go: RecipeMetadataSpec.Merge in metadata.go is what merges spec.constraints; mergeValidationPhase handles phase constraints. Signed-off-by: Yuan Chen --- pkg/recipe/dra_k8s_floor_test.go | 285 ++++++++++++++++++++++++------- 1 file changed, 219 insertions(+), 66 deletions(-) diff --git a/pkg/recipe/dra_k8s_floor_test.go b/pkg/recipe/dra_k8s_floor_test.go index e06cfa352..3c56fe88c 100644 --- a/pkg/recipe/dra_k8s_floor_test.go +++ b/pkg/recipe/dra_k8s_floor_test.go @@ -28,6 +28,7 @@ import ( "github.com/NVIDIA/aicr/pkg/constraints" "github.com/NVIDIA/aicr/pkg/recipe" + "github.com/NVIDIA/aicr/pkg/version" ) // k8sServerVersionConstraint is the measurement path whose floor this guard @@ -96,30 +97,108 @@ func TestDRAChartFloorAuditIsCurrent(t *testing.T) { } } -// subFloorProbeVersions returns the Kubernetes version strings that MUST NOT -// satisfy any catalog floor: every minor below the DRA chart's kubeVersion, -// rendered in each shape a real cluster reading or a recipe author's exact pin -// can take. +// floorMajor is the Kubernetes major version the DRA chart floors sit on. The +// audited kubeVersion is ">=1.32.0-0", so every bound is compared as +// (major, minor) against (floorMajor, floorMinor). +const floorMajor = 1 + +// termClearsFloor reports whether a single parsed term, on its own, confines +// every version that satisfies it to at or above Kubernetes +// floorMajor.floorMinor.0. +// +// Only the lower-bounding operators can do that: +// +// - ">= v" admits exactly [v, inf) +// - "> v" admits (v, inf); requiring v itself to clear the floor is one +// patch conservative and never fails open, since the patch component is +// unbounded (there is no "last" 1.31.x to fall back on) +// - "== v" and a bare exact match admit only v +// +// "<", "<=", and "!=" place no lower bound at all, so they return false: they +// can narrow an alternative but can never be what lifts it above the floor. // -// Probing the production evaluator with concrete readings — rather than -// pattern-matching the expression text — is what makes this guard independent -// of the expression's *form*. A prefix match on ">= 1." is defeated by a -// compound expression (">= 1.32 || >= 1.29" begins with a safe floor and is -// still satisfied by 1.29.7, because the shipping parser treats "||" as OR), -// and a bare-string exact pin ("1.30") carries no operator to match at all. -// Both are caught here because both admit a sub-floor reading. -func subFloorProbeVersions(floorMinor int) []string { - probes := make([]string, 0, 2+4*floorMinor) - probes = append(probes, "0.99", "0.99.99") - for minor := range floorMinor { - probes = append(probes, - fmt.Sprintf("1.%d", minor), - fmt.Sprintf("1.%d.0", minor), - fmt.Sprintf("1.%d.99", minor), - fmt.Sprintf("v1.%d.0", minor), - ) +// A value the shipping version parser cannot read, or one written with less +// than major.minor precision (">= 1"), returns an error. Comparing "1" against +// "1.32.0" at min-precision would report equal and wave a floorless expression +// through, so the guard refuses to reason about it instead. +func termClearsFloor(pc constraints.ParsedConstraint, floorMinor int) (bool, error) { + switch pc.Operator { + case constraints.OperatorLT, constraints.OperatorLTE, constraints.OperatorNE: + return false, nil + case constraints.OperatorGTE, constraints.OperatorGT, constraints.OperatorEQ, constraints.OperatorExact: + default: + return false, fmt.Errorf("unknown operator %q; the guard cannot prove a lower bound for it", pc.Operator) } - return probes + + v, err := version.ParseVersion(pc.Value) + if err != nil { + return false, fmt.Errorf("value %q is not a version the shipping parser can read: %w", pc.Value, err) + } + if v.Precision < 2 { + return false, fmt.Errorf("value %q has only major precision; a floor must name at least major.minor", pc.Value) + } + if v.Major > floorMajor { + return true, nil + } + return v.Major == floorMajor && v.Minor >= floorMinor, nil +} + +// alternativeString renders one AND group for error messages. +func alternativeString(group []constraints.ParsedConstraint) string { + terms := make([]string, 0, len(group)) + for i := range group { + terms = append(terms, group[i].String()) + } + return strings.Join(terms, " ") +} + +// proveExpressionClearsFloor proves, symbolically, that no Kubernetes cluster +// below floorMajor.floorMinor.0 can satisfy expr. It returns nil only when the +// proof succeeds, and a describing error otherwise — including for any +// expression it cannot reason about, so the guard fails closed. +// +// Why symbolic and not by probing readings: the production grammar admits +// arbitrary OR-of-AND range expressions, so no finite list of probe versions +// covers it. ">= 1.32 || > 1.31.0 < 1.31.2" is a supported shape that a probe +// sweep over 1.N / 1.N.0 / 1.N.99 misses entirely while the production +// evaluator happily accepts a 1.31.1 cluster that Helm's ">=1.32.0-0" rejects. +// +// The proof: an AND group's satisfying set is the intersection of its terms, so +// the group clears the floor as soon as ANY ONE of its terms does — a single +// ">= 1.32" makes the whole group safe no matter what the others say. A +// compound expression's satisfying set is the union of its groups, so EVERY +// group must clear the floor; one loose alternative admits a sub-floor cluster +// regardless of how strict its siblings are. +func proveExpressionClearsFloor(expr string, floorMinor int) error { + parsed, err := constraints.ParseCompoundConstraint(expr) + if err != nil { + return fmt.Errorf("the shipping constraint parser rejects it: %w", err) + } + if len(parsed.Alternatives) == 0 { + return fmt.Errorf("it parsed to zero OR alternatives, so nothing bounds it") + } + + for i, group := range parsed.Alternatives { + if len(group) == 0 { + return fmt.Errorf("OR alternative %d has no terms, so nothing bounds it", i+1) + } + cleared := false + for j := range group { + ok, termErr := termClearsFloor(group[j], floorMinor) + if termErr != nil { + return fmt.Errorf("OR alternative %d (%q): %w", i+1, alternativeString(group), termErr) + } + if ok { + cleared = true + } + } + if !cleared { + return fmt.Errorf("OR alternative %d (%q) carries no lower bound at or above %d.%d.0, "+ + "so at least one cluster below the chart floor satisfies it", + i+1, alternativeString(group), floorMajor, floorMinor) + } + } + return nil } // supportedProbeVersions returns readings at or above the floor, used only to @@ -207,28 +286,32 @@ func collectK8sFloorDeclarations(file string, raw []byte) ([]k8sFloorDeclaration // applies catalog-wide either way. // // Why every declaration and not just base.yaml: constraints merge by name with -// the LATER overlay winning and no max comparison (see mergeValidation in -// validation.go). A leaf declaring ">= 1.30" silently overwrites a higher floor -// inherited from base, so raising base alone would not hold. This is the same +// the LATER overlay winning and no max comparison (see RecipeMetadataSpec.Merge +// in metadata.go; validation-phase constraints merge the same way in +// mergeValidationPhase in validation.go). A leaf declaring ">= 1.30" silently +// overwrites a higher floor inherited from base, so raising base alone would +// not hold. This is the same // last-wins hazard documented for driver floors in #2438. // // recipes/overlays/ocp.yaml already carried >= 1.32 for exactly this reason // before the rest of the catalog was reconciled; its comment records the // diagnosis. // -// How it checks, and why not by reading the expression: each declaration is -// decoded typed, parsed with the shipping parser -// (constraints.ParseCompoundConstraint), and then EVALUATED against concrete -// sub-floor readings with the shipping evaluator. The guard therefore asserts -// the property that actually matters — "no supported-but-too-old cluster -// satisfies this" — instead of asserting the expression is spelled a -// particular way. It fails closed on any expression the parser rejects and on -// any expression no supported reading satisfies. +// How it checks: each declaration is decoded typed, parsed with the shipping +// parser (constraints.ParseCompoundConstraint), and then PROVEN — symbolically, +// over the parsed OR-of-AND structure — to carry a lower bound at or above the +// chart floor on every alternative. See proveExpressionClearsFloor. +// +// Sampling the evaluator with a list of sub-floor readings was tried and is not +// sufficient: the grammar admits arbitrary ranges, and ">= 1.32 || > 1.31.0 +// < 1.31.2" slips past any fixed probe list while admitting a 1.31.1 cluster. +// The guard fails closed on any expression the parser rejects, on any operator +// or value it cannot reason about, and on any expression no supported reading +// satisfies. func TestOverlayK8sFloorsClearDRAChartFloor(t *testing.T) { t.Parallel() floorMinor := draChartKubeVersionMinor() - subFloor := subFloorProbeVersions(floorMinor) supported := supportedProbeVersions(floorMinor) efs := recipe.GetEmbeddedFS() @@ -256,7 +339,7 @@ func TestOverlayK8sFloorsClearDRAChartFloor(t *testing.T) { for _, decl := range decls { checked++ - verifyK8sFloorDeclaration(t, decl, floorMinor, subFloor, supported) + verifyK8sFloorDeclaration(t, decl, floorMinor, supported) } return nil }) @@ -275,42 +358,29 @@ func TestOverlayK8sFloorsClearDRAChartFloor(t *testing.T) { // verifyK8sFloorDeclaration checks one declaration against the DRA chart floor // using the production parser and evaluator. -func verifyK8sFloorDeclaration(t *testing.T, decl k8sFloorDeclaration, floorMinor int, subFloor, supported []string) { +func verifyK8sFloorDeclaration(t *testing.T, decl k8sFloorDeclaration, floorMinor int, supported []string) { t.Helper() - parsed, err := constraints.ParseCompoundConstraint(decl.value) - if err != nil { - t.Errorf("%s (%s) declares K8s.server.version %q, which the shipping constraint\n"+ - " parser rejects: %v\n"+ - " An expression aicr cannot parse cannot be shown to clear the pinned\n"+ - " nvidia-dra-driver-gpu chart's kubeVersion \">=1.%d.0-0\". See #2402.", - decl.file, decl.location, decl.value, err, floorMinor) + if err := proveExpressionClearsFloor(decl.value, floorMinor); err != nil { + t.Errorf("%s (%s) declares K8s.server.version %q, which this guard cannot prove\n"+ + " clears the pinned nvidia-dra-driver-gpu chart's kubeVersion \">=1.%d.0-0\":\n"+ + " %v\n"+ + " Every recipe inherits the DRA driver from base.yaml, and Helm refuses the\n"+ + " install below the chart floor — so a recipe that admits a lower cluster\n"+ + " validates clean and then fails at `helm install`. Raise it to \">= 1.%d\",\n"+ + " or, if the expression is genuinely safe in a form the guard cannot yet\n"+ + " prove, extend proveExpressionClearsFloor rather than loosening it.\n"+ + " Raising base.yaml alone does NOT fix a leaf: constraints merge last-wins\n"+ + " with no max comparison, so a lower leaf value overwrites a higher\n"+ + " inherited one. See #2402.", + decl.file, decl.location, decl.value, floorMinor, err, floorMinor) return } - for _, reading := range subFloor { - satisfied, evalErr := parsed.Evaluate(reading) - if evalErr != nil { - t.Errorf("%s (%s) declares K8s.server.version %q, which the shipping evaluator\n"+ - " could not evaluate against the Kubernetes reading %q: %v\n"+ - " The guard fails closed: an expression whose result is unknown may admit a\n"+ - " cluster below the chart floor \">=1.%d.0-0\". See #2402.", - decl.file, decl.location, decl.value, reading, evalErr, floorMinor) - return - } - if satisfied { - t.Errorf("%s (%s) declares K8s.server.version %q, which is SATISFIED by a\n"+ - " Kubernetes %s cluster — below the pinned nvidia-dra-driver-gpu chart's\n"+ - " kubeVersion \">=1.%d.0-0\".\n"+ - " Every recipe inherits the DRA driver from base.yaml, and Helm refuses the\n"+ - " install below the chart floor — so this recipe validates clean and then\n"+ - " fails at `helm install`. Raise it to \">= 1.%d\".\n"+ - " Raising base.yaml alone does NOT fix a leaf: constraints merge last-wins\n"+ - " with no max comparison, so a lower leaf value overwrites a higher\n"+ - " inherited one. See #2402.", - decl.file, decl.location, decl.value, reading, floorMinor, floorMinor) - return - } + parsed, err := constraints.ParseCompoundConstraint(decl.value) + if err != nil { + t.Errorf("%s (%s): %v", decl.file, decl.location, err) + return } for _, reading := range supported { @@ -331,3 +401,86 @@ func verifyK8sFloorDeclaration(t *testing.T, decl k8sFloorDeclaration, floorMino " guard cannot show it is a floor rather than a typo, and fails closed. See #2402.", decl.file, decl.location, decl.value, floorMinor) } + +// TestProveExpressionClearsFloor pins the prover's behavior on the shapes the +// catalog guard has to withstand. Every "must fail" row is a permanent +// regression control: each one is an expression a real author could write that +// the production evaluator accepts for a sub-floor cluster. +// +// The last row is the shape that defeated the previous probe-sampling guard: a +// safe first alternative followed by a narrow sub-floor range. It is kept here +// permanently so no future rewrite can reintroduce sampling and stay green. +func TestProveExpressionClearsFloor(t *testing.T) { + t.Parallel() + + const floorMinor = 32 + + tests := []struct { + name string + expr string + wantErr bool + }{ + {"simple floor at the chart minor", ">= 1.32", false}, + {"simple floor with patch", ">= 1.32.4", false}, + {"floor above the chart minor", ">= 1.33.0", false}, + {"major above the floor", ">= 2.0", false}, + {"range whose lower bound clears the floor", ">= 1.32.4 < 1.35.0", false}, + {"every alternative clears the floor", ">= 1.34.3-gke.1318000 < 1.35.0 || >= 1.35.0-gke.2745000", false}, + {"upper-bounded term does not lift a cleared group", ">= 1.32 < 1.33", false}, + + {"floor below the chart minor", ">= 1.30", true}, + {"simple compound with a low alternative", ">= 1.32 || >= 1.29", true}, + {"exact pin below the chart minor", "== 1.30", true}, + {"bare exact pin below the chart minor", "1.30", true}, + {"greater-than below the chart minor", "> 1.31", true}, + {"only an upper bound", "< 1.40", true}, + {"only a not-equal", "!= 1.30", true}, + {"major-only precision", ">= 1", true}, + {"non-version value", ">= stable", true}, + {"empty expression", "", true}, + {"empty OR clause", ">= 1.32 ||", true}, + // The control: accepted by the production evaluator for a 1.31.1 + // cluster, which Helm's ">=1.32.0-0" rejects. A probe sweep over + // 1.N / 1.N.0 / 1.N.99 / v1.N.0 misses it entirely. + {"narrow sub-floor range hidden behind a safe alternative", ">= 1.32 || > 1.31.0 < 1.31.2", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + err := proveExpressionClearsFloor(tt.expr, floorMinor) + if (err != nil) != tt.wantErr { + t.Fatalf("proveExpressionClearsFloor(%q, %d) error = %v, wantErr %v", + tt.expr, floorMinor, err, tt.wantErr) + } + }) + } +} + +// TestProveExpressionRejectsWhatTheEvaluatorAdmits is the adversarial control +// for the row above: it proves independently that the production evaluator +// really does accept a sub-floor cluster for that expression, so the "must +// fail" verdict is grounded in behavior rather than in the prover's own +// opinion. Without this, a prover bug that rejected everything would still +// make the table green. +func TestProveExpressionRejectsWhatTheEvaluatorAdmits(t *testing.T) { + t.Parallel() + + const expr = ">= 1.32 || > 1.31.0 < 1.31.2" + + parsed, err := constraints.ParseCompoundConstraint(expr) + if err != nil { + t.Fatalf("ParseCompoundConstraint(%q): %v", expr, err) + } + satisfied, err := parsed.Evaluate("1.31.1") + if err != nil { + t.Fatalf("Evaluate(1.31.1): %v", err) + } + if !satisfied { + t.Fatalf("expected the production evaluator to accept 1.31.1 for %q; if this "+ + "changed, the control in TestProveExpressionClearsFloor needs rebasing", expr) + } + if err := proveExpressionClearsFloor(expr, 32); err == nil { + t.Fatalf("prover accepted %q even though the evaluator admits a 1.31.1 cluster", expr) + } +} From e2e65d257a1ba1d299ca8c8ceffd25f7629114f5 Mon Sep 17 00:00:00 2001 From: Yuan Chen Date: Fri, 28 Aug 2026 16:37:18 -0700 Subject: [PATCH 6/6] test(recipe): probe declared bounds so patch-precision floors are satisfiable The probe sweep tried only minor-precision readings, which no patch-precision range can satisfy: ">= 1.34.3 < 1.35.0" clears the prover, yet 1.34.0 is below its lower bound and 1.35.0 is excluded by its upper one, so a correct floor was reported as admitting no supported release. The catalog declares no such range today, so the defect was latent and fail-closed rather than fail-open. Extract probeReadings, which appends each declared bound to the minor-precision probes and returns a fresh slice - appending to the shared probe slice in place would write into a backing array reused by every declaration under test. TestProbeSetAdmitsPatchPrecisionRange pins this: reverting probeReadings to return the supported readings unchanged fails it. Also drop a redundant comparison in the component catalog. Signed-off-by: Yuan Chen --- docs/user/component-catalog.md | 2 +- pkg/recipe/dra_k8s_floor_test.go | 65 +++++++++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/docs/user/component-catalog.md b/docs/user/component-catalog.md index c9e7897b2..353abc051 100644 --- a/docs/user/component-catalog.md +++ b/docs/user/component-catalog.md @@ -72,7 +72,7 @@ Not every component appears in every recipe. The recipe engine selects component Production GPU leaf recipes (H100, GB200, RTX Pro 6000 on EKS / AKS / GKE / OKE / LKE) enable the NFD Topology Updater. It publishes per-node `NodeResourceTopology` CRDs that describe NUMA zones, GPU-to-NUMA affinity, and NIC-to-NUMA affinity. Runtime consumers (NUMA-aware schedulers, debugging via `kubectl get noderesourcetopologies`) can read these CRDs without further configuration. -The Topology Updater requires the kubelet `podResources` gRPC socket. The `KubeletPodResources` feature gate has been on by default since Kubernetes 1.15 (Beta) and reached GA in Kubernetes 1.28; AICR's recipe constraints require K8s ≥ 1.32 or higher, so this is satisfied in practice. Recipes targeting Kubernetes `< 1.15` must enable the feature gate explicitly. Kind / KWOK simulated clusters do not run a real kubelet and therefore leave the Topology Updater disabled — kind-based recipes will not see `NodeResourceTopology` CRDs. +The Topology Updater requires the kubelet `podResources` gRPC socket. The `KubeletPodResources` feature gate has been on by default since Kubernetes 1.15 (Beta) and reached GA in Kubernetes 1.28; AICR's recipe constraints require K8s ≥ 1.32, so this is satisfied in practice. Recipes targeting Kubernetes `< 1.15` must enable the feature gate explicitly. Kind / KWOK simulated clusters do not run a real kubelet and therefore leave the Topology Updater disabled — kind-based recipes will not see `NodeResourceTopology` CRDs. See the upstream [Topology Updater docs](https://kubernetes-sigs.github.io/node-feature-discovery/stable/usage/nfd-topology-updater.html) for runtime consumer examples. diff --git a/pkg/recipe/dra_k8s_floor_test.go b/pkg/recipe/dra_k8s_floor_test.go index 3c56fe88c..0754148da 100644 --- a/pkg/recipe/dra_k8s_floor_test.go +++ b/pkg/recipe/dra_k8s_floor_test.go @@ -207,6 +207,26 @@ func proveExpressionClearsFloor(expr string, floorMinor int) error { // also rejects every cluster the catalog claims to support — a typo, not a // floor. Failing on it keeps the guard closed against expressions it cannot // show are meaningful. +// probeReadings returns the Kubernetes readings used to show that a +// declaration admits at least one cluster. Minor-precision probes alone are +// not enough: a patch-precision range such as ">= 1.34.3 < 1.35.0" is +// satisfied by no "1.N.0" reading, so a correct floor would be reported as +// admitting nothing. The declared bounds are therefore probed as well. +// +// The result is a fresh slice. Appending to `supported` in place would write +// into a backing array shared by every declaration under test whenever it has +// spare capacity. +func probeReadings(supported []string, parsed *constraints.CompoundConstraint) []string { + readings := make([]string, 0, len(supported)) + readings = append(readings, supported...) + for _, group := range parsed.Alternatives { + for i := range group { + readings = append(readings, group[i].Value) + } + } + return readings +} + func supportedProbeVersions(floorMinor int) []string { var probes []string for minor := floorMinor; minor <= 60; minor++ { @@ -383,7 +403,7 @@ func verifyK8sFloorDeclaration(t *testing.T, decl k8sFloorDeclaration, floorMino return } - for _, reading := range supported { + for _, reading := range probeReadings(supported, parsed) { satisfied, evalErr := parsed.Evaluate(reading) if evalErr != nil { t.Errorf("%s (%s) declares K8s.server.version %q, which the shipping evaluator\n"+ @@ -484,3 +504,46 @@ func TestProveExpressionRejectsWhatTheEvaluatorAdmits(t *testing.T) { t.Fatalf("prover accepted %q even though the evaluator admits a 1.31.1 cluster", expr) } } + +// TestProbeSetAdmitsPatchPrecisionRange is the regression control for the +// probe sweep in assertDeclarationIsSatisfiable. The sweep used to try only +// "1.N.0" readings, which no patch-precision range can satisfy: ">= 1.34.3 +// < 1.35.0" clears proveExpressionClearsFloor, yet 1.34.0 is below its lower +// bound and 1.35.0 is excluded by its upper one. A correct floor was therefore +// reported as admitting no supported release. +// +// Reverting probeReadings to return `supported` unchanged makes this test +// fail, which is the point: the catalog currently declares no patch-precision +// range, so the defect is latent and nothing else would catch a regression. +func TestProbeSetAdmitsPatchPrecisionRange(t *testing.T) { + const expr = ">= 1.34.3 < 1.35.0" + + parsed, err := constraints.ParseCompoundConstraint(expr) + if err != nil { + t.Fatalf("parsing %q: %v", expr, err) + } + + for _, reading := range supportedProbeVersions(32) { + satisfied, evalErr := parsed.Evaluate(reading) + if evalErr != nil { + t.Fatalf("evaluating %q against %q: %v", expr, reading, evalErr) + } + if satisfied { + t.Fatalf("expected no minor-precision reading to satisfy %q, but %q did;\n"+ + " this test no longer proves the bound-probing loop is required", expr, reading) + } + } + + var admitted bool + for _, reading := range probeReadings(supportedProbeVersions(32), parsed) { + satisfied, evalErr := parsed.Evaluate(reading) + if evalErr != nil { + t.Fatalf("evaluating %q against probe %q: %v", expr, reading, evalErr) + } + admitted = admitted || satisfied + } + if !admitted { + t.Errorf("no declared bound of %q satisfies it, so the probe sweep would still\n"+ + " fail a correct floor. See #2402.", expr) + } +}