diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index ad7d89c55..5f3be7c21 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -130,6 +130,11 @@ jobs: with: go-version-file: tools/go-toolchain/go.mod + - name: Run ncp-local-cluster Makefile tests + run: | + make -C tools/ncp-local-cluster test-cluster-lifecycle-make + make -C tools/ncp-local-cluster test-multicluster-make + # The repo tooling modules carry tests that no workflow ran, so a pull # request could break them and still go green. tools/docs-version-sync # reached main with three failing tests that way. diff --git a/docs/dev/local-development.md b/docs/dev/local-development.md index 7943ad399..cbac96aab 100644 --- a/docs/dev/local-development.md +++ b/docs/dev/local-development.md @@ -108,8 +108,9 @@ the rationale. - [k3d](https://k3d.io/#installation) v5.x or later - `kubectl` - `helm` >= 3.12 -- `helmfile` >= 1.1.0, < 1.2.0 (Helmfile flows only) -- `helm-diff` plugin (Helmfile flows only): +- [Go](https://go.dev/doc/install) >= 1.24.0 (required to build `nvcf-cli`) +- `helmfile` >= 1.1.0, < 1.2.0 +- `helm-diff` plugin: `helm plugin install https://github.com/databus23/helm-diff` - An NGC API key with access to the NVCF chart and image registry. - `nvcf-cli` built from this repo: diff --git a/docs/user/local-development/multi-cluster-cli.md b/docs/user/local-development/multi-cluster-cli.md index 23f861ffa..b8d9a66a3 100644 --- a/docs/user/local-development/multi-cluster-cli.md +++ b/docs/user/local-development/multi-cluster-cli.md @@ -45,6 +45,9 @@ Install the following tools: - [k3d](https://k3d.io/#installation) v5.x or later - `kubectl` - `helm` >= 3.12 +- [Go](https://go.dev/doc/install) >= 1.24.0 (required to build `nvcf-cli`) +- `helmfile` >= 1.1.0, < 1.2.0 +- `helm-diff` plugin: `helm plugin install https://github.com/databus23/helm-diff` - An NGC API key from [ngc.nvidia.com](https://ngc.nvidia.com) with access to the NVCF chart and image registry. - The NGC organization and team slugs that hold the chart and image diff --git a/docs/user/local-development/multi-cluster-helmfile.md b/docs/user/local-development/multi-cluster-helmfile.md index 1300e2fbe..a69d0fa3b 100644 --- a/docs/user/local-development/multi-cluster-helmfile.md +++ b/docs/user/local-development/multi-cluster-helmfile.md @@ -45,6 +45,7 @@ Install the following tools: - [k3d](https://k3d.io/#installation) v5.x or later - `kubectl` - `helm` >= 3.12 +- [Go](https://go.dev/doc/install) >= 1.24.0 (required to build `nvcf-cli`) - `helmfile` >= 1.1.0, < 1.2.0 - `helm-diff` plugin: `helm plugin install https://github.com/databus23/helm-diff` - An NGC API key from [ngc.nvidia.com](https://ngc.nvidia.com) with diff --git a/docs/user/local-development/single-cluster-cli.md b/docs/user/local-development/single-cluster-cli.md index 96b86b31a..127c1dc7f 100644 --- a/docs/user/local-development/single-cluster-cli.md +++ b/docs/user/local-development/single-cluster-cli.md @@ -25,6 +25,9 @@ Install the following tools: - [k3d](https://k3d.io/#installation) v5.x or later - `kubectl` - `helm` >= 3.12 +- [Go](https://go.dev/doc/install) >= 1.24.0 (required to build `nvcf-cli`) +- `helmfile` >= 1.1.0, < 1.2.0 +- `helm-diff` plugin: `helm plugin install https://github.com/databus23/helm-diff` - An NGC API key from [ngc.nvidia.com](https://ngc.nvidia.com) with access to the NVCF chart and image registry. - The NGC organization and team slugs that hold the chart and image diff --git a/docs/user/local-development/single-cluster-helmfile.md b/docs/user/local-development/single-cluster-helmfile.md index c4325c26f..913f996b3 100644 --- a/docs/user/local-development/single-cluster-helmfile.md +++ b/docs/user/local-development/single-cluster-helmfile.md @@ -25,6 +25,7 @@ Install the following tools: - [k3d](https://k3d.io/#installation) v5.x or later - `kubectl` - `helm` >= 3.12 +- [Go](https://go.dev/doc/install) >= 1.24.0 (required to build `nvcf-cli`) - `helmfile` >= 1.1.0, < 1.2.0 - `helm-diff` plugin: `helm plugin install https://github.com/databus23/helm-diff` - An NGC API key from [ngc.nvidia.com](https://ngc.nvidia.com) with diff --git a/src/clis/nvcf-cli/cmd/self_hosted_compute_plane_test.go b/src/clis/nvcf-cli/cmd/self_hosted_compute_plane_test.go index a3b08b8b0..2cabc6c98 100644 --- a/src/clis/nvcf-cli/cmd/self_hosted_compute_plane_test.go +++ b/src/clis/nvcf-cli/cmd/self_hosted_compute_plane_test.go @@ -923,12 +923,14 @@ func installFakeComputePlaneHelmfile(t *testing.T) string { t.Helper() fakeBin := filepath.Join(t.TempDir(), "helmfile") body := `#!/bin/sh -last= +verb= for arg in "$@"; do printf 'arg=%s\n' "$arg" - last="$arg" + case "$arg" in + apply|template) verb="$arg" ;; + esac done -printf 'verb=%s\n' "$last" +printf 'verb=%s\n' "$verb" printf 'env:CLUSTER_NAME=%s\n' "$CLUSTER_NAME" printf 'env:NCA_ID=%s\n' "$NCA_ID" printf 'env:OUTPUT_DIR=%s\n' "$OUTPUT_DIR" diff --git a/src/clis/nvcf-cli/cmd/self_hosted_install.go b/src/clis/nvcf-cli/cmd/self_hosted_install.go index d163180f1..bb2879f44 100644 --- a/src/clis/nvcf-cli/cmd/self_hosted_install.go +++ b/src/clis/nvcf-cli/cmd/self_hosted_install.go @@ -136,15 +136,21 @@ func runSelfHostedInstall(c *cobra.Command, _ []string) error { } fmt.Fprintf(c.ErrOrStderr(), ">>> Resolving stack: %s\n", stackDescriptor(resolved)) + helmRuntimeMode, err := resolveSelfHostedHelmRuntimeMode(c.Context()) + if err != nil { + return fmt.Errorf("resolving helm runtime: %w", err) + } + if installControlPlane { if err := selfhosted.Render(selfhosted.RenderOptions{ - StackPath: resolved.Path, - Env: selfHostedEnv, - Apply: !selfHostedNoApply, - KubeContext: selfHostedControlPlaneContext, // M+9: empty in single-cluster mode - Stdout: c.OutOrStdout(), - Stderr: c.ErrOrStderr(), - Ctx: c.Context(), + StackPath: resolved.Path, + Env: selfHostedEnv, + Apply: !selfHostedNoApply, + HelmRuntimeMode: helmRuntimeMode, + KubeContext: selfHostedControlPlaneContext, // M+9: empty in single-cluster mode + Stdout: c.OutOrStdout(), + Stderr: c.ErrOrStderr(), + Ctx: c.Context(), }); err != nil { return err } @@ -238,13 +244,14 @@ func runSelfHostedInstall(c *cobra.Command, _ []string) error { } return selfhosted.Render(selfhosted.RenderOptions{ - StackPath: resolved.Path, - Env: selfHostedEnv, - Apply: !selfHostedNoApply, - KubeContext: selfHostedComputePlaneContext, // M+9: empty in single-cluster mode - Stdout: c.OutOrStdout(), - Stderr: c.ErrOrStderr(), - Ctx: c.Context(), + StackPath: resolved.Path, + Env: selfHostedEnv, + Apply: !selfHostedNoApply, + HelmRuntimeMode: helmRuntimeMode, + KubeContext: selfHostedComputePlaneContext, // M+9: empty in single-cluster mode + Stdout: c.OutOrStdout(), + Stderr: c.ErrOrStderr(), + Ctx: c.Context(), ExtraEnv: []string{ "CLUSTER_NAME=" + installClusterName, "CLUSTER_ID=" + resp.ClusterID, diff --git a/src/clis/nvcf-cli/cmd/self_hosted_install_test.go b/src/clis/nvcf-cli/cmd/self_hosted_install_test.go index 9c3cdc146..a3de40b7b 100644 --- a/src/clis/nvcf-cli/cmd/self_hosted_install_test.go +++ b/src/clis/nvcf-cli/cmd/self_hosted_install_test.go @@ -45,6 +45,10 @@ func resetInstallFlags(t *testing.T) { selfHostedToken = "" selfHostedControlPlaneContext = "" selfHostedComputePlaneContext = "" + prevRuntimeResolver := resolveSelfHostedHelmRuntimeMode + resolveSelfHostedHelmRuntimeMode = func(context.Context) (selfhosted.HelmRuntimeMode, error) { + return selfhosted.HelmRuntimeHelm3Legacy, nil + } prevFetchRootCA := fetchControlPlaneRootCAPEM fetchControlPlaneRootCAPEM = func(context.Context, string) (string, error) { return "", nil @@ -60,6 +64,7 @@ func resetInstallFlags(t *testing.T) { selfHostedToken = "" selfHostedControlPlaneContext = "" selfHostedComputePlaneContext = "" + resolveSelfHostedHelmRuntimeMode = prevRuntimeResolver fetchControlPlaneRootCAPEM = prevFetchRootCA }) } @@ -89,7 +94,7 @@ func TestSelfHostedInstall_ControlPlane_NoApply(t *testing.T) { assert.Contains(t, stdout.String(), "kind: ConfigMap") } -func TestSelfHostedInstall_ControlPlane_AppliesByDefault(t *testing.T) { +func TestSelfHostedInstall_ControlPlane_Helm4AppliesStateFilesSequentially(t *testing.T) { resetInstallFlags(t) stackDir := t.TempDir() require.NoError(t, os.MkdirAll(filepath.Join(stackDir, "helmfile.d"), 0o755)) @@ -97,7 +102,7 @@ func TestSelfHostedInstall_ControlPlane_AppliesByDefault(t *testing.T) { fakeBin := filepath.Join(t.TempDir(), "helmfile") require.NoError(t, os.WriteFile(fakeBin, - []byte("#!/bin/sh\nlast=\nfor arg in \"$@\"; do last=\"$arg\"; done\nprintf 'verb=%s\\n' \"$last\"\n"), + []byte("#!/bin/sh\nprintf '%s\\n' \"$@\"\n"), 0o755)) t.Setenv("PATH", filepath.Dir(fakeBin)+":"+os.Getenv("PATH")) t.Setenv("HOME", t.TempDir()) @@ -119,12 +124,17 @@ func TestSelfHostedInstall_ControlPlane_AppliesByDefault(t *testing.T) { } require.NoError(t, sm.Save()) + prevRuntimeResolver := resolveSelfHostedHelmRuntimeMode prevAuthProbe := authProbe prevInit := runSelfHostedInit t.Cleanup(func() { + resolveSelfHostedHelmRuntimeMode = prevRuntimeResolver authProbe = prevAuthProbe runSelfHostedInit = prevInit }) + resolveSelfHostedHelmRuntimeMode = func(context.Context) (selfhosted.HelmRuntimeMode, error) { + return selfhosted.HelmRuntimeHelm4Compat, nil + } authProbe = func(context.Context, string) (*auth.Fingerprint, error) { return &auth.Fingerprint{IssuerURL: "http://api.localhost:8080", JWKSKid: "kid", APIKeysEndpoint: "http://api-keys.localhost:8080"}, nil } @@ -141,7 +151,9 @@ func TestSelfHostedInstall_ControlPlane_AppliesByDefault(t *testing.T) { }) require.NoError(t, rootCmd.Execute()) - assert.Contains(t, stdout.String(), "verb=apply") + assert.Contains(t, stdout.String(), "--sequential-helmfiles") + assert.Contains(t, stdout.String(), "apply") + assert.Contains(t, stdout.String(), "--skip-diff-on-install") assert.Equal(t, 1, initCalls) } @@ -295,11 +307,11 @@ func TestSelfHostedInstall_ComputePlane_AppliesByDefault(t *testing.T) { stackDir := t.TempDir() require.NoError(t, os.MkdirAll(filepath.Join(stackDir, "helmfile.d"), 0o755)) - // Fake helmfile echoes the verb passed as its last arg so the test can assert - // that 'apply' (not 'template') ran when --no-apply is omitted. + // Fake helmfile echoes every argument so the test can assert that apply runs + // with the first-install diff guard when --no-apply is omitted. fakeBin := filepath.Join(t.TempDir(), "helmfile") require.NoError(t, os.WriteFile(fakeBin, - []byte("#!/bin/sh\nlast=\nfor arg in \"$@\"; do last=\"$arg\"; done\nprintf 'verb=%s\\n' \"$last\"\n"), + []byte("#!/bin/sh\nprintf '%s\\n' \"$@\"\n"), 0o755)) t.Setenv("PATH", filepath.Dir(fakeBin)+":"+os.Getenv("PATH")) @@ -314,7 +326,8 @@ func TestSelfHostedInstall_ComputePlane_AppliesByDefault(t *testing.T) { require.NoError(t, rootCmd.Execute()) assert.Equal(t, 1, fakeCC.registerCalls) - assert.Contains(t, stdout.String(), "verb=apply") + assert.Contains(t, stdout.String(), "apply") + assert.Contains(t, stdout.String(), "--skip-diff-on-install") } func TestSelfHostedInstall_ComputePlane_LocalSplitWritesExternalControlPlaneEndpoints(t *testing.T) { diff --git a/src/clis/nvcf-cli/internal/selfhosted/render.go b/src/clis/nvcf-cli/internal/selfhosted/render.go index a267163c6..a4a140ce1 100644 --- a/src/clis/nvcf-cli/internal/selfhosted/render.go +++ b/src/clis/nvcf-cli/internal/selfhosted/render.go @@ -26,7 +26,8 @@ import ( ) // RenderOptions controls a single invocation of 'helmfile template' or -// 'helmfile apply'. Stdout/Stderr writers are caller-supplied so the install +// 'helmfile apply --skip-diff-on-install'. Stdout/Stderr writers are +// caller-supplied so the install // command can split YAML output (stdout) from helmfile progress (stderr) per // spec ยง6.1. type RenderOptions struct { @@ -48,8 +49,8 @@ type RenderOptions struct { // Selector (optional) is the -l flag value for narrowing to a single // release group (e.g. "component=control-plane"). Selector string - // Apply runs 'helmfile apply' when true; otherwise runs 'helmfile template'. - // Apply=true is reserved for the up orchestrator (M5). + // Apply runs 'helmfile apply --skip-diff-on-install' when true; otherwise + // runs 'helmfile template'. Apply bool // ExtraEnv holds additional environment variables in "KEY=VALUE" form that // are appended to the helmfile subprocess environment after HELMFILE_ENV. @@ -69,7 +70,7 @@ type RenderOptions struct { Ctx context.Context } -// Render invokes 'helmfile {template,apply}' against the resolved stack tree, +// Render invokes helmfile template or apply against the resolved stack tree, // plumbing HELMFILE_ENV through the subprocess environment. Stdout and Stderr // are written to the caller-supplied writers. A non-zero exit code from // helmfile is returned as a wrapped error containing the exit status. @@ -94,6 +95,9 @@ func Render(opts RenderOptions) error { args = append(args, "--kube-context="+opts.KubeContext) } args = append(args, verb) + if opts.Apply { + args = append(args, "--skip-diff-on-install") + } ctx := opts.Ctx if ctx == nil { diff --git a/src/clis/nvcf-cli/internal/selfhosted/render_test.go b/src/clis/nvcf-cli/internal/selfhosted/render_test.go index c678017ac..5722721ad 100644 --- a/src/clis/nvcf-cli/internal/selfhosted/render_test.go +++ b/src/clis/nvcf-cli/internal/selfhosted/render_test.go @@ -191,6 +191,30 @@ func TestRender_Helm4CompatApplyUsesLegacyTrackMode(t *testing.T) { assert.Contains(t, out, "helm-legacy") } +func TestRender_ApplySkipsDiffOnInstall(t *testing.T) { + dir := t.TempDir() + fake := filepath.Join(dir, "helmfile") + require.NoError(t, os.WriteFile(fake, + []byte("#!/bin/sh\nprintf '%s\\n' \"$@\"\n"), 0o755)) + + stack := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(stack, "helmfile.d"), 0o755)) + + var stdout, stderr bytes.Buffer + err := Render(RenderOptions{ + StackPath: stack, + Env: "local", + HelmfileBin: fake, + Apply: true, + Stdout: &stdout, + Stderr: &stderr, + }) + require.NoError(t, err) + args := strings.Split(strings.TrimSpace(stdout.String()), "\n") + require.GreaterOrEqual(t, len(args), 2) + assert.Equal(t, []string{"apply", "--skip-diff-on-install"}, args[len(args)-2:]) +} + func TestRender_Helm4CompatSingleFileOmitsSequentialFlag(t *testing.T) { dir := t.TempDir() fake := filepath.Join(dir, "helmfile") diff --git a/tests/bdd/fixtures/nvcf-compute-plane-local-bdd-multi.yaml b/tests/bdd/fixtures/nvcf-compute-plane-local-bdd-multi.yaml index f59dfb2bb..9d9568bef 100644 --- a/tests/bdd/fixtures/nvcf-compute-plane-local-bdd-multi.yaml +++ b/tests/bdd/fixtures/nvcf-compute-plane-local-bdd-multi.yaml @@ -44,6 +44,10 @@ global: revalServiceURL: http://reval.nvcf.svc.cluster.local:8080 natsURL: nats://nats.nats-system.svc.cluster.local:4222 +# Local BDD does not install the optional observability stack. +observability: + profile: disabled + agentConfig: mergeConfig: | cluster: diff --git a/tests/bdd/fixtures/nvcf-compute-plane-local-bdd.yaml b/tests/bdd/fixtures/nvcf-compute-plane-local-bdd.yaml index 0ac047b2d..83472f525 100644 --- a/tests/bdd/fixtures/nvcf-compute-plane-local-bdd.yaml +++ b/tests/bdd/fixtures/nvcf-compute-plane-local-bdd.yaml @@ -38,6 +38,10 @@ global: revalServiceURL: http://reval.nvcf.svc.cluster.local:8080 natsURL: nats://nats.nats-system.svc.cluster.local:4222 +# Local BDD does not install the optional observability stack. +observability: + profile: disabled + agentConfig: mergeConfig: | cluster: diff --git a/tests/bdd/fixtures/self-managed-local-bdd-multi.yaml b/tests/bdd/fixtures/self-managed-local-bdd-multi.yaml index 2c901781e..e3b6b38e7 100644 --- a/tests/bdd/fixtures/self-managed-local-bdd-multi.yaml +++ b/tests/bdd/fixtures/self-managed-local-bdd-multi.yaml @@ -38,6 +38,10 @@ global: # NVCF API advertises this value as LLM_REQUEST_ROUTER_ADDRESS. llmRequestRouterAddress: llm-request-router.nvcf.svc.cluster.local:50071 +# Local BDD does not install the optional observability stack. +observability: + profile: disabled + cassandra: replicaCount: 1 resourcesPreset: "large" diff --git a/tests/bdd/fixtures/self-managed-local-bdd.yaml b/tests/bdd/fixtures/self-managed-local-bdd.yaml index b4ed793ca..8e5341eea 100644 --- a/tests/bdd/fixtures/self-managed-local-bdd.yaml +++ b/tests/bdd/fixtures/self-managed-local-bdd.yaml @@ -31,6 +31,10 @@ global: essServiceURL: http://ess-api.ess.svc.cluster.local:8080 invocationServiceURL: http://invocation.nvcf.svc.cluster.local:8080 +# Local BDD does not install the optional observability stack. +observability: + profile: disabled + cassandra: replicaCount: 1 resourcesPreset: "large" diff --git a/tools/ncp-local-cluster/AGENTS.md b/tools/ncp-local-cluster/AGENTS.md index efce1c983..3dc12633d 100644 --- a/tools/ncp-local-cluster/AGENTS.md +++ b/tools/ncp-local-cluster/AGENTS.md @@ -16,6 +16,7 @@ Run Makefile-only validation from `tools/ncp-local-cluster`: ```bash make validate-compute-clusters make print-compute-clusters +make test-cluster-lifecycle-make make test-multicluster-make ``` diff --git a/tools/ncp-local-cluster/Makefile b/tools/ncp-local-cluster/Makefile index 2b41a681c..2b4ddba32 100644 --- a/tools/ncp-local-cluster/Makefile +++ b/tools/ncp-local-cluster/Makefile @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -.PHONY: help clean build test test-coverage-html test-manual start stop destroy destroy-all-ncp-local status ensure-cluster ensure-context ensure-docker-config validate-compute-clusters print-compute-clusters start-control-plane deploy-control-plane-addons deploy-control-plane-endpoints build-and-deploy-control-plane-cluster destroy-control-plane start-compute-plane deploy-compute-plane-addons configure-compute-control-plane-dns deploy-compute-control-plane-endpoints build-and-deploy-compute-plane-cluster destroy-compute-plane build-and-deploy-multicluster destroy-multicluster test-destroy-all-ncp-local test-multicluster-make setup-gateway-api setup-metallb check-gateway-api deploy-sample deploy-nginx wait-for-nginx wait-for-gateway validate-gateway cleanup-nginx wait-for-deployment validate-deployment cleanup-sample build-and-deploy-cluster build-csi-smb deploy-csi-smb wait-for-csi-smb build-fake-gpu-operator deploy-fake-gpu-operator wait-for-fake-gpu-operator deploy-prometheus-crds uninstall-prometheus-crds deploy-kube-state-metrics wait-for-kube-state-metrics uninstall-kube-state-metrics build-credential-provider-multiarch +.PHONY: help clean build test test-coverage-html test-manual start stop destroy destroy-all-ncp-local status ensure-cluster ensure-context ensure-docker-config validate-compute-clusters print-compute-clusters start-control-plane deploy-control-plane-addons deploy-control-plane-endpoints build-and-deploy-control-plane-cluster destroy-control-plane start-compute-plane deploy-compute-plane-addons configure-compute-control-plane-dns deploy-compute-control-plane-endpoints build-and-deploy-compute-plane-cluster destroy-compute-plane build-and-deploy-multicluster destroy-multicluster test-cluster-lifecycle-make test-destroy-all-ncp-local test-multicluster-make setup-gateway-api setup-metallb check-gateway-api deploy-sample deploy-nginx wait-for-nginx wait-for-gateway validate-gateway cleanup-nginx wait-for-deployment validate-deployment cleanup-sample build-and-deploy-cluster build-csi-smb deploy-csi-smb wait-for-csi-smb build-fake-gpu-operator deploy-fake-gpu-operator wait-for-fake-gpu-operator deploy-prometheus-crds uninstall-prometheus-crds deploy-kube-state-metrics wait-for-kube-state-metrics uninstall-kube-state-metrics build-credential-provider-multiarch # === Cluster Configuration === CLUSTER_NAME := ncp-local @@ -96,7 +96,7 @@ ensure-cluster: ## Create cluster if it doesn't exist, otherwise skip $(call require,kubectl,See https://kubernetes.io/docs/tasks/tools/) @if ! k3d cluster get $(CLUSTER_NAME) >/dev/null 2>&1; then \ echo "Creating k3d cluster $(CLUSTER_NAME) using config $(K3D_CONFIG_FILE)..."; \ - K3D_CLUSTER_NAME="$(CLUSTER_NAME)" \ + if ! K3D_CLUSTER_NAME="$(CLUSTER_NAME)" \ CONTROL_PLANE_HTTP_PORT="$(CONTROL_PLANE_HTTP_PORT)" \ CONTROL_PLANE_HTTPS_PORT="$(CONTROL_PLANE_HTTPS_PORT)" \ CONTROL_PLANE_GRPC_PORT="$(CONTROL_PLANE_GRPC_PORT)" \ @@ -105,7 +105,10 @@ ensure-cluster: ## Create cluster if it doesn't exist, otherwise skip CONTROL_PLANE_LLM_GRPC_PORT="$(CONTROL_PLANE_LLM_GRPC_PORT)" \ CONTROL_PLANE_LLM_QUIC_PORT="$(CONTROL_PLANE_LLM_QUIC_PORT)" \ CONTROL_PLANE_NATS_PORT="$(CONTROL_PLANE_NATS_PORT)" \ - k3d cluster create $(CLUSTER_NAME) --config $(K3D_CONFIG_FILE); \ + k3d cluster create $(CLUSTER_NAME) --config $(K3D_CONFIG_FILE); then \ + echo "ERROR Failed to create k3d cluster $(CLUSTER_NAME)." >&2; \ + exit 1; \ + fi; \ echo "OK Cluster $(CLUSTER_NAME) created."; \ else \ echo "OK Cluster $(CLUSTER_NAME) already exists. Skipping creation."; \ @@ -133,7 +136,10 @@ start: ensure-docker-config @echo "========== START CLUSTER ==========" @$(MAKE) ensure-cluster @echo "Ensuring cluster $(CLUSTER_NAME) is ready..." - @k3d cluster start $(CLUSTER_NAME) 2>/dev/null || echo "Cluster already running." + @if ! k3d cluster start $(CLUSTER_NAME); then \ + echo "ERROR Failed to start k3d cluster $(CLUSTER_NAME)." >&2; \ + exit 1; \ + fi @echo "OK Cluster $(CLUSTER_NAME) is running." @$(MAKE) ensure-context @@ -295,6 +301,9 @@ destroy-multicluster: validate-compute-clusters ## Destroy the default control-p $(MAKE) CLUSTER_NAME="$$cluster" destroy || true; \ done +test-cluster-lifecycle-make: ## Run dry Makefile tests for cluster lifecycle behavior + @tests/test-cluster-lifecycle-make.sh + test-multicluster-make: ## Run dry Makefile tests for multi-cluster name derivation @tests/test-multicluster-make.sh diff --git a/tools/ncp-local-cluster/tests/test-cluster-lifecycle-make.sh b/tools/ncp-local-cluster/tests/test-cluster-lifecycle-make.sh new file mode 100755 index 000000000..8bb5e8e1c --- /dev/null +++ b/tools/ncp-local-cluster/tests/test-cluster-lifecycle-make.sh @@ -0,0 +1,236 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +# Run the real Makefile in an isolated directory. Commands resolved through +# FAKE_BIN are test doubles, so this test never creates or changes a real cluster. +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TEST_ROOT="$(mktemp -d)" +FAKE_BIN="${TEST_ROOT}/bin" +CALL_LOG="${TEST_ROOT}/calls.log" + +cleanup() { + rm -rf "${TEST_ROOT}" +} +trap cleanup EXIT + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +assert_contains() { + local expected="$1" + local file="$2" + local label="$3" + + if ! grep -Fqx "${expected}" "${file}"; then + echo "--- ${file}" >&2 + cat "${file}" >&2 + fail "${label}: missing '${expected}'" + fi +} + +# Match a substring in captured Make output. Use assert_contains above when the +# expected value must occupy a complete call-log line. +assert_output_contains() { + local expected="$1" + local file="$2" + local label="$3" + + if ! grep -Fq "${expected}" "${file}"; then + cat "${file}" >&2 + fail "${label}: missing '${expected}'" + fi +} + +# Confirm that a failed lifecycle step did not allow a later command to run. +assert_log_excludes() { + local unexpected="$1" + local label="$2" + + if grep -Fq "${unexpected}" "${CALL_LOG}"; then + cat "${CALL_LOG}" >&2 + fail "${label}: unexpectedly found '${unexpected}'" + fi +} + +# Run the copied Makefile quietly in the sandbox. "$@" forwards the target and +# every variable assignment supplied by the test scenario. +run_make() { + make --no-print-directory -s -C "${TEST_ROOT}" "$@" +} + +# The copied Makefile is the unit under test. The dummy Docker config satisfies +# the start target's existence check without using real registry credentials. +mkdir -p "${FAKE_BIN}" "${TEST_ROOT}/secrets" +cp "${ROOT_DIR}/Makefile" "${TEST_ROOT}/Makefile" +printf '{}\n' >"${TEST_ROOT}/secrets/docker-config.json" +: >"${CALL_LOG}" + +# Create a controllable k3d test double. The quoted heredoc defers variable +# expansion until the fake command runs. K3D_*_EXIT values simulate cluster +# state and failures, while K3D_CALL_LOG records commands and inputs. +cat >"${FAKE_BIN}/k3d" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +case "$*" in + cluster\ get\ *) + # Exit 0 means the cluster exists; a nonzero value means it is absent. + printf 'k3d %s\n' "$*" >>"${K3D_CALL_LOG}" + exit "${K3D_GET_EXIT:-0}" + ;; + cluster\ create\ *) + # Record the create command and every value consumed by the k3d config. + { + printf 'k3d %s\n' "$*" + printf 'K3D_CLUSTER_NAME=%s\n' "${K3D_CLUSTER_NAME-}" + printf 'CONTROL_PLANE_HTTP_PORT=%s\n' "${CONTROL_PLANE_HTTP_PORT-}" + printf 'CONTROL_PLANE_HTTPS_PORT=%s\n' "${CONTROL_PLANE_HTTPS_PORT-}" + printf 'CONTROL_PLANE_GRPC_PORT=%s\n' "${CONTROL_PLANE_GRPC_PORT-}" + printf 'CONTROL_PLANE_GRPC_PROXY_PORT=%s\n' "${CONTROL_PLANE_GRPC_PROXY_PORT-}" + printf 'CONTROL_PLANE_GRPC_WORKER_PORT=%s\n' "${CONTROL_PLANE_GRPC_WORKER_PORT-}" + printf 'CONTROL_PLANE_LLM_GRPC_PORT=%s\n' "${CONTROL_PLANE_LLM_GRPC_PORT-}" + printf 'CONTROL_PLANE_LLM_QUIC_PORT=%s\n' "${CONTROL_PLANE_LLM_QUIC_PORT-}" + printf 'CONTROL_PLANE_NATS_PORT=%s\n' "${CONTROL_PLANE_NATS_PORT-}" + } >>"${K3D_CALL_LOG}" + exit "${K3D_CREATE_EXIT:-0}" + ;; + cluster\ start\ *) + # A nonzero value simulates k3d failing to start an existing cluster. + printf 'k3d %s\n' "$*" >>"${K3D_CALL_LOG}" + exit "${K3D_START_EXIT:-0}" + ;; + kubeconfig\ merge\ *) + printf 'k3d %s\n' "$*" >>"${K3D_CALL_LOG}" + ;; + *) + echo "unexpected k3d command: $*" >&2 + exit 64 + ;; +esac +EOF + +# Provide only the kubectl operations used by ensure-context. Unexpected +# operations fail the test instead of being silently accepted. +cat >"${FAKE_BIN}/kubectl" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail + +printf 'kubectl %s\n' "$*" >>"${K3D_CALL_LOG}" +case "$*" in + "config current-context") + printf '%s\n' "${KUBECTL_CURRENT_CONTEXT-}" + ;; + config\ use-context\ *) + ;; + *) + echo "unexpected kubectl command: $*" >&2 + exit 64 + ;; +esac +EOF + +# Put the test doubles first in PATH before invoking the copied Makefile. +chmod +x "${FAKE_BIN}/k3d" "${FAKE_BIN}/kubectl" +export PATH="${FAKE_BIN}:${PATH}" +export K3D_CALL_LOG="${CALL_LOG}" + +# Scenario 1: the cluster is absent and creation succeeds. Verify that +# ensure-cluster invokes k3d create with the selected name, config, and ports. +export K3D_GET_EXIT=1 +export K3D_CREATE_EXIT=0 +run_make ensure-cluster \ + CLUSTER_NAME=lifecycle-create \ + K3D_CONFIG_FILE=lifecycle-config.yaml \ + CONTROL_PLANE_HTTP_PORT=18080 \ + CONTROL_PLANE_HTTPS_PORT=18443 \ + CONTROL_PLANE_GRPC_PORT=19090 \ + CONTROL_PLANE_GRPC_PROXY_PORT=20081 \ + CONTROL_PLANE_GRPC_WORKER_PORT=20086 \ + CONTROL_PLANE_LLM_GRPC_PORT=25071 \ + CONTROL_PLANE_LLM_QUIC_PORT=25072 \ + CONTROL_PLANE_NATS_PORT=14222 >/dev/null + +assert_contains "k3d cluster create lifecycle-create --config lifecycle-config.yaml" "${CALL_LOG}" "cluster create command" +assert_contains "K3D_CLUSTER_NAME=lifecycle-create" "${CALL_LOG}" "cluster name environment" +assert_contains "CONTROL_PLANE_HTTP_PORT=18080" "${CALL_LOG}" "HTTP port environment" +assert_contains "CONTROL_PLANE_HTTPS_PORT=18443" "${CALL_LOG}" "HTTPS port environment" +assert_contains "CONTROL_PLANE_GRPC_PORT=19090" "${CALL_LOG}" "gRPC port environment" +assert_contains "CONTROL_PLANE_GRPC_PROXY_PORT=20081" "${CALL_LOG}" "gRPC proxy port environment" +assert_contains "CONTROL_PLANE_GRPC_WORKER_PORT=20086" "${CALL_LOG}" "gRPC worker port environment" +assert_contains "CONTROL_PLANE_LLM_GRPC_PORT=25071" "${CALL_LOG}" "LLM gRPC port environment" +assert_contains "CONTROL_PLANE_LLM_QUIC_PORT=25072" "${CALL_LOG}" "LLM QUIC port environment" +assert_contains "CONTROL_PLANE_NATS_PORT=14222" "${CALL_LOG}" "NATS port environment" + +# Scenario 2: the cluster is absent and k3d create fails. Verify that +# ensure-cluster returns nonzero and reports the failed cluster name. +: >"${CALL_LOG}" +create_failure_output="${TEST_ROOT}/create-failure.out" +export K3D_CREATE_EXIT=23 +if run_make ensure-cluster CLUSTER_NAME=lifecycle-create-failure >"${create_failure_output}" 2>&1; then + cat "${create_failure_output}" >&2 + fail "ensure-cluster should fail when k3d cluster create fails" +fi +assert_output_contains \ + "ERROR Failed to create k3d cluster lifecycle-create-failure." \ + "${create_failure_output}" \ + "ensure-cluster create failure" + +# Scenario 3: start needs to create the cluster, but creation fails. Verify that +# start propagates the failure without attempting startup or context selection. +: >"${CALL_LOG}" +start_create_failure_output="${TEST_ROOT}/start-create-failure.out" +if run_make start CLUSTER_NAME=lifecycle-start-create-failure >"${start_create_failure_output}" 2>&1; then + cat "${start_create_failure_output}" >&2 + fail "start should fail when cluster creation fails" +fi +assert_log_excludes "k3d cluster start" "start after create failure" +assert_log_excludes "kubectl config current-context" "context check after create failure" + +# Scenario 4: the cluster already exists, but k3d start fails. Verify that start +# returns nonzero and stops before ensure-context. +: >"${CALL_LOG}" +start_failure_output="${TEST_ROOT}/start-failure.out" +export K3D_GET_EXIT=0 +export K3D_START_EXIT=24 +if run_make start CLUSTER_NAME=lifecycle-start-failure >"${start_failure_output}" 2>&1; then + cat "${start_failure_output}" >&2 + fail "start should fail when k3d cluster start fails" +fi +assert_output_contains \ + "ERROR Failed to start k3d cluster lifecycle-start-failure." \ + "${start_failure_output}" \ + "cluster startup failure" +assert_log_excludes "kubectl config current-context" "context check after startup failure" + +# Scenario 5: the cluster exists, starts successfully, and already has the +# expected kubectl context. Verify that the success path reaches ensure-context. +: >"${CALL_LOG}" +start_success_output="${TEST_ROOT}/start-success.out" +export K3D_START_EXIT=0 +export KUBECTL_CURRENT_CONTEXT=k3d-lifecycle-start-success +run_make start CLUSTER_NAME=lifecycle-start-success >"${start_success_output}" 2>&1 +assert_contains "k3d cluster start lifecycle-start-success" "${CALL_LOG}" "cluster startup command" +assert_contains "kubectl config current-context" "${CALL_LOG}" "ensure-context invocation" +assert_output_contains \ + "OK kubectl context is already set to k3d-lifecycle-start-success" \ + "${start_success_output}" \ + "successful context verification" + +echo "PASS: cluster lifecycle Makefile tests"