From bad255246a6a294a88c928b77b12dcb796a138a3 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Tue, 10 Mar 2026 11:22:46 +0900 Subject: [PATCH 01/24] feat: add bare metal support for Intel TDX and AMD SEV-SNP Co-Authored-By: Claude Opus 4.6 (1M context) --- README.md | 51 ++++- ansible/detect-runtime-class.yaml | 56 +++++ charts/all/baremetal/Chart.yaml | 9 + charts/all/baremetal/bm-kernel-params.yaml | 2 + charts/all/baremetal/templates/kata-nfd.yaml | 80 +++++++ .../templates/kernel-params-mco.yaml | 21 ++ .../all/baremetal/templates/nfd-instance.yaml | 12 ++ .../templates/runtimeclass-amd-snp.yaml | 12 ++ .../templates/runtimeclass-intel-tdx.yaml | 13 ++ charts/all/baremetal/templates/vsock-mco.yaml | 21 ++ charts/all/baremetal/values.yaml | 1 + charts/all/intel-dcap/Chart.yaml | 10 + .../intel-dcap/templates/intel-dpo-sgx.yaml | 11 + .../intel-dcap/templates/pccs-deployment.yaml | 69 ++++++ .../all/intel-dcap/templates/pccs-rbac.yaml | 49 +++++ .../templates/pccs-secrets-eso.yaml | 37 ++++ .../intel-dcap/templates/pccs-service.yaml | 13 ++ .../intel-dcap/templates/pccs-tls-eso.yaml | 24 +++ .../intel-dcap/templates/qgs-config-cm.yaml | 9 + charts/all/intel-dcap/templates/qgs-ds.yaml | 88 ++++++++ charts/all/intel-dcap/templates/qgs-rbac.yaml | 47 ++++ .../all/intel-dcap/templates/qgs-sgx-cm.yaml | 16 ++ charts/all/intel-dcap/values.yaml | 7 + .../hello-openshift/templates/_helpers.tpl | 9 +- .../hello-openshift/values.yaml | 5 + .../kbs-access/templates/secure-pod.yaml | 2 +- charts/coco-supported/kbs-access/values.yaml | 5 + charts/hub/storage/Chart.yaml | 9 + .../templates/hostpathprovisioner.yaml | 11 + .../storage/templates/hpp-storageclass.yaml | 13 ++ charts/hub/storage/templates/lvmcluster.yaml | 15 ++ charts/hub/storage/values.yaml | 14 ++ docs/dell-tdx-configuration.md | 149 +++++++++++++ overrides/values-BareMetal.yaml | 2 + overrides/values-None.yaml | 2 + overrides/values-storage-external.yaml | 2 + overrides/values-storage-hpp.yaml | 8 + overrides/values-storage-lvm.yaml | 8 + scripts/gen-secrets.sh | 26 +++ scripts/get-pccs-node.sh | 10 + values-baremetal.yaml | 200 ++++++++++++++++++ values-global.yaml | 4 +- values-secret.yaml.template | 36 ++++ 43 files changed, 1174 insertions(+), 14 deletions(-) create mode 100644 ansible/detect-runtime-class.yaml create mode 100644 charts/all/baremetal/Chart.yaml create mode 100644 charts/all/baremetal/bm-kernel-params.yaml create mode 100644 charts/all/baremetal/templates/kata-nfd.yaml create mode 100644 charts/all/baremetal/templates/kernel-params-mco.yaml create mode 100644 charts/all/baremetal/templates/nfd-instance.yaml create mode 100644 charts/all/baremetal/templates/runtimeclass-amd-snp.yaml create mode 100644 charts/all/baremetal/templates/runtimeclass-intel-tdx.yaml create mode 100644 charts/all/baremetal/templates/vsock-mco.yaml create mode 100644 charts/all/baremetal/values.yaml create mode 100644 charts/all/intel-dcap/Chart.yaml create mode 100644 charts/all/intel-dcap/templates/intel-dpo-sgx.yaml create mode 100644 charts/all/intel-dcap/templates/pccs-deployment.yaml create mode 100644 charts/all/intel-dcap/templates/pccs-rbac.yaml create mode 100644 charts/all/intel-dcap/templates/pccs-secrets-eso.yaml create mode 100644 charts/all/intel-dcap/templates/pccs-service.yaml create mode 100644 charts/all/intel-dcap/templates/pccs-tls-eso.yaml create mode 100644 charts/all/intel-dcap/templates/qgs-config-cm.yaml create mode 100644 charts/all/intel-dcap/templates/qgs-ds.yaml create mode 100644 charts/all/intel-dcap/templates/qgs-rbac.yaml create mode 100644 charts/all/intel-dcap/templates/qgs-sgx-cm.yaml create mode 100644 charts/all/intel-dcap/values.yaml create mode 100644 charts/hub/storage/Chart.yaml create mode 100644 charts/hub/storage/templates/hostpathprovisioner.yaml create mode 100644 charts/hub/storage/templates/hpp-storageclass.yaml create mode 100644 charts/hub/storage/templates/lvmcluster.yaml create mode 100644 charts/hub/storage/values.yaml create mode 100644 docs/dell-tdx-configuration.md create mode 100644 overrides/values-BareMetal.yaml create mode 100644 overrides/values-None.yaml create mode 100644 overrides/values-storage-external.yaml create mode 100644 overrides/values-storage-hpp.yaml create mode 100644 overrides/values-storage-lvm.yaml create mode 100755 scripts/get-pccs-node.sh create mode 100644 values-baremetal.yaml diff --git a/README.md b/README.md index 32ebe80b..f64fe596 100644 --- a/README.md +++ b/README.md @@ -2,21 +2,23 @@ Validated pattern for deploying confidential containers on OpenShift using the [Validated Patterns](https://validatedpatterns.io/) framework. -Confidential containers use hardware-backed Trusted Execution Environments (TEEs) to isolate workloads from cluster and hypervisor administrators. This pattern deploys and configures the Red Hat CoCo stack — including the sandboxed containers operator, Trustee (Key Broker Service), and peer-pod infrastructure — on Azure. +Confidential containers use hardware-backed Trusted Execution Environments (TEEs) to isolate workloads from cluster and hypervisor administrators. This pattern deploys and configures the Red Hat CoCo stack — including the sandboxed containers operator, Trustee (Key Broker Service), and peer-pod infrastructure — on Azure and bare metal. ## Topologies -The pattern provides two deployment topologies: +The pattern provides three deployment topologies: -1. **Single cluster** (`simple` clusterGroup) — deploys all components (Trustee, Vault, ACM, sandboxed containers, workloads) in one cluster. This breaks the RACI separation expected in a remote attestation architecture but simplifies testing and demonstrations. +1. **Single cluster** (`simple` clusterGroup) — deploys all components (Trustee, Vault, ACM, sandboxed containers, workloads) in one cluster on Azure. This breaks the RACI separation expected in a remote attestation architecture but simplifies testing and demonstrations. 2. **Multi-cluster** (`trusted-hub` + `spoke` clusterGroups) — separates the trusted zone from the untrusted workload zone: - **Hub** (`trusted-hub`): Runs Trustee (KBS + attestation service), HashiCorp Vault, ACM, and cert-manager. This cluster is the trust anchor. - **Spoke** (`spoke`): Runs the sandboxed containers operator and confidential workloads. The spoke is imported into ACM and managed from the hub. +3. **Bare metal** (`baremetal` clusterGroup) — deploys all components on bare metal hardware with Intel TDX or AMD SEV-SNP support. NFD (Node Feature Discovery) auto-detects the CPU architecture and configures the appropriate runtime. Supports SNO (Single Node OpenShift) and multi-node clusters. + The topology is controlled by the `main.clusterGroupName` field in `values-global.yaml`. -Currently supports Azure via peer-pods. Peer-pods provision confidential VMs (`Standard_DCas_v5` family) directly on the Azure hypervisor rather than nesting VMs inside worker nodes. +Azure deployments use peer-pods, which provision confidential VMs (`Standard_DCas_v5` family) directly on the Azure hypervisor. Bare metal deployments use layered images and hardware TEE features directly. ## Current version (4.*) @@ -42,9 +44,21 @@ All previous versions used pre-GA (Technology Preview) releases of Trustee: ### Prerequisites +**Azure deployments:** + - OpenShift 4.17+ cluster on Azure (self-managed via `openshift-install` or ARO) - Azure `Standard_DCas_v5` VM quota in your target region (these are confidential computing VMs and are not available in all regions). See the note below for more details. - Azure DNS hosting the cluster's DNS zone + +**Bare metal deployments:** + +- OpenShift 4.17+ cluster on bare metal with Intel TDX or AMD SEV-SNP hardware +- BIOS/firmware configured to enable TDX or SEV-SNP +- Available block devices for LVMS storage (auto-discovered) +- For Intel TDX: an Intel PCS API key from [api.portal.trustedservices.intel.com](https://api.portal.trustedservices.intel.com/) + +**Common:** + - Tools on your workstation: `podman`, `yq`, `jq`, `skopeo` - OpenShift pull secret saved at `~/pull-secret.json` (download from [console.redhat.com](https://console.redhat.com/openshift/downloads)) - Fork the repository — ArgoCD reconciles cluster state against your fork, so changes must be pushed to your remote @@ -53,20 +67,20 @@ All previous versions used pre-GA (Technology Preview) releases of Trustee: These scripts generate the cryptographic material and attestation measurements needed by Trustee and the peer-pod VMs. Run them once before your first deployment. -1. `bash scripts/gen-secrets.sh` — generates KBS key pairs, attestation policy seeds, and copies `values-secret.yaml.template` to `~/values-secret-coco-pattern.yaml` -2. `bash scripts/get-pcr.sh` — retrieves PCR measurements from the peer-pod VM image and stores them at `~/.coco-pattern/measurements.json` (requires `podman`, `skopeo`, and `~/pull-secret.json`) -3. Review and customise `~/values-secret-coco-pattern.yaml` — this file is loaded into Vault and provides secrets to the pattern +1. `bash scripts/gen-secrets.sh` — generates KBS key pairs, PCCS certificates/tokens (for bare metal), and copies `values-secret.yaml.template` to `~/values-secret-coco-pattern.yaml` +2. `bash scripts/get-pcr.sh` — retrieves PCR measurements from the peer-pod VM image and stores them at `~/.coco-pattern/measurements.json` (requires `podman`, `skopeo`, and `~/pull-secret.json`). **Not required for bare metal deployments.** +3. Review and customise `~/values-secret-coco-pattern.yaml` — this file is loaded into Vault and provides secrets to the pattern. For bare metal, uncomment the PCCS secrets section and provide your Intel PCS API key. > **Note:** `gen-secrets.sh` will not overwrite existing secrets. Delete `~/.coco-pattern/` if you need to regenerate. -### Single cluster deployment +### Single cluster deployment (Azure) 1. Set `main.clusterGroupName: simple` in `values-global.yaml` 2. Ensure your Azure configuration is populated in `values-global.yaml` (see `global.azure.*` fields) 3. `./pattern.sh make install` 4. Wait for the cluster to reboot all nodes (the sandboxed containers operator triggers a MachineConfig update). Monitor progress in the ArgoCD UI. -### Multi-cluster deployment +### Multi-cluster deployment (Azure) 1. Set `main.clusterGroupName: trusted-hub` in `values-global.yaml` 2. Deploy the hub cluster: `./pattern.sh make install` @@ -76,6 +90,25 @@ These scripts generate the cryptographic material and attestation measurements n (see [importing a cluster](https://validatedpatterns.io/learn/importing-a-cluster/)) 6. ACM will automatically deploy the `spoke` clusterGroup applications (sandboxed containers, workloads) to the imported cluster +### Bare metal deployment + +1. Set `main.clusterGroupName: baremetal` in `values-global.yaml` +2. Run `bash scripts/gen-secrets.sh` to generate KBS keys and PCCS secrets +3. For Intel TDX: uncomment the PCCS secrets in `~/values-secret-coco-pattern.yaml` and provide your Intel PCS API key +4. `./pattern.sh make install` +5. Wait for the cluster to reboot nodes (MachineConfig updates for TDX kernel parameters and vsock) + +The system auto-detects your hardware: + +- **NFD** discovers Intel TDX or AMD SEV-SNP capabilities and labels nodes +- **LVMS** auto-discovers available block devices for storage +- **RuntimeClass** `kata-cc` is created automatically pointing to the correct handler (`kata-tdx` or `kata-snp`) +- Both `kata-tdx` and `kata-snp` RuntimeClasses are deployed; only the one matching your hardware has schedulable nodes +- MachineConfigs are deployed for both `master` and `worker` roles (safe on SNO where only master exists) +- PCCS and QGS services deploy unconditionally; DaemonSets only schedule on Intel nodes via NFD labels + +Optional: pin PCCS to a specific node with `bash scripts/get-pccs-node.sh` and set `baremetal.pccs.nodeSelector` in the baremetal chart values. + ## Sample applications Two sample applications are deployed on the cluster running confidential workloads (the single cluster in `simple` mode, or the spoke in multi-cluster mode): diff --git a/ansible/detect-runtime-class.yaml b/ansible/detect-runtime-class.yaml new file mode 100644 index 00000000..a3618ecb --- /dev/null +++ b/ansible/detect-runtime-class.yaml @@ -0,0 +1,56 @@ +- name: Detect and configure runtime class + hosts: localhost + connection: local + gather_facts: false + tasks: + - name: Check for Intel TDX nodes + kubernetes.core.k8s_info: + api_version: v1 + kind: Node + label_selectors: + - intel.feature.node.kubernetes.io/tdx=true + register: tdx_nodes + + - name: Check for AMD SEV-SNP nodes + kubernetes.core.k8s_info: + api_version: v1 + kind: Node + label_selectors: + - amd.feature.node.kubernetes.io/snp=true + register: snp_nodes + + - name: Set runtime handler for Intel TDX + ansible.builtin.set_fact: + kata_handler: "kata-tdx" + kata_overhead: + memory: "350Mi" + cpu: "250m" + tdx.intel.com/keys: "1" + kata_node_selector: + intel.feature.node.kubernetes.io/tdx: "true" + when: tdx_nodes.resources | length > 0 + + - name: Set runtime handler for AMD SEV-SNP + ansible.builtin.set_fact: + kata_handler: "kata-snp" + kata_overhead: + memory: "350Mi" + cpu: "250m" + kata_node_selector: + amd.feature.node.kubernetes.io/snp: "true" + when: snp_nodes.resources | length > 0 + + - name: Create kata-cc RuntimeClass + kubernetes.core.k8s: + state: present + definition: + apiVersion: node.k8s.io/v1 + kind: RuntimeClass + metadata: + name: kata-cc + handler: "{{ kata_handler }}" + overhead: + podFixed: "{{ kata_overhead }}" + scheduling: + nodeSelector: "{{ kata_node_selector }}" + when: kata_handler is defined diff --git a/charts/all/baremetal/Chart.yaml b/charts/all/baremetal/Chart.yaml new file mode 100644 index 00000000..33940799 --- /dev/null +++ b/charts/all/baremetal/Chart.yaml @@ -0,0 +1,9 @@ +apiVersion: v2 +description: Bare metal platform configuration (NFD rules, MachineConfigs, RuntimeClasses, Intel device plugin). +keywords: +- pattern +- upstream +- sandbox +- baremetal +name: baremetal +version: 0.0.1 diff --git a/charts/all/baremetal/bm-kernel-params.yaml b/charts/all/baremetal/bm-kernel-params.yaml new file mode 100644 index 00000000..86f01791 --- /dev/null +++ b/charts/all/baremetal/bm-kernel-params.yaml @@ -0,0 +1,2 @@ +[hypervisor.qemu] +kernel_params="agent.aa_kbc_params=cc_kbc::http://kbs-trustee-operator-system.{{ .Values.global.hubClusterDomain }}" diff --git a/charts/all/baremetal/templates/kata-nfd.yaml b/charts/all/baremetal/templates/kata-nfd.yaml new file mode 100644 index 00000000..6196e02c --- /dev/null +++ b/charts/all/baremetal/templates/kata-nfd.yaml @@ -0,0 +1,80 @@ +apiVersion: nfd.openshift.io/v1alpha1 +kind: NodeFeatureRule +metadata: + name: consolidated-hardware-features + namespace: openshift-nfd +spec: + rules: + - name: "runtime.kata" + labels: + feature.node.kubernetes.io/runtime.kata: "true" + matchAny: + - matchFeatures: + - feature: cpu.cpuid + matchExpressions: + SSE42: { op: Exists } + VMX: { op: Exists } + - feature: kernel.loadedmodule + matchExpressions: + kvm: { op: Exists } + kvm_intel: { op: Exists } + - matchFeatures: + - feature: cpu.cpuid + matchExpressions: + SSE42: { op: Exists } + SVM: { op: Exists } + - feature: kernel.loadedmodule + matchExpressions: + kvm: { op: Exists } + kvm_amd: { op: Exists } + + - name: "amd.sev-snp" + labels: + amd.feature.node.kubernetes.io/snp: "true" + extendedResources: + sev-snp.amd.com/esids: "@cpu.security.sev.encrypted_state_ids" + matchFeatures: + - feature: cpu.cpuid + matchExpressions: + SVM: { op: Exists } + - feature: cpu.security + matchExpressions: + sev.snp.enabled: { op: Exists } + + - name: "intel.sgx" + labels: + intel.feature.node.kubernetes.io/sgx: "true" + extendedResources: + sgx.intel.com/epc: "@cpu.security.sgx.epc" + matchFeatures: + - feature: cpu.cpuid + matchExpressions: + SGX: { op: Exists } + SGXLC: { op: Exists } + - feature: cpu.security + matchExpressions: + sgx.enabled: { op: IsTrue } + - feature: kernel.config + matchExpressions: + X86_SGX: { op: Exists } + + - name: "intel.tdx" + labels: + intel.feature.node.kubernetes.io/tdx: "true" + extendedResources: + tdx.intel.com/keys: "@cpu.security.tdx.total_keys" + matchFeatures: + - feature: cpu.cpuid + matchExpressions: + VMX: { op: Exists } + - feature: cpu.security + matchExpressions: + tdx.enabled: { op: Exists } + + - name: "ibm.se.enabled" + labels: + ibm.feature.node.kubernetes.io/se: "true" + matchFeatures: + - feature: cpu.security + matchExpressions: + se.enabled: { op: IsTrue } diff --git a/charts/all/baremetal/templates/kernel-params-mco.yaml b/charts/all/baremetal/templates/kernel-params-mco.yaml new file mode 100644 index 00000000..1c372551 --- /dev/null +++ b/charts/all/baremetal/templates/kernel-params-mco.yaml @@ -0,0 +1,21 @@ +{{- range list "master" "worker" }} +--- +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + labels: + machineconfiguration.openshift.io/role: {{ . }} + name: 96-kata-kernel-config-{{ . }} + namespace: openshift-machine-config-operator +spec: + config: + ignition: + version: 3.2.0 + storage: + files: + - contents: + source: 'data:text/plain;charset=utf-8;base64,{{ tpl ($.Files.Get "bm-kernel-params.yaml") $ | b64enc }}' + mode: 420 + overwrite: true + path: /etc/kata-containers/snp/config.d/96-kata-kernel-config +{{- end }} diff --git a/charts/all/baremetal/templates/nfd-instance.yaml b/charts/all/baremetal/templates/nfd-instance.yaml new file mode 100644 index 00000000..97ce9ee1 --- /dev/null +++ b/charts/all/baremetal/templates/nfd-instance.yaml @@ -0,0 +1,12 @@ +apiVersion: nfd.openshift.io/v1 +kind: NodeFeatureDiscovery +metadata: + name: nfd-instance + namespace: openshift-nfd +spec: + operand: + image: registry.redhat.io/openshift4/ose-node-feature-discovery-rhel9:v4.20 + imagePullPolicy: Always + servicePort: 12000 + workerConfig: + configData: | diff --git a/charts/all/baremetal/templates/runtimeclass-amd-snp.yaml b/charts/all/baremetal/templates/runtimeclass-amd-snp.yaml new file mode 100644 index 00000000..c59be865 --- /dev/null +++ b/charts/all/baremetal/templates/runtimeclass-amd-snp.yaml @@ -0,0 +1,12 @@ +# apiVersion: node.k8s.io/v1 +# kind: RuntimeClass +# metadata: +# name: kata-snp +# handler: kata-snp +# overhead: +# podFixed: +# memory: "350Mi" +# cpu: "250m" +# scheduling: +# nodeSelector: +# amd.feature.node.kubernetes.io/snp: "true" diff --git a/charts/all/baremetal/templates/runtimeclass-intel-tdx.yaml b/charts/all/baremetal/templates/runtimeclass-intel-tdx.yaml new file mode 100644 index 00000000..f328dc0e --- /dev/null +++ b/charts/all/baremetal/templates/runtimeclass-intel-tdx.yaml @@ -0,0 +1,13 @@ +# apiVersion: node.k8s.io/v1 +# kind: RuntimeClass +# metadata: +# name: kata-tdx +# handler: kata-tdx +# overhead: +# podFixed: +# memory: "350Mi" +# cpu: "250m" +# tdx.intel.com/keys: 1 +# scheduling: +# nodeSelector: +# intel.feature.node.kubernetes.io/tdx: "true" diff --git a/charts/all/baremetal/templates/vsock-mco.yaml b/charts/all/baremetal/templates/vsock-mco.yaml new file mode 100644 index 00000000..7b1bbf7b --- /dev/null +++ b/charts/all/baremetal/templates/vsock-mco.yaml @@ -0,0 +1,21 @@ +{{- range list "master" "worker" }} +--- +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + labels: + machineconfiguration.openshift.io/role: {{ . }} + name: 99-enable-coco-{{ . }} +spec: + kernelArguments: + - nohibernate + config: + ignition: + version: 3.2.0 + storage: + files: + - path: /etc/modules-load.d/vsock.conf + mode: 0644 + contents: + source: data:text/plain;charset=utf-8;base64,dnNvY2stbG9vcGJhY2sK +{{- end }} diff --git a/charts/all/baremetal/values.yaml b/charts/all/baremetal/values.yaml new file mode 100644 index 00000000..b1960bfc --- /dev/null +++ b/charts/all/baremetal/values.yaml @@ -0,0 +1 @@ +# No configurable values — platform resources use fixed configurations. diff --git a/charts/all/intel-dcap/Chart.yaml b/charts/all/intel-dcap/Chart.yaml new file mode 100644 index 00000000..06095d7f --- /dev/null +++ b/charts/all/intel-dcap/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +description: Intel DCAP services (PCCS and QGS) for TDX remote attestation. +keywords: +- pattern +- intel +- tdx +- pccs +- qgs +name: intel-dcap +version: 0.0.1 diff --git a/charts/all/intel-dcap/templates/intel-dpo-sgx.yaml b/charts/all/intel-dcap/templates/intel-dpo-sgx.yaml new file mode 100644 index 00000000..2a7a8fca --- /dev/null +++ b/charts/all/intel-dcap/templates/intel-dpo-sgx.yaml @@ -0,0 +1,11 @@ +apiVersion: deviceplugin.intel.com/v1 +kind: SgxDevicePlugin +metadata: + name: sgxdeviceplugin-sample +spec: + image: registry.connect.redhat.com/intel/intel-sgx-plugin@sha256:f2c77521c6dae6b4db1896a5784ba8b06a5ebb2a01684184fc90143cfcca7bf4 + enclaveLimit: 110 + provisionLimit: 110 + logLevel: 4 + nodeSelector: + intel.feature.node.kubernetes.io/sgx: "true" diff --git a/charts/all/intel-dcap/templates/pccs-deployment.yaml b/charts/all/intel-dcap/templates/pccs-deployment.yaml new file mode 100644 index 00000000..9d5435e2 --- /dev/null +++ b/charts/all/intel-dcap/templates/pccs-deployment.yaml @@ -0,0 +1,69 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pccs + namespace: intel-dcap +spec: + replicas: 1 + selector: + matchLabels: + app: pccs + template: + metadata: + labels: + app: pccs + trustedservices.intel.com/cache: pccs + spec: + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + serviceAccountName: pccs-service-account + {{- if .Values.baremetal.pccs.nodeSelector }} + nodeSelector: + kubernetes.io/hostname: {{ .Values.baremetal.pccs.nodeSelector }} + {{- end }} + initContainers: + - name: init-seclabel + image: registry.access.redhat.com/ubi9/ubi:9.7-1764578509 + command: [ "sh", "-c", "chcon -Rt container_file_t /var/cache/pccs" ] + volumeMounts: + - name: host-database + mountPath: /var/cache/pccs + securityContext: + runAsUser: 0 + runAsGroup: 0 + privileged: true # Required for chcon to work on host files + containers: + - name: pccs + image: registry.redhat.io/openshift-sandboxed-containers/osc-pccs@sha256:de64fc7b13aaa7e466e825d62207f77e7c63a4f9da98663c3ab06abc45f2334d + envFrom: + - secretRef: + name: pccs-secrets + env: + - name: "PCCS_LOG_LEVEL" + value: "info" + - name: "CLUSTER_HTTPS_PROXY" + value: "" + - name: "PCCS_FILL_MODE" + value: "LAZY" + ports: + - containerPort: 8042 + name: pccs-port + volumeMounts: + - name: pccs-tls + mountPath: /opt/intel/pccs/ssl_key + readOnly: true + - name: host-database + mountPath: /var/cache/pccs/ + securityContext: + runAsUser: 0 + volumes: + - name: pccs-tls + secret: + secretName: pccs-tls + - name: host-database + hostPath: + path: /var/cache/pccs/ + type: DirectoryOrCreate diff --git a/charts/all/intel-dcap/templates/pccs-rbac.yaml b/charts/all/intel-dcap/templates/pccs-rbac.yaml new file mode 100644 index 00000000..2122caf0 --- /dev/null +++ b/charts/all/intel-dcap/templates/pccs-rbac.yaml @@ -0,0 +1,49 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: pccs-service-account + namespace: intel-dcap +--- +apiVersion: security.openshift.io/v1 +kind: SecurityContextConstraints +metadata: + name: pccs-scc + annotations: + kubernetes.io/description: "SCC for Intel DCAP PCCS service requiring privileged access and hostPath volumes" +allowHostDirVolumePlugin: true +allowHostIPC: false +allowHostNetwork: false +allowHostPID: false +allowHostPorts: false +allowPrivilegedContainer: true +allowedCapabilities: +- DAC_OVERRIDE +- SETGID +- SETUID +defaultAddCapabilities: null +fsGroup: + type: RunAsAny +priority: null +readOnlyRootFilesystem: false +requiredDropCapabilities: +- KILL +- MKNOD +- SETPCAP +- SYS_CHROOT +runAsUser: + type: RunAsAny +seLinuxContext: + type: MustRunAs +supplementalGroups: + type: RunAsAny +users: +- system:serviceaccount:intel-dcap:pccs-service-account +volumes: +- configMap +- downwardAPI +- emptyDir +- hostPath +- persistentVolumeClaim +- projected +- secret diff --git a/charts/all/intel-dcap/templates/pccs-secrets-eso.yaml b/charts/all/intel-dcap/templates/pccs-secrets-eso.yaml new file mode 100644 index 00000000..5ee91ab9 --- /dev/null +++ b/charts/all/intel-dcap/templates/pccs-secrets-eso.yaml @@ -0,0 +1,37 @@ +--- +apiVersion: "external-secrets.io/v1beta1" +kind: ExternalSecret +metadata: + name: pccs-secrets-eso + namespace: intel-dcap +spec: + refreshInterval: 15s + secretStoreRef: + name: {{ .Values.secretStore.name }} + kind: {{ .Values.secretStore.kind }} + target: + name: pccs-secrets + template: + type: Opaque + data: + PCCS_API_KEY: "{{ "{{ .api_key }}" }}" + PCCS_USER_TOKEN_HASH: "{{ "{{ .user_token_hash }}" }}" + USER_TOKEN: "{{ "{{ .user_token }}" }}" + PCCS_ADMIN_TOKEN_HASH: "{{ "{{ .admin_token_hash }}" }}" + data: + - secretKey: api_key + remoteRef: + key: 'secret/data/hub/pccs' + property: api_key + - secretKey: user_token_hash + remoteRef: + key: 'secret/data/hub/pccs' + property: user_token_hash + - secretKey: user_token + remoteRef: + key: 'secret/data/hub/pccs' + property: user_token + - secretKey: admin_token_hash + remoteRef: + key: 'secret/data/hub/pccs' + property: admin_token_hash diff --git a/charts/all/intel-dcap/templates/pccs-service.yaml b/charts/all/intel-dcap/templates/pccs-service.yaml new file mode 100644 index 00000000..bc83684c --- /dev/null +++ b/charts/all/intel-dcap/templates/pccs-service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: pccs-service + namespace: intel-dcap +spec: + selector: + trustedservices.intel.com/cache: pccs + ports: + - name: pccs + protocol: TCP + port: 8042 + targetPort: pccs-port diff --git a/charts/all/intel-dcap/templates/pccs-tls-eso.yaml b/charts/all/intel-dcap/templates/pccs-tls-eso.yaml new file mode 100644 index 00000000..a7212ae1 --- /dev/null +++ b/charts/all/intel-dcap/templates/pccs-tls-eso.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: "external-secrets.io/v1beta1" +kind: ExternalSecret +metadata: + name: pccs-tls-eso + namespace: intel-dcap +spec: + refreshInterval: 15s + secretStoreRef: + name: {{ .Values.secretStore.name }} + kind: {{ .Values.secretStore.kind }} + target: + name: pccs-tls + template: + type: Opaque + data: + - secretKey: private.pem + remoteRef: + key: 'secret/data/hub/pccs-tls' + property: private_key + - secretKey: file.crt + remoteRef: + key: 'secret/data/hub/pccs-tls' + property: certificate diff --git a/charts/all/intel-dcap/templates/qgs-config-cm.yaml b/charts/all/intel-dcap/templates/qgs-config-cm.yaml new file mode 100644 index 00000000..5745adeb --- /dev/null +++ b/charts/all/intel-dcap/templates/qgs-config-cm.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: qgs-config + namespace: intel-dcap +data: + qgs.conf: | + port = 4050 + number_threads = 4 diff --git a/charts/all/intel-dcap/templates/qgs-ds.yaml b/charts/all/intel-dcap/templates/qgs-ds.yaml new file mode 100644 index 00000000..788e769c --- /dev/null +++ b/charts/all/intel-dcap/templates/qgs-ds.yaml @@ -0,0 +1,88 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: tdx-qgs + namespace: intel-dcap +spec: + selector: + matchLabels: + app: tdx-qgs + template: + metadata: + labels: + app: tdx-qgs + annotations: + sgx.intel.com/quote-provider: tdx-qgs + qcnl-conf: '{"pccs_url": "https://pccs-service:8042/sgx/certification/v4/", "use_secure_cert": false, "pck_cache_expire_hours": 168}' + spec: + serviceAccountName: tdx-qgs-service-account + nodeSelector: + intel.feature.node.kubernetes.io/tdx: 'true' + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + initContainers: + - name: platform-registration + image: registry.redhat.io/openshift-sandboxed-containers/osc-tdx-qgs@sha256:86b23461c4eea073f4535a777374a54e934c37ac8c96c6180030f92ebf970524 + restartPolicy: Always + command: [ '/usr/bin/dcap-registration-flow' ] + env: + - name: PCCS_URL + value: "https://pccs-service:8042" + - name: SECURE_CERT + value: 'false' + envFrom: + - secretRef: + name: pccs-secrets + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: true + privileged: true + capabilities: + drop: + - ALL + add: + - LINUX_IMMUTABLE + volumeMounts: + - name: efivars + mountPath: /sys/firmware/efi/efivars + containers: + - name: tdx-qgs + image: registry.redhat.io/openshift-sandboxed-containers/osc-tdx-qgs@sha256:86b23461c4eea073f4535a777374a54e934c37ac8c96c6180030f92ebf970524 + args: + - -p=4050 + - -n=4 + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + resources: + limits: + sgx.intel.com/epc: "512Ki" + sgx.intel.com/enclave: 1 + sgx.intel.com/provision: 1 + env: + - name: QCNL_CONF_PATH + value: "/run/dcap/qcnl_conf" + - name: XDG_CACHE_HOME + value: "/run/dcap/cache" + volumeMounts: + - name: dcap-qcnl-cache + mountPath: /run/dcap/cache + - name: qcnl-config + mountPath: /run/dcap/ + readOnly: true + volumes: + - name: dcap-qcnl-cache + emptyDir: + sizeLimit: 50Mi + - name: qcnl-config + downwardAPI: + items: + - path: "qcnl_conf" + fieldRef: + fieldPath: metadata.annotations['qcnl-conf'] + - name: efivars + hostPath: + path: /sys/firmware/efi/efivars/ diff --git a/charts/all/intel-dcap/templates/qgs-rbac.yaml b/charts/all/intel-dcap/templates/qgs-rbac.yaml new file mode 100644 index 00000000..2dcb591e --- /dev/null +++ b/charts/all/intel-dcap/templates/qgs-rbac.yaml @@ -0,0 +1,47 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: tdx-qgs-service-account + namespace: intel-dcap +--- +apiVersion: security.openshift.io/v1 +kind: SecurityContextConstraints +metadata: + name: tdx-qgs-scc + annotations: + kubernetes.io/description: "SCC for Intel TDX Quote Generation Service requiring host network access and SGX devices" +allowHostDirVolumePlugin: true +allowHostIPC: false +allowHostNetwork: true +allowHostPID: false +allowHostPorts: false +allowPrivilegedContainer: true +allowedCapabilities: +- LINUX_IMMUTABLE +defaultAddCapabilities: null +fsGroup: + type: RunAsAny +priority: null +readOnlyRootFilesystem: false +requiredDropCapabilities: +- KILL +- MKNOD +- SETPCAP +- SYS_CHROOT +runAsUser: + type: RunAsAny +seLinuxContext: + type: MustRunAs +supplementalGroups: + type: RunAsAny +users: +- system:serviceaccount:intel-dcap:tdx-qgs-service-account +volumes: +- configMap +- downwardAPI +- emptyDir +- hostPath +- persistentVolumeClaim +- projected +- secret diff --git a/charts/all/intel-dcap/templates/qgs-sgx-cm.yaml b/charts/all/intel-dcap/templates/qgs-sgx-cm.yaml new file mode 100644 index 00000000..b715a010 --- /dev/null +++ b/charts/all/intel-dcap/templates/qgs-sgx-cm.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: sgx-default-qcnl-conf + namespace: intel-dcap +data: + sgx_default_qcnl.conf: | + { + "pccs_url": "https://pccs-service:8042/sgx/certification/v4/", + "use_secure_cert": false, + "retry_times": 6, + "retry_delay": 10, + "pck_cache_expire_hours": 168, + "verify_collateral_cache_expire_hours": 168, + "local_cache_only": false + } diff --git a/charts/all/intel-dcap/values.yaml b/charts/all/intel-dcap/values.yaml new file mode 100644 index 00000000..5910caed --- /dev/null +++ b/charts/all/intel-dcap/values.yaml @@ -0,0 +1,7 @@ +baremetal: + pccs: + nodeSelector: "" # optional: hostname for PCCS pinning. Empty = schedule anywhere with tolerations. + +secretStore: + name: "" + kind: "" diff --git a/charts/coco-supported/hello-openshift/templates/_helpers.tpl b/charts/coco-supported/hello-openshift/templates/_helpers.tpl index 6082b80a..ab49aab2 100644 --- a/charts/coco-supported/hello-openshift/templates/_helpers.tpl +++ b/charts/coco-supported/hello-openshift/templates/_helpers.tpl @@ -51,11 +51,14 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} {{/* -Determine runtime class name based on cluster platform -Returns "kata-remote" for Azure/AWS, "kata-cc" for other platforms +Determine runtime class name. +If runtimeClassName is explicitly set, use it. +Otherwise, detect from cluster platform: "kata-remote" for Azure/AWS, "kata-cc" for other platforms. */}} {{- define "hello-openshift.runtimeClassName" -}} -{{- if or (eq .Values.global.clusterPlatform "Azure") (eq .Values.global.clusterPlatform "AWS") -}} +{{- if .Values.runtimeClassName -}} +{{- .Values.runtimeClassName -}} +{{- else if or (eq .Values.global.clusterPlatform "Azure") (eq .Values.global.clusterPlatform "AWS") -}} kata-remote {{- else -}} kata-cc diff --git a/charts/coco-supported/hello-openshift/values.yaml b/charts/coco-supported/hello-openshift/values.yaml index c9dbbe0a..39f2b9ff 100644 --- a/charts/coco-supported/hello-openshift/values.yaml +++ b/charts/coco-supported/hello-openshift/values.yaml @@ -1,6 +1,11 @@ # Chart-specific values # Common values are inherited from values-global.yaml +# Runtime class for confidential containers. +# When empty, auto-detected from global.clusterPlatform (kata-remote for Azure/AWS, kata-cc otherwise). +# Bare metal: set to "kata-cc" via values-baremetal.yaml overrides. +runtimeClassName: "" + # Global values used by this chart (overridden by values-global.yaml) global: clusterPlatform: "" # Cluster platform: "Azure" or "AWS" - determines runtime class diff --git a/charts/coco-supported/kbs-access/templates/secure-pod.yaml b/charts/coco-supported/kbs-access/templates/secure-pod.yaml index 663408bd..bf5d33f8 100644 --- a/charts/coco-supported/kbs-access/templates/secure-pod.yaml +++ b/charts/coco-supported/kbs-access/templates/secure-pod.yaml @@ -7,7 +7,7 @@ metadata: annotations: peerpods: "true" spec: - runtimeClassName: kata-remote + runtimeClassName: {{ .Values.runtimeClassName }} containers: - name: python-access image: ghcr.io/butler54/kbs-access-app:latest diff --git a/charts/coco-supported/kbs-access/values.yaml b/charts/coco-supported/kbs-access/values.yaml index fdaa4d74..975ee21a 100644 --- a/charts/coco-supported/kbs-access/values.yaml +++ b/charts/coco-supported/kbs-access/values.yaml @@ -1,6 +1,11 @@ # Chart-specific values # Common values are inherited from values-global.yaml +# Runtime class for confidential containers. +# Azure/AWS peer-pods: kata-remote (default) +# Bare metal: kata-cc (set via values-baremetal.yaml overrides) +runtimeClassName: "kata-remote" + # Global values used by this chart (overridden by values-global.yaml) global: coco: diff --git a/charts/hub/storage/Chart.yaml b/charts/hub/storage/Chart.yaml new file mode 100644 index 00000000..55383ab4 --- /dev/null +++ b/charts/hub/storage/Chart.yaml @@ -0,0 +1,9 @@ +apiVersion: v2 +description: Deploy and configure storage providers (HPP/LVM) for baremetal clusters +keywords: +- pattern +- storage +- hpp +- lvm +name: storage +version: 0.0.2 diff --git a/charts/hub/storage/templates/hostpathprovisioner.yaml b/charts/hub/storage/templates/hostpathprovisioner.yaml new file mode 100644 index 00000000..909e24e5 --- /dev/null +++ b/charts/hub/storage/templates/hostpathprovisioner.yaml @@ -0,0 +1,11 @@ +{{- if eq .Values.global.storageProvider "hpp" }} +apiVersion: hostpathprovisioner.kubevirt.io/v1beta1 +kind: HostPathProvisioner +metadata: + name: hostpath-provisioner +spec: + imagePullPolicy: IfNotPresent + storagePools: + - name: local + path: {{ .Values.hpp.storagePools.path | default "/var/hpvolumes" }} +{{- end }} diff --git a/charts/hub/storage/templates/hpp-storageclass.yaml b/charts/hub/storage/templates/hpp-storageclass.yaml new file mode 100644 index 00000000..7c86c889 --- /dev/null +++ b/charts/hub/storage/templates/hpp-storageclass.yaml @@ -0,0 +1,13 @@ +{{- if eq .Values.global.storageProvider "hpp" }} +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ .Values.hpp.storageClass.name | default "hostpath-csi" }} + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: kubevirt.io.hostpath-provisioner +reclaimPolicy: {{ .Values.hpp.storageClass.reclaimPolicy | default "Delete" }} +volumeBindingMode: {{ .Values.hpp.storageClass.volumeBindingMode | default "WaitForFirstConsumer" }} +parameters: + storagePool: local +{{- end }} diff --git a/charts/hub/storage/templates/lvmcluster.yaml b/charts/hub/storage/templates/lvmcluster.yaml new file mode 100644 index 00000000..3a7724bb --- /dev/null +++ b/charts/hub/storage/templates/lvmcluster.yaml @@ -0,0 +1,15 @@ +{{- if eq .Values.global.storageProvider "lvm" }} +apiVersion: lvm.topolvm.io/v1alpha1 +kind: LVMCluster +metadata: + name: {{ .Values.lvmCluster.name | default "lvmcluster" }} + namespace: openshift-storage +spec: + storage: + deviceClasses: + - name: {{ .Values.lvmCluster.deviceClass | default "vg1" }} + thinPoolConfig: + name: thin-pool-1 + sizePercent: 90 + overprovisionRatio: 10 +{{- end }} diff --git a/charts/hub/storage/values.yaml b/charts/hub/storage/values.yaml new file mode 100644 index 00000000..705f5233 --- /dev/null +++ b/charts/hub/storage/values.yaml @@ -0,0 +1,14 @@ +global: + storageProvider: hpp + +lvmCluster: + name: "lvmcluster" + deviceClass: "vg1" + +hpp: + storagePools: + path: /var/hpvolumes + storageClass: + name: hostpath-csi + reclaimPolicy: Delete + volumeBindingMode: WaitForFirstConsumer diff --git a/docs/dell-tdx-configuration.md b/docs/dell-tdx-configuration.md new file mode 100644 index 00000000..ecf078fd --- /dev/null +++ b/docs/dell-tdx-configuration.md @@ -0,0 +1,149 @@ +# Enable Intel TDX on Dell PowerEdge via iDRAC + +This guide provides step-by-step instructions for enabling Intel Trust Domain Extensions (TDX) on Dell PowerEdge servers using the iDRAC console. + +## Prerequisites + +- Dell 16th Generation PowerEdge server: + - PowerEdge R660, R660xs + - PowerEdge R760, R760xs, R760xd2, R760XA + - PowerEdge R860, R960 + - PowerEdge XE8640, XE9640, XE9680 + - PowerEdge C6620, MX760c + - PowerEdge XR5610, XR7620, XR8610t, XR8620t + - PowerEdge T360, T560 +- 5th Gen Intel Xeon Scalable processor with TDX support +- **8 or 16 DIMMs per socket** (required memory configuration) +- Latest BIOS firmware installed + +## Step-by-Step Instructions (Order Matters) + +> **IMPORTANT:** Settings must be configured in this exact order. Some options (like "Multiple Keys") will be greyed out until prerequisite settings are applied. You may need to **save and reboot between steps** for dependent options to become available. + +### 1. Access BIOS Setup via iDRAC + +1. Log into the iDRAC web console +2. Navigate to **Configuration → BIOS Settings** +3. Alternatively, launch **Virtual Console** and press **F2** during POST to enter System Setup + +### 2. Configure Memory Settings (FIRST) + +Navigate to: **System BIOS → Memory Settings** + +| Setting | Value | +| ---------------------- | ----------- | +| **Node Interleaving** | Disabled | + +**Save and reboot** before proceeding. + +### 3. Configure Processor Prerequisites (SECOND) + +Navigate to: **System BIOS → Processor Settings** + +| Setting | Value | +| -------------------------------- | -------- | +| **Logical Processor (x2APIC)** | Enabled | +| **CPU Physical Address Limit** | Disabled | + +**Save and reboot** before proceeding. + +### 4. Enable Memory Encryption - Multiple Keys (THIRD) + +Navigate to: **System BIOS → System Security** + +| Setting | Value | +| --------------------- | -------------- | +| **Memory Encryption** | Multiple Keys | + +> If "Multiple Keys" is still greyed out, verify steps 2 and 3 were applied and the system was rebooted. + +**Save and reboot** before proceeding. + +### 5. Configure TDX Settings (FOURTH) + +Navigate to: **System BIOS → System Security** (or **Processor Settings** depending on BIOS version) + +| Setting | Value | +| ------------------------------------------------- | ------- | +| **Global Memory Integrity** | Disabled | +| **Intel TDX (Trust Domain Extension)** | Enabled | +| **TME-MT/TDX Key Split** | 1 | +| **TDX Secure Arbitration Mode Loader (SEAM)** | Enabled | + +### 6. Configure SGX Settings (FIFTH) + +Navigate to: **System BIOS → Processor Settings → Software Guard Extensions (SGX)** + +| Setting | Value | +| -------------------- | ------------------------- | +| **Intel SGX** | Enabled | +| **SGX Factory Reset** | Off | +| **SGX PRMRR Size** | As needed (e.g., 64GB) | + +### 7. Final Save and Reboot + +1. Press **Escape** to exit menus +2. Select **Save Changes and Exit** +3. System will reboot with TDX enabled + +## Configuration Summary (Order of Operations) + +```text +1. Disable Node Interleaving → Save & Reboot +2. Enable x2APIC Mode → Save & Reboot +3. Disable CPU Physical Address Limit → Save & Reboot +4. Set Memory Encryption = Multiple Keys → Save & Reboot +5. Disable Global Memory Integrity +6. Enable Intel TDX +7. Set TME-MT/TDX Key Split = 1 +8. Enable SEAM Loader +9. Enable Intel SGX → Final Save & Reboot +``` + +## Verification + +After the OS boots, verify TDX is enabled: + +```bash +# Check kernel messages for TDX +dmesg | grep -i tdx +# Should show: "virt/tdx: BIOS enabled: private KeyID range: [X, Y)" + +# Check for TDX module +ls /sys/firmware/tdx_seam/ +``` + +## Troubleshooting + +### "Multiple Keys" Option is Greyed Out + +This is typically caused by: + +1. **Node Interleaving is Enabled** - Must be disabled first +2. **x2APIC Mode is Disabled** - Must be enabled first +3. **CPU Physical Address Limit is Enabled** - Must be disabled first +4. **System not rebooted** - Some changes require reboot before dependent options appear +5. **Insufficient DIMMs** - Requires 8 or 16 DIMMs per socket + +### Settings Not Available + +If TDX-related settings are not visible: + +1. Ensure BIOS firmware is updated to the latest version +2. Verify your processor supports TDX (5th Gen Xeon Scalable required) +3. Contact Dell support for BIOS with TDX support + +### TDX Not Detected by OS + +If the OS doesn't detect TDX after configuration: + +1. Verify all settings are correctly applied in the order specified +2. Ensure the OS/kernel supports TDX (Linux 6.2+ recommended) +3. Check that Memory Encryption is set to "Multiple Keys" (not "Single Key") + +## References + +- [Dell: Enable Intel TDX on Dell 16G Intel Servers](https://www.dell.com/support/kbdoc/en-us/000226452/enableinteltdxondell16g) +- [Intel TDX Enabling Guide - Hardware Setup](https://cc-enabling.trustedservices.intel.com/intel-tdx-enabling-guide/04/hardware_setup/) +- [Dell Info Hub: Enable Intel TDX in BIOS](https://infohub.delltechnologies.com/en-us/l/securing-ai-workloads-on-dell-poweredge-with-intel-xeon-processors-using-intel-trust-domain-extensions/appendix-b-enable-intel-r-tdx-in-bios/) +- [Linux Kernel TDX Documentation](https://docs.kernel.org/arch/x86/tdx.html) diff --git a/overrides/values-BareMetal.yaml b/overrides/values-BareMetal.yaml new file mode 100644 index 00000000..59cb0ab8 --- /dev/null +++ b/overrides/values-BareMetal.yaml @@ -0,0 +1,2 @@ +# Bare metal platform overrides. +# Storage-specific values moved to overrides/values-storage-*.yaml. diff --git a/overrides/values-None.yaml b/overrides/values-None.yaml new file mode 100644 index 00000000..172b55e3 --- /dev/null +++ b/overrides/values-None.yaml @@ -0,0 +1,2 @@ +# None platform overrides. +# Storage-specific values moved to overrides/values-storage-*.yaml. diff --git a/overrides/values-storage-external.yaml b/overrides/values-storage-external.yaml new file mode 100644 index 00000000..72b69f73 --- /dev/null +++ b/overrides/values-storage-external.yaml @@ -0,0 +1,2 @@ +# External storage: uses cluster default StorageClass. +# No vault storageClass override — vault uses whatever default exists. diff --git a/overrides/values-storage-hpp.yaml b/overrides/values-storage-hpp.yaml new file mode 100644 index 00000000..1bd94b8b --- /dev/null +++ b/overrides/values-storage-hpp.yaml @@ -0,0 +1,8 @@ +vault: + server: + dataStorage: + storageClass: hostpath-csi + +global: + objectStorage: + backingStorageClass: "hostpath-csi" diff --git a/overrides/values-storage-lvm.yaml b/overrides/values-storage-lvm.yaml new file mode 100644 index 00000000..3c49baed --- /dev/null +++ b/overrides/values-storage-lvm.yaml @@ -0,0 +1,8 @@ +vault: + server: + dataStorage: + storageClass: lvms-vg1 + +global: + objectStorage: + backingStorageClass: "lvms-vg1" diff --git a/scripts/gen-secrets.sh b/scripts/gen-secrets.sh index c487bcac..902d52a2 100755 --- a/scripts/gen-secrets.sh +++ b/scripts/gen-secrets.sh @@ -28,6 +28,32 @@ if [ ! -f "${KBS_PRIVATE_KEY}" ]; then openssl pkey -in "${KBS_PRIVATE_KEY}" -pubout -out "${KBS_PUBLIC_KEY}" fi +## PCCS secrets for bare metal Intel TDX deployments +PCCS_PRIVATE_KEY="${COCO_SECRETS_DIR}/pccs_private.pem" +PCCS_CERTIFICATE="${COCO_SECRETS_DIR}/pccs_certificate.pem" +PCCS_USER_TOKEN_FILE="${COCO_SECRETS_DIR}/pccs_user_token" +PCCS_ADMIN_TOKEN_FILE="${COCO_SECRETS_DIR}/pccs_admin_token" + +if [ ! -f "${PCCS_PRIVATE_KEY}" ]; then + echo "Creating PCCS TLS certificate" + openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 \ + -keyout "${PCCS_PRIVATE_KEY}" \ + -out "${PCCS_CERTIFICATE}" \ + -subj "/C=US/ST=Denial/L=Springfield/O=Dis/CN=pccs-service.intel-dcap.svc.cluster.local" +fi + +if [ ! -f "${PCCS_USER_TOKEN_FILE}" ]; then + echo "Creating PCCS user token" + echo "usertoken" > "${PCCS_USER_TOKEN_FILE}" +fi +echo -n "usertoken" | sha512sum | tr -d '[:space:]-' > "${COCO_SECRETS_DIR}/pccs_user_token_hash" + +if [ ! -f "${PCCS_ADMIN_TOKEN_FILE}" ]; then + echo "Creating PCCS admin token" + echo "admintoken" > "${PCCS_ADMIN_TOKEN_FILE}" +fi +echo -n "admintoken" | sha512sum | tr -d '[:space:]-' > "${COCO_SECRETS_DIR}/pccs_admin_token_hash" + ## Copy a sample values file if this stuff doesn't exist if [ ! -f "${VALUES_FILE}" ]; then diff --git a/scripts/get-pccs-node.sh b/scripts/get-pccs-node.sh new file mode 100755 index 00000000..32b4313b --- /dev/null +++ b/scripts/get-pccs-node.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Detects a node with Intel TDX support for PCCS deployment. +# Usage: bash scripts/get-pccs-node.sh +NODE=$(oc get nodes -l intel.feature.node.kubernetes.io/tdx=true \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) +if [ -z "$NODE" ]; then + echo "ERROR: No TDX-capable nodes found" >&2 + exit 1 +fi +echo "$NODE" diff --git a/values-baremetal.yaml b/values-baremetal.yaml new file mode 100644 index 00000000..330f0f12 --- /dev/null +++ b/values-baremetal.yaml @@ -0,0 +1,200 @@ +# Bare metal deployment for confidential containers. +# Supports Intel TDX and AMD SEV-SNP via auto-detection (NFD). +# Set main.clusterGroupName: baremetal in values-global.yaml to use. + +clusterGroup: + name: baremetal + isHubCluster: true + namespaces: + - open-cluster-management + - vault + - golang-external-secrets + - openshift-sandboxed-containers-operator + - trustee-operator-system + - cert-manager-operator + - cert-manager + - hello-openshift + - kbs-access + - openshift-cnv + - openshift-storage + - openshift-nfd + - baremetal + + subscriptions: + acm: + name: advanced-cluster-management + namespace: open-cluster-management + sandbox: + name: sandboxed-containers-operator + namespace: openshift-sandboxed-containers-operator + source: redhat-operators + channel: stable + installPlanApproval: Manual + csv: sandboxed-containers-operator.v1.11.0 + trustee: + name: trustee-operator + namespace: trustee-operator-system + source: redhat-operators + channel: stable + installPlanApproval: Manual + csv: trustee-operator.v1.0.0 + cert-manager: + name: openshift-cert-manager-operator + namespace: cert-manager-operator + channel: stable-v1 + lvm-operator: + name: lvms-operator + namespace: openshift-storage + source: redhat-operators + channel: stable-4.20 + installPlanApproval: Automatic + cnv: + name: kubevirt-hyperconverged + namespace: openshift-cnv + source: redhat-operators + channel: stable + installPlanApproval: Automatic + nfd: + name: nfd + namespace: openshift-nfd + channel: stable + projects: + - hub + - vault + - trustee + - golang-external-secrets + - sandbox + - workloads + - default + + # Explicitly mention the cluster-state based overrides we plan to use for this pattern. + # We can use self-referential variables because the chart calls the tpl function with these variables defined + sharedValueFiles: + - '/overrides/values-{{ $.Values.global.clusterPlatform }}.yaml' + - '/overrides/values-storage-{{ $.Values.global.storageProvider }}.yaml' + + applications: + acm: + name: acm + namespace: open-cluster-management + project: hub + chart: acm + chartVersion: 0.1.* + + vault: + name: vault + namespace: vault + project: vault + chart: hashicorp-vault + chartVersion: 0.1.* + + secrets-operator: + name: golang-external-secrets + namespace: golang-external-secrets + project: golang-external-secrets + chart: golang-external-secrets + chartVersion: 0.1.* + + trustee: + name: trustee + namespace: trustee-operator-system + project: trustee + repoURL: https://github.com/butler54/trustee-chart.git + targetRevision: merge-certs + path: ./ + overrides: + - name: global.coco.secured + value: "true" + - name: global.coco.bypassAttestation + value: "true" + - name: kbs.https.enabled + value: "false" + - name: kbs.secretResources[0].name + value: kbsres1 + - name: kbs.secretResources[0].key + value: secret/data/hub/kbsres1 + - name: kbs.secretResources[1].name + value: passphrase + - name: kbs.secretResources[1].key + value: secret/data/hub/passphrase + + storage: + name: storage + namespace: openshift-storage + project: hub + path: charts/hub/storage + + baremetal: + name: baremetal + namespace: baremetal + project: hub + path: charts/all/baremetal + + sandbox: + name: sandbox + namespace: openshift-sandboxed-containers-operator + project: sandbox + repoURL: https://github.com/butler54/sandboxed-containers-chart.git + targetRevision: remove-ssh + path: ./ + overrides: + - name: global.secretStore.backend + value: vault + - name: secretStore.name + value: vault-backend + - name: secretStore.kind + value: ClusterSecretStore + - name: enablePeerPods + value: "false" + + + sandbox-policies: + name: sandbox-policies + namespace: openshift-sandboxed-containers-operator + repoURL: https://github.com/butler54/sandboxed-policies-chart.git + targetRevision: rootvolume + path: ./ + + kbs-access: + name: kbs-access + namespace: kbs-access + project: workloads + path: charts/coco-supported/kbs-access + overrides: + - name: runtimeClassName + value: "kata-cc" + + hello-openshift: + name: hello-openshift + namespace: hello-openshift + project: workloads + path: charts/coco-supported/hello-openshift + overrides: + - name: runtimeClassName + value: "kata-cc" + + imperative: + # NOTE: We *must* use lists and not hashes. As hashes lose ordering once parsed by helm + # The default schedule is every 10 minutes: imperative.schedule + # Total timeout of all jobs is 1h: imperative.activeDeadlineSeconds + # imagePullPolicy is set to always: imperative.imagePullPolicy + # For additional overrides that apply to the jobs, please refer to + # https://validatedpatterns.io/imperative-actions/#additional-job-customizations + image: ghcr.io/butler54/imperative-container:latest + serviceAccountCreate: true + adminServiceAccountCreate: true + serviceAccountName: imperative-admin-sa + jobs: + - name: install-deps + playbook: ansible/install-deps.yaml + verbosity: -vvv + timeout: 3600 + - name: init-data-gzipper + playbook: ansible/init-data-gzipper.yaml + verbosity: -vvv + timeout: 3600 + # Required for tech preview only. + # - name: detect-runtime-class + # playbook: ansible/detect-runtime-class.yaml + # verbosity: -vvv + # timeout: 600 diff --git a/values-global.yaml b/values-global.yaml index d7c202a3..9e83b312 100644 --- a/values-global.yaml +++ b/values-global.yaml @@ -1,5 +1,6 @@ global: pattern: coco-pattern + storageProvider: hpp # Options: hpp, lvm, external secretStore: # Warning: This must be present even if it is set to none. backend: vault # none, vault, kubernetes @@ -13,6 +14,7 @@ global: coco: securityPolicyFlavour: "insecure" # insecure, signed or reject is expected. secured: true # true or false. If true, the cluster will be secured. If false, the cluster will be insecure. + bypassAttestation: false # Set true for bare metal (skips PCR/initdata/RVPS) # Enable SSH key injection into podvm for debugging. Do not enable in production. # Also requires: COCO_ENABLE_SSH_DEBUG=true ./scripts/gen-secrets.sh # and uncommenting the sshKey block in values-secret.yaml.template. @@ -24,7 +26,7 @@ main: # WARNING # This default configuration uses a single cluster on azure. # It fundamentally violates the separation of duties. - clusterGroupName: simple + clusterGroupName: baremetal multiSourceConfig: enabled: true clusterGroupChartVersion: 0.9.* diff --git a/values-secret.yaml.template b/values-secret.yaml.template index 4ed9d158..c4874d6b 100644 --- a/values-secret.yaml.template +++ b/values-secret.yaml.template @@ -126,3 +126,39 @@ secrets: onMissingValue: generate vaultPolicy: validatedPatternDefaultPolicy + # PCCS secrets for bare metal Intel TDX deployments. + # Uncomment these sections for bare metal deployments. + # Run ./scripts/gen-secrets.sh first to generate tokens and certificates. + # You must provide your Intel PCS API key in the api_key field. + # Get an API key from: https://api.portal.trustedservices.intel.com/ + #- name: pccs + # vaultPrefixes: + # - hub + # fields: + # - name: api_key + # value: '' + # - name: user_token_hash + # path: ~/.coco-pattern/pccs_user_token_hash + # - name: user_token + # path: ~/.coco-pattern/pccs_user_token + # - name: admin_token_hash + # path: ~/.coco-pattern/pccs_admin_token_hash + # - name: admin_token + # path: ~/.coco-pattern/pccs_admin_token + # - name: db_username + # value: '' + # onMissingValue: generate + # vaultPolicy: validatedPatternDefaultPolicy + # - name: db_password + # value: '' + # onMissingValue: generate + # vaultPolicy: validatedPatternDefaultPolicy + #- name: pccs-tls + # vaultPrefixes: + # - hub + # fields: + # - name: private_key + # path: ~/.coco-pattern/pccs_private.pem + # - name: certificate + # path: ~/.coco-pattern/pccs_certificate.pem + From 95fbf78aee8130f944f0f77222d88ebc56eb2a62 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Tue, 10 Mar 2026 15:01:23 +0900 Subject: [PATCH 02/24] feat: update baremetal values to use released charts Replace git branch references (repoURL/targetRevision/path) with released Helm chart references (chart/chartVersion) for trustee, sandboxed-containers, and sandboxed-policies in values-baremetal.yaml. Co-Authored-By: Claude Opus 4.6 (1M context) --- values-baremetal.yaml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/values-baremetal.yaml b/values-baremetal.yaml index 330f0f12..63446f5a 100644 --- a/values-baremetal.yaml +++ b/values-baremetal.yaml @@ -99,9 +99,8 @@ clusterGroup: name: trustee namespace: trustee-operator-system project: trustee - repoURL: https://github.com/butler54/trustee-chart.git - targetRevision: merge-certs - path: ./ + chart: trustee + chartVersion: 0.2.* overrides: - name: global.coco.secured value: "true" @@ -134,9 +133,8 @@ clusterGroup: name: sandbox namespace: openshift-sandboxed-containers-operator project: sandbox - repoURL: https://github.com/butler54/sandboxed-containers-chart.git - targetRevision: remove-ssh - path: ./ + chart: sandboxed-containers + chartVersion: 0.2.* overrides: - name: global.secretStore.backend value: vault @@ -151,9 +149,8 @@ clusterGroup: sandbox-policies: name: sandbox-policies namespace: openshift-sandboxed-containers-operator - repoURL: https://github.com/butler54/sandboxed-policies-chart.git - targetRevision: rootvolume - path: ./ + chart: sandboxed-policies + chartVersion: 0.1.* kbs-access: name: kbs-access From 708c94c3cab8a8bec3308b3f55b4144f5c705100 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Tue, 10 Mar 2026 19:02:16 +0900 Subject: [PATCH 03/24] feat: add TDX kernel flag and enable intel-dcap for baremetal Add tdx.enabled flag (default true) to baremetal chart to conditionally set kvm_intel.tdx=1 kernel argument. Without this, the kvm_intel module does not activate TDX and NFD cannot detect it. Enable intel-dcap application in values-baremetal.yaml for PCCS/QGS attestation services. Co-Authored-By: Claude Opus 4.6 (1M context) --- charts/all/baremetal/templates/vsock-mco.yaml | 3 +++ charts/all/baremetal/values.yaml | 3 ++- values-baremetal.yaml | 12 ++++++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/charts/all/baremetal/templates/vsock-mco.yaml b/charts/all/baremetal/templates/vsock-mco.yaml index 7b1bbf7b..f8938f62 100644 --- a/charts/all/baremetal/templates/vsock-mco.yaml +++ b/charts/all/baremetal/templates/vsock-mco.yaml @@ -9,6 +9,9 @@ metadata: spec: kernelArguments: - nohibernate +{{- if $.Values.tdx.enabled }} + - kvm_intel.tdx=1 +{{- end }} config: ignition: version: 3.2.0 diff --git a/charts/all/baremetal/values.yaml b/charts/all/baremetal/values.yaml index b1960bfc..3942cb97 100644 --- a/charts/all/baremetal/values.yaml +++ b/charts/all/baremetal/values.yaml @@ -1 +1,2 @@ -# No configurable values — platform resources use fixed configurations. +tdx: + enabled: true diff --git a/values-baremetal.yaml b/values-baremetal.yaml index 63446f5a..2a2980e8 100644 --- a/values-baremetal.yaml +++ b/values-baremetal.yaml @@ -19,6 +19,7 @@ clusterGroup: - openshift-storage - openshift-nfd - baremetal + - intel-dcap subscriptions: acm: @@ -146,6 +147,17 @@ clusterGroup: value: "false" + intel-dcap: + name: intel-dcap + namespace: intel-dcap + project: hub + path: charts/all/intel-dcap + overrides: + - name: secretStore.name + value: vault-backend + - name: secretStore.kind + value: ClusterSecretStore + sandbox-policies: name: sandbox-policies namespace: openshift-sandboxed-containers-operator From 6dadcca1172468da4ff97a67fca3868ab7cbaca0 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Wed, 18 Mar 2026 16:26:20 +1100 Subject: [PATCH 04/24] fix: remove unused runtime class, kernel params, and commented-out templates Address PR review feedback: - Remove detect-runtime-class.yaml (OSC operator manages RuntimeClass) - Remove bm-kernel-params.yaml and kernel-params-mco.yaml (config should be provided via initdata or pod annotations to avoid inconsistencies) - Remove commented-out runtimeclass templates for AMD SNP and Intel TDX Co-Authored-By: Claude Opus 4.6 (1M context) --- ansible/detect-runtime-class.yaml | 56 ------------------- charts/all/baremetal/bm-kernel-params.yaml | 2 - .../templates/kernel-params-mco.yaml | 21 ------- .../templates/runtimeclass-amd-snp.yaml | 12 ---- .../templates/runtimeclass-intel-tdx.yaml | 13 ----- 5 files changed, 104 deletions(-) delete mode 100644 ansible/detect-runtime-class.yaml delete mode 100644 charts/all/baremetal/bm-kernel-params.yaml delete mode 100644 charts/all/baremetal/templates/kernel-params-mco.yaml delete mode 100644 charts/all/baremetal/templates/runtimeclass-amd-snp.yaml delete mode 100644 charts/all/baremetal/templates/runtimeclass-intel-tdx.yaml diff --git a/ansible/detect-runtime-class.yaml b/ansible/detect-runtime-class.yaml deleted file mode 100644 index a3618ecb..00000000 --- a/ansible/detect-runtime-class.yaml +++ /dev/null @@ -1,56 +0,0 @@ -- name: Detect and configure runtime class - hosts: localhost - connection: local - gather_facts: false - tasks: - - name: Check for Intel TDX nodes - kubernetes.core.k8s_info: - api_version: v1 - kind: Node - label_selectors: - - intel.feature.node.kubernetes.io/tdx=true - register: tdx_nodes - - - name: Check for AMD SEV-SNP nodes - kubernetes.core.k8s_info: - api_version: v1 - kind: Node - label_selectors: - - amd.feature.node.kubernetes.io/snp=true - register: snp_nodes - - - name: Set runtime handler for Intel TDX - ansible.builtin.set_fact: - kata_handler: "kata-tdx" - kata_overhead: - memory: "350Mi" - cpu: "250m" - tdx.intel.com/keys: "1" - kata_node_selector: - intel.feature.node.kubernetes.io/tdx: "true" - when: tdx_nodes.resources | length > 0 - - - name: Set runtime handler for AMD SEV-SNP - ansible.builtin.set_fact: - kata_handler: "kata-snp" - kata_overhead: - memory: "350Mi" - cpu: "250m" - kata_node_selector: - amd.feature.node.kubernetes.io/snp: "true" - when: snp_nodes.resources | length > 0 - - - name: Create kata-cc RuntimeClass - kubernetes.core.k8s: - state: present - definition: - apiVersion: node.k8s.io/v1 - kind: RuntimeClass - metadata: - name: kata-cc - handler: "{{ kata_handler }}" - overhead: - podFixed: "{{ kata_overhead }}" - scheduling: - nodeSelector: "{{ kata_node_selector }}" - when: kata_handler is defined diff --git a/charts/all/baremetal/bm-kernel-params.yaml b/charts/all/baremetal/bm-kernel-params.yaml deleted file mode 100644 index 86f01791..00000000 --- a/charts/all/baremetal/bm-kernel-params.yaml +++ /dev/null @@ -1,2 +0,0 @@ -[hypervisor.qemu] -kernel_params="agent.aa_kbc_params=cc_kbc::http://kbs-trustee-operator-system.{{ .Values.global.hubClusterDomain }}" diff --git a/charts/all/baremetal/templates/kernel-params-mco.yaml b/charts/all/baremetal/templates/kernel-params-mco.yaml deleted file mode 100644 index 1c372551..00000000 --- a/charts/all/baremetal/templates/kernel-params-mco.yaml +++ /dev/null @@ -1,21 +0,0 @@ -{{- range list "master" "worker" }} ---- -apiVersion: machineconfiguration.openshift.io/v1 -kind: MachineConfig -metadata: - labels: - machineconfiguration.openshift.io/role: {{ . }} - name: 96-kata-kernel-config-{{ . }} - namespace: openshift-machine-config-operator -spec: - config: - ignition: - version: 3.2.0 - storage: - files: - - contents: - source: 'data:text/plain;charset=utf-8;base64,{{ tpl ($.Files.Get "bm-kernel-params.yaml") $ | b64enc }}' - mode: 420 - overwrite: true - path: /etc/kata-containers/snp/config.d/96-kata-kernel-config -{{- end }} diff --git a/charts/all/baremetal/templates/runtimeclass-amd-snp.yaml b/charts/all/baremetal/templates/runtimeclass-amd-snp.yaml deleted file mode 100644 index c59be865..00000000 --- a/charts/all/baremetal/templates/runtimeclass-amd-snp.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# apiVersion: node.k8s.io/v1 -# kind: RuntimeClass -# metadata: -# name: kata-snp -# handler: kata-snp -# overhead: -# podFixed: -# memory: "350Mi" -# cpu: "250m" -# scheduling: -# nodeSelector: -# amd.feature.node.kubernetes.io/snp: "true" diff --git a/charts/all/baremetal/templates/runtimeclass-intel-tdx.yaml b/charts/all/baremetal/templates/runtimeclass-intel-tdx.yaml deleted file mode 100644 index f328dc0e..00000000 --- a/charts/all/baremetal/templates/runtimeclass-intel-tdx.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# apiVersion: node.k8s.io/v1 -# kind: RuntimeClass -# metadata: -# name: kata-tdx -# handler: kata-tdx -# overhead: -# podFixed: -# memory: "350Mi" -# cpu: "250m" -# tdx.intel.com/keys: 1 -# scheduling: -# nodeSelector: -# intel.feature.node.kubernetes.io/tdx: "true" From fbce1aaeabbf28c479a02fc942c108ca085c0603 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Mon, 20 Apr 2026 08:44:58 +0900 Subject: [PATCH 05/24] feat: update to OSC 1.12 / Trustee 1.1.0 Signed-off-by: Chris Butler --- values-baremetal.yaml | 4 ++-- values-simple.yaml | 4 ++-- values-spoke.yaml | 2 +- values-trusted-hub.yaml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/values-baremetal.yaml b/values-baremetal.yaml index 2a2980e8..8e58daaa 100644 --- a/values-baremetal.yaml +++ b/values-baremetal.yaml @@ -31,14 +31,14 @@ clusterGroup: source: redhat-operators channel: stable installPlanApproval: Manual - csv: sandboxed-containers-operator.v1.11.0 + csv: sandboxed-containers-operator.v1.12.0 trustee: name: trustee-operator namespace: trustee-operator-system source: redhat-operators channel: stable installPlanApproval: Manual - csv: trustee-operator.v1.0.0 + csv: trustee-operator.v1.1.0 cert-manager: name: openshift-cert-manager-operator namespace: cert-manager-operator diff --git a/values-simple.yaml b/values-simple.yaml index fed2b1d3..0a8fe58e 100644 --- a/values-simple.yaml +++ b/values-simple.yaml @@ -27,14 +27,14 @@ clusterGroup: source: redhat-operators channel: stable installPlanApproval: Manual - csv: sandboxed-containers-operator.v1.11.0 + csv: sandboxed-containers-operator.v1.12.0 trustee: name: trustee-operator namespace: trustee-operator-system source: redhat-operators channel: stable installPlanApproval: Manual - csv: trustee-operator.v1.0.0 + csv: trustee-operator.v1.1.0 cert-manager: name: openshift-cert-manager-operator namespace: cert-manager-operator diff --git a/values-spoke.yaml b/values-spoke.yaml index c604a5a6..0dab1011 100644 --- a/values-spoke.yaml +++ b/values-spoke.yaml @@ -17,7 +17,7 @@ clusterGroup: source: redhat-operators channel: stable installPlanApproval: Manual - csv: sandboxed-containers-operator.v1.11.0 + csv: sandboxed-containers-operator.v1.12.0 cert-manager: name: openshift-cert-manager-operator namespace: cert-manager-operator diff --git a/values-trusted-hub.yaml b/values-trusted-hub.yaml index c3531ba1..cdc4eda4 100644 --- a/values-trusted-hub.yaml +++ b/values-trusted-hub.yaml @@ -22,7 +22,7 @@ clusterGroup: source: redhat-operators channel: stable installPlanApproval: Manual - csv: trustee-operator.v1.0.0 + csv: trustee-operator.v1.1.0 cert-manager: name: openshift-cert-manager-operator namespace: cert-manager-operator From ccaee96b2fa6139616cd075e88d7216920e12806 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Wed, 22 Apr 2026 13:14:38 +0900 Subject: [PATCH 06/24] feat: integrate Kyverno and update trustee config for baremetal - Add Kyverno chart and coco-kyverno-policies to baremetal values - Update trustee chart to 0.3.* with kbs.admin.format v1.1 - Remove bypassAttestation (proper attestation via init_data) - Remove explicit runtimeClassName overrides (auto-detected by platform) - Add syncPolicy prune to hello-openshift and kbs-access - Reset default clusterGroupName to simple Co-Authored-By: Claude Opus 4.6 --- values-baremetal.yaml | 60 +++++++++++++++++++++++++++++++++++++------ values-global.yaml | 4 +-- 2 files changed, 54 insertions(+), 10 deletions(-) diff --git a/values-baremetal.yaml b/values-baremetal.yaml index 8e58daaa..f8ab2ad5 100644 --- a/values-baremetal.yaml +++ b/values-baremetal.yaml @@ -20,6 +20,7 @@ clusterGroup: - openshift-nfd - baremetal - intel-dcap + - kyverno subscriptions: acm: @@ -101,12 +102,12 @@ clusterGroup: namespace: trustee-operator-system project: trustee chart: trustee - chartVersion: 0.2.* + chartVersion: 0.3.* overrides: - name: global.coco.secured value: "true" - - name: global.coco.bypassAttestation - value: "true" + - name: kbs.admin.format + value: "v1.1" - name: kbs.https.enabled value: "false" - name: kbs.secretResources[0].name @@ -169,18 +170,61 @@ clusterGroup: namespace: kbs-access project: workloads path: charts/coco-supported/kbs-access - overrides: - - name: runtimeClassName - value: "kata-cc" + syncPolicy: + automated: + prune: true hello-openshift: name: hello-openshift namespace: hello-openshift project: workloads path: charts/coco-supported/hello-openshift + syncPolicy: + automated: + prune: true + + kyverno: + name: kyverno + namespace: kyverno + project: hub + repoURL: https://kyverno.github.io/kyverno/ + chart: kyverno + chartVersion: 3.7.* + syncPolicy: + automated: {} + retry: + limit: 20 + syncOptions: + - ServerSideApply=true overrides: - - name: runtimeClassName - value: "kata-cc" + - name: admissionController.container.securityContext + value: "null" + - name: admissionController.initContainer.securityContext + value: "null" + - name: backgroundController.securityContext + value: "null" + - name: cleanupController.securityContext + value: "null" + - name: reportsController.securityContext + value: "null" + - name: crds.migration.securityContext + value: "null" + - name: webhooksCleanup.securityContext + value: "null" + - name: test.securityContext + value: "null" + - name: crds.groups.wgpolicyk8s.policyreports + value: "false" + - name: crds.groups.wgpolicyk8s.clusterpolicyreports + value: "false" + - name: reportsController.enabled + value: "false" + + coco-kyverno-policies: + name: coco-kyverno-policies + namespace: openshift-sandboxed-containers-operator + project: sandbox + path: charts/all/coco-kyverno-policies imperative: # NOTE: We *must* use lists and not hashes. As hashes lose ordering once parsed by helm diff --git a/values-global.yaml b/values-global.yaml index 9e83b312..aa8513d4 100644 --- a/values-global.yaml +++ b/values-global.yaml @@ -14,7 +14,7 @@ global: coco: securityPolicyFlavour: "insecure" # insecure, signed or reject is expected. secured: true # true or false. If true, the cluster will be secured. If false, the cluster will be insecure. - bypassAttestation: false # Set true for bare metal (skips PCR/initdata/RVPS) + bypassAttestation: false # Enable SSH key injection into podvm for debugging. Do not enable in production. # Also requires: COCO_ENABLE_SSH_DEBUG=true ./scripts/gen-secrets.sh # and uncommenting the sshKey block in values-secret.yaml.template. @@ -26,7 +26,7 @@ main: # WARNING # This default configuration uses a single cluster on azure. # It fundamentally violates the separation of duties. - clusterGroupName: baremetal + clusterGroupName: simple multiSourceConfig: enabled: true clusterGroupChartVersion: 0.9.* From a601af03440a5c9505f977f42335e115dc05fa01 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Wed, 22 Apr 2026 13:21:45 +0900 Subject: [PATCH 07/24] fix: set clusterGroupName to baremetal for deployment testing Co-Authored-By: Claude Opus 4.6 --- values-global.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/values-global.yaml b/values-global.yaml index aa8513d4..7cfbda9f 100644 --- a/values-global.yaml +++ b/values-global.yaml @@ -26,7 +26,7 @@ main: # WARNING # This default configuration uses a single cluster on azure. # It fundamentally violates the separation of duties. - clusterGroupName: simple + clusterGroupName: baremetal multiSourceConfig: enabled: true clusterGroupChartVersion: 0.9.* From cccc0807039206015bc8adc7ad4c51fb539b2040 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Wed, 22 Apr 2026 14:18:53 +0900 Subject: [PATCH 08/24] fix: add UPDATE operation to initdata injection policy The policy only fired on Pod/Deployment CREATE, so pods created before the initdata ConfigMap existed never got the cc_init_data annotation. Adding UPDATE allows Kyverno to inject the annotation when a Deployment is updated (e.g. by ArgoCD sync), triggering a rolling restart with the correct initdata. Co-Authored-By: Claude Opus 4.6 --- .../coco-kyverno-policies/templates/inject-coco-initdata.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/charts/all/coco-kyverno-policies/templates/inject-coco-initdata.yaml b/charts/all/coco-kyverno-policies/templates/inject-coco-initdata.yaml index 397ba39b..ccb7b193 100644 --- a/charts/all/coco-kyverno-policies/templates/inject-coco-initdata.yaml +++ b/charts/all/coco-kyverno-policies/templates/inject-coco-initdata.yaml @@ -24,6 +24,7 @@ spec: - Pod operations: - CREATE + - UPDATE preconditions: all: - key: "{{ "{{" }}request.object.spec.runtimeClassName || '' {{ "}}" }}" From 27c71e5b05f64a451e6c49e8e2cee99788183a30 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Wed, 22 Apr 2026 14:28:36 +0900 Subject: [PATCH 09/24] feat: add intel-device-plugins-operator subscription for SGX/TDX quote generation Co-Authored-By: Claude Opus 4.6 --- values-baremetal.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/values-baremetal.yaml b/values-baremetal.yaml index f8ab2ad5..a90b2207 100644 --- a/values-baremetal.yaml +++ b/values-baremetal.yaml @@ -60,6 +60,11 @@ clusterGroup: name: nfd namespace: openshift-nfd channel: stable + intel-device-plugins: + name: intel-device-plugins-operator + namespace: openshift-operators + source: certified-operators + channel: stable projects: - hub - vault From e462936ac813b69bd173b171eb199d4e6ff54735 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Wed, 22 Apr 2026 14:34:39 +0900 Subject: [PATCH 10/24] fix: enable TDX config in trustee to point QCNL at local PCCS service Co-Authored-By: Claude Opus 4.6 --- values-baremetal.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/values-baremetal.yaml b/values-baremetal.yaml index a90b2207..c53cc7b1 100644 --- a/values-baremetal.yaml +++ b/values-baremetal.yaml @@ -119,6 +119,10 @@ clusterGroup: value: kbsres1 - name: kbs.secretResources[0].key value: secret/data/hub/kbsres1 + - name: kbs.tdx.enabled + value: "true" + - name: kbs.tdx.collateralService + value: "https://pccs-service.intel-dcap.svc.cluster.local:8042/sgx/certification/v4/" - name: kbs.secretResources[1].name value: passphrase - name: kbs.secretResources[1].key From 20b2b73ba31e933d4395b48bd14f8248baff53d7 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Wed, 22 Apr 2026 14:53:45 +0900 Subject: [PATCH 11/24] feat: store raw SHA-256 hash alongside PCR8 hash in initdata ConfigMaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds RAW_HASH field to both initdata and debug-initdata ConfigMaps. PCR8_HASH = SHA256(zeros || SHA256(toml)) — used by Azure vTPM attestation RAW_HASH = SHA256(toml) — used by baremetal TDX/SNP attestation Both are needed because Azure and baremetal present initdata differently in their attestation evidence. A single Trustee attestation server must accept both formats to support multi-platform deployments. Future: integrate veritas for comprehensive reference value generation. Co-Authored-By: Claude Opus 4.6 --- ansible/init-data-gzipper.yaml | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/ansible/init-data-gzipper.yaml b/ansible/init-data-gzipper.yaml index f0e10cf6..4ebc2e1a 100644 --- a/ansible/init-data-gzipper.yaml +++ b/ansible/init-data-gzipper.yaml @@ -114,21 +114,33 @@ register: debug_initdata_encoded changed_when: false + - name: Compute raw SHA-256 hash of default initdata + ansible.builtin.shell: | + set -o pipefail + sha256sum "{{ rendered_path }}" | cut -d' ' -f1 + register: raw_hash + changed_when: false + + - name: Compute raw SHA-256 hash of debug initdata + ansible.builtin.shell: | + set -o pipefail + sha256sum "{{ debug_rendered_path }}" | cut -d' ' -f1 + register: debug_raw_hash + changed_when: false + - name: Register init data pcr into a var ansible.builtin.shell: | set -o pipefail - hash=$(sha256sum "{{ rendered_path }}" | cut -d' ' -f1) initial_pcr=0000000000000000000000000000000000000000000000000000000000000000 - PCR8_HASH=$(echo -n "$initial_pcr$hash" | xxd -r -p | sha256sum | cut -d' ' -f1) && echo $PCR8_HASH + PCR8_HASH=$(echo -n "$initial_pcr{{ raw_hash.stdout }}" | xxd -r -p | sha256sum | cut -d' ' -f1) && echo $PCR8_HASH register: pcr8_hash changed_when: false - name: Register debug init data pcr into a var ansible.builtin.shell: | set -o pipefail - hash=$(sha256sum "{{ debug_rendered_path }}" | cut -d' ' -f1) initial_pcr=0000000000000000000000000000000000000000000000000000000000000000 - PCR8_HASH=$(echo -n "$initial_pcr$hash" | xxd -r -p | sha256sum | cut -d' ' -f1) && echo $PCR8_HASH + PCR8_HASH=$(echo -n "$initial_pcr{{ debug_raw_hash.stdout }}" | xxd -r -p | sha256sum | cut -d' ' -f1) && echo $PCR8_HASH register: debug_pcr8_hash changed_when: false @@ -147,6 +159,7 @@ data: INITDATA: "{{ initdata_encoded.stdout }}" PCR8_HASH: "{{ pcr8_hash.stdout }}" + RAW_HASH: "{{ raw_hash.stdout }}" version: "0.1.0" algorithm: "sha256" aa.toml: "{{ raw_aa_toml.stdout }}" @@ -168,6 +181,7 @@ data: INITDATA: "{{ debug_initdata_encoded.stdout }}" PCR8_HASH: "{{ debug_pcr8_hash.stdout }}" + RAW_HASH: "{{ debug_raw_hash.stdout }}" version: "0.1.0" algorithm: "sha256" aa.toml: "{{ raw_aa_toml.stdout }}" From 98d4a1b6eee8a46444406395208daa03a9d9f144 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Wed, 22 Apr 2026 14:56:01 +0900 Subject: [PATCH 12/24] fix: point trustee at feature branch for baremetal attestation testing Temporarily uses butler54/trustee-chart feature/baremetal-attestation branch instead of released chart. This branch includes: - Baremetal TDX and SNP attestation rules - Conditional pcr-stash (no error on baremetal without vTPM) - Raw init_data hash (zero-padded) for baremetal attestation - TDX QCNL config with use_secure_cert: false for local PCCS Revert to chartVersion after merging and releasing trustee chart. Co-Authored-By: Claude Opus 4.6 --- values-baremetal.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/values-baremetal.yaml b/values-baremetal.yaml index c53cc7b1..fb70c409 100644 --- a/values-baremetal.yaml +++ b/values-baremetal.yaml @@ -106,8 +106,9 @@ clusterGroup: name: trustee namespace: trustee-operator-system project: trustee - chart: trustee - chartVersion: 0.3.* + repoURL: https://github.com/butler54/trustee-chart.git + targetRevision: feature/baremetal-attestation + path: . overrides: - name: global.coco.secured value: "true" From 070ca0e9743d81e5439006527f4cbcb8cd7c55be Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Wed, 22 Apr 2026 15:17:44 +0900 Subject: [PATCH 13/24] fix: increase kata VM memory for kbs-access to 8192MB The kbs-access-app container image is ~1GB which causes container creation timeouts with the default 2GB kata VM memory. Co-Authored-By: Claude Opus 4.6 --- values-baremetal.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/values-baremetal.yaml b/values-baremetal.yaml index fb70c409..ee31e913 100644 --- a/values-baremetal.yaml +++ b/values-baremetal.yaml @@ -183,6 +183,9 @@ clusterGroup: syncPolicy: automated: prune: true + overrides: + - name: defaultMemory + value: "8192" hello-openshift: name: hello-openshift From 68457d021fa4e5a4e3b59c32a1542276a4a5c68f Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Thu, 23 Apr 2026 14:29:08 +0900 Subject: [PATCH 14/24] fix: target Pods only for cc_init_data injection, disable autogen MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The autogen Deployment rule causes admission failures when the initdata ConfigMap hasn't been propagated to the workload namespace yet. By targeting Pods only (autogen-controllers: none), Deployments are admitted without ConfigMap resolution. Pods get cc_init_data injected at creation time when the ConfigMap is available. A rollout restart picks up new initdata values. Also removes UPDATE operation — only CREATE is needed since a rollout restart creates new Pods. Co-Authored-By: Claude Opus 4.6 --- .../templates/inject-coco-initdata.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/charts/all/coco-kyverno-policies/templates/inject-coco-initdata.yaml b/charts/all/coco-kyverno-policies/templates/inject-coco-initdata.yaml index ccb7b193..56483fe9 100644 --- a/charts/all/coco-kyverno-policies/templates/inject-coco-initdata.yaml +++ b/charts/all/coco-kyverno-policies/templates/inject-coco-initdata.yaml @@ -6,14 +6,14 @@ metadata: policies.kyverno.io/title: Inject CoCo InitData policies.kyverno.io/category: Confidential Computing policies.kyverno.io/severity: medium - policies.kyverno.io/subject: Pod,Deployment + policies.kyverno.io/subject: Pod policies.kyverno.io/description: >- Injects cc_init_data annotation into pods with a kata runtime class by reading from a ConfigMap specified via the coco.io/initdata-configmap - annotation. Kyverno autogen extends this to Deployments, StatefulSets, - DaemonSets, and Jobs automatically. + annotation. Targets Pods only (not Deployments) so that Deployments + remain stable and a rollout restart resolves the latest initdata. argocd.argoproj.io/sync-wave: "1" - pod-policies.kyverno.io/autogen-controllers: Deployment,StatefulSet,DaemonSet,Job + pod-policies.kyverno.io/autogen-controllers: none spec: rules: - name: inject-initdata @@ -24,7 +24,6 @@ spec: - Pod operations: - CREATE - - UPDATE preconditions: all: - key: "{{ "{{" }}request.object.spec.runtimeClassName || '' {{ "}}" }}" From d61db58824fab45da169c5dc69ac8180f4fef35d Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Thu, 23 Apr 2026 14:57:20 +0900 Subject: [PATCH 15/24] fix: use ${initial_pcr} braces in PCR8 hash computation Without braces, bash treats $initial_pcr followed by the hex hash as a single undefined variable name, producing SHA-256 of empty string instead of the correct PCR extend value. Co-Authored-By: Claude Opus 4.6 --- ansible/init-data-gzipper.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/init-data-gzipper.yaml b/ansible/init-data-gzipper.yaml index 4ebc2e1a..c9de0a5f 100644 --- a/ansible/init-data-gzipper.yaml +++ b/ansible/init-data-gzipper.yaml @@ -132,7 +132,7 @@ ansible.builtin.shell: | set -o pipefail initial_pcr=0000000000000000000000000000000000000000000000000000000000000000 - PCR8_HASH=$(echo -n "$initial_pcr{{ raw_hash.stdout }}" | xxd -r -p | sha256sum | cut -d' ' -f1) && echo $PCR8_HASH + PCR8_HASH=$(echo -n "${initial_pcr}{{ raw_hash.stdout }}" | xxd -r -p | sha256sum | cut -d' ' -f1) && echo $PCR8_HASH register: pcr8_hash changed_when: false @@ -140,7 +140,7 @@ ansible.builtin.shell: | set -o pipefail initial_pcr=0000000000000000000000000000000000000000000000000000000000000000 - PCR8_HASH=$(echo -n "$initial_pcr{{ debug_raw_hash.stdout }}" | xxd -r -p | sha256sum | cut -d' ' -f1) && echo $PCR8_HASH + PCR8_HASH=$(echo -n "${initial_pcr}{{ debug_raw_hash.stdout }}" | xxd -r -p | sha256sum | cut -d' ' -f1) && echo $PCR8_HASH register: debug_pcr8_hash changed_when: false From b54884e2d2c5fd59664f65aa3a042fb9fb306772 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Fri, 24 Apr 2026 14:32:58 +0900 Subject: [PATCH 16/24] feat!: add NVIDIA H100 confidential GPU support for bare metal Enables NVIDIA confidential GPU (H100/H200) on bare metal deployments with full CoCo integration. Addresses three documented gaps in the Red Hat OSC 1.12 documentation: - Gap 1: Pin GPU Operator to v26.3.0 (v26.3.1 breaks kata state machine) - Gap 2: Include kataSandboxDevicePlugin in ClusterPolicy (required for nvidia.com/pgpu resource advertisement) - Gap 3: Add imperative job to re-reconcile KataConfig after GPU Operator labels nodes (kata-cc-nvidia-gpu RuntimeClass creation) New charts: - charts/all/nvidia-gpu: ClusterPolicy CR and IOMMU MachineConfig - charts/coco-supported/gpu-workload: CUDA vectorAdd sample deployment Also extends Kyverno initdata injection to support kata-cc-nvidia-gpu runtime class and propagate initdata to gpu-workload namespace. Co-Authored-By: Claude Opus 4.6 --- ansible/reconcile-kataconfig-gpu.yaml | 42 +++++++++++++++ .../templates/inject-coco-initdata.yaml | 2 +- charts/all/coco-kyverno-policies/values.yaml | 1 + charts/all/nvidia-gpu/Chart.yaml | 9 ++++ .../nvidia-gpu/templates/cluster-policy.yaml | 53 +++++++++++++++++++ .../all/nvidia-gpu/templates/iommu-mco.yaml | 15 ++++++ charts/all/nvidia-gpu/values.yaml | 15 ++++++ charts/coco-supported/gpu-workload/Chart.yaml | 10 ++++ .../templates/gpu-vectoradd-deployment.yaml | 37 +++++++++++++ .../coco-supported/gpu-workload/values.yaml | 6 +++ values-baremetal.yaml | 30 +++++++++++ 11 files changed, 219 insertions(+), 1 deletion(-) create mode 100644 ansible/reconcile-kataconfig-gpu.yaml create mode 100644 charts/all/nvidia-gpu/Chart.yaml create mode 100644 charts/all/nvidia-gpu/templates/cluster-policy.yaml create mode 100644 charts/all/nvidia-gpu/templates/iommu-mco.yaml create mode 100644 charts/all/nvidia-gpu/values.yaml create mode 100644 charts/coco-supported/gpu-workload/Chart.yaml create mode 100644 charts/coco-supported/gpu-workload/templates/gpu-vectoradd-deployment.yaml create mode 100644 charts/coco-supported/gpu-workload/values.yaml diff --git a/ansible/reconcile-kataconfig-gpu.yaml b/ansible/reconcile-kataconfig-gpu.yaml new file mode 100644 index 00000000..d0c721b7 --- /dev/null +++ b/ansible/reconcile-kataconfig-gpu.yaml @@ -0,0 +1,42 @@ +--- +- name: Reconcile KataConfig for GPU RuntimeClass + hosts: localhost + connection: local + become: false + gather_facts: false + tasks: + - name: Check for nodes with NVIDIA GPU labels + kubernetes.core.k8s_info: + api_version: v1 + kind: Node + label_selectors: + - "nvidia.com/gpu.present=true" + register: gpu_nodes + + - name: Check if kata-cc-nvidia-gpu RuntimeClass exists + kubernetes.core.k8s_info: + api_version: node.k8s.io/v1 + kind: RuntimeClass + name: kata-cc-nvidia-gpu + register: gpu_runtimeclass + + - name: Trigger KataConfig re-reconciliation + kubernetes.core.k8s: + state: patched + api_version: kataconfiguration.openshift.io/v1 + kind: KataConfig + name: default-kata-config + definition: + metadata: + annotations: + kata-reconcile: "{{ ansible_date_time.epoch }}" + when: + - gpu_nodes.resources | length > 0 + - gpu_runtimeclass.resources | length == 0 + + - name: Report status + ansible.builtin.debug: + msg: >- + GPU nodes: {{ gpu_nodes.resources | length }}, + RuntimeClass exists: {{ gpu_runtimeclass.resources | length > 0 }}, + Action: {{ 'triggered re-reconciliation' if (gpu_nodes.resources | length > 0 and gpu_runtimeclass.resources | length == 0) else 'no action needed' }} diff --git a/charts/all/coco-kyverno-policies/templates/inject-coco-initdata.yaml b/charts/all/coco-kyverno-policies/templates/inject-coco-initdata.yaml index 56483fe9..84c06855 100644 --- a/charts/all/coco-kyverno-policies/templates/inject-coco-initdata.yaml +++ b/charts/all/coco-kyverno-policies/templates/inject-coco-initdata.yaml @@ -28,7 +28,7 @@ spec: all: - key: "{{ "{{" }}request.object.spec.runtimeClassName || '' {{ "}}" }}" operator: AnyIn - value: ["kata", "kata-cc", "kata-remote"] + value: ["kata", "kata-cc", "kata-remote", "kata-cc-nvidia-gpu"] - key: "{{ "{{" }}request.object.metadata.annotations.\"coco.io/initdata-configmap\" || '' {{ "}}" }}" operator: NotEquals value: "" diff --git a/charts/all/coco-kyverno-policies/values.yaml b/charts/all/coco-kyverno-policies/values.yaml index ef78f81e..0f534874 100644 --- a/charts/all/coco-kyverno-policies/values.yaml +++ b/charts/all/coco-kyverno-policies/values.yaml @@ -1,5 +1,6 @@ workloadNamespaces: - hello-openshift - kbs-access + - gpu-workload initdataSourceNamespace: imperative diff --git a/charts/all/nvidia-gpu/Chart.yaml b/charts/all/nvidia-gpu/Chart.yaml new file mode 100644 index 00000000..f96cd825 --- /dev/null +++ b/charts/all/nvidia-gpu/Chart.yaml @@ -0,0 +1,9 @@ +apiVersion: v2 +description: NVIDIA GPU Operator configuration for confidential containers (ClusterPolicy, IOMMU MachineConfig). +keywords: +- pattern +- nvidia +- gpu +- confidential +name: nvidia-gpu +version: 0.0.1 diff --git a/charts/all/nvidia-gpu/templates/cluster-policy.yaml b/charts/all/nvidia-gpu/templates/cluster-policy.yaml new file mode 100644 index 00000000..c2765174 --- /dev/null +++ b/charts/all/nvidia-gpu/templates/cluster-policy.yaml @@ -0,0 +1,53 @@ +{{- if .Values.enabled }} +apiVersion: nvidia.com/v1 +kind: ClusterPolicy +metadata: + name: gpu-cluster-policy + annotations: + argocd.argoproj.io/sync-wave: "110" +spec: + operator: + defaultRuntime: crio + + sandboxWorkloads: + enabled: true + mode: kata + defaultWorkload: vm-passthrough + + kataManager: + enabled: false + + ccManager: + enabled: {{ .Values.ccManager.enabled }} + defaultMode: {{ .Values.ccManager.defaultMode | quote }} + repository: nvcr.io/nvidia/cloud-native + image: k8s-cc-manager + version: v0.1.0 + env: + - name: CC_CAPABLE_DEVICE_IDS + value: {{ .Values.ccManager.deviceIDs | quote }} + + kataSandboxDevicePlugin: + enabled: {{ .Values.kataSandboxDevicePlugin.enabled }} + repository: {{ .Values.kataSandboxDevicePlugin.repository }} + image: {{ .Values.kataSandboxDevicePlugin.image }} + version: {{ .Values.kataSandboxDevicePlugin.version | quote }} + + sandboxDevicePlugin: + enabled: true + + driver: + enabled: true + + devicePlugin: + enabled: false + + vfioManager: + enabled: true + + gfd: + enabled: true + + nfd: + nodefeaturerules: true +{{- end }} diff --git a/charts/all/nvidia-gpu/templates/iommu-mco.yaml b/charts/all/nvidia-gpu/templates/iommu-mco.yaml new file mode 100644 index 00000000..3d1508f3 --- /dev/null +++ b/charts/all/nvidia-gpu/templates/iommu-mco.yaml @@ -0,0 +1,15 @@ +{{- if .Values.iommu.enabled }} +{{- range list "master" "worker" }} +--- +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + labels: + machineconfiguration.openshift.io/role: {{ . }} + name: 100-iommu-{{ . }} +spec: + kernelArguments: + - intel_iommu=on + - iommu=pt +{{- end }} +{{- end }} diff --git a/charts/all/nvidia-gpu/values.yaml b/charts/all/nvidia-gpu/values.yaml new file mode 100644 index 00000000..23a2e897 --- /dev/null +++ b/charts/all/nvidia-gpu/values.yaml @@ -0,0 +1,15 @@ +enabled: true + +ccManager: + enabled: true + defaultMode: "on" + deviceIDs: "0x2331,0x2322" + +kataSandboxDevicePlugin: + enabled: true + repository: nvcr.io/nvidia/cloud-native + image: nvidia-sandbox-device-plugin + version: "v0.0.2" + +iommu: + enabled: true diff --git a/charts/coco-supported/gpu-workload/Chart.yaml b/charts/coco-supported/gpu-workload/Chart.yaml new file mode 100644 index 00000000..9f4398cd --- /dev/null +++ b/charts/coco-supported/gpu-workload/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +description: Sample CUDA workload for NVIDIA confidential GPU verification. +keywords: +- pattern +- nvidia +- gpu +- workload +- confidential +name: gpu-workload +version: 0.0.1 diff --git a/charts/coco-supported/gpu-workload/templates/gpu-vectoradd-deployment.yaml b/charts/coco-supported/gpu-workload/templates/gpu-vectoradd-deployment.yaml new file mode 100644 index 00000000..4ad3ea6d --- /dev/null +++ b/charts/coco-supported/gpu-workload/templates/gpu-vectoradd-deployment.yaml @@ -0,0 +1,37 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gpu-vectoradd + labels: + app: gpu-vectoradd +spec: + replicas: 1 + selector: + matchLabels: + app: gpu-vectoradd + template: + metadata: + labels: + app: gpu-vectoradd + annotations: + coco.io/initdata-configmap: initdata + {{- if .Values.defaultMemory }} + io.katacontainers.config.hypervisor.default_memory: {{ .Values.defaultMemory | quote }} + {{- end }} + spec: + runtimeClassName: {{ .Values.runtimeClassName }} + containers: + - name: cuda-vectoradd + image: nvcr.io/nvidia/k8s/cuda-sample:vectoradd-cuda11.7.1-ubuntu20.04 + resources: + limits: + nvidia.com/pgpu: 1 + securityContext: + privileged: false + allowPrivilegeEscalation: false + runAsNonRoot: true + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault diff --git a/charts/coco-supported/gpu-workload/values.yaml b/charts/coco-supported/gpu-workload/values.yaml new file mode 100644 index 00000000..46e9a266 --- /dev/null +++ b/charts/coco-supported/gpu-workload/values.yaml @@ -0,0 +1,6 @@ +runtimeClassName: "kata-cc-nvidia-gpu" + +defaultMemory: "32768" + +global: + clusterPlatform: "" diff --git a/values-baremetal.yaml b/values-baremetal.yaml index ee31e913..65d5eaf6 100644 --- a/values-baremetal.yaml +++ b/values-baremetal.yaml @@ -20,6 +20,8 @@ clusterGroup: - openshift-nfd - baremetal - intel-dcap + - nvidia-gpu-operator + - gpu-workload - kyverno subscriptions: @@ -60,6 +62,13 @@ clusterGroup: name: nfd namespace: openshift-nfd channel: stable + gpu-operator: + name: gpu-operator-certified + namespace: nvidia-gpu-operator + source: certified-operators + channel: v26.3 + installPlanApproval: Manual + csv: gpu-operator-certified.v26.3.0 intel-device-plugins: name: intel-device-plugins-operator namespace: openshift-operators @@ -128,6 +137,8 @@ clusterGroup: value: passphrase - name: kbs.secretResources[1].key value: secret/data/hub/passphrase + - name: kbs.gpu.enabled + value: "true" storage: name: storage @@ -169,6 +180,21 @@ clusterGroup: - name: secretStore.kind value: ClusterSecretStore + nvidia-gpu: + name: nvidia-gpu + namespace: nvidia-gpu-operator + project: hub + path: charts/all/nvidia-gpu + + gpu-workload: + name: gpu-workload + namespace: gpu-workload + project: workloads + path: charts/coco-supported/gpu-workload + syncPolicy: + automated: + prune: true + sandbox-policies: name: sandbox-policies namespace: openshift-sandboxed-containers-operator @@ -259,6 +285,10 @@ clusterGroup: playbook: ansible/init-data-gzipper.yaml verbosity: -vvv timeout: 3600 + - name: reconcile-kataconfig-gpu + playbook: ansible/reconcile-kataconfig-gpu.yaml + verbosity: -vvv + timeout: 600 # Required for tech preview only. # - name: detect-runtime-class # playbook: ansible/detect-runtime-class.yaml From bbcb62214d218430c1b061384c67016b4ecffc9b Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Mon, 27 Apr 2026 21:04:32 +0900 Subject: [PATCH 17/24] feat: segregated gpu bm from bm for ease of testing Signed-off-by: Chris Butler --- ansible/reconcile-kataconfig-gpu.yaml | 2 +- .../templates/hostpathprovisioner.yaml | 2 +- .../storage/templates/hpp-storageclass.yaml | 2 +- charts/hub/storage/templates/lvmcluster.yaml | 2 +- charts/hub/storage/values.yaml | 2 +- scripts/gen-secrets.sh | 4 +- values-baremetal-gpu.yaml | 297 ++++++++++++++++++ values-baremetal.yaml | 32 +- values-global.yaml | 5 +- 9 files changed, 307 insertions(+), 41 deletions(-) create mode 100644 values-baremetal-gpu.yaml diff --git a/ansible/reconcile-kataconfig-gpu.yaml b/ansible/reconcile-kataconfig-gpu.yaml index d0c721b7..c438d0a4 100644 --- a/ansible/reconcile-kataconfig-gpu.yaml +++ b/ansible/reconcile-kataconfig-gpu.yaml @@ -3,7 +3,7 @@ hosts: localhost connection: local become: false - gather_facts: false + gather_facts: true tasks: - name: Check for nodes with NVIDIA GPU labels kubernetes.core.k8s_info: diff --git a/charts/hub/storage/templates/hostpathprovisioner.yaml b/charts/hub/storage/templates/hostpathprovisioner.yaml index 909e24e5..99a318e6 100644 --- a/charts/hub/storage/templates/hostpathprovisioner.yaml +++ b/charts/hub/storage/templates/hostpathprovisioner.yaml @@ -1,4 +1,4 @@ -{{- if eq .Values.global.storageProvider "hpp" }} +{{- if eq .Values.global.baremetalStorageProvider "hpp" }} apiVersion: hostpathprovisioner.kubevirt.io/v1beta1 kind: HostPathProvisioner metadata: diff --git a/charts/hub/storage/templates/hpp-storageclass.yaml b/charts/hub/storage/templates/hpp-storageclass.yaml index 7c86c889..2fd23754 100644 --- a/charts/hub/storage/templates/hpp-storageclass.yaml +++ b/charts/hub/storage/templates/hpp-storageclass.yaml @@ -1,4 +1,4 @@ -{{- if eq .Values.global.storageProvider "hpp" }} +{{- if eq .Values.global.baremetalStorageProvider "hpp" }} apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: diff --git a/charts/hub/storage/templates/lvmcluster.yaml b/charts/hub/storage/templates/lvmcluster.yaml index 3a7724bb..45ba2851 100644 --- a/charts/hub/storage/templates/lvmcluster.yaml +++ b/charts/hub/storage/templates/lvmcluster.yaml @@ -1,4 +1,4 @@ -{{- if eq .Values.global.storageProvider "lvm" }} +{{- if eq .Values.global.baremetalStorageProvider "lvm" }} apiVersion: lvm.topolvm.io/v1alpha1 kind: LVMCluster metadata: diff --git a/charts/hub/storage/values.yaml b/charts/hub/storage/values.yaml index 705f5233..8add54b7 100644 --- a/charts/hub/storage/values.yaml +++ b/charts/hub/storage/values.yaml @@ -1,5 +1,5 @@ global: - storageProvider: hpp + baremetalStorageProvider: hpp lvmCluster: name: "lvmcluster" diff --git a/scripts/gen-secrets.sh b/scripts/gen-secrets.sh index 902d52a2..ad5fd9c4 100755 --- a/scripts/gen-secrets.sh +++ b/scripts/gen-secrets.sh @@ -46,13 +46,13 @@ if [ ! -f "${PCCS_USER_TOKEN_FILE}" ]; then echo "Creating PCCS user token" echo "usertoken" > "${PCCS_USER_TOKEN_FILE}" fi -echo -n "usertoken" | sha512sum | tr -d '[:space:]-' > "${COCO_SECRETS_DIR}/pccs_user_token_hash" +tr -d '\n' < "${PCCS_USER_TOKEN_FILE}" | sha512sum | tr -d '[:space:]-' > "${COCO_SECRETS_DIR}/pccs_user_token_hash" if [ ! -f "${PCCS_ADMIN_TOKEN_FILE}" ]; then echo "Creating PCCS admin token" echo "admintoken" > "${PCCS_ADMIN_TOKEN_FILE}" fi -echo -n "admintoken" | sha512sum | tr -d '[:space:]-' > "${COCO_SECRETS_DIR}/pccs_admin_token_hash" +tr -d '\n' < "${PCCS_ADMIN_TOKEN_FILE}" | sha512sum | tr -d '[:space:]-' > "${COCO_SECRETS_DIR}/pccs_admin_token_hash" ## Copy a sample values file if this stuff doesn't exist diff --git a/values-baremetal-gpu.yaml b/values-baremetal-gpu.yaml new file mode 100644 index 00000000..b28facc9 --- /dev/null +++ b/values-baremetal-gpu.yaml @@ -0,0 +1,297 @@ +# Bare metal deployment for confidential containers WITH NVIDIA GPU support. +# Supports Intel TDX and AMD SEV-SNP via auto-detection (NFD). +# Includes NVIDIA H100 confidential GPU components (GPU Operator, IOMMU, CC Manager). +# Set main.clusterGroupName: baremetal in values-global.yaml to use. + +clusterGroup: + name: baremetal + isHubCluster: true + namespaces: + - open-cluster-management + - vault + - golang-external-secrets + - openshift-sandboxed-containers-operator + - trustee-operator-system + - cert-manager-operator + - cert-manager + - hello-openshift + - kbs-access + - openshift-cnv + - openshift-storage + - openshift-nfd + - baremetal + - intel-dcap + - nvidia-gpu-operator + - gpu-workload + - kyverno + + subscriptions: + acm: + name: advanced-cluster-management + namespace: open-cluster-management + sandbox: + name: sandboxed-containers-operator + namespace: openshift-sandboxed-containers-operator + source: redhat-operators + channel: stable + installPlanApproval: Manual + csv: sandboxed-containers-operator.v1.12.0 + trustee: + name: trustee-operator + namespace: trustee-operator-system + source: redhat-operators + channel: stable + installPlanApproval: Manual + csv: trustee-operator.v1.1.0 + cert-manager: + name: openshift-cert-manager-operator + namespace: cert-manager-operator + channel: stable-v1 + lvm-operator: + name: lvms-operator + namespace: openshift-storage + source: redhat-operators + channel: stable-4.20 + installPlanApproval: Automatic + cnv: + name: kubevirt-hyperconverged + namespace: openshift-cnv + source: redhat-operators + channel: stable + installPlanApproval: Automatic + nfd: + name: nfd + namespace: openshift-nfd + channel: stable + gpu-operator: + name: gpu-operator-certified + namespace: nvidia-gpu-operator + source: certified-operators + channel: v26.3 + installPlanApproval: Manual + csv: gpu-operator-certified.v26.3.0 + intel-device-plugins: + name: intel-device-plugins-operator + namespace: openshift-operators + source: certified-operators + channel: stable + projects: + - hub + - vault + - trustee + - golang-external-secrets + - sandbox + - workloads + - default + + # Explicitly mention the cluster-state based overrides we plan to use for this pattern. + # We can use self-referential variables because the chart calls the tpl function with these variables defined + sharedValueFiles: + - '/overrides/values-{{ $.Values.global.clusterPlatform }}.yaml' + - '/overrides/values-storage-{{ $.Values.global.baremetalStorageProvider }}.yaml' + + applications: + acm: + name: acm + namespace: open-cluster-management + project: hub + chart: acm + chartVersion: 0.1.* + + vault: + name: vault + namespace: vault + project: vault + chart: hashicorp-vault + chartVersion: 0.1.* + + secrets-operator: + name: golang-external-secrets + namespace: golang-external-secrets + project: golang-external-secrets + chart: golang-external-secrets + chartVersion: 0.1.* + + trustee: + name: trustee + namespace: trustee-operator-system + project: trustee + repoURL: https://github.com/butler54/trustee-chart.git + targetRevision: feature/baremetal-attestation + path: . + overrides: + - name: global.coco.secured + value: "true" + - name: kbs.admin.format + value: "v1.1" + - name: kbs.https.enabled + value: "false" + - name: kbs.secretResources[0].name + value: kbsres1 + - name: kbs.secretResources[0].key + value: secret/data/hub/kbsres1 + - name: kbs.tdx.enabled + value: "true" + - name: kbs.tdx.collateralService + value: "https://pccs-service.intel-dcap.svc.cluster.local:8042/sgx/certification/v4/" + - name: kbs.secretResources[1].name + value: passphrase + - name: kbs.secretResources[1].key + value: secret/data/hub/passphrase + - name: kbs.gpu.enabled + value: "true" + + storage: + name: storage + namespace: openshift-storage + project: hub + path: charts/hub/storage + + baremetal: + name: baremetal + namespace: baremetal + project: hub + path: charts/all/baremetal + + sandbox: + name: sandbox + namespace: openshift-sandboxed-containers-operator + project: sandbox + chart: sandboxed-containers + chartVersion: 0.2.* + overrides: + - name: global.secretStore.backend + value: vault + - name: secretStore.name + value: vault-backend + - name: secretStore.kind + value: ClusterSecretStore + - name: enablePeerPods + value: "false" + + + intel-dcap: + name: intel-dcap + namespace: intel-dcap + project: hub + path: charts/all/intel-dcap + overrides: + - name: secretStore.name + value: vault-backend + - name: secretStore.kind + value: ClusterSecretStore + + nvidia-gpu: + name: nvidia-gpu + namespace: nvidia-gpu-operator + project: hub + path: charts/all/nvidia-gpu + + gpu-workload: + name: gpu-workload + namespace: gpu-workload + project: workloads + path: charts/coco-supported/gpu-workload + syncPolicy: + automated: + prune: true + + sandbox-policies: + name: sandbox-policies + namespace: openshift-sandboxed-containers-operator + chart: sandboxed-policies + chartVersion: 0.1.* + + kbs-access: + name: kbs-access + namespace: kbs-access + project: workloads + path: charts/coco-supported/kbs-access + syncPolicy: + automated: + prune: true + overrides: + - name: defaultMemory + value: "8192" + + hello-openshift: + name: hello-openshift + namespace: hello-openshift + project: workloads + path: charts/coco-supported/hello-openshift + syncPolicy: + automated: + prune: true + + kyverno: + name: kyverno + namespace: kyverno + project: hub + repoURL: https://kyverno.github.io/kyverno/ + chart: kyverno + chartVersion: 3.7.* + syncPolicy: + automated: {} + retry: + limit: 20 + syncOptions: + - ServerSideApply=true + overrides: + - name: admissionController.container.securityContext + value: "null" + - name: admissionController.initContainer.securityContext + value: "null" + - name: backgroundController.securityContext + value: "null" + - name: cleanupController.securityContext + value: "null" + - name: reportsController.securityContext + value: "null" + - name: crds.migration.securityContext + value: "null" + - name: webhooksCleanup.securityContext + value: "null" + - name: test.securityContext + value: "null" + - name: crds.groups.wgpolicyk8s.policyreports + value: "false" + - name: crds.groups.wgpolicyk8s.clusterpolicyreports + value: "false" + - name: reportsController.enabled + value: "false" + + coco-kyverno-policies: + name: coco-kyverno-policies + namespace: openshift-sandboxed-containers-operator + project: sandbox + path: charts/all/coco-kyverno-policies + + imperative: + # NOTE: We *must* use lists and not hashes. As hashes lose ordering once parsed by helm + # The default schedule is every 10 minutes: imperative.schedule + # Total timeout of all jobs is 1h: imperative.activeDeadlineSeconds + # imagePullPolicy is set to always: imperative.imagePullPolicy + # For additional overrides that apply to the jobs, please refer to + # https://validatedpatterns.io/imperative-actions/#additional-job-customizations + image: ghcr.io/butler54/imperative-container:latest + serviceAccountCreate: true + adminServiceAccountCreate: true + serviceAccountName: imperative-admin-sa + jobs: + - name: install-deps + playbook: ansible/install-deps.yaml + verbosity: -vvv + timeout: 3600 + - name: init-data-gzipper + playbook: ansible/init-data-gzipper.yaml + verbosity: -vvv + timeout: 3600 + - name: reconcile-kataconfig-gpu + playbook: ansible/reconcile-kataconfig-gpu.yaml + verbosity: -vvv + timeout: 600 + # Required for tech preview only. + # - name: detect-runtime-class + # playbook: ansible/detect-runtime-class.yaml + # verbosity: -vvv + # timeout: 600 diff --git a/values-baremetal.yaml b/values-baremetal.yaml index 65d5eaf6..8069d108 100644 --- a/values-baremetal.yaml +++ b/values-baremetal.yaml @@ -20,8 +20,6 @@ clusterGroup: - openshift-nfd - baremetal - intel-dcap - - nvidia-gpu-operator - - gpu-workload - kyverno subscriptions: @@ -62,13 +60,6 @@ clusterGroup: name: nfd namespace: openshift-nfd channel: stable - gpu-operator: - name: gpu-operator-certified - namespace: nvidia-gpu-operator - source: certified-operators - channel: v26.3 - installPlanApproval: Manual - csv: gpu-operator-certified.v26.3.0 intel-device-plugins: name: intel-device-plugins-operator namespace: openshift-operators @@ -87,7 +78,7 @@ clusterGroup: # We can use self-referential variables because the chart calls the tpl function with these variables defined sharedValueFiles: - '/overrides/values-{{ $.Values.global.clusterPlatform }}.yaml' - - '/overrides/values-storage-{{ $.Values.global.storageProvider }}.yaml' + - '/overrides/values-storage-{{ $.Values.global.baremetalStorageProvider }}.yaml' applications: acm: @@ -137,8 +128,6 @@ clusterGroup: value: passphrase - name: kbs.secretResources[1].key value: secret/data/hub/passphrase - - name: kbs.gpu.enabled - value: "true" storage: name: storage @@ -180,21 +169,6 @@ clusterGroup: - name: secretStore.kind value: ClusterSecretStore - nvidia-gpu: - name: nvidia-gpu - namespace: nvidia-gpu-operator - project: hub - path: charts/all/nvidia-gpu - - gpu-workload: - name: gpu-workload - namespace: gpu-workload - project: workloads - path: charts/coco-supported/gpu-workload - syncPolicy: - automated: - prune: true - sandbox-policies: name: sandbox-policies namespace: openshift-sandboxed-containers-operator @@ -285,10 +259,6 @@ clusterGroup: playbook: ansible/init-data-gzipper.yaml verbosity: -vvv timeout: 3600 - - name: reconcile-kataconfig-gpu - playbook: ansible/reconcile-kataconfig-gpu.yaml - verbosity: -vvv - timeout: 600 # Required for tech preview only. # - name: detect-runtime-class # playbook: ansible/detect-runtime-class.yaml diff --git a/values-global.yaml b/values-global.yaml index 7cfbda9f..ea6a5428 100644 --- a/values-global.yaml +++ b/values-global.yaml @@ -1,6 +1,6 @@ global: pattern: coco-pattern - storageProvider: hpp # Options: hpp, lvm, external + baremetalStorageProvider: hpp # Options: hpp, lvm, external secretStore: # Warning: This must be present even if it is set to none. backend: vault # none, vault, kubernetes @@ -14,7 +14,6 @@ global: coco: securityPolicyFlavour: "insecure" # insecure, signed or reject is expected. secured: true # true or false. If true, the cluster will be secured. If false, the cluster will be insecure. - bypassAttestation: false # Enable SSH key injection into podvm for debugging. Do not enable in production. # Also requires: COCO_ENABLE_SSH_DEBUG=true ./scripts/gen-secrets.sh # and uncommenting the sshKey block in values-secret.yaml.template. @@ -26,7 +25,7 @@ main: # WARNING # This default configuration uses a single cluster on azure. # It fundamentally violates the separation of duties. - clusterGroupName: baremetal + clusterGroupName: baremetal-gpu multiSourceConfig: enabled: true clusterGroupChartVersion: 0.9.* From 40f41ae83d404b7d38b77b18f6d197c04594af31 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Tue, 28 Apr 2026 11:19:41 +0800 Subject: [PATCH 18/24] feat: correct BM deployment Signed-off-by: Chris Butler --- .../nvidia-gpu/templates/cluster-policy.yaml | 98 ++++++++++++++++++- 1 file changed, 94 insertions(+), 4 deletions(-) diff --git a/charts/all/nvidia-gpu/templates/cluster-policy.yaml b/charts/all/nvidia-gpu/templates/cluster-policy.yaml index c2765174..595690d7 100644 --- a/charts/all/nvidia-gpu/templates/cluster-policy.yaml +++ b/charts/all/nvidia-gpu/templates/cluster-policy.yaml @@ -8,6 +8,76 @@ metadata: spec: operator: defaultRuntime: crio + runtimeClass: nvidia + use_ocp_driver_toolkit: true + + daemonsets: + rollingUpdate: + maxUnavailable: "1" + updateStrategy: RollingUpdate + + driver: + enabled: true + useNvidiaDriverCRD: false + kernelModuleType: auto + licensingConfig: + nlsEnabled: true + secretName: "" + certConfig: + name: "" + kernelModuleConfig: + name: "" + repoConfig: + configMapName: "" + virtualTopology: + config: "" + upgradePolicy: + autoUpgrade: true + maxParallelUpgrades: 1 + maxUnavailable: 25% + drain: + deleteEmptyDir: false + enable: false + force: false + timeoutSeconds: 300 + podDeletion: + deleteEmptyDir: false + force: false + timeoutSeconds: 300 + waitForCompletion: + timeoutSeconds: 0 + + devicePlugin: + enabled: false + config: + name: "" + default: "" + mps: + root: /run/nvidia/mps + + dcgm: + enabled: true + + dcgmExporter: + enabled: true + config: + name: "" + serviceMonitor: + enabled: true + + gfd: + enabled: true + + nodeStatusExporter: + enabled: true + + toolkit: + enabled: true + installDir: /usr/local/nvidia + + validator: + plugin: + env: [] sandboxWorkloads: enabled: true @@ -16,6 +86,8 @@ spec: kataManager: enabled: false + config: + artifactsDir: /opt/nvidia-gpu-operator/artifacts/runtimeclasses ccManager: enabled: {{ .Values.ccManager.enabled }} @@ -36,17 +108,35 @@ spec: sandboxDevicePlugin: enabled: true - driver: + vfioManager: enabled: true - devicePlugin: + vgpuManager: enabled: false - vfioManager: + vgpuDeviceManager: enabled: true + config: + default: default - gfd: + gdrcopy: + enabled: false + + gds: + enabled: false + + mig: + strategy: single + + migManager: + enabled: true + config: + default: all-disabled + + cdi: + default: false enabled: true + nriPluginEnabled: false nfd: nodefeaturerules: true From 03ff51dfb4229c220855921cd1568643c69aa153 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Tue, 28 Apr 2026 13:52:50 +0800 Subject: [PATCH 19/24] fix: align ClusterPolicy with Red Hat OCP 4.21 CC GPU reference MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rewrite ClusterPolicy to match the official Red Hat OCP 4.21.9+ documentation for NVIDIA confidential GPU support. Key changes: - Remove hardcoded cc-manager v0.1.0 (was from old v25.3.x line, caused IntelRootPort crash) — let GPU Operator manage its versions - Remove hardcoded CC_CAPABLE_DEVICE_IDS and sandbox device plugin image/version — operator fills in correct defaults - Disable host-side components not needed for CC passthrough (driver, dcgm, toolkit, migManager) — driver runs inside kata VM via initrd - Add kataSandboxDevicePlugin env vars (P_GPU_ALIAS, NVSWITCH_ALIAS) - Add vfioManager BIND_NVSWITCHES env var - Add both amd_iommu=on and intel_iommu=on to IOMMU MachineConfig Co-Authored-By: Claude Opus 4.6 --- .../nvidia-gpu/templates/cluster-policy.yaml | 168 +++++++----------- .../all/nvidia-gpu/templates/iommu-mco.yaml | 2 +- charts/all/nvidia-gpu/values.yaml | 4 - 3 files changed, 70 insertions(+), 104 deletions(-) diff --git a/charts/all/nvidia-gpu/templates/cluster-policy.yaml b/charts/all/nvidia-gpu/templates/cluster-policy.yaml index 595690d7..3f9119b2 100644 --- a/charts/all/nvidia-gpu/templates/cluster-policy.yaml +++ b/charts/all/nvidia-gpu/templates/cluster-policy.yaml @@ -6,138 +6,108 @@ metadata: annotations: argocd.argoproj.io/sync-wave: "110" spec: - operator: - defaultRuntime: crio - runtimeClass: nvidia - use_ocp_driver_toolkit: true - + ccManager: + defaultMode: {{ .Values.ccManager.defaultMode | quote }} + enabled: {{ .Values.ccManager.enabled }} + cdi: + default: false + enabled: true + nriPluginEnabled: false daemonsets: rollingUpdate: - maxUnavailable: "1" + maxUnavailable: '1' updateStrategy: RollingUpdate - + dcgm: + enabled: false + dcgmExporter: + config: + name: '' + enabled: false + serviceMonitor: + enabled: true + devicePlugin: + config: + default: '' + name: '' + enabled: false + mps: + root: /run/nvidia/mps driver: - enabled: true - useNvidiaDriverCRD: false + certConfig: + name: '' + enabled: false + kernelModuleConfig: + name: '' kernelModuleType: auto licensingConfig: + configMapName: '' nlsEnabled: true - secretName: "" - certConfig: - name: "" - kernelModuleConfig: - name: "" repoConfig: - configMapName: "" - virtualTopology: - config: "" + configMapName: '' upgradePolicy: autoUpgrade: true - maxParallelUpgrades: 1 - maxUnavailable: 25% drain: deleteEmptyDir: false enable: false force: false timeoutSeconds: 300 + maxParallelUpgrades: 1 + maxUnavailable: 25% podDeletion: deleteEmptyDir: false force: false timeoutSeconds: 300 waitForCompletion: timeoutSeconds: 0 - - devicePlugin: + useNvidiaDriverCRD: false + useOpenKernelModules: false + virtualTopology: + config: '' + gdrcopy: + enabled: false + gds: enabled: false - config: - name: "" - default: "" - mps: - root: /run/nvidia/mps - - dcgm: - enabled: true - - dcgmExporter: - enabled: true - config: - name: "" - serviceMonitor: - enabled: true - gfd: enabled: true - + kataManager: + enabled: false + mig: + strategy: single + migManager: + enabled: false nodeStatusExporter: enabled: true - - toolkit: - enabled: true - installDir: /usr/local/nvidia - - validator: - plugin: - env: [] - + operator: + defaultRuntime: crio + initContainer: {} + runtimeClass: nvidia + use_ocp_driver_toolkit: true + kataSandboxDevicePlugin: + enabled: {{ .Values.kataSandboxDevicePlugin.enabled }} + env: + - name: P_GPU_ALIAS + value: pgpu + - name: NVSWITCH_ALIAS + value: nvswitch sandboxWorkloads: + defaultWorkload: vm-passthrough enabled: true mode: kata - defaultWorkload: vm-passthrough - - kataManager: + toolkit: enabled: false - config: - artifactsDir: /opt/nvidia-gpu-operator/artifacts/runtimeclasses - - ccManager: - enabled: {{ .Values.ccManager.enabled }} - defaultMode: {{ .Values.ccManager.defaultMode | quote }} - repository: nvcr.io/nvidia/cloud-native - image: k8s-cc-manager - version: v0.1.0 - env: - - name: CC_CAPABLE_DEVICE_IDS - value: {{ .Values.ccManager.deviceIDs | quote }} - - kataSandboxDevicePlugin: - enabled: {{ .Values.kataSandboxDevicePlugin.enabled }} - repository: {{ .Values.kataSandboxDevicePlugin.repository }} - image: {{ .Values.kataSandboxDevicePlugin.image }} - version: {{ .Values.kataSandboxDevicePlugin.version | quote }} - - sandboxDevicePlugin: - enabled: true - + installDir: /usr/local/nvidia + validator: + plugin: + env: + - name: WITH_WORKLOAD + value: 'false' vfioManager: enabled: true - - vgpuManager: - enabled: false - + env: + - name: BIND_NVSWITCHES + value: 'true' vgpuDeviceManager: - enabled: true - config: - default: default - - gdrcopy: enabled: false - - gds: + vgpuManager: enabled: false - - mig: - strategy: single - - migManager: - enabled: true - config: - default: all-disabled - - cdi: - default: false - enabled: true - nriPluginEnabled: false - - nfd: - nodefeaturerules: true {{- end }} diff --git a/charts/all/nvidia-gpu/templates/iommu-mco.yaml b/charts/all/nvidia-gpu/templates/iommu-mco.yaml index 3d1508f3..d2546386 100644 --- a/charts/all/nvidia-gpu/templates/iommu-mco.yaml +++ b/charts/all/nvidia-gpu/templates/iommu-mco.yaml @@ -9,7 +9,7 @@ metadata: name: 100-iommu-{{ . }} spec: kernelArguments: + - amd_iommu=on - intel_iommu=on - - iommu=pt {{- end }} {{- end }} diff --git a/charts/all/nvidia-gpu/values.yaml b/charts/all/nvidia-gpu/values.yaml index 23a2e897..a033824f 100644 --- a/charts/all/nvidia-gpu/values.yaml +++ b/charts/all/nvidia-gpu/values.yaml @@ -3,13 +3,9 @@ enabled: true ccManager: enabled: true defaultMode: "on" - deviceIDs: "0x2331,0x2322" kataSandboxDevicePlugin: enabled: true - repository: nvcr.io/nvidia/cloud-native - image: nvidia-sandbox-device-plugin - version: "v0.0.2" iommu: enabled: true From 7bbea75fcbe66a1ed13238126d72a128cde95ca6 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Tue, 28 Apr 2026 14:10:04 +0800 Subject: [PATCH 20/24] fix: increase kyverno background controller memory to 512Mi The background controller crashes with leader election timeouts at the default 128Mi limit when processing multiple generate policies for cc_init_data propagation across namespaces. Co-Authored-By: Claude Opus 4.6 --- values-baremetal-gpu.yaml | 4 ++++ values-baremetal.yaml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/values-baremetal-gpu.yaml b/values-baremetal-gpu.yaml index b28facc9..7a59c389 100644 --- a/values-baremetal-gpu.yaml +++ b/values-baremetal-gpu.yaml @@ -259,6 +259,10 @@ clusterGroup: value: "false" - name: reportsController.enabled value: "false" + - name: backgroundController.resources.limits.memory + value: "512Mi" + - name: backgroundController.resources.requests.memory + value: "256Mi" coco-kyverno-policies: name: coco-kyverno-policies diff --git a/values-baremetal.yaml b/values-baremetal.yaml index 8069d108..8a127de3 100644 --- a/values-baremetal.yaml +++ b/values-baremetal.yaml @@ -232,6 +232,10 @@ clusterGroup: value: "false" - name: reportsController.enabled value: "false" + - name: backgroundController.resources.limits.memory + value: "512Mi" + - name: backgroundController.resources.requests.memory + value: "256Mi" coco-kyverno-policies: name: coco-kyverno-policies From ca7c81a456df6592eb4a84655be07e77664d6354 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Tue, 28 Apr 2026 15:04:48 +0800 Subject: [PATCH 21/24] fix: switch gpu-workload to debug initdata for attestation debugging GPU attestation fails at trustee ("contraindicated") while TDX CPU attestation passes. Switch to permissive agent policy to allow exec into the kata VM for further investigation. Co-Authored-By: Claude Opus 4.6 --- .../gpu-workload/templates/gpu-vectoradd-deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/coco-supported/gpu-workload/templates/gpu-vectoradd-deployment.yaml b/charts/coco-supported/gpu-workload/templates/gpu-vectoradd-deployment.yaml index 4ad3ea6d..a917ff7e 100644 --- a/charts/coco-supported/gpu-workload/templates/gpu-vectoradd-deployment.yaml +++ b/charts/coco-supported/gpu-workload/templates/gpu-vectoradd-deployment.yaml @@ -14,7 +14,7 @@ spec: labels: app: gpu-vectoradd annotations: - coco.io/initdata-configmap: initdata + coco.io/initdata-configmap: debug-initdata {{- if .Values.defaultMemory }} io.katacontainers.config.hypervisor.default_memory: {{ .Values.defaultMemory | quote }} {{- end }} From 38421d262548bcfd829fc3b5f3402b90e48ce2d1 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Tue, 28 Apr 2026 15:11:11 +0800 Subject: [PATCH 22/24] fix: update vectoradd image to CUDA 12.5.0 for driver compatibility Driver 580.105.08 inside the kata VM provides CUDA 12.8+ runtime. The old cuda11.7.1 sample fails with "CUDA driver version is insufficient for CUDA runtime version". Co-Authored-By: Claude Opus 4.6 --- .../gpu-workload/templates/gpu-vectoradd-deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/coco-supported/gpu-workload/templates/gpu-vectoradd-deployment.yaml b/charts/coco-supported/gpu-workload/templates/gpu-vectoradd-deployment.yaml index a917ff7e..74049499 100644 --- a/charts/coco-supported/gpu-workload/templates/gpu-vectoradd-deployment.yaml +++ b/charts/coco-supported/gpu-workload/templates/gpu-vectoradd-deployment.yaml @@ -22,7 +22,7 @@ spec: runtimeClassName: {{ .Values.runtimeClassName }} containers: - name: cuda-vectoradd - image: nvcr.io/nvidia/k8s/cuda-sample:vectoradd-cuda11.7.1-ubuntu20.04 + image: nvcr.io/nvidia/k8s/cuda-sample:vectoradd-cuda12.5.0 resources: limits: nvidia.com/pgpu: 1 From 5cc8303a6b84696a4b67be12f38be973af42805c Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Tue, 28 Apr 2026 17:30:13 +0800 Subject: [PATCH 23/24] fix: use Red Hat gpu-verifier image for CC GPU workload testing The NVIDIA cuda-sample images (cuda11.7.1, cuda12.5.0) have CUDA runtime version mismatches with the driver inside the kata VM GPU initrd (580.105.08). Switch to the Red Hat gpu-verifier:ubi9 image which bundles CUDA samples built for the correct driver version. Co-Authored-By: Claude Opus 4.6 --- .../templates/gpu-vectoradd-deployment.yaml | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/charts/coco-supported/gpu-workload/templates/gpu-vectoradd-deployment.yaml b/charts/coco-supported/gpu-workload/templates/gpu-vectoradd-deployment.yaml index 74049499..50fd4d55 100644 --- a/charts/coco-supported/gpu-workload/templates/gpu-vectoradd-deployment.yaml +++ b/charts/coco-supported/gpu-workload/templates/gpu-vectoradd-deployment.yaml @@ -21,17 +21,14 @@ spec: spec: runtimeClassName: {{ .Values.runtimeClassName }} containers: - - name: cuda-vectoradd - image: nvcr.io/nvidia/k8s/cuda-sample:vectoradd-cuda12.5.0 + - name: gpu-cc-verifier + image: quay.io/openshift_sandboxed_containers/gpu-verifier:ubi9 + command: ["/bin/bash"] + args: + - -c + - | + /opt/cuda-samples/Samples/0_Introduction/vectorAdd/build/vectorAdd + sleep 36000 resources: limits: nvidia.com/pgpu: 1 - securityContext: - privileged: false - allowPrivilegeEscalation: false - runAsNonRoot: true - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault From 0e565883ddfeb59eb22f3ee63c0275fed171a22a Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Thu, 30 Apr 2026 11:34:01 +0800 Subject: [PATCH 24/24] chore: gitignore changes Signed-off-by: Chris Butler --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index afdac960..8ddaa9d9 100644 --- a/.gitignore +++ b/.gitignore @@ -20,4 +20,5 @@ openshift-install* node_modules .envrc .ansible/ -__pycache__/ \ No newline at end of file +__pycache__/ +LAB.md \ No newline at end of file