From d2be9a07f44b478f37339d9aa37e47c813f1cadc Mon Sep 17 00:00:00 2001 From: Roy Kaufman Date: Thu, 21 May 2026 15:00:32 +0300 Subject: [PATCH 01/15] trustee: Add key pair as a base to use kbs API Signed-off-by: Roy Kaufman --- operator/src/main.rs | 6 ++++ operator/src/trustee.rs | 66 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/operator/src/main.rs b/operator/src/main.rs index 03ff138e..44df2504 100644 --- a/operator/src/main.rs +++ b/operator/src/main.rs @@ -172,6 +172,12 @@ async fn install_trustee_configuration( .context("Failed to create the attestation policy configmap")?; info!("Generated configmap for the attestation policy"); + match trustee::generate_trustee_auth_keys_secret(client.clone(), owner_reference.clone()).await + { + Ok(_) => info!("Generate auth keys for the KBS API",), + Err(e) => error!("Failed to create the auth keys: {e}"), + } + let kbs_port = cluster.spec.trustee_kbs_port; trustee::generate_kbs_service(client.clone(), owner_reference.clone(), kbs_port) .await diff --git a/operator/src/trustee.rs b/operator/src/trustee.rs index d8d63d16..df042070 100644 --- a/operator/src/trustee.rs +++ b/operator/src/trustee.rs @@ -42,6 +42,10 @@ pub(crate) const TRUSTEE_DATA_MAP: &str = "trustee-data"; const ATT_POLICY_MAP: &str = "attestation-policy"; const TRUSTED_AK_KEYS_VOLUME: &str = "trusted-ak-keys"; const TRUSTED_AK_KEYS_DIR: &str = "/etc/tpm/trusted_ak_keys"; +const TRUSTEE_AUTH_SECRET: &str = "trustee-auth"; +const TRUSTEE_AUTH_KEY_DIR: &str = "/opt/trustee/keys"; +const TRUSTEE_AUTH_PUB_KEY: &str = "public.pub"; +const TRUSTEE_AUTH_PRIV_KEY: &str = "private.key"; fn primitive_date_time_to_str(d: &DateTime, s: S) -> Result where @@ -111,6 +115,21 @@ pub async fn update_reference_values(client: Client) -> Result<()> { Ok(()) } +pub struct Ed25519KeyPair { + pub private_key_pem: Vec, + pub public_key_pem: Vec, +} + +fn generate_ed25519_key_pair() -> Result { + let key = openssl::pkey::PKey::generate_ed25519()?; + let private_key_pem = key.private_key_to_pem_pkcs8()?; + let public_key_pem = key.public_key_to_pem()?; + Ok(Ed25519KeyPair { + private_key_pem, + public_key_pem, + }) +} + fn generate_luks_key() -> Result> { // Constraint: 32 bytes b64-encoded, thus 24 let mut pass = [0; 24]; @@ -356,6 +375,35 @@ pub async fn generate_secret( Ok(()) } +pub async fn generate_trustee_auth_keys_secret( + client: Client, + owner_reference: OwnerReference, +) -> Result<()> { + let key_pair = generate_ed25519_key_pair()?; + let data = BTreeMap::from([ + ( + TRUSTEE_AUTH_PRIV_KEY.to_string(), + k8s_openapi::ByteString(key_pair.private_key_pem), + ), + ( + TRUSTEE_AUTH_PUB_KEY.to_string(), + k8s_openapi::ByteString(key_pair.public_key_pem), + ), + ]); + + let secret = Secret { + metadata: ObjectMeta { + name: Some(TRUSTEE_AUTH_SECRET.to_string()), + owner_references: Some(vec![owner_reference]), + ..Default::default() + }, + data: Some(data), + ..Default::default() + }; + create_or_info_if_exists!(client, Secret, secret); + Ok(()) +} + pub async fn generate_attestation_policy( client: Client, owner_reference: OwnerReference, @@ -459,7 +507,7 @@ pub async fn generate_kbs_service( Ok(()) } -fn generate_kbs_volume_templates() -> [(&'static str, &'static str, Volume); 3] { +fn generate_kbs_volume_templates() -> [(&'static str, &'static str, Volume); 4] { [ ( ATT_POLICY_MAP, @@ -494,6 +542,22 @@ fn generate_kbs_volume_templates() -> [(&'static str, &'static str, Volume); 3] ..Default::default() }, ), + ( + TRUSTEE_AUTH_SECRET, + TRUSTEE_AUTH_KEY_DIR, + Volume { + secret: Some(SecretVolumeSource { + secret_name: Some(TRUSTEE_AUTH_SECRET.to_string()), + items: Some(vec![KeyToPath { + key: "public.pub".to_string(), + path: "public.pub".to_string(), + ..Default::default() + }]), + ..Default::default() + }), + ..Default::default() + }, + ), ] } From 27057f097ffeae32d66f4eb6557518e33962d5a9 Mon Sep 17 00:00:00 2001 From: Roy Kaufman Date: Wed, 24 Jun 2026 15:10:22 +0300 Subject: [PATCH 02/15] Bump trustee to v0.20.0 & Synchronize Trustee Resources via KBS API All trustee resources, including the resource policy, attestation policy, RV, LUKS key, and AK, are now synchronized using the KBS API. The main motivation for this is replacing the patch mechanism that causes a restart of the trustee deployment. Also, this abstraction over the trustee backend simplifies the process of upgrading to future versions of trustee. Config: - Update kbs-config.toml to v0.20.0 format - Store reference values in dedicated ConfigMap (trustee-rv-data) instead of trustee-data API-driven resource management: - Add KBS API client functions: send_secret, delete_secret, register_ak, sync_resource_policy, sync_attestation_policy, sync_reference_values - Replace volume-mount approach for secrets with KBS API calls - Add trustee deployment reconciler that syncs policies, reference values, and secrets when deployment becomes available Dependencies: - Add kbs-client and jsonwebtoken crates - Bump trustee image tag to v0.20.0 Signed-off-by: Roy Kaufman --- Cargo.lock | 1269 ++++++++++++++++++++-- Makefile | 4 +- operator/Cargo.toml | 3 + operator/src/attestation_key_register.rs | 4 +- operator/src/kbs-config.toml | 35 +- operator/src/main.rs | 62 +- operator/src/reference_values.rs | 10 +- operator/src/register_server.rs | 12 +- operator/src/test_utils.rs | 25 +- operator/src/tpm.rego | 1 + operator/src/trustee.rs | 759 +++++++------ tests/no_disallowed_crypto.rs | 36 + 12 files changed, 1683 insertions(+), 537 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 47b6098a..ca60593b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,74 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common 0.1.6", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher 0.4.4", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aes" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1fc76eaeac4c9164506c466d4ffdd8ec9d0c5bf57ee97177c4d8eceb3a0e138" +dependencies = [ + "cipher 0.5.2", + "cpubits", + "cpufeatures 0.3.0", + "zeroize", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes 0.8.4", + "cipher 0.4.4", + "ctr", + "ghash", + "subtle", +] + +[[package]] +name = "aes-keywrap" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10b6f24a1f796bc46415a1d0d18dc0a8203ccba088acf5def3291c4f61225522" +dependencies = [ + "aes 0.9.1", + "byteorder", +] + +[[package]] +name = "aes-kw" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41ac571010bd60765c56085a4f1d412012a9be2663b1a2f2b19b49318653fd0d" +dependencies = [ + "aes 0.9.1", + "const-oid 0.10.2", + "zeroize", +] + [[package]] name = "ahash" version = "0.8.12" @@ -146,6 +214,18 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "async-broadcast" version = "0.7.2" @@ -216,6 +296,29 @@ dependencies = [ "uuid", ] +[[package]] +name = "attester" +version = "0.1.0" +source = "git+https://github.com/confidential-containers/guest-components.git?rev=1fcebcb66a3c21b62e852819d5b55212c90eba2a#1fcebcb66a3c21b62e852819d5b55212c90eba2a" +dependencies = [ + "anyhow", + "async-trait", + "base64 0.22.1", + "cfg-if", + "crypto", + "hex", + "kbs-types", + "num-traits", + "serde", + "serde_json", + "serde_with", + "sha2 0.11.0", + "strum 0.28.0", + "thiserror 2.0.18", + "tokio", + "tracing", +] + [[package]] name = "auto_impl" version = "1.3.0" @@ -276,6 +379,29 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "aws-lc-rs" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +dependencies = [ + "aws-lc-sys", + "untrusted 0.7.1", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "axum" version = "0.8.9" @@ -409,6 +535,12 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" +[[package]] +name = "binstring" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0669d5a35b64fdb5ab7fb19cae13148b6b5cbdf4b8247faf54ece47f699c8cef" + [[package]] name = "bitflags" version = "1.3.2" @@ -421,6 +553,17 @@ version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +[[package]] +name = "blake2b_simd" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b79834656f71332577234b50bfc009996f7449e0c056884e6a02492ded0ca2f3" +dependencies = [ + "arrayref", + "arrayvec", + "constant_time_eq", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -439,6 +582,15 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + [[package]] name = "bumpalo" version = "3.19.0" @@ -457,13 +609,25 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +[[package]] +name = "cbor-codec" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e083a023562b37c52837e850131a51b1154cceb9d149f41ee3d386737b140f46" +dependencies = [ + "byteorder", + "libc", +] + [[package]] name = "cc" -version = "1.2.45" +version = "1.2.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35900b6c8d709fb1d854671ae27aeaa9eec2f8b01b364e1619a40da3e6fe2afe" +checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] @@ -473,6 +637,17 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + [[package]] name = "chrono" version = "0.4.45" @@ -487,6 +662,33 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "cipher" version = "0.4.4" @@ -494,7 +696,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ "crypto-common 0.1.6", - "inout", + "inout 0.1.4", +] + +[[package]] +name = "cipher" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" +dependencies = [ + "crypto-common 0.2.2", + "inout 0.2.2", ] [[package]] @@ -554,6 +766,32 @@ dependencies = [ "serde", ] +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "coarsetime" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e58eb270476aa4fc7843849f8a35063e8743b4dbcdf6dd0f8ea0886980c204c2" +dependencies = [ + "libc", + "wasix", + "wasm-bindgen", +] + [[package]] name = "colorchoice" version = "1.0.4" @@ -590,6 +828,15 @@ dependencies = [ "uuid", ] +[[package]] +name = "concat-kdf" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d72c1252426a83be2092dd5884a5f6e3b8e7180f6891b6263d2c21b92ec8816" +dependencies = [ + "digest 0.10.7", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -631,6 +878,41 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "cookie" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + +[[package]] +name = "cookie_store" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b2c103cf610ec6cae3da84a766285b42fd16aad564758459e6ecf128c75206" +dependencies = [ + "cookie", + "document-features", + "idna", + "log", + "publicsuffix", + "serde", + "serde_derive", + "serde_json", + "time", + "url", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -657,6 +939,23 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cose-rust" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a140f41f55ff1f2126aed96961bad2387ae31d7f9bbd0e98ec888073beaac6f" +dependencies = [ + "cbor-codec", + "openssl", + "rand 0.8.5", +] + +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -690,6 +989,37 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto" +version = "0.1.0" +source = "git+https://github.com/confidential-containers/guest-components.git?rev=1fcebcb66a3c21b62e852819d5b55212c90eba2a#1fcebcb66a3c21b62e852819d5b55212c90eba2a" +dependencies = [ + "aes-gcm", + "aes-kw", + "anyhow", + "base64 0.22.1", + "concat-kdf", + "ctr", + "kbs-types", + "openssl", + "p256", + "p521", + "rand 0.10.1", + "rand 0.8.5", + "rsa", + "serde", + "serde_json", + "sha2 0.11.0", + "strum 0.28.0", + "zeroize", +] + [[package]] name = "crypto-bigint" version = "0.5.5" @@ -697,7 +1027,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ "generic-array", - "rand_core", + "rand_core 0.6.4", "subtle", "zeroize", ] @@ -709,6 +1039,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", + "rand_core 0.6.4", "typenum", ] @@ -718,7 +1049,33 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ + "getrandom 0.4.1", "hybrid-array", + "rand_core 0.10.1", +] + +[[package]] +name = "ct-codecs" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49fb0c6640b4507ebd99ff67677009e381ba5eee1d14df78de4a3d16eb123c39" + +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher 0.4.4", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", ] [[package]] @@ -857,6 +1214,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "der" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71fd89660b2dc699704064e59e9dba0147b903e85319429e131620d022be411b" +dependencies = [ + "const-oid 0.10.2", + "zeroize", +] + [[package]] name = "deranged" version = "0.5.5" @@ -864,6 +1231,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" dependencies = [ "powerfmt", + "serde_core", ] [[package]] @@ -951,24 +1319,58 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "dyn-clone" version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "ear" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aec2877d084955915f48086ae07f49f4c89e33e3826d1f7af33b342274f8100" +dependencies = [ + "base64 0.22.1", + "ciborium", + "cose-rust", + "hex", + "jsonwebtoken", + "lazy_static", + "openssl", + "phf", + "serde", + "serde_json", + "thiserror 2.0.18", +] + [[package]] name = "ecdsa" version = "0.16.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ - "der", + "der 0.7.10", "digest 0.10.7", "elliptic-curve", "rfc6979", - "signature", - "spki", + "signature 2.2.0", + "spki 0.7.3", ] [[package]] @@ -977,8 +1379,18 @@ version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" dependencies = [ - "pkcs8", - "signature", + "pkcs8 0.10.2", + "signature 2.2.0", +] + +[[package]] +name = "ed25519-compact" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5c0284a5d4b1a2fae017a9fe55fd7d01699711f1b572493f16593e173ea2801" +dependencies = [ + "ct-codecs", + "getrandom 0.4.1", ] [[package]] @@ -1027,8 +1439,8 @@ dependencies = [ "group", "hkdf", "pem-rfc7468", - "pkcs8", - "rand_core", + "pkcs8 0.10.2", + "rand_core 0.6.4", "sec1", "subtle", "zeroize", @@ -1144,7 +1556,7 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" dependencies = [ - "rand_core", + "rand_core 0.6.4", "subtle", ] @@ -1156,9 +1568,9 @@ checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" [[package]] name = "find-msvc-tools" -version = "0.1.4" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "flate2" @@ -1322,8 +1734,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -1347,10 +1761,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", + "rand_core 0.10.1", "wasip2", "wasip3", + "wasm-bindgen", ] [[package]] @@ -1365,6 +1782,16 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + [[package]] name = "gimli" version = "0.32.3" @@ -1391,6 +1818,19 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "git2" +version = "0.20.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b88256088d75a56f8ecfa070513a775dd9107f6530ef14919dac831af9cfe2b" +dependencies = [ + "bitflags 2.10.0", + "libc", + "libgit2-sys", + "log", + "url", +] + [[package]] name = "glob" version = "0.3.4" @@ -1426,7 +1866,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ "ff", - "rand_core", + "rand_core 0.6.4", "subtle", ] @@ -1468,6 +1908,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -1542,6 +1993,30 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "hmac-sha1-compact" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0b3ba31f6dc772cc8221ce81dbbbd64fa1e668255a6737d95eeace59b5a8823" + +[[package]] +name = "hmac-sha256" +version = "1.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec9d92d097f4749b64e8cc33d924d9f40a2d4eb91402b458014b781f5733d60f" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "hmac-sha512" +version = "1.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "019ece39bbefc17f13f677a690328cb978dbf6790e141a3c24e66372cb38588b" +dependencies = [ + "digest 0.10.7", +] + [[package]] name = "hostname" version = "0.4.1" @@ -1635,6 +2110,7 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" dependencies = [ + "ctutils", "typenum", ] @@ -1973,6 +2449,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "inout" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" +dependencies = [ + "hybrid-array", +] + [[package]] name = "ipnet" version = "2.11.0" @@ -2006,6 +2491,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45" +[[package]] +name = "is_debug" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fe266d2e243c931d8190177f20bf7f24eed45e96f39e87dc49a27b32d12d407" + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -2043,10 +2534,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" dependencies = [ "jiff-static", + "jiff-tzdb-platform", "log", "portable-atomic", "portable-atomic-util", "serde_core", + "windows-sys 0.61.2", ] [[package]] @@ -2060,6 +2553,31 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "jiff-tzdb" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c900ef84826f1338a557697dc8fc601df9ca9af4ac137c7fb61d4c6f2dfd3076" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + [[package]] name = "js-sys" version = "0.3.85" @@ -2107,10 +2625,11 @@ dependencies = [ [[package]] name = "jsonwebtoken" -version = "10.3.0" +version = "10.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0529410abe238729a60b108898784df8984c87f6054c9c4fcacc47e4803c1ce1" +checksum = "eba32bfb4ffdeaca3e34431072faf01745c9b26d25504aa7a6cf5684334fc4fc" dependencies = [ + "aws-lc-rs", "base64 0.22.1", "ed25519-dalek", "getrandom 0.2.16", @@ -2118,12 +2637,65 @@ dependencies = [ "js-sys", "p256", "p384", - "rand", + "pem", + "rand 0.8.5", "rsa", "serde", "serde_json", "sha2 0.10.9", - "signature", + "signature 2.2.0", + "simple_asn1", + "zeroize", +] + +[[package]] +name = "jsonwebtoken-openssl" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53aec0291d5d8b37fbf826ccc23f5473536dad8dafda4161a7b6b56c2b0e0202" +dependencies = [ + "jsonwebtoken", + "openssl", +] + +[[package]] +name = "jwt-simple" +version = "0.12.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3991f54af4b009bb6efe01aa5a4fcce9ca52f3de7a104a3f6b6e2ad36c852c48" +dependencies = [ + "anyhow", + "binstring", + "blake2b_simd", + "coarsetime", + "ct-codecs", + "ed25519-compact", + "hmac-sha1-compact", + "hmac-sha256", + "hmac-sha512", + "k256", + "p256", + "p384", + "rand 0.8.5", + "serde", + "serde_json", + "superboring", + "thiserror 2.0.18", + "zeroize", +] + +[[package]] +name = "k256" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" +dependencies = [ + "cfg-if", + "ecdsa", + "elliptic-curve", + "once_cell", + "sha2 0.10.9", + "signature 2.2.0", ] [[package]] @@ -2146,9 +2718,79 @@ checksum = "d9c6922f6afe80418dd6019818af5d0d34584c371780ff09b9752370c25b4abb" dependencies = [ "base64 0.22.1", "jiff", - "schemars", + "schemars 1.1.0", + "serde", + "serde_json", +] + +[[package]] +name = "kbs-client" +version = "0.1.0" +source = "git+https://github.com/confidential-containers/trustee.git?rev=e65897a9ad4eb3ac69fa2ec75ed831200eb2acd7#e65897a9ad4eb3ac69fa2ec75ed831200eb2acd7" +dependencies = [ + "anyhow", + "base64 0.22.1", + "clap", + "jsonwebtoken", + "kbs_protocol", + "reqwest 0.13.2", + "serde", + "serde_json", + "tokio", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "kbs-types" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "010924a5f65328c9609598c895ea506c723e3c72e528aeaba0a9645d8330b718" +dependencies = [ + "base64 0.22.1", + "ear", + "serde", + "serde_json", + "sha2 0.10.9", + "sm3", + "strum 0.27.2", + "thiserror 2.0.18", +] + +[[package]] +name = "kbs_protocol" +version = "0.1.0" +source = "git+https://github.com/confidential-containers/guest-components.git?rev=1fcebcb66a3c21b62e852819d5b55212c90eba2a#1fcebcb66a3c21b62e852819d5b55212c90eba2a" +dependencies = [ + "anyhow", + "async-trait", + "attester", + "base64 0.22.1", + "crypto", + "jwt-simple", + "kbs-types", + "reqwest 0.13.2", + "resource_uri", "serde", "serde_json", + "serde_json_canonicalizer", + "sha2 0.11.0", + "shadow-rs", + "thiserror 2.0.18", + "tokio", + "tracing", + "url", + "zeroize", +] + +[[package]] +name = "keccak" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e24a010dd405bd7ed803e5253182815b41bf2e6a80cc3bfc066658e03a198aa" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", ] [[package]] @@ -2283,7 +2925,7 @@ dependencies = [ "http 1.4.2", "jiff", "k8s-openapi 0.27.1", - "schemars", + "schemars 1.1.0", "serde", "serde-value", "serde_json", @@ -2302,7 +2944,7 @@ dependencies = [ "jiff", "json-patch", "k8s-openapi 0.28.0", - "schemars", + "schemars 1.1.0", "serde", "serde-value", "serde_json", @@ -2385,12 +3027,36 @@ version = "0.2.184" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" +[[package]] +name = "libgit2-sys" +version = "0.18.5+1.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "005d6ae6eac1912906073e069f7db60b1fa98e052a68227824afe3e3a1c59ca2" +dependencies = [ + "cc", + "libc", + "libz-sys", + "pkg-config", +] + [[package]] name = "libm" version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" +[[package]] +name = "libz-sys" +version = "1.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bc9657773828b90eeb625adff10eeac83cc21bbfd8e23a03eaa8a33c9e28d9" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "lief" version = "0.17.1" @@ -2467,6 +3133,12 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + [[package]] name = "lock_api" version = "0.4.14" @@ -2482,6 +3154,15 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + [[package]] name = "matchit" version = "0.8.4" @@ -2553,6 +3234,33 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "ml-dsa" +version = "0.1.0-rc.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "163f15320f3fba11760c373af52d7f69d638482c2c350d877fb06513b1c3137c" +dependencies = [ + "const-oid 0.10.2", + "crypto-common 0.2.2", + "ctutils", + "hybrid-array", + "module-lattice", + "pkcs8 0.11.0", + "sha3", + "signature 3.0.0", +] + +[[package]] +name = "module-lattice" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c61b87c9683ab7cb1c6871d261ad5479b6b10ceb52c4352aaca3b5d35a8febe" +dependencies = [ + "ctutils", + "hybrid-array", + "num-traits", +] + [[package]] name = "moveit" version = "0.6.0" @@ -2585,6 +3293,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.60.2", +] + [[package]] name = "num-bigint" version = "0.4.6" @@ -2597,25 +3314,25 @@ dependencies = [ [[package]] name = "num-bigint-dig" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82c79c15c05d4bf82b6f5ef163104cc81a760d8e874d38ac50ab67c8877b647b" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" dependencies = [ "lazy_static", "libm", "num-integer", "num-iter", "num-traits", - "rand", + "rand 0.8.5", "smallvec", "zeroize", ] [[package]] name = "num-conv" -version = "0.1.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-derive" @@ -2751,6 +3468,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + [[package]] name = "openssl" version = "0.10.81" @@ -2788,6 +3511,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "openssl-src" +version = "300.6.1+3.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46eb8fb9fb3b61ce1c0f8a026c4c1a0714d3a9e138e7fbde78753ce2babc3846" +dependencies = [ + "cc", +] + [[package]] name = "openssl-sys" version = "0.9.117" @@ -2796,6 +3528,7 @@ checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" dependencies = [ "cc", "libc", + "openssl-src", "pkg-config", "vcpkg", ] @@ -2815,7 +3548,10 @@ dependencies = [ "http 1.4.2", "json-patch", "jsonptr", + "jsonwebtoken", + "jsonwebtoken-openssl", "k8s-openapi 0.28.0", + "kbs-client", "kube 4.0.0", "log", "oci-client", @@ -2879,7 +3615,7 @@ dependencies = [ "ecdsa", "elliptic-curve", "primeorder", - "rand_core", + "rand_core 0.6.4", "sha2 0.10.9", ] @@ -2941,43 +3677,86 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" name = "pest" version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "989e7521a040efde50c3ab6bbadafbe15ab6dc042686926be59ac35d74607df4" +checksum = "989e7521a040efde50c3ab6bbadafbe15ab6dc042686926be59ac35d74607df4" +dependencies = [ + "memchr", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "187da9a3030dbafabbbfb20cb323b976dc7b7ce91fcd84f2f74d6e31d378e2de" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49b401d98f5757ebe97a26085998d6c0eecec4995cad6ab7fc30ffdf4b052843" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "pest_meta" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f27a2cfee9f9039c4d86faa5af122a0ac3851441a34865b8a043b46be0065a" +dependencies = [ + "pest", + "sha2 0.10.9", +] + +[[package]] +name = "phf" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" dependencies = [ - "memchr", - "ucd-trie", + "phf_macros", + "phf_shared", + "serde", ] [[package]] -name = "pest_derive" -version = "2.8.3" +name = "phf_generator" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "187da9a3030dbafabbbfb20cb323b976dc7b7ce91fcd84f2f74d6e31d378e2de" +checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" dependencies = [ - "pest", - "pest_generator", + "fastrand", + "phf_shared", ] [[package]] -name = "pest_generator" -version = "2.8.3" +name = "phf_macros" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49b401d98f5757ebe97a26085998d6c0eecec4995cad6ab7fc30ffdf4b052843" +checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" dependencies = [ - "pest", - "pest_meta", + "phf_generator", + "phf_shared", "proc-macro2", "quote", "syn 2.0.117", ] [[package]] -name = "pest_meta" -version = "2.8.3" +name = "phf_shared" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f27a2cfee9f9039c4d86faa5af122a0ac3851441a34865b8a043b46be0065a" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" dependencies = [ - "pest", - "sha2 0.10.9", + "siphasher", ] [[package]] @@ -3018,9 +3797,9 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der", - "pkcs8", - "spki", + "der 0.7.10", + "pkcs8 0.10.2", + "spki 0.7.3", ] [[package]] @@ -3029,8 +3808,18 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der", - "spki", + "der 0.7.10", + "spki 0.7.3", +] + +[[package]] +name = "pkcs8" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" +dependencies = [ + "der 0.8.0", + "spki 0.8.0", ] [[package]] @@ -3039,6 +3828,18 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + [[package]] name = "portable-atomic" version = "1.11.1" @@ -3152,6 +3953,22 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "psl-types" +version = "2.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac" + +[[package]] +name = "publicsuffix" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42ea446cab60335f76979ec15e12619a2165b5ae2c12166bef27d283a9fadf" +dependencies = [ + "idna", + "psl-types", +] + [[package]] name = "quote" version = "1.0.45" @@ -3175,7 +3992,18 @@ checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", "rand_chacha", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" +dependencies = [ + "chacha20", + "getrandom 0.4.1", + "rand_core 0.10.1", ] [[package]] @@ -3185,7 +4013,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -3197,6 +4025,12 @@ dependencies = [ "getrandom 0.2.16", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "redox_syscall" version = "0.5.18" @@ -3327,6 +4161,8 @@ checksum = "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801" dependencies = [ "base64 0.22.1", "bytes", + "cookie", + "cookie_store", "futures-core", "futures-util", "http 1.4.2", @@ -3358,6 +4194,17 @@ dependencies = [ "web-sys", ] +[[package]] +name = "resource_uri" +version = "0.1.0" +source = "git+https://github.com/confidential-containers/guest-components.git?rev=1fcebcb66a3c21b62e852819d5b55212c90eba2a#1fcebcb66a3c21b62e852819d5b55212c90eba2a" +dependencies = [ + "anyhow", + "serde", + "serde_json", + "url", +] + [[package]] name = "rfc6979" version = "0.4.0" @@ -3378,15 +4225,15 @@ dependencies = [ "cfg-if", "getrandom 0.2.16", "libc", - "untrusted", + "untrusted 0.9.0", "windows-sys 0.52.0", ] [[package]] name = "rsa" -version = "0.9.8" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78928ac1ed176a5ca1d17e578a1825f3d81ca54cf41053a592584b020cfd691b" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ "const-oid 0.9.6", "digest 0.10.7", @@ -3394,11 +4241,11 @@ dependencies = [ "num-integer", "num-traits", "pkcs1", - "pkcs8", - "rand_core", + "pkcs8 0.10.2", + "rand_core 0.6.4", "sha2 0.10.9", - "signature", - "spki", + "signature 2.2.0", + "spki 0.7.3", "subtle", "zeroize", ] @@ -3495,7 +4342,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ "ring", - "untrusted", + "untrusted 0.9.0", ] [[package]] @@ -3506,7 +4353,7 @@ checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" dependencies = [ "ring", "rustls-pki-types", - "untrusted", + "untrusted 0.9.0", ] [[package]] @@ -3521,6 +4368,12 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "ryu-js" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd29631678d6fb0903b69223673e122c32e9ae559d0960a38d574695ebc0ea15" + [[package]] name = "schannel" version = "0.1.28" @@ -3530,6 +4383,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + [[package]] name = "schemars" version = "1.1.0" @@ -3568,7 +4433,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ "ring", - "untrusted", + "untrusted 0.9.0", ] [[package]] @@ -3578,9 +4443,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ "base16ct", - "der", + "der 0.7.10", "generic-array", - "pkcs8", + "pkcs8 0.10.2", "subtle", "zeroize", ] @@ -3729,6 +4594,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_json_canonicalizer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe52319a927259afbfa5180c5157cd8167edfd3e8c254f9558c7fef44c5649f2" +dependencies = [ + "ryu-js", + "serde", + "serde_json", +] + [[package]] name = "serde_path_to_error" version = "0.1.20" @@ -3761,6 +4637,38 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" +dependencies = [ + "base64 0.22.1", + "bs58", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.14.0", + "schemars 0.9.0", + "schemars 1.1.0", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" +dependencies = [ + "darling 0.23.0", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "serde_yaml" version = "0.9.34+deprecated" @@ -3796,11 +4704,42 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "sha3" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be176f1a57ce4e3d31c1a166222d9768de5954f811601fb7ca06fc8203905ce1" +dependencies = [ + "digest 0.11.3", + "keccak", +] + +[[package]] +name = "shadow-rs" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dd39b4b2077bd36e60ca28c31d494046e747759cb9b507a7d177bb64787c39e" +dependencies = [ + "const_format", + "git2", + "is_debug", + "jiff", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "shlex" -version = "1.3.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" [[package]] name = "signal-hook-registry" @@ -3818,7 +4757,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ "digest 0.10.7", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "signature" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" +dependencies = [ + "digest 0.11.3", + "rand_core 0.10.1", ] [[package]] @@ -3827,12 +4776,39 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" +[[package]] +name = "simple_asn1" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" +dependencies = [ + "num-bigint", + "num-traits", + "thiserror 2.0.18", + "time", +] + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + [[package]] name = "slab" version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" +[[package]] +name = "sm3" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebb9a3b702d0a7e33bc4d85a14456633d2b165c2ad839c5fd9a8417c1ab15860" +dependencies = [ + "digest 0.10.7", +] + [[package]] name = "smallvec" version = "1.15.1" @@ -3878,7 +4854,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der", + "der 0.7.10", +] + +[[package]] +name = "spki" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f" +dependencies = [ + "base64ct", + "der 0.8.0", ] [[package]] @@ -3887,7 +4873,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caac132742f0d33c3af65bfcde7f6aa8f62f0e991d80db99149eb9d44708784f" dependencies = [ - "cipher", + "cipher 0.4.4", "ssh-encoding", ] @@ -3912,11 +4898,11 @@ dependencies = [ "p256", "p384", "p521", - "rand_core", + "rand_core 0.6.4", "rsa", "sec1", "sha2 0.10.9", - "signature", + "signature 2.2.0", "ssh-cipher", "ssh-encoding", "subtle", @@ -3940,6 +4926,9 @@ name = "strum" version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros 0.27.2", +] [[package]] name = "strum" @@ -3980,6 +4969,22 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "superboring" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efc6310a69b44420f3bf53d518077615b7d466cc57df7a80e404e7feb8c510f7" +dependencies = [ + "aes-gcm", + "aes-keywrap", + "getrandom 0.2.16", + "hmac-sha256", + "hmac-sha512", + "ml-dsa", + "rand 0.8.5", + "rsa", +] + [[package]] name = "supports-color" version = "2.1.0" @@ -4162,24 +5167,45 @@ dependencies = [ "syn 3.0.2", ] +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + [[package]] name = "time" -version = "0.3.44" +version = "0.3.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" dependencies = [ "deranged", + "itoa", "num-conv", "powerfmt", - "serde", + "serde_core", "time-core", + "time-macros", ] [[package]] name = "time-core" -version = "0.1.6" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] [[package]] name = "tinystr" @@ -4380,9 +5406,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.41" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ "log", "pin-project-lite", @@ -4392,9 +5418,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", @@ -4403,11 +5429,41 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.34" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" dependencies = [ + "matchers", + "nu-ansi-term", "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", ] [[package]] @@ -4445,7 +5501,7 @@ dependencies = [ "kube 4.0.0", "log", "percent-encoding", - "rand_core", + "rand_core 0.6.4", "serde", "serde_json", "serde_yaml", @@ -4557,12 +5613,28 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common 0.1.6", + "subtle", +] + [[package]] name = "unsafe-libyaml" version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "untrusted" version = "0.9.0" @@ -4605,6 +5677,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "vcpkg" version = "0.2.15" @@ -4650,6 +5728,15 @@ dependencies = [ "wit-bindgen 0.51.0", ] +[[package]] +name = "wasix" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae86f02046da16a333a9129d31451423e1657737ecdafed4193838a5f54c5cfe" +dependencies = [ + "wasi", +] + [[package]] name = "wasm-bindgen" version = "0.2.108" @@ -5275,6 +6362,20 @@ name = "zeroize" version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] [[package]] name = "zerotrie" diff --git a/Makefile b/Makefile index e29e750b..e938e8a9 100644 --- a/Makefile +++ b/Makefile @@ -28,8 +28,10 @@ OPERATOR_IMAGE ?= $(REGISTRY)/trusted-cluster-operator:$(TAG) COMPUTE_PCRS_IMAGE=$(REGISTRY)/compute-pcrs:$(TAG) REG_SERVER_IMAGE=$(REGISTRY)/registration-server:$(TAG) ATTESTATION_KEY_REGISTER_IMAGE=$(REGISTRY)/attestation-key-register:$(TAG) -TRUSTEE_IMAGE ?= quay.io/trusted-execution-clusters/key-broker-service:v0.17.0 + +TRUSTEE_IMAGE ?= quay.io/trusted-execution-clusters/key-broker-service:v0.20.0 TEST_IMAGE ?= quay.io/trusted-execution-clusters/fedora-coreos-kubevirt:42.20260622 + # tagged as 42.20251012.2.0 APPROVED_IMAGE ?= quay.io/trusted-execution-clusters/fedora-coreos@sha256:6997f51fd27d1be1b5fc2e6cc3ebf16c17eb94d819b5d44ea8d6cf5f826ee773 diff --git a/operator/Cargo.toml b/operator/Cargo.toml index 1496094a..507029f3 100644 --- a/operator/Cargo.toml +++ b/operator/Cargo.toml @@ -31,6 +31,9 @@ serde_json.workspace = true thiserror = "2.0.18" tokio.workspace = true toml = "1.1.2" +kbs-client = {git = "https://github.com/confidential-containers/trustee.git", rev = "e65897a9ad4eb3ac69fa2ec75ed831200eb2acd7", default-features = false, features = ["native-tls"] } +jsonwebtoken = { version = "10.4.0", default-features = false, features = ["use_pem"] } +jsonwebtoken-openssl = "1.0.0" [dev-dependencies] http.workspace = true diff --git a/operator/src/attestation_key_register.rs b/operator/src/attestation_key_register.rs index 332e5124..2fb6955b 100644 --- a/operator/src/attestation_key_register.rs +++ b/operator/src/attestation_key_register.rs @@ -331,7 +331,7 @@ async fn secret_reconcile( match ev { Event::Apply(_secret) => { // On creation/update, just update the trustee deployment volumes - trustee::update_attestation_keys(&ctx) + trustee::update_attestation_keys(ctx.client.clone()) .await .map(|_| Action::await_change()) .map_err(|e| { @@ -345,7 +345,7 @@ async fn secret_reconcile( "AttestationKey secret {secret_name} is being deleted, updating trustee deployment volumes" ); // Update trustee deployment - secrets with deletion_timestamp will be filtered out - trustee::update_attestation_keys(&ctx) + trustee::update_attestation_keys(ctx.client.clone()) .await .map(|_| Action::await_change()) .map_err(|e| { diff --git a/operator/src/kbs-config.toml b/operator/src/kbs-config.toml index fc1005d5..20b4a588 100644 --- a/operator/src/kbs-config.toml +++ b/operator/src/kbs-config.toml @@ -2,36 +2,35 @@ sockets = ["0.0.0.0:8080"] [admin] -type = "DenyAll" +authorization_mode = "AuthenticatedAuthorization" + + [admin.authentication.bearer_jwt] + identity_providers = [ + { public_key_uri = "/opt/trustee/keys/public.pub" } + ] + + [admin.authorization.regex_acl] + acls = [{ role = "admin", allowed_endpoints = "^/kbs/.+$" }] [attestation_token] -insecure_key = true -attestation_token_type = "CoCo" +insecure_header_jwk = true [attestation_service] type = "coco_as_builtin" -work_dir = "/opt/trustee" -policy_engine = "opa" +timeout = 5 [attestation_service.attestation_token_broker] - type = "Ear" - policy_dir = "/opt/trustee/policies" - - [attestation_service.attestation_token_config] duration_min = 5 [attestation_service.rvps_config] type = "BuiltIn" - [attestation_service.rvps_config.storage] - type = "LocalJson" - file_path = "/opt/trustee/reference-values.json" - - [[plugins]] name = "resource" -type = "LocalFs" -dir_path = "/opt/trustee/kbs-repository" +storage_backend_type = "kvstorage" + +[storage_backend] +storage_type = "LocalJson" -[policy_engine] -policy_path = "/opt/trustee/policy.rego" + [storage_backend.backends.local_json] + file_dir_path = "/opt/trustee/storage" diff --git a/operator/src/main.rs b/operator/src/main.rs index 44df2504..dcd14515 100644 --- a/operator/src/main.rs +++ b/operator/src/main.rs @@ -28,12 +28,11 @@ mod register_server; #[cfg(test)] mod test_utils; mod trustee; - use crate::conditions::*; use operator::*; /// Default tag for Trustee image -const TRUSTEE_VERSION: &str = "v0.17.0"; +const TRUSTEE_VERSION: &str = "v0.20.0"; /// Default version tag for operator-managed component images const COMPONENT_VERSION: &str = "v0.2.0"; /// Default registry @@ -167,17 +166,18 @@ async fn install_trustee_configuration( .context("Failed to create the KBS configuration configmap")?; info!("Generated configmap for the KBS configuration"); - trustee::generate_attestation_policy(client.clone(), owner_reference.clone()) + reference_values::create_pcrs_config_map(client.clone()) .await - .context("Failed to create the attestation policy configmap")?; - info!("Generated configmap for the attestation policy"); - - match trustee::generate_trustee_auth_keys_secret(client.clone(), owner_reference.clone()).await - { - Ok(_) => info!("Generate auth keys for the KBS API",), - Err(e) => error!("Failed to create the auth keys: {e}"), - } - + .context("Failed to create the PCRs configmap")?; + info!("Created bare configmap for PCRs"); + trustee::generate_trustee_auth_keys_secret(client.clone(), owner_reference.clone()) + .await + .context("Failed to create the auth keys")?; + info!("Generate auth keys for the KBS API"); + trustee::generate_rv_data(client.clone(), owner_reference.clone()) + .await + .context("Failed to create the reference values configmap")?; + info!("Created configmap for reference values"); let kbs_port = cluster.spec.trustee_kbs_port; trustee::generate_kbs_service(client.clone(), owner_reference.clone(), kbs_port) .await @@ -253,7 +253,7 @@ async fn install_attestation_key_register( #[tokio::main] async fn main() -> Result<()> { env_logger::Builder::from_env(Env::default().default_filter_or("info")).init(); - + let _ = jsonwebtoken_openssl::install_default(); let kube_client = Client::try_default().await?; info!("trusted execution clusters operator",); @@ -284,9 +284,9 @@ async fn main() -> Result<()> { attestation_key_register::launch_ak_controller(ak_ctx.clone()).await; attestation_key_register::launch_machine_ak_controller(ak_ctx.clone()).await; attestation_key_register::launch_secret_ak_controller(ak_ctx).await; - reference_values::create_pcrs_config_map(kube_client.clone()).await?; reference_values::launch_rv_image_controller(kube_client.clone()).await; reference_values::launch_rv_job_controller(kube_client.clone()).await; + trustee::launch_trustee_sync_controller(kube_client.clone()).await; Controller::new(cl, watcher::Config::default()) .run(reconcile, controller_error_policy, ctx) @@ -300,7 +300,7 @@ async fn main() -> Result<()> { mod tests { use http::{Method, Request, StatusCode}; use k8s_openapi::api::apps::v1::Deployment; - use k8s_openapi::api::core::v1::{ConfigMap, Service}; + use k8s_openapi::api::core::v1::{ConfigMap, Secret, Service}; use k8s_openapi::{apimachinery::pkg::apis::meta::v1::Time, jiff::Timestamp}; use kube::api::ObjectList; use kube::client::Body; @@ -442,31 +442,33 @@ mod tests { }; let clos = async |req: Request, ctr| { - if ctr < 8 && req.method() == Method::POST { + if ctr < 10 && req.method() == Method::POST { use serde_json::to_string; let resp = match ctr { - // Trustee - 0 => to_string(&ConfigMap::default()), - 1 => to_string(&ConfigMap::default()), - 2 => to_string(&Service::default()), - 3 => to_string(&Deployment::default()), - // Registration server - 4 => to_string(&Deployment::default()), - 5 => to_string(&Service::default()), - // Attestation key register server - 6 => to_string(&Deployment::default()), - 7 => to_string(&Service::default()), + // install_trustee_configuration + 0 => to_string(&ConfigMap::default()), // trustee-data + 1 => to_string(&ConfigMap::default()), // image-pcrs + 2 => to_string(&Secret::default()), // trustee-auth + 3 => to_string(&ConfigMap::default()), // trustee-rv-data + 4 => to_string(&Service::default()), // kbs-service + 5 => to_string(&Deployment::default()), // trustee-deployment + // install_register_server + 6 => to_string(&Deployment::default()), // register-server + 7 => to_string(&Service::default()), // register-server-svc + // install_attestation_key_register + 8 => to_string(&Deployment::default()), // ak-register + 9 => to_string(&Service::default()), // ak-register-svc _ => unreachable!("unexpected counter {ctr}"), }; Ok(resp.unwrap()) - } else if ctr == 8 && req.method() == Method::GET { + } else if ctr == 10 && req.method() == Method::GET { let object_list = ObjectList:: { items: Vec::new(), types: Default::default(), metadata: Default::default(), }; Ok(serde_json::to_string(&object_list).unwrap()) - } else if ctr == 9 && req.method() == Method::PATCH { + } else if ctr == 11 && req.method() == Method::PATCH { let body = req.into_body().collect_bytes().await.unwrap().to_vec(); let body = String::from_utf8_lossy(&body); assert!(body.contains("ForeignCondition"),); @@ -497,7 +499,7 @@ mod tests { cluster.status = Some(TrustedExecutionClusterStatus { conditions: Some(vec![pre_existing_installed, foreign_condition]), }); - count_check!(10, clos, |client| { + count_check!(12, clos, |client| { let result = reconcile(Arc::new(cluster), Arc::new(dummy_cluster_ctx(client))).await; assert_eq!(result.unwrap(), Action::await_change()); }); diff --git a/operator/src/reference_values.rs b/operator/src/reference_values.rs index 4077d9bb..636e9e94 100644 --- a/operator/src/reference_values.rs +++ b/operator/src/reference_values.rs @@ -495,12 +495,13 @@ mod tests { Ok(serde_json::to_string(&dummy_pcrs_map()).unwrap()) } (2, &Method::GET) | (3, &Method::PUT) => { - assert!(req.uri().path().contains(trustee::TRUSTEE_DATA_MAP)); + assert!(req.uri().path().contains(trustee::TRUSTEE_RV_MAP)); Ok(serde_json::to_string(&dummy_trustee_map()).unwrap()) } + (4, &Method::GET) => Err(StatusCode::NOT_FOUND), _ => panic!("unexpected API interaction: {req:?}, counter {ctr}"), }; - count_check!(4, clos, |client| { + count_check!(5, clos, |client| { let job = Arc::new(dummy_job()); let result = job_reconcile(job, Arc::new(client)).await.unwrap(); assert_eq!(result, Action::await_change()); @@ -612,12 +613,13 @@ mod tests { Ok(serde_json::to_string(&dummy_pcrs_map()).unwrap()) } (3, &Method::GET) | (4, &Method::PUT) => { - assert!(req.uri().path().contains(trustee::TRUSTEE_DATA_MAP)); + assert!(req.uri().path().contains(trustee::TRUSTEE_RV_MAP)); Ok(serde_json::to_string(&dummy_trustee_map()).unwrap()) } + (5, &Method::GET) => Err(StatusCode::NOT_FOUND), _ => panic!("unexpected API interaction: {req:?}, counter {ctr}"), }; - count_check!(5, clos, |client| { + count_check!(6, clos, |client| { assert!(image_remove_reconcile(client, image, cluster).await.is_ok()); }); } diff --git a/operator/src/register_server.rs b/operator/src/register_server.rs index 2a57ded2..96b689e3 100644 --- a/operator/src/register_server.rs +++ b/operator/src/register_server.rs @@ -139,7 +139,7 @@ async fn keygen_reconcile( async { let owner_reference = generate_owner_reference(&Arc::unwrap_or_clone(machine))?; trustee::generate_secret(kube_client.clone(), id, owner_reference).await?; - trustee::mount_secret(kube_client, id).await + trustee::send_secret(kube_client, id).await } .await .map(|_| Action::await_change()) @@ -185,10 +185,12 @@ async fn keygen_reconcile( } } - trustee::unmount_secret(kube_client, id) - .await - .map(|_| Action::await_change()) - .map_err(|e| finalizer::Error::::CleanupFailed(e.into())) + trustee::delete_secret(kube_client, id).await.map_err(|e| { + finalizer::Error::::CleanupFailed( + anyhow!("failed to delete secret for machine {id}: {e}").into(), + ) + })?; + Ok(Action::await_change()) } } }) diff --git a/operator/src/test_utils.rs b/operator/src/test_utils.rs index af4ed28c..96e5512d 100644 --- a/operator/src/test_utils.rs +++ b/operator/src/test_utils.rs @@ -3,7 +3,10 @@ // SPDX-License-Identifier: MIT use compute_pcrs_lib::Pcr; -use k8s_openapi::{api::core::v1::ConfigMap, jiff::Timestamp}; +use k8s_openapi::{ + api::core::v1::{ConfigMap, Secret}, + jiff::Timestamp, +}; use std::collections::BTreeMap; use crate::trustee; @@ -41,6 +44,26 @@ pub fn dummy_trustee_map() -> ConfigMap { } } +pub fn dummy_trustee_auth() -> Secret { + let key_pair = + trustee::generate_ed25519_key_pair().expect("Failed to generate ed25519 key pair"); + let data = BTreeMap::from([ + ( + trustee::TRUSTEE_AUTH_PRIV_KEY.to_string(), + k8s_openapi::ByteString(key_pair.private_key_pem), + ), + ( + trustee::TRUSTEE_AUTH_PUB_KEY.to_string(), + k8s_openapi::ByteString(key_pair.public_key_pem), + ), + ]); + + Secret { + data: Some(data), + ..Default::default() + } +} + pub fn dummy_pcrs_map() -> ConfigMap { let data = BTreeMap::from([( PCR_CONFIG_FILE.to_string(), diff --git a/operator/src/tpm.rego b/operator/src/tpm.rego index abf6ea6d..f94b74a5 100644 --- a/operator/src/tpm.rego +++ b/operator/src/tpm.rego @@ -9,6 +9,7 @@ executables := 3 if { input.tpm.pcr04 in query_reference_value("tpm_pcr4") input.tpm.pcr14 in query_reference_value("tpm_pcr14") + input.tpm.ak_public in query_reference_value("trusted_aks") } # Azure SNP vTPM validation executables := 3 if { diff --git a/operator/src/trustee.rs b/operator/src/trustee.rs index df042070..ab179530 100644 --- a/operator/src/trustee.rs +++ b/operator/src/trustee.rs @@ -4,48 +4,56 @@ // // SPDX-License-Identifier: MIT -use crate::attestation_key_register::AkContextData; use anyhow::{Context, Result}; use base64::{Engine as _, engine::general_purpose}; use chrono::{DateTime, Utc}; use clevis_pin_trustee_lib::Key as ClevisKey; +use futures_util::StreamExt; use k8s_openapi::api::apps::v1::{Deployment, DeploymentSpec}; use k8s_openapi::api::core::v1::{ - ConfigMap, ConfigMapVolumeSource, Container, ContainerPort, EmptyDirVolumeSource, EnvVar, - KeyToPath, PodSpec, PodTemplateSpec, ProjectedVolumeSource, Secret, SecretProjection, - SecretVolumeSource, Service, ServicePort, ServiceSpec, Volume, VolumeMount, VolumeProjection, + ConfigMap, ConfigMapVolumeSource, Container, ContainerPort, EnvVar, KeyToPath, PodSpec, + PodTemplateSpec, Secret, SecretVolumeSource, Service, ServicePort, ServiceSpec, Volume, + VolumeMount, }; use k8s_openapi::apimachinery::pkg::{ apis::meta::v1::{LabelSelector, OwnerReference}, util::intstr::IntOrString, }; + use kube::{ Api, Client, Resource, - api::{ObjectMeta, Patch, PatchParams}, - runtime::reflector::ObjectRef, + api::ObjectMeta, + runtime::{ + controller::{Action, Controller}, + watcher, + }, +}; +use log::{info, warn}; +use operator::{ + ControllerError, TLS_DIR, controller_error_policy, controller_info, create_or_info_if_exists, + read_certificate, }; -use log::info; -use operator::{TLS_DIR, create_or_info_if_exists, read_certificate}; -use serde::{Serialize, Serializer}; + +use jsonwebtoken::{Algorithm, EncodingKey, Header, encode}; +use serde::{Deserialize, Serialize, Serializer}; use serde_json::{Value::String as JsonString, json}; use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::Duration; -use trusted_cluster_operator_lib::endpoints::*; use trusted_cluster_operator_lib::reference_values::*; +use trusted_cluster_operator_lib::{Machine, endpoints::*}; -const TRUSTEE_DATA_DIR: &str = "/opt/trustee"; -pub const TRUSTEE_SECRETS_PATH: &str = "/opt/trustee/kbs-repository/default"; +const TRUSTEE_DATA_DIR: &str = "/etc/kbs"; const KBS_CONFIG_FILE: &str = "kbs-config.toml"; -pub(crate) const REFERENCE_VALUES_FILE: &str = "reference-values.json"; pub(crate) const TRUSTEE_DATA_MAP: &str = "trustee-data"; -const ATT_POLICY_MAP: &str = "attestation-policy"; -const TRUSTED_AK_KEYS_VOLUME: &str = "trusted-ak-keys"; -const TRUSTED_AK_KEYS_DIR: &str = "/etc/tpm/trusted_ak_keys"; +pub(crate) const TRUSTEE_RV_MAP: &str = "trustee-rv-data"; +pub(crate) const REFERENCE_VALUES_FILE: &str = "reference-values.json"; const TRUSTEE_AUTH_SECRET: &str = "trustee-auth"; const TRUSTEE_AUTH_KEY_DIR: &str = "/opt/trustee/keys"; -const TRUSTEE_AUTH_PUB_KEY: &str = "public.pub"; -const TRUSTEE_AUTH_PRIV_KEY: &str = "private.key"; +pub(crate) const TRUSTEE_AUTH_PUB_KEY: &str = "public.pub"; +pub(crate) const TRUSTEE_AUTH_PRIV_KEY: &str = "private.key"; fn primitive_date_time_to_str(d: &DateTime, s: S) -> Result where @@ -58,7 +66,7 @@ where /// reference_value_provider_service::reference_value::ReferenceValue /// (cannot import directly because its expiration doesn't serialize /// right) -#[derive(Serialize)] +#[derive(Serialize, Deserialize)] struct ReferenceValue { pub version: String, pub name: String, @@ -97,30 +105,70 @@ fn recompute_reference_values(image_pcrs: ImagePcrs) -> Vec { } pub async fn update_reference_values(client: Client) -> Result<()> { - let config_maps: Api = Api::default_namespaced(client); + let config_maps: Api = Api::default_namespaced(client.clone()); let image_pcrs_map = config_maps.get(PCR_CONFIG_MAP).await?; let reference_values = recompute_reference_values(get_image_pcrs(image_pcrs_map)?); let rv_json = serde_json::to_string(&reference_values)?; - let mut trustee_map = config_maps.get(TRUSTEE_DATA_MAP).await?; - let err = format!("ConfigMap {TRUSTEE_DATA_MAP} existed, but had no data"); - let trustee_data = trustee_map.data.as_mut().context(err)?; - trustee_data.insert(REFERENCE_VALUES_FILE.to_string(), rv_json); - + let mut rv_map = config_maps.get(TRUSTEE_RV_MAP).await?; + let err = format!("ConfigMap {TRUSTEE_RV_MAP} existed, but had no data"); + let rv_data = rv_map.data.as_mut().context(err)?; + rv_data.insert(REFERENCE_VALUES_FILE.to_string(), rv_json); config_maps - .replace(TRUSTEE_DATA_MAP, &Default::default(), &trustee_map) + .replace(TRUSTEE_RV_MAP, &Default::default(), &rv_map) .await?; + + if let Err(e) = sync_reference_values(&client, &reference_values).await { + warn!( + "Failed to sync reference values to KBS (will retry on next deployment reconcile): {e}" + ); + } info!("Recomputed reference values"); Ok(()) } +async fn sync_reference_values(client: &Client, reference_values: &[ReferenceValue]) -> Result<()> { + let auth_token = get_auth_key_token(client).await?; + let (url, certs) = get_kbs_connection(client).await?; + for rv in reference_values { + kbs_client::set_sample_rv( + url.clone(), + rv.name.clone(), + rv.value.clone(), + Some(auth_token.clone()), + certs.clone(), + ) + .await?; + } + info!("Sent {} reference values to KBS", reference_values.len()); + Ok(()) +} + +async fn sync_reference_values_from_configmap(client: &Client) -> Result<()> { + let config_maps: Api = Api::default_namespaced(client.clone()); + let rv_map = config_maps.get(TRUSTEE_RV_MAP).await?; + let data = rv_map.data.context("RV ConfigMap has no data")?; + let rv_json = match data.get(REFERENCE_VALUES_FILE) { + Some(json) => json, + None => { + info!("No reference values in ConfigMap yet, skipping sync"); + return Ok(()); + } + }; + let reference_values: Vec = serde_json::from_str(rv_json)?; + if reference_values.is_empty() { + return Ok(()); + } + sync_reference_values(client, &reference_values).await +} + pub struct Ed25519KeyPair { pub private_key_pem: Vec, pub public_key_pem: Vec, } -fn generate_ed25519_key_pair() -> Result { +pub fn generate_ed25519_key_pair() -> Result { let key = openssl::pkey::PKey::generate_ed25519()?; let private_key_pem = key.private_key_to_pem_pkcs8()?; let public_key_pem = key.public_key_to_pem()?; @@ -141,88 +189,225 @@ fn generate_luks_key() -> Result> { }; serde_json::to_vec(&jwk).map_err(Into::into) } +async fn get_auth_key_token(client: &Client) -> Result { + let secret_api: Api = Api::default_namespaced(client.clone()); + let auth_secret = secret_api.get(TRUSTEE_AUTH_SECRET).await?; + let auth_data = auth_secret.data.context("Auth secret has no data")?; + let auth_key_bytes = auth_data + .get("private.key") + .context("Auth secret missing private.key")?; + + let claims = json!({ + "role": "admin", + "exp": i32::MAX + }); + + let encoding_key = EncodingKey::from_ed_pem(auth_key_bytes.0.as_slice())?; + + let token = encode(&Header::new(Algorithm::EdDSA), &claims, &encoding_key)?; + Ok(token) +} +async fn get_kbs_connection(client: &Client) -> Result<(String, Vec)> { + let tec = trusted_cluster_operator_lib::get_trusted_execution_cluster(client.clone()).await?; + let secret_api: Api = Api::default_namespaced(client.clone()); + + if let Some(secret_name) = &tec.spec.trustee_secret + && let Ok(secret) = secret_api.get(secret_name).await + && let Some(ca_crt) = secret.data.as_ref().and_then(|d| d.get("ca.crt")) + { + let ca_pem = + String::from_utf8(ca_crt.0.clone()).context("ca certificate is not valid UTF-8")?; + let trustee_addr = format!( + "https://{}", + tec.spec + .public_trustee_addr + .as_ref() + .context("TrustedExecutionCluster missing public_trustee_addr HTTPS")? + ); + return Ok((trustee_addr, vec![ca_pem])); + } -fn generate_secret_volume(id: &str) -> (Volume, VolumeMount) { - ( - Volume { - name: id.to_string(), - secret: Some(SecretVolumeSource { - secret_name: Some(id.to_string()), - ..Default::default() - }), - ..Default::default() - }, - VolumeMount { - name: id.to_string(), - mount_path: format!("{TRUSTEE_SECRETS_PATH}/{id}"), - ..Default::default() - }, + Ok(( + format!( + "http://{}", + tec.spec + .public_trustee_addr + .as_ref() + .context("TrustedExecutionCluster missing public_trustee_addr HTTP")? + ), + vec![], + )) +} + +pub fn secret_path(id: &str) -> String { + format!("default/{id}/root") +} + +pub async fn send_secret(client: Client, id: &str) -> Result<()> { + let secret_api: Api = Api::default_namespaced(client.clone()); + let auth_key_token = get_auth_key_token(&client).await?; + let (url, certs) = get_kbs_connection(&client).await?; + let secret = secret_api.get(id).await?; + let secret_data = secret.data.context("Secret has no data")?; + let resource_bytes = secret_data + .get("root") + .context("Secret missing root key")? + .0 + .clone(); + let path = secret_path(id); + info!("Sending secret {id} to KBS API..."); + kbs_client::set_resource(&url, Some(auth_key_token), resource_bytes, &path, certs).await?; + info!("Secret {id} sent successfully"); + Ok(()) +} + +pub async fn delete_secret(client: Client, id: &str) -> Result<()> { + let auth_key_token = get_auth_key_token(&client).await?; + let (url, certs) = get_kbs_connection(&client).await?; + let path = secret_path(id); + info!("Deleting secret {id} to KBS API..."); + kbs_client::delete_resource(&url, Some(auth_key_token), &path, certs).await?; + info!("Secret {id} deleted successfully"); + Ok(()) +} + +pub async fn register_ak(client: Client, ak_secrets: &[String]) -> Result<()> { + let auth_key_token = get_auth_key_token(&client).await?; + let (url, certs) = get_kbs_connection(&client).await?; + let ak_der: Vec = ak_secrets + .iter() + .map(|ak| { + let der = ak + .lines() + .filter(|l| !l.starts_with("-----")) + .collect::(); + JsonString(der) + }) + .collect(); + info!("Registering AK to KBS API..."); + kbs_client::set_sample_rv( + url.to_string(), + "trusted_aks".to_string(), + serde_json::Value::Array(ak_der), + Some(auth_key_token), + certs, ) + .await?; + info!("AK registered successfully"); + Ok(()) } -pub async fn mount_secret(client: Client, id: &str) -> Result<()> { - let result = do_mount_secret(client, id, true).await; - info!("Mounted secret {id} to {TRUSTEE_DEPLOYMENT}"); - result +pub async fn sync_resource_policy(client: Client) -> Result<()> { + let auth_token = get_auth_key_token(&client).await?; + let (url, certs) = get_kbs_connection(&client).await?; + let policy = include_str!("resource.rego"); + info!("Sending resource policy to KBS API..."); + kbs_client::set_resource_policy(&url, Some(auth_token), policy.as_bytes().to_vec(), certs) + .await?; + info!("Resource policy set successfully"); + Ok(()) } -pub async fn unmount_secret(client: Client, id: &str) -> Result<()> { - let result = do_mount_secret(client, id, false).await; - info!("Unmounted secret {id} from {TRUSTEE_DEPLOYMENT}"); - result +pub async fn sync_attestation_policy(client: Client) -> Result<()> { + let auth_token = get_auth_key_token(&client).await?; + let (url, certs) = get_kbs_connection(&client).await?; + let policy = include_str!("tpm.rego"); + info!("Sending attestation policy to KBS API..."); + kbs_client::set_attestation_policy( + &url, + Some(auth_token), + policy.as_bytes().to_vec(), + Some("rego".to_string()), + Some("default_cpu".to_string()), + certs, + ) + .await?; + info!("Attestation policy set successfully"); + Ok(()) } -pub async fn do_mount_secret(client: Client, id: &str, add: bool) -> Result<()> { - let deployments: Api = Api::default_namespaced(client); - let mut deployment = deployments.get(TRUSTEE_DEPLOYMENT).await?; +pub async fn sync_all_machine_luks_key(client: Client) -> Result<()> { + let machine_api: Api = Api::default_namespaced(client.clone()); + let machine_list = machine_api.list(&Default::default()).await?; - let err = format!("Deployment {TRUSTEE_DEPLOYMENT} existed, but had no spec"); - let depl_spec = deployment.spec.as_mut().context(err)?; - let err = format!("Deployment {TRUSTEE_DEPLOYMENT} existed, but had no pod spec"); - let pod_spec = depl_spec.template.spec.as_mut().context(err)?; - let err = format!("Deployment {TRUSTEE_DEPLOYMENT} existed, but had no containers"); - let container = pod_spec.containers.get_mut(0).context(err)?; - let vol_mounts = container.volume_mounts.get_or_insert_default(); + let machine_ids: Vec = machine_list + .items + .iter() + .map(|machine| machine.spec.id.clone()) + .collect(); - if add { - let (volume, volume_mount) = generate_secret_volume(id); - pod_spec.volumes.get_or_insert_default().push(volume); - vol_mounts.push(volume_mount); - } else { - let vol_result = pod_spec.volumes.as_mut().and_then(|vs| { - let pos = vs.iter().position(|v| v.name == id); - pos.map(|p| vs.swap_remove(p)) - }); - if vol_result.is_none() { - info!("Secret {id} was to be dropped, but volume had already been removed"); + info!("Syncing {} machine luks key to KBS", machine_ids.len()); + for id in &machine_ids { + send_secret(client.clone(), id).await?; + } + Ok(()) +} + +async fn trustee_deployment_reconcile( + deployment: Arc, + client: Arc, +) -> Result { + if let Some(status) = &deployment.status + && let Some(is_available) = &status.conditions + && is_available + .iter() + .any(|c| c.type_ == "Available" && c.status == "True") + { + let c = Arc::unwrap_or_clone(client.clone()); + if let Err(e) = sync_resource_policy(c.clone()).await { + warn!("Failed to sync resource policy to KBS: {e}"); + return Ok(Action::requeue(Duration::from_secs(30))); } - let vol_mount_result = container.volume_mounts.as_mut().and_then(|vms| { - let pos = vms.iter().position(|v| v.name == id); - pos.map(|p| vms.swap_remove(p)) - }); - if vol_mount_result.is_none() { - info!("Secret {id} was to be dropped, but volume mount had already been removed"); + if let Err(e) = sync_attestation_policy(c.clone()).await { + warn!("Failed to sync attestation policy to KBS: {e}"); + return Ok(Action::requeue(Duration::from_secs(30))); + } + if let Err(e) = sync_reference_values_from_configmap(&c).await { + warn!("Failed to sync reference values to KBS: {e}"); + return Ok(Action::requeue(Duration::from_secs(30))); + } + if let Err(e) = sync_all_machine_luks_key(c.clone()).await { + warn!("Failed to sync machine luks keys to KBS: {e}"); + return Ok(Action::requeue(Duration::from_secs(30))); + } + if let Err(e) = update_attestation_keys(c).await { + warn!("Failed to update attestation keys to KBS: {e}"); + return Ok(Action::requeue(Duration::from_secs(30))); } } - deployments - .replace(TRUSTEE_DEPLOYMENT, &Default::default(), &deployment) - .await?; - Ok(()) + Ok(Action::await_change()) } -pub async fn update_attestation_keys(ctx: &AkContextData) -> Result<()> { - let client = &ctx.client; - let ak_secrets: Vec = ctx - .secret_store - .state() - .into_iter() +pub async fn launch_trustee_sync_controller(client: Client) { + let deployments: Api = Api::default_namespaced(client.clone()); + let watcher_config = watcher::Config { + label_selector: Some(format!("app={TRUSTEE_APP_LABEL}")), + ..Default::default() + }; + tokio::spawn( + Controller::new(deployments, watcher_config) + .run( + trustee_deployment_reconcile, + controller_error_policy, + Arc::new(client), + ) + .for_each(controller_info), + ); +} + +pub async fn update_attestation_keys(client: Client) -> Result<()> { + let secrets: Api = Api::default_namespaced(client.clone()); + let secret_list = secrets.list(&Default::default()).await?; + + let ak_secrets: Vec = secret_list + .items + .iter() .filter(|secret| { // Filter out secrets that are being deleted if secret.metadata.deletion_timestamp.is_some() { return false; } - secret .metadata .owner_references @@ -230,125 +415,17 @@ pub async fn update_attestation_keys(ctx: &AkContextData) -> Result<()> { .map(|owners| owners.iter().any(|owner| owner.kind == "AttestationKey")) .unwrap_or(false) }) - .filter_map(|secret| secret.metadata.name.clone()) - .collect(); - - let ns = client.default_namespace().to_string(); - let Some(deployment) = ctx - .deployment_store - .get(&ObjectRef::new(TRUSTEE_DEPLOYMENT).within(&ns)) - .map(std::sync::Arc::unwrap_or_clone) - else { - // Trustee deployment is not (yet or no longer) present — nothing to patch. - info!("{TRUSTEE_DEPLOYMENT} not found in cache, skipping attestation key volume update"); - return Ok(()); - }; - let deployments: Api = Api::default_namespaced(client.clone()); - let err = format!("Deployment {TRUSTEE_DEPLOYMENT} existed, but had no spec"); - let depl_spec = deployment.spec.as_ref().context(err)?; - let err = format!("Deployment {TRUSTEE_DEPLOYMENT} existed, but had no pod spec"); - let pod_spec = depl_spec.template.spec.as_ref().context(err)?; - - // Get existing volumes and volumeMounts, filtering out the attestation key volume - let mut volumes: Vec = pod_spec - .volumes - .as_ref() - .map(|v| { - v.iter() - .filter(|vol| vol.name != TRUSTED_AK_KEYS_VOLUME) - .cloned() - .collect() - }) - .unwrap_or_default(); - - let err = format!("Deployment {TRUSTEE_DEPLOYMENT} existed, but had no containers"); - let container = pod_spec.containers.first().context(err)?; - let mut vol_mounts: Vec = container - .volume_mounts - .as_ref() - .map(|vm| { - vm.iter() - .filter(|mount| mount.name != TRUSTED_AK_KEYS_VOLUME) - .cloned() - .collect() + .filter_map(|secret| { + secret + .data + .as_ref() + .and_then(|d| d.get("public_key")) + .and_then(|pk| String::from_utf8(pk.0.clone()).ok()) }) - .unwrap_or_default(); - - if ak_secrets.is_empty() { - info!( - "No AttestationKey secrets found, removing projected volume from {TRUSTEE_DEPLOYMENT}" - ); - } else { - // Build the projected volume with all AttestationKey secrets - let projections: Vec = ak_secrets - .iter() - .map(|secret_name| VolumeProjection { - secret: Some(SecretProjection { - name: secret_name.to_string(), - items: Some(vec![KeyToPath { - key: "public_key".to_string(), - path: format!("{secret_name}.pub"), - ..Default::default() - }]), - ..Default::default() - }), - ..Default::default() - }) - .collect(); - - let projected_volume = Volume { - name: TRUSTED_AK_KEYS_VOLUME.to_string(), - projected: Some(ProjectedVolumeSource { - sources: Some(projections), - ..Default::default() - }), - ..Default::default() - }; - - volumes.push(projected_volume); - - vol_mounts.push(VolumeMount { - name: TRUSTED_AK_KEYS_VOLUME.to_string(), - mount_path: TRUSTED_AK_KEYS_DIR.to_string(), - ..Default::default() - }); - } - - // Check if volumes or volumeMounts have changed - let volumes_changed = pod_spec.volumes.as_ref() != Some(&volumes); - let vol_mounts_changed = container.volume_mounts.as_ref() != Some(&vol_mounts); - - if volumes_changed || vol_mounts_changed { - // Patch the deployment with updated volumes and volumeMounts - let patch = json!({ - "apiVersion": "apps/v1", - "kind": "Deployment", - "metadata": { - "name": TRUSTEE_DEPLOYMENT - }, - "spec": { - "template": { - "spec": { - "volumes": volumes, - "containers": [{ - "name": "kbs", - "volumeMounts": vol_mounts - }] - } - } - } - }); + .collect(); - deployments - .patch( - TRUSTEE_DEPLOYMENT, - &PatchParams::apply("trusted-cluster-operator").force(), - &Patch::Apply(&patch), - ) - .await?; - info!("Successfully patched {TRUSTEE_DEPLOYMENT} with attestation key volumes"); - } else { - info!("No changes to attestation key volumes, skipping deployment update"); + if let Err(e) = register_ak(client.clone(), &ak_secrets).await { + warn!("Failed to register AK to KBS: {e}"); } Ok(()) @@ -404,30 +481,6 @@ pub async fn generate_trustee_auth_keys_secret( Ok(()) } -pub async fn generate_attestation_policy( - client: Client, - owner_reference: OwnerReference, -) -> Result<()> { - let policy_rego = include_str!("tpm.rego"); - let data = BTreeMap::from([ - ("default_cpu.rego".to_string(), policy_rego.to_string()), - // Must create GPU policy or Trustee will attempt to write one to the read-only mount - ("default_gpu.rego".to_string(), String::new()), - ]); - - let config_map = ConfigMap { - metadata: ObjectMeta { - name: Some(ATT_POLICY_MAP.to_string()), - owner_references: Some(vec![owner_reference]), - ..Default::default() - }, - data: Some(data), - ..Default::default() - }; - create_or_info_if_exists!(client, ConfigMap, config_map); - Ok(()) -} - fn generate_kbs_config(has_certificate: bool) -> Result { let kbs_config_template = include_str!("kbs-config.toml"); let mut config: toml::Table = toml::from_str(kbs_config_template)?; @@ -443,6 +496,9 @@ fn generate_kbs_config(has_certificate: bool) -> Result { let tls_cert = toml::Value::String(format!("{TLS_DIR}/tls.crt")); http_server.insert("certificate".to_string(), tls_cert); } else { + warn!( + "Trustee deployment has no TLS certificate, starting KBS with insecure HTTP (not recommended for production)" + ); http_server.insert("insecure_http".to_string(), toml::Value::Boolean(true)); } @@ -456,13 +512,8 @@ pub async fn generate_trustee_data( ) -> Result<()> { let has_certificate = read_certificate(client.clone(), secret).await?.is_some(); let kbs_config = generate_kbs_config(has_certificate)?; - let policy_rego = include_str!("resource.rego"); - let data = BTreeMap::from([ - ("kbs-config.toml".to_string(), kbs_config), - ("policy.rego".to_string(), policy_rego.to_string()), - (REFERENCE_VALUES_FILE.to_string(), "[]".to_string()), - ]); + let data = BTreeMap::from([("kbs-config.toml".to_string(), kbs_config)]); let config_map = ConfigMap { metadata: ObjectMeta { @@ -477,6 +528,21 @@ pub async fn generate_trustee_data( Ok(()) } +pub async fn generate_rv_data(client: Client, owner_reference: OwnerReference) -> Result<()> { + let data = BTreeMap::from([(REFERENCE_VALUES_FILE.to_string(), "[]".to_string())]); + let config_map = ConfigMap { + metadata: ObjectMeta { + name: Some(TRUSTEE_RV_MAP.to_string()), + owner_references: Some(vec![owner_reference]), + ..Default::default() + }, + data: Some(data), + ..Default::default() + }; + create_or_info_if_exists!(client, ConfigMap, config_map); + Ok(()) +} + pub async fn generate_kbs_service( client: Client, owner_reference: OwnerReference, @@ -507,19 +573,8 @@ pub async fn generate_kbs_service( Ok(()) } -fn generate_kbs_volume_templates() -> [(&'static str, &'static str, Volume); 4] { +fn generate_kbs_volume_templates() -> [(&'static str, &'static str, Volume); 2] { [ - ( - ATT_POLICY_MAP, - "/opt/trustee/policies/opa", - Volume { - config_map: Some(ConfigMapVolumeSource { - name: ATT_POLICY_MAP.to_string(), - ..Default::default() - }), - ..Default::default() - }, - ), ( TRUSTEE_DATA_MAP, TRUSTEE_DATA_DIR, @@ -531,17 +586,6 @@ fn generate_kbs_volume_templates() -> [(&'static str, &'static str, Volume); 4] ..Default::default() }, ), - ( - "resource-dir", - TRUSTEE_SECRETS_PATH, - Volume { - empty_dir: Some(EmptyDirVolumeSource { - medium: Some("Memory".to_string()), - ..Default::default() - }), - ..Default::default() - }, - ), ( TRUSTEE_AUTH_SECRET, TRUSTEE_AUTH_KEY_DIR, @@ -617,7 +661,10 @@ pub async fn generate_kbs_deployment( image: &str, secret: &Option, ) -> Result<()> { - let selector = Some(BTreeMap::from([("app".to_string(), "kbs".to_string())])); + let selector = Some(BTreeMap::from([( + "app".to_string(), + TRUSTEE_APP_LABEL.to_string(), + )])); let tls_volumes = read_certificate(client.clone(), secret).await?; let pod_spec = generate_kbs_pod_spec(image, tls_volumes); @@ -625,6 +672,7 @@ pub async fn generate_kbs_deployment( let deployment = Deployment { metadata: ObjectMeta { name: Some(TRUSTEE_DEPLOYMENT.to_string()), + labels: selector.clone(), owner_references: Some(vec![owner_reference]), ..Default::default() }, @@ -654,7 +702,6 @@ mod tests { use super::*; use crate::test_utils::*; use http::{Method, Request, StatusCode}; - use kube::client::Body; use trusted_cluster_operator_test_utils::mock_client::*; use trusted_cluster_operator_test_utils::test_error_method; @@ -705,22 +752,27 @@ mod tests { #[tokio::test] async fn test_update_rvs_success() { + let _ = jsonwebtoken_openssl::install_default(); let clos = async |req: Request<_>, ctr| match (ctr, req.method()) { (0, &Method::GET) => { assert!(req.uri().path().contains(PCR_CONFIG_MAP)); Ok(serde_json::to_string(&dummy_pcrs_map()).unwrap()) } (1, &Method::GET) | (2, &Method::PUT) => { - assert!(req.uri().path().contains(TRUSTEE_DATA_MAP)); + assert!(req.uri().path().contains(TRUSTEE_RV_MAP)); Ok(serde_json::to_string(&dummy_trustee_map()).unwrap()) } + (3, &Method::GET) => { + assert!(req.uri().path().contains(TRUSTEE_AUTH_SECRET)); + Ok(serde_json::to_string(&dummy_trustee_auth()).unwrap()) + } + (4, &Method::GET) => Ok(serde_json::to_string(&dummy_cluster()).unwrap()), _ => panic!("unexpected API interaction: {req:?}, counter {ctr}"), }; - count_check!(3, clos, |client| { + count_check!(5, clos, |client| { assert!(update_reference_values(client).await.is_ok()); }); } - #[tokio::test] async fn test_update_rvs_no_pcr_map() { let clos = async |req: Request<_>, _| match (req.uri().path(), req.method()) { @@ -733,12 +785,12 @@ mod tests { } #[tokio::test] - async fn test_update_rvs_no_trustee_map() { + async fn test_update_rvs_no_rvs_map() { let clos = async |req: Request<_>, ctr| match (ctr, req.uri().path()) { (0, p) if p.contains(PCR_CONFIG_MAP) => { Ok(serde_json::to_string(&dummy_pcrs_map()).unwrap()) } - (1, p) if p.contains(TRUSTEE_DATA_MAP) => Err(StatusCode::NOT_FOUND), + (1, p) if p.contains(TRUSTEE_RV_MAP) => Err(StatusCode::NOT_FOUND), _ => panic!("unexpected API interaction: {req:?}, counter {ctr}"), }; count_check!(2, clos, |client| { @@ -747,12 +799,12 @@ mod tests { } #[tokio::test] - async fn test_update_rvs_no_trustee_data() { + async fn test_update_rvs_no_rvs_data() { let clos = async |req: Request<_>, ctr| match (ctr, req.uri().path()) { (0, p) if p.contains(PCR_CONFIG_MAP) => { Ok(serde_json::to_string(&dummy_pcrs_map()).unwrap()) } - (1, p) if p.contains(TRUSTEE_DATA_MAP) => { + (1, p) if p.contains(TRUSTEE_RV_MAP) => { Ok(serde_json::to_string(&ConfigMap::default()).unwrap()) } _ => panic!("unexpected API interaction: {req:?}, counter {ctr}"), @@ -769,133 +821,21 @@ mod tests { assert_eq!(jwk.key.len(), 32); } - fn dummy_deployment() -> Deployment { - Deployment { - spec: Some(DeploymentSpec { - replicas: Some(1), - template: PodTemplateSpec { - spec: Some(PodSpec { - containers: vec![Container::default()], - ..Default::default() - }), - ..Default::default() - }, - ..Default::default() - }), - ..Default::default() - } - } - - #[tokio::test] - async fn test_mount_secret_success() { - let clos = async |req: Request<_>, ctr| match (ctr, req.method()) { - (0, &Method::GET) | (1, &Method::PUT) => { - Ok(serde_json::to_string(&dummy_deployment()).unwrap()) - } - _ => panic!("unexpected API interaction: {req:?}, counter {ctr}"), - }; - count_check!(2, clos, |client| { - assert!(mount_secret(client, "id").await.is_ok()); - }); - } - - #[tokio::test] - async fn test_mount_secret_no_depl() { - let clos = async |_, _| Err(StatusCode::NOT_FOUND); - count_check!(1, clos, |client| { - assert!(mount_secret(client, "id").await.is_err()); - }); - } - - #[tokio::test] - async fn test_mount_secret_no_spec() { - let clos = async |_, _| { - let mut depl = dummy_deployment(); - depl.spec = None; - Ok(serde_json::to_string(&depl).unwrap()) - }; - count_check!(1, clos, |client| { - let err = mount_secret(client, "id").await.err().unwrap(); - assert!(err.to_string().contains("but had no spec")); - }); - } - - #[tokio::test] - async fn test_mount_secret_no_pod_spec() { - let clos = async |_, _| { - let mut depl = dummy_deployment(); - let spec = depl.spec.as_mut().unwrap(); - spec.template.spec = None; - Ok(serde_json::to_string(&depl).unwrap()) - }; - count_check!(1, clos, |client| { - let err = mount_secret(client, "id").await.err().unwrap(); - assert!(err.to_string().contains("but had no pod spec")); - }); - } - - #[tokio::test] - async fn test_mount_secret_no_containers() { - let clos = async |_, _| { - let mut depl = dummy_deployment(); - let spec = depl.spec.as_mut().unwrap(); - let pod_spec = spec.template.spec.as_mut().unwrap(); - pod_spec.containers = vec![]; - Ok(serde_json::to_string(&depl).unwrap()) - }; - count_check!(1, clos, |client| { - let err = mount_secret(client, "id").await.err().unwrap(); - assert!(err.to_string().contains("but had no containers")); - }); - } - - #[tokio::test] - async fn test_unmount_secret() { - let clos = async |req: Request, ctr| match (ctr, req.method()) { - (0, &Method::GET) => { - let mut depl = dummy_deployment(); - let spec = depl.spec.as_mut().unwrap(); - let pod_spec = spec.template.spec.as_mut().unwrap(); - pod_spec.volumes = Some(vec![Volume { - name: "id".to_string(), - ..Default::default() - }]); - let container = pod_spec.containers.get_mut(0).unwrap(); - container.volume_mounts = Some(vec![VolumeMount { - name: "id".to_string(), - ..Default::default() - }]); - Ok(serde_json::to_string(&depl).unwrap()) - } - (1, &Method::PUT) => { - let bytes = req.into_body().collect_bytes().await.unwrap().to_vec(); - let body = String::from_utf8_lossy(&bytes); - assert!(!body.contains("id")); - Ok(serde_json::to_string(&dummy_deployment()).unwrap()) - } - _ => panic!("unexpected API interaction: {req:?}, counter {ctr}"), - }; - count_check!(2, clos, |client| { - assert!(unmount_secret(client, "id").await.is_ok()); - }); - } - - #[tokio::test] - async fn test_generate_att_policy_success() { - let clos = |client| generate_attestation_policy(client, Default::default()); - test_create_success::<_, _, ConfigMap>(clos).await; - } - - #[tokio::test] - async fn test_generate_att_policy_already_exists() { - let clos = |client| generate_attestation_policy(client, Default::default()); - test_create_already_exists(clos).await; + #[test] + fn test_generate_ed25519_key_pair() { + let pair = generate_ed25519_key_pair().unwrap(); + let priv_pem = String::from_utf8(pair.private_key_pem).unwrap(); + let pub_pem = String::from_utf8(pair.public_key_pem).unwrap(); + assert!(priv_pem.starts_with("-----BEGIN PRIVATE KEY-----")); + assert!(pub_pem.starts_with("-----BEGIN PUBLIC KEY-----")); } - #[tokio::test] - async fn test_generate_att_policy_error() { - let clos = |client| generate_attestation_policy(client, Default::default()); - test_error_method!(clos, Method::POST); + #[test] + fn test_generate_ed25519_key_pair_unique() { + let pair1 = generate_ed25519_key_pair().unwrap(); + let pair2 = generate_ed25519_key_pair().unwrap(); + assert_ne!(pair1.private_key_pem, pair2.private_key_pem); + assert_ne!(pair1.public_key_pem, pair2.public_key_pem); } #[tokio::test] @@ -957,4 +897,39 @@ mod tests { let clos = |client| generate_kbs_deployment(client, Default::default(), "image", &None); test_error_method!(clos, Method::POST); } + + #[tokio::test] + async fn test_generate_rv_data_success() { + let clos = |client| generate_rv_data(client, Default::default()); + test_create_success::<_, _, ConfigMap>(clos).await; + } + + #[tokio::test] + async fn test_generate_rv_data_already_exists() { + let clos = |client| generate_rv_data(client, Default::default()); + test_create_already_exists(clos).await; + } + + #[tokio::test] + async fn test_generate_rv_data_error() { + let clos = |client| generate_rv_data(client, Default::default()); + test_error_method!(clos, Method::POST); + } + + #[test] + fn test_recompute_reference_values_includes_svn() { + let result = recompute_reference_values(dummy_pcrs()); + let svn = result.iter().find(|rv| rv.name == "tpm_svn").unwrap(); + let vals = svn.value.as_array().unwrap(); + assert_eq!(vals.len(), 1); + assert_eq!(vals[0].as_str().unwrap(), "1"); + } + + #[test] + fn test_recompute_reference_values_version() { + let result = recompute_reference_values(dummy_pcrs()); + for rv in &result { + assert_eq!(rv.version, "0.1.0"); + } + } } diff --git a/tests/no_disallowed_crypto.rs b/tests/no_disallowed_crypto.rs index f1754db7..5c0fcc21 100644 --- a/tests/no_disallowed_crypto.rs +++ b/tests/no_disallowed_crypto.rs @@ -58,6 +58,42 @@ const ALLOWED_CRYPTO_CRATES: &[&str] = &[ "spki", "subtle", "zeroize", + // Pulled in by kbs-client (via kbs_protocol / jwt-simple / cose-rust). + // kbs-client uses reqwest with native-tls (OpenSSL) for actual TLS. + // These crates handle JWS/JWE/COSE token processing, not network TLS. + "aead", + "aes", + "aes-gcm", + "aes-keywrap", + "aes-kw", + "aws-lc-rs", + "chacha20", + "cipher", + "cmov", + "concat-kdf", + "constant_time_eq", + "cose-rust", + "ct-codecs", + "ctr", + "ctutils", + "ed25519-compact", + "ghash", + "hmac-sha1-compact", + "hmac-sha256", + "hmac-sha512", + "jwt-simple", + "k256", + "keccak", + "ml-dsa", + "module-lattice", + "p521", + "polyval", + "sha3", + "siphasher", + "sm3", + "superboring", + "universal-hash", + "zeroize_derive", ]; // Target we actually build and ship for; keeps platform-only crates. From 888d85fb9a86d97a350eba03c3257d8123a40ed5 Mon Sep 17 00:00:00 2001 From: Roy Kaufman Date: Wed, 17 Jun 2026 16:11:20 +0300 Subject: [PATCH 03/15] test: add LUKS and Attestation Key sync tests test_luks_key_sync - verify the initial LUKS key upload, re-sync after trustee restart, and LUKS key deletion on machine removal. test_attestation_key_sync - verify that attestation keys are registered with the KBS and re-registered after trustee restarts. Signed-off-by: Roy Kaufman --- Cargo.lock | 1 + test_utils/src/lib.rs | 46 +++- tests/Cargo.toml | 1 + tests/trusted_execution_cluster.rs | 407 ++++++++++++++++++++++++++++- 4 files changed, 430 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ca60593b..7c5c7d53 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5519,6 +5519,7 @@ version = "0.2.1" dependencies = [ "anyhow", "cfg-if", + "chrono", "compute-pcrs-lib", "k8s-openapi 0.28.0", "kube 4.0.0", diff --git a/test_utils/src/lib.rs b/test_utils/src/lib.rs index a9899fd2..7cf5384d 100644 --- a/test_utils/src/lib.rs +++ b/test_utils/src/lib.rs @@ -5,7 +5,7 @@ use anyhow::{Context, Result, anyhow}; use fs_extra::dir; -use k8s_openapi::api::apps::v1::{Deployment, DeploymentCondition, DeploymentStatus}; +use k8s_openapi::api::apps::v1::Deployment; use k8s_openapi::api::core::v1::{ ConfigMap, LoadBalancerStatus, Namespace, Secret, Service, ServicePort, ServiceSpec, ServiceStatus, @@ -633,6 +633,35 @@ impl TestContext { Ok(()) } + pub async fn wait_for_deployment_ready( + &self, + deployments_api: &Api, + deployment_name: &str, + timeout_secs: u64, + ) -> Result<()> { + test_info!( + &self.test_name, + "Waiting for deployment {} to be ready", + deployment_name + ); + let has_available_replica = |d: Option<&Deployment>| { + d.and_then(|d| d.status.as_ref()) + .and_then(|s| s.available_replicas) + .is_some_and(|r| r >= 1) + }; + let done = await_condition( + deployments_api.clone(), + deployment_name, + has_available_replica, + ); + timeout(Duration::from_secs(timeout_secs), done) + .await + .context(format!( + "{deployment_name} deployment does not have 1 available replica after {timeout_secs} seconds" + ))??; + Ok(()) + } + async fn create_certificate( &self, service_name: &str, @@ -963,14 +992,6 @@ impl TestContext { "Applying ApprovedImage manifest" ); - let depl_ready = |depl: Option<&Deployment>| { - let chk_cond = |c: &DeploymentCondition| c.type_ == "Available" && c.status == "True"; - let chk_status = - |st: &DeploymentStatus| st.conditions.as_ref().map(|cs| cs.iter().any(chk_cond)); - let chk = |depl: &Deployment| depl.status.as_ref().and_then(chk_status); - depl.and_then(chk).unwrap_or(false) - }; - let depls: Api = Api::namespaced(self.client.clone(), ns); for depl in [ "trusted-cluster-operator", @@ -978,11 +999,8 @@ impl TestContext { TRUSTEE_DEPLOYMENT, ATTESTATION_KEY_REGISTER_DEPLOYMENT, ] { - let info = format!("Waiting for deployment {depl} to be ready"); - test_info!(&self.test_name, "{info}"); - let done = await_condition(depls.clone(), depl, depl_ready); - let ctx = format!("waiting for deployment {depl} to be ready"); - timeout(scaled_duration(300), done).await.context(ctx)??; + self.wait_for_deployment_ready(&depls, depl, scaled_timeout(300)) + .await?; } let svc = ATTESTATION_KEY_REGISTER_SERVICE; diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 01d434c0..169dc653 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -15,6 +15,7 @@ virtualization = [] [dependencies] anyhow.workspace = true cfg-if = "1.0.4" +chrono.workspace = true compute-pcrs-lib.workspace = true k8s-openapi.workspace = true kube = { workspace = true } diff --git a/tests/trusted_execution_cluster.rs b/tests/trusted_execution_cluster.rs index fb36dcef..781164da 100644 --- a/tests/trusted_execution_cluster.rs +++ b/tests/trusted_execution_cluster.rs @@ -4,26 +4,30 @@ // SPDX-License-Identifier: MIT use anyhow::Context; +use chrono::Utc; use compute_pcrs_lib::{Part, Pcr}; use k8s_openapi::api::apps::v1::Deployment; -use k8s_openapi::api::core::v1::{ConfigMap, Secret}; +use k8s_openapi::api::core::v1::{ConfigMap, Pod, Secret}; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, OwnerReference}; -use kube::api::ObjectMeta; +use kube::api::{ListParams, LogParams, Patch, PatchParams}; use kube::runtime::wait::await_condition; use kube::{Api, api::DeleteParams}; +use serde_json::json; use std::time::Duration; use tokio::time::timeout; use trusted_cluster_operator_lib::conditions::NOT_COMMITTED_REASON_PENDING; -use trusted_cluster_operator_lib::endpoints::{REGISTER_SERVER_DEPLOYMENT, TRUSTEE_DEPLOYMENT}; +use trusted_cluster_operator_lib::endpoints::REGISTER_SERVER_DEPLOYMENT; +use trusted_cluster_operator_lib::endpoints::TRUSTEE_DEPLOYMENT; use trusted_cluster_operator_lib::reference_values::ImagePcrs; use trusted_cluster_operator_lib::{ ApprovedImage, AttestationKey, Machine, TrustedExecutionCluster, generate_owner_reference, }; use trusted_cluster_operator_test_utils::*; - const EXPECTED_PCR4: &str = "ff2b357be4a4bc66be796d4e7b2f1f27077dc89b96220aae60b443bcf4672525"; const TEC_NAME: &str = "trusted-execution-cluster"; -const TRUSTEE_CONFIG_MAP: &str = "trustee-data"; +const APPROVED_IMAGE_NAME: &str = "coreos"; +const TRUSTEE_RV_MAP: &str = "trustee-rv-data"; const RV_JSON_KEY: &str = "reference-values.json"; fn ak_approved(ak: Option<&AttestationKey>) -> bool { @@ -230,8 +234,8 @@ async fn test_image_disallow() -> anyhow::Result<()> { let json = data.and_then(|data| data.get(RV_JSON_KEY)); json.map(|json| !json.contains(EXPECTED_PCR4)).unwrap_or(false) }; - let rv_removed = await_condition(configmap_api, TRUSTEE_CONFIG_MAP, chk_removed); - let ctx = format!("waiting for ConfigMap {TRUSTEE_CONFIG_MAP} to not contain PCR value"); + let rv_removed = await_condition(configmap_api, TRUSTEE_RV_MAP, chk_removed); + let ctx = format!("waiting for ConfigMap {TRUSTEE_RV_MAP} to not contain PCR value"); timeout(scaled_duration(180), rv_removed).await.context(ctx)??; test_ctx.cleanup().await?; @@ -368,6 +372,387 @@ async fn test_nonexistent_approved_image() -> anyhow::Result<()> { let ctx = "waiting for ApprovedImage coreos1 to be PodPending"; timeout(scaled_duration(30), done).await.context(ctx)??; + test_ctx.cleanup().await?; + Ok(()) +} +} + +named_test! { +async fn test_luks_key_sync() -> anyhow::Result<()> { + let test_ctx = setup!().await?; + let client = test_ctx.client(); + let namespace = test_ctx.namespace(); + let tec_name = "trusted-execution-cluster"; + + let tec_api: Api = Api::namespaced(client.clone(), namespace); + let tec = tec_api.get(tec_name).await?; + let owner_reference = generate_owner_reference(&tec)?; + + // Create two machines + let machine1_uuid = uuid::Uuid::new_v4().to_string(); + let machine1_name = format!("test-machine-{}", &machine1_uuid[..8]); + let machine2_uuid = uuid::Uuid::new_v4().to_string(); + let machine2_name = format!("test-machine-{}", &machine2_uuid[..8]); + + let machines: Api = Api::namespaced(client.clone(), namespace); + + let machine1 = Machine { + metadata: k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta { + name: Some(machine1_name.clone()), + namespace: Some(namespace.to_string()), + owner_references: Some(vec![owner_reference.clone()]), + ..Default::default() + }, + spec: trusted_cluster_operator_lib::MachineSpec { + id: machine1_uuid.clone(), + }, + status: None, + }; + machines.create(&Default::default(), &machine1).await?; + test_ctx.info(format!("Created Machine 1: {machine1_name}")); + + let machine2 = Machine { + metadata: k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta { + name: Some(machine2_name.clone()), + namespace: Some(namespace.to_string()), + owner_references: Some(vec![owner_reference.clone()]), + ..Default::default() + }, + spec: trusted_cluster_operator_lib::MachineSpec { + id: machine2_uuid.clone(), + }, + status: None, + }; + machines.create(&Default::default(), &machine2).await?; + test_ctx.info(format!("Created Machine 2: {machine2_name}")); + + // Wait for both K8s secrets to be created by the keygen controller + let secrets_api: Api = Api::namespaced(client.clone(), namespace); + wait_for_resource_created(&secrets_api, &machine1_uuid, scaled_timeout(60)).await?; + wait_for_resource_created(&secrets_api, &machine2_uuid, scaled_timeout(60)).await?; + test_ctx.info("Both machine secrets created"); + + // Wait for the operator to send both secrets to the KBS + let pods_api: Api = Api::namespaced(client.clone(), namespace); + let poller = Poller::new() + .with_timeout(scaled_duration(60)) + .with_interval(scaled_duration(2)) + .with_error_message("Secrets not sent to KBS".to_string()); + + poller + .poll_async(|| { + let api = pods_api.clone(); + let id1 = machine1_uuid.clone(); + let id2 = machine2_uuid.clone(); + async move { + let lp = ListParams::default().labels("app=trusted-cluster-operator"); + let operator_pods = api.list(&lp).await?; + let pod_name = operator_pods + .items + .first() + .and_then(|p| p.metadata.name.as_ref()) + .ok_or_else(|| anyhow::anyhow!("Operator pod not found"))? + .clone(); + let logs = api.logs(&pod_name, &LogParams::default()).await?; + if logs.contains(&format!("{id1} sent successfully")) + && logs.contains(&format!("{id2} sent successfully")) + { + return Ok(()); + } + Err(anyhow::anyhow!("Not all secrets sent to KBS yet")) + } + }) + .await?; + test_ctx.info("Both secrets sent to KBS"); + + + let now = Utc::now().to_rfc3339(); + let patch = json!({ + "spec": { + "template": { + "metadata": { + "annotations": { + "kubectl.kubernetes.io/restartedAt": now + } + } + } + } + }); + + test_ctx.info(format!("Triggering rollout restart for deployment: {TRUSTEE_DEPLOYMENT}")); + let deployments: Api = Api::namespaced(client.clone(), namespace); + // Apply the patch + deployments + .patch( + TRUSTEE_DEPLOYMENT, + &PatchParams::default(), + &Patch::Strategic(patch), + ) + .await?; + + test_ctx.wait_for_deployment_ready(&deployments, TRUSTEE_DEPLOYMENT, 120).await?; + + // Wait for the new pod to be ready + test_ctx.info("Trustee deployment is ready after restart"); + + // Verify both secrets are re-synced to KBS after the trustee restart + let poller = Poller::new() + .with_timeout(scaled_duration(60)) + .with_interval(scaled_duration(2)) + .with_error_message("Secrets not re-synced to KBS after restart".to_string()); + + poller + .poll_async(|| { + let api = pods_api.clone(); + async move { + let lp = ListParams::default().labels("app=trusted-cluster-operator"); + let operator_pods = api.list(&lp).await?; + let pod_name = operator_pods + .items + .first() + .and_then(|p| p.metadata.name.as_ref()) + .ok_or_else(|| anyhow::anyhow!("Operator pod not found"))? + .clone(); + let logs = api.logs(&pod_name, &LogParams::default()).await?; + if logs.contains("Syncing 2 machine luks key to KBS") { + return Ok(()); + } + Err(anyhow::anyhow!("Secrets not yet re-synced to KBS after restart.")) + } + }) + .await?; + test_ctx.info("Both secrets re-synced to KBS after trustee restart"); + + // Delete machine1 and verify its secret is removed from both K8s and KBS + machines + .delete(&machine1_name, &Default::default()) + .await?; + test_ctx.info(format!("Deleted Machine 1: {machine1_name}")); + + let poller = Poller::new() + .with_timeout(scaled_duration(60)) + .with_interval(scaled_duration(2)) + .with_error_message("Machine1 secret not deleted from KBS".to_string()); + + poller + .poll_async(|| { + let api = pods_api.clone(); + let id1 = machine1_uuid.clone(); + async move { + let lp = ListParams::default().labels("app=trusted-cluster-operator"); + let operator_pods = api.list(&lp).await?; + let pod_name = operator_pods + .items + .first() + .and_then(|p| p.metadata.name.as_ref()) + .ok_or_else(|| anyhow::anyhow!("Operator pod not found"))? + .clone(); + let logs = api.logs(&pod_name, &LogParams::default()).await?; + if logs.contains(&format!("Secret {id1} deleted successfully")) { + return Ok(()); + } + Err(anyhow::anyhow!("Machine1 secret not yet deleted from KBS")) + } + }) + .await?; + test_ctx.info("Machine1 secret deleted from KBS"); + + // Verify the K8s Secret for machine1 is also deleted + wait_for_resource_deleted(&secrets_api, &machine1_uuid, 60).await?; + test_ctx.info("Machine1 K8s secret deleted"); + + test_ctx.cleanup().await?; + Ok(()) +} +} + +named_test! { +async fn test_attestation_key_sync() -> anyhow::Result<()> { + let test_ctx = setup!().await?; + let client = test_ctx.client(); + let namespace = test_ctx.namespace(); + let tec_name = "trusted-execution-cluster"; + + let tec_api: Api = Api::namespaced(client.clone(), namespace); + let tec = tec_api.get(tec_name).await?; + let owner_reference = generate_owner_reference(&tec)?; + + // Create two machines + let machine1_uuid = uuid::Uuid::new_v4().to_string(); + let machine1_name = format!("test-machine-{}", &machine1_uuid[..8]); + let machine2_uuid = uuid::Uuid::new_v4().to_string(); + let machine2_name = format!("test-machine-{}", &machine2_uuid[..8]); + + let machines: Api = Api::namespaced(client.clone(), namespace); + let machine1 = Machine { + metadata: k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta { + name: Some(machine1_name.clone()), + namespace: Some(namespace.to_string()), + owner_references: Some(vec![owner_reference.clone()]), + ..Default::default() + }, + spec: trusted_cluster_operator_lib::MachineSpec { + id: machine1_uuid.clone(), + }, + status: None, + }; + machines.create(&Default::default(), &machine1).await?; + test_ctx.info(format!("Created Machine 1: {machine1_name}")); + + let machine2 = Machine { + metadata: k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta { + name: Some(machine2_name.clone()), + namespace: Some(namespace.to_string()), + owner_references: Some(vec![owner_reference.clone()]), + ..Default::default() + }, + spec: trusted_cluster_operator_lib::MachineSpec { + id: machine2_uuid.clone(), + }, + status: None, + }; + machines.create(&Default::default(), &machine2).await?; + test_ctx.info(format!("Created Machine 2: {machine2_name}")); + + // Create two AttestationKeys with matching UUIDs + let ak1_name = format!("test-ak-{}", &machine1_uuid[..8]); + let ak1_public_key = uuid::Uuid::new_v4().to_string(); + let ak2_name = format!("test-ak-{}", &machine2_uuid[..8]); + let ak2_public_key = uuid::Uuid::new_v4().to_string(); + + let attestation_keys: Api = Api::namespaced(client.clone(), namespace); + + let ak1 = AttestationKey { + metadata: k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta { + name: Some(ak1_name.clone()), + namespace: Some(namespace.to_string()), + owner_references: Some(vec![owner_reference.clone()]), + ..Default::default() + }, + spec: trusted_cluster_operator_lib::AttestationKeySpec { + public_key: ak1_public_key, + uuid: Some(machine1_uuid.clone()), + }, + status: None, + }; + attestation_keys.create(&Default::default(), &ak1).await?; + test_ctx.info(format!("Created AttestationKey 1: {ak1_name}")); + + let ak2 = AttestationKey { + metadata: k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta { + name: Some(ak2_name.clone()), + namespace: Some(namespace.to_string()), + owner_references: Some(vec![owner_reference.clone()]), + ..Default::default() + }, + spec: trusted_cluster_operator_lib::AttestationKeySpec { + public_key: ak2_public_key, + uuid: Some(machine2_uuid.clone()), + }, + status: None, + }; + attestation_keys.create(&Default::default(), &ak2).await?; + test_ctx.info(format!("Created AttestationKey 2: {ak2_name}")); + + // Wait for both AKs to be approved and have secrets created + let secrets_api: Api = Api::namespaced(client.clone(), namespace); + let ak1_done = await_condition(attestation_keys.clone(), &ak1_name, ak_approved); + let ak2_done = await_condition(attestation_keys.clone(), &ak2_name, ak_approved); + wait_for_resource_created(&secrets_api, &ak1_name, scaled_timeout(60)).await?; + wait_for_resource_created(&secrets_api, &ak2_name, scaled_timeout(60)).await?; + timeout(scaled_duration(60), async { + tokio::try_join!(ak1_done, ak2_done) + }) + .await + .context("waiting for AttestationKeys to be approved with secrets")??; + test_ctx.info("Both AttestationKeys approved and secrets created"); + + // Wait for both AKs to be registered with KBS + let pods_api: Api = Api::namespaced(client.clone(), namespace); + let poller = Poller::new() + .with_timeout(scaled_duration(60)) + .with_interval(scaled_duration(2)) + .with_error_message("AKs not registered with KBS".to_string()); + + poller + .poll_async(|| { + let api = pods_api.clone(); + async move { + let lp = ListParams::default().labels("app=trusted-cluster-operator"); + let operator_pods = api.list(&lp).await?; + let pod_name = operator_pods + .items + .first() + .and_then(|p| p.metadata.name.as_ref()) + .ok_or_else(|| anyhow::anyhow!("Operator pod not found"))? + .clone(); + let logs = api.logs(&pod_name, &LogParams::default()).await?; + let count = logs.matches("AK registered successfully").count(); + if count >= 1 { + return Ok(()); + } + Err(anyhow::anyhow!("Only {count} AK registrations found, need at least 1")) + } + }) + .await?; + test_ctx.info("Both AKs registered with KBS"); + + // Restart the trustee deployment + let now = Utc::now().to_rfc3339(); + let patch = json!({ + "spec": { + "template": { + "metadata": { + "annotations": { + "kubectl.kubernetes.io/restartedAt": now + } + } + } + } + }); + + test_ctx.info(format!("Triggering rollout restart for deployment: {TRUSTEE_DEPLOYMENT}")); + let deployments: Api = Api::namespaced(client.clone(), namespace); + deployments + .patch( + TRUSTEE_DEPLOYMENT, + &PatchParams::default(), + &Patch::Strategic(patch), + ) + .await?; + + test_ctx.wait_for_deployment_ready(&deployments, TRUSTEE_DEPLOYMENT, 120).await?; + test_ctx.info("Trustee deployment is ready after restart"); + + // Verify both AKs are re-registered to KBS after the trustee restart + let poller = Poller::new() + .with_timeout(scaled_duration(60)) + .with_interval(scaled_duration(2)) + .with_error_message("AKs not re-registered with KBS after restart".to_string()); + + poller + .poll_async(|| { + let api = pods_api.clone(); + async move { + let lp = ListParams::default().labels("app=trusted-cluster-operator"); + let operator_pods = api.list(&lp).await?; + let pod_name = operator_pods + .items + .first() + .and_then(|p| p.metadata.name.as_ref()) + .ok_or_else(|| anyhow::anyhow!("Operator pod not found"))? + .clone(); + let logs = api.logs(&pod_name, &LogParams::default()).await?; + let count = logs.matches("AK registered successfully").count(); + if count >= 2 { + return Ok(()); + } + Err(anyhow::anyhow!("Only {count} AK registrations after restart, need at least 2")) + } + }) + .await?; + test_ctx.info("Both AKs re-registered with KBS after trustee restart"); + test_ctx.cleanup().await?; Ok(()) } @@ -396,9 +781,9 @@ async fn test_approved_image_readoption() -> anyhow::Result<()> { test_ctx.info(format!("Deleting TrustedExecutionCluster {TEC_NAME}")); clusters.delete(TEC_NAME, &Default::default()).await?; - wait_for_resource_deleted(&configmaps, TRUSTEE_CONFIG_MAP, scaled_timeout(60)).await?; + wait_for_resource_deleted(&configmaps, TRUSTEE_RV_MAP, scaled_timeout(60)).await?; wait_for_resource_deleted(&images, APPROVED_IMAGE_NAME, scaled_timeout(60)).await?; - test_ctx.info(format!("Configmap {TRUSTEE_CONFIG_MAP} was removed")); + test_ctx.info(format!("Configmap {TRUSTEE_RV_MAP} was removed")); let image = ApprovedImage { spec: image_spec, @@ -427,8 +812,8 @@ async fn test_approved_image_readoption() -> anyhow::Result<()> { let json = data.and_then(|data| data.get(RV_JSON_KEY)); json.map(|json| json.contains(EXPECTED_PCR4)).unwrap_or(false) }; - let rv_added = await_condition(configmaps, TRUSTEE_CONFIG_MAP, chk_added); - let ctx = format!("waiting for ConfigMap {TRUSTEE_CONFIG_MAP} to contain PCR value"); + let rv_added = await_condition(configmaps, TRUSTEE_RV_MAP, chk_added); + let ctx = format!("waiting for ConfigMap {TRUSTEE_RV_MAP} to contain PCR value"); timeout(scaled_duration(180), rv_added).await.context(ctx)??; test_ctx.info("Reference values regenerated"); From e9031f9edea661cac5e4f8b6c4cce7326b6dc578 Mon Sep 17 00:00:00 2001 From: Roy Kaufman Date: Thu, 25 Jun 2026 18:56:36 +0300 Subject: [PATCH 04/15] trustee: Add unit tests Tested functions: - sync_all_machine_luks_key - update_attestation_keys Signed-off-by: Roy Kaufman --- Cargo.lock | 45 ++++++++++++-- operator/src/test_utils.rs | 37 +++++++++++- operator/src/trustee.rs | 119 +++++++++++++++++++++++++++++++++++++ 3 files changed, 193 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7c5c7d53..69b0ea97 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1203,6 +1203,38 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "defmt" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6e524506490a1953d237cb87b1cfc1e46f88c18f10a22dfe0f507dc6bfc7f7f" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0a27770e9c8f719a79d8b638281f4d828f77d8fd61e0bd94451b9b85e576a0b" +dependencies = [ + "defmt-parser", + "proc-macro-error2", + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror 2.0.18", +] + [[package]] name = "der" version = "0.7.10" @@ -2529,24 +2561,25 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "jiff" -version = "0.2.23" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" +checksum = "34f877a98676d2fb664698d74cc6a51ce6c484ce8c770f05d0108ec9090aeb46" dependencies = [ + "defmt", "jiff-static", "jiff-tzdb-platform", "log", "portable-atomic", "portable-atomic-util", "serde_core", - "windows-sys 0.61.2", + "windows-link 0.2.1", ] [[package]] name = "jiff-static" -version = "0.2.23" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" +checksum = "0666b5ab5ecaca213fc2a85b8c0083d9004e84ee2d5f9a7e0017aaf50986f25f" dependencies = [ "proc-macro2", "quote", @@ -3299,7 +3332,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] diff --git a/operator/src/test_utils.rs b/operator/src/test_utils.rs index 96e5512d..fa4a3f28 100644 --- a/operator/src/test_utils.rs +++ b/operator/src/test_utils.rs @@ -2,15 +2,17 @@ // // SPDX-License-Identifier: MIT +use crate::trustee; use compute_pcrs_lib::Pcr; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::OwnerReference; use k8s_openapi::{ api::core::v1::{ConfigMap, Secret}, jiff::Timestamp, }; +use kube::api::ObjectMeta; use std::collections::BTreeMap; - -use crate::trustee; use trusted_cluster_operator_lib::reference_values::{ImagePcr, ImagePcrs, PCR_CONFIG_FILE}; +use trusted_cluster_operator_lib::{Machine, MachineSpec}; pub fn dummy_pcrs() -> ImagePcrs { ImagePcrs(BTreeMap::from([( @@ -74,3 +76,34 @@ pub fn dummy_pcrs_map() -> ConfigMap { ..Default::default() } } + +pub fn dummy_machine(id: &str) -> Machine { + Machine { + metadata: ObjectMeta { + name: Some(id.to_string()), + ..Default::default() + }, + spec: MachineSpec { id: id.to_string() }, + status: None, + } +} + +pub fn dummy_ak_secret(name: &str) -> Secret { + Secret { + metadata: ObjectMeta { + name: Some(name.to_string()), + owner_references: Some(vec![OwnerReference { + kind: "AttestationKey".to_string(), + name: name.to_string(), + uid: "ak-uid".to_string(), + ..Default::default() + }]), + ..Default::default() + }, + data: Some(BTreeMap::from([( + "public_key".to_string(), + k8s_openapi::ByteString(b"test-ak-public-key".to_vec()), + )])), + ..Default::default() + } +} diff --git a/operator/src/trustee.rs b/operator/src/trustee.rs index ab179530..f6e1173c 100644 --- a/operator/src/trustee.rs +++ b/operator/src/trustee.rs @@ -702,6 +702,7 @@ mod tests { use super::*; use crate::test_utils::*; use http::{Method, Request, StatusCode}; + use k8s_openapi::apimachinery::pkg::apis::meta::v1::Time; use trusted_cluster_operator_test_utils::mock_client::*; use trusted_cluster_operator_test_utils::test_error_method; @@ -932,4 +933,122 @@ mod tests { assert_eq!(rv.version, "0.1.0"); } } + + #[tokio::test] + async fn test_sync_all_machine_luks_key_empty() { + let clos = async |req: Request<_>, ctr| match (ctr, req.method()) { + (0, &Method::GET) => { + let list = kube::api::ObjectList { + items: Vec::::new(), + types: Default::default(), + metadata: Default::default(), + }; + Ok(serde_json::to_string(&list).unwrap()) + } + _ => panic!("unexpected API interaction: {req:?}, counter {ctr}"), + }; + count_check!(1, clos, |client| { + assert!(sync_all_machine_luks_key(client).await.is_ok()); + }); + } + + #[tokio::test] + async fn test_sync_all_machine_luks_key_send_error() { + let clos = async |req: Request<_>, ctr| match (ctr, req.method()) { + (0, &Method::GET) => { + let list = kube::api::ObjectList { + items: vec![dummy_machine("m1")], + types: Default::default(), + metadata: Default::default(), + }; + Ok(serde_json::to_string(&list).unwrap()) + } + (_, &Method::GET) => Err(StatusCode::NOT_FOUND), + _ => panic!("unexpected API interaction: {req:?}, counter {ctr}"), + }; + count_check!(2, clos, |client| { + assert!(sync_all_machine_luks_key(client).await.is_err()); + }); + } + + #[tokio::test] + async fn test_update_attestation_keys_empty() { + let clos = async |_, ctr| match ctr { + 0 => { + let list = kube::api::ObjectList { + items: Vec::::new(), + types: Default::default(), + metadata: Default::default(), + }; + Ok(serde_json::to_string(&list).unwrap()) + } + _ => Err(StatusCode::NOT_FOUND), + }; + count_check!(2, clos, |client| { + assert!(update_attestation_keys(client).await.is_ok()); + }); + } + + #[tokio::test] + async fn test_update_attestation_keys_register_fails_gracefully() { + let clos = async |_, ctr| match ctr { + 0 => { + let list = kube::api::ObjectList { + items: vec![dummy_ak_secret("ak1"), dummy_ak_secret("ak2")], + types: Default::default(), + metadata: Default::default(), + }; + Ok(serde_json::to_string(&list).unwrap()) + } + _ => Err(StatusCode::NOT_FOUND), + }; + count_check!(2, clos, |client| { + assert!(update_attestation_keys(client).await.is_ok()); + }); + } + + #[tokio::test] + async fn test_update_attestation_keys_register_success() { + let _ = jsonwebtoken_openssl::install_default(); + let clos = async |req: Request<_>, ctr| match (ctr, req.method()) { + (0, &Method::GET) => { + let list = kube::api::ObjectList { + items: vec![dummy_ak_secret("ak1")], + types: Default::default(), + metadata: Default::default(), + }; + Ok(serde_json::to_string(&list).unwrap()) + } + (1, &Method::GET) => { + assert!(req.uri().path().contains(TRUSTEE_AUTH_SECRET)); + Ok(serde_json::to_string(&dummy_trustee_auth()).unwrap()) + } + (2, &Method::GET) => Ok(serde_json::to_string(&dummy_cluster()).unwrap()), + _ => panic!("unexpected API interaction: {req:?}, counter {ctr}"), + }; + count_check!(3, clos, |client| { + assert!(update_attestation_keys(client).await.is_ok()); + }); + } + + #[tokio::test] + async fn test_update_attestation_keys_filters_deleting() { + let clos = async |_, ctr| match ctr { + 0 => { + let mut deleting = dummy_ak_secret("ak-deleting"); + deleting.metadata.deletion_timestamp = + Some(Time(k8s_openapi::jiff::Timestamp::now())); + let list = kube::api::ObjectList { + items: vec![deleting], + types: Default::default(), + metadata: Default::default(), + }; + Ok(serde_json::to_string(&list).unwrap()) + } + _ => Err(StatusCode::NOT_FOUND), + }; + count_check!(2, clos, |client| { + assert!(update_attestation_keys(client).await.is_ok()); + }); + } } From 39b0245ae57e92351f330dcbea3c981e70a274a5 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Fri, 3 Jul 2026 10:38:36 +0200 Subject: [PATCH 05/15] trustee: Add empty directory volume for storage required on OpenShift SCC Signed-off-by: Jakob Naucke --- Cargo.lock | 14 +++++++------- operator/src/trustee.rs | 18 ++++++++++++++---- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 69b0ea97..3be3ebcc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -314,7 +314,7 @@ dependencies = [ "serde_with", "sha2 0.11.0", "strum 0.28.0", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tracing", ] @@ -1232,7 +1232,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" dependencies = [ - "thiserror 2.0.18", + "thiserror 2.0.19", ] [[package]] @@ -1388,7 +1388,7 @@ dependencies = [ "phf", "serde", "serde_json", - "thiserror 2.0.18", + "thiserror 2.0.19", ] [[package]] @@ -2713,7 +2713,7 @@ dependencies = [ "serde", "serde_json", "superboring", - "thiserror 2.0.18", + "thiserror 2.0.19", "zeroize", ] @@ -2787,7 +2787,7 @@ dependencies = [ "sha2 0.10.9", "sm3", "strum 0.27.2", - "thiserror 2.0.18", + "thiserror 2.0.19", ] [[package]] @@ -2809,7 +2809,7 @@ dependencies = [ "serde_json_canonicalizer", "sha2 0.11.0", "shadow-rs", - "thiserror 2.0.18", + "thiserror 2.0.19", "tokio", "tracing", "url", @@ -4817,7 +4817,7 @@ checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" dependencies = [ "num-bigint", "num-traits", - "thiserror 2.0.18", + "thiserror 2.0.19", "time", ] diff --git a/operator/src/trustee.rs b/operator/src/trustee.rs index f6e1173c..7732bd7a 100644 --- a/operator/src/trustee.rs +++ b/operator/src/trustee.rs @@ -11,9 +11,9 @@ use clevis_pin_trustee_lib::Key as ClevisKey; use futures_util::StreamExt; use k8s_openapi::api::apps::v1::{Deployment, DeploymentSpec}; use k8s_openapi::api::core::v1::{ - ConfigMap, ConfigMapVolumeSource, Container, ContainerPort, EnvVar, KeyToPath, PodSpec, - PodTemplateSpec, Secret, SecretVolumeSource, Service, ServicePort, ServiceSpec, Volume, - VolumeMount, + ConfigMap, ConfigMapVolumeSource, Container, ContainerPort, EmptyDirVolumeSource, EnvVar, + KeyToPath, PodSpec, PodTemplateSpec, Secret, SecretVolumeSource, Service, ServicePort, + ServiceSpec, Volume, VolumeMount, }; use k8s_openapi::apimachinery::pkg::{ apis::meta::v1::{LabelSelector, OwnerReference}, @@ -52,6 +52,8 @@ pub(crate) const TRUSTEE_RV_MAP: &str = "trustee-rv-data"; pub(crate) const REFERENCE_VALUES_FILE: &str = "reference-values.json"; const TRUSTEE_AUTH_SECRET: &str = "trustee-auth"; const TRUSTEE_AUTH_KEY_DIR: &str = "/opt/trustee/keys"; +const TRUSTEE_STORAGE_VOLUME: &str = "trustee-storage"; +const TRUSTEE_STORAGE_DIR: &str = "/opt/trustee/storage"; pub(crate) const TRUSTEE_AUTH_PUB_KEY: &str = "public.pub"; pub(crate) const TRUSTEE_AUTH_PRIV_KEY: &str = "private.key"; @@ -573,7 +575,7 @@ pub async fn generate_kbs_service( Ok(()) } -fn generate_kbs_volume_templates() -> [(&'static str, &'static str, Volume); 2] { +fn generate_kbs_volume_templates() -> [(&'static str, &'static str, Volume); 3] { [ ( TRUSTEE_DATA_MAP, @@ -602,6 +604,14 @@ fn generate_kbs_volume_templates() -> [(&'static str, &'static str, Volume); 2] ..Default::default() }, ), + ( + TRUSTEE_STORAGE_VOLUME, + TRUSTEE_STORAGE_DIR, + Volume { + empty_dir: Some(EmptyDirVolumeSource::default()), + ..Default::default() + }, + ), ] } From c93be8dba80e48a985a2fe2464e07fd99fbdf331 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Thu, 16 Jul 2026 12:38:07 +0200 Subject: [PATCH 06/15] trustee: try pad-token Signed-off-by: Jakob Naucke --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e938e8a9..4d8062f4 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ COMPUTE_PCRS_IMAGE=$(REGISTRY)/compute-pcrs:$(TAG) REG_SERVER_IMAGE=$(REGISTRY)/registration-server:$(TAG) ATTESTATION_KEY_REGISTER_IMAGE=$(REGISTRY)/attestation-key-register:$(TAG) -TRUSTEE_IMAGE ?= quay.io/trusted-execution-clusters/key-broker-service:v0.20.0 +TRUSTEE_IMAGE ?= quay.io/trusted-execution-clusters/key-broker-service:0.20-pad-token TEST_IMAGE ?= quay.io/trusted-execution-clusters/fedora-coreos-kubevirt:42.20260622 # tagged as 42.20251012.2.0 From b534fcbd626aa65d8fce6190b31279ba64635597 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Mon, 13 Jul 2026 17:15:53 +0200 Subject: [PATCH 07/15] Add timestamps to logging for crates and testing Fixes: #227 Signed-off-by: Jakob Naucke --- Cargo.toml | 2 +- test_utils/src/lib.rs | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 02db0a13..025ebf45 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ chrono = "0.4.45" clap = { version = "4.6.1", features = ["derive"] } clevis-pin-trustee-lib = { git = "https://github.com/latchset/clevis-pin-trustee" } compute-pcrs-lib = { git = "https://github.com/trusted-execution-clusters/compute-pcrs" } -env_logger = { version = "0.11.10", default-features = false } +env_logger = { version = "0.11.10", default-features = false, features = ["humantime"] } http = "1.4.2" ignition-config = "0.6.1" k8s-openapi = { version = "0.28.0", features = ["v1_35", "schemars"] } diff --git a/test_utils/src/lib.rs b/test_utils/src/lib.rs index 7cf5384d..79377437 100644 --- a/test_utils/src/lib.rs +++ b/test_utils/src/lib.rs @@ -89,6 +89,11 @@ pub fn scaled_duration(secs: u64) -> Duration { Duration::from_secs(scaled_timeout(secs)) } +fn log_time() -> String { + let fmt = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"); + fmt.to_string() +} + // Large warning frame, e.g. for paid cloud resources that may not have been shut down correctly pub fn warn_frame(msg: &str) -> String { format!("{YELLOW}=== WARNING ===\n{msg}{ANSI_RESET}") @@ -98,14 +103,14 @@ pub fn warn_frame(msg: &str) -> String { macro_rules! test_info { ($test_name:expr, $($arg:tt)*) => {{ const GREEN: &str = "\x1b[32m"; - println!("{}INFO{}: {}: {}", GREEN, ANSI_RESET, $test_name, format!($($arg)*)); + println!("{} {}INFO{}: {}: {}", log_time(), GREEN, ANSI_RESET, $test_name, format!($($arg)*)); }} } #[macro_export] macro_rules! test_warn { ($test_name:expr, $($arg:tt)*) => {{ - println!("{YELLOW}WARN{ANSI_RESET}: {}: {}", $test_name, format!($($arg)*)); + println!("{} {YELLOW}WARN{ANSI_RESET}: {}: {}", log_time(), $test_name, format!($($arg)*)); }} } From 79ce36a8bae3fd157c757165d5b27a7f31629083 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Thu, 9 Jul 2026 12:38:23 +0200 Subject: [PATCH 08/15] rvs: Reconcile images on jobs too so that status is updated in a timely manner Signed-off-by: Jakob Naucke --- operator/src/reference_values.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/operator/src/reference_values.rs b/operator/src/reference_values.rs index 636e9e94..7737a308 100644 --- a/operator/src/reference_values.rs +++ b/operator/src/reference_values.rs @@ -341,8 +341,11 @@ async fn image_remove_reconcile( pub async fn launch_rv_image_controller(client: Client) { let images: Api = Api::default_namespaced(client.clone()); + let jobs: Api = Api::default_namespaced(client.clone()); + let wc = watcher::Config::default().labels(&format!("{JOB_LABEL_KEY}={PCR_COMMAND_NAME}")); tokio::spawn( Controller::new(images, Default::default()) + .owns(jobs, wc) .run(image_reconcile, controller_error_policy, Arc::new(client)) .for_each(controller_info), ); From c7fe62c2f37a9f783a874c0daea650a1786c5d80 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Wed, 29 Jul 2026 15:36:38 +0200 Subject: [PATCH 09/15] Use a read-timeout client for non-watch operations Default kube-rs client has no read timeout so that watch operations are supported. Some non-watch operations have been seen to hang. Use a timeout of 30 seconds for such operations. Introduce a structure to pass both clients where needed. Signed-off-by: Jakob Naucke Assisted-by: AI --- attestation-key-register/src/main.rs | 5 +- compute-pcrs/src/main.rs | 4 +- lib/src/lib.rs | 25 ++++++ operator/src/attestation_key_register.rs | 25 +++--- operator/src/main.rs | 34 +++++--- operator/src/reference_values.rs | 16 ++-- operator/src/register_server.rs | 9 +- operator/src/trustee.rs | 8 +- register-server/src/main.rs | 4 +- test_utils/src/lib.rs | 104 +++++++++++++---------- test_utils/src/virt/kubevirt.rs | 12 ++- test_utils/src/virt/mod.rs | 14 +-- tests/attestation.rs | 14 +-- tests/trusted_execution_cluster.rs | 64 ++++++++------ 14 files changed, 207 insertions(+), 131 deletions(-) diff --git a/attestation-key-register/src/main.rs b/attestation-key-register/src/main.rs index 3effee0a..476adc72 100644 --- a/attestation-key-register/src/main.rs +++ b/attestation-key-register/src/main.rs @@ -17,7 +17,8 @@ use uuid::Uuid; use trusted_cluster_operator_lib::endpoints::ATTESTATION_KEY_REGISTER_RESOURCE; use trusted_cluster_operator_lib::{ - generate_owner_reference, get_trusted_execution_cluster, AttestationKey, AttestationKeySpec, + generate_owner_reference, get_trusted_execution_cluster, timed_client, AttestationKey, + AttestationKeySpec, }; #[derive(Parser)] @@ -131,7 +132,7 @@ async fn main() { let args = Args::parse(); let endpoint = format!("/{ATTESTATION_KEY_REGISTER_RESOURCE}"); let err = "failed to create Kubernetes client"; - let client = Client::try_default().await.expect(err); + let client = timed_client().await.expect(err); let app = Router::new() .route(&endpoint, put(handle_registration)) .with_state(client); diff --git a/compute-pcrs/src/main.rs b/compute-pcrs/src/main.rs index 0c1bfacf..cf543662 100644 --- a/compute-pcrs/src/main.rs +++ b/compute-pcrs/src/main.rs @@ -7,7 +7,7 @@ use anyhow::{Context, Result, anyhow}; use clap::Parser; use compute_pcrs_lib::*; use k8s_openapi::{api::core::v1::ConfigMap, jiff::Timestamp}; -use kube::{Api, Client}; +use kube::Api; use std::{fs::File, io::Read}; use trusted_cluster_operator_lib::{conditions::INSTALLED_REASON, reference_values::*, *}; @@ -52,7 +52,7 @@ async fn main() -> Result<()> { compute_pcr14(&mokvars), ]; - let client = Client::try_default().await?; + let client = timed_client().await?; let config_maps: Api = Api::default_namespaced(client.clone()); let mut image_pcrs_map = config_maps.get(PCR_CONFIG_MAP).await?; diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 03adb539..e01ae23d 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -27,6 +27,31 @@ use anyhow::{Context, Result, anyhow}; use conditions::*; use k8s_openapi::apimachinery::pkg::apis::meta::v1::{Condition, OwnerReference, Time}; use kube::{Api, Client, Resource}; +use std::time::Duration; + +/// Read timeout for non-watch Kubernetes API calls. +pub const KUBE_READ_TIMEOUT: Duration = Duration::from_secs(30); + +/// Create a [`Client`] with [`KUBE_READ_TIMEOUT`] applied. +/// +/// Use this for request/response API calls. For long-lived watch streams, use +/// [`Client::try_default`] instead so the connection never times out on idle. +pub async fn timed_client() -> Result { + let mut config = kube::Config::infer().await?; + config.read_timeout = Some(KUBE_READ_TIMEOUT); + Ok(Client::try_from(config)?) +} + +/// A pair of Kubernetes clients +/// +/// `client` has a read timeout for request/response API calls. +/// `watch` has no read timeout, keeping it safe for long-lived watch streams +/// (Controllers, Reflectors, `await_condition`). +#[derive(Clone)] +pub struct KubeClients { + pub client: Client, + pub watch: Client, +} #[macro_export] macro_rules! update_status { diff --git a/operator/src/attestation_key_register.rs b/operator/src/attestation_key_register.rs index 2fb6955b..1595b9f6 100644 --- a/operator/src/attestation_key_register.rs +++ b/operator/src/attestation_key_register.rs @@ -30,8 +30,8 @@ use serde_json::json; use std::{collections::BTreeMap, sync::Arc, time::Duration}; use trusted_cluster_operator_lib::conditions::ATTESTATION_KEY_MACHINE_APPROVE; -use trusted_cluster_operator_lib::endpoints::*; -use trusted_cluster_operator_lib::{AttestationKey, AttestationKeyStatus, Machine, update_status}; +use trusted_cluster_operator_lib::{AttestationKey, AttestationKeyStatus, KubeClients, Machine}; +use trusted_cluster_operator_lib::{endpoints::*, update_status}; use crate::conditions::attestation_key_approved_condition; use crate::trustee; @@ -44,6 +44,7 @@ use operator::{ /// Stores give local cache access to avoid repeated API-server reads. pub struct AkContextData { pub client: Client, + watch_client: Client, pub machine_store: Store, pub ak_store: Store, pub secret_store: Store, @@ -51,19 +52,21 @@ pub struct AkContextData { } impl AkContextData { - pub fn new(client: Client) -> Self { + pub fn new(clients: &KubeClients) -> Self { let (machine_store, machine_writer) = reflector::store::(); let (ak_store, ak_writer) = reflector::store::(); let (secret_store, secret_writer) = reflector::store::(); let (deployment_store, deployment_writer) = reflector::store::(); - crate::spawn_reflector::(machine_writer, client.clone(), "Machine"); - crate::spawn_reflector::(ak_writer, client.clone(), "AttestationKey"); - crate::spawn_reflector::(secret_writer, client.clone(), "Secret"); - crate::spawn_reflector::(deployment_writer, client.clone(), "Deployment"); + let watch = &clients.watch; + crate::spawn_reflector::(machine_writer, watch.clone(), "Machine"); + crate::spawn_reflector::(ak_writer, watch.clone(), "AttestationKey"); + crate::spawn_reflector::(secret_writer, watch.clone(), "Secret"); + crate::spawn_reflector::(deployment_writer, watch.clone(), "Deployment"); Self { - client, + client: clients.client.clone(), + watch_client: watch.clone(), machine_store, ak_store, secret_store, @@ -362,7 +365,7 @@ async fn secret_reconcile( } pub async fn launch_ak_controller(ctx: Arc) { - let aks: Api = Api::default_namespaced(ctx.client.clone()); + let aks: Api = Api::default_namespaced(ctx.watch_client.clone()); tokio::spawn( Controller::new(aks, watcher::Config::default()) .run(ak_reconcile, controller_error_policy, ctx) @@ -376,7 +379,7 @@ pub async fn launch_ak_controller(ctx: Arc) { } pub async fn launch_machine_ak_controller(ctx: Arc) { - let machines: Api = Api::default_namespaced(ctx.client.clone()); + let machines: Api = Api::default_namespaced(ctx.watch_client.clone()); tokio::spawn( Controller::new(machines, watcher::Config::default()) .run(machine_reconcile, controller_error_policy, ctx) @@ -390,7 +393,7 @@ pub async fn launch_machine_ak_controller(ctx: Arc) { } pub async fn launch_secret_ak_controller(ctx: Arc) { - let secrets: Api = Api::default_namespaced(ctx.client.clone()); + let secrets: Api = Api::default_namespaced(ctx.watch_client.clone()); tokio::spawn( Controller::new(secrets, watcher::Config::default()) .run(secret_reconcile, controller_error_policy, ctx) diff --git a/operator/src/main.rs b/operator/src/main.rs index dcd14515..aba3805d 100644 --- a/operator/src/main.rs +++ b/operator/src/main.rs @@ -18,8 +18,10 @@ use kube::{Api, Client}; use log::{info, warn}; use operator::{generate_owner_reference, upsert_condition}; -use trusted_cluster_operator_lib::{TrustedExecutionCluster, TrustedExecutionClusterStatus}; -use trusted_cluster_operator_lib::{conditions::*, images::*, update_status}; +use trusted_cluster_operator_lib::{ + KubeClients, TrustedExecutionCluster, TrustedExecutionClusterStatus, conditions::*, images::*, + timed_client, update_status, +}; mod attestation_key_register; mod conditions; @@ -44,15 +46,16 @@ struct ClusterContext { } impl ClusterContext { - fn new(client: Client) -> Self { + fn new(clients: &KubeClients) -> Self { let (tec_store, tec_writer) = reflector::store::(); spawn_reflector::( tec_writer, - client.clone(), + clients.watch.clone(), "TrustedExecutionCluster", ); + let client = clients.client.clone(); Self { client, tec_store } } @@ -254,19 +257,22 @@ async fn install_attestation_key_register( async fn main() -> Result<()> { env_logger::Builder::from_env(Env::default().default_filter_or("info")).init(); let _ = jsonwebtoken_openssl::install_default(); - let kube_client = Client::try_default().await?; + let kube_client = timed_client().await?; + let watch_client = Client::try_default().await?; + let clients = KubeClients { + client: kube_client.clone(), + watch: watch_client.clone(), + }; info!("trusted execution clusters operator",); const CACHE_SYNC_TIMEOUT: Duration = Duration::from_secs(60); // Launch controllers that do not depend on reflector caches first. - register_server::launch_keygen_controller(kube_client.clone()).await; + register_server::launch_keygen_controller(clients.clone()).await; // Spawn reflectors (starts background list-watch immediately). - let ak_ctx = Arc::new(attestation_key_register::AkContextData::new( - kube_client.clone(), - )); - let ctx = Arc::new(ClusterContext::new(kube_client.clone())); + let ak_ctx = Arc::new(attestation_key_register::AkContextData::new(&clients)); + let ctx = Arc::new(ClusterContext::new(&clients)); // Best-effort wait for caches; controllers will work with // partially-filled stores if the sync times out. @@ -279,14 +285,14 @@ async fn main() -> Result<()> { info!("Starting controllers"); - let cl: Api = Api::default_namespaced(kube_client.clone()); + let cl: Api = Api::default_namespaced(watch_client); attestation_key_register::launch_ak_controller(ak_ctx.clone()).await; attestation_key_register::launch_machine_ak_controller(ak_ctx.clone()).await; attestation_key_register::launch_secret_ak_controller(ak_ctx).await; - reference_values::launch_rv_image_controller(kube_client.clone()).await; - reference_values::launch_rv_job_controller(kube_client.clone()).await; - trustee::launch_trustee_sync_controller(kube_client.clone()).await; + reference_values::launch_rv_image_controller(clients.clone()).await; + reference_values::launch_rv_job_controller(clients.clone()).await; + trustee::launch_trustee_sync_controller(clients).await; Controller::new(cl, watcher::Config::default()) .run(reconcile, controller_error_policy, ctx) diff --git a/operator/src/reference_values.rs b/operator/src/reference_values.rs index 7737a308..9b4267d0 100644 --- a/operator/src/reference_values.rs +++ b/operator/src/reference_values.rs @@ -133,15 +133,16 @@ async fn job_reconcile(job: Arc, client: Arc) -> Result = Api::default_namespaced(client.clone()); +pub async fn launch_rv_job_controller(clients: KubeClients) { + let jobs: Api = Api::default_namespaced(clients.watch); let watcher = watcher::Config { label_selector: Some(format!("{JOB_LABEL_KEY}={PCR_COMMAND_NAME}")), ..Default::default() }; + let client = Arc::new(clients.client); tokio::spawn( Controller::new(jobs, watcher) - .run(job_reconcile, controller_error_policy, Arc::new(client)) + .run(job_reconcile, controller_error_policy, client) .for_each(controller_info), ); } @@ -339,14 +340,15 @@ async fn image_remove_reconcile( Ok(Action::await_change()) } -pub async fn launch_rv_image_controller(client: Client) { - let images: Api = Api::default_namespaced(client.clone()); - let jobs: Api = Api::default_namespaced(client.clone()); +pub async fn launch_rv_image_controller(clients: KubeClients) { + let images: Api = Api::default_namespaced(clients.watch.clone()); + let jobs: Api = Api::default_namespaced(clients.watch); let wc = watcher::Config::default().labels(&format!("{JOB_LABEL_KEY}={PCR_COMMAND_NAME}")); + let client = Arc::new(clients.client); tokio::spawn( Controller::new(images, Default::default()) .owns(jobs, wc) - .run(image_reconcile, controller_error_policy, Arc::new(client)) + .run(image_reconcile, controller_error_policy, client) .for_each(controller_info), ); } diff --git a/operator/src/register_server.rs b/operator/src/register_server.rs index 96b689e3..08fbeca8 100644 --- a/operator/src/register_server.rs +++ b/operator/src/register_server.rs @@ -24,7 +24,7 @@ use std::{collections::BTreeMap, sync::Arc}; use crate::trustee; use operator::*; -use trusted_cluster_operator_lib::{Machine, TrustedExecutionCluster, endpoints::*}; +use trusted_cluster_operator_lib::{KubeClients, Machine, TrustedExecutionCluster, endpoints::*}; /// Finalizer name to discard decryption keys when a machine is deleted const MACHINE_FINALIZER: &str = "finalizer.machine.trusted-execution-clusters.io"; @@ -198,11 +198,12 @@ async fn keygen_reconcile( .map_err(|e| anyhow!("failed to reconcile on machine: {e}").into()) } -pub async fn launch_keygen_controller(client: Client) { - let machines: Api = Api::default_namespaced(client.clone()); +pub async fn launch_keygen_controller(clients: KubeClients) { + let machines: Api = Api::default_namespaced(clients.watch); + let client = Arc::new(clients.client); tokio::spawn( Controller::new(machines, Default::default()) - .run(keygen_reconcile, controller_error_policy, Arc::new(client)) + .run(keygen_reconcile, controller_error_policy, client) .for_each(controller_info), ); } diff --git a/operator/src/trustee.rs b/operator/src/trustee.rs index 7732bd7a..eded4607 100644 --- a/operator/src/trustee.rs +++ b/operator/src/trustee.rs @@ -41,7 +41,7 @@ use std::collections::BTreeMap; use std::sync::Arc; use std::time::Duration; -use trusted_cluster_operator_lib::reference_values::*; +use trusted_cluster_operator_lib::{KubeClients, reference_values::*}; use trusted_cluster_operator_lib::{Machine, endpoints::*}; const TRUSTEE_DATA_DIR: &str = "/etc/kbs"; @@ -381,8 +381,8 @@ async fn trustee_deployment_reconcile( Ok(Action::await_change()) } -pub async fn launch_trustee_sync_controller(client: Client) { - let deployments: Api = Api::default_namespaced(client.clone()); +pub async fn launch_trustee_sync_controller(clients: KubeClients) { + let deployments: Api = Api::default_namespaced(clients.watch.clone()); let watcher_config = watcher::Config { label_selector: Some(format!("app={TRUSTEE_APP_LABEL}")), ..Default::default() @@ -392,7 +392,7 @@ pub async fn launch_trustee_sync_controller(client: Client) { .run( trustee_deployment_reconcile, controller_error_policy, - Arc::new(client), + Arc::new(clients.client), ) .for_each(controller_info), ); diff --git a/register-server/src/main.rs b/register-server/src/main.rs index 6db836b2..def96601 100644 --- a/register-server/src/main.rs +++ b/register-server/src/main.rs @@ -25,7 +25,7 @@ use uuid::Uuid; use trusted_cluster_operator_lib::endpoints::*; use trusted_cluster_operator_lib::{ - generate_owner_reference, get_trusted_execution_cluster, Machine, MachineSpec, + generate_owner_reference, get_trusted_execution_cluster, timed_client, Machine, MachineSpec, }; #[derive(Parser)] @@ -237,7 +237,7 @@ async fn main() { let err = "failed to create Kubernetes client"; let app = Router::new() .route(&endpoint, get(register_handler)) - .with_state(Client::try_default().await.expect(err)); + .with_state(timed_client().await.expect(err)); let addr = SocketAddr::from(([0, 0, 0, 0], args.port)); let service = app.into_make_service(); diff --git a/test_utils/src/lib.rs b/test_utils/src/lib.rs index 79377437..96e68fda 100644 --- a/test_utils/src/lib.rs +++ b/test_utils/src/lib.rs @@ -26,7 +26,9 @@ use trusted_cluster_operator_lib::conditions::COMMITTED_CONDITION; use trusted_cluster_operator_lib::issuers::{Issuer, IssuerCa, IssuerSpec}; use trusted_cluster_operator_lib::{ApprovedImage, ApprovedImageStatus, AttestationKey, Machine}; -use trusted_cluster_operator_lib::{TrustedExecutionCluster, endpoints::*, images::*}; +use trusted_cluster_operator_lib::{ + KubeClients, TrustedExecutionCluster, endpoints::*, images::*, timed_client, +}; pub mod timer; pub use timer::Poller; @@ -194,30 +196,30 @@ trait K8sPlatform: Send + Sync { struct Kind { public: bool, - client: Client, + clients: KubeClients, namespace: String, } struct OpenShift { - client: Client, + clients: KubeClients, namespace: String, } struct OtherK8s {} -fn get_k8s_platform(client: &Client, namespace: &str) -> Box { - let client = client.clone(); +fn get_k8s_platform(clients: &KubeClients, namespace: &str) -> Box { + let clients = clients.clone(); let namespace = namespace.to_string(); match env::var(PLATFORM_ENV).as_deref().unwrap_or("kind") { "kind" => Box::new(Kind { public: false, - client, + clients, namespace, }), "kind_public" => Box::new(Kind { public: true, - client, + clients, namespace, }), - "openshift" => Box::new(OpenShift { client, namespace }), + "openshift" => Box::new(OpenShift { clients, namespace }), _ => Box::new(OtherK8s {}), } } @@ -249,7 +251,7 @@ impl K8sPlatform for Kind { port, ..Default::default() }; - let services: Api = Api::namespaced(self.client.clone(), &self.namespace); + let services: Api = Api::namespaced(self.clients.client.clone(), &self.namespace); let service = Service { metadata: ObjectMeta { name: Some(format!("{service}-forward")), @@ -284,7 +286,7 @@ enum OpenShiftHost { impl OpenShift { async fn get_url(&self, service: &str) -> OpenShiftHost { - let services: Api = Api::namespaced(self.client.clone(), &self.namespace); + let services: Api = Api::namespaced(self.clients.client.clone(), &self.namespace); let Ok(svc) = services.get(service).await else { return OpenShiftHost::None; }; @@ -313,7 +315,9 @@ impl K8sPlatform for OpenShift { cert_name: &str, _: &str, ) -> Result<()> { - let services: Api = Api::namespaced(self.client.clone(), &self.namespace); + let services: Api = Api::namespaced(self.clients.client.clone(), &self.namespace); + let watch_services: Api = + Api::namespaced(self.clients.watch.clone(), &self.namespace); let pp = Default::default(); let duration = scaled_duration(120); let lb = json!({ "spec": { "type": "LoadBalancer" } }); @@ -333,7 +337,7 @@ impl K8sPlatform for OpenShift { let ctx = format!( "waiting for ingress on {service} (attempt {attempt}/{EXPOSE_MAX_ATTEMPTS})" ); - let ingress_ready = await_condition(services.clone(), service, has_ingress); + let ingress_ready = await_condition(watch_services.clone(), service, has_ingress); match timeout(duration, ingress_ready).await { Err(_) => { last_err = Some(anyhow!(ctx)); @@ -369,7 +373,9 @@ impl K8sPlatform for OpenShift { return Err(e); } - let certs: Api = Api::namespaced(self.client.clone(), &self.namespace); + let certs: Api = Api::namespaced(self.clients.client.clone(), &self.namespace); + let watch_certs: Api = + Api::namespaced(self.clients.watch.clone(), &self.namespace); let cert = certs.get(cert_name).await?; let old_revision = cert.status.and_then(|st| st.revision).unwrap_or(0); let cert_patch = match url { @@ -397,11 +403,12 @@ impl K8sPlatform for OpenShift { cert.and_then(|c| c.status.as_ref().and_then(chk)) .unwrap_or(false) }; - let cert_done = await_condition(certs, cert_name, cert_reissued); + let cert_done = await_condition(watch_certs, cert_name, cert_reissued); let ctx = format!("waiting for cert {cert_name} to have a rev newer than {old_revision}"); timeout(duration, cert_done).await.context(ctx)??; - let deployments: Api = Api::namespaced(self.client.clone(), &self.namespace); + let deployments: Api = + Api::namespaced(self.clients.client.clone(), &self.namespace); deployments.restart(deployment).await?; Ok(()) @@ -438,7 +445,7 @@ impl K8sPlatform for OtherK8s { } pub async fn get_cluster_url( - client: &Client, + clients: &KubeClients, namespace: &str, service: &str, port: Option, @@ -450,7 +457,7 @@ pub async fn get_cluster_url( None => full_url, }); } - get_k8s_platform(client, namespace) + get_k8s_platform(clients, namespace) .get_cluster_url(service, port) .await } @@ -458,7 +465,7 @@ pub async fn get_cluster_url( static INIT: Once = Once::new(); pub struct TestContext { - client: Client, + clients: KubeClients, test_namespace: String, manifests_dir: String, test_name: String, @@ -471,11 +478,11 @@ impl TestContext { let _ = env_logger::builder().is_test(true).try_init(); }); - let client = setup_test_client().await?; + let clients = setup_test_clients().await?; let namespace = test_namespace_name(); let ctx = Self { - client, + clients, test_namespace: namespace, manifests_dir: String::new(), test_name: test_name.to_string(), @@ -499,7 +506,15 @@ impl TestContext { } pub fn client(&self) -> &Client { - &self.client + &self.clients.client + } + + pub fn watch_client(&self) -> &Client { + &self.clients.watch + } + + pub fn clients(&self) -> &KubeClients { + &self.clients } pub fn namespace(&self) -> &str { @@ -537,7 +552,7 @@ impl TestContext { "Creating test namespace: {}", self.test_namespace ); - let namespace_api: Api = Api::all(self.client.clone()); + let namespace_api: Api = Api::all(self.clients.client.clone()); let namespace = Namespace { metadata: k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta { name: Some(self.test_namespace.clone()), @@ -558,7 +573,7 @@ impl TestContext { K: kube::Resource + Clone, K: k8s_openapi::serde::de::DeserializeOwned + std::fmt::Debug + Send + 'static, { - let api: Api = Api::namespaced(self.client.clone(), &self.test_namespace); + let api: Api = Api::namespaced(self.clients.client.clone(), &self.test_namespace); let list = api.list(&Default::default()).await?; if let Some(item) = list.items.first() { return Err(anyhow!("Resource still present: {item:?}")); @@ -568,7 +583,7 @@ impl TestContext { async fn delete_trusted_execution_cluster(&self) -> Result<()> { let tec_api: Api = - Api::namespaced(self.client.clone(), &self.test_namespace); + Api::namespaced(self.clients.client.clone(), &self.test_namespace); let dp = DeleteParams::default(); let tec_list = tec_api.list(&Default::default()).await?; @@ -596,14 +611,15 @@ impl TestContext { } async fn cleanup_namespace(&self) -> Result<()> { - let namespace_api: Api = Api::all(self.client.clone()); + let namespace_api: Api = Api::all(self.clients.client.clone()); + let watch_namespaces: Api = Api::all(self.clients.watch.clone()); let dp = DeleteParams::default(); match namespace_api.get(&self.test_namespace).await { Ok(_) => { namespace_api.delete(&self.test_namespace, &dp).await?; let timeout = scaled_timeout(300); - wait_for_resource_deleted(&namespace_api, &self.test_namespace, timeout).await?; + wait_for_resource_deleted(&watch_namespaces, &self.test_namespace, timeout).await?; test_info!(&self.test_name, "Deleted namespace {}", self.test_namespace); } Err(kube::Error::Api(ae)) if ae.code == 404 => { @@ -675,8 +691,8 @@ impl TestContext { issuer_name: &str, ) -> Result<()> { let ns = &self.test_namespace; - let domain = get_cluster_url(&self.client, ns, service_name, None).await?; - let certs: Api = Api::namespaced(self.client.clone(), ns); + let domain = get_cluster_url(&self.clients, ns, service_name, None).await?; + let certs: Api = Api::namespaced(self.clients.client.clone(), ns); let cert = Certificate { metadata: ObjectMeta { name: Some(cert_name.to_string()), @@ -711,7 +727,7 @@ impl TestContext { }, ..Default::default() }; - let issuers: Api = Api::namespaced(self.client.clone(), ns); + let issuers: Api = Api::namespaced(self.clients.client.clone(), ns); issuers.create(&Default::default(), &root_issuer).await?; let root_cert = Certificate { metadata: ObjectMeta { @@ -730,7 +746,7 @@ impl TestContext { }, ..Default::default() }; - let certs: Api = Api::namespaced(self.client.clone(), ns); + let certs: Api = Api::namespaced(self.clients.client.clone(), ns); certs.create(&Default::default(), &root_cert).await?; let issuer_name = "issuer"; let issuer = Issuer { @@ -758,7 +774,8 @@ impl TestContext { self.create_certificate(svc, ATT_REG_CERT, ATT_REG_SECRET, issuer_name) .await?; - let secrets: Api = Api::namespaced(self.client.clone(), &self.test_namespace); + let secrets: Api = + Api::namespaced(self.clients.client.clone(), &self.test_namespace); for secret in [REG_SECRET, TRUSTEE_SECRET, ATT_REG_SECRET] { wait_for_resource_created(&secrets, secret, scaled_timeout(60)).await?; } @@ -905,7 +922,7 @@ impl TestContext { std::fs::write(&le_rb_dst, le_rb_content)?; test_info!(&self.test_name, "Preparing RBAC kustomization"); - let platform = get_k8s_platform(&self.client, &self.test_namespace); + let platform = get_k8s_platform(&self.clients, &self.test_namespace); let kustomization_src = workspace_root.join("config/rbac/kustomization.yaml.in"); let kustomization_content = std::fs::read_to_string(&kustomization_src)?; let mut kustom_value: serde_yaml::Value = serde_yaml::from_str(&kustomization_content)?; @@ -971,7 +988,7 @@ impl TestContext { ); if get_virt_provider()? == VirtProvider::Kubevirt { - let platform = get_k8s_platform(&self.client, &self.test_namespace); + let platform = get_k8s_platform(&self.clients, &self.test_namespace); let port = ATTESTATION_KEY_REGISTER_PORT; let address = platform.get_cluster_url(ATTESTATION_KEY_REGISTER_SERVICE, Some(port)); spec_map.insert( @@ -997,7 +1014,7 @@ impl TestContext { "Applying ApprovedImage manifest" ); - let depls: Api = Api::namespaced(self.client.clone(), ns); + let depls: Api = Api::namespaced(self.clients.watch.clone(), ns); for depl in [ "trusted-cluster-operator", REGISTER_SERVER_DEPLOYMENT, @@ -1009,14 +1026,14 @@ impl TestContext { } let svc = ATTESTATION_KEY_REGISTER_SERVICE; - let services: Api = Api::namespaced(self.client.clone(), ns); + let services: Api = Api::namespaced(self.clients.watch.clone(), ns); for svc in [REGISTER_SERVER_SERVICE, TRUSTEE_SERVICE, svc] { let done = await_condition(services.clone(), svc, |s: Option<&Service>| s.is_some()); let ctx = format!("waiting for service {svc} to exist"); timeout(scaled_duration(60), done).await.context(ctx)??; } - let platform = get_k8s_platform(&self.client, &self.test_namespace); + let platform = get_k8s_platform(&self.clients, &self.test_namespace); let svc = REGISTER_SERVER_SERVICE; let depl = REGISTER_SERVER_DEPLOYMENT; let test_name = &self.test_name; @@ -1028,9 +1045,9 @@ impl TestContext { let depl = ATTESTATION_KEY_REGISTER_DEPLOYMENT; platform.expose(svc, depl, ATT_REG_CERT, test_name).await?; - let tecs: Api = Api::namespaced(self.client.clone(), ns); + let tecs: Api = Api::namespaced(self.clients.client.clone(), ns); let trustee_addr = - get_cluster_url(&self.client, ns, TRUSTEE_SERVICE, Some(TRUSTEE_PORT)).await?; + get_cluster_url(&self.clients, ns, TRUSTEE_SERVICE, Some(TRUSTEE_PORT)).await?; let json = json!({ "spec": { "publicTrusteeAddr": trustee_addr @@ -1046,12 +1063,12 @@ impl TestContext { &self.test_name, "Waiting for image-pcrs ConfigMap to be created" ); - let configmap_api: Api = Api::namespaced(self.client.clone(), ns); + let configmap_api: Api = Api::namespaced(self.clients.client.clone(), ns); wait_for_resource_created(&configmap_api, "image-pcrs", scaled_timeout(60)).await?; let info = format!("Waiting for ApprovedImage {APPROVED_IMAGE_NAME} to be Committed"); test_info!(&self.test_name, "{info}"); - let images: Api = Api::namespaced(self.client.clone(), ns); + let images: Api = Api::namespaced(self.clients.watch.clone(), ns); let image_ready = |img: Option<&ApprovedImage>| { let chk_cond = |c: &Condition| c.type_ == COMMITTED_CONDITION && c.status == "True"; let chk_status = @@ -1096,9 +1113,10 @@ macro_rules! setup { (delayed_approved_image) => {{ $crate::TestContext::new(TEST_NAME, true) }}; } -async fn setup_test_client() -> Result { - let client = Client::try_default().await?; - Ok(client) +async fn setup_test_clients() -> Result { + let client = timed_client().await?; + let watch = Client::try_default().await?; + Ok(KubeClients { client, watch }) } fn test_namespace_name() -> String { diff --git a/test_utils/src/virt/kubevirt.rs b/test_utils/src/virt/kubevirt.rs index 25c2d84a..2113b83a 100644 --- a/test_utils/src/virt/kubevirt.rs +++ b/test_utils/src/virt/kubevirt.rs @@ -36,7 +36,8 @@ impl VmBackend for KubevirtBackend { ..Default::default() }; - let secrets: Api = Api::namespaced(self.0.client.clone(), &self.0.namespace); + let secrets: Api = + Api::namespaced(self.0.clients.client.clone(), &self.0.namespace); secrets.create(&Default::default(), &secret).await?; let vm = VirtualMachine { @@ -134,14 +135,16 @@ impl VmBackend for KubevirtBackend { ..Default::default() }; - let vms: Api = Api::namespaced(self.0.client.clone(), &self.0.namespace); + let vms: Api = + Api::namespaced(self.0.clients.client.clone(), &self.0.namespace); vms.create(&Default::default(), &vm).await?; Ok(()) } async fn wait_for_running(&self, timeout_secs: u64) -> Result<()> { - let api: Api = Api::namespaced(self.0.client.clone(), &self.0.namespace); + let api: Api = + Api::namespaced(self.0.clients.watch.clone(), &self.0.namespace); let machine_running = |m: Option<&VirtualMachine>| { let status = m.as_ref().and_then(|m| m.status.as_ref()); let print_status = status.and_then(|s| s.printable_status.as_ref()); @@ -180,7 +183,8 @@ impl VmBackend for KubevirtBackend { } // Use the UUID to get the secret (secrets are named with just the UUID) - let secrets: Api = Api::namespaced(self.0.client.clone(), &self.0.namespace); + let secrets: Api = + Api::namespaced(self.0.clients.client.clone(), &self.0.namespace); let secret = secrets .get(uuid) .await diff --git a/test_utils/src/virt/mod.rs b/test_utils/src/virt/mod.rs index 77f849f6..8de55029 100644 --- a/test_utils/src/virt/mod.rs +++ b/test_utils/src/virt/mod.rs @@ -9,7 +9,7 @@ pub mod kubevirt; use anyhow::{Result, anyhow}; use clevis_pin_trustee_lib::Key as ClevisKey; use k8s_openapi::api::core::v1::Secret; -use kube::{Api, Client}; +use kube::Api; use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode}; use std::{env, path::PathBuf, time::Duration}; use tokio::process::Command; @@ -22,7 +22,7 @@ use crate::{ROOT_SECRET, VirtProvider, get_cluster_url, get_env, get_virt_provid #[derive(Clone)] pub struct VmConfig { - pub client: Client, + pub clients: KubeClients, pub namespace: String, pub vm_name: String, pub ssh_public_key: String, @@ -76,10 +76,10 @@ pub fn generate_ssh_key_pair() -> Result<(String, PathBuf)> { pub async fn generate_ignition(config: &VmConfig) -> Result { use ignition_config::v3_6::*; - let client = config.client.clone(); let ns = &config.namespace; let port = Some(REGISTER_SERVER_PORT); - let register_server_url = get_cluster_url(&client, ns, REGISTER_SERVER_SERVICE, port).await?; + let register_server_url = + get_cluster_url(&config.clients, ns, REGISTER_SERVER_SERVICE, port).await?; let root_pem_encoded = utf8_percent_encode(&config.ca_pem, NON_ALPHANUMERIC); let ignition = Ignition { version: "3.6.0".to_string(), @@ -148,11 +148,11 @@ pub async fn ssh_exec(command: &str) -> Result { } pub async fn create_backend( - client: Client, + clients: KubeClients, namespace: &str, vm_name: &str, ) -> Result> { - let secrets: Api = Api::namespaced(client.clone(), namespace); + let secrets: Api = Api::namespaced(clients.client.clone(), namespace); let root_secret = secrets.get(ROOT_SECRET).await?; let root_secret_data = root_secret.data.unwrap(); let ca_pem_bytes = root_secret_data.get("ca.crt").unwrap(); @@ -162,7 +162,7 @@ pub async fn create_backend( let (public_key, key_path) = generate_ssh_key_pair()?; let image = get_env("TEST_IMAGE")?; let config = VmConfig { - client, + clients, namespace: namespace.to_string(), vm_name: vm_name.to_string(), ssh_public_key: public_key, diff --git a/tests/attestation.rs b/tests/attestation.rs index a1e1ef29..6ae0f950 100644 --- a/tests/attestation.rs +++ b/tests/attestation.rs @@ -36,10 +36,10 @@ impl SingleAttestationContext { impl SingleAttestationContext { async fn new(vm_name: &str, test_ctx: &TestContext) -> Result { - let client = test_ctx.client(); let namespace = test_ctx.namespace(); - let backend = virt::create_backend(client.clone(), namespace, vm_name).await?; + let backend = + virt::create_backend(test_ctx.clients().clone(), namespace, vm_name).await?; test_ctx.info(format!("Creating VM: {vm_name}")); backend.create_vm().await?; @@ -83,15 +83,15 @@ async fn test_attestation() -> anyhow::Result<()> { virt_test! { async fn test_parallel_vm_attestation() -> anyhow::Result<()> { let test_ctx = setup!().await?; - let client = test_ctx.client(); + let clients = test_ctx.clients().clone(); let namespace = test_ctx.namespace(); test_ctx.info("Testing parallel VM attestation - launching 2 VMs simultaneously"); // Launch both VMs in parallel let vm1_name = "test-coreos-vm1"; let vm2_name = "test-coreos-vm2"; - let backend1 = virt::create_backend(client.clone(), namespace, vm1_name).await?; - let backend2 = virt::create_backend(client.clone(), namespace, vm2_name).await?; + let backend1 = virt::create_backend(clients.clone(), namespace, vm1_name).await?; + let backend2 = virt::create_backend(clients.clone(), namespace, vm2_name).await?; test_ctx.info("Creating VM1 and VM2 in parallel"); let (vm1_result, vm2_result) = tokio::join!(backend1.create_vm(), backend2.create_vm()); @@ -198,10 +198,12 @@ async fn test_vm_reboot_delete_machine() -> anyhow::Result<()> { let att_ctx = SingleAttestationContext::new(vm_name, &test_ctx).await?; let machines: Api = Api::namespaced(test_ctx.client().clone(), test_ctx.namespace()); + let watch_machines: Api = + Api::namespaced(test_ctx.watch_client().clone(), test_ctx.namespace()); let list = machines.list(&Default::default()).await?; let name = list.items[0].metadata.name.as_ref().unwrap(); machines.delete(name, &Default::default()).await?; - wait_for_resource_deleted(&machines, name, scaled_timeout(120)).await?; + wait_for_resource_deleted(&watch_machines, name, scaled_timeout(120)).await?; test_ctx.info("Performing reboot, expecting missing resource"); let boot_id = att_ctx.backend.get_boot_id().await?; diff --git a/tests/trusted_execution_cluster.rs b/tests/trusted_execution_cluster.rs index 781164da..9dfd85d7 100644 --- a/tests/trusted_execution_cluster.rs +++ b/tests/trusted_execution_cluster.rs @@ -40,6 +40,7 @@ named_test!( async fn test_trusted_execution_cluster_uninstall() -> anyhow::Result<()> { let test_ctx = setup!().await?; let client = test_ctx.client(); + let watch = test_ctx.watch_client(); let namespace = test_ctx.namespace(); let tec_api: Api = Api::namespaced(client.clone(), namespace); @@ -95,7 +96,8 @@ named_test!( )); // Wait for the AttestationKey to be approved (operator should match Machine IP and approve it) - let done = await_condition(attestation_keys.clone(), &ak_name, ak_approved); + let watch_aks: Api = Api::namespaced(watch.clone(), namespace); + let done = await_condition(watch_aks.clone(), &ak_name, ak_approved); let ctx = format!("waiting for AttestationKey {ak_name} to be approved"); timeout(scaled_duration(30), done).await.context(ctx)??; @@ -107,20 +109,22 @@ named_test!( api.delete(TEC_NAME, &dp).await?; // Wait until it disappears - wait_for_resource_deleted(&api, TEC_NAME, scaled_timeout(120)).await?; + let watch_tec: Api = Api::namespaced(watch.clone(), namespace); + wait_for_resource_deleted(&watch_tec, TEC_NAME, scaled_timeout(120)).await?; - let deployments_api: Api = Api::namespaced(client.clone(), namespace); + let watch_depls: Api = Api::namespaced(watch.clone(), namespace); let timeout = scaled_timeout(120); - wait_for_resource_deleted(&deployments_api, TRUSTEE_DEPLOYMENT, timeout).await?; - wait_for_resource_deleted(&deployments_api, REGISTER_SERVER_DEPLOYMENT, timeout).await?; + wait_for_resource_deleted(&watch_depls, TRUSTEE_DEPLOYMENT, timeout).await?; + wait_for_resource_deleted(&watch_depls, REGISTER_SERVER_DEPLOYMENT, timeout).await?; - let images_api: Api = Api::namespaced(client.clone(), namespace); - wait_for_resource_deleted(&images_api, APPROVED_IMAGE_NAME, scaled_timeout(120)).await?; + let watch_images: Api = Api::namespaced(watch.clone(), namespace); + wait_for_resource_deleted(&watch_images, APPROVED_IMAGE_NAME, scaled_timeout(120)).await?; - wait_for_resource_deleted(&machines, &machine_name, scaled_timeout(120)).await?; - wait_for_resource_deleted(&attestation_keys, &ak_name, scaled_timeout(120)).await?; - let secrets_api: Api = Api::namespaced(client.clone(), namespace); - wait_for_resource_deleted(&secrets_api, &ak_name, scaled_timeout(120)).await?; + let watch_machines: Api = Api::namespaced(watch.clone(), namespace); + wait_for_resource_deleted(&watch_machines, &machine_name, scaled_timeout(120)).await?; + wait_for_resource_deleted(&watch_aks, &ak_name, scaled_timeout(120)).await?; + let watch_secrets: Api = Api::namespaced(watch.clone(), namespace); + wait_for_resource_deleted(&watch_secrets, &ak_name, scaled_timeout(120)).await?; test_ctx.cleanup().await?; @@ -132,9 +136,10 @@ named_test! { async fn test_image_pcrs_configmap_updates() -> anyhow::Result<()> { let test_ctx = setup!().await?; let client = test_ctx.client(); + let watch = test_ctx.watch_client(); let namespace = test_ctx.namespace(); - let configmap_api: Api = Api::namespaced(client.clone(), namespace); + let configmap_api: Api = Api::namespaced(watch.clone(), namespace); let populated = |cm: Option<&ConfigMap>| { let data = cm.and_then(|cm| cm.data.as_ref()); let json = data.and_then(|data| data.get("image-pcrs.json")); @@ -145,7 +150,8 @@ async fn test_image_pcrs_configmap_updates() -> anyhow::Result<()> { let ctx = "waiting for ConfigMap image-pcrs to be populated"; timeout(scaled_duration(180), done).await.context(ctx)??; - let image_pcrs_cm = configmap_api.get("image-pcrs").await?; + let cm_api: Api = Api::namespaced(client.clone(), namespace); + let image_pcrs_cm = cm_api.get("image-pcrs").await?; assert_eq!(image_pcrs_cm.metadata.name.as_deref(), Some("image-pcrs")); let data = image_pcrs_cm.data.as_ref() @@ -223,12 +229,13 @@ named_test! { async fn test_image_disallow() -> anyhow::Result<()> { let test_ctx = setup!().await?; let client = test_ctx.client(); + let watch = test_ctx.watch_client(); let namespace = test_ctx.namespace(); let images: Api = Api::namespaced(client.clone(), namespace); images.delete(APPROVED_IMAGE_NAME, &DeleteParams::default()).await?; - let configmap_api: Api = Api::namespaced(client.clone(), namespace); + let configmap_api: Api = Api::namespaced(watch.clone(), namespace); let chk_removed = |cm: Option<&ConfigMap>| { let data = cm.and_then(|cm| cm.data.as_ref()); let json = data.and_then(|data| data.get(RV_JSON_KEY)); @@ -247,6 +254,7 @@ named_test! { async fn test_attestation_key_lifecycle() -> anyhow::Result<()> { let test_ctx = setup!().await?; let client = test_ctx.client(); + let watch = test_ctx.watch_client(); let namespace = test_ctx.namespace(); let tec_api: Api = Api::namespaced(client.clone(), namespace); @@ -301,7 +309,8 @@ async fn test_attestation_key_lifecycle() -> anyhow::Result<()> { )); // Timeout for the AttestationKey to be approved, have owner reference, and have a Secret created - let approved = await_condition(attestation_keys.clone(), &ak_name, ak_approved); + let watch_aks: Api = Api::namespaced(watch.clone(), namespace); + let approved = await_condition(watch_aks.clone(), &ak_name, ak_approved); let ctx = format!("waiting for AttestationKey {ak_name} to be approved"); timeout(scaled_duration(30), approved).await.context(ctx)??; let chk_machine_owner = |ak: Option<&AttestationKey>| { @@ -309,16 +318,16 @@ async fn test_attestation_key_lifecycle() -> anyhow::Result<()> { let refs = ak.and_then(|ak| ak.metadata.owner_references.as_ref()); refs.map(|refs| refs.iter().any(chk_owner)).unwrap_or(false) }; - let has_machine_owner = await_condition(attestation_keys.clone(), &ak_name, chk_machine_owner); + let has_machine_owner = await_condition(watch_aks.clone(), &ak_name, chk_machine_owner); let ctx = format!("waiting for AttestationKey {ak_name} to be owned by Machine {machine_name}"); timeout(scaled_duration(30), has_machine_owner).await.context(ctx)??; - let secrets_api: Api = Api::namespaced(client.clone(), namespace); + let secrets: Api = Api::namespaced(watch.clone(), namespace); let chk_ak_owner = |secret: Option<&Secret>| { let chk_owner = |owner: &OwnerReference| owner.kind == "AttestationKey" && owner.name == ak_name; let refs = secret.and_then(|s| s.metadata.owner_references.as_ref()); refs.map(|refs| refs.iter().any(chk_owner)).unwrap_or(false) }; - let has_ak_owner = await_condition(secrets_api.clone(), &ak_name, chk_ak_owner); + let has_ak_owner = await_condition(secrets.clone(), &ak_name, chk_ak_owner); let ctx = format!("waiting for Secret {ak_name} to be owned by AttestationKey {ak_name}"); timeout(scaled_duration(30), has_ak_owner).await.context(ctx)??; @@ -331,11 +340,12 @@ async fn test_attestation_key_lifecycle() -> anyhow::Result<()> { machines.delete(&machine_name, &dp).await?; test_ctx.info(format!("Deleted Machine: {machine_name}")); - wait_for_resource_deleted(&machines, &machine_name, scaled_timeout(120)).await?; + let watch_machines: Api = Api::namespaced(watch.clone(), namespace); + wait_for_resource_deleted(&watch_machines, &machine_name, scaled_timeout(120)).await?; test_ctx.info("Machine successfully deleted"); - wait_for_resource_deleted(&attestation_keys, &ak_name, scaled_timeout(120)).await?; + wait_for_resource_deleted(&watch_aks, &ak_name, scaled_timeout(120)).await?; test_ctx.info("AttestationKey successfully deleted"); - wait_for_resource_deleted(&secrets_api, &ak_name, scaled_timeout(120)).await?; + wait_for_resource_deleted(&secrets, &ak_name, scaled_timeout(120)).await?; test_ctx.info("Secret successfully deleted"); test_ctx.cleanup().await?; @@ -348,6 +358,7 @@ named_test! { async fn test_nonexistent_approved_image() -> anyhow::Result<()> { let test_ctx = setup!().await?; let client = test_ctx.client(); + let watch = test_ctx.watch_client(); let namespace = test_ctx.namespace(); let images: Api = Api::namespaced(client.clone(), namespace); @@ -368,7 +379,8 @@ async fn test_nonexistent_approved_image() -> anyhow::Result<()> { let cs = img.and_then(|img| img.status.as_ref()).and_then(|s| s.conditions.as_ref()); cs.map(|cs| cs.iter().any(pending)).unwrap_or(false) }; - let done = await_condition(images, "coreos1", is_pending); + let watch_images: Api = Api::namespaced(watch.clone(), namespace); + let done = await_condition(watch_images, "coreos1", is_pending); let ctx = "waiting for ApprovedImage coreos1 to be PodPending"; timeout(scaled_duration(30), done).await.context(ctx)??; @@ -762,11 +774,13 @@ named_test! { async fn test_approved_image_readoption() -> anyhow::Result<()> { let test_ctx = setup!(delayed_approved_image).await?; let client = test_ctx.client(); + let watch = test_ctx.watch_client(); let namespace = test_ctx.namespace(); let clusters: Api = Api::namespaced(client.clone(), namespace); let images: Api = Api::namespaced(client.clone(), namespace); - let configmaps: Api = Api::namespaced(client.clone(), namespace); + let watch_images: Api = Api::namespaced(watch.clone(), namespace); + let configmaps: Api = Api::namespaced(watch.clone(), namespace); let cluster_spec = clusters.get(TEC_NAME).await?.spec; let image_spec = images.get(APPROVED_IMAGE_NAME).await?.spec; @@ -775,14 +789,14 @@ async fn test_approved_image_readoption() -> anyhow::Result<()> { let refs = img.and_then(|img| img.metadata.owner_references.as_ref()); refs.is_some_and(|refs| refs.iter().any(|o| o.kind == "TrustedExecutionCluster")) }; - let done = await_condition(images.clone(), APPROVED_IMAGE_NAME, owned); + let done = await_condition(watch_images.clone(), APPROVED_IMAGE_NAME, owned); let ctx = "waiting for ApprovedImage to be owned by TrustedExecutionCluster"; timeout(scaled_duration(30), done).await.context(ctx)??; test_ctx.info(format!("Deleting TrustedExecutionCluster {TEC_NAME}")); clusters.delete(TEC_NAME, &Default::default()).await?; wait_for_resource_deleted(&configmaps, TRUSTEE_RV_MAP, scaled_timeout(60)).await?; - wait_for_resource_deleted(&images, APPROVED_IMAGE_NAME, scaled_timeout(60)).await?; + wait_for_resource_deleted(&watch_images, APPROVED_IMAGE_NAME, scaled_timeout(60)).await?; test_ctx.info(format!("Configmap {TRUSTEE_RV_MAP} was removed")); let image = ApprovedImage { From c94e955c0e485d648263afe8fa1b84a117bca1ae Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Wed, 29 Jul 2026 19:58:26 +0200 Subject: [PATCH 10/15] operator/reg: Pass error if cleanup fails GET on trustedexecutionclusters is known to fail occasionally. This should return an error, which will lead to a requeue. Thus, also increase base deletion timeout to 120s (30s timeout on failure, 60s requeue). Signed-off-by: Jakob Naucke --- operator/src/register_server.rs | 4 ++++ test_utils/src/lib.rs | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/operator/src/register_server.rs b/operator/src/register_server.rs index 08fbeca8..99791070 100644 --- a/operator/src/register_server.rs +++ b/operator/src/register_server.rs @@ -179,6 +179,10 @@ async fn keygen_reconcile( ); return Ok(Action::await_change()); } + Err(e) => { + let err = anyhow!("{e}").into(); + return Err(finalizer::Error::::CleanupFailed(err)); + } _ => { // TEC exists and is not being deleted, proceed with unmount_secret } diff --git a/test_utils/src/lib.rs b/test_utils/src/lib.rs index 96e68fda..2df83a80 100644 --- a/test_utils/src/lib.rs +++ b/test_utils/src/lib.rs @@ -531,7 +531,7 @@ impl TestContext { pub async fn cleanup(&self) -> Result<()> { self.delete_trusted_execution_cluster().await?; - let timeout = scaled_duration(60); + let timeout = scaled_duration(120); let msg = format!("Resources were left behind after {timeout:?}"); let poller = Poller::new().with_timeout(timeout).with_error_message(msg); let chk = || async move { From 309438736c405691710b8d64751e889377d9d169 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Thu, 16 Jul 2026 20:04:29 +0200 Subject: [PATCH 11/15] kv: Forward console Signed-off-by: Jakob Naucke --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 4d8062f4..438d5b14 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ REG_SERVER_IMAGE=$(REGISTRY)/registration-server:$(TAG) ATTESTATION_KEY_REGISTER_IMAGE=$(REGISTRY)/attestation-key-register:$(TAG) TRUSTEE_IMAGE ?= quay.io/trusted-execution-clusters/key-broker-service:0.20-pad-token -TEST_IMAGE ?= quay.io/trusted-execution-clusters/fedora-coreos-kubevirt:42.20260622 +TEST_IMAGE ?= quay.io/trusted-execution-clusters/fedora-coreos-kubevirt:42.20251012.2.0-console-fwd # tagged as 42.20251012.2.0 APPROVED_IMAGE ?= quay.io/trusted-execution-clusters/fedora-coreos@sha256:6997f51fd27d1be1b5fc2e6cc3ebf16c17eb94d819b5d44ea8d6cf5f826ee773 From 0373165533ec410a1f84e4025e1467d9bee7ed02 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Fri, 17 Jul 2026 11:27:31 +0200 Subject: [PATCH 12/15] must-gather: Add kube-system logging Signed-off-by: Jakob Naucke --- must-gather/gather | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/must-gather/gather b/must-gather/gather index c899bd00..14942e5b 100755 --- a/must-gather/gather +++ b/must-gather/gather @@ -31,3 +31,7 @@ for ns in $(${KUBECTL} get namespaces -o jsonpath={..metadata.name}); do "${COLLECTION_PATH}/logs/${pod}-${ns}.log" done done + +for pod in $(${KUBECTL} get pods -o jsonpath={..metadata.name} -n kube-system); do + ${KUBECTL} logs "$pod" -n kube-system &> "${COLLECTION_PATH}/logs/${pod}-kube-system.log" +done From 35748362cb590ac7c65064ce8ca22dd4f67c4b8c Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Wed, 15 Jul 2026 17:13:11 +0200 Subject: [PATCH 13/15] gha: run & upload must-gather Signed-off-by: Jakob Naucke Assisted-by: AI --- .github/workflows/integration-tests.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 7ab5ac7b..c49b84bb 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -76,3 +76,14 @@ jobs: run: | eval $(ssh-agent -s) make integration-tests + - name: "Gather must-gather" + if: always() + run: must-gather/gather + env: + COLLECTION_PATH: must-gather-output + - name: "Upload must-gather" + if: always() + uses: actions/upload-artifact@v7 + with: + name: must-gather + path: must-gather-output/ From de6b5a92a984a8b70cf5152604864c299ca35d42 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Tue, 28 Jul 2026 13:57:56 +0200 Subject: [PATCH 14/15] gha: repeat ad inf Signed-off-by: Jakob Naucke --- .github/workflows/integration-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index c49b84bb..2e0cbe63 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -32,7 +32,6 @@ jobs: github.event.pull_request.author_association == 'COLLABORATOR' || contains(github.event.pull_request.labels.*.name, 'ok-to-test') runs-on: "ubuntu-24.04" - timeout-minutes: 120 steps: - name: "Check out repository" uses: actions/checkout@v7 @@ -75,7 +74,8 @@ jobs: - name: "Run integration tests" run: | eval $(ssh-agent -s) - make integration-tests + while make integration-tests; do :; done + exit 1 - name: "Gather must-gather" if: always() run: must-gather/gather From 690f50a62d0d3c55a39c303030386c1cd6567672 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Tue, 28 Jul 2026 12:36:15 +0200 Subject: [PATCH 15/15] gha: Use self-built for debugging to fail fast Signed-off-by: Jakob Naucke --- .github/workflows/integration-tests.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 2e0cbe63..953c4b5d 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -21,7 +21,8 @@ env: CARGO_TERM_COLOR: always RUNTIME: docker CONTAINER_CLI: docker - REGISTRY: localhost:5000/trusted-execution-clusters + REGISTRY: quay.io/jnaucke + TAG: 20260730-cd170d1 jobs: integration-tests: @@ -67,8 +68,6 @@ jobs: sudo mv virtctl /usr/local/bin/ - name: "Create KinD cluster" run: make cluster-up - - name: "Build and push images" - run: make push - name: "Install KubeVirt" run: make install-kubevirt - name: "Run integration tests"