From 0441d3475e727a910e5ac6a2c6b290acca3d4578 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Mon, 13 Jul 2026 10:28:50 +0200 Subject: [PATCH 1/6] operator/ak-reg: Use warn!, not eprintln! in consistency with remainder codebase Signed-off-by: Jakob Naucke --- operator/src/attestation_key_register.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/operator/src/attestation_key_register.rs b/operator/src/attestation_key_register.rs index 332e5124..aea0d450 100644 --- a/operator/src/attestation_key_register.rs +++ b/operator/src/attestation_key_register.rs @@ -25,7 +25,7 @@ use kube::{ watcher, }, }; -use log::info; +use log::{info, warn}; use serde_json::json; use std::{collections::BTreeMap, sync::Arc, time::Duration}; @@ -335,7 +335,7 @@ async fn secret_reconcile( .await .map(|_| Action::await_change()) .map_err(|e| { - eprintln!("Error updating attestation key volumes on secret apply: {e}"); + warn!("Error updating attestation key volumes on secret apply: {e}"); finalizer::Error::::ApplyFailed(e.into()) }) } @@ -349,9 +349,7 @@ async fn secret_reconcile( .await .map(|_| Action::await_change()) .map_err(|e| { - eprintln!( - "Error updating attestation key volumes during secret deletion: {e}" - ); + warn!("Error updating attestation key volumes during secret deletion: {e}"); finalizer::Error::::CleanupFailed(e.into()) }) } From 1f7f43c31123c8971ca8a9e5a61b3c1ee2075af4 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Wed, 29 Jul 2026 15:42:02 +0200 Subject: [PATCH 2/6] operator/ak-reg: Fix a log message Signed-off-by: Jakob Naucke --- operator/src/attestation_key_register.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operator/src/attestation_key_register.rs b/operator/src/attestation_key_register.rs index aea0d450..6f1e30d0 100644 --- a/operator/src/attestation_key_register.rs +++ b/operator/src/attestation_key_register.rs @@ -213,7 +213,7 @@ async fn machine_reconcile( // Check if the machine is being deleted if machine.metadata.deletion_timestamp.is_some() { info!( - "Machine {} is being deleted, updating attestation key volumes", + "Machine {} is being deleted, skipping update of attestation key volumes", machine.metadata.name.clone().unwrap_or_default() ); return Ok(Action::await_change()); From e547a7511d0c604c5a9dfce8b0177ea0d8e72f24 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Mon, 13 Jul 2026 17:15:53 +0200 Subject: [PATCH 3/6] Add timestamps to logging for crates and testing Fixes: #227 Signed-off-by: Jakob Naucke --- Cargo.toml | 2 +- test_utils/src/lib.rs | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index cae6e2ea..178fab6f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ chrono = "0.4.45" clap = { version = "4.6.1", features = ["derive"] } clevis-pin-trustee-lib = { git = "https://github.com/latchset/clevis-pin-trustee" } compute-pcrs-lib = { git = "https://github.com/trusted-execution-clusters/compute-pcrs" } -env_logger = { version = "0.11.10", default-features = false } +env_logger = { version = "0.11.10", default-features = false, features = ["humantime"] } http = "1.4.2" hex = "0.4.3" ignition-config = "0.6.1" diff --git a/test_utils/src/lib.rs b/test_utils/src/lib.rs index 9dc75538..d467259c 100644 --- a/test_utils/src/lib.rs +++ b/test_utils/src/lib.rs @@ -91,6 +91,11 @@ pub fn scaled_duration(secs: u64) -> Duration { Duration::from_secs(scaled_timeout(secs)) } +fn log_time() -> String { + let fmt = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ"); + fmt.to_string() +} + // Large warning frame, e.g. for paid cloud resources that may not have been shut down correctly pub fn warn_frame(msg: &str) -> String { format!("{YELLOW}=== WARNING ===\n{msg}{ANSI_RESET}") @@ -100,14 +105,14 @@ pub fn warn_frame(msg: &str) -> String { macro_rules! test_info { ($test_name:expr, $($arg:tt)*) => {{ const GREEN: &str = "\x1b[32m"; - println!("{}INFO{}: {}: {}", GREEN, ANSI_RESET, $test_name, format!($($arg)*)); + println!("{} {}INFO{}: {}: {}", log_time(), GREEN, ANSI_RESET, $test_name, format!($($arg)*)); }} } #[macro_export] macro_rules! test_warn { ($test_name:expr, $($arg:tt)*) => {{ - println!("{YELLOW}WARN{ANSI_RESET}: {}: {}", $test_name, format!($($arg)*)); + println!("{} {YELLOW}WARN{ANSI_RESET}: {}: {}", log_time(), $test_name, format!($($arg)*)); }} } From 7a18178d9754aec9e506ecbe24d634da5fe8350a Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Wed, 15 Jul 2026 17:13:11 +0200 Subject: [PATCH 4/6] gha: run & upload must-gather Signed-off-by: Jakob Naucke Assisted-by: AI --- .github/workflows/integration-tests.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 89130403..a373eebd 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -78,3 +78,14 @@ jobs: run: | eval $(ssh-agent -s) make integration-tests + - name: "Gather must-gather" + if: always() + run: must-gather/gather + env: + COLLECTION_PATH: must-gather-output + - name: "Upload must-gather" + if: always() + uses: actions/upload-artifact@v7 + with: + name: must-gather-${GITHUB_HEAD_REF} + path: must-gather-output/ From ef6924cfb586a29779429dfc243e2a537743c367 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Mon, 10 Aug 2026 10:54:59 +0200 Subject: [PATCH 5/6] rust: Bump to 1.92 as per UBI-supported version Signed-off-by: Jakob Naucke --- .github/workflows/lint.yml | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 92f0b965..0966f440 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -22,7 +22,7 @@ concurrency: env: CARGO_TERM_COLOR: always # Pinned toolchain for linting - ACTIONS_LINTS_TOOLCHAIN: 1.88.0 + ACTIONS_LINTS_TOOLCHAIN: 1.92.0 jobs: linting: diff --git a/Cargo.toml b/Cargo.toml index 178fab6f..2c8e2591 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ resolver = "3" [workspace.package] edition = "2024" -rust-version = "1.88" +rust-version = "1.92" [workspace.dependencies] anyhow = "1.0.102" From 67631240822d81cfd4ea1815f0a7efddc94f2887 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Fri, 26 Jun 2026 16:03:27 +0200 Subject: [PATCH 6/6] Remove await_change As per kube-rs docs, await_change is unadvisable for cases where eventual consistency is desired because reconcile events can be lost occasionally. Requeue after an hour instead. Signed-off-by: Jakob Naucke --- operator/src/attestation_key_register.rs | 20 +++++++++----------- operator/src/lib.rs | 4 ++++ operator/src/main.rs | 12 ++++++------ operator/src/reference_values.rs | 10 +++++----- operator/src/register_server.rs | 8 ++++---- 5 files changed, 28 insertions(+), 26 deletions(-) diff --git a/operator/src/attestation_key_register.rs b/operator/src/attestation_key_register.rs index 6f1e30d0..cd502ae4 100644 --- a/operator/src/attestation_key_register.rs +++ b/operator/src/attestation_key_register.rs @@ -35,10 +35,8 @@ use trusted_cluster_operator_lib::{AttestationKey, AttestationKeyStatus, Machine use crate::conditions::attestation_key_approved_condition; use crate::trustee; -use operator::{ - ControllerError, TLS_DIR, controller_error_policy, create_or_info_if_exists, read_certificate, - upsert_condition, -}; +use operator::{ControllerError, LONG_REQUEUE, TLS_DIR, controller_error_policy}; +use operator::{create_or_info_if_exists, read_certificate, upsert_condition}; /// Shared context for the three attestation-key controllers. /// Stores give local cache access to avoid repeated API-server reads. @@ -195,10 +193,10 @@ async fn ak_reconcile( for machine in ctx.machine_store.state() { if ak.spec.uuid.as_ref() == Some(&machine.spec.id) { approve_ak(&ak, &machine, &ctx).await?; - return Ok(Action::await_change()); + return Ok(LONG_REQUEUE); } } - Ok(Action::await_change()) + Ok(LONG_REQUEUE) } async fn machine_reconcile( @@ -216,7 +214,7 @@ async fn machine_reconcile( "Machine {} is being deleted, skipping update of attestation key volumes", machine.metadata.name.clone().unwrap_or_default() ); - return Ok(Action::await_change()); + return Ok(LONG_REQUEUE); } for ak in ctx.ak_store.state() { @@ -224,10 +222,10 @@ async fn machine_reconcile( && *ak_uuid == machine.spec.id { approve_ak(&ak, &machine, &ctx).await?; - return Ok(Action::await_change()); + return Ok(LONG_REQUEUE); } } - Ok(Action::await_change()) + Ok(LONG_REQUEUE) } async fn approve_ak(ak: &AttestationKey, machine: &Machine, ctx: &AkContextData) -> Result<()> { @@ -333,7 +331,7 @@ async fn secret_reconcile( // On creation/update, just update the trustee deployment volumes trustee::update_attestation_keys(&ctx) .await - .map(|_| Action::await_change()) + .map(|_| LONG_REQUEUE) .map_err(|e| { warn!("Error updating attestation key volumes on secret apply: {e}"); finalizer::Error::::ApplyFailed(e.into()) @@ -347,7 +345,7 @@ async fn secret_reconcile( // Update trustee deployment - secrets with deletion_timestamp will be filtered out trustee::update_attestation_keys(&ctx) .await - .map(|_| Action::await_change()) + .map(|_| LONG_REQUEUE) .map_err(|e| { warn!("Error updating attestation key volumes during secret deletion: {e}"); finalizer::Error::::CleanupFailed(e.into()) diff --git a/operator/src/lib.rs b/operator/src/lib.rs index 5cc6caa1..d77bc31e 100644 --- a/operator/src/lib.rs +++ b/operator/src/lib.rs @@ -59,6 +59,10 @@ macro_rules! create_or_info_if_exists { } pub const TLS_DIR: &str = "/etc/tls"; +/// As per kube-rs docs, it's possible to miss events and requeue_after = None should only be used +/// when it is known another requeue is imminent. Use this requeue duration for cases where no +/// further action is usually needed, but eventual consistency is desired. +pub const LONG_REQUEUE: Action = Action::requeue(Duration::from_hours(1)); /// Reads a TLS certificate secret and returns the Volume and VolumeMount for it. /// Returns None if the secret name is not provided or the secret does not exist. diff --git a/operator/src/main.rs b/operator/src/main.rs index e160e31d..18d59665 100644 --- a/operator/src/main.rs +++ b/operator/src/main.rs @@ -104,11 +104,11 @@ async fn reconcile( if changed { update_status!(clusters, name, TrustedExecutionClusterStatus { conditions })?; } - return Ok(Action::await_change()); + return Ok(LONG_REQUEUE); } if is_installed(cluster.status.clone()) { - return Ok(Action::await_change()); + return Ok(LONG_REQUEUE); } if ctx.tec_store.state().len() > 1 { @@ -150,7 +150,7 @@ async fn reconcile( let status = TrustedExecutionClusterStatus { conditions }; update_status!(clusters, name, status)?; } - Ok(Action::await_change()) + Ok(LONG_REQUEUE) } async fn install_components(client: &Client, cluster: &TrustedExecutionCluster) -> Result<()> { @@ -341,7 +341,7 @@ mod tests { let mut cluster = dummy_cluster(); cluster.metadata.deletion_timestamp = Some(Time(Timestamp::now())); let result = reconcile(Arc::new(cluster), Arc::new(dummy_cluster_ctx(client))).await; - assert_eq!(result.unwrap(), Action::await_change()); + assert_eq!(result.unwrap(), LONG_REQUEUE); }); } @@ -421,7 +421,7 @@ mod tests { conditions: Some(vec![foreign_condition]), }); let result = reconcile(Arc::new(cluster), Arc::new(dummy_cluster_ctx(client))).await; - assert_eq!(result.unwrap(), Action::await_change()); + assert_eq!(result.unwrap(), LONG_REQUEUE); }); } @@ -498,7 +498,7 @@ mod tests { }); count_check!(10, clos, |client| { let result = reconcile(Arc::new(cluster), Arc::new(dummy_cluster_ctx(client))).await; - assert_eq!(result.unwrap(), Action::await_change()); + assert_eq!(result.unwrap(), LONG_REQUEUE); }); } diff --git a/operator/src/reference_values.rs b/operator/src/reference_values.rs index 4077d9bb..946315c2 100644 --- a/operator/src/reference_values.rs +++ b/operator/src/reference_values.rs @@ -32,7 +32,7 @@ use std::{collections::BTreeMap, sync::Arc, time::Duration}; use crate::COMPONENT_VERSION; use crate::trustee::{self, get_image_pcrs}; -use operator::{ControllerError, upsert_condition}; +use operator::{ControllerError, LONG_REQUEUE, upsert_condition}; use operator::{controller_error_policy, controller_info, create_or_info_if_exists}; use trusted_cluster_operator_lib::{conditions::*, reference_values::*, *}; @@ -295,7 +295,7 @@ async fn image_add_reconcile( return Ok(Action::requeue(Duration::from_secs(5))); } let (action, reason) = match handle_new_image(client.clone(), image).await { - Ok(reason) => (Action::await_change(), reason), + Ok(reason) => (LONG_REQUEUE, reason), Err(e) => { warn!("PCR computation for {name} failed: {e}"); let action = Action::requeue(Duration::from_secs(60)); @@ -324,7 +324,7 @@ async fn image_remove_reconcile( let name = image.metadata.name.as_ref().unwrap_or(&default); if cluster.is_none() { info!("No TrustedExecutionCluster found, skipping disallow_image for {name}"); - return Ok(Action::await_change()); + return Ok(LONG_REQUEUE); } let cluster = cluster.unwrap(); let tec_name = cluster.metadata.name.unwrap_or("".to_string()); @@ -333,10 +333,10 @@ async fn image_remove_reconcile( "TrustedExecutionCluster {tec_name} is being deleted, \ skipping disallow_image for {name}" ); - return Ok(Action::await_change()); + return Ok(LONG_REQUEUE); } disallow_image(client, name).await?; - Ok(Action::await_change()) + Ok(LONG_REQUEUE) } pub async fn launch_rv_image_controller(client: Client) { diff --git a/operator/src/register_server.rs b/operator/src/register_server.rs index 2a57ded2..8b733ef6 100644 --- a/operator/src/register_server.rs +++ b/operator/src/register_server.rs @@ -142,7 +142,7 @@ async fn keygen_reconcile( trustee::mount_secret(kube_client, id).await } .await - .map(|_| Action::await_change()) + .map(|_| LONG_REQUEUE) .map_err(|e| finalizer::Error::::ApplyFailed(e.into())) } Event::Cleanup(machine) => { @@ -168,7 +168,7 @@ async fn keygen_reconcile( skipping unmount_secret for Machine {}", machine.metadata.name.as_deref().unwrap_or("unknown") ); - return Ok(Action::await_change()); + return Ok(LONG_REQUEUE); } Err(kube::Error::Api(ae)) if ae.code == 404 => { // TEC already deleted, skip unmount_secret @@ -177,7 +177,7 @@ async fn keygen_reconcile( skipping unmount_secret for Machine {}", machine.metadata.name.as_deref().unwrap_or("unknown") ); - return Ok(Action::await_change()); + return Ok(LONG_REQUEUE); } _ => { // TEC exists and is not being deleted, proceed with unmount_secret @@ -187,7 +187,7 @@ async fn keygen_reconcile( trustee::unmount_secret(kube_client, id) .await - .map(|_| Action::await_change()) + .map(|_| LONG_REQUEUE) .map_err(|e| finalizer::Error::::CleanupFailed(e.into())) } }