Requeue instead of awaiting change

.github/workflows/integration-tests.yml

@@ -78,3 +78,14 @@ jobs:
         run: |
           eval $(ssh-agent -s)
           make integration-tests
+      - name: "Gather must-gather"
+        if: always()
+        run: must-gather/gather
+        env:
+          COLLECTION_PATH: must-gather-output
+      - name: "Upload must-gather"
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: must-gather-${GITHUB_HEAD_REF}
+          path: must-gather-output/

.github/workflows/lint.yml

@@ -22,7 +22,7 @@ concurrency:
 env:
   CARGO_TERM_COLOR: always
   # Pinned toolchain for linting
-  ACTIONS_LINTS_TOOLCHAIN: 1.88.0
+  ACTIONS_LINTS_TOOLCHAIN: 1.92.0
 
 jobs:
   linting:

Cargo.toml

@@ -9,7 +9,7 @@ resolver = "3"
 
 [workspace.package]
 edition = "2024"
-rust-version = "1.88"
+rust-version = "1.92"
 
 [workspace.dependencies]
 anyhow = "1.0.102"
@@ -19,7 +19,7 @@ chrono = "0.4.45"
 clap = { version = "4.6.1", features = ["derive"] }
 clevis-pin-trustee-lib = { git = "https://github.com/latchset/clevis-pin-trustee" }
 compute-pcrs-lib = { git = "https://github.com/trusted-execution-clusters/compute-pcrs" }
-env_logger = { version = "0.11.10", default-features = false }
+env_logger = { version = "0.11.10", default-features = false, features = ["humantime"] }
 http = "1.4.2"
 hex = "0.4.3"
 ignition-config = "0.6.1"

operator/src/attestation_key_register.rs

@@ -25,7 +25,7 @@ use kube::{
         watcher,
     },
 };
-use log::info;
+use log::{info, warn};
 use serde_json::json;
 use std::{collections::BTreeMap, sync::Arc, time::Duration};
 
@@ -35,10 +35,8 @@ use trusted_cluster_operator_lib::{AttestationKey, AttestationKeyStatus, Machine
 
 use crate::conditions::attestation_key_approved_condition;
 use crate::trustee;
-use operator::{
-    ControllerError, TLS_DIR, controller_error_policy, create_or_info_if_exists, read_certificate,
-    upsert_condition,
-};
+use operator::{ControllerError, LONG_REQUEUE, TLS_DIR, controller_error_policy};
+use operator::{create_or_info_if_exists, read_certificate, upsert_condition};
 
 /// Shared context for the three attestation-key controllers.
 /// Stores give local cache access to avoid repeated API-server reads.
@@ -195,10 +193,10 @@ async fn ak_reconcile(
     for machine in ctx.machine_store.state() {
         if ak.spec.uuid.as_ref() == Some(&machine.spec.id) {
             approve_ak(&ak, &machine, &ctx).await?;
-            return Ok(Action::await_change());
+            return Ok(LONG_REQUEUE);
         }
     }
-    Ok(Action::await_change())
+    Ok(LONG_REQUEUE)
 }
 
 async fn machine_reconcile(
@@ -213,21 +211,21 @@ async fn machine_reconcile(
     // Check if the machine is being deleted
     if machine.metadata.deletion_timestamp.is_some() {
         info!(
-            "Machine {} is being deleted, updating attestation key volumes",
+            "Machine {} is being deleted, skipping update of attestation key volumes",
             machine.metadata.name.clone().unwrap_or_default()
         );
-        return Ok(Action::await_change());
+        return Ok(LONG_REQUEUE);
     }
 
     for ak in ctx.ak_store.state() {
         if let Some(ak_uuid) = &ak.spec.uuid
             && *ak_uuid == machine.spec.id
         {
             approve_ak(&ak, &machine, &ctx).await?;
-            return Ok(Action::await_change());
+            return Ok(LONG_REQUEUE);
         }
     }
-    Ok(Action::await_change())
+    Ok(LONG_REQUEUE)
 }
 
 async fn approve_ak(ak: &AttestationKey, machine: &Machine, ctx: &AkContextData) -> Result<()> {
@@ -333,9 +331,9 @@ async fn secret_reconcile(
                 // On creation/update, just update the trustee deployment volumes
                 trustee::update_attestation_keys(&ctx)
                     .await
-                    .map(|_| Action::await_change())
+                    .map(|_| LONG_REQUEUE)
                     .map_err(|e| {
-                        eprintln!("Error updating attestation key volumes on secret apply: {e}");
+                        warn!("Error updating attestation key volumes on secret apply: {e}");
                         finalizer::Error::<ControllerError>::ApplyFailed(e.into())
                     })
             }
@@ -347,11 +345,9 @@ async fn secret_reconcile(
                 // Update trustee deployment - secrets with deletion_timestamp will be filtered out
                 trustee::update_attestation_keys(&ctx)
                     .await
-                    .map(|_| Action::await_change())
+                    .map(|_| LONG_REQUEUE)
                     .map_err(|e| {
-                        eprintln!(
-                            "Error updating attestation key volumes during secret deletion: {e}"
-                        );
+                        warn!("Error updating attestation key volumes during secret deletion: {e}");
                         finalizer::Error::<ControllerError>::CleanupFailed(e.into())
                     })
             }

operator/src/lib.rs

@@ -59,6 +59,10 @@ macro_rules! create_or_info_if_exists {
 }
 
 pub const TLS_DIR: &str = "/etc/tls";
+/// As per kube-rs docs, it's possible to miss events and requeue_after = None should only be used
+/// when it is known another requeue is imminent. Use this requeue duration for cases where no
+/// further action is usually needed, but eventual consistency is desired.
+pub const LONG_REQUEUE: Action = Action::requeue(Duration::from_hours(1));
 
 /// Reads a TLS certificate secret and returns the Volume and VolumeMount for it.
 /// Returns None if the secret name is not provided or the secret does not exist.

operator/src/main.rs

@@ -104,11 +104,11 @@ async fn reconcile(
         if changed {
             update_status!(clusters, name, TrustedExecutionClusterStatus { conditions })?;
         }
-        return Ok(Action::await_change());
+        return Ok(LONG_REQUEUE);
     }
 
     if is_installed(cluster.status.clone()) {
-        return Ok(Action::await_change());
+        return Ok(LONG_REQUEUE);
     }
 
     if ctx.tec_store.state().len() > 1 {
@@ -150,7 +150,7 @@ async fn reconcile(
         let status = TrustedExecutionClusterStatus { conditions };
         update_status!(clusters, name, status)?;
     }
-    Ok(Action::await_change())
+    Ok(LONG_REQUEUE)
 }
 
 async fn install_components(client: &Client, cluster: &TrustedExecutionCluster) -> Result<()> {
@@ -341,7 +341,7 @@ mod tests {
             let mut cluster = dummy_cluster();
             cluster.metadata.deletion_timestamp = Some(Time(Timestamp::now()));
             let result = reconcile(Arc::new(cluster), Arc::new(dummy_cluster_ctx(client))).await;
-            assert_eq!(result.unwrap(), Action::await_change());
+            assert_eq!(result.unwrap(), LONG_REQUEUE);
         });
     }
 
@@ -421,7 +421,7 @@ mod tests {
                 conditions: Some(vec![foreign_condition]),
             });
             let result = reconcile(Arc::new(cluster), Arc::new(dummy_cluster_ctx(client))).await;
-            assert_eq!(result.unwrap(), Action::await_change());
+            assert_eq!(result.unwrap(), LONG_REQUEUE);
         });
     }
 
@@ -498,7 +498,7 @@ mod tests {
         });
         count_check!(10, clos, |client| {
             let result = reconcile(Arc::new(cluster), Arc::new(dummy_cluster_ctx(client))).await;
-            assert_eq!(result.unwrap(), Action::await_change());
+            assert_eq!(result.unwrap(), LONG_REQUEUE);
         });
     }
 

operator/src/reference_values.rs

@@ -32,7 +32,7 @@ use std::{collections::BTreeMap, sync::Arc, time::Duration};
 
 use crate::COMPONENT_VERSION;
 use crate::trustee::{self, get_image_pcrs};
-use operator::{ControllerError, upsert_condition};
+use operator::{ControllerError, LONG_REQUEUE, upsert_condition};
 use operator::{controller_error_policy, controller_info, create_or_info_if_exists};
 use trusted_cluster_operator_lib::{conditions::*, reference_values::*, *};
 
@@ -295,7 +295,7 @@ async fn image_add_reconcile(
         return Ok(Action::requeue(Duration::from_secs(5)));
     }
     let (action, reason) = match handle_new_image(client.clone(), image).await {
-        Ok(reason) => (Action::await_change(), reason),
+        Ok(reason) => (LONG_REQUEUE, reason),
         Err(e) => {
             warn!("PCR computation for {name} failed: {e}");
             let action = Action::requeue(Duration::from_secs(60));
@@ -324,7 +324,7 @@ async fn image_remove_reconcile(
     let name = image.metadata.name.as_ref().unwrap_or(&default);
     if cluster.is_none() {
         info!("No TrustedExecutionCluster found, skipping disallow_image for {name}");
-        return Ok(Action::await_change());
+        return Ok(LONG_REQUEUE);
     }
     let cluster = cluster.unwrap();
     let tec_name = cluster.metadata.name.unwrap_or("<no name>".to_string());
@@ -333,10 +333,10 @@ async fn image_remove_reconcile(
             "TrustedExecutionCluster {tec_name} is being deleted, \
              skipping disallow_image for {name}"
         );
-        return Ok(Action::await_change());
+        return Ok(LONG_REQUEUE);
     }
     disallow_image(client, name).await?;
-    Ok(Action::await_change())
+    Ok(LONG_REQUEUE)
 }
 
 pub async fn launch_rv_image_controller(client: Client) {

operator/src/register_server.rs

@@ -142,7 +142,7 @@ async fn keygen_reconcile(
                     trustee::mount_secret(kube_client, id).await
                 }
                 .await
-                .map(|_| Action::await_change())
+                .map(|_| LONG_REQUEUE)
                 .map_err(|e| finalizer::Error::<ControllerError>::ApplyFailed(e.into()))
             }
             Event::Cleanup(machine) => {
@@ -168,7 +168,7 @@ async fn keygen_reconcile(
                                      skipping unmount_secret for Machine {}",
                                 machine.metadata.name.as_deref().unwrap_or("unknown")
                             );
-                            return Ok(Action::await_change());
+                            return Ok(LONG_REQUEUE);
                         }
                         Err(kube::Error::Api(ae)) if ae.code == 404 => {
                             // TEC already deleted, skip unmount_secret
@@ -177,7 +177,7 @@ async fn keygen_reconcile(
                                      skipping unmount_secret for Machine {}",
                                 machine.metadata.name.as_deref().unwrap_or("unknown")
                             );
-                            return Ok(Action::await_change());
+                            return Ok(LONG_REQUEUE);
                         }
                         _ => {
                             // TEC exists and is not being deleted, proceed with unmount_secret
@@ -187,7 +187,7 @@ async fn keygen_reconcile(
 
                 trustee::unmount_secret(kube_client, id)
                     .await
-                    .map(|_| Action::await_change())
+                    .map(|_| LONG_REQUEUE)
                     .map_err(|e| finalizer::Error::<ControllerError>::CleanupFailed(e.into()))
             }
         }

test_utils/src/lib.rs

@@ -91,6 +91,11 @@ pub fn scaled_duration(secs: u64) -> Duration {
     Duration::from_secs(scaled_timeout(secs))
 }
 
+fn log_time() -> String {
+    let fmt = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ");
+    fmt.to_string()
+}
+
 // Large warning frame, e.g. for paid cloud resources that may not have been shut down correctly
 pub fn warn_frame(msg: &str) -> String {
     format!("{YELLOW}=== WARNING ===\n{msg}{ANSI_RESET}")
@@ -100,14 +105,14 @@ pub fn warn_frame(msg: &str) -> String {
 macro_rules! test_info {
     ($test_name:expr, $($arg:tt)*) => {{
         const GREEN: &str = "\x1b[32m";
-        println!("{}INFO{}: {}: {}", GREEN, ANSI_RESET, $test_name, format!($($arg)*));
+        println!("{} {}INFO{}: {}: {}", log_time(), GREEN, ANSI_RESET, $test_name, format!($($arg)*));
     }}
 }
 
 #[macro_export]
 macro_rules! test_warn {
     ($test_name:expr, $($arg:tt)*) => {{
-        println!("{YELLOW}WARN{ANSI_RESET}: {}: {}", $test_name, format!($($arg)*));
+        println!("{} {YELLOW}WARN{ANSI_RESET}: {}: {}", log_time(), $test_name, format!($($arg)*));
     }}
 }