diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 727a3d6..ed231e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,7 +62,7 @@ jobs: - uses: taiki-e/install-action@v2.75.27 with: tool: cargo-nextest - - run: cargo nextest run --locked --workspace + - run: cargo nextest run --locked --workspace --all-features build: runs-on: ubuntu-latest @@ -70,7 +70,7 @@ jobs: - uses: actions/checkout@v6.0.2 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2.9.1 - - run: cargo build --locked --workspace + - run: cargo build --locked --workspace --all-features bench: runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index f4ae014..8f996f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -571,6 +571,7 @@ dependencies = [ "contextforge-gateway-rs-cpex", "contextforge-gateway-rs-lib", "cpex-payload-marker", + "cpex-secrets-detection", "cpex-text-prefixer", "cpex-tool-namespace", "http", @@ -585,7 +586,6 @@ dependencies = [ "rmcp", "rmp-serde", "rustls", - "secrets_detection_rust", "serde_json", "tikv-jemallocator", "tokio", @@ -637,6 +637,7 @@ dependencies = [ "contextforge-gateway-rs-apis", "contextforge-gateway-rs-cpex", "cpex", + "cpex-secrets-detection", "futures", "http", "hyper", @@ -653,7 +654,6 @@ dependencies = [ "rustls", "rustls-pki-types", "secret-string", - "secrets_detection_rust", "serde", "serde_json", "test-log", @@ -789,12 +789,14 @@ dependencies = [ ] [[package]] -name = "cpex-sdk" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba20e81ab4924231882adc31a437000fa045f890b80d7cce4fc1396848e710f0" +name = "cpex-secrets-detection" +version = "0.1.0" dependencies = [ - "cpex-core", + "cpex", + "regex", + "serde", + "serde_json", + "tokio", ] [[package]] @@ -3119,18 +3121,6 @@ version = "0.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "068d960589a13890bdb2a9ea5e7ae0974b89249101117f2e9a5e166bf3af44b9" -[[package]] -name = "secrets_detection_rust" -version = "0.0.0" -source = "git+https://github.com/IBM/cpex-plugins?rev=6ff7af74587574fe6115ce87427519b63f6062da#6ff7af74587574fe6115ce87427519b63f6062da" -dependencies = [ - "cpex-core", - "cpex-sdk", - "regex", - "serde", - "serde_json", -] - [[package]] name = "security-framework" version = "3.7.0" diff --git a/Cargo.toml b/Cargo.toml index ea9a3f8..47091cc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "crates/contextforge-gateway-rs-cpex", "crates/contextforge-gateway-rs-lib", "crates/contextforge-load-test", + "crates/plugins/cpex-secrets-detection", ] resolver = "3" @@ -56,7 +57,7 @@ cpex = "=0.2.2" uuid = { version = "1.23.1", features = ["v4"] } axum = "0.8" openport = { version = "0.4.0", features = ["rand"] } -secrets_detection_rust = { git = "https://github.com/IBM/cpex-plugins", rev = "6ff7af74587574fe6115ce87427519b63f6062da", package = "secrets_detection_rust" } +cpex-secrets-detection = { path = "./crates/plugins/cpex-secrets-detection" } [profile.release] codegen-units = 1 diff --git a/README.md b/README.md index 67a4ef2..b7acc6f 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,8 @@ curl --request POST \ Runtime CPEX plugins are disabled by default. Enable hook execution when starting the gateway: ```bash -cargo run --release --bin contextforge-gateway-rs -- \ +cargo run --release \ + --bin contextforge-gateway-rs -- \ --address 0.0.0.0:8001 \ --redis-port 6379 \ --redis-address 127.0.0.1 \ @@ -92,6 +93,38 @@ Plugin configuration is stored in Redis at key `ContextForgeGatewayRuntimePlugin This integration currently passes only tool payloads. CPEX configs that enable route-based plugin selection, plugin directories, global policies/defaults, non-tool hooks, or plugin conditions are rejected in this PR. Redis write access to this key is a control-plane trust boundary because it controls which registered hooks run. +### Experimental Secrets Detection Plugin + +The bundled secrets detection CPEX plugin is experimental. It is compiled into +the gateway with `contextforge-gateway-rs/plugins`; Redis config only activates +plugin factories that are already present in the binary. + +Activation requires all three pieces: + +- Compile-time feature: `contextforge-gateway-rs/plugins` +- Runtime flag: `--runtime-plugins-enabled true` +- Redis config key: `ContextForgeGatewayRuntimePluginConfig` + +The plugin kind is `validator/secrets-detection`. The dataplane currently wires +only `cmf.tool_pre_invoke` and `cmf.tool_post_invoke`. + +Example run command: + +```bash +cargo run --release \ + --features contextforge-gateway-rs/plugins \ + --bin contextforge-gateway-rs -- \ + --address 0.0.0.0:8001 \ + --redis-port 6379 \ + --redis-address 127.0.0.1 \ + --token-verification-public-key assets/jwt.key.pub \ + --token-verification-private-key assets/jwt.key \ + --number-of-cpus 16 \ + --redis-mode=plain-text \ + --upstream-connection-mode=plain-text-or-tls \ + --runtime-plugins-enabled true +``` + ### Payload Marker Demo This demo uses the `test-plugins` feature, which includes the demo plugin crates from `cpex-plugins-rs`. The plugin must be included in the gateway build before the gateway starts. Redis runtime registration activates already-registered factories; it does not load new Rust code into a running process. diff --git a/crates/contextforge-gateway-rs-lib/Cargo.toml b/crates/contextforge-gateway-rs-lib/Cargo.toml index 09ce27e..bf09b2d 100644 --- a/crates/contextforge-gateway-rs-lib/Cargo.toml +++ b/crates/contextforge-gateway-rs-lib/Cargo.toml @@ -53,7 +53,7 @@ with_tools = [] opentelemetry_sdk.workspace = true cpex.workspace = true openport.workspace = true -secrets_detection_rust.workspace = true +cpex-secrets-detection.workspace = true test-log = "0.2.20" axum-server = { version = "0.8.0", features = ["tls-rustls"] } diff --git a/crates/contextforge-gateway-rs-lib/tests/gateway_plugins.rs b/crates/contextforge-gateway-rs-lib/tests/gateway_plugins.rs index 8273937..18c4b20 100644 --- a/crates/contextforge-gateway-rs-lib/tests/gateway_plugins.rs +++ b/crates/contextforge-gateway-rs-lib/tests/gateway_plugins.rs @@ -147,12 +147,12 @@ fn reflect_text_request(text: String) -> CallToolRequestParams { async fn runtime_with_secrets_detection(hooks: Vec<&'static str>, plugin_config: Value) -> Arc { let mut runtime = CpexRuntimeRegistry::default(); runtime - .register_factory(secrets_detection_rust::KIND, Box::new(secrets_detection_rust::SecretsDetectionFactory)) + .register_factory(cpex_secrets_detection::KIND, Box::new(cpex_secrets_detection::SecretsDetectionFactory)) .expect("secrets detection factory registers"); let config: CpexConfig = serde_json::from_value(json!({ "plugins": [{ "name": "secrets-detection", - "kind": secrets_detection_rust::KIND, + "kind": cpex_secrets_detection::KIND, "hooks": hooks, "config": plugin_config, }] diff --git a/crates/contextforge-gateway-rs/Cargo.toml b/crates/contextforge-gateway-rs/Cargo.toml index 07918a4..f3b140f 100644 --- a/crates/contextforge-gateway-rs/Cargo.toml +++ b/crates/contextforge-gateway-rs/Cargo.toml @@ -13,7 +13,7 @@ contextforge-gateway-rs-lib = { path = "../contextforge-gateway-rs-lib" } cpex-payload-marker = { git = "https://github.com/contextforge-gateway-rs/cpex-plugins-rs", rev = "ab47801daccfbba44ea07b033034a347e7b5afdd", optional = true } cpex-text-prefixer = { git = "https://github.com/contextforge-gateway-rs/cpex-plugins-rs", rev = "ab47801daccfbba44ea07b033034a347e7b5afdd", optional = true } cpex-tool-namespace = { git = "https://github.com/contextforge-gateway-rs/cpex-plugins-rs", rev = "ab47801daccfbba44ea07b033034a347e7b5afdd", optional = true } -secrets_detection_rust = { workspace = true, optional = true } +cpex-secrets-detection = { workspace = true, optional = true } clap.workspace = true tracing.workspace = true tracing-appender = "0.2.3" @@ -30,7 +30,7 @@ tikv-jemallocator = "0.7.0" rustls.workspace = true [features] -secrets-detection-plugin = ["dep:secrets_detection_rust"] +plugins = ["dep:cpex-secrets-detection"] test-plugins = ["dep:cpex-payload-marker", "dep:cpex-text-prefixer", "dep:cpex-tool-namespace"] [dev-dependencies] diff --git a/crates/contextforge-gateway-rs/src/main.rs b/crates/contextforge-gateway-rs/src/main.rs index 30a5734..bfa7c49 100644 --- a/crates/contextforge-gateway-rs/src/main.rs +++ b/crates/contextforge-gateway-rs/src/main.rs @@ -58,12 +58,12 @@ fn plugin_runtime_from_config( ) -> Result> { let redis_client = RedisClient::try_from(RedisConfig::try_from(config)?)?; let plugin_runtime = CpexRuntimeRegistry::with_redis_config(redis_client); - #[cfg(any(feature = "test-plugins", feature = "secrets-detection-plugin"))] + #[cfg(any(feature = "test-plugins", feature = "plugins"))] let plugin_runtime = register_builtin_factories(plugin_runtime)?; Ok(plugin_runtime) } -#[cfg(any(feature = "test-plugins", feature = "secrets-detection-plugin"))] +#[cfg(any(feature = "test-plugins", feature = "plugins"))] fn register_builtin_factories( mut plugin_runtime: CpexRuntimeRegistry, ) -> Result> { @@ -71,11 +71,11 @@ fn register_builtin_factories( { test_plugins::register(&mut plugin_runtime)?; } - #[cfg(feature = "secrets-detection-plugin")] + #[cfg(feature = "plugins")] { plugin_runtime.register_factory( - secrets_detection_rust::KIND, - Box::new(secrets_detection_rust::SecretsDetectionFactory), + cpex_secrets_detection::KIND, + Box::new(cpex_secrets_detection::SecretsDetectionFactory), )?; } Ok(plugin_runtime) diff --git a/crates/contextforge-gateway-rs/tests/secrets_detection_e2e.rs b/crates/contextforge-gateway-rs/tests/secrets_detection_e2e.rs index a9b35ef..bf8203e 100644 --- a/crates/contextforge-gateway-rs/tests/secrets_detection_e2e.rs +++ b/crates/contextforge-gateway-rs/tests/secrets_detection_e2e.rs @@ -1,7 +1,7 @@ // Copyright 2026 // SPDX-License-Identifier: Apache-2.0 -#![cfg(feature = "secrets-detection-plugin")] +#![cfg(feature = "plugins")] use std::{ collections::HashMap, diff --git a/crates/plugins/cpex-secrets-detection/Cargo.toml b/crates/plugins/cpex-secrets-detection/Cargo.toml new file mode 100644 index 0000000..002ffca --- /dev/null +++ b/crates/plugins/cpex-secrets-detection/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "cpex-secrets-detection" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish = false + +[lib] +doctest = false + +[dependencies] +cpex.workspace = true +regex = "1.12.3" +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true + +[dev-dependencies] +tokio.workspace = true + +[lints] +workspace = true diff --git a/crates/plugins/cpex-secrets-detection/README.md b/crates/plugins/cpex-secrets-detection/README.md new file mode 100644 index 0000000..a36bcf9 --- /dev/null +++ b/crates/plugins/cpex-secrets-detection/README.md @@ -0,0 +1,79 @@ +# ContextForge Gateway Secrets Detection + +Rust CPEX secrets detection plugin for the ContextForge dataplane. + +This crate is a first-class member of the `contextforge-data-plane` workspace. +It provides the `SecretsDetectionFactory` registered by the gateway binary when +the `plugins` Cargo feature is enabled. + +## Runtime Activation + +The crate is compiled into the gateway binary. Runtime configuration still comes +from the dataplane plugin config document stored in Redis. + +Example config: + +```json +{ + "version": 1, + "cpex": { + "plugins": [ + { + "name": "secrets-detection", + "kind": "validator/secrets-detection", + "hooks": ["cmf.tool_pre_invoke", "cmf.tool_post_invoke"], + "config": { + "redact": true, + "redaction_text": "[redacted]", + "block_on_detection": false + } + } + ] + } +} +``` + +The dataplane integration currently wires the tool-call path: + +- `cmf.tool_pre_invoke`: scans tool arguments before the backend receives them. +- `cmf.tool_post_invoke`: scans tool results before the client receives them. + +The crate also keeps prompt/resource stage handling for CPEX parity and future +hosts, but the current dataplane runtime config only uses the tool pre/post +hooks. + +## Behavior + +The scanner detects common secret-shaped values in JSON payloads and direct text +content. Depending on config, it can: + +- redact detected values +- deny payloads when `block_on_detection` is enabled and the threshold is met +- apply dotted field allowlists/denylists to JSON arguments and results +- emit non-sensitive metadata from direct handlers when trace context exists + +The CPEX plugin kind is: + +```text +validator/secrets-detection +``` + +## Known CPEX 0.2.2 Gaps + +- `PluginResult.metadata` is not propagated through `PluginManager`. +- A denied result cannot surface a redacted payload through `PluginManager`. + +The direct handler can return both metadata and a redacted payload on block, but +those fields are lost at the manager/executor boundary in CPEX 0.2.2. + +## Verification + +From the workspace root: + +```bash +cargo +1.96 test -p cpex-secrets-detection +cargo +1.96 check -p contextforge-gateway-rs --features plugins +cargo +1.96 test -p contextforge-gateway-rs-cpex +cargo +1.96 test -p contextforge-gateway-rs-lib --test gateway_plugins -- --nocapture +cargo +1.96 test -p contextforge-gateway-rs --features plugins --test secrets_detection_e2e -- --ignored --nocapture +``` diff --git a/crates/plugins/cpex-secrets-detection/src/config.rs b/crates/plugins/cpex-secrets-detection/src/config.rs new file mode 100644 index 0000000..f78d378 --- /dev/null +++ b/crates/plugins/cpex-secrets-detection/src/config.rs @@ -0,0 +1,349 @@ +// Copyright 2026 +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::HashMap; + +use serde::Deserialize; +use serde_json::Value; + +use crate::patterns::PATTERNS; + +const BROAD_PATTERNS: [&str; 4] = ["generic_api_key_assignment", "jwt_like", "hex_secret_32", "base64_24"]; + +#[derive(Debug, Clone)] +pub struct SecretsDetectionConfig { + pub enabled: HashMap, + pub redact: bool, + pub redaction_text: String, + pub block_on_detection: bool, + pub min_findings_to_block: usize, + pub field_allowlist: Vec, + pub field_denylist: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FieldPath { + segments: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ConfigError { + message: String, +} + +impl ConfigError { + fn new(message: impl Into) -> Self { + Self { message: message.into() } + } +} + +impl std::fmt::Display for ConfigError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.message) + } +} + +impl std::error::Error for ConfigError {} + +#[derive(Debug, Default, Deserialize)] +#[serde(default)] +struct RawSecretsDetectionConfig { + enabled: Option>, + redact: Option, + redaction_text: Option, + block_on_detection: Option, + min_findings_to_block: Option, + field_allowlist: Option, + field_denylist: Option, +} + +impl FieldPath { + pub fn parse(path: &str, field_name: &str) -> Result { + if path.trim().is_empty() { + return Err(ConfigError::new(format!("{field_name} entries must not be empty or whitespace-only"))); + } + if path.starts_with('.') || path.ends_with('.') { + return Err(ConfigError::new(format!("{field_name} path {path:?} must not start or end with '.'"))); + } + + let segments: Vec = path.split('.').map(str::to_owned).collect(); + if segments.iter().any(|segment| segment.trim().is_empty()) { + return Err(ConfigError::new(format!( + "{field_name} path {path:?} must not contain empty or whitespace-only segments" + ))); + } + + Ok(Self { segments }) + } + + pub fn segments(&self) -> &[String] { + &self.segments + } + + fn matches_path_or_descendant(&self, path: &[String]) -> bool { + path_starts_with(path, &self.segments) + } + + fn can_be_reached_from(&self, path: &[String]) -> bool { + path_starts_with(&self.segments, path) + } +} + +impl SecretsDetectionConfig { + pub fn from_value(value: Option<&Value>) -> Result { + let raw = match value { + Some(Value::Null) | None => RawSecretsDetectionConfig::default(), + Some(value) => serde_json::from_value::(value.clone()) + .map_err(|err| ConfigError::new(format!("invalid secrets_detection config: {err}")))?, + }; + + Ok(Self { + enabled: raw.enabled.map_or_else(default_enabled_map, merge_enabled_map), + redact: raw.redact.unwrap_or(false), + redaction_text: raw.redaction_text.unwrap_or_else(|| "***REDACTED***".to_owned()), + block_on_detection: raw.block_on_detection.unwrap_or(true), + min_findings_to_block: raw.min_findings_to_block.unwrap_or(1), + field_allowlist: parse_field_paths(raw.field_allowlist.as_ref(), "field_allowlist")?, + field_denylist: parse_field_paths(raw.field_denylist.as_ref(), "field_denylist")?, + }) + } + + pub fn is_enabled(&self, name: &str) -> bool { + self.enabled.get(name).copied().unwrap_or(false) + } + + pub fn should_scan_field_path(&self, path: &[String], direct_scalar_root: bool) -> bool { + if direct_scalar_root && path.is_empty() { + return true; + } + if self.path_is_denied(path) { + return false; + } + self.field_allowlist.is_empty() + || self.field_allowlist.iter().any(|allow_path| allow_path.matches_path_or_descendant(path)) + } + + pub fn should_traverse_field_path(&self, path: &[String]) -> bool { + if path.is_empty() { + return true; + } + if self.path_is_denied(path) { + return false; + } + self.field_allowlist.is_empty() + || self + .field_allowlist + .iter() + .any(|allow_path| allow_path.matches_path_or_descendant(path) || allow_path.can_be_reached_from(path)) + } + + fn path_is_denied(&self, path: &[String]) -> bool { + self.field_denylist.iter().any(|deny_path| deny_path.matches_path_or_descendant(path)) + } +} + +impl Default for SecretsDetectionConfig { + fn default() -> Self { + Self { + enabled: default_enabled_map(), + redact: false, + redaction_text: "***REDACTED***".to_owned(), + block_on_detection: true, + min_findings_to_block: 1, + field_allowlist: Vec::new(), + field_denylist: Vec::new(), + } + } +} + +fn parse_field_paths(value: Option<&Value>, field_name: &str) -> Result, ConfigError> { + let Some(value) = value else { + return Ok(Vec::new()); + }; + let entries = value + .as_array() + .ok_or_else(|| ConfigError::new(format!("{field_name} must be a list of dotted field path strings")))?; + entries + .iter() + .map(|entry| { + let path = entry + .as_str() + .ok_or_else(|| ConfigError::new(format!("{field_name} must be a list of dotted field path strings")))?; + FieldPath::parse(path, field_name) + }) + .collect() +} + +fn path_starts_with(path: &[String], prefix: &[String]) -> bool { + path.len() >= prefix.len() && path.iter().zip(prefix.iter()).all(|(left, right)| left == right) +} + +fn default_enabled_map() -> HashMap { + PATTERNS.keys().map(|&name| (name.to_owned(), !BROAD_PATTERNS.contains(&name))).collect() +} + +fn merge_enabled_map(overrides: HashMap) -> HashMap { + let mut enabled = default_enabled_map(); + enabled.extend(overrides); + enabled +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn broad_patterns_default_to_disabled() { + let config = SecretsDetectionConfig::default(); + assert!(!config.is_enabled("generic_api_key_assignment")); + assert!(!config.is_enabled("jwt_like")); + assert!(config.is_enabled("aws_access_key_id")); + assert!(config.field_allowlist.is_empty()); + assert!(config.field_denylist.is_empty()); + } + + #[test] + fn from_value_merges_overrides_with_defaults() { + let value = json!({ + "enabled": { + "jwt_like": true, + "aws_access_key_id": false + }, + "redact": true, + "redaction_text": "[SECRET]", + "block_on_detection": false, + "min_findings_to_block": 3, + "field_allowlist": ["layer1", "accounts.credentials"], + "field_denylist": ["layer1.layer2.layer3", "accounts.credentials.test_token"] + }); + + let config = SecretsDetectionConfig::from_value(Some(&value)).unwrap(); + + assert!(config.is_enabled("jwt_like")); + assert!(!config.is_enabled("aws_access_key_id")); + assert!(config.is_enabled("github_token")); + assert!(config.redact); + assert_eq!(config.redaction_text, "[SECRET]"); + assert!(!config.block_on_detection); + assert_eq!(config.min_findings_to_block, 3); + assert_eq!(segments(&config.field_allowlist[0]), vec!["layer1"]); + assert_eq!(segments(&config.field_allowlist[1]), vec!["accounts", "credentials"]); + assert_eq!(segments(&config.field_denylist[0]), vec!["layer1", "layer2", "layer3"]); + assert_eq!(segments(&config.field_denylist[1]), vec!["accounts", "credentials", "test_token"]); + } + + #[test] + fn from_value_rejects_invalid_field_paths() { + for (field_name, path, expected) in [ + ("field_allowlist", "", "must not be empty or whitespace-only"), + ("field_allowlist", " ", "must not be empty or whitespace-only"), + ("field_denylist", ".token", "must not start or end with '.'"), + ("field_denylist", "token.", "must not start or end with '.'"), + ("field_allowlist", "layer1..layer3", "must not contain empty or whitespace-only segments"), + ("field_denylist", "layer1. .layer3", "must not contain empty or whitespace-only segments"), + ] { + let value = json!({ field_name: [path] }); + + let err = SecretsDetectionConfig::from_value(Some(&value)) + .expect_err("invalid field path should fail config parsing"); + + assert!(err.to_string().contains(expected), "{field_name}={path:?}: {err}"); + } + } + + #[test] + fn from_value_rejects_non_string_field_path_entries() { + let value = json!({ "field_allowlist": [1] }); + + let err = + SecretsDetectionConfig::from_value(Some(&value)).expect_err("non-string path should fail config parsing"); + + assert!(err.to_string().contains("field_allowlist must be a list of dotted field path strings"), "{err}"); + } + + #[test] + fn field_path_matcher_scans_and_traverses_everything_by_default() { + let config = SecretsDetectionConfig::default(); + + assert!(config.should_scan_field_path(&path(&[]), true)); + assert!(config.should_scan_field_path(&path(&["layer1"]), false)); + assert!(config.should_scan_field_path(&path(&["accounts", "credentials", "token"]), false)); + assert!(config.should_traverse_field_path(&path(&[]))); + assert!(config.should_traverse_field_path(&path(&["layer1"]))); + assert!(config.should_traverse_field_path(&path(&["accounts", "credentials"]))); + } + + #[test] + fn field_path_matcher_allows_listed_paths_and_descendants() { + let config = config_with_field_paths(&["layer1"], &[]); + + assert!(config.should_scan_field_path(&path(&[]), true)); + assert!(!config.should_scan_field_path(&path(&[]), false)); + assert!(config.should_scan_field_path(&path(&["layer1"]), false)); + assert!(config.should_scan_field_path(&path(&["layer1", "public"]), false)); + assert!(config.should_traverse_field_path(&path(&[]))); + assert!(config.should_traverse_field_path(&path(&["layer1"]))); + assert!(config.should_traverse_field_path(&path(&["layer1", "public"]))); + assert!(!config.should_scan_field_path(&path(&["layer10"]), false)); + assert!(!config.should_traverse_field_path(&path(&["layer10"]))); + } + + #[test] + fn field_path_matcher_traverses_unselected_parents_to_reach_nested_allowlist() { + let config = config_with_field_paths(&["accounts.credentials.token"], &[]); + + assert!(config.should_traverse_field_path(&path(&[]))); + assert!(config.should_traverse_field_path(&path(&["accounts"]))); + assert!(config.should_traverse_field_path(&path(&["accounts", "credentials"]))); + assert!(!config.should_scan_field_path(&path(&["accounts"]), false)); + assert!(!config.should_scan_field_path(&path(&["accounts", "credentials"]), false)); + assert!(config.should_scan_field_path(&path(&["accounts", "credentials", "token"]), false)); + assert!(config.should_scan_field_path(&path(&["accounts", "credentials", "token", "value"]), false)); + assert!(!config.should_traverse_field_path(&path(&["profile"]))); + } + + #[test] + fn field_path_matcher_denylist_takes_precedence() { + let config = config_with_field_paths(&["layer1"], &["layer1.layer2.layer3"]); + + assert!(config.should_scan_field_path(&path(&["layer1", "public"]), false)); + assert!(config.should_traverse_field_path(&path(&["layer1", "layer2"]))); + assert!(config.should_scan_field_path(&path(&["layer1", "layer2", "safe"]), false)); + assert!(!config.should_scan_field_path(&path(&["layer1", "layer2", "layer3"]), false)); + assert!(!config.should_traverse_field_path(&path(&["layer1", "layer2", "layer3"]))); + assert!(!config.should_scan_field_path(&path(&["layer1", "layer2", "layer3", "token"]), false)); + assert!(!config.should_traverse_field_path(&path(&["layer1", "layer2", "layer3", "token"]))); + } + + #[test] + fn field_path_matcher_uses_segment_aware_matching() { + let allow_config = config_with_field_paths(&["layer1"], &[]); + assert!(allow_config.should_scan_field_path(&path(&["layer1"]), false)); + assert!(!allow_config.should_scan_field_path(&path(&["layer10"]), false)); + assert!(!allow_config.should_traverse_field_path(&path(&["layer10"]))); + + let deny_config = config_with_field_paths(&[], &["layer1"]); + assert!(!deny_config.should_scan_field_path(&path(&["layer1"]), false)); + assert!(!deny_config.should_scan_field_path(&path(&["layer1", "secret"]), false)); + assert!(deny_config.should_scan_field_path(&path(&["layer10"]), false)); + assert!(deny_config.should_traverse_field_path(&path(&["layer10"]))); + } + + fn segments(path: &FieldPath) -> Vec<&str> { + path.segments().iter().map(String::as_str).collect() + } + + fn config_with_field_paths(allow: &[&str], deny: &[&str]) -> SecretsDetectionConfig { + SecretsDetectionConfig { + field_allowlist: allow.iter().map(|path| FieldPath::parse(path, "field_allowlist").unwrap()).collect(), + field_denylist: deny.iter().map(|path| FieldPath::parse(path, "field_denylist").unwrap()).collect(), + ..Default::default() + } + } + + fn path(parts: &[&str]) -> Vec { + parts.iter().map(|part| (*part).to_owned()).collect() + } +} diff --git a/crates/plugins/cpex-secrets-detection/src/lib.rs b/crates/plugins/cpex-secrets-detection/src/lib.rs new file mode 100644 index 0000000..09f0e26 --- /dev/null +++ b/crates/plugins/cpex-secrets-detection/src/lib.rs @@ -0,0 +1,795 @@ +// Copyright 2026 +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::HashMap; +use std::sync::Arc; + +use config::SecretsDetectionConfig; +use cpex::cpex_core::{ + cmf::{CmfHook, ContentPart, MessagePayload}, + context::PluginContext, + error::{PluginError, PluginViolation}, + factory::{PluginFactory, PluginInstance}, + hooks::{Extensions, HookHandler, PluginResult, TypedHandlerAdapter}, + plugin::{Plugin, PluginConfig}, + registry::AnyHookHandler, +}; +use scanner::{Finding, scan_direct_text, scan_json_value}; +use serde_json::{Map, Value, json}; + +pub mod config; +pub mod patterns; +pub mod scanner; + +pub const KIND: &str = "validator/secrets-detection"; +const VIOLATION_CODE: &str = "SECRETS_DETECTED"; +const MAX_SECRET_TYPES: usize = 32; + +#[derive(Debug)] +pub struct SecretsDetectionCore { + config: PluginConfig, + scanner_config: SecretsDetectionConfig, +} + +impl SecretsDetectionCore { + pub fn new(config: PluginConfig) -> Result { + let scanner_config = SecretsDetectionConfig::from_value(config.config.as_ref())?; + Ok(Self { config, scanner_config }) + } + + fn scanner_config(&self) -> &SecretsDetectionConfig { + &self.scanner_config + } + + fn should_block(&self, count: usize) -> bool { + self.scanner_config.block_on_detection && count >= self.scanner_config.min_findings_to_block + } +} + +impl Plugin for SecretsDetectionCore { + fn config(&self) -> &PluginConfig { + &self.config + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Stage { + // The dataplane runtime config currently uses the tool pre/post stages. + // Prompt/resource stages are kept for CPEX parity and future hosts. + PromptPreFetch, + ToolPreInvoke, + ToolPostInvoke, + ResourcePostFetch, +} + +pub struct StageHandler { + core: Arc, + stage: Stage, +} + +impl StageHandler { + fn new(core: Arc, stage: Stage) -> Self { + Self { core, stage } + } +} + +impl Plugin for StageHandler { + fn config(&self) -> &PluginConfig { + self.core.config() + } +} + +impl HookHandler for StageHandler { + async fn handle( + &self, + payload: &MessagePayload, + extensions: &Extensions, + _ctx: &mut PluginContext, + ) -> PluginResult { + let scan = self.scan_payload(payload); + + if self.core.should_block(scan.count) { + let violation = build_violation(self.stage, scan.count, &scan.findings); + let mut result = PluginResult::deny(violation); + result.modified_payload = scan.modified_payload.or_else(|| Some(payload.clone())); + attach_metrics(&mut result, extensions, scan.count, &scan.findings, DetectionOutcome::Blocked); + return result; + } + + if let Some(modified_payload) = scan.modified_payload { + let mut result = PluginResult::modify_payload(modified_payload); + attach_metrics(&mut result, extensions, scan.count, &scan.findings, DetectionOutcome::Masked); + return result; + } + + let mut result = PluginResult::allow(); + if scan.count > 0 { + attach_metrics(&mut result, extensions, scan.count, &scan.findings, DetectionOutcome::None); + } + result + } +} + +impl StageHandler { + fn scan_payload(&self, payload: &MessagePayload) -> PayloadScan { + let config = self.core.scanner_config(); + let mut modified_payload = payload.clone(); + let mut count = 0usize; + let mut findings = Vec::new(); + let mut redacted_any = false; + + for part in &mut modified_payload.message.content { + let Some(mut report) = self.scan_content_part(part, config) else { + continue; + }; + count += report.count; + findings.append(&mut report.findings); + if config.redact && report.redacted { + redacted_any = true; + } + } + + PayloadScan { count, findings, modified_payload: redacted_any.then_some(modified_payload) } + } + + fn scan_content_part(&self, part: &mut ContentPart, config: &SecretsDetectionConfig) -> Option { + match (self.stage, part) { + (Stage::PromptPreFetch, ContentPart::PromptRequest { content }) => { + let value = Value::Object(Map::from_iter(content.arguments.clone())); + let report = scan_json_value(&value, config); + let redacted = config.redact && report.count > 0; + if redacted { + let Value::Object(arguments) = report.redacted else { + unreachable!("prompt arguments scan preserves JSON object shape"); + }; + content.arguments = arguments.into_iter().collect(); + } + Some(PartScanReport { count: report.count, findings: report.findings, redacted }) + }, + (Stage::ToolPreInvoke, ContentPart::ToolCall { content }) => { + let value = Value::Object(Map::from_iter(content.arguments.clone())); + let report = scan_json_value(&value, config); + let redacted = config.redact && report.count > 0; + if redacted { + let Value::Object(arguments) = report.redacted else { + unreachable!("tool arguments scan preserves JSON object shape"); + }; + content.arguments = arguments.into_iter().collect(); + } + Some(PartScanReport { count: report.count, findings: report.findings, redacted }) + }, + (Stage::ToolPostInvoke, ContentPart::ToolResult { content }) => { + let report = scan_json_value(&content.content, config); + let redacted = config.redact && report.count > 0; + if redacted { + content.content = report.redacted; + } + Some(PartScanReport { count: report.count, findings: report.findings, redacted }) + }, + (Stage::ResourcePostFetch, ContentPart::Resource { content }) => { + let text = content.content.as_ref()?; + let report = scan_direct_text(text, config); + let redacted = config.redact && report.count > 0; + if redacted { + let Value::String(redacted_text) = report.redacted else { + unreachable!("direct text scan returns a string value"); + }; + content.content = Some(redacted_text); + } + Some(PartScanReport { count: report.count, findings: report.findings, redacted }) + }, + _ => None, + } + } +} + +struct PayloadScan { + count: usize, + findings: Vec, + modified_payload: Option, +} + +struct PartScanReport { + count: usize, + findings: Vec, + redacted: bool, +} + +#[derive(Clone, Copy)] +enum DetectionOutcome { + Masked, + Blocked, + None, +} + +fn build_violation(stage: Stage, count: usize, findings: &[Finding]) -> PluginViolation { + let details = + HashMap::from([("count".to_owned(), json!(count)), ("examples".to_owned(), sanitized_findings(findings))]); + PluginViolation::new(VIOLATION_CODE, "Secrets detected") + .with_description(stage.block_description()) + .with_details(details) +} + +fn sanitized_findings(findings: &[Finding]) -> Value { + Value::Array(findings.iter().map(|finding| json!({ "type": finding.pii_type })).collect()) +} + +fn attach_metrics( + result: &mut PluginResult, + extensions: &Extensions, + count: usize, + findings: &[Finding], + outcome: DetectionOutcome, +) { + let Some(metadata) = build_metrics(extensions, count, findings, outcome) else { + return; + }; + result.metadata = Some(metadata); +} + +fn build_metrics( + extensions: &Extensions, + count: usize, + findings: &[Finding], + outcome: DetectionOutcome, +) -> Option { + let trace_id = extensions.request.as_ref().and_then(|request| request.trace_id.as_deref())?; + if trace_id.is_empty() || count == 0 { + return None; + } + + let mut secret_types = findings.iter().map(|finding| finding.pii_type.as_str()).collect::>(); + secret_types.sort_unstable(); + secret_types.dedup(); + secret_types.truncate(MAX_SECRET_TYPES); + + let (masked, blocked) = match outcome { + DetectionOutcome::Masked => (count, 0), + DetectionOutcome::Blocked => (0, count), + DetectionOutcome::None => (0, 0), + }; + + Some(json!({ + "secrets_detection": { + "total_detections": count, + "total_masked": masked, + "total_blocked": blocked, + "secret_types": secret_types, + } + })) +} + +impl Stage { + fn block_description(self) -> &'static str { + match self { + Stage::PromptPreFetch => "Potential secrets detected in prompt arguments", + Stage::ToolPreInvoke => "Potential secrets detected in tool arguments", + Stage::ToolPostInvoke => "Potential secrets detected in tool result", + Stage::ResourcePostFetch => "Potential secrets detected in resource content", + } + } +} + +pub struct SecretsDetectionFactory; + +impl PluginFactory for SecretsDetectionFactory { + fn create(&self, config: &PluginConfig) -> Result> { + let core = Arc::new( + SecretsDetectionCore::new(config.clone()) + .map_err(|err| PluginError::Config { message: err.to_string() })?, + ); + let handlers = config.hooks.iter().filter_map(|hook| handler_for_hook(hook, Arc::clone(&core))).collect(); + + Ok(PluginInstance { plugin: core, handlers }) + } +} + +fn handler_for_hook(hook: &str, core: Arc) -> Option<(&'static str, Arc)> { + let stage = match hook { + "cmf.prompt_pre_fetch" => Stage::PromptPreFetch, + "cmf.tool_pre_invoke" => Stage::ToolPreInvoke, + "cmf.tool_post_invoke" => Stage::ToolPostInvoke, + "cmf.resource_post_fetch" => Stage::ResourcePostFetch, + _ => return None, + }; + let hook_name: &'static str = Box::leak(hook.to_owned().into_boxed_str()); + let handler = StageHandler::new(core, stage); + let adapter: Arc = Arc::new(TypedHandlerAdapter::::new(Arc::new(handler))); + Some((hook_name, adapter)) +} + +#[cfg(test)] +mod tests { + use std::collections::{HashMap, HashSet}; + use std::sync::Arc; + + use cpex::{ + PluginManager, + cpex_core::{ + cmf::{ContentPart, Message, PromptRequest, Resource, ResourceType, Role, ToolCall, ToolResult}, + executor::PipelineResult, + extensions::RequestExtension, + hooks::Extensions, + plugin::{OnError, PluginMode}, + }, + }; + use serde_json::{Value, json}; + + use super::*; + + #[tokio::test] + async fn factory_registers_cmf_tool_pre_invoke_handler() { + let payload = tool_call_payload(HashMap::from([("message".to_owned(), json!("hello"))])); + + let result = invoke_manager("cmf.tool_pre_invoke", "", payload, Extensions::default()).await; + + assert!(result.continue_processing); + assert!(result.violation.is_none()); + } + + #[tokio::test] + async fn tool_post_invoke_redacts_json_content_through_manager() { + let payload = tool_result_payload(json!({ + "message": "AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE" + })); + + let result = invoke_manager( + "cmf.tool_post_invoke", + r#" block_on_detection: false + redact: true + redaction_text: "[REDACTED]" +"#, + payload, + Extensions::default(), + ) + .await; + + assert!(result.continue_processing); + assert!(result.violation.is_none()); + assert_eq!(tool_result_content(pipeline_payload(&result))["message"], json!("AWS_ACCESS_KEY_ID=[REDACTED]")); + } + + #[tokio::test] + async fn prompt_pre_fetch_redacts_arguments_through_manager() { + let payload = prompt_request_payload(HashMap::from([( + "token".to_owned(), + json!("AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE"), + )])); + + let result = invoke_manager( + "cmf.prompt_pre_fetch", + r#" block_on_detection: false + redact: true + redaction_text: "[REDACTED]" +"#, + payload, + Extensions::default(), + ) + .await; + + assert!(result.continue_processing); + assert_eq!(prompt_argument(pipeline_payload(&result), "token"), &json!("AWS_ACCESS_KEY_ID=[REDACTED]")); + } + + #[tokio::test] + async fn resource_post_fetch_direct_text_ignores_field_filters_through_manager() { + let payload = resource_payload("AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE"); + + let result = invoke_manager( + "cmf.resource_post_fetch", + r#" block_on_detection: false + redact: true + redaction_text: "[REDACTED]" + field_allowlist: + - different.path + field_denylist: + - content.text +"#, + payload, + Extensions::default(), + ) + .await; + + assert!(result.continue_processing); + assert_eq!(resource_text(pipeline_payload(&result)), "AWS_ACCESS_KEY_ID=[REDACTED]"); + } + + #[tokio::test] + async fn tool_pre_invoke_threshold_counts_only_eligible_fields() { + let payload = tool_call_payload(HashMap::from([ + ("allowed".to_owned(), json!("AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE")), + ("ignored".to_owned(), json!("AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE")), + ])); + + let result = invoke_manager( + "cmf.tool_pre_invoke", + r" block_on_detection: true + min_findings_to_block: 2 + field_allowlist: + - allowed +", + payload, + Extensions::default(), + ) + .await; + + assert!(result.continue_processing); + assert!(result.violation.is_none()); + } + + #[tokio::test] + async fn direct_handler_returns_metadata_and_redacted_payload_on_block() { + let payload = + tool_call_payload(HashMap::from([("message".to_owned(), json!("AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE"))])); + + let result = invoke_direct_handler( + Stage::ToolPreInvoke, + json!({ + "block_on_detection": true, + "redact": true, + "redaction_text": "[REDACTED]", + "min_findings_to_block": 1 + }), + payload, + extensions_with_trace("trace-1"), + ) + .await; + + let metadata = result.metadata.as_ref().unwrap(); + assert!(!result.continue_processing); + assert_eq!(result.violation.as_ref().unwrap().code, VIOLATION_CODE); + assert_eq!(metadata["secrets_detection"]["total_blocked"], json!(1)); + assert_eq!( + tool_call_argument(result.modified_payload.as_ref().unwrap(), "message"), + &json!("AWS_ACCESS_KEY_ID=[REDACTED]") + ); + assert!(!metadata.to_string().contains("AKIAFAKE12345EXAMPLE")); + } + + #[tokio::test] + async fn direct_handler_omits_metadata_without_trace_id() { + let payload = + tool_call_payload(HashMap::from([("message".to_owned(), json!("AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE"))])); + + let result = invoke_direct_handler( + Stage::ToolPreInvoke, + json!({ + "block_on_detection": false, + "redact": true, + "redaction_text": "[REDACTED]" + }), + payload, + Extensions::default(), + ) + .await; + + assert!(result.continue_processing); + assert!(result.metadata.is_none()); + assert_eq!( + tool_call_argument(result.modified_payload.as_ref().unwrap(), "message"), + &json!("AWS_ACCESS_KEY_ID=[REDACTED]") + ); + } + + #[tokio::test] + async fn direct_handler_omits_metadata_for_clean_payload_with_trace_id() { + let payload = tool_call_payload(HashMap::from([("message".to_owned(), json!("hello"))])); + + let result = invoke_direct_handler( + Stage::ToolPreInvoke, + json!({ + "block_on_detection": false, + "redact": true, + "redaction_text": "[REDACTED]" + }), + payload, + extensions_with_trace("trace-1"), + ) + .await; + + assert!(result.continue_processing); + assert!(result.violation.is_none()); + assert!(result.modified_payload.is_none()); + assert!(result.metadata.is_none()); + } + + #[tokio::test] + async fn direct_handler_metadata_reports_masked_outcome() { + let payload = + tool_call_payload(HashMap::from([("message".to_owned(), json!("AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE"))])); + + let result = invoke_direct_handler( + Stage::ToolPreInvoke, + json!({ + "block_on_detection": false, + "redact": true, + "redaction_text": "[REDACTED]" + }), + payload, + extensions_with_trace("trace-1"), + ) + .await; + + let metadata = result.metadata.as_ref().unwrap(); + assert!(result.continue_processing); + assert_eq!(metadata["secrets_detection"]["total_detections"], json!(1)); + assert_eq!(metadata["secrets_detection"]["total_masked"], json!(1)); + assert_eq!(metadata["secrets_detection"]["total_blocked"], json!(0)); + assert_eq!(metadata["secrets_detection"]["secret_types"], json!(["aws_access_key_id"])); + assert!(!metadata.to_string().contains("AKIAFAKE12345EXAMPLE")); + } + + #[tokio::test] + async fn direct_handler_metadata_reports_findings_only_outcome() { + let payload = + tool_call_payload(HashMap::from([("message".to_owned(), json!("AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE"))])); + + let result = invoke_direct_handler( + Stage::ToolPreInvoke, + json!({ + "block_on_detection": false, + "redact": false + }), + payload, + extensions_with_trace("trace-1"), + ) + .await; + + let metadata = result.metadata.as_ref().unwrap(); + assert!(result.continue_processing); + assert!(result.violation.is_none()); + assert!(result.modified_payload.is_none()); + assert_eq!(metadata["secrets_detection"]["total_detections"], json!(1)); + assert_eq!(metadata["secrets_detection"]["total_masked"], json!(0)); + assert_eq!(metadata["secrets_detection"]["total_blocked"], json!(0)); + assert!(!metadata.to_string().contains("AKIAFAKE12345EXAMPLE")); + } + + #[tokio::test] + async fn direct_handler_metadata_omits_raw_secret_values() { + let payload = + tool_call_payload(HashMap::from([("message".to_owned(), json!("AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE"))])); + + let result = invoke_direct_handler( + Stage::ToolPreInvoke, + json!({ + "block_on_detection": false, + "redact": false + }), + payload, + extensions_with_trace("trace-1"), + ) + .await; + + let dumped = result.metadata.as_ref().unwrap().to_string(); + assert!(dumped.contains("aws_access_key_id")); + assert!(!dumped.contains("AKIAFAKE12345EXAMPLE")); + assert!(!dumped.contains("AWS_ACCESS_KEY_ID=")); + } + + async fn invoke_direct_handler( + stage: Stage, + config: Value, + payload: MessagePayload, + extensions: Extensions, + ) -> PluginResult { + let core = Arc::new(SecretsDetectionCore::new(plugin_config(config)).unwrap()); + let handler = StageHandler::new(core, stage); + let mut ctx = PluginContext::new(); + handler.handle(&payload, &extensions, &mut ctx).await + } + + #[tokio::test] + async fn manager_drops_plugin_metadata_in_cpex_0_2_2() { + let payload = + tool_call_payload(HashMap::from([("message".to_owned(), json!("AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE"))])); + + let result = invoke_manager( + "cmf.tool_pre_invoke", + r#" block_on_detection: false + redact: true + redaction_text: "[REDACTED]" +"#, + payload, + extensions_with_trace("trace-1"), + ) + .await; + + assert!(result.continue_processing); + assert_eq!(tool_call_argument(pipeline_payload(&result), "message"), &json!("AWS_ACCESS_KEY_ID=[REDACTED]")); + assert!( + result.metadata.is_none(), + "CPEX 0.2.2 erase_result does not carry PluginResult.metadata into PipelineResult" + ); + } + + #[tokio::test] + async fn manager_deny_drops_modified_payload_in_cpex_0_2_2() { + let payload = + tool_call_payload(HashMap::from([("message".to_owned(), json!("AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE"))])); + + let result = invoke_manager( + "cmf.tool_pre_invoke", + r#" block_on_detection: true + redact: true + redaction_text: "[REDACTED]" + min_findings_to_block: 1 +"#, + payload, + extensions_with_trace("trace-1"), + ) + .await; + + assert!(!result.continue_processing); + assert_eq!(result.violation.as_ref().unwrap().code, VIOLATION_CODE); + assert!( + result.modified_payload.is_none(), + "CPEX 0.2.2 PipelineResult::denied does not surface denied modified_payload" + ); + assert!( + result.metadata.is_none(), + "CPEX 0.2.2 erase_result does not carry PluginResult.metadata into PipelineResult" + ); + } + + async fn invoke_manager( + hook: &str, + config_block: &str, + payload: MessagePayload, + extensions: Extensions, + ) -> PipelineResult { + let manager = Arc::new(PluginManager::default()); + manager.register_factory(KIND, Box::new(SecretsDetectionFactory)); + let yaml = plugin_yaml(hook, config_block); + manager.load_config_yaml(&yaml).expect("config should load"); + manager.initialize().await.expect("initialize"); + + let (result, background) = manager.invoke_named::(hook, payload, extensions, None).await; + background.wait_for_background_tasks().await; + result + } + + fn plugin_yaml(hook: &str, config_block: &str) -> String { + let config = if config_block.trim().is_empty() { + " config: {}\n".to_owned() + } else { + format!(" config:\n{config_block}") + }; + format!( + r#"plugins: + - name: secrets-detection + kind: validator/secrets-detection + hooks: ["{hook}"] + mode: sequential +{config}"# + ) + } + + fn plugin_config(config: Value) -> PluginConfig { + PluginConfig { + name: "secrets-detection".to_owned(), + kind: KIND.to_owned(), + description: None, + author: None, + version: None, + hooks: vec!["cmf.tool_pre_invoke".to_owned()], + mode: PluginMode::Sequential, + priority: 100, + on_error: OnError::Fail, + capabilities: HashSet::new(), + tags: Vec::new(), + conditions: Vec::new(), + config: Some(config), + } + } + + fn extensions_with_trace(trace_id: &str) -> Extensions { + Extensions { + request: Some(Arc::new(RequestExtension { trace_id: Some(trace_id.to_owned()), ..Default::default() })), + ..Default::default() + } + } + + fn tool_call_payload(arguments: HashMap) -> MessagePayload { + MessagePayload { + message: Message::with_content( + Role::User, + vec![ContentPart::ToolCall { + content: ToolCall { + tool_call_id: "tool-call-1".into(), + name: "echo".into(), + arguments, + namespace: None, + }, + }], + ), + } + } + + fn prompt_request_payload(arguments: HashMap) -> MessagePayload { + MessagePayload { + message: Message::with_content( + Role::User, + vec![ContentPart::PromptRequest { + content: PromptRequest { + prompt_request_id: "prompt-1".into(), + name: "summarize".into(), + arguments, + server_id: None, + }, + }], + ), + } + } + + fn tool_result_payload(content: Value) -> MessagePayload { + MessagePayload { + message: Message::with_content( + Role::Tool, + vec![ContentPart::ToolResult { + content: ToolResult { + tool_call_id: "tool-call-1".into(), + tool_name: "echo".into(), + content, + is_error: false, + }, + }], + ), + } + } + + fn resource_payload(text: &str) -> MessagePayload { + MessagePayload { + message: Message::with_content( + Role::Tool, + vec![ContentPart::Resource { + content: Resource { + resource_request_id: "resource-1".into(), + uri: "file:///tmp/secret.txt".into(), + resource_type: ResourceType::File, + content: Some(text.to_owned()), + ..Default::default() + }, + }], + ), + } + } + + fn pipeline_payload(result: &PipelineResult) -> &MessagePayload { + result + .modified_payload + .as_ref() + .expect("pipeline returns final payload on allow") + .as_any() + .downcast_ref::() + .expect("CMF payload type") + } + + fn tool_call_argument<'a>(payload: &'a MessagePayload, key: &str) -> &'a Value { + let ContentPart::ToolCall { content } = &payload.message.content[0] else { + panic!("expected tool call content"); + }; + content.arguments.get(key).expect("argument exists") + } + + fn prompt_argument<'a>(payload: &'a MessagePayload, key: &str) -> &'a Value { + let ContentPart::PromptRequest { content } = &payload.message.content[0] else { + panic!("expected prompt request content"); + }; + content.arguments.get(key).expect("argument exists") + } + + fn tool_result_content(payload: &MessagePayload) -> &Value { + let ContentPart::ToolResult { content } = &payload.message.content[0] else { + panic!("expected tool result content"); + }; + &content.content + } + + fn resource_text(payload: &MessagePayload) -> &str { + let ContentPart::Resource { content } = &payload.message.content[0] else { + panic!("expected resource content"); + }; + content.content.as_deref().expect("text content exists") + } +} diff --git a/crates/plugins/cpex-secrets-detection/src/patterns.rs b/crates/plugins/cpex-secrets-detection/src/patterns.rs new file mode 100644 index 0000000..35401c9 --- /dev/null +++ b/crates/plugins/cpex-secrets-detection/src/patterns.rs @@ -0,0 +1,65 @@ +// Copyright 2026 +// SPDX-License-Identifier: Apache-2.0 + +use regex::Regex; +use std::collections::{HashMap, HashSet}; +use std::sync::LazyLock; + +pub static PATTERNS: LazyLock> = LazyLock::new(|| { + let mut patterns = HashMap::new(); + patterns.insert("aws_access_key_id", Regex::new(r"\bAKIA[0-9A-Z]{16}\b").expect("valid aws_access_key_id regex")); + patterns.insert( + "aws_secret_access_key", + Regex::new(r#"(?i)aws.{0,20}(?:secret|access).{0,20}[:=]\s*["']?([A-Za-z0-9/+=]{40})["']?"#) + .expect("valid aws_secret_access_key regex"), + ); + patterns.insert("google_api_key", Regex::new(r"\bAIza[0-9A-Za-z\-_]{35}\b").expect("valid google_api_key regex")); + patterns.insert( + "github_token", + Regex::new(r"\b(?:gh[opusr]_[A-Za-z0-9]{36}|github_pat_[A-Za-z0-9_]{20,})\b") + .expect("valid github_token regex"), + ); + patterns.insert( + "stripe_secret_key", + Regex::new(r"\b(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{16,}\b").expect("valid stripe_secret_key regex"), + ); + patterns.insert( + "generic_api_key_assignment", + Regex::new( + r#"(?ix)\b(?:(?:x[-_])?api[-_]?key|apikey|api[_-]?token|access[_-]?token|bearer[_-]?token|auth[_-]?token)\b\s*[:=]\s*['"]?[A-Za-z0-9_\-]{20,}['"]?"#, + ) + .expect("valid generic_api_key_assignment regex"), + ); + patterns + .insert("slack_token", Regex::new(r"\bxox[abpqr]-[0-9A-Za-z\-]{10,80}\b").expect("valid slack_token regex")); + patterns.insert( + "private_key_block", + Regex::new(r"-----BEGIN (?:RSA|DSA|EC|OPENSSH) PRIVATE KEY-----").expect("valid private_key_block regex"), + ); + patterns.insert( + "jwt_like", + Regex::new(r"\beyJ[a-zA-Z0-9_\-]{10,}\.eyJ[a-zA-Z0-9_\-]{10,}\.[a-zA-Z0-9_\-]{10,}\b") + .expect("valid jwt_like regex"), + ); + patterns.insert("hex_secret_32", Regex::new(r"(?i)\b[a-f0-9]{32,}\b").expect("valid hex_secret_32 regex")); + patterns.insert( + "base64_24", + Regex::new(r"(?:^|[^A-Za-z0-9+/])((?:[A-Za-z0-9+/]{24,}={0,2}|[A-Za-z0-9+/]{22}==|[A-Za-z0-9+/]{23}=))") + .expect("valid base64_24 regex"), + ); + patterns +}); + +pub static CAPTURE_PATTERNS: LazyLock> = LazyLock::new(|| HashSet::from(["base64_24"])); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn loads_all_patterns() { + assert_eq!(PATTERNS.len(), 11); + assert!(PATTERNS.contains_key("aws_access_key_id")); + assert!(PATTERNS.contains_key("generic_api_key_assignment")); + } +} diff --git a/crates/plugins/cpex-secrets-detection/src/scanner.rs b/crates/plugins/cpex-secrets-detection/src/scanner.rs new file mode 100644 index 0000000..c26593a --- /dev/null +++ b/crates/plugins/cpex-secrets-detection/src/scanner.rs @@ -0,0 +1,448 @@ +// Copyright 2026 +// SPDX-License-Identifier: Apache-2.0 + +use serde_json::{Map, Value}; + +use crate::config::SecretsDetectionConfig; +use crate::patterns::{CAPTURE_PATTERNS, PATTERNS}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Finding { + pub pii_type: String, + pub preview: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ScanReport { + pub count: usize, + pub redacted: Value, + pub findings: Vec, +} + +struct MatchCandidate<'a> { + name: &'static str, + start: usize, + end: usize, + text: &'a str, +} + +pub fn scan_json_value(value: &Value, config: &SecretsDetectionConfig) -> ScanReport { + let mut path = Vec::new(); + scan_json_value_inner(value, config, &mut path, true) +} + +pub fn scan_direct_text(text: &str, config: &SecretsDetectionConfig) -> ScanReport { + let (findings, redacted) = detect_and_redact(text, config); + ScanReport { count: findings.len(), redacted: Value::String(redacted), findings } +} + +pub fn detect_and_redact(text: &str, config: &SecretsDetectionConfig) -> (Vec, String) { + let mut candidates = Vec::new(); + + for (name, pattern) in PATTERNS.iter() { + if !config.is_enabled(name) { + continue; + } + + if CAPTURE_PATTERNS.contains(name) { + for captures in pattern.captures_iter(text) { + let Some(matched) = captures.get(1) else { + continue; + }; + if is_base64_boundary_char(text[matched.end()..].chars().next()) { + continue; + } + candidates.push(MatchCandidate { + name, + start: matched.start(), + end: matched.end(), + text: matched.as_str(), + }); + } + } else { + for matched in pattern.find_iter(text) { + candidates.push(MatchCandidate { + name, + start: matched.start(), + end: matched.end(), + text: matched.as_str(), + }); + } + } + } + + candidates.sort_by(|left, right| { + left.start + .cmp(&right.start) + .then_with(|| pattern_specificity(left.name).cmp(&pattern_specificity(right.name))) + .then_with(|| (right.end - right.start).cmp(&(left.end - left.start))) + .then_with(|| left.name.cmp(right.name)) + }); + + let mut selected = Vec::new(); + for candidate in candidates { + let Some(current) = selected.last_mut() else { + selected.push(candidate); + continue; + }; + + if candidate.start >= current.end { + selected.push(candidate); + continue; + } + + let candidate_specificity = pattern_specificity(candidate.name); + let current_specificity = pattern_specificity(current.name); + let candidate_len = candidate.end - candidate.start; + let current_len = current.end - current.start; + if candidate_specificity < current_specificity + || (candidate_specificity == current_specificity && candidate_len > current_len) + { + current.name = candidate.name; + current.text = candidate.text; + } + + if candidate.end > current.end { + current.end = candidate.end; + } + } + + let findings = selected + .iter() + .map(|matched| { + let preview = if matched.text.chars().count() > 8 { + format!("{}...", matched.text.chars().take(8).collect::()) + } else { + matched.text.to_owned() + }; + Finding { pii_type: matched.name.to_owned(), preview } + }) + .collect::>(); + + let redacted = if config.redact && !selected.is_empty() { + let mut redacted = String::with_capacity(text.len()); + let mut cursor = 0usize; + for matched in &selected { + redacted.push_str(&text[cursor..matched.start]); + redacted.push_str(&config.redaction_text); + cursor = matched.end; + } + redacted.push_str(&text[cursor..]); + redacted + } else { + text.to_owned() + }; + + (findings, redacted) +} + +fn scan_json_value_inner( + value: &Value, + config: &SecretsDetectionConfig, + path: &mut Vec, + direct_scalar_root: bool, +) -> ScanReport { + match value { + Value::String(text) => { + if !config.should_scan_field_path(path, direct_scalar_root) { + return clean_report(value); + } + let (findings, redacted) = detect_and_redact(text, config); + ScanReport { count: findings.len(), redacted: Value::String(redacted), findings } + }, + Value::Array(items) => { + if !config.should_traverse_field_path(path) { + return clean_report(value); + } + let mut total = 0usize; + let mut redacted_items = Vec::with_capacity(items.len()); + let mut findings = Vec::new(); + + for item in items { + let mut child = scan_json_value_inner(item, config, path, false); + total += child.count; + redacted_items.push(child.redacted); + findings.append(&mut child.findings); + } + + ScanReport { count: total, redacted: Value::Array(redacted_items), findings } + }, + Value::Object(entries) => { + if !config.should_traverse_field_path(path) { + return clean_report(value); + } + let mut total = 0usize; + let mut redacted_entries = Map::with_capacity(entries.len()); + let mut findings = Vec::new(); + + for (key, value) in entries { + path.push(key.clone()); + let mut child = scan_json_value_inner(value, config, path, false); + path.pop(); + total += child.count; + redacted_entries.insert(key.clone(), child.redacted); + findings.append(&mut child.findings); + } + + ScanReport { count: total, redacted: Value::Object(redacted_entries), findings } + }, + _ => clean_report(value), + } +} + +fn clean_report(value: &Value) -> ScanReport { + ScanReport { count: 0, redacted: value.clone(), findings: Vec::new() } +} + +fn pattern_specificity(name: &str) -> usize { + match name { + "generic_api_key_assignment" | "jwt_like" => 1, + "hex_secret_32" => 2, + "base64_24" => 3, + _ => 0, + } +} + +fn is_base64_boundary_char(ch: Option) -> bool { + ch.is_some_and(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '+' | '/' | '=')) +} + +#[cfg(test)] +mod tests { + use std::collections::{HashMap, HashSet}; + + use serde_json::json; + + use super::*; + use crate::config::FieldPath; + + const SECRET_FIXTURE: &str = "FAKESecretAccessKeyForTestingEXAMPLE0000"; + + #[test] + fn detects_aws_secret_access_key() { + let config = SecretsDetectionConfig::default(); + let (findings, _) = + detect_and_redact("AWS_SECRET_ACCESS_KEY=FAKESecretAccessKeyForTestingEXAMPLE0000", &config); + assert!(findings.iter().any(|finding| finding.pii_type == "aws_secret_access_key")); + } + + #[test] + fn detects_aws_secret_access_key_assignment_formats() { + let config = + SecretsDetectionConfig { redact: true, redaction_text: "[REDACTED]".to_owned(), ..Default::default() }; + + let cases = [ + ("equals-unquoted", format!("AWS_SECRET_ACCESS_KEY={SECRET_FIXTURE}")), + ("equals-double-quoted", format!("aws_secret_access_key = \"{SECRET_FIXTURE}\"")), + ("equals-single-quoted", format!("aws_secret_access_key = '{SECRET_FIXTURE}'")), + ("yaml-unquoted", format!("aws_secret_access_key: {SECRET_FIXTURE}")), + ("yaml-double-quoted", format!("aws_secret_access_key: \"{SECRET_FIXTURE}\"")), + ("yaml-single-quoted", format!("aws_secret_access_key: '{SECRET_FIXTURE}'")), + ("json-spaced", format!(r#""aws_secret_access_key": "{SECRET_FIXTURE}""#)), + ("json-compact", format!(r#""aws_secret_access_key":"{SECRET_FIXTURE}""#)), + ("mixed-case", format!("AwsSecretAccessKey: \"{SECRET_FIXTURE}\"")), + ]; + + for (name, text) in cases { + let (findings, redacted) = detect_and_redact(&text, &config); + + assert_eq!(findings.len(), 1, "{name}: {findings:?}"); + assert_eq!(findings[0].pii_type, "aws_secret_access_key", "{name}"); + assert!(!redacted.contains(SECRET_FIXTURE), "{name}: {redacted}"); + assert!(redacted.contains(&config.redaction_text), "{name}: {redacted}"); + } + } + + #[test] + fn redaction_works() { + let config = + SecretsDetectionConfig { redact: true, redaction_text: "[REDACTED]".to_owned(), ..Default::default() }; + let (findings, redacted) = detect_and_redact("AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE", &config); + assert_eq!(findings.len(), 1); + assert_eq!(redacted, "AWS_ACCESS_KEY_ID=[REDACTED]"); + } + + #[test] + fn redacts_each_supported_secret_as_one_replacement_with_all_patterns_enabled() { + let config = SecretsDetectionConfig { + enabled: crate::patterns::PATTERNS.keys().map(|&name| (name.to_owned(), true)).collect(), + redact: true, + redaction_text: "[TESTING-REDACTED]".to_owned(), + ..Default::default() + }; + + for (name, secret) in [ + ("aws_access_key_id", "AKIAFAKE12345EXAMPLE".to_owned()), + ("aws_secret_access_key", "AWS_SECRET_ACCESS_KEY=FAKESecretAccessKeyForTestingEXAMPLE0000".to_owned()), + ("google_api_key", "AIzaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_owned()), + ("github_token", "ghp_abcdefghijklmnopqrstuvwxyz0123456789".to_owned()), + ("stripe_secret_key", "sk_test_abcdefghijklmnopqrstuvwxyz".to_owned()), + ("generic_api_key_assignment", "api_key=test12345678901234567890".to_owned()), + ("slack_token", ["xoxb", "123456789012", "123456789012", "abcdefghijklmnopqrstuvwx"].join("-")), + ("private_key_block", "-----BEGIN RSA PRIVATE KEY-----".to_owned()), + ("jwt_like", "eyJaaaaaaaaaaa.eyJbbbbbbbbbbb.cccccccccccccc".to_owned()), + ("hex_secret_32", "0123456789abcdef0123456789abcdef".to_owned()), + ("base64_24", "QUJDREVGR0hJSktMTU5PUFFSU1RVVldY".to_owned()), + ] { + let (findings, redacted) = detect_and_redact(&secret, &config); + assert_eq!(findings.len(), 1, "{name}: {findings:?}"); + assert_eq!(findings[0].pii_type, name, "{name}: {findings:?}"); + assert_eq!(redacted, config.redaction_text, "{name}"); + } + } + + #[test] + fn overlapping_broad_match_keeps_specific_finding_type() { + let config = SecretsDetectionConfig { + enabled: crate::patterns::PATTERNS.keys().map(|&name| (name.to_owned(), true)).collect(), + redact: true, + redaction_text: "[TESTING-REDACTED]".to_owned(), + ..Default::default() + }; + let secret = "AIzaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA/BBBBBBBB"; + + let (findings, redacted) = detect_and_redact(secret, &config); + + assert_eq!(findings.len(), 1, "{findings:?}"); + assert_eq!(findings[0].pii_type, "google_api_key", "{findings:?}"); + assert_eq!(redacted, config.redaction_text); + } + + #[test] + fn json_scan_handles_nested_structures() { + let redact_config = + SecretsDetectionConfig { redact: true, redaction_text: "[REDACTED]".to_owned(), ..Default::default() }; + let value = json!({ + "users": [ + { + "name": "Alice", + "key": "AKIAFAKE12345EXAMPLE" + }, + { + "name": "Bob", + "token": "xoxr-fake-000000000-fake000000000-fakefakefakefake" + } + ] + }); + + let report = scan_json_value(&value, &redact_config); + + assert_eq!(report.count, 2); + assert_eq!( + report.redacted, + json!({ + "users": [ + { + "name": "Alice", + "key": "[REDACTED]" + }, + { + "name": "Bob", + "token": "[REDACTED]" + } + ] + }) + ); + assert_eq!(report.findings.len(), 2); + let finding_types: HashSet<_> = report.findings.iter().map(|finding| finding.pii_type.as_str()).collect(); + assert_eq!(finding_types, HashSet::from(["aws_access_key_id", "slack_token"])); + } + + #[test] + fn field_filters_apply_to_json_objects() { + let payload = json!({ + "layer1": { + "public": "AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE", + "layer2": { + "layer3": "AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE" + } + }, + "layer10": "AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE" + }); + + let config = config_with_field_filters(&["layer1"], &["layer1.layer2.layer3"], true); + let report = scan_json_value(&payload, &config); + + assert_eq!(report.count, 1); + assert_eq!(report.redacted["layer1"]["public"], json!("AWS_ACCESS_KEY_ID=[REDACTED]")); + assert_eq!(report.redacted["layer1"]["layer2"]["layer3"], json!("AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE")); + assert_eq!(report.redacted["layer10"], json!("AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE")); + } + + #[test] + fn field_filters_reach_nested_allowlisted_paths_through_lists() { + let payload = json!({ + "users": [ + { + "credentials": { + "token": "AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE", + "other": "AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE" + } + }, + { + "credentials": { + "token": "AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE" + } + } + ], + "outside": "AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE" + }); + + let config = config_with_field_filters(&["users.credentials.token"], &[], true); + let report = scan_json_value(&payload, &config); + + assert_eq!(report.count, 2); + assert_eq!(report.redacted["users"][0]["credentials"]["token"], json!("AWS_ACCESS_KEY_ID=[REDACTED]")); + assert_eq!( + report.redacted["users"][0]["credentials"]["other"], + json!("AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE") + ); + assert_eq!(report.redacted["users"][1]["credentials"]["token"], json!("AWS_ACCESS_KEY_ID=[REDACTED]")); + assert_eq!(report.redacted["outside"], json!("AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE")); + } + + #[test] + fn generic_api_key_assignment_detection_is_opt_in() { + let config = SecretsDetectionConfig { + enabled: HashMap::from([("generic_api_key_assignment".to_owned(), true)]), + ..Default::default() + }; + let (findings, _) = detect_and_redact("X-API-Key: test12345678901234567890", &config); + assert!(findings.iter().any(|finding| finding.pii_type == "generic_api_key_assignment")); + } + + #[test] + fn broad_patterns_are_opt_in() { + let config = SecretsDetectionConfig { redact: true, ..Default::default() }; + let (findings, redacted) = detect_and_redact("access_token = 'abcdefghijklmnopqrstuvwx'", &config); + assert!(findings.is_empty()); + assert_eq!(redacted, "access_token = 'abcdefghijklmnopqrstuvwx'"); + } + + #[test] + fn redacts_padded_base64_secret_without_leaving_padding() { + let config = SecretsDetectionConfig { + enabled: HashMap::from([("base64_24".to_owned(), true)]), + redact: true, + redaction_text: "[REDACTED]".to_owned(), + ..Default::default() + }; + + let sample = "mZ8qL2vYwT1pNc4Rb6HxUg=="; + let (findings, redacted) = detect_and_redact(&format!("token={sample}"), &config); + + assert_eq!(findings.len(), 1, "{findings:?}"); + assert_eq!(findings[0].pii_type, "base64_24"); + assert_eq!(redacted, "token=[REDACTED]"); + } + + fn config_with_field_filters(allowlist: &[&str], denylist: &[&str], redact: bool) -> SecretsDetectionConfig { + SecretsDetectionConfig { + redact, + redaction_text: "[REDACTED]".to_owned(), + field_allowlist: allowlist.iter().map(|path| FieldPath::parse(path, "field_allowlist").unwrap()).collect(), + field_denylist: denylist.iter().map(|path| FieldPath::parse(path, "field_denylist").unwrap()).collect(), + ..Default::default() + } + } +} diff --git a/crates/plugins/cpex-secrets-detection/tests/plugin_manager.rs b/crates/plugins/cpex-secrets-detection/tests/plugin_manager.rs new file mode 100644 index 0000000..bda206c --- /dev/null +++ b/crates/plugins/cpex-secrets-detection/tests/plugin_manager.rs @@ -0,0 +1,432 @@ +// Copyright 2026 +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::HashMap; +use std::sync::Arc; + +use cpex::{ + PluginManager, + cpex_core::{ + cmf::{ + CmfHook, ContentPart, Message, MessagePayload, PromptRequest, Resource, ResourceType, Role, ToolCall, + ToolResult, + }, + executor::PipelineResult, + hooks::Extensions, + }, +}; +use cpex_secrets_detection::{KIND, SecretsDetectionFactory}; +use serde_json::{Value, json}; + +#[tokio::test] +async fn prompt_pre_fetch_blocks_without_redaction() { + let payload = + prompt_request_payload(HashMap::from([("token".to_owned(), json!("AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE"))])); + + let result = invoke_manager( + "cmf.prompt_pre_fetch", + r" block_on_detection: true + redact: false + min_findings_to_block: 1 +", + payload, + ) + .await; + + assert!(!result.continue_processing); + assert_eq!(result.violation.as_ref().unwrap().code, "SECRETS_DETECTED"); + assert!(result.modified_payload.is_none()); +} + +#[tokio::test] +async fn prompt_pre_fetch_clean_payload_allows_without_modification() { + let payload = prompt_request_payload(HashMap::from([("message".to_owned(), json!("hello"))])); + + let result = invoke_manager( + "cmf.prompt_pre_fetch", + r" block_on_detection: true + redact: true +", + payload, + ) + .await; + + assert!(result.continue_processing); + assert!(result.violation.is_none()); + assert_eq!(prompt_argument(pipeline_payload(&result), "message"), &json!("hello")); +} + +#[tokio::test] +async fn prompt_pre_fetch_field_filters_apply_to_arguments() { + let payload = prompt_request_payload(HashMap::from([ + ("allowed".to_owned(), json!("AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE")), + ("ignored".to_owned(), json!("AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE")), + ])); + + let result = invoke_manager( + "cmf.prompt_pre_fetch", + r#" block_on_detection: false + redact: true + redaction_text: "[REDACTED]" + field_allowlist: + - allowed +"#, + payload, + ) + .await; + + assert!(result.continue_processing); + let payload = pipeline_payload(&result); + assert_eq!(prompt_argument(payload, "allowed"), &json!("AWS_ACCESS_KEY_ID=[REDACTED]")); + assert_eq!(prompt_argument(payload, "ignored"), &json!("AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE")); +} + +#[tokio::test] +async fn tool_pre_invoke_redacts_without_blocking() { + let payload = + tool_call_payload(HashMap::from([("token".to_owned(), json!("AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE"))])); + + let result = invoke_manager( + "cmf.tool_pre_invoke", + r#" block_on_detection: false + redact: true + redaction_text: "[REDACTED]" +"#, + payload, + ) + .await; + + assert!(result.continue_processing); + assert!(result.violation.is_none()); + assert_eq!(tool_call_argument(pipeline_payload(&result), "token"), &json!("AWS_ACCESS_KEY_ID=[REDACTED]")); +} + +#[tokio::test] +async fn tool_pre_invoke_blocks_without_redaction() { + let payload = + tool_call_payload(HashMap::from([("token".to_owned(), json!("AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE"))])); + + let result = invoke_manager( + "cmf.tool_pre_invoke", + r" block_on_detection: true + redact: false + min_findings_to_block: 1 +", + payload, + ) + .await; + + assert!(!result.continue_processing); + assert_eq!(result.violation.as_ref().unwrap().code, "SECRETS_DETECTED"); + assert!(result.modified_payload.is_none()); +} + +#[tokio::test] +async fn tool_pre_invoke_clean_payload_allows_without_modification() { + let payload = tool_call_payload(HashMap::from([("message".to_owned(), json!("hello"))])); + + let result = invoke_manager( + "cmf.tool_pre_invoke", + r" block_on_detection: true + redact: true +", + payload, + ) + .await; + + assert!(result.continue_processing); + assert!(result.violation.is_none()); + assert_eq!(tool_call_argument(pipeline_payload(&result), "message"), &json!("hello")); +} + +#[tokio::test] +async fn tool_pre_invoke_rejects_invalid_field_allowlist_at_load() { + let manager = Arc::new(PluginManager::default()); + manager.register_factory(KIND, Box::new(SecretsDetectionFactory)); + let yaml = plugin_yaml( + "cmf.tool_pre_invoke", + r" field_allowlist: + - bad. +", + ); + + let err = match manager.load_config_yaml(&yaml) { + Ok(()) => panic!("invalid field_allowlist should fail config loading"), + Err(err) => err.to_string(), + }; + + assert!( + err.contains("field_allowlist path \"bad.\" must not start or end with '.'"), + "unexpected config error: {err}" + ); +} + +#[tokio::test] +async fn tool_pre_invoke_nested_filters_match_crate_smoke() { + let payload = tool_call_payload(HashMap::from([ + ( + "accounts".to_owned(), + json!({ + "keep": "AWS_ACCESS_KEY_ID=AKIATEST12345EXAMPLE", + "skip": "AWS_ACCESS_KEY_ID=AKIASKIP12345EXAMPLE" + }), + ), + ("ignored".to_owned(), json!("AWS_ACCESS_KEY_ID=AKIAIGNR12345EXAMPLE")), + ])); + let original = payload.clone(); + + let result = invoke_manager( + "cmf.tool_pre_invoke", + r#" block_on_detection: false + redact: true + redaction_text: "[REDACTED]" + field_allowlist: + - accounts + field_denylist: + - accounts.skip +"#, + payload, + ) + .await; + + assert!(result.continue_processing); + assert!(result.violation.is_none()); + + let modified = pipeline_payload(&result); + assert_eq!(tool_call_argument(modified, "accounts")["keep"], json!("AWS_ACCESS_KEY_ID=[REDACTED]")); + assert_eq!(tool_call_argument(modified, "accounts")["skip"], json!("AWS_ACCESS_KEY_ID=AKIASKIP12345EXAMPLE")); + assert_eq!(tool_call_argument(modified, "ignored"), &json!("AWS_ACCESS_KEY_ID=AKIAIGNR12345EXAMPLE")); + assert_eq!(tool_call_argument(&original, "accounts")["keep"], json!("AWS_ACCESS_KEY_ID=AKIATEST12345EXAMPLE")); +} + +#[tokio::test] +async fn tool_post_invoke_blocks_json_content() { + let payload = tool_result_payload(json!({ + "token": "AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE" + })); + + let result = invoke_manager( + "cmf.tool_post_invoke", + r" block_on_detection: true + redact: false + min_findings_to_block: 1 +", + payload, + ) + .await; + + assert!(!result.continue_processing); + assert_eq!(result.violation.as_ref().unwrap().code, "SECRETS_DETECTED"); + assert!(result.modified_payload.is_none()); +} + +#[tokio::test] +async fn tool_post_invoke_field_filters_apply_to_result_content() { + let payload = tool_result_payload(json!({ + "allowed": "AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE", + "ignored": "AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE" + })); + + let result = invoke_manager( + "cmf.tool_post_invoke", + r#" block_on_detection: false + redact: true + redaction_text: "[REDACTED]" + field_allowlist: + - allowed +"#, + payload, + ) + .await; + + assert!(result.continue_processing); + let content = tool_result_content(pipeline_payload(&result)); + assert_eq!(content["allowed"], json!("AWS_ACCESS_KEY_ID=[REDACTED]")); + assert_eq!(content["ignored"], json!("AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE")); +} + +#[tokio::test] +async fn tool_post_invoke_clean_result_allows_without_modification() { + let payload = tool_result_payload(json!({ "message": "hello" })); + + let result = invoke_manager( + "cmf.tool_post_invoke", + r" block_on_detection: true + redact: true +", + payload, + ) + .await; + + assert!(result.continue_processing); + assert!(result.violation.is_none()); + assert_eq!(tool_result_content(pipeline_payload(&result))["message"], json!("hello")); +} + +#[tokio::test] +async fn resource_post_fetch_blocks_text_content() { + let payload = resource_payload("AWS_ACCESS_KEY_ID=AKIAFAKE12345EXAMPLE"); + + let result = invoke_manager( + "cmf.resource_post_fetch", + r" block_on_detection: true + redact: false + min_findings_to_block: 1 +", + payload, + ) + .await; + + assert!(!result.continue_processing); + assert_eq!(result.violation.as_ref().unwrap().code, "SECRETS_DETECTED"); + assert!(result.modified_payload.is_none()); +} + +#[tokio::test] +async fn resource_post_fetch_clean_payload_allows_without_modification() { + let payload = resource_payload("hello"); + + let result = invoke_manager( + "cmf.resource_post_fetch", + r" block_on_detection: true + redact: true +", + payload, + ) + .await; + + assert!(result.continue_processing); + assert!(result.violation.is_none()); + assert_eq!(resource_text(pipeline_payload(&result)), "hello"); +} + +async fn invoke_manager(hook: &str, config_block: &str, payload: MessagePayload) -> PipelineResult { + let manager = Arc::new(PluginManager::default()); + manager.register_factory(KIND, Box::new(SecretsDetectionFactory)); + let yaml = plugin_yaml(hook, config_block); + manager.load_config_yaml(&yaml).expect("config should load"); + manager.initialize().await.expect("initialize"); + + let (result, background) = manager.invoke_named::(hook, payload, Extensions::default(), None).await; + background.wait_for_background_tasks().await; + result +} + +fn plugin_yaml(hook: &str, config_block: &str) -> String { + let config = if config_block.trim().is_empty() { + " config: {}\n".to_owned() + } else { + format!(" config:\n{config_block}") + }; + format!( + r#"plugins: + - name: secrets-detection + kind: validator/secrets-detection + hooks: ["{hook}"] + mode: sequential +{config}"# + ) +} + +fn tool_call_payload(arguments: HashMap) -> MessagePayload { + MessagePayload { + message: Message::with_content( + Role::User, + vec![ContentPart::ToolCall { + content: ToolCall { + tool_call_id: "tool-call-1".into(), + name: "echo".into(), + arguments, + namespace: None, + }, + }], + ), + } +} + +fn prompt_request_payload(arguments: HashMap) -> MessagePayload { + MessagePayload { + message: Message::with_content( + Role::User, + vec![ContentPart::PromptRequest { + content: PromptRequest { + prompt_request_id: "prompt-1".into(), + name: "summarize".into(), + arguments, + server_id: None, + }, + }], + ), + } +} + +fn tool_result_payload(content: Value) -> MessagePayload { + MessagePayload { + message: Message::with_content( + Role::Tool, + vec![ContentPart::ToolResult { + content: ToolResult { + tool_call_id: "tool-call-1".into(), + tool_name: "echo".into(), + content, + is_error: false, + }, + }], + ), + } +} + +fn resource_payload(text: &str) -> MessagePayload { + MessagePayload { + message: Message::with_content( + Role::Tool, + vec![ContentPart::Resource { + content: Resource { + resource_request_id: "resource-1".into(), + uri: "file:///tmp/secret.txt".into(), + resource_type: ResourceType::File, + content: Some(text.to_owned()), + ..Default::default() + }, + }], + ), + } +} + +fn pipeline_payload(result: &PipelineResult) -> &MessagePayload { + result + .modified_payload + .as_ref() + .expect("pipeline returns final payload on allow") + .as_any() + .downcast_ref::() + .expect("CMF payload type") +} + +fn tool_call_argument<'a>(payload: &'a MessagePayload, key: &str) -> &'a Value { + let ContentPart::ToolCall { content } = &payload.message.content[0] else { + panic!("expected tool call content"); + }; + content.arguments.get(key).expect("argument exists") +} + +fn prompt_argument<'a>(payload: &'a MessagePayload, key: &str) -> &'a Value { + let ContentPart::PromptRequest { content } = &payload.message.content[0] else { + panic!("expected prompt request content"); + }; + content.arguments.get(key).expect("argument exists") +} + +fn tool_result_content(payload: &MessagePayload) -> &Value { + let ContentPart::ToolResult { content } = &payload.message.content[0] else { + panic!("expected tool result content"); + }; + &content.content +} + +fn resource_text(payload: &MessagePayload) -> &str { + let ContentPart::Resource { content } = &payload.message.content[0] else { + panic!("expected resource content"); + }; + content.content.as_deref().expect("text content exists") +} diff --git a/docker/Dockerfile b/docker/Dockerfile index db21aa6..408275e 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -12,7 +12,7 @@ RUN --mount=type=cache,id=cargo,target=/usr/local/cargo/registry \ cargo fetch --locked RUN --mount=type=cache,id=cargo,target=/usr/local/cargo/registry \ --mount=type=cache,id=cargo-git,target=/usr/local/cargo/git \ - cargo build --release --features contextforge-gateway-rs-lib/with_tools + cargo build --release --features "contextforge-gateway-rs-lib/with_tools contextforge-gateway-rs/plugins" FROM debian:trixie-slim RUN <