diff --git a/simplyblock_core/controllers/host_auth.py b/simplyblock_core/controllers/host_auth.py index 1862094e7..6f7cb4679 100644 --- a/simplyblock_core/controllers/host_auth.py +++ b/simplyblock_core/controllers/host_auth.py @@ -1,6 +1,10 @@ # coding=utf-8 +from pydantic import SecretStr + from simplyblock_core import constants, utils from simplyblock_core.db_controller import DBController +from simplyblock_core.rpc_client import RPCException +from simplyblock_core.snode_client import SNodeClientException logger = utils.get_logger(__name__) @@ -22,8 +26,55 @@ def _get_dhchap_group(cluster, pool=None): return "null" +def _register_key_on_node(snode, rpc_client, key_name, key_value): + """Register a single key in SPDK's keyring on a storage node. + + Prefers the spdk-proxy's keyring_add_key interceptor, which delivers the + key material inline and keeps it on a memory-backed volume. A proxy that + predates interception support forwards keyring_add_key to SPDK, which + rejects the unknown method (-32601); we then fall back to the deprecated + SNodeAPI write_key_file + keyring_file_add_key path (key file on persistent + host storage) so nodes not yet restarted after an upgrade keep working. + + "File exists" (code -17) means the key is already registered, which is + fine (e.g. same host on another volume) and handled via allow_existing. + + Returns True if the key is registered (or already was), False on failure. + """ + try: + rpc_client.keyring_add_key(key_name, key_value, allow_existing=True) + return True + except RPCException as e: + if e.code != -32601: # anything but "Method not found" is a real failure + logger.error("Failed to register key %s in SPDK keyring on node %s: %s", + key_name, snode.get_id(), e.message) + return False + logger.warning("Node %s uses a spdk-proxy without keyring_add_key support; " + "falling back to the deprecated on-disk key path — restart " + "the storage node to enable in-memory key delivery", + snode.get_id()) + + try: + key_path, error = snode.client().write_key_file(key_name, key_value) + except SNodeClientException as e: + logger.error("Failed to write key file %s on node %s: %s", + key_name, snode.get_id(), e.message) + return False + if error: + logger.error("Failed to write key file %s on node %s: %s", + key_name, snode.get_id(), error) + return False + try: + rpc_client.keyring_file_add_key(key_name, key_path, allow_existing=True) + return True + except RPCException as e: + logger.error("Failed to register key %s in SPDK keyring on node %s: %s", + key_name, snode.get_id(), e.message) + return False + + def _register_pool_dhchap_keys_on_node(pool, snode, rpc_client): - """Write pool-level DHCHAP key files to a storage node and register in SPDK keyring. + """Register pool-level DHCHAP keys on a storage node's SPDK keyring. All LVols in a DHCHAP pool share one key pair stored on the pool. Key names are pool-scoped so a single registration serves all LVols. @@ -31,46 +82,29 @@ def _register_pool_dhchap_keys_on_node(pool, snode, rpc_client): Returns a dict with 'dhchap_key' and 'dhchap_ctrlr_key' keyring names, or an empty dict on failure. """ - snode_api = snode.client() safe_pool = pool.get_id().replace("-", "_") key_names = {} for key_type, key_value in ( - ("dhchap_key", pool.dhchap_key.get_secret_value()), - ("dhchap_ctrlr_key", pool.dhchap_ctrlr_key.get_secret_value()), + ("dhchap_key", pool.dhchap_key), + ("dhchap_ctrlr_key", pool.dhchap_ctrlr_key), ): - if not key_value: + if not key_value or not key_value.get_secret_value(): continue key_name = f"pool_{safe_pool}_{key_type}" - result, error = snode_api.write_key_file(key_name, key_value) - if error: - logger.error("Failed to write pool key %s on node %s: %s", - key_name, snode.get_id(), error) - continue - key_path = result - ret, err = rpc_client._request2("keyring_file_add_key", - {"name": key_name, "path": key_path}) - if not ret and err: - if err.get("code") == -17: - logger.info("Pool key %s already in SPDK keyring on node %s, reusing", - key_name, snode.get_id()) - else: - logger.error("Failed to register pool key %s in SPDK keyring on node %s: %s", - key_name, snode.get_id(), err.get("message", err)) - continue - key_names[key_type] = key_name + if _register_key_on_node(snode, rpc_client, key_name, key_value): + key_names[key_type] = key_name return key_names def _register_dhchap_keys_on_node(snode, host_nqn, host_entry, rpc_client): - """Write DHCHAP key files to a storage node and register them in SPDK's keyring. + """Register per-host DHCHAP/PSK keys on a storage node's SPDK keyring. Returns a dict mapping key type ('dhchap_key', 'dhchap_ctrlr_key', 'psk') to the SPDK keyring name for use in subsystem_add_host. """ - snode_api = snode.client() - # Sanitize host NQN for use as filename + # Sanitize host NQN for use as key name safe_host = host_nqn.replace(":", "_").replace(".", "_") key_names = {} @@ -79,24 +113,8 @@ def _register_dhchap_keys_on_node(snode, host_nqn, host_entry, rpc_client): if not key_value: continue key_name = f"{key_type}_{safe_host}" - result, error = snode_api.write_key_file(key_name, key_value) - if error: - logger.error("Failed to write key file %s on node %s: %s", key_name, snode.get_id(), error) - continue - key_path = result - # Register in SPDK keyring — "File exists" (code -17) means the key - # is already registered, which is fine (e.g. same host on another volume). - ret, err = rpc_client._request2("keyring_file_add_key", - {"name": key_name, "path": key_path}) - if not ret and err: - if err.get("code") == -17: - logger.info("Key %s already in SPDK keyring on node %s, reusing", - key_name, snode.get_id()) - else: - logger.error("Failed to register key %s in SPDK keyring on node %s: %s", - key_name, snode.get_id(), err.get("message", err)) - continue - key_names[key_type] = key_name + if _register_key_on_node(snode, rpc_client, key_name, SecretStr(key_value)): + key_names[key_type] = key_name return key_names diff --git a/simplyblock_core/rpc_client.py b/simplyblock_core/rpc_client.py index ec5b7fda8..12e8cd165 100755 --- a/simplyblock_core/rpc_client.py +++ b/simplyblock_core/rpc_client.py @@ -235,6 +235,11 @@ def subsystem_create(self, nqn, serial_number, model_number, min_cntlid=1, max_n def keyring_file_add_key(self, name: str, path: str, *, allow_existing: bool = False): """Register a file-based key in SPDK's keyring by path. + Deprecated: requires the key file to be placed on the node beforehand + (via the deprecated SNodeAPI write_key_file endpoint). Use + keyring_add_key instead, which delivers the key material through the + spdk-proxy without touching persistent storage. + Args: name: Key name for the SPDK keyring. path: Path to the key file on the storage node. @@ -250,6 +255,34 @@ def keyring_file_add_key(self, name: str, path: str, *, allow_existing: bool = F return None raise + def keyring_add_key(self, name: str, key: SecretStr, *, allow_existing: bool = False): + """Register a key in SPDK's keyring, delivering the material inline. + + Intercepted by the spdk-proxy, which writes the key to a memory-backed + secrets directory shared with the SPDK container and forwards a + keyring_file_add_key referencing that file. A proxy that predates + interception forwards this to SPDK, which rejects the unknown method + with -32601; callers fall back to keyring_file_add_key in that case. + + Args: + name: Key name for the SPDK keyring. + key: The key material (e.g. DHCHAP/PSK key in its wire format). + allow_existing: When True, silently succeed if the key is already + registered with the same material (SPDK errno -17). When False + (default) an existing key raises RPCException. + + Raises: + RPCException: On proxy or SPDK errors, including a key of the same + name existing with different material. + """ + try: + return self._request3("keyring_add_key", name=name, key=key) + except RPCException as e: + if allow_existing and e.code == -17: + logger.debug("Key %s already in SPDK keyring, reusing", name) + return None + raise + def keyring_file_remove_key(self, name): """Remove a key from SPDK's keyring.""" params = {"name": name} diff --git a/simplyblock_core/services/spdk_http_proxy_server.py b/simplyblock_core/services/spdk_http_proxy_server.py index 75f74bebe..6e24151c3 100644 --- a/simplyblock_core/services/spdk_http_proxy_server.py +++ b/simplyblock_core/services/spdk_http_proxy_server.py @@ -4,6 +4,7 @@ import json import logging import os +import re import socket import sys import threading @@ -204,6 +205,145 @@ def _rpc_call_inner(req, req_data, req_time, sock_timeout): pass +KEY_NAME_RE = re.compile(r'^[a-zA-Z0-9_-]+$') + + +def _jsonrpc_error(req_id, code, message): + return json.dumps({'jsonrpc': '2.0', 'id': req_id, 'error': {'code': code, 'message': message}}) + + +def _response_error(response): + """Extract the JSON-RPC error object from a raw response string, if any.""" + if response is None: + return None + try: + parsed = json.loads(response) + except ValueError: + return None + if not isinstance(parsed, dict): + return None + return parsed.get('error') + + +def _key_path(name): + return os.path.join(SPDK_PROXY_KEY_DIR, name) + + +def _remove_key_file(name): + try: + os.unlink(_key_path(name)) + except FileNotFoundError: + pass # ignore error for idempotency + + +def _handle_keyring_add_key(req_data, client_timeout): + """Virtual method {name, key}: write the key material to the memory-backed + secrets dir and forward a rewritten keyring_file_add_key {name, path} to + SPDK. Key material must never be forwarded or logged (rpc_call logs params). + """ + req_id = req_data.get('id') + params = req_data.get('params') or {} + name = params.get('name') + key = params.get('key') + if not isinstance(name, str) or not KEY_NAME_RE.match(name): + return _jsonrpc_error(req_id, -32602, 'Invalid key name') + if not isinstance(key, str) or not key or not key.isascii(): + return _jsonrpc_error(req_id, -32602, 'Invalid key material') + + path = _key_path(name) + pre_existing = os.path.exists(path) + if pre_existing: + with open(path, encoding='ascii') as f: + if f.read() != key: + return _jsonrpc_error( + req_id, -32000, + f"Key '{name}' already exists with different material; remove it first") + else: + os.makedirs(SPDK_PROXY_KEY_DIR, mode=0o700, exist_ok=True) + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + os.write(fd, key.encode('ascii')) + finally: + os.close(fd) + logger.info(f"Intercepted keyring_add_key for key {name}") + + fwd = {k: v for k, v in req_data.items() if k != 'params'} + fwd['method'] = 'keyring_file_add_key' + fwd['params'] = {'name': name, 'path': path} + try: + response = rpc_call(json.dumps(fwd).encode('ascii'), client_timeout) + except Exception: + # The forward failed outright (e.g. SPDK timeout). A key file freshly + # written on behalf of this request must not linger on the shared volume. + if not pre_existing: + _remove_key_file(name) + raise + + error = _response_error(response) + if error is not None and error.get('code') != -17 and not pre_existing: + # We wrote the key file for this request but SPDK rejected it with + # something other than "already exists": never leave an orphaned secret. + _remove_key_file(name) + # On -17 the key is registered in SPDK, so its backing file must stay: the + # client's allow_existing treats -17 as reuse. A pre-existing file with + # matching content is likewise kept whether SPDK reports success (keyring + # lost, e.g. SPDK restarted while the tmpfs survived) or -17. + return response + + +def _handle_keyring_file_remove_key(req_data, client_timeout): + """Pass keyring_file_remove_key through to SPDK, then delete the key file + so removed keys don't linger in the secrets dir.""" + params = req_data.get('params') or {} + name = params.get('name') + response = rpc_call(json.dumps(req_data).encode('ascii'), client_timeout) + if isinstance(name, str) and KEY_NAME_RE.match(name): + error = _response_error(response) + if error is None or error.get('code') == -2: # removed, or unknown to SPDK + _remove_key_file(name) + return response + + +def _handle_proxy_get_capabilities(req_data, client_timeout): + """Answered locally: lets clients probe whether this proxy supports + interception before sending any key material (older proxies forward this + to SPDK and return 'Method not found' without logging secrets).""" + return json.dumps({ + 'jsonrpc': '2.0', + 'id': req_data.get('id'), + 'result': {'interceptors': sorted(RPC_INTERCEPTORS.keys())}, + }) + + +RPC_INTERCEPTORS = { + 'keyring_add_key': _handle_keyring_add_key, + 'keyring_file_remove_key': _handle_keyring_file_remove_key, + 'proxy_get_capabilities': _handle_proxy_get_capabilities, +} + + +def dispatch_rpc(data_string, client_timeout=None): + """Route a JSON-RPC payload through a method interceptor if one is + registered, otherwise forward it to SPDK unchanged. + + Interception happens before rpc_call so secret-bearing params are + rewritten before any request logging. + """ + try: + req_data = json.loads(data_string.decode('ascii')) + method = req_data.get('method') if isinstance(req_data, dict) else None + except (ValueError, UnicodeDecodeError): + return rpc_call(data_string, client_timeout) + handler = RPC_INTERCEPTORS.get(method) if isinstance(method, str) else None + if handler is None: + return rpc_call(data_string, client_timeout) + try: + return handler(req_data, client_timeout) + except OSError as e: + logger.error(f"Interceptor for {method} failed: {e}") + return _jsonrpc_error(req_data.get('id'), -32000, f'proxy interceptor error: {e}') + + class ServerHandler(BaseHTTPRequestHandler): server_session: list[int] = [] key = "" @@ -261,7 +401,7 @@ def do_POST(self): logger.info(f"read_line_time_diff: {time_diff}") read_line_time_diff[read_line_time_start] = time_diff try: - response = rpc_call(data_string, self.headers.get('X-RPC-Timeout')) + response = dispatch_rpc(data_string, self.headers.get('X-RPC-Timeout')) if response is not None: self.do_HEAD() self.wfile.write(bytes(response.encode(encoding='ascii'))) @@ -304,6 +444,11 @@ def run_server(host, port, user, password, is_threading_enabled=False): # instead of being aborted; small enough that abandoned/stuck RPCs free their # slot quickly. See _resolve_sock_timeout. SPDK_TIMEOUT_MARGIN = float(get_env_var("SPDK_TIMEOUT_MARGIN", is_required=False, default=2)) +# Directory for key files written by the keyring_add_key interceptor. Must be +# a memory-backed volume shared with the SPDK container so key material never +# touches persistent storage. +SPDK_PROXY_KEY_DIR = get_env_var("SPDK_PROXY_KEY_DIR", is_required=False, + default="/var/run/secrets/simplyblock/keys") is_threading_enabled = get_env_var("MULTI_THREADING_ENABLED", is_required=False, default=False) server_ip = get_env_var("SERVER_IP", is_required=True, default="") rpc_port = get_env_var("RPC_PORT", is_required=True) diff --git a/simplyblock_core/snode_client.py b/simplyblock_core/snode_client.py index 08eb7cd99..a8092f3d2 100644 --- a/simplyblock_core/snode_client.py +++ b/simplyblock_core/snode_client.py @@ -115,7 +115,13 @@ def info(self): return self._request("GET", "info") def write_key_file(self, name, content): - """Write a DHCHAP key file on the storage node for SPDK keyring.""" + """Write a DHCHAP key file on the storage node for SPDK keyring. + + Deprecated: use RPCClient.keyring_add_key, which delivers the key + material through the spdk-proxy onto a memory-backed volume instead + of persistent host storage. Kept one release as a fallback for + proxies without interception support. + """ return self._request("POST", "write_key_file", {"name": name, "content": content}) def read_allowed_list(self): diff --git a/simplyblock_web/api/internal/storage_node/docker.py b/simplyblock_web/api/internal/storage_node/docker.py index a7457a533..aca25b9e8 100644 --- a/simplyblock_web/api/internal/storage_node/docker.py +++ b/simplyblock_web/api/internal/storage_node/docker.py @@ -144,6 +144,15 @@ def spdk_process_start(body: SPDKParams): else: log_config = LogConfig(type=LogConfig.types.JOURNALD) + # Memory-backed volume shared between the SPDK and spdk-proxy containers: + # the proxy's keyring_add_key interceptor writes key files here, so key + # material never touches persistent host storage. volumes.create is + # idempotent by name; contents evaporate once both containers are gone. + secrets_volume = f"spdk_keys_{body.rpc_port}" + node_docker.volumes.create( + name=secrets_volume, driver='local', + driver_opts={'type': 'tmpfs', 'device': 'tmpfs', 'o': 'size=16m,mode=0700'}) + container = node_docker.containers.run( body.spdk_image, f"sudo -E /root/scripts/run_distr_with_ssd.sh {body.l_cores} {spdk_mem_mib} {spdk_debug}", @@ -161,6 +170,7 @@ def spdk_process_start(body: SPDKParams): '/var/lib/systemd/coredump/:/var/lib/systemd/coredump/', '/sys:/sys', '/mnt/ramdisk:/mnt/ramdisk', + f'{secrets_volume}:/var/run/secrets/simplyblock', ], environment=[ f"RPC_PORT={body.rpc_port}", @@ -183,6 +193,7 @@ def spdk_process_start(body: SPDKParams): volumes=[ f'/var/tmp/spdk_{body.rpc_port}:/var/tmp', '/mnt/ramdisk:/mnt/ramdisk', + f'{secrets_volume}:/var/run/secrets/simplyblock', ], environment=[ f"SERVER_IP={body.server_ip}", @@ -191,6 +202,7 @@ def spdk_process_start(body: SPDKParams): f"RPC_PASSWORD={body.rpc_password}", f"MULTI_THREADING_ENABLED={body.multi_threading_enabled}", f"TIMEOUT={body.timeout}", + "SPDK_PROXY_KEY_DIR=/var/run/secrets/simplyblock/keys", ] # restart_policy={"Name": "always"} ) @@ -439,7 +451,15 @@ class WriteKeyFileBody(BaseModel): })}}}, }) def write_key_file(body: WriteKeyFileBody): - """Write a DHCHAP key file for SPDK keyring_file module.""" + """Write a DHCHAP key file for SPDK keyring_file module. + + Deprecated: keys are now delivered through the spdk-proxy's + keyring_add_key interceptor onto a memory-backed volume. This endpoint + remains one release as a fallback for proxies without interception + support and will be removed. + """ + logger.warning("write_key_file is deprecated; keys should be registered " + "via the spdk-proxy keyring_add_key method") import re if not re.match(r'^[a-zA-Z0-9_\-]+$', body.name): return utils.get_response(None, "Invalid key name") diff --git a/simplyblock_web/api/internal/storage_node/kubernetes.py b/simplyblock_web/api/internal/storage_node/kubernetes.py index db86a6fe1..3d843f57c 100644 --- a/simplyblock_web/api/internal/storage_node/kubernetes.py +++ b/simplyblock_web/api/internal/storage_node/kubernetes.py @@ -589,9 +589,17 @@ class WriteKeyFileBody(BaseModel): })}}}, }) def write_key_file(body: WriteKeyFileBody): - """Write a DHCHAP key file for SPDK keyring_file module.""" + """Write a DHCHAP key file for SPDK keyring_file module. + + Deprecated: keys are now delivered through the spdk-proxy's + keyring_add_key interceptor onto a memory-backed volume. This endpoint + remains one release as a fallback for proxies without interception + support and will be removed. + """ + logger.warning("write_key_file is deprecated; keys should be registered " + "via the spdk-proxy keyring_add_key method") import re - if not re.match(r'^[a-zA-Z0-9_\\-]+$', body.name): + if not re.match(r'^[a-zA-Z0-9_-]+$', body.name): return utils.get_response(None, "Invalid key name") os.makedirs(DHCHAP_KEY_DIR, mode=0o700, exist_ok=True) key_path = os.path.join(DHCHAP_KEY_DIR, body.name) diff --git a/simplyblock_web/templates/storage_deploy_spdk.yaml.j2 b/simplyblock_web/templates/storage_deploy_spdk.yaml.j2 index b10eb318a..cbf04edd1 100755 --- a/simplyblock_web/templates/storage_deploy_spdk.yaml.j2 +++ b/simplyblock_web/templates/storage_deploy_spdk.yaml.j2 @@ -27,6 +27,10 @@ spec: emptyDir: medium: Memory sizeLimit: 1Gi + - name: spdk-secrets + emptyDir: + medium: Memory + sizeLimit: 16Mi - name: host-sys hostPath: path: /sys @@ -128,6 +132,8 @@ spec: mountPath: /etc/simplyblock - name: var-crash mountPath: /var/crash + - name: spdk-secrets + mountPath: /var/run/secrets/simplyblock resources: limits: hugepages-2Mi: {{ MEM_MEGA }}Mi @@ -154,6 +160,8 @@ spec: mountPath: /mnt/ramdisk - name: tmp-simplyblock mountPath: /var/run/simplyblock + - name: spdk-secrets + mountPath: /var/run/secrets/simplyblock {% if TLS_SERVE or TLS_CONNECT != "disabled" %} - name: tls mountPath: /etc/simplyblock/tls @@ -172,6 +180,8 @@ spec: value: "True" - name: TIMEOUT value: "300" + - name: SPDK_PROXY_KEY_DIR + value: "/var/run/secrets/simplyblock/keys" - name: SB_TLS_SERVE value: "{{ TLS_SERVE }}" - name: SB_TLS_CONNECT diff --git a/tests/integration/expansion_sim/_rpc_sim.py b/tests/integration/expansion_sim/_rpc_sim.py index 47732430e..4c25cd912 100644 --- a/tests/integration/expansion_sim/_rpc_sim.py +++ b/tests/integration/expansion_sim/_rpc_sim.py @@ -318,6 +318,9 @@ def jc_suspend_compression(self, jm_vuid=None, suspend=False, **_): def keyring_file_add_key(self, *_, **__): return True + def keyring_add_key(self, *_, **__): + return True + # -- Misc --------------------------------------------------------------- def bdev_lvol_create(self, *_, **__): diff --git a/tests/integration/ftt2/mock_cluster.py b/tests/integration/ftt2/mock_cluster.py index 26aab9172..dcc6170f6 100644 --- a/tests/integration/ftt2/mock_cluster.py +++ b/tests/integration/ftt2/mock_cluster.py @@ -522,6 +522,8 @@ def _STATIC_EMPTY_DICT(s, p): return {} 'lvol_crypto_key_create': _lvol_crypto_key_create, 'lvol_crypto_create': _lvol_crypto_create, 'keyring_file_add_key': _keyring_file_add_key, + 'keyring_add_key': _STATIC_TRUE, + 'keyring_file_remove_key': _STATIC_TRUE, # LVol operations 'bdev_lvol_create': _bdev_lvol_create, diff --git a/tests/integration/test_dhchap_e2e.py b/tests/integration/test_dhchap_e2e.py index b05bd4b8d..ecbe55b7a 100644 --- a/tests/integration/test_dhchap_e2e.py +++ b/tests/integration/test_dhchap_e2e.py @@ -2,8 +2,11 @@ """ test_dhchap_e2e.py – end-to-end tests for DHCHAP key registration flow. -Uses a real SNodeAPI (running locally) and a mock SPDK JSON-RPC server. -The test exercises the full path: +Uses a real SNodeAPI (running locally) and a mock SPDK JSON-RPC server. The +mock rejects the virtual keyring_add_key with -32601, impersonating SPDK behind +an OLD spdk-proxy without interception support, so this suite exercises the +deprecated on-disk fallback path (the in-memory keyring_add_key path is covered +by test_spdk_proxy_e2e.py against the real proxy): 1. generate_dhchap_key() produces DHHC-1:01:...: keys 2. SNodeClient.write_key_file() writes key to storage node via SNodeAPI @@ -64,6 +67,14 @@ def do_POST(self): self.wfile.write(payload) def _dispatch(self, method, params): + if method == "keyring_add_key": + # This mock impersonates SPDK reached through an OLD proxy (no + # interception support): the virtual keyring_add_key method is + # forwarded verbatim and rejected with "Method not found", so + # controller helpers fall back to the deprecated write_key_file + + # keyring_file_add_key path. + return {"error": {"code": -32601, "message": "Method not found"}} + if method == "keyring_file_add_key": name = params.get("name") path = params.get("path") diff --git a/tests/integration/test_spdk_proxy_e2e.py b/tests/integration/test_spdk_proxy_e2e.py index df9c95991..74951ed37 100644 --- a/tests/integration/test_spdk_proxy_e2e.py +++ b/tests/integration/test_spdk_proxy_e2e.py @@ -45,13 +45,12 @@ def handle(self): except (ValueError, UnicodeDecodeError): continue - result = self.server.dispatch(req) + resp = self.server.dispatch_full(req) if 'id' not in req: # fire-and-forget break - resp = json.dumps({"jsonrpc": "2.0", "id": req["id"], "result": result}) - self.request.sendall(resp.encode('ascii')) + self.request.sendall(json.dumps(resp).encode('ascii')) break @@ -65,15 +64,41 @@ def __init__(self, sock_path, ready=True, delay=0): self._ready = ready self._delay = delay self._call_log = [] + self._param_log = [] + self._keyring = {} # name -> path, mimicking SPDK's keyring state self._lock = threading.Lock() super().__init__(sock_path, MockSPDKHandler) - def dispatch(self, req): + def dispatch_full(self, req): + """Return a complete JSON-RPC response dict (result or error).""" method = req.get("method", "") + params = req.get("params") or {} + req_id = req.get("id") with self._lock: self._call_log.append(method) + self._param_log.append((method, params)) if self._delay > 0: time.sleep(self._delay) + + if method == "keyring_file_add_key": + name = params.get("name") + if name in self._keyring: + return {"jsonrpc": "2.0", "id": req_id, + "error": {"code": -17, "message": "File exists"}} + self._keyring[name] = params.get("path") + return {"jsonrpc": "2.0", "id": req_id, "result": True} + if method == "keyring_file_remove_key": + name = params.get("name") + if name not in self._keyring: + return {"jsonrpc": "2.0", "id": req_id, + "error": {"code": -2, "message": "No such file or directory"}} + del self._keyring[name] + return {"jsonrpc": "2.0", "id": req_id, "result": True} + + return {"jsonrpc": "2.0", "id": req_id, "result": self.dispatch(req)} + + def dispatch(self, req): + method = req.get("method", "") if method == "spdk_get_version": return {"version": "24.01", "fields": {}} if method == "bdev_get_bdevs": @@ -87,6 +112,16 @@ def call_log(self): with self._lock: return list(self._call_log) + @property + def param_log(self): + with self._lock: + return list(self._param_log) + + @property + def keyring(self): + with self._lock: + return dict(self._keyring) + def _start_mock_spdk(sock_path, **kwargs): """Start a mock SPDK server in a background thread.""" @@ -264,6 +299,157 @@ def test_session_count_returns_to_baseline(self): self.assertEqual(len(self._mod.ServerHandler.server_session), 0) +class TestProxyKeyringInterception(unittest.TestCase): + """End-to-end tests for the keyring_add_key interception: full HTTP + round-trip through the real proxy against a mock SPDK unix socket.""" + + KEY_MATERIAL = "DHHC-1:00:c2VjcmV0LWtleS1tYXRlcmlhbA==:" + + @classmethod + def setUpClass(cls): + cls._tmpdir = tempfile.mkdtemp() + cls._sock_path = os.path.join(cls._tmpdir, "spdk_keyring.sock") + cls._key_dir = os.path.join(cls._tmpdir, "keys") + cls._http_port = 18201 + + cls._spdk_server = _start_mock_spdk(cls._sock_path) + time.sleep(0.1) + + cls._proxy_thread, cls._stop_event, cls._mod = _start_proxy( + cls._sock_path, cls._http_port, max_concurrent=4, timeout=5) + cls._mod.SPDK_PROXY_KEY_DIR = cls._key_dir + + for _ in range(30): + try: + r = requests.post( + f"http://127.0.0.1:{cls._http_port}/", + data=json.dumps({"id": 1, "method": "spdk_get_version"}), + auth=("test", "test"), + timeout=2, + ) + if r.status_code == 200: + break + except requests.ConnectionError: + pass # proxy may still be starting up + time.sleep(0.2) + + @classmethod + def tearDownClass(cls): + cls._stop_event.set() + cls._spdk_server.shutdown() + cls._spdk_server.server_close() + import shutil + shutil.rmtree(cls._tmpdir, ignore_errors=True) + + def setUp(self): + self._spdk_server._keyring.clear() + self._spdk_server._call_log.clear() + self._spdk_server._param_log.clear() + import shutil + shutil.rmtree(self._key_dir, ignore_errors=True) + + def _post(self, method, params=None, req_id=1): + payload = {"id": req_id, "method": method} + if params: + payload["params"] = params + return requests.post( + f"http://127.0.0.1:{self._http_port}/", + data=json.dumps(payload), + auth=("test", "test"), + timeout=5, + ) + + def test_keyring_add_key_roundtrip(self): + r = self._post("keyring_add_key", {"name": "key_e2e", "key": self.KEY_MATERIAL}) + self.assertEqual(r.status_code, 200) + self.assertTrue(r.json().get("result")) + + # File written into the proxy key dir with the exact material + key_path = os.path.join(self._key_dir, "key_e2e") + self.assertTrue(os.path.isfile(key_path)) + with open(key_path) as f: + self.assertEqual(f.read(), self.KEY_MATERIAL) + self.assertEqual(os.stat(key_path).st_mode & 0o777, 0o600) + + # SPDK saw the rewritten call: path instead of key material + adds = [p for m, p in self._spdk_server.param_log if m == "keyring_file_add_key"] + self.assertEqual(adds, [{"name": "key_e2e", "path": key_path}]) + self.assertEqual(self._spdk_server.keyring, {"key_e2e": key_path}) + for _, params in self._spdk_server.param_log: + self.assertNotIn(self.KEY_MATERIAL, json.dumps(params)) + + def test_duplicate_add_same_material_returns_eexist(self): + self._post("keyring_add_key", {"name": "key_dup", "key": self.KEY_MATERIAL}) + r = self._post("keyring_add_key", {"name": "key_dup", "key": self.KEY_MATERIAL}) + self.assertEqual(r.json()["error"]["code"], -17) + # File kept: the key is still valid and registered + self.assertTrue(os.path.isfile(os.path.join(self._key_dir, "key_dup"))) + + def test_duplicate_add_different_material_is_rejected(self): + self._post("keyring_add_key", {"name": "key_conflict", "key": self.KEY_MATERIAL}) + r = self._post("keyring_add_key", + {"name": "key_conflict", "key": "DHHC-1:00:b3RoZXI=:"}) + self.assertEqual(r.json()["error"]["code"], -32000) + with open(os.path.join(self._key_dir, "key_conflict")) as f: + self.assertEqual(f.read(), self.KEY_MATERIAL) + + def test_remove_deletes_file(self): + self._post("keyring_add_key", {"name": "key_rm", "key": self.KEY_MATERIAL}) + key_path = os.path.join(self._key_dir, "key_rm") + self.assertTrue(os.path.isfile(key_path)) + + r = self._post("keyring_file_remove_key", {"name": "key_rm"}) + self.assertTrue(r.json().get("result")) + self.assertFalse(os.path.exists(key_path)) + self.assertEqual(self._spdk_server.keyring, {}) + + def test_capabilities_answered_locally(self): + r = self._post("proxy_get_capabilities", req_id=42) + data = r.json() + self.assertEqual(data["id"], 42) + self.assertIn("keyring_add_key", data["result"]["interceptors"]) + self.assertNotIn("proxy_get_capabilities", self._spdk_server.call_log) + + def test_register_dhchap_keys_on_node_uses_interception(self): + """The controller helper takes the keyring_add_key path end-to-end: + RPCClient -> real proxy -> interceptor -> mock SPDK, with no SNodeAPI.""" + from pydantic import SecretStr + + from simplyblock_core.controllers.host_auth import _register_dhchap_keys_on_node + from simplyblock_core.models.storage_node import StorageNode + from simplyblock_core.rpc_client import RPCClient + from simplyblock_core.utils import generate_dhchap_key + + snode = StorageNode() + snode.uuid = "node-keyring-e2e" + # Unreachable SNodeAPI: proves the fallback path is not taken. + snode.api_endpoint = "127.0.0.1:1" + + rpc_client = RPCClient("127.0.0.1", self._http_port, "test", SecretStr("test")) + + host_nqn = "nqn.2014-08.org.nvmexpress:uuid:proxy-e2e-host" + host_entry = { + "nqn": host_nqn, + "dhchap_key": generate_dhchap_key(), + "dhchap_ctrlr_key": generate_dhchap_key(), + } + + key_names = _register_dhchap_keys_on_node(snode, host_nqn, host_entry, rpc_client) + + self.assertIn("dhchap_key", key_names) + self.assertIn("dhchap_ctrlr_key", key_names) + for key_type, key_name in key_names.items(): + key_path = os.path.join(self._key_dir, key_name) + self.assertEqual(self._spdk_server.keyring[key_name], key_path) + with open(key_path) as f: + self.assertEqual(f.read(), host_entry[key_type]) + # Key material never reached SPDK's params + for _, params in self._spdk_server.param_log: + payload = json.dumps(params) + self.assertNotIn(host_entry["dhchap_key"], payload) + self.assertNotIn(host_entry["dhchap_ctrlr_key"], payload) + + class TestProxyReadinessGate(unittest.TestCase): """Test that the proxy blocks until SPDK is ready.""" diff --git a/tests/unit/test_client_secret_logging.py b/tests/unit/test_client_secret_logging.py index 68ade107f..304bfc7a9 100644 --- a/tests/unit/test_client_secret_logging.py +++ b/tests/unit/test_client_secret_logging.py @@ -58,6 +58,23 @@ def test_rpc_client_request_body_carries_unwrapped_param(rpc_client, caplog): assert "ctor-secret" not in _captured_logs_text(caplog) +def test_rpc_client_keyring_add_key_wire_unwraps_and_logs_mask(rpc_client, caplog): + rpc_client._fake_session.post.return_value = _make_json_response({ + "jsonrpc": "2.0", "id": 1, "result": True, + }) + + with caplog.at_level(logging.DEBUG): + rpc_client.keyring_add_key("key_host_1", SecretStr("DHHC-1:00:S0VZVkFMVUU=:")) + + posted_body = rpc_client._fake_session.post.call_args.kwargs["data"] + parsed = json.loads(posted_body) + assert parsed["method"] == "keyring_add_key" + assert parsed["params"] == {"name": "key_host_1", "key": "DHHC-1:00:S0VZVkFMVUU=:"} + + assert "DHHC-1:00:S0VZVkFMVUU=:" not in _captured_logs_text(caplog) + assert "ctor-secret" not in _captured_logs_text(caplog) + + def test_rpc_client_response_body_hidden_by_default(rpc_client, caplog): rpc_client._fake_session.post.return_value = _make_json_response({ "jsonrpc": "2.0", "id": 1, "result": {"sensitive": "RESPVALUE"}, diff --git a/tests/unit/test_host_auth_key_registration.py b/tests/unit/test_host_auth_key_registration.py new file mode 100644 index 000000000..e9b393b66 --- /dev/null +++ b/tests/unit/test_host_auth_key_registration.py @@ -0,0 +1,75 @@ +# coding=utf-8 +"""Unit tests for host_auth._register_key_on_node key-delivery selection. + +The control plane prefers the spdk-proxy's keyring_add_key interceptor and only +falls back to the deprecated write_key_file + keyring_file_add_key path when the +proxy predates interception support — signalled by SPDK rejecting the forwarded +keyring_add_key with -32601 ("Method not found"). +""" + +from unittest.mock import MagicMock + +from pydantic import SecretStr + +from simplyblock_core.controllers.host_auth import _register_key_on_node +from simplyblock_core.rpc_client import RPCException + +KEY = SecretStr("DHHC-1:00:c2VjcmV0LWtleS1tYXRlcmlhbA==:") + + +def _snode(): + snode = MagicMock() + snode.get_id.return_value = "node-1" + return snode + + +def test_uses_interceptor_when_supported(): + """keyring_add_key succeeds → key registered inline, no on-disk fallback.""" + snode = _snode() + rpc = MagicMock() + rpc.keyring_add_key.return_value = None + + assert _register_key_on_node(snode, rpc, "key_1", KEY) is True + + rpc.keyring_add_key.assert_called_once_with("key_1", KEY, allow_existing=True) + snode.client.assert_not_called() + rpc.keyring_file_add_key.assert_not_called() + + +def test_falls_back_on_method_not_found(): + """-32601 → old proxy: fall back to write_key_file + keyring_file_add_key.""" + snode = _snode() + snode.client.return_value.write_key_file.return_value = ("/keys/key_1", None) + rpc = MagicMock() + rpc.keyring_add_key.side_effect = RPCException("Method not found", code=-32601) + rpc.keyring_file_add_key.return_value = None + + assert _register_key_on_node(snode, rpc, "key_1", KEY) is True + + snode.client.return_value.write_key_file.assert_called_once_with("key_1", KEY) + rpc.keyring_file_add_key.assert_called_once_with( + "key_1", "/keys/key_1", allow_existing=True) + + +def test_no_fallback_on_other_rpc_error(): + """A non -32601 error is a real failure: no fallback, returns False.""" + snode = _snode() + rpc = MagicMock() + rpc.keyring_add_key.side_effect = RPCException("bad params", code=-32602) + + assert _register_key_on_node(snode, rpc, "key_1", KEY) is False + + snode.client.assert_not_called() + rpc.keyring_file_add_key.assert_not_called() + + +def test_fallback_write_key_file_error_returns_false(): + """When the fallback write_key_file reports an error, registration fails.""" + snode = _snode() + snode.client.return_value.write_key_file.return_value = (None, "disk full") + rpc = MagicMock() + rpc.keyring_add_key.side_effect = RPCException("Method not found", code=-32601) + + assert _register_key_on_node(snode, rpc, "key_1", KEY) is False + + rpc.keyring_file_add_key.assert_not_called() diff --git a/tests/unit/test_spdk_proxy_interceptors.py b/tests/unit/test_spdk_proxy_interceptors.py new file mode 100644 index 000000000..c9566ef5f --- /dev/null +++ b/tests/unit/test_spdk_proxy_interceptors.py @@ -0,0 +1,276 @@ +# coding=utf-8 +"""Unit tests for the spdk_http_proxy_server RPC interceptors. + +The keyring_add_key virtual method must write key material to the (memory- +backed) key dir and forward a rewritten keyring_file_add_key to SPDK without +the material ever reaching the forwarded payload or the logs. +""" + +import json +import os +import stat +from unittest.mock import patch + +import pytest + +from tests.conftest_proxy import import_proxy_module + +proxy_mod = import_proxy_module() + +KEY_MATERIAL = "DHHC-1:00:c2VjcmV0LWtleS1tYXRlcmlhbA==:" + + +@pytest.fixture +def key_dir(tmp_path): + d = tmp_path / "keys" + with patch.object(proxy_mod, "SPDK_PROXY_KEY_DIR", str(d)): + yield d + + +def _ok_response(req_id=7): + return json.dumps({"jsonrpc": "2.0", "id": req_id, "result": True}) + + +def _error_response(code, req_id=7): + return json.dumps({"jsonrpc": "2.0", "id": req_id, "error": {"code": code, "message": "err"}}) + + +def _add_request(name="key_1", key=KEY_MATERIAL, req_id=7): + return {"id": req_id, "method": "keyring_add_key", "params": {"name": name, "key": key}} + + +def _remove_request(name="key_1", req_id=7): + return {"id": req_id, "method": "keyring_file_remove_key", "params": {"name": name}} + + +class TestKeyringAddKey: + + def test_happy_path_rewrites_and_writes_file(self, key_dir): + with patch.object(proxy_mod, "rpc_call", return_value=_ok_response()) as mock_rpc: + response = proxy_mod._handle_keyring_add_key(_add_request(), None) + + forwarded = json.loads(mock_rpc.call_args[0][0].decode("ascii")) + key_path = key_dir / "key_1" + assert forwarded["method"] == "keyring_file_add_key" + assert forwarded["id"] == 7 + assert forwarded["params"] == {"name": "key_1", "path": str(key_path)} + assert KEY_MATERIAL not in json.dumps(forwarded) + + assert key_path.read_text() == KEY_MATERIAL + assert stat.S_IMODE(os.stat(key_path).st_mode) == 0o600 + assert stat.S_IMODE(os.stat(key_dir).st_mode) == 0o700 + assert json.loads(response)["result"] is True + + @pytest.mark.parametrize("name", ["../evil", "a/b", "a b", "", None, 5]) + def test_invalid_name_rejected(self, key_dir, name): + with patch.object(proxy_mod, "rpc_call") as mock_rpc: + response = proxy_mod._handle_keyring_add_key(_add_request(name=name), None) + + assert json.loads(response)["error"]["code"] == -32602 + mock_rpc.assert_not_called() + assert not key_dir.exists() + + @pytest.mark.parametrize("key", ["", None, 5, "ключ"]) + def test_invalid_key_material_rejected(self, key_dir, key): + with patch.object(proxy_mod, "rpc_call") as mock_rpc: + response = proxy_mod._handle_keyring_add_key(_add_request(key=key), None) + + assert json.loads(response)["error"]["code"] == -32602 + mock_rpc.assert_not_called() + assert not key_dir.exists() + + def test_existing_file_different_content_is_error(self, key_dir): + key_dir.mkdir(mode=0o700) + (key_dir / "key_1").write_text("DHHC-1:00:b3RoZXI=:") + + with patch.object(proxy_mod, "rpc_call") as mock_rpc: + response = proxy_mod._handle_keyring_add_key(_add_request(), None) + + assert json.loads(response)["error"]["code"] == -32000 + mock_rpc.assert_not_called() + assert (key_dir / "key_1").read_text() == "DHHC-1:00:b3RoZXI=:" + + def test_existing_file_matching_content_spdk_eexist_passes_through(self, key_dir): + key_dir.mkdir(mode=0o700) + (key_dir / "key_1").write_text(KEY_MATERIAL) + + with patch.object(proxy_mod, "rpc_call", return_value=_error_response(-17)): + response = proxy_mod._handle_keyring_add_key(_add_request(), None) + + assert json.loads(response)["error"]["code"] == -17 + assert (key_dir / "key_1").read_text() == KEY_MATERIAL + + def test_existing_file_matching_content_spdk_success_passes_through(self, key_dir): + # SPDK restarted while the tmpfs survived: keyring empty, file present. + key_dir.mkdir(mode=0o700) + (key_dir / "key_1").write_text(KEY_MATERIAL) + + with patch.object(proxy_mod, "rpc_call", return_value=_ok_response()): + response = proxy_mod._handle_keyring_add_key(_add_request(), None) + + assert json.loads(response)["result"] is True + assert (key_dir / "key_1").read_text() == KEY_MATERIAL + + def test_fresh_write_spdk_eexist_keeps_file_and_passes_through(self, key_dir): + # SPDK already has the key registered; the freshly written backing file + # must stay (the client treats -17 as reuse), and -17 passes through. + with patch.object(proxy_mod, "rpc_call", return_value=_error_response(-17)): + response = proxy_mod._handle_keyring_add_key(_add_request(), None) + + assert json.loads(response)["error"]["code"] == -17 + assert (key_dir / "key_1").read_text() == KEY_MATERIAL + + def test_fresh_write_spdk_error_deletes_file(self, key_dir): + with patch.object(proxy_mod, "rpc_call", return_value=_error_response(-22)): + response = proxy_mod._handle_keyring_add_key(_add_request(), None) + + assert json.loads(response)["error"]["code"] == -22 + assert not (key_dir / "key_1").exists() + + def test_fresh_write_rpc_call_raises_deletes_file(self, key_dir): + # The forward itself failed (e.g. SPDK timeout raises ValueError): the + # freshly written secret must not linger, and the error propagates. + with patch.object(proxy_mod, "rpc_call", side_effect=ValueError("SPDK response timeout")): + with pytest.raises(ValueError): + proxy_mod._handle_keyring_add_key(_add_request(), None) + + assert not (key_dir / "key_1").exists() + + def test_existing_file_spdk_error_keeps_file(self, key_dir): + key_dir.mkdir(mode=0o700) + (key_dir / "key_1").write_text(KEY_MATERIAL) + + with patch.object(proxy_mod, "rpc_call", return_value=_error_response(-22)): + response = proxy_mod._handle_keyring_add_key(_add_request(), None) + + assert json.loads(response)["error"]["code"] == -22 + assert (key_dir / "key_1").read_text() == KEY_MATERIAL + + def test_existing_file_rpc_call_raises_keeps_file(self, key_dir): + # We did not write the pre-existing file, so a failing forward must not + # remove it. + key_dir.mkdir(mode=0o700) + (key_dir / "key_1").write_text(KEY_MATERIAL) + + with patch.object(proxy_mod, "rpc_call", side_effect=ValueError("boom")): + with pytest.raises(ValueError): + proxy_mod._handle_keyring_add_key(_add_request(), None) + + assert (key_dir / "key_1").read_text() == KEY_MATERIAL + + +class TestKeyringFileRemoveKey: + + def test_removal_deletes_file(self, key_dir): + key_dir.mkdir(mode=0o700) + (key_dir / "key_1").write_text(KEY_MATERIAL) + + with patch.object(proxy_mod, "rpc_call", return_value=_ok_response()) as mock_rpc: + response = proxy_mod._handle_keyring_file_remove_key(_remove_request(), None) + + forwarded = json.loads(mock_rpc.call_args[0][0].decode("ascii")) + assert forwarded == _remove_request() + assert json.loads(response)["result"] is True + assert not (key_dir / "key_1").exists() + + def test_key_unknown_to_spdk_still_deletes_file(self, key_dir): + key_dir.mkdir(mode=0o700) + (key_dir / "key_1").write_text(KEY_MATERIAL) + + with patch.object(proxy_mod, "rpc_call", return_value=_error_response(-2)): + proxy_mod._handle_keyring_file_remove_key(_remove_request(), None) + + assert not (key_dir / "key_1").exists() + + def test_other_spdk_error_keeps_file(self, key_dir): + key_dir.mkdir(mode=0o700) + (key_dir / "key_1").write_text(KEY_MATERIAL) + + with patch.object(proxy_mod, "rpc_call", return_value=_error_response(-22)): + response = proxy_mod._handle_keyring_file_remove_key(_remove_request(), None) + + assert json.loads(response)["error"]["code"] == -22 + assert (key_dir / "key_1").read_text() == KEY_MATERIAL + + def test_missing_file_is_tolerated(self, key_dir): + with patch.object(proxy_mod, "rpc_call", return_value=_ok_response()): + response = proxy_mod._handle_keyring_file_remove_key(_remove_request(), None) + + assert json.loads(response)["result"] is True + + def test_invalid_name_still_forwarded(self, key_dir): + with patch.object(proxy_mod, "rpc_call", return_value=_error_response(-2)) as mock_rpc: + proxy_mod._handle_keyring_file_remove_key(_remove_request(name="../evil"), None) + + mock_rpc.assert_called_once() + + +class TestProxyGetCapabilities: + + def test_answered_locally(self): + with patch.object(proxy_mod, "rpc_call") as mock_rpc: + response = proxy_mod._handle_proxy_get_capabilities({"id": 3, "method": "proxy_get_capabilities"}, None) + + mock_rpc.assert_not_called() + parsed = json.loads(response) + assert parsed["id"] == 3 + assert set(parsed["result"]["interceptors"]) == { + "keyring_add_key", "keyring_file_remove_key", "proxy_get_capabilities", + } + + +class TestDispatchRpc: + + def test_unknown_method_passes_through_unmodified(self): + raw = json.dumps({"id": 1, "method": "bdev_get_bdevs"}).encode("ascii") + with patch.object(proxy_mod, "rpc_call", return_value=_ok_response(1)) as mock_rpc: + proxy_mod.dispatch_rpc(raw, "5") + + mock_rpc.assert_called_once_with(raw, "5") + + def test_raw_keyring_file_add_key_passes_through(self, key_dir): + raw = json.dumps({"id": 1, "method": "keyring_file_add_key", + "params": {"name": "k", "path": "/etc/simplyblock/dhchap_keys/k"}}).encode("ascii") + with patch.object(proxy_mod, "rpc_call", return_value=_ok_response(1)) as mock_rpc: + proxy_mod.dispatch_rpc(raw, None) + + mock_rpc.assert_called_once_with(raw, None) + + def test_malformed_json_passes_through(self): + raw = b"{not json" + with patch.object(proxy_mod, "rpc_call", side_effect=ValueError("bad")) as mock_rpc: + with pytest.raises(ValueError): + proxy_mod.dispatch_rpc(raw, None) + + mock_rpc.assert_called_once_with(raw, None) + + def test_routes_keyring_add_key(self, key_dir): + raw = json.dumps(_add_request()).encode("ascii") + with patch.object(proxy_mod, "rpc_call", return_value=_ok_response()): + proxy_mod.dispatch_rpc(raw, None) + + assert (key_dir / "key_1").read_text() == KEY_MATERIAL + + def test_oserror_becomes_jsonrpc_error(self, tmp_path): + blocker = tmp_path / "not-a-dir" + blocker.write_text("") + with patch.object(proxy_mod, "SPDK_PROXY_KEY_DIR", str(blocker / "keys")): + with patch.object(proxy_mod, "rpc_call") as mock_rpc: + response = proxy_mod.dispatch_rpc(json.dumps(_add_request()).encode("ascii"), None) + + mock_rpc.assert_not_called() + assert json.loads(response)["error"]["code"] == -32000 + + def test_key_material_never_logged_or_forwarded(self, key_dir, caplog): + forwarded_payloads = [] + + def capture(req, client_timeout=None): + forwarded_payloads.append(req.decode("ascii")) + return _ok_response() + + with caplog.at_level("DEBUG"): + with patch.object(proxy_mod, "rpc_call", side_effect=capture): + proxy_mod.dispatch_rpc(json.dumps(_add_request()).encode("ascii"), None) + + assert forwarded_payloads and all(KEY_MATERIAL not in p for p in forwarded_payloads) + assert KEY_MATERIAL not in caplog.text