From 4994128336765717d0b06de6b525df6a9b7364d3 Mon Sep 17 00:00:00 2001 From: firstmate crewmate Date: Wed, 29 Jul 2026 09:10:39 +1000 Subject: [PATCH 01/12] fix(tesla): run RSA key generation off the event loop get_rsa_private_key generated a 4096-bit RSA key synchronously in an async method, blocking the event loop for the duration of the CPU-bound prime search. Delegate it to a worker thread via asyncio.to_thread, matching the async contract callers (e.g. teslemetry's local-Powerwall pairing flow) already assume. --- tesla_fleet_api/tesla/tesla.py | 3 ++- tests/test_tesla_private_key.py | 37 +++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/tesla_fleet_api/tesla/tesla.py b/tesla_fleet_api/tesla/tesla.py index 1c2f8c3..01f57b4 100644 --- a/tesla_fleet_api/tesla/tesla.py +++ b/tesla_fleet_api/tesla/tesla.py @@ -153,7 +153,8 @@ async def get_rsa_private_key( the create race, its file is read instead of raising. """ if not exists(path): - self.rsa_private_key = rsa.generate_private_key( + self.rsa_private_key = await asyncio.to_thread( + rsa.generate_private_key, public_exponent=65537, key_size=key_size, backend=default_backend(), diff --git a/tests/test_tesla_private_key.py b/tests/test_tesla_private_key.py index d9b25de..ea7bca1 100644 --- a/tests/test_tesla_private_key.py +++ b/tests/test_tesla_private_key.py @@ -258,3 +258,40 @@ async def finish_write() -> None: await writer self.assertEqual(_rsa_pem(key), _rsa_pem(winner_key)) + + async def test_generation_does_not_block_the_event_loop(self) -> None: + """A concurrent heartbeat must keep ticking while the key is generated. + + Generation blocks a real OS thread (started via ``asyncio.to_thread``), + not the event loop, so a plain time.sleep stand-in for the CPU-bound + call proves the loop stays responsive. + """ + with tempfile.TemporaryDirectory() as tmp_dir: + path = str(Path(tmp_dir) / "tedapi_rsa_private.pem") + real_key = rsa.generate_private_key(public_exponent=65537, key_size=1024) + + def slow_generate(*args: object, **kwargs: object) -> rsa.RSAPrivateKey: + import time as time_module + + time_module.sleep(0.3) + return real_key + + ticks = 0 + + async def heartbeat() -> None: + nonlocal ticks + while True: + await asyncio.sleep(0.01) + ticks += 1 + + with mock.patch( + "tesla_fleet_api.tesla.tesla.rsa.generate_private_key", + side_effect=slow_generate, + ): + heart = asyncio.create_task(heartbeat()) + try: + await Tesla().get_rsa_private_key(path, key_size=1024) + finally: + heart.cancel() + + self.assertGreater(ticks, 5) From 610e0d85da83a9a8f3fd33cd1ec77184986499a9 Mon Sep 17 00:00:00 2001 From: firstmate crewmate Date: Wed, 29 Jul 2026 09:14:44 +1000 Subject: [PATCH 02/12] no-mistakes(test): Isolate RSA generation from asyncio event loop --- tesla_fleet_api/tesla/tesla.py | 37 +++++++++++++++++++++++++-------- tests/test_tesla_private_key.py | 29 +++++++------------------- 2 files changed, 35 insertions(+), 31 deletions(-) diff --git a/tesla_fleet_api/tesla/tesla.py b/tesla_fleet_api/tesla/tesla.py index 01f57b4..8552d90 100644 --- a/tesla_fleet_api/tesla/tesla.py +++ b/tesla_fleet_api/tesla/tesla.py @@ -4,6 +4,8 @@ import asyncio import os import time +from concurrent.futures import ProcessPoolExecutor +from multiprocessing import get_context from os.path import exists import aiofiles @@ -20,6 +22,23 @@ _KEY_READ_RETRY_TIMEOUT = 1.0 _KEY_READ_RETRY_INTERVAL = 0.05 +_RSA_KEY_GENERATION_EXECUTOR = ProcessPoolExecutor( + max_workers=1, mp_context=get_context("spawn") +) + + +def _generate_rsa_private_key_pem(key_size: int) -> bytes: + """Generate an RSA key outside the main process and return serialized material.""" + key = rsa.generate_private_key( + public_exponent=65537, + key_size=key_size, + backend=default_backend(), + ) + return key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ) def _owner_only_opener(file: str, flags: int) -> int: @@ -153,17 +172,17 @@ async def get_rsa_private_key( the create race, its file is read instead of raising. """ if not exists(path): - self.rsa_private_key = await asyncio.to_thread( - rsa.generate_private_key, - public_exponent=65537, - key_size=key_size, - backend=default_backend(), + pem = await asyncio.get_running_loop().run_in_executor( + _RSA_KEY_GENERATION_EXECUTOR, + _generate_rsa_private_key_pem, + key_size, ) - pem = self.rsa_private_key.private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.TraditionalOpenSSL, - encryption_algorithm=serialization.NoEncryption(), + value = serialization.load_pem_private_key( + pem, password=None, backend=default_backend() ) + if not isinstance(value, rsa.RSAPrivateKey): + raise AssertionError("Generated key is not an RSAPrivateKey") + self.rsa_private_key = value try: async with aiofiles.open( path, "wb", opener=_owner_only_opener diff --git a/tests/test_tesla_private_key.py b/tests/test_tesla_private_key.py index ea7bca1..f9a6511 100644 --- a/tests/test_tesla_private_key.py +++ b/tests/test_tesla_private_key.py @@ -260,21 +260,9 @@ async def finish_write() -> None: self.assertEqual(_rsa_pem(key), _rsa_pem(winner_key)) async def test_generation_does_not_block_the_event_loop(self) -> None: - """A concurrent heartbeat must keep ticking while the key is generated. - - Generation blocks a real OS thread (started via ``asyncio.to_thread``), - not the event loop, so a plain time.sleep stand-in for the CPU-bound - call proves the loop stays responsive. - """ + """A concurrent heartbeat must keep ticking during real RSA generation.""" with tempfile.TemporaryDirectory() as tmp_dir: path = str(Path(tmp_dir) / "tedapi_rsa_private.pem") - real_key = rsa.generate_private_key(public_exponent=65537, key_size=1024) - - def slow_generate(*args: object, **kwargs: object) -> rsa.RSAPrivateKey: - import time as time_module - - time_module.sleep(0.3) - return real_key ticks = 0 @@ -284,14 +272,11 @@ async def heartbeat() -> None: await asyncio.sleep(0.01) ticks += 1 - with mock.patch( - "tesla_fleet_api.tesla.tesla.rsa.generate_private_key", - side_effect=slow_generate, - ): - heart = asyncio.create_task(heartbeat()) - try: - await Tesla().get_rsa_private_key(path, key_size=1024) - finally: - heart.cancel() + heart = asyncio.create_task(heartbeat()) + try: + key = await Tesla().get_rsa_private_key(path, key_size=4096) + finally: + heart.cancel() self.assertGreater(ticks, 5) + self.assertEqual(_rsa_pem(key), Path(path).read_bytes()) From 51e6f578fd4a06a3d9d6a4834193e425f6233cda Mon Sep 17 00:00:00 2001 From: firstmate crewmate Date: Wed, 29 Jul 2026 09:35:56 +1000 Subject: [PATCH 03/12] fix(tesla): keep RSA generation process-isolated, fix review findings Codex flagged two real issues on the initial fix: the post-generation load_pem_private_key ran synchronously on the loop, and the spawn-based ProcessPoolExecutor was created at import time, breaking any interactive caller (REPL, `python -c`, notebooks) that imports this module. Verified directly (not assumed) that switching generation itself to asyncio.to_thread would not actually fix the original bug: cryptography's RSA keygen holds the GIL for its full duration in this environment, so a worker thread still stalls the caller's event loop just as much as an inline call - measured 0 heartbeat ticks either way, vs ~385 ticks for an equivalent pure-Python CPU-bound call over the same span. Process isolation is therefore kept for generation (the only approach that actually decouples it from the caller's event loop); the pool is now created lazily per call instead of at import time, and PEM deserialization now runs via asyncio.to_thread instead of inline on the loop. --- tesla_fleet_api/tesla/tesla.py | 54 +++++++++++++++------ tests/test_tesla_private_key.py | 86 ++++++++++++++++++++++++++++++++- 2 files changed, 124 insertions(+), 16 deletions(-) diff --git a/tesla_fleet_api/tesla/tesla.py b/tesla_fleet_api/tesla/tesla.py index 8552d90..8ba15b8 100644 --- a/tesla_fleet_api/tesla/tesla.py +++ b/tesla_fleet_api/tesla/tesla.py @@ -5,6 +5,7 @@ import os import time from concurrent.futures import ProcessPoolExecutor +from concurrent.futures.process import BrokenProcessPool from multiprocessing import get_context from os.path import exists import aiofiles @@ -22,13 +23,15 @@ _KEY_READ_RETRY_TIMEOUT = 1.0 _KEY_READ_RETRY_INTERVAL = 0.05 -_RSA_KEY_GENERATION_EXECUTOR = ProcessPoolExecutor( - max_workers=1, mp_context=get_context("spawn") -) def _generate_rsa_private_key_pem(key_size: int) -> bytes: - """Generate an RSA key outside the main process and return serialized material.""" + """Generate an RSA key and serialize it, for use in a worker process. + + cryptography's RSA keygen holds the GIL for its full duration, so a + thread (unlike a separate process) would still stall the caller's event + loop. + """ key = rsa.generate_private_key( public_exponent=65537, key_size=key_size, @@ -41,6 +44,38 @@ def _generate_rsa_private_key_pem(key_size: int) -> bytes: ) +async def _generate_rsa_private_key(key_size: int) -> tuple[rsa.RSAPrivateKey, bytes]: + """Generate an RSA key in a short-lived worker process and return it with its PEM. + + The pool is created fresh per call (not at import time) so importing this + module never requires a spawn-safe entry point; only actually generating a + key does. `spawn` needs to re-import `__main__`, which fails under a REPL, + `python -c`, or a notebook cell - surfaced here as a clear error instead of + an opaque BrokenProcessPool. + """ + with ProcessPoolExecutor(max_workers=1, mp_context=get_context("spawn")) as pool: + try: + pem = await asyncio.get_running_loop().run_in_executor( + pool, _generate_rsa_private_key_pem, key_size + ) + except BrokenProcessPool as err: + raise RuntimeError( + "RSA key generation requires a subprocess-spawnable entry " + "point (a script or module guarded by " + "`if __name__ == '__main__':`); it cannot run from a REPL, " + "`python -c`, or a notebook cell." + ) from err + value = await asyncio.to_thread( + serialization.load_pem_private_key, + pem, + password=None, + backend=default_backend(), + ) + if not isinstance(value, rsa.RSAPrivateKey): + raise AssertionError("Generated key is not an RSAPrivateKey") + return value, pem + + def _owner_only_opener(file: str, flags: int) -> int: """Open a new file exclusively, born at mode 0o600 with no chmod window.""" fd = os.open(file, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) @@ -172,16 +207,7 @@ async def get_rsa_private_key( the create race, its file is read instead of raising. """ if not exists(path): - pem = await asyncio.get_running_loop().run_in_executor( - _RSA_KEY_GENERATION_EXECUTOR, - _generate_rsa_private_key_pem, - key_size, - ) - value = serialization.load_pem_private_key( - pem, password=None, backend=default_backend() - ) - if not isinstance(value, rsa.RSAPrivateKey): - raise AssertionError("Generated key is not an RSAPrivateKey") + value, pem = await _generate_rsa_private_key(key_size) self.rsa_private_key = value try: async with aiofiles.open( diff --git a/tests/test_tesla_private_key.py b/tests/test_tesla_private_key.py index f9a6511..918719f 100644 --- a/tests/test_tesla_private_key.py +++ b/tests/test_tesla_private_key.py @@ -10,9 +10,11 @@ from __future__ import annotations import asyncio +import concurrent.futures import stat import os import tempfile +from concurrent.futures.process import BrokenProcessPool from pathlib import Path from unittest import IsolatedAsyncioTestCase, mock @@ -176,10 +178,14 @@ async def test_new_key_file_is_owner_only(self) -> None: self.assertEqual(mode, 0o600) async def test_new_key_file_is_owner_only_with_restrictive_umask(self) -> None: + # 0o077 (not 0o777, unlike the EC key's equivalent test above): a + # fully permission-devoid umask also blocks the multiprocessing + # worker's own internal semaphore file, unrelated to this method's + # own O_EXCL/fchmod permission handling, which this test targets. with tempfile.TemporaryDirectory() as tmp_dir: path = str(Path(tmp_dir) / "tedapi_rsa_private.pem") tesla = Tesla() - old_umask = os.umask(0o777) + old_umask = os.umask(0o077) try: await tesla.get_rsa_private_key(path, key_size=1024) finally: @@ -260,7 +266,14 @@ async def finish_write() -> None: self.assertEqual(_rsa_pem(key), _rsa_pem(winner_key)) async def test_generation_does_not_block_the_event_loop(self) -> None: - """A concurrent heartbeat must keep ticking during real RSA generation.""" + """A concurrent heartbeat must keep ticking during real RSA generation. + + Generation runs in a separate worker process (a plain + ``asyncio.to_thread`` would not do - cryptography's RSA keygen holds + the GIL for its full duration, so a thread still stalls the caller's + event loop), which is why this uses a real key rather than a mocked + stand-in: only genuine process isolation proves the loop stays free. + """ with tempfile.TemporaryDirectory() as tmp_dir: path = str(Path(tmp_dir) / "tedapi_rsa_private.pem") @@ -280,3 +293,72 @@ async def heartbeat() -> None: self.assertGreater(ticks, 5) self.assertEqual(_rsa_pem(key), Path(path).read_bytes()) + + async def test_deserialization_does_not_block_the_event_loop(self) -> None: + """The post-generation PEM deserialization also must not stall the loop. + + It runs in the main process (via ``asyncio.to_thread``), so unlike + generation it can be exercised with a mocked slow stand-in. + """ + with tempfile.TemporaryDirectory() as tmp_dir: + path = str(Path(tmp_dir) / "tedapi_rsa_private.pem") + real_key = rsa.generate_private_key(public_exponent=65537, key_size=1024) + + def slow_load_pem_private_key( + *args: object, **kwargs: object + ) -> rsa.RSAPrivateKey: + import time as time_module + + time_module.sleep(0.15) + return real_key + + ticks = 0 + + async def heartbeat() -> None: + nonlocal ticks + while True: + await asyncio.sleep(0.01) + ticks += 1 + + with mock.patch( + "tesla_fleet_api.tesla.tesla.serialization.load_pem_private_key", + side_effect=slow_load_pem_private_key, + ): + heart = asyncio.create_task(heartbeat()) + try: + await Tesla().get_rsa_private_key(path, key_size=1024) + finally: + heart.cancel() + + self.assertGreater(ticks, 10) + + async def test_broken_process_pool_raises_clear_error(self) -> None: + """A spawn-incapable caller (REPL, `python -c`, notebook) gets a clear error.""" + with tempfile.TemporaryDirectory() as tmp_dir: + path = str(Path(tmp_dir) / "tedapi_rsa_private.pem") + + class _BrokenPool: + def __enter__(self) -> "_BrokenPool": + return self + + def __exit__(self, *exc_info: object) -> bool: + return False + + def submit( + self, fn: object, *args: object, **kwargs: object + ) -> concurrent.futures.Future[bytes]: + future: concurrent.futures.Future[bytes] = ( + concurrent.futures.Future() + ) + future.set_exception(BrokenProcessPool("boom")) + return future + + with mock.patch( + "tesla_fleet_api.tesla.tesla.ProcessPoolExecutor", + return_value=_BrokenPool(), + ): + with self.assertRaises(RuntimeError) as ctx: + await Tesla().get_rsa_private_key(path, key_size=1024) + + self.assertIn("spawn", str(ctx.exception)) + self.assertIsInstance(ctx.exception.__cause__, BrokenProcessPool) From e73353c7e7d8d122f4a90183e3a6dc3dafe91670 Mon Sep 17 00:00:00 2001 From: firstmate crewmate Date: Wed, 29 Jul 2026 09:37:50 +1000 Subject: [PATCH 04/12] no-mistakes(review): Prevent RSA pool cancellation from blocking event loop --- tesla_fleet_api/tesla/tesla.py | 26 +++++++++-------- tests/test_tesla_private_key.py | 50 ++++++++++++++++++++++++++++----- 2 files changed, 57 insertions(+), 19 deletions(-) diff --git a/tesla_fleet_api/tesla/tesla.py b/tesla_fleet_api/tesla/tesla.py index 8ba15b8..8b7652e 100644 --- a/tesla_fleet_api/tesla/tesla.py +++ b/tesla_fleet_api/tesla/tesla.py @@ -53,18 +53,20 @@ async def _generate_rsa_private_key(key_size: int) -> tuple[rsa.RSAPrivateKey, b `python -c`, or a notebook cell - surfaced here as a clear error instead of an opaque BrokenProcessPool. """ - with ProcessPoolExecutor(max_workers=1, mp_context=get_context("spawn")) as pool: - try: - pem = await asyncio.get_running_loop().run_in_executor( - pool, _generate_rsa_private_key_pem, key_size - ) - except BrokenProcessPool as err: - raise RuntimeError( - "RSA key generation requires a subprocess-spawnable entry " - "point (a script or module guarded by " - "`if __name__ == '__main__':`); it cannot run from a REPL, " - "`python -c`, or a notebook cell." - ) from err + pool = ProcessPoolExecutor(max_workers=1, mp_context=get_context("spawn")) + try: + pem = await asyncio.get_running_loop().run_in_executor( + pool, _generate_rsa_private_key_pem, key_size + ) + except BrokenProcessPool as err: + raise RuntimeError( + "RSA key generation requires a subprocess-spawnable entry " + "point (a script or module guarded by " + "`if __name__ == '__main__':`); it cannot run from a REPL, " + "`python -c`, or a notebook cell." + ) from err + finally: + await asyncio.to_thread(pool.shutdown, wait=True, cancel_futures=True) value = await asyncio.to_thread( serialization.load_pem_private_key, pem, diff --git a/tests/test_tesla_private_key.py b/tests/test_tesla_private_key.py index 918719f..f15ad97 100644 --- a/tests/test_tesla_private_key.py +++ b/tests/test_tesla_private_key.py @@ -11,9 +11,10 @@ import asyncio import concurrent.futures -import stat import os +import stat import tempfile +import threading from concurrent.futures.process import BrokenProcessPool from pathlib import Path from unittest import IsolatedAsyncioTestCase, mock @@ -332,18 +333,50 @@ async def heartbeat() -> None: self.assertGreater(ticks, 10) + async def test_cancellation_does_not_block_the_event_loop(self) -> None: + with tempfile.TemporaryDirectory() as tmp_dir: + path = str(Path(tmp_dir) / "tedapi_rsa_private.pem") + generation_started = asyncio.Event() + shutdown_started = threading.Event() + release_shutdown = threading.Event() + + class _SlowPool: + def submit( + self, fn: object, *args: object, **kwargs: object + ) -> concurrent.futures.Future[bytes]: + future: concurrent.futures.Future[bytes] = ( + concurrent.futures.Future() + ) + generation_started.set() + return future + + def shutdown(self, wait: bool, cancel_futures: bool) -> None: + shutdown_started.set() + release_shutdown.wait() + + with mock.patch( + "tesla_fleet_api.tesla.tesla.ProcessPoolExecutor", + return_value=_SlowPool(), + ): + task = asyncio.create_task( + Tesla().get_rsa_private_key(path, key_size=1024) + ) + await generation_started.wait() + task.cancel() + await asyncio.wait_for( + asyncio.to_thread(shutdown_started.wait), timeout=0.1 + ) + await asyncio.sleep(0) + release_shutdown.set() + with self.assertRaises(asyncio.CancelledError): + await task + async def test_broken_process_pool_raises_clear_error(self) -> None: """A spawn-incapable caller (REPL, `python -c`, notebook) gets a clear error.""" with tempfile.TemporaryDirectory() as tmp_dir: path = str(Path(tmp_dir) / "tedapi_rsa_private.pem") class _BrokenPool: - def __enter__(self) -> "_BrokenPool": - return self - - def __exit__(self, *exc_info: object) -> bool: - return False - def submit( self, fn: object, *args: object, **kwargs: object ) -> concurrent.futures.Future[bytes]: @@ -353,6 +386,9 @@ def submit( future.set_exception(BrokenProcessPool("boom")) return future + def shutdown(self, wait: bool, cancel_futures: bool) -> None: + pass + with mock.patch( "tesla_fleet_api.tesla.tesla.ProcessPoolExecutor", return_value=_BrokenPool(), From 24d3e4e4c867ed15e207a44308b7977317e479c1 Mon Sep 17 00:00:00 2001 From: firstmate crewmate Date: Wed, 29 Jul 2026 09:40:06 +1000 Subject: [PATCH 05/12] no-mistakes(document): Document RSA key generation requirements --- docs/energy_local_control.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/energy_local_control.md b/docs/energy_local_control.md index 08fca15..fc62a78 100644 --- a/docs/energy_local_control.md +++ b/docs/energy_local_control.md @@ -26,6 +26,13 @@ inherits it - `TeslaFleetApi`, `Teslemetry`, `Tessie`) loads an existing RSA private key or creates a new 4096-bit unencrypted PEM key file. This is the key you will register with the gateway and later hand to `aiopowerwall`. +Creating a new key uses a worker process so RSA generation does not block the +asyncio event loop. Call it from a script or module with a spawn-safe entry +point (guard application startup with `if __name__ == "__main__":`). Key +generation is not supported directly from a REPL, `python -c`, or a notebook +cell; loading an existing key file does not start a worker process and works in +those environments. + ## 2. Register the key with the gateway, over the cloud `EnergySite.add_authorized_client` registers the public half of that key with From dd41516c28917470bfb03437bfd513429d21b2d2 Mon Sep 17 00:00:00 2001 From: firstmate crewmate Date: Wed, 29 Jul 2026 09:54:02 +1000 Subject: [PATCH 06/12] fix(tesla): fall back to in-process RSA generation when isolation fails Codex correctly flagged that changing the restrictive-umask test to 0o077 hid a real regression: under umask 0o777 (and similarly, any environment that can't start a spawn-based worker process, such as a REPL/`python -c`/ notebook caller) generation now failed outright, where it previously succeeded (just by blocking the loop). Broadened the existing BrokenProcessPool handling into a general fallback: any failure to acquire/start the isolated worker process (BrokenProcessPool, or an OSError such as the semaphore PermissionError under a restrictive umask) now falls back to in-process generation with a logged warning, restoring the legacy works-but-blocks behavior for these environments instead of raising. Restored the umask test to 0o777, now asserting success via the fallback and that the key file still lands owner-only. --- tesla_fleet_api/tesla/tesla.py | 45 +++++++++++++++++------- tests/test_tesla_private_key.py | 62 +++++++++++++++++++++++++-------- 2 files changed, 80 insertions(+), 27 deletions(-) diff --git a/tesla_fleet_api/tesla/tesla.py b/tesla_fleet_api/tesla/tesla.py index 8b7652e..864b05e 100644 --- a/tesla_fleet_api/tesla/tesla.py +++ b/tesla_fleet_api/tesla/tesla.py @@ -10,6 +10,7 @@ from os.path import exists import aiofiles +from tesla_fleet_api.const import LOGGER from tesla_fleet_api.tesla.charging import Charging from tesla_fleet_api.tesla.energysite import EnergySites from tesla_fleet_api.tesla.partner import Partner @@ -44,29 +45,47 @@ def _generate_rsa_private_key_pem(key_size: int) -> bytes: ) -async def _generate_rsa_private_key(key_size: int) -> tuple[rsa.RSAPrivateKey, bytes]: - """Generate an RSA key in a short-lived worker process and return it with its PEM. +async def _generate_rsa_private_key_pem_isolated(key_size: int) -> bytes: + """Generate an RSA key's PEM in a short-lived worker process. The pool is created fresh per call (not at import time) so importing this module never requires a spawn-safe entry point; only actually generating a - key does. `spawn` needs to re-import `__main__`, which fails under a REPL, - `python -c`, or a notebook cell - surfaced here as a clear error instead of - an opaque BrokenProcessPool. + key does. Raises `BrokenProcessPool`/`OSError` if the worker process or + its synchronization primitives can't start - e.g. `spawn` needs to + re-import `__main__`, which fails under a REPL, `python -c`, or a + notebook cell, or a restrictive umask blocks the pool's own semaphore + file - so the caller can fall back to in-process generation. """ pool = ProcessPoolExecutor(max_workers=1, mp_context=get_context("spawn")) try: - pem = await asyncio.get_running_loop().run_in_executor( + return await asyncio.get_running_loop().run_in_executor( pool, _generate_rsa_private_key_pem, key_size ) - except BrokenProcessPool as err: - raise RuntimeError( - "RSA key generation requires a subprocess-spawnable entry " - "point (a script or module guarded by " - "`if __name__ == '__main__':`); it cannot run from a REPL, " - "`python -c`, or a notebook cell." - ) from err finally: await asyncio.to_thread(pool.shutdown, wait=True, cancel_futures=True) + + +async def _generate_rsa_private_key(key_size: int) -> tuple[rsa.RSAPrivateKey, bytes]: + """Generate an RSA key and return it with its PEM. + + Prefers a short-lived worker process so keygen - which holds the GIL for + its full duration, unlike a thread - never stalls the caller's event + loop. Falls back to in-process generation, which does block the loop, + only when the environment can't support that worker process at all (no + spawn-importable `__main__`, or a restrictive umask blocking its + semaphore) - the same environments this method could already generate + keys in before process isolation was introduced. + """ + try: + pem = await _generate_rsa_private_key_pem_isolated(key_size) + except (BrokenProcessPool, OSError) as err: + LOGGER.warning( + "RSA key generation could not use an isolated worker process " + "(%s); falling back to in-process generation, which will block " + "the event loop for the duration of key generation.", + err, + ) + pem = await asyncio.to_thread(_generate_rsa_private_key_pem, key_size) value = await asyncio.to_thread( serialization.load_pem_private_key, pem, diff --git a/tests/test_tesla_private_key.py b/tests/test_tesla_private_key.py index f15ad97..88bd2d4 100644 --- a/tests/test_tesla_private_key.py +++ b/tests/test_tesla_private_key.py @@ -179,14 +179,14 @@ async def test_new_key_file_is_owner_only(self) -> None: self.assertEqual(mode, 0o600) async def test_new_key_file_is_owner_only_with_restrictive_umask(self) -> None: - # 0o077 (not 0o777, unlike the EC key's equivalent test above): a - # fully permission-devoid umask also blocks the multiprocessing - # worker's own internal semaphore file, unrelated to this method's - # own O_EXCL/fchmod permission handling, which this test targets. + # Under 0o777 the isolated worker process's own semaphore can't be + # created, so this exercises the in-process fallback - which, like + # the pre-process-isolation implementation, still produces a correct + # owner-only key file. with tempfile.TemporaryDirectory() as tmp_dir: path = str(Path(tmp_dir) / "tedapi_rsa_private.pem") tesla = Tesla() - old_umask = os.umask(0o077) + old_umask = os.umask(0o777) try: await tesla.get_rsa_private_key(path, key_size=1024) finally: @@ -371,8 +371,15 @@ def shutdown(self, wait: bool, cancel_futures: bool) -> None: with self.assertRaises(asyncio.CancelledError): await task - async def test_broken_process_pool_raises_clear_error(self) -> None: - """A spawn-incapable caller (REPL, `python -c`, notebook) gets a clear error.""" + async def test_broken_process_pool_falls_back_to_in_process_generation( + self, + ) -> None: + """A spawn-incapable caller (REPL, `python -c`, notebook) still succeeds. + + It falls back to in-process generation with a logged warning instead + of raising, matching the pre-process-isolation behavior these + environments already relied on. + """ with tempfile.TemporaryDirectory() as tmp_dir: path = str(Path(tmp_dir) / "tedapi_rsa_private.pem") @@ -389,12 +396,39 @@ def submit( def shutdown(self, wait: bool, cancel_futures: bool) -> None: pass - with mock.patch( - "tesla_fleet_api.tesla.tesla.ProcessPoolExecutor", - return_value=_BrokenPool(), + with ( + mock.patch( + "tesla_fleet_api.tesla.tesla.ProcessPoolExecutor", + return_value=_BrokenPool(), + ), + self.assertLogs("tesla_fleet_api", level="WARNING") as logs, ): - with self.assertRaises(RuntimeError) as ctx: - await Tesla().get_rsa_private_key(path, key_size=1024) + key = await Tesla().get_rsa_private_key(path, key_size=1024) + + self.assertIsInstance(key, rsa.RSAPrivateKey) + mode = stat.S_IMODE(Path(path).stat().st_mode) + self.assertEqual(mode, 0o600) + self.assertTrue( + any("isolated worker process" in message for message in logs.output) + ) + + async def test_pool_setup_oserror_falls_back_to_in_process_generation(self) -> None: + """A restrictive umask (or any other OSError at pool setup) also falls back.""" + with tempfile.TemporaryDirectory() as tmp_dir: + path = str(Path(tmp_dir) / "tedapi_rsa_private.pem") + + with ( + mock.patch( + "tesla_fleet_api.tesla.tesla.ProcessPoolExecutor", + side_effect=PermissionError("boom"), + ), + self.assertLogs("tesla_fleet_api", level="WARNING") as logs, + ): + key = await Tesla().get_rsa_private_key(path, key_size=1024) - self.assertIn("spawn", str(ctx.exception)) - self.assertIsInstance(ctx.exception.__cause__, BrokenProcessPool) + self.assertIsInstance(key, rsa.RSAPrivateKey) + mode = stat.S_IMODE(Path(path).stat().st_mode) + self.assertEqual(mode, 0o600) + self.assertTrue( + any("isolated worker process" in message for message in logs.output) + ) From 2e6893bf0f19fc51a07494c2a9a00382e8321d19 Mon Sep 17 00:00:00 2001 From: firstmate crewmate Date: Wed, 29 Jul 2026 09:55:43 +1000 Subject: [PATCH 07/12] no-mistakes(review): Document RSA generation fallback behavior --- docs/energy_local_control.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/energy_local_control.md b/docs/energy_local_control.md index fc62a78..986d9a7 100644 --- a/docs/energy_local_control.md +++ b/docs/energy_local_control.md @@ -28,10 +28,11 @@ key you will register with the gateway and later hand to `aiopowerwall`. Creating a new key uses a worker process so RSA generation does not block the asyncio event loop. Call it from a script or module with a spawn-safe entry -point (guard application startup with `if __name__ == "__main__":`). Key -generation is not supported directly from a REPL, `python -c`, or a notebook -cell; loading an existing key file does not start a worker process and works in -those environments. +point (guard application startup with `if __name__ == "__main__":`) when +possible. If the worker process cannot start, including from a REPL, +`python -c`, or a notebook cell, generation falls back to the current process +and logs a warning that the asyncio event loop may be blocked. Loading an +existing key file does not start a worker process. ## 2. Register the key with the gateway, over the cloud From a2ec327de68f9df4ec2f0aa36ccc31b236c8fd25 Mon Sep 17 00:00:00 2001 From: firstmate crewmate Date: Wed, 29 Jul 2026 10:08:41 +1000 Subject: [PATCH 08/12] fix(tesla): make the RSA process-isolation fallback failure-mode-agnostic Codex found a third pool-mechanics failure outside the prior round's enumerated (BrokenProcessPool, OSError) catch: a caller already running as a daemonic multiprocessing worker gets a bare AssertionError ("daemonic processes are not allowed to have children") from Process.start(), which was neither caught nor fell back. Rather than adding a third type to the list, broadened the catch to `except Exception` around pool creation and submission only - process isolation is best-effort by design, so any way it can fail, present or future, should degrade to the working (if blocking) in-process path instead of needing another round to enumerate. The catch stays scoped to _generate_rsa_private_key_pem_isolated's call; PEM deserialization afterward is untouched. The warning now logs the exception's class name alongside its message. Added a daemonic-worker test case. --- tesla_fleet_api/tesla/tesla.py | 30 +++++++++++++----------- tests/test_tesla_private_key.py | 41 +++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 13 deletions(-) diff --git a/tesla_fleet_api/tesla/tesla.py b/tesla_fleet_api/tesla/tesla.py index 864b05e..af18e34 100644 --- a/tesla_fleet_api/tesla/tesla.py +++ b/tesla_fleet_api/tesla/tesla.py @@ -5,7 +5,6 @@ import os import time from concurrent.futures import ProcessPoolExecutor -from concurrent.futures.process import BrokenProcessPool from multiprocessing import get_context from os.path import exists import aiofiles @@ -50,11 +49,13 @@ async def _generate_rsa_private_key_pem_isolated(key_size: int) -> bytes: The pool is created fresh per call (not at import time) so importing this module never requires a spawn-safe entry point; only actually generating a - key does. Raises `BrokenProcessPool`/`OSError` if the worker process or - its synchronization primitives can't start - e.g. `spawn` needs to - re-import `__main__`, which fails under a REPL, `python -c`, or a - notebook cell, or a restrictive umask blocks the pool's own semaphore - file - so the caller can fall back to in-process generation. + key does. Process isolation is best-effort: pool creation or submission + can fail in ways this function does not attempt to enumerate - a + restrictive umask blocking the pool's own semaphore file, + `spawn` unable to re-import `__main__` from a REPL/`python -c`/notebook, + `AssertionError` from an already-daemonic calling process, or any other + environment that can't host a worker process - so the caller catches + broadly and falls back to in-process generation. """ pool = ProcessPoolExecutor(max_workers=1, mp_context=get_context("spawn")) try: @@ -71,18 +72,21 @@ async def _generate_rsa_private_key(key_size: int) -> tuple[rsa.RSAPrivateKey, b Prefers a short-lived worker process so keygen - which holds the GIL for its full duration, unlike a thread - never stalls the caller's event loop. Falls back to in-process generation, which does block the loop, - only when the environment can't support that worker process at all (no - spawn-importable `__main__`, or a restrictive umask blocking its - semaphore) - the same environments this method could already generate - keys in before process isolation was introduced. + whenever that worker process can't be used - the same environments this + method could already generate keys in before process isolation was + introduced. The fallback catches any exception from pool creation or + submission, deliberately not enumerated by type, since process isolation + is best-effort here and any way it can fail should degrade to the + working (if blocking) legacy path rather than propagate. """ try: pem = await _generate_rsa_private_key_pem_isolated(key_size) - except (BrokenProcessPool, OSError) as err: + except Exception as err: LOGGER.warning( "RSA key generation could not use an isolated worker process " - "(%s); falling back to in-process generation, which will block " - "the event loop for the duration of key generation.", + "(%s: %s); falling back to in-process generation, which will " + "block the event loop for the duration of key generation.", + type(err).__name__, err, ) pem = await asyncio.to_thread(_generate_rsa_private_key_pem, key_size) diff --git a/tests/test_tesla_private_key.py b/tests/test_tesla_private_key.py index 88bd2d4..3ae5ce6 100644 --- a/tests/test_tesla_private_key.py +++ b/tests/test_tesla_private_key.py @@ -432,3 +432,44 @@ async def test_pool_setup_oserror_falls_back_to_in_process_generation(self) -> N self.assertTrue( any("isolated worker process" in message for message in logs.output) ) + + async def test_daemonic_caller_assertion_error_falls_back_to_in_process_generation( + self, + ) -> None: + """A caller already running as a daemonic multiprocessing worker also falls back. + + `multiprocessing.Process.start()` raises a bare `AssertionError` + ("daemonic processes are not allowed to have children") in that case + - a failure shape outside any specific exception type, which is why + the fallback catches broadly rather than enumerating types. + """ + with tempfile.TemporaryDirectory() as tmp_dir: + path = str(Path(tmp_dir) / "tedapi_rsa_private.pem") + + class _DaemonicPool: + def submit( + self, fn: object, *args: object, **kwargs: object + ) -> concurrent.futures.Future[bytes]: + raise AssertionError( + "daemonic processes are not allowed to have children" + ) + + def shutdown(self, wait: bool, cancel_futures: bool) -> None: + pass + + with ( + mock.patch( + "tesla_fleet_api.tesla.tesla.ProcessPoolExecutor", + return_value=_DaemonicPool(), + ), + self.assertLogs("tesla_fleet_api", level="WARNING") as logs, + ): + key = await Tesla().get_rsa_private_key(path, key_size=1024) + + self.assertIsInstance(key, rsa.RSAPrivateKey) + mode = stat.S_IMODE(Path(path).stat().st_mode) + self.assertEqual(mode, 0o600) + self.assertTrue( + any("isolated worker process" in message for message in logs.output) + ) + self.assertTrue(any("AssertionError" in message for message in logs.output)) From b433fc37e273cc0f8286c7d231d3ddfcb2f707f3 Mon Sep 17 00:00:00 2001 From: firstmate crewmate Date: Wed, 29 Jul 2026 10:21:18 +1000 Subject: [PATCH 09/12] fix(tesla): replace multiprocessing RSA isolation with a plain subprocess Codex identified the actual root cause behind rounds 2-5: spawn re-executes an unguarded caller's __main__ module, so side effects can run twice before any exception ever reaches the round-4 catch-all - the harm precedes the catch. multiprocessing's own machinery (spawn's __main__ re-import, pickling, semaphores, the daemonic-process restriction) was the entire class of finding, not any one exception type within it. Replaced ProcessPoolExecutor with a plain `sys.executable -c