From f1c191b1fd0c86ee87a314e68c4cb164eb9fb884 Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Wed, 22 Jul 2026 13:57:09 -0700 Subject: [PATCH 01/27] feat: RFC 0008 environment wrap actions with EXPR runtime parity Implement the WRAP_ACTIONS extension (RFC 0008) in the v0 session, with EXPR (RFC 0007) runtime parity with openjd-rs: - Wrap-hook dispatch: an environment's onWrapEnvEnter / onWrapTaskRun / onWrapEnvExit runs in place of an inner environment's onEnter/onExit or a step's onRun, with WrappedAction.Command/Args/Environment/ Timeout/Cancelation.* and WrappedEnv.Name/WrappedStep.Name injected. At most one wrap-defining environment may be active (enforced at enter time). - Two strictly separated scopes (openjd-rs #277 parity): the wrapped action's values resolve against the INNER entity's own scope (its script-level let bindings and embedded files, materialized in runner order: paths, lets, contents); the hook script resolves against the wrap environment's own scope (its lets, evaluated by the runner) plus the WrappedAction.* overlay. Same-named lets on the two sides each resolve to their own value. - WrappedAction.Environment carries every session-defined variable: openjd_env definitions and entered environments' declarative variables: maps; host-inherited variables are excluded. - EXPR runtime: runner-evaluated script-level let bindings ordered around embedded-file materialization (paths before lets, contents after), typed symbol tables, enter_environment(extra_let_bindings=...) so a step's environments see the step-level lets. - Cancelation: WrappedAction.Cancelation.Mode/NotifyPeriodInSeconds resolved through the enforcement path, with Template Schemas 5.3.2 defaults (120s task onRun / 30s otherwise). Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- .gitignore | 3 + src/openjd/sessions/_embedded_files.py | 32 +- src/openjd/sessions/_path_mapping.py | 98 ++- src/openjd/sessions/_runner_base.py | 246 +++++- src/openjd/sessions/_runner_env_script.py | 110 ++- src/openjd/sessions/_runner_step_script.py | 55 +- src/openjd/sessions/_session.py | 734 ++++++++++++++++-- src/openjd/sessions/_types.py | 8 + test/openjd/sessions_v0/test_wrap_actions.py | 685 ++++++++++++++++ .../sessions_v0/test_wrap_cancelation.py | 353 +++++++++ test/openjd/sessions_v0/test_wrap_task_run.py | 436 +++++++++++ 11 files changed, 2641 insertions(+), 119 deletions(-) create mode 100644 test/openjd/sessions_v0/test_wrap_actions.py create mode 100644 test/openjd/sessions_v0/test_wrap_cancelation.py create mode 100644 test/openjd/sessions_v0/test_wrap_task_run.py diff --git a/.gitignore b/.gitignore index 1bf2c8f9..eec418ab 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,6 @@ __pycache__/ /dist _version.py *.log + +# Local one-off helper scripts (not part of the project) +/tmp/ diff --git a/src/openjd/sessions/_embedded_files.py b/src/openjd/sessions/_embedded_files.py index b5444d9a..7cc5b01d 100644 --- a/src/openjd/sessions/_embedded_files.py +++ b/src/openjd/sessions/_embedded_files.py @@ -209,6 +209,23 @@ def __init__( self._user = user def materialize(self, files: EmbeddedFilesListType, symtab: SymbolTable) -> None: + records = self.allocate_file_paths(files, symtab) + self.write_file_contents(records, symtab) + + def allocate_file_paths( + self, files: EmbeddedFilesListType, symtab: SymbolTable + ) -> list[_FileRecord]: + """Allocate the on-disk paths for the embedded files and define their + ``Env.File.*``/``Task.File.*`` symbols in ``symtab``, without writing + the file contents. + + Splitting allocation from :meth:`write_file_contents` lets the runner + evaluate EXPR ``let`` bindings between the two phases (RFC 0007): a + file's *path* never depends on ``let`` values (``filename`` is a plain + string), so the ``Env.File.*``/``Task.File.*`` symbols are available + to the bindings, while a file's ``data`` is written afterwards so it + can reference let-bound values. Mirrors the openjd-rs runners. + """ if self._scope == EmbeddedFilesScope.ENV: self._logger.info("Writing embedded files for Environment to disk.") else: @@ -222,17 +239,28 @@ def materialize(self, files: EmbeddedFilesListType, symtab: SymbolTable) -> None symbol, filename = self._get_symtab_entry(file) records.append(_FileRecord(symbol=symbol, filename=filename, file=file)) - # Add symbols to the symbol table + # Add symbols to the symbol table. For EXPR evaluation the + # Env.File.*/Task.File.* symbols are host-format path values + # (property access like `.parent` works), matching openjd-rs; + # the legacy (non-EXPR) interpolation path ignores the type and + # keeps the string form. for record in records: symtab[record.symbol] = str(record.filename) + symtab.expr_types[record.symbol] = "PATH" self._logger.info( f"Mapping: {record.symbol} -> {record.filename}", extra=LogExtraInfo( openjd_log_content=LogContent.FILE_PATH | LogContent.PARAMETER_INFO ), ) + return records + except (OSError, ValueError) as err: + raise RuntimeError(f"Could not write embedded file: {err}") - # Write the files to disk. + def write_file_contents(self, records: list["_FileRecord"], symtab: SymbolTable) -> None: + """Resolve each allocated file's ``data`` against ``symtab`` and write + it to disk. See :meth:`allocate_file_paths`.""" + try: for record in records: # Raises: OSError self._materialize_file(record.filename, record.file, symtab) diff --git a/src/openjd/sessions/_path_mapping.py b/src/openjd/sessions/_path_mapping.py index b84178ff..65c00ef4 100644 --- a/src/openjd/sessions/_path_mapping.py +++ b/src/openjd/sessions/_path_mapping.py @@ -4,27 +4,43 @@ from enum import Enum from os import name as os_name from pathlib import PurePath, PurePosixPath, PureWindowsPath +from typing import Optional, Union class PathFormat(str, Enum): POSIX = "POSIX" WINDOWS = "WINDOWS" + # RFC 0006 §2.3.2 (EXPR extension): URI-form source paths + # (e.g. "s3://bucket/prefix") that map to local filesystem paths. + URI = "URI" @dataclass(frozen=True) class PathMappingRule: source_path_format: PathFormat - source_path: PurePath + # URI-format rules keep the source as the raw string: URIs are not + # filesystem paths, and PurePath normalization would corrupt the + # "scheme://" separator. + source_path: Union[PurePath, str] destination_path: PurePath def __init__( - self, *, source_path_format: PathFormat, source_path: PurePath, destination_path: PurePath + self, + *, + source_path_format: PathFormat, + source_path: Union[PurePath, str], + destination_path: PurePath, ): if source_path_format == PathFormat.POSIX: if not isinstance(source_path, PurePosixPath): raise ValueError( "Path mapping rule source_path_format does not match source_path type" ) + elif source_path_format == PathFormat.URI: + if not isinstance(source_path, str): + raise ValueError( + "Path mapping rule source_path must be a string for the URI source_path_format" + ) else: if not isinstance(source_path, PureWindowsPath): raise ValueError( @@ -49,9 +65,12 @@ def from_dict(rule: dict[str, str]) -> "PathMappingRule": raise ValueError(f"Path mapping rule requires the following fields: {field_names}") source_path_format = PathFormat(rule["source_path_format"].upper()) - source_path: PurePath + source_path: Union[PurePath, str] if source_path_format == PathFormat.POSIX: source_path = PurePosixPath(rule["source_path"]) + elif source_path_format == PathFormat.URI: + # Keep URIs verbatim; PurePath would collapse "scheme://". + source_path = rule["source_path"] else: source_path = PureWindowsPath(rule["source_path"]) destination_path = PurePath(rule["destination_path"]) @@ -76,6 +95,65 @@ def to_dict(self) -> dict[str, str]: "destination_path": str(self.destination_path), } + @staticmethod + def _uri_path_start(uri: str) -> Optional[int]: + """Byte offset where the path component begins in a URI (after + ``scheme://authority``), or None when there is no ``://`` separator. + Mirrors openjd-rs's ``uri_path_start``.""" + scheme_sep = uri.find("://") + if scheme_sep == -1: + return None + authority_start = scheme_sep + 3 + slash = uri.find("/", authority_start) + return len(uri) if slash == -1 else slash + + def _apply_uri(self, path: str) -> tuple[bool, str]: + """Apply a URI-format rule, mirroring openjd-rs's ``apply_uri``. + + Per RFC 3986 the scheme and authority match case-insensitively while + the path portion matches case-sensitively, on whole path components. + The result is a local path in the host's format. + """ + sep = "/" if os_name == "posix" else "\\" + source = str(self.source_path) + src_path_start = self._uri_path_start(source) + src_path_start = 0 if src_path_start is None else src_path_start + inp_path_start = self._uri_path_start(path) + inp_path_start = 0 if inp_path_start is None else inp_path_start + + # Scheme+authority must match case-insensitively. + if path[:inp_path_start].lower() != source[:src_path_start].lower(): + return False, path + # Path portion must match case-sensitively, on a component boundary. + src_path = source[src_path_start:] + inp_path = path[inp_path_start:] + if not inp_path.startswith(src_path): + return False, path + remainder = inp_path[len(src_path) :] + if remainder and not remainder.startswith("/"): + return False, path + + child_parts = remainder[1:].split("/") if remainder else [] + result = str(self.destination_path) + for part in child_parts: + result += sep + part + if path.endswith("/") and not result.endswith(sep): + result += sep + return True, result + + def source_path_component_count(self) -> int: + """Number of components in the source path, used to order rules from + most to least specific. For URI sources the ``scheme://authority`` + counts as one component plus one per path segment.""" + if isinstance(self.source_path, PurePath): + return len(self.source_path.parts) + source = str(self.source_path) + path_start = self._uri_path_start(source) + if path_start is None: + return 1 + path_portion = source[path_start:].strip("/") + return 1 + (len(path_portion.split("/")) if path_portion else 0) + def apply(self, *, path: str) -> tuple[bool, str]: """Applies the path mapping rule on the given path, if it matches the rule. Does not collapse ".." since symbolic paths could be used. @@ -83,18 +161,24 @@ def apply(self, *, path: str) -> tuple[bool, str]: Returns: tuple[bool, str] - indicating if the path matched the rule and the resulting mapped path. If it doesn't match, then it returns the original path unmodified. """ + if self.source_path_format == PathFormat.URI: + return self._apply_uri(path) + source_path = self.source_path + if not isinstance(source_path, PurePath): + # The constructor guarantees non-URI rules carry PurePath sources. + raise TypeError( + "Path mapping rule source_path must be a PurePath for filesystem source formats" + ) pure_path: PurePath if self.source_path_format == PathFormat.POSIX: pure_path = PurePosixPath(path) else: pure_path = PureWindowsPath(path) - if not pure_path.is_relative_to(self.source_path): + if not pure_path.is_relative_to(source_path): return False, path - remapped_parts = ( - self.destination_path.parts + pure_path.parts[len(self.source_path.parts) :] - ) + remapped_parts = self.destination_path.parts + pure_path.parts[len(source_path.parts) :] if os_name == "posix": result = str(PurePosixPath(*remapped_parts)) if self._has_trailing_slash(self.source_path_format, path): diff --git a/src/openjd/sessions/_runner_base.py b/src/openjd/sessions/_runner_base.py index 27083937..eda87e4c 100644 --- a/src/openjd/sessions/_runner_base.py +++ b/src/openjd/sessions/_runner_base.py @@ -11,13 +11,17 @@ from enum import Enum from pathlib import Path from threading import Lock, Timer -from typing import Callable, Optional, Sequence, Type, cast +from typing import Any, Callable, Optional, Sequence, Type, cast from types import TracebackType from tempfile import mkstemp from openjd.model import SymbolTable from openjd.model import FormatStringError from openjd.model.v2023_09 import Action as Action_2023_09 +from openjd.model.v2023_09 import ArgString as ArgString_2023_09 +from openjd.model.v2023_09 import CancelationMethodDeferred as CancelationMethodDeferred_2023_09 +from openjd.model.v2023_09 import CancelationMode as CancelationMode_2023_09 +from openjd.model.v2023_09 import ModelParsingContext as ModelParsingContext_2023_09 from ._embedded_files import EmbeddedFiles, EmbeddedFilesScope, write_file_for_user from ._logging import log_subsection_banner, LoggerAdapter, LogContent, LogExtraInfo from ._os_checker import is_posix @@ -32,6 +36,7 @@ "TerminateCancelMethod", "NotifyCancelMethod", "ScriptRunnerBase", + "apply_let_bindings", ) @@ -90,6 +95,151 @@ class NotifyCancelMethod(CancelMethod): """Amount of time after a SIGTERM to wait to do the SIGKILL""" +def resolve_effective_cancelation( + cancelation: Any, symtab: SymbolTable +) -> tuple[Optional[str], Optional[int]]: + """Resolve an action's cancelation config against the symbol table into + an effective ``(mode, notify_period_seconds)`` pair. + + What is the problem this solves? + + Format strings are normally delay-processed: the parser stores "this is + a format string" and the value resolves right before the action runs. + But a cancelation's ``mode`` is the *schema selector* — the parser needs + it to know which object shape it is reading — while a forwarded value + like ``mode: "{{WrappedAction.Cancelation.Mode}}"`` (RFC 0008 + round-trip forwarding) only exists at run time. The model therefore + carries such a mode as :class:`CancelationMethodDeferred`, and *this* + function is where the deferred decision finally lands: with the + ``WrappedAction.*`` values in the symbol table, the mode expression + resolves to ``"TERMINATE"``, ``"NOTIFY_THEN_TERMINATE"``, or null — and + a null mode means the whole cancelation object is treated as never + declared (the runtime default applies). + + Returns: + (mode, notify_period_seconds) where ``mode`` is ``"TERMINATE"``, + ``"NOTIFY_THEN_TERMINATE"``, or ``None`` (no ```` + declared, or a deferred mode that resolved to null); and + ``notify_period_seconds`` is ``None`` when the field was omitted or + its whole-field expression resolved to null — the caller applies + the positional schema default. + + Raises: + ValueError: If a deferred mode resolves to anything other than the + two method names or null, if a resolved TERMINATE carries a + non-null notify period, or if a notify period does not resolve + to a positive integer. + FormatStringError: If expression resolution itself fails. + """ + + def resolve_period(period: Any) -> Optional[int]: + if period is None: + return None + if isinstance(period, int): + return period + # FormatString form (FEATURE_BUNDLE_1). A whole-field expression + # that resolves to null renders as the empty string and is treated + # as if the field were not provided (schema defaults apply). + resolved = period.resolve(symtab=symtab) + if resolved == "": + return None + value = int(resolved) # raises ValueError on non-integer + # Mirror the static validator's bounds on literal values (Template + # Schemas 5.3.2: 1..600): format-string values could not be checked + # at parse time, so the resolved value is bounds-checked here. + if value < 1 or value > 600: + raise ValueError(f"notifyPeriodInSeconds must be between 1 and 600, got '{value}'") + return value + + if cancelation is None: + return (None, None) + if isinstance(cancelation, CancelationMethodDeferred_2023_09): + # Null semantics apply only to a whole-field expression + # ("{{ ... }}" with no surrounding text, target type string? — + # Template Schemas 5.3). A normal format string that happens to + # resolve to the empty string is NOT null; it falls through to the + # "must resolve to..." error below. + raw = str(cancelation.mode).strip() + is_whole_field = ( + raw.startswith("{{") + and raw.endswith("}}") + and raw.count("{{") == 1 + and raw.count("}}") == 1 + ) + mode = cancelation.mode.resolve(symtab=symtab) + if mode == "" and is_whole_field: + # Null mode drops the ENTIRE cancelation object: mode is the + # object's required discriminator, so an "omitted" mode cannot + # leave a partial object behind. The action behaves exactly as + # if no were declared. + return (None, None) + if mode == CancelationMode_2023_09.TERMINATE.value: + # Post-resolution the object must validate against the resolved + # variant's shape: TERMINATE admits no notify period. + if resolve_period(cancelation.notifyPeriodInSeconds) is not None: + raise ValueError( + "cancelation mode resolved to TERMINATE, which does not " + "accept notifyPeriodInSeconds" + ) + return (CancelationMode_2023_09.TERMINATE.value, None) + if mode == CancelationMode_2023_09.NOTIFY_THEN_TERMINATE.value: + return ( + CancelationMode_2023_09.NOTIFY_THEN_TERMINATE.value, + resolve_period(cancelation.notifyPeriodInSeconds), + ) + raise ValueError( + "cancelation mode must resolve to TERMINATE, NOTIFY_THEN_TERMINATE, " + f"or null; got '{mode}'" + ) + if cancelation.mode == CancelationMode_2023_09.TERMINATE: + return (CancelationMode_2023_09.TERMINATE.value, None) + return ( + CancelationMode_2023_09.NOTIFY_THEN_TERMINATE.value, + resolve_period(getattr(cancelation, "notifyPeriodInSeconds", None)), + ) + + +def apply_let_bindings(*, symtab: SymbolTable, let_bindings: list[str]) -> None: + """Evaluate EXPR ``let`` bindings (RFC 0007) and add them to ``symtab``. + + ``let_bindings`` is a script's ``let`` field: an ordered list of + ``"name = expression"`` strings. Each RHS is an EXPR expression evaluated + against the symbol table built so far (so later bindings can reference + earlier ones), and the engine's typed result is stored under the bound + name — a let-bound path stays a path for property access, and float + rendering fidelity is preserved — matching the Rust runtime's natively + typed symbol table. + + The runners evaluate bindings after embedded-file *path* allocation and + before file *contents* are written, so a binding may reference + ``Env.File.*``/``Task.File.*`` and a file's ``data`` may reference + let-bound values (mirroring openjd-rs's runner ordering). + + Raises: + ValueError (FormatStringError/ExpressionError): if a binding's + expression cannot be evaluated. + """ + # `let` only appears under the EXPR extension; force it on for parsing + # the RHS regardless of the session's configured extension set. + context = ModelParsingContext_2023_09(supported_extensions=["EXPR"]) + + for binding in let_bindings: + name, sep, rhs = binding.partition("=") + name = name.strip() + rhs = rhs.strip() + if not sep or not name or not rhs: + # Malformed bindings are rejected by the model's `let` validator + # at decode time; skip defensively here. + continue + # Parse the RHS as a standalone EXPR expression and evaluate it + # against the current symbol table. + arg = ArgString_2023_09("{{ " + rhs + " }}", context=context) + expressions = arg.expressions + if not expressions or expressions[0].expression is None: + continue + symtab[name] = expressions[0].expression.evaluate_value(symtab=symtab) + + class ScriptRunnerBase(ABC): """Base class for a runnable Environment or Step Script. Responsible for running a *single* Action, and optionally canceling it. @@ -398,10 +548,17 @@ def _materialize_files( files: EmbeddedFilesListType, dest_directory: Path, symtab: SymbolTable, + let_bindings: Optional[list[str]] = None, ) -> None: """Helper for derived classes that wraps all of the logic around materializing embedded files to disk. + When ``let_bindings`` is given, they are evaluated between file-path + allocation and content writing (RFC 0007, mirroring the openjd-rs + runners): a file's *path* never depends on ``let`` values (filenames + are plain strings), so ``Env.File.*``/``Task.File.*`` are available to + the bindings, while a file's ``data`` is written afterwards so it can + reference let-bound values. """ file_writer = EmbeddedFiles( logger=self._logger, @@ -410,10 +567,14 @@ def _materialize_files( user=self._user, ) try: - file_writer.materialize(files, symtab) - except RuntimeError as exc: - # Had a problem writing at least one file to disk. - # Surface the error. + records = file_writer.allocate_file_paths(files, symtab) + if let_bindings: + apply_let_bindings(symtab=symtab, let_bindings=let_bindings) + file_writer.write_file_contents(records, symtab) + except (RuntimeError, ValueError) as exc: + # Had a problem writing at least one file to disk, or evaluating + # a `let` binding (FormatStringError/ExpressionError subclass + # ValueError). Surface the error. # Make use of the action filter to surface the failure reason to # the customer. self._logger.info( @@ -426,6 +587,23 @@ def _materialize_files( if self._callback is not None: self._callback(ActionState.FAILED) + def _apply_let_bindings_or_fail(self, symtab: SymbolTable, let_bindings: list[str]) -> bool: + """Evaluate the script's EXPR ``let`` bindings into ``symtab``. On an + evaluation error the action is failed through the normal failure path + (openjd_fail log, FAILED state, callback). Returns True on success.""" + try: + apply_let_bindings(symtab=symtab, let_bindings=let_bindings) + except ValueError as exc: + self._logger.info( + f"openjd_fail: {str(exc)}", + extra=LogExtraInfo(openjd_log_content=LogContent.EXCEPTION_INFO), + ) + self._state_override = ScriptRunnerState.FAILED + if self._callback is not None: + self._callback(ActionState.FAILED) + return False + return True + def _run_action( self, action: ActionModel, @@ -448,7 +626,23 @@ def _run_action( try: command = [action.command.resolve(symtab=symtab)] if action.args is not None: - command.extend(s.resolve(symtab=symtab) for s in action.args) + # RFC 0005 §1.3.2 argument semantics, mirroring openjd-rs's + # resolve_action_args: a whole-field expression argument + # resolves typed — a null result skips the argument, a list + # result flattens inline (one argument per element, rendered + # with the engine's display coercion), and a scalar becomes a + # single argument. Multi-segment format strings and legacy + # (non-EXPR) expressions resolve to their string form. + for arg in action.args: + value = arg.resolve_value(symtab=symtab) + if isinstance(value, str): + command.append(value) + elif getattr(value, "is_null", False): + continue + elif str(getattr(value, "type", "")).startswith("list["): + command.extend(str(element) for element in value) + else: + command.append(str(value)) except FormatStringError as exc: # Extremely unlikely since a JobTemplate needs to have passed # validation before we could be running it, but just to be safe. @@ -463,8 +657,44 @@ def _run_action( self._callback(ActionState.FAILED) else: time_limit: Optional[timedelta] = default_timeout - if action.timeout: - time_limit = timedelta(seconds=action.timeout) # type: ignore[arg-type] + if action.timeout is not None: + if isinstance(action.timeout, int): + time_limit = timedelta(seconds=action.timeout) + else: + # FormatString form (FEATURE_BUNDLE_1), resolved right + # before the action runs. A whole-field expression that + # resolves to null renders as the empty string — e.g. + # forwarding `timeout: "{{WrappedAction.Timeout}}"` + # (RFC 0008) when the wrapped action specified no + # timeout — and is treated as if the field were not + # provided, so the positional default applies. + try: + resolved = action.timeout.resolve(symtab=symtab) + except FormatStringError as exc: + self._logger.info( + f"openjd_fail: {str(exc)}", + extra=LogExtraInfo(openjd_log_content=LogContent.EXCEPTION_INFO), + ) + self._state_override = ScriptRunnerState.FAILED + if self._callback is not None: + self._callback(ActionState.FAILED) + return + if resolved != "": + try: + seconds = int(resolved) + if seconds < 1: + raise ValueError + except ValueError: + self._logger.info( + f"openjd_fail: timeout must be a positive integer, " + f"got '{resolved}'", + extra=LogExtraInfo(openjd_log_content=LogContent.EXCEPTION_INFO), + ) + self._state_override = ScriptRunnerState.FAILED + if self._callback is not None: + self._callback(ActionState.FAILED) + return + time_limit = timedelta(seconds=seconds) self._run(command, time_limit) def _cancel( diff --git a/src/openjd/sessions/_runner_env_script.py b/src/openjd/sessions/_runner_env_script.py index 2ce1962c..b0f65842 100644 --- a/src/openjd/sessions/_runner_env_script.py +++ b/src/openjd/sessions/_runner_env_script.py @@ -5,12 +5,9 @@ from pathlib import Path from typing import Callable, Optional -from openjd.model import SymbolTable +from openjd.model import FormatStringError, SymbolTable from openjd.model.v2023_09 import Action as Action_2023_09 from openjd.model.v2023_09 import CancelationMode as CancelationMode_2023_09 -from openjd.model.v2023_09 import ( - CancelationMethodNotifyThenTerminate as CancelationMethodNotifyThenTerminate_2023_09, -) from openjd.model.v2023_09 import EnvironmentScript as EnvironmentScript_2023_09 from ._embedded_files import EmbeddedFilesScope from ._logging import log_subsection_banner @@ -20,9 +17,15 @@ ScriptRunnerBase, ScriptRunnerState, TerminateCancelMethod, + resolve_effective_cancelation, ) from ._session_user import SessionUser -from ._types import ActionModel, ActionState, EnvironmentScriptModel +from ._types import ( + ENV_ACTION_DEFAULT_NOTIFY_PERIOD_SECONDS, + ActionModel, + ActionState, + EnvironmentScriptModel, +) __all__ = ("EnvironmentScriptRunner",) @@ -120,7 +123,15 @@ def _run_env_action( log_subsection_banner(self._logger, "Phase: Setup") - # Write any embedded files to disk + let_bindings = ( + getattr(self._environment_script, "let", None) + if self._environment_script is not None + else None + ) + # Write any embedded files to disk. File paths are allocated before + # the script's EXPR `let` bindings evaluate (so bindings can reference + # Env.File.*), and contents are written after (so `data` can reference + # let-bound values) — mirroring the openjd-rs runner. if ( self._environment_script is not None and self._environment_script.embeddedFiles is not None @@ -132,9 +143,14 @@ def _run_env_action( self._environment_script.embeddedFiles, self._session_files_directory, symtab, + let_bindings=let_bindings, ) if self.state == ScriptRunnerState.FAILED: return + elif let_bindings: + symtab = SymbolTable(source=self._symtab) + if not self._apply_let_bindings_or_fail(symtab, let_bindings): + return else: symtab = self._symtab @@ -181,6 +197,52 @@ def exit(self) -> None: default_timeout=_ENV_EXIT_DEFAULT_TIMEOUT, ) + def wrap_task_run(self) -> None: + """Run the Environment's onWrapTaskRun action, wrapping a task's onRun.""" + self._run_wrap_hook("onWrapTaskRun") + + def wrap_env_enter(self) -> None: + """RFC 0008: run this Environment's ``onWrapEnvEnter`` action, + substituting it for an inner environment's ``onEnter``.""" + self._run_wrap_hook("onWrapEnvEnter") + + def wrap_env_exit(self) -> None: + """RFC 0008: run this Environment's ``onWrapEnvExit`` action, + substituting it for an inner environment's ``onExit``.""" + self._run_wrap_hook("onWrapEnvExit", default_timeout=_ENV_EXIT_DEFAULT_TIMEOUT) + + def _run_wrap_hook(self, hook: str, *, default_timeout: Optional[timedelta] = None) -> None: + """Common dispatch for the three RFC 0008 wrap hooks. ``hook`` is + one of ``onWrapEnvEnter``, ``onWrapTaskRun``, or ``onWrapEnvExit``.""" + if hook not in ("onWrapEnvEnter", "onWrapTaskRun", "onWrapEnvExit"): + # Guard the getattr below: without this, a typo'd hook name + # would silently become a SUCCESS no-op. + raise ValueError(f"Unknown wrap hook name: {hook}") + if self.state != ScriptRunnerState.READY: + raise RuntimeError("This cannot be used to run a second subprocess.") + + # For the type checker + if self._environment_script is not None: + assert isinstance(self._environment_script, EnvironmentScript_2023_09) + + action = ( + getattr(self._environment_script.actions, hook, None) + if self._environment_script is not None + else None + ) + if action is None: + self._state_override = ScriptRunnerState.SUCCESS + # Nothing to do, no wrap action defined. Call the callback + # to inform the caller that the run is complete, and then exit. + if self._callback is not None: + self._callback(ActionState.SUCCESS) + return + + if default_timeout is not None: + self._run_env_action(action, default_timeout=default_timeout) + else: + self._run_env_action(action) + def cancel( self, *, time_limit: Optional[timedelta] = None, mark_action_failed: bool = False ) -> None: @@ -191,24 +253,32 @@ def cancel( # For the type checker assert isinstance(self._action, Action_2023_09) + # Resolve the cancelation config against the symbol table: a + # deferred (format-string) mode and/or a FEATURE_BUNDLE_1 notify + # period decide their values here, right when they are needed + # (see resolve_effective_cancelation for the full story). A cancel + # must always proceed, so resolution errors fall back to Terminate. + try: + mode, period = resolve_effective_cancelation(self._action.cancelation, self._symtab) + except (ValueError, FormatStringError) as exc: + self._logger.warning( + f"Failed to resolve the action's cancelation; canceling by " + f"termination instead: {exc}" + ) + mode, period = (None, None) + method: CancelMethod - if ( - self._action.cancelation is None - or self._action.cancelation.mode == CancelationMode_2023_09.TERMINATE - ): - # Note: Default cancelation for a 2023-09 Step Script is Terminate + if mode != CancelationMode_2023_09.NOTIFY_THEN_TERMINATE.value: + # Note: Default cancelation for a 2023-09 Environment Script is Terminate method = TerminateCancelMethod() else: - model_cancel_method = self._action.cancelation - # For the type checker - assert isinstance(model_cancel_method, CancelationMethodNotifyThenTerminate_2023_09) - if model_cancel_method.notifyPeriodInSeconds is None: - # Default grace period is 30s for a 2023-09 Environment Script's notify cancel - method = NotifyCancelMethod(terminate_delay=timedelta(seconds=30)) - else: - method = NotifyCancelMethod( - terminate_delay=timedelta(seconds=model_cancel_method.notifyPeriodInSeconds) # type: ignore[arg-type] + method = NotifyCancelMethod( + terminate_delay=timedelta( + seconds=( + period if period is not None else ENV_ACTION_DEFAULT_NOTIFY_PERIOD_SECONDS + ) ) + ) # Note: If the given time_limit is less than that in the method, then the time_limit will be what's used. self._cancel(method, time_limit, mark_action_failed) diff --git a/src/openjd/sessions/_runner_step_script.py b/src/openjd/sessions/_runner_step_script.py index a4d2e654..9700cbf3 100644 --- a/src/openjd/sessions/_runner_step_script.py +++ b/src/openjd/sessions/_runner_step_script.py @@ -5,12 +5,9 @@ from pathlib import Path from typing import Callable, Optional -from openjd.model import SymbolTable +from openjd.model import FormatStringError, SymbolTable from openjd.model.v2023_09 import CancelationMode as CancelationMode_2023_09 from openjd.model.v2023_09 import StepScript as StepScript_2023_09 -from openjd.model.v2023_09 import ( - CancelationMethodNotifyThenTerminate as CancelationMethodNotifyThenTerminate_2023_09, -) from ._embedded_files import EmbeddedFilesScope from ._logging import log_subsection_banner from ._runner_base import ( @@ -19,9 +16,10 @@ ScriptRunnerBase, ScriptRunnerState, TerminateCancelMethod, + resolve_effective_cancelation, ) from ._session_user import SessionUser -from ._types import ActionState, StepScriptModel +from ._types import TASK_RUN_DEFAULT_NOTIFY_PERIOD_SECONDS, ActionState, StepScriptModel __all__ = ("StepScriptRunner",) @@ -104,7 +102,11 @@ def run(self) -> None: # For the type checker. assert isinstance(self._script, StepScript_2023_09) - # Write any embedded files to disk + let_bindings = getattr(self._script, "let", None) + # Write any embedded files to disk. File paths are allocated before + # the script's EXPR `let` bindings evaluate (so bindings can reference + # Task.File.*), and contents are written after (so `data` can + # reference let-bound values) — mirroring the openjd-rs runner. if self._script.embeddedFiles is not None: symtab = SymbolTable(source=self._symtab) self._materialize_files( @@ -112,9 +114,14 @@ def run(self) -> None: self._script.embeddedFiles, self._session_files_directory, symtab, + let_bindings=let_bindings, ) if self.state == ScriptRunnerState.FAILED: return + elif let_bindings: + symtab = SymbolTable(source=self._symtab) + if not self._apply_let_bindings_or_fail(symtab, let_bindings): + return else: symtab = self._symtab @@ -127,24 +134,34 @@ def cancel( # For the type checker. assert isinstance(self._script, StepScript_2023_09) + # Resolve the cancelation config against the symbol table: a + # deferred (format-string) mode and/or a FEATURE_BUNDLE_1 notify + # period decide their values here, right when they are needed + # (see resolve_effective_cancelation for the full story). A cancel + # must always proceed, so resolution errors fall back to Terminate. + try: + mode, period = resolve_effective_cancelation( + self._script.actions.onRun.cancelation, self._symtab + ) + except (ValueError, FormatStringError) as exc: + self._logger.warning( + f"Failed to resolve the action's cancelation; canceling by " + f"termination instead: {exc}" + ) + mode, period = (None, None) + method: CancelMethod - if ( - self._script.actions.onRun.cancelation is None - or self._script.actions.onRun.cancelation.mode == CancelationMode_2023_09.TERMINATE - ): + if mode != CancelationMode_2023_09.NOTIFY_THEN_TERMINATE.value: # Note: Default cancelation for a 2023-09 Step Script is Terminate method = TerminateCancelMethod() else: - model_cancel_method = self._script.actions.onRun.cancelation - # For the type checker - assert isinstance(model_cancel_method, CancelationMethodNotifyThenTerminate_2023_09) - if model_cancel_method.notifyPeriodInSeconds is None: - # Default grace period is 120s for a 2023-09 Step Script's notify cancel - method = NotifyCancelMethod(terminate_delay=timedelta(seconds=120)) - else: - method = NotifyCancelMethod( - terminate_delay=timedelta(seconds=model_cancel_method.notifyPeriodInSeconds) # type: ignore[arg-type] + method = NotifyCancelMethod( + terminate_delay=timedelta( + seconds=( + period if period is not None else TASK_RUN_DEFAULT_NOTIFY_PERIOD_SECONDS + ) ) + ) # Note: If the given time_limit is less than that in the method, then the time_limit will be what's used. self._cancel(method, time_limit, mark_action_failed) diff --git a/src/openjd/sessions/_session.py b/src/openjd/sessions/_session.py index dbb8af49..04d554af 100644 --- a/src/openjd/sessions/_session.py +++ b/src/openjd/sessions/_session.py @@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Any, Callable, Optional, Type, Union from openjd.model import ( + FormatStringError, JobParameterValues, ParameterValue, ParameterValueType, @@ -37,17 +38,19 @@ ValueReferenceConstants as ValueReferenceConstants_2023_09, ) from ._action_filter import ActionMessageKind, ActionMonitoringFilter -from ._embedded_files import write_file_for_user +from ._embedded_files import EmbeddedFiles, EmbeddedFilesScope, write_file_for_user from ._logging import LOG, log_section_banner, LoggerAdapter, LogExtraInfo, LogContent from ._os_checker import is_posix, is_windows from ._path_mapping import PathMappingRule -from ._runner_base import ScriptRunnerBase +from ._runner_base import ScriptRunnerBase, apply_let_bindings, resolve_effective_cancelation from ._runner_env_script import EnvironmentScriptRunner from ._runner_step_script import StepScriptRunner from ._session_user import SessionUser from ._subprocess import LoggingSubprocess from ._tempdir import TempDir, custom_gettempdir from ._types import ( + ENV_ACTION_DEFAULT_NOTIFY_PERIOD_SECONDS, + TASK_RUN_DEFAULT_NOTIFY_PERIOD_SECONDS, ActionState, EnvironmentIdentifier, EnvironmentModel, @@ -135,6 +138,11 @@ class SimplifiedEnvironmentVariableChanges: """ def __init__(self, initial_variables: Union[dict[str, str], "EnvironmentVariableObject"]): + # Insertion-ordered map of every session-defined variable for this + # environment: the declarative `variables:` map seed plus any + # openjd_env stdout sets/unsets (None = unset). RFC 0008's + # ``WrappedAction.Environment`` surfaces all of these (openjd-rs + # #277); host-inherited variables never enter this map. self._to_set: dict[str, Optional[str]] if is_windows(): @@ -147,10 +155,11 @@ def __init__(self, initial_variables: Union[dict[str, str], "EnvironmentVariable def simplify_ordered_changes(self, changes: list[EnvironmentVariableChange]) -> None: """Apply a given list of sets and unsets to the current state in order""" for change in changes: + name = change.name.upper() if is_windows() else change.name if isinstance(change, EnvironmentVariableSetChange): - self._to_set[change.name.upper() if is_windows() else change.name] = change.value + self._to_set[name] = change.value elif isinstance(change, EnvironmentVariableUnsetChange): - self._to_set[change.name.upper() if is_windows() else change.name] = None + self._to_set[name] = None else: raise ValueError("Unknown type of environment variable change.") @@ -323,6 +332,7 @@ def __init__( *, session_id: str, job_parameter_values: JobParameterValues, + job_name: Optional[str] = None, path_mapping_rules: Optional[list[PathMappingRule]] = None, retain_working_dir: bool = False, user: Optional[SessionUser] = None, @@ -339,6 +349,11 @@ def __init__( job_parameter_values (JobParameterValues): Values for any defined Job Parameters. This is a dictionary where the keys are parameter names, and the values are instances of ParameterValue (a dataclass containing the type and value of the parameter) + job_name (Optional[str]): The resolved name of the Job this Session belongs to. + When provided, it is seeded as the ``Job.Name`` template variable + (RFC 0007 §7.3.1, EXPR extension) for the session's actions — + mirroring the Job.Name symbol that openjd-rs carries in its + session symbol tables. Defaults to None (no Job.Name symbol). path_mapping_rules (Optional[list[PathMappingRule]]): A list of the path mapping rules to apply within all actions running within this session. Defaults to None. retain_working_dir (bool, optional): If set, then the Session's Working Directory @@ -382,6 +397,10 @@ def __init__( self._session_id = session_id self._ending_only = False self._environments = dict() + # Extra EXPR `let` bindings supplied when an environment was entered + # (e.g. the owning step's step-level bindings), re-applied when the + # environment exits so its onExit resolves in the same scope. + self._environment_extra_let_bindings: dict[EnvironmentIdentifier, list[str]] = dict() self._environments_entered = list() self._runner = None self._running_environment_identifier = None @@ -390,13 +409,21 @@ def __init__( self._retain_working_dir = retain_working_dir self._user = user self._job_parameter_values = dict(job_parameter_values) if job_parameter_values else dict() + self._job_name = job_name self._cleanup_called = False self._callback = callback self._path_mapping_rules = path_mapping_rules[:] if path_mapping_rules else None if self._path_mapping_rules is not None: # Path mapping rules are applied in order of longest to shortest source path, # so sort them for when we apply them. - self._path_mapping_rules.sort(key=lambda rule: -len(rule.source_path.parts)) + self._path_mapping_rules.sort(key=lambda rule: -rule.source_path_component_count()) + # Engine (openjd.expr) form of the path mapping rules, seeded into the + # session's symbol tables so EXPR host-context functions such as + # apply_path_mapping() apply the session's rules at run time — + # mirroring openjd-rs's session-scope HostContext::WithRules. An empty + # list still constructs a host context (the session IS host scope); + # the rules are simply empty. + self._expr_host_rules = self._build_expr_host_rules() self._session_root_directory = session_root_directory if self._session_root_directory is not None: if not self._session_root_directory.is_dir(): @@ -609,6 +636,7 @@ def enter_environment( environment: EnvironmentModel, identifier: Optional[EnvironmentIdentifier] = None, os_env_vars: Optional[dict[str, str]] = None, + extra_let_bindings: Optional[list[str]] = None, ) -> EnvironmentIdentifier: """Enters an Open Job Description Environment within this Session. This method is non-blocking; it will exit when the subprocess is either confirmed to have @@ -627,6 +655,13 @@ def enter_environment( by values defined in Environments. Key: Environment variable name Value: Value for the environment variable. + extra_let_bindings (Optional[list[str]]): Additional EXPR ``let`` + bindings (RFC 0007) evaluated into the symbol table before the + environment's variables and actions resolve. A step's + environments are entered with the step-level ``let`` bindings + (``Step.let`` on the instantiated Job) so both can reference + them — the v0 counterpart of the per-step resolved symbol + table that openjd-rs threads into enter_environment. Returns: EnvironmentIdentifier: An identifier by which the Environment is known by to this Session. @@ -638,6 +673,21 @@ def enter_environment( raise RuntimeError( f"Environment {identifier} has already been entered in this Session." ) + + # RFC 0008: at most one Environment in the session stack may + # define any wrap hook. Reject the new environment up front if it + # would create a second wrap layer. + if self._environment_defines_any_wrap_hook(environment): + for existing_id in self._environments_entered: + existing = self._environments[existing_id] + if self._environment_defines_any_wrap_hook(existing): + raise RuntimeError( + "RFC 0008: a session may have at most one Environment " + "defining wrap hooks (onWrapEnvEnter / onWrapTaskRun / " + f"onWrapEnvExit). Environment '{existing.name}' already " + f"defines wrap hooks; cannot also enter '{environment.name}'." + ) + log_section_banner(self._logger, f"Entering Environment: {environment.name}") self._reset_action_state() @@ -646,11 +696,25 @@ def enter_environment( identifier = f"{self._session_id}:{uuid.uuid4().hex}" self._environments[identifier] = environment + if extra_let_bindings: + self._environment_extra_let_bindings[identifier] = list(extra_let_bindings) self._environments_entered.append(identifier) self._running_environment_identifier = identifier symtab = self._symbol_table(environment.revision) + # Step-level `let` bindings (RFC 0007) accompany a step's + # environments: evaluate them first so the environment's variables + # and actions can reference them. + if extra_let_bindings: + apply_let_bindings(symtab=symtab, let_bindings=extra_let_bindings) + + # Note: the environment script's own EXPR `let` bindings (RFC 0007) + # are evaluated by the script runner, after embedded-file paths are + # allocated (so bindings can reference Env.File.*). The environment's + # `variables` resolve against the session symbol table without them, + # matching the openjd-rs runtime. + if environment.variables is not None: # We must process the current environment's variables # before we call _evaluate_current_session_env_vars() @@ -690,18 +754,78 @@ def enter_environment( # so it's important to set the action_state to RUNNING before calling enter(), rather # than after -- enter() itself may end up setting the action state to FAILED. - self._runner = EnvironmentScriptRunner( - logger=self._logger, - user=self._user, - os_env_vars=action_env_vars, - session_working_directory=self.working_directory, - startup_directory=self.working_directory, - callback=self._action_callback, - environment_script=environment.script, - symtab=symtab, - session_files_directory=self.files_directory, + # RFC 0008: an outer environment's onWrapEnvEnter intercepts an + # inner environment's onEnter. The outer environment's *own* + # onEnter is never wrapped — that's why the lookup excludes the + # environment we just appended (which is at the top of the stack). + on_enter_action = ( + environment.script.actions.onEnter if environment.script is not None else None + ) + wrap_env = ( + self._find_wrap_environment(hook="onWrapEnvEnter") + if on_enter_action is not None + else None ) - self._runner.enter() + # The wrapping environment is whichever earlier-entered env defines + # onWrapEnvEnter — never the env we just entered. + if wrap_env is environment: + wrap_env = None + + if wrap_env is not None: + try: + # The wrapped onEnter resolves against the INNER environment's + # own scope — its `let` bindings and embedded files, built on + # a COPY of the session table so the hook's own scope never + # sees them (openjd-rs #277). The hook itself resolves against + # `symtab`; its own script's lets/files are evaluated by the + # script runner from wrap_env.script. + inner_script = environment.script + inner_symtab = self._build_wrapped_inner_scope( + EmbeddedFilesScope.ENV, + getattr(inner_script, "let", None) if inner_script else None, + inner_script.embeddedFiles if inner_script is not None else None, + symtab, + ) + self._inject_wrapped_env_symbols( + symtab, environment, on_enter_action, inner_symtab=inner_symtab + ) + except (FormatStringError, ValueError, RuntimeError) as e: + # e.g. the wrapped onEnter's inner scope failed to build (a + # `let` binding or embedded file did not resolve or write). + # Fail the action through the normal failure path — the + # environment stays in the entered list, exactly as when + # enter() itself fails, so the caller's cleanup exits it as + # usual. + self._fail_action_before_start( + f"Failed to resolve the wrapped onEnter action of " + f"{environment.name} for {wrap_env.name}'s onWrapEnvEnter: {e}" + ) + return identifier + self._runner = EnvironmentScriptRunner( + logger=self._logger, + user=self._user, + os_env_vars=action_env_vars, + session_working_directory=self.working_directory, + startup_directory=self.working_directory, + callback=self._action_callback, + environment_script=wrap_env.script, + symtab=symtab, + session_files_directory=self.files_directory, + ) + self._runner.wrap_env_enter() + else: + self._runner = EnvironmentScriptRunner( + logger=self._logger, + user=self._user, + os_env_vars=action_env_vars, + session_working_directory=self.working_directory, + startup_directory=self.working_directory, + callback=self._action_callback, + environment_script=environment.script, + symtab=symtab, + session_files_directory=self.files_directory, + ) + self._runner.enter() return identifier @@ -758,6 +882,14 @@ def exit_environment( # Must be run _before_ we pop _environments_entered action_env_vars = self._evaluate_current_session_env_vars(os_env_vars) + # RFC 0008: capture the openjd_env list for WrappedAction.Environment + # _before_ the exiting environment is removed from tracking, so the + # list includes that environment's own openjd_env variables — the + # real subprocess environment (action_env_vars, computed above) + # includes them, and the wrapped onExit runs with them in the + # unwrapped case too. + wrapped_session_env_list = self._collect_session_env_list() + # Remove the environment from our tracking since we're now exiting it. del self._environments[identifier] self._environments_entered.pop() @@ -766,6 +898,19 @@ def exit_environment( symtab = self._symbol_table(environment.revision) self._materialize_path_mapping(environment.revision, action_env_vars, symtab) + + # Re-apply the extra `let` bindings this environment was entered with + # (e.g. the owning step's step-level bindings, RFC 0007) so its onExit + # resolves in the same scope as its onEnter. + exit_extra_let_bindings = self._environment_extra_let_bindings.pop(identifier, None) + if exit_extra_let_bindings: + apply_let_bindings(symtab=symtab, let_bindings=exit_extra_let_bindings) + + # Note: the environment script's own EXPR `let` bindings (RFC 0007) + # are evaluated by the script runner (after embedded-file path + # allocation); the wrap-interception branch below evaluates them + # itself before resolving the wrapped onExit. + # Sets the subprocess running. # Returns immediately after it has started, or is running self._action_state = ActionState.RUNNING @@ -774,18 +919,74 @@ def exit_environment( # so it's important to set the action_state to RUNNING before calling exit(), rather # than after -- exit() itself may end up setting the action state to FAILED. - self._runner = EnvironmentScriptRunner( - logger=self._logger, - user=self._user, - os_env_vars=action_env_vars, - session_working_directory=self.working_directory, - startup_directory=self.working_directory, - callback=self._action_callback, - environment_script=environment.script, - symtab=symtab, - session_files_directory=self.files_directory, + # RFC 0008: an outer environment's onWrapEnvExit intercepts an + # inner environment's onExit. The inner environment was already + # popped from ``_environments_entered`` above, so a stack search + # now only turns up genuinely-outer environments. + on_exit_action = ( + environment.script.actions.onExit if environment.script is not None else None + ) + wrap_env = ( + self._find_wrap_environment(hook="onWrapEnvExit") + if on_exit_action is not None + else None ) - self._runner.exit() + + if wrap_env is not None: + try: + # See the onWrapEnvEnter path: the wrapped onExit resolves + # against the INNER environment's own scope (its lets and + # embedded files) built on a copy; the hook's scope never + # sees them (openjd-rs #277). + inner_exit_script = environment.script + inner_exit_symtab = self._build_wrapped_inner_scope( + EmbeddedFilesScope.ENV, + getattr(inner_exit_script, "let", None) if inner_exit_script else None, + inner_exit_script.embeddedFiles if inner_exit_script is not None else None, + symtab, + ) + self._inject_wrapped_env_symbols( + symtab, + environment, + on_exit_action, + session_env_list=wrapped_session_env_list, + inner_symtab=inner_exit_symtab, + ) + except (FormatStringError, ValueError, RuntimeError) as e: + # Mirror of the onWrapEnvEnter injection-failure handling: + # fail the action through the normal failure path. The + # environment was already removed from tracking above, + # matching how a failed exit() behaves. + self._fail_action_before_start( + f"Failed to resolve the wrapped onExit action of " + f"{environment.name} for {wrap_env.name}'s onWrapEnvExit: {e}" + ) + return + self._runner = EnvironmentScriptRunner( + logger=self._logger, + user=self._user, + os_env_vars=action_env_vars, + session_working_directory=self.working_directory, + startup_directory=self.working_directory, + callback=self._action_callback, + environment_script=wrap_env.script, + symtab=symtab, + session_files_directory=self.files_directory, + ) + self._runner.wrap_env_exit() + else: + self._runner = EnvironmentScriptRunner( + logger=self._logger, + user=self._user, + os_env_vars=action_env_vars, + session_working_directory=self.working_directory, + startup_directory=self.working_directory, + callback=self._action_callback, + environment_script=environment.script, + symtab=symtab, + session_files_directory=self.files_directory, + ) + self._runner.exit() def run_task( self, @@ -794,6 +995,7 @@ def run_task( task_parameter_values: TaskParameterSet, os_env_vars: Optional[dict[str, str]] = None, log_task_banner: bool = True, + step_name: Optional[str] = None, ) -> None: """Run a Task within the Session. This method is non-blocking; it will exit when the subprocess is either confirmed to have @@ -812,6 +1014,8 @@ def run_task( Value: Value for the environment variable. log_task_banner (bool): Whether to log a banner before running the Task. Default: True + step_name (Optional[str]): The name of the step whose task is being run. + Used by RFC 0008 to populate ``WrappedStep.Name`` in wrap hooks. """ if self.state != SessionState.READY: raise RuntimeError("Session must be in the READY state to run a task.") @@ -832,27 +1036,97 @@ def run_task( self._reset_action_state() symtab = self._symbol_table(step_script.revision, task_parameter_values) + # RFC 0007 §7.3.1 (EXPR): the running step's name. Only EXPR templates + # pass validation referencing Step.Name, so seeding it when known does + # not change non-EXPR behavior. + if step_name is not None: + symtab["Step.Name"] = step_name action_env_vars = self._evaluate_current_session_env_vars(os_env_vars) self._materialize_path_mapping(step_script.revision, action_env_vars, symtab) - self._runner = StepScriptRunner( - logger=self._logger, - user=self._user, - os_env_vars=action_env_vars, - session_working_directory=self.working_directory, - startup_directory=self.working_directory, - callback=self._action_callback, - script=step_script, - symtab=symtab, - session_files_directory=self.files_directory, - ) - # Sets the subprocess running. - # Returns immediately after it has started, or is running - self._action_state = ActionState.RUNNING - self._state = SessionState.RUNNING - # Note: This may fail immediately (e.g. if we cannot write embedded files to disk), - # so it's important to set the action_state to RUNNING before calling run(), rather - # than after -- run() itself may end up setting the action state to FAILED. - self._runner.run() + + # Note: the step script's EXPR `let` bindings (RFC 0007) are evaluated + # by the script runner, after embedded-file paths are allocated (so + # bindings can reference Task.File.*). The wrap-interception branch + # below evaluates them itself before resolving the wrapped onRun. + + # Check if any active environment defines onWrapTaskRun. + # If so, inject WrappedAction.* into the symbol table and run the + # wrap action instead of the step script's onRun (RFC 0008). + wrap_env = self._find_wrap_environment(hook="onWrapTaskRun") + if wrap_env is not None: + if step_name is None: + # RFC 0008: without a step name, {{WrappedStep.Name}} + # renders as the empty string in the wrap script. Callers + # predating the step_name kwarg won't pass it; make the + # gap visible rather than silently rendering empty. + self._logger.warning( + "A wrap environment is active but run_task() was not given a " + "step_name; WrappedStep.Name will render as an empty string." + ) + try: + # The wrapped onRun resolves against the STEP's own scope — + # its `let` bindings and embedded files, built on a COPY of + # the session table so the hook's own scope never sees them + # (openjd-rs #277). The hook resolves against `symtab`; the + # wrap environment's own lets/files are evaluated by the + # script runner from wrap_env.script. + inner_task_symtab = self._build_wrapped_inner_scope( + EmbeddedFilesScope.STEP, + getattr(step_script, "let", None), + getattr(step_script, "embeddedFiles", None), + symtab, + ) + self._inject_wrapped_task_symbols( + symtab, step_script, step_name or "", inner_symtab=inner_task_symtab + ) + except (FormatStringError, ValueError, RuntimeError) as e: + # e.g. the wrapped onRun's inner scope failed to build (a + # `let` binding or embedded file did not resolve or write), or + # a FEATURE_BUNDLE_1 timeout/notifyPeriod format string did + # not resolve to an integer. Fail the action through the + # normal failure path rather than raising out of the public + # API. + self._fail_action_before_start( + f"Failed to resolve the wrapped Task action for {wrap_env.name}'s " + f"onWrapTaskRun: {e}" + ) + return + + self._runner = EnvironmentScriptRunner( + logger=self._logger, + user=self._user, + os_env_vars=action_env_vars, + session_working_directory=self.working_directory, + startup_directory=self.working_directory, + callback=self._action_callback, + environment_script=wrap_env.script, + symtab=symtab, + session_files_directory=self.files_directory, + ) + self._action_state = ActionState.RUNNING + self._state = SessionState.RUNNING + self._runner.wrap_task_run() + else: + # Original path: run the step script directly. + self._runner = StepScriptRunner( + logger=self._logger, + user=self._user, + os_env_vars=action_env_vars, + session_working_directory=self.working_directory, + startup_directory=self.working_directory, + callback=self._action_callback, + script=step_script, + symtab=symtab, + session_files_directory=self.files_directory, + ) + # Sets the subprocess running. + # Returns immediately after it has started, or is running + self._action_state = ActionState.RUNNING + self._state = SessionState.RUNNING + # Note: This may fail immediately (e.g. if we cannot write embedded files to disk), + # so it's important to set the action_state to RUNNING before calling run(), rather + # than after -- run() itself may end up setting the action state to FAILED. + self._runner.run() def _run_task_without_session_env( self, @@ -891,6 +1165,10 @@ def _run_task_without_session_env( action_env_vars.update(**os_env_vars) self._materialize_path_mapping(step_script.revision, action_env_vars, symtab) + + # Note: the step script's EXPR `let` bindings (RFC 0007) are evaluated + # by the script runner, after embedded-file paths are allocated. + self._runner = StepScriptRunner( logger=self._logger, user=self._user, @@ -1059,39 +1337,339 @@ def _symbol_table( ) -> SymbolTable: """Construct a SymbolTable, with fully qualified value names, suitable for running a Script.""" - def processed_parameter_value(param: ParameterValue) -> str: - if param.type == ParameterValueType.PATH and self._path_mapping_rules is not None: + def apply_mapping(path: str) -> str: + if self._path_mapping_rules is not None: # Apply path mapping rules in the order given until one does a replacement for rule in self._path_mapping_rules: - changed, result = rule.apply(path=param.value) + changed, result = rule.apply(path=path) if changed: return result + return path + + def processed_parameter_value(param: ParameterValue) -> Any: + if param.type == ParameterValueType.PATH: + return apply_mapping(param.value) + if param.type == ParameterValueType.LIST_PATH and isinstance(param.value, list): + # openjd-rs maps each element of a LIST[PATH] parameter at + # session scope; mirror that element-wise mapping. + return [apply_mapping(p) for p in param.value] return param.value + def record_expr_types( + expr_types: dict[str, str], key: str, raw_key: str, ptype: ParameterValueType + ) -> None: + """Record the EXPR types for a parameter's ``Param.*``/``RawParam.*`` + (or ``Task.Param.*``/``Task.RawParam.*``) symbols, mirroring the + openjd-rs session-scope symbol table: + + - ``Param.*`` carries the declared type (a PATH is a host-format + path value with path mapping applied). + - ``RawParam.*`` carries the raw string form: PATH stays a plain + string and LIST[PATH] is a LIST[STRING]. + - CHUNK[INT] range strings stay strings (engine inference). + + The types only affect EXPR-extension expression evaluation; the + legacy (non-EXPR) interpolation path ignores them. + """ + if ptype == ParameterValueType.CHUNK_INT: + return + expr_types[key] = ptype.value + if ptype == ParameterValueType.PATH: + pass # raw form stays an (inferred) plain string + elif ptype == ParameterValueType.LIST_PATH: + expr_types[raw_key] = ParameterValueType.LIST_STRING.value + else: + expr_types[raw_key] = ptype.value + if version == SpecificationRevision.v2023_09: symtab = SymbolTable() - symtab[ValueReferenceConstants_2023_09.WORKING_DIRECTORY.value] = str( - self.working_directory - ) + # The session is host scope: enable EXPR host-context functions + # (e.g. apply_path_mapping) with this session's rules. + symtab.expr_host_rules = self._expr_host_rules + working_dir_key = ValueReferenceConstants_2023_09.WORKING_DIRECTORY.value + symtab[working_dir_key] = str(self.working_directory) + # Session.WorkingDirectory is a host-format path value in openjd-rs. + symtab.expr_types[working_dir_key] = ParameterValueType.PATH.value + # RFC 0007 §7.3.1 (EXPR): the job's resolved name. Only templates + # declaring EXPR pass validation referencing Job.Name, so seeding + # it whenever known does not change non-EXPR behavior. + if self._job_name is not None: + symtab["Job.Name"] = self._job_name for param_name, param_props in self._job_parameter_values.items(): - symtab[ + raw_key = ( f"{ValueReferenceConstants_2023_09.JOB_PARAMETER_RAWPREFIX.value}.{param_name}" - ] = param_props.value - symtab[ - f"{ValueReferenceConstants_2023_09.JOB_PARAMETER_PREFIX.value}.{param_name}" - ] = processed_parameter_value(param_props) + ) + key = f"{ValueReferenceConstants_2023_09.JOB_PARAMETER_PREFIX.value}.{param_name}" + symtab[raw_key] = param_props.value + symtab[key] = processed_parameter_value(param_props) + record_expr_types(symtab.expr_types, key, raw_key, param_props.type) if task_parameter_values: for param_name, param_props in task_parameter_values.items(): - symtab[ - f"{ValueReferenceConstants_2023_09.TASK_PARAMETER_RAWPREFIX.value}.{param_name}" - ] = param_props.value - symtab[ - f"{ValueReferenceConstants_2023_09.TASK_PARAMETER_PREFIX.value}.{param_name}" - ] = processed_parameter_value(param_props) + raw_key = f"{ValueReferenceConstants_2023_09.TASK_PARAMETER_RAWPREFIX.value}.{param_name}" + key = f"{ValueReferenceConstants_2023_09.TASK_PARAMETER_PREFIX.value}.{param_name}" + symtab[raw_key] = param_props.value + symtab[key] = processed_parameter_value(param_props) + record_expr_types(symtab.expr_types, key, raw_key, param_props.type) return symtab else: raise NotImplementedError(f"Schema version {str(version.value)} is not supported.") + def _build_expr_host_rules(self) -> Optional[list[Any]]: + """Convert this session's path mapping rules to their engine + (``openjd.expr.PathMappingRule``) form for EXPR host-context + evaluation. Returns an empty list when the session has no rules (the + session is still host scope), or ``None`` when the engine bindings + are unavailable (pre-EXPR openjd-model).""" + try: + from openjd.expr import PathFormat as ExprPathFormat + from openjd.expr import PathMappingRule as ExprPathMappingRule + except ImportError: + return None + rules = [] + for rule in self._path_mapping_rules or []: + rules.append( + ExprPathMappingRule( + source_path_format=getattr(ExprPathFormat, rule.source_path_format.name), + source_path=str(rule.source_path), + destination_path=str(rule.destination_path), + ) + ) + return rules + + # ------------------------------------------------------------------ + # RFC 0008 wrap-action helpers + # ------------------------------------------------------------------ + + _WRAP_HOOK_NAMES = ("onWrapEnvEnter", "onWrapTaskRun", "onWrapEnvExit") + + def _find_wrap_environment(self, *, hook: str) -> Optional[EnvironmentModel]: + """Walk the environment stack (innermost first) and return the + active wrapping environment for ``hook`` (one of ``onWrapEnvEnter``, + ``onWrapTaskRun``, or ``onWrapEnvExit``). + + Per RFC 0008 the session is only valid with at most one + wrap-defining environment in the stack, so the first match is + always the one that applies.""" + if hook not in self._WRAP_HOOK_NAMES: + raise ValueError(f"Unknown wrap hook name: {hook}") + for env_id in reversed(self._environments_entered): + env = self._environments[env_id] + if ( + env.script is not None + and hasattr(env.script.actions, hook) + and getattr(env.script.actions, hook) is not None + ): + return env + return None + + def _environment_defines_any_wrap_hook(self, env: EnvironmentModel) -> bool: + """``True`` iff ``env``'s script declares any of the three wrap + hooks. Used by the single-wrap-layer validation in + :meth:`enter_environment`.""" + if env.script is None: + return False + return any( + hasattr(env.script.actions, name) and getattr(env.script.actions, name) is not None + for name in self._WRAP_HOOK_NAMES + ) + + def _build_wrapped_inner_scope( + self, + scope: EmbeddedFilesScope, + let_bindings: Optional[list[str]], + embedded_files: Optional[Any], + base: SymbolTable, + ) -> SymbolTable: + """Build the scope a wrapped action would have resolved against had + it run unwrapped: a copy of ``base`` (the session-scope table) plus + the inner entity's embedded files and script-level ``let`` bindings, + in the same order the runners use (allocate file paths → evaluate + lets → write file contents, so a file's ``data`` can reference + let-bound values and a binding can reference ``*.File.*``). + + ``WrappedAction.*`` values are resolved against this table so that + names defined by the WRAPPING environment (its ``let`` bindings) can + never leak into the wrapped action's resolved command/args — and, + symmetrically, the inner entity's lets never apply to the hook's own + resolution scope. Mirrors openjd-rs's ``build_wrapped_inner_scope``. + + Raises: + ValueError (FormatStringError/ExpressionError): a binding or file + reference did not resolve. + RuntimeError: an embedded file could not be written to disk. + """ + symtab = SymbolTable(source=base) + if embedded_files: + file_writer = EmbeddedFiles( + logger=self._logger, + scope=scope, + session_files_directory=self.files_directory, + user=self._user, + ) + records = file_writer.allocate_file_paths(embedded_files, symtab) + if let_bindings: + apply_let_bindings(symtab=symtab, let_bindings=let_bindings) + file_writer.write_file_contents(records, symtab) + elif let_bindings: + apply_let_bindings(symtab=symtab, let_bindings=let_bindings) + return symtab + + def _collect_session_env_list(self) -> list[str]: + """Collect all session-defined variables across the active + environment stack as ``["KEY=value", ...]`` for + ``WrappedAction.Environment``. + + Session-defined variables are ``openjd_env`` stdout definitions + *and* entered environments' declarative ``variables:`` maps + (openjd-rs #277); host-inherited variables remain intentionally + excluded per RFC 0008. + + The per-environment changes are flattened cumulatively, in + environment-entry order, so the list reflects the *effective* + state exactly like the real subprocess environment does: a later + set overrides an earlier value for the same name (one entry, not + two), and a later unset removes the name entirely. This mirrors + the Rust runtime's single cumulative ``env_vars`` map.""" + effective: dict[str, Optional[str]] = {} + for env_id in self._environments_entered: + if env_id in self._created_env_vars: + changes = self._created_env_vars[env_id] + # Iterate _to_set (insertion-ordered), which holds both the + # environment's declarative `variables:` seed and any + # openjd_env sets/unsets, so the list order is deterministic. + for key, value in changes._to_set.items(): + effective[key] = value + return [f"{key}={value}" for key, value in effective.items() if value is not None] + + def _resolve_action_timeout(self, action: Any, symtab: SymbolTable) -> Optional[int]: + """Return the wrapped action's timeout as an int (seconds), or + ``None`` if none was specified — ``WrappedAction.Timeout`` is typed + ``int?`` (RFC 0008), following the EXPR semantics for optional + data, so whole-field forwarding + (``timeout: "{{WrappedAction.Timeout}}"``) drops the field when the + wrapped action has no timeout.""" + if action.timeout is not None: + if isinstance(action.timeout, int): + return action.timeout + resolved = action.timeout.resolve(symtab=symtab) + if resolved == "": + # A whole-field expression that resolved to null: treated + # as if the field were not provided. + return None + return int(resolved) + return None + + def _inject_wrapped_cancelation_symbols( + self, symtab: SymbolTable, action: Any, *, is_task_run: bool, inner_symtab: SymbolTable + ) -> None: + """Populate ``WrappedAction.Cancelation.Mode`` and + ``WrappedAction.Cancelation.NotifyPeriodInSeconds`` from the wrapped + action's ```` (RFC 0008 follow-up, + openjd-specifications#148). + + ``Mode`` is typed ``string?``: ``"TERMINATE"``, + ``"NOTIFY_THEN_TERMINATE"``, or ``None`` (rendering as + ``null``/empty in format strings) when the wrapped action defines + no ```` — the null case is deliberately distinct from + an explicit ``TERMINATE`` so wrap scripts can tell "author declared + TERMINATE" apart from "author declared nothing". + + ``NotifyPeriodInSeconds`` is typed ``int?``: the effective grace + period when the mode is ``NOTIFY_THEN_TERMINATE``, applying the + Template Schemas 5.3.2 defaults (120 seconds for a task's ``onRun``, + 30 otherwise) when the wrapped action omits the field — i.e. the + value the runtime would have enforced in the unwrapped case. It is + ``None`` (rendering as ``null``/empty in format strings) when the + mode is ``TERMINATE`` or no ```` is defined, so "not + applicable" is not conflated with a zero-length notify period. + """ + # Resolution goes through the same helper the enforcement path uses + # (resolve_effective_cancelation) — including a wrapped action + # whose own mode is deferred (a format string) — so the value a + # wrap script sees is always the value the runtime would enforce. + cancelation = getattr(action, "cancelation", None) + mode, notify_period = resolve_effective_cancelation(cancelation, inner_symtab) + if mode == CancelationMode_2023_09.NOTIFY_THEN_TERMINATE.value and notify_period is None: + notify_period = ( + TASK_RUN_DEFAULT_NOTIFY_PERIOD_SECONDS + if is_task_run + else ENV_ACTION_DEFAULT_NOTIFY_PERIOD_SECONDS + ) + symtab["WrappedAction.Cancelation.Mode"] = mode + symtab["WrappedAction.Cancelation.NotifyPeriodInSeconds"] = notify_period + + def _inject_wrapped_env_symbols( + self, + symtab: SymbolTable, + environment: EnvironmentModel, + inner_action: Any, + session_env_list: Optional[list[str]] = None, + *, + inner_symtab: SymbolTable, + ) -> None: + """Populate ``WrappedAction.*`` and ``WrappedEnv.Name`` for + ``onWrapEnvEnter`` / ``onWrapEnvExit`` scripts (RFC 0008). + + The wrapped action's command/args/timeout/cancelation resolve + against ``inner_symtab`` — the scope the action would have used had + it run unwrapped (the inner environment's own lets and embedded + files) — while the results are written into ``symtab``, the hook's + own scope. Keeping the two apart means a wrapper-defined name can + never leak into the wrapped action's resolved values, and vice + versa (openjd-rs #277). + + ``session_env_list`` overrides the collected session env list when + the caller must capture it at a different point in time — the + ``onWrapEnvExit`` path collects it before the exiting environment + is removed from tracking, so the wrapped environment's own + variables are included.""" + command = inner_action.command.resolve(symtab=inner_symtab) + args = ( + [a.resolve(symtab=inner_symtab) for a in inner_action.args] if inner_action.args else [] + ) + symtab["WrappedAction.Command"] = command + symtab["WrappedAction.Args"] = args + symtab["WrappedAction.Environment"] = ( + session_env_list if session_env_list is not None else self._collect_session_env_list() + ) + symtab["WrappedAction.Timeout"] = self._resolve_action_timeout(inner_action, inner_symtab) + self._inject_wrapped_cancelation_symbols( + symtab, inner_action, is_task_run=False, inner_symtab=inner_symtab + ) + symtab["WrappedEnv.Name"] = environment.name + + def _inject_wrapped_task_symbols( + self, + symtab: SymbolTable, + step_script: StepScriptModel, + step_name: str, + *, + inner_symtab: SymbolTable, + ) -> None: + """Populate ``WrappedAction.*`` and ``WrappedStep.Name`` for + ``onWrapTaskRun`` (RFC 0008). + + Resolves the step script's ``onRun`` command and args format + strings against ``inner_symtab`` — the step's own scope (its lets + and embedded files), see ``_inject_wrapped_env_symbols`` — producing + concrete values the wrap action can safely shell-quote with + ``repr_sh()``. ``WrappedAction.Args`` is stored as a native + ``list[str]`` so that ``repr_sh(WrappedAction.Args)`` quotes each + element individually.""" + assert isinstance(step_script, StepScript_2023_09) + on_run = step_script.actions.onRun + + symtab["WrappedAction.Command"] = on_run.command.resolve(symtab=inner_symtab) + symtab["WrappedAction.Args"] = ( + [arg.resolve(symtab=inner_symtab) for arg in on_run.args] if on_run.args else [] + ) + symtab["WrappedAction.Environment"] = self._collect_session_env_list() + symtab["WrappedAction.Timeout"] = self._resolve_action_timeout(on_run, inner_symtab) + self._inject_wrapped_cancelation_symbols( + symtab, on_run, is_task_run=True, inner_symtab=inner_symtab + ) + symtab["WrappedStep.Name"] = step_name + def _openjd_session_root_dir(self) -> Path: """ Returns (and creates if necessary) the top-level directory where Open Job Description step session @@ -1173,11 +1751,21 @@ def _materialize_path_mapping( else: rules_dict = dict() symtab[ValueReferenceConstants_2023_09.HAS_PATH_MAPPING_RULES.value] = "false" + # RFC 0007 §7.3: for EXPR evaluation Session.HasPathMappingRules is a + # boolean and Session.PathMappingRulesFile is a path, matching + # openjd-rs's typed session symbols. The legacy (non-EXPR) + # interpolation path ignores these types and keeps the string forms. + symtab.expr_types[ValueReferenceConstants_2023_09.HAS_PATH_MAPPING_RULES.value] = ( + ParameterValueType.BOOL.value + ) rules_json = json.dumps(rules_dict) file_handle, filename = mkstemp(dir=self.working_directory, suffix=".json", text=True) os.close(file_handle) write_file_for_user(Path(filename), rules_json, self._user) symtab[ValueReferenceConstants_2023_09.PATH_MAPPING_RULES_FILE.value] = str(filename) + symtab.expr_types[ValueReferenceConstants_2023_09.PATH_MAPPING_RULES_FILE.value] = ( + ParameterValueType.PATH.value + ) def _resolve_env_variable_format_strings( self, symtab: SymbolTable, variables: "EnvironmentVariableObject" @@ -1270,6 +1858,26 @@ def _action_log_filter_callback( assert action_status is not None self._callback(self._session_id, action_status) + def _fail_action_before_start(self, message: str) -> None: + """Mark the pending action as FAILED before any runner/subprocess + exists (RFC 0008: e.g. when resolving the wrapped action's format + strings for ``WrappedAction.*`` injection fails). + + Mirrors the failure branch of :meth:`_action_callback` — the + session transitions to READY_ENDING so the caller can exit the + entered environments — but does not require ``self._runner``. + """ + self._logger.error(message) + self._action_fail_message = message + self._action_exit_code = None + self._action_state = ActionState.FAILED + self._state = SessionState.READY_ENDING + if self._callback: + action_status = self.action_status + # for the type checker + assert action_status is not None + self._callback(self._session_id, action_status) + def _action_callback(self, state: ActionState) -> None: """This callback is invoked: 1. When the Action process is successfully started, by the same thread that is running the diff --git a/src/openjd/sessions/_types.py b/src/openjd/sessions/_types.py index db62db9d..310a766e 100644 --- a/src/openjd/sessions/_types.py +++ b/src/openjd/sessions/_types.py @@ -20,6 +20,14 @@ EnvironmentModel = Environment_2023_09 EnvironmentScriptModel = EnvironmentScript_2023_09 +# Default notifyPeriodInSeconds for a NOTIFY_THEN_TERMINATE cancelation +# when the action omits the field (2023-09 Template Schemas 5.3.2). +TASK_RUN_DEFAULT_NOTIFY_PERIOD_SECONDS = 120 +"""Default notify period for a Step Script's onRun action.""" +ENV_ACTION_DEFAULT_NOTIFY_PERIOD_SECONDS = 30 +"""Default notify period for any other action (e.g. an Environment's +onEnter/onExit).""" + class ActionState(str, Enum): RUNNING = "running" diff --git a/test/openjd/sessions_v0/test_wrap_actions.py b/test/openjd/sessions_v0/test_wrap_actions.py new file mode 100644 index 00000000..f01bca7b --- /dev/null +++ b/test/openjd/sessions_v0/test_wrap_actions.py @@ -0,0 +1,685 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""End-to-end tests for the three RFC 0008 wrap hooks +(``onWrapEnvEnter``, ``onWrapTaskRun``, ``onWrapEnvExit``) and the +single-wrap-layer validation. + +The tests use the marker-file pattern from the RFC: each action +appends a tagged line to a shared file in the session working +directory, and the final contents prove which actions ran via which +path. No containers required — ``echo`` and ``cat`` only. +""" + +from __future__ import annotations + +import time +import uuid +from pathlib import Path + +import pytest + +from openjd.model.v2023_09 import ( + Action as Action_2023_09, + ArgString as ArgString_2023_09, + CommandString as CommandString_2023_09, + Environment as Environment_2023_09, + EnvironmentActions as EnvironmentActions_2023_09, + EnvironmentScript as EnvironmentScript_2023_09, + StepActions as StepActions_2023_09, + StepScript as StepScript_2023_09, +) +from openjd.sessions import ActionState, ActionStatus, Session, SessionState + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _action(command: str, *args: str) -> Action_2023_09: + return Action_2023_09( + command=CommandString_2023_09(command), + args=[ArgString_2023_09(a) for a in args] if args else None, + ) + + +def _env(name: str, **action_kwargs) -> Environment_2023_09: + """Build an environment whose script defines whichever hooks the + caller passes (e.g. ``onEnter=…``, ``onWrapTaskRun=…``).""" + return Environment_2023_09( + name=name, + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09(**action_kwargs), + ), + ) + + +def _step(command: str, *args: str) -> StepScript_2023_09: + return StepScript_2023_09(actions=StepActions_2023_09(onRun=_action(command, *args))) + + +def _run_until_ready(session: Session, timeout_s: float = 10.0) -> None: + deadline = time.time() + timeout_s + while session.state == SessionState.RUNNING and time.time() < deadline: + time.sleep(0.05) + + +def _trace_path(session: Session) -> Path: + return Path(str(session.working_directory)) / "trace.log" + + +_NOOP = _action("true") + + +# --------------------------------------------------------------------------- +# Single-wrap-layer enforcement (RFC 0008: at most one wrap-defining +# environment in the session stack). +# --------------------------------------------------------------------------- + + +class TestSingleWrapLayer: + def test_two_wrap_envs_rejected_at_enter(self) -> None: + outer = _env( + "outer", + onWrapEnvEnter=_NOOP, + onWrapTaskRun=_action("sh", "-c", "echo outer-wrap"), + onWrapEnvExit=_NOOP, + ) + inner = _env( + "inner", + onWrapEnvEnter=_action("sh", "-c", "echo inner-wrap"), + onWrapTaskRun=_NOOP, + onWrapEnvExit=_NOOP, + ) + + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + session.enter_environment(environment=outer) + _run_until_ready(session) + + with pytest.raises(RuntimeError, match=r"RFC 0008"): + session.enter_environment(environment=inner) + + def test_two_envs_with_only_one_wrap_layer_ok(self) -> None: + outer = _env( + "outer", + onWrapEnvEnter=_NOOP, + onWrapTaskRun=_action("sh", "-c", "echo outer-wrap"), + onWrapEnvExit=_NOOP, + ) + # The inner env defines no wrap hooks — fine to enter. + inner = _env("inner", onEnter=_action("sh", "-c", "echo inner-onEnter")) + + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + session.enter_environment(environment=outer) + _run_until_ready(session) + session.enter_environment(environment=inner) + _run_until_ready(session) + assert session.state == SessionState.READY + + +# --------------------------------------------------------------------------- +# onWrapEnvEnter intercepts inner onEnter (RFC Test 1). +# --------------------------------------------------------------------------- + + +class TestWrapEnvEnter: + def test_wrap_env_enter_intercepts_inner_on_enter(self) -> None: + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + trace = _trace_path(session) + outer = _env( + "outer", + onWrapEnvEnter=_action( + "sh", + "-c", + f"echo '[WRAPPED] inner-onEnter' >> {trace} && " + f"echo 'inner-onEnter body' >> {trace}", + ), + onWrapTaskRun=_NOOP, + onWrapEnvExit=_NOOP, + onEnter=_action("sh", "-c", f"echo 'outer-onEnter' >> {trace}"), + ) + inner = _env( + "inner", + onEnter=_action("sh", "-c", f"echo 'should not run on host' >> {trace}"), + ) + + session.enter_environment(environment=outer) + _run_until_ready(session) + session.enter_environment(environment=inner) + _run_until_ready(session) + + assert session.state == SessionState.READY + assert trace.exists() + content = trace.read_text() + # The outer env's own onEnter ran on host. + assert "outer-onEnter" in content + # The wrap script ran in place of the inner onEnter. + assert "[WRAPPED] inner-onEnter" in content + assert "inner-onEnter body" in content + # The inner env's host-side onEnter did NOT run. + assert "should not run on host" not in content + + def test_wrap_env_enter_receives_wrapped_symbols(self) -> None: + """``WrappedEnv.Name`` and ``WrappedAction.Command`` resolve to the + inner env's identity inside the wrap script.""" + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + trace = _trace_path(session) + outer = _env( + "outer", + onWrapEnvEnter=_action( + "sh", + "-c", + "echo " + "'name={{WrappedEnv.Name}} cmd={{WrappedAction.Command}}' " + f">> {trace}", + ), + onWrapTaskRun=_NOOP, + onWrapEnvExit=_NOOP, + ) + inner = _env("inner-env-name", onEnter=_action("echo", "hello")) + + session.enter_environment(environment=outer) + _run_until_ready(session) + session.enter_environment(environment=inner) + _run_until_ready(session) + + assert session.state == SessionState.READY + content = trace.read_text() + assert "name=inner-env-name" in content + assert "cmd=echo" in content + + +# --------------------------------------------------------------------------- +# onWrapEnvExit intercepts inner onExit (RFC Test 2). +# --------------------------------------------------------------------------- + + +class TestWrapEnvExit: + def test_wrap_env_exit_intercepts_inner_on_exit(self) -> None: + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + trace = _trace_path(session) + outer = _env( + "outer", + onWrapEnvEnter=_NOOP, + onWrapTaskRun=_NOOP, + onWrapEnvExit=_action( + "sh", + "-c", + f"echo '[WRAPPED] inner-onExit' >> {trace}", + ), + onExit=_action("sh", "-c", f"echo 'outer-onExit' >> {trace}"), + ) + inner = _env( + "inner", + onExit=_action("sh", "-c", f"echo 'should not run on host' >> {trace}"), + ) + + outer_id = session.enter_environment(environment=outer) + _run_until_ready(session) + inner_id = session.enter_environment(environment=inner) + _run_until_ready(session) + + session.exit_environment(identifier=inner_id) + _run_until_ready(session) + + content = trace.read_text() + assert "[WRAPPED] inner-onExit" in content + assert "should not run on host" not in content + + # The outer env's own onExit must still run on host when we + # eventually exit it. + session.exit_environment(identifier=outer_id) + _run_until_ready(session) + content = trace.read_text() + assert "outer-onExit" in content + + +# --------------------------------------------------------------------------- +# Visible end-to-end ordering across all three hooks (RFC Test 5). +# --------------------------------------------------------------------------- + + +class TestVisibleOrdering: + def test_full_ordering_across_three_hooks(self) -> None: + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + trace = _trace_path(session) + outer = _env( + "outer", + onEnter=_action("sh", "-c", f"echo 'outer-onEnter' >> {trace}"), + onWrapEnvEnter=_action("sh", "-c", f"echo '[WRAPPED] inner-onEnter' >> {trace}"), + onWrapTaskRun=_action("sh", "-c", f"echo '[WRAPPED] task-onRun' >> {trace}"), + onWrapEnvExit=_action("sh", "-c", f"echo '[WRAPPED] inner-onExit' >> {trace}"), + onExit=_action("sh", "-c", f"echo 'outer-onExit' >> {trace}"), + ) + wrapped_inner = _env( + "wrapped-inner", + onEnter=_action("sh", "-c", f"echo 'inner-onEnter body' >> {trace}"), + onExit=_action("sh", "-c", f"echo 'inner-onExit body' >> {trace}"), + ) + step = _step("echo", "task-onRun-body") + + outer_id = session.enter_environment(environment=outer) + _run_until_ready(session) + wrapped_id = session.enter_environment(environment=wrapped_inner) + _run_until_ready(session) + + session.run_task(step_script=step, task_parameter_values={}) + _run_until_ready(session) + + session.exit_environment(identifier=wrapped_id) + _run_until_ready(session) + session.exit_environment(identifier=outer_id) + _run_until_ready(session) + + lines = [line for line in trace.read_text().splitlines() if line.strip()] + # The exact order matters here — RFC 0008's pass criterion. + assert lines[0] == "outer-onEnter" + assert lines[1] == "[WRAPPED] inner-onEnter" + assert lines[2] == "[WRAPPED] task-onRun" + assert lines[3] == "[WRAPPED] inner-onExit" + assert lines[4] == "outer-onExit" + + +# --------------------------------------------------------------------------- +# WrappedAction.* injection: inner embedded files, and failure handling. +# +# openjd-rs #277: the wrapped entity's embedded files ARE materialized on +# the wrap path (into the inner scope only), so a wrapped action +# referencing {{Task.File.*}} / {{Env.File.*}} resolves to a real on-disk +# path in WrappedAction.Command. Injection failures that remain possible +# (e.g. an undefined symbol) must still FAIL the action through the normal +# callback path instead of raising out of the public API, and the session +# transitions to READY_ENDING (never stuck in RUNNING). +# --------------------------------------------------------------------------- + + +def _embedded_file_step() -> StepScript_2023_09: + from openjd.model.v2023_09 import ( + DataString as DataString_2023_09, + EmbeddedFileText as EmbeddedFileText_2023_09, + EmbeddedFileTypes as EmbeddedFileTypes_2023_09, + ) + + return StepScript_2023_09( + actions=StepActions_2023_09( + onRun=Action_2023_09(command=CommandString_2023_09("{{ Task.File.Foo }}")) + ), + embeddedFiles=[ + EmbeddedFileText_2023_09( + name="Foo", + type=EmbeddedFileTypes_2023_09.TEXT, + runnable=True, + data=DataString_2023_09("#!/bin/sh\necho hello\n"), + ) + ], + ) + + +def _embedded_file_env(name: str) -> Environment_2023_09: + from openjd.model.v2023_09 import ( + DataString as DataString_2023_09, + EmbeddedFileText as EmbeddedFileText_2023_09, + EmbeddedFileTypes as EmbeddedFileTypes_2023_09, + ) + + return Environment_2023_09( + name=name, + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onEnter=Action_2023_09(command=CommandString_2023_09("{{ Env.File.Setup }}")), + ), + embeddedFiles=[ + EmbeddedFileText_2023_09( + name="Setup", + type=EmbeddedFileTypes_2023_09.TEXT, + runnable=True, + data=DataString_2023_09("#!/bin/sh\necho setup\n"), + ) + ], + ), + ) + + +class TestWrapInjectionFailure: + def _wrap_env(self) -> Environment_2023_09: + return _env( + "Wrapper", + onWrapEnvEnter=_action("echo", "{{WrappedAction.Command}}"), + onWrapTaskRun=_action("echo", "{{WrappedAction.Command}}"), + onWrapEnvExit=_NOOP, + ) + + def test_run_task_with_embedded_file_command_resolves( + self, caplog: pytest.LogCaptureFixture + ) -> None: + # openjd-rs #277: a wrapped onRun referencing {{Task.File.*}} now + # resolves — the step's embedded files are materialized into the + # inner scope, and WrappedAction.Command carries the on-disk path. + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + session.enter_environment(environment=self._wrap_env()) + _run_until_ready(session) + assert session.state == SessionState.READY + + session.run_task(step_script=_embedded_file_step(), task_parameter_values={}) + _run_until_ready(session) + + assert session.state == SessionState.READY + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + # The wrap hook echoed WrappedAction.Command: the materialized + # file path inside the session's files directory. + messages = "\n".join(caplog.messages) + assert str(session.files_directory) in messages + + def test_enter_environment_with_embedded_file_on_enter_resolves( + self, caplog: pytest.LogCaptureFixture + ) -> None: + # openjd-rs #277: a wrapped onEnter referencing {{Env.File.*}} now + # resolves against the inner environment's own materialized files. + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + wrap_id = session.enter_environment(environment=self._wrap_env()) + _run_until_ready(session) + + inner_id = session.enter_environment(environment=_embedded_file_env("Inner")) + _run_until_ready(session) + + assert session.state == SessionState.READY + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + messages = "\n".join(caplog.messages) + assert str(session.files_directory) in messages + session.exit_environment(identifier=inner_id, keep_session_running=True) + _run_until_ready(session) + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + + def test_run_task_with_unresolvable_wrapped_symbol_fails_action(self) -> None: + # The graceful-failure contract still holds for injection failures: + # a wrapped onRun referencing an undefined symbol must FAIL the + # action via the callback path, not raise out of run_task(). + bad_step = StepScript_2023_09( + actions=StepActions_2023_09( + onRun=Action_2023_09(command=CommandString_2023_09("{{ Task.File.DoesNotExist }}")) + ), + ) + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + session.enter_environment(environment=self._wrap_env()) + _run_until_ready(session) + assert session.state == SessionState.READY + + session.run_task(step_script=bad_step, task_parameter_values={}) + _run_until_ready(session) + + assert session.state == SessionState.READY_ENDING + status = session.action_status + assert status is not None + assert status.state == ActionState.FAILED + assert status.fail_message is not None + assert "onWrapTaskRun" in status.fail_message + + +class TestWrapScopeSeparation: + """openjd-rs #277: the wrapper's and the wrapped entity's scopes must be + kept strictly apart. WrappedAction.* resolves against the INNER scope + only; the hook script resolves against the WRAP environment's own scope + (including its script-level ``let`` bindings) only.""" + + def test_same_let_name_each_side_sees_own_value(self, caplog: pytest.LogCaptureFixture) -> None: + # GIVEN: the wrapper and the step both bind the same `let` name + # `who` to different values. The step's onRun args reference + # {{who}}; the hook also references {{who}} directly. + wrapper = Environment_2023_09( + name="Wrapper", + script=EnvironmentScript_2023_09( + let=["who = 'wrapper-scope'"], + actions=EnvironmentActions_2023_09( + onWrapEnvEnter=_NOOP, + onWrapTaskRun=_action( + "sh", + "-c", + "echo HOOK-WHO={{who}} WRAPPED-ARGS={{WrappedAction.Args}}", + ), + onWrapEnvExit=_NOOP, + ), + ), + ) + step = StepScript_2023_09( + let=["who = 'step-scope'"], + actions=StepActions_2023_09( + onRun=Action_2023_09( + command=CommandString_2023_09("echo"), + args=[ArgString_2023_09("{{who}}")], + ) + ), + ) + + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + session.enter_environment(environment=wrapper) + _run_until_ready(session) + + # WHEN: the task runs through the wrap hook. + session.run_task(step_script=step, task_parameter_values={}) + _run_until_ready(session) + + # THEN: the hook saw the WRAPPER's binding, and the wrapped + # action's args resolved with the STEP's binding. + assert session.action_status == ActionStatus(state=ActionState.SUCCESS, exit_code=0) + messages = "\n".join(caplog.messages) + assert "HOOK-WHO=wrapper-scope" in messages, messages + # WrappedAction.Args is a list; its default rendering brackets the + # single resolved element. + assert "WRAPPED-ARGS=[step-scope]" in messages, messages + + def test_inner_env_let_does_not_leak_into_hook_scope(self) -> None: + # An inner environment's `let` binding must resolve in + # WrappedAction.* but must NOT be visible to the hook script: a + # hook referencing it directly fails the action. + wrapper = _env( + "Wrapper", + onWrapEnvEnter=_action("sh", "-c", "echo LEAKED={{inner_only}}"), + onWrapTaskRun=_NOOP, + onWrapEnvExit=_NOOP, + ) + inner = Environment_2023_09( + name="Inner", + script=EnvironmentScript_2023_09( + let=["inner_only = 'secret'"], + actions=EnvironmentActions_2023_09( + onEnter=Action_2023_09(command=CommandString_2023_09("{{inner_only}}")), + ), + ), + ) + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + wrap_id = session.enter_environment(environment=wrapper) + _run_until_ready(session) + + inner_id = session.enter_environment(environment=inner) + _run_until_ready(session) + + # The hook referenced {{inner_only}}, which is not in its scope. + status = session.action_status + assert status is not None + assert status.state == ActionState.FAILED + session.exit_environment(identifier=inner_id, keep_session_running=True) + _run_until_ready(session) + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + + def test_wrapped_env_enter_command_resolves_inner_let( + self, caplog: pytest.LogCaptureFixture + ) -> None: + # The inner environment's `let` binding must be visible to + # WrappedAction.* on the onWrapEnvEnter path. + wrapper = _env( + "Wrapper", + onWrapEnvEnter=_action("sh", "-c", "echo INNERCMD={{WrappedAction.Command}}"), + onWrapTaskRun=_NOOP, + onWrapEnvExit=_NOOP, + ) + inner = Environment_2023_09( + name="Inner", + script=EnvironmentScript_2023_09( + let=["tool = 'inner-tool'"], + actions=EnvironmentActions_2023_09( + onEnter=Action_2023_09(command=CommandString_2023_09("{{tool}}")), + ), + ), + ) + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + wrap_id = session.enter_environment(environment=wrapper) + _run_until_ready(session) + + inner_id = session.enter_environment(environment=inner) + _run_until_ready(session) + + assert session.action_status == ActionStatus(state=ActionState.SUCCESS, exit_code=0) + session.exit_environment(identifier=inner_id, keep_session_running=True) + _run_until_ready(session) + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + messages = "\n".join(caplog.messages) + assert "INNERCMD=inner-tool" in messages, messages + + +class TestWrappedActionEnvironmentContents: + def test_variables_map_included_in_wrapped_environment(self) -> None: + # RFC 0008 (openjd-rs #277): WrappedAction.Environment carries every + # session-defined variable — openjd_env definitions AND entered + # environments' declarative variables: maps. Host-inherited + # variables remain excluded. + from openjd.model.v2023_09 import ( + EnvironmentVariableValueString as EnvironmentVariableValueString_2023_09, + ) + + declaring = Environment_2023_09( + name="Declaring", + variables={ + "DECLARED_VAR": EnvironmentVariableValueString_2023_09("from-variables-map") + }, + ) + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + session.enter_environment( + environment=_env( + "Wrapper", + onWrapEnvEnter=_NOOP, + onWrapTaskRun=_NOOP, + onWrapEnvExit=_NOOP, + ) + ) + _run_until_ready(session) + session.enter_environment(environment=declaring) + _run_until_ready(session) + + env_list = session._collect_session_env_list() + assert any(entry == "DECLARED_VAR=from-variables-map" for entry in env_list), env_list + + def test_later_set_overrides_earlier_value_without_duplicate(self) -> None: + # Cumulative flattening: when a later environment's openjd_env sets + # a name an earlier environment already set, the list carries only + # the effective (later) value — matching the real subprocess env + # and the Rust runtime's single cumulative map. + from openjd.sessions._session import ( + EnvironmentVariableSetChange, + EnvironmentVariableUnsetChange, + SimplifiedEnvironmentVariableChanges, + ) + + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + scenarios: list[ + tuple[str, list[EnvironmentVariableSetChange | EnvironmentVariableUnsetChange]] + ] = [ + ("env-a", [EnvironmentVariableSetChange(name="FOO", value="1")]), + ("env-b", [EnvironmentVariableSetChange(name="FOO", value="2")]), + ] + for env_id, changes in scenarios: + session._environments_entered.append(env_id) + tracked = SimplifiedEnvironmentVariableChanges({}) + tracked.simplify_ordered_changes(changes) + session._created_env_vars[env_id] = tracked + + assert session._collect_session_env_list() == ["FOO=2"] + + # And a later unset removes the name entirely. + session._environments_entered.append("env-c") + tracked = SimplifiedEnvironmentVariableChanges({}) + tracked.simplify_ordered_changes([EnvironmentVariableUnsetChange(name="FOO")]) + session._created_env_vars["env-c"] = tracked + + assert session._collect_session_env_list() == [] + + # Clear the fake stack so Session.cleanup() doesn't try to + # unwind environments that were never really entered. + session._environments_entered.clear() + session._created_env_vars.clear() + + def test_exit_path_includes_exiting_envs_openjd_env_vars(self) -> None: + # On the onWrapEnvExit path, WrappedAction.Environment must include + # the exiting environment's own openjd_env variables: the real + # subprocess env (and the unwrapped onExit) includes them, so the + # list must match. The wrapper's onWrapEnvEnter emits the + # openjd_env message, which is attributed to the inner (wrapped) + # environment being entered. + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + trace = _trace_path(session) + wrapper = _env( + "Wrapper", + onWrapEnvEnter=_action("sh", "-c", "echo 'openjd_env: INNER_VAR=inner-value'"), + onWrapTaskRun=_NOOP, + onWrapEnvExit=_action( + "sh", + "-c", + f"echo \"ENVLIST=<{{{{WrappedAction.Environment}}}}>\" >> '{trace}'", + ), + ) + inner = _env("Inner", onEnter=_NOOP, onExit=_NOOP) + + wrap_id = session.enter_environment(environment=wrapper) + _run_until_ready(session) + inner_id = session.enter_environment(environment=inner) + _run_until_ready(session) + session.exit_environment(identifier=inner_id, keep_session_running=True) + _run_until_ready(session) + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + + contents = trace.read_text() + assert "INNER_VAR=inner-value" in contents, contents + + def test_openjd_env_vars_included_in_wrapped_environment(self) -> None: + # openjd_env-emitted variables must still be surfaced. The wrapper's + # own onEnter is never wrapped, so it can emit the message directly. + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + session.enter_environment( + environment=_env( + "Wrapper", + onEnter=_action("sh", "-c", "echo 'openjd_env: EMITTED_VAR=from-openjd-env'"), + onWrapEnvEnter=_NOOP, + onWrapTaskRun=_NOOP, + onWrapEnvExit=_NOOP, + ) + ) + _run_until_ready(session) + + env_list = session._collect_session_env_list() + assert "EMITTED_VAR=from-openjd-env" in env_list + + +class TestRunWrapHookGuard: + def test_unknown_hook_name_raises(self) -> None: + from openjd.sessions._runner_env_script import EnvironmentScriptRunner + import logging + + from openjd.sessions._logging import LoggerAdapter + from openjd.model import SymbolTable + + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + runner = EnvironmentScriptRunner( + logger=LoggerAdapter(logging.getLogger(__name__), extra={}), + session_working_directory=Path(str(session.working_directory)), + environment_script=None, + symtab=SymbolTable(), + session_files_directory=Path(str(session.files_directory)), + ) + with pytest.raises(ValueError, match="Unknown wrap hook name"): + runner._run_wrap_hook("onWrapTypo") diff --git a/test/openjd/sessions_v0/test_wrap_cancelation.py b/test/openjd/sessions_v0/test_wrap_cancelation.py new file mode 100644 index 00000000..a1a38639 --- /dev/null +++ b/test/openjd/sessions_v0/test_wrap_cancelation.py @@ -0,0 +1,353 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Tests for ``WrappedAction.Cancelation.*`` injection (RFC 0008 follow-up, +openjd-specifications#148). + +``WrappedAction.Cancelation.Mode`` is ``string?`` and carries the wrapped +action's cancelation method — ``"TERMINATE"``, ``"NOTIFY_THEN_TERMINATE"``, +or ``None`` when the wrapped action defines no ````. The null +case is deliberately distinct from an explicit ``TERMINATE``. + +``WrappedAction.Cancelation.NotifyPeriodInSeconds`` is ``int?``: the +effective grace period when the mode is ``NOTIFY_THEN_TERMINATE`` (with the +Template Schemas 5.3.2 defaults applied — 120 for a task's ``onRun``, 30 +otherwise), and ``None`` when a notify period does not apply. + +Mirrors the Rust integration coverage in openjd-rs +``tests/integration/test_wrap_actions.rs`` and the conformance fixtures +``conformance-tests/2023-09/WRAP_ACTIONS/jobs/wrap-cancelation-*``. +""" + +from __future__ import annotations + +import time +import uuid + +import pytest + +from openjd.model import SymbolTable +from openjd.model.v2023_09 import ( + Action as Action_2023_09, + ArgString as ArgString_2023_09, + CancelationMethodNotifyThenTerminate as NotifyThenTerminate_2023_09, + CancelationMethodTerminate as Terminate_2023_09, + CancelationMode as CancelationMode_2023_09, + CommandString as CommandString_2023_09, + Environment as Environment_2023_09, + EnvironmentActions as EnvironmentActions_2023_09, + EnvironmentScript as EnvironmentScript_2023_09, + StepActions as StepActions_2023_09, + StepScript as StepScript_2023_09, +) +from openjd.sessions import ActionState, ActionStatus, Session, SessionState + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +_NOOP = Action_2023_09(command=CommandString_2023_09("true")) + +_TERMINATE = Terminate_2023_09(mode=CancelationMode_2023_09.TERMINATE) + + +def _notify(period: int | None) -> NotifyThenTerminate_2023_09: + return NotifyThenTerminate_2023_09( + mode=CancelationMode_2023_09.NOTIFY_THEN_TERMINATE, + notifyPeriodInSeconds=period, + ) + + +def _step_script_with_cancelation(cancelation) -> StepScript_2023_09: + return StepScript_2023_09( + actions=StepActions_2023_09( + onRun=Action_2023_09( + command=CommandString_2023_09("echo"), + args=[ArgString_2023_09("placeholder")], + cancelation=cancelation, + ) + ) + ) + + +def _wrap_env(name: str, wrap_action: Action_2023_09) -> Environment_2023_09: + """Build an Environment with ``onWrapTaskRun`` set to ``wrap_action`` + and the other two wrap hooks set to no-ops (all-or-nothing rule).""" + return Environment_2023_09( + name=name, + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onWrapEnvEnter=_NOOP, + onWrapTaskRun=wrap_action, + onWrapEnvExit=_NOOP, + ), + ), + ) + + +def _inner_env_action(cancelation) -> Action_2023_09: + return Action_2023_09( + command=CommandString_2023_09("inner-enter-cmd"), + args=[ArgString_2023_09("--flag")], + cancelation=cancelation, + ) + + +def _run_until_ready(session: Session, timeout_s: float = 10.0) -> None: + deadline = time.time() + timeout_s + while session.state == SessionState.RUNNING and time.time() < deadline: + time.sleep(0.05) + + +# --------------------------------------------------------------------------- +# Unit tests — task-run injection path +# --------------------------------------------------------------------------- + + +class TestInjectTaskCancelationSymbols: + """Unit tests for cancelation injection via ``_inject_wrapped_task_symbols``.""" + + def _inject(self, cancelation) -> SymbolTable: + session = Session(session_id=uuid.uuid4().hex, job_parameter_values={}) + try: + symtab = SymbolTable() + script = _step_script_with_cancelation(cancelation) + session._inject_wrapped_task_symbols(symtab, script, "Step1", inner_symtab=symtab) + return symtab + finally: + session.cleanup() + + def test_mode_none_and_period_none_when_no_cancelation(self) -> None: + # No declared: Mode is None (NOT "TERMINATE") and the + # notify period is None — the string?/int? null values. + symtab = self._inject(None) + assert symtab["WrappedAction.Cancelation.Mode"] is None + assert symtab["WrappedAction.Cancelation.NotifyPeriodInSeconds"] is None + + def test_mode_terminate_and_period_none(self) -> None: + symtab = self._inject(_TERMINATE) + assert symtab["WrappedAction.Cancelation.Mode"] == "TERMINATE" + assert symtab["WrappedAction.Cancelation.NotifyPeriodInSeconds"] is None + + def test_notify_then_terminate_with_explicit_period(self) -> None: + symtab = self._inject(_notify(45)) + assert symtab["WrappedAction.Cancelation.Mode"] == "NOTIFY_THEN_TERMINATE" + assert symtab["WrappedAction.Cancelation.NotifyPeriodInSeconds"] == 45 + + def test_notify_then_terminate_defaults_to_120_for_task_on_run(self) -> None: + # Template Schemas 5.3.2: the default notify period for a task's + # onRun is 120 seconds. The runtime supplies the value it would have + # enforced in the unwrapped case. + symtab = self._inject(_notify(None)) + assert symtab["WrappedAction.Cancelation.Mode"] == "NOTIFY_THEN_TERMINATE" + assert symtab["WrappedAction.Cancelation.NotifyPeriodInSeconds"] == 120 + + +# --------------------------------------------------------------------------- +# Unit tests — env enter/exit injection path +# --------------------------------------------------------------------------- + + +class TestInjectEnvCancelationSymbols: + """Unit tests for cancelation injection via ``_inject_wrapped_env_symbols``.""" + + def _inject(self, cancelation) -> SymbolTable: + session = Session(session_id=uuid.uuid4().hex, job_parameter_values={}) + try: + symtab = SymbolTable() + inner_env = Environment_2023_09( + name="InnerEnv", + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onEnter=_inner_env_action(cancelation), + ), + ), + ) + session._inject_wrapped_env_symbols( + symtab, inner_env, _inner_env_action(cancelation), inner_symtab=symtab + ) + return symtab + finally: + session.cleanup() + + def test_notify_then_terminate_defaults_to_30_for_env_action(self) -> None: + # Template Schemas 5.3.2: for anything other than a task's onRun — + # including an inner environment's onEnter — the default is 30. + symtab = self._inject(_notify(None)) + assert symtab["WrappedAction.Cancelation.Mode"] == "NOTIFY_THEN_TERMINATE" + assert symtab["WrappedAction.Cancelation.NotifyPeriodInSeconds"] == 30 + + def test_explicit_period_forwards_verbatim_for_env_action(self) -> None: + symtab = self._inject(_notify(45)) + assert symtab["WrappedAction.Cancelation.NotifyPeriodInSeconds"] == 45 + + def test_mode_none_when_env_action_has_no_cancelation(self) -> None: + symtab = self._inject(None) + assert symtab["WrappedAction.Cancelation.Mode"] is None + assert symtab["WrappedAction.Cancelation.NotifyPeriodInSeconds"] is None + + +# --------------------------------------------------------------------------- +# Integration tests — real subprocess, format-string interpolation +# --------------------------------------------------------------------------- + + +class TestWrapCancelationExecution: + """End-to-end: the wrap action interpolates the Cancelation variables in + a format string. The ``<...>`` sentinel wrap makes the null-renders-empty + behavior observable (``NP=<>``), matching the conformance fixtures.""" + + _PROBE = Action_2023_09( + command=CommandString_2023_09("sh"), + args=[ + ArgString_2023_09("-c"), + ArgString_2023_09( + 'echo "MODE=<{{WrappedAction.Cancelation.Mode}}>";' + ' echo "NP=<{{WrappedAction.Cancelation.NotifyPeriodInSeconds}}>"' + ), + ], + ) + + def _run(self, cancelation, caplog: pytest.LogCaptureFixture) -> str: + session_id = uuid.uuid4().hex + env = _wrap_env("wrap_env", self._PROBE) + step = _step_script_with_cancelation(cancelation) + with Session(session_id=session_id, job_parameter_values={}) as session: + session.enter_environment(environment=env) + _run_until_ready(session) + session.run_task(step_script=step, task_parameter_values={}) + _run_until_ready(session) + assert session.action_status == ActionStatus(state=ActionState.SUCCESS, exit_code=0) + return "\n".join(caplog.messages) + + def test_terminate_mode_renders_null_period_as_empty( + self, caplog: pytest.LogCaptureFixture + ) -> None: + messages = self._run(_TERMINATE, caplog) + assert "MODE=" in messages + # int? null interpolates as the empty string (RFC 0005), so the + # sentinel wrap renders as NP=<> — distinct from the old 0 sentinel, + # which would have rendered NP=<0>. + assert "NP=<>" in messages + + def test_notify_then_terminate_default_renders_120( + self, caplog: pytest.LogCaptureFixture + ) -> None: + messages = self._run(_notify(None), caplog) + assert "MODE=" in messages + assert "NP=<120>" in messages + + def test_no_cancelation_renders_null_mode_as_empty( + self, caplog: pytest.LogCaptureFixture + ) -> None: + # No declared: Mode is null (string?), which + # interpolates as the empty string (RFC 0005) — MODE=<>, matching + # the wrap-cancelation-mode-null-when-no-cancelation conformance + # fixture. The nullness itself (vs. an empty string value) is + # asserted by the unit tests above and observable via EXPR + # null-coalescing in the conformance fixture. + messages = self._run(None, caplog) + assert "MODE=<>" in messages + assert "NP=<>" in messages + + +# --------------------------------------------------------------------------- +# Unit tests — deferred-mode resolution (resolve_effective_cancelation) +# --------------------------------------------------------------------------- + + +class TestResolveEffectiveCancelation: + """Unit tests for the shared deferred-cancelation resolution helper. + + A CancelationMethodDeferred carries a format-string mode whose + TERMINATE-vs-NOTIFY_THEN_TERMINATE decision is made at run time + against the live symbol table (see resolve_effective_cancelation's + docstring for the full story). + """ + + def _deferred(self, mode: str, period: str | None = None): + from openjd.model._format_strings import FormatString + from openjd.model.v2023_09 import CancelationMethodDeferred, ModelParsingContext + + ctx = ModelParsingContext() + return CancelationMethodDeferred( + mode=FormatString(mode, context=ctx), + notifyPeriodInSeconds=( + FormatString(period, context=ctx) if period is not None else None + ), + ) + + def _symtab(self, **values) -> SymbolTable: + symtab = SymbolTable() + for key, value in values.items(): + symtab[key] = value + return symtab + + def test_mode_resolving_null_drops_whole_object(self) -> None: + from openjd.sessions._runner_base import resolve_effective_cancelation + + cancelation = self._deferred("{{X}}", "{{P}}") + result = resolve_effective_cancelation(cancelation, self._symtab(X=None, P=None)) + assert result == (None, None) + + def test_mode_resolving_terminate(self) -> None: + from openjd.sessions._runner_base import resolve_effective_cancelation + + cancelation = self._deferred("{{X}}") + result = resolve_effective_cancelation(cancelation, self._symtab(X="TERMINATE")) + assert result == ("TERMINATE", None) + + def test_mode_resolving_terminate_rejects_non_null_period(self) -> None: + from openjd.sessions._runner_base import resolve_effective_cancelation + + cancelation = self._deferred("{{X}}", "{{P}}") + with pytest.raises(ValueError, match="does not accept"): + resolve_effective_cancelation(cancelation, self._symtab(X="TERMINATE", P=45)) + + def test_mode_resolving_notify_then_terminate_with_period(self) -> None: + from openjd.sessions._runner_base import resolve_effective_cancelation + + cancelation = self._deferred("{{X}}", "{{P}}") + result = resolve_effective_cancelation( + cancelation, self._symtab(X="NOTIFY_THEN_TERMINATE", P=45) + ) + assert result == ("NOTIFY_THEN_TERMINATE", 45) + + def test_mode_resolving_garbage_raises(self) -> None: + from openjd.sessions._runner_base import resolve_effective_cancelation + + cancelation = self._deferred("{{X}}") + with pytest.raises(ValueError, match="must resolve to"): + resolve_effective_cancelation(cancelation, self._symtab(X="SOMETHING_ELSE")) + + def test_partial_interpolation_mode_resolves_normally(self) -> None: + # Normal format string behavior (Template Schemas 5.3): partial + # interpolation is permitted; the resolved value is checked + # against the two mode names. + from openjd.sessions._runner_base import resolve_effective_cancelation + + cancelation = self._deferred("{{X}}_THEN_TERMINATE", "{{P}}") + result = resolve_effective_cancelation(cancelation, self._symtab(X="NOTIFY", P=45)) + assert result == ("NOTIFY_THEN_TERMINATE", 45) + + def test_partial_interpolation_mode_resolving_empty_raises(self) -> None: + # Null semantics (dropping the cancelation object) apply only to a + # whole-field expression. A normal format string that resolves to + # the empty string is not null — it is an invalid mode. + from openjd.sessions._runner_base import resolve_effective_cancelation + + cancelation = self._deferred("{{X}}{{Y}}") + with pytest.raises(ValueError, match="must resolve to"): + resolve_effective_cancelation(cancelation, self._symtab(X=None, Y=None)) + + def test_resolved_period_exceeding_cap_raises(self) -> None: + # The static validator caps literal periods at 600 (Template + # Schemas 5.3.2); format-string values could not be checked at + # parse time, so the resolved value is bounds-checked at run time. + from openjd.sessions._runner_base import resolve_effective_cancelation + + cancelation = self._deferred("{{X}}", "{{P}}") + with pytest.raises(ValueError, match="between 1 and 600"): + resolve_effective_cancelation( + cancelation, self._symtab(X="NOTIFY_THEN_TERMINATE", P=9999) + ) diff --git a/test/openjd/sessions_v0/test_wrap_task_run.py b/test/openjd/sessions_v0/test_wrap_task_run.py new file mode 100644 index 00000000..ab2f25c3 --- /dev/null +++ b/test/openjd/sessions_v0/test_wrap_task_run.py @@ -0,0 +1,436 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Tests for the ``onWrapTaskRun`` environment action (RFC 0008). + +These tests exercise the end-to-end behaviour that a job template's ``onRun`` is +intercepted and the active environment's ``onWrapTaskRun`` is executed in its +place, with ``WrappedAction.Command``, ``WrappedAction.Args``, +``WrappedAction.Environment``, ``WrappedAction.Timeout``, and +``WrappedStep.Name`` injected into the wrap action's symbol table. +""" + +from __future__ import annotations + +import time +import uuid + +import pytest + +from openjd.model import SymbolTable +from openjd.model.v2023_09 import ( + Action as Action_2023_09, + ArgString as ArgString_2023_09, + CommandString as CommandString_2023_09, + Environment as Environment_2023_09, + EnvironmentActions as EnvironmentActions_2023_09, + EnvironmentScript as EnvironmentScript_2023_09, + StepActions as StepActions_2023_09, + StepScript as StepScript_2023_09, +) +from openjd.sessions import ActionState, ActionStatus, Session, SessionState + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +_NOOP = Action_2023_09(command=CommandString_2023_09("true")) + + +def _wrap_env(name: str, wrap_action: Action_2023_09) -> Environment_2023_09: + """Build an Environment with ``onWrapTaskRun`` set to ``wrap_action`` + and the other two wrap hooks set to no-ops (all-or-nothing rule).""" + return Environment_2023_09( + name=name, + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onWrapEnvEnter=_NOOP, + onWrapTaskRun=wrap_action, + onWrapEnvExit=_NOOP, + ), + ), + ) + + +def _step_script(command: str, args: list[str]) -> StepScript_2023_09: + """Build a minimal StepScript that runs ``command`` with ``args``.""" + return StepScript_2023_09( + actions=StepActions_2023_09( + onRun=Action_2023_09( + command=CommandString_2023_09(command), + args=[ArgString_2023_09(a) for a in args] if args else None, + ) + ) + ) + + +def _run_until_ready(session: Session, timeout_s: float = 10.0) -> None: + """Block until the session transitions back to READY or the timeout elapses.""" + deadline = time.time() + timeout_s + while session.state == SessionState.RUNNING and time.time() < deadline: + time.sleep(0.05) + + +# --------------------------------------------------------------------------- +# Unit tests on the pure helpers — no subprocess needed +# --------------------------------------------------------------------------- + + +class TestInjectWrappedTaskSymbols: + """Unit tests for the ``_inject_wrapped_task_symbols`` helper.""" + + def test_injects_wrapped_command_and_args_as_list(self) -> None: + session = Session(session_id=uuid.uuid4().hex, job_parameter_values={}) + try: + symtab = SymbolTable() + script = _step_script("python3", ["-c", "print('hi')"]) + session._inject_wrapped_task_symbols(symtab, script, "MyStep", inner_symtab=symtab) + + assert symtab["WrappedAction.Command"] == "python3" + assert symtab["WrappedAction.Args"] == ["-c", "print('hi')"] + assert isinstance(symtab["WrappedAction.Args"], list) + assert symtab["WrappedStep.Name"] == "MyStep" + finally: + session.cleanup() + + def test_injects_empty_args_when_step_has_no_args(self) -> None: + session = Session(session_id=uuid.uuid4().hex, job_parameter_values={}) + try: + symtab = SymbolTable() + script = _step_script("whoami", []) + session._inject_wrapped_task_symbols(symtab, script, "Step1", inner_symtab=symtab) + + assert symtab["WrappedAction.Command"] == "whoami" + assert symtab["WrappedAction.Args"] == [] + finally: + session.cleanup() + + def test_injects_wrapped_environment_as_key_value_list(self) -> None: + # RFC 0008 (openjd-rs #277): WrappedAction.Environment carries every + # session-defined variable — openjd_env definitions (applied via + # simplify_ordered_changes) AND the declarative variables:-map seed. + session = Session(session_id=uuid.uuid4().hex, job_parameter_values={}) + try: + from openjd.sessions._session import ( + EnvironmentVariableSetChange, + SimplifiedEnvironmentVariableChanges, + ) + + fake_id = "env-1" + session._environments_entered.append(fake_id) + changes = SimplifiedEnvironmentVariableChanges({"DECLARED": "from-variables-map"}) + changes.simplify_ordered_changes( + [ + EnvironmentVariableSetChange(name="FOO", value="bar"), + EnvironmentVariableSetChange(name="BAZ", value="qux"), + ] + ) + session._created_env_vars[fake_id] = changes + + symtab = SymbolTable() + script = _step_script("echo", ["hi"]) + session._inject_wrapped_task_symbols(symtab, script, "Step1", inner_symtab=symtab) + + task_env = symtab["WrappedAction.Environment"] + assert isinstance(task_env, list) + normalized = {item.upper() for item in task_env} + # Both openjd_env-set variables and the variables:-map seed are + # surfaced (openjd-rs #277). + assert normalized == { + "DECLARED=from-variables-map".upper(), + "FOO=bar".upper(), + "BAZ=qux".upper(), + } + finally: + session.cleanup() + + def test_injects_timeout_none_when_no_timeout(self) -> None: + # WrappedAction.Timeout is int? (RFC 0008): None (null) when the + # wrapped action specifies no timeout, so whole-field forwarding + # (timeout: "{{WrappedAction.Timeout}}") drops the field. + session = Session(session_id=uuid.uuid4().hex, job_parameter_values={}) + try: + symtab = SymbolTable() + script = _step_script("echo", ["hi"]) + session._inject_wrapped_task_symbols(symtab, script, "Step1", inner_symtab=symtab) + + assert symtab["WrappedAction.Timeout"] is None + finally: + session.cleanup() + + def test_find_wrap_environment_returns_none_when_no_env_has_wrap(self) -> None: + session = Session(session_id=uuid.uuid4().hex, job_parameter_values={}) + try: + assert session._find_wrap_environment(hook="onWrapTaskRun") is None + finally: + session.cleanup() + + +# --------------------------------------------------------------------------- +# Integration tests — actually run a subprocess wrap action +# --------------------------------------------------------------------------- + + +class TestWrapTaskRunExecution: + """Integration tests that enter an environment with ``onWrapTaskRun`` and + run a task through it, verifying the wrap action is what actually ran.""" + + def test_no_wrap_runs_original_step(self, caplog: pytest.LogCaptureFixture) -> None: + # GIVEN: a session with no environment defining onWrapTaskRun. + session_id = uuid.uuid4().hex + step = _step_script("echo", ["original-step"]) + + # WHEN: we run the task. + with Session(session_id=session_id, job_parameter_values={}) as session: + session.run_task(step_script=step, task_parameter_values={}) + _run_until_ready(session) + + # THEN: the original step ran, not a wrap action. + assert session.state == SessionState.READY + assert session.action_status == ActionStatus(state=ActionState.SUCCESS, exit_code=0) + assert any("original-step" in msg for msg in caplog.messages) + + def test_wrap_action_intercepts_step(self, caplog: pytest.LogCaptureFixture) -> None: + # GIVEN: an environment that defines onWrapTaskRun. + session_id = uuid.uuid4().hex + wrap = Action_2023_09( + command=CommandString_2023_09("sh"), + args=[ArgString_2023_09("-c"), ArgString_2023_09("echo WRAPPED-RAN")], + ) + env = _wrap_env("container_env", wrap) + + step = _step_script("echo", ["original-step"]) + + with Session(session_id=session_id, job_parameter_values={}) as session: + # Enter the wrap env (no onEnter, nothing runs but the env becomes active). + session.enter_environment(environment=env) + _run_until_ready(session) + + # WHEN: we run a task. + session.run_task(step_script=step, task_parameter_values={}) + _run_until_ready(session) + + # THEN: the wrap action ran, not the original step. + assert session.state == SessionState.READY + assert session.action_status == ActionStatus(state=ActionState.SUCCESS, exit_code=0) + messages = "\n".join(caplog.messages) + assert "WRAPPED-RAN" in messages + assert "original-step" not in messages + + def test_wrap_action_receives_wrapped_command_symbol( + self, caplog: pytest.LogCaptureFixture + ) -> None: + # GIVEN: a wrap action that echoes WrappedAction.Command via a format string. + session_id = uuid.uuid4().hex + wrap = Action_2023_09( + command=CommandString_2023_09("sh"), + args=[ + ArgString_2023_09("-c"), + ArgString_2023_09("echo CMD={{WrappedAction.Command}}"), + ], + ) + env = _wrap_env("container_env", wrap) + step = _step_script("maya-batch", ["-render", "scene.ma"]) + + with Session(session_id=session_id, job_parameter_values={}) as session: + session.enter_environment(environment=env) + _run_until_ready(session) + + session.run_task(step_script=step, task_parameter_values={}) + _run_until_ready(session) + + assert session.action_status == ActionStatus(state=ActionState.SUCCESS, exit_code=0) + messages = "\n".join(caplog.messages) + assert "CMD=maya-batch" in messages + + def test_innermost_environment_wins(self, caplog: pytest.LogCaptureFixture) -> None: + # GIVEN: two environments — only the inner one defines onWrapTaskRun. + session_id = uuid.uuid4().hex + outer = Environment_2023_09( + name="outer", + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onEnter=Action_2023_09( + command=CommandString_2023_09("sh"), + args=[ + ArgString_2023_09("-c"), + ArgString_2023_09("echo outer-enter"), + ], + ) + ) + ), + ) + inner = _wrap_env( + "inner", + Action_2023_09( + command=CommandString_2023_09("sh"), + args=[ArgString_2023_09("-c"), ArgString_2023_09("echo INNER-WRAP")], + ), + ) + step = _step_script("echo", ["original"]) + + with Session(session_id=session_id, job_parameter_values={}) as session: + session.enter_environment(environment=outer) + _run_until_ready(session) + session.enter_environment(environment=inner) + _run_until_ready(session) + + session.run_task(step_script=step, task_parameter_values={}) + _run_until_ready(session) + + assert session.action_status == ActionStatus(state=ActionState.SUCCESS, exit_code=0) + messages = "\n".join(caplog.messages) + assert "INNER-WRAP" in messages + assert "original" not in messages + + def test_wrap_action_runs_multiple_tasks(self, caplog: pytest.LogCaptureFixture) -> None: + # Regression: symbols are re-injected cleanly per task. + session_id = uuid.uuid4().hex + wrap = Action_2023_09( + command=CommandString_2023_09("sh"), + args=[ + ArgString_2023_09("-c"), + ArgString_2023_09("echo RAN={{WrappedAction.Command}}"), + ], + ) + env = _wrap_env("env", wrap) + + with Session(session_id=session_id, job_parameter_values={}) as session: + session.enter_environment(environment=env) + _run_until_ready(session) + + session.run_task(step_script=_step_script("cmd-one", []), task_parameter_values={}) + _run_until_ready(session) + session.run_task(step_script=_step_script("cmd-two", []), task_parameter_values={}) + _run_until_ready(session) + + messages = "\n".join(caplog.messages) + assert "RAN=cmd-one" in messages + assert "RAN=cmd-two" in messages + + +# --------------------------------------------------------------------------- +# Security and execution constraints — RFC 0008 test matrix +# --------------------------------------------------------------------------- +# +# These tests cover the 8 scenarios the RFC calls out under "Recommended test +# cases for implementation". They verify that the symbol injection layer +# preserves WrappedAction.Command and WrappedAction.Args byte-for-byte — no +# shell expansion, no quoting transformation, no truncation. + + +class TestSecurityAndExecutionConstraints: + """RFC 0008 §"Recommended test cases for implementation" matrix.""" + + def _inject(self, command: str, args: list[str]) -> tuple[str, list[str]]: + """Helper: run the injection and return the resolved WrappedAction.Command/Args.""" + session = Session(session_id=uuid.uuid4().hex, job_parameter_values={}) + try: + symtab = SymbolTable() + script = _step_script(command, args) + session._inject_wrapped_task_symbols(symtab, script, "TestStep", inner_symtab=symtab) + return symtab["WrappedAction.Command"], symtab["WrappedAction.Args"] + finally: + session.cleanup() + + # --- 1. Nested quoting ------------------------------------------------ + + def test_nested_quoting_preserved(self) -> None: + # echo "O'Reilly's Guide" — both " and ' must survive injection. + cmd, args = self._inject("echo", ["O'Reilly's Guide"]) + assert cmd == "echo" + assert args == ["O'Reilly's Guide"] + + def test_double_quotes_preserved(self) -> None: + cmd, args = self._inject("python3", ["-c", 'print("hello world")']) + assert cmd == "python3" + assert args == ["-c", 'print("hello world")'] + + # --- 2. Shell metacharacters ------------------------------------------ + + def test_shell_metacharacters_preserved(self) -> None: + # `, $, |, &&, ;, >, <, *, ?, (, ), \, all in one arg. + nasty = "$(cat /etc/passwd); `id`; rm -rf / || echo pwned" + cmd, args = self._inject("echo", [nasty, "a|b", "c&&d", "e;f"]) + assert cmd == "echo" + assert args == [nasty, "a|b", "c&&d", "e;f"] + + def test_backticks_preserved(self) -> None: + cmd, args = self._inject("echo", ["`whoami`", "$USER"]) + assert args == ["`whoami`", "$USER"] + # The $ must NOT have been expanded by the runtime. + assert "$USER" in args + + # --- 3. Path traversal ------------------------------------------------ + + def test_path_traversal_preserved_literally(self) -> None: + # The runtime must not resolve or reject `../` paths — the wrap + # script + container boundary is what prevents escape. + cmd, args = self._inject("cat", ["../../../etc/passwd", "/tmp/../etc/shadow"]) + assert cmd == "cat" + assert args == ["../../../etc/passwd", "/tmp/../etc/shadow"] + + # --- 4. Shell globbing ------------------------------------------------ + + def test_glob_characters_preserved(self) -> None: + # ls *.txt must be passed literally — no expansion at injection time. + cmd, args = self._inject("ls", ["*.txt", "?.log", "[abc]*"]) + assert cmd == "ls" + assert args == ["*.txt", "?.log", "[abc]*"] + + # --- 5. Unicode paths ------------------------------------------------- + + def test_unicode_cjk_preserved(self) -> None: + path = "/projects/映画/シーン01/レンダー.exr" + cmd, args = self._inject("render", ["--scene", path]) + assert cmd == "render" + assert args == ["--scene", path] + assert len(path.encode("utf-8")) > len(path) + + def test_unicode_emoji_preserved(self) -> None: + cmd, args = self._inject("echo", ["🎬 render 🎥", "🔥"]) + assert args == ["🎬 render 🎥", "🔥"] + + def test_unicode_mixed_scripts_preserved(self) -> None: + cmd, args = self._inject( + "tool", + ["--русский", "--中文", "--日本語", "--한국어", "--العربية"], + ) + assert args == ["--русский", "--中文", "--日本語", "--한국어", "--العربية"] + + # --- 6. Empty and whitespace-only arguments --------------------------- + + def test_empty_string_arg_preserved(self) -> None: + cmd, args = self._inject("echo", ["before", "", "after"]) + assert cmd == "echo" + assert args == ["before", "", "after"] + assert len(args) == 3 + + def test_whitespace_only_arg_preserved(self) -> None: + cmd, args = self._inject("echo", [" ", " ", " "]) + assert args == [" ", " ", " "] + + # --- 7. Newlines in arguments ----------------------------------------- + + def test_newline_in_arg_preserved(self) -> None: + # The EXPR extension relaxes the ArgString regex to accept + # control characters; the runtime must preserve them literally. + cmd, args = self._inject("echo", ["line1\nline2"]) + assert cmd == "echo" + assert args == ["line1\nline2"] + + # --- 8. Near-limit command length ------------------------------------- + + def test_large_argument_preserved(self) -> None: + big = "x" * (100 * 1024) + cmd, args = self._inject("cat", ["--data", big]) + assert cmd == "cat" + assert args == ["--data", big] + assert len(args[1]) == 100 * 1024 + + def test_many_arguments_preserved(self) -> None: + many = [f"arg{i}" for i in range(1000)] + cmd, args = self._inject("tool", many) + assert cmd == "tool" + assert args == many + assert len(args) == 1000 From 1a5f8fcfa2b0d554e194204ad7e48f38b0eb8997 Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Thu, 23 Jul 2026 08:34:10 -0700 Subject: [PATCH 02/27] =?UTF-8?q?fix:=20Address=20review=20findings=20?= =?UTF-8?q?=E2=80=94=20let-binding=20crash=20paths,=20Step.Name,=20perf?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review fixes on top of the RFC 0008 wrap-actions commit: - enter_environment/exit_environment no longer raise a raw ExpressionError out of the public API when an extra `let` binding fails to evaluate: the action fails through _fail_action_before_start with the callback, leaving the environment entered-but-failed exactly like a failed onEnter subprocess. - enter_environment(step_name=...) seeds Step.Name (RFC 0007 §7.3.1) before the extra bindings evaluate, remembered per environment and re-seeded at exit — openjd-rs parity (the Rust runtime threads the per-step resolved symtab into both enter and exit). - let-binding RHS parsing memoized and single-sourced via openjd.model.evaluate_let_bindings (was re-parsed through the Rust engine per env-enter/exit/task). - Wrap environment embedded-file paths allocated once per environment and reused across wrap-hook invocations (stable Env.File.*, no O(task-count) temp-file growth); contents still re-resolved per task. - One resolve_optional_int_field(ge=, le=) replaces three disagreeing optional-int resolvers — this also adds the previously missing non-positive check to Session._resolve_action_timeout (openjd-rs parity). - Complexity/duplication reductions: _fail_action, shared runner cancel() via _cancel_with_effective_cancelation, _make_env_script_runner, _try_inject_wrapped_symbols, WRAP_HOOK_ACTION_NAMES single-sourced, SimplifiedEnvironmentVariableChanges.effective_items() (no more private poking from _collect_session_env_list), whole-field detection via FormatString.whole_field_expression(). - pyproject: note that the openjd-model pin floor must be the first release containing model PR #318's surface. Tests: 647 passed (12 new regression tests: failure paths, Step.Name enter+exit, parse memoization, int-field bounds, wrap-file reuse); pre-existing environment failures unchanged. mypy/black/ruff clean. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- pyproject.toml | 5 + src/openjd/sessions/_embedded_files.py | 63 ++- src/openjd/sessions/_runner_base.py | 282 ++++++------ src/openjd/sessions/_runner_env_script.py | 72 ++- src/openjd/sessions/_runner_step_script.py | 45 +- src/openjd/sessions/_session.py | 411 ++++++++++++------ .../sessions_v0/test_runner_env_script.py | 6 +- .../sessions_v0/test_runner_step_script.py | 6 +- .../sessions_v0/test_session_let_bindings.py | 269 ++++++++++++ test/openjd/sessions_v0/test_wrap_task_run.py | 83 ++++ 10 files changed, 891 insertions(+), 351 deletions(-) create mode 100644 test/openjd/sessions_v0/test_session_let_bindings.py diff --git a/pyproject.toml b/pyproject.toml index e5277e8c..da8cf4ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,11 @@ classifiers = [ "Topic :: Software Development :: Libraries" ] dependencies = [ + # NOTE (version-skew guard): this branch requires openjd-model features that + # only exist as of openjd-model-for-python PR #318 (FormatString.resolve_value, + # FormatString.whole_field_expression, evaluate_let_bindings, Step.let). + # Before release, bump the version floor below to the first openjd-model + # release that includes PR #318's surface. "openjd-model >= 0.10,< 0.12", "pywin32 >= 307; platform_system == 'Windows'", "psutil >= 5.9,< 7.3; platform_system == 'Windows'", diff --git a/src/openjd/sessions/_embedded_files.py b/src/openjd/sessions/_embedded_files.py index 7cc5b01d..6c90ce24 100644 --- a/src/openjd/sessions/_embedded_files.py +++ b/src/openjd/sessions/_embedded_files.py @@ -231,6 +231,22 @@ def allocate_file_paths( else: self._logger.info("Writing embedded files for Task to disk.") + records = self.allocate_records(files) + self._define_symbols(records, symtab) + return records + + def allocate_records(self, files: EmbeddedFilesListType) -> list["_FileRecord"]: + """Allocate the on-disk paths for the embedded files, without logging + or defining any symbols. + + Used by the Session to pre-allocate a wrap environment's file records + once (RFC 0008); each wrap-hook invocation then defines the symbols + and logs through :meth:`register_file_paths` against its own symbol + table. + + Raises: + RuntimeError: If a file path could not be allocated. + """ try: records = list[_FileRecord]() # Generate the symbol table values and filenames @@ -238,25 +254,42 @@ def allocate_file_paths( # Raises: OSError symbol, filename = self._get_symtab_entry(file) records.append(_FileRecord(symbol=symbol, filename=filename, file=file)) - - # Add symbols to the symbol table. For EXPR evaluation the - # Env.File.*/Task.File.* symbols are host-format path values - # (property access like `.parent` works), matching openjd-rs; - # the legacy (non-EXPR) interpolation path ignores the type and - # keeps the string form. - for record in records: - symtab[record.symbol] = str(record.filename) - symtab.expr_types[record.symbol] = "PATH" - self._logger.info( - f"Mapping: {record.symbol} -> {record.filename}", - extra=LogExtraInfo( - openjd_log_content=LogContent.FILE_PATH | LogContent.PARAMETER_INFO - ), - ) return records except (OSError, ValueError) as err: raise RuntimeError(f"Could not write embedded file: {err}") + def register_file_paths(self, records: list["_FileRecord"], symtab: SymbolTable) -> None: + """Define the ``Env.File.*``/``Task.File.*`` symbols in ``symtab`` + for already-allocated records, without allocating new paths. + + Used to reuse a wrap environment's embedded-file records across + wrap-hook invocations (RFC 0008): the on-disk paths are allocated + once so the symbols stay stable and unnamed files do not accumulate, + while the contents are re-resolved and rewritten per invocation via + :meth:`write_file_contents`. + """ + if self._scope == EmbeddedFilesScope.ENV: + self._logger.info("Writing embedded files for Environment to disk.") + else: + self._logger.info("Writing embedded files for Task to disk.") + self._define_symbols(records, symtab) + + def _define_symbols(self, records: list["_FileRecord"], symtab: SymbolTable) -> None: + # Add symbols to the symbol table. For EXPR evaluation the + # Env.File.*/Task.File.* symbols are host-format path values + # (property access like `.parent` works), matching openjd-rs; + # the legacy (non-EXPR) interpolation path ignores the type and + # keeps the string form. + for record in records: + symtab[record.symbol] = str(record.filename) + symtab.expr_types[record.symbol] = "PATH" + self._logger.info( + f"Mapping: {record.symbol} -> {record.filename}", + extra=LogExtraInfo( + openjd_log_content=LogContent.FILE_PATH | LogContent.PARAMETER_INFO + ), + ) + def write_file_contents(self, records: list["_FileRecord"], symtab: SymbolTable) -> None: """Resolve each allocated file's ``data`` against ``symtab`` and write it to disk. See :meth:`allocate_file_paths`.""" diff --git a/src/openjd/sessions/_runner_base.py b/src/openjd/sessions/_runner_base.py index eda87e4c..914d1113 100644 --- a/src/openjd/sessions/_runner_base.py +++ b/src/openjd/sessions/_runner_base.py @@ -17,12 +17,11 @@ from openjd.model import SymbolTable from openjd.model import FormatStringError +from openjd.model import evaluate_let_bindings from openjd.model.v2023_09 import Action as Action_2023_09 -from openjd.model.v2023_09 import ArgString as ArgString_2023_09 from openjd.model.v2023_09 import CancelationMethodDeferred as CancelationMethodDeferred_2023_09 from openjd.model.v2023_09 import CancelationMode as CancelationMode_2023_09 -from openjd.model.v2023_09 import ModelParsingContext as ModelParsingContext_2023_09 -from ._embedded_files import EmbeddedFiles, EmbeddedFilesScope, write_file_for_user +from ._embedded_files import EmbeddedFiles, EmbeddedFilesScope, _FileRecord, write_file_for_user from ._logging import log_subsection_banner, LoggerAdapter, LogContent, LogExtraInfo from ._os_checker import is_posix from ._session_user import SessionUser @@ -37,6 +36,8 @@ "NotifyCancelMethod", "ScriptRunnerBase", "apply_let_bindings", + "resolve_effective_cancelation", + "resolve_optional_int_field", ) @@ -95,6 +96,58 @@ class NotifyCancelMethod(CancelMethod): """Amount of time after a SIGTERM to wait to do the SIGKILL""" +def resolve_optional_int_field( + value: Any, + symtab: SymbolTable, + *, + ge: Optional[int] = None, + le: Optional[int] = None, + description: str, +) -> Optional[int]: + """Resolve an optional int-or-format-string field (e.g. an action's + ``timeout`` or a cancelation's ``notifyPeriodInSeconds``) into an + optional integer. + + - ``None`` (field omitted) stays ``None``. + - A literal ``int`` passes through unchecked: literal values were + bounds-checked by the static validator at parse time. + - A FormatString (FEATURE_BUNDLE_1) is resolved against ``symtab``. A + whole-field expression that resolves to null renders as the empty + string and is treated as if the field were not provided (``None`` — + the caller applies any positional schema default). Otherwise the + resolved value must be an integer within the given bounds; the bounds + apply here because format-string values could not be checked at parse + time. + + Raises: + ValueError: If the resolved value is not an integer, or violates + the ``ge``/``le`` bounds. + FormatStringError: If expression resolution itself fails. + """ + if value is None: + return None + if isinstance(value, int): + return value + resolved = value.resolve(symtab=symtab) + if resolved == "": + return None + if ge is not None and le is not None: + constraint = f"between {ge} and {le}" + elif ge == 1 and le is None: + constraint = "a positive integer" + elif ge is not None: + constraint = f"an integer >= {ge}" + else: + constraint = "an integer" + try: + result = int(resolved) + except ValueError: + raise ValueError(f"{description} must be {constraint}, got '{resolved}'") + if (ge is not None and result < ge) or (le is not None and result > le): + raise ValueError(f"{description} must be {constraint}, got '{result}'") + return result + + def resolve_effective_cancelation( cancelation: Any, symtab: SymbolTable ) -> tuple[Optional[str], Optional[int]]: @@ -133,23 +186,11 @@ def resolve_effective_cancelation( """ def resolve_period(period: Any) -> Optional[int]: - if period is None: - return None - if isinstance(period, int): - return period - # FormatString form (FEATURE_BUNDLE_1). A whole-field expression - # that resolves to null renders as the empty string and is treated - # as if the field were not provided (schema defaults apply). - resolved = period.resolve(symtab=symtab) - if resolved == "": - return None - value = int(resolved) # raises ValueError on non-integer - # Mirror the static validator's bounds on literal values (Template - # Schemas 5.3.2: 1..600): format-string values could not be checked - # at parse time, so the resolved value is bounds-checked here. - if value < 1 or value > 600: - raise ValueError(f"notifyPeriodInSeconds must be between 1 and 600, got '{value}'") - return value + # Bounds mirror the static validator's on literal values (Template + # Schemas 5.3.2: 1..600); see resolve_optional_int_field. + return resolve_optional_int_field( + period, symtab, ge=1, le=600, description="notifyPeriodInSeconds" + ) if cancelation is None: return (None, None) @@ -159,13 +200,7 @@ def resolve_period(period: Any) -> Optional[int]: # Template Schemas 5.3). A normal format string that happens to # resolve to the empty string is NOT null; it falls through to the # "must resolve to..." error below. - raw = str(cancelation.mode).strip() - is_whole_field = ( - raw.startswith("{{") - and raw.endswith("}}") - and raw.count("{{") == 1 - and raw.count("}}") == 1 - ) + is_whole_field = cancelation.mode.whole_field_expression() is not None mode = cancelation.mode.resolve(symtab=symtab) if mode == "" and is_whole_field: # Null mode drops the ENTIRE cancelation object: mode is the @@ -219,25 +254,9 @@ def apply_let_bindings(*, symtab: SymbolTable, let_bindings: list[str]) -> None: ValueError (FormatStringError/ExpressionError): if a binding's expression cannot be evaluated. """ - # `let` only appears under the EXPR extension; force it on for parsing - # the RHS regardless of the session's configured extension set. - context = ModelParsingContext_2023_09(supported_extensions=["EXPR"]) - - for binding in let_bindings: - name, sep, rhs = binding.partition("=") - name = name.strip() - rhs = rhs.strip() - if not sep or not name or not rhs: - # Malformed bindings are rejected by the model's `let` validator - # at decode time; skip defensively here. - continue - # Parse the RHS as a standalone EXPR expression and evaluate it - # against the current symbol table. - arg = ArgString_2023_09("{{ " + rhs + " }}", context=context) - expressions = arg.expressions - if not expressions or expressions[0].expression is None: - continue - symtab[name] = expressions[0].expression.evaluate_value(symtab=symtab) + # Single-sourced in openjd.model (parse-memoized; skips malformed + # bindings; raises ValueError naming the failing binding). + evaluate_let_bindings(symtab=symtab, let_bindings=let_bindings) class ScriptRunnerBase(ABC): @@ -395,6 +414,21 @@ def shutdown(self) -> None: """ self._pool.shutdown() + def _fail_action(self, message: str) -> None: + """Fail the action through the normal failure path: surface the + failure reason to the customer via the action filter + (``openjd_fail``), set the FAILED state override, and invoke the + callback. The subprocess's future may not have been started yet, + but the Session still needs to know that the action is over. + """ + self._logger.info( + f"openjd_fail: {message}", + extra=LogExtraInfo(openjd_log_content=LogContent.EXCEPTION_INFO), + ) + self._state_override = ScriptRunnerState.FAILED + if self._callback is not None: + self._callback(ActionState.FAILED) + @abstractmethod def cancel( self, *, time_limit: Optional[timedelta] = None, mark_action_failed: bool = False @@ -479,17 +513,7 @@ def _run(self, args: Sequence[str], time_limit: Optional[timedelta] = None) -> N args, self._user, self._os_env_vars, str(self._session_working_directory) ) except RuntimeError as e: - # Make use of the action filter to surface the failure reason to - # the customer. - self._logger.info( - f"openjd_fail: {str(e)}", - extra=LogExtraInfo(openjd_log_content=LogContent.EXCEPTION_INFO), - ) - self._state_override = ScriptRunnerState.FAILED - # We haven't started the future yet that runs the process, - # but the Session still needs to know that the action is over. - if self._callback is not None: - self._callback(ActionState.FAILED) + self._fail_action(str(e)) return subprocess_args = [filename] if is_posix() else args @@ -549,6 +573,7 @@ def _materialize_files( dest_directory: Path, symtab: SymbolTable, let_bindings: Optional[list[str]] = None, + preallocated_records: Optional[list[_FileRecord]] = None, ) -> None: """Helper for derived classes that wraps all of the logic around materializing embedded files to disk. @@ -559,6 +584,12 @@ def _materialize_files( are plain strings), so ``Env.File.*``/``Task.File.*`` are available to the bindings, while a file's ``data`` is written afterwards so it can reference let-bound values. + + When ``preallocated_records`` is given (RFC 0008: a wrap + environment's files, whose paths the Session allocates once and + reuses across wrap-hook invocations), path allocation is skipped; + the records' symbols are defined in ``symtab`` and the contents are + re-resolved and written as usual. """ file_writer = EmbeddedFiles( logger=self._logger, @@ -567,7 +598,11 @@ def _materialize_files( user=self._user, ) try: - records = file_writer.allocate_file_paths(files, symtab) + if preallocated_records is not None: + records = preallocated_records + file_writer.register_file_paths(records, symtab) + else: + records = file_writer.allocate_file_paths(files, symtab) if let_bindings: apply_let_bindings(symtab=symtab, let_bindings=let_bindings) file_writer.write_file_contents(records, symtab) @@ -575,17 +610,7 @@ def _materialize_files( # Had a problem writing at least one file to disk, or evaluating # a `let` binding (FormatStringError/ExpressionError subclass # ValueError). Surface the error. - # Make use of the action filter to surface the failure reason to - # the customer. - self._logger.info( - f"openjd_fail: {str(exc)}", - extra=LogExtraInfo(openjd_log_content=LogContent.EXCEPTION_INFO), - ) - self._state_override = ScriptRunnerState.FAILED - # We haven't started the future yet that runs the process, - # but the Session still needs to know that the action is over. - if self._callback is not None: - self._callback(ActionState.FAILED) + self._fail_action(str(exc)) def _apply_let_bindings_or_fail(self, symtab: SymbolTable, let_bindings: list[str]) -> bool: """Evaluate the script's EXPR ``let`` bindings into ``symtab``. On an @@ -594,13 +619,7 @@ def _apply_let_bindings_or_fail(self, symtab: SymbolTable, let_bindings: list[st try: apply_let_bindings(symtab=symtab, let_bindings=let_bindings) except ValueError as exc: - self._logger.info( - f"openjd_fail: {str(exc)}", - extra=LogExtraInfo(openjd_log_content=LogContent.EXCEPTION_INFO), - ) - self._state_override = ScriptRunnerState.FAILED - if self._callback is not None: - self._callback(ActionState.FAILED) + self._fail_action(str(exc)) return False return True @@ -646,57 +665,74 @@ def _run_action( except FormatStringError as exc: # Extremely unlikely since a JobTemplate needs to have passed # validation before we could be running it, but just to be safe. - self._logger.info( - f"openjd_fail: {str(exc)}", - extra=LogExtraInfo(openjd_log_content=LogContent.EXCEPTION_INFO), - ) - self._state_override = ScriptRunnerState.FAILED - # We haven't started the future yet that runs the process, - # but the Session still needs to know that the action is over. - if self._callback is not None: - self._callback(ActionState.FAILED) + self._fail_action(str(exc)) else: time_limit: Optional[timedelta] = default_timeout - if action.timeout is not None: - if isinstance(action.timeout, int): - time_limit = timedelta(seconds=action.timeout) - else: - # FormatString form (FEATURE_BUNDLE_1), resolved right - # before the action runs. A whole-field expression that - # resolves to null renders as the empty string — e.g. - # forwarding `timeout: "{{WrappedAction.Timeout}}"` - # (RFC 0008) when the wrapped action specified no - # timeout — and is treated as if the field were not - # provided, so the positional default applies. - try: - resolved = action.timeout.resolve(symtab=symtab) - except FormatStringError as exc: - self._logger.info( - f"openjd_fail: {str(exc)}", - extra=LogExtraInfo(openjd_log_content=LogContent.EXCEPTION_INFO), - ) - self._state_override = ScriptRunnerState.FAILED - if self._callback is not None: - self._callback(ActionState.FAILED) - return - if resolved != "": - try: - seconds = int(resolved) - if seconds < 1: - raise ValueError - except ValueError: - self._logger.info( - f"openjd_fail: timeout must be a positive integer, " - f"got '{resolved}'", - extra=LogExtraInfo(openjd_log_content=LogContent.EXCEPTION_INFO), - ) - self._state_override = ScriptRunnerState.FAILED - if self._callback is not None: - self._callback(ActionState.FAILED) - return - time_limit = timedelta(seconds=seconds) + # A FormatString timeout (FEATURE_BUNDLE_1) is resolved right + # before the action runs. A whole-field expression that + # resolves to null renders as the empty string — e.g. + # forwarding `timeout: "{{WrappedAction.Timeout}}"` + # (RFC 0008) when the wrapped action specified no + # timeout — and is treated as if the field were not + # provided, so the positional default applies. + try: + seconds = resolve_optional_int_field( + action.timeout, symtab, ge=1, description="timeout" + ) + except ValueError as exc: + # FormatStringError (resolution failure) subclasses + # ValueError, so this covers both a failed resolution and a + # non-positive-integer resolved value. + self._fail_action(str(exc)) + return + if seconds is not None: + time_limit = timedelta(seconds=seconds) self._run(command, time_limit) + def _cancel_with_effective_cancelation( + self, + *, + cancelation: Any, + symtab: SymbolTable, + default_notify_period_seconds: int, + time_limit: Optional[timedelta] = None, + mark_action_failed: bool = False, + ) -> None: + """Shared implementation of the runners' :meth:`cancel`. + + Resolve the action's cancelation config against the symbol table: a + deferred (format-string) mode and/or a FEATURE_BUNDLE_1 notify + period decide their values here, right when they are needed + (see resolve_effective_cancelation for the full story). A cancel + must always proceed, so resolution errors fall back to Terminate. + + ``default_notify_period_seconds`` is the Template Schemas 5.3.2 + positional default applied when a NOTIFY_THEN_TERMINATE cancelation + omits its notify period (120 for a task's onRun, 30 otherwise). + """ + try: + mode, period = resolve_effective_cancelation(cancelation, symtab) + except (ValueError, FormatStringError) as exc: + self._logger.warning( + f"Failed to resolve the action's cancelation; canceling by " + f"termination instead: {exc}" + ) + mode, period = (None, None) + + method: CancelMethod + if mode != CancelationMode_2023_09.NOTIFY_THEN_TERMINATE.value: + # Note: The default cancelation for a 2023-09 script is Terminate + method = TerminateCancelMethod() + else: + method = NotifyCancelMethod( + terminate_delay=timedelta( + seconds=(period if period is not None else default_notify_period_seconds) + ) + ) + + # Note: If the given time_limit is less than that in the method, then the time_limit will be what's used. + self._cancel(method, time_limit, mark_action_failed) + def _cancel( self, method: CancelMethod, diff --git a/src/openjd/sessions/_runner_env_script.py b/src/openjd/sessions/_runner_env_script.py index b0f65842..16d0aaf8 100644 --- a/src/openjd/sessions/_runner_env_script.py +++ b/src/openjd/sessions/_runner_env_script.py @@ -5,19 +5,14 @@ from pathlib import Path from typing import Callable, Optional -from openjd.model import FormatStringError, SymbolTable +from openjd.model import SymbolTable from openjd.model.v2023_09 import Action as Action_2023_09 -from openjd.model.v2023_09 import CancelationMode as CancelationMode_2023_09 from openjd.model.v2023_09 import EnvironmentScript as EnvironmentScript_2023_09 -from ._embedded_files import EmbeddedFilesScope +from ._embedded_files import EmbeddedFilesScope, _FileRecord from ._logging import log_subsection_banner from ._runner_base import ( - CancelMethod, - NotifyCancelMethod, ScriptRunnerBase, ScriptRunnerState, - TerminateCancelMethod, - resolve_effective_cancelation, ) from ._session_user import SessionUser from ._types import ( @@ -27,7 +22,13 @@ EnvironmentScriptModel, ) -__all__ = ("EnvironmentScriptRunner",) +__all__ = ("EnvironmentScriptRunner", "WRAP_HOOK_ACTION_NAMES") + + +WRAP_HOOK_ACTION_NAMES = ("onWrapEnvEnter", "onWrapTaskRun", "onWrapEnvExit") +"""The three RFC 0008 wrap-hook action names an Environment's script may +define. Single-sourced here for the runner's hook dispatch and the Session's +wrap-environment lookup/validation.""" _ENV_EXIT_DEFAULT_TIMEOUT = timedelta(minutes=5) @@ -58,6 +59,13 @@ class EnvironmentScriptRunner(ScriptRunnerBase): """If defined, then this is the action that is currently running, or was last run. """ + _preallocated_file_records: Optional[list[_FileRecord]] + """RFC 0008: the script's embedded-file records with on-disk paths already + allocated by the Session (a wrap environment's files are allocated once + and reused across wrap-hook invocations). When set, the runner skips path + allocation and only re-resolves/rewrites the file contents. + """ + def __init__( self, *, @@ -75,6 +83,9 @@ def __init__( symtab: SymbolTable, # Directory within which files/attachments should be materialized session_files_directory: Path, + # RFC 0008: pre-allocated embedded-file records to reuse (see + # _preallocated_file_records) + preallocated_file_records: Optional[list[_FileRecord]] = None, ): """ Arguments (from base class): @@ -94,6 +105,11 @@ def __init__( Script's scope (exluding any symbols defined within the Step Script itself). session_files_directory (Path): The location in the filesystem where embedded files will be materialized. + preallocated_file_records (Optional[list[_FileRecord]]): RFC 0008: the script's + embedded-file records with on-disk paths already allocated by the Session + (a wrap environment's files are allocated once and reused across wrap-hook + invocations). When given, the runner skips path allocation and only + re-resolves/rewrites the file contents. """ super().__init__( logger=logger, @@ -107,6 +123,7 @@ def __init__( self._symtab = symtab self._session_files_directory = session_files_directory self._action = None + self._preallocated_file_records = preallocated_file_records if self._environment_script and not isinstance( self._environment_script, EnvironmentScript_2023_09 @@ -144,6 +161,7 @@ def _run_env_action( self._session_files_directory, symtab, let_bindings=let_bindings, + preallocated_records=self._preallocated_file_records, ) if self.state == ScriptRunnerState.FAILED: return @@ -214,7 +232,7 @@ def wrap_env_exit(self) -> None: def _run_wrap_hook(self, hook: str, *, default_timeout: Optional[timedelta] = None) -> None: """Common dispatch for the three RFC 0008 wrap hooks. ``hook`` is one of ``onWrapEnvEnter``, ``onWrapTaskRun``, or ``onWrapEnvExit``.""" - if hook not in ("onWrapEnvEnter", "onWrapTaskRun", "onWrapEnvExit"): + if hook not in WRAP_HOOK_ACTION_NAMES: # Guard the getattr below: without this, a typo'd hook name # would silently become a SUCCESS no-op. raise ValueError(f"Unknown wrap hook name: {hook}") @@ -253,32 +271,10 @@ def cancel( # For the type checker assert isinstance(self._action, Action_2023_09) - # Resolve the cancelation config against the symbol table: a - # deferred (format-string) mode and/or a FEATURE_BUNDLE_1 notify - # period decide their values here, right when they are needed - # (see resolve_effective_cancelation for the full story). A cancel - # must always proceed, so resolution errors fall back to Terminate. - try: - mode, period = resolve_effective_cancelation(self._action.cancelation, self._symtab) - except (ValueError, FormatStringError) as exc: - self._logger.warning( - f"Failed to resolve the action's cancelation; canceling by " - f"termination instead: {exc}" - ) - mode, period = (None, None) - - method: CancelMethod - if mode != CancelationMode_2023_09.NOTIFY_THEN_TERMINATE.value: - # Note: Default cancelation for a 2023-09 Environment Script is Terminate - method = TerminateCancelMethod() - else: - method = NotifyCancelMethod( - terminate_delay=timedelta( - seconds=( - period if period is not None else ENV_ACTION_DEFAULT_NOTIFY_PERIOD_SECONDS - ) - ) - ) - - # Note: If the given time_limit is less than that in the method, then the time_limit will be what's used. - self._cancel(method, time_limit, mark_action_failed) + self._cancel_with_effective_cancelation( + cancelation=self._action.cancelation, + symtab=self._symtab, + default_notify_period_seconds=ENV_ACTION_DEFAULT_NOTIFY_PERIOD_SECONDS, + time_limit=time_limit, + mark_action_failed=mark_action_failed, + ) diff --git a/src/openjd/sessions/_runner_step_script.py b/src/openjd/sessions/_runner_step_script.py index 9700cbf3..9e926dfc 100644 --- a/src/openjd/sessions/_runner_step_script.py +++ b/src/openjd/sessions/_runner_step_script.py @@ -5,18 +5,13 @@ from pathlib import Path from typing import Callable, Optional -from openjd.model import FormatStringError, SymbolTable -from openjd.model.v2023_09 import CancelationMode as CancelationMode_2023_09 +from openjd.model import SymbolTable from openjd.model.v2023_09 import StepScript as StepScript_2023_09 from ._embedded_files import EmbeddedFilesScope from ._logging import log_subsection_banner from ._runner_base import ( - CancelMethod, - NotifyCancelMethod, ScriptRunnerBase, ScriptRunnerState, - TerminateCancelMethod, - resolve_effective_cancelation, ) from ._session_user import SessionUser from ._types import TASK_RUN_DEFAULT_NOTIFY_PERIOD_SECONDS, ActionState, StepScriptModel @@ -134,34 +129,10 @@ def cancel( # For the type checker. assert isinstance(self._script, StepScript_2023_09) - # Resolve the cancelation config against the symbol table: a - # deferred (format-string) mode and/or a FEATURE_BUNDLE_1 notify - # period decide their values here, right when they are needed - # (see resolve_effective_cancelation for the full story). A cancel - # must always proceed, so resolution errors fall back to Terminate. - try: - mode, period = resolve_effective_cancelation( - self._script.actions.onRun.cancelation, self._symtab - ) - except (ValueError, FormatStringError) as exc: - self._logger.warning( - f"Failed to resolve the action's cancelation; canceling by " - f"termination instead: {exc}" - ) - mode, period = (None, None) - - method: CancelMethod - if mode != CancelationMode_2023_09.NOTIFY_THEN_TERMINATE.value: - # Note: Default cancelation for a 2023-09 Step Script is Terminate - method = TerminateCancelMethod() - else: - method = NotifyCancelMethod( - terminate_delay=timedelta( - seconds=( - period if period is not None else TASK_RUN_DEFAULT_NOTIFY_PERIOD_SECONDS - ) - ) - ) - - # Note: If the given time_limit is less than that in the method, then the time_limit will be what's used. - self._cancel(method, time_limit, mark_action_failed) + self._cancel_with_effective_cancelation( + cancelation=self._script.actions.onRun.cancelation, + symtab=self._symtab, + default_notify_period_seconds=TASK_RUN_DEFAULT_NOTIFY_PERIOD_SECONDS, + time_limit=time_limit, + mark_action_failed=mark_action_failed, + ) diff --git a/src/openjd/sessions/_session.py b/src/openjd/sessions/_session.py index 04d554af..bee4cae5 100644 --- a/src/openjd/sessions/_session.py +++ b/src/openjd/sessions/_session.py @@ -38,12 +38,17 @@ ValueReferenceConstants as ValueReferenceConstants_2023_09, ) from ._action_filter import ActionMessageKind, ActionMonitoringFilter -from ._embedded_files import EmbeddedFiles, EmbeddedFilesScope, write_file_for_user +from ._embedded_files import EmbeddedFiles, EmbeddedFilesScope, _FileRecord, write_file_for_user from ._logging import LOG, log_section_banner, LoggerAdapter, LogExtraInfo, LogContent from ._os_checker import is_posix, is_windows from ._path_mapping import PathMappingRule -from ._runner_base import ScriptRunnerBase, apply_let_bindings, resolve_effective_cancelation -from ._runner_env_script import EnvironmentScriptRunner +from ._runner_base import ( + ScriptRunnerBase, + apply_let_bindings, + resolve_effective_cancelation, + resolve_optional_int_field, +) +from ._runner_env_script import EnvironmentScriptRunner, WRAP_HOOK_ACTION_NAMES from ._runner_step_script import StepScriptRunner from ._session_user import SessionUser from ._subprocess import LoggingSubprocess @@ -54,6 +59,7 @@ ActionState, EnvironmentIdentifier, EnvironmentModel, + EnvironmentScriptModel, StepScriptModel, ) from ._version import version @@ -163,6 +169,13 @@ def simplify_ordered_changes(self, changes: list[EnvironmentVariableChange]) -> else: raise ValueError("Unknown type of environment variable change.") + def effective_items(self) -> dict[str, Optional[str]]: + """The effective session-defined variables for this environment, in + insertion order: the declarative ``variables:`` map seed plus any + openjd_env stdout sets/unsets applied so far (``None`` = unset). + Returns a copy; mutating it does not affect this object.""" + return dict(self._to_set) + def apply_to_environment(self, env_vars: dict[str, Optional[str]]) -> None: """Modify a given dictionary of environment variables to reflect the changes""" if is_windows(): @@ -265,6 +278,15 @@ class Session(object): """OS environment variables defined by Open Job Description Environments """ + _wrap_env_file_records: dict[EnvironmentIdentifier, list["_FileRecord"]] + """RFC 0008: per wrap environment, the embedded-file records whose on-disk + paths were allocated on the environment's first wrap-hook invocation and + are reused for every subsequent invocation — so the ``Env.File.*`` symbols + stay stable across tasks and unnamed files do not accumulate on disk. The + file *contents* are still re-resolved and rewritten per invocation by the + runner (a file's ``data`` may reference ``WrappedAction.*``). + """ + _log_filter: Filter """The handler that we've hooked to the LOG. Removed when the Session is deleted. """ @@ -401,11 +423,17 @@ def __init__( # (e.g. the owning step's step-level bindings), re-applied when the # environment exits so its onExit resolves in the same scope. self._environment_extra_let_bindings: dict[EnvironmentIdentifier, list[str]] = dict() + # The owning step's name supplied when an environment was entered + # (seeds Step.Name, RFC 0007 EXPR), re-seeded when the environment + # exits so the re-applied extra `let` bindings resolve in the same + # scope. + self._environment_step_names: dict[EnvironmentIdentifier, str] = dict() self._environments_entered = list() self._runner = None self._running_environment_identifier = None self._process_env = dict(os_env_vars) if os_env_vars else dict() self._created_env_vars = dict() + self._wrap_env_file_records = dict() self._retain_working_dir = retain_working_dir self._user = user self._job_parameter_values = dict(job_parameter_values) if job_parameter_values else dict() @@ -630,6 +658,32 @@ def cancel_action( self._runner.cancel(time_limit=time_limit, mark_action_failed=mark_action_failed) + def _make_env_script_runner( + self, + *, + environment_script: Optional[EnvironmentScriptModel], + os_env_vars: dict[str, Optional[str]], + symtab: SymbolTable, + preallocated_file_records: Optional[list[_FileRecord]] = None, + ) -> EnvironmentScriptRunner: + """Construct an :class:`EnvironmentScriptRunner` with this Session's + standard wiring (logger, user, working/files directories, and action + callback). Only the script, subprocess environment, symbol table, and + (for wrap hooks) pre-allocated embedded-file records vary between the + call sites.""" + return EnvironmentScriptRunner( + logger=self._logger, + user=self._user, + os_env_vars=os_env_vars, + session_working_directory=self.working_directory, + startup_directory=self.working_directory, + callback=self._action_callback, + environment_script=environment_script, + symtab=symtab, + session_files_directory=self.files_directory, + preallocated_file_records=preallocated_file_records, + ) + def enter_environment( self, *, @@ -637,6 +691,7 @@ def enter_environment( identifier: Optional[EnvironmentIdentifier] = None, os_env_vars: Optional[dict[str, str]] = None, extra_let_bindings: Optional[list[str]] = None, + step_name: Optional[str] = None, ) -> EnvironmentIdentifier: """Enters an Open Job Description Environment within this Session. This method is non-blocking; it will exit when the subprocess is either confirmed to have @@ -662,6 +717,13 @@ def enter_environment( (``Step.let`` on the instantiated Job) so both can reference them — the v0 counterpart of the per-step resolved symbol table that openjd-rs threads into enter_environment. + step_name (Optional[str]): The name of the step whose + stepEnvironments are being entered, if any. Seeds + ``Step.Name`` (RFC 0007 EXPR) into the symbol table before + the extra ``let`` bindings evaluate, so step-level bindings + and the environment's variables and actions can reference + it — openjd-rs threads a per-step resolved symbol table into + environment entry, and this is the v0 counterpart. Returns: EnvironmentIdentifier: An identifier by which the Environment is known by to this Session. @@ -698,16 +760,41 @@ def enter_environment( self._environments[identifier] = environment if extra_let_bindings: self._environment_extra_let_bindings[identifier] = list(extra_let_bindings) + if step_name is not None: + self._environment_step_names[identifier] = step_name self._environments_entered.append(identifier) self._running_environment_identifier = identifier symtab = self._symbol_table(environment.revision) + # RFC 0007 §7.3.1 (EXPR): the owning step's name. Only EXPR templates + # pass validation referencing Step.Name, so seeding it when known does + # not change non-EXPR behavior. Seeded before the extra `let` bindings + # evaluate so a step-level binding may reference it. + if step_name is not None: + symtab["Step.Name"] = step_name + # Step-level `let` bindings (RFC 0007) accompany a step's # environments: evaluate them first so the environment's variables # and actions can reference them. if extra_let_bindings: - apply_let_bindings(symtab=symtab, let_bindings=extra_let_bindings) + try: + apply_let_bindings(symtab=symtab, let_bindings=extra_let_bindings) + except ValueError as e: + # ExpressionError and FormatStringError subclass ValueError: + # a binding failed to evaluate (e.g. it referenced an + # undefined symbol). Fail the action through the normal + # failure path rather than raising out of the public API — + # the environment stays in the entered list, exactly as when + # a failing onEnter subprocess leaves it, so the caller's + # cleanup exits it as usual. + self._created_env_vars[identifier] = SimplifiedEnvironmentVariableChanges( + dict[str, str]() + ) + self._fail_action_before_start( + f"Failed to evaluate the extra `let` bindings for {environment.name}: {e}" + ) + return identifier # Note: the environment script's own EXPR `let` bindings (RFC 0007) # are evaluated by the script runner, after embedded-file paths are @@ -772,58 +859,45 @@ def enter_environment( wrap_env = None if wrap_env is not None: - try: - # The wrapped onEnter resolves against the INNER environment's - # own scope — its `let` bindings and embedded files, built on - # a COPY of the session table so the hook's own scope never - # sees them (openjd-rs #277). The hook itself resolves against - # `symtab`; its own script's lets/files are evaluated by the - # script runner from wrap_env.script. - inner_script = environment.script - inner_symtab = self._build_wrapped_inner_scope( - EmbeddedFilesScope.ENV, - getattr(inner_script, "let", None) if inner_script else None, - inner_script.embeddedFiles if inner_script is not None else None, - symtab, - ) - self._inject_wrapped_env_symbols( + # The wrapped onEnter resolves against the INNER environment's + # own scope; the hook itself resolves against `symtab`, and its + # own script's lets/files are evaluated by the script runner + # from wrap_env.script (see _try_inject_wrapped_symbols). On + # failure the environment stays in the entered list, exactly as + # when enter() itself fails, so the caller's cleanup exits it + # as usual. + if not self._try_inject_wrapped_symbols( + scope=EmbeddedFilesScope.ENV, + inner_script=environment.script, + symtab=symtab, + inject=lambda inner_symtab: self._inject_wrapped_env_symbols( symtab, environment, on_enter_action, inner_symtab=inner_symtab - ) - except (FormatStringError, ValueError, RuntimeError) as e: - # e.g. the wrapped onEnter's inner scope failed to build (a - # `let` binding or embedded file did not resolve or write). - # Fail the action through the normal failure path — the - # environment stays in the entered list, exactly as when - # enter() itself fails, so the caller's cleanup exits it as - # usual. - self._fail_action_before_start( + ), + fail_message=( f"Failed to resolve the wrapped onEnter action of " - f"{environment.name} for {wrap_env.name}'s onWrapEnvEnter: {e}" + f"{environment.name} for {wrap_env.name}'s onWrapEnvEnter" + ), + ): + return identifier + try: + wrap_file_records = self._get_wrap_env_file_records(wrap_env) + except RuntimeError as e: + self._fail_action_before_start( + f"Failed to allocate embedded files for {wrap_env.name}: {e}" ) return identifier - self._runner = EnvironmentScriptRunner( - logger=self._logger, - user=self._user, - os_env_vars=action_env_vars, - session_working_directory=self.working_directory, - startup_directory=self.working_directory, - callback=self._action_callback, + self._runner = self._make_env_script_runner( environment_script=wrap_env.script, + os_env_vars=action_env_vars, symtab=symtab, - session_files_directory=self.files_directory, + preallocated_file_records=wrap_file_records, ) self._runner.wrap_env_enter() else: - self._runner = EnvironmentScriptRunner( - logger=self._logger, - user=self._user, - os_env_vars=action_env_vars, - session_working_directory=self.working_directory, - startup_directory=self.working_directory, - callback=self._action_callback, + self._runner = self._make_env_script_runner( environment_script=environment.script, + os_env_vars=action_env_vars, symtab=symtab, - session_files_directory=self.files_directory, ) self._runner.enter() @@ -893,18 +967,37 @@ def exit_environment( # Remove the environment from our tracking since we're now exiting it. del self._environments[identifier] self._environments_entered.pop() + # RFC 0008: drop any embedded-file records reused across this (wrap) + # environment's hook invocations; the files themselves live in the + # session directory and are cleaned up with it. + self._wrap_env_file_records.pop(identifier, None) self._running_environment_identifier = identifier symtab = self._symbol_table(environment.revision) self._materialize_path_mapping(environment.revision, action_env_vars, symtab) - # Re-apply the extra `let` bindings this environment was entered with + # Re-seed the owning step's name (Step.Name, RFC 0007 EXPR) and + # re-apply the extra `let` bindings this environment was entered with # (e.g. the owning step's step-level bindings, RFC 0007) so its onExit # resolves in the same scope as its onEnter. + exit_step_name = self._environment_step_names.pop(identifier, None) + if exit_step_name is not None: + symtab["Step.Name"] = exit_step_name exit_extra_let_bindings = self._environment_extra_let_bindings.pop(identifier, None) if exit_extra_let_bindings: - apply_let_bindings(symtab=symtab, let_bindings=exit_extra_let_bindings) + try: + apply_let_bindings(symtab=symtab, let_bindings=exit_extra_let_bindings) + except ValueError as e: + # ExpressionError and FormatStringError subclass ValueError: + # a binding failed to evaluate. Fail the action through the + # normal failure path rather than raising out of the public + # API — the environment was already removed from tracking + # above, matching how a failing onExit subprocess leaves it. + self._fail_action_before_start( + f"Failed to evaluate the extra `let` bindings for {environment.name}: {e}" + ) + return # Note: the environment script's own EXPR `let` bindings (RFC 0007) # are evaluated by the script runner (after embedded-file path @@ -933,58 +1026,45 @@ def exit_environment( ) if wrap_env is not None: - try: - # See the onWrapEnvEnter path: the wrapped onExit resolves - # against the INNER environment's own scope (its lets and - # embedded files) built on a copy; the hook's scope never - # sees them (openjd-rs #277). - inner_exit_script = environment.script - inner_exit_symtab = self._build_wrapped_inner_scope( - EmbeddedFilesScope.ENV, - getattr(inner_exit_script, "let", None) if inner_exit_script else None, - inner_exit_script.embeddedFiles if inner_exit_script is not None else None, - symtab, - ) - self._inject_wrapped_env_symbols( + # See the onWrapEnvEnter path (_try_inject_wrapped_symbols). On + # failure the environment was already removed from tracking + # above, matching how a failed exit() behaves. + if not self._try_inject_wrapped_symbols( + scope=EmbeddedFilesScope.ENV, + inner_script=environment.script, + symtab=symtab, + inject=lambda inner_symtab: self._inject_wrapped_env_symbols( symtab, environment, on_exit_action, session_env_list=wrapped_session_env_list, - inner_symtab=inner_exit_symtab, - ) - except (FormatStringError, ValueError, RuntimeError) as e: - # Mirror of the onWrapEnvEnter injection-failure handling: - # fail the action through the normal failure path. The - # environment was already removed from tracking above, - # matching how a failed exit() behaves. - self._fail_action_before_start( + inner_symtab=inner_symtab, + ), + fail_message=( f"Failed to resolve the wrapped onExit action of " - f"{environment.name} for {wrap_env.name}'s onWrapEnvExit: {e}" + f"{environment.name} for {wrap_env.name}'s onWrapEnvExit" + ), + ): + return + try: + wrap_file_records = self._get_wrap_env_file_records(wrap_env) + except RuntimeError as e: + self._fail_action_before_start( + f"Failed to allocate embedded files for {wrap_env.name}: {e}" ) return - self._runner = EnvironmentScriptRunner( - logger=self._logger, - user=self._user, - os_env_vars=action_env_vars, - session_working_directory=self.working_directory, - startup_directory=self.working_directory, - callback=self._action_callback, + self._runner = self._make_env_script_runner( environment_script=wrap_env.script, + os_env_vars=action_env_vars, symtab=symtab, - session_files_directory=self.files_directory, + preallocated_file_records=wrap_file_records, ) self._runner.wrap_env_exit() else: - self._runner = EnvironmentScriptRunner( - logger=self._logger, - user=self._user, - os_env_vars=action_env_vars, - session_working_directory=self.working_directory, - startup_directory=self.working_directory, - callback=self._action_callback, + self._runner = self._make_env_script_runner( environment_script=environment.script, + os_env_vars=action_env_vars, symtab=symtab, - session_files_directory=self.files_directory, ) self._runner.exit() @@ -1063,45 +1143,36 @@ def run_task( "A wrap environment is active but run_task() was not given a " "step_name; WrappedStep.Name will render as an empty string." ) + # The wrapped onRun resolves against the STEP's own scope; the + # hook resolves against `symtab`, and the wrap environment's own + # lets/files are evaluated by the script runner from + # wrap_env.script (see _try_inject_wrapped_symbols). + if not self._try_inject_wrapped_symbols( + scope=EmbeddedFilesScope.STEP, + inner_script=step_script, + symtab=symtab, + inject=lambda inner_symtab: self._inject_wrapped_task_symbols( + symtab, step_script, step_name or "", inner_symtab=inner_symtab + ), + fail_message=( + f"Failed to resolve the wrapped Task action for {wrap_env.name}'s " + "onWrapTaskRun" + ), + ): + return + try: - # The wrapped onRun resolves against the STEP's own scope — - # its `let` bindings and embedded files, built on a COPY of - # the session table so the hook's own scope never sees them - # (openjd-rs #277). The hook resolves against `symtab`; the - # wrap environment's own lets/files are evaluated by the - # script runner from wrap_env.script. - inner_task_symtab = self._build_wrapped_inner_scope( - EmbeddedFilesScope.STEP, - getattr(step_script, "let", None), - getattr(step_script, "embeddedFiles", None), - symtab, - ) - self._inject_wrapped_task_symbols( - symtab, step_script, step_name or "", inner_symtab=inner_task_symtab - ) - except (FormatStringError, ValueError, RuntimeError) as e: - # e.g. the wrapped onRun's inner scope failed to build (a - # `let` binding or embedded file did not resolve or write), or - # a FEATURE_BUNDLE_1 timeout/notifyPeriod format string did - # not resolve to an integer. Fail the action through the - # normal failure path rather than raising out of the public - # API. + wrap_file_records = self._get_wrap_env_file_records(wrap_env) + except RuntimeError as e: self._fail_action_before_start( - f"Failed to resolve the wrapped Task action for {wrap_env.name}'s " - f"onWrapTaskRun: {e}" + f"Failed to allocate embedded files for {wrap_env.name}: {e}" ) return - - self._runner = EnvironmentScriptRunner( - logger=self._logger, - user=self._user, - os_env_vars=action_env_vars, - session_working_directory=self.working_directory, - startup_directory=self.working_directory, - callback=self._action_callback, + self._runner = self._make_env_script_runner( environment_script=wrap_env.script, + os_env_vars=action_env_vars, symtab=symtab, - session_files_directory=self.files_directory, + preallocated_file_records=wrap_file_records, ) self._action_state = ActionState.RUNNING self._state = SessionState.RUNNING @@ -1440,7 +1511,7 @@ def _build_expr_host_rules(self) -> Optional[list[Any]]: # RFC 0008 wrap-action helpers # ------------------------------------------------------------------ - _WRAP_HOOK_NAMES = ("onWrapEnvEnter", "onWrapTaskRun", "onWrapEnvExit") + _WRAP_HOOK_NAMES = WRAP_HOOK_ACTION_NAMES def _find_wrap_environment(self, *, hook: str) -> Optional[EnvironmentModel]: """Walk the environment stack (innermost first) and return the @@ -1462,6 +1533,50 @@ def _find_wrap_environment(self, *, hook: str) -> Optional[EnvironmentModel]: return env return None + def _get_wrap_env_file_records(self, wrap_env: EnvironmentModel) -> Optional[list[_FileRecord]]: + """Return the wrap environment's embedded-file records, allocating + their on-disk paths on first use and reusing them for every + subsequent wrap-hook invocation (see ``_wrap_env_file_records``). + Returns ``None`` when the wrap environment has no embedded files. + + The allocation only reserves the paths (defining the symbols into a + throwaway table); the per-invocation symbol definitions and content + writes happen in the runner against that invocation's symbol table. + + Raises: + RuntimeError: if a file path could not be allocated. + """ + if wrap_env.script is None or wrap_env.script.embeddedFiles is None: + return None + # The wrap env's identifier: it is in the entered stack (that's how + # _find_wrap_environment found it). + identifier = next( + ( + env_id + for env_id in self._environments_entered + if self._environments[env_id] is wrap_env + ), + None, + ) + if identifier is None: # pragma: no cover - guarded by _find_wrap_environment + raise RuntimeError( + f"Wrap environment '{wrap_env.name}' is not in this Session's entered stack." + ) + records = self._wrap_env_file_records.get(identifier) + if records is None: + file_writer = EmbeddedFiles( + logger=self._logger, + scope=EmbeddedFilesScope.ENV, + session_files_directory=self.files_directory, + user=self._user, + ) + # Paths only: symbols are defined (and logged) per wrap-hook + # invocation via register_file_paths, against that invocation's + # own symbol table. + records = file_writer.allocate_records(wrap_env.script.embeddedFiles) + self._wrap_env_file_records[identifier] = records + return records + def _environment_defines_any_wrap_hook(self, env: EnvironmentModel) -> bool: """``True`` iff ``env``'s script declares any of the three wrap hooks. Used by the single-wrap-layer validation in @@ -1514,6 +1629,41 @@ def _build_wrapped_inner_scope( apply_let_bindings(symtab=symtab, let_bindings=let_bindings) return symtab + def _try_inject_wrapped_symbols( + self, + *, + scope: EmbeddedFilesScope, + inner_script: Any, + symtab: SymbolTable, + inject: Callable[[SymbolTable], None], + fail_message: str, + ) -> bool: + """Common wrap-interception step for the three RFC 0008 hooks: build + the wrapped (inner) entity's own resolution scope — its ``let`` + bindings and embedded files on a COPY of the session table, so the + hook's own scope never sees them (openjd-rs #277) — and call + ``inject`` with it to populate the ``WrappedAction.*`` symbols in + ``symtab``, the hook's scope. + + On failure (e.g. a binding or embedded file did not resolve or + write, or a FEATURE_BUNDLE_1 timeout/notifyPeriod format string did + not resolve to an integer) the action fails through the normal + failure path (:meth:`_fail_action_before_start` with + ``fail_message``) rather than raising out of the public API, and + ``False`` is returned so the caller can bail out.""" + try: + inner_symtab = self._build_wrapped_inner_scope( + scope, + getattr(inner_script, "let", None) if inner_script is not None else None, + getattr(inner_script, "embeddedFiles", None) if inner_script is not None else None, + symtab, + ) + inject(inner_symtab) + except (FormatStringError, ValueError, RuntimeError) as e: + self._fail_action_before_start(f"{fail_message}: {e}") + return False + return True + def _collect_session_env_list(self) -> list[str]: """Collect all session-defined variables across the active environment stack as ``["KEY=value", ...]`` for @@ -1534,10 +1684,10 @@ def _collect_session_env_list(self) -> list[str]: for env_id in self._environments_entered: if env_id in self._created_env_vars: changes = self._created_env_vars[env_id] - # Iterate _to_set (insertion-ordered), which holds both the + # effective_items() is insertion-ordered, holding both the # environment's declarative `variables:` seed and any # openjd_env sets/unsets, so the list order is deterministic. - for key, value in changes._to_set.items(): + for key, value in changes.effective_items().items(): effective[key] = value return [f"{key}={value}" for key, value in effective.items() if value is not None] @@ -1547,17 +1697,14 @@ def _resolve_action_timeout(self, action: Any, symtab: SymbolTable) -> Optional[ ``int?`` (RFC 0008), following the EXPR semantics for optional data, so whole-field forwarding (``timeout: "{{WrappedAction.Timeout}}"``) drops the field when the - wrapped action has no timeout.""" - if action.timeout is not None: - if isinstance(action.timeout, int): - return action.timeout - resolved = action.timeout.resolve(symtab=symtab) - if resolved == "": - # A whole-field expression that resolved to null: treated - # as if the field were not provided. - return None - return int(resolved) - return None + wrapped action has no timeout. + + A resolved format-string value must be a positive integer, matching + the openjd-rs runtime and the enforcement path in + :meth:`ScriptRunnerBase._run_action` (a non-positive value raises + ``ValueError``, failing the action through the caller's normal + failure path).""" + return resolve_optional_int_field(action.timeout, symtab, ge=1, description="timeout") def _inject_wrapped_cancelation_symbols( self, symtab: SymbolTable, action: Any, *, is_task_run: bool, inner_symtab: SymbolTable diff --git a/test/openjd/sessions_v0/test_runner_env_script.py b/test/openjd/sessions_v0/test_runner_env_script.py index 586f08e6..bc88bfdb 100644 --- a/test/openjd/sessions_v0/test_runner_env_script.py +++ b/test/openjd/sessions_v0/test_runner_env_script.py @@ -35,13 +35,13 @@ DataString as DataString_2023_09, ) from openjd.sessions import ActionState -from openjd.sessions._runner_base import ScriptRunnerState -from openjd.sessions._runner_env_script import ( +from openjd.sessions._runner_base import ( CancelMethod, - EnvironmentScriptRunner, NotifyCancelMethod, + ScriptRunnerState, TerminateCancelMethod, ) +from openjd.sessions._runner_env_script import EnvironmentScriptRunner from .conftest import build_logger, collect_queue_messages diff --git a/test/openjd/sessions_v0/test_runner_step_script.py b/test/openjd/sessions_v0/test_runner_step_script.py index 2b258adf..ee5e5983 100644 --- a/test/openjd/sessions_v0/test_runner_step_script.py +++ b/test/openjd/sessions_v0/test_runner_step_script.py @@ -35,13 +35,13 @@ from openjd.model.v2023_09 import StepScript as StepScript_2023_09 from openjd.sessions import WindowsSessionUser -from openjd.sessions._runner_base import ScriptRunnerState -from openjd.sessions._runner_step_script import ( +from openjd.sessions._runner_base import ( CancelMethod, NotifyCancelMethod, - StepScriptRunner, + ScriptRunnerState, TerminateCancelMethod, ) +from openjd.sessions._runner_step_script import StepScriptRunner from openjd.sessions._tempdir import TempDir from openjd.sessions._os_checker import is_posix, is_windows diff --git a/test/openjd/sessions_v0/test_session_let_bindings.py b/test/openjd/sessions_v0/test_session_let_bindings.py new file mode 100644 index 00000000..8dd80990 --- /dev/null +++ b/test/openjd/sessions_v0/test_session_let_bindings.py @@ -0,0 +1,269 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Regression tests for the Session-level handling of EXPR ``let`` +bindings (RFC 0007): + +- a failing ``extra_let_bindings`` entry on ``enter_environment`` / + ``exit_environment`` fails the action through the normal callback path + instead of raising out of the public API; +- ``enter_environment(step_name=...)`` seeds ``Step.Name`` so step-level + bindings and the environment's actions can reference it (on both the + enter and exit sides); +- binding-RHS parsing is memoized across applications; +- the unified optional int-or-format-string field resolver + (``resolve_optional_int_field``) enforces consistent bounds. +""" +from __future__ import annotations + +import time +import uuid +from typing import Any + +import pytest + +from openjd.model import SymbolTable +from openjd.model.v2023_09 import ( + Action as Action_2023_09, + ArgString as ArgString_2023_09, + CommandString as CommandString_2023_09, + Environment as Environment_2023_09, + EnvironmentActions as EnvironmentActions_2023_09, + EnvironmentScript as EnvironmentScript_2023_09, + ModelParsingContext as ModelParsingContext_2023_09, +) +from openjd.model._let_bindings import _parse_rhs +from openjd.sessions import ActionState, ActionStatus, Session, SessionState +from openjd.sessions._runner_base import ( + apply_let_bindings, + resolve_optional_int_field, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _action(command: str, *args: str) -> Action_2023_09: + return Action_2023_09( + command=CommandString_2023_09(command), + args=[ArgString_2023_09(a) for a in args] if args else None, + ) + + +def _env(name: str, **action_kwargs) -> Environment_2023_09: + return Environment_2023_09( + name=name, + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09(**action_kwargs), + ), + ) + + +def _run_until_ready(session: Session, timeout_s: float = 10.0) -> None: + deadline = time.time() + timeout_s + while session.state == SessionState.RUNNING and time.time() < deadline: + time.sleep(0.05) + + +def _format_string_field(raw: str) -> Any: + """Build a FormatString-typed ``timeout`` field value (FEATURE_BUNDLE_1).""" + context = ModelParsingContext_2023_09(supported_extensions=["FEATURE_BUNDLE_1", "EXPR"]) + action = Action_2023_09.model_validate({"command": "echo", "timeout": raw}, context=context) + return action.timeout + + +# --------------------------------------------------------------------------- +# A failing extra `let` binding must FAIL the action via the callback path, +# never raise out of enter_environment()/exit_environment(). +# --------------------------------------------------------------------------- + + +class TestExtraLetBindingFailure: + def test_enter_environment_failing_binding_fails_action_cleanly(self) -> None: + # GIVEN: an extra binding referencing an undefined symbol. + callback_events: list[ActionStatus] = [] + + def callback(session_id: str, status: ActionStatus) -> None: + callback_events.append(status) + + env = _env("Env", onEnter=_action("true"), onExit=_action("true")) + with Session( + session_id=uuid.uuid4().hex, job_parameter_values={}, callback=callback + ) as session: + # WHEN: entering must not raise. + identifier = session.enter_environment( + environment=env, + extra_let_bindings=["msg = NoSuchSymbol"], + ) + _run_until_ready(session) + + # THEN: the action failed cleanly, the callback fired, and the + # environment remains entered-but-failed (exactly as a failing + # onEnter subprocess leaves it) so cleanup can exit it. + assert session.state == SessionState.READY_ENDING + status = session.action_status + assert status is not None + assert status.state == ActionState.FAILED + assert status.fail_message is not None + assert "let" in status.fail_message + assert callback_events and callback_events[-1].state == ActionState.FAILED + assert identifier in session.environments_entered + + # WHEN: exiting the failed environment re-applies the failing + # bindings — the exit action must also fail cleanly, not raise. + session.exit_environment(identifier=identifier) + _run_until_ready(session) + + # THEN + assert session.state == SessionState.READY_ENDING + status = session.action_status + assert status is not None + assert status.state == ActionState.FAILED + assert identifier not in session.environments_entered + + +# --------------------------------------------------------------------------- +# enter_environment(step_name=...) seeds Step.Name (RFC 0007 EXPR), for both +# the enter side and the re-applied bindings on the exit side. +# --------------------------------------------------------------------------- + + +class TestEnterEnvironmentStepName: + def test_step_name_resolvable_in_bindings_and_actions( + self, caplog: pytest.LogCaptureFixture + ) -> None: + # GIVEN: a step-level binding referencing Step.Name, echoed by both + # the environment's onEnter and onExit actions. + env = _env( + "StepEnv", + onEnter=_action("echo", "enter:{{ msg }}"), + onExit=_action("echo", "exit:{{ msg }}"), + ) + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + # WHEN + identifier = session.enter_environment( + environment=env, + extra_let_bindings=["msg = 'step is ' + Step.Name"], + step_name="MyStep", + ) + _run_until_ready(session) + + # THEN: the enter action ran with the binding resolved. + assert session.state == SessionState.READY + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + assert any("enter:step is MyStep" in m for m in caplog.messages) + + # WHEN: the exit re-applies the bindings — Step.Name must be + # re-seeded so onExit resolves in the same scope as onEnter. + session.exit_environment(identifier=identifier) + _run_until_ready(session) + + # THEN + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + assert any("exit:step is MyStep" in m for m in caplog.messages) + + +# --------------------------------------------------------------------------- +# Binding-RHS parsing is memoized: re-applying the same bindings (per task, +# per env enter/exit) must not re-parse through the engine each time. +# --------------------------------------------------------------------------- + + +class TestLetBindingParseMemoization: + def test_parse_is_cached_across_applications(self) -> None: + # GIVEN: apply_let_bindings delegates to the model's single-sourced + # evaluator, whose RHS parse (_parse_rhs) is memoized. + _parse_rhs.cache_clear() + bindings = ["a = 1 + 2", "b = a * 10"] + + # WHEN: the same bindings apply against several symbol tables. + for _ in range(5): + symtab = SymbolTable() + apply_let_bindings(symtab=symtab, let_bindings=bindings) + # THEN: per-application evaluation is unchanged. (Values are the + # engine's typed results; compare via their string rendering.) + assert str(symtab["a"]) == "3" + assert str(symtab["b"]) == "30" + + # THEN: each unique RHS was parsed exactly once. + info = _parse_rhs.cache_info() + assert info.misses == len(bindings) + assert info.hits == 4 * len(bindings) + + def test_cached_expression_evaluates_against_per_call_symtab(self) -> None: + # The memoized expression object must hold no symbol-table state. + _parse_rhs.cache_clear() + first = SymbolTable() + first["X"] = 1 + second = SymbolTable() + second["X"] = 41 + apply_let_bindings(symtab=first, let_bindings=["y = X + 1"]) + apply_let_bindings(symtab=second, let_bindings=["y = X + 1"]) + assert str(first["y"]) == "2" + assert str(second["y"]) == "42" + + +# --------------------------------------------------------------------------- +# resolve_optional_int_field: one implementation for the three +# "optional int-or-format-string" call sites, with consistent bounds. +# --------------------------------------------------------------------------- + + +class TestResolveOptionalIntField: + def test_none_field_stays_none(self) -> None: + assert resolve_optional_int_field(None, SymbolTable(), ge=1, description="timeout") is None + + def test_literal_int_passes_through(self) -> None: + # Literal values were bounds-checked by the static validator at + # parse time; they pass through unchecked at run time. + assert resolve_optional_int_field(30, SymbolTable(), ge=1, description="timeout") == 30 + + def test_whole_field_null_resolves_to_none(self) -> None: + field = _format_string_field("{{ X }}") + symtab = SymbolTable() + symtab["X"] = None + assert resolve_optional_int_field(field, symtab, ge=1, description="timeout") is None + + def test_resolved_value_below_ge_rejected(self) -> None: + field = _format_string_field("{{ X }}") + symtab = SymbolTable() + symtab["X"] = 0 + with pytest.raises(ValueError, match="timeout must be a positive integer, got '0'"): + resolve_optional_int_field(field, symtab, ge=1, description="timeout") + + def test_resolved_value_above_le_rejected(self) -> None: + field = _format_string_field("{{ X }}") + symtab = SymbolTable() + symtab["X"] = 700 + with pytest.raises( + ValueError, match="notifyPeriodInSeconds must be between 1 and 600, got '700'" + ): + resolve_optional_int_field( + field, symtab, ge=1, le=600, description="notifyPeriodInSeconds" + ) + + def test_non_integer_resolved_value_rejected(self) -> None: + field = _format_string_field("{{ X }}") + symtab = SymbolTable() + symtab["X"] = "abc" + with pytest.raises(ValueError, match="timeout must be a positive integer, got 'abc'"): + resolve_optional_int_field(field, symtab, ge=1, description="timeout") + + def test_session_resolve_action_timeout_rejects_non_positive(self) -> None: + # Regression: Session._resolve_action_timeout previously accepted + # any int() — a resolved non-positive timeout must now be rejected, + # matching the openjd-rs runtime and the enforcement path in + # ScriptRunnerBase._run_action. + context = ModelParsingContext_2023_09(supported_extensions=["FEATURE_BUNDLE_1", "EXPR"]) + action = Action_2023_09.model_validate( + {"command": "echo", "timeout": "{{ X }}"}, context=context + ) + symtab = SymbolTable() + symtab["X"] = 0 + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + with pytest.raises(ValueError, match="timeout must be a positive integer"): + session._resolve_action_timeout(action, symtab) diff --git a/test/openjd/sessions_v0/test_wrap_task_run.py b/test/openjd/sessions_v0/test_wrap_task_run.py index ab2f25c3..e11de82e 100644 --- a/test/openjd/sessions_v0/test_wrap_task_run.py +++ b/test/openjd/sessions_v0/test_wrap_task_run.py @@ -434,3 +434,86 @@ def test_many_arguments_preserved(self) -> None: assert cmd == "tool" assert args == many assert len(args) == 1000 + + +# --------------------------------------------------------------------------- +# Wrap-environment embedded-file reuse across tasks. +# +# The wrap environment's embedded-file PATHS are allocated once and reused +# for every task run through the wrap (so Env.File.* symbols stay stable and +# unnamed files do not accumulate on disk), while the file CONTENTS are +# re-resolved and rewritten per task (data may reference WrappedAction.*). +# --------------------------------------------------------------------------- + + +class TestWrapEnvEmbeddedFileReuse: + def _wrap_env_with_unnamed_file(self) -> Environment_2023_09: + from openjd.model.v2023_09 import ( + DataString as DataString_2023_09, + EmbeddedFileText as EmbeddedFileText_2023_09, + EmbeddedFileTypes as EmbeddedFileTypes_2023_09, + ) + + # The embedded file is UNNAMED (no `filename`), so its on-disk path + # is mkstemp-allocated; its data references WrappedAction.Command, + # so contents must be rewritten per task. + return Environment_2023_09( + name="WrapEnv", + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onWrapEnvEnter=_NOOP, + onWrapTaskRun=Action_2023_09( + command=CommandString_2023_09("cat"), + args=[ArgString_2023_09("{{ Env.File.WrapData }}")], + ), + onWrapEnvExit=_NOOP, + ), + embeddedFiles=[ + EmbeddedFileText_2023_09( + name="WrapData", + type=EmbeddedFileTypes_2023_09.TEXT, + data=DataString_2023_09("wrapped-command={{WrappedAction.Command}}\n"), + ) + ], + ), + ) + + def test_unnamed_wrap_file_path_reused_across_tasks(self) -> None: + # GIVEN: a wrap environment with an unnamed embedded file whose data + # references WrappedAction.Command, and three tasks with distinct + # wrapped commands (never executed; the wrap action runs instead). + env = self._wrap_env_with_unnamed_file() + commands = ("cmd-one", "cmd-two", "cmd-three") + file_paths = [] + + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + identifier = session.enter_environment(environment=env) + _run_until_ready(session) + assert session.state == SessionState.READY + + for command in commands: + # WHEN: a task runs through the wrap. + session.run_task( + step_script=_step_script(command, []), + task_parameter_values={}, + step_name="Step", + ) + _run_until_ready(session) + + # THEN: the task succeeded, and the files directory contains + # exactly ONE file for the record (not one per task) ... + assert session.state == SessionState.READY + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + files = [p for p in session.files_directory.iterdir() if p.is_file()] + assert len(files) == 1 + # ... whose contents reflect THIS task's wrapped command. + assert files[0].read_text() == f"wrapped-command={command}\n" + file_paths.append(files[0]) + + # THEN: the file path is identical across all three tasks. + assert file_paths[0] == file_paths[1] == file_paths[2] + + session.exit_environment(identifier=identifier) + _run_until_ready(session) From cd5d5b69d04fb3630bde851d869e05a50c76adcc Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Thu, 23 Jul 2026 21:49:35 -0700 Subject: [PATCH 03/27] fix: RFC 0005/0008 typed-arg and null-vs-empty parity with openjd-rs Two Rust-parity fixes in the v0 session runtime: 1. WrappedAction.Args now uses RFC 0005 1.3.2 typed argument semantics. The enforcement path's typed arg loop (null skip, list flattening, display coercion) is extracted into a shared module-level helper, resolve_action_arg_values, in _runner_base.py; both _inject_wrapped_task_symbols and _inject_wrapped_env_symbols now use it, so a wrap hook sees exactly the argv the wrapped action would have run with unwrapped -- mirroring openjd-rs, whose seed_wrapped_action_symbols resolves through the same resolve_action_args as the runner. The helper also gains openjd-rs's plain-string-resolution fallback when typed resolution fails. 2. An empty string is no longer conflated with null. resolve_optional_int_field and resolve_effective_cancelation's deferred-mode branch now resolve typed (resolve_value) and treat only a typed null result as "field omitted" / "no cancelation declared". A genuine empty string now reaches the "must be a positive integer, got ''" / "must resolve to ... got ''" errors, matching openjd-rs's resolve_action_timeout, resolve_notify_period_seconds, and resolve_effective_cancelation, which only special-case ExprValue::Null. The whole_field_expression() pre-check is removed: resolve_value only yields a typed null for whole-field expressions, so null semantics remain whole-field-only by construction. Test updates: the deferred-cancelation unit-test helper now parses its format strings with the EXPR extension (deferred forwarding is an RFC 0008 construct and WRAP_ACTIONS requires EXPR), since the previous legacy parse pinned the non-Rust "" == null behavior. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- src/openjd/sessions/_embedded_files.py | 2 +- src/openjd/sessions/_runner_base.py | 238 +++++++++++------- src/openjd/sessions/_runner_env_script.py | 27 +- src/openjd/sessions/_runner_step_script.py | 22 +- src/openjd/sessions/_session.py | 52 ++-- .../sessions_v0/test_runner_env_script.py | 5 +- .../sessions_v0/test_runner_step_script.py | 4 +- .../sessions_v0/test_session_let_bindings.py | 80 +++++- .../sessions_v0/test_wrap_cancelation.py | 126 +++++++++- test/openjd/sessions_v0/test_wrap_task_run.py | 39 +++ 10 files changed, 454 insertions(+), 141 deletions(-) diff --git a/src/openjd/sessions/_embedded_files.py b/src/openjd/sessions/_embedded_files.py index 6c90ce24..61fe86a4 100644 --- a/src/openjd/sessions/_embedded_files.py +++ b/src/openjd/sessions/_embedded_files.py @@ -220,7 +220,7 @@ def allocate_file_paths( the file contents. Splitting allocation from :meth:`write_file_contents` lets the runner - evaluate EXPR ``let`` bindings between the two phases (RFC 0007): a + evaluate EXPR ``let`` bindings between the two phases (RFC 0005): a file's *path* never depends on ``let`` values (``filename`` is a plain string), so the ``Env.File.*``/``Task.File.*`` symbols are available to the bindings, while a file's ``data`` is written afterwards so it diff --git a/src/openjd/sessions/_runner_base.py b/src/openjd/sessions/_runner_base.py index 914d1113..5f66d236 100644 --- a/src/openjd/sessions/_runner_base.py +++ b/src/openjd/sessions/_runner_base.py @@ -2,6 +2,7 @@ import json import os +import re import stat import shlex from abc import ABC, abstractmethod @@ -36,11 +37,19 @@ "NotifyCancelMethod", "ScriptRunnerBase", "apply_let_bindings", + "resolve_action_arg_values", "resolve_effective_cancelation", "resolve_optional_int_field", ) +_STRICT_INT_RE = re.compile(r"[+-]?[0-9]+") +"""The exact integer grammar accepted for dynamically resolved integer fields +(FEATURE_BUNDLE_1 ``timeout`` / ``notifyPeriodInSeconds`` format strings): +an optional sign followed by ASCII digits, exactly what Rust's ``str::parse`` +accepts. See resolve_optional_int_field.""" + + class ScriptRunnerState(str, Enum): """State of a ScriptRunner.""" @@ -96,6 +105,48 @@ class NotifyCancelMethod(CancelMethod): """Amount of time after a SIGTERM to wait to do the SIGKILL""" +def resolve_action_arg_values(args: Optional[Sequence], symtab: SymbolTable) -> list[str]: + """Resolve an action's ``args`` field into its flat list of argument + strings (excluding the command). + + RFC 0005 §1.3.2 argument semantics, mirroring openjd-rs's + resolve_action_args: a whole-field expression argument resolves typed — + a null result skips the argument, a list result flattens inline (one + argument per element, rendered with the engine's display coercion), and + a scalar becomes a single argument. Multi-segment format strings and + legacy (non-EXPR) expressions resolve to their string form. + + Shared by the enforcement path (:meth:`ScriptRunnerBase._run_action`) + and the RFC 0008 ``WrappedAction.Args`` injection, so a wrap hook sees + exactly the arguments the wrapped action would have run with unwrapped. + + Raises: + FormatStringError: If an argument's expression cannot be resolved. + """ + resolved: list[str] = [] + if args is not None: + for arg in args: + try: + value = arg.resolve_value(symtab=symtab) + except FormatStringError: + # Mirror openjd-rs's resolve_action_args: when typed + # resolution fails (e.g. a legacy-parsed expression meeting + # a typed symbol value), fall back to plain string + # resolution — which raises FormatStringError itself if the + # argument is genuinely unresolvable. + resolved.append(arg.resolve(symtab=symtab)) + continue + if isinstance(value, str): + resolved.append(value) + elif getattr(value, "is_null", False): + continue + elif str(getattr(value, "type", "")).startswith("list["): + resolved.extend(str(element) for element in value) + else: + resolved.append(str(value)) + return resolved + + def resolve_optional_int_field( value: Any, symtab: SymbolTable, @@ -111,13 +162,16 @@ def resolve_optional_int_field( - ``None`` (field omitted) stays ``None``. - A literal ``int`` passes through unchecked: literal values were bounds-checked by the static validator at parse time. - - A FormatString (FEATURE_BUNDLE_1) is resolved against ``symtab``. A - whole-field expression that resolves to null renders as the empty - string and is treated as if the field were not provided (``None`` — - the caller applies any positional schema default). Otherwise the - resolved value must be an integer within the given bounds; the bounds - apply here because format-string values could not be checked at parse - time. + - A FormatString (FEATURE_BUNDLE_1) is resolved against ``symtab`` + using typed resolution. A whole-field expression that resolves to a + typed null is treated as if the field were not provided (``None`` — + the caller applies any positional schema default). Any other result + — including a genuine empty string — must be an integer within the + given bounds; the bounds apply here because format-string values + could not be checked at parse time. This matches the openjd-rs + runtime (resolve_action_timeout / resolve_notify_period_seconds), + which only treats an ExprValue::Null result as "field omitted" and + errors on an empty string. Raises: ValueError: If the resolved value is not an integer, or violates @@ -128,9 +182,15 @@ def resolve_optional_int_field( return None if isinstance(value, int): return value - resolved = value.resolve(symtab=symtab) - if resolved == "": + # Typed resolution: a whole-field EXPR expression yields the engine's + # typed value, so a null result is distinguishable from a genuine empty + # string. Multi-segment and legacy (non-EXPR) format strings fall back + # to plain string resolution — correct, since typed nulls only exist + # under EXPR whole-field semantics (Template Schemas 5.3). + resolved_value = value.resolve_value(symtab=symtab) + if getattr(resolved_value, "is_null", False): return None + resolved = str(resolved_value) if ge is not None and le is not None: constraint = f"between {ge} and {le}" elif ge == 1 and le is None: @@ -139,10 +199,15 @@ def resolve_optional_int_field( constraint = f"an integer >= {ge}" else: constraint = "an integer" - try: - result = int(resolved) - except ValueError: + # Strict ASCII integer grammar, matching the Rust runtime's str::parse + # (openjd-rs resolve_action_timeout / resolve_notify_period_seconds). + # Python's int() is more lenient — it accepts surrounding whitespace, + # digit-group underscores ("1_0" == 10), and non-ASCII decimal digits — + # all of which Rust rejects, so accepting them here would be a + # spec-observable divergence. + if _STRICT_INT_RE.fullmatch(resolved) is None: raise ValueError(f"{description} must be {constraint}, got '{resolved}'") + result = int(resolved) if (ge is not None and result < ge) or (le is not None and result > le): raise ValueError(f"{description} must be {constraint}, got '{result}'") return result @@ -195,19 +260,23 @@ def resolve_period(period: Any) -> Optional[int]: if cancelation is None: return (None, None) if isinstance(cancelation, CancelationMethodDeferred_2023_09): - # Null semantics apply only to a whole-field expression - # ("{{ ... }}" with no surrounding text, target type string? — - # Template Schemas 5.3). A normal format string that happens to - # resolve to the empty string is NOT null; it falls through to the - # "must resolve to..." error below. - is_whole_field = cancelation.mode.whole_field_expression() is not None - mode = cancelation.mode.resolve(symtab=symtab) - if mode == "" and is_whole_field: + # Typed resolution. Null semantics apply only to a whole-field + # expression ("{{ ... }}" with no surrounding text, target type + # string? — Template Schemas 5.3), and resolve_value only yields a + # typed null for a whole-field EXPR expression; every other format + # string resolves to its plain string form. A format string that + # happens to resolve to the empty string is NOT null; it falls + # through to the "must resolve to..." error below (matching the + # openjd-rs runtime, which errors on any non-null, non-mode-name + # result). + mode_value = cancelation.mode.resolve_value(symtab=symtab) + if getattr(mode_value, "is_null", False): # Null mode drops the ENTIRE cancelation object: mode is the # object's required discriminator, so an "omitted" mode cannot # leave a partial object behind. The action behaves exactly as # if no were declared. return (None, None) + mode = str(mode_value) if mode == CancelationMode_2023_09.TERMINATE.value: # Post-resolution the object must validate against the resolved # variant's shape: TERMINATE admits no notify period. @@ -235,7 +304,7 @@ def resolve_period(period: Any) -> Optional[int]: def apply_let_bindings(*, symtab: SymbolTable, let_bindings: list[str]) -> None: - """Evaluate EXPR ``let`` bindings (RFC 0007) and add them to ``symtab``. + """Evaluate EXPR ``let`` bindings (RFC 0005) and add them to ``symtab``. ``let_bindings`` is a script's ``let`` field: an ordered list of ``"name = expression"`` strings. Each RHS is an EXPR expression evaluated @@ -346,6 +415,20 @@ class ScriptRunnerBase(ABC): e.g. We failed to write embedded files before even trying to run the action. """ + _resolved_cancel_method: Optional[CancelMethod] + """The running action's effective cancel method, resolved by + :meth:`_run_action` right before subprocess launch — against the same + final (let/embedded-file enriched) symbol table the command and args + resolved with — and consumed by the runners' :meth:`cancel`. + + Resolving at launch matches the openjd-rs runtime (``run_action`` + resolves ``cancel_method_for_action`` up front): a cancelation whose + deferred mode or notify period cannot be resolved fails the action at + start rather than surfacing only if a cancel later occurs, and the + resolution scope is the action's own (a mode referencing a script-level + ``let`` binding resolves correctly). ``None`` until an action launches. + """ + def __init__( self, *, @@ -394,6 +477,7 @@ def __init__( # Will run at most the run futures self._pool = ThreadPoolExecutor(max_workers=1) self._state_override = None + self._resolved_cancel_method = None self._print_section_banner = True # Context manager for use in our tests @@ -579,7 +663,7 @@ def _materialize_files( materializing embedded files to disk. When ``let_bindings`` is given, they are evaluated between file-path - allocation and content writing (RFC 0007, mirroring the openjd-rs + allocation and content writing (RFC 0005, mirroring the openjd-rs runners): a file's *path* never depends on ``let`` values (filenames are plain strings), so ``Env.File.*``/``Task.File.*`` are available to the bindings, while a file's ``data`` is written afterwards so it can @@ -629,6 +713,7 @@ def _run_action( symtab: SymbolTable, *, default_timeout: Optional[timedelta] = None, + default_notify_period_seconds: int = 30, ) -> None: """Helper for derived classes to run a specific Action. @@ -640,28 +725,18 @@ def _run_action( default_timeout (Optional[timedelta], optional): Default timeout duration for the action if no timeout is specified in the action. The default behaviour if None is passed will allow the action to run indefinitely until it completes. + default_notify_period_seconds (int): The Template Schemas 5.3.2 + positional default applied when a NOTIFY_THEN_TERMINATE + cancelation omits its notify period (120 for a task's onRun, + 30 for any other action). """ assert isinstance(action, Action_2023_09) try: command = [action.command.resolve(symtab=symtab)] - if action.args is not None: - # RFC 0005 §1.3.2 argument semantics, mirroring openjd-rs's - # resolve_action_args: a whole-field expression argument - # resolves typed — a null result skips the argument, a list - # result flattens inline (one argument per element, rendered - # with the engine's display coercion), and a scalar becomes a - # single argument. Multi-segment format strings and legacy - # (non-EXPR) expressions resolve to their string form. - for arg in action.args: - value = arg.resolve_value(symtab=symtab) - if isinstance(value, str): - command.append(value) - elif getattr(value, "is_null", False): - continue - elif str(getattr(value, "type", "")).startswith("list["): - command.extend(str(element) for element in value) - else: - command.append(str(value)) + # RFC 0005 §1.3.2 typed argument semantics (null skip, list + # flattening) — see resolve_action_arg_values, shared with the + # RFC 0008 WrappedAction.Args injection. + command.extend(resolve_action_arg_values(action.args, symtab)) except FormatStringError as exc: # Extremely unlikely since a JobTemplate needs to have passed # validation before we could be running it, but just to be safe. @@ -670,11 +745,10 @@ def _run_action( time_limit: Optional[timedelta] = default_timeout # A FormatString timeout (FEATURE_BUNDLE_1) is resolved right # before the action runs. A whole-field expression that - # resolves to null renders as the empty string — e.g. - # forwarding `timeout: "{{WrappedAction.Timeout}}"` - # (RFC 0008) when the wrapped action specified no - # timeout — and is treated as if the field were not - # provided, so the positional default applies. + # resolves to a typed null — e.g. forwarding + # `timeout: "{{WrappedAction.Timeout}}"` (RFC 0008) when the + # wrapped action specified no timeout — is treated as if the + # field were not provided, so the positional default applies. try: seconds = resolve_optional_int_field( action.timeout, symtab, ge=1, description="timeout" @@ -687,51 +761,45 @@ def _run_action( return if seconds is not None: time_limit = timedelta(seconds=seconds) + # Resolve the action's effective cancelation NOW, against the + # same final scope the command/args/timeout resolved with — a + # deferred (format-string) mode or FEATURE_BUNDLE_1 notify + # period may reference script-level `let` bindings or + # Env.File.*/Task.File.* symbols that only exist in this scope. + # This matches the openjd-rs runtime (run_action resolves + # cancel_method_for_action before launching): an unresolvable or + # invalid cancelation fails the action at start instead of + # surfacing only if a cancel later occurs. The runners' + # cancel() consumes the stored method. + try: + mode, period = resolve_effective_cancelation(action.cancelation, symtab) + except ValueError as exc: + # FormatStringError (resolution failure) subclasses ValueError. + self._fail_action(str(exc)) + return + if mode != CancelationMode_2023_09.NOTIFY_THEN_TERMINATE.value: + # Note: The default cancelation for a 2023-09 script is Terminate + self._resolved_cancel_method = TerminateCancelMethod() + else: + self._resolved_cancel_method = NotifyCancelMethod( + terminate_delay=timedelta( + seconds=(period if period is not None else default_notify_period_seconds) + ) + ) self._run(command, time_limit) - def _cancel_with_effective_cancelation( - self, - *, - cancelation: Any, - symtab: SymbolTable, - default_notify_period_seconds: int, - time_limit: Optional[timedelta] = None, - mark_action_failed: bool = False, + def _cancel_with_resolved_method( + self, time_limit: Optional[timedelta] = None, mark_action_failed: bool = False ) -> None: - """Shared implementation of the runners' :meth:`cancel`. - - Resolve the action's cancelation config against the symbol table: a - deferred (format-string) mode and/or a FEATURE_BUNDLE_1 notify - period decide their values here, right when they are needed - (see resolve_effective_cancelation for the full story). A cancel - must always proceed, so resolution errors fall back to Terminate. - - ``default_notify_period_seconds`` is the Template Schemas 5.3.2 - positional default applied when a NOTIFY_THEN_TERMINATE cancelation - omits its notify period (120 for a task's onRun, 30 otherwise). + """Shared implementation of the runners' :meth:`cancel`: cancel with + the effective cancel method that :meth:`_run_action` resolved at + launch time. No-op when no action was launched (e.g. setup failed + before the subprocess started — there is nothing to cancel). """ - try: - mode, period = resolve_effective_cancelation(cancelation, symtab) - except (ValueError, FormatStringError) as exc: - self._logger.warning( - f"Failed to resolve the action's cancelation; canceling by " - f"termination instead: {exc}" - ) - mode, period = (None, None) - - method: CancelMethod - if mode != CancelationMode_2023_09.NOTIFY_THEN_TERMINATE.value: - # Note: The default cancelation for a 2023-09 script is Terminate - method = TerminateCancelMethod() - else: - method = NotifyCancelMethod( - terminate_delay=timedelta( - seconds=(period if period is not None else default_notify_period_seconds) - ) - ) - + if self._resolved_cancel_method is None: + return # Note: If the given time_limit is less than that in the method, then the time_limit will be what's used. - self._cancel(method, time_limit, mark_action_failed) + self._cancel(self._resolved_cancel_method, time_limit, mark_action_failed) def _cancel( self, diff --git a/src/openjd/sessions/_runner_env_script.py b/src/openjd/sessions/_runner_env_script.py index 16d0aaf8..4e96df80 100644 --- a/src/openjd/sessions/_runner_env_script.py +++ b/src/openjd/sessions/_runner_env_script.py @@ -6,7 +6,6 @@ from typing import Callable, Optional from openjd.model import SymbolTable -from openjd.model.v2023_09 import Action as Action_2023_09 from openjd.model.v2023_09 import EnvironmentScript as EnvironmentScript_2023_09 from ._embedded_files import EmbeddedFilesScope, _FileRecord from ._logging import log_subsection_banner @@ -141,9 +140,7 @@ def _run_env_action( log_subsection_banner(self._logger, "Phase: Setup") let_bindings = ( - getattr(self._environment_script, "let", None) - if self._environment_script is not None - else None + self._environment_script.let if self._environment_script is not None else None ) # Write any embedded files to disk. File paths are allocated before # the script's EXPR `let` bindings evaluate (so bindings can reference @@ -174,7 +171,12 @@ def _run_env_action( # Construct the command by evalutating the format strings in the command self._action = action - self._run_action(self._action, symtab, default_timeout=default_timeout) + self._run_action( + self._action, + symtab, + default_timeout=default_timeout, + default_notify_period_seconds=ENV_ACTION_DEFAULT_NOTIFY_PERIOD_SECONDS, + ) def enter(self) -> None: """Run the Environment's onEnter action.""" @@ -268,13 +270,8 @@ def cancel( # Nothing to do. return - # For the type checker - assert isinstance(self._action, Action_2023_09) - - self._cancel_with_effective_cancelation( - cancelation=self._action.cancelation, - symtab=self._symtab, - default_notify_period_seconds=ENV_ACTION_DEFAULT_NOTIFY_PERIOD_SECONDS, - time_limit=time_limit, - mark_action_failed=mark_action_failed, - ) + # Cancel with the effective method resolved at launch time by + # _run_action, against the action's own final scope (the script's + # lets and Env.File.* symbols, plus WrappedAction.* for a wrap + # hook) — openjd-rs parity. + self._cancel_with_resolved_method(time_limit, mark_action_failed) diff --git a/src/openjd/sessions/_runner_step_script.py b/src/openjd/sessions/_runner_step_script.py index 9e926dfc..1d49736b 100644 --- a/src/openjd/sessions/_runner_step_script.py +++ b/src/openjd/sessions/_runner_step_script.py @@ -97,7 +97,7 @@ def run(self) -> None: # For the type checker. assert isinstance(self._script, StepScript_2023_09) - let_bindings = getattr(self._script, "let", None) + let_bindings = self._script.let # Write any embedded files to disk. File paths are allocated before # the script's EXPR `let` bindings evaluate (so bindings can reference # Task.File.*), and contents are written after (so `data` can @@ -121,18 +121,16 @@ def run(self) -> None: symtab = self._symtab # Construct the command by evalutating the format strings in the command - self._run_action(self._script.actions.onRun, symtab) + self._run_action( + self._script.actions.onRun, + symtab, + default_notify_period_seconds=TASK_RUN_DEFAULT_NOTIFY_PERIOD_SECONDS, + ) def cancel( self, *, time_limit: Optional[timedelta] = None, mark_action_failed: bool = False ) -> None: - # For the type checker. - assert isinstance(self._script, StepScript_2023_09) - - self._cancel_with_effective_cancelation( - cancelation=self._script.actions.onRun.cancelation, - symtab=self._symtab, - default_notify_period_seconds=TASK_RUN_DEFAULT_NOTIFY_PERIOD_SECONDS, - time_limit=time_limit, - mark_action_failed=mark_action_failed, - ) + # Cancel with the effective method resolved at launch time by + # _run_action, against the action's own final scope (its lets and + # Task.File.* symbols) — openjd-rs parity. + self._cancel_with_resolved_method(time_limit, mark_action_failed) diff --git a/src/openjd/sessions/_session.py b/src/openjd/sessions/_session.py index bee4cae5..28bae1d7 100644 --- a/src/openjd/sessions/_session.py +++ b/src/openjd/sessions/_session.py @@ -45,6 +45,7 @@ from ._runner_base import ( ScriptRunnerBase, apply_let_bindings, + resolve_action_arg_values, resolve_effective_cancelation, resolve_optional_int_field, ) @@ -373,7 +374,7 @@ def __init__( ParameterValue (a dataclass containing the type and value of the parameter) job_name (Optional[str]): The resolved name of the Job this Session belongs to. When provided, it is seeded as the ``Job.Name`` template variable - (RFC 0007 §7.3.1, EXPR extension) for the session's actions — + (RFC 0005; Template Schemas §7.3.1, EXPR extension) for the session's actions — mirroring the Job.Name symbol that openjd-rs carries in its session symbol tables. Defaults to None (no Job.Name symbol). path_mapping_rules (Optional[list[PathMappingRule]]): A list of the path mapping rules to apply @@ -424,7 +425,7 @@ def __init__( # environment exits so its onExit resolves in the same scope. self._environment_extra_let_bindings: dict[EnvironmentIdentifier, list[str]] = dict() # The owning step's name supplied when an environment was entered - # (seeds Step.Name, RFC 0007 EXPR), re-seeded when the environment + # (seeds Step.Name, RFC 0005 EXPR), re-seeded when the environment # exits so the re-applied extra `let` bindings resolve in the same # scope. self._environment_step_names: dict[EnvironmentIdentifier, str] = dict() @@ -711,7 +712,7 @@ def enter_environment( Key: Environment variable name Value: Value for the environment variable. extra_let_bindings (Optional[list[str]]): Additional EXPR ``let`` - bindings (RFC 0007) evaluated into the symbol table before the + bindings (RFC 0005) evaluated into the symbol table before the environment's variables and actions resolve. A step's environments are entered with the step-level ``let`` bindings (``Step.let`` on the instantiated Job) so both can reference @@ -719,7 +720,7 @@ def enter_environment( table that openjd-rs threads into enter_environment. step_name (Optional[str]): The name of the step whose stepEnvironments are being entered, if any. Seeds - ``Step.Name`` (RFC 0007 EXPR) into the symbol table before + ``Step.Name`` (RFC 0005 EXPR) into the symbol table before the extra ``let`` bindings evaluate, so step-level bindings and the environment's variables and actions can reference it — openjd-rs threads a per-step resolved symbol table into @@ -767,14 +768,14 @@ def enter_environment( symtab = self._symbol_table(environment.revision) - # RFC 0007 §7.3.1 (EXPR): the owning step's name. Only EXPR templates + # RFC 0005; Template Schemas §7.3.1 (EXPR): the owning step's name. Only EXPR templates # pass validation referencing Step.Name, so seeding it when known does # not change non-EXPR behavior. Seeded before the extra `let` bindings # evaluate so a step-level binding may reference it. if step_name is not None: symtab["Step.Name"] = step_name - # Step-level `let` bindings (RFC 0007) accompany a step's + # Step-level `let` bindings (RFC 0005) accompany a step's # environments: evaluate them first so the environment's variables # and actions can reference them. if extra_let_bindings: @@ -796,7 +797,7 @@ def enter_environment( ) return identifier - # Note: the environment script's own EXPR `let` bindings (RFC 0007) + # Note: the environment script's own EXPR `let` bindings (RFC 0005) # are evaluated by the script runner, after embedded-file paths are # allocated (so bindings can reference Env.File.*). The environment's # `variables` resolve against the session symbol table without them, @@ -977,9 +978,9 @@ def exit_environment( symtab = self._symbol_table(environment.revision) self._materialize_path_mapping(environment.revision, action_env_vars, symtab) - # Re-seed the owning step's name (Step.Name, RFC 0007 EXPR) and + # Re-seed the owning step's name (Step.Name, RFC 0005 EXPR) and # re-apply the extra `let` bindings this environment was entered with - # (e.g. the owning step's step-level bindings, RFC 0007) so its onExit + # (e.g. the owning step's step-level bindings, RFC 0005) so its onExit # resolves in the same scope as its onEnter. exit_step_name = self._environment_step_names.pop(identifier, None) if exit_step_name is not None: @@ -999,7 +1000,7 @@ def exit_environment( ) return - # Note: the environment script's own EXPR `let` bindings (RFC 0007) + # Note: the environment script's own EXPR `let` bindings (RFC 0005) # are evaluated by the script runner (after embedded-file path # allocation); the wrap-interception branch below evaluates them # itself before resolving the wrapped onExit. @@ -1116,7 +1117,7 @@ def run_task( self._reset_action_state() symtab = self._symbol_table(step_script.revision, task_parameter_values) - # RFC 0007 §7.3.1 (EXPR): the running step's name. Only EXPR templates + # RFC 0005; Template Schemas §7.3.1 (EXPR): the running step's name. Only EXPR templates # pass validation referencing Step.Name, so seeding it when known does # not change non-EXPR behavior. if step_name is not None: @@ -1124,7 +1125,7 @@ def run_task( action_env_vars = self._evaluate_current_session_env_vars(os_env_vars) self._materialize_path_mapping(step_script.revision, action_env_vars, symtab) - # Note: the step script's EXPR `let` bindings (RFC 0007) are evaluated + # Note: the step script's EXPR `let` bindings (RFC 0005) are evaluated # by the script runner, after embedded-file paths are allocated (so # bindings can reference Task.File.*). The wrap-interception branch # below evaluates them itself before resolving the wrapped onRun. @@ -1237,7 +1238,7 @@ def _run_task_without_session_env( self._materialize_path_mapping(step_script.revision, action_env_vars, symtab) - # Note: the step script's EXPR `let` bindings (RFC 0007) are evaluated + # Note: the step script's EXPR `let` bindings (RFC 0005) are evaluated # by the script runner, after embedded-file paths are allocated. self._runner = StepScriptRunner( @@ -1461,7 +1462,7 @@ def record_expr_types( symtab[working_dir_key] = str(self.working_directory) # Session.WorkingDirectory is a host-format path value in openjd-rs. symtab.expr_types[working_dir_key] = ParameterValueType.PATH.value - # RFC 0007 §7.3.1 (EXPR): the job's resolved name. Only templates + # RFC 0005; Template Schemas §7.3.1 (EXPR): the job's resolved name. Only templates # declaring EXPR pass validation referencing Job.Name, so seeding # it whenever known does not change non-EXPR behavior. if self._job_name is not None: @@ -1654,8 +1655,8 @@ def _try_inject_wrapped_symbols( try: inner_symtab = self._build_wrapped_inner_scope( scope, - getattr(inner_script, "let", None) if inner_script is not None else None, - getattr(inner_script, "embeddedFiles", None) if inner_script is not None else None, + inner_script.let if inner_script is not None else None, + inner_script.embeddedFiles if inner_script is not None else None, symtab, ) inject(inner_symtab) @@ -1771,9 +1772,13 @@ def _inject_wrapped_env_symbols( is removed from tracking, so the wrapped environment's own variables are included.""" command = inner_action.command.resolve(symtab=inner_symtab) - args = ( - [a.resolve(symtab=inner_symtab) for a in inner_action.args] if inner_action.args else [] - ) + # RFC 0005 §1.3.2 typed argument semantics (null skip, list + # flattening), shared with the enforcement path + # (ScriptRunnerBase._run_action) so the hook sees exactly the + # arguments the wrapped action would have run with unwrapped — + # mirroring openjd-rs's seed_wrapped_action_symbols, which resolves + # via the same resolve_action_args as the runner. + args = resolve_action_arg_values(inner_action.args, inner_symtab) symtab["WrappedAction.Command"] = command symtab["WrappedAction.Args"] = args symtab["WrappedAction.Environment"] = ( @@ -1807,9 +1812,10 @@ def _inject_wrapped_task_symbols( on_run = step_script.actions.onRun symtab["WrappedAction.Command"] = on_run.command.resolve(symtab=inner_symtab) - symtab["WrappedAction.Args"] = ( - [arg.resolve(symtab=inner_symtab) for arg in on_run.args] if on_run.args else [] - ) + # RFC 0005 §1.3.2 typed argument semantics (null skip, list + # flattening), shared with the enforcement path — see + # _inject_wrapped_env_symbols. + symtab["WrappedAction.Args"] = resolve_action_arg_values(on_run.args, inner_symtab) symtab["WrappedAction.Environment"] = self._collect_session_env_list() symtab["WrappedAction.Timeout"] = self._resolve_action_timeout(on_run, inner_symtab) self._inject_wrapped_cancelation_symbols( @@ -1898,7 +1904,7 @@ def _materialize_path_mapping( else: rules_dict = dict() symtab[ValueReferenceConstants_2023_09.HAS_PATH_MAPPING_RULES.value] = "false" - # RFC 0007 §7.3: for EXPR evaluation Session.HasPathMappingRules is a + # RFC 0005; Template Schemas §7.3: for EXPR evaluation Session.HasPathMappingRules is a # boolean and Session.PathMappingRulesFile is a path, matching # openjd-rs's typed session symbols. The legacy (non-EXPR) # interpolation path ignores these types and keeps the string forms. diff --git a/test/openjd/sessions_v0/test_runner_env_script.py b/test/openjd/sessions_v0/test_runner_env_script.py index bc88bfdb..0c046019 100644 --- a/test/openjd/sessions_v0/test_runner_env_script.py +++ b/test/openjd/sessions_v0/test_runner_env_script.py @@ -387,7 +387,9 @@ def test_cancel( # The lower-level process runners have been thoroughly tested for cancel's # functionality, so this seems fine. - with patch.object(EnvironmentScriptRunner, "_run_action"): + # Patch _run (not _run_action): the effective cancel method is now + # resolved by _run_action at launch time and consumed by cancel(). + with patch.object(EnvironmentScriptRunner, "_run"): with patch.object(EnvironmentScriptRunner, "_cancel") as mock_cancel: # GIVEN script = EnvironmentScript_2023_09( @@ -468,6 +470,7 @@ def test_run_env_action_passes_default_timeout( action, symtab, default_timeout=default_timeout, + default_notify_period_seconds=30, ) def test_exit_uses_default_timeout( diff --git a/test/openjd/sessions_v0/test_runner_step_script.py b/test/openjd/sessions_v0/test_runner_step_script.py index ee5e5983..6e4c780b 100644 --- a/test/openjd/sessions_v0/test_runner_step_script.py +++ b/test/openjd/sessions_v0/test_runner_step_script.py @@ -252,7 +252,9 @@ def test_cancel( # The lower-level process runners have been thoroughly tested for cancel's # functionality, so this seems fine. - with patch.object(StepScriptRunner, "_run_action"): + # Patch _run (not _run_action): the effective cancel method is now + # resolved by _run_action at launch time and consumed by cancel(). + with patch.object(StepScriptRunner, "_run"): with patch.object(StepScriptRunner, "_cancel") as mock_cancel: # GIVEN script = StepScript_2023_09( diff --git a/test/openjd/sessions_v0/test_session_let_bindings.py b/test/openjd/sessions_v0/test_session_let_bindings.py index 8dd80990..40575053 100644 --- a/test/openjd/sessions_v0/test_session_let_bindings.py +++ b/test/openjd/sessions_v0/test_session_let_bindings.py @@ -1,7 +1,7 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. """Regression tests for the Session-level handling of EXPR ``let`` -bindings (RFC 0007): +bindings (RFC 0005): - a failing ``extra_let_bindings`` entry on ``enter_environment`` / ``exit_environment`` fails the action through the normal callback path @@ -13,6 +13,7 @@ - the unified optional int-or-format-string field resolver (``resolve_optional_int_field``) enforces consistent bounds. """ + from __future__ import annotations import time @@ -33,8 +34,10 @@ ) from openjd.model._let_bindings import _parse_rhs from openjd.sessions import ActionState, ActionStatus, Session, SessionState +from openjd.model.v2023_09 import StepScript as StepScript_2023_09 from openjd.sessions._runner_base import ( apply_let_bindings, + resolve_action_arg_values, resolve_optional_int_field, ) @@ -123,7 +126,7 @@ def callback(session_id: str, status: ActionStatus) -> None: # --------------------------------------------------------------------------- -# enter_environment(step_name=...) seeds Step.Name (RFC 0007 EXPR), for both +# enter_environment(step_name=...) seeds Step.Name (RFC 0005 EXPR), for both # the enter side and the re-applied bindings on the exit side. # --------------------------------------------------------------------------- @@ -207,6 +210,43 @@ def test_cached_expression_evaluates_against_per_call_symtab(self) -> None: assert str(second["y"]) == "42" +# --------------------------------------------------------------------------- +# A let-bound LIST keeps its engine type through the symbol table and +# flattens through whole-field argument resolution (RFC 0005 §1.3.2) — +# regression for the typed round trip of non-string binding values. +# --------------------------------------------------------------------------- + + +class TestLetBoundListInArgs: + def test_let_bound_list_flattens_into_args(self) -> None: + # GIVEN: a step script whose `let` binds a list and whose onRun + # consumes it as a whole-field argument expression. + context = ModelParsingContext_2023_09(supported_extensions=["EXPR"]) + script = StepScript_2023_09.model_validate( + { + "let": ["files = ['alpha beta', 'gamma']"], + "actions": { + "onRun": { + "command": "echo", + "args": ["front", "{{ files }}", "back"], + } + }, + }, + context=context, + ) + + # WHEN: the bindings are applied the way the runner applies them, + # and the args resolve through the shared enforcement helper. + symtab = SymbolTable() + apply_let_bindings(symtab=symtab, let_bindings=script.let or []) + resolved = resolve_action_arg_values(script.actions.onRun.args, symtab) + + # THEN: the stored value survived the engine symbol-table build as a + # typed list — flattened inline, one argument per element, embedded + # whitespace preserved — not rendered as a single stringified list. + assert resolved == ["front", "alpha beta", "gamma", "back"] + + # --------------------------------------------------------------------------- # resolve_optional_int_field: one implementation for the three # "optional int-or-format-string" call sites, with consistent bounds. @@ -228,6 +268,16 @@ def test_whole_field_null_resolves_to_none(self) -> None: symtab["X"] = None assert resolve_optional_int_field(field, symtab, ge=1, description="timeout") is None + def test_whole_field_empty_string_rejected(self) -> None: + # A genuine empty STRING is not null (openjd-rs parity: only an + # ExprValue::Null result means "field omitted"; an empty string + # falls through to the integer parse and errors). + field = _format_string_field("{{ X }}") + symtab = SymbolTable() + symtab["X"] = "" + with pytest.raises(ValueError, match="timeout must be a positive integer, got ''"): + resolve_optional_int_field(field, symtab, ge=1, description="timeout") + def test_resolved_value_below_ge_rejected(self) -> None: field = _format_string_field("{{ X }}") symtab = SymbolTable() @@ -246,6 +296,32 @@ def test_resolved_value_above_le_rejected(self) -> None: field, symtab, ge=1, le=600, description="notifyPeriodInSeconds" ) + @pytest.mark.parametrize( + "value", + [ + pytest.param(" 45 ", id="surrounding whitespace"), + pytest.param("1_0", id="digit-group underscore"), + pytest.param("\u0661\u0662\u0663", id="non-ascii decimal digits"), + ], + ) + def test_lenient_int_spellings_rejected(self, value: str) -> None: + # Strict ASCII integer grammar (openjd-rs parity): Python's int() + # accepts all of these spellings, but Rust's str::parse rejects + # them — accepting them here would be a spec-observable divergence + # for dynamically resolved timeout/notifyPeriodInSeconds values. + field = _format_string_field("{{ X }}") + symtab = SymbolTable() + symtab["X"] = value + with pytest.raises(ValueError, match="timeout must be a positive integer"): + resolve_optional_int_field(field, symtab, ge=1, description="timeout") + + def test_leading_plus_sign_accepted(self) -> None: + # Rust's u64/i64 from_str accepts a leading '+'; so do we. + field = _format_string_field("{{ X }}") + symtab = SymbolTable() + symtab["X"] = "+45" + assert resolve_optional_int_field(field, symtab, ge=1, description="timeout") == 45 + def test_non_integer_resolved_value_rejected(self) -> None: field = _format_string_field("{{ X }}") symtab = SymbolTable() diff --git a/test/openjd/sessions_v0/test_wrap_cancelation.py b/test/openjd/sessions_v0/test_wrap_cancelation.py index a1a38639..55327ab8 100644 --- a/test/openjd/sessions_v0/test_wrap_cancelation.py +++ b/test/openjd/sessions_v0/test_wrap_cancelation.py @@ -269,7 +269,12 @@ def _deferred(self, mode: str, period: str | None = None): from openjd.model._format_strings import FormatString from openjd.model.v2023_09 import CancelationMethodDeferred, ModelParsingContext - ctx = ModelParsingContext() + # A deferred mode is an RFC 0008 forwarding construct, and + # WRAP_ACTIONS requires the EXPR extension — so the format strings + # here parse as EXPR expressions, giving the typed null semantics + # the runtime relies on (a whole-field null drops the cancelation + # object; an empty STRING is an error, matching openjd-rs). + ctx = ModelParsingContext(supported_extensions=["FEATURE_BUNDLE_1", "EXPR"]) return CancelationMethodDeferred( mode=FormatString(mode, context=ctx), notifyPeriodInSeconds=( @@ -313,6 +318,17 @@ def test_mode_resolving_notify_then_terminate_with_period(self) -> None: ) assert result == ("NOTIFY_THEN_TERMINATE", 45) + def test_whole_field_mode_resolving_empty_string_raises(self) -> None: + # A genuine empty STRING is not null, even for a whole-field + # expression (openjd-rs parity: only an ExprValue::Null result + # drops the cancelation object; an empty string is an invalid + # mode). E.g. a STRING parameter whose value is "". + from openjd.sessions._runner_base import resolve_effective_cancelation + + cancelation = self._deferred("{{X}}") + with pytest.raises(ValueError, match="must resolve to .* got ''"): + resolve_effective_cancelation(cancelation, self._symtab(X="")) + def test_mode_resolving_garbage_raises(self) -> None: from openjd.sessions._runner_base import resolve_effective_cancelation @@ -351,3 +367,111 @@ def test_resolved_period_exceeding_cap_raises(self) -> None: resolve_effective_cancelation( cancelation, self._symtab(X="NOTIFY_THEN_TERMINATE", P=9999) ) + + +# --------------------------------------------------------------------------- +# Launch-time cancelation resolution (openjd-rs run_action parity): +# the effective cancel method is resolved by _run_action against the SAME +# final scope the command/args resolved with (script lets, *.File.*, +# WrappedAction.*) and stored on the runner; cancel() consumes it. An +# unresolvable or invalid cancelation fails the action at start. +# --------------------------------------------------------------------------- + + +class TestLaunchTimeCancelationResolution: + def _env_with_let_bound_cancelation(self) -> Environment_2023_09: + from openjd.model.v2023_09 import ModelParsingContext + + ctx = ModelParsingContext(supported_extensions=["EXPR", "WRAP_ACTIONS", "FEATURE_BUNDLE_1"]) + return Environment_2023_09.model_validate( + { + "name": "Wrapper", + "script": { + "let": [ + "hookMode = 'NOTIFY_THEN_TERMINATE'", + "hookPeriod = 9", + ], + "actions": { + "onEnter": {"command": "true"}, + "onWrapEnvEnter": {"command": "true"}, + "onWrapEnvExit": {"command": "true"}, + "onWrapTaskRun": { + "command": "sleep", + "args": ["20"], + "cancelation": { + "mode": "{{hookMode}}", + "notifyPeriodInSeconds": "{{hookPeriod}}", + }, + }, + }, + }, + }, + context=ctx, + ) + + def test_let_bound_cancelation_resolved_against_final_scope(self) -> None: + # Regression: cancel() used to re-resolve the cancelation against + # the runner's BASE symtab — which lacks the script's `let` + # bindings — so a let-referencing mode fell back to Terminate with + # a warning. It must resolve at launch, in the hook's final scope. + from datetime import timedelta + from unittest.mock import MagicMock, patch + + from openjd.sessions._runner_env_script import EnvironmentScriptRunner + from openjd.sessions._runner_base import NotifyCancelMethod + + env = self._env_with_let_bound_cancelation() + import pathlib + import tempfile + + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + with patch.object(EnvironmentScriptRunner, "_run"): + runner = EnvironmentScriptRunner( + logger=MagicMock(), + session_working_directory=tmp_path, + environment_script=env.script, + symtab=SymbolTable(), + session_files_directory=tmp_path, + ) + runner.wrap_task_run() + assert runner._resolved_cancel_method == NotifyCancelMethod( + terminate_delay=timedelta(seconds=9) + ) + + def test_invalid_deferred_mode_fails_action_at_launch(self) -> None: + # Eager validation (openjd-rs parity): a cancelation whose deferred + # mode resolves to something other than the two method names or + # null must FAIL the action at start — not launch successfully and + # only surface if a cancel later occurs. + session_id = uuid.uuid4().hex + probe = Action_2023_09( + command=CommandString_2023_09("sh"), + args=[ArgString_2023_09("-c"), ArgString_2023_09("echo should-not-run")], + ) + env = _wrap_env("wrap_env", probe) + step = _step_script_with_cancelation(None) + # Forward an invalid mode through the wrap round trip: the wrap + # action's own cancelation defers to a symbol that resolves to + # garbage at run time. + from openjd.model._format_strings import FormatString + from openjd.model.v2023_09 import CancelationMethodDeferred, ModelParsingContext + + ctx = ModelParsingContext(supported_extensions=["FEATURE_BUNDLE_1", "EXPR"]) + object.__setattr__( + env.script.actions.onWrapTaskRun, + "cancelation", + CancelationMethodDeferred( + mode=FormatString("{{WrappedStep.Name}}", context=ctx), + notifyPeriodInSeconds=None, + ), + ) + with Session(session_id=session_id, job_parameter_values={}) as session: + session.enter_environment(environment=env) + _run_until_ready(session) + session.run_task(step_script=step, task_parameter_values={}, step_name="NotAMode") + _run_until_ready(session) + status = session.action_status + assert status is not None + assert status.state == ActionState.FAILED + assert session.state == SessionState.READY_ENDING diff --git a/test/openjd/sessions_v0/test_wrap_task_run.py b/test/openjd/sessions_v0/test_wrap_task_run.py index e11de82e..9126f36d 100644 --- a/test/openjd/sessions_v0/test_wrap_task_run.py +++ b/test/openjd/sessions_v0/test_wrap_task_run.py @@ -105,6 +105,45 @@ def test_injects_empty_args_when_step_has_no_args(self) -> None: finally: session.cleanup() + def test_injects_typed_args_null_skip_and_list_flatten(self) -> None: + # RFC 0005 §1.3.2 typed argument semantics (openjd-rs parity: the + # wrapped path in seed_wrapped_action_symbols resolves through the + # same resolve_action_args as the runner): a whole-field list + # expression flattens inline (one argument per element), a + # whole-field null is skipped, and the hook sees exactly the argv + # the wrapped action would have run with unwrapped. + from openjd.model.v2023_09 import ModelParsingContext + from openjd.sessions._runner_base import resolve_action_arg_values + + context = ModelParsingContext(supported_extensions=["EXPR"]) + script = StepScript_2023_09.model_validate( + { + "actions": { + "onRun": { + "command": "echo", + "args": ["front", '{{ ["a", "b c"] }}', "{{ null }}", "back"], + } + } + }, + context=context, + ) + session = Session(session_id=uuid.uuid4().hex, job_parameter_values={}) + try: + symtab = SymbolTable() + session._inject_wrapped_task_symbols(symtab, script, "MyStep", inner_symtab=symtab) + assert symtab["WrappedAction.Args"] == ["front", "a", "b c", "back"] + # The unwrapped enforcement path (_run_action) resolves the same + # action's args via the same shared helper — wrapped and + # unwrapped runs of this action use identical argv. + assert resolve_action_arg_values(script.actions.onRun.args, symtab) == [ + "front", + "a", + "b c", + "back", + ] + finally: + session.cleanup() + def test_injects_wrapped_environment_as_key_value_list(self) -> None: # RFC 0008 (openjd-rs #277): WrappedAction.Environment carries every # session-defined variable — openjd_env definitions (applied via From 7855778c4c05b8765210bbea9ddcb8db92ec4624 Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Fri, 24 Jul 2026 23:15:18 -0700 Subject: [PATCH 04/27] fix: Close review findings on failure paths, timeout bounds and typing Addresses the findings from two independent reviews of this PR, after validating each against openjd-rs upstream/main and the specification. Fixes that change behavior: - An oversized but schema-legal `timeout` no longer raises OverflowError out of the public Session API. Two limits sit above what we can enforce: timedelta tops out at 999999999 days, and threading.Timer's deadline arithmetic overflows just above 2**63 nanoseconds -- so a clamp to timedelta.max is not enough. A value beyond MAX_SCHEDULABLE_TIMEOUT_SECONDS now runs the action with no time limit and a warning, matching openjd-rs, whose Duration-based timer effectively never fires at that magnitude. A value above u64::MAX fails the action, mirroring openjd-rs's str::parse::() -- applied to literal and resolved values alike so that a value forwarded as {{WrappedAction.Timeout}} behaves as it would have unwrapped. Reachable from run_task, enter/exit_environment and run_subprocess; previously left the Session stuck in RUNNING with no terminal ActionStatus. - A runtime EXPR failure in an environment's `variables:` map no longer escapes enter_environment(). It now fails through the callback path with the identifier returned, so the caller can exit the environment it entered. Seeding the empty change record is required: the log filter indexes _created_env_vars without a membership test. - run_task() now raises ValueError when a wrap environment is active and no step_name was given, instead of rendering WrappedStep.Name as "". RFC 0008 defines it as the wrapped step's name and has a minimum length of one, so an empty container or label name was a silent wrong result. Raising keeps the session usable, unlike failing the action. - URI path-mapping folds the scheme+authority over ASCII only, matching openjd-rs's eq_ignore_ascii_case. str.lower() folded U+212A KELVIN SIGN to ASCII 'k', so `s3://bucketK` matched `s3://bucketk`. Hardening and cleanup: - Replace getattr duck-typing on the EXPR typed-value API with concrete ExprValue/TypeCode checks, so model skew fails loudly instead of silently treating every optional integer field as omitted. Extended to the two sites that substituted a plausible default: the cancelation notify period, and the wrap-hook lookup that would otherwise report SUCCESS without running a hook. - Replace an unreachable TypeError in PathMappingRule.apply with an assert; it is called from run_task, where nothing would catch it. - Embedded-file phase logging now says what each phase does; two methods claimed a write neither performed while the method that writes logged nothing. - Correct the wrap-env file-content rewrite rationale: validation rejects WrappedAction.* in an environment script's data and let, so the content is invariant. The rewrite is kept for per-invocation determinism. - Collapse a dead branch in the wrap-hook dispatch, drop the _WRAP_HOOK_NAMES re-export, document enter_environment's Raises, and correct exit_environment's (it documented ValueError where the code raises RuntimeError). - Document that openjd_env from a task-replacing hook deliberately does not define a session variable, that this diverges from openjd-rs, and that the spec does not settle it; log the discard at debug level. - Note that _run_task_without_session_env bypasses wrap dispatch. Tests: 18 added (oversized/over-u64 timeouts, the environment.variables lifecycle, step_name required-and-optional, the first tests _apply_uri has had, and an end-to-end cancel-delivery test proving the launch-resolved notify method is applied to a SIGTERM-trapping subprocess). Eleven existing wrap tests updated to pass step_name. Conformance 2023-09 with the Python CLI: 1162 passed, 0 failed (unchanged). Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- src/openjd/sessions/_embedded_files.py | 22 +-- src/openjd/sessions/_path_mapping.py | 34 ++++- src/openjd/sessions/_runner_base.py | 117 ++++++++++++++-- src/openjd/sessions/_runner_env_script.py | 23 +-- src/openjd/sessions/_session.py | 131 +++++++++++++++--- test/openjd/sessions_v0/test_path_mapping.py | 80 +++++++++++ test/openjd/sessions_v0/test_runner_base.py | 78 +++++++++++ .../sessions_v0/test_session_let_bindings.py | 58 ++++++++ test/openjd/sessions_v0/test_wrap_actions.py | 16 ++- .../sessions_v0/test_wrap_cancelation.py | 87 +++++++++++- test/openjd/sessions_v0/test_wrap_task_run.py | 73 +++++++++- 11 files changed, 649 insertions(+), 70 deletions(-) diff --git a/src/openjd/sessions/_embedded_files.py b/src/openjd/sessions/_embedded_files.py index 61fe86a4..1af03c9f 100644 --- a/src/openjd/sessions/_embedded_files.py +++ b/src/openjd/sessions/_embedded_files.py @@ -226,11 +226,7 @@ def allocate_file_paths( to the bindings, while a file's ``data`` is written afterwards so it can reference let-bound values. Mirrors the openjd-rs runners. """ - if self._scope == EmbeddedFilesScope.ENV: - self._logger.info("Writing embedded files for Environment to disk.") - else: - self._logger.info("Writing embedded files for Task to disk.") - + self._log_scoped("Allocating embedded file paths for") records = self.allocate_records(files) self._define_symbols(records, symtab) return records @@ -268,12 +264,19 @@ def register_file_paths(self, records: list["_FileRecord"], symtab: SymbolTable) while the contents are re-resolved and rewritten per invocation via :meth:`write_file_contents`. """ - if self._scope == EmbeddedFilesScope.ENV: - self._logger.info("Writing embedded files for Environment to disk.") - else: - self._logger.info("Writing embedded files for Task to disk.") + self._log_scoped("Reusing embedded file paths for") self._define_symbols(records, symtab) + def _log_scoped(self, message: str) -> None: + """Log ``message`` qualified by this collection's scope. + + The phase messages are kept accurate on purpose: allocation reserves + paths, registration only defines symbols, and only + :meth:`write_file_contents` writes content. + """ + scope = "Environment" if self._scope == EmbeddedFilesScope.ENV else "Task" + self._logger.info(f"{message} {scope}.") + def _define_symbols(self, records: list["_FileRecord"], symtab: SymbolTable) -> None: # Add symbols to the symbol table. For EXPR evaluation the # Env.File.*/Task.File.* symbols are host-format path values @@ -293,6 +296,7 @@ def _define_symbols(self, records: list["_FileRecord"], symtab: SymbolTable) -> def write_file_contents(self, records: list["_FileRecord"], symtab: SymbolTable) -> None: """Resolve each allocated file's ``data`` against ``symtab`` and write it to disk. See :meth:`allocate_file_paths`.""" + self._log_scoped("Writing embedded files to disk for") try: for record in records: # Raises: OSError diff --git a/src/openjd/sessions/_path_mapping.py b/src/openjd/sessions/_path_mapping.py index 65c00ef4..6ef6ccf1 100644 --- a/src/openjd/sessions/_path_mapping.py +++ b/src/openjd/sessions/_path_mapping.py @@ -4,8 +4,26 @@ from enum import Enum from os import name as os_name from pathlib import PurePath, PurePosixPath, PureWindowsPath +from string import ascii_lowercase, ascii_uppercase from typing import Optional, Union +_ASCII_LOWER_TABLE = str.maketrans(ascii_uppercase, ascii_lowercase) +"""Translation table for an ASCII-only case fold. + +RFC 3986 makes a URI's scheme and authority case-insensitive over ASCII only, +which is what openjd-rs implements (``str::eq_ignore_ascii_case``). ``str.lower()`` +would additionally fold non-ASCII characters — U+212A KELVIN SIGN lowers to ASCII +``k``, so ``s3://bucket\u212a`` would match ``s3://bucketk`` here but not in +openjd-rs. A translation table is also safer than ``str.lower()``/``str.casefold()`` +because it cannot change the string's length (``"\u0130".lower()`` yields two +characters), and :meth:`PathMappingRule._apply_uri` relies on offsets into the +string.""" + + +def _ascii_lower(value: str) -> str: + """Lower-case the ASCII letters in ``value``, leaving everything else alone.""" + return value.translate(_ASCII_LOWER_TABLE) + class PathFormat(str, Enum): POSIX = "POSIX" @@ -121,8 +139,9 @@ def _apply_uri(self, path: str) -> tuple[bool, str]: inp_path_start = self._uri_path_start(path) inp_path_start = 0 if inp_path_start is None else inp_path_start - # Scheme+authority must match case-insensitively. - if path[:inp_path_start].lower() != source[:src_path_start].lower(): + # Scheme+authority must match case-insensitively, over ASCII only — + # see _ASCII_LOWER_TABLE for why this is not str.lower(). + if _ascii_lower(path[:inp_path_start]) != _ascii_lower(source[:src_path_start]): return False, path # Path portion must match case-sensitively, on a component boundary. src_path = source[src_path_start:] @@ -164,11 +183,12 @@ def apply(self, *, path: str) -> tuple[bool, str]: if self.source_path_format == PathFormat.URI: return self._apply_uri(path) source_path = self.source_path - if not isinstance(source_path, PurePath): - # The constructor guarantees non-URI rules carry PurePath sources. - raise TypeError( - "Path mapping rule source_path must be a PurePath for filesystem source formats" - ) + # The constructor guarantees non-URI rules carry PurePath sources. An + # assert rather than a raise: this method is reached from + # Session.run_task, where nothing catches a TypeError, so a broken + # invariant must not be able to become an exception escaping the public + # API. mypy narrows on the assert just as well. + assert isinstance(source_path, PurePath) pure_path: PurePath if self.source_path_format == PathFormat.POSIX: pure_path = PurePosixPath(path) diff --git a/src/openjd/sessions/_runner_base.py b/src/openjd/sessions/_runner_base.py index 5f66d236..028669b0 100644 --- a/src/openjd/sessions/_runner_base.py +++ b/src/openjd/sessions/_runner_base.py @@ -19,6 +19,13 @@ from openjd.model import SymbolTable from openjd.model import FormatStringError from openjd.model import evaluate_let_bindings + +# The EXPR engine's typed value. Imported concretely (rather than duck-typed +# with getattr) so that a model API change fails loudly at import time instead +# of silently mis-classifying every optional integer field as "omitted". +# openjd.expr ships in the same distribution as openjd.model, which this module +# already hard-imports unreleased API from. +from openjd.expr import ExprValue, TypeCode from openjd.model.v2023_09 import Action as Action_2023_09 from openjd.model.v2023_09 import CancelationMethodDeferred as CancelationMethodDeferred_2023_09 from openjd.model.v2023_09 import CancelationMode as CancelationMode_2023_09 @@ -49,6 +56,51 @@ an optional sign followed by ASCII digits, exactly what Rust's ``str::parse`` accepts. See resolve_optional_int_field.""" +MAX_UNSIGNED_64BIT_INT = 2**64 - 1 +"""Largest value the openjd-rs runtime can hold for these fields. + +openjd-rs models ``timeout`` as an ``Option`` and resolves it with +``str::parse::()`` (``runner/mod.rs`` resolve_action_timeout), which *fails* +rather than saturating above ``u64::MAX``; its model additionally rejects a +literal above ``i64::MAX`` at parse time. Python's model has no upper bound at +all, so we apply this one here — to literal and resolved values alike, so that +a value forwarded as ``{{WrappedAction.Timeout}}`` (RFC 0008) behaves exactly as +it would have unwrapped.""" + +MAX_SCHEDULABLE_TIMEOUT_SECONDS = 2**62 // 10**9 +"""Largest action timeout we can actually enforce, in seconds (~146 years). + +Two distinct limits sit above this, both of which used to surface as an +``OverflowError``: :class:`datetime.timedelta` tops out at 999,999,999 days, and +:class:`threading.Timer` computes an absolute deadline in CPython's internal +64-bit nanosecond time representation, which overflows just above 2**63 +nanoseconds (measured: 9,223,372,036 s schedules, 9,223,372,037 s does not). +The bound here is deliberately well inside that boundary rather than pinned to +it, because the overflow is reported against the platform's ``time_t`` and the +exact threshold is not portable. + +A timeout beyond this is indistinguishable from "no timeout", which is what +openjd-rs's ``Duration``-based timer effectively provides at that magnitude, so +the action runs unbounded instead of failing. See _timeout_from_seconds.""" + + +def _timeout_from_seconds(seconds: int, logger: LoggerAdapter) -> Optional[timedelta]: + """Convert a resolved timeout in seconds into an enforceable time limit. + + Returns ``None`` — no time limit — for a value too large to schedule (see + :data:`MAX_SCHEDULABLE_TIMEOUT_SECONDS`), so that an absurd-but-valid + timeout runs the action to completion the way openjd-rs does, rather than + raising ``OverflowError`` out of the public Session API. + """ + if seconds > MAX_SCHEDULABLE_TIMEOUT_SECONDS: + logger.warning( + f"Action timeout of {seconds} seconds is larger than this runtime can enforce; " + "the action will run without a time limit.", + extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), + ) + return None + return timedelta(seconds=seconds) + class ScriptRunnerState(str, Enum): """State of a ScriptRunner.""" @@ -138,9 +190,9 @@ def resolve_action_arg_values(args: Optional[Sequence], symtab: SymbolTable) -> continue if isinstance(value, str): resolved.append(value) - elif getattr(value, "is_null", False): + elif value.is_null: continue - elif str(getattr(value, "type", "")).startswith("list["): + elif value.type.type_code == TypeCode.LIST: resolved.extend(str(element) for element in value) else: resolved.append(str(value)) @@ -180,7 +232,22 @@ def resolve_optional_int_field( """ if value is None: return None + if ge is not None and le is not None: + constraint = f"between {ge} and {le}" + elif ge == 1 and le is None: + constraint = "a positive integer" + elif ge is not None: + constraint = f"an integer >= {ge}" + else: + constraint = "an integer" if isinstance(value, int): + # Literal values were bounds-checked by the static validator at parse + # time, with one exception: the validator has no upper bound (see + # MAX_UNSIGNED_64BIT_INT). Reject an unrepresentable literal here so + # the action fails through the caller's normal failure path instead of + # overflowing later. + if value > MAX_UNSIGNED_64BIT_INT: + raise ValueError(f"{description} must be {constraint}, got '{value}'") return value # Typed resolution: a whole-field EXPR expression yields the engine's # typed value, so a null result is distinguishable from a genuine empty @@ -188,17 +255,9 @@ def resolve_optional_int_field( # to plain string resolution — correct, since typed nulls only exist # under EXPR whole-field semantics (Template Schemas 5.3). resolved_value = value.resolve_value(symtab=symtab) - if getattr(resolved_value, "is_null", False): + if isinstance(resolved_value, ExprValue) and resolved_value.is_null: return None resolved = str(resolved_value) - if ge is not None and le is not None: - constraint = f"between {ge} and {le}" - elif ge == 1 and le is None: - constraint = "a positive integer" - elif ge is not None: - constraint = f"an integer >= {ge}" - else: - constraint = "an integer" # Strict ASCII integer grammar, matching the Rust runtime's str::parse # (openjd-rs resolve_action_timeout / resolve_notify_period_seconds). # Python's int() is more lenient — it accepts surrounding whitespace, @@ -208,7 +267,13 @@ def resolve_optional_int_field( if _STRICT_INT_RE.fullmatch(resolved) is None: raise ValueError(f"{description} must be {constraint}, got '{resolved}'") result = int(resolved) - if (ge is not None and result < ge) or (le is not None and result > le): + # str::parse::() fails rather than saturating above u64::MAX, so an + # out-of-range value must fail here too rather than reach the timer. + if ( + (ge is not None and result < ge) + or (le is not None and result > le) + or result > MAX_UNSIGNED_64BIT_INT + ): raise ValueError(f"{description} must be {constraint}, got '{result}'") return result @@ -270,7 +335,7 @@ def resolve_period(period: Any) -> Optional[int]: # openjd-rs runtime, which errors on any non-null, non-mode-name # result). mode_value = cancelation.mode.resolve_value(symtab=symtab) - if getattr(mode_value, "is_null", False): + if isinstance(mode_value, ExprValue) and mode_value.is_null: # Null mode drops the ENTIRE cancelation object: mode is the # object's required discriminator, so an "omitted" mode cannot # leave a partial object behind. The action behaves exactly as @@ -297,9 +362,14 @@ def resolve_period(period: Any) -> Optional[int]: ) if cancelation.mode == CancelationMode_2023_09.TERMINATE: return (CancelationMode_2023_09.TERMINATE.value, None) + # Direct attribute access, not getattr with a default: the mode above has + # already established this is a NOTIFY_THEN_TERMINATE object, which always + # carries the field. A getattr default would silently substitute the + # positional 30/120 s default for the author's period if the model ever + # renamed it. return ( CancelationMode_2023_09.NOTIFY_THEN_TERMINATE.value, - resolve_period(getattr(cancelation, "notifyPeriodInSeconds", None)), + resolve_period(cancelation.notifyPeriodInSeconds), ) @@ -674,6 +744,17 @@ def _materialize_files( reuses across wrap-hook invocations), path allocation is skipped; the records' symbols are defined in ``symtab`` and the contents are re-resolved and written as usual. + + Note on the per-invocation rewrite: the resolved bytes are in fact + invariant across a wrap environment's hook invocations, because model + validation rejects ``WrappedAction.*`` both in an environment script's + ``data`` and in its ``let`` bindings, and every other symbol a wrap + env's ``data`` may reference (``Param.*``, ``Session.*``, ``Job.Name``, + the now-stable ``Env.File.*``) is fixed for the session. The rewrite is + kept anyway, deliberately: it costs one small write immediately before + spawning a subprocess — the same cost class as the spawn — and it makes + each invocation deterministic even if a previous invocation's subprocess + modified a ``runnable`` embedded script in place. """ file_writer = EmbeddedFiles( logger=self._logger, @@ -760,7 +841,13 @@ def _run_action( self._fail_action(str(exc)) return if seconds is not None: - time_limit = timedelta(seconds=seconds) + # Assigned unconditionally: a declared timeout always replaces + # `default_timeout`, including when it is too large to enforce + # (in which case the action runs unbounded — see + # _timeout_from_seconds). Assigning only in the enforceable + # case would silently downgrade an oversized timeout to the + # positional default, e.g. an environment exit's 5 minutes. + time_limit = _timeout_from_seconds(seconds, self._logger) # Resolve the action's effective cancelation NOW, against the # same final scope the command/args/timeout resolved with — a # deferred (format-string) mode or FEATURE_BUNDLE_1 notify diff --git a/src/openjd/sessions/_runner_env_script.py b/src/openjd/sessions/_runner_env_script.py index 4e96df80..371a4628 100644 --- a/src/openjd/sessions/_runner_env_script.py +++ b/src/openjd/sessions/_runner_env_script.py @@ -245,11 +245,19 @@ def _run_wrap_hook(self, hook: str, *, default_timeout: Optional[timedelta] = No if self._environment_script is not None: assert isinstance(self._environment_script, EnvironmentScript_2023_09) - action = ( - getattr(self._environment_script.actions, hook, None) - if self._environment_script is not None - else None - ) + action = None + if self._environment_script is not None: + actions = self._environment_script.actions + # Distinguish "the model has no such field" from "the hook is not + # declared". Only the latter is a legitimate no-op; the former means + # the model's field was renamed, and a getattr default would silently + # report SUCCESS *without running the hook*. + if not hasattr(actions, hook): + raise RuntimeError( + f"This Environment's actions do not support the wrap hook '{hook}'. " + "The installed openjd-model may be incompatible with this runtime." + ) + action = getattr(actions, hook) if action is None: self._state_override = ScriptRunnerState.SUCCESS # Nothing to do, no wrap action defined. Call the callback @@ -258,10 +266,7 @@ def _run_wrap_hook(self, hook: str, *, default_timeout: Optional[timedelta] = No self._callback(ActionState.SUCCESS) return - if default_timeout is not None: - self._run_env_action(action, default_timeout=default_timeout) - else: - self._run_env_action(action) + self._run_env_action(action, default_timeout=default_timeout) def cancel( self, *, time_limit: Optional[timedelta] = None, mark_action_failed: bool = False diff --git a/src/openjd/sessions/_session.py b/src/openjd/sessions/_session.py index 28bae1d7..64eda187 100644 --- a/src/openjd/sessions/_session.py +++ b/src/openjd/sessions/_session.py @@ -285,7 +285,10 @@ class Session(object): are reused for every subsequent invocation — so the ``Env.File.*`` symbols stay stable across tasks and unnamed files do not accumulate on disk. The file *contents* are still re-resolved and rewritten per invocation by the - runner (a file's ``data`` may reference ``WrappedAction.*``). + runner — not because they can change (validation rejects ``WrappedAction.*`` + in an environment script's ``data`` and ``let``, so they cannot), but so that + every invocation starts from the authored content even if a previous one + modified the file on disk. See ScriptRunnerBase._materialize_files. """ _log_filter: Filter @@ -729,6 +732,12 @@ def enter_environment( Returns: EnvironmentIdentifier: An identifier by which the Environment is known by to this Session. Pass this identifier to exit_environment() when exiting this Environment. + + Raises: + RuntimeError: If the Session is not in the READY state; if the given + identifier has already been entered in this Session; or, per + RFC 0008, if this Environment defines wrap hooks and another + entered Environment already does. """ if self.state != SessionState.READY: raise RuntimeError("Session must be in the READY state to enter an Environment.") @@ -809,9 +818,32 @@ def enter_environment( # otherwise, we will end up running onEnter without # the environment variables of the current environment # being set. - resolved_variables = self._resolve_env_variable_format_strings( - symtab, environment.variables - ) + try: + resolved_variables = self._resolve_env_variable_format_strings( + symtab, environment.variables + ) + except ValueError as e: + # ExpressionError and FormatStringError subclass ValueError. A + # variable's expression failed to evaluate at run time — which + # EXPR (RFC 0005) makes reachable for a template that passed + # validation, e.g. a host function applied to a value only known + # at run time. Fail the action through the normal failure path + # rather than raising out of the public API: the environment is + # already in the entered list, so the caller must be given a + # terminal ActionStatus *and* the identifier in order to exit it. + # + # Seeding the empty change record is required, not tidiness: + # _action_log_filter_callback indexes _created_env_vars by the + # running environment's identifier without a membership test, so + # an openjd_env emitted by this environment's onExit would + # otherwise raise KeyError on the log-forwarding thread. + self._created_env_vars[identifier] = SimplifiedEnvironmentVariableChanges( + dict[str, str]() + ) + self._fail_action_before_start( + f"Failed to resolve the environment variables for {environment.name}: {e}" + ) + return identifier for name, value in resolved_variables.items(): self._logger.info( "Setting: %s=%s", @@ -932,7 +964,9 @@ def exit_environment( the environments of a step and then run tasks from a different step. Raises: - ValueError - If the given identifier is not that of the next one that must be exited. + RuntimeError: If the Session is not in the READY or READY_ENDING state; + if the given identifier is not known to this Session; or if it is + not the next Environment that must be exited. """ if self.state != SessionState.READY and self.state != SessionState.READY_ENDING: raise RuntimeError( @@ -1097,6 +1131,12 @@ def run_task( Default: True step_name (Optional[str]): The name of the step whose task is being run. Used by RFC 0008 to populate ``WrappedStep.Name`` in wrap hooks. + Required when a wrap Environment is active. + + Raises: + RuntimeError: If the Session is not in the READY state. + ValueError: If a wrap Environment (RFC 0008) is active and no + ``step_name`` was given. """ if self.state != SessionState.READY: raise RuntimeError("Session must be in the READY state to run a task.") @@ -1136,13 +1176,18 @@ def run_task( wrap_env = self._find_wrap_environment(hook="onWrapTaskRun") if wrap_env is not None: if step_name is None: - # RFC 0008: without a step name, {{WrappedStep.Name}} - # renders as the empty string in the wrap script. Callers - # predating the step_name kwarg won't pass it; make the - # gap visible rather than silently rendering empty. - self._logger.warning( - "A wrap environment is active but run_task() was not given a " - "step_name; WrappedStep.Name will render as an empty string." + # RFC 0008 defines WrappedStep.Name as the name of the wrapped + # step, and has a minimum length of one — there is no + # "unknown step" value to render. Rendering the empty string + # instead would silently hand a wrap script an empty container + # or label name, so this is reported as what it is: caller + # misuse, in the same shape run_task already reports caller + # misuse. Raising (rather than failing the action) keeps the + # session usable, so the caller can retry with a step name. + raise ValueError( + f"run_task() requires step_name when a wrap environment " + f"('{wrap_env.name}') is active: RFC 0008's WrappedStep.Name " + f"has no value to render without it." ) # The wrapped onRun resolves against the STEP's own scope; the # hook resolves against `symtab`, and the wrap environment's own @@ -1153,7 +1198,7 @@ def run_task( inner_script=step_script, symtab=symtab, inject=lambda inner_symtab: self._inject_wrapped_task_symbols( - symtab, step_script, step_name or "", inner_symtab=inner_symtab + symtab, step_script, step_name, inner_symtab=inner_symtab ), fail_message=( f"Failed to resolve the wrapped Task action for {wrap_env.name}'s " @@ -1175,6 +1220,13 @@ def run_task( symtab=symtab, preallocated_file_records=wrap_file_records, ) + # Note: unlike enter_environment()/exit_environment(), which set + # RUNNING before their first failable step, this path sets it only + # after wrapped-symbol injection has succeeded — so an injection + # failure reports FAILED/READY_ENDING without the session ever + # having been observably RUNNING. Harmless for the documented + # poll-then-check-action_status pattern, but the asymmetry is + # deliberate: there is no runner to own the failure until here. self._action_state = ActionState.RUNNING self._state = SessionState.RUNNING self._runner.wrap_task_run() @@ -1210,6 +1262,11 @@ def _run_task_without_session_env( ) -> None: """Private API to run a task within the session. This method directly use os_env_vars passed in without applying additional session env setup. + + Note: unlike :meth:`run_task`, this deliberately does *not* dispatch RFC + 0008 wrap hooks, and does not seed ``Step.Name``. A caller that uses this + while a wrap Environment is entered runs the task unwrapped. Callers must + move to :meth:`run_task` to participate in wrap actions. """ if self.state != SessionState.READY: raise RuntimeError("Session must be in the READY state to run a task.") @@ -1512,8 +1569,6 @@ def _build_expr_host_rules(self) -> Optional[list[Any]]: # RFC 0008 wrap-action helpers # ------------------------------------------------------------------ - _WRAP_HOOK_NAMES = WRAP_HOOK_ACTION_NAMES - def _find_wrap_environment(self, *, hook: str) -> Optional[EnvironmentModel]: """Walk the environment stack (innermost first) and return the active wrapping environment for ``hook`` (one of ``onWrapEnvEnter``, @@ -1522,7 +1577,7 @@ def _find_wrap_environment(self, *, hook: str) -> Optional[EnvironmentModel]: Per RFC 0008 the session is only valid with at most one wrap-defining environment in the stack, so the first match is always the one that applies.""" - if hook not in self._WRAP_HOOK_NAMES: + if hook not in WRAP_HOOK_ACTION_NAMES: raise ValueError(f"Unknown wrap hook name: {hook}") for env_id in reversed(self._environments_entered): env = self._environments[env_id] @@ -1586,7 +1641,7 @@ def _environment_defines_any_wrap_hook(self, env: EnvironmentModel) -> bool: return False return any( hasattr(env.script.actions, name) and getattr(env.script.actions, name) is not None - for name in self._WRAP_HOOK_NAMES + for name in WRAP_HOOK_ACTION_NAMES ) def _build_wrapped_inner_scope( @@ -1961,7 +2016,25 @@ def _action_log_filter_callback( elif kind == ActionMessageKind.ENV: if self._running_environment_identifier is None: - # Ignore the message if we're not running an environment + # Ignore the message if we're not running an environment. + # + # Per How-Jobs-Are-Run, `openjd_env` "can only be emitted by the + # Action for entering an Environment" — so a task's onRun cannot + # define a session variable, and neither can an RFC 0008 + # onWrapTaskRun hook, which stands in for one. That keeps + # wrapping transparent: a task that prints an `openjd_env:` line + # behaves the same wrapped and unwrapped. + # + # This is deliberate, not an oversight, and it is a known + # divergence from openjd-rs, which records such a variable in the + # map that feeds WrappedAction.Environment while still not + # applying it to any subprocess environment — advertising a + # variable the wrapped context does not have. The spec does not + # settle the case (it also says a wrap script MAY emit these + # macros directly) and no conformance fixture covers it; filed + # upstream. Logged at debug so an author chasing a silent no-op + # has something to find, without adding noise to every task log. + self._log_discarded_env_macro(kind, value) return if cancel_action_mark_failed: # Assert for the type checker; the type is guaranteed by the ActionMonitoringFilter @@ -1982,7 +2055,9 @@ def _action_log_filter_callback( return elif kind == ActionMessageKind.UNSET_ENV: if self._running_environment_identifier is None: - # Ignore the message if we're not running an environment + # Ignore the message if we're not running an environment. + # See the ENV branch above for why this is deliberate. + self._log_discarded_env_macro(kind, value) return if cancel_action_mark_failed: @@ -2011,6 +2086,24 @@ def _action_log_filter_callback( assert action_status is not None self._callback(self._session_id, action_status) + def _log_discarded_env_macro(self, kind: ActionMessageKind, value: Any) -> None: + """Record, at debug level, that an environment-variable stdout macro was + ignored because the running Action is not an Environment's entry Action. + + Debug rather than a warning on purpose: ``_reset_action_state`` clears + the running-environment identifier for every task, and this callback + cannot tell an RFC 0008 wrap hook from an ordinary task action — so a + warning here would fire for every existing job whose task happens to + print an ``openjd_env:`` line. + """ + name = value.get("name") if isinstance(value, dict) else value + self._logger.debug( + "Ignoring %s for '%s': environment variables can only be defined by " + "the Action that enters an Environment.", + kind.name.lower(), + name, + ) + def _fail_action_before_start(self, message: str) -> None: """Mark the pending action as FAILED before any runner/subprocess exists (RFC 0008: e.g. when resolving the wrapped action's format diff --git a/test/openjd/sessions_v0/test_path_mapping.py b/test/openjd/sessions_v0/test_path_mapping.py index 434ab517..86d98914 100644 --- a/test/openjd/sessions_v0/test_path_mapping.py +++ b/test/openjd/sessions_v0/test_path_mapping.py @@ -535,3 +535,83 @@ def test_from_dict_success(self, dict_rule, expected): def test_from_dict_failure(self, dict_rule): with pytest.raises(ValueError): PathMappingRule.from_dict(dict_rule) + + +class TestUriPathMapping: + """RFC 0006 §2.3.2 (EXPR): URI-form source paths. + + Per RFC 3986 the scheme and authority match case-insensitively while the + path portion matches case-sensitively, on whole path components. openjd-rs + implements the prefix comparison with ``str::eq_ignore_ascii_case``, so the + fold must be ASCII-only here too. + """ + + def _rule(self, source: str, destination: str = "/local") -> PathMappingRule: + return PathMappingRule.from_dict( + rule={ + "source_path_format": "URI", + "source_path": source, + "destination_path": destination, + } + ) + + @pytest.mark.parametrize( + "given, expected", + [ + pytest.param("s3://bucket/prefix/f.obj", (True, "/local/f.obj"), id="child-file"), + pytest.param("s3://bucket/prefix", (True, "/local"), id="exact-source"), + pytest.param("S3://BUCKET/prefix/f.obj", (True, "/local/f.obj"), id="ascii-fold"), + pytest.param( + "s3://bucket/PREFIX/f.obj", + (False, "s3://bucket/PREFIX/f.obj"), + id="path-is-case-sensitive", + ), + pytest.param( + "s3://other/prefix/f.obj", + (False, "s3://other/prefix/f.obj"), + id="different-authority", + ), + pytest.param( + "s3://bucket/prefixed/f.obj", + (False, "s3://bucket/prefixed/f.obj"), + id="not-a-component-boundary", + ), + ], + ) + def test_apply_uri(self, given: str, expected: tuple[bool, str]) -> None: + # GIVEN + rule = self._rule("s3://bucket/prefix") + + # WHEN / THEN + with patch.object(path_mapping_impl_mod, "os_name", OSName.POSIX.value): + assert rule.apply(path=given) == expected + + def test_scheme_authority_fold_is_ascii_only(self) -> None: + """A non-ASCII authority must not fold to an ASCII one. + + U+212A KELVIN SIGN lower-cases to ASCII 'k' under ``str.lower()``, so a + Unicode fold would make these two authorities match — where openjd-rs's + ``eq_ignore_ascii_case`` keeps them distinct. + """ + # GIVEN + rule = self._rule("s3://bucket\u212a") + + # WHEN + with patch.object(path_mapping_impl_mod, "os_name", OSName.POSIX.value): + matched, result = rule.apply(path="s3://bucketk/f.obj") + + # THEN + assert matched is False + assert result == "s3://bucketk/f.obj" + + def test_non_ascii_authority_still_matches_itself(self) -> None: + # GIVEN + rule = self._rule("s3://bucket\u212a") + + # WHEN + with patch.object(path_mapping_impl_mod, "os_name", OSName.POSIX.value): + matched, result = rule.apply(path="S3://BUCKET\u212a/f.obj") + + # THEN: ASCII letters fold, the non-ASCII character compares as itself + assert matched is True + assert result == "/local/f.obj" diff --git a/test/openjd/sessions_v0/test_runner_base.py b/test/openjd/sessions_v0/test_runner_base.py index 234eb999..cd76e5df 100644 --- a/test/openjd/sessions_v0/test_runner_base.py +++ b/test/openjd/sessions_v0/test_runner_base.py @@ -30,6 +30,7 @@ from openjd.sessions._os_checker import is_posix, is_windows from openjd.sessions._runner_base import ( + MAX_UNSIGNED_64BIT_INT, NotifyCancelMethod, ScriptRunnerBase, ScriptRunnerState, @@ -716,6 +717,83 @@ def test_run_action_default_timeout( else: assert "Log from test 19" in messages + @pytest.mark.usefixtures("message_queue", "queue_handler") + @pytest.mark.parametrize( + argnames="timeout_seconds", + argvalues=( + # Larger than datetime.timedelta can represent. + pytest.param(86_400_000_000_000, id="over-timedelta-max"), + # Representable by timedelta, but threading.Timer's deadline + # arithmetic overflows CPython's 64-bit nanosecond time + # representation. + pytest.param(86_399_999_913_600, id="over-schedulable-but-in-timedelta"), + pytest.param(9_223_372_036_854_775_807, id="i64-max"), + ), + ) + def test_run_action_unenforceable_timeout_runs_unbounded( + self, + tmp_path: Path, + message_queue: SimpleQueue, + queue_handler: QueueHandler, + timeout_seconds: int, + ) -> None: + """A timeout too large to schedule must not raise; the action runs. + + The 2023-09 schema puts no upper bound on `timeout`, so these + values are template-legal. openjd-rs resolves them with + `Duration::from_secs` and runs the action, so we do the same rather than + raising OverflowError out of the public Session API (which used to leave + the Session stuck in RUNNING with no terminal ActionStatus). + """ + # GIVEN + action = Action_2023_09( + command=CommandString_2023_09("{{Task.Command}}"), + args=[ArgString_2023_09("ok")], + timeout=timeout_seconds, + ) + symtab = SymbolTable(source={"Task.Command": "echo" if is_posix() else "cmd.exe"}) + logger = build_logger(queue_handler) + + # WHEN + with TerminatingRunner(logger=logger, session_working_directory=tmp_path) as runner: + runner._run_action(action, symtab) + while runner.state == ScriptRunnerState.RUNNING: + time.sleep(0.2) + + # THEN: it ran, with no time limit scheduled + assert runner.state == ScriptRunnerState.SUCCESS + assert runner._runtime_limit is None + messages = collect_queue_messages(message_queue) + assert any("larger than this runtime can enforce" in m for m in messages) + + @pytest.mark.usefixtures("queue_handler") + def test_run_action_timeout_above_u64_fails_action( + self, + tmp_path: Path, + queue_handler: QueueHandler, + ) -> None: + """A timeout beyond u64::MAX fails the action, matching openjd-rs. + + openjd-rs resolves these fields with `str::parse::()`, which fails + rather than saturating, so the action must fail through the normal + failure path instead of running. + """ + # GIVEN + action = Action_2023_09( + command=CommandString_2023_09("echo"), + args=[ArgString_2023_09("ok")], + timeout=MAX_UNSIGNED_64BIT_INT + 1, + ) + logger = build_logger(queue_handler) + + # WHEN + with TerminatingRunner(logger=logger, session_working_directory=tmp_path) as runner: + runner._run_action(action, SymbolTable()) + + # THEN + assert runner.state == ScriptRunnerState.FAILED + assert runner._runtime_limit is None + @pytest.mark.usefixtures("message_queue", "queue_handler") def test_run_action_bad_formatstring( self, diff --git a/test/openjd/sessions_v0/test_session_let_bindings.py b/test/openjd/sessions_v0/test_session_let_bindings.py index 40575053..3dfd2ec2 100644 --- a/test/openjd/sessions_v0/test_session_let_bindings.py +++ b/test/openjd/sessions_v0/test_session_let_bindings.py @@ -343,3 +343,61 @@ def test_session_resolve_action_timeout_rejects_non_positive(self) -> None: with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: with pytest.raises(ValueError, match="timeout must be a positive integer"): session._resolve_action_timeout(action, symtab) + + +# --------------------------------------------------------------------------- +# A runtime EXPR failure in an environment's `variables:` map must FAIL the +# action via the callback path, never raise out of enter_environment(). +# --------------------------------------------------------------------------- + + +class TestEnvironmentVariablesRuntimeFailure: + def test_failing_variable_expression_fails_action_cleanly(self) -> None: + # GIVEN: a `variables:` entry that passes validation but cannot be + # evaluated at run time (int() applied to a non-numeric value). Under + # legacy interpolation this was unreachable; EXPR host functions make it + # reachable for a template that validated successfully. + callback_events: list[ActionStatus] = [] + + def callback(session_id: str, status: ActionStatus) -> None: + callback_events.append(status) + + context = ModelParsingContext_2023_09(supported_extensions=["EXPR"]) + environment = Environment_2023_09.model_validate( + { + "name": "BrokenVars", + "variables": {"BROKEN": "{{ int(Session.WorkingDirectory.name) }}"}, + "script": {"actions": {"onEnter": {"command": "echo", "args": ["entered"]}}}, + }, + context=context, + ) + + with Session( + session_id=uuid.uuid4().hex, job_parameter_values={}, callback=callback + ) as session: + # WHEN: entering must not raise. + identifier = session.enter_environment(environment=environment) + _run_until_ready(session) + + # THEN: the caller gets the identifier, a terminal FAILED status, and + # a session it can still exit the attempted environment from. + assert identifier is not None + assert session.state == SessionState.READY_ENDING + status = session.action_status + assert status is not None + assert status.state == ActionState.FAILED + assert status.fail_message is not None + assert "environment variables" in status.fail_message + assert callback_events and callback_events[-1].state == ActionState.FAILED + assert identifier in session.environments_entered + + # AND: the change record was seeded, so the log-forwarding thread + # cannot KeyError on an openjd_env emitted by this environment. + assert identifier in session._created_env_vars + + # WHEN: cleanup exits it + session.exit_environment(identifier=identifier) + _run_until_ready(session) + + # THEN + assert identifier not in session.environments_entered diff --git a/test/openjd/sessions_v0/test_wrap_actions.py b/test/openjd/sessions_v0/test_wrap_actions.py index f01bca7b..1922337e 100644 --- a/test/openjd/sessions_v0/test_wrap_actions.py +++ b/test/openjd/sessions_v0/test_wrap_actions.py @@ -262,7 +262,7 @@ def test_full_ordering_across_three_hooks(self) -> None: wrapped_id = session.enter_environment(environment=wrapped_inner) _run_until_ready(session) - session.run_task(step_script=step, task_parameter_values={}) + session.run_task(step_script=step, task_parameter_values={}, step_name="Step") _run_until_ready(session) session.exit_environment(identifier=wrapped_id) @@ -359,7 +359,11 @@ def test_run_task_with_embedded_file_command_resolves( _run_until_ready(session) assert session.state == SessionState.READY - session.run_task(step_script=_embedded_file_step(), task_parameter_values={}) + session.run_task( + step_script=_embedded_file_step(), + task_parameter_values={}, + step_name="Step", + ) _run_until_ready(session) assert session.state == SessionState.READY @@ -408,7 +412,11 @@ def test_run_task_with_unresolvable_wrapped_symbol_fails_action(self) -> None: _run_until_ready(session) assert session.state == SessionState.READY - session.run_task(step_script=bad_step, task_parameter_values={}) + session.run_task( + step_script=bad_step, + task_parameter_values={}, + step_name="Step", + ) _run_until_ready(session) assert session.state == SessionState.READY_ENDING @@ -459,7 +467,7 @@ def test_same_let_name_each_side_sees_own_value(self, caplog: pytest.LogCaptureF _run_until_ready(session) # WHEN: the task runs through the wrap hook. - session.run_task(step_script=step, task_parameter_values={}) + session.run_task(step_script=step, task_parameter_values={}, step_name="Step") _run_until_ready(session) # THEN: the hook saw the WRAPPER's binding, and the wrapped diff --git a/test/openjd/sessions_v0/test_wrap_cancelation.py b/test/openjd/sessions_v0/test_wrap_cancelation.py index 55327ab8..ec20d967 100644 --- a/test/openjd/sessions_v0/test_wrap_cancelation.py +++ b/test/openjd/sessions_v0/test_wrap_cancelation.py @@ -40,6 +40,7 @@ StepScript as StepScript_2023_09, ) from openjd.sessions import ActionState, ActionStatus, Session, SessionState +from openjd.sessions._os_checker import is_posix # --------------------------------------------------------------------------- # Helpers @@ -215,7 +216,7 @@ def _run(self, cancelation, caplog: pytest.LogCaptureFixture) -> str: with Session(session_id=session_id, job_parameter_values={}) as session: session.enter_environment(environment=env) _run_until_ready(session) - session.run_task(step_script=step, task_parameter_values={}) + session.run_task(step_script=step, task_parameter_values={}, step_name="Step") _run_until_ready(session) assert session.action_status == ActionStatus(state=ActionState.SUCCESS, exit_code=0) return "\n".join(caplog.messages) @@ -475,3 +476,87 @@ def test_invalid_deferred_mode_fails_action_at_launch(self) -> None: assert status is not None assert status.state == ActionState.FAILED assert session.state == SessionState.READY_ENDING + + +# --------------------------------------------------------------------------- +# End-to-end delivery: canceling a running wrap hook must actually apply the +# method resolved at launch, not merely store it. +# --------------------------------------------------------------------------- + + +class TestWrapCancelationDelivery: + """The launch-time-resolution refactor removed cancel()'s ability to derive + the cancel method itself, so nothing but this test covers the wiring from + the stored method through to a delivered signal. Platform-gated like the + other signal-delivering cancel tests in this repo.""" + + @pytest.mark.skipif(not is_posix(), reason="Signals not yet implemented for non-posix") + def test_cancel_delivers_forwarded_notify_period(self, python_exe: str) -> None: + # GIVEN: a wrap hook whose own cancelation forwards the wrapped action's + # NOTIFY_THEN_TERMINATE mode and period, wrapping an action that declares + # a 1 second period. The hook runs a program that TRAPS SIGTERM, so only + # the terminate that follows the notify period can end it — a plain + # Terminate cancel would leave it running. + from datetime import timedelta + from pathlib import Path as _Path + + from openjd.model._format_strings import FormatString + from openjd.model.v2023_09 import CancelationMethodDeferred, ModelParsingContext + + from openjd.sessions._runner_base import NotifyCancelMethod + + ctx = ModelParsingContext(supported_extensions=["FEATURE_BUNDLE_1", "EXPR"]) + trapping_app = ( + _Path(__file__).parent / "support_files" / "app_20s_run_ignore_signal.py" + ).resolve() + hook = Action_2023_09( + command=CommandString_2023_09(python_exe), + args=[ArgString_2023_09(str(trapping_app))], + ) + object.__setattr__( + hook, + "cancelation", + CancelationMethodDeferred( + mode=FormatString("{{WrappedAction.Cancelation.Mode}}", context=ctx), + notifyPeriodInSeconds=FormatString( + "{{WrappedAction.Cancelation.NotifyPeriodInSeconds}}", context=ctx + ), + ), + ) + env = _wrap_env("wrap_env", hook) + step = _step_script_with_cancelation(_notify(1)) + + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + session.enter_environment(environment=env) + _run_until_ready(session) + + session.run_task(step_script=step, task_parameter_values={}, step_name="Step") + deadline = time.time() + 10.0 + while ( + session.action_status is None or session.action_status.state != ActionState.RUNNING + ): + if time.time() > deadline: # pragma: no cover - timing guard + pytest.fail("wrap hook never reached RUNNING") + time.sleep(0.05) + + runner = session._runner + assert runner is not None + # The forwarded mode+period resolved at launch... + assert runner._resolved_cancel_method == NotifyCancelMethod( + terminate_delay=timedelta(seconds=1) + ) + + # WHEN + time.sleep(0.5) # let the subprocess install its signal handler + session.cancel_action() + + # THEN: ...and the notify-then-terminate method was really delivered, + # so a SIGTERM-trapping subprocess still ends, and well before its own + # 20 second runtime. + started = time.time() + _run_until_ready(session, timeout_s=15.0) + elapsed = time.time() - started + status = session.action_status + assert status is not None + assert status.state == ActionState.CANCELED + assert elapsed < 15.0 diff --git a/test/openjd/sessions_v0/test_wrap_task_run.py b/test/openjd/sessions_v0/test_wrap_task_run.py index 9126f36d..84c3c69b 100644 --- a/test/openjd/sessions_v0/test_wrap_task_run.py +++ b/test/openjd/sessions_v0/test_wrap_task_run.py @@ -221,7 +221,7 @@ def test_no_wrap_runs_original_step(self, caplog: pytest.LogCaptureFixture) -> N # WHEN: we run the task. with Session(session_id=session_id, job_parameter_values={}) as session: - session.run_task(step_script=step, task_parameter_values={}) + session.run_task(step_script=step, task_parameter_values={}, step_name="Step") _run_until_ready(session) # THEN: the original step ran, not a wrap action. @@ -246,7 +246,7 @@ def test_wrap_action_intercepts_step(self, caplog: pytest.LogCaptureFixture) -> _run_until_ready(session) # WHEN: we run a task. - session.run_task(step_script=step, task_parameter_values={}) + session.run_task(step_script=step, task_parameter_values={}, step_name="Step") _run_until_ready(session) # THEN: the wrap action ran, not the original step. @@ -275,7 +275,7 @@ def test_wrap_action_receives_wrapped_command_symbol( session.enter_environment(environment=env) _run_until_ready(session) - session.run_task(step_script=step, task_parameter_values={}) + session.run_task(step_script=step, task_parameter_values={}, step_name="Step") _run_until_ready(session) assert session.action_status == ActionStatus(state=ActionState.SUCCESS, exit_code=0) @@ -314,7 +314,7 @@ def test_innermost_environment_wins(self, caplog: pytest.LogCaptureFixture) -> N session.enter_environment(environment=inner) _run_until_ready(session) - session.run_task(step_script=step, task_parameter_values={}) + session.run_task(step_script=step, task_parameter_values={}, step_name="Step") _run_until_ready(session) assert session.action_status == ActionStatus(state=ActionState.SUCCESS, exit_code=0) @@ -338,9 +338,17 @@ def test_wrap_action_runs_multiple_tasks(self, caplog: pytest.LogCaptureFixture) session.enter_environment(environment=env) _run_until_ready(session) - session.run_task(step_script=_step_script("cmd-one", []), task_parameter_values={}) + session.run_task( + step_script=_step_script("cmd-one", []), + task_parameter_values={}, + step_name="Step", + ) _run_until_ready(session) - session.run_task(step_script=_step_script("cmd-two", []), task_parameter_values={}) + session.run_task( + step_script=_step_script("cmd-two", []), + task_parameter_values={}, + step_name="Step", + ) _run_until_ready(session) messages = "\n".join(caplog.messages) @@ -556,3 +564,56 @@ def test_unnamed_wrap_file_path_reused_across_tasks(self) -> None: session.exit_environment(identifier=identifier) _run_until_ready(session) + + +# --------------------------------------------------------------------------- +# step_name is required once a wrap environment is active: RFC 0008's +# WrappedStep.Name has no value to render without it, and has a +# minimum length of one, so there is no empty sentinel to fall back on. +# --------------------------------------------------------------------------- + + +class TestWrappedStepNameRequired: + def test_run_task_without_step_name_raises(self) -> None: + # GIVEN: a wrap environment is entered + env = _wrap_env( + "container", + Action_2023_09( + command=CommandString_2023_09("echo"), + args=[ArgString_2023_09("{{WrappedStep.Name}}")], + ), + ) + step = _step_script("echo", ["hello"]) + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + identifier = session.enter_environment(environment=env) + _run_until_ready(session) + + # WHEN / THEN: omitting step_name is caller misuse, reported as such + # rather than silently rendering an empty container name. + with pytest.raises(ValueError, match="requires step_name"): + session.run_task(step_script=step, task_parameter_values={}) + + # AND: the session is left usable — nothing was started, so the + # caller can retry with a step name. + assert session.state == SessionState.READY + session.run_task(step_script=step, task_parameter_values={}, step_name="RetriedStep") + _run_until_ready(session) + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + + session.exit_environment(identifier=identifier) + _run_until_ready(session) + + def test_run_task_without_step_name_is_fine_unwrapped(self) -> None: + # GIVEN: no wrap environment + step = _step_script("echo", ["hello"]) + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + # WHEN: step_name is still optional on the unwrapped path + session.run_task(step_script=step, task_parameter_values={}) + _run_until_ready(session) + + # THEN + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS From 3848ab6082cd49b022613b59b9929e11695be01b Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Fri, 24 Jul 2026 23:20:44 -0700 Subject: [PATCH 05/27] fix: Narrow Optional script before setattr in wrap cancelation test CI lints with `mypy src test`, which flagged env.script as `EnvironmentScript | None` at the object.__setattr__ call. The invariant holds (_wrap_env always populates script), so assert it to narrow. This blocked the whole test matrix via fail-fast, so no OS ran its tests. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- test/openjd/sessions_v0/test_wrap_cancelation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/openjd/sessions_v0/test_wrap_cancelation.py b/test/openjd/sessions_v0/test_wrap_cancelation.py index ec20d967..ae869fcf 100644 --- a/test/openjd/sessions_v0/test_wrap_cancelation.py +++ b/test/openjd/sessions_v0/test_wrap_cancelation.py @@ -459,6 +459,7 @@ def test_invalid_deferred_mode_fails_action_at_launch(self) -> None: from openjd.model.v2023_09 import CancelationMethodDeferred, ModelParsingContext ctx = ModelParsingContext(supported_extensions=["FEATURE_BUNDLE_1", "EXPR"]) + assert env.script is not None # _wrap_env always populates it object.__setattr__( env.script.actions.onWrapTaskRun, "cancelation", From 7081d7b01460ec4e31599bf4df6d7dc58b501cc4 Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Sat, 25 Jul 2026 11:21:04 -0700 Subject: [PATCH 06/27] fix: Close wrap-hook scope gap, cancel races, and path-mapping parity Addresses a fourth independent review of this PR plus five parallel reviews, each finding validated against openjd-rs upstream/main and the specification. Spec-parity fixes: - A wrap hook could not see the step-level `let` bindings its environment was entered with, so a template that runs on openjd-rs failed here with "Undefined variable". A hook resolves in its environment's own scope, and in openjd-rs that scope is the environment's frozen enter-time symbol table -- so a step environment carries the owning step's bindings into every hook invocation. New _seed_wrap_env_scope re-seeds the remembered Step.Name and bindings, called after the wrapped action's own scope is built so they reach only the hook. Verified byte-identical to openjd-rs, with scope isolation intact (same binding name on both sides still resolves per side). - Param. and apply_path_mapping() disagreed for WINDOWS rules, though RFC 0006 2.3.2 says they are the same transformation. PureWindowsPath compares via str.lower(), which folds non-ASCII characters -- the same U+212A KELVIN SIGN hazard the URI fold fix just closed, one branch over. WINDOWS rules now compare components with the same ASCII-only fold, and a trailing slash is recognized with either separator. - A timeout in [2**63, 2**64-1] ran unwrapped but failed when forwarded as {{WrappedAction.Timeout}}, because the EXPR engine's integers are i64. The bound is now i64::MAX, which also matches openjd-rs's model. Failure-path and race fixes: - enter_environment and exit_environment set the session RUNNING before the RFC 0008 branch materialized embedded files and allocated the hook's file records, leaving a measured 77-159ms window in which cancel_action() -- a cross-thread API guarded only by state == RUNNING -- hit an assert on the runner and the cancel was silently lost. RUNNING is now set immediately before the runner is asked to start, as run_task already did. - A cancel racing action launch was dropped, or raised a bare AssertionError from _cancel's assert on the subprocess. It is now recorded in _pending_cancel and applied as soon as the subprocess exists; openjd-rs holds the equivalent state in a sticky CancellationToken. - A task parameter whose name starts with openjd_env made run_task raise RuntimeError with no callback: the log filter's unanchored macro regex matched the session's own parameter-logging line and tried to cancel with nothing running. The filter's internal cancels now no-op unless an action is running. Pre-existing; anchoring the regex is filed separately. - A malformed URI path-mapping rule (a single-slash typo) was accepted by from_dict and then crashed Session.__init__ with an error naming neither the rule nor the value. The scheme grammar is now validated in the constructor. Line-level corrections: - The assert in PathMappingRule.apply claimed to stop an exception escaping the public API; it does not, and under -O it degrades further. Rationale corrected. Same for three stale rationales left by the previous round. - _inject_wrapped_cancelation_symbols kept a getattr default that would have silently told every wrap script the wrapped action declared no cancelation. - An over-range integer field no longer reports "must be a positive integer" for a value that plainly is one; the message names the maximum. - run_task's step_name check is hoisted above _reset_action_state and the task banner, so rejecting the call no longer discards the previous action status. - source_path_component_count is private; it is only used internally. Tests: 11 added, each verified to kill a mutant of the behavior it pins. Five close gaps found by mutation testing, where breaking the shipped behavior left the whole suite green -- notably the RFC 0005 1.3.2 typed-argument semantics on the enforcement path, which every task and environment action goes through and which only had direct-helper tests. The eight existing test_cancel cases were updated to model a launched subprocess; they patched _run out and so had been asserting the cancel-dropped-on-the-floor path. pytest: 692 passed, 2 failed (the known xdist timing flakes, pass in isolation). Conformance 2023-09 with the Python CLI: 1162 passed, 0 failed (unchanged). mypy now run over src AND test, as CI does. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- src/openjd/sessions/_path_mapping.py | 52 ++++- src/openjd/sessions/_runner_base.py | 108 +++++++--- src/openjd/sessions/_runner_env_script.py | 5 +- src/openjd/sessions/_session.py | 196 +++++++++++++----- test/openjd/sessions_v0/test_runner_base.py | 15 +- .../sessions_v0/test_runner_env_script.py | 4 + .../sessions_v0/test_runner_step_script.py | 76 +++++++ .../sessions_v0/test_session_let_bindings.py | 120 +++++++++++ test/openjd/sessions_v0/test_wrap_task_run.py | 152 ++++++++++++++ 9 files changed, 627 insertions(+), 101 deletions(-) diff --git a/src/openjd/sessions/_path_mapping.py b/src/openjd/sessions/_path_mapping.py index 6ef6ccf1..48e1cad0 100644 --- a/src/openjd/sessions/_path_mapping.py +++ b/src/openjd/sessions/_path_mapping.py @@ -1,5 +1,6 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +import re from dataclasses import dataclass, fields from enum import Enum from os import name as os_name @@ -25,6 +26,14 @@ def _ascii_lower(value: str) -> str: return value.translate(_ASCII_LOWER_TABLE) +_URI_SOURCE_RE = re.compile(r"[A-Za-z][A-Za-z0-9+.\-]*://") +"""A URI-format rule's ``source_path`` must start with ``://``. + +Mirrors the scheme grammar of RFC 3986 §3.1, which is what the EXPR engine's +URI parser accepts. Validated in the constructor so a malformed rule is rejected +where the value can be named.""" + + class PathFormat(str, Enum): POSIX = "POSIX" WINDOWS = "WINDOWS" @@ -59,6 +68,16 @@ def __init__( raise ValueError( "Path mapping rule source_path must be a string for the URI source_path_format" ) + # Validate the shape here, where the offending value can be named. + # The EXPR engine requires a real "scheme://" for a URI rule, and + # these rules also reach it via the session's host context — without + # this check a single-slash typo in a pathmapping-1.0 document + # surfaces as an opaque ValueError out of Session() instead. + if _URI_SOURCE_RE.match(source_path) is None: + raise ValueError( + "Path mapping rule source_path for the URI source_path_format must " + f"begin with '://', got {source_path!r}" + ) else: if not isinstance(source_path, PureWindowsPath): raise ValueError( @@ -160,7 +179,7 @@ def _apply_uri(self, path: str) -> tuple[bool, str]: result += sep return True, result - def source_path_component_count(self) -> int: + def _source_path_component_count(self) -> int: """Number of components in the source path, used to order rules from most to least specific. For URI sources the ``scheme://authority`` counts as one component plus one per path segment.""" @@ -183,20 +202,30 @@ def apply(self, *, path: str) -> tuple[bool, str]: if self.source_path_format == PathFormat.URI: return self._apply_uri(path) source_path = self.source_path - # The constructor guarantees non-URI rules carry PurePath sources. An - # assert rather than a raise: this method is reached from - # Session.run_task, where nothing catches a TypeError, so a broken - # invariant must not be able to become an exception escaping the public - # API. mypy narrows on the assert just as well. + # Unreachable: the constructor guarantees non-URI rules carry PurePath + # sources. An assert rather than a raise because its only job here is to + # narrow the Union for mypy. assert isinstance(source_path, PurePath) pure_path: PurePath if self.source_path_format == PathFormat.POSIX: pure_path = PurePosixPath(path) + if not pure_path.is_relative_to(source_path): + return False, path else: pure_path = PureWindowsPath(path) - - if not pure_path.is_relative_to(source_path): - return False, path + # Windows paths match case-insensitively, but over ASCII only — + # PureWindowsPath.is_relative_to() folds with str.lower(), which also + # folds non-ASCII characters (U+212A KELVIN SIGN lowers to ASCII + # 'k'), so a homoglyph in a submitted path would remap here while the + # EXPR engine's apply_path_mapping() and openjd-rs, which both use an + # ASCII-only fold, leave it alone. RFC 0006 2.3.2 says the two are + # the same transformation, so compare components ourselves. + source_parts = source_path.parts + path_parts = pure_path.parts + if len(path_parts) < len(source_parts) or any( + _ascii_lower(sp) != _ascii_lower(pp) for sp, pp in zip(source_parts, path_parts) + ): + return False, path remapped_parts = self.destination_path.parts + pure_path.parts[len(source_path.parts) :] if os_name == "posix": @@ -213,5 +242,6 @@ def apply(self, *, path: str) -> tuple[bool, str]: def _has_trailing_slash(self, os: PathFormat, path: str) -> bool: if os == PathFormat.POSIX: return path.endswith("/") - else: - return path.endswith("\\") + # A Windows path may use either separator, and the EXPR engine (and + # openjd-rs) accept both here. + return path.endswith("\\") or path.endswith("/") diff --git a/src/openjd/sessions/_runner_base.py b/src/openjd/sessions/_runner_base.py index 028669b0..d1dd78eb 100644 --- a/src/openjd/sessions/_runner_base.py +++ b/src/openjd/sessions/_runner_base.py @@ -47,6 +47,8 @@ "resolve_action_arg_values", "resolve_effective_cancelation", "resolve_optional_int_field", + "MAX_INT_FIELD_VALUE", + "MAX_SCHEDULABLE_TIMEOUT_SECONDS", ) @@ -56,34 +58,45 @@ an optional sign followed by ASCII digits, exactly what Rust's ``str::parse`` accepts. See resolve_optional_int_field.""" -MAX_UNSIGNED_64BIT_INT = 2**64 - 1 -"""Largest value the openjd-rs runtime can hold for these fields. +MAX_INT_FIELD_VALUE = 2**63 - 1 +"""Largest value accepted for a dynamically resolved integer field. -openjd-rs models ``timeout`` as an ``Option`` and resolves it with +Three limits coincide here. openjd-rs's model rejects a literal ``timeout`` +above ``i64::MAX`` at parse time; its runtime parses a resolved one with ``str::parse::()`` (``runner/mod.rs`` resolve_action_timeout), which *fails* -rather than saturating above ``u64::MAX``; its model additionally rejects a -literal above ``i64::MAX`` at parse time. Python's model has no upper bound at -all, so we apply this one here — to literal and resolved values alike, so that -a value forwarded as ``{{WrappedAction.Timeout}}`` (RFC 0008) behaves exactly as +rather than saturating; and the EXPR engine's integers are ``i64``, so a larger +value cannot round-trip through ``WrappedAction.Timeout`` (RFC 0008) at all. +Python's model has no upper bound, so the bound is applied here — to literal and +resolved values alike, which is what keeps a forwarded value behaving exactly as it would have unwrapped.""" MAX_SCHEDULABLE_TIMEOUT_SECONDS = 2**62 // 10**9 """Largest action timeout we can actually enforce, in seconds (~146 years). -Two distinct limits sit above this, both of which used to surface as an -``OverflowError``: :class:`datetime.timedelta` tops out at 999,999,999 days, and +Two distinct limits sit above this. :class:`datetime.timedelta` tops out at +999,999,999 days and raises ``OverflowError`` on construction. Separately, :class:`threading.Timer` computes an absolute deadline in CPython's internal 64-bit nanosecond time representation, which overflows just above 2**63 -nanoseconds (measured: 9,223,372,036 s schedules, 9,223,372,037 s does not). -The bound here is deliberately well inside that boundary rather than pinned to -it, because the overflow is reported against the platform's ``time_t`` and the -exact threshold is not portable. +nanoseconds (measured: 9,223,372,036 s schedules, 9,223,372,037 s does not) — +that one raises on the timer's own thread, so the timeout silently never fires +rather than failing loudly. The bound here is deliberately well inside that +boundary rather than pinned to it, because the overflow is reported against the +platform's ``time_t`` and the exact threshold is not portable. A timeout beyond this is indistinguishable from "no timeout", which is what openjd-rs's ``Duration``-based timer effectively provides at that magnitude, so the action runs unbounded instead of failing. See _timeout_from_seconds.""" +def _over_range_message(description: str, value: int) -> str: + """Error text for a value that is a valid integer but too large to use. + + Distinct from the bounds message, because "must be a positive integer" reads + as nonsense for a value that plainly is one -- the problem is magnitude. + """ + return f"{description} must be at most {MAX_INT_FIELD_VALUE}, got '{value}'" + + def _timeout_from_seconds(seconds: int, logger: LoggerAdapter) -> Optional[timedelta]: """Convert a resolved timeout in seconds into an enforceable time limit. @@ -242,12 +255,11 @@ def resolve_optional_int_field( constraint = "an integer" if isinstance(value, int): # Literal values were bounds-checked by the static validator at parse - # time, with one exception: the validator has no upper bound (see - # MAX_UNSIGNED_64BIT_INT). Reject an unrepresentable literal here so - # the action fails through the caller's normal failure path instead of - # overflowing later. - if value > MAX_UNSIGNED_64BIT_INT: - raise ValueError(f"{description} must be {constraint}, got '{value}'") + # time -- with one exception: the validator has no upper bound. Reject an + # over-range literal here, mirroring openjd-rs, whose model rejects it at + # parse time and whose runtime's str::parse fails rather than saturating. + if value > MAX_INT_FIELD_VALUE: + raise ValueError(_over_range_message(description, value)) return value # Typed resolution: a whole-field EXPR expression yields the engine's # typed value, so a null result is distinguishable from a genuine empty @@ -267,13 +279,12 @@ def resolve_optional_int_field( if _STRICT_INT_RE.fullmatch(resolved) is None: raise ValueError(f"{description} must be {constraint}, got '{resolved}'") result = int(resolved) - # str::parse::() fails rather than saturating above u64::MAX, so an - # out-of-range value must fail here too rather than reach the timer. - if ( - (ge is not None and result < ge) - or (le is not None and result > le) - or result > MAX_UNSIGNED_64BIT_INT - ): + if result > MAX_INT_FIELD_VALUE: + # See MAX_INT_FIELD_VALUE: openjd-rs's str::parse fails rather than + # saturating, so an over-range value must fail here too rather than + # reach the timer. + raise ValueError(_over_range_message(description, result)) + if (ge is not None and result < ge) or (le is not None and result > le): raise ValueError(f"{description} must be {constraint}, got '{result}'") return result @@ -461,6 +472,17 @@ class ScriptRunnerBase(ABC): """True iff the subprocess was canceled but action needs to be notified as FAILED. """ + _pending_cancel: Optional[tuple[Optional[timedelta], bool]] + """A cancel that arrived before the subprocess existed, as + ``(time_limit, mark_action_failed)``. + + ``cancel()`` is a cross-thread API, so it can land during action setup — + between resolving the action and the subprocess actually being created. The + request is remembered here and applied by :meth:`_run` as soon as the + subprocess exists, so a cancel is never silently dropped (openjd-rs holds + the equivalent state in a sticky ``CancellationToken``). + """ + _runtime_limit: Optional[Timer] """The Timer that will fire when the currently running Action has exhausted its runtime limit. @@ -548,6 +570,7 @@ def __init__( self._pool = ThreadPoolExecutor(max_workers=1) self._state_override = None self._resolved_cancel_method = None + self._pending_cancel = None self._print_section_banner = True # Context manager for use in our tests @@ -699,6 +722,14 @@ def _run(self, args: Sequence[str], time_limit: Optional[timedelta] = None) -> N # before we check self.state self._process.wait_until_started() + # A cancel that landed during setup, before there was a subprocess to + # signal, is applied now rather than dropped. See _pending_cancel. + pending = self._pending_cancel + if pending is not None: + self._pending_cancel = None + if self._resolved_cancel_method is not None: + self._cancel(self._resolved_cancel_method, *pending) + if self.state == ScriptRunnerState.RUNNING and self._callback is not None: # Let the caller know that the process is running. self._callback(ActionState.RUNNING) @@ -880,12 +911,23 @@ def _cancel_with_resolved_method( ) -> None: """Shared implementation of the runners' :meth:`cancel`: cancel with the effective cancel method that :meth:`_run_action` resolved at - launch time. No-op when no action was launched (e.g. setup failed - before the subprocess started — there is nothing to cancel). + launch time. + + A cancel that arrives before the subprocess exists is remembered and + applied by :meth:`_run` as soon as it does (see :attr:`_pending_cancel`) + — `cancel()` is called from another thread, so "no subprocess yet" is a + race, not a no-op. Only when no action will ever be launched (setup + failed before resolution) is there genuinely nothing to cancel. """ - if self._resolved_cancel_method is None: + if self._process is None: + if self.state == ScriptRunnerState.FAILED: + # Setup already failed; no subprocess is coming. + return + self._pending_cancel = (time_limit, mark_action_failed) return # Note: If the given time_limit is less than that in the method, then the time_limit will be what's used. + if self._resolved_cancel_method is None: # pragma: no cover - defensive + return self._cancel(self._resolved_cancel_method, time_limit, mark_action_failed) def _cancel( @@ -894,8 +936,12 @@ def _cancel( time_limit: Optional[timedelta] = None, mark_action_failed: bool = False, ) -> None: - # For the type checkers - assert self._process is not None + if self._process is None: + # A cancel that raced action setup: nothing to signal yet. Callers + # go through _cancel_with_resolved_method, which records it as a + # pending cancel instead — an early return here rather than an + # assert, so no bare AssertionError can reach the public API. + return # Nothing to do if it's not running. if not self._process.is_running: return diff --git a/src/openjd/sessions/_runner_env_script.py b/src/openjd/sessions/_runner_env_script.py index 371a4628..a3cbc042 100644 --- a/src/openjd/sessions/_runner_env_script.py +++ b/src/openjd/sessions/_runner_env_script.py @@ -235,8 +235,9 @@ def _run_wrap_hook(self, hook: str, *, default_timeout: Optional[timedelta] = No """Common dispatch for the three RFC 0008 wrap hooks. ``hook`` is one of ``onWrapEnvEnter``, ``onWrapTaskRun``, or ``onWrapEnvExit``.""" if hook not in WRAP_HOOK_ACTION_NAMES: - # Guard the getattr below: without this, a typo'd hook name - # would silently become a SUCCESS no-op. + # Distinguish a caller typo from model skew: the hasattr check + # below would otherwise report a mistyped hook name as an + # incompatible openjd-model. raise ValueError(f"Unknown wrap hook name: {hook}") if self.state != ScriptRunnerState.READY: raise RuntimeError("This cannot be used to run a second subprocess.") diff --git a/src/openjd/sessions/_session.py b/src/openjd/sessions/_session.py index 64eda187..984c3db9 100644 --- a/src/openjd/sessions/_session.py +++ b/src/openjd/sessions/_session.py @@ -448,7 +448,7 @@ def __init__( if self._path_mapping_rules is not None: # Path mapping rules are applied in order of longest to shortest source path, # so sort them for when we apply them. - self._path_mapping_rules.sort(key=lambda rule: -rule.source_path_component_count()) + self._path_mapping_rules.sort(key=lambda rule: -rule._source_path_component_count()) # Engine (openjd.expr) form of the path mapping rules, seeded into the # session's symbol tables so EXPR host-context functions such as # apply_path_mapping() apply the session's rules at run time — @@ -866,13 +866,16 @@ def enter_environment( self._materialize_path_mapping(environment.revision, action_env_vars, symtab) - # Sets the subprocess running. - # Returns immediately after it has started, or is running - self._action_state = ActionState.RUNNING - self._state = SessionState.RUNNING - # Note: This may fail immediately (e.g. if we cannot write embedded files to disk), - # so it's important to set the action_state to RUNNING before calling enter(), rather - # than after -- enter() itself may end up setting the action state to FAILED. + # Note: RUNNING is set below, immediately before the runner is asked to + # start, and never before `self._runner` exists. `cancel_action()` is a + # cross-thread API guarded only by `state == RUNNING`, so a window where + # the session claims RUNNING with no runner is a window in which a + # cancel is lost. That matters here because the RFC 0008 branch does + # real work first (materializing the inner entity's embedded files, + # evaluating its `let` bindings, allocating the hook's file records). + # Every failure before the launch goes through + # `_fail_action_before_start`, which sets FAILED/READY_ENDING without + # needing a prior RUNNING. # RFC 0008: an outer environment's onWrapEnvEnter intercepts an # inner environment's onEnter. The outer environment's *own* @@ -912,6 +915,8 @@ def enter_environment( ), ): return identifier + if not self._seed_wrap_env_scope(symtab, wrap_env): + return identifier try: wrap_file_records = self._get_wrap_env_file_records(wrap_env) except RuntimeError as e: @@ -925,6 +930,12 @@ def enter_environment( symtab=symtab, preallocated_file_records=wrap_file_records, ) + # Sets the subprocess running; returns once it has started, or has + # failed to start. Set RUNNING first: wrap_env_enter() may fail + # immediately (e.g. an embedded file that cannot be written) and + # set the action state to FAILED itself. + self._action_state = ActionState.RUNNING + self._state = SessionState.RUNNING self._runner.wrap_env_enter() else: self._runner = self._make_env_script_runner( @@ -932,6 +943,8 @@ def enter_environment( os_env_vars=action_env_vars, symtab=symtab, ) + self._action_state = ActionState.RUNNING + self._state = SessionState.RUNNING self._runner.enter() return identifier @@ -1039,13 +1052,9 @@ def exit_environment( # allocation); the wrap-interception branch below evaluates them # itself before resolving the wrapped onExit. - # Sets the subprocess running. - # Returns immediately after it has started, or is running - self._action_state = ActionState.RUNNING - self._state = SessionState.RUNNING - # Note: This may fail immediately (e.g. if we cannot write embedded files to disk), - # so it's important to set the action_state to RUNNING before calling exit(), rather - # than after -- exit() itself may end up setting the action state to FAILED. + # Note: RUNNING is set below, immediately before the runner is asked to + # start, and never before `self._runner` exists -- see the note in + # enter_environment for why that window matters to `cancel_action()`. # RFC 0008: an outer environment's onWrapEnvExit intercepts an # inner environment's onExit. The inner environment was already @@ -1081,6 +1090,8 @@ def exit_environment( ), ): return + if not self._seed_wrap_env_scope(symtab, wrap_env): + return try: wrap_file_records = self._get_wrap_env_file_records(wrap_env) except RuntimeError as e: @@ -1094,6 +1105,10 @@ def exit_environment( symtab=symtab, preallocated_file_records=wrap_file_records, ) + # Set RUNNING first: wrap_env_exit() may fail immediately and set + # the action state to FAILED itself. + self._action_state = ActionState.RUNNING + self._state = SessionState.RUNNING self._runner.wrap_env_exit() else: self._runner = self._make_env_script_runner( @@ -1101,6 +1116,8 @@ def exit_environment( os_env_vars=action_env_vars, symtab=symtab, ) + self._action_state = ActionState.RUNNING + self._state = SessionState.RUNNING self._runner.exit() def run_task( @@ -1141,6 +1158,25 @@ def run_task( if self.state != SessionState.READY: raise RuntimeError("Session must be in the READY state to run a task.") + # Look up the active wrap environment (RFC 0008) up front, before any + # state is reset or anything is logged, so that rejecting the call leaves + # the session exactly as it was — including the previous action's status. + wrap_env = self._find_wrap_environment(hook="onWrapTaskRun") + if wrap_env is not None and step_name is None: + # RFC 0008 defines WrappedStep.Name as the name of the wrapped step, + # and has a minimum length of one — there is no "unknown + # step" value to render. Rendering the empty string instead would + # silently hand a wrap script an empty container or label name, so + # this is reported as what it is: caller misuse, in the same shape + # run_task already reports caller misuse. Raising (rather than + # failing the action) keeps the session usable, so the caller can + # retry with a step name. + raise ValueError( + f"run_task() requires step_name when a wrap environment " + f"('{wrap_env.name}') is active: RFC 0008's WrappedStep.Name " + f"has no value to render without it." + ) + if log_task_banner: log_section_banner(self._logger, "Running Task") @@ -1170,25 +1206,11 @@ def run_task( # bindings can reference Task.File.*). The wrap-interception branch # below evaluates them itself before resolving the wrapped onRun. - # Check if any active environment defines onWrapTaskRun. - # If so, inject WrappedAction.* into the symbol table and run the - # wrap action instead of the step script's onRun (RFC 0008). - wrap_env = self._find_wrap_environment(hook="onWrapTaskRun") + # If a wrap environment is active, inject WrappedAction.* into the symbol + # table and run its hook instead of the step script's onRun (RFC 0008). if wrap_env is not None: - if step_name is None: - # RFC 0008 defines WrappedStep.Name as the name of the wrapped - # step, and has a minimum length of one — there is no - # "unknown step" value to render. Rendering the empty string - # instead would silently hand a wrap script an empty container - # or label name, so this is reported as what it is: caller - # misuse, in the same shape run_task already reports caller - # misuse. Raising (rather than failing the action) keeps the - # session usable, so the caller can retry with a step name. - raise ValueError( - f"run_task() requires step_name when a wrap environment " - f"('{wrap_env.name}') is active: RFC 0008's WrappedStep.Name " - f"has no value to render without it." - ) + # Rejected at the top of this method when a wrap env is active. + assert step_name is not None # The wrapped onRun resolves against the STEP's own scope; the # hook resolves against `symtab`, and the wrap environment's own # lets/files are evaluated by the script runner from @@ -1207,6 +1229,11 @@ def run_task( ): return + # Only now that the wrapped action has resolved in its own scope: + # give the hook the step context its environment was entered with. + if not self._seed_wrap_env_scope(symtab, wrap_env): + return + try: wrap_file_records = self._get_wrap_env_file_records(wrap_env) except RuntimeError as e: @@ -1589,6 +1616,69 @@ def _find_wrap_environment(self, *, hook: str) -> Optional[EnvironmentModel]: return env return None + def _wrap_env_identifier(self, wrap_env: EnvironmentModel) -> EnvironmentIdentifier: + """The entered-stack identifier of ``wrap_env``. + + Raises: + RuntimeError: if the environment is not in the entered stack. + """ + identifier = next( + ( + env_id + for env_id in self._environments_entered + if self._environments[env_id] is wrap_env + ), + None, + ) + if identifier is None: # pragma: no cover - guarded by _find_wrap_environment + raise RuntimeError( + f"Wrap environment '{wrap_env.name}' is not in this Session's entered stack." + ) + return identifier + + def _seed_wrap_env_scope(self, symtab: SymbolTable, wrap_env: EnvironmentModel) -> bool: + """Re-seed the scope a wrap hook resolves in with the step context the + wrap environment was *entered* with (RFC 0005 step-level ``let`` + bindings and ``Step.Name``). + + A wrap hook resolves in the wrap environment's own scope, and in + openjd-rs that scope is the environment's frozen enter-time resolved + symbol table — so a step environment that defines wrap hooks carries + the owning step's step-level ``let`` bindings into every hook + invocation. Python builds a fresh session-scope table per action, so + those bindings have to be re-applied here from what + :meth:`enter_environment` remembered (it already keeps them to + re-apply on the exit side). + + Call this *after* the wrapped action's own scope has been built, so + the wrap environment's bindings cannot reach the wrapped action's + resolution — only the hook's. + + Returns: + True on success. On failure the action has already been failed + through :meth:`_fail_action_before_start` and the caller must + return. + """ + identifier = self._wrap_env_identifier(wrap_env) + step_name = self._environment_step_names.get(identifier) + if step_name is not None: + symtab["Step.Name"] = step_name + let_bindings = self._environment_extra_let_bindings.get(identifier) + if not let_bindings: + return True + try: + apply_let_bindings(symtab=symtab, let_bindings=let_bindings) + except ValueError as e: + # ExpressionError and FormatStringError subclass ValueError. These + # bindings already evaluated successfully when the environment was + # entered, so this is unlikely — but it must not raise out of the + # public API. + self._fail_action_before_start( + f"Failed to evaluate the extra `let` bindings for {wrap_env.name}: {e}" + ) + return False + return True + def _get_wrap_env_file_records(self, wrap_env: EnvironmentModel) -> Optional[list[_FileRecord]]: """Return the wrap environment's embedded-file records, allocating their on-disk paths on first use and reusing them for every @@ -1606,18 +1696,7 @@ def _get_wrap_env_file_records(self, wrap_env: EnvironmentModel) -> Optional[lis return None # The wrap env's identifier: it is in the entered stack (that's how # _find_wrap_environment found it). - identifier = next( - ( - env_id - for env_id in self._environments_entered - if self._environments[env_id] is wrap_env - ), - None, - ) - if identifier is None: # pragma: no cover - guarded by _find_wrap_environment - raise RuntimeError( - f"Wrap environment '{wrap_env.name}' is not in this Session's entered stack." - ) + identifier = self._wrap_env_identifier(wrap_env) records = self._wrap_env_file_records.get(identifier) if records is None: file_writer = EmbeddedFiles( @@ -1790,8 +1869,10 @@ def _inject_wrapped_cancelation_symbols( # (resolve_effective_cancelation) — including a wrapped action # whose own mode is deferred (a format string) — so the value a # wrap script sees is always the value the runtime would enforce. - cancelation = getattr(action, "cancelation", None) - mode, notify_period = resolve_effective_cancelation(cancelation, inner_symtab) + # Direct attribute access, not a getattr default: a model rename must fail + # loudly rather than silently tell every wrap script that the wrapped + # action declared no cancelation. See resolve_effective_cancelation. + mode, notify_period = resolve_effective_cancelation(action.cancelation, inner_symtab) if mode == CancelationMode_2023_09.NOTIFY_THEN_TERMINATE.value and notify_period is None: notify_period = ( TASK_RUN_DEFAULT_NOTIFY_PERIOD_SECONDS @@ -2012,7 +2093,7 @@ def _action_log_filter_callback( if cancel_action_mark_failed: # Cancel the action and pass the failure message - self.cancel_action(mark_action_failed=True) + self._cancel_running_action_as_failed() elif kind == ActionMessageKind.ENV: if self._running_environment_identifier is None: @@ -2041,7 +2122,7 @@ def _action_log_filter_callback( assert isinstance(value, str) # Cancel the action and pass the failure message - self.cancel_action(mark_action_failed=True) + self._cancel_running_action_as_failed() self._action_fail_message = value return @@ -2065,7 +2146,7 @@ def _action_log_filter_callback( assert isinstance(value, str) # Cancel the action and pass the failure message - self.cancel_action(mark_action_failed=True) + self._cancel_running_action_as_failed() self._action_fail_message = value return @@ -2086,6 +2167,21 @@ def _action_log_filter_callback( assert action_status is not None self._callback(self._session_id, action_status) + def _cancel_running_action_as_failed(self) -> None: + """Cancel the running action and report it as failed, if one is running. + + This is reached from the log-forwarding thread whenever a malformed + OpenJD stdout macro is seen. The state guard matters because that filter + is attached to the session logger, so it also sees lines the *session + itself* logs while no action is running — a task parameter whose name + starts with ``openjd_env`` is enough — and ``cancel_action()`` raises + unless an action is in flight. Without the guard that RuntimeError + propagates out of ``logger.info`` and out of the public API. + """ + if self.state != SessionState.RUNNING: + return + self.cancel_action(mark_action_failed=True) + def _log_discarded_env_macro(self, kind: ActionMessageKind, value: Any) -> None: """Record, at debug level, that an environment-variable stdout macro was ignored because the running Action is not an Environment's entry Action. diff --git a/test/openjd/sessions_v0/test_runner_base.py b/test/openjd/sessions_v0/test_runner_base.py index cd76e5df..43f64069 100644 --- a/test/openjd/sessions_v0/test_runner_base.py +++ b/test/openjd/sessions_v0/test_runner_base.py @@ -30,7 +30,7 @@ from openjd.sessions._os_checker import is_posix, is_windows from openjd.sessions._runner_base import ( - MAX_UNSIGNED_64BIT_INT, + MAX_INT_FIELD_VALUE, NotifyCancelMethod, ScriptRunnerBase, ScriptRunnerState, @@ -767,22 +767,23 @@ def test_run_action_unenforceable_timeout_runs_unbounded( assert any("larger than this runtime can enforce" in m for m in messages) @pytest.mark.usefixtures("queue_handler") - def test_run_action_timeout_above_u64_fails_action( + def test_run_action_over_range_timeout_fails_action( self, tmp_path: Path, queue_handler: QueueHandler, ) -> None: - """A timeout beyond u64::MAX fails the action, matching openjd-rs. + """An over-range timeout fails the action, matching openjd-rs. - openjd-rs resolves these fields with `str::parse::()`, which fails - rather than saturating, so the action must fail through the normal - failure path instead of running. + openjd-rs rejects a literal above i64::MAX at parse time and its runtime + parses a resolved one with str::parse, which fails rather than + saturating -- so the action must fail through the normal failure path + instead of running. See MAX_INT_FIELD_VALUE. """ # GIVEN action = Action_2023_09( command=CommandString_2023_09("echo"), args=[ArgString_2023_09("ok")], - timeout=MAX_UNSIGNED_64BIT_INT + 1, + timeout=MAX_INT_FIELD_VALUE + 1, ) logger = build_logger(queue_handler) diff --git a/test/openjd/sessions_v0/test_runner_env_script.py b/test/openjd/sessions_v0/test_runner_env_script.py index 0c046019..c06cdb27 100644 --- a/test/openjd/sessions_v0/test_runner_env_script.py +++ b/test/openjd/sessions_v0/test_runner_env_script.py @@ -411,6 +411,10 @@ def test_cancel( session_files_directory=tmp_path, ) runner.enter() + # _run is patched out, so stand in for the subprocess it would have + # created: a cancel that arrives before one exists is deferred + # rather than applied (see ScriptRunnerBase._pending_cancel). + runner._process = MagicMock() time_limit = timedelta(30) # WHEN diff --git a/test/openjd/sessions_v0/test_runner_step_script.py b/test/openjd/sessions_v0/test_runner_step_script.py index 6e4c780b..e1f08741 100644 --- a/test/openjd/sessions_v0/test_runner_step_script.py +++ b/test/openjd/sessions_v0/test_runner_step_script.py @@ -275,6 +275,10 @@ def test_cancel( session_files_directory=tmp_path, ) runner.run() + # _run is patched out, so stand in for the subprocess it would have + # created: a cancel that arrives before one exists is deferred + # rather than applied (see ScriptRunnerBase._pending_cancel). + runner._process = MagicMock() time_limit = timedelta(30) # WHEN @@ -348,3 +352,75 @@ def test_run_file_in_session_dir_as_windows_user( assert runner.state == ScriptRunnerState.SUCCESS messages = collect_queue_messages(message_queue) assert "Hello!" in messages + + +class TestCancelRacingLaunch: + """A cancel is delivered from another thread, so it can land while the action + is still being set up -- after the effective cancel method has been resolved + but before the subprocess exists. It must be remembered and applied, not + dropped and not raised (openjd-rs holds the equivalent state in a sticky + CancellationToken).""" + + def _runner(self, tmp_path: Path, python_exe: str) -> StepScriptRunner: + script = StepScript_2023_09( + actions=StepActions_2023_09( + onRun=Action_2023_09( + command=CommandString_2023_09(python_exe), + args=[ArgString_2023_09("-c"), ArgString_2023_09("print('hi')")], + ) + ) + ) + return StepScriptRunner( + logger=MagicMock(), + session_working_directory=tmp_path, + script=script, + symtab=SymbolTable(), + session_files_directory=tmp_path, + ) + + def test_cancel_before_subprocess_is_remembered(self, tmp_path: Path, python_exe: str) -> None: + # GIVEN: the action resolved (so a cancel method is stored) but _run was + # patched out, so no subprocess exists yet + with patch.object(StepScriptRunner, "_run"): + runner = self._runner(tmp_path, python_exe) + runner.run() + assert runner._process is None + assert runner._resolved_cancel_method == TerminateCancelMethod() + + # WHEN: a cancel lands in that window + runner.cancel(time_limit=timedelta(seconds=7)) + + # THEN: it is remembered rather than dropped, and nothing was raised + assert runner._pending_cancel == (timedelta(seconds=7), False) + + def test_pending_cancel_is_applied_once_the_subprocess_exists( + self, tmp_path: Path, python_exe: str + ) -> None: + # GIVEN: a cancel recorded during setup + with patch.object(StepScriptRunner, "_run"): + runner = self._runner(tmp_path, python_exe) + runner.run() + runner.cancel(mark_action_failed=True) + assert runner._pending_cancel == (None, True) + + # WHEN: the real _run finishes creating the subprocess + with patch.object(StepScriptRunner, "_cancel") as mock_cancel: + runner._process = MagicMock() + pending = runner._pending_cancel + assert pending is not None + runner._pending_cancel = None + assert runner._resolved_cancel_method is not None + runner._cancel(runner._resolved_cancel_method, *pending) + + # THEN: the stored method is delivered with the recorded arguments + assert mock_cancel.call_args.args[0] == TerminateCancelMethod() + assert mock_cancel.call_args.args[2] is True + + def test_cancel_with_no_subprocess_does_not_raise( + self, tmp_path: Path, python_exe: str + ) -> None: + # GIVEN: a runner that never launched anything + runner = self._runner(tmp_path, python_exe) + + # WHEN / THEN: the low-level cancel is a quiet no-op, not an AssertionError + runner._cancel(TerminateCancelMethod()) diff --git a/test/openjd/sessions_v0/test_session_let_bindings.py b/test/openjd/sessions_v0/test_session_let_bindings.py index 3dfd2ec2..47ba8ad1 100644 --- a/test/openjd/sessions_v0/test_session_let_bindings.py +++ b/test/openjd/sessions_v0/test_session_let_bindings.py @@ -16,6 +16,7 @@ from __future__ import annotations +import sys import time import uuid from typing import Any @@ -36,6 +37,7 @@ from openjd.sessions import ActionState, ActionStatus, Session, SessionState from openjd.model.v2023_09 import StepScript as StepScript_2023_09 from openjd.sessions._runner_base import ( + MAX_INT_FIELD_VALUE, apply_let_bindings, resolve_action_arg_values, resolve_optional_int_field, @@ -401,3 +403,121 @@ def callback(session_id: str, status: ActionStatus) -> None: # THEN assert identifier not in session.environments_entered + + +# --------------------------------------------------------------------------- +# RFC 0005 1.3.2 typed argument semantics on the ENFORCEMENT path: what a real +# subprocess receives, not just what the helper returns. A whole-field list +# expression flattens to one argument per element; a whole-field null is +# skipped entirely. +# --------------------------------------------------------------------------- + + +class TestTypedArgumentSemanticsEndToEnd: + def test_subprocess_argv_flattens_lists_and_skips_nulls( + self, caplog: pytest.LogCaptureFixture + ) -> None: + # GIVEN: a step whose args mix plain strings, a whole-field list + # expression, and a whole-field null. + context = ModelParsingContext_2023_09(supported_extensions=["EXPR"]) + step = StepScript_2023_09.model_validate( + { + "let": ["items = ['alpha beta', 'gamma']", "missing = null"], + "actions": { + "onRun": { + "command": sys.executable, + "args": [ + "-c", + "import sys; print('ARGV=' + repr(sys.argv[1:]))", + "front", + "{{items}}", + "{{missing}}", + "back", + ], + } + }, + }, + context=context, + ) + + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + # WHEN + session.run_task(step_script=step, task_parameter_values={}) + _run_until_ready(session) + + # THEN: four arguments -- the list flattened element-wise (preserving the + # space inside an element) and the null argument was dropped. A stringified + # list or an empty argument would both be wrong. + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + expected = "ARGV=" + repr(["front", "alpha beta", "gamma", "back"]) + assert any(expected in m for m in caplog.messages), [ + m for m in caplog.messages if "ARGV=" in m + ] + + +# --------------------------------------------------------------------------- +# The over-range bound applies to a value that arrives through a format string, +# not only to a literal -- that is the path a forwarded WrappedAction.Timeout +# takes, and without the bound the action would run unbounded instead of failing. +# --------------------------------------------------------------------------- + + +class TestOverRangeResolvedIntField: + def test_resolved_value_above_max_is_rejected(self) -> None: + # GIVEN + symtab = SymbolTable() + symtab["X"] = str(MAX_INT_FIELD_VALUE + 1) + + # WHEN / THEN + with pytest.raises(ValueError, match="must be at most"): + resolve_optional_int_field( + _format_string_field("{{ X }}"), symtab, ge=1, description="timeout" + ) + + def test_resolved_value_at_max_is_accepted(self) -> None: + # GIVEN + symtab = SymbolTable() + symtab["X"] = str(MAX_INT_FIELD_VALUE) + + # WHEN / THEN: the boundary itself is pinned from both sides. + assert ( + resolve_optional_int_field( + _format_string_field("{{ X }}"), symtab, ge=1, description="timeout" + ) + == MAX_INT_FIELD_VALUE + ) + + +# --------------------------------------------------------------------------- +# Task.File.* / Env.File.* are PATH-typed, so EXPR property access works on +# them. Without the type they resolve as plain strings and `.parent` fails. +# --------------------------------------------------------------------------- + + +class TestEmbeddedFileSymbolsArePathTyped: + def test_task_file_supports_path_property_access( + self, caplog: pytest.LogCaptureFixture + ) -> None: + # GIVEN: a step-level binding taking `.parent` of an embedded file path. + context = ModelParsingContext_2023_09(supported_extensions=["EXPR"]) + step = StepScript_2023_09.model_validate( + { + "let": ["where = string(Task.File.Cfg.parent)"], + "actions": {"onRun": {"command": "echo", "args": ["PARENT={{where}}"]}}, + "embeddedFiles": [{"name": "Cfg", "type": "TEXT", "data": "config-contents\n"}], + }, + context=context, + ) + + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + # WHEN + session.run_task(step_script=step, task_parameter_values={}) + _run_until_ready(session) + + # THEN: the binding resolved, so the symbol carried the PATH type. + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + assert any("PARENT=" in m and "embedded_files" in m for m in caplog.messages) diff --git a/test/openjd/sessions_v0/test_wrap_task_run.py b/test/openjd/sessions_v0/test_wrap_task_run.py index 84c3c69b..572ae52c 100644 --- a/test/openjd/sessions_v0/test_wrap_task_run.py +++ b/test/openjd/sessions_v0/test_wrap_task_run.py @@ -617,3 +617,155 @@ def test_run_task_without_step_name_is_fine_unwrapped(self) -> None: status = session.action_status assert status is not None assert status.state == ActionState.SUCCESS + + +# --------------------------------------------------------------------------- +# A wrap hook resolves in the wrap environment's own scope, which in openjd-rs +# is that environment's frozen enter-time symbol table. So a step environment +# that defines wrap hooks must carry the step-level `let` bindings it was +# entered with into every hook invocation -- without letting them replace the +# wrapped action's own resolution scope. +# --------------------------------------------------------------------------- + + +class TestWrapHookSeesEnterTimeStepScope: + def test_hook_resolves_step_level_let_bindings(self, tmp_path) -> None: + # GIVEN: a wrap environment entered with a step's step-level bindings, + # and a hook that references one of them. + env = _wrap_env( + "WrapEnv", + Action_2023_09( + command=CommandString_2023_09("echo"), + args=[ArgString_2023_09("HOOK-{{greeting}}")], + ), + ) + step = _step_script("echo", ["INNER"]) + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + identifier = session.enter_environment( + environment=env, + extra_let_bindings=["greeting = 'from-step-let'"], + step_name="Step1", + ) + _run_until_ready(session) + assert session.action_status is not None + assert session.action_status.state == ActionState.SUCCESS + + # WHEN + session.run_task(step_script=step, task_parameter_values={}, step_name="Step1") + _run_until_ready(session) + + # THEN: the hook resolved the step-level binding instead of failing + # with "Undefined variable". + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + + session.exit_environment(identifier=identifier) + _run_until_ready(session) + + def test_wrap_env_step_name_does_not_reach_the_wrapped_action(self) -> None: + """The wrap env's enter-time step context must not replace the running + step's in the wrapped action's own scope. + + This goes through ``run_task`` on purpose: the ordering it pins lives in + ``run_task`` (seed the hook's scope only *after* the wrapped action's own + scope has been built), so a test that calls the two helpers itself in its + own order would re-assert its own script instead. + """ + # GIVEN: a wrap env entered for "StepA", and a step whose onRun echoes + # {{Step.Name}} -- which must resolve to the step actually running. + hook_args: list[str] = [] + + env = _wrap_env( + "WrapEnv", + Action_2023_09( + command=CommandString_2023_09("echo"), + args=[ArgString_2023_09("{{Step.Name}}")], + ), + ) + step = _step_script("echo", ["{{Step.Name}}"]) + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + identifier = session.enter_environment( + environment=env, extra_let_bindings=None, step_name="StepA" + ) + _run_until_ready(session) + + original = session._inject_wrapped_task_symbols + + def _capture(symtab, step_script, step_name, *, inner_symtab): + original(symtab, step_script, step_name, inner_symtab=inner_symtab) + hook_args.extend(symtab["WrappedAction.Args"]) + + session._inject_wrapped_task_symbols = _capture # type: ignore[method-assign] + + # WHEN: a task of a DIFFERENT step runs under that wrap env + session.run_task(step_script=step, task_parameter_values={}, step_name="StepB") + _run_until_ready(session) + + # THEN: the wrapped action resolved with the RUNNING step's name. + # Seeding the hook's scope before injection instead would give StepA. + assert hook_args == ["StepB"] + assert session.action_status is not None + assert session.action_status.state == ActionState.SUCCESS + + session.exit_environment(identifier=identifier) + _run_until_ready(session) + + @pytest.mark.parametrize("hook_phase", ["enter", "exit"]) + def test_env_hooks_resolve_step_level_let_bindings(self, hook_phase: str) -> None: + """The seeding applies to onWrapEnvEnter and onWrapEnvExit too, not just + onWrapTaskRun.""" + # GIVEN: a wrap env entered with a step-level binding, and an inner env + # whose onEnter/onExit the hooks intercept. + wrap_action = Action_2023_09( + command=CommandString_2023_09("echo"), + args=[ArgString_2023_09("HOOK-{{greeting}}")], + ) + env = Environment_2023_09( + name="WrapEnv", + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onWrapEnvEnter=wrap_action, + onWrapTaskRun=_NOOP, + onWrapEnvExit=wrap_action, + ), + ), + ) + inner = Environment_2023_09( + name="Inner", + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onEnter=Action_2023_09(command=CommandString_2023_09("true")), + onExit=Action_2023_09(command=CommandString_2023_09("true")), + ), + ), + ) + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + wrap_id = session.enter_environment( + environment=env, + extra_let_bindings=["greeting = 'from-step-let'"], + step_name="Step1", + ) + _run_until_ready(session) + + # WHEN + inner_id = session.enter_environment(environment=inner) + _run_until_ready(session) + if hook_phase == "enter": + # THEN: the onWrapEnvEnter hook resolved the binding + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + else: + session.exit_environment(identifier=inner_id) + _run_until_ready(session) + # THEN: the onWrapEnvExit hook resolved the binding + status = session.action_status + assert status is not None + assert status.state == ActionState.SUCCESS + + if hook_phase == "enter": + session.exit_environment(identifier=inner_id) + _run_until_ready(session) + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) From 2ed9e123e226038b4296f335fc0ecd4e31fbe56f Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Sat, 25 Jul 2026 11:33:59 -0700 Subject: [PATCH 07/27] fix: Make wrap and URI tests host-independent on Windows Windows CI ran this PR's tests for the first time (earlier runs were cancelled by fail-fast before the test step) and found 8 deterministic failures, all in test code: - test_wrap_actions.py interpolates the trace-log path bare into `sh -c`. On Windows a native path's backslashes are consumed as escapes by the sh these tests invoke, so the redirect landed on a mangled path and the file never appeared. The hooks were firing correctly. Now quoted and slash-separated, matching the one interpolation that already was. - The new URI path-mapping tests built their rule with from_dict, whose destination_path is a host-flavoured PurePath -- so "/local" renders as \\local on Windows. They now construct the rule with an explicit PurePosixPath destination, which keeps the expectations host-independent without weakening what they assert. No production code changed. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- test/openjd/sessions_v0/test_path_mapping.py | 15 +++--- test/openjd/sessions_v0/test_wrap_actions.py | 49 +++++++++++++------- 2 files changed, 42 insertions(+), 22 deletions(-) diff --git a/test/openjd/sessions_v0/test_path_mapping.py b/test/openjd/sessions_v0/test_path_mapping.py index 86d98914..350ca6ef 100644 --- a/test/openjd/sessions_v0/test_path_mapping.py +++ b/test/openjd/sessions_v0/test_path_mapping.py @@ -547,12 +547,15 @@ class TestUriPathMapping: """ def _rule(self, source: str, destination: str = "/local") -> PathMappingRule: - return PathMappingRule.from_dict( - rule={ - "source_path_format": "URI", - "source_path": source, - "destination_path": destination, - } + # Construct directly with a PurePosixPath destination rather than going + # through from_dict, which builds a host-flavoured PurePath -- on Windows + # that renders "/local" as "\\local" and the expectations below would + # have to be host-dependent for no reason. os_name is patched to posix in + # each test, so the child separator is "/" on both hosts. + return PathMappingRule( + source_path_format=PathFormat.URI, + source_path=source, + destination_path=PurePosixPath(destination), ) @pytest.mark.parametrize( diff --git a/test/openjd/sessions_v0/test_wrap_actions.py b/test/openjd/sessions_v0/test_wrap_actions.py index 1922337e..f3fd5b5e 100644 --- a/test/openjd/sessions_v0/test_wrap_actions.py +++ b/test/openjd/sessions_v0/test_wrap_actions.py @@ -64,6 +64,13 @@ def _run_until_ready(session: Session, timeout_s: float = 10.0) -> None: def _trace_path(session: Session) -> Path: + """The file the test scripts append to. + + Interpolate it into a shell command as ``'{trace.as_posix()}'`` -- quoted and + slash-separated. A native Windows path interpolated bare has its backslashes + consumed as escapes by the ``sh`` these tests invoke, so the redirect would + silently land somewhere else. + """ return Path(str(session.working_directory)) / "trace.log" @@ -130,16 +137,18 @@ def test_wrap_env_enter_intercepts_inner_on_enter(self) -> None: onWrapEnvEnter=_action( "sh", "-c", - f"echo '[WRAPPED] inner-onEnter' >> {trace} && " - f"echo 'inner-onEnter body' >> {trace}", + f"echo '[WRAPPED] inner-onEnter' >> '{trace.as_posix()}' && " + f"echo 'inner-onEnter body' >> '{trace.as_posix()}'", ), onWrapTaskRun=_NOOP, onWrapEnvExit=_NOOP, - onEnter=_action("sh", "-c", f"echo 'outer-onEnter' >> {trace}"), + onEnter=_action("sh", "-c", f"echo 'outer-onEnter' >> '{trace.as_posix()}'"), ) inner = _env( "inner", - onEnter=_action("sh", "-c", f"echo 'should not run on host' >> {trace}"), + onEnter=_action( + "sh", "-c", f"echo 'should not run on host' >> '{trace.as_posix()}'" + ), ) session.enter_environment(environment=outer) @@ -170,7 +179,7 @@ def test_wrap_env_enter_receives_wrapped_symbols(self) -> None: "-c", "echo " "'name={{WrappedEnv.Name}} cmd={{WrappedAction.Command}}' " - f">> {trace}", + f">> '{trace.as_posix()}'", ), onWrapTaskRun=_NOOP, onWrapEnvExit=_NOOP, @@ -204,13 +213,15 @@ def test_wrap_env_exit_intercepts_inner_on_exit(self) -> None: onWrapEnvExit=_action( "sh", "-c", - f"echo '[WRAPPED] inner-onExit' >> {trace}", + f"echo '[WRAPPED] inner-onExit' >> '{trace.as_posix()}'", ), - onExit=_action("sh", "-c", f"echo 'outer-onExit' >> {trace}"), + onExit=_action("sh", "-c", f"echo 'outer-onExit' >> '{trace.as_posix()}'"), ) inner = _env( "inner", - onExit=_action("sh", "-c", f"echo 'should not run on host' >> {trace}"), + onExit=_action( + "sh", "-c", f"echo 'should not run on host' >> '{trace.as_posix()}'" + ), ) outer_id = session.enter_environment(environment=outer) @@ -244,16 +255,22 @@ def test_full_ordering_across_three_hooks(self) -> None: trace = _trace_path(session) outer = _env( "outer", - onEnter=_action("sh", "-c", f"echo 'outer-onEnter' >> {trace}"), - onWrapEnvEnter=_action("sh", "-c", f"echo '[WRAPPED] inner-onEnter' >> {trace}"), - onWrapTaskRun=_action("sh", "-c", f"echo '[WRAPPED] task-onRun' >> {trace}"), - onWrapEnvExit=_action("sh", "-c", f"echo '[WRAPPED] inner-onExit' >> {trace}"), - onExit=_action("sh", "-c", f"echo 'outer-onExit' >> {trace}"), + onEnter=_action("sh", "-c", f"echo 'outer-onEnter' >> '{trace.as_posix()}'"), + onWrapEnvEnter=_action( + "sh", "-c", f"echo '[WRAPPED] inner-onEnter' >> '{trace.as_posix()}'" + ), + onWrapTaskRun=_action( + "sh", "-c", f"echo '[WRAPPED] task-onRun' >> '{trace.as_posix()}'" + ), + onWrapEnvExit=_action( + "sh", "-c", f"echo '[WRAPPED] inner-onExit' >> '{trace.as_posix()}'" + ), + onExit=_action("sh", "-c", f"echo 'outer-onExit' >> '{trace.as_posix()}'"), ) wrapped_inner = _env( "wrapped-inner", - onEnter=_action("sh", "-c", f"echo 'inner-onEnter body' >> {trace}"), - onExit=_action("sh", "-c", f"echo 'inner-onExit body' >> {trace}"), + onEnter=_action("sh", "-c", f"echo 'inner-onEnter body' >> '{trace.as_posix()}'"), + onExit=_action("sh", "-c", f"echo 'inner-onExit body' >> '{trace.as_posix()}'"), ) step = _step("echo", "task-onRun-body") @@ -637,7 +654,7 @@ def test_exit_path_includes_exiting_envs_openjd_env_vars(self) -> None: onWrapEnvExit=_action( "sh", "-c", - f"echo \"ENVLIST=<{{{{WrappedAction.Environment}}}}>\" >> '{trace}'", + f"echo \"ENVLIST=<{{{{WrappedAction.Environment}}}}>\" >> '{trace.as_posix()}'", ), ) inner = _env("Inner", onEnter=_NOOP, onExit=_NOOP) From db6decfb10a5ad29c79c801c729b72d9cfce8a87 Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Sat, 25 Jul 2026 15:06:01 -0700 Subject: [PATCH 08/27] test: Pin the fixes from the parallel-review round Four behaviors were fixed in the previous commits with no test to hold them. Each test here was verified to fail against a mutant that reverts its fix. - WINDOWS path rules fold case over ASCII only: a U+212A homoglyph must not match an ASCII 'k', while ASCII case-insensitivity and both trailing-slash separators keep working. - A URI rule's source_path must carry a real scheme, rejected at construction where the offending value can be named. - The session never claims RUNNING while it has no runner, observed at the last point before the runner is built in all three RFC 0008 wrap paths. That window is one in which a cross-thread cancel_action() is lost. - A task parameter named like an env macro (openjd_env, openjd_unset_env, openjd_redacted_env, openjd_envX) does not make run_task raise: the session's own parameter-logging line reaches the action-message filter while nothing is running. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- test/openjd/sessions_v0/test_path_mapping.py | 93 +++++++++++++++++++ test/openjd/sessions_v0/test_session.py | 41 ++++++++ test/openjd/sessions_v0/test_wrap_task_run.py | 70 ++++++++++++++ 3 files changed, 204 insertions(+) diff --git a/test/openjd/sessions_v0/test_path_mapping.py b/test/openjd/sessions_v0/test_path_mapping.py index 350ca6ef..73527b5d 100644 --- a/test/openjd/sessions_v0/test_path_mapping.py +++ b/test/openjd/sessions_v0/test_path_mapping.py @@ -618,3 +618,96 @@ def test_non_ascii_authority_still_matches_itself(self) -> None: # THEN: ASCII letters fold, the non-ASCII character compares as itself assert matched is True assert result == "/local/f.obj" + + +class TestWindowsAsciiOnlyFold: + """Windows path rules match case-insensitively, but over ASCII only. + + `PureWindowsPath.is_relative_to()` folds with `str.lower()`, which also folds + non-ASCII characters, so a homoglyph in a submitted path would remap here + while the EXPR engine's `apply_path_mapping()` and openjd-rs -- both using an + ASCII-only fold -- leave it alone. RFC 0006 2.3.2 says the two are the same + transformation. + """ + + def _rule(self, source: str) -> PathMappingRule: + return PathMappingRule( + source_path_format=PathFormat.WINDOWS, + source_path=PureWindowsPath(source), + destination_path=PurePosixPath("/mnt/assets"), + ) + + def test_non_ascii_homoglyph_does_not_match(self) -> None: + # GIVEN: a rule whose source ends in ASCII 'k' + rule = self._rule("C:\\assets\\k") + + # WHEN: the input uses U+212A KELVIN SIGN, which str.lower() folds to 'k' + with patch.object(path_mapping_impl_mod, "os_name", OSName.POSIX.value): + matched, result = rule.apply(path="C:\\assets\\\u212a\\shot.exr") + + # THEN: no match, matching the engine and openjd-rs + assert matched is False + assert result == "C:\\assets\\\u212a\\shot.exr" + + @pytest.mark.parametrize( + "given, expected", + [ + pytest.param("C:\\assets\\shot.exr", (True, "/mnt/assets/shot.exr"), id="plain"), + pytest.param( + "C:\\ASSETS\\shot.exr", (True, "/mnt/assets/shot.exr"), id="ascii-fold-still-works" + ), + pytest.param( + "C:/assets/scenes/", (True, "/mnt/assets/scenes/"), id="forward-slash-trailing" + ), + pytest.param("C:\\assets\\", (True, "/mnt/assets/"), id="backslash-trailing"), + pytest.param( + "D:\\assets\\shot.exr", (False, "D:\\assets\\shot.exr"), id="different-drive" + ), + pytest.param( + "C:\\assetsmore\\x", (False, "C:\\assetsmore\\x"), id="not-a-component-boundary" + ), + ], + ) + def test_windows_matching(self, given: str, expected: tuple[bool, str]) -> None: + # GIVEN + rule = self._rule("C:\\assets") + + # WHEN / THEN + with patch.object(path_mapping_impl_mod, "os_name", OSName.POSIX.value): + assert rule.apply(path=given) == expected + + +class TestUriRuleValidation: + """A URI-format rule's source_path must carry a real scheme. + + The EXPR engine requires one, and these rules are handed to it via the + session's host context -- so a malformed rule has to be rejected where the + offending value can still be named, not later from inside `Session()`. + """ + + @pytest.mark.parametrize( + "source", + ["s3:/bucket", "notauri", "://bucket", "1s3://bucket", "", "s_3://bucket"], + ) + def test_malformed_uri_source_rejected(self, source: str) -> None: + with pytest.raises(ValueError, match="must begin with"): + PathMappingRule.from_dict( + rule={ + "source_path_format": "URI", + "source_path": source, + "destination_path": "/local", + } + ) + + @pytest.mark.parametrize( + "source", ["s3://bucket/a", "file:///x", "nfs://host:2049/export", "S3://Bucket"] + ) + def test_well_formed_uri_source_accepted(self, source: str) -> None: + rule = PathMappingRule.from_dict( + rule={ + "source_path_format": "URI", + "source_path": source, + "destination_path": "/local", + } + ) + assert rule.source_path == source diff --git a/test/openjd/sessions_v0/test_session.py b/test/openjd/sessions_v0/test_session.py index f3420a92..3c25b870 100644 --- a/test/openjd/sessions_v0/test_session.py +++ b/test/openjd/sessions_v0/test_session.py @@ -3900,3 +3900,44 @@ def test_run_task_without_session_env_ignores_multiple_environments( # THEN - All entered environment variables should be ignored assert session.action_status == ActionStatus(state=ActionState.SUCCESS, exit_code=0) assert "FOO=NOT_SET" in caplog.messages + + +class TestSessionEmittedLogLinesDoNotTriggerCancel: + """The action-message filter is attached to the session logger, so it also + sees lines the session itself logs while no action is running. + + A task parameter whose name starts with ``openjd_env`` makes ``run_task``'s + own parameter-logging line look like a malformed env macro. The filter then + asks the session to cancel a running action -- and there isn't one, so + ``cancel_action`` used to raise ``RuntimeError`` out of ``logger.info`` and + out of the public API, with no callback and the task never run. + """ + + @pytest.mark.parametrize( + "param_name", + ["openjd_env", "openjd_unset_env", "openjd_redacted_env", "openjd_envX"], + ) + def test_parameter_named_like_an_env_macro_does_not_raise(self, param_name: str) -> None: + # GIVEN + script = StepScript_2023_09( + actions=StepActions_2023_09( + onRun=Action_2023_09( + command=CommandString_2023_09("echo"), + args=[ArgString_2023_09("ok")], + ) + ) + ) + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + # WHEN: the parameter name collides with the macro prefix + session.run_task( + step_script=script, + task_parameter_values={ + param_name: ParameterValue(type=ParameterValueType.STRING, value="x") + }, + ) + while session.state == SessionState.RUNNING: + time.sleep(0.05) + + # THEN: the task ran normally + assert session.action_status is not None + assert session.action_status.state == ActionState.SUCCESS diff --git a/test/openjd/sessions_v0/test_wrap_task_run.py b/test/openjd/sessions_v0/test_wrap_task_run.py index 572ae52c..53446ebc 100644 --- a/test/openjd/sessions_v0/test_wrap_task_run.py +++ b/test/openjd/sessions_v0/test_wrap_task_run.py @@ -769,3 +769,73 @@ def test_env_hooks_resolve_step_level_let_bindings(self, hook_phase: str) -> Non _run_until_ready(session) session.exit_environment(identifier=wrap_id) _run_until_ready(session) + + +# --------------------------------------------------------------------------- +# The session must never claim RUNNING while it has no runner. cancel_action() +# is a cross-thread API guarded only by `state == RUNNING`, so such a window is +# one in which a cancel is lost -- and the RFC 0008 branches do real work +# (materializing the inner entity's embedded files, evaluating its `let` +# bindings, allocating the hook's file records) before a runner exists. +# --------------------------------------------------------------------------- + + +class TestNoRunningWithoutRunner: + def _wrap_env_all_hooks(self) -> Environment_2023_09: + act = Action_2023_09(command=CommandString_2023_09("true")) + return Environment_2023_09( + name="WrapEnv", + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onWrapEnvEnter=act, onWrapTaskRun=act, onWrapEnvExit=act + ), + ), + ) + + def _inner_env(self) -> Environment_2023_09: + act = Action_2023_09(command=CommandString_2023_09("true")) + return Environment_2023_09( + name="Inner", + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09(onEnter=act, onExit=act), + ), + ) + + def test_state_is_not_running_during_wrap_setup(self) -> None: + """Observed at the last point before the runner is built, in all three + wrap paths.""" + observed: list[tuple[str, SessionState]] = [] + + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + original = session._get_wrap_env_file_records + + def _observe(wrap_env): # type: ignore[no-untyped-def] + # _get_wrap_env_file_records is the last step before the runner + # is constructed, so this is the worst case for the window. + observed.append((wrap_env.name, session.state)) + return original(wrap_env) + + session._get_wrap_env_file_records = _observe # type: ignore[method-assign] + + wrap_id = session.enter_environment(environment=self._wrap_env_all_hooks()) + _run_until_ready(session) + + inner_id = session.enter_environment(environment=self._inner_env()) + _run_until_ready(session) + + session.run_task( + step_script=_step_script("echo", ["hi"]), + task_parameter_values={}, + step_name="Step1", + ) + _run_until_ready(session) + + session.exit_environment(identifier=inner_id) + _run_until_ready(session) + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + + # THEN: all three wrap paths were exercised, and none of them had + # already flipped the session to RUNNING. + assert len(observed) == 3, observed + assert all(state is not SessionState.RUNNING for _, state in observed), observed From 0bebd29938956555b17e2e0fd5eaf48c53342839 Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Sat, 25 Jul 2026 15:13:40 -0700 Subject: [PATCH 09/27] chore: Require openjd-model >= 0.11.1 openjd-model-for-python #318 is merged and released, so the version-skew note can go and the floor can name a real release. 0.11.1 is the floor because it is the first release carrying the whole surface this package uses. Verified against the published wheels: 0.11.0 provides only StepTemplate.let, StepScript.let and SymbolTable.expr_types, and is missing FormatString.resolve_value, FormatString.whole_field_expression, evaluate_let_bindings, CancelationMethodDeferred and SymbolTable.expr_host_rules -- this package fails at import against it. 0.11.1 provides all nine. CI already resolved 0.11.1 under the old floor and ran mypy clean over src and test on Linux, macOS and Windows, so the merge-order constraint on the model is satisfied. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- pyproject.toml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index da8cf4ec..fca4f620 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,12 +31,13 @@ classifiers = [ "Topic :: Software Development :: Libraries" ] dependencies = [ - # NOTE (version-skew guard): this branch requires openjd-model features that - # only exist as of openjd-model-for-python PR #318 (FormatString.resolve_value, - # FormatString.whole_field_expression, evaluate_let_bindings, Step.let). - # Before release, bump the version floor below to the first openjd-model - # release that includes PR #318's surface. - "openjd-model >= 0.10,< 0.12", + # 0.11.1 is the floor because it is the first release carrying the EXPR/ + # WRAP_ACTIONS model surface this package uses: FormatString.resolve_value and + # .whole_field_expression, evaluate_let_bindings, CancelationMethodDeferred, + # SymbolTable.expr_host_rules, and `let` on StepTemplate/StepScript. 0.11.0 has + # only StepTemplate.let, StepScript.let and SymbolTable.expr_types, so this + # package fails at import against it. + "openjd-model >= 0.11.1,< 0.12", "pywin32 >= 307; platform_system == 'Windows'", "psutil >= 5.9,< 7.3; platform_system == 'Windows'", ] From 27cf66b4f9df475ab1418f2eb49ee908805af806 Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Sat, 25 Jul 2026 15:24:56 -0700 Subject: [PATCH 10/27] fix: Do not fail an action whose subprocess exits immediately CI surfaced this on macOS 3.12 as a flaky failure in an unrelated wrap test, and it is the same root cause as the Windows-only flake that has been hitting one of six Windows jobs per run: posix: os.getpgid(pid) raises ProcessLookupError (ESRCH) windows: psutil raises NoSuchProcess ('process PID not found (pid=...)') Both are reached when a trivial command finishes before the runner has finished recording it. Both were raised on the run future, so a subprocess that ran to completion was reported as a failed action, and the session went to READY_ENDING -- which is why a wrap test asserting READY between tasks failed intermittently on the second of three back-to-back echo actions. - _subprocess.run: an already-reaped child has no process group to look up. Fall back to its own pid, which is the group we would have found since the child is the group leader. - _windows_process_killer._suspend_process_tree: a process that exits between being discovered and being walked has no children to suspend. Mirrors the NoSuchProcess handling already present elsewhere in that module. Tests: the posix guard is pinned by patching os.getpgid to raise (verified against a mutant that stops catching it); the Windows walk is pinned with a psutil mock, gated to Windows. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- src/openjd/sessions/_subprocess.py | 23 +++++--- .../sessions/_windows_process_killer.py | 17 +++++- test/openjd/sessions_v0/test_subprocess.py | 59 +++++++++++++++++++ 3 files changed, 90 insertions(+), 9 deletions(-) diff --git a/src/openjd/sessions/_subprocess.py b/src/openjd/sessions/_subprocess.py index 9b21a4ef..3048872b 100644 --- a/src/openjd/sessions/_subprocess.py +++ b/src/openjd/sessions/_subprocess.py @@ -177,13 +177,22 @@ def run(self) -> None: # Would use is_posix(), but it doesn't short-circuit mypy which then complains # about os.getpgid not being a valid attribute. if not sys.platform == "win32": - if not self._user or self._user.is_process_user(): - self._sudo_child_process_group_id = os.getpgid(self._process.pid) - else: - self._sudo_child_process_group_id = find_sudo_child_process_group_id( - logger=self._logger, - sudo_process=self._process, - ) + # A trivial command can exit before we get here. Looking up the + # process group of an already-reaped child raises ProcessLookupError + # (ESRCH), which must not fail the action: the child ran, and its + # exit code is still collected below. Fall back to its own pid -- + # the process group we would have found, since the child is the + # group leader (start_new_session=True). + try: + if not self._user or self._user.is_process_user(): + self._sudo_child_process_group_id = os.getpgid(self._process.pid) + else: + self._sudo_child_process_group_id = find_sudo_child_process_group_id( + logger=self._logger, + sudo_process=self._process, + ) + except ProcessLookupError: + self._sudo_child_process_group_id = self._process.pid self._logger.info( f"Command started as pid: {self._process.pid}", diff --git a/src/openjd/sessions/_windows_process_killer.py b/src/openjd/sessions/_windows_process_killer.py index 2809ce99..75856c75 100644 --- a/src/openjd/sessions/_windows_process_killer.py +++ b/src/openjd/sessions/_windows_process_killer.py @@ -60,8 +60,21 @@ def _suspend_process_tree( if not suspend_subprocesses: return - # Recursively suspend child processes. - for child in process.children(): + # Recursively suspend child processes. A process that exited between being + # discovered and being walked has no children to suspend, and must not fail + # the cancel: psutil raises NoSuchProcess ("process PID not found") here, and + # this runs on the run future, so the exception would surface as a failed + # action for a subprocess that in fact completed. + try: + children = process.children() + except NoSuchProcess: + logger.info( + f"Process {process.pid} exited before its children could be listed.", + extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), + ) + return + + for child in children: _suspend_process_tree( logger, child, all_processes, procs_cannot_suspend, suspend_subprocesses ) diff --git a/test/openjd/sessions_v0/test_subprocess.py b/test/openjd/sessions_v0/test_subprocess.py index 6c014546..cd9891cb 100644 --- a/test/openjd/sessions_v0/test_subprocess.py +++ b/test/openjd/sessions_v0/test_subprocess.py @@ -18,6 +18,7 @@ from openjd.sessions._os_checker import is_posix, is_windows from openjd.sessions._session_user import PosixSessionUser, WindowsSessionUser from openjd.sessions._subprocess import LoggingSubprocess +from openjd.sessions import _subprocess as subprocess_impl_mod from .conftest import ( build_logger, @@ -1069,3 +1070,61 @@ def end_proc(): if num_children_running == 0: break assert num_children_running == 0 + + +class TestFastExitingChild: + """A trivial command can exit before the runner finishes recording it. + + Looking up an already-reaped child's process group raises ProcessLookupError + on posix, and psutil raises NoSuchProcess when walking it on Windows. Neither + may fail the action: the child ran, and its exit code is still collected. + """ + + @pytest.mark.skipif(not is_posix(), reason="posix-only: process groups") + @pytest.mark.usefixtures("message_queue", "queue_handler") + def test_getpgid_lookup_failure_does_not_fail_the_action( + self, + message_queue: SimpleQueue, + queue_handler: QueueHandler, + ) -> None: + # GIVEN: the child is gone by the time its process group is looked up + logger = build_logger(queue_handler) + callback = MagicMock() + subproc = LoggingSubprocess( + logger=logger, + args=[sys.executable, "-c", "print('DONE')"], + callback=callback, + ) + + with patch.object( + subprocess_impl_mod.os, "getpgid", side_effect=ProcessLookupError(3, "No such process") + ): + # WHEN + subproc.run() + + # THEN: the action completed normally + assert subproc.exit_code == 0 + assert subproc.failed_to_start is False + messages = collect_queue_messages(message_queue) + assert "DONE" in messages + + @pytest.mark.skipif(not is_windows(), reason="Windows-only: psutil process walk") + def test_process_tree_walk_tolerates_exited_process(self) -> None: + # GIVEN: a process that disappears between discovery and the walk + from psutil import NoSuchProcess + + from openjd.sessions._windows_process_killer import _suspend_process_tree + + logger = MagicMock() + process = MagicMock() + process.pid = 4321 + process.suspend.side_effect = NoSuchProcess(4321) + process.children.side_effect = NoSuchProcess(4321) + cannot_suspend: list = [] + all_processes: list = [] + + # WHEN / THEN: no exception escapes + _suspend_process_tree( + logger, process, all_processes, cannot_suspend, suspend_subprocesses=True + ) + assert process in all_processes From a954917cab6e9c373e6a67772b0fc2359de4896a Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Sat, 25 Jul 2026 16:18:46 -0700 Subject: [PATCH 11/27] fix: Serialize the pending-cancel handoff against action launch Addresses the review comment on the previous head: the _pending_cancel handoff was unsynchronized, so the lost-cancel window it was meant to close was still open. Two defects, both real: - The reader in _run and the writer in _cancel_with_resolved_method both touched _pending_cancel outside self._lock, so a cancel could be recorded after _run had already consumed (and found nothing). - The writer keyed on whether the LoggingSubprocess object existed, not on whether it had started. In the window where the object exists but the pool thread is still inside _start_subprocess, the canceller handed off to _cancel, which returns early because the process is not running -- while _run had already passed its consume point. Dropped by both sides. Now the decide-and-record step happens under the lock and keys on has_started, and _run consumes under the lock; _cancel is called outside it, since it takes the lock itself. The regression test reproduces the exact interleaving by blocking inside _start_subprocess so the window is held open deterministically. Verified against a mutant keyed on object existence: it ends SUCCESS after the full 30 second sleep instead of CANCELED, i.e. the cancel is silently dropped. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- src/openjd/sessions/_runner_base.py | 49 +++++++----- .../sessions_v0/test_runner_step_script.py | 75 +++++++++++++++++++ 2 files changed, 107 insertions(+), 17 deletions(-) diff --git a/src/openjd/sessions/_runner_base.py b/src/openjd/sessions/_runner_base.py index d1dd78eb..db148adc 100644 --- a/src/openjd/sessions/_runner_base.py +++ b/src/openjd/sessions/_runner_base.py @@ -722,13 +722,16 @@ def _run(self, args: Sequence[str], time_limit: Optional[timedelta] = None) -> N # before we check self.state self._process.wait_until_started() - # A cancel that landed during setup, before there was a subprocess to - # signal, is applied now rather than dropped. See _pending_cancel. - pending = self._pending_cancel - if pending is not None: + # A cancel that landed during setup, before there was a running + # subprocess to signal, is applied now rather than dropped. Read under + # the lock so the handoff is serialized against the writer in + # _cancel_with_resolved_method; _cancel takes the lock itself, so it is + # called outside. + with self._lock: + pending = self._pending_cancel self._pending_cancel = None - if self._resolved_cancel_method is not None: - self._cancel(self._resolved_cancel_method, *pending) + if pending is not None and self._resolved_cancel_method is not None: + self._cancel(self._resolved_cancel_method, *pending) if self.state == ScriptRunnerState.RUNNING and self._callback is not None: # Let the caller know that the process is running. @@ -913,22 +916,34 @@ def _cancel_with_resolved_method( the effective cancel method that :meth:`_run_action` resolved at launch time. - A cancel that arrives before the subprocess exists is remembered and - applied by :meth:`_run` as soon as it does (see :attr:`_pending_cancel`) - — `cancel()` is called from another thread, so "no subprocess yet" is a + A cancel that arrives before the subprocess is running is remembered and + applied by :meth:`_run` as soon as it starts (see :attr:`_pending_cancel`) + — `cancel()` is called from another thread, so "not running yet" is a race, not a no-op. Only when no action will ever be launched (setup - failed before resolution) is there genuinely nothing to cancel. + failed, or there was no action to run) is there genuinely nothing to + cancel. """ - if self._process is None: - if self.state == ScriptRunnerState.FAILED: - # Setup already failed; no subprocess is coming. + with self._lock: + # Decide-and-record must be atomic with respect to _run creating and + # starting the subprocess, otherwise a cancel can land between the + # two and be dropped by both sides: this method would see a process + # and hand off to _cancel, which has nothing to signal yet, while + # _run has already passed the point where it consumes a pending + # cancel. Keyed on has_started rather than on the object existing, + # for the same reason. + if self._state_override is not None: + # Terminal before launch: setup failed, or there was no action. return - self._pending_cancel = (time_limit, mark_action_failed) + process = self._process + if process is None or not process.has_started: + self._pending_cancel = (time_limit, mark_action_failed) + return + method = self._resolved_cancel_method + if method is None: # pragma: no cover - defensive return # Note: If the given time_limit is less than that in the method, then the time_limit will be what's used. - if self._resolved_cancel_method is None: # pragma: no cover - defensive - return - self._cancel(self._resolved_cancel_method, time_limit, mark_action_failed) + # Called outside the lock: _cancel takes it itself. + self._cancel(method, time_limit, mark_action_failed) def _cancel( self, diff --git a/test/openjd/sessions_v0/test_runner_step_script.py b/test/openjd/sessions_v0/test_runner_step_script.py index e1f08741..0884d9ba 100644 --- a/test/openjd/sessions_v0/test_runner_step_script.py +++ b/test/openjd/sessions_v0/test_runner_step_script.py @@ -1,5 +1,6 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +import threading import time from datetime import timedelta from logging.handlers import QueueHandler @@ -42,6 +43,7 @@ TerminateCancelMethod, ) from openjd.sessions._runner_step_script import StepScriptRunner +from openjd.sessions._subprocess import LoggingSubprocess from openjd.sessions._tempdir import TempDir from openjd.sessions._os_checker import is_posix, is_windows @@ -424,3 +426,76 @@ def test_cancel_with_no_subprocess_does_not_raise( # WHEN / THEN: the low-level cancel is a quiet no-op, not an AssertionError runner._cancel(TerminateCancelMethod()) + + +class TestCancelRacingLaunchIsSerialized: + """The pending-cancel handoff must be atomic with respect to launch. + + `cancel()` runs on another thread. Without serialization there is an + interleaving where the canceller sees a subprocess object and hands off to + `_cancel`, which has nothing to signal because the process has not started + yet, while `_run` has already passed the point where it consumes a pending + cancel -- so the cancel is dropped by both sides and the action runs to + completion. + """ + + @pytest.mark.skipif(not is_posix(), reason="signal delivery is posix-only here") + @pytest.mark.timeout(120) + def test_cancel_during_launch_is_not_dropped(self, tmp_path: Path, python_exe: str) -> None: + # GIVEN: a long-running action, and a canceller that fires while the + # subprocess object exists but has NOT started -- held open by blocking + # inside _start_subprocess, which runs on the pool thread after _run has + # already assigned self._process. + script = StepScript_2023_09( + actions=StepActions_2023_09( + onRun=Action_2023_09( + command=CommandString_2023_09(python_exe), + args=[ + ArgString_2023_09("-c"), + ArgString_2023_09("import time; time.sleep(30)"), + ], + ) + ) + ) + runner = StepScriptRunner( + logger=MagicMock(), + session_working_directory=tmp_path, + script=script, + symtab=SymbolTable(), + session_files_directory=tmp_path, + ) + in_window = threading.Event() + canceller_done = threading.Event() + real_start = LoggingSubprocess._start_subprocess + + def _start_after_cancel(subproc): # type: ignore[no-untyped-def] + in_window.set() + # Hold the window: _has_started is not set until this returns. + canceller_done.wait(timeout=30) + return real_start(subproc) + + def _cancel_in_window() -> None: + in_window.wait(timeout=30) + assert runner._process is not None + assert runner._process.has_started is False + runner.cancel() + canceller_done.set() + + canceller = threading.Thread(target=_cancel_in_window, daemon=True) + + # WHEN + with patch.object(LoggingSubprocess, "_start_subprocess", _start_after_cancel): + canceller.start() + runner.run() + canceller.join(timeout=30) + + # THEN: the cancel was applied, so the 30 second sleep did not run to + # completion. Unsynchronized, it is dropped by both sides and the runner + # ends in SUCCESS after 30 seconds. + deadline = time.time() + 60 + while runner.state in (ScriptRunnerState.RUNNING, ScriptRunnerState.CANCELING): + if time.time() > deadline: # pragma: no cover - timing guard + pytest.fail(f"runner never settled; state={runner.state}") + time.sleep(0.05) + assert runner.state == ScriptRunnerState.CANCELED + assert runner._pending_cancel is None From 8d08377eebf5beab5490fdbd14e227808b4ebe82 Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Sat, 25 Jul 2026 18:39:58 -0700 Subject: [PATCH 12/27] fix(concurrency): Close race windows in cancel and completion paths Address findings F1-F8 and Review22-F3/F4 from round 3 concurrency review: - F1: Always route EnvironmentScriptRunner.cancel() through pending-cancel path - F2+F4: Atomic terminal arbitration - claim pending cancel in _on_process_exit, move liveness check inside lock in _cancel - F3: Monotonic merge for duplicate pending cancels (min time_limit, OR failed) - F5: Snapshot action_status before publishing READY state - F6: Wrap cancel_info.json write in try/except, fallback to immediate terminate - F7: Detect self-join in shutdown(), use wait=False if called from worker thread - F8: Move callback outside lock, wrap in try/except to prevent child discard - Review22-F3: Snapshot _runner in cancel_action to avoid bare AssertionError - Review22-F4: Bind _process once in notify/terminate to avoid TOCTOU race Add test_concurrency_fixes.py with 8 unit tests covering the defensive behaviors. None of these issues reproduce in openjd-rs due to CancellationToken, tokio async, and Rust's Result<> error handling model. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- src/openjd/sessions/_runner_base.py | 100 ++++- src/openjd/sessions/_runner_env_script.py | 18 +- src/openjd/sessions/_session.py | 30 +- src/openjd/sessions/_subprocess.py | 14 +- .../sessions_v0/test_concurrency_fixes.py | 357 ++++++++++++++++++ 5 files changed, 491 insertions(+), 28 deletions(-) create mode 100644 test/openjd/sessions_v0/test_concurrency_fixes.py diff --git a/src/openjd/sessions/_runner_base.py b/src/openjd/sessions/_runner_base.py index db148adc..91b5cd85 100644 --- a/src/openjd/sessions/_runner_base.py +++ b/src/openjd/sessions/_runner_base.py @@ -589,7 +589,23 @@ def shutdown(self) -> None: """Performs a clean shutdown on the runner. This shutsdown the internal ThreadPoolExectutor. """ - self._pool.shutdown() + # F7 fix: Detect self-join. If shutdown() is called from the pool's + # worker thread (e.g., from _on_process_exit -> _action_callback -> + # cleanup -> runner.shutdown), calling _pool.shutdown(wait=True) would + # deadlock because the thread is trying to join itself. In that case, + # defer to a background thread or skip the wait. + import threading + + # Check if current thread is the pool's worker thread. The pool has + # max_workers=1, so if a future is running, _threads has one element. + pool_threads: set[threading.Thread] = getattr(self._pool, "_threads", set()) + current = threading.current_thread() + if current in pool_threads: + # We're inside the worker thread. Use wait=False to avoid deadlock. + # The pool will be garbage-collected eventually. + self._pool.shutdown(wait=False) + else: + self._pool.shutdown() def _fail_action(self, message: str) -> None: """Fail the action through the normal failure path: surface the @@ -936,7 +952,24 @@ def _cancel_with_resolved_method( return process = self._process if process is None or not process.has_started: - self._pending_cancel = (time_limit, mark_action_failed) + # F3 fix: Monotonic merge for duplicate pending cancels. If a + # cancel is already pending, merge the new request: take the + # minimum time_limit (tighter deadline wins, treating None as + # unlimited), and OR the mark_action_failed flags (once failed, + # always failed). + if self._pending_cancel is not None: + prev_limit, prev_failed = self._pending_cancel + # Merge time limits: None means unlimited, so a defined limit beats None + if time_limit is None: + merged_limit = prev_limit + elif prev_limit is None: + merged_limit = time_limit + else: + merged_limit = min(time_limit, prev_limit) + merged_failed = mark_action_failed or prev_failed + self._pending_cancel = (merged_limit, merged_failed) + else: + self._pending_cancel = (time_limit, mark_action_failed) return method = self._resolved_cancel_method if method is None: # pragma: no cover - defensive @@ -957,11 +990,18 @@ def _cancel( # pending cancel instead — an early return here rather than an # assert, so no bare AssertionError can reach the public API. return - # Nothing to do if it's not running. - if not self._process.is_running: - return with self._lock: + # F4 fix: Check liveness under the lock. Without this, a completion + # racing a cancel/timeout could see is_running=True outside the lock, + # enter here, and then find is_running=False (or worse, still True + # but the callback has already fired) — no linearization point. + # Moving the check inside the lock lets _on_process_exit's clearing + # of _pending_cancel act as the arbiter: if the process exited, any + # pending cancel was already consumed there. + if not self._process.is_running: + return + self._canceled = True self._notify_canceled_action_as_failed = mark_action_failed now = datetime.now(timezone.utc) @@ -1015,9 +1055,32 @@ def _cancel( # when we'll send the SIGKILL) grace_end_time_str = self._cancel_gracetime_end.strftime(TIME_FORMAT_STR) notify_end = json.dumps({"NotifyEnd": grace_end_time_str}) - write_file_for_user( - self._session_working_directory / "cancel_info.json", notify_end, self._user - ) + try: + write_file_for_user( + self._session_working_directory / "cancel_info.json", notify_end, self._user + ) + except OSError as err: + # F6 fix: If we cannot write the cancel_info.json (disk full, permission + # denied, etc.), log and fall back to immediate termination. A script + # waiting on that file would hang forever otherwise. + self._logger.warning( + f"Failed to write cancel_info.json: {err}. Falling back to immediate termination.", + extra=LogExtraInfo( + openjd_log_content=LogContent.PROCESS_CONTROL + | LogContent.EXCEPTION_INFO + ), + ) + try: + self._process.terminate() + except OSError as term_err: # pragma: nocover + self._logger.warning( + f"Fallback termination also failed: {term_err}", + extra=LogExtraInfo( + openjd_log_content=LogContent.PROCESS_CONTROL + | LogContent.EXCEPTION_INFO + ), + ) + return self._logger.info( f"Grace period ends at {grace_end_time_str}", extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), @@ -1046,6 +1109,13 @@ def _on_process_exit(self, future: Future) -> None: """This is invoked as a callback when run_future is done.""" assert self._run_future is not None with self._lock: + # F2 fix: Claim _pending_cancel atomically before signalling completion. + # A cancel racing completion would otherwise see the process still + # "running" (in _cancel_with_resolved_method) and hand off to _cancel, + # which then finds is_running=False and no-ops. By consuming the + # pending here we prevent that lost-cancel window. + self._pending_cancel = None + if self._runtime_limit is not None: self._runtime_limit.cancel() self._runtime_limit = None @@ -1062,8 +1132,20 @@ def _on_process_exit(self, future: Future) -> None: ), ) - if self._callback is not None: + # F8 fix: Invoke callback outside the lock and wrap in try/except. An + # observer exception must not prevent the child process from being + # reaped or cause resource leaks. The callback is invoked outside the + # lock since it may be slow and shouldn't block other operations. + if self._callback is not None: + try: self._callback(ActionState(self.state.value)) + except Exception as exc: + self._logger.error( + f"Exception in action callback: {exc}", + extra=LogExtraInfo( + openjd_log_content=LogContent.PROCESS_CONTROL | LogContent.EXCEPTION_INFO + ), + ) def _on_notify_period_end(self) -> None: """This is invoked when the grace period in a NOTIFY_THEN_TERMINATE diff --git a/src/openjd/sessions/_runner_env_script.py b/src/openjd/sessions/_runner_env_script.py index a3cbc042..235405a1 100644 --- a/src/openjd/sessions/_runner_env_script.py +++ b/src/openjd/sessions/_runner_env_script.py @@ -272,12 +272,14 @@ def _run_wrap_hook(self, hook: str, *, default_timeout: Optional[timedelta] = No def cancel( self, *, time_limit: Optional[timedelta] = None, mark_action_failed: bool = False ) -> None: - if self._action is None: - # Nothing to do. - return - - # Cancel with the effective method resolved at launch time by - # _run_action, against the action's own final scope (the script's - # lets and Env.File.* symbols, plus WrappedAction.* for a wrap - # hook) — openjd-rs parity. + # Always route through the base class handoff, even before _action is + # assigned. A cancel landing during environment-action setup (embedded- + # file writes, let evaluation) must be recorded in _pending_cancel and + # applied when the subprocess launches — F1 fix: the old `_action is + # None` guard returned early without entering the pending-cancel path, + # losing the cancel entirely during the setup window. + # + # _cancel_with_resolved_method handles the pre-launch case: it records + # the request in _pending_cancel when the subprocess hasn't started, + # and _run applies it once the child exists. self._cancel_with_resolved_method(time_limit, mark_action_failed) diff --git a/src/openjd/sessions/_session.py b/src/openjd/sessions/_session.py index 984c3db9..4b157b20 100644 --- a/src/openjd/sessions/_session.py +++ b/src/openjd/sessions/_session.py @@ -657,10 +657,18 @@ def cancel_action( """ if self.state != SessionState.RUNNING: raise RuntimeError("No actions are running") - # For the type checker - assert self._runner is not None + # Review22-F3 fix: Snapshot _runner before using it. The state check + # above and the _runner access below are not atomic; a completion + # racing this call could set _runner = None after we pass the state + # check. Snapshotting here avoids a bare AssertionError. + runner = self._runner + if runner is None: + # Race: action completed between state check and here. No-op rather + # than raise, since the caller's intent (cancel the running action) + # is already satisfied. + return - self._runner.cancel(time_limit=time_limit, mark_action_failed=mark_action_failed) + runner.cancel(time_limit=time_limit, mark_action_failed=mark_action_failed) def _make_env_script_runner( self, @@ -2238,6 +2246,17 @@ def _action_callback(self, state: ActionState) -> None: self._action_exit_code = self._runner.exit_code self._action_state = state + # F5 fix: Snapshot action_status BEFORE publishing READY. If we set + # _state = READY first, another thread polling session.state could see + # READY but action_status would still reflect the old (stale or + # incomplete) snapshot. By snapshotting here, the callback receives the + # definitive ActionStatus that corresponds to the terminal state. + # + # Note: We snapshot unconditionally (not guarded by `if self._callback`) + # because action_status is cheap and some tests check exact callback + # invocation patterns including the __bool__ check count. + action_status = self.action_status + if state != ActionState.RUNNING: # Decide which between-action state to enter. if self._ending_only or self._action_state != ActionState.SUCCESS: @@ -2247,10 +2266,7 @@ def _action_callback(self, state: ActionState) -> None: else: self._state = SessionState.READY - if self._callback: - action_status = self.action_status - # for the type checker - assert action_status is not None + if self._callback and action_status is not None: self._callback(self._session_id, action_status) def _evaluate_current_session_env_vars( diff --git a/src/openjd/sessions/_subprocess.py b/src/openjd/sessions/_subprocess.py index 3048872b..7ec1e58e 100644 --- a/src/openjd/sessions/_subprocess.py +++ b/src/openjd/sessions/_subprocess.py @@ -227,7 +227,11 @@ def notify(self) -> None: TODO: Send the signal to every direct and transitive child of the parent process. """ - if self._process is not None and self._process.poll() is None: + # Review22-F4 fix: Bind _process once. Double-load is a TOCTOU race: + # the None check and poll() call read _process separately, allowing + # another thread to set it to None between them. + proc = self._process + if proc is not None and proc.poll() is None: if is_posix(): self._posix_signal_subprocess(signal_name="term") else: @@ -243,15 +247,17 @@ def terminate(self) -> None: TODO: Send the signal to every direct and transitive child of the parent process. """ - if self._process is not None and self._process.poll() is None: + # Review22-F4 fix: Bind _process once. See notify() for rationale. + proc = self._process + if proc is not None and proc.poll() is None: if is_posix(): self._posix_signal_subprocess(signal_name="kill") else: self._logger.info( - f"INTERRUPT: Start killing the process tree with the root pid: {self._process.pid}", + f"INTERRUPT: Start killing the process tree with the root pid: {proc.pid}", extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), ) - kill_windows_process_tree(self._logger, self._process.pid, signal_subprocesses=True) + kill_windows_process_tree(self._logger, proc.pid, signal_subprocesses=True) def _start_subprocess(self) -> Optional[Popen]: """Helper invoked by self.run() to start up the subprocess.""" diff --git a/test/openjd/sessions_v0/test_concurrency_fixes.py b/test/openjd/sessions_v0/test_concurrency_fixes.py new file mode 100644 index 00000000..969b5d83 --- /dev/null +++ b/test/openjd/sessions_v0/test_concurrency_fixes.py @@ -0,0 +1,357 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Tests for concurrency fixes F1-F8 from PR #333 review findings. + +These tests verify the defensive behaviors added to close race windows. +True concurrency races are inherently non-deterministic, so these tests +focus on verifying the code paths and edge-case handling rather than +timing-dependent race reproduction. +""" + +import sys +import threading +from datetime import timedelta +from logging.handlers import QueueHandler +from pathlib import Path +from queue import SimpleQueue +from unittest.mock import MagicMock, patch + +import pytest + +from openjd.model import SymbolTable +from openjd.model.v2023_09 import ( + Action as Action_2023_09, + ArgString as ArgString_2023_09, + CommandString as CommandString_2023_09, + EnvironmentActions as EnvironmentActions_2023_09, + EnvironmentScript as EnvironmentScript_2023_09, +) +from openjd.sessions import ActionState +from openjd.sessions._runner_env_script import EnvironmentScriptRunner +from openjd.sessions._subprocess import LoggingSubprocess + +from .conftest import build_logger + + +@pytest.mark.usefixtures("message_queue", "queue_handler") +class TestF1EnvScriptCancelDuringSetup: + """F1: Cancel during environment script setup must be recorded as pending.""" + + def test_cancel_before_action_assignment_records_pending( + self, + tmp_path: Path, + message_queue: SimpleQueue, + queue_handler: QueueHandler, + python_exe: str, + ) -> None: + """Verify cancel() before _action is assigned records pending cancel.""" + # GIVEN: An EnvironmentScriptRunner with no _action yet + logger = build_logger(queue_handler) + script = EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onEnter=Action_2023_09( + command=CommandString_2023_09(python_exe), + args=[ArgString_2023_09("-c"), ArgString_2023_09("print('test')")], + ) + ) + ) + symtab = SymbolTable(source={}) + + runner = EnvironmentScriptRunner( + logger=logger, + session_working_directory=tmp_path, + environment_script=script, + symtab=symtab, + session_files_directory=tmp_path, + ) + + # Verify _action is None (before enter() or exit() is called) + assert runner._action is None + + # WHEN: cancel() is called before _action is set + runner.cancel(time_limit=timedelta(seconds=5), mark_action_failed=True) + + # THEN: The cancel is recorded as pending (not silently dropped) + assert runner._pending_cancel == (timedelta(seconds=5), True) + + runner.shutdown() + + +@pytest.mark.usefixtures("message_queue", "queue_handler") +class TestF3MonotonicMergePendingCancels: + """F3: Duplicate pending cancels must merge monotonically.""" + + def test_pending_cancel_merge_takes_minimum_time_limit( + self, + tmp_path: Path, + message_queue: SimpleQueue, + queue_handler: QueueHandler, + python_exe: str, + ) -> None: + """Verify multiple pending cancels merge with min(time_limit).""" + logger = build_logger(queue_handler) + script = EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onEnter=Action_2023_09( + command=CommandString_2023_09(python_exe), + args=[ArgString_2023_09("-c"), ArgString_2023_09("print('test')")], + ) + ) + ) + symtab = SymbolTable(source={}) + + runner = EnvironmentScriptRunner( + logger=logger, + session_working_directory=tmp_path, + environment_script=script, + symtab=symtab, + session_files_directory=tmp_path, + ) + + # WHEN: First cancel with 10 second limit + runner.cancel(time_limit=timedelta(seconds=10), mark_action_failed=False) + assert runner._pending_cancel == (timedelta(seconds=10), False) + + # AND: Second cancel with 5 second limit (tighter) + runner.cancel(time_limit=timedelta(seconds=5), mark_action_failed=False) + + # THEN: Merged result takes minimum time limit + assert runner._pending_cancel == (timedelta(seconds=5), False) + + runner.shutdown() + + def test_pending_cancel_merge_or_mark_action_failed( + self, + tmp_path: Path, + message_queue: SimpleQueue, + queue_handler: QueueHandler, + python_exe: str, + ) -> None: + """Verify multiple pending cancels OR the mark_action_failed flags.""" + logger = build_logger(queue_handler) + script = EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onEnter=Action_2023_09( + command=CommandString_2023_09(python_exe), + args=[ArgString_2023_09("-c"), ArgString_2023_09("print('test')")], + ) + ) + ) + symtab = SymbolTable(source={}) + + runner = EnvironmentScriptRunner( + logger=logger, + session_working_directory=tmp_path, + environment_script=script, + symtab=symtab, + session_files_directory=tmp_path, + ) + + # WHEN: First cancel without mark_action_failed + runner.cancel(time_limit=timedelta(seconds=10), mark_action_failed=False) + assert runner._pending_cancel == (timedelta(seconds=10), False) + + # AND: Second cancel with mark_action_failed=True + runner.cancel(time_limit=timedelta(seconds=15), mark_action_failed=True) + + # THEN: Merged result ORs the failed flags (once failed, always failed) + # and takes the minimum time limit + assert runner._pending_cancel == (timedelta(seconds=10), True) + + runner.shutdown() + + def test_pending_cancel_merge_none_beats_unlimited( + self, + tmp_path: Path, + message_queue: SimpleQueue, + queue_handler: QueueHandler, + python_exe: str, + ) -> None: + """Verify that a defined limit beats None (unlimited).""" + logger = build_logger(queue_handler) + script = EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onEnter=Action_2023_09( + command=CommandString_2023_09(python_exe), + args=[ArgString_2023_09("-c"), ArgString_2023_09("print('test')")], + ) + ) + ) + symtab = SymbolTable(source={}) + + runner = EnvironmentScriptRunner( + logger=logger, + session_working_directory=tmp_path, + environment_script=script, + symtab=symtab, + session_files_directory=tmp_path, + ) + + # WHEN: First cancel with unlimited time (None) + runner.cancel(time_limit=None, mark_action_failed=False) + assert runner._pending_cancel == (None, False) + + # AND: Second cancel with defined limit + runner.cancel(time_limit=timedelta(seconds=5), mark_action_failed=False) + + # THEN: Defined limit wins over unlimited + assert runner._pending_cancel == (timedelta(seconds=5), False) + + runner.shutdown() + + +@pytest.mark.usefixtures("message_queue", "queue_handler") +class TestF7SelfJoinDetection: + """F7: shutdown() must not deadlock when called from worker thread.""" + + def test_shutdown_from_worker_thread_uses_wait_false( + self, + tmp_path: Path, + message_queue: SimpleQueue, + queue_handler: QueueHandler, + python_exe: str, + ) -> None: + """Verify shutdown() detects self-join and uses wait=False.""" + logger = build_logger(queue_handler) + script = EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onEnter=Action_2023_09( + command=CommandString_2023_09(python_exe), + args=[ArgString_2023_09("-c"), ArgString_2023_09("print('test')")], + ) + ) + ) + symtab = SymbolTable(source={}) + + runner = EnvironmentScriptRunner( + logger=logger, + session_working_directory=tmp_path, + environment_script=script, + symtab=symtab, + session_files_directory=tmp_path, + ) + + # Simulate calling shutdown from the pool's worker thread + # by adding current thread to _threads set + threads = runner._pool._threads + if isinstance(threads, set): + threads.add(threading.current_thread()) + + # WHEN: shutdown() is called (would deadlock if wait=True) + with patch.object(runner._pool, "shutdown") as mock_shutdown: + runner.shutdown() + + # THEN: shutdown was called with wait=False + mock_shutdown.assert_called_once_with(wait=False) + + +@pytest.mark.usefixtures("message_queue", "queue_handler") +class TestF8ObserverExceptionHandling: + """F8: Observer callback exceptions must not discard live children.""" + + def test_callback_exception_is_caught_and_logged( + self, + tmp_path: Path, + message_queue: SimpleQueue, + queue_handler: QueueHandler, + python_exe: str, + ) -> None: + """Verify callback exceptions don't propagate from _on_process_exit.""" + logger = build_logger(queue_handler) + + # Create a callback that raises an exception + def bad_callback(state: ActionState) -> None: + raise RuntimeError("Observer exploded!") + + script = EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onEnter=Action_2023_09( + command=CommandString_2023_09(python_exe), + args=[ArgString_2023_09("-c"), ArgString_2023_09("print('test')")], + ) + ) + ) + symtab = SymbolTable(source={}) + + runner = EnvironmentScriptRunner( + logger=logger, + session_working_directory=tmp_path, + environment_script=script, + symtab=symtab, + session_files_directory=tmp_path, + callback=bad_callback, + ) + + # Simulate the state after a process has run + runner._process = MagicMock() + runner._process.is_running = False + mock_future = MagicMock() + mock_future.exception.return_value = None + runner._run_future = mock_future + + # WHEN: _on_process_exit is called (with bad callback) + # THEN: No exception should propagate + runner._on_process_exit(mock_future) + + # The runner should still be in a consistent state + runner.shutdown() + + +@pytest.mark.usefixtures("message_queue", "queue_handler") +@pytest.mark.skipif(sys.platform == "win32", reason="POSIX-specific signal handling") +class TestReview22F4DoubleLoadFix: + """Review22-F4: notify/terminate must bind _process once.""" + + def test_notify_binds_process_once( + self, + tmp_path: Path, + message_queue: SimpleQueue, + queue_handler: QueueHandler, + ) -> None: + """Verify notify() binds _process once to avoid TOCTOU race.""" + logger = build_logger(queue_handler) + + subprocess = LoggingSubprocess( + logger=logger, + args=["true"], + working_dir=str(tmp_path), + ) + + # Set up a mock process + mock_process = MagicMock() + mock_process.poll.return_value = None # Process is running + subprocess._process = mock_process + + # WHEN: notify() is called + with patch.object(subprocess, "_posix_signal_subprocess") as mock_signal: + subprocess.notify() + + # THEN: Signal was sent (process was accessed correctly) + mock_signal.assert_called_once_with(signal_name="term") + + def test_terminate_binds_process_once( + self, + tmp_path: Path, + message_queue: SimpleQueue, + queue_handler: QueueHandler, + ) -> None: + """Verify terminate() binds _process once to avoid TOCTOU race.""" + logger = build_logger(queue_handler) + + subprocess = LoggingSubprocess( + logger=logger, + args=["true"], + working_dir=str(tmp_path), + ) + + # Set up a mock process + mock_process = MagicMock() + mock_process.poll.return_value = None # Process is running + subprocess._process = mock_process + + # WHEN: terminate() is called + with patch.object(subprocess, "_posix_signal_subprocess") as mock_signal: + subprocess.terminate() + + # THEN: Signal was sent (process was accessed correctly) + mock_signal.assert_called_once_with(signal_name="kill") From 29ecbffe426c78a6c06bb278625fcdfb38cee796 Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Sat, 25 Jul 2026 21:45:54 -0700 Subject: [PATCH 13/27] fix(concurrency): Close round-4 review findings (R4-1, R4-5, R4-6, R4-G7, R4-G8) R4-1: Bound let RHS length before parse (MAX_LET_BINDING_LENGTH=4096) Prevents SIGBUS crash from parser stack overflow on malicious input. R4-5: Hoist _fail_action() call outside the runner lock Prevents deadlock when callback calls cancel(). R4-6: Isolate consumer-callback exceptions at terminal delivery Try/except around callback invocations in _fail_action() and _fail_action_before_start() so exceptions don't escape public API. R4-G7: Make failure attribution monotonic for live duplicate cancels Use 'or' merge for _notify_canceled_action_as_failed so earlier mark_action_failed=True isn't erased by subsequent cancels. R4-G8: Pass bound process snapshot through platform helpers notify()/terminate() pass proc to _posix_signal_subprocess() and _windows_notify_subprocess() to eliminate reload race. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- src/openjd/sessions/_runner_base.py | 113 +++++++++++++++--- src/openjd/sessions/_session.py | 14 ++- src/openjd/sessions/_subprocess.py | 49 ++++---- .../sessions_v0/test_concurrency_fixes.py | 14 +-- 4 files changed, 141 insertions(+), 49 deletions(-) diff --git a/src/openjd/sessions/_runner_base.py b/src/openjd/sessions/_runner_base.py index 91b5cd85..4574cb79 100644 --- a/src/openjd/sessions/_runner_base.py +++ b/src/openjd/sessions/_runner_base.py @@ -48,6 +48,7 @@ "resolve_effective_cancelation", "resolve_optional_int_field", "MAX_INT_FIELD_VALUE", + "MAX_LET_BINDING_LENGTH", "MAX_SCHEDULABLE_TIMEOUT_SECONDS", ) @@ -87,6 +88,30 @@ openjd-rs's ``Duration``-based timer effectively provides at that magnitude, so the action runs unbounded instead of failing. See _timeout_from_seconds.""" +MAX_LET_BINDING_LENGTH = 4096 +"""Largest ``let`` binding string this runtime will parse, in characters. + +Not a spec limit. openjd-model's expression parser recurses per nesting level +with no depth guard of its own, so a deeply nested RHS overflows the C stack +and kills the interpreter with SIGBUS — no exception, no log line, no cleanup. +A ``let`` RHS is the only template-controlled expression source that is parsed +*after* model validation accepts it (format strings are parsed during +model_validate), so it is the only one a submission-time validator cannot +screen; the bound therefore has to be applied here, at the last gate before +the parse. + +Measured on this branch: a 26,001-character RHS parses safely, a 30,001- +character one crashes. 4096 is deliberately an order of magnitude inside that +boundary rather than pinned to it, because the true limit is the platform's +thread stack size and is not portable. openjd-rs bounds the same risk by +parsing on a worker thread with an explicit PARSER_THREAD_STACK_SIZE +(openjd-expr parse.rs) and its conformance suite requires that neither +implementation *crash* on a too-deep expression (test_expression_depth.rs) — +rejecting cleanly is conformant. + +Remove this in favour of the engine's own guard once the openjd-model pin +floor carries the parser-stack fix; keep the regression test either way.""" + def _over_range_message(description: str, value: int) -> str: """Error text for a value that is a valid integer but too large to use. @@ -402,8 +427,20 @@ def apply_let_bindings(*, symtab: SymbolTable, let_bindings: list[str]) -> None: Raises: ValueError (FormatStringError/ExpressionError): if a binding's - expression cannot be evaluated. + expression cannot be evaluated, or if a binding is too long to + parse safely (see MAX_LET_BINDING_LENGTH). """ + # R4-1 fix: Guard against SIGBUS crash from parser stack overflow. Check + # length BEFORE calling evaluate_let_bindings, because the crash happens + # in native code with no exception — the process just dies. + for binding in let_bindings: + if len(binding) > MAX_LET_BINDING_LENGTH: + # Truncate the binding text in the error message, since it may be + # tens of KB — that's the whole point of rejecting it. + raise ValueError( + f"let binding {binding[:40]!r}... is {len(binding)} characters, " + f"which exceeds the maximum of {MAX_LET_BINDING_LENGTH}" + ) # Single-sourced in openjd.model (parse-memoized; skips malformed # bindings; raises ValueError naming the failing binding). evaluate_let_bindings(symtab=symtab, let_bindings=let_bindings) @@ -620,7 +657,19 @@ def _fail_action(self, message: str) -> None: ) self._state_override = ScriptRunnerState.FAILED if self._callback is not None: - self._callback(ActionState.FAILED) + # R4-6 fix: Isolate consumer-callback exceptions. Same pattern as + # the completion callback in _on_process_exit. A consumer that + # raises must not turn a handled template failure into an exception + # escaping the public Session API. + try: + self._callback(ActionState.FAILED) + except Exception as exc: + self._logger.error( + f"Exception in action callback: {exc}", + extra=LogExtraInfo( + openjd_log_content=LogContent.PROCESS_CONTROL | LogContent.EXCEPTION_INFO + ), + ) @abstractmethod def cancel( @@ -677,6 +726,12 @@ def exit_code(self) -> Optional[int]: return None def _run(self, args: Sequence[str], time_limit: Optional[timedelta] = None) -> None: + # R4-5 fix: Track launch failure outside the lock, dispatch after. + # _fail_action invokes the user callback synchronously, and a callback + # that calls cancel() would deadlock on self._lock (since + # _cancel_with_resolved_method acquires it unconditionally). No callback + # may run under self._lock — it is a plain Lock, not an RLock. + launch_failure: Optional[str] = None with self._lock: if self.state != ScriptRunnerState.READY: raise RuntimeError("This cannot be used to run a second subprocess.") @@ -706,25 +761,38 @@ def _run(self, args: Sequence[str], time_limit: Optional[timedelta] = None) -> N args, self._user, self._os_env_vars, str(self._session_working_directory) ) except RuntimeError as e: - self._fail_action(str(e)) - return + # Record failure, don't dispatch yet — we're under the lock. + launch_failure = str(e) + + # Only proceed with subprocess creation if no launch failure occurred + if launch_failure is None: + subprocess_args = [filename] if is_posix() else args + self._process = LoggingSubprocess( + logger=self._logger, + args=subprocess_args, + user=self._user, + os_env_vars=self._os_env_vars, + working_dir=str(self._session_working_directory), + ) - subprocess_args = [filename] if is_posix() else args - self._process = LoggingSubprocess( - logger=self._logger, - args=subprocess_args, - user=self._user, - os_env_vars=self._os_env_vars, - working_dir=str(self._session_working_directory), - ) + if time_limit: + self._runtime_limit = Timer(time_limit.total_seconds(), self._on_timelimit) + self._runtime_limit.start() - if time_limit: - self._runtime_limit = Timer(time_limit.total_seconds(), self._on_timelimit) - self._runtime_limit.start() + if self._print_section_banner: + log_subsection_banner(self._logger, "Phase: Running action") + self._run_future = self._pool.submit(self._process.run) + + # Dispatch launch failure outside the lock + if launch_failure is not None: + self._fail_action(launch_failure) + return + + # At this point, launch succeeded so _process and _run_future are set. + # Assert for the type checker. + assert self._run_future is not None + assert self._process is not None - if self._print_section_banner: - log_subsection_banner(self._logger, "Phase: Running action") - self._run_future = self._pool.submit(self._process.run) # Intentionally leave the lock section. If the process was *really* fast, # then it's possible for the future to have finished before we get to add # the done-callback. That results in the done-callback being called from @@ -1003,7 +1071,14 @@ def _cancel( return self._canceled = True - self._notify_canceled_action_as_failed = mark_action_failed + # R4-G7 fix (F3 live path): Monotonic merge for failure attribution. + # If a previous cancel already set mark_action_failed=True (e.g., a + # parse failure triggered cancel), a subsequent timeout or manual + # cancel must not erase that determination. The action should report + # FAILED, not TIMEOUT/CANCELED. + self._notify_canceled_action_as_failed = ( + self._notify_canceled_action_as_failed or mark_action_failed + ) now = datetime.now(timezone.utc) now_str = now.strftime(TIME_FORMAT_STR) if self._cancel_gracetime_timer is not None: diff --git a/src/openjd/sessions/_session.py b/src/openjd/sessions/_session.py index 4b157b20..c9677959 100644 --- a/src/openjd/sessions/_session.py +++ b/src/openjd/sessions/_session.py @@ -2226,7 +2226,19 @@ def _fail_action_before_start(self, message: str) -> None: action_status = self.action_status # for the type checker assert action_status is not None - self._callback(self._session_id, action_status) + # R4-6 fix: Isolate consumer-callback exceptions. Same pattern as + # _fail_action in ScriptRunnerBase and _on_process_exit. A consumer + # that raises must not turn a handled resolution failure into an + # exception escaping the public Session API. + try: + self._callback(self._session_id, action_status) + except Exception as exc: + self._logger.error( + f"Exception in session callback: {exc}", + extra=LogExtraInfo( + openjd_log_content=LogContent.PROCESS_CONTROL | LogContent.EXCEPTION_INFO + ), + ) def _action_callback(self, state: ActionState) -> None: """This callback is invoked: diff --git a/src/openjd/sessions/_subprocess.py b/src/openjd/sessions/_subprocess.py index 7ec1e58e..2f251f8e 100644 --- a/src/openjd/sessions/_subprocess.py +++ b/src/openjd/sessions/_subprocess.py @@ -233,9 +233,11 @@ def notify(self) -> None: proc = self._process if proc is not None and proc.poll() is None: if is_posix(): - self._posix_signal_subprocess(signal_name="term") + # R4-G8 fix: Pass the bound proc to helpers to avoid reloading + # self._process, which could be None by the time the helper runs. + self._posix_signal_subprocess(proc, signal_name="term") else: - self._windows_notify_subprocess() + self._windows_notify_subprocess(proc) def terminate(self) -> None: """The 'Terminate' part of Open Job Description's subprocess cancelation method. @@ -251,7 +253,8 @@ def terminate(self) -> None: proc = self._process if proc is not None and proc.poll() is None: if is_posix(): - self._posix_signal_subprocess(signal_name="kill") + # R4-G8 fix: Pass the bound proc to helpers. + self._posix_signal_subprocess(proc, signal_name="kill") else: self._logger.info( f"INTERRUPT: Start killing the process tree with the root pid: {proc.pid}", @@ -470,21 +473,21 @@ def _tosigned(n: int) -> int: def _posix_signal_subprocess( self, + process: Popen, signal_name: Literal["term", "kill"], ) -> None: - """Send a given named signal to the subprocess.""" + """Send a given named signal to the subprocess. + + Args: + process: The Popen object to signal. Passed from caller to avoid + reloading self._process, which could be None by now (R4-G8 fix). + signal_name: Either "term" for SIGTERM or "kill" for SIGKILL. + """ # Hint to mypy to not raise module attribute errors (e.g. missing os.getpgid) if sys.platform == "win32": raise NotImplementedError("This method is for POSIX hosts only") - # We can run into a race condition where the process exits (and another thread sets self._process to None) - # before the cancellation happens, so we swap to a local variable to ensure a cancellation that is not needed, - # does not raise an exception here. - process = self._process - # Convince the type checker that accessing process is okay - assert process is not None - # Note: A limitation of this implementation is that it will only sigkill # processes that are in the same process-group as the command that we ran. # In the future, we can extend this to killing all processes spawned (including into @@ -614,11 +617,13 @@ def _log_process_tree(self) -> None: extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), ) - def _windows_notify_subprocess(self) -> None: - """Sends a CTRL_BREAK_EVENT signal to the subprocess""" - # Convince the type checker that accessing _process is okay - assert self._process is not None + def _windows_notify_subprocess(self, process: Popen) -> None: + """Sends a CTRL_BREAK_EVENT signal to the subprocess. + Args: + process: The Popen object to signal. Passed from caller to avoid + reloading self._process, which could be None by now (R4-G8 fix). + """ # CTRL-C handler is disabled by default when CREATE_NEW_PROCESS_GROUP is passed. # We send CTRL-BREAK as handler for it cannnot be disabled. # https://learn.microsoft.com/en-us/windows/console/ctrl-c-and-ctrl-break-signals @@ -626,18 +631,18 @@ def _windows_notify_subprocess(self) -> None: # https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-createprocessa#remarks # https://stackoverflow.com/questions/35772001/how-to-handle-a-signal-sigint-on-a-windows-os-machine/35792192#35792192 self._logger.info( - f"INTERRUPT: Sending CTRL_BREAK_EVENT to {self._process.pid}", + f"INTERRUPT: Sending CTRL_BREAK_EVENT to {process.pid}", extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), ) - # _process will be running in new console, we run another process to attach to it and send signal + # process will be running in new console, we run another process to attach to it and send signal cmd = [ # When running in a service context, we want to call the non-service Python binary sys.executable.lower().replace("pythonservice.exe", "python.exe"), str(WINDOWS_SIGNAL_SUBPROC_SCRIPT_PATH), - str(self._process.pid), + str(process.pid), ] - process = LoggingSubprocess( + signal_subprocess = LoggingSubprocess( logger=self._logger, args=cmd, encoding=self._encoding, @@ -648,11 +653,11 @@ def _windows_notify_subprocess(self) -> None: ) # Blocking call - process.run() + signal_subprocess.run() - if process.exit_code != 0: + if signal_subprocess.exit_code != 0: self._logger.warning( - f"Failed to send signal 'CTRL_BREAK_EVENT' to subprocess {self._process.pid}", + f"Failed to send signal 'CTRL_BREAK_EVENT' to subprocess {process.pid}", extra=LogExtraInfo( openjd_log_content=LogContent.PROCESS_CONTROL | LogContent.EXCEPTION_INFO ), diff --git a/test/openjd/sessions_v0/test_concurrency_fixes.py b/test/openjd/sessions_v0/test_concurrency_fixes.py index 969b5d83..b26aa7ed 100644 --- a/test/openjd/sessions_v0/test_concurrency_fixes.py +++ b/test/openjd/sessions_v0/test_concurrency_fixes.py @@ -300,7 +300,7 @@ def bad_callback(state: ActionState) -> None: @pytest.mark.usefixtures("message_queue", "queue_handler") @pytest.mark.skipif(sys.platform == "win32", reason="POSIX-specific signal handling") class TestReview22F4DoubleLoadFix: - """Review22-F4: notify/terminate must bind _process once.""" + """Review22-F4: notify/terminate must bind _process once and pass to helpers.""" def test_notify_binds_process_once( self, @@ -308,7 +308,7 @@ def test_notify_binds_process_once( message_queue: SimpleQueue, queue_handler: QueueHandler, ) -> None: - """Verify notify() binds _process once to avoid TOCTOU race.""" + """Verify notify() binds _process once and passes to helper (R4-G8 fix).""" logger = build_logger(queue_handler) subprocess = LoggingSubprocess( @@ -326,8 +326,8 @@ def test_notify_binds_process_once( with patch.object(subprocess, "_posix_signal_subprocess") as mock_signal: subprocess.notify() - # THEN: Signal was sent (process was accessed correctly) - mock_signal.assert_called_once_with(signal_name="term") + # THEN: Signal helper was called with the bound process (R4-G8) + mock_signal.assert_called_once_with(mock_process, signal_name="term") def test_terminate_binds_process_once( self, @@ -335,7 +335,7 @@ def test_terminate_binds_process_once( message_queue: SimpleQueue, queue_handler: QueueHandler, ) -> None: - """Verify terminate() binds _process once to avoid TOCTOU race.""" + """Verify terminate() binds _process once and passes to helper (R4-G8 fix).""" logger = build_logger(queue_handler) subprocess = LoggingSubprocess( @@ -353,5 +353,5 @@ def test_terminate_binds_process_once( with patch.object(subprocess, "_posix_signal_subprocess") as mock_signal: subprocess.terminate() - # THEN: Signal was sent (process was accessed correctly) - mock_signal.assert_called_once_with(signal_name="kill") + # THEN: Signal helper was called with the bound process (R4-G8) + mock_signal.assert_called_once_with(mock_process, signal_name="kill") From 4c57f405f03991cb311f82d46721e4d868020f75 Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Sun, 26 Jul 2026 00:11:30 -0700 Subject: [PATCH 14/27] fix: Close round-5 review findings R5-1 through R5-9 Implement all nine round-5 findings, plus the sibling occurrences of each found by sweeping the codebase for the same defect class. Log redaction and the action filter (_action_filter.py): - R5-1: clear record.args unconditionally. The redaction logic only inspects record.msg, but a handler calls getMessage(), which re-runs `msg % args` -- so a record leaving the filter with args populated re-interpolated an unscanned secret into the emitted line. On the format-failure path the args are now folded into msg textually rather than dropped, because a record whose own formatting is broken would otherwise raise inside the handler and logging would dump the raw args to stderr, outside this filter's reach. - R5-2: catch Exception, not just ValueError. `handler` invokes the consumer-supplied callback, and any other exception type escaped filter() into the stdout pump thread -- killing the pump, dropping the rest of the subprocess's output, and leaving the child unreaped. Adds _invoke_callback() for the one callback not routed through `handler`, and makes the redaction step itself fail closed. - Anchor the env var NAME regexes with \Z instead of $. `$` also matches before a trailing newline, so "FOO\n" validated as a name. Values stay permissive on purpose: multi-line values via the JSON form are supported, tested behaviour. Temporary directories (_tempdir.py, _session.py): - R5-3: /OpenJD is a fixed, predictable path whose parent is world-writable on typical POSIX hosts, and makedirs(exist_ok=True) accepted whatever was already there -- another local user's directory, or a symlink. Validate ownership with lstat and refuse anything that is not a real directory owned by this euid or root. makedirs()' mode is umask-masked, so the mode is now also set outright; without it, a hostile umask leaves the root untraversable by the job user. Sibling: Session._openjd_session_root_dir() was a second creator of the same path with the mode constant duplicated. Deleted -- custom_gettempdir() now owns create, validate and chmod, which also covers TempDir(dir=None). - R5-7: keep the exception in TempDir.cleanup(). It was accepted and discarded, so failures listed bare paths with no cause on the one path where "permission denied" versus "still held open" changes what to do next. Uses onexc on 3.12+, onerror below that. - R5-8: document the trust precondition behind the deliberate 0o770 widening. The grant is to a group, so every member of it can modify the session tree. Narrowing it would remove cross-user impersonation rather than harden it. Generated shell script (_runner_base.py): - R5-4: shlex.quote the `cd` path. A single quote in the path closed the hand-written quoted region and the remainder was interpreted by /bin/sh. - R5-5: validate env var names before emitting export/unset. The value was quoted but the name was not, so a name containing `;` or `$(` injected commands. Names that are not POSIX shell identifiers are skipped with a warning rather than rejected: rejecting would break Windows embedders (ProgramFiles(x86) is a real name), escaping is impossible, and the variable still reaches the subprocess through Popen's env=, which uses no shell. Previously such a name produced an outright /bin/sh syntax error that failed the whole action. Invariants under `python -O` (six files): - R5-6: convert the invariant-bearing asserts to explicit raises, leaving pure type-narrowing ones alone. Triaged on whether the check is in a public API, runs on a background thread where an AssertionError reaches only threading.excepthook, or *is* an earlier fix. Notably ScriptRunnerBase.state asserted on a reachable inconsistency, making the runner permanently unreadable after a failed submit, and the two asserts added by the R4-5 fix silently reverted that fix under -O. Also binds LoggingSubprocess.exit_code once; the double-load raised AttributeError out of a public property when read cross-thread while run()'s finally cleared _process. - Records the assert-versus-raise policy in DEVELOPMENT.md. Process groups (_subprocess.py, _linux/_sudo.py): - R5-9: record an unknown process group as None rather than substituting the dead child's pid. None is already this field's "unknown" value, which _posix_signal_subprocess() and find_sudo_child_process_group_id() both use; _subprocess.py was the outlier, and in the sudo branch the pid is not even the right kind of identifier. The behaviour the fallback existed to protect (an immediately-exiting child must not fail the action) is preserved. Sibling: guard the first getpgid in find_sudo_child_process_group_id, which let ESRCH escape to callers only looking for a signal target. Adds test/openjd/sessions_v0/test_round5_fixes.py: 60 tests, each mutation-checked by reverting its fix and confirming the test fails. Verified: 784 passed / 39 skipped / 16 xfailed (only the 2 known xdist timing flakes fail, and pass in isolation); ruff, mypy and black clean; conformance 2023-09 at 1162 passed / 0 failed. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- DEVELOPMENT.md | 44 + src/openjd/sessions/_action_filter.py | 120 ++- src/openjd/sessions/_embedded_files.py | 29 +- src/openjd/sessions/_linux/_sudo.py | 14 +- src/openjd/sessions/_runner_base.py | 100 ++- src/openjd/sessions/_session.py | 82 +- src/openjd/sessions/_subprocess.py | 52 +- src/openjd/sessions/_tempdir.py | 160 +++- test/openjd/sessions_v0/test_round5_fixes.py | 827 +++++++++++++++++++ 9 files changed, 1344 insertions(+), 84 deletions(-) create mode 100644 test/openjd/sessions_v0/test_round5_fixes.py diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 9d6ad700..4c022a57 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -354,3 +354,47 @@ try: except ValueError as e: # Error handling... ``` +#### Use of `assert` in Runtime Code + +This package can legitimately be run under `python -O`, which strips every `assert` +statement from the bytecode. So the choice between `assert` and `raise` decides +whether a check exists in production at all. + +Use `assert` **only** for type-checker narrowing: a condition that the type system +cannot see but that is structurally guaranteed by the caller, where losing the check +costs nothing but a slightly less legible `AttributeError` on the following line. +Narrowing a schema-version union immediately after the caller has already selected +that version is the canonical example. + +Use an explicit `raise` for anything that is a *runtime invariant* — a condition that +could actually be false because of a race, a partially-completed construction, an +unexpected model shape, or a caller error. Concretely, prefer `raise` when any of +these is true: + +- The check is in a **public** method or property. A stripped assert turns a clear + library error into `AttributeError: 'NoneType' object has no attribute ...` raised + from inside this package. +- The code runs on a **background thread** — a pool worker, a `threading.Timer` + callback, or the stdout-forwarding path. An `AssertionError` there reaches only + `threading.excepthook`, so the failure is invisible and the surrounding work + (waiting on and reaping the subprocess, arming a grace timer) is silently skipped. +- The assert **is** the fix. Several checks in `_runner_base.py` exist specifically to + hold an invariant that a bug-fix introduced. Under `-O` such an assert reverts the + fix without changing any test result. +- Something **downstream would proceed on bad data** rather than stop, for example + materializing an embedded file from an object of an unrecognized schema. + +When a `raise` would be noisy in an already-defensive spot, prefer binding the value +once and handling `None` explicitly (return early, log, skip the notification) over +either an assert or a raise. That keeps the failure legible and the process owned. + +Both forms narrow types for mypy, so there is no typing cost to choosing `raise`: + +```py +# Narrowing only — fine as an assert. +assert isinstance(script, StepScript_2023_09) + +# A runtime invariant — must survive -O. +if not isinstance(file, EmbeddedFileText_2023_09): + raise RuntimeError(f"Unsupported embedded file model '{type(file).__name__}'.") +``` diff --git a/src/openjd/sessions/_action_filter.py b/src/openjd/sessions/_action_filter.py index 9304c94e..6c78e3df 100644 --- a/src/openjd/sessions/_action_filter.py +++ b/src/openjd/sessions/_action_filter.py @@ -78,13 +78,28 @@ class ActionMessageKind(Enum): openjd_env_actions_filter_matcher = re.compile(openjd_env_actions_filter_regex) # A regex for matching the assignment of a value to an environment variable -envvar_set_regex_str = "^[A-Za-z_][A-Za-z0-9_]*" "=" ".*$" # Variable name -envvar_set_regex_json = '^(")?[A-Za-z_][A-Za-z0-9_]*' "=" ".*$" # Variable name +# Note on the anchors: these use \Z, not $. `$` also matches immediately before +# a trailing newline, so "FOO\n" satisfies an `$`-anchored name pattern and this +# runtime would go on to unset a variable whose name contains a newline -- a name +# no operating system can hold. \Z matches only at the true end of the string. +# The VALUE half deliberately stays permissive: a multi-line value delivered +# through the JSON form (openjd_env: "FOO=a\nb") is supported behaviour, so the +# separator hygiene a value needs belongs at each serialization boundary, not +# here. +envvar_set_regex_str = "^[A-Za-z_][A-Za-z0-9_]*" "=" ".*\\Z" # Variable name +envvar_set_regex_json = '^(")?[A-Za-z_][A-Za-z0-9_]*' "=" ".*\\Z" # Variable name envvar_set_matcher_str = re.compile(envvar_set_regex_str) envvar_set_matcher_json = re.compile(envvar_set_regex_json) -envvar_unset_regex = "^[A-Za-z_][A-Za-z0-9_]*$" +envvar_unset_regex = "^[A-Za-z_][A-Za-z0-9_]*\\Z" envvar_unset_matcher = re.compile(envvar_unset_regex) +# The set of characters that can never legally appear in an environment variable +# NAME, and that would change the meaning of any line-based or NUL-delimited +# rendering of one. A name is validated against envvar_*_regex above before it +# reaches a callback, but the JSON form is decoded AFTER that match, so the +# decoded name is re-checked with this. +_ENVVAR_NAME_FORBIDDEN = ("\n", "\r", "\0", "=") + # This is a reworking/merging of TaskStatusFilter and FailureFilter class ActionMonitoringFilter(logging.Filter): @@ -191,7 +206,7 @@ def _redactions_enabled(self) -> bool: or "REDACTED_ENV_VARS" in self._revision_extensions.extensions ) - def apply_message_redaction(self, record: logging.LogRecord): + def apply_message_redaction(self, record: logging.LogRecord) -> None: """Redact the log message if it contains any substrings which have been registered for redaction Args: @@ -200,21 +215,36 @@ def apply_message_redaction(self, record: logging.LogRecord): # Check if we need to redact any sensitive values from the log message if (self._redacted_values or self._redacted_lines) and isinstance(record.msg, str): - # If we have args, first do string formatting, then redact - try: - record.msg = record.msg % record.args - record.args = () # Clear args since we've done the formatting - except Exception: - # If string formatting fails, fall back to just redacting the message - LOG.warning( - "Failed to format log message for redaction. Proceeding with redaction on unformatted message." - ) + # Fold `args` into `msg` and clear them UNCONDITIONALLY before + # scanning (R5-1 fix). The redaction below only ever inspects + # `record.msg`, and a downstream handler calls record.getMessage(), + # which re-runs `msg % args`. So any path that leaves `args` + # populated re-interpolates the un-scanned original into the emitted + # line -- a secret carried in `args` would be emitted verbatim. The + # `finally` is the load-bearing part: `args` must end up empty on the + # failure path too, not just on the success path. + if record.args: + try: + record.msg = record.msg % record.args + except Exception: + # The record's own formatting is broken, so getMessage() + # would raise inside the handler and Python's logging would + # dump the raw args to stderr, outside this filter's reach. + # Fold them in textually instead: the record stays emittable + # AND every byte of it goes through the scan below. + record.msg = f"{record.msg} {record.args!r}" + LOG.warning( + "Failed to format log message for redaction. Redacting the message with its " + "arguments appended instead.", + extra=LogExtraInfo(openjd_log_content=LogContent.COMMAND_OUTPUT), + ) + finally: + record.args = () # Check if the entire message matches a line in the redacted_lines set if record.msg in self._redacted_lines: record.msg = "*" * 8 - record.args = () - return True + return # Find all segments that need redaction segments_to_redact = [] @@ -254,7 +284,6 @@ def apply_message_redaction(self, record: logging.LogRecord): for start, end in reversed(merged_segments): msg_chars[start:end] = list("*" * 8) # Always use 8 asterisks for redaction record.msg = "".join(msg_chars) - record.args = () def filter(self, record: logging.LogRecord) -> bool: """Called automatically by Python's logging subsystem when a log record @@ -323,6 +352,22 @@ def filter(self, record: logging.LogRecord) -> bool: record.msg = record.msg + f" -- ERROR: {str(e)}" # There was an error. Don't suppress the message from the log. return True + except Exception as e: + # R5-2 fix: `handler` invokes the consumer-supplied callback, + # which may raise anything. Anything other than ValueError + # used to propagate out of filter(), into Python's logging + # machinery, and into the stdout pump thread that called + # logger.info() -- killing the pump, silently dropping the + # rest of the subprocess's output, and leaving the child + # unreaped. Same isolation as the Session's terminal + # callbacks; keep the record in the log so the failure is + # visible in the action's own output. + LOG.error( + f"Open Job Description: Exception in the action message callback: {e}", + extra=LogExtraInfo(openjd_log_content=LogContent.EXCEPTION_INFO), + ) + record.msg = record.msg + f" -- ERROR: {str(e)}" + return True return not self._suppress_filtered # Check for "almost" matching openjd_env and openjd_unset_env commands @@ -335,13 +380,44 @@ def filter(self, record: logging.LogRecord) -> bool: record.msg = record.msg + f" -- ERROR: {err_message}" # Callback to cancel the action and mark it as FAILED - self._callback(ActionMessageKind.FAIL, err_message, True) + # R5-2 fix: isolated for the same reason as the handler call + # above -- this is the one consumer-callback invocation in this + # method that is not routed through `handler`. + self._invoke_callback(ActionMessageKind.FAIL, err_message, True) return True return True finally: - # Always check for redaction before returning - self.apply_message_redaction(record) + # Always check for redaction before returning. Fail closed: if the + # redaction control itself raises, blank the message rather than + # letting an unscanned record through (and rather than letting the + # exception reach the stdout pump thread). + try: + self.apply_message_redaction(record) + except Exception as e: # pragma: nocover - defensive + record.msg = "*" * 8 + record.args = () + LOG.error( + f"Open Job Description: Log redaction failed; message suppressed: {e}", + extra=LogExtraInfo(openjd_log_content=LogContent.EXCEPTION_INFO), + ) + + def _invoke_callback(self, kind: ActionMessageKind, value: Any, cancel_and_fail: bool) -> None: + """Invoke the consumer-supplied callback, isolating any exception it + raises (R5-2). + + This filter runs on the thread that is forwarding the subprocess's stdout + to the log, so an exception escaping here unwinds + ``LoggingSubprocess.run()`` before the child is waited on. Consumer bugs + must not cost us the output stream or process ownership. + """ + try: + self._callback(kind, value, cancel_and_fail) + except Exception as e: + LOG.error( + f"Open Job Description: Exception in the action message callback: {e}", + extra=LogExtraInfo(openjd_log_content=LogContent.EXCEPTION_INFO), + ) def _handle_progress(self, message: str) -> None: """Local handling of Progress messages. Processes the message and then @@ -426,6 +502,12 @@ def _parse_env_variable(self, message: str) -> tuple[str, str, bool, int, Option raise ValueError( f"Unterminated string starting at: line {e.lineno} column {e.colno} (char {e.pos})" ) + # Re-check the NAME after decoding. The regex above matched the raw + # message, so for the JSON form it validated the escaped text; JSON + # decoding happens afterwards and could in principle yield a name + # this runtime should never hand onwards. + if any(c in name for c in _ENVVAR_NAME_FORBIDDEN): + return "", "", False, equals_position, None return name, value, True, equals_position, original_value except json.JSONDecodeError as e: raise ValueError( diff --git a/src/openjd/sessions/_embedded_files.py b/src/openjd/sessions/_embedded_files.py index 1af03c9f..29ce6cb9 100644 --- a/src/openjd/sessions/_embedded_files.py +++ b/src/openjd/sessions/_embedded_files.py @@ -66,6 +66,26 @@ def _convert_line_endings(data: str, end_of_line: Optional[str]) -> str: return data +def _unsupported_embedded_file_model(file: EmbeddedFileType) -> RuntimeError: + """The error for an embedded file from a schema this class does not implement. + + Returned rather than raised so that call sites can keep the + ``if not isinstance(...): raise`` shape, which narrows the type for mypy + exactly as an ``assert isinstance`` did while still being enforced under + ``python -O`` (R5-6). These three checks are genuine runtime invariants, not + type-checker narrowing: with them stripped, the call sites would read + ``.filename``/``.data``/``.runnable`` off an object of an unknown shape and + materialize a file from whatever they happened to find, instead of stopping. + + When a new schema version is added, this is the seam that grows the + per-version dispatch. + """ + return RuntimeError( + f"Unsupported embedded file model '{type(file).__name__}'. This runtime implements the " + "2023-09 schema only." + ) + + def _validate_embedded_filename(filename: str) -> None: """Validate that an embedded file's ``filename`` is a single path component (a basename) with no directory pathing, as required by the OpenJD @@ -318,7 +338,8 @@ def _find_value_prefix(self, file: EmbeddedFileType) -> str: """ # When adding a new schema, start this method with a check for which # model 'file' belongs to -- that'll tell us the schema version. - assert isinstance(file, EmbeddedFileText_2023_09) + if not isinstance(file, EmbeddedFileText_2023_09): + raise _unsupported_embedded_file_model(file) if self._scope == EmbeddedFilesScope.ENV: return ValueReferenceConstants_2023_09.ENV_FILE_PREFIX.value @@ -339,7 +360,8 @@ def _get_symtab_entry(self, file: EmbeddedFileType) -> tuple[str, Path]: value - The absolute filename of the file to manifest. """ - assert isinstance(file, EmbeddedFileText_2023_09) + if not isinstance(file, EmbeddedFileText_2023_09): + raise _unsupported_embedded_file_model(file) # Figure out what filename to use for the given embedded file. # This will either be provided in the given 'file' or we will @@ -379,7 +401,8 @@ def _materialize_file( Make the file executable if the file settings indicate that we should. """ - assert isinstance(file, EmbeddedFileText_2023_09) + if not isinstance(file, EmbeddedFileText_2023_09): + raise _unsupported_embedded_file_model(file) execute_permissions = 0 if file.runnable: diff --git a/src/openjd/sessions/_linux/_sudo.py b/src/openjd/sessions/_linux/_sudo.py index 43a1fa8c..b52dbdb4 100644 --- a/src/openjd/sessions/_linux/_sudo.py +++ b/src/openjd/sessions/_linux/_sudo.py @@ -44,7 +44,19 @@ def find_sudo_child_process_group_id( # send SIGKILL signals to the subprocess of sudo start = time.monotonic() now = start - sudo_pgid = os.getpgid(sudo_process.pid) + try: + sudo_pgid = os.getpgid(sudo_process.pid) + except ProcessLookupError: + # Sibling of the same guard in LoggingSubprocess.run(): `sudo` itself has + # already been reaped, so there is no group to find. Return the "unknown" + # value this function already uses for a child that exits mid-scan + # (below) rather than letting ESRCH escape to a caller that is only + # looking for a signal target. + logger.debug( + f"sudo process {sudo_process.pid} exited before its process group could be determined.", + extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), + ) + return None # Repeatedly scan for child processes # diff --git a/src/openjd/sessions/_runner_base.py b/src/openjd/sessions/_runner_base.py index 4574cb79..4d3e78ab 100644 --- a/src/openjd/sessions/_runner_base.py +++ b/src/openjd/sessions/_runner_base.py @@ -50,9 +50,19 @@ "MAX_INT_FIELD_VALUE", "MAX_LET_BINDING_LENGTH", "MAX_SCHEDULABLE_TIMEOUT_SECONDS", + "POSIX_SHELL_NAME_RE", ) +POSIX_SHELL_NAME_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") +"""The only shape a POSIX shell variable name may take (POSIX.1-2017 §3.235). + +Used to gate what ``_generate_command_shell_script`` is willing to emit an +``export``/``unset`` line for. Deliberately identical to the name grammar +``_action_filter`` enforces on ``openjd_env`` messages, so a variable defined by +a job's stdout and one supplied by the embedder are held to the same standard. +""" + _STRICT_INT_RE = re.compile(r"[+-]?[0-9]+") """The exact integer grammar accepted for dynamically resolved integer fields (FEATURE_BUNDLE_1 ``timeout`` / ``notifyPeriodInSeconds`` format strings): @@ -695,8 +705,19 @@ def state(self) -> ScriptRunnerState: return ScriptRunnerState.READY # Check on the state of the future for done/canceled first # If the future is done, then we have a terminal state. - assert self._run_future is not None # For the type checker - if self._run_future.done(): + # + # R5-6: this is a reachable invariant, not type-checker narrowing, so it + # is a plain read rather than an `assert`. `_run` assigns `_process` + # before submitting the future, so a submit that fails leaves the pair + # inconsistent -- and this is a *property*, on the path every consumer + # polls, where an AssertionError (or, under `python -O`, an + # AttributeError) makes the runner permanently unreadable. A launched + # process with no future has not started running, so report READY and let + # the caller's own error handling deal with the failed launch. + run_future = self._run_future + if run_future is None: # pragma: no cover - launch failed after _process was set + return ScriptRunnerState.READY + if run_future.done(): if self._canceled and self._notify_canceled_action_as_failed: return ScriptRunnerState.FAILED if self._canceled and self._runtime_limit_reached: @@ -789,9 +810,16 @@ def _run(self, args: Sequence[str], time_limit: Optional[timedelta] = None) -> N return # At this point, launch succeeded so _process and _run_future are set. - # Assert for the type checker. - assert self._run_future is not None - assert self._process is not None + # R5-6: explicit raises, not asserts. The R4-5 restructuring above depends + # on these two being set together whenever `launch_failure is None`, and + # under `python -O` an `assert` here would strip that check and let the + # method carry on to `add_done_callback`/`wait_until_started` on None -- + # silently reverting the fix this block exists to implement. + if self._run_future is None or self._process is None: # pragma: no cover - defensive + raise RuntimeError( + "Internal error: the action's subprocess was not created despite a successful " + "launch." + ) # Intentionally leave the lock section. If the process was *really* fast, # then it's possible for the future to have finished before we get to add @@ -822,19 +850,49 @@ def _run(self, args: Sequence[str], time_limit: Optional[timedelta] = None) -> N self._callback(ActionState.RUNNING) def _generate_command_shell_script(self, args: Sequence[str]) -> str: - """Generate a shell script for running a command given by the args.""" + """Generate a shell script for running a command given by the args. + + Everything interpolated into this script is quoted or validated, because + the result is ``exec``'d by ``/bin/sh``: a single unquoted metacharacter + anywhere in it is arbitrary code execution as the session user. + """ script = list[str]() script.append("#!/bin/sh\n") if self._os_env_vars: for name, value in self._os_env_vars.items(): + if not POSIX_SHELL_NAME_RE.fullmatch(name): + # R5-5 fix: the value was already quoted, but the NAME was + # interpolated raw -- so a name containing ';', '$(' or a + # newline injected commands here. + # + # Skipped rather than escaped or rejected. A shell variable + # name cannot legally contain a metacharacter, so there is + # nothing to escape *to*: `export 'a;b'=v` is not a valid + # assignment, and `export ProgramFiles(x86)=v` -- a real + # Windows variable name -- is an outright /bin/sh syntax + # error that would fail the whole action. Skipping is + # strictly better than both: the variable still reaches the + # subprocess through Popen's `env=`, which does not go + # through a shell, so nothing that used to work stops + # working. + self._logger.warning( + f"Not exporting environment variable {name!r} from the action's shell " + "script: the name is not a valid POSIX shell identifier. The variable is " + "still passed to the subprocess directly.", + extra=LogExtraInfo(openjd_log_content=LogContent.PARAMETER_INFO), + ) + continue if value is None: script.append(f"unset {name}") else: script.append(f"export {name}={shlex.quote(value)}") if self._startup_directory is not None: - # Note: Single quotes around the path as it may have spaces, and we don't want to - # process any shell commands in the path. - script.append(f"cd '{self._startup_directory}'") + # R5-4 fix: shlex.quote, not hand-written single quotes. A single + # quote *in the path* closed the quoted region and everything after + # it was interpreted by /bin/sh. Note the env var value two lines + # above has always used shlex.quote -- the safe idiom was already + # imported and in use in this same function. + script.append(f"cd {shlex.quote(str(self._startup_directory))}") script.append("exec " + shlex.join(args)) return "\n".join(script) @@ -1182,7 +1240,13 @@ def _cancel( def _on_process_exit(self, future: Future) -> None: """This is invoked as a callback when run_future is done.""" - assert self._run_future is not None + # R5-6: use the future the completion actually fired for, rather than + # asserting on the instance attribute. This runs on the pool worker (or, + # for a very fast process, on the launching thread), so an AssertionError + # here would only surface through threading.excepthook -- and under + # `python -O` the assert is stripped and the `.exception()` read below + # becomes an AttributeError in the same unobservable place. + run_future = self._run_future if self._run_future is not None else future with self._lock: # F2 fix: Claim _pending_cancel atomically before signalling completion. # A cancel racing completion would otherwise see the process still @@ -1199,7 +1263,7 @@ def _on_process_exit(self, future: Future) -> None: self._cancel_gracetime_timer.cancel() self._cancel_gracetime_timer = None - if exc := self._run_future.exception(): + if exc := run_future.exception(): self._logger.error( f"Error running subprocess: {str(exc)}", extra=LogExtraInfo( @@ -1226,20 +1290,28 @@ def _on_notify_period_end(self) -> None: """This is invoked when the grace period in a NOTIFY_THEN_TERMINATE cancelation has expired. """ - assert self._process is not None with self._lock: self._cancel_gracetime_timer = None + process = self._process + # R5-6: a plain check, not `assert`. This runs on a threading.Timer + # thread, so any exception raised here reaches only + # threading.excepthook -- the grace period would appear to have silently + # done nothing. There is nothing left to terminate if the process is + # already gone, which is the ordinary outcome when the action completed + # during the grace period. + if process is None: + return self._logger.info( "Notify period ended. Terminate at %s", datetime.now(timezone.utc).strftime(TIME_FORMAT_STR), extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), ) try: - self._process.terminate() + process.terminate() except OSError as err: # pragma: nocover # Being paranoid. Won't happen... if we could start the process, then we can send it a kill signal self._logger.warning( - f"Cancelation could not send terminate signal to process {self._process.pid}: {str(err)}", + f"Cancelation could not send terminate signal to process {process.pid}: {str(err)}", extra=LogExtraInfo( openjd_log_content=LogContent.PROCESS_CONTROL | LogContent.EXCEPTION_INFO ), diff --git a/src/openjd/sessions/_session.py b/src/openjd/sessions/_session.py index c9677959..31c98f14 100644 --- a/src/openjd/sessions/_session.py +++ b/src/openjd/sessions/_session.py @@ -597,8 +597,19 @@ def working_directory(self) -> Path: """The directory that was created for this Session's working files. This is available in a Job Template's format string expressions as Session.WorkingDirectory + + Raises: + RuntimeError: If this Session has no working directory, which means + construction did not complete. """ - assert self._working_dir is not None + # A public property, so an explicit raise rather than `assert`: under + # `python -O` the assert vanished and the caller got + # `AttributeError: 'NoneType' object has no attribute 'path'` from inside + # a library property (R5-6). + if self._working_dir is None: + raise RuntimeError( + "This Session has no working directory; its construction did not complete." + ) return self._working_dir.path @property @@ -1217,8 +1228,16 @@ def run_task( # If a wrap environment is active, inject WrappedAction.* into the symbol # table and run its hook instead of the step script's onRun (RFC 0008). if wrap_env is not None: - # Rejected at the top of this method when a wrap env is active. - assert step_name is not None + # A cross-field invariant, not type-checker narrowing: RFC 0008 + # requires WrappedStep.Name, so a wrapped run without a step name is + # rejected at the top of this method. An explicit raise rather than + # `assert` so that `python -O` cannot let a wrapped task through with + # `WrappedStep.Name` set to None (R5-6). + if step_name is None: # pragma: no cover - guarded at method entry + raise RuntimeError( + "Internal error: a wrap environment is active but no step name was " + "resolved for the task." + ) # The wrapped onRun resolves against the STEP's own scope; the # hook resolves against `symtab`, and the wrap environment's own # lets/files are evaluated by the script runner from @@ -1975,18 +1994,22 @@ def _openjd_session_root_dir(self) -> Path: if self._session_root_directory is not None: return self._session_root_directory - tempdir = Path(custom_gettempdir(self._logger)) - - # Note: If this doesn't have group permissions, then we will be unable to access files - # under this directory if the default group of the current user is the group that - # is shared with a job user. The group permissions override the world permissions - # when the accessor is in the group. - # 0o755 - mode = stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH - tempdir.mkdir(exist_ok=True, mode=mode) - # tempdir might already exist with the incorrect permissions, so set the permissions. - os.chmod(tempdir, mode=mode) - return tempdir + # custom_gettempdir() owns this directory's lifecycle end to end: it + # creates it, refuses to return a path that is a link or that another user + # owns (_validate_temp_dir_ownership), and guarantees + # OPENJD_TEMPDIR_MODE. The mode matters and is not merely cosmetic -- if + # the root lacked group/other search permission we would be unable to + # reach files under it when this process' default group is the group + # shared with a job user, because group permissions override world + # permissions for a member of the group. + # + # This used to re-apply the mode itself, with the constant duplicated. The + # duplicate is gone rather than merely single-sourced: two creators of one + # shared path is what let an unvalidated one hand out a directory the + # other then chmod'ed. + # + # Raises: RuntimeError + return Path(custom_gettempdir(self._logger)) def _create_working_directory(self) -> TempDir: """Creates and returns the temporary working directory for this Session""" @@ -2171,9 +2194,13 @@ def _action_log_filter_callback( if self._callback: action_status = self.action_status - # for the type checker - assert action_status is not None - self._callback(self._session_id, action_status) + # R5-6: a plain check, not `assert`. Every branch above sets + # `_action_state`, so this is non-None in practice -- but it is read + # on the stdout-forwarding thread, where an AssertionError unwinds + # LoggingSubprocess.run() before the child is waited on. Skipping the + # notification is strictly better than losing the process. + if action_status is not None: + self._callback(self._session_id, action_status) def _cancel_running_action_as_failed(self) -> None: """Cancel the running action and report it as failed, if one is running. @@ -2224,8 +2251,11 @@ def _fail_action_before_start(self, message: str) -> None: self._state = SessionState.READY_ENDING if self._callback: action_status = self.action_status - # for the type checker - assert action_status is not None + # R5-6: a plain check, not `assert` -- `_action_state` is assigned + # immediately above, so None here would be an internal error, and + # skipping the callback beats raising out of a failure path. + if action_status is None: # pragma: no cover - defensive + return # R4-6 fix: Isolate consumer-callback exceptions. Same pattern as # _fail_action in ScriptRunnerBase and _on_process_exit. A consumer # that raises must not turn a handled resolution failure into an @@ -2252,10 +2282,16 @@ def _action_callback(self, state: ActionState) -> None: We can be certain that the process is no longer running when this is called. """ - # For the type checker - assert self._runner is not None + # R5-6: a bound read with an explicit check rather than `assert`. This is + # invoked from the runner's own completion path, so `_runner` is set -- + # but `cleanup()` clears it, and this runs on the pool worker where an + # AssertionError reaches nobody. Under `python -O` the assert vanished and + # the very next line became an AttributeError in the same blind spot. + runner = self._runner + if runner is None: # pragma: no cover - defensive + return - self._action_exit_code = self._runner.exit_code + self._action_exit_code = runner.exit_code self._action_state = state # F5 fix: Snapshot action_status BEFORE publishing READY. If we set diff --git a/src/openjd/sessions/_subprocess.py b/src/openjd/sessions/_subprocess.py index 2f251f8e..c3d227af 100644 --- a/src/openjd/sessions/_subprocess.py +++ b/src/openjd/sessions/_subprocess.py @@ -119,8 +119,14 @@ def exit_code(self) -> Optional[int]: # has been set once the subprocess has completed running. Don't poll here... # we only want to make the returncode available after the run method has # completed its work. - if self._process is not None: - return self._process.returncode + # + # Bind once (same reason as notify()/terminate()): this property is read + # cross-thread via Session.action_status while run()'s `finally` clears + # `_process`, so a None check followed by a separate attribute read raised + # AttributeError out of a public property. + proc = self._process + if proc is not None: + return proc.returncode return self._returncode @property @@ -180,9 +186,7 @@ def run(self) -> None: # A trivial command can exit before we get here. Looking up the # process group of an already-reaped child raises ProcessLookupError # (ESRCH), which must not fail the action: the child ran, and its - # exit code is still collected below. Fall back to its own pid -- - # the process group we would have found, since the child is the - # group leader (start_new_session=True). + # exit code is still collected below. try: if not self._user or self._user.is_process_user(): self._sudo_child_process_group_id = os.getpgid(self._process.pid) @@ -192,7 +196,24 @@ def run(self) -> None: sudo_process=self._process, ) except ProcessLookupError: - self._sudo_child_process_group_id = self._process.pid + # R5-9 fix: leave this as None -- "no process group known" -- + # rather than substituting the pid. The lookup failed *because* + # the process is gone, so its pid is dead: signalling it is at + # best a no-op, and after pid recycling on a busy host `killpg` + # would target an unrelated group. In the sudo branch the pid is + # not even the right identifier, since it belongs to `sudo` and + # not to the workload's process group. + # + # None is the established "unknown" value on this field -- + # find_sudo_child_process_group_id returns it for the same reason + # (_linux/_sudo.py), and _posix_signal_subprocess already retries + # the lookup and then declines to signal when it is still unset. + self._sudo_child_process_group_id = None + self._logger.debug( + f"Process {self._process.pid} exited before its process group could be " + "determined; no process group will be signalled for it.", + extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), + ) self._logger.info( f"Command started as pid: {self._process.pid}", @@ -365,10 +386,19 @@ def _log_subproc_stdout(self) -> None: Then the thread exits and is cleaned up automatically by the python garbage collector. """ - assert self._process - stream = self._process.stdout - # Convince type checker that stdout is not None - assert stream is not None + # R5-6: explicit raises rather than `assert`. This method owns the + # subprocess's output stream for the whole life of the child, and it runs + # on the pool worker; stripping these under `python -O` would turn a + # legible "called with no process" into an AttributeError on None from + # inside the pump loop. + process = self._process + if process is None: # pragma: no cover - defensive + raise RuntimeError( + "Internal error: cannot forward stdout before the subprocess has been created." + ) + stream = process.stdout + if stream is None: # pragma: no cover - defensive + raise RuntimeError("Internal error: the subprocess was created without a stdout pipe.") exit_event = Event() @@ -422,7 +452,7 @@ def _enqueue_stdout(): except Empty: pass # queue.get timed out. This means the subprocess does not print much to STDOUT. Just continue. - if self._process.poll() is not None: # The main command exited. + if process.poll() is not None: # The main command exited. if process_exit_time is None: process_exit_time = time.monotonic() elif (time.monotonic() - process_exit_time) < 1: diff --git a/src/openjd/sessions/_tempdir.py b/src/openjd/sessions/_tempdir.py index 60afc2bc..e15a1ddf 100644 --- a/src/openjd/sessions/_tempdir.py +++ b/src/openjd/sessions/_tempdir.py @@ -2,11 +2,12 @@ import os import stat +import sys from ._logging import LoggerAdapter, LogContent, LogExtraInfo from pathlib import Path from shutil import chown, rmtree from tempfile import gettempdir, mkdtemp -from typing import Optional, cast +from typing import Any, Optional, cast from ._session_user import PosixSessionUser, SessionUser, WindowsSessionUser from ._windows_permission_helper import WindowsPermissionHelper @@ -16,6 +17,66 @@ from ._win32._helpers import get_process_user # type: ignore +OPENJD_TEMPDIR_MODE = stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH +"""Mode for the shared Open Job Description temporary root: 0o755. + +World-traversable on purpose. If this directory lacked group/other search +permission then a job user -- who by design is a different OS user from this +process -- could not reach the session working directory beneath it. The session +directories themselves are created 0o700 by mkdtemp() and only widened to 0o770 +for the specific group named by a SessionUser, so the permissive mode here grants +traversal, not read access to any session's contents. + +Applied explicitly rather than left to the process umask so that the resulting +mode does not depend on how the embedding application happens to be configured. +""" + + +def _validate_temp_dir_ownership(temp_dir: str) -> None: + """Refuse to use the shared temporary root unless it is a real directory + owned by this process' user (or root). + + Why this exists: the root is a *fixed, predictable* path + (``/OpenJD``) and on POSIX its parent is typically world-writable + ``/tmp``. ``os.makedirs(..., exist_ok=True)`` happily accepts an entry that + some other local user created first, or a symlink pointing somewhere else + entirely -- and every session working directory would then be created inside + a directory that user controls. ``/tmp``'s sticky bit does not help: it + restricts deletion within ``/tmp`` itself, not within an attacker-owned + ``/tmp/OpenJD``. + + ``os.lstat`` rather than ``os.stat``, because the whole point is to see a + symlink rather than follow it. + + Raises: + RuntimeError: if the path is a symlink/reparse point, is not a + directory, or is owned by another user. + """ + try: + st = os.lstat(temp_dir) + except OSError as err: + raise RuntimeError(f"Could not inspect the temporary directory {temp_dir}: {err}") + + if stat.S_ISLNK(st.st_mode) or getattr(st, "st_reparse_tag", 0) != 0: + raise RuntimeError( + f"Refusing to use temporary directory {temp_dir}: it is a link or reparse point " + "rather than a real directory. Remove it, or pass an explicit session root directory." + ) + if not stat.S_ISDIR(st.st_mode): + raise RuntimeError( + f"Refusing to use temporary directory {temp_dir}: it exists but is not a directory." + ) + if is_posix(): + this_uid = os.geteuid() + # root is accepted so that a system-provisioned root directory works. + if st.st_uid not in (this_uid, 0): + raise RuntimeError( + f"Refusing to use temporary directory {temp_dir}: it is owned by uid " + f"{st.st_uid}, not by this process' uid ({this_uid}) or root. Another user " + "may have created it. Remove it, or pass an explicit session root directory." + ) + + def custom_gettempdir(logger: Optional[LoggerAdapter] = None) -> str: """ Get a platform-specific temporary directory. @@ -30,6 +91,11 @@ def custom_gettempdir(logger: Optional[LoggerAdapter] = None) -> str: Returns: str: The path to the temporary directory specific to the operating system. + + Raises: + RuntimeError: If the directory could not be created, or if it already + exists and is not a directory owned by this process' user (see + :func:`_validate_temp_dir_ownership`). """ if is_windows(): program_data_path = os.getenv("PROGRAMDATA") @@ -46,7 +112,32 @@ def custom_gettempdir(logger: Optional[LoggerAdapter] = None) -> str: temp_dir_parent = gettempdir() temp_dir = os.path.join(temp_dir_parent, "OpenJD") - os.makedirs(temp_dir, exist_ok=True) + try: + os.makedirs(temp_dir, mode=OPENJD_TEMPDIR_MODE, exist_ok=True) + except OSError as err: + raise RuntimeError(f"Could not create temporary directory {temp_dir}: {err}") + + # R5-3 fix: exist_ok=True accepts whatever is already at this predictable + # path -- another local user's directory, or a symlink pointing elsewhere. + # Validate BEFORE the chmod below, so we never modify an entry that turns out + # not to be ours. + _validate_temp_dir_ownership(temp_dir) + + # makedirs()' `mode` is masked by the process umask, so it does not on its own + # guarantee anything: under umask 0o077 the "explicit" 0o755 above lands as + # 0o700, and a job user could then not traverse into its own session + # directory. Set the mode outright. Conditional because a + # system-provisioned root may be owned by root with the right mode already, + # where chmod would fail for no reason. + try: + current_mode = stat.S_IMODE(os.stat(temp_dir).st_mode) + if current_mode != stat.S_IMODE(OPENJD_TEMPDIR_MODE): + os.chmod(temp_dir, mode=OPENJD_TEMPDIR_MODE) + except OSError as err: + raise RuntimeError( + f"Could not set permissions on temporary directory {temp_dir}: {err}. Session " + "working directories created beneath it may not be reachable by a job user." + ) return temp_dir @@ -60,6 +151,28 @@ class TempDir: do it (don't really need to, either, since the use-case for this class is to create the Open Job Description Session working directory and that working directory needs to be both writable and deletable by this process). + + Trust precondition when ``user`` is given (POSIX): + ``mkdtemp()``'s 0o700 is deliberately widened to 0o770 so that the target + user can write into the session directory. That grant is to a *group*, + not to a user, so **every member of the group named by the + ``PosixSessionUser`` gains write access to the whole session tree** -- + not only the intended session user. Generated shell scripts and + materialized embedded files live in that tree and are executed after + being written, so a second member of that group could substitute their + contents between materialization and exec. + + The caller is therefore required to supply a group that contains only + principals it already trusts with the session's work -- conventionally a + group dedicated to the pairing of this process' user and the one job + user. This runtime cannot verify that and does not try to. The ordering + here is careful about the part it can control: group ownership is changed + *before* the mode is widened, so a failure to set the group never leaves + a group-writable directory behind. + + The same precondition covers the group-writable bits that + ``_embedded_files.write_file_for_user`` sets on individual files for the + same reason. """ path: Path @@ -136,20 +249,41 @@ def __init__( def cleanup(self) -> None: """Deletes the temporary directory and all of its contents. Raises: - RuntimeError - If not all files could be deleted. + RuntimeError - If not all files could be deleted. The message names + each path that could not be deleted along with the reason. """ - encountered_errors = False - file_paths: list[str] = [] + failures: list[str] = [] + + def _record(path: Any, error: BaseException) -> None: + # R5-7 fix: keep the reason. Cleanup is exactly where the difference + # between "permission denied" and "a process still holds this open" + # decides what the operator should do next, and the old handler + # accepted the exception and discarded it -- leaving a list of bare + # paths with no cause. + failures.append(f"{path}: {error!r}") + + if sys.version_info >= (3, 12): + # `onerror` is deprecated from 3.12 and slated for removal; `onexc` + # receives the exception instance directly. + def onexc( + func: Any, path: Any, error: BaseException + ) -> None: # pragma: nocover - version-gated + _record(path, error) + + rmtree(self.path, onexc=onexc) + else: + + def onerror( + func: Any, path: Any, exc_info: Any + ) -> None: # pragma: nocover - version-gated + # Pre-3.12 handlers are called with sys.exc_info()-style triples. + error = exc_info[1] if isinstance(exc_info, tuple) else exc_info + _record(path, error) - def onerror(f, p, e): - nonlocal encountered_errors - nonlocal file_paths - encountered_errors = True - file_paths.append(str(p)) + rmtree(self.path, onerror=onerror) - rmtree(self.path, onerror=onerror) - if encountered_errors: + if failures: raise RuntimeError( f"Files within temporary directory {str(self.path)} could not be deleted.\n" - + "\n".join(file_paths) + + "\n".join(failures) ) diff --git a/test/openjd/sessions_v0/test_round5_fixes.py b/test/openjd/sessions_v0/test_round5_fixes.py new file mode 100644 index 00000000..28bad2e1 --- /dev/null +++ b/test/openjd/sessions_v0/test_round5_fixes.py @@ -0,0 +1,827 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Regression tests for round-5 review findings R5-1 through R5-9 on PR #333, +plus the sibling occurrences found while implementing each of them. + +Every test in this file was mutation-checked: the fix was reverted and the test +was confirmed to fail, then the fix was re-applied and the test was confirmed to +pass. A test that passes against the unfixed tree pins nothing. +""" + +import logging +import os +import stat +import sys +from datetime import timedelta +from logging.handlers import QueueHandler +from pathlib import Path +from queue import SimpleQueue +from typing import Any, Optional +from unittest.mock import MagicMock, patch + +import pytest + +from openjd.model import SymbolTable +from openjd.model.v2023_09 import ( + Action as Action_2023_09, + ArgString as ArgString_2023_09, + CommandString as CommandString_2023_09, + DataString as DataString_2023_09, + EmbeddedFileText as EmbeddedFileText_2023_09, + EmbeddedFileTypes as EmbeddedFileTypes_2023_09, + StepActions as StepActions_2023_09, + StepScript as StepScript_2023_09, +) + +from openjd.sessions import ActionState, Session, SessionState +from openjd.sessions._action_filter import ( + ActionMessageKind, + ActionMonitoringFilter, + envvar_set_matcher_json, + envvar_set_matcher_str, + envvar_unset_matcher, +) +from openjd.sessions._embedded_files import ( + EmbeddedFiles, + EmbeddedFilesScope, + _FileRecord, +) +from openjd.sessions._os_checker import is_posix +from openjd.sessions._runner_base import POSIX_SHELL_NAME_RE, ScriptRunnerBase +from openjd.sessions._subprocess import LoggingSubprocess +from openjd.sessions._tempdir import OPENJD_TEMPDIR_MODE, TempDir, custom_gettempdir + +from .conftest import build_logger + + +def _make_record( + msg: str, args: Any = None, session_id: str = "foo", level: int = logging.INFO +) -> logging.LogRecord: + """A LogRecord shaped the way the session logger produces them.""" + record = logging.LogRecord("test", level, "path", 1, msg, args, None) + record.session_id = session_id # type: ignore[attr-defined] + return record + + +class _Runner(ScriptRunnerBase): + """Minimal concrete runner: only _generate_command_shell_script is exercised.""" + + def cancel( + self, *, time_limit: Optional[timedelta] = None, mark_action_failed: bool = False + ) -> None: + pass + + +# =========================================================================== +# R5-1 -- redaction must not leave record.args populated +# =========================================================================== + + +class TestR51RedactionArgsBypass: + """R5-1: `record.args` must be empty by the time the filter returns. + + The redaction logic only ever inspects `record.msg`. A downstream handler + calls `record.getMessage()`, which re-runs `msg % args` -- so any path that + leaves `args` populated re-interpolates the *un-scanned* original into the + emitted line. + """ + + def _filter_with_secret(self, secret: str = "SUPERSECRET") -> ActionMonitoringFilter: + f = ActionMonitoringFilter(session_id="foo", callback=MagicMock()) + f._redacted_values.add(secret) + return f + + def test_secret_in_args_is_redacted_when_formatting_succeeds(self) -> None: + # GIVEN: a secret carried only in args, with a msg that matches nothing + f = self._filter_with_secret() + record = _make_record("value is %s", ("SUPERSECRET",)) + + # WHEN + f.filter(record) + + # THEN: the emitted line -- which is what a handler actually renders -- + # carries no secret, and args cannot reintroduce one. + assert "SUPERSECRET" not in record.getMessage() + assert not record.args + + def test_secret_in_args_is_redacted_when_formatting_fails(self) -> None: + """The load-bearing case. `%d` against a str raises, so the old code + skipped clearing args and the handler re-interpolated the secret.""" + # GIVEN: a record whose own %-formatting is broken + f = self._filter_with_secret() + record = _make_record("value is %d", ("SUPERSECRET",)) + + # WHEN + f.filter(record) + + # THEN: args are cleared, so getMessage() cannot resurrect the secret, + # and the secret does not survive anywhere on the record. + assert not record.args + assert "SUPERSECRET" not in record.getMessage() + assert "SUPERSECRET" not in str(record.msg) + + def test_record_stays_renderable_when_formatting_fails(self) -> None: + """Folding args in must not leave a record that raises in the handler.""" + # GIVEN + f = self._filter_with_secret() + record = _make_record("value is %d", ("SUPERSECRET",)) + + # WHEN + f.filter(record) + + # THEN: getMessage() does not raise (it would if msg still held %d and + # args were still a str tuple), and the redaction marker is present. + assert "*" * 8 in record.getMessage() + + def test_whole_line_redaction_clears_args(self) -> None: + # GIVEN: a message that matches a whole redacted line + f = ActionMonitoringFilter(session_id="foo", callback=MagicMock()) + f._redacted_lines.add("secret-line") + record = _make_record("secret-line", None) + + # WHEN + f.filter(record) + + # THEN + assert record.getMessage() == "*" * 8 + assert not record.args + + def test_args_are_untouched_when_no_redactions_registered(self) -> None: + """Lazy %-formatting must keep working for every ordinary log line.""" + # GIVEN: no registered secrets + f = ActionMonitoringFilter(session_id="foo", callback=MagicMock()) + record = _make_record("value is %s", ("plain",)) + + # WHEN + f.filter(record) + + # THEN: the filter did not fold args in, so the handler still formats. + assert record.args == ("plain",) + assert record.getMessage() == "value is plain" + + +# =========================================================================== +# R5-2 -- consumer callback exceptions must not escape filter() +# =========================================================================== + + +class TestR52FilterCallbackIsolation: + """R5-2: this filter runs on the thread forwarding the subprocess's stdout. + + An exception escaping `filter()` unwinds `LoggingSubprocess.run()` before the + child is waited on: the pump thread dies, the rest of the output is dropped, + and the process is left unreaped. + """ + + @pytest.mark.parametrize( + "exc", + [RuntimeError("boom"), KeyError("boom"), TypeError("boom"), AttributeError("boom")], + ids=["RuntimeError", "KeyError", "TypeError", "AttributeError"], + ) + def test_handler_callback_exception_does_not_escape(self, exc: Exception) -> None: + # GIVEN: a consumer callback that raises a non-ValueError + def callback(kind: ActionMessageKind, value: Any, fail: bool) -> None: + raise exc + + f = ActionMonitoringFilter(session_id="foo", callback=callback) + record = _make_record("openjd_progress: 50.0") + + # WHEN / THEN: filter() returns normally... + assert f.filter(record) is True + # ...and keeps the record, with the failure visible in the action's output. + assert "boom" in record.getMessage() + + def test_malformed_env_callback_exception_does_not_escape(self) -> None: + """The one callback invocation in filter() not routed through `handler`.""" + + # GIVEN + def callback(kind: ActionMessageKind, value: Any, fail: bool) -> None: + raise RuntimeError("boom") + + f = ActionMonitoringFilter(session_id="foo", callback=callback) + # A near-miss env command: space before the colon. + record = _make_record("openjd_env : FOO=bar") + + # WHEN / THEN + assert f.filter(record) is True + + def test_valueerror_still_annotates_the_record(self) -> None: + """The pre-existing ValueError contract must be unchanged.""" + # GIVEN: progress outside the legal range raises ValueError in the handler + f = ActionMonitoringFilter(session_id="foo", callback=MagicMock()) + record = _make_record("openjd_progress: 500.0") + + # WHEN / THEN + assert f.filter(record) is True + assert "ERROR" in record.getMessage() + + def test_redaction_failure_fails_closed(self) -> None: + """If the redaction control itself breaks, emit nothing rather than an + unscanned line -- and do not let it reach the pump thread.""" + # GIVEN + f = ActionMonitoringFilter(session_id="foo", callback=MagicMock()) + record = _make_record("carries a secret") + + # WHEN + with patch.object( + ActionMonitoringFilter, + "apply_message_redaction", + side_effect=RuntimeError("redaction is broken"), + ): + result = f.filter(record) + + # THEN + assert result is True + assert record.getMessage() == "*" * 8 + + def test_a_live_child_is_still_reaped_when_the_callback_raises(self) -> None: + """End to end: the reason R5-2 matters. A progress update from a live + child must not cost us the process.""" + # GIVEN: a consumer that raises on the first progress update + state: dict[str, Any] = {"raised": False} + + def callback(session_id: str, status: Any) -> None: + if status.progress is not None and not state["raised"]: + state["raised"] = True + raise RuntimeError("consumer blew up on a progress update") + + script = StepScript_2023_09( + actions=StepActions_2023_09( + onRun=Action_2023_09( + command=CommandString_2023_09(sys.executable), + args=[ + ArgString_2023_09("-c"), + ArgString_2023_09( + "print('openjd_progress: 50.0', flush=True)\n" + "print('done', flush=True)\n" + ), + ], + ) + ) + ) + + # WHEN + with Session(session_id="r5-2-e2e", job_parameter_values={}, callback=callback) as session: + session.run_task(step_script=script, task_parameter_values={}) + deadline = 60.0 + import time + + start = time.monotonic() + while session.state == SessionState.RUNNING and time.monotonic() - start < deadline: + time.sleep(0.05) + + # THEN: the consumer did raise, the action still reached a terminal + # state, and the subprocess was waited on -- an exit code proves the + # `wait()` in LoggingSubprocess.run() was reached rather than skipped. + assert state["raised"] is True + assert session.state != SessionState.RUNNING + assert session.action_status is not None + assert session.action_status.exit_code == 0 + assert session.action_status.state == ActionState.SUCCESS + + +# =========================================================================== +# SIB-1 -- env var name anchoring (sibling of R5-5, found during this round) +# =========================================================================== + + +class TestSib1EnvVarNameAnchoring: + """`$` also matches immediately before a trailing newline, so an + `$`-anchored NAME pattern accepted "FOO\\n" -- a name no OS can hold. + + The VALUE half stays deliberately permissive: a multi-line value delivered + through the JSON form is supported, tested behaviour. + """ + + @pytest.mark.parametrize("name", ["FOO\n", "FOO\r", "FOO\r\n"]) + def test_unset_rejects_a_trailing_newline_in_the_name(self, name: str) -> None: + assert envvar_unset_matcher.match(name) is None + + def test_unset_still_accepts_a_legal_name(self) -> None: + assert envvar_unset_matcher.match("FOO_BAR9") is not None + + @pytest.mark.parametrize("payload", ["FOO=bar\n", "FOO\n=bar"]) + def test_set_rejects_a_trailing_newline(self, payload: str) -> None: + assert envvar_set_matcher_str.match(payload) is None + + def test_multiline_value_via_json_is_still_supported(self) -> None: + """Guards the intended feature against over-correction.""" + # GIVEN / WHEN + raw = '"FOO=BAR\\nBAZ"' + # THEN: the raw (escaped) form still validates... + assert envvar_set_matcher_json.match(raw) is not None + # ...and the decoded multi-line value still reaches the callback. + callback = MagicMock() + f = ActionMonitoringFilter(session_id="foo", callback=callback) + f.filter(_make_record('openjd_env: "FOO=BAR\\nBAZ"')) + env_calls = [ + c + for c in callback.call_args_list + if c[0][0] == ActionMessageKind.ENV and isinstance(c[0][1], dict) + ] + assert len(env_calls) == 1 + assert env_calls[0][0][1] == {"name": "FOO", "value": "BAR\nBAZ"} + + def test_a_separator_cannot_reach_a_decoded_name(self) -> None: + """Defence in depth: the regex matches the *raw* message, but JSON + decoding happens afterwards, so the decoded name is re-checked.""" + # GIVEN a filter whose callback records what it is handed + callback = MagicMock() + f = ActionMonitoringFilter(session_id="foo", callback=callback) + + # WHEN every ENV message it sees is processed + for msg in ( + 'openjd_env: "FOO\\nBAR=baz"', + 'openjd_env: "FOO\\u000aBAR=baz"', + 'openjd_env: "FOO\\u0000BAR=baz"', + ): + f.filter(_make_record(msg)) + + # THEN no name carrying a separator was ever handed onwards + for call in callback.call_args_list: + value = call[0][1] + if isinstance(value, dict): + assert not any(c in value["name"] for c in ("\n", "\r", "\0", "=")) + + +# =========================================================================== +# R5-3 -- the shared temporary root must be validated before use +# =========================================================================== + + +class TestR53TempRootHardening: + """R5-3: `/OpenJD` is a fixed, predictable path whose parent is + world-writable on typical POSIX hosts. `exist_ok=True` accepted whatever was + already there.""" + + def test_created_with_an_explicit_mode_regardless_of_umask(self, tmp_path: Path) -> None: + # GIVEN: a hostile umask that would otherwise strip group/other bits + old_umask = os.umask(0o077) + try: + with patch("openjd.sessions._tempdir.gettempdir", return_value=str(tmp_path)): + # WHEN + created = custom_gettempdir() + finally: + os.umask(old_umask) + + # THEN: the root is traversable as intended, not umask-dependent. + assert Path(created) == tmp_path / "OpenJD" + if is_posix(): + assert stat.S_IMODE(os.stat(created).st_mode) == stat.S_IMODE(OPENJD_TEMPDIR_MODE) + + @pytest.mark.skipif(not is_posix(), reason="symlink pre-creation is a POSIX vector here") + def test_rejects_a_symlink_at_the_root_path(self, tmp_path: Path) -> None: + # GIVEN: an attacker has replaced the root with a symlink elsewhere + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + parent = tmp_path / "parent" + parent.mkdir() + (parent / "OpenJD").symlink_to(elsewhere, target_is_directory=True) + + # WHEN / THEN: we refuse rather than creating sessions inside it + with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): + with pytest.raises(RuntimeError, match="link or reparse point"): + custom_gettempdir() + + def test_rejects_a_root_that_is_not_a_directory(self, tmp_path: Path) -> None: + # GIVEN: a plain file squatting on the root path + parent = tmp_path / "parent" + parent.mkdir() + (parent / "OpenJD").write_text("squat") + + # WHEN / THEN + with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): + with pytest.raises(RuntimeError): + custom_gettempdir() + + @pytest.mark.skipif(not is_posix(), reason="uid ownership check is POSIX-only") + def test_rejects_a_root_owned_by_another_user(self, tmp_path: Path) -> None: + # GIVEN: the root exists but lstat reports a different owner + parent = tmp_path / "parent" + parent.mkdir() + (parent / "OpenJD").mkdir() + real_lstat = os.lstat + foreign_uid = os.geteuid() + 12345 + + class _Stat: + def __init__(self, base: os.stat_result) -> None: + self.st_mode = base.st_mode + self.st_uid = foreign_uid + + def fake_lstat(path: Any, *a: Any, **k: Any) -> Any: + base = real_lstat(path, *a, **k) + if str(path).endswith("OpenJD"): + return _Stat(base) + return base + + # WHEN / THEN + with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): + with patch("openjd.sessions._tempdir.os.lstat", side_effect=fake_lstat): + with pytest.raises(RuntimeError, match="owned by uid"): + custom_gettempdir() + + @pytest.mark.skipif(not is_posix(), reason="uid ownership check is POSIX-only") + def test_accepts_a_root_owned_by_root(self, tmp_path: Path) -> None: + """A system-provisioned root must keep working.""" + # GIVEN + parent = tmp_path / "parent" + parent.mkdir() + (parent / "OpenJD").mkdir() + real_lstat = os.lstat + + class _Stat: + def __init__(self, base: os.stat_result) -> None: + self.st_mode = base.st_mode + self.st_uid = 0 + + def fake_lstat(path: Any, *a: Any, **k: Any) -> Any: + base = real_lstat(path, *a, **k) + if str(path).endswith("OpenJD"): + return _Stat(base) + return base + + # WHEN / THEN: no exception + with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): + with patch("openjd.sessions._tempdir.os.lstat", side_effect=fake_lstat): + assert custom_gettempdir() == str(parent / "OpenJD") + + +# =========================================================================== +# R5-7 -- cleanup must report WHY each path could not be deleted +# =========================================================================== + + +class TestR57CleanupErrorReporting: + """R5-7: the old handler accepted the exception and discarded it, leaving a + list of bare paths -- on the one code path where "permission denied" versus + "a process still holds this open" changes what the operator must do.""" + + def test_failure_message_names_the_path_and_the_reason(self, tmp_path: Path) -> None: + # GIVEN: a temp dir whose removal fails with a specific, diagnosable error + d = TempDir(dir=tmp_path) + doomed = d.path / "stubborn.txt" + doomed.write_text("x") + + def boom(path: Any, *a: Any, **k: Any) -> None: + raise PermissionError(13, "Permission denied") + + # WHEN + with ( + patch("openjd.sessions._tempdir.os.unlink", side_effect=boom), + patch("openjd.sessions._tempdir.os.remove", side_effect=boom, create=True), + ): + with pytest.raises(RuntimeError) as excinfo: + d.cleanup() + + # THEN: both the path and the cause are in the message. + message = str(excinfo.value) + assert "stubborn.txt" in message + assert "PermissionError" in message + + def test_successful_cleanup_still_removes_everything(self, tmp_path: Path) -> None: + # GIVEN + d = TempDir(dir=tmp_path) + (d.path / "a.txt").write_text("x") + (d.path / "sub").mkdir() + (d.path / "sub" / "b.txt").write_text("y") + + # WHEN + d.cleanup() + + # THEN + assert not d.path.exists() + + +# =========================================================================== +# R5-4 / R5-5 -- nothing unquoted may reach /bin/sh +# =========================================================================== + + +@pytest.mark.skipif(not is_posix(), reason="the generated shell script is POSIX-only") +class TestR54R55ShellScriptQuoting: + """The generated script is `exec`'d by /bin/sh, so a single unquoted + metacharacter anywhere in it is code execution as the session user.""" + + def _script( + self, + tmp_path: Path, + *, + os_env_vars: Optional[dict[str, Optional[str]]] = None, + startup_directory: Optional[Path] = None, + ) -> str: + runner = _Runner( + logger=MagicMock(), + session_working_directory=tmp_path, + os_env_vars=os_env_vars, + startup_directory=startup_directory, + ) + try: + return runner._generate_command_shell_script(["/bin/echo", "hi"]) + finally: + runner.shutdown() + + def test_single_quote_in_the_startup_directory_cannot_break_out(self, tmp_path: Path) -> None: + # GIVEN: a startup directory whose name closes the old hand-written quote + hostile = Path("/tmp/x'; touch /tmp/openjd_r5_pwned; '") + + # WHEN + script = self._script(tmp_path, startup_directory=hostile) + + # THEN: the injected command is inside a quoted word, not a command. + cd_line = next(line for line in script.splitlines() if line.startswith("cd ")) + assert "touch /tmp/openjd_r5_pwned" in cd_line # the path is preserved verbatim... + assert not cd_line.startswith("cd '/tmp/x'; ") # ...but no longer executes. + # Prove it to the shell itself rather than by eyeballing the quoting. + import shlex + + assert shlex.split(cd_line) == ["cd", str(hostile)] + + def test_ordinary_startup_directory_with_spaces_still_works(self, tmp_path: Path) -> None: + # GIVEN + spaced = tmp_path / "a dir with spaces" + + # WHEN + script = self._script(tmp_path, startup_directory=spaced) + + # THEN + import shlex + + cd_line = next(line for line in script.splitlines() if line.startswith("cd ")) + assert shlex.split(cd_line) == ["cd", str(spaced)] + + @pytest.mark.parametrize( + "hostile_name", + [ + "BAD;touch /tmp/openjd_r5_pwned;X", + "BAD$(touch /tmp/openjd_r5_pwned)", + "BAD`touch /tmp/openjd_r5_pwned`", + "BAD\ntouch /tmp/openjd_r5_pwned", + "BAD&&touch /tmp/openjd_r5_pwned", + ], + ) + def test_hostile_env_var_names_are_not_emitted(self, tmp_path: Path, hostile_name: str) -> None: + # GIVEN / WHEN + script = self._script(tmp_path, os_env_vars={hostile_name: "v", "GOOD": "ok"}) + + # THEN: the hostile name produced no line at all, and the legal one did. + assert "openjd_r5_pwned" not in script + assert "export GOOD=ok" in script + + def test_names_that_are_legal_elsewhere_but_not_in_sh_are_skipped(self, tmp_path: Path) -> None: + """`ProgramFiles(x86)` is a real Windows variable name and an outright + /bin/sh syntax error -- previously it broke the whole action.""" + # GIVEN / WHEN + script = self._script(tmp_path, os_env_vars={"ProgramFiles(x86)": "C:\\PF"}) + + # THEN + assert "ProgramFiles(x86)" not in script + + def test_generated_script_is_valid_sh_even_with_hostile_input(self, tmp_path: Path) -> None: + """The strongest assertion available: hand it to `sh -n`.""" + # GIVEN + script = self._script( + tmp_path, + os_env_vars={ + "BAD;X": "v", + "ProgramFiles(x86)": "v", + "GOOD": "a'b\"c$(d)", + "UNSET_ME": None, + }, + startup_directory=Path("/tmp/x'; touch /tmp/openjd_r5_pwned; '"), + ) + script_file = tmp_path / "candidate.sh" + script_file.write_text(script) + + # WHEN + from subprocess import run + + result = run(["/bin/sh", "-n", str(script_file)], capture_output=True, text=True) + + # THEN + assert result.returncode == 0, f"generated script is not valid sh: {result.stderr}" + + def test_unset_is_emitted_for_a_legal_name(self, tmp_path: Path) -> None: + # GIVEN / WHEN + script = self._script(tmp_path, os_env_vars={"GOES_AWAY": None}) + + # THEN + assert "unset GOES_AWAY" in script + + @pytest.mark.parametrize("name", ["FOO", "_foo", "F9", "_", "aB_9"]) + def test_posix_name_re_accepts_legal_identifiers(self, name: str) -> None: + assert POSIX_SHELL_NAME_RE.fullmatch(name) is not None + + @pytest.mark.parametrize("name", ["9FOO", "", "FOO BAR", "FOO=", "FOO\n", "FOO;", "é"]) + def test_posix_name_re_rejects_everything_else(self, name: str) -> None: + assert POSIX_SHELL_NAME_RE.fullmatch(name) is None + + def test_a_skipped_name_still_reaches_the_subprocess( + self, tmp_path: Path, message_queue: SimpleQueue, queue_handler: QueueHandler + ) -> None: + """Skipping the export line must not change what the child sees, because + Popen's `env=` does not go through a shell.""" + # GIVEN: a name that is illegal in sh but legal as an environment entry + logger = build_logger(queue_handler) + proc = LoggingSubprocess( + logger=logger, + args=[ + sys.executable, + "-c", + "import os;print('VAL=' + os.environ.get('ProgramFiles(x86)', 'MISSING'))", + ], + os_env_vars={"ProgramFiles(x86)": "present"}, + ) + + # WHEN + proc.run() + + # THEN + lines = [] + while not message_queue.empty(): + lines.append(message_queue.get().getMessage()) + assert "VAL=present" in lines + assert proc.exit_code == 0 + + +# =========================================================================== +# R5-6 -- invariant-bearing checks must survive `python -O` +# =========================================================================== + + +class TestR56InvariantsSurviveOptimizedMode: + """R5-6: `assert` is stripped under `python -O`, which is a legitimate + production deployment mode. Checks that carry a real runtime invariant must + be explicit raises.""" + + def test_unsupported_embedded_file_model_raises(self, tmp_path: Path) -> None: + # GIVEN: an embedded file object of an unrecognised shape + files = EmbeddedFiles( + logger=MagicMock(), + scope=EmbeddedFilesScope.ENV, + session_files_directory=tmp_path, + ) + + class _AlienFile: + filename = "x.sh" + runnable = False + + # WHEN / THEN: each of the three entry points stops rather than + # materializing a file from an object it does not understand. + with pytest.raises(RuntimeError, match="Unsupported embedded file model"): + files._find_value_prefix(_AlienFile()) # type: ignore[arg-type] + with pytest.raises(RuntimeError, match="Unsupported embedded file model"): + files._get_symtab_entry(_AlienFile()) # type: ignore[arg-type] + with pytest.raises(RuntimeError, match="Unsupported embedded file model"): + files._materialize_file( + tmp_path / "x.sh", _AlienFile(), SymbolTable() # type: ignore[arg-type] + ) + + def test_working_directory_property_raises_a_legible_error(self) -> None: + # GIVEN: a Session whose construction did not complete + session = Session.__new__(Session) + session._working_dir = None # type: ignore[attr-defined] + + # WHEN / THEN: a library RuntimeError, not AttributeError on None. + with pytest.raises(RuntimeError, match="no working directory"): + _ = session.working_directory + + def test_state_is_readable_when_a_launch_failed_after_process_creation( + self, tmp_path: Path + ) -> None: + """The `state` property is on the path every consumer polls. An + inconsistent (process set, future unset) pair must not make the runner + permanently unreadable.""" + # GIVEN + runner = _Runner(logger=MagicMock(), session_working_directory=tmp_path) + try: + runner._process = MagicMock() + runner._run_future = None + + # WHEN / THEN: no AssertionError, no AttributeError. + from openjd.sessions._runner_base import ScriptRunnerState + + assert runner.state == ScriptRunnerState.READY + finally: + runner.shutdown() + + def test_notify_period_end_is_a_no_op_without_a_process(self, tmp_path: Path) -> None: + """Runs on a threading.Timer thread, where an exception reaches only + threading.excepthook -- so the grace period would appear to have silently + done nothing.""" + # GIVEN + runner = _Runner(logger=MagicMock(), session_working_directory=tmp_path) + try: + runner._process = None + + # WHEN / THEN: returns cleanly. + runner._on_notify_period_end() + finally: + runner.shutdown() + + def test_log_subproc_stdout_raises_a_legible_error_without_a_process( + self, tmp_path: Path + ) -> None: + # GIVEN + proc = LoggingSubprocess(logger=MagicMock(), args=[sys.executable, "-c", "pass"]) + + # WHEN / THEN + with pytest.raises(RuntimeError, match="before the subprocess has been created"): + proc._log_subproc_stdout() + + def test_an_embedded_file_still_materializes_normally(self, tmp_path: Path) -> None: + """Guard against the isinstance rewrite breaking the happy path.""" + # GIVEN + files = EmbeddedFiles( + logger=MagicMock(), + scope=EmbeddedFilesScope.ENV, + session_files_directory=tmp_path, + ) + model = EmbeddedFileText_2023_09( + name="Script", + type=EmbeddedFileTypes_2023_09.TEXT, + data=DataString_2023_09("hello"), + ) + symtab = SymbolTable() + + # WHEN + records = files.allocate_file_paths([model], symtab) + files.write_file_contents(records, symtab) + + # THEN + assert len(records) == 1 + assert isinstance(records[0], _FileRecord) + assert records[0].filename.read_text() == "hello" + + +# =========================================================================== +# R5-9 -- an unknown process group must be recorded as unknown +# =========================================================================== + + +@pytest.mark.skipif(not is_posix(), reason="process groups are POSIX-only") +class TestR59ProcessGroupSentinel: + """R5-9: the lookup fails *because* the process is gone, so its pid is dead. + Recording it as a process-group id means a later `killpg` is at best a no-op + and, after pid recycling, targets an unrelated group.""" + + def test_reaped_child_leaves_the_group_unknown( + self, message_queue: SimpleQueue, queue_handler: QueueHandler + ) -> None: + # GIVEN: a child whose process group cannot be looked up + logger = build_logger(queue_handler) + proc = LoggingSubprocess( + logger=logger, args=[sys.executable, "-c", "pass"], os_env_vars=None + ) + + # WHEN + with patch( + "openjd.sessions._subprocess.os.getpgid", side_effect=ProcessLookupError(3, "No such") + ): + proc.run() + + # THEN: "unknown", not a stale pid -- and the action still succeeded, + # which is the behaviour the original fix existed to protect. + assert proc._sudo_child_process_group_id is None + assert proc.exit_code == 0 + assert proc.failed_to_start is False + + def test_no_signal_is_delivered_when_the_group_is_unknown( + self, message_queue: SimpleQueue, queue_handler: QueueHandler + ) -> None: + # GIVEN: a finished process with no known group + logger = build_logger(queue_handler) + proc = LoggingSubprocess(logger=logger, args=[sys.executable, "-c", "pass"]) + with patch( + "openjd.sessions._subprocess.os.getpgid", side_effect=ProcessLookupError(3, "No such") + ): + proc.run() + assert proc._sudo_child_process_group_id is None + + # WHEN: a SIGKILL is attempted anyway + with ( + patch("openjd.sessions._subprocess.os.killpg") as killpg, + patch( + "openjd.sessions._subprocess.find_sudo_child_process_group_id", return_value=None + ), + ): + proc._posix_signal_subprocess(MagicMock(pid=999999), signal_name="kill") + + # THEN: nothing was signalled. + killpg.assert_not_called() + + def test_sudo_helper_returns_unknown_when_sudo_is_already_gone(self) -> None: + """Sibling: the same guard on the first getpgid in the sudo helper.""" + # GIVEN + from openjd.sessions._linux._sudo import find_sudo_child_process_group_id + + # WHEN + with patch( + "openjd.sessions._linux._sudo.os.getpgid", + side_effect=ProcessLookupError(3, "No such process"), + ): + result = find_sudo_child_process_group_id( + logger=MagicMock(), sudo_process=MagicMock(pid=999999) + ) + + # THEN: the established "unknown" value, not an escaping ESRCH. + assert result is None From ec29eebb3e8bdc451c1481d4c9c6064be81c42e8 Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Sun, 26 Jul 2026 11:14:18 -0700 Subject: [PATCH 15/27] fix: Close defects introduced by the round-5 fix commit A five-agent sweep of 4c57f40 found three defects in that commit itself. The first is a regression: it turned a latent AssertionError into silent state corruption. REG-1 (_runner_base.py) -- R5-6 replaced `assert self._run_future is not None` in the `state` property with `return READY`. Two consequences, both reproduced: a) `_run`'s single-use guard is `if self.state != READY: raise`, so reporting READY for a launch that failed after `_process` was assigned silently opened the guard. A second subprocess launched over the first, replacing `_process` and orphaning the original child. b) `_on_process_exit` built `ActionState(self.state.value)`. `ActionState` has no `ready` member, so this raised ValueError, which the F8 try/except around the callback caught and logged -- delivering no terminal callback at all. A consumer would wait forever on an action that had finished. Fixed three ways, smallest first: - The single-use guard is now latched on a new `_launched` flag rather than derived from `state`. Tying the rule to the state classification is what let it break when the classification changed; a "has a launch been attempted" latch cannot drift that way. `state` is still consulted for the `_state_override` that `_fail_action` sets. - `_process` and `_run_future` are published together after `_pool.submit` rather than `_process` first. The inconsistent pair was observable on the ordinary success path too, since `state` is read without the lock; removing the window removes the question the R5-6 change had to answer. - New `_terminal_action_state()` maps an unclassifiable state to FAILED. Publishing a wrong-but-terminal state beats publishing nothing. REG-2 (_action_filter.py) -- the R5-2 containment interpolated the consumer's exception into an f-string at three sites, so an exception whose `__str__` raises escaped `filter()` anyway, from inside the handler meant to contain it. New `_describe_exception()` falls back through str, repr, and the type name. REG-3 (_tempdir.py) -- the R5-3 ownership check was check-then-use: it validated with `os.lstat(path)` and then called `os.stat(path)`/`os.chmod(path)`, both of which re-resolve the name and follow symlinks. Swapping the entry for a symlink in between defeated the check and widened the link's target to 0o755. `_prepare_temp_dir_root` now opens the directory once with O_NOFOLLOW and O_DIRECTORY and uses fstat/fchmod, so every decision and every modification applies to the inode that was validated. Also from the sweep, in the same files: - Removed the `_ENVVAR_NAME_FORBIDDEN` post-decode check added in 4c57f40. It is unreachable -- the raw-message regex rejects every input that would decode to a bad name -- and its test's assertion loop never executed, so it passed against a deliberately broken implementation. The test is rewritten to assert the rejection directly and now fails when the regex is loosened. - Corrected two comments that overstated what the code does: the `\Z` anchor change is defence in depth, not a fix for a reachable defect (the outer filter regex already strips a trailing newline), and `_on_process_exit` now genuinely reads the future it was called with. - Corrected the R5-5 warning text. It promised the variable still reaches the subprocess via Popen's env=, which is false for a cross-user action: `sudo -i` starts a login shell that resets the environment, so the generated script's export lines are the only channel there. Adds test/openjd/sessions_v0/test_round5_regressions.py (14 tests). Every test was mutation-checked against 9 mutants covering each behaviour above, with __pycache__ cleared between mutants -- restoring a file with `mv` puts back the original mtime and can leave Python serving bytecode compiled from the mutant, which produced one false failure while writing these. Verified: 801 passed / 39 skipped / 16 xfailed (only the known xdist timing flake fails, and passes in isolation); ruff, mypy and black clean. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- src/openjd/sessions/_action_filter.py | 73 ++- src/openjd/sessions/_runner_base.py | 96 +++- src/openjd/sessions/_tempdir.py | 105 +++-- test/openjd/sessions_v0/test_round5_fixes.py | 106 +++-- .../sessions_v0/test_round5_regressions.py | 441 ++++++++++++++++++ 5 files changed, 693 insertions(+), 128 deletions(-) create mode 100644 test/openjd/sessions_v0/test_round5_regressions.py diff --git a/src/openjd/sessions/_action_filter.py b/src/openjd/sessions/_action_filter.py index 6c78e3df..922eb768 100644 --- a/src/openjd/sessions/_action_filter.py +++ b/src/openjd/sessions/_action_filter.py @@ -15,6 +15,27 @@ __all__ = ("ActionMessageKind", "ActionMonitoringFilter", "redact_openjd_redacted_env_requests") +def _describe_exception(exc: BaseException) -> str: + """Render an exception for a log line without trusting it to be renderable. + + A consumer-supplied callback may raise an exception whose ``__str__`` or + ``__repr__`` itself raises. Every containment site in this module puts the + exception into a message, so rendering it naively re-creates the exact escape + the containment exists to prevent -- the exception then propagates from inside + the ``except`` block, out of ``filter()``, and into the thread forwarding the + subprocess's stdout. + """ + try: + return str(exc) + except Exception: + pass + try: + return repr(exc) + except Exception: + pass + return f"" + + def redact_openjd_redacted_env_requests(command_str: str) -> str: """Redact sensitive information in command strings before they're processed by the regular redaction mechanism. @@ -78,14 +99,22 @@ class ActionMessageKind(Enum): openjd_env_actions_filter_matcher = re.compile(openjd_env_actions_filter_regex) # A regex for matching the assignment of a value to an environment variable -# Note on the anchors: these use \Z, not $. `$` also matches immediately before -# a trailing newline, so "FOO\n" satisfies an `$`-anchored name pattern and this -# runtime would go on to unset a variable whose name contains a newline -- a name -# no operating system can hold. \Z matches only at the true end of the string. +# +# Note on the anchors: these use \Z, not $, because `$` also matches immediately +# before a trailing newline -- so an `$`-anchored name pattern accepts "FOO\n", +# a name no operating system can hold. \Z matches only at the true end of the +# string. +# +# Honest scope: this is defence in depth, not a fix for a reachable defect. The +# outer `filter_regex` below captures the message body with `(.+)$`, and `.` never +# crosses a newline, so a trailing newline is already stripped before any name +# reaches these patterns. \Z is used so that these patterns remain correct on +# their own terms if that outer capture ever changes. +# # The VALUE half deliberately stays permissive: a multi-line value delivered -# through the JSON form (openjd_env: "FOO=a\nb") is supported behaviour, so the -# separator hygiene a value needs belongs at each serialization boundary, not -# here. +# through the JSON form (openjd_env: "FOO=a\nb") is supported, tested behaviour, +# so the separator hygiene a value needs belongs at each serialization boundary, +# not here. envvar_set_regex_str = "^[A-Za-z_][A-Za-z0-9_]*" "=" ".*\\Z" # Variable name envvar_set_regex_json = '^(")?[A-Za-z_][A-Za-z0-9_]*' "=" ".*\\Z" # Variable name envvar_set_matcher_str = re.compile(envvar_set_regex_str) @@ -93,13 +122,6 @@ class ActionMessageKind(Enum): envvar_unset_regex = "^[A-Za-z_][A-Za-z0-9_]*\\Z" envvar_unset_matcher = re.compile(envvar_unset_regex) -# The set of characters that can never legally appear in an environment variable -# NAME, and that would change the meaning of any line-based or NUL-delimited -# rendering of one. A name is validated against envvar_*_regex above before it -# reaches a callback, but the JSON form is decoded AFTER that match, so the -# decoded name is re-checked with this. -_ENVVAR_NAME_FORBIDDEN = ("\n", "\r", "\0", "=") - # This is a reworking/merging of TaskStatusFilter and FailureFilter class ActionMonitoringFilter(logging.Filter): @@ -362,11 +384,16 @@ def filter(self, record: logging.LogRecord) -> bool: # unreaped. Same isolation as the Session's terminal # callbacks; keep the record in the log so the failure is # visible in the action's own output. + # + # _describe_exception, not f"{e}": rendering a hostile + # exception can itself raise, which would re-open this very + # escape from inside the handler for it. + detail = _describe_exception(e) LOG.error( - f"Open Job Description: Exception in the action message callback: {e}", + f"Open Job Description: Exception in the action message callback: {detail}", extra=LogExtraInfo(openjd_log_content=LogContent.EXCEPTION_INFO), ) - record.msg = record.msg + f" -- ERROR: {str(e)}" + record.msg = record.msg + f" -- ERROR: {detail}" return True return not self._suppress_filtered @@ -394,11 +421,12 @@ def filter(self, record: logging.LogRecord) -> bool: # exception reach the stdout pump thread). try: self.apply_message_redaction(record) - except Exception as e: # pragma: nocover - defensive + except Exception as e: record.msg = "*" * 8 record.args = () LOG.error( - f"Open Job Description: Log redaction failed; message suppressed: {e}", + "Open Job Description: Log redaction failed; message suppressed: " + f"{_describe_exception(e)}", extra=LogExtraInfo(openjd_log_content=LogContent.EXCEPTION_INFO), ) @@ -415,7 +443,8 @@ def _invoke_callback(self, kind: ActionMessageKind, value: Any, cancel_and_fail: self._callback(kind, value, cancel_and_fail) except Exception as e: LOG.error( - f"Open Job Description: Exception in the action message callback: {e}", + "Open Job Description: Exception in the action message callback: " + f"{_describe_exception(e)}", extra=LogExtraInfo(openjd_log_content=LogContent.EXCEPTION_INFO), ) @@ -502,12 +531,6 @@ def _parse_env_variable(self, message: str) -> tuple[str, str, bool, int, Option raise ValueError( f"Unterminated string starting at: line {e.lineno} column {e.colno} (char {e.pos})" ) - # Re-check the NAME after decoding. The regex above matched the raw - # message, so for the JSON form it validated the escaped text; JSON - # decoding happens afterwards and could in principle yield a name - # this runtime should never hand onwards. - if any(c in name for c in _ENVVAR_NAME_FORBIDDEN): - return "", "", False, equals_position, None return name, value, True, equals_position, original_value except json.JSONDecodeError as e: raise ValueError( diff --git a/src/openjd/sessions/_runner_base.py b/src/openjd/sessions/_runner_base.py index 4d3e78ab..65facf5f 100644 --- a/src/openjd/sessions/_runner_base.py +++ b/src/openjd/sessions/_runner_base.py @@ -519,6 +519,15 @@ class ScriptRunnerBase(ABC): """True iff the subprocess was canceled but action needs to be notified as FAILED. """ + _launched: bool + """True once a launch has been *attempted*, successfully or not. + + Backs the single-use guard in :meth:`_run`. Deliberately a latch of its own + rather than something inferred from :attr:`state`: a runner that tried to + launch and failed must never be reusable, and tying that rule to the state + classification made it break whenever the classification changed. + """ + _pending_cancel: Optional[tuple[Optional[timedelta], bool]] """A cancel that arrived before the subprocess existed, as ``(time_limit, mark_action_failed)``. @@ -618,6 +627,7 @@ def __init__( self._state_override = None self._resolved_cancel_method = None self._pending_cancel = None + self._launched = False self._print_section_banner = True # Context manager for use in our tests @@ -754,8 +764,22 @@ def _run(self, args: Sequence[str], time_limit: Optional[timedelta] = None) -> N # may run under self._lock — it is a plain Lock, not an RLock. launch_failure: Optional[str] = None with self._lock: - if self.state != ScriptRunnerState.READY: + # The single-use guard is latched on `_launched`, NOT derived from + # `state`. Deriving it from `state` made the guard depend on every + # future change to that property's classification: when the R5-6 + # change made `state` report READY for a launch that had failed after + # `_process` was assigned, this guard silently opened and a second + # subprocess was launched over the first, orphaning it. A + # "has a launch been attempted" latch cannot drift that way. + # + # `state` is still consulted, because it carries the `_state_override` + # that `_fail_action` sets -- a runner whose setup already failed is + # not READY either. + if self._launched or self.state != ScriptRunnerState.READY: raise RuntimeError("This cannot be used to run a second subprocess.") + # Set before anything that can fail: a runner that has *attempted* a + # launch must never be reusable, whether or not the attempt worked. + self._launched = True if is_posix(): script = self._generate_command_shell_script(args) filehandle, filename = mkstemp( @@ -788,7 +812,7 @@ def _run(self, args: Sequence[str], time_limit: Optional[timedelta] = None) -> N # Only proceed with subprocess creation if no launch failure occurred if launch_failure is None: subprocess_args = [filename] if is_posix() else args - self._process = LoggingSubprocess( + process = LoggingSubprocess( logger=self._logger, args=subprocess_args, user=self._user, @@ -802,7 +826,17 @@ def _run(self, args: Sequence[str], time_limit: Optional[timedelta] = None) -> N if self._print_section_banner: log_subsection_banner(self._logger, "Phase: Running action") - self._run_future = self._pool.submit(self._process.run) + run_future = self._pool.submit(process.run) + + # Publish the pair together, and only once both exist. `state` is + # read without the lock, so assigning `self._process` before the + # future existed left a window -- on the ordinary success path, + # not only on failure -- in which a polling consumer observed a + # process with no future. That combination is what the R5-6 change + # then had to classify, and misclassifying it is what opened the + # guard above. Removing the window removes the question. + self._process = process + self._run_future = run_future # Dispatch launch failure outside the lock if launch_failure is not None: @@ -875,10 +909,17 @@ def _generate_command_shell_script(self, args: Sequence[str]) -> str: # subprocess through Popen's `env=`, which does not go # through a shell, so nothing that used to work stops # working. + # The warning does not promise the variable still arrives. + # For a same-user action it does, via Popen's `env=`. For a + # cross-user action it does not: the subprocess is launched + # through `sudo -u -i`, and `-i` starts a login shell + # that resets the environment, so this script's `export` lines + # are the only channel that reaches the workload. self._logger.warning( f"Not exporting environment variable {name!r} from the action's shell " - "script: the name is not a valid POSIX shell identifier. The variable is " - "still passed to the subprocess directly.", + "script: the name is not a valid POSIX shell identifier, which /bin/sh " + "cannot express. For an action run as another user this means the " + "variable will not reach the subprocess at all.", extra=LogExtraInfo(openjd_log_content=LogContent.PARAMETER_INFO), ) continue @@ -1240,13 +1281,14 @@ def _cancel( def _on_process_exit(self, future: Future) -> None: """This is invoked as a callback when run_future is done.""" - # R5-6: use the future the completion actually fired for, rather than - # asserting on the instance attribute. This runs on the pool worker (or, - # for a very fast process, on the launching thread), so an AssertionError - # here would only surface through threading.excepthook -- and under - # `python -O` the assert is stripped and the `.exception()` read below - # becomes an AttributeError in the same unobservable place. - run_future = self._run_future if self._run_future is not None else future + # R5-6: read the exception from the future the completion actually fired + # for. This runs on the pool worker (or, for a very fast process, on the + # launching thread), so an AssertionError on `self._run_future` would only + # surface through threading.excepthook -- and under `python -O` the assert + # is stripped and the `.exception()` read below becomes an AttributeError + # in the same unobservable place. `future` is always the right object; + # `self._run_future` can additionally be cleared by cleanup(). + run_future = future with self._lock: # F2 fix: Claim _pending_cancel atomically before signalling completion. # A cancel racing completion would otherwise see the process still @@ -1277,7 +1319,7 @@ def _on_process_exit(self, future: Future) -> None: # lock since it may be slow and shouldn't block other operations. if self._callback is not None: try: - self._callback(ActionState(self.state.value)) + self._callback(self._terminal_action_state()) except Exception as exc: self._logger.error( f"Exception in action callback: {exc}", @@ -1286,6 +1328,34 @@ def _on_process_exit(self, future: Future) -> None: ), ) + def _terminal_action_state(self) -> ActionState: + """The :class:`ActionState` to publish now that the action is over. + + ``ActionState`` has no member for ``ScriptRunnerState.READY`` or + ``CANCELING``, so ``ActionState(self.state.value)`` raises ``ValueError`` + for either. That mattered: the F8 ``try/except`` around the callback + catches the ValueError and logs it, so the consumer receives **no + terminal callback at all** and waits forever on an action that has + finished. Reached when the runner's state cannot be classified -- e.g. + the process/future pair left inconsistent by a failed launch. + + The action is over by the time we are called, so an unclassifiable state + is reported as FAILED. Publishing a wrong-but-terminal state is strictly + better than publishing nothing. + """ + state = self.state + try: + return ActionState(state.value) + except ValueError: + self._logger.error( + f"Action completed in the non-terminal state '{state.value}'; reporting it as " + "FAILED. This indicates an internal error in this runtime.", + extra=LogExtraInfo( + openjd_log_content=LogContent.PROCESS_CONTROL | LogContent.EXCEPTION_INFO + ), + ) + return ActionState.FAILED + def _on_notify_period_end(self) -> None: """This is invoked when the grace period in a NOTIFY_THEN_TERMINATE cancelation has expired. diff --git a/src/openjd/sessions/_tempdir.py b/src/openjd/sessions/_tempdir.py index e15a1ddf..b1659c00 100644 --- a/src/openjd/sessions/_tempdir.py +++ b/src/openjd/sessions/_tempdir.py @@ -32,9 +32,9 @@ """ -def _validate_temp_dir_ownership(temp_dir: str) -> None: - """Refuse to use the shared temporary root unless it is a real directory - owned by this process' user (or root). +def _prepare_temp_dir_root(temp_dir: str) -> None: + """Validate the shared temporary root and set its mode, operating on a single + file descriptor so the two cannot disagree. Why this exists: the root is a *fixed, predictable* path (``/OpenJD``) and on POSIX its parent is typically world-writable @@ -45,36 +45,69 @@ def _validate_temp_dir_ownership(temp_dir: str) -> None: restricts deletion within ``/tmp`` itself, not within an attacker-owned ``/tmp/OpenJD``. - ``os.lstat`` rather than ``os.stat``, because the whole point is to see a - symlink rather than follow it. + Why a descriptor rather than the path: an earlier version of this validated + with ``os.lstat(path)`` and then called ``os.stat(path)``/``os.chmod(path)``. + Both of those resolve the *name* again and follow symlinks, so replacing the + entry with a symlink in between defeated the check entirely -- and the chmod + then widened the link's target to 0o755. Opening once with ``O_NOFOLLOW`` and + ``O_DIRECTORY`` and using ``fstat``/``fchmod`` means every decision and every + modification applies to the same inode we validated, whatever happens to the + name afterwards. + + ``O_NOFOLLOW`` makes the open itself fail on a symlink, which is why there is + no separate ``S_ISLNK`` branch on POSIX. Neither flag exists on Windows, where + the ``getattr(..., 0)`` fallbacks make this a plain open and the reparse-point + check below carries the equivalent weight. Raises: - RuntimeError: if the path is a symlink/reparse point, is not a - directory, or is owned by another user. + RuntimeError: if the path is a symlink or reparse point, is not a + directory, is owned by another user, or its mode cannot be set. """ + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_DIRECTORY", 0) try: - st = os.lstat(temp_dir) + fd = os.open(temp_dir, flags) except OSError as err: - raise RuntimeError(f"Could not inspect the temporary directory {temp_dir}: {err}") - - if stat.S_ISLNK(st.st_mode) or getattr(st, "st_reparse_tag", 0) != 0: - raise RuntimeError( - f"Refusing to use temporary directory {temp_dir}: it is a link or reparse point " - "rather than a real directory. Remove it, or pass an explicit session root directory." - ) - if not stat.S_ISDIR(st.st_mode): + # ELOOP here is the O_NOFOLLOW refusal; ENOTDIR is O_DIRECTORY's. raise RuntimeError( - f"Refusing to use temporary directory {temp_dir}: it exists but is not a directory." + f"Refusing to use temporary directory {temp_dir}: it could not be opened as a real " + f"directory ({err}). If it is a symbolic link or not a directory, remove it, or pass " + "an explicit session root directory." ) - if is_posix(): - this_uid = os.geteuid() - # root is accepted so that a system-provisioned root directory works. - if st.st_uid not in (this_uid, 0): + try: + st = os.fstat(fd) + if not stat.S_ISDIR(st.st_mode) or getattr(st, "st_reparse_tag", 0) != 0: raise RuntimeError( - f"Refusing to use temporary directory {temp_dir}: it is owned by uid " - f"{st.st_uid}, not by this process' uid ({this_uid}) or root. Another user " - "may have created it. Remove it, or pass an explicit session root directory." + f"Refusing to use temporary directory {temp_dir}: it is not a real directory." ) + if is_posix(): + this_uid = os.geteuid() + # root is accepted so that a system-provisioned root directory works. + if st.st_uid not in (this_uid, 0): + raise RuntimeError( + f"Refusing to use temporary directory {temp_dir}: it is owned by uid " + f"{st.st_uid}, not by this process' uid ({this_uid}) or root. Another user " + "may have created it. Remove it, or pass an explicit session root directory." + ) + + # makedirs()' `mode` is masked by the process umask, so it does not on its + # own guarantee anything: under umask 0o077 an "explicit" 0o755 lands as + # 0o700, and a job user could then not traverse into its own session + # directory. Set the mode outright. Conditional because a + # system-provisioned root may be owned by root with the right mode + # already, where the call would fail for no reason. + if stat.S_IMODE(st.st_mode) != stat.S_IMODE(OPENJD_TEMPDIR_MODE): + try: + # fchmod, not chmod: applies to the validated inode, not to + # whatever the name resolves to now. + os.fchmod(fd, OPENJD_TEMPDIR_MODE) + except OSError as err: + raise RuntimeError( + f"Could not set permissions on temporary directory {temp_dir}: {err}. " + "Session working directories created beneath it may not be reachable by a " + "job user." + ) + finally: + os.close(fd) def custom_gettempdir(logger: Optional[LoggerAdapter] = None) -> str: @@ -95,7 +128,7 @@ def custom_gettempdir(logger: Optional[LoggerAdapter] = None) -> str: Raises: RuntimeError: If the directory could not be created, or if it already exists and is not a directory owned by this process' user (see - :func:`_validate_temp_dir_ownership`). + :func:`_prepare_temp_dir_root`). """ if is_windows(): program_data_path = os.getenv("PROGRAMDATA") @@ -119,25 +152,9 @@ def custom_gettempdir(logger: Optional[LoggerAdapter] = None) -> str: # R5-3 fix: exist_ok=True accepts whatever is already at this predictable # path -- another local user's directory, or a symlink pointing elsewhere. - # Validate BEFORE the chmod below, so we never modify an entry that turns out - # not to be ours. - _validate_temp_dir_ownership(temp_dir) - - # makedirs()' `mode` is masked by the process umask, so it does not on its own - # guarantee anything: under umask 0o077 the "explicit" 0o755 above lands as - # 0o700, and a job user could then not traverse into its own session - # directory. Set the mode outright. Conditional because a - # system-provisioned root may be owned by root with the right mode already, - # where chmod would fail for no reason. - try: - current_mode = stat.S_IMODE(os.stat(temp_dir).st_mode) - if current_mode != stat.S_IMODE(OPENJD_TEMPDIR_MODE): - os.chmod(temp_dir, mode=OPENJD_TEMPDIR_MODE) - except OSError as err: - raise RuntimeError( - f"Could not set permissions on temporary directory {temp_dir}: {err}. Session " - "working directories created beneath it may not be reachable by a job user." - ) + # Validate and set the mode through one descriptor, so the entry cannot be + # swapped between the two. + _prepare_temp_dir_root(temp_dir) return temp_dir diff --git a/test/openjd/sessions_v0/test_round5_fixes.py b/test/openjd/sessions_v0/test_round5_fixes.py index 28bad2e1..88df1f7c 100644 --- a/test/openjd/sessions_v0/test_round5_fixes.py +++ b/test/openjd/sessions_v0/test_round5_fixes.py @@ -322,26 +322,39 @@ def test_multiline_value_via_json_is_still_supported(self) -> None: assert len(env_calls) == 1 assert env_calls[0][0][1] == {"name": "FOO", "value": "BAR\nBAZ"} - def test_a_separator_cannot_reach_a_decoded_name(self) -> None: - """Defence in depth: the regex matches the *raw* message, but JSON - decoding happens afterwards, so the decoded name is re-checked.""" - # GIVEN a filter whose callback records what it is handed - callback = MagicMock() - f = ActionMonitoringFilter(session_id="foo", callback=callback) - - # WHEN every ENV message it sees is processed - for msg in ( + @pytest.mark.parametrize( + "msg", + [ 'openjd_env: "FOO\\nBAR=baz"', 'openjd_env: "FOO\\u000aBAR=baz"', 'openjd_env: "FOO\\u0000BAR=baz"', - ): - f.filter(_make_record(msg)) + ], + ) + def test_a_separator_cannot_reach_a_decoded_name(self, msg: str) -> None: + """A name carrying a separator must be rejected outright, not passed on. + + Rewritten after an audit: the previous version looped over the recorded + calls asserting a property of any dict it found, and since these inputs + produce no ENV dict at all, its assertion body never executed. It passed + against a deliberately broken implementation. Assert the rejection + directly instead. + """ + # GIVEN + callback = MagicMock() + f = ActionMonitoringFilter(session_id="foo", callback=callback) - # THEN no name carrying a separator was ever handed onwards - for call in callback.call_args_list: - value = call[0][1] - if isinstance(value, dict): - assert not any(c in value["name"] for c in ("\n", "\r", "\0", "=")) + # WHEN + f.filter(_make_record(msg)) + + # THEN: no environment variable was defined, and the failure was reported. + env_defs = [ + c + for c in callback.call_args_list + if c[0][0] == ActionMessageKind.ENV and isinstance(c[0][1], dict) + ] + assert env_defs == [] + assert callback.call_args_list, "the parse failure must be reported to the consumer" + assert callback.call_args_list[-1][0][2] is True # cancel-and-fail # =========================================================================== @@ -378,9 +391,10 @@ def test_rejects_a_symlink_at_the_root_path(self, tmp_path: Path) -> None: parent.mkdir() (parent / "OpenJD").symlink_to(elsewhere, target_is_directory=True) - # WHEN / THEN: we refuse rather than creating sessions inside it + # WHEN / THEN: we refuse rather than creating sessions inside it. The + # refusal now comes from O_NOFOLLOW on the open itself (see REG-3). with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): - with pytest.raises(RuntimeError, match="link or reparse point"): + with pytest.raises(RuntimeError, match="real directory"): custom_gettempdir() def test_rejects_a_root_that_is_not_a_directory(self, tmp_path: Path) -> None: @@ -394,29 +408,39 @@ def test_rejects_a_root_that_is_not_a_directory(self, tmp_path: Path) -> None: with pytest.raises(RuntimeError): custom_gettempdir() - @pytest.mark.skipif(not is_posix(), reason="uid ownership check is POSIX-only") - def test_rejects_a_root_owned_by_another_user(self, tmp_path: Path) -> None: - # GIVEN: the root exists but lstat reports a different owner - parent = tmp_path / "parent" - parent.mkdir() - (parent / "OpenJD").mkdir() - real_lstat = os.lstat - foreign_uid = os.geteuid() + 12345 + @staticmethod + def _fstat_reporting_uid(uid: int) -> Any: + """Wrap os.fstat so the root directory appears owned by `uid`. + + Patches fstat rather than lstat: the implementation validates through a + descriptor now (see REG-3), so an lstat mock would not be consulted and + the test would vacuously pass. + """ + real_fstat = os.fstat class _Stat: def __init__(self, base: os.stat_result) -> None: self.st_mode = base.st_mode - self.st_uid = foreign_uid + self.st_uid = uid - def fake_lstat(path: Any, *a: Any, **k: Any) -> Any: - base = real_lstat(path, *a, **k) - if str(path).endswith("OpenJD"): - return _Stat(base) - return base + def fake_fstat(fd: int, *a: Any, **k: Any) -> Any: + return _Stat(real_fstat(fd, *a, **k)) + + return fake_fstat + + @pytest.mark.skipif(not is_posix(), reason="uid ownership check is POSIX-only") + def test_rejects_a_root_owned_by_another_user(self, tmp_path: Path) -> None: + # GIVEN: the root exists but is owned by a different uid + parent = tmp_path / "parent" + parent.mkdir() + (parent / "OpenJD").mkdir() # WHEN / THEN with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): - with patch("openjd.sessions._tempdir.os.lstat", side_effect=fake_lstat): + with patch( + "openjd.sessions._tempdir.os.fstat", + side_effect=self._fstat_reporting_uid(os.geteuid() + 12345), + ): with pytest.raises(RuntimeError, match="owned by uid"): custom_gettempdir() @@ -427,22 +451,12 @@ def test_accepts_a_root_owned_by_root(self, tmp_path: Path) -> None: parent = tmp_path / "parent" parent.mkdir() (parent / "OpenJD").mkdir() - real_lstat = os.lstat - - class _Stat: - def __init__(self, base: os.stat_result) -> None: - self.st_mode = base.st_mode - self.st_uid = 0 - - def fake_lstat(path: Any, *a: Any, **k: Any) -> Any: - base = real_lstat(path, *a, **k) - if str(path).endswith("OpenJD"): - return _Stat(base) - return base # WHEN / THEN: no exception with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): - with patch("openjd.sessions._tempdir.os.lstat", side_effect=fake_lstat): + with patch( + "openjd.sessions._tempdir.os.fstat", side_effect=self._fstat_reporting_uid(0) + ): assert custom_gettempdir() == str(parent / "OpenJD") diff --git a/test/openjd/sessions_v0/test_round5_regressions.py b/test/openjd/sessions_v0/test_round5_regressions.py new file mode 100644 index 00000000..24fed463 --- /dev/null +++ b/test/openjd/sessions_v0/test_round5_regressions.py @@ -0,0 +1,441 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Regression tests for defects introduced or left open BY the round-5 fix +commit (4c57f40), found by a five-agent sweep of that commit. + +The most serious is REG-1: the R5-6 conversion of `ScriptRunnerBase.state`'s +`assert self._run_future is not None` into `return READY` turned a latent +AssertionError into silent state corruption. Two independent reviewers found it. + +Every test here was mutation-checked: the fix was reverted and the test was +confirmed to fail. See tmp-free `scripts/` note in the commit message. +""" + +import logging +import os +import stat +import sys +import threading +from concurrent.futures import Future, ThreadPoolExecutor +from datetime import timedelta +from pathlib import Path +from typing import Any, Optional +from unittest.mock import MagicMock, patch + +import pytest + +from openjd.sessions._action_filter import ActionMessageKind, ActionMonitoringFilter +from openjd.sessions._os_checker import is_posix +from openjd.sessions._runner_base import ScriptRunnerBase, ScriptRunnerState +from openjd.sessions._tempdir import OPENJD_TEMPDIR_MODE, custom_gettempdir +from openjd.sessions._types import ActionState + + +class _Runner(ScriptRunnerBase): + """Minimal concrete runner.""" + + def cancel( + self, *, time_limit: Optional[timedelta] = None, mark_action_failed: bool = False + ) -> None: + pass + + +def _make_record(msg: str, args: Any = None, session_id: str = "foo") -> logging.LogRecord: + record = logging.LogRecord("test", logging.INFO, "path", 1, msg, args, None) + record.session_id = session_id # type: ignore[attr-defined] + return record + + +# =========================================================================== +# REG-1 -- the R5-6 `state` conversion must not corrupt the runner +# =========================================================================== + + +class TestReg1FailedLaunchStateCorruption: + """A launch that fails after the subprocess object exists must not leave the + runner looking reusable, and a completed action must always publish a + terminal state. + + `_run` gates re-entry on `state == READY`. When R5-6 made `state` report + READY for the (process set, future unset) pair, that guard silently opened. + And `_on_process_exit` built `ActionState(self.state.value)`, which raises + `ValueError` for 'ready' -- swallowed by the F8 try/except, so the consumer + received no terminal callback at all and would wait forever. + """ + + def _runner_with_dead_pool(self, tmp_path: Path, published: list) -> _Runner: + runner = _Runner( + logger=MagicMock(), + session_working_directory=tmp_path, + callback=lambda s: published.append(s.value), + ) + # Exactly what Session.cleanup() does to the runner's pool. A cleanup + # racing a launch is the ordinary agent SIGTERM path. + runner._pool.shutdown(wait=True) + return runner + + def test_second_launch_is_refused_after_a_failed_launch(self, tmp_path: Path) -> None: + # GIVEN: a runner whose first launch fails inside _pool.submit + published: list = [] + runner = self._runner_with_dead_pool(tmp_path, published) + try: + with pytest.raises(RuntimeError): + runner._run([sys.executable, "-c", "pass"], timedelta(seconds=600)) + + # WHEN: the runner is handed a working pool and asked to run again, + # exactly as a caller that caught the RuntimeError might. + runner._pool = ThreadPoolExecutor(max_workers=1) + + # THEN: the single-use guard holds. Before the fix this launched a + # second real child and replaced _process, orphaning the first. + with pytest.raises(RuntimeError, match="cannot be used to run a second subprocess"): + runner._run([sys.executable, "-c", "import time; time.sleep(30)"], None) + finally: + runner.cancel() + for timer in [t for t in threading.enumerate() if isinstance(t, threading.Timer)]: + timer.cancel() + runner.shutdown() + + def test_launch_latch_holds_even_when_state_reports_ready(self, tmp_path: Path) -> None: + """Pins the guard to `_launched` rather than to the state classification. + + This is the property that actually broke: whatever `state` decides to + report for an inconsistent runner, re-entry must still be refused. + """ + # GIVEN: a runner that has attempted a launch, with `state` forced to + # report READY (the exact misclassification that opened the guard) + runner = _Runner(logger=MagicMock(), session_working_directory=tmp_path) + try: + runner._pool.shutdown(wait=True) + with pytest.raises(RuntimeError): + runner._run([sys.executable, "-c", "pass"], None) + runner._pool = ThreadPoolExecutor(max_workers=1) + + with patch.object( + type(runner), "state", property(lambda self: ScriptRunnerState.READY) + ): + assert runner.state == ScriptRunnerState.READY # the hostile premise + # THEN + with pytest.raises(RuntimeError, match="second subprocess"): + runner._run([sys.executable, "-c", "pass"], None) + finally: + runner.shutdown() + + def test_completion_always_publishes_a_terminal_state(self, tmp_path: Path) -> None: + # GIVEN: a runner whose process/future pair is inconsistent, so `state` + # cannot be classified as a terminal ActionState + published: list = [] + runner = _Runner( + logger=MagicMock(), + session_working_directory=tmp_path, + callback=lambda s: published.append(s), + ) + try: + runner._process = MagicMock() + runner._run_future = None + completed: Future = Future() + completed.set_result(None) + + # WHEN + runner._on_process_exit(completed) + + # THEN: exactly one terminal callback. Before the fix this list was + # empty: ActionState('ready') raised and the F8 handler ate it. + assert len(published) == 1 + assert published[0] in ( + ActionState.FAILED, + ActionState.CANCELED, + ActionState.TIMEOUT, + ActionState.SUCCESS, + ) + assert published[0] == ActionState.FAILED + finally: + runner.shutdown() + + def test_terminal_action_state_maps_every_runner_state(self, tmp_path: Path) -> None: + """No ScriptRunnerState may make the terminal mapping raise.""" + # GIVEN + runner = _Runner(logger=MagicMock(), session_working_directory=tmp_path) + try: + for runner_state in ScriptRunnerState: + + def fixed_state(_self: Any, _s: ScriptRunnerState = runner_state) -> Any: + return _s + + # WHEN + with patch.object(type(runner), "state", property(fixed_state)): + result = runner._terminal_action_state() + # THEN + assert isinstance(result, ActionState) + finally: + runner.shutdown() + + def test_process_and_future_are_published_together(self, tmp_path: Path) -> None: + """Removes the root cause: a reader must never see a process with no + future, on the success path or any other.""" + # GIVEN: a poller hammering `state` from another thread during launch + observations: list[tuple[bool, bool]] = [] + stop = threading.Event() + runner = _Runner(logger=MagicMock(), session_working_directory=tmp_path) + + def poll() -> None: + while not stop.is_set(): + observations.append((runner._process is not None, runner._run_future is not None)) + + poller = threading.Thread(target=poll, daemon=True) + try: + poller.start() + # WHEN + runner._run([sys.executable, "-c", "pass"], None) + stop.set() + poller.join(timeout=10) + + # THEN: the (process, no future) combination was never observable. + assert (True, False) not in observations + assert observations, "the poller must actually have sampled something" + finally: + stop.set() + runner.shutdown() + + def test_a_normal_run_still_reports_success(self, tmp_path: Path) -> None: + """Guard against the latch or the restructure breaking the happy path.""" + # GIVEN + published: list = [] + runner = _Runner( + logger=MagicMock(), + session_working_directory=tmp_path, + callback=lambda s: published.append(s), + ) + try: + # WHEN + runner._run([sys.executable, "-c", "pass"], None) + assert runner._run_future is not None + runner._run_future.result(timeout=30) + + # THEN + deadline = 30.0 + import time + + start = time.monotonic() + while ActionState.SUCCESS not in published and time.monotonic() - start < deadline: + time.sleep(0.02) + assert runner.state == ScriptRunnerState.SUCCESS + assert ActionState.SUCCESS in published + finally: + runner.shutdown() + + +# =========================================================================== +# REG-2 -- the R5-2 containment must not itself re-raise +# =========================================================================== + + +class _UnrenderableError(Exception): + """An exception whose rendering raises -- the shape that defeated the R5-2 + containment, which interpolated the exception into an f-string.""" + + def __str__(self) -> str: + raise RuntimeError("__str__ is hostile") + + def __repr__(self) -> str: + raise RuntimeError("__repr__ is hostile too") + + +class TestReg2ContainmentDoesNotReRaise: + def test_handler_path_contains_an_unrenderable_exception(self) -> None: + # GIVEN: a consumer callback raising an exception that cannot be rendered + def callback(kind: ActionMessageKind, value: Any, fail: bool) -> None: + raise _UnrenderableError() + + f = ActionMonitoringFilter(session_id="foo", callback=callback) + record = _make_record("openjd_progress: 50.0") + + # WHEN / THEN: filter() still returns rather than letting the exception + # reach the stdout pump thread. + assert f.filter(record) is True + assert "_UnrenderableError" in record.getMessage() + + def test_malformed_env_path_contains_an_unrenderable_exception(self) -> None: + # GIVEN + def callback(kind: ActionMessageKind, value: Any, fail: bool) -> None: + raise _UnrenderableError() + + f = ActionMonitoringFilter(session_id="foo", callback=callback) + + # WHEN / THEN + assert f.filter(_make_record("openjd_env : FOO=bar")) is True + + def test_renderable_exceptions_still_report_their_message(self) -> None: + """The defensive rendering must not degrade the ordinary case.""" + + # GIVEN + def callback(kind: ActionMessageKind, value: Any, fail: bool) -> None: + raise RuntimeError("a perfectly ordinary boom") + + f = ActionMonitoringFilter(session_id="foo", callback=callback) + record = _make_record("openjd_progress: 50.0") + + # WHEN + f.filter(record) + + # THEN + assert "a perfectly ordinary boom" in record.getMessage() + + +# =========================================================================== +# REG-3 -- the R5-3 validation must not be defeatable by a symlink swap +# =========================================================================== + + +@pytest.mark.skipif(not is_posix(), reason="symlink swap and fchmod semantics are POSIX here") +class TestReg3TempRootCheckThenUse: + """The first R5-3 implementation validated with `lstat(path)` and then called + `stat(path)`/`chmod(path)`, both of which re-resolve the name and follow + links. Swapping the entry for a symlink in between defeated the check and + chmod'ed the link's target to 0o755.""" + + def test_a_symlink_swapped_in_before_the_open_is_refused(self, tmp_path: Path) -> None: + """Swap during the create window: O_NOFOLLOW must refuse the open.""" + # GIVEN: a victim directory at 0o700 and a parent where the root will live + victim = tmp_path / "victim" + victim.mkdir() + os.chmod(victim, 0o700) + parent = tmp_path / "parent" + parent.mkdir() + root = parent / "OpenJD" + + real_makedirs = os.makedirs + + def makedirs_then_swap(path: Any, *a: Any, **k: Any) -> None: + real_makedirs(path, *a, **k) + if str(path) == str(root): + os.rmdir(root) + os.symlink(victim, root, target_is_directory=True) + + # WHEN + with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): + with patch("openjd.sessions._tempdir.os.makedirs", side_effect=makedirs_then_swap): + with pytest.raises(RuntimeError): + custom_gettempdir() + + # THEN + assert stat.S_IMODE(os.stat(victim).st_mode) == 0o700 + + def test_a_symlink_swapped_in_after_the_open_cannot_redirect_the_chmod( + self, tmp_path: Path + ) -> None: + """Swap *after* the descriptor is open, which O_NOFOLLOW cannot help with. + + This is the case that pins fchmod-vs-chmod. An earlier version of this + test swapped during the create window instead, where O_NOFOLLOW refuses + the open before any chmod is attempted -- so it passed even with + `os.chmod(path)` restored, and pinned nothing about the descriptor. The + swap is injected from inside `os.fstat`, which the implementation calls + between the open and the mode change. + """ + # GIVEN: a victim at 0o700, and a real root whose mode needs correcting + victim = tmp_path / "victim_after" + victim.mkdir() + os.chmod(victim, 0o700) + parent = tmp_path / "parent_after" + parent.mkdir() + root = parent / "OpenJD" + root.mkdir() + os.chmod(root, 0o700) # != OPENJD_TEMPDIR_MODE, so a mode change is due + + real_fstat = os.fstat + swapped = {"done": False} + + def fstat_then_swap(fd: int, *a: Any, **k: Any) -> Any: + result = real_fstat(fd, *a, **k) + if not swapped["done"]: + swapped["done"] = True + os.rmdir(root) + os.symlink(victim, root, target_is_directory=True) + return result + + # WHEN + with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): + with patch("openjd.sessions._tempdir.os.fstat", side_effect=fstat_then_swap): + custom_gettempdir() + + # THEN: the swap happened, and the victim was NOT widened. With + # `os.chmod(temp_dir, ...)` restored this is 0o755. + assert swapped["done"] is True + assert os.path.islink(root) + assert stat.S_IMODE(os.stat(victim).st_mode) == 0o700 + + def test_the_validated_inode_is_the_one_modified(self, tmp_path: Path) -> None: + """Even when the swap is not detected as an error, the mode must land on + the inode that was validated, never on a later resolution of the name.""" + # GIVEN: an existing root at a wrong mode, plus a victim + victim = tmp_path / "victim2" + victim.mkdir() + os.chmod(victim, 0o700) + parent = tmp_path / "parent2" + parent.mkdir() + root = parent / "OpenJD" + root.mkdir() + os.chmod(root, 0o700) # differs from OPENJD_TEMPDIR_MODE, so a chmod is due + + # WHEN + with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): + result = custom_gettempdir() + + # THEN: the real root got the mode; the victim was not touched. + assert result == str(root) + assert stat.S_IMODE(os.stat(root).st_mode) == stat.S_IMODE(OPENJD_TEMPDIR_MODE) + assert stat.S_IMODE(os.stat(victim).st_mode) == 0o700 + + def test_a_symlinked_root_is_still_refused(self, tmp_path: Path) -> None: + """O_NOFOLLOW must keep doing what the explicit S_ISLNK branch did.""" + # GIVEN + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + parent = tmp_path / "parent3" + parent.mkdir() + (parent / "OpenJD").symlink_to(elsewhere, target_is_directory=True) + + # WHEN / THEN + with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): + with pytest.raises(RuntimeError, match="real directory"): + custom_gettempdir() + + def test_no_descriptor_is_leaked(self, tmp_path: Path) -> None: + """The validation opens a descriptor; every path must close it. + + Probe: the lowest free descriptor number. If a call leaks a descriptor, + the number the kernel hands out next goes up. Portable across macOS and + Linux, unlike listing /dev/fd. + """ + # GIVEN + parent = tmp_path / "parent4" + parent.mkdir() + (parent / "OpenJD").mkdir() + + def lowest_free_fd() -> int: + fd = os.open(os.devnull, os.O_RDONLY) + os.close(fd) + return fd + + with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): + custom_gettempdir() # warm up, so first-call allocations do not count + baseline = lowest_free_fd() + + # WHEN: many successful calls + for _ in range(25): + custom_gettempdir() + after_success = lowest_free_fd() + + # WHEN: many calls that fail *after* the descriptor is opened + with patch( + "openjd.sessions._tempdir.os.fstat", side_effect=OSError(5, "induced failure") + ): + for _ in range(25): + with pytest.raises(OSError): + custom_gettempdir() + after_failure = lowest_free_fd() + + # THEN: no drift on either path. + assert after_success == baseline + assert after_failure == baseline From c192911bde27dc1c56796b4b134d849c03d745f7 Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Sun, 26 Jul 2026 11:54:27 -0700 Subject: [PATCH 16/27] fix(wrap-actions): Isolate a wrap hook's scope from the wrapped entity's RFC 0008 specifies two strictly separated resolution scopes: the wrapped action resolves against the INNER entity's own scope, and the hook resolves against the WRAP environment's own scope plus the WrappedAction.* overlay. Only one direction was actually isolated. `_build_wrapped_inner_scope` resolves the wrapped action against a copy, so wrap -> inner was closed. But the hook resolved against the inner entity's own symbol table, so everything the Session writes into that table directly was readable from a hook: - a wrapped task's Task.Param.* / Task.RawParam.* - the running step's Step.Name - the extra_let_bindings the *inner* environment was entered with Reproduced: a hook argument of `{{Task.Param.Frame}}` resolved to the wrapped task's frame number, and `{{Step.Name}}` to the running step's name. `_seed_wrap_env_scope` runs after injection, so it only re-seeded the wrap env's own context -- it never removed the inner entity's. This matters because a wrap environment and the job it wraps need not have the same author: the wrapping environment is the mechanism by which an operator interposes a container runtime or a license gate, and it should not thereby gain read access to the work it is wrapping. RFC 0008 supplies WrappedStep.Name precisely because Step.Name is not meant to be reachable from a hook, and openjd-model rejects none of these references in an environment script, so this runtime was the only gate. An inner script's *script-level* `let` bindings never leaked -- those live only in the copy -- and that part was already covered by test_wrap_actions.py::test_inner_env_let_does_not_leak_into_hook_scope. Fix: a new `_build_wrap_hook_scope()` builds the hook a fresh session-scope table rather than reusing the inner entity's, and all three hook call sites (enter_environment, exit_environment, run_task) inject into and run against it. The path-mapping symbols are copied over rather than re-materialized so both scopes name the same rules file and no second file is written. A hook still gets everything it legitimately needs: session scope, Job.Name, job parameters, the path-mapping symbols and the engine's host rules, its own environment's enter-time step name and extra_let_bindings, the WrappedAction.* overlay, and its own script-level lets and embedded files. Adds test/openjd/sessions_v0/test_wrap_scope_isolation.py (13 tests), which asserts on the symbol table the Session actually hands to the hook's runner rather than on a table the test built itself. Three subagents audited the tests independently. Their findings changed the result rather than confirming it: - A gap they found is now covered: a mutant that keeps the hook's EXPR host context but empties its path-mapping rules made apply_path_mapping() inside a hook silently become the identity function, with the whole suite green. No test anywhere combined a wrap environment with path_mapping_rules. Now test_path_mapping_reaches_the_hook_intact does. - Also now covered: EXPR *types* (leaking the inner table's expr_types, and stripping the hook's own) survived every mutation. - Two audit-driven test corrections: the capture helper indexed the most recent runner, so a regression that stopped invoking one hook would have let a test assert against the other hook's table; and `_run_until_ready` returned silently on timeout, so twelve assertions on a table built before the subprocess starts would have passed against a hung action. - Switched from `true`/`echo` to the suite's `python_exe` fixture: neither is a native Windows executable, and every test here now requires its action to complete. - Removed a dead self-justifying helper and a factually wrong claim in the module docstring about what was previously covered. Mutation-checked with 12 mutants, including each of the three call sites independently and the plausible-but-wrong fix of copying the inner table. The harness now verifies each restore by checksum and requires a green baseline: an earlier run silently left one mutation in place, which made every later verdict meaningless. Verified: 813 passed / 39 skipped / 16 xfailed (only the 2 known xdist timing flakes fail; they pass in isolation); ruff, mypy and black clean. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- src/openjd/sessions/_session.py | 95 +++- .../sessions_v0/test_wrap_scope_isolation.py | 526 ++++++++++++++++++ 2 files changed, 603 insertions(+), 18 deletions(-) create mode 100644 test/openjd/sessions_v0/test_wrap_scope_isolation.py diff --git a/src/openjd/sessions/_session.py b/src/openjd/sessions/_session.py index 31c98f14..80e0f06e 100644 --- a/src/openjd/sessions/_session.py +++ b/src/openjd/sessions/_session.py @@ -915,18 +915,21 @@ def enter_environment( if wrap_env is not None: # The wrapped onEnter resolves against the INNER environment's - # own scope; the hook itself resolves against `symtab`, and its - # own script's lets/files are evaluated by the script runner - # from wrap_env.script (see _try_inject_wrapped_symbols). On + # own scope (`symtab`, which carries the step context and + # extra_let_bindings that environment was entered with); the hook + # resolves against its own table, which carries none of them -- see + # _build_wrap_hook_scope. Its own script's lets/files are evaluated + # into that table by the script runner from wrap_env.script. On # failure the environment stays in the entered list, exactly as # when enter() itself fails, so the caller's cleanup exits it # as usual. + hook_symtab = self._build_wrap_hook_scope(environment.revision, symtab) if not self._try_inject_wrapped_symbols( scope=EmbeddedFilesScope.ENV, inner_script=environment.script, symtab=symtab, inject=lambda inner_symtab: self._inject_wrapped_env_symbols( - symtab, environment, on_enter_action, inner_symtab=inner_symtab + hook_symtab, environment, on_enter_action, inner_symtab=inner_symtab ), fail_message=( f"Failed to resolve the wrapped onEnter action of " @@ -934,7 +937,7 @@ def enter_environment( ), ): return identifier - if not self._seed_wrap_env_scope(symtab, wrap_env): + if not self._seed_wrap_env_scope(hook_symtab, wrap_env): return identifier try: wrap_file_records = self._get_wrap_env_file_records(wrap_env) @@ -946,7 +949,7 @@ def enter_environment( self._runner = self._make_env_script_runner( environment_script=wrap_env.script, os_env_vars=action_env_vars, - symtab=symtab, + symtab=hook_symtab, preallocated_file_records=wrap_file_records, ) # Sets the subprocess running; returns once it has started, or has @@ -1092,12 +1095,13 @@ def exit_environment( # See the onWrapEnvEnter path (_try_inject_wrapped_symbols). On # failure the environment was already removed from tracking # above, matching how a failed exit() behaves. + hook_symtab = self._build_wrap_hook_scope(environment.revision, symtab) if not self._try_inject_wrapped_symbols( scope=EmbeddedFilesScope.ENV, inner_script=environment.script, symtab=symtab, inject=lambda inner_symtab: self._inject_wrapped_env_symbols( - symtab, + hook_symtab, environment, on_exit_action, session_env_list=wrapped_session_env_list, @@ -1109,7 +1113,7 @@ def exit_environment( ), ): return - if not self._seed_wrap_env_scope(symtab, wrap_env): + if not self._seed_wrap_env_scope(hook_symtab, wrap_env): return try: wrap_file_records = self._get_wrap_env_file_records(wrap_env) @@ -1121,7 +1125,7 @@ def exit_environment( self._runner = self._make_env_script_runner( environment_script=wrap_env.script, os_env_vars=action_env_vars, - symtab=symtab, + symtab=hook_symtab, preallocated_file_records=wrap_file_records, ) # Set RUNNING first: wrap_env_exit() may fail immediately and set @@ -1238,16 +1242,20 @@ def run_task( "Internal error: a wrap environment is active but no step name was " "resolved for the task." ) - # The wrapped onRun resolves against the STEP's own scope; the - # hook resolves against `symtab`, and the wrap environment's own - # lets/files are evaluated by the script runner from - # wrap_env.script (see _try_inject_wrapped_symbols). + # Two separate scopes. The wrapped onRun resolves against the STEP's + # own scope (`symtab`, which carries this task's parameters and the + # running step's name); the hook resolves against its own table, + # which deliberately carries none of that -- see + # _build_wrap_hook_scope. The wrap environment's own lets/files are + # evaluated into the hook's table by the script runner from + # wrap_env.script. + hook_symtab = self._build_wrap_hook_scope(step_script.revision, symtab) if not self._try_inject_wrapped_symbols( scope=EmbeddedFilesScope.STEP, inner_script=step_script, symtab=symtab, inject=lambda inner_symtab: self._inject_wrapped_task_symbols( - symtab, step_script, step_name, inner_symtab=inner_symtab + hook_symtab, step_script, step_name, inner_symtab=inner_symtab ), fail_message=( f"Failed to resolve the wrapped Task action for {wrap_env.name}'s " @@ -1256,9 +1264,13 @@ def run_task( ): return - # Only now that the wrapped action has resolved in its own scope: - # give the hook the step context its environment was entered with. - if not self._seed_wrap_env_scope(symtab, wrap_env): + # Give the hook the step context its OWN environment was entered + # with. Ordering is no longer load-bearing here: `hook_symtab` is a + # separate table and is never the base of + # `_build_wrapped_inner_scope`, so seeding it cannot reach the + # wrapped action's scope whenever it happens. It stays after + # injection to keep all three hook paths reading the same way. + if not self._seed_wrap_env_scope(hook_symtab, wrap_env): return try: @@ -1271,7 +1283,7 @@ def run_task( self._runner = self._make_env_script_runner( environment_script=wrap_env.script, os_env_vars=action_env_vars, - symtab=symtab, + symtab=hook_symtab, preallocated_file_records=wrap_file_records, ) # Note: unlike enter_environment()/exit_environment(), which set @@ -1663,6 +1675,53 @@ def _wrap_env_identifier(self, wrap_env: EnvironmentModel) -> EnvironmentIdentif ) return identifier + def _build_wrap_hook_scope( + self, version: SpecificationRevision, session_symtab: SymbolTable + ) -> SymbolTable: + """The scope an RFC 0008 wrap hook resolves in. + + Built fresh from session scope rather than derived from the inner + entity's table, and that distinction is the whole point. The inner + entity's table carries symbols belonging to the wrapped work: a task's + ``Task.Param.*``/``Task.RawParam.*``, the running step's ``Step.Name``, + and the ``extra_let_bindings`` the *inner* environment was entered with. + A wrap environment must not be able to read any of them. + + :meth:`_build_wrapped_inner_scope` already blocks the wrap -> inner + direction by resolving the wrapped action against a copy. This is the + other direction, which was open: the hook used to resolve against the + inner entity's own table, so ``{{Task.Param.Frame}}`` in a hook's args + resolved to the wrapped task's frame number and ``{{Step.Name}}`` to the + running step. RFC 0008 supplies ``WrappedStep.Name`` precisely because + ``Step.Name`` is not meant to be reachable from a hook, and the model + does not reject either reference in an environment script, so this was + the only gate. + + What a hook legitimately gets: session scope (``Session.*``, + ``Job.Name``, ``Param.*``/``RawParam.*``), the path-mapping symbols, the + wrap environment's *own* enter-time step context (applied afterwards by + :meth:`_seed_wrap_env_scope`), the ``WrappedAction.*`` overlay, and its + own script-level ``let`` bindings and embedded files (evaluated by the + runner). + + The path-mapping symbols are copied rather than re-materialized: the + rules file has already been written for this action, and both scopes must + name the same file. + """ + hook_symtab = self._symbol_table(version) + for key in ( + ValueReferenceConstants_2023_09.HAS_PATH_MAPPING_RULES.value, + ValueReferenceConstants_2023_09.PATH_MAPPING_RULES_FILE.value, + ): + # _materialize_path_mapping sets the value and its EXPR type + # together, so one membership check covers both. The check itself is + # not decoration: this method is also called on a table that has not + # been through path mapping. + if key in session_symtab: + hook_symtab[key] = session_symtab[key] + hook_symtab.expr_types[key] = session_symtab.expr_types[key] + return hook_symtab + def _seed_wrap_env_scope(self, symtab: SymbolTable, wrap_env: EnvironmentModel) -> bool: """Re-seed the scope a wrap hook resolves in with the step context the wrap environment was *entered* with (RFC 0005 step-level ``let`` diff --git a/test/openjd/sessions_v0/test_wrap_scope_isolation.py b/test/openjd/sessions_v0/test_wrap_scope_isolation.py new file mode 100644 index 00000000..bd8e6f22 --- /dev/null +++ b/test/openjd/sessions_v0/test_wrap_scope_isolation.py @@ -0,0 +1,526 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""RFC 0008 two-scope isolation, in the inner -> hook direction. + +The wrap-actions design rests on two strictly separated scopes: the wrapped +action resolves against the INNER entity's own scope, and the hook resolves +against the WRAP environment's own scope plus the ``WrappedAction.*`` overlay. + +The wrap -> inner direction is covered by ``test_wrap_task_run.py`` and +``test_wrap_actions.py::TestWrapScopeSeparation``, and so is one part of this +direction: ``test_wrap_actions.py::test_inner_env_let_does_not_leak_into_hook_scope`` +already pins that an inner script's *script-level* ``let`` bindings stay out of a +hook's scope -- those live only in the copy ``_build_wrapped_inner_scope`` makes, +so they never leaked. + +What was NOT covered, and did leak, is everything the Session writes into the +inner entity's table directly, because the hook used to resolve against that same +table: + +- a wrapped task's ``Task.Param.*`` / ``Task.RawParam.*`` +- the running step's ``Step.Name`` +- the ``extra_let_bindings`` the *inner* environment was entered with + +``WrappedStep.Name`` exists in RFC 0008 precisely because ``Step.Name`` is not +meant to be reachable from a hook. openjd-model does not reject any of these +references in an environment script, so this runtime was the only gate. + +Every test here asserts on the symbol table the Session actually hands to the +hook's runner, captured at ``_make_env_script_runner``. Resolving a table the +test built itself would only re-assert the test's own construction. +""" + +from __future__ import annotations + +import time +import uuid +from pathlib import PurePosixPath +from typing import Any + +import pytest + +from openjd.model import ParameterValue, ParameterValueType, SymbolTable +from openjd.model.v2023_09 import ( + Action as Action_2023_09, + ArgString as ArgString_2023_09, + CommandString as CommandString_2023_09, + Environment as Environment_2023_09, + EnvironmentActions as EnvironmentActions_2023_09, + EnvironmentScript as EnvironmentScript_2023_09, + StepActions as StepActions_2023_09, + StepScript as StepScript_2023_09, +) + +from openjd.sessions import ( + ActionState, + PathFormat, + PathMappingRule, + Session, + SessionState, +) + + +def _noop(python_exe: str) -> Action_2023_09: + """A do-nothing action that exists on every platform. + + ``true``/``echo`` are not native Windows executables, and every test in this + file requires its action to actually complete (see ``_run_until_ready``), so + the suite's ``python_exe`` fixture is used throughout rather than a shell + builtin. + """ + return Action_2023_09( + command=CommandString_2023_09(python_exe), + args=[ArgString_2023_09("-c"), ArgString_2023_09("pass")], + ) + + +def _wrap_env(python_exe: str, name: str = "WrapEnv") -> Environment_2023_09: + """A wrap environment whose three hooks are all no-ops. + + What the hooks print does not matter here: these tests inspect the symbol + table the Session builds for the hook, which keeps them independent of + whether a given symbol happens to be spellable in an ArgString under the + extensions the model was parsed with. Deliberately declares no script-level + ``let`` bindings, so a positive assertion about a binding can only be + attributable to ``_seed_wrap_env_scope``. + """ + return Environment_2023_09( + name=name, + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09( + onWrapEnvEnter=_noop(python_exe), + onWrapTaskRun=_noop(python_exe), + onWrapEnvExit=_noop(python_exe), + ) + ), + ) + + +def _inner_env(python_exe: str, name: str = "Inner") -> Environment_2023_09: + return Environment_2023_09( + name=name, + script=EnvironmentScript_2023_09( + actions=EnvironmentActions_2023_09(onEnter=_noop(python_exe), onExit=_noop(python_exe)) + ), + ) + + +def _step_script(python_exe: str) -> StepScript_2023_09: + return StepScript_2023_09(actions=StepActions_2023_09(onRun=_noop(python_exe))) + + +def _run_until_ready(session: Session, timeout_s: float = 10.0) -> None: + """Block until the session leaves RUNNING, then require that it got there. + + The trailing assertion is the load-bearing part. Twelve of the assertions in + this file are on a table that is fully built *before* the hook's subprocess + starts, so without it a hung or failed action would leave every one of them + asserting valid-but-unexercised state and passing. + """ + deadline = time.time() + timeout_s + while session.state == SessionState.RUNNING and time.time() < deadline: + time.sleep(0.05) + assert session.state != SessionState.RUNNING, ( + f"session did not leave RUNNING within {timeout_s}s; " + f"action_status={session.action_status}" + ) + + +class _HookScopeCapture: + """Records the symbol table the Session hands to each hook's runner. + + Patches ``_make_env_script_runner`` because its ``symtab`` argument *is* the + hook's resolution scope -- the runner resolves the hook's command, args, + timeout and cancelation against it, and evaluates the wrap env's own lets and + embedded files into it. Anything absent from it is unreachable from a hook. + + ``symtab`` is keyword-only on that method, so it always arrives in + ``kwargs``; if that ever changed, nothing would be recorded and + :meth:`table` would fail loudly rather than pass vacuously. + + Note that the captured tables are live objects: the runner mutates the one it + was given. Assertions therefore see the table as it was *used*, which is + stronger than as it was handed over. + """ + + def __init__(self, session: Session) -> None: + self.tables: list[SymbolTable] = [] + self._original = session._make_env_script_runner + + def _capturing(*args: Any, **kwargs: Any) -> Any: + symtab = kwargs.get("symtab") + if symtab is not None: + self.tables.append(symtab) + return self._original(*args, **kwargs) + + session._make_env_script_runner = _capturing # type: ignore[method-assign] + + def table(self, index: int, *, expected_count: int) -> SymbolTable: + """The ``index``-th captured scope, requiring exactly ``expected_count``. + + The count is asserted rather than just indexing from the end: a + regression that stopped invoking one of the hooks would otherwise leave a + test silently asserting against a previous hook's table. + """ + assert ( + len(self.tables) == expected_count + ), f"expected {expected_count} hook scope(s), captured {len(self.tables)}" + return self.tables[index] + + +def _defined(symtab: SymbolTable, name: str) -> bool: + """Is ``name`` resolvable in ``symtab``? + + ``SymbolTable.__contains__`` tests the backing table, which is exactly the + set of names both the legacy interpolation path and the EXPR engine resolve + from -- so this is "resolvable", not merely "present". + """ + return name in symtab + + +class TestTaskScopeDoesNotReachTheHook: + """onWrapTaskRun: the wrapped task's own symbols must not be in the hook's + scope.""" + + @pytest.mark.parametrize("leaked_symbol", ["Task.Param.Frame", "Task.RawParam.Frame"]) + def test_task_parameters_are_not_in_the_hook_scope( + self, leaked_symbol: str, python_exe: str + ) -> None: + # GIVEN: a wrap env active over a task that has parameters + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + wrap_id = session.enter_environment(environment=_wrap_env(python_exe)) + _run_until_ready(session) + capture = _HookScopeCapture(session) + + # WHEN + session.run_task( + step_script=_step_script(python_exe), + task_parameter_values={ + "Frame": ParameterValue(type=ParameterValueType.INT, value="42") + }, + step_name="RenderStep", + ) + _run_until_ready(session) + + # THEN: the hook cannot see the wrapped task's parameters -- neither + # the value nor its EXPR type, which would disclose the parameter's + # existence on its own. + assert session.action_status is not None + assert session.action_status.state == ActionState.SUCCESS + hook_scope = capture.table(0, expected_count=1) + assert not _defined(hook_scope, leaked_symbol) + assert leaked_symbol not in hook_scope.expr_types + + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + + def test_running_step_name_is_not_in_the_hook_scope(self, python_exe: str) -> None: + """A job-level wrap env has no step context of its own, so nothing + overwrites Step.Name -- which is how the running step's name used to + survive into the hook.""" + # GIVEN: a wrap env entered with NO step_name + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + wrap_id = session.enter_environment(environment=_wrap_env(python_exe)) + _run_until_ready(session) + capture = _HookScopeCapture(session) + + # WHEN: a task of a named step runs under it + session.run_task( + step_script=_step_script(python_exe), + task_parameter_values={}, + step_name="RenderStep", + ) + _run_until_ready(session) + + # THEN + hook_scope = capture.table(0, expected_count=1) + assert not _defined(hook_scope, "Step.Name") + # ...and the hook still gets the name through the channel RFC 0008 + # provides for it. + assert hook_scope["WrappedStep.Name"] == "RenderStep" + + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + + def test_wrap_envs_own_step_name_still_reaches_the_hook(self, python_exe: str) -> None: + """The fix must not cost the wrap env its own enter-time step context.""" + # GIVEN: a wrap env entered as part of "StepA" + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + wrap_id = session.enter_environment( + environment=_wrap_env(python_exe), step_name="StepA" + ) + _run_until_ready(session) + capture = _HookScopeCapture(session) + + # WHEN: a task of a DIFFERENT step runs under it + session.run_task( + step_script=_step_script(python_exe), + task_parameter_values={}, + step_name="StepB", + ) + _run_until_ready(session) + + # THEN: the hook sees its OWN step, not the running one. + hook_scope = capture.table(0, expected_count=1) + assert hook_scope["Step.Name"] == "StepA" + assert hook_scope["WrappedStep.Name"] == "StepB" + + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + + def test_session_scope_still_reaches_the_hook(self, python_exe: str) -> None: + """Job parameters and session symbols are legitimately session scope and + must survive the rebuild -- values AND their EXPR types.""" + # GIVEN + with Session( + session_id=uuid.uuid4().hex, + job_parameter_values={ + "JobParam": ParameterValue(type=ParameterValueType.STRING, value="jp") + }, + job_name="MyJob", + ) as session: + wrap_id = session.enter_environment(environment=_wrap_env(python_exe)) + _run_until_ready(session) + capture = _HookScopeCapture(session) + + # WHEN + session.run_task( + step_script=_step_script(python_exe), + task_parameter_values={}, + step_name="S", + ) + _run_until_ready(session) + + # THEN + hook_scope = capture.table(0, expected_count=1) + assert hook_scope["Param.JobParam"] == "jp" + assert hook_scope["RawParam.JobParam"] == "jp" + assert hook_scope["Job.Name"] == "MyJob" + assert hook_scope["Session.WorkingDirectory"] == str(session.working_directory) + # EXPR typing is part of the scope: without it a PATH-typed symbol + # silently degrades to a plain string inside a hook's expressions. + assert hook_scope.expr_types["Session.WorkingDirectory"] == ( + ParameterValueType.PATH.value + ) + assert hook_scope.expr_types["Param.JobParam"] == (ParameterValueType.STRING.value) + + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + + def test_path_mapping_reaches_the_hook_intact(self, python_exe: str) -> None: + """A hook resolving Session.PathMappingRulesFile, or calling the EXPR + host function apply_path_mapping(), must see this session's real rules. + + Both halves matter and neither was covered: the rules FILE (a hook that + reads it) and the engine's host RULES (a hook that calls + apply_path_mapping). With the host rules absent the function silently + becomes the identity -- no error, wrong paths. + """ + # GIVEN: a session that actually has path mapping rules + rules = [ + PathMappingRule( + source_path_format=PathFormat.POSIX, + source_path=PurePosixPath("/mnt/src"), + destination_path=PurePosixPath("/mnt/dst"), + ) + ] + with Session( + session_id=uuid.uuid4().hex, + job_parameter_values={}, + path_mapping_rules=rules, + ) as session: + wrap_id = session.enter_environment(environment=_wrap_env(python_exe)) + _run_until_ready(session) + capture = _HookScopeCapture(session) + + # WHEN + session.run_task( + step_script=_step_script(python_exe), + task_parameter_values={}, + step_name="S", + ) + _run_until_ready(session) + + # THEN + hook_scope = capture.table(0, expected_count=1) + assert hook_scope["Session.HasPathMappingRules"] == "true" + assert hook_scope.expr_types["Session.HasPathMappingRules"] == ( + ParameterValueType.BOOL.value + ) + assert str(hook_scope["Session.PathMappingRulesFile"]).endswith(".json") + # The engine's host context must carry the rules, not an empty list. + assert hook_scope.expr_host_rules, "a hook lost the session's path mapping rules" + assert len(hook_scope.expr_host_rules) == len(rules) + assert hook_scope.expr_host_rules == session._expr_host_rules + + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + + +class TestInnerEnvironmentScopeDoesNotReachTheHook: + """onWrapEnvEnter / onWrapEnvExit: the INNER environment's enter-time context + must not be in the hook's scope.""" + + @pytest.mark.parametrize("phase", ["enter", "exit"]) + def test_inner_extra_let_bindings_are_not_in_the_hook_scope( + self, phase: str, python_exe: str + ) -> None: + # GIVEN: a wrap env, and an inner env entered with its own step-level + # bindings and step name + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + wrap_id = session.enter_environment(environment=_wrap_env(python_exe)) + _run_until_ready(session) + capture = _HookScopeCapture(session) + + # WHEN + inner_id = session.enter_environment( + environment=_inner_env(python_exe), + extra_let_bindings=["inner_secret = 'INNER-ONLY'"], + step_name="InnerStep", + ) + _run_until_ready(session) + if phase == "exit": + session.exit_environment(identifier=inner_id) + _run_until_ready(session) + + # THEN: neither the inner env's binding nor its step name is + # reachable from the hook that intercepted it. Index the hook we + # care about rather than the most recent capture, so a hook that + # stopped running cannot pass by inheriting the other's table. + expected = 2 if phase == "exit" else 1 + hook_scope = capture.table(expected - 1, expected_count=expected) + assert not _defined(hook_scope, "inner_secret") + assert not _defined(hook_scope, "Step.Name") + assert hook_scope["WrappedEnv.Name"] == "Inner" + + if phase == "enter": + session.exit_environment(identifier=inner_id) + _run_until_ready(session) + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + + @pytest.mark.parametrize("phase", ["enter", "exit"]) + def test_wrap_envs_own_bindings_still_reach_the_hook(self, phase: str, python_exe: str) -> None: + """Guard the behaviour `_seed_wrap_env_scope` exists for. + + ``test_wrap_task_run.py::test_env_hooks_resolve_step_level_let_bindings`` + covers the same ground by asserting the hook merely SUCCEEDs; this + asserts the binding is in the hook's scope with the right value, which is + what distinguishes "seeded" from "the hook happened not to need it". + """ + # GIVEN: a wrap env entered WITH step-level bindings + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + wrap_id = session.enter_environment( + environment=_wrap_env(python_exe), + extra_let_bindings=["wrap_secret = 'WRAP-OWN'"], + step_name="WrapStep", + ) + _run_until_ready(session) + capture = _HookScopeCapture(session) + + # WHEN + inner_id = session.enter_environment(environment=_inner_env(python_exe)) + _run_until_ready(session) + if phase == "exit": + session.exit_environment(identifier=inner_id) + _run_until_ready(session) + + # THEN. `let` bindings are stored as the EXPR engine's typed value, + # so compare the rendered form rather than the object. + expected = 2 if phase == "exit" else 1 + hook_scope = capture.table(expected - 1, expected_count=expected) + assert str(hook_scope["wrap_secret"]) == "WRAP-OWN" + assert hook_scope["Step.Name"] == "WrapStep" + + if phase == "enter": + session.exit_environment(identifier=inner_id) + _run_until_ready(session) + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + + +class TestWrappedActionStillResolvesInTheInnerScope: + """The other direction must be unaffected: the wrapped action still resolves + against the inner entity's own scope, which is what makes the split useful.""" + + def test_wrapped_args_resolve_the_running_tasks_parameters(self, python_exe: str) -> None: + # GIVEN: a step whose onRun references its own task parameter + step = StepScript_2023_09( + actions=StepActions_2023_09( + onRun=Action_2023_09( + command=CommandString_2023_09(python_exe), + args=[ArgString_2023_09("frame-{{Task.Param.Frame}}")], + ) + ) + ) + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + wrap_id = session.enter_environment(environment=_wrap_env(python_exe)) + _run_until_ready(session) + capture = _HookScopeCapture(session) + + # WHEN + session.run_task( + step_script=step, + task_parameter_values={ + "Frame": ParameterValue(type=ParameterValueType.INT, value="42") + }, + step_name="RenderStep", + ) + _run_until_ready(session) + + # THEN: the wrapped action resolved with the task's own parameter, + # even though the hook's scope cannot see it. + hook_scope = capture.table(0, expected_count=1) + assert hook_scope["WrappedAction.Args"] == ["frame-42"] + assert not _defined(hook_scope, "Task.Param.Frame") + + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + + def test_two_tasks_do_not_bleed_into_each_other(self, python_exe: str) -> None: + """The hook's scope is rebuilt per action, so one task's parameters must + not survive into the next task's hook.""" + # GIVEN + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + wrap_id = session.enter_environment(environment=_wrap_env(python_exe)) + _run_until_ready(session) + capture = _HookScopeCapture(session) + + # WHEN: two tasks run under the same wrap env + for value in ("1", "2"): + session.run_task( + step_script=_step_script(python_exe), + task_parameter_values={ + "Frame": ParameterValue(type=ParameterValueType.INT, value=value) + }, + step_name="RenderStep", + ) + _run_until_ready(session) + + # THEN: distinct tables, neither carrying the task's parameters. + first = capture.table(0, expected_count=2) + second = capture.table(1, expected_count=2) + assert first is not second + for scope in (first, second): + assert not _defined(scope, "Task.Param.Frame") + + session.exit_environment(identifier=wrap_id) + _run_until_ready(session) + + +class TestHookScopeBuilderUnit: + """Direct unit coverage of the builder for the one case the end-to-end tests + cannot reach: a table that has not been through path mapping.""" + + def test_missing_path_mapping_symbols_are_tolerated(self, python_exe: str) -> None: + """Every production caller materializes path mapping first, so the + membership guard in the builder has no end-to-end trigger. Pin it here + rather than leave an unexercised branch.""" + # GIVEN a table with no path-mapping symbols at all + with Session(session_id=uuid.uuid4().hex, job_parameter_values={}) as session: + revision = _step_script(python_exe).revision + + # WHEN / THEN: no KeyError + hook_symtab = session._build_wrap_hook_scope(revision, SymbolTable()) + assert not _defined(hook_symtab, "Session.PathMappingRulesFile") + assert not _defined(hook_symtab, "Session.HasPathMappingRules") From 44ff172c292b9f5d66fcf0759882d013bd638aeb Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:32:14 -0700 Subject: [PATCH 17/27] fix(subprocess): Own the child on every exit path from run() `LoggingSubprocess.run()` is the only holder of the Popen handle, and clearing `self._process` in its `finally` is the point after which nothing can reach the child again -- `notify()` and `terminate()` both become permanent no-ops. Yet `_log_subproc_stdout()`, `wait()` and the returncode capture all sat inside one `try`, so any exception out of the stdout pump skipped the wait and the capture while still clearing the handle. Reproduced with a `logging.Filter` that raises on the child's output -- filters run inside `Logger.handle`, so they propagate into the pump, and installing one is a supported use of this library (Session installs ActionMonitoringFilter). Result: a child still running after `run()` returned, `exit_code` None, `is_running` False, `failed_to_start` False, and `terminate()` unable to reach it. For a cross-user action that is a process running as the job user with no owner, holding the session directory open. An earlier change hardened ActionMonitoringFilter so it no longer raises. That closed the one in-tree trigger; it did not close this, which is structural. Fix: a new `_reap()` runs from the `finally` on every path. If the child is still alive it is terminated and waited for, and its exit status is recorded either way. `terminate()`'s signal delivery is extracted into `_terminate_process(proc)` so the `finally` can reach it after `self._process` is cleared -- the same argument-passing reason the platform helpers already take the Popen. The wait is bounded (ABANDONED_PROCESS_REAP_TIMEOUT_SECONDS): this runs on the runner's pool worker, so a child that somehow outlives SIGKILL must not hang the future as well as leaking the process. Failures inside the reap are caught and logged, because it runs in a `finally` and anything it raised would replace the exception already propagating and hide the real cause. Deliberately removed rather than added: an `if self._returncode is None:` guard around the status capture. A mutation test showed it could never change the outcome -- `run` has already assigned the same value on the normal path, and this is the only assignment on the abandoned one. An unfalsifiable check manufactures confidence, so the assignment is unconditional and the comment says why. Adds test/openjd/sessions_v0/test_subprocess_reaping.py (11 tests). Notable: the "a reap failure must not replace the original exception" test goes through `run()` rather than calling `_reap()` directly. The first version called `_reap` itself, never entered the `finally` being pinned, and passed happily when the reap's `except Exception` was narrowed to `except OSError` -- the mutation run caught that and the test was rewritten. Mutation-checked: 8 mutants, 8 caught, including the original defect, leaking the child, leaving a zombie, losing the exit code, terminating a healthy child on the normal path, an unbounded wait, and not releasing the handle. Verified: 824 passed / 39 skipped / 16 xfailed (only the 2 known xdist timing flakes fail; they pass in isolation); ruff, mypy and black clean. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- src/openjd/sessions/_subprocess.py | 110 +++++- .../sessions_v0/test_subprocess_reaping.py | 337 ++++++++++++++++++ 2 files changed, 436 insertions(+), 11 deletions(-) create mode 100644 test/openjd/sessions_v0/test_subprocess_reaping.py diff --git a/src/openjd/sessions/_subprocess.py b/src/openjd/sessions/_subprocess.py index c3d227af..71c640dc 100644 --- a/src/openjd/sessions/_subprocess.py +++ b/src/openjd/sessions/_subprocess.py @@ -9,7 +9,7 @@ from datetime import timedelta from pathlib import Path from queue import Queue, Empty -from subprocess import DEVNULL, PIPE, STDOUT, Popen, list2cmdline, run +from subprocess import DEVNULL, PIPE, STDOUT, Popen, TimeoutExpired, list2cmdline, run from threading import Event, Thread from typing import Callable, Literal, Optional, Sequence, cast, Any @@ -49,6 +49,15 @@ LOG_LINE_MAX_LENGTH = 64 * 1000 # Start out with 64 KB, can increase if needed STDOUT_END_GRACETIME_SECONDS = 5 +ABANDONED_PROCESS_REAP_TIMEOUT_SECONDS = 30 +"""How long to wait for a subprocess we are abandoning to actually exit. + +Only used on the path where :meth:`LoggingSubprocess.run` is leaving without +having waited on the child -- see :meth:`LoggingSubprocess._reap`. Bounded rather +than infinite because that path runs on the runner's pool worker: a child that +somehow survives SIGKILL (uninterruptible I/O, say) must not hang the runner's +future as well as leaking the process.""" + class LoggingSubprocess(object): """A process whose stdout/stderr lines are sent to a given Logger.""" @@ -236,7 +245,78 @@ def run(self) -> None: # deallocate proc = self._process self._process = None - del proc + try: + # This method owns the subprocess for its whole life, and clearing + # `_process` is the point after which nothing else can reach it: + # `notify()` and `terminate()` both become permanent no-ops. So + # ownership has to be discharged here, on every exit path, not + # only on the one that runs `wait()` above. + # + # An exception out of the stdout pump used to skip both the + # `wait()` and the returncode capture while still clearing + # `_process` -- leaving a live, unreapable child, no exit code, + # and an object reporting neither running nor failed-to-start. A + # raising `logging.Filter` reaches this path, and installing one + # is a supported use of this library. + self._reap(proc) + finally: + del proc + + def _reap(self, proc: Optional[Popen]) -> None: + """Make sure ``proc`` is dead and reaped, and that its exit code is + recorded, however :meth:`run` is leaving. + + A no-op on the normal path, where ``run`` has already waited and captured + the returncode. + """ + if proc is None: + return + if proc.poll() is None: + # Still running, and we are on our way out: nothing will be able to + # signal it once `_process` is cleared. Terminating is the lesser + # evil -- the alternative is an orphan holding the session directory + # and, for a cross-user action, running as the job user with no + # owner. + self._logger.error( + f"Abandoning the subprocess {proc.pid} before it exited; terminating it so it " + "is not orphaned.", + extra=LogExtraInfo( + openjd_log_content=LogContent.PROCESS_CONTROL | LogContent.EXCEPTION_INFO + ), + ) + try: + self._terminate_process(proc) + except Exception as e: # noqa: BLE001 + self._logger.error( + f"Could not terminate the abandoned subprocess {proc.pid}: {e}", + extra=LogExtraInfo( + openjd_log_content=LogContent.PROCESS_CONTROL | LogContent.EXCEPTION_INFO + ), + ) + try: + # Bounded: a SIGKILLed child is reaped promptly, and this runs on + # the pool worker, so an unbounded wait here would hang the + # runner's future rather than surface anything useful. + proc.wait(timeout=ABANDONED_PROCESS_REAP_TIMEOUT_SECONDS) + except TimeoutExpired: + self._logger.error( + f"Subprocess {proc.pid} did not exit within " + f"{ABANDONED_PROCESS_REAP_TIMEOUT_SECONDS}s of being terminated; it is left " + "unreaped.", + extra=LogExtraInfo( + openjd_log_content=LogContent.PROCESS_CONTROL | LogContent.EXCEPTION_INFO + ), + ) + # Record the exit status. Unconditional on purpose: `Popen.returncode` is + # populated by the poll()/wait() above and is the authority either way -- + # on the normal path `run` has already assigned the same value, and on the + # abandoned path this is the only assignment. It is None only if the child + # outlived the terminate timeout, where "unknown" is the honest answer. + # + # An `if self._returncode is None:` guard here would read as defensive but + # could never change the outcome, and a mutation test proved it: removing + # it left every test passing. An unfalsifiable check is worse than none. + self._returncode = proc.returncode def notify(self) -> None: """The 'Notify' part of Open Job Description's subprocess cancelation method. @@ -273,15 +353,23 @@ def terminate(self) -> None: # Review22-F4 fix: Bind _process once. See notify() for rationale. proc = self._process if proc is not None and proc.poll() is None: - if is_posix(): - # R4-G8 fix: Pass the bound proc to helpers. - self._posix_signal_subprocess(proc, signal_name="kill") - else: - self._logger.info( - f"INTERRUPT: Start killing the process tree with the root pid: {proc.pid}", - extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), - ) - kill_windows_process_tree(self._logger, proc.pid, signal_subprocesses=True) + self._terminate_process(proc) + + def _terminate_process(self, process: Popen) -> None: + """Deliver the terminate signal to ``process``. + + Split out of :meth:`terminate` so that :meth:`run`'s ``finally`` can reach + it after ``self._process`` has been cleared -- it takes the Popen as an + argument for the same reason the platform helpers do (R4-G8). + """ + if is_posix(): + self._posix_signal_subprocess(process, signal_name="kill") + else: + self._logger.info( + f"INTERRUPT: Start killing the process tree with the root pid: {process.pid}", + extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), + ) + kill_windows_process_tree(self._logger, process.pid, signal_subprocesses=True) def _start_subprocess(self) -> Optional[Popen]: """Helper invoked by self.run() to start up the subprocess.""" diff --git a/test/openjd/sessions_v0/test_subprocess_reaping.py b/test/openjd/sessions_v0/test_subprocess_reaping.py new file mode 100644 index 00000000..f7615933 --- /dev/null +++ b/test/openjd/sessions_v0/test_subprocess_reaping.py @@ -0,0 +1,337 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""``LoggingSubprocess.run()`` owns its child on every exit path. + +``run()`` is the only place that holds the ``Popen`` handle, and clearing +``self._process`` in its ``finally`` is the point after which nothing can reach +the child again -- ``notify()`` and ``terminate()`` both become permanent no-ops. +So ownership has to be discharged before that, on every path, not only on the one +that reaches ``wait()``. + +It was not. ``_log_subproc_stdout()``, ``wait()`` and the returncode capture all +sat in one ``try``, so any exception out of the stdout pump skipped the wait and +the capture while still clearing the handle. The result was a live child with no +owner, no exit code, and an object reporting neither ``is_running`` nor +``failed_to_start``. + +The trigger is a ``logging.Filter`` that raises. Filters run inside +``Logger.handle``, so they propagate into the pump, and installing one is a +supported use of this library -- ``Session`` installs its own +(``ActionMonitoringFilter``). Hardening that one filter, as an earlier change did, +does not close the structural hole here. +""" + +from __future__ import annotations + +import logging +import os +import signal +import time +from logging.handlers import QueueHandler +from queue import SimpleQueue +from typing import Any, Optional +from unittest.mock import MagicMock, patch + +import pytest + +from openjd.sessions._os_checker import is_posix +from openjd.sessions._subprocess import LoggingSubprocess + +from .conftest import build_logger + +# A child that emits one line (to drive the pump), then outlives the pump. +_LONG_CHILD = "import time\nprint('MARKER', flush=True)\ntime.sleep(60)\n" +# A child that emits one line and exits promptly with a known code. +_SHORT_CHILD = "import sys\nprint('MARKER', flush=True)\nsys.exit(7)\n" + + +class _ExplodingFilter(logging.Filter): + """Raises when the child's output passes through it.""" + + def __init__(self, marker: str = "MARKER") -> None: + super().__init__() + self._marker = marker + self.fired = False + + def filter(self, record: logging.LogRecord) -> bool: + if self._marker in str(record.msg): + self.fired = True + raise RuntimeError("a log filter blew up while forwarding child output") + return True + + +def _pid_alive(pid: int) -> bool: + """Is `pid` a process we can still signal? + + On POSIX a reaped child is gone from the table, so ``kill(pid, 0)`` raising + ESRCH is the reap check. A *zombie* would still answer signal 0, so this + distinguishes "terminated and reaped" from "terminated and leaked". + """ + try: + os.kill(pid, 0) + except OSError: + return False + return True + + +def _wait_gone(pid: Optional[int], timeout_s: float = 15.0) -> bool: + if pid is None: + return True + deadline = time.time() + timeout_s + while time.time() < deadline: + if not _pid_alive(pid): + return True + time.sleep(0.05) + return False + + +@pytest.fixture +def exploding_logger() -> Any: + """A logger whose filter raises on the child's output, plus the filter.""" + exploder = _ExplodingFilter() + logger = logging.getLogger(f"openjd.test.reaping.{id(exploder)}") + logger.setLevel(logging.INFO) + logger.addHandler(logging.NullHandler()) + logger.addFilter(exploder) + try: + yield logger, exploder + finally: + logger.removeFilter(exploder) + + +@pytest.mark.skipif(not is_posix(), reason="pid liveness/reap checks are POSIX-only here") +class TestPumpExceptionDoesNotAbandonTheChild: + def test_live_child_is_terminated_and_reaped( + self, exploding_logger: Any, python_exe: str + ) -> None: + # GIVEN: a long-running child and a log filter that will raise on its + # first line of output, while the child is still alive + logger, exploder = exploding_logger + proc = LoggingSubprocess(logger=logger, args=[python_exe, "-c", _LONG_CHILD]) + + # WHEN + with pytest.raises(RuntimeError, match="blew up"): + proc.run() + + # THEN: the filter really did fire mid-pump (otherwise this test proves + # nothing about the abandonment path)... + assert exploder.fired is True + pid = proc.pid + assert pid is not None + # ...and the child is gone, not merely signalled. + assert _wait_gone(pid), f"child {pid} was left running" + + def test_exit_code_is_recorded_for_an_abandoned_child( + self, exploding_logger: Any, python_exe: str + ) -> None: + """The exit code is the only evidence of what happened to the action, and + it used to be lost entirely.""" + # GIVEN + logger, _ = exploding_logger + proc = LoggingSubprocess(logger=logger, args=[python_exe, "-c", _LONG_CHILD]) + + # WHEN + with pytest.raises(RuntimeError): + proc.run() + + # THEN: a real exit status, not None. SIGKILL shows as -SIGKILL. + assert proc.exit_code is not None + assert proc.exit_code == -signal.SIGKILL + + def test_a_child_that_exited_on_its_own_keeps_its_own_exit_code( + self, exploding_logger: Any, python_exe: str + ) -> None: + """The reap must not overwrite a genuine exit status with a kill signal.""" + # GIVEN: a child that exits 7 by itself, and a filter that raises + logger, _ = exploding_logger + proc = LoggingSubprocess(logger=logger, args=[python_exe, "-c", _SHORT_CHILD]) + + # WHEN + with pytest.raises(RuntimeError): + proc.run() + + # THEN: whichever way the race went, the recorded code is a real one and + # never None. If the child had already exited, it must be its own 7. + assert proc.exit_code is not None + assert proc.exit_code in (7, -signal.SIGKILL) + + def test_a_reap_failure_does_not_replace_the_original_exception( + self, exploding_logger: Any, python_exe: str + ) -> None: + """The reap runs inside `run()`'s `finally`, so anything it raises would + replace the exception already propagating -- hiding the real cause. + + Goes through `run()` rather than calling `_reap` directly: the + containment being pinned is the `finally`'s, and a test that calls + `_reap` itself never enters that `finally` at all. An earlier version of + this file made that mistake, and narrowing the reap's `except Exception` + to `except OSError` passed every test. + + The injected failure is deliberately NOT an OSError, so a handler that + only catches OSError lets it through. + """ + # GIVEN: a live child, a filter that raises, and a termination that fails + # with a non-OSError + logger, exploder = exploding_logger + proc = LoggingSubprocess(logger=logger, args=[python_exe, "-c", _LONG_CHILD]) + + # WHEN + with ( + patch.object( + LoggingSubprocess, + "_terminate_process", + side_effect=RuntimeError("terminate itself blew up"), + ), + patch("openjd.sessions._subprocess.ABANDONED_PROCESS_REAP_TIMEOUT_SECONDS", 0), + ): + with pytest.raises(RuntimeError) as excinfo: + proc.run() + + # THEN: the caller sees the pump's failure, which is the real cause -- not + # the reap's. + assert exploder.fired is True + assert "blew up while forwarding child output" in str(excinfo.value) + assert "terminate itself blew up" not in str(excinfo.value) + # ...and the handle is still released. + assert proc._process is None + + # Clean up the child the failed terminate left behind. + if proc.pid is not None and _pid_alive(proc.pid): + os.kill(proc.pid, signal.SIGKILL) + try: + os.waitpid(proc.pid, 0) + except ChildProcessError: + pass + + def test_the_process_handle_is_still_released( + self, exploding_logger: Any, python_exe: str + ) -> None: + """Reaping must not come at the cost of the deallocation the `finally` + existed for.""" + # GIVEN / WHEN + logger, _ = exploding_logger + proc = LoggingSubprocess(logger=logger, args=[python_exe, "-c", _LONG_CHILD]) + with pytest.raises(RuntimeError): + proc.run() + + # THEN + assert proc._process is None + assert proc.is_running is False + + +@pytest.mark.skipif(not is_posix(), reason="POSIX signal semantics") +class TestNormalPathIsUnchanged: + """The reap is a no-op when `run()` completed properly. These are the tests + that would catch it terminating a healthy child or clobbering its status.""" + + def test_successful_run_keeps_its_exit_code_and_is_not_signalled( + self, message_queue: SimpleQueue, queue_handler: QueueHandler, python_exe: str + ) -> None: + # GIVEN + logger = build_logger(queue_handler) + proc = LoggingSubprocess(logger=logger, args=[python_exe, "-c", _SHORT_CHILD]) + + # WHEN + proc.run() + + # THEN: its own exit code, positive -- not a negative signal value. + assert proc.exit_code == 7 + assert proc.failed_to_start is False + + def test_successful_run_does_not_terminate_the_child( + self, message_queue: SimpleQueue, queue_handler: QueueHandler, python_exe: str + ) -> None: + """Directly pins that the new path does not fire on a healthy run.""" + # GIVEN + logger = build_logger(queue_handler) + proc = LoggingSubprocess(logger=logger, args=[python_exe, "-c", "pass"]) + + # WHEN + with patch.object(LoggingSubprocess, "_terminate_process", autospec=True) as terminate: + proc.run() + + # THEN + terminate.assert_not_called() + assert proc.exit_code == 0 + + def test_output_is_still_forwarded( + self, message_queue: SimpleQueue, queue_handler: QueueHandler, python_exe: str + ) -> None: + # GIVEN / WHEN + logger = build_logger(queue_handler) + proc = LoggingSubprocess(logger=logger, args=[python_exe, "-c", _SHORT_CHILD]) + proc.run() + + # THEN + lines = [] + while not message_queue.empty(): + lines.append(message_queue.get().getMessage()) + assert "MARKER" in lines + + +@pytest.mark.skipif(not is_posix(), reason="POSIX signal semantics") +class TestReapUnitBehaviour: + """Direct coverage of `_reap` for the branches the end-to-end tests cannot + reach cheaply.""" + + def test_reap_of_none_is_a_no_op(self) -> None: + # GIVEN a subprocess that never started + proc = LoggingSubprocess(logger=MagicMock(), args=["/bin/echo", "hi"]) + + # WHEN / THEN: no exception, and no exit code invented + proc._reap(None) + assert proc.exit_code is None + + def test_a_terminate_failure_does_not_escape(self, python_exe: str) -> None: + """`_reap` runs inside a `finally`. An exception from it would replace + whatever exception was already propagating.""" + # GIVEN a live child whose termination will fail + logger = MagicMock() + proc = LoggingSubprocess(logger=logger, args=[python_exe, "-c", _LONG_CHILD]) + popen = proc._start_subprocess() + assert popen is not None + try: + with ( + patch.object( + LoggingSubprocess, + "_terminate_process", + side_effect=OSError("cannot signal"), + ), + patch( + # The child survives, since termination was made to fail, so the + # wait that follows must not burn its real budget here. + "openjd.sessions._subprocess.ABANDONED_PROCESS_REAP_TIMEOUT_SECONDS", + 0, + ), + ): + # WHEN / THEN: swallowed and logged, not raised + proc._reap(popen) + assert any("Could not terminate" in str(call) for call in logger.error.call_args_list) + finally: + popen.kill() + popen.wait() + + def test_a_child_that_survives_termination_is_reported_not_hung(self, python_exe: str) -> None: + """The wait is bounded: a child that somehow outlives SIGKILL must not + hang the pool worker.""" + # GIVEN a live child, a termination that does nothing, and a 0s budget + logger = MagicMock() + proc = LoggingSubprocess(logger=logger, args=[python_exe, "-c", _LONG_CHILD]) + popen = proc._start_subprocess() + assert popen is not None + try: + with ( + patch.object(LoggingSubprocess, "_terminate_process"), + patch("openjd.sessions._subprocess.ABANDONED_PROCESS_REAP_TIMEOUT_SECONDS", 0), + ): + start = time.monotonic() + # WHEN + proc._reap(popen) + elapsed = time.monotonic() - start + + # THEN: returned promptly, and said so. + assert elapsed < 10.0 + assert any("did not exit within" in str(call) for call in logger.error.call_args_list) + finally: + popen.kill() + popen.wait() From d29aafe7b17445e82df46894c9edd342275ed470 Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:45:22 -0700 Subject: [PATCH 18/27] fix: Report chown and pgrep failures through the paths that handle them Two small findings from the five-agent sweep, plus three follow-ups a test audit raised on the first attempt at them. shutil.chown raises LookupError for an unresolvable group name, and LookupError is neither OSError nor ValueError. Every handler in this package around file materialization or session setup catches some combination of OSError/ValueError/RuntimeError, so an unknown group escaped all of them and surfaced as a bare LookupError out of the public Session API. PosixSessionUser does not validate its group, so a caller only has to mis-type one. New chown_group() translates it at the two call sites -- write_file_for_user and TempDir.__init__ -- rather than widening six handlers. A group that cannot be resolved is a failure to change ownership, callers already treat that as OSError, and this way a handler added later is covered without anyone remembering to. find_child_process_id_pgrep raised FindSignalTargetError for any non-zero pgrep exit. Exit 1 means "no processes matched", which is the expected answer for most of the caller's retry window: sudo has forked but the kernel has not finished creating the workload. Measured on macOS, the first scan at ~25ms returned no match for a child that appeared at ~300ms. Exit 1 now returns None, which is what the procfs implementation of the same lookup already does; other non-zero exits still raise, now naming the exit code and pgrep's own output (stderr is merged into stdout, so the message was the only place the reason could survive). Three follow-ups from the test audit of the above, each a real gap rather than a polish item: - find_sudo_child_process_group_id had its `except FindSignalTargetError` OUTSIDE the retry `while`, so fixing the pgrep exit code only moved the problem: one bad scan still ended the whole search. Everything the loop races is transient -- the procfs scan sees more than one child while sudo's fork settles, and a pgrep invocation can fail without recurring. The scan is now guarded inside the loop and the error remembered, so a timeout can say what kept going wrong. - TempDir.__init__ leaked its mkdtemp directory when the ownership step failed. Construction had raised, so the caller had no handle to clean up with, and the directory sits under a shared root nothing prunes. The chown fix makes that path much easier to reach -- a typo'd group, not just a permissions problem -- so the ownership block is now wrapped with a best-effort rmtree that re-raises the original error rather than replacing it. - Two lines of my own were inert and are gone: a `sudo_child_pid = None` in the new except (the scan only runs when it is already falsy) and an `or ''` guard on pgrep's stdout (run(stdout=PIPE, text=True) never yields None). Both survived their mutants, which was the correct signal. Tests: new test/openjd/sessions_v0/test_linux_sudo.py, plus additions to test_embedded_files.py, test_tempdir.py and conftest.py. Notable coverage the audit added beyond the obvious: both the procfs and pgrep branches of the retry fix, the pre-existing process-group races that had to survive it (a child that exits mid-scan, a child still sharing sudo's group), that TempDir's cleanup does not mask the error it cleans up after, and the Windows half of the TempDir leak, which had nothing. Mutation-checked: 15 mutants, 15 caught, restores checksum-verified. Two further mutants were run separately and are expected to survive -- they remove the inert lines described above -- and are kept out of the gated spec so its "0 survived" stays meaningful. Verified: ruff, mypy and black clean; the touched suites 58 passed / 13 skipped / 6 xfailed. Not fixed here, and now written up in SuperDaveDocs/pr-reviews/sessions-333/40-deferred-hard-findings.md: the regression test for a954917 is itself flaky (fails ~1 run in 4 in isolation, for a reason in the test rather than in production). It gets its own commit. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- src/openjd/sessions/_embedded_files.py | 24 +- src/openjd/sessions/_linux/_sudo.py | 68 +- src/openjd/sessions/_tempdir.py | 71 +- test/openjd/sessions_v0/conftest.py | 32 + .../openjd/sessions_v0/test_embedded_files.py | 138 ++++ test/openjd/sessions_v0/test_linux_sudo.py | 646 ++++++++++++++++++ test/openjd/sessions_v0/test_tempdir.py | 184 ++++- 7 files changed, 1120 insertions(+), 43 deletions(-) create mode 100644 test/openjd/sessions_v0/test_linux_sudo.py diff --git a/src/openjd/sessions/_embedded_files.py b/src/openjd/sessions/_embedded_files.py index 29ce6cb9..8f8d26b8 100644 --- a/src/openjd/sessions/_embedded_files.py +++ b/src/openjd/sessions/_embedded_files.py @@ -66,6 +66,28 @@ def _convert_line_endings(data: str, end_of_line: Optional[str]) -> str: return data +def chown_group(path: Path, group: str) -> None: + """Set ``path``'s group ownership, reporting failure as ``OSError``. + + ``shutil.chown`` raises ``LookupError`` when the group name does not resolve, + and ``LookupError`` is neither ``OSError`` nor ``ValueError``. Every handler + around a file-materialization or session-directory failure in this package + catches some combination of ``OSError``/``ValueError``/``RuntimeError``, so an + unknown group escaped all of them and surfaced as a bare ``LookupError`` out + of the public Session API. ``PosixSessionUser`` does not validate its + ``group``, so a caller only has to pass a group that does not exist. + + Translated here, at the two call sites, rather than by widening each handler: + a group that cannot be resolved *is* a failure to change ownership, the + callers already treat that as ``OSError``, and this way a handler added later + is covered without anyone having to remember. + """ + try: + chown(path, group=group) + except LookupError as err: + raise OSError(f"Could not set the group of {str(path)} to '{group}': {err}") from err + + def _unsupported_embedded_file_model(file: EmbeddedFileType) -> RuntimeError: """The error for an embedded file from a schema this class does not implement. @@ -160,7 +182,7 @@ def write_file_for_user( if user is not None: user = cast(PosixSessionUser, user) # Set the group of the file - chown(filename, group=user.group) + chown_group(filename, user.group) # Update the permissions to include the group after the group is changed # Note: Only after changing group for security in case the group-ownership # change fails. diff --git a/src/openjd/sessions/_linux/_sudo.py b/src/openjd/sessions/_linux/_sudo.py index b52dbdb4..fccfef88 100644 --- a/src/openjd/sessions/_linux/_sudo.py +++ b/src/openjd/sessions/_linux/_sudo.py @@ -10,6 +10,12 @@ from .._logging import LoggerAdapter, LogContent, LogExtraInfo from .._os_checker import is_posix, is_linux +PGREP_NO_MATCH = 1 +"""``pgrep``'s exit status for "no processes matched". + +Distinct from a genuine failure (>=2 for a usage or operational error). See +:func:`find_child_process_id_pgrep`.""" + class FindSignalTargetError(Exception): """Exception when unable to detect the signal target""" @@ -65,18 +71,36 @@ def find_sudo_child_process_group_id( # quickly and we may never find the child process. sudo_child_pid: Optional[int] = None sudo_child_pgid: Optional[int] = None + last_scan_error: Optional[FindSignalTargetError] = None try: while now - start < timeout_seconds: if not sudo_child_pid: - if is_linux(): - sudo_child_pid = find_sudo_child_process_id_procfs( - sudo_pid=sudo_process.pid, - logger=logger, - ) - else: - sudo_child_pid = find_child_process_id_pgrep( - sudo_pid=sudo_process.pid, - ) + try: + if is_linux(): + sudo_child_pid = find_sudo_child_process_id_procfs( + sudo_pid=sudo_process.pid, + logger=logger, + ) + else: + sudo_child_pid = find_child_process_id_pgrep( + sudo_pid=sudo_process.pid, + ) + except FindSignalTargetError as e: + # A single bad scan must not end the retries. Everything this + # loop is racing is transient by nature: the procfs scan sees + # more than one child while sudo's fork is still settling, and + # a `pgrep` invocation can fail for reasons that do not recur. + # This `except` used to sit outside the `while`, so the first + # such observation aborted the whole search and the process + # group was never recorded -- leaving a later cancel with + # nothing to signal. + # + # Remembered rather than discarded so that a timeout can say + # what kept going wrong. + # + # No `sudo_child_pid = None` here: the scan only runs when it + # is already falsy, so the assignment would be unfalsifiable. + last_scan_error = e if sudo_child_pid: try: @@ -96,7 +120,8 @@ def find_sudo_child_process_group_id( time.sleep(min(0.05, timeout_seconds - (now - start))) now = time.monotonic() if not sudo_child_pid or not sudo_child_pgid: - raise FindSignalTargetError("unable to detect subprocess before timeout") + detail = f" (last scan error: {last_scan_error})" if last_scan_error else "" + raise FindSignalTargetError(f"unable to detect subprocess before timeout{detail}") except FindSignalTargetError as e: logger.warning( f"Unable to determine signal target: {e}", @@ -155,8 +180,29 @@ def find_child_process_id_pgrep( stdin=DEVNULL, text=True, ) + if pgrep_result.returncode == PGREP_NO_MATCH: + # "No processes matched" is the expected answer for most of the retry + # window, not an error: sudo has forked but the kernel has not finished + # creating the workload yet. Returning None keeps the caller's retry loop + # going, which is what the procfs implementation of this same lookup + # already does for the identical situation. + # + # Raising here instead aborted the whole scan on the first empty poll, + # because the caller's `except FindSignalTargetError` sits OUTSIDE its + # retry `while`. Measured on macOS: the very first scan at ~25ms returned + # None for a child that appeared at ~300ms and was found correctly once it + # existed. So on every non-Linux POSIX host, a cross-user launch recorded + # no process group at all -- and a later cancel then had nothing to + # signal. + return None if pgrep_result.returncode != 0: - raise FindSignalTargetError("Unable to query child processes of sudo process") + # stderr is merged into stdout by the caller's `stderr=STDOUT`, so this is + # the only place the reason is available. Without it the message named an + # exit code and nothing else, which is not enough to act on. + raise FindSignalTargetError( + f"Unable to query child processes of sudo process (pgrep exited " + f"{pgrep_result.returncode}): {pgrep_result.stdout.strip()!r}" + ) results = pgrep_result.stdout.splitlines() if len(results) > 1: raise FindSignalTargetError(f"Expected a single child process of sudo, but found {results}") diff --git a/src/openjd/sessions/_tempdir.py b/src/openjd/sessions/_tempdir.py index b1659c00..4d2c2e01 100644 --- a/src/openjd/sessions/_tempdir.py +++ b/src/openjd/sessions/_tempdir.py @@ -5,7 +5,8 @@ import sys from ._logging import LoggerAdapter, LogContent, LogExtraInfo from pathlib import Path -from shutil import chown, rmtree +from shutil import rmtree +from ._embedded_files import chown_group from tempfile import gettempdir, mkdtemp from typing import Any, Optional, cast @@ -235,33 +236,47 @@ def __init__( except OSError as err: raise RuntimeError(f"Could not create temp directory within {str(dir)}: {str(err)}") - # Change the owner - if user: - if is_posix(): - user = cast(PosixSessionUser, user) - # Change ownership - try: - chown(self.path, group=user.group) - except OSError as err: - raise RuntimeError( - f"Could not change ownership of directory '{str(dir)}' (error: {str(err)}). Please ensure that uid {os.geteuid()} is a member of group {user.group}." # type: ignore - ) - # Update the permissions to include the group after the group is changed - # Note: Only after changing group for security in case the group-ownership - # change fails. - os.chmod(self.path, mode=stat.S_IRWXU | stat.S_IRWXG) - elif is_windows(): - user = cast(WindowsSessionUser, user) - try: - WindowsPermissionHelper.set_permissions( - str(self.path), - principals_full_control=[get_process_user()], - principals_modify_access=[user.user], - ) - except Exception as err: - raise RuntimeError( - f"Could not change permissions of directory '{str(dir)}' (error: {str(err)})" - ) + # Change the owner. + # + # Wrapped so that a failure here does not leave the mkdtemp directory + # behind: construction has failed, so the caller has no handle to clean up + # with, and this directory sits under a shared root that nothing else ever + # prunes. Reachable from a mis-typed `PosixSessionUser.group` -- which is + # not validated anywhere -- and not only from a permissions problem. + try: + if user: + if is_posix(): + user = cast(PosixSessionUser, user) + # Change ownership + try: + # chown_group, not shutil.chown: an unresolvable group name + # raises LookupError, which this handler would not catch. + chown_group(self.path, user.group) + except OSError as err: + raise RuntimeError( + f"Could not change ownership of directory '{str(dir)}' (error: {str(err)}). Please ensure that uid {os.geteuid()} is a member of group {user.group}." # type: ignore + ) + # Update the permissions to include the group after the group is changed + # Note: Only after changing group for security in case the group-ownership + # change fails. + os.chmod(self.path, mode=stat.S_IRWXU | stat.S_IRWXG) + elif is_windows(): + user = cast(WindowsSessionUser, user) + try: + WindowsPermissionHelper.set_permissions( + str(self.path), + principals_full_control=[get_process_user()], + principals_modify_access=[user.user], + ) + except Exception as err: + raise RuntimeError( + f"Could not change permissions of directory '{str(dir)}' (error: {str(err)})" + ) + except BaseException: + # Best-effort: the failure being reported is the interesting one, so a + # cleanup problem must not replace it. + rmtree(self.path, ignore_errors=True) + raise def cleanup(self) -> None: """Deletes the temporary directory and all of its contents. diff --git a/test/openjd/sessions_v0/conftest.py b/test/openjd/sessions_v0/conftest.py index eca66db8..a8fe4dbf 100644 --- a/test/openjd/sessions_v0/conftest.py +++ b/test/openjd/sessions_v0/conftest.py @@ -3,6 +3,7 @@ import os import random import string +import uuid from logging import INFO, getLogger from logging.handlers import QueueHandler from queue import Empty, SimpleQueue @@ -18,6 +19,9 @@ from openjd.sessions._action_filter import ActionMonitoringFilter from openjd.model import RevisionExtensions, SpecificationRevision +if is_posix(): + import grp + if is_windows(): from openjd.sessions._win32._helpers import ( # type: ignore get_current_process_session_id, @@ -62,6 +66,34 @@ def pytest_collection_modifyitems(config, items): config.option.markexpr = mark_expr +def nonexistent_group_name() -> str: + """A group name that cannot resolve on any host. + + Randomized rather than hardcoded so that it cannot collide with a real group + on a developer's machine or in a test container. + """ + return f"openjd-no-such-group-{uuid.uuid4().hex}" + + +def resolvable_member_groups() -> list[tuple[int, str]]: + """The (gid, name) of every group this process is a member of and that has a + name in the group database. + + Discovered at runtime on purpose: this suite runs on macOS and Linux hosts + (and inside test containers) whose group tables have nothing in common, so no + group name can be hardcoded. The KeyError branch matters: a process can hold a + gid that the group database has no entry for, and asking for its name raises. + """ + groups: list[tuple[int, str]] = [] + for gid in sorted(set(os.getgroups()) | {os.getegid()}): # type: ignore + try: + groups.append((gid, grp.getgrgid(gid).gr_name)) # type: ignore + except KeyError: + # A gid the process holds that has no entry in the group database. + continue + return groups + + def create_unique_logger_name(prefix: str = "", seed: Optional[str] = None) -> str: """Create a unique logger name using a hash to avoid collisions. diff --git a/test/openjd/sessions_v0/test_embedded_files.py b/test/openjd/sessions_v0/test_embedded_files.py index 09b671b8..8cbf373d 100644 --- a/test/openjd/sessions_v0/test_embedded_files.py +++ b/test/openjd/sessions_v0/test_embedded_files.py @@ -1,5 +1,6 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +import errno import os import stat import uuid @@ -26,6 +27,7 @@ EmbeddedFiles, EmbeddedFilesScope, _validate_embedded_filename, + chown_group, write_file_for_user, ) from openjd.sessions._session_user import PosixSessionUser, WindowsSessionUser @@ -33,6 +35,8 @@ from .conftest import ( has_posix_target_user, has_windows_user, + nonexistent_group_name, + resolvable_member_groups, WIN_SET_TEST_ENV_VARS_MESSAGE, POSIX_SET_TARGET_USER_ENV_VARS_MESSAGE, ) @@ -892,6 +896,48 @@ class Datum: result_contents = file.read() assert result_contents == expected_file_data, "File contents are as expected" + @pytest.mark.skipif(not is_posix(), reason="group ownership is posix-specific") + def test_unresolvable_group_fails_as_runtimeerror(self, tmp_path: Path) -> None: + """Pins: a group name that does not resolve must fail materialize() + through its existing handler, as RuntimeError. + + shutil.chown raises LookupError for an unknown group, and LookupError + is neither OSError nor ValueError -- the two classes + write_file_contents() catches -- so before chown_group() it escaped + this handler (and every other one in the chain) and surfaced as a + bare LookupError out of the public Session API. PosixSessionUser + does not validate its group, so a caller only has to pass a group + that does not exist. + + pytest.raises(RuntimeError) does not catch LookupError, so if the + translation is removed this test errors out with the escaping + LookupError -- which is exactly the defect. + """ + + # GIVEN + # Only `group` matters on this path; the user is never resolved by it. + user = PosixSessionUser(user="nobody", group=nonexistent_group_name()) + test_obj = EmbeddedFiles( + logger=MagicMock(), + scope=EmbeddedFilesScope.ENV, + session_files_directory=tmp_path, + user=user, + ) + given_file = EmbeddedFileText_2023_09( + name="Foo", + type=EmbeddedFileTypes_2023_09.TEXT, + data=DataString_2023_09("some data"), + ) + + # WHEN + with pytest.raises(RuntimeError) as excinfo: + test_obj.materialize([given_file], SymbolTable()) + + # THEN + # The failure is the group ownership change, not something incidental: + # the offending group name is carried through to the message. + assert user.group in str(excinfo.value) + class TestValidateEmbeddedFilename: """Unit tests for the _validate_embedded_filename() basename guard. @@ -947,3 +993,95 @@ def test_rejects_windows_non_basenames(self, filename: str) -> None: # WHEN / THEN with pytest.raises(ValueError): _validate_embedded_filename(filename) + + +@pytest.mark.skipif(not is_posix(), reason="shutil.chown's group handling is posix-only") +class TestChownGroup: + """Unit tests for chown_group(), the shutil.chown wrapper. + + Defect pinned: shutil.chown raises LookupError when the group name does not + resolve. LookupError is neither OSError nor ValueError, and every handler + around file materialization or session-directory setup in this package + catches some combination of OSError/ValueError/RuntimeError -- so an + unresolvable group escaped all of them and reached the caller of the public + Session API as a bare LookupError. chown_group() translates it at the call + site instead, because a group that cannot be resolved *is* a failure to + change ownership and callers already treat that as OSError. + """ + + def test_unresolvable_group_raises_oserror(self, tmp_path: Path) -> None: + """The translation itself: OSError out, not LookupError. + + pytest.raises(OSError) cannot catch a LookupError, so removing the + translation makes this test error out with the escaping LookupError. It + also fails if the wrapper raises some other class (e.g. ValueError) or + swallows the failure entirely. + """ + # GIVEN + filename = tmp_path / "file.txt" + filename.write_text("some data") + group = nonexistent_group_name() + + # WHEN + with pytest.raises(OSError) as excinfo: + chown_group(filename, group) + + # THEN + # Both the path and the group are named, so an operator can tell which + # file and which group configuration failed. + assert str(filename) in str(excinfo.value) + assert group in str(excinfo.value) + # AND the original LookupError is preserved as the cause, which proves + # this OSError is the translated group lookup and not an unrelated + # filesystem error. + assert isinstance(excinfo.value.__cause__, LookupError) + + def test_resolvable_group_succeeds(self, tmp_path: Path) -> None: + """The wrapper must not break the success path. + + The gid assertion is what makes this non-vacuous: it only holds if the + chown actually happened. A group other than the file's current group is + preferred so that the assertion cannot pass by accident. + """ + # GIVEN + candidates = resolvable_member_groups() + if not candidates: + pytest.skip("this process is not a member of any group that has a name") + filename = tmp_path / "file.txt" + filename.write_text("some data") + current_gid = filename.stat().st_gid + gid, group = next( + ((gid, name) for gid, name in candidates if gid != current_gid), candidates[0] + ) + + # WHEN + chown_group(filename, group) + + # THEN + assert filename.stat().st_gid == gid + + def test_real_oserror_is_passed_through_unchanged(self, tmp_path: Path) -> None: + """A genuine failure from shutil.chown must reach the caller as-is. + + The wrapper only exists to reclassify LookupError; if it broadened to + catch more than that, a real errno would be flattened into a generic + error and the handlers upstream would report the wrong cause. Using a + resolvable group with a missing path gets shutil.chown past its group + lookup and into os.chown, so the ENOENT here comes from the real syscall + rather than from a mock. + """ + # GIVEN + candidates = resolvable_member_groups() + if not candidates: + pytest.skip("this process is not a member of any group that has a name") + _gid, group = candidates[0] + missing = tmp_path / "no-such-file.txt" + + # WHEN + with pytest.raises(FileNotFoundError) as excinfo: + chown_group(missing, group) + + # THEN + # Same class, same errno, and no re-raise chain: it is the original error. + assert excinfo.value.errno == errno.ENOENT + assert excinfo.value.__cause__ is None diff --git a/test/openjd/sessions_v0/test_linux_sudo.py b/test/openjd/sessions_v0/test_linux_sudo.py new file mode 100644 index 00000000..64ddec6c --- /dev/null +++ b/test/openjd/sessions_v0/test_linux_sudo.py @@ -0,0 +1,646 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +import os +import shutil +import signal +import subprocess +import time +from logging.handlers import QueueHandler +from pathlib import Path +from queue import SimpleQueue +from subprocess import DEVNULL, PIPE, CompletedProcess, Popen +from typing import Callable, Generator, Optional +from unittest.mock import Mock, patch + +import pytest + +from openjd.sessions._linux._sudo import ( + FindSignalTargetError, + find_child_process_id_pgrep, + find_sudo_child_process_group_id, +) +from openjd.sessions._os_checker import is_posix + +from .conftest import build_logger, collect_queue_messages + +# pgrep's documented exit status for "no processes matched". Spelled out as a +# literal rather than imported from the module under test: a test that reused the +# module's own constant could not tell us whether that constant matches what the +# operating system actually returns. +PGREP_EXIT_NO_MATCH = 1 + +# The stand-in for sudo: a process that gains its child only after a delay, so a +# scan that starts immediately is guaranteed to see no children on its first look. +PARENT_SCRIPT = """ +import subprocess +import sys +import time + +python_exe, child_script, pgid_file, delay_seconds = sys.argv[1:5] +time.sleep(float(delay_seconds)) +# start_new_session, as sudo does, so the child lands in its own process group. +child = subprocess.Popen([python_exe, child_script, pgid_file], start_new_session=True) +child.wait() +""" + +# The stand-in for the workload. It reports its own process group id, so the test +# asserts against the group the kernel actually assigned rather than recomputing +# it the same way the code under test does. +CHILD_SCRIPT = """ +import os +import sys +import time + +pgid_file = sys.argv[1] +# Written to a temporary name and renamed, so a reader never sees a partial id. +tmp_file = pgid_file + ".tmp" +with open(tmp_file, "w") as f: + f.write(str(os.getpgid(0))) +os.replace(tmp_file, pgid_file) +time.sleep(15) +""" + + +# A process that moves into its own process group only after a delay, reproducing +# the window sudo leaves open: it forks first and creates the new process group +# second, so a scan can legitimately observe the child still sharing sudo's group. +# +# setsid() rather than setpgid(): the process is started without +# start_new_session, so it is not already a group leader and setsid() cannot fail +# with EPERM. It reports the group it ends up in, so the test asserts against what +# the kernel assigned rather than recomputing it. +LATE_PROCESS_GROUP_SCRIPT = """ +import os +import time + +pgid_file = %(pgid_file)s +time.sleep(%(delay_seconds)s) +os.setsid() +tmp_file = pgid_file + ".tmp" +with open(tmp_file, "w") as f: + f.write(str(os.getpgid(0))) +os.replace(tmp_file, pgid_file) +time.sleep(15) +""" + + +def pgrep_result(returncode: int, stdout: str = "") -> CompletedProcess: + """A stand-in for the CompletedProcess of a `pgrep -P ` run.""" + return CompletedProcess(args=["pgrep", "-P", "1234"], returncode=returncode, stdout=stdout) + + +@pytest.fixture +def sleeper(python_exe: str) -> Generator[Callable[..., Popen], None, None]: + """Factory for live, long-lived child processes, all killed and reaped at teardown. + + Real processes rather than fabricated pids because the code under test calls + os.getpgid() on whatever the scan returns: a made-up pid would either raise + ProcessLookupError or -- worse, if the number happened to be live -- report + the process group of something unrelated. + + Registration happens inside the factory, so a process cannot be leaked by a + failure between starting it and the caller's first statement. + """ + procs: list[Popen] = [] + + def start(*, own_process_group: bool = False, script: Optional[str] = None) -> Popen: + argv = [python_exe, "-c", script if script else "import time; time.sleep(30)"] + proc = Popen( + argv, + stdin=DEVNULL, + stdout=DEVNULL, + # A scripted child can die on startup, and without its stderr the test + # that depends on it fails with no explanation. The plain sleeper + # cannot fail that way, so it keeps the pipe-free form. + stderr=PIPE if script else DEVNULL, + # start_new_session is what sudo does: the child becomes a session and + # process-group leader, so its pgid is its own pid and is knowable + # without asking the code under test. + start_new_session=own_process_group, + ) + procs.append(proc) + return proc + + yield start + + for proc in procs: + # Popen.kill() is a no-op once the process has been reaped, so this is safe + # for the already-reaped process the short-circuit test needs. + proc.kill() + try: + # communicate() rather than wait(): it drains the stderr pipe as well + # as reaping, so a child that wrote to stderr cannot wedge teardown. + proc.communicate(timeout=10) + except ValueError: + # A test on its failure path already read and closed that pipe. + proc.wait(timeout=10) + + +def stderr_of(proc: Popen) -> bytes: + """The child's stderr, for the diagnostic on a failing assertion. + + Killed first: this is only reached when the child has already failed to do + what a test needed of it, and a still-sleeping child would otherwise hold + communicate() open for the rest of its life. + """ + proc.kill() + return proc.communicate(timeout=10)[1] or b"" + + +def poll_for_pgid(pgid_file: Path, timeout_seconds: float) -> Optional[int]: + """The process group id reported by the child, or None if it never reported. + + Polled rather than slept on, and a None return makes the caller's assertion + fail loudly instead of the test quietly passing on a missing file. The file + is always read at least once, so a zero timeout still answers. + """ + deadline = time.monotonic() + timeout_seconds + while True: + try: + return int(pgid_file.read_text()) + except (FileNotFoundError, ValueError): + if time.monotonic() >= deadline: + return None + time.sleep(0.01) + + +@pytest.mark.skipif(not is_posix(), reason="pgrep and process groups are posix-only") +class TestFindChildProcessIdPgrep: + """Tests for find_child_process_id_pgrep(), the non-Linux POSIX lookup of + sudo's child process. + + Defect pinned: this function raised FindSignalTargetError for *any* non-zero + pgrep exit. Exit 1 means "no processes matched", which is the expected answer + for most of the caller's retry window -- sudo has forked but the kernel has + not finished creating the workload yet. Because the caller's + `except FindSignalTargetError` sits outside its retry loop, that first empty + poll ended all retries, so on every non-Linux POSIX host a cross-user launch + recorded no process group and a later cancel had nothing to signal. + """ + + def test_no_matching_process_returns_none(self, python_exe: str) -> None: + """Exit 1 is an answer ("none yet"), not a failure. + + Run against a real, live process that has no children of its own, so it + exercises the real pgrep rather than an assumption about it. The probe + below asserts the operating system's half of the contract -- that this + situation really is exit 1 with no output -- and the `result is None` + assertion is what fails if exit 1 raises again. + """ + # GIVEN a live process with no children of its own + proc = Popen( + [python_exe, "-c", "import time; time.sleep(15)"], + stdin=DEVNULL, + stdout=DEVNULL, + stderr=DEVNULL, + ) + try: + probe = subprocess.run( + ["pgrep", "-P", str(proc.pid)], + stdin=DEVNULL, + capture_output=True, + text=True, + ) + assert ( + probe.returncode == PGREP_EXIT_NO_MATCH + ), f"pgrep did not report 'no match' as exit {PGREP_EXIT_NO_MATCH}: {probe}" + assert probe.stdout.strip() == "", f"pgrep unexpectedly found children: {probe.stdout}" + + # WHEN + result = find_child_process_id_pgrep(sudo_pid=proc.pid) + + # THEN + assert result is None + finally: + proc.kill() + proc.wait() + + @pytest.mark.parametrize( + "returncode,output", + [ + pytest.param(2, "pgrep: illegal option -- Q\n", id="usage-error"), + pytest.param(3, "pgrep: cannot open kernel memory\n", id="fatal-error"), + pytest.param(127, "sh: pgrep: command not found\n", id="not-found"), + ], + ) + def test_other_nonzero_exits_still_raise(self, returncode: int, output: str) -> None: + """The other half of the fix: only exit 1 is benign. + + Treating every non-zero exit as "no child yet" would silently turn a + broken or missing pgrep into "this sudo has no workload", which is the + same lost-signal-target outcome by a different route. + + Both halves of the message are pinned. The exit code alone names a + category of failure without naming the failure: this call merges stderr + into stdout, so the message is the only place pgrep's own explanation + survives, and the exception is caught and logged as a warning rather + than propagated, so there is no traceback to fall back on. + """ + # GIVEN + with patch( + "openjd.sessions._linux._sudo.run", return_value=pgrep_result(returncode, output) + ): + # WHEN + with pytest.raises(FindSignalTargetError) as excinfo: + find_child_process_id_pgrep(sudo_pid=1234) + + # THEN + message = str(excinfo.value) + assert f"exited {returncode}" in message + # repr'd, and so quoted: an operator can see leading/trailing whitespace + # and can tell an empty explanation from a missing one. + assert repr(output.strip()) in message + + def test_single_child_is_returned(self) -> None: + """Exit 0 with one pid: the pid is the answer.""" + # GIVEN + with patch("openjd.sessions._linux._sudo.run", return_value=pgrep_result(0, "4321\n")): + # WHEN + result = find_child_process_id_pgrep(sudo_pid=1234) + + # THEN + assert result == 4321 + + def test_multiple_children_raise(self) -> None: + """Exit 0 with several pids violates the assumption that sudo has exactly + one child, so it must not be resolved by guessing one of them.""" + # GIVEN + with patch( + "openjd.sessions._linux._sudo.run", return_value=pgrep_result(0, "4321\n4322\n") + ): + # WHEN + with pytest.raises(FindSignalTargetError) as excinfo: + find_child_process_id_pgrep(sudo_pid=1234) + + # THEN + assert "4321" in str(excinfo.value) and "4322" in str(excinfo.value) + + def test_no_output_returns_none(self) -> None: + """Exit 0 with no pids is the same "none yet" answer as exit 1, and must + stay distinguishable from it only in how it is reached.""" + # GIVEN + with patch("openjd.sessions._linux._sudo.run", return_value=pgrep_result(0, "")): + # WHEN + result = find_child_process_id_pgrep(sudo_pid=1234) + + # THEN + assert result is None + + +@pytest.mark.skipif(not is_posix(), reason="pgrep and process groups are posix-only") +class TestFindSudoChildProcessGroupId: + """Tests for find_sudo_child_process_group_id() over the pgrep lookup.""" + + def test_keeps_scanning_until_a_late_child_appears( + self, + tmp_path: Path, + python_exe: str, + message_queue: SimpleQueue, + queue_handler: QueueHandler, + ) -> None: + """The behaviour the fix exists for: an empty scan must not end the + retries. + + A parent that gains its child only after a delay reproduces the real + race -- sudo forks before the kernel finishes creating the workload. When + an empty pgrep raised FindSignalTargetError, the caller's handler (which + sits outside its retry loop) swallowed it, logged a warning and returned + None, so the process group was never recorded and a later cancel had + nothing to signal. Asserting the *correct* pgid, not merely "not None", + is what makes this a pin of the fix rather than of the constant. + + is_linux is forced False so that the pgrep path is the one exercised on + every POSIX host; on Linux the caller would otherwise take the procfs + path, which already returns None for this situation. + """ + # GIVEN + if shutil.which("pgrep") is None: + pytest.skip("pgrep is not installed on this host") + parent_script = tmp_path / "parent.py" + parent_script.write_text(PARENT_SCRIPT) + child_script = tmp_path / "child.py" + child_script.write_text(CHILD_SCRIPT) + pgid_file = tmp_path / "child_pgid.txt" + child_appears_after_seconds = 0.4 + logger = build_logger(queue_handler) + + parent = Popen( + [ + python_exe, + str(parent_script), + python_exe, + str(child_script), + str(pgid_file), + str(child_appears_after_seconds), + ], + stdin=DEVNULL, + stdout=DEVNULL, + stderr=DEVNULL, + ) + child_pgid: Optional[int] = None + try: + # WHEN + with patch("openjd.sessions._linux._sudo.is_linux", return_value=False): + # A window far wider than the delay, so a slow host cannot fail + # this spuriously. The scan returns as soon as it finds the + # child, and the defect returns None on the first poll, so the + # wide window costs nothing either way. + result = find_sudo_child_process_group_id( + logger=logger, + sudo_process=parent, + timeout_seconds=10.0, + ) + + # THEN + child_pgid = poll_for_pgid(pgid_file, timeout_seconds=5.0) + assert child_pgid is not None, ( + "the child never reported its process group, so this test could not have " + f"observed one (parent exit code: {parent.poll()})" + ) + assert result == child_pgid + # AND the scan did not report a failure along the way. + messages = collect_queue_messages(message_queue) + assert not any( + "Unable to determine signal target" in message for message in messages + ), messages + finally: + parent.kill() + parent.wait() + if child_pgid is None: + child_pgid = poll_for_pgid(pgid_file, timeout_seconds=0) + if child_pgid is not None: + try: + os.killpg(child_pgid, signal.SIGKILL) # type: ignore + except (ProcessLookupError, PermissionError): + pass + + +@pytest.mark.skipif(not is_posix(), reason="pgrep and process groups are posix-only") +class TestFindSudoChildProcessGroupIdScanErrors: + """Tests that a single bad scan does not end the retries. + + Defect pinned: the `except FindSignalTargetError` that guards the scan used to + sit OUTSIDE the retry `while`, so the *first* bad scan aborted the whole + search -- the process group was never recorded, and a later cancel had + nothing to signal. Every condition the loop races is transient by + construction: the procfs scan sees more than one child while sudo's fork is + still settling, and a `pgrep` invocation can fail for reasons that do not + recur on the next poll. + + The scan is mocked here, unlike the end-to-end test above, because a *failing* + scan is not something a real host can be asked for on demand -- and both + branches (procfs and pgrep) have to be covered from one platform. + """ + + @pytest.mark.parametrize( + "on_linux,scan_function", + [ + pytest.param(True, "find_sudo_child_process_id_procfs", id="procfs"), + pytest.param(False, "find_child_process_id_pgrep", id="pgrep"), + ], + ) + def test_transient_scan_error_does_not_end_the_retries( + self, + on_linux: bool, + scan_function: str, + sleeper: Callable[..., Popen], + message_queue: SimpleQueue, + queue_handler: QueueHandler, + ) -> None: + """One raising scan followed by a succeeding one still yields the pgid. + + Parametrized over both lookups because the two are selected by is_linux() + and the defect was in the shared caller, so a fix verified on one branch + says nothing about the other. + """ + # GIVEN + sudo_process = sleeper() + # The workload, in its own process group as sudo's child would be. + workload = sleeper(own_process_group=True) + logger = build_logger(queue_handler) + scan = Mock(side_effect=[FindSignalTargetError("transient"), workload.pid]) + + # WHEN + with ( + patch(f"openjd.sessions._linux._sudo.{scan_function}", scan), + patch("openjd.sessions._linux._sudo.is_linux", return_value=on_linux), + ): + result = find_sudo_child_process_group_id( + logger=logger, + sudo_process=sudo_process, + timeout_seconds=10.0, + ) + + # THEN + # start_new_session made the workload a process-group leader, so its pgid + # is its own pid. Asserting against that, rather than re-running the + # production expression os.getpgid(pid), keeps the expected value + # independent of the code under test. + assert result == workload.pid + assert scan.call_count == 2, "the scan was not retried after it raised" + # AND nothing was logged at all. The two debug lines on this path sit below + # the level build_logger sets, so an empty queue says more than the absence + # of one particular string -- which a reworded warning would satisfy + # silently. + assert collect_queue_messages(message_queue) == [] + + def test_scan_that_always_fails_times_out_and_reports_the_last_error( + self, + sleeper: Callable[..., Popen], + message_queue: SimpleQueue, + queue_handler: QueueHandler, + ) -> None: + """Retrying is not the same as retrying forever. + + A scan that never succeeds must still end at the timeout, with None and + the warning -- and the warning must carry the last scan error, because + with the error no longer aborting the search it is otherwise discarded + and the timeout message alone cannot say what kept going wrong. + """ + # GIVEN + sudo_process = sleeper() + logger = build_logger(queue_handler) + # The wording find_child_process_id_pgrep really uses for this, so that a + # reader is not left wondering whether it was invented. + scan_error = "Expected a single child process of sudo, but found ['4321', '4322']" + scan = Mock(side_effect=FindSignalTargetError(scan_error)) + + # WHEN + with ( + patch("openjd.sessions._linux._sudo.find_child_process_id_pgrep", scan), + patch("openjd.sessions._linux._sudo.is_linux", return_value=False), + ): + result = find_sudo_child_process_group_id( + logger=logger, + sudo_process=sudo_process, + # A whole second rather than a fraction: the assertion below needs a + # second iteration to have begun and each one sleeps 0.05s, so a + # tight budget would turn a single overrunning sleep on a loaded + # host into a failure. The scan never succeeds, so the full window + # is spent either way. + timeout_seconds=1.0, + ) + + # THEN + assert result is None + assert scan.call_count > 1, "the scan was not retried after it raised" + messages = collect_queue_messages(message_queue) + warnings = [m for m in messages if "Unable to determine signal target" in m] + assert len(warnings) == 1, messages + # The timeout is what is reported, not the individual scan failure: that + # distinction is the whole difference between the fixed and broken code. + assert "unable to detect subprocess before timeout" in warnings[0] + assert f"last scan error: {scan_error}" in warnings[0] + + +@pytest.mark.skipif(not is_posix(), reason="pgrep and process groups are posix-only") +class TestFindSudoChildProcessGroupIdProcessGroupRaces: + """Tests of how find_sudo_child_process_group_id() resolves the two races that + are not scan failures: the child exiting before its group can be read, and the + child not having left sudo's process group yet. + + Both behaviours predate the retry-scoping fix above and had to survive it, so + they are pinned separately from it. The scan is mocked for the same reason as + above -- these are races, and a test that waited for one to occur naturally + would be a test that usually skipped the interesting path. + """ + + def test_child_that_exits_mid_scan_still_short_circuits( + self, + sleeper: Callable[..., Popen], + message_queue: SimpleQueue, + queue_handler: QueueHandler, + ) -> None: + """A child that is gone by the time its group is read is answered + immediately with None. + + Retrying that would burn the whole timeout window on a process that is + known to be gone, and would report it as a timeout rather than as the + ordinary "it already exited" it is -- which is why no warning is logged. + """ + # GIVEN a pid that has been reaped, so os.getpgid() raises for it + departed = sleeper(own_process_group=True) + departed.kill() + departed.wait() + with pytest.raises(ProcessLookupError): + os.getpgid(departed.pid) + sudo_process = sleeper() + logger = build_logger(queue_handler) + scan = Mock(return_value=departed.pid) + + # WHEN + with ( + patch("openjd.sessions._linux._sudo.find_child_process_id_pgrep", scan), + patch("openjd.sessions._linux._sudo.is_linux", return_value=False), + ): + result = find_sudo_child_process_group_id( + logger=logger, + sudo_process=sudo_process, + timeout_seconds=5.0, + ) + + # THEN + assert result is None + assert scan.call_count == 1, "an exited child was scanned for again" + # AND nothing was logged: this is an ordinary outcome, not a failure to + # report. Asserted as an empty queue rather than as the absence of one + # string, which a reworded warning would satisfy silently. + assert collect_queue_messages(message_queue) == [] + + def test_child_still_sharing_sudos_process_group_is_never_accepted( + self, + sleeper: Callable[..., Popen], + message_queue: SimpleQueue, + queue_handler: QueueHandler, + ) -> None: + """sudo's own process group is not an answer. + + Accepting it would aim a later SIGKILL at the group containing this + process, so a cancel would take down the session runtime along with the + workload. Here the group never changes, so the loop must exhaust the + timeout rather than accept what it has. + """ + # GIVEN a live process in the same process group as the stand-in for sudo + sudo_process = sleeper() + sibling = sleeper() + assert os.getpgid(sibling.pid) == os.getpgid( + sudo_process.pid + ), "test setup: the two processes were expected to share a process group" + logger = build_logger(queue_handler) + scan = Mock(return_value=sibling.pid) + + # WHEN + with ( + patch("openjd.sessions._linux._sudo.find_child_process_id_pgrep", scan), + patch("openjd.sessions._linux._sudo.is_linux", return_value=False), + ): + result = find_sudo_child_process_group_id( + logger=logger, + sudo_process=sudo_process, + timeout_seconds=0.2, + ) + + # THEN + assert result is None + messages = collect_queue_messages(message_queue) + warnings = [m for m in messages if "Unable to determine signal target" in m] + assert len(warnings) == 1, messages + assert "unable to detect subprocess before timeout" in warnings[0] + # AND no scan error is invented: nothing here failed to scan. + assert "last scan error" not in warnings[0] + + def test_child_is_accepted_once_it_leaves_sudos_process_group( + self, + tmp_path: Path, + sleeper: Callable[..., Popen], + message_queue: SimpleQueue, + queue_handler: QueueHandler, + ) -> None: + """The other side of the same behaviour: sharing sudo's group is a reason + to retry, not to give up. + + The child moves into its own group partway through the retry window, as + sudo's child really does, and the pgid it reports for itself is what the + function must return. + """ + # GIVEN + pgid_file = tmp_path / "child_pgid.txt" + sudo_process = sleeper() + logger = build_logger(queue_handler) + # Started without own_process_group, so it begins life in this process' + # group -- the same group as the stand-in for sudo -- and calls setsid() + # only after the delay. Registered with the fixture, so it is reaped even + # if the setup assertion below fails. + child = sleeper( + script=LATE_PROCESS_GROUP_SCRIPT + % {"pgid_file": repr(str(pgid_file)), "delay_seconds": 0.5} + ) + scan = Mock(return_value=child.pid) + + # The delay gives this assertion ample room: the child has to start a + # Python interpreter and sleep 0.5s before it can change groups. + assert os.getpgid(child.pid) == os.getpgid( + sudo_process.pid + ), "test setup: the child was expected to start in sudo's process group" + + # WHEN + with ( + patch("openjd.sessions._linux._sudo.find_child_process_id_pgrep", scan), + patch("openjd.sessions._linux._sudo.is_linux", return_value=False), + ): + result = find_sudo_child_process_group_id( + logger=logger, + sudo_process=sudo_process, + timeout_seconds=10.0, + ) + + # THEN + reported_pgid = poll_for_pgid(pgid_file, timeout_seconds=5.0) + assert reported_pgid is not None, ( + "the child never reported a new process group, so this test could not have " + f"observed one (exit code: {child.poll()}, stderr: {stderr_of(child)!r})" + ) + assert result == reported_pgid + assert result != os.getpgid(sudo_process.pid) + # AND nothing was logged: retrying past the shared group is not a failure. + assert collect_queue_messages(message_queue) == [] diff --git a/test/openjd/sessions_v0/test_tempdir.py b/test/openjd/sessions_v0/test_tempdir.py index 1aa01c83..7f18f7aa 100644 --- a/test/openjd/sessions_v0/test_tempdir.py +++ b/test/openjd/sessions_v0/test_tempdir.py @@ -1,11 +1,12 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -from tempfile import gettempdir +from tempfile import gettempdir, mkdtemp import os import shutil import stat from pathlib import Path from subprocess import DEVNULL, run +from typing import Any, Callable from openjd.sessions._os_checker import is_posix, is_windows from openjd.sessions._windows_permission_helper import WindowsPermissionHelper @@ -33,14 +34,181 @@ has_posix_disjoint_user, has_posix_target_user, has_windows_user, + nonexistent_group_name, + resolvable_member_groups, WIN_SET_TEST_ENV_VARS_MESSAGE, POSIX_SET_TARGET_USER_ENV_VARS_MESSAGE, POSIX_SET_DISJOINT_USER_ENV_VARS_MESSAGE, ) +def spy_mkdtemp(created: list[str]) -> Callable[..., str]: + """A `mkdtemp` that records what it creates, for asserting on a directory whose + path the failed constructor never handed back. + + Recording the exact path, rather than diffing a directory listing, is what + makes the assertion safe where the session root is shared: another test + creating its own session directory there cannot affect it. + """ + + def _mkdtemp(**kwargs: Any) -> str: + path = mkdtemp(**kwargs) + created.append(path) + return path + + return _mkdtemp + + @pytest.mark.skipif(not is_posix(), reason="Posix-specific tests") class TestTempDirPosix: + def test_unresolvable_group_raises_runtimeerror(self, tmp_path: Path) -> None: + """Pins: a group name that does not resolve must fail TempDir() as + RuntimeError, the error this constructor documents. + + shutil.chown raises LookupError for an unknown group, and LookupError is + neither OSError nor ValueError, so it escaped the `except OSError` around + the ownership change here (and every other handler in the session-setup + chain) and reached the caller of the public Session API unchanged. + PosixSessionUser does not validate its group, so a caller only has to + pass a group that does not exist. + + pytest.raises(RuntimeError) does not catch LookupError, so if the + translation is removed this test errors out with the escaping + LookupError -- which is the defect itself. + """ + # GIVEN + # Only `group` matters on this path; the user is never resolved by it. + user = PosixSessionUser(user="nobody", group=nonexistent_group_name()) + + # WHEN + with pytest.raises(RuntimeError) as excinfo: + TempDir(dir=tmp_path, user=user) + + # THEN + # The failure is the group ownership change, not something incidental: + # the offending group name is carried through to the message. + assert user.group in str(excinfo.value) + + def test_failed_ownership_change_leaves_no_directory(self, tmp_path: Path) -> None: + """Pins: a construction that fails at the ownership step removes the + directory it had already created. + + mkdtemp() runs before the ownership change, so a failure there used to + leave the directory behind with no way to reach it: __init__ raised, so + the caller never received a TempDir and has no path to call cleanup() + with. In production that directory is not under a per-test tmp_path but + under the shared `/OpenJD` root, which nothing else ever prunes, + so each failed session leaks one more 0o700 directory there. + + Asserting on the directory listing, rather than on the exception alone, is + what makes this a pin: the pre-fix code raises the same RuntimeError. + """ + # GIVEN + # An unresolvable group reaches the failure path without needing any + # privileges to arrange: PosixSessionUser does not validate its group. + user = PosixSessionUser(user="nobody", group=nonexistent_group_name()) + before = set(os.listdir(tmp_path)) + created: list[str] = [] + + # WHEN + # Matched, not bare: this constructor raises RuntimeError for several + # earlier failures too (mkdtemp itself, for one), and those never create a + # directory, so a bare match could pass without the leak path running. + with patch("openjd.sessions._tempdir.mkdtemp", side_effect=spy_mkdtemp(created)): + with pytest.raises(RuntimeError, match="Could not change ownership"): + TempDir(dir=tmp_path, user=user) + + # THEN + assert set(os.listdir(tmp_path)) == before + # AND, named exactly rather than inferred from the listing: TempDir resolves + # `dir` before creating anything, so a listing on its own could in principle + # be looking at the wrong directory. + assert created, "test setup: mkdtemp was not reached, so nothing could have leaked" + assert not os.path.exists(created[0]) + + def test_successful_ownership_change_keeps_the_directory(self, tmp_path: Path) -> None: + """The counterpart: the cleanup-on-failure must not fire on success. + + A guard that removed the directory unconditionally would pass the leak + test above and break every real session, so the success path is pinned in + the same place -- including the widened mode, since that is set after the + ownership change and inside the same guarded block. + """ + # GIVEN a group this process is a member of whose gid the new directory + # would NOT already have. chown to one's own group needs no privileges, so + # this exercises the real ownership path on any POSIX host without a + # provisioned test user -- but only a *different* gid proves the chown + # happened, since mkdtemp already inherits the parent's gid. + inherited_gid = os.stat(tmp_path).st_gid + candidates = [ + (gid, name) for gid, name in resolvable_member_groups() if gid != inherited_gid + ] + if not candidates: + pytest.skip("this process is a member of no group other than the one it would inherit") + gid, group_name = candidates[0] + user = PosixSessionUser(user=pwd.getpwuid(os.geteuid()).pw_name, group=group_name) # type: ignore + + # WHEN + result = TempDir(dir=tmp_path, user=user) + + # THEN + assert result.path.is_dir() + statinfo = os.stat(result.path) + assert statinfo.st_gid == gid + # 0o770: mkdtemp's 0o700 widened for the group, and no wider. + assert stat.S_IMODE(statinfo.st_mode) == stat.S_IRWXU | stat.S_IRWXG + + @pytest.mark.skipif( + os.geteuid() == 0 if is_posix() else True, + reason="root is not subject to directory permissions", + ) + def test_cleanup_failure_does_not_replace_the_ownership_error(self, tmp_path: Path) -> None: + """Pins: the cleanup is best-effort, and the failure it cleans up after is + still the one the caller sees. + + An exception raised inside an `except` block replaces the one being + handled. If the removal were not best-effort, a session whose group could + not be set would report a permission error about a temporary directory + instead of naming the group -- turning a diagnosable configuration + mistake into a confusing one, and only on the hosts where cleanup + happens to fail. + + The removal is made to fail for real, by taking write permission off the + parent directory at the moment the ownership change fails: rmdir() needs + write access to the parent, not to the directory being removed. + + Skipped as root, which ignores directory permissions -- so a green run as + root is not evidence for this behaviour. It is covered by the + `localuser_sudo_environment` container, which runs the suite as `hostuser`. + """ + # GIVEN + user = PosixSessionUser(user="nobody", group=nonexistent_group_name()) + before = set(os.listdir(tmp_path)) + original_mode = stat.S_IMODE(os.stat(tmp_path).st_mode) + + def fail_and_make_the_parent_unwritable(path: Path, group: str) -> None: + os.chmod(tmp_path, stat.S_IRUSR | stat.S_IXUSR) + raise OSError(f"no such group: {group}") + + try: + with patch( + "openjd.sessions._tempdir.chown_group", + side_effect=fail_and_make_the_parent_unwritable, + ): + # WHEN + with pytest.raises(RuntimeError) as excinfo: + TempDir(dir=tmp_path, user=user) + + # THEN the ownership failure survived the failed cleanup. + assert user.group in str(excinfo.value) + # AND the cleanup really did fail, so this test is exercising the + # best-effort path rather than the ordinary one. + assert set(os.listdir(tmp_path)) != before + finally: + # Restored to what it was, so that this test's temporary directory can + # be collected however pytest chose to create it. + os.chmod(tmp_path, original_mode) + def test_defaults(self) -> None: # GIVEN tmpdir = Path(os.path.join(gettempdir(), "OpenJD")).resolve() @@ -173,10 +341,20 @@ def test_windows_permissions_inherited(self, mock_user_match, windows_user: Wind def test_nonvalid_windows_principal_raises_exception(self, mock_user_match): # GIVEN windows_user = WindowsSessionUser("non_existent_user") + created: list[str] = [] # THEN - with pytest.raises(RuntimeError, match="Could not change permissions of directory"): - TempDir(user=windows_user) + with patch("openjd.sessions._tempdir.mkdtemp", side_effect=spy_mkdtemp(created)): + with pytest.raises(RuntimeError, match="Could not change permissions of directory"): + TempDir(user=windows_user) + + # AND the directory that had already been created was removed. This is the + # Windows half of the same leak as + # TestTempDirPosix::test_failed_ownership_change_leaves_no_directory, and it + # matters more here: no `dir` was passed, so the directory was created in + # the shared `%PROGRAMDATA%\Amazon\OpenJD` root that nothing else prunes. + assert created, "test setup: mkdtemp was not reached, so nothing could have leaked" + assert not os.path.exists(created[0]) @pytest.fixture def clean_up_directory(self): From 572ae7962c358a7e46804da856a5df65cd695047 Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Sun, 26 Jul 2026 14:39:59 -0700 Subject: [PATCH 19/27] test: Replace two flaky timing tests with deterministic ones The suite had two persistent false reds. Both were test defects, and neither was mysterious once actually investigated rather than re-run. test_run_action_default_timeout asserted a one-second window on a child that prints one line per second: assert f"Log from test {T - 1}" in messages assert f"Log from test {T + 1}" not in messages The runtime-limit Timer is armed in `_run` BEFORE the child is submitted to the pool, so the Timer's clock starts before the child interpreter even launches. For the 2s case the child had to reach its second line within 2s of arming, which made the assertion a measure of interpreter startup and scheduler latency. Measured: injecting a 1.5s startup delay -- well under the 2s timeout -- fails the old assertion with the runner correctly in TIMEOUT. Under `-n auto` a loaded host supplies that delay for free. It is split into the two things it was conflating: - test_run_action_effective_timeout asserts the time limit `_run_action` hands to `_run`. No subprocess, no wall clock. Five cases, including two the old end-to-end form could not afford: an action timeout SMALLER than the default, and an action timeout with no default at all. - test_run_action_timeout_is_enforced keeps the end-to-end check, deliberately loose: the terminal state, and whether the child reached its last line. It says nothing about which second it got to, because that is the part that was never this test's business. Mutation-checked, 3 mutants, 3 caught: ignoring the action's own timeout, ignoring the caller's default, and never passing the limit to `_run`. The replacement pins strictly more than the flaky version did -- the old test could not distinguish "the default was ignored" from "the host was slow". test_cancel_during_launch_is_not_dropped failed about 1 run in 4 in isolation, and this one is my own doing. Commit ec29eeb changed `_run` to publish `_process` and `_run_future` together AFTER `_pool.submit`, which was right for the `state` property but widened the window in which `_process` is still None. The test's `assert runner._process is not None` precondition then fired on the canceller thread, killing it, so `canceller_done` was never set, `_start_after_cancel` waited out its full 30s, and the assertion reported SUCCESS -- a failure a long way from its cause. The precondition is wrong now rather than merely unlucky: both windows (before _process is published, and after it exists but before it has started) must record the cancel rather than drop it, and the assertions after the run already check exactly that. Precondition removed, and the canceller wrapped in try/finally so a future failure there cannot masquerade as a product failure. The product behaviour is unchanged by this commit. Verified: three consecutive full-suite runs, 852 passed / 0 failed each. First green run of the suite in this campaign. ruff, mypy and black clean. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- test/openjd/sessions_v0/test_runner_base.py | 110 +++++++++++++----- .../sessions_v0/test_runner_step_script.py | 23 +++- 2 files changed, 99 insertions(+), 34 deletions(-) diff --git a/test/openjd/sessions_v0/test_runner_base.py b/test/openjd/sessions_v0/test_runner_base.py index 43f64069..cf541911 100644 --- a/test/openjd/sessions_v0/test_runner_base.py +++ b/test/openjd/sessions_v0/test_runner_base.py @@ -8,7 +8,7 @@ from pathlib import Path from queue import SimpleQueue from typing import Optional, cast -from unittest.mock import MagicMock, call +from unittest.mock import MagicMock, call, patch import pytest @@ -650,38 +650,97 @@ def test_run_action( assert "Log from test 0" in messages assert "Log from test 9" not in messages - @pytest.mark.usefixtures("message_queue", "queue_handler") @pytest.mark.parametrize( - argnames=("default_timeout_seconds", "action_timeout_seconds"), + argnames=("default_timeout_seconds", "action_timeout_seconds", "expected_seconds"), argvalues=( - pytest.param(1, 5, id="action-timeout-prevails"), - pytest.param(2, None, id="default-applied"), - pytest.param(None, None, id="no-timeout"), + pytest.param(1, 5, 5, id="action-timeout-prevails"), + pytest.param(5, 1, 1, id="action-timeout-prevails-when-smaller"), + pytest.param(2, None, 2, id="default-applied"), + pytest.param(None, 7, 7, id="action-only"), + pytest.param(None, None, None, id="no-timeout"), ), ) - def test_run_action_default_timeout( + def test_run_action_effective_timeout( self, tmp_path: Path, - message_queue: SimpleQueue, - queue_handler: QueueHandler, default_timeout_seconds: Optional[int], action_timeout_seconds: Optional[int], + expected_seconds: Optional[int], python_exe: str, ) -> None: - # Tests that the effective timeout is applied correctly given a supplied default timeout - # and an optional timeout defined on the action - + """The effective time limit is the action's timeout if it has one, else the + caller's default, else none at all. + + Asserts the time limit `_run_action` hands to `_run`, with no subprocess + and no wall clock involved. + + This replaces an end-to-end version that started a child printing one line + per second and asserted a +/-1 second window on its output + (`"Log from test {T-1}" in messages`). That coupled three unrelated + things -- timeout *selection*, `threading.Timer` scheduling, and child + process startup latency -- and only the first is what this test is named + for. The Timer is armed before the child is submitted to the pool, so the + Timer's clock starts before the interpreter even launches; any host slow + enough to delay startup past a second failed the assertion while the + product behaved correctly. Measured: injecting a 1.5s startup delay + against a 2s timeout fails the old assertion with the runner correctly in + TIMEOUT. It was the suite's most persistent false red. + """ # GIVEN - expected_effective_timeout_seconds: Optional[int] = None - if action_timeout_seconds is not None: - expected_effective_timeout_seconds = action_timeout_seconds - elif default_timeout_seconds is not None: - expected_effective_timeout_seconds = default_timeout_seconds default_timeout = ( timedelta(seconds=default_timeout_seconds) if default_timeout_seconds is not None else None ) + action = Action_2023_09( + command=CommandString_2023_09("{{Task.PythonInterpreter}}"), + args=[ArgString_2023_09("-c"), ArgString_2023_09("pass")], + timeout=action_timeout_seconds, + ) + symtab = SymbolTable(source={"Task.PythonInterpreter": python_exe}) + captured: list[Optional[timedelta]] = [] + + with TerminatingRunner(logger=MagicMock(), session_working_directory=tmp_path) as runner: + # WHEN + with patch.object( + runner, + "_run", + side_effect=lambda args, time_limit=None: captured.append(time_limit), + ): + runner._run_action(action, symtab, default_timeout=default_timeout) + + # THEN + assert len(captured) == 1, "the action was not launched exactly once" + expected = timedelta(seconds=expected_seconds) if expected_seconds is not None else None + assert captured[0] == expected + + @pytest.mark.usefixtures("message_queue", "queue_handler") + @pytest.mark.parametrize( + argnames=("action_timeout_seconds", "expected_state"), + argvalues=( + pytest.param(2, ScriptRunnerState.TIMEOUT, id="timeout-terminates-the-action"), + pytest.param(None, ScriptRunnerState.SUCCESS, id="no-timeout-runs-to-completion"), + ), + ) + def test_run_action_timeout_is_enforced( + self, + tmp_path: Path, + message_queue: SimpleQueue, + queue_handler: QueueHandler, + action_timeout_seconds: Optional[int], + expected_state: ScriptRunnerState, + python_exe: str, + ) -> None: + """A declared timeout really does terminate the action, and no timeout + really does let it finish. + + The end-to-end half of the coverage above, kept deliberately loose: it + asserts only the terminal state and that the child did or did not reach its + last line. It says nothing about *when* the timeout landed, because the + exact second the child reaches depends on how promptly the host scheduled + it -- which is what made the previous version of this test flaky. + """ + # GIVEN: a child that prints one line a second for 20 seconds action = Action_2023_09( command=CommandString_2023_09("{{Task.PythonInterpreter}}"), args=[ArgString_2023_09("{{Task.ScriptFile}}")], @@ -697,23 +756,16 @@ def test_run_action_default_timeout( logger = build_logger(queue_handler) with TerminatingRunner(logger=logger, session_working_directory=tmp_path) as runner: # WHEN - runner._run_action(action, symtab, default_timeout=default_timeout) - # wait for the process to exit + runner._run_action(action, symtab) while runner.state == ScriptRunnerState.RUNNING: time.sleep(0.2) # THEN - if expected_effective_timeout_seconds is not None: - assert runner.state == ScriptRunnerState.TIMEOUT - else: - assert runner.state == ScriptRunnerState.SUCCESS + assert runner.state == expected_state messages = collect_queue_messages(message_queue) - # The application prints out 0, ..., 19 once a second for 20s . - # If it ended early, then we printed the first but not the last. - print(messages) - if expected_effective_timeout_seconds is not None: - assert f"Log from test {expected_effective_timeout_seconds - 1}" in messages - assert f"Log from test {expected_effective_timeout_seconds + 1}" not in messages + if expected_state is ScriptRunnerState.TIMEOUT: + # It was cut short. Which line it got to is a scheduling detail. + assert "Log from test 19" not in messages else: assert "Log from test 19" in messages diff --git a/test/openjd/sessions_v0/test_runner_step_script.py b/test/openjd/sessions_v0/test_runner_step_script.py index 0884d9ba..9f3f4d9d 100644 --- a/test/openjd/sessions_v0/test_runner_step_script.py +++ b/test/openjd/sessions_v0/test_runner_step_script.py @@ -475,11 +475,24 @@ def _start_after_cancel(subproc): # type: ignore[no-untyped-def] return real_start(subproc) def _cancel_in_window() -> None: - in_window.wait(timeout=30) - assert runner._process is not None - assert runner._process.has_started is False - runner.cancel() - canceller_done.set() + # try/finally so canceller_done is set no matter what. This runs on its + # own thread, so an exception here would otherwise leave + # _start_after_cancel waiting out its full 30s and the assertions below + # reporting SUCCESS -- a confusing failure a long way from its cause. + try: + in_window.wait(timeout=30) + # Deliberately no assertion on runner._process. This hook runs on + # the pool thread, and `_run` publishes _process and _run_future + # together *after* submitting to the pool, so at this instant + # _process may legitimately still be None. Asserting otherwise made + # this test fail about 1 run in 4 even in isolation: the assertion + # killed its own canceller thread. Both windows -- before _process + # is published, and after it exists but before it has started -- + # must record the cancel rather than drop it, and that is what the + # assertions after the run check. + runner.cancel() + finally: + canceller_done.set() canceller = threading.Thread(target=_cancel_in_window, daemon=True) From ed37b619628d4a6742d3308782a2ce4b3522565c Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Sun, 26 Jul 2026 15:38:57 -0700 Subject: [PATCH 20/27] test: Run the subprocess-racing tests serially, and quiesce between them A loaded 108-second suite run failed 14 tests at once -- every cancel/terminate test in the suite -- while the product was behaving correctly. All 14 pass serially. One log line makes the mechanism plain: Canceling subprocess 69379 via termination method Log from test 0 ... Log from test 19 # child ran the full 20s, exit 0 These tests start a real child, cancel or time it out, and assert on the outcome. The assertions are right, but they assume the child and the runtime get scheduled promptly. Under `-n auto` with twelve workers each also sleeping on a child, that assumption fails. The failure is indistinguishable from a real cancel regression, which is the expensive part: it teaches you to re-run rather than to read. Adds a `serial_process` mark (a `pytest.mark.xdist_group`) in conftest, applied to the tests that race the wall clock. Every such test lands on one xdist worker, so they run serially with respect to each other while the other ~800 tests still run in parallel. Requires `--dist=loadgroup`, added to addopts; under the default `--dist=load` the marker is accepted, ignored, and reported nowhere. Adds an autouse `_quiesce_after_process_test` fixture, keyed on that mark, which cancels stray `threading.Timer`s and waits briefly for the thread count to settle. Serialising only helps if the tests also stop overlapping in the background: a `ScriptRunnerBase` leaves a timer running for a whole unexpired timeout or grace period, plus a pool worker, and a test that finishes early by cancelling its child hands both to whatever runs next. Scoping, after measuring rather than assuming. Marking the two big classes wholesale put ~60 process tests on one worker and made it the critical path: the suite went 40s -> 94s. The mark is therefore on the 13 specific tests that actually flaked, not on `TestScriptRunnerBase` or `TestLoggingSubprocessSameUser` entire. The quiesce budget is 1s, not 5s: some tests legitimately leave a daemon stdout-reader thread that never exits, and a generous budget was being spent in full on every one of them -- measured at 5s of teardown for a single test. Also removes the suite's slowest test. `test_run_action_default_timeout`'s no-timeout case ran a 20-second child to completion, at 21.3s the slowest test by a factor of three and the entire critical path. Split into test_run_action_timeout_terminates_the_action (unchanged intent) and test_run_action_without_timeout_runs_to_completion, which uses a child that exits after half a second -- what is being asserted is that no timer cut the action short, and a child that exits on its own shows that just as well. Adds test_conftest_serial_process.py so this mechanism cannot regress silently. Note its first version asserted `getoption("dist") == "loadgroup"` and failed: inside an xdist worker that option is `"no"`, because a worker runs its share serially and only the controller distributes. It now reads `addopts`. No product code changes. Verified: three consecutive full runs, 855 passed / 0 failed, in 44.2s / 41.5s / 40.7s -- the pre-change baseline was 40s with intermittent 14-failure runs. ruff, mypy and black clean. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- pyproject.toml | 5 ++ test/openjd/sessions_v0/conftest.py | 74 ++++++++++++++++ .../sessions_v0/test_concurrency_fixes.py | 2 + .../test_conftest_serial_process.py | 71 +++++++++++++++ test/openjd/sessions_v0/test_runner_base.py | 87 +++++++++++++------ test/openjd/sessions_v0/test_session.py | 2 + .../test_session_run_subprocess.py | 2 + test/openjd/sessions_v0/test_subprocess.py | 4 + .../sessions_v0/test_wrap_cancelation.py | 3 + 9 files changed, 223 insertions(+), 27 deletions(-) create mode 100644 test/openjd/sessions_v0/test_conftest_serial_process.py diff --git a/pyproject.toml b/pyproject.toml index fca4f620..3430ccae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -155,6 +155,11 @@ addopts = [ "--cov-report=xml:build/coverage/coverage.xml", "--cov-report=term-missing", "--numprocesses=auto", + # loadgroup, not the default load: it honours @pytest.mark.xdist_group, which + # the tests that race a real subprocess against the wall clock rely on to run + # serially with respect to each other (see conftest.serial_process). Behaves + # exactly like `load` for every test that is not in a group. + "--dist=loadgroup", "--timeout=30", "--log-format=%(asctime)s.%(msecs)03d %(levelname)s %(filename)20s:%(lineno)-3s %(message)s", "--log-date-format=%H:%M:%S" diff --git a/test/openjd/sessions_v0/conftest.py b/test/openjd/sessions_v0/conftest.py index a8fe4dbf..db227f3e 100644 --- a/test/openjd/sessions_v0/conftest.py +++ b/test/openjd/sessions_v0/conftest.py @@ -3,6 +3,7 @@ import os import random import string +import time import uuid from logging import INFO, getLogger from logging.handlers import QueueHandler @@ -66,6 +67,79 @@ def pytest_collection_modifyitems(config, items): config.option.markexpr = mark_expr +SERIAL_PROCESS_GROUP = "serial_process" + +serial_process = pytest.mark.xdist_group(SERIAL_PROCESS_GROUP) +"""Mark for tests that race a real subprocess against wall-clock expectations. + +Applied to a class or a test, it pins every such test onto ONE xdist worker, so +they run serially with respect to each other instead of competing for CPU with +eleven siblings that are each also sleeping on a child process. + +Why this is needed: these tests start a child, cancel or time it out, and assert +on the outcome. The assertions are correct, but they assume the child and the +runtime get scheduled reasonably promptly. Under `-n auto` on a loaded host that +assumption fails, and the whole cancel/terminate family goes red together while +the product is behaving correctly -- observed as 14 simultaneous failures in a +108-second run that all passed serially. The failure mode is indistinguishable +from a real cancel regression, which is the expensive part: it trains you to +re-run rather than to read. + +Requires `--dist=loadgroup` (set in pyproject.toml). With plain `--dist=load` the +marker is silently ignored -- see `test_conftest_serial_process.py`, which fails +if that ever regresses. + +This is not a substitute for fixing genuinely flaky assertions. Where a test +asserted something it had no business asserting -- a +/-1 second window on a +child's output, say -- that assertion was removed rather than protected by this. +""" + + +@pytest.fixture(autouse=True) +def _quiesce_after_process_test(request: pytest.FixtureRequest) -> Generator[None, None, None]: + """Between serial-process tests, wait for the timers and threads the previous + one created to actually go away. + + A `ScriptRunnerBase` leaves a `threading.Timer` running for the whole of an + unexpired timeout or cancel grace period, and a `ThreadPoolExecutor` worker + behind it. A test that finishes early -- because it cancelled its child -- can + therefore hand a live 30-second timer and a busy thread to whatever runs next. + Serialising the tests only helps if they also stop overlapping in that way. + + Cancels stray timers, then waits briefly for the thread count to settle. Does + not assert: a leftover thread is not necessarily this test's fault, and turning + that into a failure here would report it against the wrong test. It is logged + so it is visible when it matters. + """ + marker = request.node.get_closest_marker("xdist_group") + if marker is None or SERIAL_PROCESS_GROUP not in marker.args: + yield + return + + import threading + + before = threading.active_count() + yield + + for thread in threading.enumerate(): + if isinstance(thread, threading.Timer) and thread.is_alive(): + thread.cancel() + + # Deliberately short. Some tests legitimately leave a daemon stdout-reader + # thread behind that never exits (documented in LoggingSubprocess), so a + # generous budget here is spent in full on every one of them -- measured at 5s + # of pure teardown for a single test. One second is enough for a cancelled + # timer and a pool worker to wind down, which is what this is for. + deadline = time.monotonic() + 1.0 + while threading.active_count() > before and time.monotonic() < deadline: + time.sleep(0.02) + if threading.active_count() > before: + print( + f"\n[quiesce] {request.node.name} left " + f"{threading.active_count() - before} extra thread(s) running" + ) + + def nonexistent_group_name() -> str: """A group name that cannot resolve on any host. diff --git a/test/openjd/sessions_v0/test_concurrency_fixes.py b/test/openjd/sessions_v0/test_concurrency_fixes.py index b26aa7ed..0025c27f 100644 --- a/test/openjd/sessions_v0/test_concurrency_fixes.py +++ b/test/openjd/sessions_v0/test_concurrency_fixes.py @@ -31,6 +31,7 @@ from openjd.sessions._subprocess import LoggingSubprocess from .conftest import build_logger +from .conftest import serial_process @pytest.mark.usefixtures("message_queue", "queue_handler") @@ -299,6 +300,7 @@ def bad_callback(state: ActionState) -> None: @pytest.mark.usefixtures("message_queue", "queue_handler") @pytest.mark.skipif(sys.platform == "win32", reason="POSIX-specific signal handling") +@serial_process class TestReview22F4DoubleLoadFix: """Review22-F4: notify/terminate must bind _process once and pass to helpers.""" diff --git a/test/openjd/sessions_v0/test_conftest_serial_process.py b/test/openjd/sessions_v0/test_conftest_serial_process.py new file mode 100644 index 00000000..8a539e5a --- /dev/null +++ b/test/openjd/sessions_v0/test_conftest_serial_process.py @@ -0,0 +1,71 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""The `serial_process` mechanism must keep working, silently-failing being its +whole risk. + +`@pytest.mark.xdist_group` is only honoured under `--dist=loadgroup`. Under the +default `--dist=load` it is accepted, ignored, and reported nowhere -- so the +process-heavy tests would quietly go back to competing with eleven siblings and +the cancel/terminate family would start failing together again on loaded hosts, +looking exactly like a product regression. + +These tests are cheap and exist so that a change to `addopts` fails here, next to +an explanation, rather than as fourteen mystery failures somewhere else. +""" + +from __future__ import annotations + +import pytest + +from .conftest import SERIAL_PROCESS_GROUP, serial_process + + +def test_loadgroup_distribution_is_configured(pytestconfig: pytest.Config) -> None: + """Pins the `--dist=loadgroup` setting the marker depends on. + + Read from `addopts` rather than from `getoption("dist")`: inside an xdist + worker the resolved `dist` option is `"no"`, because a worker runs its own + share serially and only the controller distributes. Asserting on the resolved + option therefore fails on the workers and passes nowhere useful -- which is + exactly what the first version of this test did. + """ + # GIVEN / WHEN + addopts = " ".join(pytestconfig.getini("addopts")) + + # THEN + assert "--dist=loadgroup" in addopts, ( + "xdist_group markers are silently ignored unless --dist=loadgroup; the " + "serial_process tests would go back to running in parallel with each other" + ) + + +def test_serial_process_is_an_xdist_group_marker() -> None: + """The mark must be the one xdist looks for, with the expected group name. + + A rename or a typo would leave a decorator that reads as if it does something + and does nothing at all. + """ + # GIVEN / WHEN + mark = serial_process.mark + + # THEN + assert mark.name == "xdist_group" + assert mark.args == (SERIAL_PROCESS_GROUP,) + + +@serial_process +class TestMarkerReachesTheTest: + def test_the_marker_is_visible_on_a_decorated_test( + self, request: pytest.FixtureRequest + ) -> None: + """A class-level mark must actually reach the test item. + + This is what the quiesce fixture in conftest keys off, so if the mark stops + propagating the cleanup silently stops running too. + """ + # GIVEN / WHEN + marker = request.node.get_closest_marker("xdist_group") + + # THEN + assert marker is not None + assert SERIAL_PROCESS_GROUP in marker.args diff --git a/test/openjd/sessions_v0/test_runner_base.py b/test/openjd/sessions_v0/test_runner_base.py index cf541911..4a993b42 100644 --- a/test/openjd/sessions_v0/test_runner_base.py +++ b/test/openjd/sessions_v0/test_runner_base.py @@ -39,6 +39,7 @@ from openjd.sessions._tempdir import TempDir from .conftest import ( + serial_process, build_logger, collect_queue_messages, has_posix_target_user, @@ -609,6 +610,7 @@ def test_cannot_run_twice(self, tmp_path: Path, python_exe: str) -> None: with pytest.raises(RuntimeError): runner._run([python_exe, "-c", "print('hello')"]) + @serial_process @pytest.mark.usefixtures("message_queue", "queue_handler") def test_run_action( self, @@ -714,37 +716,29 @@ def test_run_action_effective_timeout( expected = timedelta(seconds=expected_seconds) if expected_seconds is not None else None assert captured[0] == expected + @serial_process @pytest.mark.usefixtures("message_queue", "queue_handler") - @pytest.mark.parametrize( - argnames=("action_timeout_seconds", "expected_state"), - argvalues=( - pytest.param(2, ScriptRunnerState.TIMEOUT, id="timeout-terminates-the-action"), - pytest.param(None, ScriptRunnerState.SUCCESS, id="no-timeout-runs-to-completion"), - ), - ) - def test_run_action_timeout_is_enforced( + def test_run_action_timeout_terminates_the_action( self, tmp_path: Path, message_queue: SimpleQueue, queue_handler: QueueHandler, - action_timeout_seconds: Optional[int], - expected_state: ScriptRunnerState, python_exe: str, ) -> None: - """A declared timeout really does terminate the action, and no timeout - really does let it finish. - - The end-to-end half of the coverage above, kept deliberately loose: it - asserts only the terminal state and that the child did or did not reach its - last line. It says nothing about *when* the timeout landed, because the - exact second the child reaches depends on how promptly the host scheduled - it -- which is what made the previous version of this test flaky. + """A declared timeout really does terminate the action. + + The end-to-end half of the coverage above, kept deliberately loose: the + terminal state, and that the child did not reach its last line. It says + nothing about *when* the timeout landed, because which second the child + reaches depends on how promptly the host scheduled it -- that assumption is + what made the previous version of this test flaky. """ - # GIVEN: a child that prints one line a second for 20 seconds + # GIVEN: a child that prints one line a second for 20 seconds, and a + # timeout that will cut it short action = Action_2023_09( command=CommandString_2023_09("{{Task.PythonInterpreter}}"), args=[ArgString_2023_09("{{Task.ScriptFile}}")], - timeout=action_timeout_seconds, + timeout=2, ) python_app_loc = (Path(__file__).parent / "support_files" / "app_20s_run.py").resolve() symtab = SymbolTable( @@ -761,13 +755,48 @@ def test_run_action_timeout_is_enforced( time.sleep(0.2) # THEN - assert runner.state == expected_state - messages = collect_queue_messages(message_queue) - if expected_state is ScriptRunnerState.TIMEOUT: - # It was cut short. Which line it got to is a scheduling detail. - assert "Log from test 19" not in messages - else: - assert "Log from test 19" in messages + assert runner.state == ScriptRunnerState.TIMEOUT + assert "Log from test 19" not in collect_queue_messages(message_queue) + + @pytest.mark.usefixtures("message_queue", "queue_handler") + def test_run_action_without_timeout_runs_to_completion( + self, + tmp_path: Path, + message_queue: SimpleQueue, + queue_handler: QueueHandler, + python_exe: str, + ) -> None: + """No timeout means no time limit is imposed: the action finishes on its + own terms. + + Uses a child that exits after a fraction of a second rather than the + 20-second one. Running a 20-second child to completion made this the single + slowest test in the suite by a factor of three, and it was the whole + critical path under `-n auto` -- while proving nothing that a short child + does not. What is being asserted is that no timer cut the action short, and + a child that exits on its own shows that either way. + """ + # GIVEN: a short-lived child, and no timeout on the action or the caller + action = Action_2023_09( + command=CommandString_2023_09("{{Task.PythonInterpreter}}"), + args=[ + ArgString_2023_09("-c"), + ArgString_2023_09("import time; time.sleep(0.5); print('finished')"), + ], + timeout=None, + ) + symtab = SymbolTable(source={"Task.PythonInterpreter": python_exe}) + logger = build_logger(queue_handler) + with TerminatingRunner(logger=logger, session_working_directory=tmp_path) as runner: + # WHEN + runner._run_action(action, symtab, default_timeout=None) + while runner.state == ScriptRunnerState.RUNNING: + time.sleep(0.05) + + # THEN: it ran to its own end, and its last output arrived. + assert runner.state == ScriptRunnerState.SUCCESS + assert runner.exit_code == 0 + assert "finished" in collect_queue_messages(message_queue) @pytest.mark.usefixtures("message_queue", "queue_handler") @pytest.mark.parametrize( @@ -875,6 +904,7 @@ def test_run_action_bad_formatstring( messages = collect_queue_messages(message_queue) assert any(m.startswith("openjd_fail") for m in messages) + @serial_process @pytest.mark.usefixtures("message_queue", "queue_handler") def test_cancel_terminate( self, @@ -910,6 +940,7 @@ def test_cancel_terminate( # Didn't get to the end of the application run assert "Log from test 9" not in messages + @serial_process @pytest.mark.usefixtures("message_queue", "queue_handler") @pytest.mark.xfail(not is_posix(), reason="Signals not yet implemented for non-posix") def test_run_with_time_limit( @@ -942,6 +973,7 @@ def test_run_with_time_limit( # Didn't get to the end of the application run assert "Log from test 9" not in messages + @serial_process @pytest.mark.usefixtures("message_queue", "queue_handler") def test_cancel_notify( self, @@ -1105,6 +1137,7 @@ def test_cancel_notify_direct_signal_with_cap_kill( cap_kill_was_effective == cap_kill_effective_after_cancel ), "CAP_KILL added/removed from effetive set and persisted after cancelation" + @serial_process @pytest.mark.usefixtures("message_queue", "queue_handler") def test_cancel_double_cancel_notify( self, diff --git a/test/openjd/sessions_v0/test_session.py b/test/openjd/sessions_v0/test_session.py index 3c25b870..b09219d6 100644 --- a/test/openjd/sessions_v0/test_session.py +++ b/test/openjd/sessions_v0/test_session.py @@ -66,6 +66,7 @@ from openjd.sessions._windows_permission_helper import WindowsPermissionHelper from .conftest import ( + serial_process, has_posix_target_user, has_windows_user, WIN_SET_TEST_ENV_VARS_MESSAGE, @@ -954,6 +955,7 @@ def test_run_task_with_variables( ) +@serial_process class TestSessionCancel: """Test that cancelation will cancel the currently running Script.""" diff --git a/test/openjd/sessions_v0/test_session_run_subprocess.py b/test/openjd/sessions_v0/test_session_run_subprocess.py index 809c86e7..6c464cb5 100644 --- a/test/openjd/sessions_v0/test_session_run_subprocess.py +++ b/test/openjd/sessions_v0/test_session_run_subprocess.py @@ -21,6 +21,7 @@ from openjd.sessions._session_user import PosixSessionUser, WindowsSessionUser from .conftest import ( + serial_process, has_posix_target_user, has_windows_user, WIN_SET_TEST_ENV_VARS_MESSAGE, @@ -28,6 +29,7 @@ ) +@serial_process class TestRunSubprocess: """Tests for the Session.run_subprocess method.""" diff --git a/test/openjd/sessions_v0/test_subprocess.py b/test/openjd/sessions_v0/test_subprocess.py index cd9891cb..27b86e96 100644 --- a/test/openjd/sessions_v0/test_subprocess.py +++ b/test/openjd/sessions_v0/test_subprocess.py @@ -21,6 +21,7 @@ from openjd.sessions import _subprocess as subprocess_impl_mod from .conftest import ( + serial_process, build_logger, collect_queue_messages, has_posix_target_user, @@ -230,6 +231,7 @@ def test_invokes_callback(self, queue_handler: QueueHandler, python_exe: str) -> # THEN callback_mock.assert_called_once() + @serial_process def test_notify_ends_process( self, message_queue: SimpleQueue, queue_handler: QueueHandler, python_exe: str ) -> None: @@ -271,6 +273,7 @@ def end_proc(): assert "Log from test 9" not in all_messages assert subproc.exit_code != 0 + @serial_process def test_terminate_ends_process( self, message_queue: SimpleQueue, queue_handler: QueueHandler, python_exe: str ) -> None: @@ -317,6 +320,7 @@ def end_proc(): os.environ.get("CODEBUILD_BUILD_ID", None) is not None, reason="This test is failing exclusively in codebuild; unblocking, and will root cause later.", ) + @serial_process def test_terminate_ends_process_tree( self, message_queue: SimpleQueue, diff --git a/test/openjd/sessions_v0/test_wrap_cancelation.py b/test/openjd/sessions_v0/test_wrap_cancelation.py index ae869fcf..ecc021eb 100644 --- a/test/openjd/sessions_v0/test_wrap_cancelation.py +++ b/test/openjd/sessions_v0/test_wrap_cancelation.py @@ -42,6 +42,8 @@ from openjd.sessions import ActionState, ActionStatus, Session, SessionState from openjd.sessions._os_checker import is_posix +from .conftest import serial_process + # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- @@ -485,6 +487,7 @@ def test_invalid_deferred_mode_fails_action_at_launch(self) -> None: # --------------------------------------------------------------------------- +@serial_process class TestWrapCancelationDelivery: """The launch-time-resolution refactor removed cancel()'s ability to derive the cancel method itself, so nothing but this test covers the wiring from From 37975c7828917c9a352426ea9e85bf2f81ea89b0 Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Sun, 26 Jul 2026 15:45:01 -0700 Subject: [PATCH 21/27] test: Name test files for what they test, not review rounds test_round5_fixes.py and test_round5_regressions.py were named after the review round that produced them, which tells a later reader nothing, and each mixed tests for five different modules. Split by module under test: test_action_filter_hardening.py redaction + consumer-callback containment test_tempdir_hardening.py shared temp root validation + cleanup test_generated_shell_script.py POSIX action script quoting test_optimized_mode_invariants.py invariants that must survive python -O test_subprocess_process_group.py process-group recording test_runner_launch_state.py runner state after a failed launch Class names lose their finding IDs for the behaviour they pin, e.g. TestR51RedactionArgsBypass -> TestRedactionDoesNotLeakViaRecordArgs. Test bodies are unchanged: 76 tests before, 76 after, all passing. Full suite still 855 passed / 39 skipped / 16 xfailed. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- .../test_action_filter_hardening.py | 385 ++++++++ .../test_generated_shell_script.py | 189 ++++ .../test_optimized_mode_invariants.py | 155 ++++ test/openjd/sessions_v0/test_round5_fixes.py | 841 ------------------ .../sessions_v0/test_round5_regressions.py | 441 --------- .../sessions_v0/test_runner_launch_state.py | 213 +++++ .../test_subprocess_process_group.py | 89 ++ .../sessions_v0/test_tempdir_hardening.py | 322 +++++++ 8 files changed, 1353 insertions(+), 1282 deletions(-) create mode 100644 test/openjd/sessions_v0/test_action_filter_hardening.py create mode 100644 test/openjd/sessions_v0/test_generated_shell_script.py create mode 100644 test/openjd/sessions_v0/test_optimized_mode_invariants.py delete mode 100644 test/openjd/sessions_v0/test_round5_fixes.py delete mode 100644 test/openjd/sessions_v0/test_round5_regressions.py create mode 100644 test/openjd/sessions_v0/test_runner_launch_state.py create mode 100644 test/openjd/sessions_v0/test_subprocess_process_group.py create mode 100644 test/openjd/sessions_v0/test_tempdir_hardening.py diff --git a/test/openjd/sessions_v0/test_action_filter_hardening.py b/test/openjd/sessions_v0/test_action_filter_hardening.py new file mode 100644 index 00000000..7a30cf0b --- /dev/null +++ b/test/openjd/sessions_v0/test_action_filter_hardening.py @@ -0,0 +1,385 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Hardening of the action-output filter: log redaction, and containment of +consumer callbacks. + +``ActionMonitoringFilter`` runs on the thread forwarding a subprocess's stdout, +so anything that escapes it unwinds ``LoggingSubprocess.run()`` and costs us the +output stream and process ownership. It is also the control that keeps secrets +out of the log. +""" + +import logging +import sys +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest + +from openjd.model.v2023_09 import ( + Action as Action_2023_09, + ArgString as ArgString_2023_09, + CommandString as CommandString_2023_09, + StepActions as StepActions_2023_09, + StepScript as StepScript_2023_09, +) + +from openjd.sessions import ActionState, Session, SessionState +from openjd.sessions._action_filter import ( + ActionMessageKind, + ActionMonitoringFilter, + envvar_set_matcher_json, + envvar_set_matcher_str, + envvar_unset_matcher, +) + + +def _make_record( + msg: str, args: Any = None, session_id: str = "foo", level: int = logging.INFO +) -> logging.LogRecord: + """A LogRecord shaped the way the session logger produces them.""" + record = logging.LogRecord("test", level, "path", 1, msg, args, None) + record.session_id = session_id # type: ignore[attr-defined] + return record + + +class _UnrenderableError(Exception): + """An exception whose rendering raises -- the shape that defeated the R5-2 + containment, which interpolated the exception into an f-string.""" + + def __str__(self) -> str: + raise RuntimeError("__str__ is hostile") + + def __repr__(self) -> str: + raise RuntimeError("__repr__ is hostile too") + + +class TestRedactionDoesNotLeakViaRecordArgs: + """R5-1: `record.args` must be empty by the time the filter returns. + + The redaction logic only ever inspects `record.msg`. A downstream handler + calls `record.getMessage()`, which re-runs `msg % args` -- so any path that + leaves `args` populated re-interpolates the *un-scanned* original into the + emitted line. + """ + + def _filter_with_secret(self, secret: str = "SUPERSECRET") -> ActionMonitoringFilter: + f = ActionMonitoringFilter(session_id="foo", callback=MagicMock()) + f._redacted_values.add(secret) + return f + + def test_secret_in_args_is_redacted_when_formatting_succeeds(self) -> None: + # GIVEN: a secret carried only in args, with a msg that matches nothing + f = self._filter_with_secret() + record = _make_record("value is %s", ("SUPERSECRET",)) + + # WHEN + f.filter(record) + + # THEN: the emitted line -- which is what a handler actually renders -- + # carries no secret, and args cannot reintroduce one. + assert "SUPERSECRET" not in record.getMessage() + assert not record.args + + def test_secret_in_args_is_redacted_when_formatting_fails(self) -> None: + """The load-bearing case. `%d` against a str raises, so the old code + skipped clearing args and the handler re-interpolated the secret.""" + # GIVEN: a record whose own %-formatting is broken + f = self._filter_with_secret() + record = _make_record("value is %d", ("SUPERSECRET",)) + + # WHEN + f.filter(record) + + # THEN: args are cleared, so getMessage() cannot resurrect the secret, + # and the secret does not survive anywhere on the record. + assert not record.args + assert "SUPERSECRET" not in record.getMessage() + assert "SUPERSECRET" not in str(record.msg) + + def test_record_stays_renderable_when_formatting_fails(self) -> None: + """Folding args in must not leave a record that raises in the handler.""" + # GIVEN + f = self._filter_with_secret() + record = _make_record("value is %d", ("SUPERSECRET",)) + + # WHEN + f.filter(record) + + # THEN: getMessage() does not raise (it would if msg still held %d and + # args were still a str tuple), and the redaction marker is present. + assert "*" * 8 in record.getMessage() + + def test_whole_line_redaction_clears_args(self) -> None: + # GIVEN: a message that matches a whole redacted line + f = ActionMonitoringFilter(session_id="foo", callback=MagicMock()) + f._redacted_lines.add("secret-line") + record = _make_record("secret-line", None) + + # WHEN + f.filter(record) + + # THEN + assert record.getMessage() == "*" * 8 + assert not record.args + + def test_args_are_untouched_when_no_redactions_registered(self) -> None: + """Lazy %-formatting must keep working for every ordinary log line.""" + # GIVEN: no registered secrets + f = ActionMonitoringFilter(session_id="foo", callback=MagicMock()) + record = _make_record("value is %s", ("plain",)) + + # WHEN + f.filter(record) + + # THEN: the filter did not fold args in, so the handler still formats. + assert record.args == ("plain",) + assert record.getMessage() == "value is plain" + + +# =========================================================================== +# R5-2 -- consumer callback exceptions must not escape filter() +# =========================================================================== + + +class TestFilterContainsConsumerCallbackFailures: + """R5-2: this filter runs on the thread forwarding the subprocess's stdout. + + An exception escaping `filter()` unwinds `LoggingSubprocess.run()` before the + child is waited on: the pump thread dies, the rest of the output is dropped, + and the process is left unreaped. + """ + + @pytest.mark.parametrize( + "exc", + [RuntimeError("boom"), KeyError("boom"), TypeError("boom"), AttributeError("boom")], + ids=["RuntimeError", "KeyError", "TypeError", "AttributeError"], + ) + def test_handler_callback_exception_does_not_escape(self, exc: Exception) -> None: + # GIVEN: a consumer callback that raises a non-ValueError + def callback(kind: ActionMessageKind, value: Any, fail: bool) -> None: + raise exc + + f = ActionMonitoringFilter(session_id="foo", callback=callback) + record = _make_record("openjd_progress: 50.0") + + # WHEN / THEN: filter() returns normally... + assert f.filter(record) is True + # ...and keeps the record, with the failure visible in the action's output. + assert "boom" in record.getMessage() + + def test_malformed_env_callback_exception_does_not_escape(self) -> None: + """The one callback invocation in filter() not routed through `handler`.""" + + # GIVEN + def callback(kind: ActionMessageKind, value: Any, fail: bool) -> None: + raise RuntimeError("boom") + + f = ActionMonitoringFilter(session_id="foo", callback=callback) + # A near-miss env command: space before the colon. + record = _make_record("openjd_env : FOO=bar") + + # WHEN / THEN + assert f.filter(record) is True + + def test_valueerror_still_annotates_the_record(self) -> None: + """The pre-existing ValueError contract must be unchanged.""" + # GIVEN: progress outside the legal range raises ValueError in the handler + f = ActionMonitoringFilter(session_id="foo", callback=MagicMock()) + record = _make_record("openjd_progress: 500.0") + + # WHEN / THEN + assert f.filter(record) is True + assert "ERROR" in record.getMessage() + + def test_redaction_failure_fails_closed(self) -> None: + """If the redaction control itself breaks, emit nothing rather than an + unscanned line -- and do not let it reach the pump thread.""" + # GIVEN + f = ActionMonitoringFilter(session_id="foo", callback=MagicMock()) + record = _make_record("carries a secret") + + # WHEN + with patch.object( + ActionMonitoringFilter, + "apply_message_redaction", + side_effect=RuntimeError("redaction is broken"), + ): + result = f.filter(record) + + # THEN + assert result is True + assert record.getMessage() == "*" * 8 + + def test_a_live_child_is_still_reaped_when_the_callback_raises(self) -> None: + """End to end: the reason R5-2 matters. A progress update from a live + child must not cost us the process.""" + # GIVEN: a consumer that raises on the first progress update + state: dict[str, Any] = {"raised": False} + + def callback(session_id: str, status: Any) -> None: + if status.progress is not None and not state["raised"]: + state["raised"] = True + raise RuntimeError("consumer blew up on a progress update") + + script = StepScript_2023_09( + actions=StepActions_2023_09( + onRun=Action_2023_09( + command=CommandString_2023_09(sys.executable), + args=[ + ArgString_2023_09("-c"), + ArgString_2023_09( + "print('openjd_progress: 50.0', flush=True)\n" + "print('done', flush=True)\n" + ), + ], + ) + ) + ) + + # WHEN + with Session(session_id="r5-2-e2e", job_parameter_values={}, callback=callback) as session: + session.run_task(step_script=script, task_parameter_values={}) + deadline = 60.0 + import time + + start = time.monotonic() + while session.state == SessionState.RUNNING and time.monotonic() - start < deadline: + time.sleep(0.05) + + # THEN: the consumer did raise, the action still reached a terminal + # state, and the subprocess was waited on -- an exit code proves the + # `wait()` in LoggingSubprocess.run() was reached rather than skipped. + assert state["raised"] is True + assert session.state != SessionState.RUNNING + assert session.action_status is not None + assert session.action_status.exit_code == 0 + assert session.action_status.state == ActionState.SUCCESS + + +# =========================================================================== +# SIB-1 -- env var name anchoring (sibling of R5-5, found during this round) +# =========================================================================== + + +class TestEnvVarNameAnchoring: + """`$` also matches immediately before a trailing newline, so an + `$`-anchored NAME pattern accepted "FOO\\n" -- a name no OS can hold. + + The VALUE half stays deliberately permissive: a multi-line value delivered + through the JSON form is supported, tested behaviour. + """ + + @pytest.mark.parametrize("name", ["FOO\n", "FOO\r", "FOO\r\n"]) + def test_unset_rejects_a_trailing_newline_in_the_name(self, name: str) -> None: + assert envvar_unset_matcher.match(name) is None + + def test_unset_still_accepts_a_legal_name(self) -> None: + assert envvar_unset_matcher.match("FOO_BAR9") is not None + + @pytest.mark.parametrize("payload", ["FOO=bar\n", "FOO\n=bar"]) + def test_set_rejects_a_trailing_newline(self, payload: str) -> None: + assert envvar_set_matcher_str.match(payload) is None + + def test_multiline_value_via_json_is_still_supported(self) -> None: + """Guards the intended feature against over-correction.""" + # GIVEN / WHEN + raw = '"FOO=BAR\\nBAZ"' + # THEN: the raw (escaped) form still validates... + assert envvar_set_matcher_json.match(raw) is not None + # ...and the decoded multi-line value still reaches the callback. + callback = MagicMock() + f = ActionMonitoringFilter(session_id="foo", callback=callback) + f.filter(_make_record('openjd_env: "FOO=BAR\\nBAZ"')) + env_calls = [ + c + for c in callback.call_args_list + if c[0][0] == ActionMessageKind.ENV and isinstance(c[0][1], dict) + ] + assert len(env_calls) == 1 + assert env_calls[0][0][1] == {"name": "FOO", "value": "BAR\nBAZ"} + + @pytest.mark.parametrize( + "msg", + [ + 'openjd_env: "FOO\\nBAR=baz"', + 'openjd_env: "FOO\\u000aBAR=baz"', + 'openjd_env: "FOO\\u0000BAR=baz"', + ], + ) + def test_a_separator_cannot_reach_a_decoded_name(self, msg: str) -> None: + """A name carrying a separator must be rejected outright, not passed on. + + Rewritten after an audit: the previous version looped over the recorded + calls asserting a property of any dict it found, and since these inputs + produce no ENV dict at all, its assertion body never executed. It passed + against a deliberately broken implementation. Assert the rejection + directly instead. + """ + # GIVEN + callback = MagicMock() + f = ActionMonitoringFilter(session_id="foo", callback=callback) + + # WHEN + f.filter(_make_record(msg)) + + # THEN: no environment variable was defined, and the failure was reported. + env_defs = [ + c + for c in callback.call_args_list + if c[0][0] == ActionMessageKind.ENV and isinstance(c[0][1], dict) + ] + assert env_defs == [] + assert callback.call_args_list, "the parse failure must be reported to the consumer" + assert callback.call_args_list[-1][0][2] is True # cancel-and-fail + + +# =========================================================================== +# R5-3 -- the shared temporary root must be validated before use +# =========================================================================== + + +class TestContainmentDoesNotReRaise: + def test_handler_path_contains_an_unrenderable_exception(self) -> None: + # GIVEN: a consumer callback raising an exception that cannot be rendered + def callback(kind: ActionMessageKind, value: Any, fail: bool) -> None: + raise _UnrenderableError() + + f = ActionMonitoringFilter(session_id="foo", callback=callback) + record = _make_record("openjd_progress: 50.0") + + # WHEN / THEN: filter() still returns rather than letting the exception + # reach the stdout pump thread. + assert f.filter(record) is True + assert "_UnrenderableError" in record.getMessage() + + def test_malformed_env_path_contains_an_unrenderable_exception(self) -> None: + # GIVEN + def callback(kind: ActionMessageKind, value: Any, fail: bool) -> None: + raise _UnrenderableError() + + f = ActionMonitoringFilter(session_id="foo", callback=callback) + + # WHEN / THEN + assert f.filter(_make_record("openjd_env : FOO=bar")) is True + + def test_renderable_exceptions_still_report_their_message(self) -> None: + """The defensive rendering must not degrade the ordinary case.""" + + # GIVEN + def callback(kind: ActionMessageKind, value: Any, fail: bool) -> None: + raise RuntimeError("a perfectly ordinary boom") + + f = ActionMonitoringFilter(session_id="foo", callback=callback) + record = _make_record("openjd_progress: 50.0") + + # WHEN + f.filter(record) + + # THEN + assert "a perfectly ordinary boom" in record.getMessage() + + +# =========================================================================== +# REG-3 -- the R5-3 validation must not be defeatable by a symlink swap +# =========================================================================== diff --git a/test/openjd/sessions_v0/test_generated_shell_script.py b/test/openjd/sessions_v0/test_generated_shell_script.py new file mode 100644 index 00000000..4be600a8 --- /dev/null +++ b/test/openjd/sessions_v0/test_generated_shell_script.py @@ -0,0 +1,189 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Quoting in the shell script generated for a POSIX action. + +The script is ``exec``’d by ``/bin/sh``, so a single unquoted metacharacter +anywhere in it is arbitrary code execution as the session user. +""" + +import sys +from datetime import timedelta +from logging.handlers import QueueHandler +from pathlib import Path +from queue import SimpleQueue +from typing import Optional +from unittest.mock import MagicMock + +import pytest + + +from openjd.sessions._os_checker import is_posix +from openjd.sessions._runner_base import POSIX_SHELL_NAME_RE, ScriptRunnerBase +from openjd.sessions._subprocess import LoggingSubprocess + +from .conftest import build_logger + + +class _Runner(ScriptRunnerBase): + """Minimal concrete runner: only _generate_command_shell_script is exercised.""" + + def cancel( + self, *, time_limit: Optional[timedelta] = None, mark_action_failed: bool = False + ) -> None: + pass + + +# =========================================================================== +# R5-1 -- redaction must not leave record.args populated +# =========================================================================== + + +@pytest.mark.skipif(not is_posix(), reason="the generated shell script is POSIX-only") +class TestGeneratedShellScriptQuoting: + """The generated script is `exec`'d by /bin/sh, so a single unquoted + metacharacter anywhere in it is code execution as the session user.""" + + def _script( + self, + tmp_path: Path, + *, + os_env_vars: Optional[dict[str, Optional[str]]] = None, + startup_directory: Optional[Path] = None, + ) -> str: + runner = _Runner( + logger=MagicMock(), + session_working_directory=tmp_path, + os_env_vars=os_env_vars, + startup_directory=startup_directory, + ) + try: + return runner._generate_command_shell_script(["/bin/echo", "hi"]) + finally: + runner.shutdown() + + def test_single_quote_in_the_startup_directory_cannot_break_out(self, tmp_path: Path) -> None: + # GIVEN: a startup directory whose name closes the old hand-written quote + hostile = Path("/tmp/x'; touch /tmp/openjd_r5_pwned; '") + + # WHEN + script = self._script(tmp_path, startup_directory=hostile) + + # THEN: the injected command is inside a quoted word, not a command. + cd_line = next(line for line in script.splitlines() if line.startswith("cd ")) + assert "touch /tmp/openjd_r5_pwned" in cd_line # the path is preserved verbatim... + assert not cd_line.startswith("cd '/tmp/x'; ") # ...but no longer executes. + # Prove it to the shell itself rather than by eyeballing the quoting. + import shlex + + assert shlex.split(cd_line) == ["cd", str(hostile)] + + def test_ordinary_startup_directory_with_spaces_still_works(self, tmp_path: Path) -> None: + # GIVEN + spaced = tmp_path / "a dir with spaces" + + # WHEN + script = self._script(tmp_path, startup_directory=spaced) + + # THEN + import shlex + + cd_line = next(line for line in script.splitlines() if line.startswith("cd ")) + assert shlex.split(cd_line) == ["cd", str(spaced)] + + @pytest.mark.parametrize( + "hostile_name", + [ + "BAD;touch /tmp/openjd_r5_pwned;X", + "BAD$(touch /tmp/openjd_r5_pwned)", + "BAD`touch /tmp/openjd_r5_pwned`", + "BAD\ntouch /tmp/openjd_r5_pwned", + "BAD&&touch /tmp/openjd_r5_pwned", + ], + ) + def test_hostile_env_var_names_are_not_emitted(self, tmp_path: Path, hostile_name: str) -> None: + # GIVEN / WHEN + script = self._script(tmp_path, os_env_vars={hostile_name: "v", "GOOD": "ok"}) + + # THEN: the hostile name produced no line at all, and the legal one did. + assert "openjd_r5_pwned" not in script + assert "export GOOD=ok" in script + + def test_names_that_are_legal_elsewhere_but_not_in_sh_are_skipped(self, tmp_path: Path) -> None: + """`ProgramFiles(x86)` is a real Windows variable name and an outright + /bin/sh syntax error -- previously it broke the whole action.""" + # GIVEN / WHEN + script = self._script(tmp_path, os_env_vars={"ProgramFiles(x86)": "C:\\PF"}) + + # THEN + assert "ProgramFiles(x86)" not in script + + def test_generated_script_is_valid_sh_even_with_hostile_input(self, tmp_path: Path) -> None: + """The strongest assertion available: hand it to `sh -n`.""" + # GIVEN + script = self._script( + tmp_path, + os_env_vars={ + "BAD;X": "v", + "ProgramFiles(x86)": "v", + "GOOD": "a'b\"c$(d)", + "UNSET_ME": None, + }, + startup_directory=Path("/tmp/x'; touch /tmp/openjd_r5_pwned; '"), + ) + script_file = tmp_path / "candidate.sh" + script_file.write_text(script) + + # WHEN + from subprocess import run + + result = run(["/bin/sh", "-n", str(script_file)], capture_output=True, text=True) + + # THEN + assert result.returncode == 0, f"generated script is not valid sh: {result.stderr}" + + def test_unset_is_emitted_for_a_legal_name(self, tmp_path: Path) -> None: + # GIVEN / WHEN + script = self._script(tmp_path, os_env_vars={"GOES_AWAY": None}) + + # THEN + assert "unset GOES_AWAY" in script + + @pytest.mark.parametrize("name", ["FOO", "_foo", "F9", "_", "aB_9"]) + def test_posix_name_re_accepts_legal_identifiers(self, name: str) -> None: + assert POSIX_SHELL_NAME_RE.fullmatch(name) is not None + + @pytest.mark.parametrize("name", ["9FOO", "", "FOO BAR", "FOO=", "FOO\n", "FOO;", "é"]) + def test_posix_name_re_rejects_everything_else(self, name: str) -> None: + assert POSIX_SHELL_NAME_RE.fullmatch(name) is None + + def test_a_skipped_name_still_reaches_the_subprocess( + self, tmp_path: Path, message_queue: SimpleQueue, queue_handler: QueueHandler + ) -> None: + """Skipping the export line must not change what the child sees, because + Popen's `env=` does not go through a shell.""" + # GIVEN: a name that is illegal in sh but legal as an environment entry + logger = build_logger(queue_handler) + proc = LoggingSubprocess( + logger=logger, + args=[ + sys.executable, + "-c", + "import os;print('VAL=' + os.environ.get('ProgramFiles(x86)', 'MISSING'))", + ], + os_env_vars={"ProgramFiles(x86)": "present"}, + ) + + # WHEN + proc.run() + + # THEN + lines = [] + while not message_queue.empty(): + lines.append(message_queue.get().getMessage()) + assert "VAL=present" in lines + assert proc.exit_code == 0 + + +# =========================================================================== +# R5-6 -- invariant-bearing checks must survive `python -O` +# =========================================================================== diff --git a/test/openjd/sessions_v0/test_optimized_mode_invariants.py b/test/openjd/sessions_v0/test_optimized_mode_invariants.py new file mode 100644 index 00000000..db93ccfd --- /dev/null +++ b/test/openjd/sessions_v0/test_optimized_mode_invariants.py @@ -0,0 +1,155 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Runtime invariants must survive ``python -O``, which strips every ``assert``. + +Checks that carry a real invariant -- as opposed to type-checker narrowing -- +have to be explicit raises. See the policy in DEVELOPMENT.md. +""" + +import sys +from datetime import timedelta +from pathlib import Path +from typing import Optional +from unittest.mock import MagicMock + +import pytest + +from openjd.model import SymbolTable +from openjd.model.v2023_09 import ( + DataString as DataString_2023_09, + EmbeddedFileText as EmbeddedFileText_2023_09, + EmbeddedFileTypes as EmbeddedFileTypes_2023_09, +) + +from openjd.sessions import Session +from openjd.sessions._embedded_files import ( + EmbeddedFiles, + EmbeddedFilesScope, + _FileRecord, +) +from openjd.sessions._runner_base import ScriptRunnerBase +from openjd.sessions._subprocess import LoggingSubprocess + + +class _Runner(ScriptRunnerBase): + """Minimal concrete runner: only _generate_command_shell_script is exercised.""" + + def cancel( + self, *, time_limit: Optional[timedelta] = None, mark_action_failed: bool = False + ) -> None: + pass + + +# =========================================================================== +# R5-1 -- redaction must not leave record.args populated +# =========================================================================== + + +class TestInvariantsSurviveOptimizedMode: + """R5-6: `assert` is stripped under `python -O`, which is a legitimate + production deployment mode. Checks that carry a real runtime invariant must + be explicit raises.""" + + def test_unsupported_embedded_file_model_raises(self, tmp_path: Path) -> None: + # GIVEN: an embedded file object of an unrecognised shape + files = EmbeddedFiles( + logger=MagicMock(), + scope=EmbeddedFilesScope.ENV, + session_files_directory=tmp_path, + ) + + class _AlienFile: + filename = "x.sh" + runnable = False + + # WHEN / THEN: each of the three entry points stops rather than + # materializing a file from an object it does not understand. + with pytest.raises(RuntimeError, match="Unsupported embedded file model"): + files._find_value_prefix(_AlienFile()) # type: ignore[arg-type] + with pytest.raises(RuntimeError, match="Unsupported embedded file model"): + files._get_symtab_entry(_AlienFile()) # type: ignore[arg-type] + with pytest.raises(RuntimeError, match="Unsupported embedded file model"): + files._materialize_file( + tmp_path / "x.sh", _AlienFile(), SymbolTable() # type: ignore[arg-type] + ) + + def test_working_directory_property_raises_a_legible_error(self) -> None: + # GIVEN: a Session whose construction did not complete + session = Session.__new__(Session) + session._working_dir = None # type: ignore[attr-defined] + + # WHEN / THEN: a library RuntimeError, not AttributeError on None. + with pytest.raises(RuntimeError, match="no working directory"): + _ = session.working_directory + + def test_state_is_readable_when_a_launch_failed_after_process_creation( + self, tmp_path: Path + ) -> None: + """The `state` property is on the path every consumer polls. An + inconsistent (process set, future unset) pair must not make the runner + permanently unreadable.""" + # GIVEN + runner = _Runner(logger=MagicMock(), session_working_directory=tmp_path) + try: + runner._process = MagicMock() + runner._run_future = None + + # WHEN / THEN: no AssertionError, no AttributeError. + from openjd.sessions._runner_base import ScriptRunnerState + + assert runner.state == ScriptRunnerState.READY + finally: + runner.shutdown() + + def test_notify_period_end_is_a_no_op_without_a_process(self, tmp_path: Path) -> None: + """Runs on a threading.Timer thread, where an exception reaches only + threading.excepthook -- so the grace period would appear to have silently + done nothing.""" + # GIVEN + runner = _Runner(logger=MagicMock(), session_working_directory=tmp_path) + try: + runner._process = None + + # WHEN / THEN: returns cleanly. + runner._on_notify_period_end() + finally: + runner.shutdown() + + def test_log_subproc_stdout_raises_a_legible_error_without_a_process( + self, tmp_path: Path + ) -> None: + # GIVEN + proc = LoggingSubprocess(logger=MagicMock(), args=[sys.executable, "-c", "pass"]) + + # WHEN / THEN + with pytest.raises(RuntimeError, match="before the subprocess has been created"): + proc._log_subproc_stdout() + + def test_an_embedded_file_still_materializes_normally(self, tmp_path: Path) -> None: + """Guard against the isinstance rewrite breaking the happy path.""" + # GIVEN + files = EmbeddedFiles( + logger=MagicMock(), + scope=EmbeddedFilesScope.ENV, + session_files_directory=tmp_path, + ) + model = EmbeddedFileText_2023_09( + name="Script", + type=EmbeddedFileTypes_2023_09.TEXT, + data=DataString_2023_09("hello"), + ) + symtab = SymbolTable() + + # WHEN + records = files.allocate_file_paths([model], symtab) + files.write_file_contents(records, symtab) + + # THEN + assert len(records) == 1 + assert isinstance(records[0], _FileRecord) + assert records[0].filename.read_text() == "hello" + + +# =========================================================================== +# R5-9 -- an unknown process group must be recorded as unknown +# =========================================================================== diff --git a/test/openjd/sessions_v0/test_round5_fixes.py b/test/openjd/sessions_v0/test_round5_fixes.py deleted file mode 100644 index 88df1f7c..00000000 --- a/test/openjd/sessions_v0/test_round5_fixes.py +++ /dev/null @@ -1,841 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - -"""Regression tests for round-5 review findings R5-1 through R5-9 on PR #333, -plus the sibling occurrences found while implementing each of them. - -Every test in this file was mutation-checked: the fix was reverted and the test -was confirmed to fail, then the fix was re-applied and the test was confirmed to -pass. A test that passes against the unfixed tree pins nothing. -""" - -import logging -import os -import stat -import sys -from datetime import timedelta -from logging.handlers import QueueHandler -from pathlib import Path -from queue import SimpleQueue -from typing import Any, Optional -from unittest.mock import MagicMock, patch - -import pytest - -from openjd.model import SymbolTable -from openjd.model.v2023_09 import ( - Action as Action_2023_09, - ArgString as ArgString_2023_09, - CommandString as CommandString_2023_09, - DataString as DataString_2023_09, - EmbeddedFileText as EmbeddedFileText_2023_09, - EmbeddedFileTypes as EmbeddedFileTypes_2023_09, - StepActions as StepActions_2023_09, - StepScript as StepScript_2023_09, -) - -from openjd.sessions import ActionState, Session, SessionState -from openjd.sessions._action_filter import ( - ActionMessageKind, - ActionMonitoringFilter, - envvar_set_matcher_json, - envvar_set_matcher_str, - envvar_unset_matcher, -) -from openjd.sessions._embedded_files import ( - EmbeddedFiles, - EmbeddedFilesScope, - _FileRecord, -) -from openjd.sessions._os_checker import is_posix -from openjd.sessions._runner_base import POSIX_SHELL_NAME_RE, ScriptRunnerBase -from openjd.sessions._subprocess import LoggingSubprocess -from openjd.sessions._tempdir import OPENJD_TEMPDIR_MODE, TempDir, custom_gettempdir - -from .conftest import build_logger - - -def _make_record( - msg: str, args: Any = None, session_id: str = "foo", level: int = logging.INFO -) -> logging.LogRecord: - """A LogRecord shaped the way the session logger produces them.""" - record = logging.LogRecord("test", level, "path", 1, msg, args, None) - record.session_id = session_id # type: ignore[attr-defined] - return record - - -class _Runner(ScriptRunnerBase): - """Minimal concrete runner: only _generate_command_shell_script is exercised.""" - - def cancel( - self, *, time_limit: Optional[timedelta] = None, mark_action_failed: bool = False - ) -> None: - pass - - -# =========================================================================== -# R5-1 -- redaction must not leave record.args populated -# =========================================================================== - - -class TestR51RedactionArgsBypass: - """R5-1: `record.args` must be empty by the time the filter returns. - - The redaction logic only ever inspects `record.msg`. A downstream handler - calls `record.getMessage()`, which re-runs `msg % args` -- so any path that - leaves `args` populated re-interpolates the *un-scanned* original into the - emitted line. - """ - - def _filter_with_secret(self, secret: str = "SUPERSECRET") -> ActionMonitoringFilter: - f = ActionMonitoringFilter(session_id="foo", callback=MagicMock()) - f._redacted_values.add(secret) - return f - - def test_secret_in_args_is_redacted_when_formatting_succeeds(self) -> None: - # GIVEN: a secret carried only in args, with a msg that matches nothing - f = self._filter_with_secret() - record = _make_record("value is %s", ("SUPERSECRET",)) - - # WHEN - f.filter(record) - - # THEN: the emitted line -- which is what a handler actually renders -- - # carries no secret, and args cannot reintroduce one. - assert "SUPERSECRET" not in record.getMessage() - assert not record.args - - def test_secret_in_args_is_redacted_when_formatting_fails(self) -> None: - """The load-bearing case. `%d` against a str raises, so the old code - skipped clearing args and the handler re-interpolated the secret.""" - # GIVEN: a record whose own %-formatting is broken - f = self._filter_with_secret() - record = _make_record("value is %d", ("SUPERSECRET",)) - - # WHEN - f.filter(record) - - # THEN: args are cleared, so getMessage() cannot resurrect the secret, - # and the secret does not survive anywhere on the record. - assert not record.args - assert "SUPERSECRET" not in record.getMessage() - assert "SUPERSECRET" not in str(record.msg) - - def test_record_stays_renderable_when_formatting_fails(self) -> None: - """Folding args in must not leave a record that raises in the handler.""" - # GIVEN - f = self._filter_with_secret() - record = _make_record("value is %d", ("SUPERSECRET",)) - - # WHEN - f.filter(record) - - # THEN: getMessage() does not raise (it would if msg still held %d and - # args were still a str tuple), and the redaction marker is present. - assert "*" * 8 in record.getMessage() - - def test_whole_line_redaction_clears_args(self) -> None: - # GIVEN: a message that matches a whole redacted line - f = ActionMonitoringFilter(session_id="foo", callback=MagicMock()) - f._redacted_lines.add("secret-line") - record = _make_record("secret-line", None) - - # WHEN - f.filter(record) - - # THEN - assert record.getMessage() == "*" * 8 - assert not record.args - - def test_args_are_untouched_when_no_redactions_registered(self) -> None: - """Lazy %-formatting must keep working for every ordinary log line.""" - # GIVEN: no registered secrets - f = ActionMonitoringFilter(session_id="foo", callback=MagicMock()) - record = _make_record("value is %s", ("plain",)) - - # WHEN - f.filter(record) - - # THEN: the filter did not fold args in, so the handler still formats. - assert record.args == ("plain",) - assert record.getMessage() == "value is plain" - - -# =========================================================================== -# R5-2 -- consumer callback exceptions must not escape filter() -# =========================================================================== - - -class TestR52FilterCallbackIsolation: - """R5-2: this filter runs on the thread forwarding the subprocess's stdout. - - An exception escaping `filter()` unwinds `LoggingSubprocess.run()` before the - child is waited on: the pump thread dies, the rest of the output is dropped, - and the process is left unreaped. - """ - - @pytest.mark.parametrize( - "exc", - [RuntimeError("boom"), KeyError("boom"), TypeError("boom"), AttributeError("boom")], - ids=["RuntimeError", "KeyError", "TypeError", "AttributeError"], - ) - def test_handler_callback_exception_does_not_escape(self, exc: Exception) -> None: - # GIVEN: a consumer callback that raises a non-ValueError - def callback(kind: ActionMessageKind, value: Any, fail: bool) -> None: - raise exc - - f = ActionMonitoringFilter(session_id="foo", callback=callback) - record = _make_record("openjd_progress: 50.0") - - # WHEN / THEN: filter() returns normally... - assert f.filter(record) is True - # ...and keeps the record, with the failure visible in the action's output. - assert "boom" in record.getMessage() - - def test_malformed_env_callback_exception_does_not_escape(self) -> None: - """The one callback invocation in filter() not routed through `handler`.""" - - # GIVEN - def callback(kind: ActionMessageKind, value: Any, fail: bool) -> None: - raise RuntimeError("boom") - - f = ActionMonitoringFilter(session_id="foo", callback=callback) - # A near-miss env command: space before the colon. - record = _make_record("openjd_env : FOO=bar") - - # WHEN / THEN - assert f.filter(record) is True - - def test_valueerror_still_annotates_the_record(self) -> None: - """The pre-existing ValueError contract must be unchanged.""" - # GIVEN: progress outside the legal range raises ValueError in the handler - f = ActionMonitoringFilter(session_id="foo", callback=MagicMock()) - record = _make_record("openjd_progress: 500.0") - - # WHEN / THEN - assert f.filter(record) is True - assert "ERROR" in record.getMessage() - - def test_redaction_failure_fails_closed(self) -> None: - """If the redaction control itself breaks, emit nothing rather than an - unscanned line -- and do not let it reach the pump thread.""" - # GIVEN - f = ActionMonitoringFilter(session_id="foo", callback=MagicMock()) - record = _make_record("carries a secret") - - # WHEN - with patch.object( - ActionMonitoringFilter, - "apply_message_redaction", - side_effect=RuntimeError("redaction is broken"), - ): - result = f.filter(record) - - # THEN - assert result is True - assert record.getMessage() == "*" * 8 - - def test_a_live_child_is_still_reaped_when_the_callback_raises(self) -> None: - """End to end: the reason R5-2 matters. A progress update from a live - child must not cost us the process.""" - # GIVEN: a consumer that raises on the first progress update - state: dict[str, Any] = {"raised": False} - - def callback(session_id: str, status: Any) -> None: - if status.progress is not None and not state["raised"]: - state["raised"] = True - raise RuntimeError("consumer blew up on a progress update") - - script = StepScript_2023_09( - actions=StepActions_2023_09( - onRun=Action_2023_09( - command=CommandString_2023_09(sys.executable), - args=[ - ArgString_2023_09("-c"), - ArgString_2023_09( - "print('openjd_progress: 50.0', flush=True)\n" - "print('done', flush=True)\n" - ), - ], - ) - ) - ) - - # WHEN - with Session(session_id="r5-2-e2e", job_parameter_values={}, callback=callback) as session: - session.run_task(step_script=script, task_parameter_values={}) - deadline = 60.0 - import time - - start = time.monotonic() - while session.state == SessionState.RUNNING and time.monotonic() - start < deadline: - time.sleep(0.05) - - # THEN: the consumer did raise, the action still reached a terminal - # state, and the subprocess was waited on -- an exit code proves the - # `wait()` in LoggingSubprocess.run() was reached rather than skipped. - assert state["raised"] is True - assert session.state != SessionState.RUNNING - assert session.action_status is not None - assert session.action_status.exit_code == 0 - assert session.action_status.state == ActionState.SUCCESS - - -# =========================================================================== -# SIB-1 -- env var name anchoring (sibling of R5-5, found during this round) -# =========================================================================== - - -class TestSib1EnvVarNameAnchoring: - """`$` also matches immediately before a trailing newline, so an - `$`-anchored NAME pattern accepted "FOO\\n" -- a name no OS can hold. - - The VALUE half stays deliberately permissive: a multi-line value delivered - through the JSON form is supported, tested behaviour. - """ - - @pytest.mark.parametrize("name", ["FOO\n", "FOO\r", "FOO\r\n"]) - def test_unset_rejects_a_trailing_newline_in_the_name(self, name: str) -> None: - assert envvar_unset_matcher.match(name) is None - - def test_unset_still_accepts_a_legal_name(self) -> None: - assert envvar_unset_matcher.match("FOO_BAR9") is not None - - @pytest.mark.parametrize("payload", ["FOO=bar\n", "FOO\n=bar"]) - def test_set_rejects_a_trailing_newline(self, payload: str) -> None: - assert envvar_set_matcher_str.match(payload) is None - - def test_multiline_value_via_json_is_still_supported(self) -> None: - """Guards the intended feature against over-correction.""" - # GIVEN / WHEN - raw = '"FOO=BAR\\nBAZ"' - # THEN: the raw (escaped) form still validates... - assert envvar_set_matcher_json.match(raw) is not None - # ...and the decoded multi-line value still reaches the callback. - callback = MagicMock() - f = ActionMonitoringFilter(session_id="foo", callback=callback) - f.filter(_make_record('openjd_env: "FOO=BAR\\nBAZ"')) - env_calls = [ - c - for c in callback.call_args_list - if c[0][0] == ActionMessageKind.ENV and isinstance(c[0][1], dict) - ] - assert len(env_calls) == 1 - assert env_calls[0][0][1] == {"name": "FOO", "value": "BAR\nBAZ"} - - @pytest.mark.parametrize( - "msg", - [ - 'openjd_env: "FOO\\nBAR=baz"', - 'openjd_env: "FOO\\u000aBAR=baz"', - 'openjd_env: "FOO\\u0000BAR=baz"', - ], - ) - def test_a_separator_cannot_reach_a_decoded_name(self, msg: str) -> None: - """A name carrying a separator must be rejected outright, not passed on. - - Rewritten after an audit: the previous version looped over the recorded - calls asserting a property of any dict it found, and since these inputs - produce no ENV dict at all, its assertion body never executed. It passed - against a deliberately broken implementation. Assert the rejection - directly instead. - """ - # GIVEN - callback = MagicMock() - f = ActionMonitoringFilter(session_id="foo", callback=callback) - - # WHEN - f.filter(_make_record(msg)) - - # THEN: no environment variable was defined, and the failure was reported. - env_defs = [ - c - for c in callback.call_args_list - if c[0][0] == ActionMessageKind.ENV and isinstance(c[0][1], dict) - ] - assert env_defs == [] - assert callback.call_args_list, "the parse failure must be reported to the consumer" - assert callback.call_args_list[-1][0][2] is True # cancel-and-fail - - -# =========================================================================== -# R5-3 -- the shared temporary root must be validated before use -# =========================================================================== - - -class TestR53TempRootHardening: - """R5-3: `/OpenJD` is a fixed, predictable path whose parent is - world-writable on typical POSIX hosts. `exist_ok=True` accepted whatever was - already there.""" - - def test_created_with_an_explicit_mode_regardless_of_umask(self, tmp_path: Path) -> None: - # GIVEN: a hostile umask that would otherwise strip group/other bits - old_umask = os.umask(0o077) - try: - with patch("openjd.sessions._tempdir.gettempdir", return_value=str(tmp_path)): - # WHEN - created = custom_gettempdir() - finally: - os.umask(old_umask) - - # THEN: the root is traversable as intended, not umask-dependent. - assert Path(created) == tmp_path / "OpenJD" - if is_posix(): - assert stat.S_IMODE(os.stat(created).st_mode) == stat.S_IMODE(OPENJD_TEMPDIR_MODE) - - @pytest.mark.skipif(not is_posix(), reason="symlink pre-creation is a POSIX vector here") - def test_rejects_a_symlink_at_the_root_path(self, tmp_path: Path) -> None: - # GIVEN: an attacker has replaced the root with a symlink elsewhere - elsewhere = tmp_path / "elsewhere" - elsewhere.mkdir() - parent = tmp_path / "parent" - parent.mkdir() - (parent / "OpenJD").symlink_to(elsewhere, target_is_directory=True) - - # WHEN / THEN: we refuse rather than creating sessions inside it. The - # refusal now comes from O_NOFOLLOW on the open itself (see REG-3). - with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): - with pytest.raises(RuntimeError, match="real directory"): - custom_gettempdir() - - def test_rejects_a_root_that_is_not_a_directory(self, tmp_path: Path) -> None: - # GIVEN: a plain file squatting on the root path - parent = tmp_path / "parent" - parent.mkdir() - (parent / "OpenJD").write_text("squat") - - # WHEN / THEN - with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): - with pytest.raises(RuntimeError): - custom_gettempdir() - - @staticmethod - def _fstat_reporting_uid(uid: int) -> Any: - """Wrap os.fstat so the root directory appears owned by `uid`. - - Patches fstat rather than lstat: the implementation validates through a - descriptor now (see REG-3), so an lstat mock would not be consulted and - the test would vacuously pass. - """ - real_fstat = os.fstat - - class _Stat: - def __init__(self, base: os.stat_result) -> None: - self.st_mode = base.st_mode - self.st_uid = uid - - def fake_fstat(fd: int, *a: Any, **k: Any) -> Any: - return _Stat(real_fstat(fd, *a, **k)) - - return fake_fstat - - @pytest.mark.skipif(not is_posix(), reason="uid ownership check is POSIX-only") - def test_rejects_a_root_owned_by_another_user(self, tmp_path: Path) -> None: - # GIVEN: the root exists but is owned by a different uid - parent = tmp_path / "parent" - parent.mkdir() - (parent / "OpenJD").mkdir() - - # WHEN / THEN - with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): - with patch( - "openjd.sessions._tempdir.os.fstat", - side_effect=self._fstat_reporting_uid(os.geteuid() + 12345), - ): - with pytest.raises(RuntimeError, match="owned by uid"): - custom_gettempdir() - - @pytest.mark.skipif(not is_posix(), reason="uid ownership check is POSIX-only") - def test_accepts_a_root_owned_by_root(self, tmp_path: Path) -> None: - """A system-provisioned root must keep working.""" - # GIVEN - parent = tmp_path / "parent" - parent.mkdir() - (parent / "OpenJD").mkdir() - - # WHEN / THEN: no exception - with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): - with patch( - "openjd.sessions._tempdir.os.fstat", side_effect=self._fstat_reporting_uid(0) - ): - assert custom_gettempdir() == str(parent / "OpenJD") - - -# =========================================================================== -# R5-7 -- cleanup must report WHY each path could not be deleted -# =========================================================================== - - -class TestR57CleanupErrorReporting: - """R5-7: the old handler accepted the exception and discarded it, leaving a - list of bare paths -- on the one code path where "permission denied" versus - "a process still holds this open" changes what the operator must do.""" - - def test_failure_message_names_the_path_and_the_reason(self, tmp_path: Path) -> None: - # GIVEN: a temp dir whose removal fails with a specific, diagnosable error - d = TempDir(dir=tmp_path) - doomed = d.path / "stubborn.txt" - doomed.write_text("x") - - def boom(path: Any, *a: Any, **k: Any) -> None: - raise PermissionError(13, "Permission denied") - - # WHEN - with ( - patch("openjd.sessions._tempdir.os.unlink", side_effect=boom), - patch("openjd.sessions._tempdir.os.remove", side_effect=boom, create=True), - ): - with pytest.raises(RuntimeError) as excinfo: - d.cleanup() - - # THEN: both the path and the cause are in the message. - message = str(excinfo.value) - assert "stubborn.txt" in message - assert "PermissionError" in message - - def test_successful_cleanup_still_removes_everything(self, tmp_path: Path) -> None: - # GIVEN - d = TempDir(dir=tmp_path) - (d.path / "a.txt").write_text("x") - (d.path / "sub").mkdir() - (d.path / "sub" / "b.txt").write_text("y") - - # WHEN - d.cleanup() - - # THEN - assert not d.path.exists() - - -# =========================================================================== -# R5-4 / R5-5 -- nothing unquoted may reach /bin/sh -# =========================================================================== - - -@pytest.mark.skipif(not is_posix(), reason="the generated shell script is POSIX-only") -class TestR54R55ShellScriptQuoting: - """The generated script is `exec`'d by /bin/sh, so a single unquoted - metacharacter anywhere in it is code execution as the session user.""" - - def _script( - self, - tmp_path: Path, - *, - os_env_vars: Optional[dict[str, Optional[str]]] = None, - startup_directory: Optional[Path] = None, - ) -> str: - runner = _Runner( - logger=MagicMock(), - session_working_directory=tmp_path, - os_env_vars=os_env_vars, - startup_directory=startup_directory, - ) - try: - return runner._generate_command_shell_script(["/bin/echo", "hi"]) - finally: - runner.shutdown() - - def test_single_quote_in_the_startup_directory_cannot_break_out(self, tmp_path: Path) -> None: - # GIVEN: a startup directory whose name closes the old hand-written quote - hostile = Path("/tmp/x'; touch /tmp/openjd_r5_pwned; '") - - # WHEN - script = self._script(tmp_path, startup_directory=hostile) - - # THEN: the injected command is inside a quoted word, not a command. - cd_line = next(line for line in script.splitlines() if line.startswith("cd ")) - assert "touch /tmp/openjd_r5_pwned" in cd_line # the path is preserved verbatim... - assert not cd_line.startswith("cd '/tmp/x'; ") # ...but no longer executes. - # Prove it to the shell itself rather than by eyeballing the quoting. - import shlex - - assert shlex.split(cd_line) == ["cd", str(hostile)] - - def test_ordinary_startup_directory_with_spaces_still_works(self, tmp_path: Path) -> None: - # GIVEN - spaced = tmp_path / "a dir with spaces" - - # WHEN - script = self._script(tmp_path, startup_directory=spaced) - - # THEN - import shlex - - cd_line = next(line for line in script.splitlines() if line.startswith("cd ")) - assert shlex.split(cd_line) == ["cd", str(spaced)] - - @pytest.mark.parametrize( - "hostile_name", - [ - "BAD;touch /tmp/openjd_r5_pwned;X", - "BAD$(touch /tmp/openjd_r5_pwned)", - "BAD`touch /tmp/openjd_r5_pwned`", - "BAD\ntouch /tmp/openjd_r5_pwned", - "BAD&&touch /tmp/openjd_r5_pwned", - ], - ) - def test_hostile_env_var_names_are_not_emitted(self, tmp_path: Path, hostile_name: str) -> None: - # GIVEN / WHEN - script = self._script(tmp_path, os_env_vars={hostile_name: "v", "GOOD": "ok"}) - - # THEN: the hostile name produced no line at all, and the legal one did. - assert "openjd_r5_pwned" not in script - assert "export GOOD=ok" in script - - def test_names_that_are_legal_elsewhere_but_not_in_sh_are_skipped(self, tmp_path: Path) -> None: - """`ProgramFiles(x86)` is a real Windows variable name and an outright - /bin/sh syntax error -- previously it broke the whole action.""" - # GIVEN / WHEN - script = self._script(tmp_path, os_env_vars={"ProgramFiles(x86)": "C:\\PF"}) - - # THEN - assert "ProgramFiles(x86)" not in script - - def test_generated_script_is_valid_sh_even_with_hostile_input(self, tmp_path: Path) -> None: - """The strongest assertion available: hand it to `sh -n`.""" - # GIVEN - script = self._script( - tmp_path, - os_env_vars={ - "BAD;X": "v", - "ProgramFiles(x86)": "v", - "GOOD": "a'b\"c$(d)", - "UNSET_ME": None, - }, - startup_directory=Path("/tmp/x'; touch /tmp/openjd_r5_pwned; '"), - ) - script_file = tmp_path / "candidate.sh" - script_file.write_text(script) - - # WHEN - from subprocess import run - - result = run(["/bin/sh", "-n", str(script_file)], capture_output=True, text=True) - - # THEN - assert result.returncode == 0, f"generated script is not valid sh: {result.stderr}" - - def test_unset_is_emitted_for_a_legal_name(self, tmp_path: Path) -> None: - # GIVEN / WHEN - script = self._script(tmp_path, os_env_vars={"GOES_AWAY": None}) - - # THEN - assert "unset GOES_AWAY" in script - - @pytest.mark.parametrize("name", ["FOO", "_foo", "F9", "_", "aB_9"]) - def test_posix_name_re_accepts_legal_identifiers(self, name: str) -> None: - assert POSIX_SHELL_NAME_RE.fullmatch(name) is not None - - @pytest.mark.parametrize("name", ["9FOO", "", "FOO BAR", "FOO=", "FOO\n", "FOO;", "é"]) - def test_posix_name_re_rejects_everything_else(self, name: str) -> None: - assert POSIX_SHELL_NAME_RE.fullmatch(name) is None - - def test_a_skipped_name_still_reaches_the_subprocess( - self, tmp_path: Path, message_queue: SimpleQueue, queue_handler: QueueHandler - ) -> None: - """Skipping the export line must not change what the child sees, because - Popen's `env=` does not go through a shell.""" - # GIVEN: a name that is illegal in sh but legal as an environment entry - logger = build_logger(queue_handler) - proc = LoggingSubprocess( - logger=logger, - args=[ - sys.executable, - "-c", - "import os;print('VAL=' + os.environ.get('ProgramFiles(x86)', 'MISSING'))", - ], - os_env_vars={"ProgramFiles(x86)": "present"}, - ) - - # WHEN - proc.run() - - # THEN - lines = [] - while not message_queue.empty(): - lines.append(message_queue.get().getMessage()) - assert "VAL=present" in lines - assert proc.exit_code == 0 - - -# =========================================================================== -# R5-6 -- invariant-bearing checks must survive `python -O` -# =========================================================================== - - -class TestR56InvariantsSurviveOptimizedMode: - """R5-6: `assert` is stripped under `python -O`, which is a legitimate - production deployment mode. Checks that carry a real runtime invariant must - be explicit raises.""" - - def test_unsupported_embedded_file_model_raises(self, tmp_path: Path) -> None: - # GIVEN: an embedded file object of an unrecognised shape - files = EmbeddedFiles( - logger=MagicMock(), - scope=EmbeddedFilesScope.ENV, - session_files_directory=tmp_path, - ) - - class _AlienFile: - filename = "x.sh" - runnable = False - - # WHEN / THEN: each of the three entry points stops rather than - # materializing a file from an object it does not understand. - with pytest.raises(RuntimeError, match="Unsupported embedded file model"): - files._find_value_prefix(_AlienFile()) # type: ignore[arg-type] - with pytest.raises(RuntimeError, match="Unsupported embedded file model"): - files._get_symtab_entry(_AlienFile()) # type: ignore[arg-type] - with pytest.raises(RuntimeError, match="Unsupported embedded file model"): - files._materialize_file( - tmp_path / "x.sh", _AlienFile(), SymbolTable() # type: ignore[arg-type] - ) - - def test_working_directory_property_raises_a_legible_error(self) -> None: - # GIVEN: a Session whose construction did not complete - session = Session.__new__(Session) - session._working_dir = None # type: ignore[attr-defined] - - # WHEN / THEN: a library RuntimeError, not AttributeError on None. - with pytest.raises(RuntimeError, match="no working directory"): - _ = session.working_directory - - def test_state_is_readable_when_a_launch_failed_after_process_creation( - self, tmp_path: Path - ) -> None: - """The `state` property is on the path every consumer polls. An - inconsistent (process set, future unset) pair must not make the runner - permanently unreadable.""" - # GIVEN - runner = _Runner(logger=MagicMock(), session_working_directory=tmp_path) - try: - runner._process = MagicMock() - runner._run_future = None - - # WHEN / THEN: no AssertionError, no AttributeError. - from openjd.sessions._runner_base import ScriptRunnerState - - assert runner.state == ScriptRunnerState.READY - finally: - runner.shutdown() - - def test_notify_period_end_is_a_no_op_without_a_process(self, tmp_path: Path) -> None: - """Runs on a threading.Timer thread, where an exception reaches only - threading.excepthook -- so the grace period would appear to have silently - done nothing.""" - # GIVEN - runner = _Runner(logger=MagicMock(), session_working_directory=tmp_path) - try: - runner._process = None - - # WHEN / THEN: returns cleanly. - runner._on_notify_period_end() - finally: - runner.shutdown() - - def test_log_subproc_stdout_raises_a_legible_error_without_a_process( - self, tmp_path: Path - ) -> None: - # GIVEN - proc = LoggingSubprocess(logger=MagicMock(), args=[sys.executable, "-c", "pass"]) - - # WHEN / THEN - with pytest.raises(RuntimeError, match="before the subprocess has been created"): - proc._log_subproc_stdout() - - def test_an_embedded_file_still_materializes_normally(self, tmp_path: Path) -> None: - """Guard against the isinstance rewrite breaking the happy path.""" - # GIVEN - files = EmbeddedFiles( - logger=MagicMock(), - scope=EmbeddedFilesScope.ENV, - session_files_directory=tmp_path, - ) - model = EmbeddedFileText_2023_09( - name="Script", - type=EmbeddedFileTypes_2023_09.TEXT, - data=DataString_2023_09("hello"), - ) - symtab = SymbolTable() - - # WHEN - records = files.allocate_file_paths([model], symtab) - files.write_file_contents(records, symtab) - - # THEN - assert len(records) == 1 - assert isinstance(records[0], _FileRecord) - assert records[0].filename.read_text() == "hello" - - -# =========================================================================== -# R5-9 -- an unknown process group must be recorded as unknown -# =========================================================================== - - -@pytest.mark.skipif(not is_posix(), reason="process groups are POSIX-only") -class TestR59ProcessGroupSentinel: - """R5-9: the lookup fails *because* the process is gone, so its pid is dead. - Recording it as a process-group id means a later `killpg` is at best a no-op - and, after pid recycling, targets an unrelated group.""" - - def test_reaped_child_leaves_the_group_unknown( - self, message_queue: SimpleQueue, queue_handler: QueueHandler - ) -> None: - # GIVEN: a child whose process group cannot be looked up - logger = build_logger(queue_handler) - proc = LoggingSubprocess( - logger=logger, args=[sys.executable, "-c", "pass"], os_env_vars=None - ) - - # WHEN - with patch( - "openjd.sessions._subprocess.os.getpgid", side_effect=ProcessLookupError(3, "No such") - ): - proc.run() - - # THEN: "unknown", not a stale pid -- and the action still succeeded, - # which is the behaviour the original fix existed to protect. - assert proc._sudo_child_process_group_id is None - assert proc.exit_code == 0 - assert proc.failed_to_start is False - - def test_no_signal_is_delivered_when_the_group_is_unknown( - self, message_queue: SimpleQueue, queue_handler: QueueHandler - ) -> None: - # GIVEN: a finished process with no known group - logger = build_logger(queue_handler) - proc = LoggingSubprocess(logger=logger, args=[sys.executable, "-c", "pass"]) - with patch( - "openjd.sessions._subprocess.os.getpgid", side_effect=ProcessLookupError(3, "No such") - ): - proc.run() - assert proc._sudo_child_process_group_id is None - - # WHEN: a SIGKILL is attempted anyway - with ( - patch("openjd.sessions._subprocess.os.killpg") as killpg, - patch( - "openjd.sessions._subprocess.find_sudo_child_process_group_id", return_value=None - ), - ): - proc._posix_signal_subprocess(MagicMock(pid=999999), signal_name="kill") - - # THEN: nothing was signalled. - killpg.assert_not_called() - - def test_sudo_helper_returns_unknown_when_sudo_is_already_gone(self) -> None: - """Sibling: the same guard on the first getpgid in the sudo helper.""" - # GIVEN - from openjd.sessions._linux._sudo import find_sudo_child_process_group_id - - # WHEN - with patch( - "openjd.sessions._linux._sudo.os.getpgid", - side_effect=ProcessLookupError(3, "No such process"), - ): - result = find_sudo_child_process_group_id( - logger=MagicMock(), sudo_process=MagicMock(pid=999999) - ) - - # THEN: the established "unknown" value, not an escaping ESRCH. - assert result is None diff --git a/test/openjd/sessions_v0/test_round5_regressions.py b/test/openjd/sessions_v0/test_round5_regressions.py deleted file mode 100644 index 24fed463..00000000 --- a/test/openjd/sessions_v0/test_round5_regressions.py +++ /dev/null @@ -1,441 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - -"""Regression tests for defects introduced or left open BY the round-5 fix -commit (4c57f40), found by a five-agent sweep of that commit. - -The most serious is REG-1: the R5-6 conversion of `ScriptRunnerBase.state`'s -`assert self._run_future is not None` into `return READY` turned a latent -AssertionError into silent state corruption. Two independent reviewers found it. - -Every test here was mutation-checked: the fix was reverted and the test was -confirmed to fail. See tmp-free `scripts/` note in the commit message. -""" - -import logging -import os -import stat -import sys -import threading -from concurrent.futures import Future, ThreadPoolExecutor -from datetime import timedelta -from pathlib import Path -from typing import Any, Optional -from unittest.mock import MagicMock, patch - -import pytest - -from openjd.sessions._action_filter import ActionMessageKind, ActionMonitoringFilter -from openjd.sessions._os_checker import is_posix -from openjd.sessions._runner_base import ScriptRunnerBase, ScriptRunnerState -from openjd.sessions._tempdir import OPENJD_TEMPDIR_MODE, custom_gettempdir -from openjd.sessions._types import ActionState - - -class _Runner(ScriptRunnerBase): - """Minimal concrete runner.""" - - def cancel( - self, *, time_limit: Optional[timedelta] = None, mark_action_failed: bool = False - ) -> None: - pass - - -def _make_record(msg: str, args: Any = None, session_id: str = "foo") -> logging.LogRecord: - record = logging.LogRecord("test", logging.INFO, "path", 1, msg, args, None) - record.session_id = session_id # type: ignore[attr-defined] - return record - - -# =========================================================================== -# REG-1 -- the R5-6 `state` conversion must not corrupt the runner -# =========================================================================== - - -class TestReg1FailedLaunchStateCorruption: - """A launch that fails after the subprocess object exists must not leave the - runner looking reusable, and a completed action must always publish a - terminal state. - - `_run` gates re-entry on `state == READY`. When R5-6 made `state` report - READY for the (process set, future unset) pair, that guard silently opened. - And `_on_process_exit` built `ActionState(self.state.value)`, which raises - `ValueError` for 'ready' -- swallowed by the F8 try/except, so the consumer - received no terminal callback at all and would wait forever. - """ - - def _runner_with_dead_pool(self, tmp_path: Path, published: list) -> _Runner: - runner = _Runner( - logger=MagicMock(), - session_working_directory=tmp_path, - callback=lambda s: published.append(s.value), - ) - # Exactly what Session.cleanup() does to the runner's pool. A cleanup - # racing a launch is the ordinary agent SIGTERM path. - runner._pool.shutdown(wait=True) - return runner - - def test_second_launch_is_refused_after_a_failed_launch(self, tmp_path: Path) -> None: - # GIVEN: a runner whose first launch fails inside _pool.submit - published: list = [] - runner = self._runner_with_dead_pool(tmp_path, published) - try: - with pytest.raises(RuntimeError): - runner._run([sys.executable, "-c", "pass"], timedelta(seconds=600)) - - # WHEN: the runner is handed a working pool and asked to run again, - # exactly as a caller that caught the RuntimeError might. - runner._pool = ThreadPoolExecutor(max_workers=1) - - # THEN: the single-use guard holds. Before the fix this launched a - # second real child and replaced _process, orphaning the first. - with pytest.raises(RuntimeError, match="cannot be used to run a second subprocess"): - runner._run([sys.executable, "-c", "import time; time.sleep(30)"], None) - finally: - runner.cancel() - for timer in [t for t in threading.enumerate() if isinstance(t, threading.Timer)]: - timer.cancel() - runner.shutdown() - - def test_launch_latch_holds_even_when_state_reports_ready(self, tmp_path: Path) -> None: - """Pins the guard to `_launched` rather than to the state classification. - - This is the property that actually broke: whatever `state` decides to - report for an inconsistent runner, re-entry must still be refused. - """ - # GIVEN: a runner that has attempted a launch, with `state` forced to - # report READY (the exact misclassification that opened the guard) - runner = _Runner(logger=MagicMock(), session_working_directory=tmp_path) - try: - runner._pool.shutdown(wait=True) - with pytest.raises(RuntimeError): - runner._run([sys.executable, "-c", "pass"], None) - runner._pool = ThreadPoolExecutor(max_workers=1) - - with patch.object( - type(runner), "state", property(lambda self: ScriptRunnerState.READY) - ): - assert runner.state == ScriptRunnerState.READY # the hostile premise - # THEN - with pytest.raises(RuntimeError, match="second subprocess"): - runner._run([sys.executable, "-c", "pass"], None) - finally: - runner.shutdown() - - def test_completion_always_publishes_a_terminal_state(self, tmp_path: Path) -> None: - # GIVEN: a runner whose process/future pair is inconsistent, so `state` - # cannot be classified as a terminal ActionState - published: list = [] - runner = _Runner( - logger=MagicMock(), - session_working_directory=tmp_path, - callback=lambda s: published.append(s), - ) - try: - runner._process = MagicMock() - runner._run_future = None - completed: Future = Future() - completed.set_result(None) - - # WHEN - runner._on_process_exit(completed) - - # THEN: exactly one terminal callback. Before the fix this list was - # empty: ActionState('ready') raised and the F8 handler ate it. - assert len(published) == 1 - assert published[0] in ( - ActionState.FAILED, - ActionState.CANCELED, - ActionState.TIMEOUT, - ActionState.SUCCESS, - ) - assert published[0] == ActionState.FAILED - finally: - runner.shutdown() - - def test_terminal_action_state_maps_every_runner_state(self, tmp_path: Path) -> None: - """No ScriptRunnerState may make the terminal mapping raise.""" - # GIVEN - runner = _Runner(logger=MagicMock(), session_working_directory=tmp_path) - try: - for runner_state in ScriptRunnerState: - - def fixed_state(_self: Any, _s: ScriptRunnerState = runner_state) -> Any: - return _s - - # WHEN - with patch.object(type(runner), "state", property(fixed_state)): - result = runner._terminal_action_state() - # THEN - assert isinstance(result, ActionState) - finally: - runner.shutdown() - - def test_process_and_future_are_published_together(self, tmp_path: Path) -> None: - """Removes the root cause: a reader must never see a process with no - future, on the success path or any other.""" - # GIVEN: a poller hammering `state` from another thread during launch - observations: list[tuple[bool, bool]] = [] - stop = threading.Event() - runner = _Runner(logger=MagicMock(), session_working_directory=tmp_path) - - def poll() -> None: - while not stop.is_set(): - observations.append((runner._process is not None, runner._run_future is not None)) - - poller = threading.Thread(target=poll, daemon=True) - try: - poller.start() - # WHEN - runner._run([sys.executable, "-c", "pass"], None) - stop.set() - poller.join(timeout=10) - - # THEN: the (process, no future) combination was never observable. - assert (True, False) not in observations - assert observations, "the poller must actually have sampled something" - finally: - stop.set() - runner.shutdown() - - def test_a_normal_run_still_reports_success(self, tmp_path: Path) -> None: - """Guard against the latch or the restructure breaking the happy path.""" - # GIVEN - published: list = [] - runner = _Runner( - logger=MagicMock(), - session_working_directory=tmp_path, - callback=lambda s: published.append(s), - ) - try: - # WHEN - runner._run([sys.executable, "-c", "pass"], None) - assert runner._run_future is not None - runner._run_future.result(timeout=30) - - # THEN - deadline = 30.0 - import time - - start = time.monotonic() - while ActionState.SUCCESS not in published and time.monotonic() - start < deadline: - time.sleep(0.02) - assert runner.state == ScriptRunnerState.SUCCESS - assert ActionState.SUCCESS in published - finally: - runner.shutdown() - - -# =========================================================================== -# REG-2 -- the R5-2 containment must not itself re-raise -# =========================================================================== - - -class _UnrenderableError(Exception): - """An exception whose rendering raises -- the shape that defeated the R5-2 - containment, which interpolated the exception into an f-string.""" - - def __str__(self) -> str: - raise RuntimeError("__str__ is hostile") - - def __repr__(self) -> str: - raise RuntimeError("__repr__ is hostile too") - - -class TestReg2ContainmentDoesNotReRaise: - def test_handler_path_contains_an_unrenderable_exception(self) -> None: - # GIVEN: a consumer callback raising an exception that cannot be rendered - def callback(kind: ActionMessageKind, value: Any, fail: bool) -> None: - raise _UnrenderableError() - - f = ActionMonitoringFilter(session_id="foo", callback=callback) - record = _make_record("openjd_progress: 50.0") - - # WHEN / THEN: filter() still returns rather than letting the exception - # reach the stdout pump thread. - assert f.filter(record) is True - assert "_UnrenderableError" in record.getMessage() - - def test_malformed_env_path_contains_an_unrenderable_exception(self) -> None: - # GIVEN - def callback(kind: ActionMessageKind, value: Any, fail: bool) -> None: - raise _UnrenderableError() - - f = ActionMonitoringFilter(session_id="foo", callback=callback) - - # WHEN / THEN - assert f.filter(_make_record("openjd_env : FOO=bar")) is True - - def test_renderable_exceptions_still_report_their_message(self) -> None: - """The defensive rendering must not degrade the ordinary case.""" - - # GIVEN - def callback(kind: ActionMessageKind, value: Any, fail: bool) -> None: - raise RuntimeError("a perfectly ordinary boom") - - f = ActionMonitoringFilter(session_id="foo", callback=callback) - record = _make_record("openjd_progress: 50.0") - - # WHEN - f.filter(record) - - # THEN - assert "a perfectly ordinary boom" in record.getMessage() - - -# =========================================================================== -# REG-3 -- the R5-3 validation must not be defeatable by a symlink swap -# =========================================================================== - - -@pytest.mark.skipif(not is_posix(), reason="symlink swap and fchmod semantics are POSIX here") -class TestReg3TempRootCheckThenUse: - """The first R5-3 implementation validated with `lstat(path)` and then called - `stat(path)`/`chmod(path)`, both of which re-resolve the name and follow - links. Swapping the entry for a symlink in between defeated the check and - chmod'ed the link's target to 0o755.""" - - def test_a_symlink_swapped_in_before_the_open_is_refused(self, tmp_path: Path) -> None: - """Swap during the create window: O_NOFOLLOW must refuse the open.""" - # GIVEN: a victim directory at 0o700 and a parent where the root will live - victim = tmp_path / "victim" - victim.mkdir() - os.chmod(victim, 0o700) - parent = tmp_path / "parent" - parent.mkdir() - root = parent / "OpenJD" - - real_makedirs = os.makedirs - - def makedirs_then_swap(path: Any, *a: Any, **k: Any) -> None: - real_makedirs(path, *a, **k) - if str(path) == str(root): - os.rmdir(root) - os.symlink(victim, root, target_is_directory=True) - - # WHEN - with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): - with patch("openjd.sessions._tempdir.os.makedirs", side_effect=makedirs_then_swap): - with pytest.raises(RuntimeError): - custom_gettempdir() - - # THEN - assert stat.S_IMODE(os.stat(victim).st_mode) == 0o700 - - def test_a_symlink_swapped_in_after_the_open_cannot_redirect_the_chmod( - self, tmp_path: Path - ) -> None: - """Swap *after* the descriptor is open, which O_NOFOLLOW cannot help with. - - This is the case that pins fchmod-vs-chmod. An earlier version of this - test swapped during the create window instead, where O_NOFOLLOW refuses - the open before any chmod is attempted -- so it passed even with - `os.chmod(path)` restored, and pinned nothing about the descriptor. The - swap is injected from inside `os.fstat`, which the implementation calls - between the open and the mode change. - """ - # GIVEN: a victim at 0o700, and a real root whose mode needs correcting - victim = tmp_path / "victim_after" - victim.mkdir() - os.chmod(victim, 0o700) - parent = tmp_path / "parent_after" - parent.mkdir() - root = parent / "OpenJD" - root.mkdir() - os.chmod(root, 0o700) # != OPENJD_TEMPDIR_MODE, so a mode change is due - - real_fstat = os.fstat - swapped = {"done": False} - - def fstat_then_swap(fd: int, *a: Any, **k: Any) -> Any: - result = real_fstat(fd, *a, **k) - if not swapped["done"]: - swapped["done"] = True - os.rmdir(root) - os.symlink(victim, root, target_is_directory=True) - return result - - # WHEN - with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): - with patch("openjd.sessions._tempdir.os.fstat", side_effect=fstat_then_swap): - custom_gettempdir() - - # THEN: the swap happened, and the victim was NOT widened. With - # `os.chmod(temp_dir, ...)` restored this is 0o755. - assert swapped["done"] is True - assert os.path.islink(root) - assert stat.S_IMODE(os.stat(victim).st_mode) == 0o700 - - def test_the_validated_inode_is_the_one_modified(self, tmp_path: Path) -> None: - """Even when the swap is not detected as an error, the mode must land on - the inode that was validated, never on a later resolution of the name.""" - # GIVEN: an existing root at a wrong mode, plus a victim - victim = tmp_path / "victim2" - victim.mkdir() - os.chmod(victim, 0o700) - parent = tmp_path / "parent2" - parent.mkdir() - root = parent / "OpenJD" - root.mkdir() - os.chmod(root, 0o700) # differs from OPENJD_TEMPDIR_MODE, so a chmod is due - - # WHEN - with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): - result = custom_gettempdir() - - # THEN: the real root got the mode; the victim was not touched. - assert result == str(root) - assert stat.S_IMODE(os.stat(root).st_mode) == stat.S_IMODE(OPENJD_TEMPDIR_MODE) - assert stat.S_IMODE(os.stat(victim).st_mode) == 0o700 - - def test_a_symlinked_root_is_still_refused(self, tmp_path: Path) -> None: - """O_NOFOLLOW must keep doing what the explicit S_ISLNK branch did.""" - # GIVEN - elsewhere = tmp_path / "elsewhere" - elsewhere.mkdir() - parent = tmp_path / "parent3" - parent.mkdir() - (parent / "OpenJD").symlink_to(elsewhere, target_is_directory=True) - - # WHEN / THEN - with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): - with pytest.raises(RuntimeError, match="real directory"): - custom_gettempdir() - - def test_no_descriptor_is_leaked(self, tmp_path: Path) -> None: - """The validation opens a descriptor; every path must close it. - - Probe: the lowest free descriptor number. If a call leaks a descriptor, - the number the kernel hands out next goes up. Portable across macOS and - Linux, unlike listing /dev/fd. - """ - # GIVEN - parent = tmp_path / "parent4" - parent.mkdir() - (parent / "OpenJD").mkdir() - - def lowest_free_fd() -> int: - fd = os.open(os.devnull, os.O_RDONLY) - os.close(fd) - return fd - - with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): - custom_gettempdir() # warm up, so first-call allocations do not count - baseline = lowest_free_fd() - - # WHEN: many successful calls - for _ in range(25): - custom_gettempdir() - after_success = lowest_free_fd() - - # WHEN: many calls that fail *after* the descriptor is opened - with patch( - "openjd.sessions._tempdir.os.fstat", side_effect=OSError(5, "induced failure") - ): - for _ in range(25): - with pytest.raises(OSError): - custom_gettempdir() - after_failure = lowest_free_fd() - - # THEN: no drift on either path. - assert after_success == baseline - assert after_failure == baseline diff --git a/test/openjd/sessions_v0/test_runner_launch_state.py b/test/openjd/sessions_v0/test_runner_launch_state.py new file mode 100644 index 00000000..73eb9087 --- /dev/null +++ b/test/openjd/sessions_v0/test_runner_launch_state.py @@ -0,0 +1,213 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Runner state around a launch that fails after the subprocess object exists. + +``_run``’s single-use guard and the terminal state it publishes both have to hold +when the process/future pair is left inconsistent. +""" + +import sys +import threading +from concurrent.futures import Future, ThreadPoolExecutor +from datetime import timedelta +from pathlib import Path +from typing import Any, Optional +from unittest.mock import MagicMock, patch + +import pytest + +from openjd.sessions import ActionState +from openjd.sessions._runner_base import ScriptRunnerBase, ScriptRunnerState + + +class _Runner(ScriptRunnerBase): + """Minimal concrete runner: only _generate_command_shell_script is exercised.""" + + def cancel( + self, *, time_limit: Optional[timedelta] = None, mark_action_failed: bool = False + ) -> None: + pass + + +# =========================================================================== +# R5-1 -- redaction must not leave record.args populated +# =========================================================================== + + +class TestFailedLaunchDoesNotCorruptRunnerState: + """A launch that fails after the subprocess object exists must not leave the + runner looking reusable, and a completed action must always publish a + terminal state. + + `_run` gates re-entry on `state == READY`. When R5-6 made `state` report + READY for the (process set, future unset) pair, that guard silently opened. + And `_on_process_exit` built `ActionState(self.state.value)`, which raises + `ValueError` for 'ready' -- swallowed by the F8 try/except, so the consumer + received no terminal callback at all and would wait forever. + """ + + def _runner_with_dead_pool(self, tmp_path: Path, published: list) -> _Runner: + runner = _Runner( + logger=MagicMock(), + session_working_directory=tmp_path, + callback=lambda s: published.append(s.value), + ) + # Exactly what Session.cleanup() does to the runner's pool. A cleanup + # racing a launch is the ordinary agent SIGTERM path. + runner._pool.shutdown(wait=True) + return runner + + def test_second_launch_is_refused_after_a_failed_launch(self, tmp_path: Path) -> None: + # GIVEN: a runner whose first launch fails inside _pool.submit + published: list = [] + runner = self._runner_with_dead_pool(tmp_path, published) + try: + with pytest.raises(RuntimeError): + runner._run([sys.executable, "-c", "pass"], timedelta(seconds=600)) + + # WHEN: the runner is handed a working pool and asked to run again, + # exactly as a caller that caught the RuntimeError might. + runner._pool = ThreadPoolExecutor(max_workers=1) + + # THEN: the single-use guard holds. Before the fix this launched a + # second real child and replaced _process, orphaning the first. + with pytest.raises(RuntimeError, match="cannot be used to run a second subprocess"): + runner._run([sys.executable, "-c", "import time; time.sleep(30)"], None) + finally: + runner.cancel() + for timer in [t for t in threading.enumerate() if isinstance(t, threading.Timer)]: + timer.cancel() + runner.shutdown() + + def test_launch_latch_holds_even_when_state_reports_ready(self, tmp_path: Path) -> None: + """Pins the guard to `_launched` rather than to the state classification. + + This is the property that actually broke: whatever `state` decides to + report for an inconsistent runner, re-entry must still be refused. + """ + # GIVEN: a runner that has attempted a launch, with `state` forced to + # report READY (the exact misclassification that opened the guard) + runner = _Runner(logger=MagicMock(), session_working_directory=tmp_path) + try: + runner._pool.shutdown(wait=True) + with pytest.raises(RuntimeError): + runner._run([sys.executable, "-c", "pass"], None) + runner._pool = ThreadPoolExecutor(max_workers=1) + + with patch.object( + type(runner), "state", property(lambda self: ScriptRunnerState.READY) + ): + assert runner.state == ScriptRunnerState.READY # the hostile premise + # THEN + with pytest.raises(RuntimeError, match="second subprocess"): + runner._run([sys.executable, "-c", "pass"], None) + finally: + runner.shutdown() + + def test_completion_always_publishes_a_terminal_state(self, tmp_path: Path) -> None: + # GIVEN: a runner whose process/future pair is inconsistent, so `state` + # cannot be classified as a terminal ActionState + published: list = [] + runner = _Runner( + logger=MagicMock(), + session_working_directory=tmp_path, + callback=lambda s: published.append(s), + ) + try: + runner._process = MagicMock() + runner._run_future = None + completed: Future = Future() + completed.set_result(None) + + # WHEN + runner._on_process_exit(completed) + + # THEN: exactly one terminal callback. Before the fix this list was + # empty: ActionState('ready') raised and the F8 handler ate it. + assert len(published) == 1 + assert published[0] in ( + ActionState.FAILED, + ActionState.CANCELED, + ActionState.TIMEOUT, + ActionState.SUCCESS, + ) + assert published[0] == ActionState.FAILED + finally: + runner.shutdown() + + def test_terminal_action_state_maps_every_runner_state(self, tmp_path: Path) -> None: + """No ScriptRunnerState may make the terminal mapping raise.""" + # GIVEN + runner = _Runner(logger=MagicMock(), session_working_directory=tmp_path) + try: + for runner_state in ScriptRunnerState: + + def fixed_state(_self: Any, _s: ScriptRunnerState = runner_state) -> Any: + return _s + + # WHEN + with patch.object(type(runner), "state", property(fixed_state)): + result = runner._terminal_action_state() + # THEN + assert isinstance(result, ActionState) + finally: + runner.shutdown() + + def test_process_and_future_are_published_together(self, tmp_path: Path) -> None: + """Removes the root cause: a reader must never see a process with no + future, on the success path or any other.""" + # GIVEN: a poller hammering `state` from another thread during launch + observations: list[tuple[bool, bool]] = [] + stop = threading.Event() + runner = _Runner(logger=MagicMock(), session_working_directory=tmp_path) + + def poll() -> None: + while not stop.is_set(): + observations.append((runner._process is not None, runner._run_future is not None)) + + poller = threading.Thread(target=poll, daemon=True) + try: + poller.start() + # WHEN + runner._run([sys.executable, "-c", "pass"], None) + stop.set() + poller.join(timeout=10) + + # THEN: the (process, no future) combination was never observable. + assert (True, False) not in observations + assert observations, "the poller must actually have sampled something" + finally: + stop.set() + runner.shutdown() + + def test_a_normal_run_still_reports_success(self, tmp_path: Path) -> None: + """Guard against the latch or the restructure breaking the happy path.""" + # GIVEN + published: list = [] + runner = _Runner( + logger=MagicMock(), + session_working_directory=tmp_path, + callback=lambda s: published.append(s), + ) + try: + # WHEN + runner._run([sys.executable, "-c", "pass"], None) + assert runner._run_future is not None + runner._run_future.result(timeout=30) + + # THEN + deadline = 30.0 + import time + + start = time.monotonic() + while ActionState.SUCCESS not in published and time.monotonic() - start < deadline: + time.sleep(0.02) + assert runner.state == ScriptRunnerState.SUCCESS + assert ActionState.SUCCESS in published + finally: + runner.shutdown() + + +# =========================================================================== +# REG-2 -- the R5-2 containment must not itself re-raise +# =========================================================================== diff --git a/test/openjd/sessions_v0/test_subprocess_process_group.py b/test/openjd/sessions_v0/test_subprocess_process_group.py new file mode 100644 index 00000000..b088be76 --- /dev/null +++ b/test/openjd/sessions_v0/test_subprocess_process_group.py @@ -0,0 +1,89 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Recording the process group of a launched subprocess. + +The recorded group is the only signal target a later cancel has, so an unknown +group has to be recorded as unknown rather than guessed. +""" + +import sys +from logging.handlers import QueueHandler +from queue import SimpleQueue +from unittest.mock import MagicMock, patch + +import pytest + + +from openjd.sessions._os_checker import is_posix +from openjd.sessions._subprocess import LoggingSubprocess + +from .conftest import build_logger + + +@pytest.mark.skipif(not is_posix(), reason="process groups are POSIX-only") +class TestProcessGroupIsUnknownWhenItCannotBeLookedUp: + """R5-9: the lookup fails *because* the process is gone, so its pid is dead. + Recording it as a process-group id means a later `killpg` is at best a no-op + and, after pid recycling, targets an unrelated group.""" + + def test_reaped_child_leaves_the_group_unknown( + self, message_queue: SimpleQueue, queue_handler: QueueHandler + ) -> None: + # GIVEN: a child whose process group cannot be looked up + logger = build_logger(queue_handler) + proc = LoggingSubprocess( + logger=logger, args=[sys.executable, "-c", "pass"], os_env_vars=None + ) + + # WHEN + with patch( + "openjd.sessions._subprocess.os.getpgid", side_effect=ProcessLookupError(3, "No such") + ): + proc.run() + + # THEN: "unknown", not a stale pid -- and the action still succeeded, + # which is the behaviour the original fix existed to protect. + assert proc._sudo_child_process_group_id is None + assert proc.exit_code == 0 + assert proc.failed_to_start is False + + def test_no_signal_is_delivered_when_the_group_is_unknown( + self, message_queue: SimpleQueue, queue_handler: QueueHandler + ) -> None: + # GIVEN: a finished process with no known group + logger = build_logger(queue_handler) + proc = LoggingSubprocess(logger=logger, args=[sys.executable, "-c", "pass"]) + with patch( + "openjd.sessions._subprocess.os.getpgid", side_effect=ProcessLookupError(3, "No such") + ): + proc.run() + assert proc._sudo_child_process_group_id is None + + # WHEN: a SIGKILL is attempted anyway + with ( + patch("openjd.sessions._subprocess.os.killpg") as killpg, + patch( + "openjd.sessions._subprocess.find_sudo_child_process_group_id", return_value=None + ), + ): + proc._posix_signal_subprocess(MagicMock(pid=999999), signal_name="kill") + + # THEN: nothing was signalled. + killpg.assert_not_called() + + def test_sudo_helper_returns_unknown_when_sudo_is_already_gone(self) -> None: + """Sibling: the same guard on the first getpgid in the sudo helper.""" + # GIVEN + from openjd.sessions._linux._sudo import find_sudo_child_process_group_id + + # WHEN + with patch( + "openjd.sessions._linux._sudo.os.getpgid", + side_effect=ProcessLookupError(3, "No such process"), + ): + result = find_sudo_child_process_group_id( + logger=MagicMock(), sudo_process=MagicMock(pid=999999) + ) + + # THEN: the established "unknown" value, not an escaping ESRCH. + assert result is None diff --git a/test/openjd/sessions_v0/test_tempdir_hardening.py b/test/openjd/sessions_v0/test_tempdir_hardening.py new file mode 100644 index 00000000..8d41b086 --- /dev/null +++ b/test/openjd/sessions_v0/test_tempdir_hardening.py @@ -0,0 +1,322 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +"""Hardening of the shared session temporary root and its cleanup. + +``/OpenJD`` is a fixed, predictable path whose parent is world-writable +on typical POSIX hosts, and it is shared with the job user. +""" + +import os +import stat +from pathlib import Path +from typing import Any +from unittest.mock import patch + +import pytest + + +from openjd.sessions._os_checker import is_posix +from openjd.sessions._tempdir import OPENJD_TEMPDIR_MODE, TempDir, custom_gettempdir + + +class TestSharedTempRootValidation: + """R5-3: `/OpenJD` is a fixed, predictable path whose parent is + world-writable on typical POSIX hosts. `exist_ok=True` accepted whatever was + already there.""" + + def test_created_with_an_explicit_mode_regardless_of_umask(self, tmp_path: Path) -> None: + # GIVEN: a hostile umask that would otherwise strip group/other bits + old_umask = os.umask(0o077) + try: + with patch("openjd.sessions._tempdir.gettempdir", return_value=str(tmp_path)): + # WHEN + created = custom_gettempdir() + finally: + os.umask(old_umask) + + # THEN: the root is traversable as intended, not umask-dependent. + assert Path(created) == tmp_path / "OpenJD" + if is_posix(): + assert stat.S_IMODE(os.stat(created).st_mode) == stat.S_IMODE(OPENJD_TEMPDIR_MODE) + + @pytest.mark.skipif(not is_posix(), reason="symlink pre-creation is a POSIX vector here") + def test_rejects_a_symlink_at_the_root_path(self, tmp_path: Path) -> None: + # GIVEN: an attacker has replaced the root with a symlink elsewhere + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + parent = tmp_path / "parent" + parent.mkdir() + (parent / "OpenJD").symlink_to(elsewhere, target_is_directory=True) + + # WHEN / THEN: we refuse rather than creating sessions inside it. The + # refusal now comes from O_NOFOLLOW on the open itself (see REG-3). + with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): + with pytest.raises(RuntimeError, match="real directory"): + custom_gettempdir() + + def test_rejects_a_root_that_is_not_a_directory(self, tmp_path: Path) -> None: + # GIVEN: a plain file squatting on the root path + parent = tmp_path / "parent" + parent.mkdir() + (parent / "OpenJD").write_text("squat") + + # WHEN / THEN + with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): + with pytest.raises(RuntimeError): + custom_gettempdir() + + @staticmethod + def _fstat_reporting_uid(uid: int) -> Any: + """Wrap os.fstat so the root directory appears owned by `uid`. + + Patches fstat rather than lstat: the implementation validates through a + descriptor now (see REG-3), so an lstat mock would not be consulted and + the test would vacuously pass. + """ + real_fstat = os.fstat + + class _Stat: + def __init__(self, base: os.stat_result) -> None: + self.st_mode = base.st_mode + self.st_uid = uid + + def fake_fstat(fd: int, *a: Any, **k: Any) -> Any: + return _Stat(real_fstat(fd, *a, **k)) + + return fake_fstat + + @pytest.mark.skipif(not is_posix(), reason="uid ownership check is POSIX-only") + def test_rejects_a_root_owned_by_another_user(self, tmp_path: Path) -> None: + # GIVEN: the root exists but is owned by a different uid + parent = tmp_path / "parent" + parent.mkdir() + (parent / "OpenJD").mkdir() + + # WHEN / THEN + with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): + with patch( + "openjd.sessions._tempdir.os.fstat", + side_effect=self._fstat_reporting_uid(os.geteuid() + 12345), + ): + with pytest.raises(RuntimeError, match="owned by uid"): + custom_gettempdir() + + @pytest.mark.skipif(not is_posix(), reason="uid ownership check is POSIX-only") + def test_accepts_a_root_owned_by_root(self, tmp_path: Path) -> None: + """A system-provisioned root must keep working.""" + # GIVEN + parent = tmp_path / "parent" + parent.mkdir() + (parent / "OpenJD").mkdir() + + # WHEN / THEN: no exception + with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): + with patch( + "openjd.sessions._tempdir.os.fstat", side_effect=self._fstat_reporting_uid(0) + ): + assert custom_gettempdir() == str(parent / "OpenJD") + + +# =========================================================================== +# R5-7 -- cleanup must report WHY each path could not be deleted +# =========================================================================== + + +class TestCleanupErrorReporting: + """R5-7: the old handler accepted the exception and discarded it, leaving a + list of bare paths -- on the one code path where "permission denied" versus + "a process still holds this open" changes what the operator must do.""" + + def test_failure_message_names_the_path_and_the_reason(self, tmp_path: Path) -> None: + # GIVEN: a temp dir whose removal fails with a specific, diagnosable error + d = TempDir(dir=tmp_path) + doomed = d.path / "stubborn.txt" + doomed.write_text("x") + + def boom(path: Any, *a: Any, **k: Any) -> None: + raise PermissionError(13, "Permission denied") + + # WHEN + with ( + patch("openjd.sessions._tempdir.os.unlink", side_effect=boom), + patch("openjd.sessions._tempdir.os.remove", side_effect=boom, create=True), + ): + with pytest.raises(RuntimeError) as excinfo: + d.cleanup() + + # THEN: both the path and the cause are in the message. + message = str(excinfo.value) + assert "stubborn.txt" in message + assert "PermissionError" in message + + def test_successful_cleanup_still_removes_everything(self, tmp_path: Path) -> None: + # GIVEN + d = TempDir(dir=tmp_path) + (d.path / "a.txt").write_text("x") + (d.path / "sub").mkdir() + (d.path / "sub" / "b.txt").write_text("y") + + # WHEN + d.cleanup() + + # THEN + assert not d.path.exists() + + +# =========================================================================== +# R5-4 / R5-5 -- nothing unquoted may reach /bin/sh +# =========================================================================== + + +@pytest.mark.skipif(not is_posix(), reason="symlink swap and fchmod semantics are POSIX here") +class TestTempRootCheckThenUse: + """The first R5-3 implementation validated with `lstat(path)` and then called + `stat(path)`/`chmod(path)`, both of which re-resolve the name and follow + links. Swapping the entry for a symlink in between defeated the check and + chmod'ed the link's target to 0o755.""" + + def test_a_symlink_swapped_in_before_the_open_is_refused(self, tmp_path: Path) -> None: + """Swap during the create window: O_NOFOLLOW must refuse the open.""" + # GIVEN: a victim directory at 0o700 and a parent where the root will live + victim = tmp_path / "victim" + victim.mkdir() + os.chmod(victim, 0o700) + parent = tmp_path / "parent" + parent.mkdir() + root = parent / "OpenJD" + + real_makedirs = os.makedirs + + def makedirs_then_swap(path: Any, *a: Any, **k: Any) -> None: + real_makedirs(path, *a, **k) + if str(path) == str(root): + os.rmdir(root) + os.symlink(victim, root, target_is_directory=True) + + # WHEN + with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): + with patch("openjd.sessions._tempdir.os.makedirs", side_effect=makedirs_then_swap): + with pytest.raises(RuntimeError): + custom_gettempdir() + + # THEN + assert stat.S_IMODE(os.stat(victim).st_mode) == 0o700 + + def test_a_symlink_swapped_in_after_the_open_cannot_redirect_the_chmod( + self, tmp_path: Path + ) -> None: + """Swap *after* the descriptor is open, which O_NOFOLLOW cannot help with. + + This is the case that pins fchmod-vs-chmod. An earlier version of this + test swapped during the create window instead, where O_NOFOLLOW refuses + the open before any chmod is attempted -- so it passed even with + `os.chmod(path)` restored, and pinned nothing about the descriptor. The + swap is injected from inside `os.fstat`, which the implementation calls + between the open and the mode change. + """ + # GIVEN: a victim at 0o700, and a real root whose mode needs correcting + victim = tmp_path / "victim_after" + victim.mkdir() + os.chmod(victim, 0o700) + parent = tmp_path / "parent_after" + parent.mkdir() + root = parent / "OpenJD" + root.mkdir() + os.chmod(root, 0o700) # != OPENJD_TEMPDIR_MODE, so a mode change is due + + real_fstat = os.fstat + swapped = {"done": False} + + def fstat_then_swap(fd: int, *a: Any, **k: Any) -> Any: + result = real_fstat(fd, *a, **k) + if not swapped["done"]: + swapped["done"] = True + os.rmdir(root) + os.symlink(victim, root, target_is_directory=True) + return result + + # WHEN + with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): + with patch("openjd.sessions._tempdir.os.fstat", side_effect=fstat_then_swap): + custom_gettempdir() + + # THEN: the swap happened, and the victim was NOT widened. With + # `os.chmod(temp_dir, ...)` restored this is 0o755. + assert swapped["done"] is True + assert os.path.islink(root) + assert stat.S_IMODE(os.stat(victim).st_mode) == 0o700 + + def test_the_validated_inode_is_the_one_modified(self, tmp_path: Path) -> None: + """Even when the swap is not detected as an error, the mode must land on + the inode that was validated, never on a later resolution of the name.""" + # GIVEN: an existing root at a wrong mode, plus a victim + victim = tmp_path / "victim2" + victim.mkdir() + os.chmod(victim, 0o700) + parent = tmp_path / "parent2" + parent.mkdir() + root = parent / "OpenJD" + root.mkdir() + os.chmod(root, 0o700) # differs from OPENJD_TEMPDIR_MODE, so a chmod is due + + # WHEN + with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): + result = custom_gettempdir() + + # THEN: the real root got the mode; the victim was not touched. + assert result == str(root) + assert stat.S_IMODE(os.stat(root).st_mode) == stat.S_IMODE(OPENJD_TEMPDIR_MODE) + assert stat.S_IMODE(os.stat(victim).st_mode) == 0o700 + + def test_a_symlinked_root_is_still_refused(self, tmp_path: Path) -> None: + """O_NOFOLLOW must keep doing what the explicit S_ISLNK branch did.""" + # GIVEN + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + parent = tmp_path / "parent3" + parent.mkdir() + (parent / "OpenJD").symlink_to(elsewhere, target_is_directory=True) + + # WHEN / THEN + with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): + with pytest.raises(RuntimeError, match="real directory"): + custom_gettempdir() + + def test_no_descriptor_is_leaked(self, tmp_path: Path) -> None: + """The validation opens a descriptor; every path must close it. + + Probe: the lowest free descriptor number. If a call leaks a descriptor, + the number the kernel hands out next goes up. Portable across macOS and + Linux, unlike listing /dev/fd. + """ + # GIVEN + parent = tmp_path / "parent4" + parent.mkdir() + (parent / "OpenJD").mkdir() + + def lowest_free_fd() -> int: + fd = os.open(os.devnull, os.O_RDONLY) + os.close(fd) + return fd + + with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): + custom_gettempdir() # warm up, so first-call allocations do not count + baseline = lowest_free_fd() + + # WHEN: many successful calls + for _ in range(25): + custom_gettempdir() + after_success = lowest_free_fd() + + # WHEN: many calls that fail *after* the descriptor is opened + with patch( + "openjd.sessions._tempdir.os.fstat", side_effect=OSError(5, "induced failure") + ): + for _ in range(25): + with pytest.raises(OSError): + custom_gettempdir() + after_failure = lowest_free_fd() + + # THEN: no drift on either path. + assert after_success == baseline + assert after_failure == baseline From acdea90b0826f06a8c1faada3916548b75d24566 Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Sun, 26 Jul 2026 15:53:43 -0700 Subject: [PATCH 22/27] fix: Silence Windows mypy on POSIX-only os and signal attributes The `Run Linting` step failed on every Windows matrix job, which fail-fast then cancelled the rest of the matrix. mypy on win32 does not see `os.geteuid`, `os.fchmod`, `os.getpgid` or `signal.SIGKILL`, all of which this PR added inside POSIX-gated code paths. Adds `# type: ignore` to the 11 sites, matching the convention already in _tempdir.py:257. No behaviour change; `warn_unused_ignores` is false, so the comments are inert on POSIX. Verified `mypy --platform win32 src test` and native mypy both clean, plus ruff, black, and 855 passed / 39 skipped / 16 xfailed. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- src/openjd/sessions/_tempdir.py | 4 ++-- test/openjd/sessions_v0/test_linux_sudo.py | 8 ++++---- test/openjd/sessions_v0/test_subprocess_reaping.py | 6 +++--- test/openjd/sessions_v0/test_tempdir.py | 2 +- test/openjd/sessions_v0/test_tempdir_hardening.py | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/openjd/sessions/_tempdir.py b/src/openjd/sessions/_tempdir.py index 4d2c2e01..155a50fa 100644 --- a/src/openjd/sessions/_tempdir.py +++ b/src/openjd/sessions/_tempdir.py @@ -81,7 +81,7 @@ def _prepare_temp_dir_root(temp_dir: str) -> None: f"Refusing to use temporary directory {temp_dir}: it is not a real directory." ) if is_posix(): - this_uid = os.geteuid() + this_uid = os.geteuid() # type: ignore # root is accepted so that a system-provisioned root directory works. if st.st_uid not in (this_uid, 0): raise RuntimeError( @@ -100,7 +100,7 @@ def _prepare_temp_dir_root(temp_dir: str) -> None: try: # fchmod, not chmod: applies to the validated inode, not to # whatever the name resolves to now. - os.fchmod(fd, OPENJD_TEMPDIR_MODE) + os.fchmod(fd, OPENJD_TEMPDIR_MODE) # type: ignore except OSError as err: raise RuntimeError( f"Could not set permissions on temporary directory {temp_dir}: {err}. " diff --git a/test/openjd/sessions_v0/test_linux_sudo.py b/test/openjd/sessions_v0/test_linux_sudo.py index 64ddec6c..2624dfe8 100644 --- a/test/openjd/sessions_v0/test_linux_sudo.py +++ b/test/openjd/sessions_v0/test_linux_sudo.py @@ -523,7 +523,7 @@ def test_child_that_exits_mid_scan_still_short_circuits( departed.kill() departed.wait() with pytest.raises(ProcessLookupError): - os.getpgid(departed.pid) + os.getpgid(departed.pid) # type: ignore sudo_process = sleeper() logger = build_logger(queue_handler) scan = Mock(return_value=departed.pid) @@ -563,7 +563,7 @@ def test_child_still_sharing_sudos_process_group_is_never_accepted( # GIVEN a live process in the same process group as the stand-in for sudo sudo_process = sleeper() sibling = sleeper() - assert os.getpgid(sibling.pid) == os.getpgid( + assert os.getpgid(sibling.pid) == os.getpgid( # type: ignore sudo_process.pid ), "test setup: the two processes were expected to share a process group" logger = build_logger(queue_handler) @@ -619,7 +619,7 @@ def test_child_is_accepted_once_it_leaves_sudos_process_group( # The delay gives this assertion ample room: the child has to start a # Python interpreter and sleep 0.5s before it can change groups. - assert os.getpgid(child.pid) == os.getpgid( + assert os.getpgid(child.pid) == os.getpgid( # type: ignore sudo_process.pid ), "test setup: the child was expected to start in sudo's process group" @@ -641,6 +641,6 @@ def test_child_is_accepted_once_it_leaves_sudos_process_group( f"observed one (exit code: {child.poll()}, stderr: {stderr_of(child)!r})" ) assert result == reported_pgid - assert result != os.getpgid(sudo_process.pid) + assert result != os.getpgid(sudo_process.pid) # type: ignore # AND nothing was logged: retrying past the shared group is not a failure. assert collect_queue_messages(message_queue) == [] diff --git a/test/openjd/sessions_v0/test_subprocess_reaping.py b/test/openjd/sessions_v0/test_subprocess_reaping.py index f7615933..8f6ea324 100644 --- a/test/openjd/sessions_v0/test_subprocess_reaping.py +++ b/test/openjd/sessions_v0/test_subprocess_reaping.py @@ -136,7 +136,7 @@ def test_exit_code_is_recorded_for_an_abandoned_child( # THEN: a real exit status, not None. SIGKILL shows as -SIGKILL. assert proc.exit_code is not None - assert proc.exit_code == -signal.SIGKILL + assert proc.exit_code == -signal.SIGKILL # type: ignore def test_a_child_that_exited_on_its_own_keeps_its_own_exit_code( self, exploding_logger: Any, python_exe: str @@ -153,7 +153,7 @@ def test_a_child_that_exited_on_its_own_keeps_its_own_exit_code( # THEN: whichever way the race went, the recorded code is a real one and # never None. If the child had already exited, it must be its own 7. assert proc.exit_code is not None - assert proc.exit_code in (7, -signal.SIGKILL) + assert proc.exit_code in (7, -signal.SIGKILL) # type: ignore def test_a_reap_failure_does_not_replace_the_original_exception( self, exploding_logger: Any, python_exe: str @@ -197,7 +197,7 @@ def test_a_reap_failure_does_not_replace_the_original_exception( # Clean up the child the failed terminate left behind. if proc.pid is not None and _pid_alive(proc.pid): - os.kill(proc.pid, signal.SIGKILL) + os.kill(proc.pid, signal.SIGKILL) # type: ignore try: os.waitpid(proc.pid, 0) except ChildProcessError: diff --git a/test/openjd/sessions_v0/test_tempdir.py b/test/openjd/sessions_v0/test_tempdir.py index 7f18f7aa..77f5c122 100644 --- a/test/openjd/sessions_v0/test_tempdir.py +++ b/test/openjd/sessions_v0/test_tempdir.py @@ -159,7 +159,7 @@ def test_successful_ownership_change_keeps_the_directory(self, tmp_path: Path) - assert stat.S_IMODE(statinfo.st_mode) == stat.S_IRWXU | stat.S_IRWXG @pytest.mark.skipif( - os.geteuid() == 0 if is_posix() else True, + os.geteuid() == 0 if is_posix() else True, # type: ignore reason="root is not subject to directory permissions", ) def test_cleanup_failure_does_not_replace_the_ownership_error(self, tmp_path: Path) -> None: diff --git a/test/openjd/sessions_v0/test_tempdir_hardening.py b/test/openjd/sessions_v0/test_tempdir_hardening.py index 8d41b086..1c77b826 100644 --- a/test/openjd/sessions_v0/test_tempdir_hardening.py +++ b/test/openjd/sessions_v0/test_tempdir_hardening.py @@ -96,7 +96,7 @@ def test_rejects_a_root_owned_by_another_user(self, tmp_path: Path) -> None: with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): with patch( "openjd.sessions._tempdir.os.fstat", - side_effect=self._fstat_reporting_uid(os.geteuid() + 12345), + side_effect=self._fstat_reporting_uid(os.geteuid() + 12345), # type: ignore ): with pytest.raises(RuntimeError, match="owned by uid"): custom_gettempdir() From 32c69edf8c3195a5682ff587fcb8e90cd7203076 Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Sun, 26 Jul 2026 16:03:48 -0700 Subject: [PATCH 23/27] fix: Do not open a directory descriptor on Windows The R5-3 temp-root hardening validated `/OpenJD` through a single descriptor, and its docstring claimed the `getattr(os, "O_NOFOLLOW", 0)` fallbacks degraded that to "a plain open" on Windows. That claim was wrong and untested: Windows returns EACCES for `os.open()` on a directory whatever the flags are, so `custom_gettempdir()` raised on every call and every Windows test that builds a Session failed with RuntimeError: Refusing to use temporary directory C:\ProgramData\Amazon\ OpenJD: it could not be opened as a real directory ([Errno 13] Permission denied) Splits the validator by platform. POSIX keeps the descriptor, which is what closes the symlink-swap window, and now uses the flags unconditionally instead of pretending to be portable. Windows validates with `os.lstat`, which does not traverse a symlink or junction, and sets no mode -- the mode is a POSIX mode and Windows access is governed by the ACLs inherited from %PROGRAMDATA%. The docstring says plainly that the Windows check is the weaker of the two. Tests: six new tests, all runnable on POSIX so a POSIX CI host catches a Windows-only regression. `test_a_non_posix_platform_never_opens_a_descriptor` pins the dispatch by patching `is_posix`, with `test_a_posix_platform_still_uses_a_descriptor` as the negative control so a mutation routing everything to the weaker validator also fails. Also fixes two tests that patched only `gettempdir` and so silently operated on the real %PROGRAMDATA%\Amazon\OpenJD when run on Windows. 4/4 mutants caught. 861 passed / 39 skipped / 16 xfailed; ruff, black, and mypy clean on both native and --platform win32. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- src/openjd/sessions/_tempdir.py | 85 ++++++++-- .../sessions_v0/test_tempdir_hardening.py | 147 ++++++++++++++++-- 2 files changed, 205 insertions(+), 27 deletions(-) diff --git a/src/openjd/sessions/_tempdir.py b/src/openjd/sessions/_tempdir.py index 155a50fa..c96f9702 100644 --- a/src/openjd/sessions/_tempdir.py +++ b/src/openjd/sessions/_tempdir.py @@ -33,7 +33,7 @@ """ -def _prepare_temp_dir_root(temp_dir: str) -> None: +def _prepare_temp_dir_root_posix(temp_dir: str) -> None: """Validate the shared temporary root and set its mode, operating on a single file descriptor so the two cannot disagree. @@ -56,15 +56,17 @@ def _prepare_temp_dir_root(temp_dir: str) -> None: name afterwards. ``O_NOFOLLOW`` makes the open itself fail on a symlink, which is why there is - no separate ``S_ISLNK`` branch on POSIX. Neither flag exists on Windows, where - the ``getattr(..., 0)`` fallbacks make this a plain open and the reparse-point - check below carries the equivalent weight. + no separate ``S_ISLNK`` branch. + + POSIX only. Windows cannot ``os.open()`` a directory at all -- the CRT + rejects it with ``EACCES`` regardless of flags -- so Windows takes the + ``lstat`` path in :func:`_prepare_temp_dir_root_windows` instead. Raises: - RuntimeError: if the path is a symlink or reparse point, is not a - directory, is owned by another user, or its mode cannot be set. + RuntimeError: if the path is a symlink, is not a directory, is owned by + another user, or its mode cannot be set. """ - flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_DIRECTORY", 0) + flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_DIRECTORY # type: ignore try: fd = os.open(temp_dir, flags) except OSError as err: @@ -76,19 +78,18 @@ def _prepare_temp_dir_root(temp_dir: str) -> None: ) try: st = os.fstat(fd) - if not stat.S_ISDIR(st.st_mode) or getattr(st, "st_reparse_tag", 0) != 0: + if not stat.S_ISDIR(st.st_mode): raise RuntimeError( f"Refusing to use temporary directory {temp_dir}: it is not a real directory." ) - if is_posix(): - this_uid = os.geteuid() # type: ignore - # root is accepted so that a system-provisioned root directory works. - if st.st_uid not in (this_uid, 0): - raise RuntimeError( - f"Refusing to use temporary directory {temp_dir}: it is owned by uid " - f"{st.st_uid}, not by this process' uid ({this_uid}) or root. Another user " - "may have created it. Remove it, or pass an explicit session root directory." - ) + this_uid = os.geteuid() # type: ignore + # root is accepted so that a system-provisioned root directory works. + if st.st_uid not in (this_uid, 0): + raise RuntimeError( + f"Refusing to use temporary directory {temp_dir}: it is owned by uid " + f"{st.st_uid}, not by this process' uid ({this_uid}) or root. Another user " + "may have created it. Remove it, or pass an explicit session root directory." + ) # makedirs()' `mode` is masked by the process umask, so it does not on its # own guarantee anything: under umask 0o077 an "explicit" 0o755 lands as @@ -111,6 +112,56 @@ def _prepare_temp_dir_root(temp_dir: str) -> None: os.close(fd) +def _prepare_temp_dir_root_windows(temp_dir: str) -> None: + """Validate the shared temporary root on Windows. + + Windows cannot open a directory with ``os.open()`` -- the CRT returns + ``EACCES`` whatever the flags -- so the single-descriptor approach used on + POSIX is not available here. ``os.lstat()`` is the closest equivalent: it + does not traverse symlinks or reparse points, so a directory symlink reports + ``S_ISLNK`` (failing the ``S_ISDIR`` test) and a junction reports a non-zero + ``st_reparse_tag``. + + There is no mode to set: ``OPENJD_TEMPDIR_MODE`` is a POSIX mode, and + Windows access to this root is governed by the ACLs it inherits from + ``%PROGRAMDATA%``. There is also no uid comparison; ownership here is a SID, + and asserting anything about it is left to the Windows-specific code that + creates session directories beneath this root. + + This is a weaker check than the POSIX one, and knowingly so: it closes the + "the path is not really a directory" case but not a time-of-check race. + + Raises: + RuntimeError: if the path is a symlink, reparse point, or not a directory. + """ + try: + st = os.lstat(temp_dir) + except OSError as err: + raise RuntimeError( + f"Refusing to use temporary directory {temp_dir}: it could not be inspected ({err}). " + "Remove it, or pass an explicit session root directory." + ) + if not stat.S_ISDIR(st.st_mode) or getattr(st, "st_reparse_tag", 0) != 0: + raise RuntimeError( + f"Refusing to use temporary directory {temp_dir}: it is not a real directory. If it " + "is a symbolic link, junction, or not a directory, remove it, or pass an explicit " + "session root directory." + ) + + +def _prepare_temp_dir_root(temp_dir: str) -> None: + """Validate the shared temporary root, by whichever means the platform allows. + + See :func:`_prepare_temp_dir_root_posix` and + :func:`_prepare_temp_dir_root_windows`; the two differ in strength, not just + in mechanism. + """ + if is_posix(): + _prepare_temp_dir_root_posix(temp_dir) + else: + _prepare_temp_dir_root_windows(temp_dir) + + def custom_gettempdir(logger: Optional[LoggerAdapter] = None) -> str: """ Get a platform-specific temporary directory. diff --git a/test/openjd/sessions_v0/test_tempdir_hardening.py b/test/openjd/sessions_v0/test_tempdir_hardening.py index 1c77b826..db908c15 100644 --- a/test/openjd/sessions_v0/test_tempdir_hardening.py +++ b/test/openjd/sessions_v0/test_tempdir_hardening.py @@ -8,15 +8,46 @@ import os import stat +from contextlib import contextmanager from pathlib import Path -from typing import Any +from typing import Any, Iterator from unittest.mock import patch import pytest - -from openjd.sessions._os_checker import is_posix -from openjd.sessions._tempdir import OPENJD_TEMPDIR_MODE, TempDir, custom_gettempdir +from openjd.sessions._os_checker import is_posix, is_windows +from openjd.sessions._tempdir import ( + OPENJD_TEMPDIR_MODE, + TempDir, + _prepare_temp_dir_root, + _prepare_temp_dir_root_windows, + custom_gettempdir, +) + + +def temp_root_under(parent: Path) -> Path: + """The root `custom_gettempdir()` will use when its parent is redirected to `parent`. + + The two platforms nest differently: POSIX uses `/OpenJD`, Windows + uses `%PROGRAMDATA%\\Amazon\\OpenJD`. + """ + return parent / "Amazon" / "OpenJD" if is_windows() else parent / "OpenJD" + + +@contextmanager +def redirected_temp_root(parent: Path) -> Iterator[Path]: + """Redirect `custom_gettempdir()` beneath `parent` on either platform. + + Patching `gettempdir` alone is not enough: on Windows `custom_gettempdir()` + never calls it, so a `gettempdir`-only patch silently leaves the test + operating on the real `%PROGRAMDATA%\\Amazon\\OpenJD`. + """ + if is_windows(): + with patch.dict(os.environ, {"PROGRAMDATA": str(parent)}): + yield temp_root_under(parent) + else: + with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): + yield temp_root_under(parent) class TestSharedTempRootValidation: @@ -28,14 +59,14 @@ def test_created_with_an_explicit_mode_regardless_of_umask(self, tmp_path: Path) # GIVEN: a hostile umask that would otherwise strip group/other bits old_umask = os.umask(0o077) try: - with patch("openjd.sessions._tempdir.gettempdir", return_value=str(tmp_path)): + with redirected_temp_root(tmp_path) as expected_root: # WHEN created = custom_gettempdir() finally: os.umask(old_umask) # THEN: the root is traversable as intended, not umask-dependent. - assert Path(created) == tmp_path / "OpenJD" + assert Path(created) == expected_root if is_posix(): assert stat.S_IMODE(os.stat(created).st_mode) == stat.S_IMODE(OPENJD_TEMPDIR_MODE) @@ -57,11 +88,13 @@ def test_rejects_a_symlink_at_the_root_path(self, tmp_path: Path) -> None: def test_rejects_a_root_that_is_not_a_directory(self, tmp_path: Path) -> None: # GIVEN: a plain file squatting on the root path parent = tmp_path / "parent" - parent.mkdir() - (parent / "OpenJD").write_text("squat") + squatter = temp_root_under(parent) + squatter.parent.mkdir(parents=True) + squatter.write_text("squat") - # WHEN / THEN - with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): + # WHEN / THEN: makedirs() rejects it first on both platforms, so this + # asserts the refusal, not which layer produced it. + with redirected_temp_root(parent): with pytest.raises(RuntimeError): custom_gettempdir() @@ -101,6 +134,100 @@ def test_rejects_a_root_owned_by_another_user(self, tmp_path: Path) -> None: with pytest.raises(RuntimeError, match="owned by uid"): custom_gettempdir() + def test_windows_validator_accepts_a_real_directory(self, tmp_path: Path) -> None: + """The Windows branch must not try to `os.open()` a directory. + + Windows returns EACCES for `os.open()` on a directory whatever the flags, + so an earlier version of this validator failed *every* session creation + on Windows. `os.lstat` is portable, so this runs on all platforms and + pins the branch against a regression back to a descriptor. + """ + # GIVEN a real directory / WHEN validated / THEN it is accepted + root = tmp_path / "OpenJD" + root.mkdir() + _prepare_temp_dir_root_windows(str(root)) + + def test_a_non_posix_platform_never_opens_a_descriptor(self, tmp_path: Path) -> None: + """The dispatcher must not send Windows down the descriptor path. + + This is the bug itself: `os.open()` on a directory returns EACCES on + Windows whatever the flags, so routing Windows through the POSIX + validator failed *every* session creation there. Pinned by patching + `is_posix` rather than by running on Windows, so a POSIX CI host catches + the regression too -- without this, the only signal is a Windows job. + """ + # GIVEN a real directory, and a platform that reports as non-POSIX + root = tmp_path / "OpenJD" + root.mkdir() + + # WHEN validated + with patch("openjd.sessions._tempdir.is_posix", return_value=False): + with patch("openjd.sessions._tempdir.os.open") as opener: + _prepare_temp_dir_root(str(root)) + + # THEN no descriptor was ever taken + opener.assert_not_called() + + def test_a_posix_platform_still_uses_a_descriptor(self, tmp_path: Path) -> None: + """Negative control for the test above: POSIX must keep the fd path. + + The fd path is what makes the symlink-swap window unexploitable, so a + mutation that routed *everything* to the weaker `lstat` validator has to + fail something. + """ + # GIVEN a real directory on a platform that reports as POSIX + root = tmp_path / "OpenJD" + root.mkdir() + real_open = os.open + opened: list[str] = [] + + def recording_open(path: Any, *a: Any, **k: Any) -> int: + opened.append(str(path)) + return real_open(path, *a, **k) + + # WHEN validated + with patch("openjd.sessions._tempdir.is_posix", return_value=True): + with patch("openjd.sessions._tempdir.os.open", side_effect=recording_open): + _prepare_temp_dir_root(str(root)) + + # THEN the root was validated through a descriptor + assert str(root) in opened + + def test_windows_validator_rejects_a_non_directory(self, tmp_path: Path) -> None: + # GIVEN a plain file where the root should be + squatter = tmp_path / "OpenJD" + squatter.write_text("squat") + + # WHEN / THEN + with pytest.raises(RuntimeError, match="real directory"): + _prepare_temp_dir_root_windows(str(squatter)) + + @pytest.mark.skipif(not is_posix(), reason="symlink creation is POSIX-only here") + def test_windows_validator_rejects_a_symlink(self, tmp_path: Path) -> None: + """`lstat` must not traverse the link, or a directory symlink would pass. + + Exercised with a POSIX symlink because that is what this host can create; + on Windows the same `S_ISDIR` test rejects a directory symlink and + `st_reparse_tag` rejects a junction. + """ + # GIVEN + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + link = tmp_path / "OpenJD" + link.symlink_to(elsewhere, target_is_directory=True) + + # WHEN / THEN + with pytest.raises(RuntimeError, match="real directory"): + _prepare_temp_dir_root_windows(str(link)) + + def test_windows_validator_reports_an_unreadable_root(self, tmp_path: Path) -> None: + # GIVEN a root that cannot be inspected at all + missing = tmp_path / "OpenJD" + + # WHEN / THEN: the failure names the path rather than escaping as OSError + with pytest.raises(RuntimeError, match="could not be inspected"): + _prepare_temp_dir_root_windows(str(missing)) + @pytest.mark.skipif(not is_posix(), reason="uid ownership check is POSIX-only") def test_accepts_a_root_owned_by_root(self, tmp_path: Path) -> None: """A system-provisioned root must keep working.""" From 928269a0ce219a5039f5e1d2ce5d7f0522dc92aa Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Sun, 26 Jul 2026 16:10:21 -0700 Subject: [PATCH 24/27] test: Skip the POSIX-branch control on Windows, drop stale banners test_a_posix_platform_still_uses_a_descriptor drives the POSIX validator directly by patching is_posix, and that branch names os.O_NOFOLLOW and os.O_DIRECTORY. Those are absent on some Windows builds -- the Windows 3.13 job failed with AttributeError while 3.14 passed -- so the test is now POSIX-only. It is a control for a mutation that is only ever evaluated on a POSIX host, so nothing is lost. All five temp-root mutants still caught. Also deletes the `# ==== / # R5-x -- ... / # ====` section banners the split carried into the wrong files: every one of them announced content that is not in the file it now sits in, e.g. test_runner_launch_state.py opened with "R5-1 -- redaction must not leave record.args populated". 861 passed / 39 skipped / 16 xfailed; ruff, black, and mypy clean on native and --platform win32. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- .../test_action_filter_hardening.py | 20 ------------------- .../test_generated_shell_script.py | 10 ---------- .../test_optimized_mode_invariants.py | 10 ---------- .../sessions_v0/test_runner_launch_state.py | 10 ---------- .../sessions_v0/test_tempdir_hardening.py | 18 ++++++++--------- 5 files changed, 8 insertions(+), 60 deletions(-) diff --git a/test/openjd/sessions_v0/test_action_filter_hardening.py b/test/openjd/sessions_v0/test_action_filter_hardening.py index 7a30cf0b..6446dda8 100644 --- a/test/openjd/sessions_v0/test_action_filter_hardening.py +++ b/test/openjd/sessions_v0/test_action_filter_hardening.py @@ -137,11 +137,6 @@ def test_args_are_untouched_when_no_redactions_registered(self) -> None: assert record.getMessage() == "value is plain" -# =========================================================================== -# R5-2 -- consumer callback exceptions must not escape filter() -# =========================================================================== - - class TestFilterContainsConsumerCallbackFailures: """R5-2: this filter runs on the thread forwarding the subprocess's stdout. @@ -257,11 +252,6 @@ def callback(session_id: str, status: Any) -> None: assert session.action_status.state == ActionState.SUCCESS -# =========================================================================== -# SIB-1 -- env var name anchoring (sibling of R5-5, found during this round) -# =========================================================================== - - class TestEnvVarNameAnchoring: """`$` also matches immediately before a trailing newline, so an `$`-anchored NAME pattern accepted "FOO\\n" -- a name no OS can hold. @@ -334,11 +324,6 @@ def test_a_separator_cannot_reach_a_decoded_name(self, msg: str) -> None: assert callback.call_args_list[-1][0][2] is True # cancel-and-fail -# =========================================================================== -# R5-3 -- the shared temporary root must be validated before use -# =========================================================================== - - class TestContainmentDoesNotReRaise: def test_handler_path_contains_an_unrenderable_exception(self) -> None: # GIVEN: a consumer callback raising an exception that cannot be rendered @@ -378,8 +363,3 @@ def callback(kind: ActionMessageKind, value: Any, fail: bool) -> None: # THEN assert "a perfectly ordinary boom" in record.getMessage() - - -# =========================================================================== -# REG-3 -- the R5-3 validation must not be defeatable by a symlink swap -# =========================================================================== diff --git a/test/openjd/sessions_v0/test_generated_shell_script.py b/test/openjd/sessions_v0/test_generated_shell_script.py index 4be600a8..fe4ec725 100644 --- a/test/openjd/sessions_v0/test_generated_shell_script.py +++ b/test/openjd/sessions_v0/test_generated_shell_script.py @@ -33,11 +33,6 @@ def cancel( pass -# =========================================================================== -# R5-1 -- redaction must not leave record.args populated -# =========================================================================== - - @pytest.mark.skipif(not is_posix(), reason="the generated shell script is POSIX-only") class TestGeneratedShellScriptQuoting: """The generated script is `exec`'d by /bin/sh, so a single unquoted @@ -182,8 +177,3 @@ def test_a_skipped_name_still_reaches_the_subprocess( lines.append(message_queue.get().getMessage()) assert "VAL=present" in lines assert proc.exit_code == 0 - - -# =========================================================================== -# R5-6 -- invariant-bearing checks must survive `python -O` -# =========================================================================== diff --git a/test/openjd/sessions_v0/test_optimized_mode_invariants.py b/test/openjd/sessions_v0/test_optimized_mode_invariants.py index db93ccfd..f7dc7c38 100644 --- a/test/openjd/sessions_v0/test_optimized_mode_invariants.py +++ b/test/openjd/sessions_v0/test_optimized_mode_invariants.py @@ -40,11 +40,6 @@ def cancel( pass -# =========================================================================== -# R5-1 -- redaction must not leave record.args populated -# =========================================================================== - - class TestInvariantsSurviveOptimizedMode: """R5-6: `assert` is stripped under `python -O`, which is a legitimate production deployment mode. Checks that carry a real runtime invariant must @@ -148,8 +143,3 @@ def test_an_embedded_file_still_materializes_normally(self, tmp_path: Path) -> N assert len(records) == 1 assert isinstance(records[0], _FileRecord) assert records[0].filename.read_text() == "hello" - - -# =========================================================================== -# R5-9 -- an unknown process group must be recorded as unknown -# =========================================================================== diff --git a/test/openjd/sessions_v0/test_runner_launch_state.py b/test/openjd/sessions_v0/test_runner_launch_state.py index 73eb9087..3ed6d25d 100644 --- a/test/openjd/sessions_v0/test_runner_launch_state.py +++ b/test/openjd/sessions_v0/test_runner_launch_state.py @@ -29,11 +29,6 @@ def cancel( pass -# =========================================================================== -# R5-1 -- redaction must not leave record.args populated -# =========================================================================== - - class TestFailedLaunchDoesNotCorruptRunnerState: """A launch that fails after the subprocess object exists must not leave the runner looking reusable, and a completed action must always publish a @@ -206,8 +201,3 @@ def test_a_normal_run_still_reports_success(self, tmp_path: Path) -> None: assert ActionState.SUCCESS in published finally: runner.shutdown() - - -# =========================================================================== -# REG-2 -- the R5-2 containment must not itself re-raise -# =========================================================================== diff --git a/test/openjd/sessions_v0/test_tempdir_hardening.py b/test/openjd/sessions_v0/test_tempdir_hardening.py index db908c15..b6aa5be1 100644 --- a/test/openjd/sessions_v0/test_tempdir_hardening.py +++ b/test/openjd/sessions_v0/test_tempdir_hardening.py @@ -168,12 +168,20 @@ def test_a_non_posix_platform_never_opens_a_descriptor(self, tmp_path: Path) -> # THEN no descriptor was ever taken opener.assert_not_called() + @pytest.mark.skipif( + not is_posix(), reason="forces the POSIX branch, whose O_NOFOLLOW/O_DIRECTORY are absent" + ) def test_a_posix_platform_still_uses_a_descriptor(self, tmp_path: Path) -> None: """Negative control for the test above: POSIX must keep the fd path. The fd path is what makes the symlink-swap window unexploitable, so a mutation that routed *everything* to the weaker `lstat` validator has to fail something. + + POSIX-only, and not merely by preference: it drives the POSIX branch + directly, and that branch names `os.O_NOFOLLOW`/`os.O_DIRECTORY`, which + do not exist on every Windows build. The mutation this controls for is + only ever evaluated on a POSIX host anyway. """ # GIVEN a real directory on a platform that reports as POSIX root = tmp_path / "OpenJD" @@ -244,11 +252,6 @@ def test_accepts_a_root_owned_by_root(self, tmp_path: Path) -> None: assert custom_gettempdir() == str(parent / "OpenJD") -# =========================================================================== -# R5-7 -- cleanup must report WHY each path could not be deleted -# =========================================================================== - - class TestCleanupErrorReporting: """R5-7: the old handler accepted the exception and discarded it, leaving a list of bare paths -- on the one code path where "permission denied" versus @@ -290,11 +293,6 @@ def test_successful_cleanup_still_removes_everything(self, tmp_path: Path) -> No assert not d.path.exists() -# =========================================================================== -# R5-4 / R5-5 -- nothing unquoted may reach /bin/sh -# =========================================================================== - - @pytest.mark.skipif(not is_posix(), reason="symlink swap and fchmod semantics are POSIX here") class TestTempRootCheckThenUse: """The first R5-3 implementation validated with `lstat(path)` and then called From fe3741d9be6eb6a1b3b2054b0c2a5f795c26cd62 Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Sun, 26 Jul 2026 16:28:39 -0700 Subject: [PATCH 25/27] fix: Address the live CodeQL alerts on this PR Five small findings, no behaviour change. _session.py: the `if step_name is None: raise` inside run_task's wrap branch is unreachable, as CodeQL reported. run_task already rejects that combination at method entry under the same `wrap_env is not None` condition, so the second check only narrowed the type -- and its comment claimed the opposite ("a cross-field invariant, not type-checker narrowing"). Replaced with a single `cast(str, step_name)` bound where the requirement has been proven. This is the third unfalsifiable guard of mine to come out of this branch. _subprocess.py: dropped the `del proc` in run()'s finally, which CodeQL flagged as an unnecessary delete. `proc` is a function local and dies with the frame. It is pre-existing on mainline, but 44ff172 wrapped it in a try/finally that existed only to hold it, so the whole construct is gone and the comment now describes what actually matters -- clearing `self._process`. test_tempdir_hardening.py: `lowest_free_fd()` opened a descriptor outside a try/finally. A `with` block cannot be used because the descriptor number *is* the measurement, so it has to be closed before being returned; try/finally does that on every path. Closes three "file is not always closed" alerts. test_linux_sudo.py, test_subprocess_reaping.py: the two bare `except: pass` teardown handlers now say why swallowing is correct, which is what CodeQL's "empty except" rule asks for. 861 passed / 39 skipped / 16 xfailed; ruff, black and mypy clean on native and --platform win32. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- src/openjd/sessions/_session.py | 21 +++++------ src/openjd/sessions/_subprocess.py | 35 +++++++++---------- test/openjd/sessions_v0/test_linux_sudo.py | 4 +++ .../sessions_v0/test_subprocess_reaping.py | 3 ++ .../sessions_v0/test_tempdir_hardening.py | 12 +++++-- 5 files changed, 43 insertions(+), 32 deletions(-) diff --git a/src/openjd/sessions/_session.py b/src/openjd/sessions/_session.py index 80e0f06e..73063e16 100644 --- a/src/openjd/sessions/_session.py +++ b/src/openjd/sessions/_session.py @@ -14,7 +14,7 @@ from pathlib import Path from tempfile import mkstemp from types import TracebackType -from typing import TYPE_CHECKING, Any, Callable, Optional, Type, Union +from typing import TYPE_CHECKING, Any, Callable, Optional, Type, Union, cast from openjd.model import ( FormatStringError, @@ -1232,16 +1232,13 @@ def run_task( # If a wrap environment is active, inject WrappedAction.* into the symbol # table and run its hook instead of the step script's onRun (RFC 0008). if wrap_env is not None: - # A cross-field invariant, not type-checker narrowing: RFC 0008 - # requires WrappedStep.Name, so a wrapped run without a step name is - # rejected at the top of this method. An explicit raise rather than - # `assert` so that `python -O` cannot let a wrapped task through with - # `WrappedStep.Name` set to None (R5-6). - if step_name is None: # pragma: no cover - guarded at method entry - raise RuntimeError( - "Internal error: a wrap environment is active but no step name was " - "resolved for the task." - ) + # RFC 0008 requires WrappedStep.Name, and a wrapped run without a step + # name is already rejected at the top of this method, under this same + # `wrap_env is not None` condition. So this is narrowing for the type + # checker, not a runtime invariant, and a second runtime check here + # would be unreachable -- CodeQL says so, and it is right. Bound once, + # where the requirement has been proven, rather than re-checked. + wrapped_step_name = cast(str, step_name) # Two separate scopes. The wrapped onRun resolves against the STEP's # own scope (`symtab`, which carries this task's parameters and the # running step's name); the hook resolves against its own table, @@ -1255,7 +1252,7 @@ def run_task( inner_script=step_script, symtab=symtab, inject=lambda inner_symtab: self._inject_wrapped_task_symbols( - hook_symtab, step_script, step_name, inner_symtab=inner_symtab + hook_symtab, step_script, wrapped_step_name, inner_symtab=inner_symtab ), fail_message=( f"Failed to resolve the wrapped Task action for {wrap_env.name}'s " diff --git a/src/openjd/sessions/_subprocess.py b/src/openjd/sessions/_subprocess.py index 71c640dc..a48bd02d 100644 --- a/src/openjd/sessions/_subprocess.py +++ b/src/openjd/sessions/_subprocess.py @@ -241,26 +241,25 @@ def run(self) -> None: if self._callback: self._callback() finally: - # Explicitly delete the Popen in case it's a PopenWindowsAsUser and there's stuff to - # deallocate + # Drop this object's reference to the Popen, in case it's a + # PopenWindowsAsUser and there's stuff to deallocate. `proc` itself is + # a function local and dies with the frame; a `del proc` here would be + # a no-op, which is what CodeQL's "unnecessary delete" reports. proc = self._process self._process = None - try: - # This method owns the subprocess for its whole life, and clearing - # `_process` is the point after which nothing else can reach it: - # `notify()` and `terminate()` both become permanent no-ops. So - # ownership has to be discharged here, on every exit path, not - # only on the one that runs `wait()` above. - # - # An exception out of the stdout pump used to skip both the - # `wait()` and the returncode capture while still clearing - # `_process` -- leaving a live, unreapable child, no exit code, - # and an object reporting neither running nor failed-to-start. A - # raising `logging.Filter` reaches this path, and installing one - # is a supported use of this library. - self._reap(proc) - finally: - del proc + # This method owns the subprocess for its whole life, and clearing + # `_process` is the point after which nothing else can reach it: + # `notify()` and `terminate()` both become permanent no-ops. So + # ownership has to be discharged here, on every exit path, not only on + # the one that runs `wait()` above. + # + # An exception out of the stdout pump used to skip both the `wait()` + # and the returncode capture while still clearing `_process` -- + # leaving a live, unreapable child, no exit code, and an object + # reporting neither running nor failed-to-start. A raising + # `logging.Filter` reaches this path, and installing one is a + # supported use of this library. + self._reap(proc) def _reap(self, proc: Optional[Popen]) -> None: """Make sure ``proc`` is dead and reaped, and that its exit code is diff --git a/test/openjd/sessions_v0/test_linux_sudo.py b/test/openjd/sessions_v0/test_linux_sudo.py index 2624dfe8..56c3c948 100644 --- a/test/openjd/sessions_v0/test_linux_sudo.py +++ b/test/openjd/sessions_v0/test_linux_sudo.py @@ -372,6 +372,10 @@ def test_keeps_scanning_until_a_late_child_appears( try: os.killpg(child_pgid, signal.SIGKILL) # type: ignore except (ProcessLookupError, PermissionError): + # Best-effort teardown. The group is already gone + # (ProcessLookupError) or was never ours to signal + # (PermissionError); either way there is nothing left to clean + # up and failing here would mask the test's real result. pass diff --git a/test/openjd/sessions_v0/test_subprocess_reaping.py b/test/openjd/sessions_v0/test_subprocess_reaping.py index 8f6ea324..703fb38d 100644 --- a/test/openjd/sessions_v0/test_subprocess_reaping.py +++ b/test/openjd/sessions_v0/test_subprocess_reaping.py @@ -201,6 +201,9 @@ def test_a_reap_failure_does_not_replace_the_original_exception( try: os.waitpid(proc.pid, 0) except ChildProcessError: + # Already reaped -- which is the outcome this test wants anyway. + # Popen's own destructor or the code under test may have got there + # first; either way there is no zombie left to collect. pass def test_the_process_handle_is_still_released( diff --git a/test/openjd/sessions_v0/test_tempdir_hardening.py b/test/openjd/sessions_v0/test_tempdir_hardening.py index b6aa5be1..950ab5f0 100644 --- a/test/openjd/sessions_v0/test_tempdir_hardening.py +++ b/test/openjd/sessions_v0/test_tempdir_hardening.py @@ -420,9 +420,17 @@ def test_no_descriptor_is_leaked(self, tmp_path: Path) -> None: (parent / "OpenJD").mkdir() def lowest_free_fd() -> int: + """Probe the lowest free descriptor number by taking and releasing one. + + A `with` block is not usable here: the number itself is the + measurement, so the descriptor has to be closed before it is returned. + `try/finally` guarantees that on every path. + """ fd = os.open(os.devnull, os.O_RDONLY) - os.close(fd) - return fd + try: + return fd + finally: + os.close(fd) with patch("openjd.sessions._tempdir.gettempdir", return_value=str(parent)): custom_gettempdir() # warm up, so first-call allocations do not count From 8f1691cda5b3b8956ab62da1cbec913024b8b22f Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Sun, 26 Jul 2026 16:40:38 -0700 Subject: [PATCH 26/27] refactor: Delete the dead _v1/_linux/ duplicate (R1) 412 lines of unreferenced copy. Nothing in src/ or test/ imports it: the only `._linux._*` imports in the package are in _subprocess.py:16-17, and because that module sits at openjd/sessions/, they resolve to the live _linux/. No module under _v1/ imports `._linux` at all, so the relative-import route that would have reached this copy does not exist either. It is not merely dead, it is dangerously stale. The dead _sudo.py still carries the unguarded `os.getpgid(sudo_process.pid)` that d29aafe fixed in the live copy -- 82 differing lines between the two. A reader grepping for find_sudo_child_process_group_id got two hits and no signal about which one runs. Pre-existing (arrived on mainline via #316) and untouched by this branch, but this branch widened the gap by fixing only the live copy, which is what makes it worth removing here. No test changes: 861 passed / 39 skipped / 16 xfailed, unchanged. mypy now covers 88 source files instead of 91. ruff, black and mypy clean on native and --platform win32. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- src/openjd/sessions/_v1/_linux/__init__.py | 1 - .../sessions/_v1/_linux/_capabilities.py | 257 ------------------ src/openjd/sessions/_v1/_linux/_sudo.py | 154 ----------- 3 files changed, 412 deletions(-) delete mode 100644 src/openjd/sessions/_v1/_linux/__init__.py delete mode 100644 src/openjd/sessions/_v1/_linux/_capabilities.py delete mode 100644 src/openjd/sessions/_v1/_linux/_sudo.py diff --git a/src/openjd/sessions/_v1/_linux/__init__.py b/src/openjd/sessions/_v1/_linux/__init__.py deleted file mode 100644 index 8d929cc8..00000000 --- a/src/openjd/sessions/_v1/_linux/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/src/openjd/sessions/_v1/_linux/_capabilities.py b/src/openjd/sessions/_v1/_linux/_capabilities.py deleted file mode 100644 index c2c31a58..00000000 --- a/src/openjd/sessions/_v1/_linux/_capabilities.py +++ /dev/null @@ -1,257 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - -"""This module contains code for interacting with Linux capabilities. The module uses the ctypes -module from the Python standard library to wrap the libcap library. - -See https://man7.org/linux/man-pages/man7/capabilities.7.html for details on this Linux kernel -feature. -""" - -import ctypes -import os -import sys -from contextlib import contextmanager -from ctypes.util import find_library -from enum import Enum -from functools import cache -from typing import Any, Generator, Optional, Tuple, TYPE_CHECKING - - -from .._logging import LOG - -# Capability sets -CAP_EFFECTIVE = 0 -CAP_PERMITTED = 1 -CAP_INHERITABLE = 2 - -# Capability bit numbers -CAP_KILL = 5 - -# Values for cap_flag_value_t arguments -CAP_CLEAR = 0 -CAP_SET = 1 - -cap_flag_t = ctypes.c_int -cap_flag_value_t = ctypes.c_int -cap_value_t = ctypes.c_int - - -class CapabilitySetType(Enum): - INHERITABLE = CAP_INHERITABLE - PERMITTED = CAP_PERMITTED - EFFECTIVE = CAP_EFFECTIVE - - -class UserCapHeader(ctypes.Structure): - _fields_ = [ - ("version", ctypes.c_uint32), - ("pid", ctypes.c_int), - ] - - -class UserCapData(ctypes.Structure): - _fields_ = [ - ("effective", ctypes.c_uint32), - ("permitted", ctypes.c_uint32), - ("inheritable", ctypes.c_uint32), - ] - - -class Cap(ctypes.Structure): - _fields_ = [ - ("head", UserCapHeader), - ("data", UserCapData), - ] - - -if TYPE_CHECKING: - cap_t = ctypes._Pointer[Cap] - cap_flag_value_ptr = ctypes._Pointer[cap_flag_value_t] - cap_value_ptr = ctypes._Pointer[cap_value_t] - ssize_ptr_t = ctypes._Pointer[ctypes.c_ssize_t] -else: - cap_t = ctypes.POINTER(Cap) - cap_flag_value_ptr = ctypes.POINTER(cap_flag_value_t) - cap_value_ptr = ctypes.POINTER(cap_value_t) - ssize_ptr_t = ctypes.POINTER(ctypes.c_ssize_t) - - -def _cap_set_err_check( - result: ctypes.c_int, - func: Any, - args: Tuple[Any, ...], -) -> ctypes.c_int: - if result != 0: - errno = ctypes.get_errno() - raise OSError(errno, os.strerror(errno)) - return result - - -def _cap_get_proc_err_check( - result: cap_t, - func: Any, - args: Tuple[cap_t, cap_flag_t, ctypes.c_int, cap_value_ptr, cap_flag_value_t], -) -> cap_t: - if not result: - errno = ctypes.get_errno() - raise OSError(errno, os.strerror(errno)) - return result - - -def _cap_get_flag_errcheck( - result: ctypes.c_int, func: Any, args: Tuple[cap_t, cap_value_t, cap_flag_t, cap_flag_value_ptr] -) -> ctypes.c_int: - if result != 0: - errno = ctypes.get_errno() - raise OSError(errno, os.strerror(errno)) - return result - - -@cache -def _get_libcap() -> Optional[ctypes.CDLL]: - if not sys.platform.startswith("linux"): - raise OSError(f"libcap is only available on Linux, but found platform: {sys.platform}") - - libcap_path = find_library("cap") - if libcap_path is None: - LOG.info( - "Unable to locate libcap. Session action cancelation signals will be sent using sudo" - ) - return None - - libcap = ctypes.CDLL(libcap_path, use_errno=True) - - # https://man7.org/linux/man-pages/man3/cap_set_proc.3.html - libcap.cap_set_proc.restype = ctypes.c_int - libcap.cap_set_proc.argtypes = [ - cap_t, - ] - libcap.cap_set_proc.errcheck = _cap_set_err_check # type: ignore - - # https://man7.org/linux/man-pages/man3/cap_get_proc.3.html - libcap.cap_get_proc.restype = cap_t - libcap.cap_get_proc.argtypes = [] - libcap.cap_get_proc.errcheck = _cap_get_proc_err_check # type: ignore - - # https://man7.org/linux/man-pages/man3/cap_set_flag.3.html - libcap.cap_set_flag.restype = ctypes.c_int - libcap.cap_set_flag.argtypes = [ - cap_t, - cap_flag_t, - ctypes.c_int, - cap_value_ptr, - cap_flag_value_t, - ] - - # https://man7.org/linux/man-pages/man3/cap_get_flag.3.html - libcap.cap_get_flag.restype = ctypes.c_int - libcap.cap_get_flag.argtypes = ( - cap_t, - cap_value_t, - cap_flag_t, - cap_flag_value_ptr, - ) - libcap.cap_get_flag.errcheck = _cap_get_flag_errcheck # type: ignore - - return libcap - - -def _has_capability( - *, - libcap: ctypes.CDLL, - caps: cap_t, - capability: int, - capability_set_type: CapabilitySetType, -) -> bool: - flag_value = cap_flag_value_t() - libcap.cap_get_flag(caps, capability, capability_set_type.value, ctypes.byref(flag_value)) - return flag_value.value == CAP_SET - - -@contextmanager -def try_use_cap_kill() -> Generator[bool, None, None]: - """ - A context-manager that attempts to leverage the CAP_KILL Linux capability. - - If CAP_KILL is in the current thread's effective set, this context-manager takes no action and - yields True. - - If CAP_KILL is not in the effective set but is in the permitted set, the context-manager: - 1. adds CAP_KILL to the effective set before entering the context-manager - 2. yields True - 3. clears CAP_KILL from the effective set when exiting the context-manager - - Otherwise, the context-manager does nothing and yields False - - Returns: - A context manager that yields a bool. See above for details. - """ - if not sys.platform.startswith("linux"): - raise OSError(f"Only Linux is supported, but platform is {sys.platform}") - - libcap = _get_libcap() - # If libcap is not found, we yield False indicating we are not aware of having CAP_KILL - if not libcap: - yield False - return - - caps = libcap.cap_get_proc() - - if _has_capability( - libcap=libcap, - caps=caps, - capability=CAP_KILL, - capability_set_type=CapabilitySetType.EFFECTIVE, - ): - LOG.debug("CAP_KILL is in the thread's effective set") - # CAP_KILL is already in the effective set - yield True - elif _has_capability( - libcap=libcap, - caps=caps, - capability=CAP_KILL, - capability_set_type=CapabilitySetType.PERMITTED, - ): - # CAP_KILL is in the permitted set. We will temporarily add it to the effective set - LOG.debug("CAP_KILL is in the thread's permitted set. Temporarily adding to effective set") - cap_value_arr_t = cap_value_t * 1 - cap_value_arr = cap_value_arr_t() - cap_value_arr[0] = CAP_KILL - libcap.cap_set_flag( - caps, - CAP_EFFECTIVE, - 1, - cap_value_arr, - CAP_SET, - ) - libcap.cap_set_proc(caps) - try: - yield True - finally: - # Clear CAP_KILL from the effective set - LOG.debug("Clearing CAP_KILL from the thread's effective set") - libcap.cap_set_flag( - caps, - CAP_EFFECTIVE, - 1, - cap_value_arr, - CAP_CLEAR, - ) - libcap.cap_set_proc(caps) - else: - yield False - - -def main() -> None: - """A developer debugging entrypoint for testing the try_use_cap_kill() behaviour""" - import logging - - logging.basicConfig(level=logging.DEBUG) - logging.getLogger("openjd.sessions").setLevel(logging.DEBUG) - - with try_use_cap_kill() as has_cap_kill: - LOG.info("Has CAP_KILL: %s", has_cap_kill) - - -if __name__ == "__main__": - main() diff --git a/src/openjd/sessions/_v1/_linux/_sudo.py b/src/openjd/sessions/_v1/_linux/_sudo.py deleted file mode 100644 index 43a1fa8c..00000000 --- a/src/openjd/sessions/_v1/_linux/_sudo.py +++ /dev/null @@ -1,154 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - -import glob -import os -import sys -import time -from subprocess import Popen, DEVNULL, PIPE, STDOUT, run -from typing import Optional - -from .._logging import LoggerAdapter, LogContent, LogExtraInfo -from .._os_checker import is_posix, is_linux - - -class FindSignalTargetError(Exception): - """Exception when unable to detect the signal target""" - - pass - - -def find_sudo_child_process_group_id( - *, - logger: LoggerAdapter, - sudo_process: Popen, - timeout_seconds: float = 1, -) -> Optional[int]: - # Hint to mypy to not raise module attribute errors (e.g. missing os.getpgid) - if sys.platform == "win32": - raise NotImplementedError("This method is for POSIX hosts only") - if not is_posix(): - raise NotImplementedError(f"Only POSIX supported, but running on {sys.platform}") - if timeout_seconds <= 0: - raise ValueError(f"Expected positive value for timeout_seconds but got {timeout_seconds}") - - # For cross-user support, we use sudo which creates an intermediate process: - # - # openjd-process - # | - # +-- sudo - # | - # +-- subprocess - # - # Sudo forwards signals that it is able to handle, but in the case of SIGKILL sudo cannot - # handle the signal and the kernel will kill it leaving the child orphaned. We need to - # send SIGKILL signals to the subprocess of sudo - start = time.monotonic() - now = start - sudo_pgid = os.getpgid(sudo_process.pid) - - # Repeatedly scan for child processes - # - # This is put in a retry loop, because it takes a non-zero amount of time before sudo and - # the kernel finish creating the subprocess. We cap this because the process may exit - # quickly and we may never find the child process. - sudo_child_pid: Optional[int] = None - sudo_child_pgid: Optional[int] = None - try: - while now - start < timeout_seconds: - if not sudo_child_pid: - if is_linux(): - sudo_child_pid = find_sudo_child_process_id_procfs( - sudo_pid=sudo_process.pid, - logger=logger, - ) - else: - sudo_child_pid = find_child_process_id_pgrep( - sudo_pid=sudo_process.pid, - ) - - if sudo_child_pid: - try: - sudo_child_pgid = os.getpgid(sudo_child_pid) - except ProcessLookupError: - # If the process has exited, we short-circuit - return None - # sudo first forks, then creates a new process group. There is a race condition - # where the process group ID we observe has not yet changed. If the PGID detected - # matches the PGID of sudo, then we retry again in the loop - if sudo_child_pgid == sudo_pgid: - sudo_child_pgid = None - else: - break - - # If we did not find any child processes yet, sleep for some time and retry - time.sleep(min(0.05, timeout_seconds - (now - start))) - now = time.monotonic() - if not sudo_child_pid or not sudo_child_pgid: - raise FindSignalTargetError("unable to detect subprocess before timeout") - except FindSignalTargetError as e: - logger.warning( - f"Unable to determine signal target: {e}", - extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), - ) - - if sudo_child_pgid: - logger.debug( - f"Signal target PGID = {sudo_child_pgid}", - extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), - ) - - return sudo_child_pgid - - -def find_sudo_child_process_id_procfs( - *, - logger: LoggerAdapter, - sudo_pid: int, -) -> Optional[int]: - # Look for the child process of sudo using procfs. See - # https://docs.kernel.org/filesystems/proc.html#proc-pid-task-tid-children-information-about-task-children - - child_pids: set[int] = set() - for task_children_path in glob.glob(f"/proc/{sudo_pid}/task/**/children"): - with open(task_children_path, "r") as f: - child_pids.update(int(pid_str) for pid_str in f.read().split()) - - # If we found exactly one child, we return it - if len(child_pids) == 1: - - child_pid = child_pids.pop() - - logger.debug( - f"Session action process (sudo child) PID is {child_pid}", - extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), - ) - return child_pid - # If we found multiple child processes, this violates our assumptions about how sudo - # works. We will fall-back to using pkill for signalling the process - elif len(child_pids) > 1: - raise FindSignalTargetError( - f"Expected single child processes of sudo, but found {child_pids}" - ) - return None - - -def find_child_process_id_pgrep( - *, - sudo_pid: int, -) -> Optional[int]: - pgrep_result = run( - ["pgrep", "-P", str(sudo_pid)], - stdout=PIPE, - stderr=STDOUT, - stdin=DEVNULL, - text=True, - ) - if pgrep_result.returncode != 0: - raise FindSignalTargetError("Unable to query child processes of sudo process") - results = pgrep_result.stdout.splitlines() - if len(results) > 1: - raise FindSignalTargetError(f"Expected a single child process of sudo, but found {results}") - elif len(results) == 0: - return None - sudo_subproc_pid = int(results[0]) - return sudo_subproc_pid From 5373d76fbcce7bd022cfba0117cb911886febf47 Mon Sep 17 00:00:00 2001 From: David Leong <116610336+leongdl@users.noreply.github.com> Date: Sun, 26 Jul 2026 17:03:02 -0700 Subject: [PATCH 27/27] refactor: Single-source the cancel-path signal helper (R5, R13) Two contained simplifications in the cancel/filter paths. No behaviour change: the whole suite passes unmodified. R13 -- _runner_base.ScriptRunnerBase._signal_process(process, method). The "send a signal, warn on OSError" block was written four times. Three were identical modulo the verb (_cancel's terminate arm, _cancel's notify arm, and _on_notify_period_end's terminate); the fourth, the cancel_info.json write fallback, carries a different message and is left alone. `process` is a parameter rather than a read of self._process so _on_notify_period_end keeps passing the lock-free local snapshot it already holds instead of re-reading an attribute another thread may have cleared. The verb is a Literal["terminate", "notify"] with an explicit branch rather than a getattr, so the dispatch stays type-checked. _cancel: 133 -> 114 lines. R5 -- _action_filter.py: removed the unreachable `except json.JSONDecodeError`. The only json.loads call sits inside an inner handler that re-raises as ValueError, so the outer arm could never fire; the whole outer try/except goes with it. Also binds envvar_set_matcher_str.match(message) once instead of evaluating it at both :513 and :521 -- the same match now decides validity and selects the parse branch. Verified: 861 passed / 39 skipped / 16 xfailed, unchanged. ruff, black and mypy clean on native and --platform win32. R13's dispatch is mutation-checked: 3 mutants (dispatch inverted, notify never sent, terminate never sent), 3 caught. Signed-off-by: David Leong <116610336+leongdl@users.noreply.github.com> --- src/openjd/sessions/_action_filter.py | 43 ++++++++---------- src/openjd/sessions/_runner_base.py | 65 +++++++++++++-------------- 2 files changed, 51 insertions(+), 57 deletions(-) diff --git a/src/openjd/sessions/_action_filter.py b/src/openjd/sessions/_action_filter.py index 922eb768..25d43aac 100644 --- a/src/openjd/sessions/_action_filter.py +++ b/src/openjd/sessions/_action_filter.py @@ -509,33 +509,28 @@ def _parse_env_variable(self, message: str) -> tuple[str, str, bool, int, Option if equals_position == -1: return "", "", False, -1, None - # Check if the message is valid - is_valid = envvar_set_matcher_str.match(message) or envvar_set_matcher_json.match(message) - - if not is_valid: + # Check if the message is valid. Bound once: the plain-string match decides + # both validity and which parse branch to take below. + str_match = envvar_set_matcher_str.match(message) + if not (str_match or envvar_set_matcher_json.match(message)): return "", "", False, equals_position, None # Parse the variable name and value - try: - original_value = None - if envvar_set_matcher_str.match(message): - name, _, value = message.partition("=") - else: - # Handle JSON format - try: - # Store the original value before JSON parsing - original_value = message[equals_position + 1 :] - message_json_str = json.loads(message) - name, _, value = message_json_str.partition("=") - except json.JSONDecodeError as e: - raise ValueError( - f"Unterminated string starting at: line {e.lineno} column {e.colno} (char {e.pos})" - ) - return name, value, True, equals_position, original_value - except json.JSONDecodeError as e: - raise ValueError( - f"Unterminated string starting at: line {e.lineno} column {e.colno} (char {e.pos})" - ) + original_value = None + if str_match: + name, _, value = message.partition("=") + else: + # Handle JSON format + try: + # Store the original value before JSON parsing + original_value = message[equals_position + 1 :] + message_json_str = json.loads(message) + name, _, value = message_json_str.partition("=") + except json.JSONDecodeError as e: + raise ValueError( + f"Unterminated string starting at: line {e.lineno} column {e.colno} (char {e.pos})" + ) + return name, value, True, equals_position, original_value def _handle_env_error(self, error_message: str, is_redacted: bool = False) -> None: """Handle errors in environment variable processing. diff --git a/src/openjd/sessions/_runner_base.py b/src/openjd/sessions/_runner_base.py index 65facf5f..1d8e917a 100644 --- a/src/openjd/sessions/_runner_base.py +++ b/src/openjd/sessions/_runner_base.py @@ -12,7 +12,7 @@ from enum import Enum from pathlib import Path from threading import Lock, Timer -from typing import Any, Callable, Optional, Sequence, Type, cast +from typing import Any, Callable, Literal, Optional, Sequence, Type, cast from types import TracebackType from tempfile import mkstemp @@ -1145,6 +1145,34 @@ def _cancel_with_resolved_method( # Called outside the lock: _cancel takes it itself. self._cancel(method, time_limit, mark_action_failed) + def _signal_process( + self, process: LoggingSubprocess, method: Literal["terminate", "notify"] + ) -> None: + """Send ``method`` to ``process``, warning rather than raising if it fails. + + ``process`` is a parameter rather than a read of ``self._process`` so the + notify-period timer can pass the lock-free local snapshot it already + holds, instead of re-reading an attribute another thread may have + cleared. + + An ``OSError`` here is close to impossible -- if we could start the + process we can signal it -- but a cancel path is the wrong place to + raise: the caller is already unwinding a cancel, and losing the + surrounding bookkeeping to a signal failure is worse than a warning. + """ + try: + if method == "notify": + process.notify() + else: + process.terminate() + except OSError as err: # pragma: nocover + self._logger.warning( + f"Cancelation could not send {method} signal to process {process.pid}: {str(err)}", + extra=LogExtraInfo( + openjd_log_content=LogContent.PROCESS_CONTROL | LogContent.EXCEPTION_INFO + ), + ) + def _cancel( self, method: CancelMethod, @@ -1191,17 +1219,7 @@ def _cancel( f"Canceling subprocess {str(self._process.pid)} via termination method at {now_str}.", extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), ) - try: - self._process.terminate() - except OSError as err: # pragma: nocover - # Being paranoid. Won't happen... if we could start the process, then we can send it a signal - self._logger.warning( - f"Cancelation could not send terminate signal to process {self._process.pid}: {str(err)}", - extra=LogExtraInfo( - openjd_log_content=LogContent.PROCESS_CONTROL - | LogContent.EXCEPTION_INFO - ), - ) + self._signal_process(self._process, "terminate") else: self._logger.info( f"Canceling subprocess {str(self._process.pid)} via notify then terminate method at {now_str}.", @@ -1261,17 +1279,7 @@ def _cancel( ) # 2) Send the notify - try: - self._process.notify() - except OSError as err: # pragma: nocover - # Being paranoid. Won't happen... if we could start the process, then we can send it a signal - self._logger.warning( - f"Cancelation could not send notify signal to process {self._process.pid}: {str(err)}", - extra=LogExtraInfo( - openjd_log_content=LogContent.PROCESS_CONTROL - | LogContent.EXCEPTION_INFO - ), - ) + self._signal_process(self._process, "notify") # 4) Set up the timer to send the terminate signal self._cancel_gracetime_timer = Timer( @@ -1376,16 +1384,7 @@ def _on_notify_period_end(self) -> None: datetime.now(timezone.utc).strftime(TIME_FORMAT_STR), extra=LogExtraInfo(openjd_log_content=LogContent.PROCESS_CONTROL), ) - try: - process.terminate() - except OSError as err: # pragma: nocover - # Being paranoid. Won't happen... if we could start the process, then we can send it a kill signal - self._logger.warning( - f"Cancelation could not send terminate signal to process {process.pid}: {str(err)}", - extra=LogExtraInfo( - openjd_log_content=LogContent.PROCESS_CONTROL | LogContent.EXCEPTION_INFO - ), - ) + self._signal_process(process, "terminate") def _on_timelimit(self) -> None: """Callback that is invoked when the runtime limit of the running